mirror of
https://github.com/towfiqi/serpbear
synced 2025-06-26 18:15:54 +00:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
3690e97fe7 | ||
|
|
17fb2c40cc | ||
|
|
faa3519a29 | ||
|
|
bc02c929ba | ||
|
|
d9d7c6347e | ||
|
|
c5714c00ae | ||
|
|
1bef7587cc |
17
CHANGELOG.md
17
CHANGELOG.md
@@ -2,6 +2,23 @@
|
||||
|
||||
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
||||
|
||||
### [2.0.5](https://github.com/towfiqi/serpbear/compare/v2.0.4...v2.0.5) (2024-11-12)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Fixes broken scrape result issue for keywords with special characters. ([d9d7c63](https://github.com/towfiqi/serpbear/commit/d9d7c6347e99e35f1e48f20b1e8f999612d69a72)), closes [#221](https://github.com/towfiqi/serpbear/issues/221)
|
||||
* Fixes misaligned Keywords table UI content. ([faa3519](https://github.com/towfiqi/serpbear/commit/faa3519a29fc61ef8bd2ce9275a6674f1c7946e0))
|
||||
* Resolves "Add Domain" UI confusion. ([17fb2c4](https://github.com/towfiqi/serpbear/commit/17fb2c40cc6b57ab2fe6aeb940dececd1a83411f))
|
||||
* Resolves broken Scrapingrobot scraper on new installs. ([bc02c92](https://github.com/towfiqi/serpbear/commit/bc02c929ba7efd6b4b6a09495af7310c155a26bd)), closes [#243](https://github.com/towfiqi/serpbear/issues/243)
|
||||
|
||||
### [2.0.4](https://github.com/towfiqi/serpbear/compare/v2.0.3...v2.0.4) (2024-11-10)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Fixes Docker build issue. ([1bef758](https://github.com/towfiqi/serpbear/commit/1bef7587cccada6df48cfa3d208bf123a5d00c30))
|
||||
|
||||
### [2.0.3](https://github.com/towfiqi/serpbear/compare/v2.0.2...v2.0.3) (2024-11-10)
|
||||
|
||||
|
||||
|
||||
15
Dockerfile
15
Dockerfile
@@ -1,5 +1,6 @@
|
||||
FROM node:lts-alpine AS deps
|
||||
|
||||
FROM node:22.11.0-alpine3.20 AS deps
|
||||
ENV NPM_VERSION=10.3.0
|
||||
RUN npm install -g npm@"${NPM_VERSION}"
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json ./
|
||||
@@ -7,8 +8,10 @@ RUN npm install
|
||||
COPY . .
|
||||
|
||||
|
||||
FROM node:lts-alpine AS builder
|
||||
FROM node:22.11.0-alpine3.20 AS builder
|
||||
WORKDIR /app
|
||||
ENV NPM_VERSION=10.3.0
|
||||
RUN npm install -g npm@"${NPM_VERSION}"
|
||||
COPY --from=deps /app ./
|
||||
RUN rm -rf /app/data
|
||||
RUN rm -rf /app/__tests__
|
||||
@@ -16,9 +19,11 @@ RUN rm -rf /app/__mocks__
|
||||
RUN npm run build
|
||||
|
||||
|
||||
FROM node:lts-alpine AS runner
|
||||
FROM node:22.11.0-alpine3.20 AS runner
|
||||
WORKDIR /app
|
||||
ENV NODE_ENV production
|
||||
ENV NPM_VERSION=10.3.0
|
||||
RUN npm install -g npm@"${NPM_VERSION}"
|
||||
ENV NODE_ENV=production
|
||||
RUN addgroup --system --gid 1001 nodejs
|
||||
RUN adduser --system --uid 1001 nextjs
|
||||
RUN set -xe && mkdir -p /app/data && chown nextjs:nodejs /app/data
|
||||
|
||||
@@ -36,7 +36,7 @@ const AddDomain = ({ closeModal, domains = [] }: AddDomainProps) => {
|
||||
}
|
||||
});
|
||||
if (invalidDomains.length > 0) {
|
||||
setNewDomainError(`Please Insert Valid Domain URL. Invalid URLs: ${invalidDomains.join(', ')}`);
|
||||
setNewDomainError(`Please Insert Valid Website URL. ${invalidDomains.length > 1 ? `Invalid URLs: ${invalidDomains.join(', ')}` : ''}`);
|
||||
} else if (domainsTobeAdded.length > 0) {
|
||||
console.log('domainsTobeAdded :', domainsTobeAdded);
|
||||
addMutate(domainsTobeAdded);
|
||||
@@ -51,11 +51,11 @@ const AddDomain = ({ closeModal, domains = [] }: AddDomainProps) => {
|
||||
return (
|
||||
<Modal closeModal={() => { closeModal(false); }} title={'Add New Domain'}>
|
||||
<div data-testid="adddomain_modal">
|
||||
<h4 className='text-sm mt-4'>Domain URL</h4>
|
||||
<h4 className='text-sm mt-4 pb-2'>Website URL(s)</h4>
|
||||
<textarea
|
||||
className={`w-full h-40 border rounded border-gray-200 p-4 outline-none
|
||||
focus:border-indigo-300 ${newDomainError ? ' border-red-400 focus:border-red-400' : ''}`}
|
||||
placeholder="Type or Paste URLs here. Insert Each URL in a New line."
|
||||
placeholder={'Type or Paste URLs here. Insert Each URL in a New line. eg: \nhttps://mysite.com/ \nhttps://anothersite.com/ '}
|
||||
value={newDomain}
|
||||
autoFocus={true}
|
||||
onChange={handleDomainInput}>
|
||||
|
||||
@@ -97,7 +97,7 @@ const Keyword = (props: KeywordProps) => {
|
||||
className={`keyword relative py-5 px-4 text-gray-600 border-b-[1px] border-gray-200 lg:py-4 lg:px-6 lg:border-0
|
||||
lg:flex lg:justify-between lg:items-center ${selected ? ' bg-indigo-50 keyword--selected' : ''} ${lastItem ? 'border-b-0' : ''}`}>
|
||||
|
||||
<div className=' w-3/4 font-semibold cursor-pointer lg:flex-1 lg:shrink-0 lg:basis-20 lg:w-auto lg:flex lg:items-center'>
|
||||
<div className=' w-3/4 font-semibold cursor-pointer lg:flex-1 lg:shrink-0 lg:basis-28 lg:w-auto lg:flex lg:items-center'>
|
||||
<button
|
||||
className={`p-0 mr-2 leading-[0px] inline-block rounded-sm pt-0 px-[1px] pb-[3px] border
|
||||
${selected ? ' bg-blue-700 border-blue-700 text-white' : 'text-transparent'}`}
|
||||
|
||||
@@ -66,7 +66,10 @@ const KeywordFilters = (props: KeywordFilterProps) => {
|
||||
const optionObject:{label:string, value:string}[] = [];
|
||||
|
||||
if (!isConsole) {
|
||||
const allCountries = keywords.reduce((acc: string[], keyword) => [...acc, keyword.country], []).filter((t) => t && t.trim() !== '');
|
||||
const allCountries = Array.from(keywords as KeywordType[])
|
||||
.map((keyword) => keyword.country)
|
||||
.reduce<string[]>((acc, country) => [...acc, country], [])
|
||||
.filter((t) => t && t.trim() !== '');
|
||||
[...new Set(allCountries)].forEach((c) => optionObject.push({ label: countries[c][0], value: c }));
|
||||
} else {
|
||||
Object.keys(countries).forEach((countryISO:string) => {
|
||||
|
||||
145
package-lock.json
generated
145
package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "serpbear",
|
||||
"version": "2.0.3",
|
||||
"version": "2.0.5",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "serpbear",
|
||||
"version": "2.0.3",
|
||||
"version": "2.0.5",
|
||||
"dependencies": {
|
||||
"@googleapis/searchconsole": "^1.0.0",
|
||||
"@isaacs/ttlcache": "^1.4.1",
|
||||
@@ -14,7 +14,7 @@
|
||||
"axios": "^1.1.3",
|
||||
"axios-retry": "^3.3.1",
|
||||
"chart.js": "^3.9.1",
|
||||
"cheerio": "^1.0.0-rc.12",
|
||||
"cheerio": "^1.0.0",
|
||||
"concurrently": "^7.6.0",
|
||||
"cookies": "^0.8.0",
|
||||
"croner": "^5.3.5",
|
||||
@@ -53,8 +53,8 @@
|
||||
"@types/jsonwebtoken": "^8.5.9",
|
||||
"@types/node": "18.11.0",
|
||||
"@types/nodemailer": "^6.4.6",
|
||||
"@types/react": "18.0.21",
|
||||
"@types/react-dom": "18.0.6",
|
||||
"@types/react": "18.2.0",
|
||||
"@types/react-dom": "18.2.0",
|
||||
"@types/react-timeago": "^4.1.3",
|
||||
"@types/react-window": "^1.8.5",
|
||||
"autoprefixer": "^10.4.12",
|
||||
@@ -2447,9 +2447,10 @@
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@types/react": {
|
||||
"version": "18.0.21",
|
||||
"resolved": "https://registry.npmjs.org/@types/react/-/react-18.0.21.tgz",
|
||||
"integrity": "sha512-7QUCOxvFgnD5Jk8ZKlUAhVcRj7GuJRjnjjiY/IUBWKgOlnvDvTMLD4RTF7NPyVmbRhNrbomZiOepg7M/2Kj1mA==",
|
||||
"version": "18.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.0.tgz",
|
||||
"integrity": "sha512-0FLj93y5USLHdnhIhABk83rm8XEGA7kH3cr+YUlvxoUGp1xNt/DINUMvqPxLyOQMzLmZe8i4RTHbvb8MC7NmrA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/prop-types": "*",
|
||||
"@types/scheduler": "*",
|
||||
@@ -2457,10 +2458,11 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@types/react-dom": {
|
||||
"version": "18.0.6",
|
||||
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.0.6.tgz",
|
||||
"integrity": "sha512-/5OFZgfIPSwy+YuIBP/FgJnQnsxhZhjjrnxudMddeblOouIodEQ75X14Rr4wGSG/bknL+Omy9iWlLo1u/9GzAA==",
|
||||
"version": "18.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.0.tgz",
|
||||
"integrity": "sha512-8yQrvS6sMpSwIovhPOwfyNf2Wz6v/B62LFSVYQ85+Rq3tLsBIG7rP5geMxaijTUxSkrO6RzN/IRuIAADYQsleA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/react": "*"
|
||||
}
|
||||
@@ -3395,7 +3397,8 @@
|
||||
"node_modules/boolbase": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
|
||||
"integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="
|
||||
"integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/brace-expansion": {
|
||||
"version": "1.1.11",
|
||||
@@ -3708,20 +3711,25 @@
|
||||
"integrity": "sha512-Ro2JbLmvg83gXF5F4sniaQ+lTbSv18E+TIf2cOeiH1Iqd2PGFOtem+DUufMZsCJwFE7ywPOpfXFBwRTGq7dh6w=="
|
||||
},
|
||||
"node_modules/cheerio": {
|
||||
"version": "1.0.0-rc.12",
|
||||
"resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz",
|
||||
"integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==",
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0.tgz",
|
||||
"integrity": "sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cheerio-select": "^2.1.0",
|
||||
"dom-serializer": "^2.0.0",
|
||||
"domhandler": "^5.0.3",
|
||||
"domutils": "^3.0.1",
|
||||
"htmlparser2": "^8.0.1",
|
||||
"parse5": "^7.0.0",
|
||||
"parse5-htmlparser2-tree-adapter": "^7.0.0"
|
||||
"domutils": "^3.1.0",
|
||||
"encoding-sniffer": "^0.2.0",
|
||||
"htmlparser2": "^9.1.0",
|
||||
"parse5": "^7.1.2",
|
||||
"parse5-htmlparser2-tree-adapter": "^7.0.0",
|
||||
"parse5-parser-stream": "^7.1.2",
|
||||
"undici": "^6.19.5",
|
||||
"whatwg-mimetype": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
"node": ">=18.17"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/cheeriojs/cheerio?sponsor=1"
|
||||
@@ -3731,6 +3739,7 @@
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz",
|
||||
"integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"boolbase": "^1.0.0",
|
||||
"css-select": "^5.1.0",
|
||||
@@ -3743,6 +3752,15 @@
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
},
|
||||
"node_modules/cheerio/node_modules/whatwg-mimetype": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz",
|
||||
"integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/chokidar": {
|
||||
"version": "3.5.3",
|
||||
"resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz",
|
||||
@@ -4490,6 +4508,7 @@
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz",
|
||||
"integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"boolbase": "^1.0.0",
|
||||
"css-what": "^6.1.0",
|
||||
@@ -4505,6 +4524,7 @@
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz",
|
||||
"integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
},
|
||||
@@ -4930,6 +4950,7 @@
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
|
||||
"integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"domelementtype": "^2.3.0",
|
||||
"domhandler": "^5.0.2",
|
||||
@@ -4948,7 +4969,8 @@
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
]
|
||||
],
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/domexception": {
|
||||
"version": "4.0.0",
|
||||
@@ -4966,6 +4988,7 @@
|
||||
"version": "5.0.3",
|
||||
"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
|
||||
"integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"domelementtype": "^2.3.0"
|
||||
},
|
||||
@@ -4980,6 +5003,7 @@
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz",
|
||||
"integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"dom-serializer": "^2.0.0",
|
||||
"domelementtype": "^2.3.0",
|
||||
@@ -5221,6 +5245,43 @@
|
||||
"iconv-lite": "^0.6.2"
|
||||
}
|
||||
},
|
||||
"node_modules/encoding-sniffer": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.0.tgz",
|
||||
"integrity": "sha512-ju7Wq1kg04I3HtiYIOrUrdfdDvkyO9s5XM8QAj/bN61Yo/Vb4vgJxy5vi4Yxk01gWHbrofpPtpxM8bKger9jhg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"iconv-lite": "^0.6.3",
|
||||
"whatwg-encoding": "^3.1.1"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/encoding-sniffer?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/encoding-sniffer/node_modules/iconv-lite": {
|
||||
"version": "0.6.3",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
|
||||
"integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"safer-buffer": ">= 2.1.2 < 3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/encoding-sniffer/node_modules/whatwg-encoding": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz",
|
||||
"integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"iconv-lite": "0.6.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/encoding/node_modules/iconv-lite": {
|
||||
"version": "0.6.3",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
|
||||
@@ -7145,9 +7206,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/htmlparser2": {
|
||||
"version": "8.0.2",
|
||||
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz",
|
||||
"integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==",
|
||||
"version": "9.1.0",
|
||||
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz",
|
||||
"integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==",
|
||||
"funding": [
|
||||
"https://github.com/fb55/htmlparser2?sponsor=1",
|
||||
{
|
||||
@@ -7155,11 +7216,12 @@
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"domelementtype": "^2.3.0",
|
||||
"domhandler": "^5.0.3",
|
||||
"domutils": "^3.0.1",
|
||||
"entities": "^4.4.0"
|
||||
"domutils": "^3.1.0",
|
||||
"entities": "^4.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/http-cache-semantics": {
|
||||
@@ -11559,6 +11621,7 @@
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
|
||||
"integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"boolbase": "^1.0.0"
|
||||
},
|
||||
@@ -11927,11 +11990,24 @@
|
||||
}
|
||||
},
|
||||
"node_modules/parse5-htmlparser2-tree-adapter": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz",
|
||||
"integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==",
|
||||
"version": "7.1.0",
|
||||
"resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz",
|
||||
"integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"domhandler": "^5.0.3",
|
||||
"parse5": "^7.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/inikulin/parse5?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/parse5-parser-stream": {
|
||||
"version": "7.1.2",
|
||||
"resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz",
|
||||
"integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"domhandler": "^5.0.2",
|
||||
"parse5": "^7.0.0"
|
||||
},
|
||||
"funding": {
|
||||
@@ -15174,6 +15250,15 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/undici": {
|
||||
"version": "6.20.1",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-6.20.1.tgz",
|
||||
"integrity": "sha512-AjQF1QsmqfJys+LXfGTNum+qw4S88CojRInG/6t31W/1fk6G59s92bnAvGz5Cmur+kQv2SURXEvvudLmbrE8QA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18.17"
|
||||
}
|
||||
},
|
||||
"node_modules/unified": {
|
||||
"version": "11.0.4",
|
||||
"resolved": "https://registry.npmjs.org/unified/-/unified-11.0.4.tgz",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "serpbear",
|
||||
"version": "2.0.3",
|
||||
"version": "2.0.5",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
@@ -24,7 +24,7 @@
|
||||
"axios": "^1.1.3",
|
||||
"axios-retry": "^3.3.1",
|
||||
"chart.js": "^3.9.1",
|
||||
"cheerio": "^1.0.0-rc.12",
|
||||
"cheerio": "^1.0.0",
|
||||
"concurrently": "^7.6.0",
|
||||
"cookies": "^0.8.0",
|
||||
"croner": "^5.3.5",
|
||||
|
||||
@@ -20,11 +20,10 @@ const hasdata:ScraperSettings = {
|
||||
scrapeURL: (keyword, settings) => {
|
||||
const country = keyword.country || 'US';
|
||||
const countryName = countries[country][0];
|
||||
const location = keyword.city && countryName ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
|
||||
return `https://api.scrape-it.cloud/scrape/google/serp?q=${encodeURI(keyword.keyword)}${location}&num=100&gl=${country.toLowerCase()}&deviceType=${keyword.device}`;
|
||||
const location = keyword.city && countryName ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
|
||||
return `https://api.scrape-it.cloud/scrape/google/serp?q=${encodeURIComponent(keyword.keyword)}${location}&num=100&gl=${country.toLowerCase()}&deviceType=${keyword.device}`;
|
||||
},
|
||||
resultObjectKey: 'organicResults',
|
||||
|
||||
serpExtractor: (content) => {
|
||||
const extractedResult = [];
|
||||
const results: HasDataResult[] = (typeof content === 'string') ? JSON.parse(content) : content as HasDataResult[];
|
||||
|
||||
@@ -20,8 +20,8 @@ const searchapi:ScraperSettings = {
|
||||
scrapeURL: (keyword) => {
|
||||
const country = keyword.country || 'US';
|
||||
const countryName = countries[country][0];
|
||||
const location = keyword.city && countryName ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
|
||||
return `https://www.searchapi.io/api/v1/search?engine=google&q=${encodeURI(keyword.keyword)}&num=100&gl=${country}&device=${keyword.device}${location}`;
|
||||
const location = keyword.city && countryName ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
|
||||
return `https://www.searchapi.io/api/v1/search?engine=google&q=${encodeURIComponent(keyword.keyword)}&num=100&gl=${country}&device=${keyword.device}${location}`;
|
||||
},
|
||||
resultObjectKey: 'organic_results',
|
||||
serpExtractor: (content) => {
|
||||
|
||||
@@ -19,8 +19,8 @@ const serpapi:ScraperSettings = {
|
||||
},
|
||||
scrapeURL: (keyword, settings) => {
|
||||
const countryName = countries[keyword.country || 'US'][0];
|
||||
const location = keyword.city && keyword.country ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
|
||||
return `https://serpapi.com/search?q=${encodeURI(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}${location}&api_key=${settings.scaping_api}`;
|
||||
const location = keyword.city && keyword.country ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
|
||||
return `https://serpapi.com/search?q=${encodeURIComponent(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}${location}&api_key=${settings.scaping_api}`;
|
||||
},
|
||||
resultObjectKey: 'organic_results',
|
||||
serpExtractor: (content) => {
|
||||
|
||||
@@ -12,7 +12,8 @@ const serper:ScraperSettings = {
|
||||
scrapeURL: (keyword, settings, countryData) => {
|
||||
const country = keyword.country || 'US';
|
||||
const lang = countryData[country][2];
|
||||
return `https://google.serper.dev/search?q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`;
|
||||
console.log('Serper URL :', `https://google.serper.dev/search?q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`);
|
||||
return `https://google.serper.dev/search?q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`;
|
||||
},
|
||||
resultObjectKey: 'organic',
|
||||
serpExtractor: (content) => {
|
||||
|
||||
@@ -20,7 +20,7 @@ const serply:ScraperSettings = {
|
||||
},
|
||||
scrapeURL: (keyword) => {
|
||||
const country = scraperCountries.includes(keyword.country.toUpperCase()) ? keyword.country : 'US';
|
||||
return `https://api.serply.io/v1/search/q=${encodeURI(keyword.keyword)}&num=100&hl=${country}`;
|
||||
return `https://api.serply.io/v1/search/q=${encodeURIComponent(keyword.keyword)}&num=100&hl=${country}`;
|
||||
},
|
||||
resultObjectKey: 'result',
|
||||
serpExtractor: (content) => {
|
||||
|
||||
@@ -15,10 +15,10 @@ const spaceSerp:ScraperSettings = {
|
||||
scrapeURL: (keyword, settings, countryData) => {
|
||||
const country = keyword.country || 'US';
|
||||
const countryName = countries[country][0];
|
||||
const location = keyword.city ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
|
||||
const location = keyword.city ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
|
||||
const device = keyword.device === 'mobile' ? '&device=mobile' : '';
|
||||
const lang = countryData[country][2];
|
||||
return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${location}${device}&resultBlocks=`;
|
||||
return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${location}${device}&resultBlocks=`;
|
||||
},
|
||||
resultObjectKey: 'organic_results',
|
||||
serpExtractor: (content) => {
|
||||
|
||||
@@ -15,11 +15,11 @@ const valueSerp:ScraperSettings = {
|
||||
scrapeURL: (keyword, settings, countryData) => {
|
||||
const country = keyword.country || 'US';
|
||||
const countryName = countries[country][0];
|
||||
const location = keyword.city ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
|
||||
const location = keyword.city ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
|
||||
const device = keyword.device === 'mobile' ? '&device=mobile' : '';
|
||||
const lang = countryData[country][2];
|
||||
console.log(`https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`);
|
||||
return `https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`;
|
||||
console.log(`https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`);
|
||||
return `https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`;
|
||||
},
|
||||
resultObjectKey: 'organic_results',
|
||||
serpExtractor: (content) => {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import axios, { AxiosResponse, CreateAxiosDefaults } from 'axios';
|
||||
import cheerio from 'cheerio';
|
||||
import * as cheerio from 'cheerio';
|
||||
import { readFile, writeFile } from 'fs/promises';
|
||||
import HttpsProxyAgent from 'https-proxy-agent';
|
||||
import countries from './countries';
|
||||
@@ -124,7 +124,7 @@ export const scrapeKeywordFromGoogle = async (keyword:KeywordType, settings:Sett
|
||||
throw new Error(res);
|
||||
}
|
||||
} catch (error:any) {
|
||||
refreshedResults.error = scraperError;
|
||||
refreshedResults.error = scraperError || 'Unknown Error';
|
||||
if (settings.scraper_type === 'proxy' && error && error.response && error.response.statusText) {
|
||||
refreshedResults.error = `[${error.response.status}] ${error.response.statusText}`;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user