mirror of
https://github.com/towfiqi/serpbear
synced 2025-06-26 18:15:54 +00:00
Compare commits
21 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
54522b2261 | ||
|
|
495d872bb9 | ||
|
|
07eb4bd94f | ||
|
|
36ed4cf800 | ||
|
|
c8601ebb84 | ||
|
|
56d8b660c5 | ||
|
c34c8260c7 | ||
|
|
cab8f518bb | ||
|
|
6e47a6fba7 | ||
|
|
bf911b4e45 | ||
|
|
d7279512cf | ||
|
|
4fef1a9abc | ||
|
|
aeed1f8559 | ||
|
|
12eac2b012 | ||
|
|
649f412303 | ||
|
|
a2edabbdf9 | ||
|
|
3690e97fe7 | ||
|
|
17fb2c40cc | ||
|
|
faa3519a29 | ||
|
|
bc02c929ba | ||
|
|
d9d7c6347e |
28
CHANGELOG.md
28
CHANGELOG.md
@@ -2,6 +2,34 @@
|
||||
|
||||
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
||||
|
||||
### [2.0.7](https://github.com/towfiqi/serpbear/compare/v2.0.6...v2.0.7) (2025-02-23)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Resolves AdWords integration issue. ([36ed4cf](https://github.com/towfiqi/serpbear/commit/36ed4cf800c1fd0e3df4e807faaa1fdb863df5e4))
|
||||
* resolves broken CDN images. ([bf911b4](https://github.com/towfiqi/serpbear/commit/bf911b4e45b9007a05ce6399838da3276161c61d))
|
||||
|
||||
### [2.0.6](https://github.com/towfiqi/serpbear/compare/v2.0.5...v2.0.6) (2024-11-15)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Ensures Docker build uses matching npm package versions from package.json ([4fef1a9](https://github.com/towfiqi/serpbear/commit/4fef1a9abc737da67ab1ea0c4efce8194890545e))
|
||||
* Resolves broken Docker build due to croner package version mismatch. ([aeed1f8](https://github.com/towfiqi/serpbear/commit/aeed1f8559e044bf658d930a22fa91f38cfedc6b)), closes [#247](https://github.com/towfiqi/serpbear/issues/247)
|
||||
* Resolves broken Proxy Scraper functionality. ([649f412](https://github.com/towfiqi/serpbear/commit/649f412303dd50127b3736740962863f735f76eb)), closes [#248](https://github.com/towfiqi/serpbear/issues/248)
|
||||
* Resolves Google Ads search volume data loading issue. ([12eac2b](https://github.com/towfiqi/serpbear/commit/12eac2b01235e9eae06882d6a2c50c793b890661))
|
||||
|
||||
### [2.0.5](https://github.com/towfiqi/serpbear/compare/v2.0.4...v2.0.5) (2024-11-12)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Fixes broken scrape result issue for keywords with special characters. ([d9d7c63](https://github.com/towfiqi/serpbear/commit/d9d7c6347e99e35f1e48f20b1e8f999612d69a72)), closes [#221](https://github.com/towfiqi/serpbear/issues/221)
|
||||
* Fixes misaligned Keywords table UI content. ([faa3519](https://github.com/towfiqi/serpbear/commit/faa3519a29fc61ef8bd2ce9275a6674f1c7946e0))
|
||||
* Resolves "Add Domain" UI confusion. ([17fb2c4](https://github.com/towfiqi/serpbear/commit/17fb2c40cc6b57ab2fe6aeb940dececd1a83411f))
|
||||
* Resolves broken Scrapingrobot scraper on new installs. ([bc02c92](https://github.com/towfiqi/serpbear/commit/bc02c929ba7efd6b4b6a09495af7310c155a26bd)), closes [#243](https://github.com/towfiqi/serpbear/issues/243)
|
||||
|
||||
### [2.0.4](https://github.com/towfiqi/serpbear/compare/v2.0.3...v2.0.4) (2024-11-10)
|
||||
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ COPY --from=builder --chown=nextjs:nodejs /app/.sequelizerc ./.sequelizerc
|
||||
COPY --from=builder --chown=nextjs:nodejs /app/entrypoint.sh ./entrypoint.sh
|
||||
RUN rm package.json
|
||||
RUN npm init -y
|
||||
RUN npm i cryptr dotenv croner @googleapis/searchconsole sequelize-cli @isaacs/ttlcache
|
||||
RUN npm i cryptr@6.0.3 dotenv@16.0.3 croner@9.0.0 @googleapis/searchconsole@1.0.5 sequelize-cli@6.6.2 @isaacs/ttlcache@1.4.1
|
||||
RUN npm i -g concurrently
|
||||
|
||||
USER nextjs
|
||||
|
||||
@@ -2,13 +2,13 @@
|
||||
|
||||
# SerpBear
|
||||
|
||||
   
|
||||
    [](https://www.youtube.com/watch?v=bjtDsd0g468&rco=1)
|
||||
|
||||
#### [Documentation](https://docs.serpbear.com/) | [Changelog](https://github.com/towfiqi/serpbear/blob/main/CHANGELOG.md) | [Docker Image](https://hub.docker.com/r/towfiqi/serpbear)
|
||||
|
||||
SerpBear is an Open Source Search Engine Position Tracking and Keyword Research App. It allows you to track your website's keyword positions in Google and get notified of their position change.
|
||||
|
||||
|
||||
|
||||
|
||||
#### Features
|
||||
|
||||
|
||||
@@ -1,22 +1,26 @@
|
||||
import { render } from '@testing-library/react';
|
||||
import { rest } from 'msw';
|
||||
import { http } from 'msw';
|
||||
import * as React from 'react';
|
||||
import { QueryClient, QueryClientProvider } from 'react-query';
|
||||
|
||||
export const handlers = [
|
||||
rest.get(
|
||||
http.get(
|
||||
'*/react-query',
|
||||
(req, res, ctx) => {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
({ request, params }) => {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
name: 'mocked-react-query',
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
},
|
||||
);
|
||||
},
|
||||
),
|
||||
];
|
||||
|
||||
const createTestQueryClient = () => new QueryClient({
|
||||
defaultOptions: {
|
||||
queries: {
|
||||
|
||||
@@ -36,7 +36,7 @@ const AddDomain = ({ closeModal, domains = [] }: AddDomainProps) => {
|
||||
}
|
||||
});
|
||||
if (invalidDomains.length > 0) {
|
||||
setNewDomainError(`Please Insert Valid Domain URL. Invalid URLs: ${invalidDomains.join(', ')}`);
|
||||
setNewDomainError(`Please Insert Valid Website URL. ${invalidDomains.length > 1 ? `Invalid URLs: ${invalidDomains.join(', ')}` : ''}`);
|
||||
} else if (domainsTobeAdded.length > 0) {
|
||||
console.log('domainsTobeAdded :', domainsTobeAdded);
|
||||
addMutate(domainsTobeAdded);
|
||||
@@ -51,11 +51,11 @@ const AddDomain = ({ closeModal, domains = [] }: AddDomainProps) => {
|
||||
return (
|
||||
<Modal closeModal={() => { closeModal(false); }} title={'Add New Domain'}>
|
||||
<div data-testid="adddomain_modal">
|
||||
<h4 className='text-sm mt-4'>Domain URL</h4>
|
||||
<h4 className='text-sm mt-4 pb-2'>Website URL(s)</h4>
|
||||
<textarea
|
||||
className={`w-full h-40 border rounded border-gray-200 p-4 outline-none
|
||||
focus:border-indigo-300 ${newDomainError ? ' border-red-400 focus:border-red-400' : ''}`}
|
||||
placeholder="Type or Paste URLs here. Insert Each URL in a New line."
|
||||
placeholder={'Type or Paste URLs here. Insert Each URL in a New line. eg: \nhttps://mysite.com/ \nhttps://anothersite.com/ '}
|
||||
value={newDomain}
|
||||
autoFocus={true}
|
||||
onChange={handleDomainInput}>
|
||||
|
||||
@@ -97,7 +97,7 @@ const Keyword = (props: KeywordProps) => {
|
||||
className={`keyword relative py-5 px-4 text-gray-600 border-b-[1px] border-gray-200 lg:py-4 lg:px-6 lg:border-0
|
||||
lg:flex lg:justify-between lg:items-center ${selected ? ' bg-indigo-50 keyword--selected' : ''} ${lastItem ? 'border-b-0' : ''}`}>
|
||||
|
||||
<div className=' w-3/4 font-semibold cursor-pointer lg:flex-1 lg:shrink-0 lg:basis-20 lg:w-auto lg:flex lg:items-center'>
|
||||
<div className=' w-3/4 font-semibold cursor-pointer lg:flex-1 lg:shrink-0 lg:basis-28 lg:w-auto lg:flex lg:items-center'>
|
||||
<button
|
||||
className={`p-0 mr-2 leading-[0px] inline-block rounded-sm pt-0 px-[1px] pb-[3px] border
|
||||
${selected ? ' bg-blue-700 border-blue-700 text-white' : 'text-transparent'}`}
|
||||
|
||||
11
cron.js
11
cron.js
@@ -1,7 +1,8 @@
|
||||
/* eslint-disable no-new */
|
||||
const Cryptr = require('cryptr');
|
||||
const { promises } = require('fs');
|
||||
const { readFile } = require('fs');
|
||||
const Cron = require('croner');
|
||||
const { Cron } = require('croner');
|
||||
require('dotenv').config({ path: './.env.local' });
|
||||
|
||||
const getAppSettings = async () => {
|
||||
@@ -71,7 +72,7 @@ const runAppCronJobs = () => {
|
||||
const scrape_interval = settings.scrape_interval || 'daily';
|
||||
if (scrape_interval !== 'never') {
|
||||
const scrapeCronTime = generateCronTime(scrape_interval);
|
||||
Cron(scrapeCronTime, () => {
|
||||
new Cron(scrapeCronTime, () => {
|
||||
// console.log('### Running Keyword Position Cron Job!');
|
||||
const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
|
||||
fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/cron`, fetchOpts)
|
||||
@@ -89,7 +90,7 @@ const runAppCronJobs = () => {
|
||||
if (notif_interval) {
|
||||
const cronTime = generateCronTime(notif_interval === 'daily' ? 'daily_morning' : notif_interval);
|
||||
if (cronTime) {
|
||||
Cron(cronTime, () => {
|
||||
new Cron(cronTime, () => {
|
||||
// console.log('### Sending Notification Email...');
|
||||
const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
|
||||
fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/notify`, fetchOpts)
|
||||
@@ -106,7 +107,7 @@ const runAppCronJobs = () => {
|
||||
|
||||
// Run Failed scraping CRON (Every Hour)
|
||||
const failedCronTime = generateCronTime('hourly');
|
||||
Cron(failedCronTime, () => {
|
||||
new Cron(failedCronTime, () => {
|
||||
// console.log('### Retrying Failed Scrapes...');
|
||||
|
||||
readFile(`${process.cwd()}/data/failed_queue.json`, { encoding: 'utf-8' }, (err, data) => {
|
||||
@@ -135,7 +136,7 @@ const runAppCronJobs = () => {
|
||||
// Run Google Search Console Scraper Daily
|
||||
if (process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL) {
|
||||
const searchConsoleCRONTime = generateCronTime('daily');
|
||||
Cron(searchConsoleCRONTime, () => {
|
||||
new Cron(searchConsoleCRONTime, () => {
|
||||
const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
|
||||
fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/searchconsole`, fetchOpts)
|
||||
.then((res) => res.json())
|
||||
|
||||
2904
package-lock.json
generated
2904
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
26
package.json
26
package.json
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "serpbear",
|
||||
"version": "2.0.4",
|
||||
"version": "2.0.7",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
@@ -18,16 +18,16 @@
|
||||
"release": "standard-version"
|
||||
},
|
||||
"dependencies": {
|
||||
"@googleapis/searchconsole": "^1.0.0",
|
||||
"@googleapis/searchconsole": "^1.0.5",
|
||||
"@isaacs/ttlcache": "^1.4.1",
|
||||
"@types/react-transition-group": "^4.4.5",
|
||||
"axios": "^1.1.3",
|
||||
"axios": "^1.7.7",
|
||||
"axios-retry": "^3.3.1",
|
||||
"chart.js": "^3.9.1",
|
||||
"cheerio": "^1.0.0-rc.12",
|
||||
"cheerio": "^1.0.0",
|
||||
"concurrently": "^7.6.0",
|
||||
"cookies": "^0.8.0",
|
||||
"croner": "^5.3.5",
|
||||
"croner": "^9.0.0",
|
||||
"cryptr": "^6.0.3",
|
||||
"dayjs": "^1.11.5",
|
||||
"dotenv": "^16.0.3",
|
||||
@@ -35,7 +35,6 @@
|
||||
"https-proxy-agent": "^5.0.1",
|
||||
"isomorphic-fetch": "^3.0.0",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"msw": "^0.49.0",
|
||||
"next": "^12.3.4",
|
||||
"nodemailer": "^6.9.9",
|
||||
"react": "18.2.0",
|
||||
@@ -50,11 +49,11 @@
|
||||
"reflect-metadata": "^0.1.13",
|
||||
"sequelize": "^6.34.0",
|
||||
"sequelize-typescript": "^2.1.6",
|
||||
"sqlite3": "^5.1.6",
|
||||
"umzug": "^3.6.1"
|
||||
"sqlite3": "^5.1.7",
|
||||
"umzug": "^3.8.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@testing-library/jest-dom": "^6.1.4",
|
||||
"@testing-library/jest-dom": "^6.6.3",
|
||||
"@testing-library/react": "^14.0.0",
|
||||
"@types/cookies": "^0.7.7",
|
||||
"@types/cryptr": "^4.0.1",
|
||||
@@ -74,15 +73,16 @@
|
||||
"jest": "^29.7.0",
|
||||
"jest-environment-jsdom": "^29.7.0",
|
||||
"jest-fetch-mock": "^3.0.3",
|
||||
"msw": "^2.6.4",
|
||||
"next-router-mock": "^0.9.10",
|
||||
"postcss": "^8.4.31",
|
||||
"postcss": "^8.4.49",
|
||||
"prettier": "^2.7.1",
|
||||
"resize-observer-polyfill": "^1.5.1",
|
||||
"sass": "^1.55.0",
|
||||
"sass": "^1.80.7",
|
||||
"sequelize-cli": "^6.6.2",
|
||||
"standard-version": "^9.5.0",
|
||||
"stylelint-config-standard": "^29.0.0",
|
||||
"tailwindcss": "^3.1.8",
|
||||
"typescript": "4.8.4"
|
||||
"tailwindcss": "^3.4.14",
|
||||
"typescript": "^4.8.4"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,14 +36,13 @@ const updatekeywordVolume = async (req: NextApiRequest, res: NextApiResponse<Key
|
||||
keywordsToSend = parseKeywords(allKeywords.map((e) => e.get({ plain: true })));
|
||||
}
|
||||
if (domain) {
|
||||
// const allDomain = domain === 'all';
|
||||
// const allKeywords:Keyword[] = allDomain ? await Keyword.findAll() : await Keyword.findAll(allDomain ? {} : { where: { domain } });
|
||||
// keywordsToSend = parseKeywords(allKeywords.map((e) => e.get({ plain: true })));
|
||||
const allDomain = domain === 'all';
|
||||
const allKeywords:Keyword[] = allDomain ? await Keyword.findAll() : await Keyword.findAll(allDomain ? {} : { where: { domain } });
|
||||
keywordsToSend = parseKeywords(allKeywords.map((e) => e.get({ plain: true })));
|
||||
}
|
||||
|
||||
if (keywordsToSend.length > 0) {
|
||||
const keywordsVolumeData = await getKeywordsVolume(keywordsToSend);
|
||||
// console.log('keywordsVolumeData :', keywordsVolumeData);
|
||||
if (keywordsVolumeData.error) {
|
||||
return res.status(400).json({ keywords: [], error: keywordsVolumeData.error });
|
||||
}
|
||||
|
||||
@@ -20,11 +20,10 @@ const hasdata:ScraperSettings = {
|
||||
scrapeURL: (keyword, settings) => {
|
||||
const country = keyword.country || 'US';
|
||||
const countryName = countries[country][0];
|
||||
const location = keyword.city && countryName ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
|
||||
return `https://api.scrape-it.cloud/scrape/google/serp?q=${encodeURI(keyword.keyword)}${location}&num=100&gl=${country.toLowerCase()}&deviceType=${keyword.device}`;
|
||||
const location = keyword.city && countryName ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
|
||||
return `https://api.scrape-it.cloud/scrape/google/serp?q=${encodeURIComponent(keyword.keyword)}${location}&num=100&gl=${country.toLowerCase()}&deviceType=${keyword.device}`;
|
||||
},
|
||||
resultObjectKey: 'organicResults',
|
||||
|
||||
serpExtractor: (content) => {
|
||||
const extractedResult = [];
|
||||
const results: HasDataResult[] = (typeof content === 'string') ? JSON.parse(content) : content as HasDataResult[];
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import cheerio from 'cheerio';
|
||||
import * as cheerio from 'cheerio';
|
||||
|
||||
const proxy:ScraperSettings = {
|
||||
id: 'proxy',
|
||||
@@ -16,6 +16,13 @@ const proxy:ScraperSettings = {
|
||||
|
||||
const $ = cheerio.load(content);
|
||||
let lastPosition = 0;
|
||||
const hasValidContent = $('body').find('#main');
|
||||
if (hasValidContent.length === 0) {
|
||||
const msg = '[ERROR] Scraped search results from proxy do not adhere to expected format. Unable to parse results';
|
||||
console.log(msg);
|
||||
throw new Error(msg);
|
||||
}
|
||||
|
||||
const mainContent = $('body').find('#main');
|
||||
const children = $(mainContent).find('h3');
|
||||
|
||||
|
||||
@@ -20,8 +20,8 @@ const searchapi:ScraperSettings = {
|
||||
scrapeURL: (keyword) => {
|
||||
const country = keyword.country || 'US';
|
||||
const countryName = countries[country][0];
|
||||
const location = keyword.city && countryName ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
|
||||
return `https://www.searchapi.io/api/v1/search?engine=google&q=${encodeURI(keyword.keyword)}&num=100&gl=${country}&device=${keyword.device}${location}`;
|
||||
const location = keyword.city && countryName ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
|
||||
return `https://www.searchapi.io/api/v1/search?engine=google&q=${encodeURIComponent(keyword.keyword)}&num=100&gl=${country}&device=${keyword.device}${location}`;
|
||||
},
|
||||
resultObjectKey: 'organic_results',
|
||||
serpExtractor: (content) => {
|
||||
|
||||
@@ -19,8 +19,8 @@ const serpapi:ScraperSettings = {
|
||||
},
|
||||
scrapeURL: (keyword, settings) => {
|
||||
const countryName = countries[keyword.country || 'US'][0];
|
||||
const location = keyword.city && keyword.country ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
|
||||
return `https://serpapi.com/search?q=${encodeURI(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}${location}&api_key=${settings.scaping_api}`;
|
||||
const location = keyword.city && keyword.country ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
|
||||
return `https://serpapi.com/search?q=${encodeURIComponent(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}${location}&api_key=${settings.scaping_api}`;
|
||||
},
|
||||
resultObjectKey: 'organic_results',
|
||||
serpExtractor: (content) => {
|
||||
|
||||
@@ -12,7 +12,8 @@ const serper:ScraperSettings = {
|
||||
scrapeURL: (keyword, settings, countryData) => {
|
||||
const country = keyword.country || 'US';
|
||||
const lang = countryData[country][2];
|
||||
return `https://google.serper.dev/search?q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`;
|
||||
console.log('Serper URL :', `https://google.serper.dev/search?q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`);
|
||||
return `https://google.serper.dev/search?q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`;
|
||||
},
|
||||
resultObjectKey: 'organic',
|
||||
serpExtractor: (content) => {
|
||||
|
||||
@@ -20,7 +20,7 @@ const serply:ScraperSettings = {
|
||||
},
|
||||
scrapeURL: (keyword) => {
|
||||
const country = scraperCountries.includes(keyword.country.toUpperCase()) ? keyword.country : 'US';
|
||||
return `https://api.serply.io/v1/search/q=${encodeURI(keyword.keyword)}&num=100&hl=${country}`;
|
||||
return `https://api.serply.io/v1/search/q=${encodeURIComponent(keyword.keyword)}&num=100&hl=${country}`;
|
||||
},
|
||||
resultObjectKey: 'result',
|
||||
serpExtractor: (content) => {
|
||||
|
||||
@@ -15,10 +15,10 @@ const spaceSerp:ScraperSettings = {
|
||||
scrapeURL: (keyword, settings, countryData) => {
|
||||
const country = keyword.country || 'US';
|
||||
const countryName = countries[country][0];
|
||||
const location = keyword.city ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
|
||||
const location = keyword.city ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
|
||||
const device = keyword.device === 'mobile' ? '&device=mobile' : '';
|
||||
const lang = countryData[country][2];
|
||||
return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${location}${device}&resultBlocks=`;
|
||||
return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${location}${device}&resultBlocks=`;
|
||||
},
|
||||
resultObjectKey: 'organic_results',
|
||||
serpExtractor: (content) => {
|
||||
|
||||
@@ -15,11 +15,11 @@ const valueSerp:ScraperSettings = {
|
||||
scrapeURL: (keyword, settings, countryData) => {
|
||||
const country = keyword.country || 'US';
|
||||
const countryName = countries[country][0];
|
||||
const location = keyword.city ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
|
||||
const location = keyword.city ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
|
||||
const device = keyword.device === 'mobile' ? '&device=mobile' : '';
|
||||
const lang = countryData[country][2];
|
||||
console.log(`https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`);
|
||||
return `https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`;
|
||||
console.log(`https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`);
|
||||
return `https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`;
|
||||
},
|
||||
resultObjectKey: 'organic_results',
|
||||
serpExtractor: (content) => {
|
||||
|
||||
@@ -163,7 +163,7 @@ export const getAdwordsKeywordIdeas = async (credentials:AdwordsCredentials, adw
|
||||
}
|
||||
|
||||
try {
|
||||
// API: https://developers.google.com/google-ads/api/rest/reference/rest/v16/customers/generateKeywordIdeas
|
||||
// API: https://developers.google.com/google-ads/api/rest/reference/rest/v18/customers/generateKeywordIdeas
|
||||
const customerID = account_id.replaceAll('-', '');
|
||||
const geoTargetConstants = countries[country][3]; // '2840';
|
||||
const reqPayload: Record<string, any> = {
|
||||
@@ -178,7 +178,7 @@ export const getAdwordsKeywordIdeas = async (credentials:AdwordsCredentials, adw
|
||||
reqPayload.siteSeed = { site: domain };
|
||||
}
|
||||
|
||||
const resp = await fetch(`https://googleads.googleapis.com/v16/customers/${customerID}:generateKeywordIdeas`, {
|
||||
const resp = await fetch(`https://googleads.googleapis.com/v18/customers/${customerID}:generateKeywordIdeas`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
@@ -297,7 +297,7 @@ export const getKeywordsVolume = async (keywords: KeywordType[]): Promise<{error
|
||||
for (const country in keywordRequests) {
|
||||
if (Object.hasOwn(keywordRequests, country) && keywordRequests[country].length > 0) {
|
||||
try {
|
||||
// API: https://developers.google.com/google-ads/api/rest/reference/rest/v16/customers/generateKeywordHistoricalMetrics
|
||||
// API: https://developers.google.com/google-ads/api/rest/reference/rest/v18/customers/generateKeywordHistoricalMetrics
|
||||
const customerID = account_id.replaceAll('-', '');
|
||||
const geoTargetConstants = countries[country][3]; // '2840';
|
||||
const reqKeywords = keywordRequests[country].map((kw) => kw.keyword);
|
||||
@@ -306,7 +306,7 @@ export const getKeywordsVolume = async (keywords: KeywordType[]): Promise<{error
|
||||
geoTargetConstants: `geoTargetConstants/${geoTargetConstants}`,
|
||||
// language: `languageConstants/${language}`,
|
||||
};
|
||||
const resp = await fetch(`https://googleads.googleapis.com/v16/customers/${customerID}:generateKeywordHistoricalMetrics`, {
|
||||
const resp = await fetch(`https://googleads.googleapis.com/v18/customers/${customerID}:generateKeywordHistoricalMetrics`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
|
||||
@@ -4,10 +4,10 @@ import path from 'path';
|
||||
import { getKeywordsInsight, getPagesInsight } from './insight';
|
||||
import { readLocalSCData } from './searchConsole';
|
||||
|
||||
const serpBearLogo = 'https://erevanto.sirv.com/Images/serpbear/ikAdjQq.png';
|
||||
const mobileIcon = 'https://erevanto.sirv.com/Images/serpbear/SqXD9rd.png';
|
||||
const desktopIcon = 'https://erevanto.sirv.com/Images/serpbear/Dx3u0XD.png';
|
||||
const googleIcon = 'https://erevanto.sirv.com/Images/serpbear/Sx3u0X9.png';
|
||||
const serpBearLogo = 'https://serpbear.b-cdn.net/ikAdjQq.png';
|
||||
const mobileIcon = 'https://serpbear.b-cdn.net/SqXD9rd.png';
|
||||
const desktopIcon = 'https://serpbear.b-cdn.net/Dx3u0XD.png';
|
||||
const googleIcon = 'https://serpbear.b-cdn.net/Sx3u0X9.png';
|
||||
|
||||
type SCStatsObject = {
|
||||
[key:string]: {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import axios, { AxiosResponse, CreateAxiosDefaults } from 'axios';
|
||||
import cheerio from 'cheerio';
|
||||
import * as cheerio from 'cheerio';
|
||||
import { readFile, writeFile } from 'fs/promises';
|
||||
import HttpsProxyAgent from 'https-proxy-agent';
|
||||
import countries from './countries';
|
||||
@@ -124,14 +124,18 @@ export const scrapeKeywordFromGoogle = async (keyword:KeywordType, settings:Sett
|
||||
throw new Error(res);
|
||||
}
|
||||
} catch (error:any) {
|
||||
refreshedResults.error = scraperError;
|
||||
refreshedResults.error = scraperError || 'Unknown Error';
|
||||
if (settings.scraper_type === 'proxy' && error && error.response && error.response.statusText) {
|
||||
refreshedResults.error = `[${error.response.status}] ${error.response.statusText}`;
|
||||
} else if (settings.scraper_type === 'proxy' && error) {
|
||||
refreshedResults.error = error;
|
||||
}
|
||||
|
||||
console.log('[ERROR] Scraping Keyword : ', keyword.keyword, '. Error: ', error && error.response && error.response.statusText);
|
||||
console.log('[ERROR] Scraping Keyword : ', keyword.keyword);
|
||||
if (!(error && error.response && error.response.statusText)) {
|
||||
console.log('[ERROR_MESSAGE]: ', error);
|
||||
} else {
|
||||
console.log('[ERROR_MESSAGE]: ', error && error.response && error.response.statusText);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,9 +152,17 @@ export const extractScrapedResult = (content: string, device: string): SearchRes
|
||||
const extractedResult = [];
|
||||
|
||||
const $ = cheerio.load(content);
|
||||
const hasValidContent = [...$('body').find('#search'), ...$('body').find('#rso')];
|
||||
if (hasValidContent.length === 0) {
|
||||
const msg = '[ERROR] Scraped search results do not adhere to expected format. Unable to parse results';
|
||||
console.log(msg);
|
||||
throw new Error(msg);
|
||||
}
|
||||
|
||||
const hasNumberofResult = $('body').find('#search > div > div');
|
||||
const searchResultItems = hasNumberofResult.find('h3');
|
||||
let lastPosition = 0;
|
||||
console.log('Scraped search results contain ', searchResultItems.length, ' desktop results.');
|
||||
|
||||
for (let i = 0; i < searchResultItems.length; i += 1) {
|
||||
if (searchResultItems[i]) {
|
||||
@@ -161,11 +173,12 @@ export const extractScrapedResult = (content: string, device: string): SearchRes
|
||||
extractedResult.push({ title, url, position: lastPosition });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Mobile Scraper
|
||||
if (extractedResult.length === 0 && device === 'mobile') {
|
||||
// Mobile Scraper
|
||||
if (extractedResult.length === 0 && device === 'mobile') {
|
||||
const items = $('body').find('#rso > div');
|
||||
console.log('Scraped search results contain ', items.length, ' mobile results.');
|
||||
for (let i = 0; i < items.length; i += 1) {
|
||||
const item = $(items[i]);
|
||||
const linkDom = item.find('a[role="presentation"]');
|
||||
@@ -181,7 +194,7 @@ export const extractScrapedResult = (content: string, device: string): SearchRes
|
||||
}
|
||||
}
|
||||
|
||||
return extractedResult;
|
||||
return extractedResult;
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user