diff --git a/README.md b/README.md index bf4e860..1a16b38 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ SerpBear is an Open Source Search Engine Position Tracking App. It allows you to - **Zero Cost to RUN:** Run the App on mogenius.com or Fly.io for free. #### How it Works -The App uses third party website scrapers like ScrapingAnt, ScrapingRobot, SerpApi or Your given Proxy ips to scrape google search results to see if your domain appears in the search result for the given keyword. Also, When you connect your Googel Search Console account, the app shows actual search visits for each tracked keywords. You can also discover new keywords, and find the most performing keywords, countries, pages. +The App uses third party website scrapers like ScrapingAnt, ScrapingRobot, SearchApi, SerpApi or Your given Proxy ips to scrape google search results to see if your domain appears in the search result for the given keyword. Also, When you connect your Googel Search Console account, the app shows actual search visits for each tracked keywords. You can also discover new keywords, and find the most performing keywords, countries, pages. #### Getting Started - **Step 1:** Deploy & Run the App. @@ -41,6 +41,7 @@ The App uses third party website scrapers like ScrapingAnt, ScrapingRobot, SerpA | serply.io | $49/mo | 5000/mo | Yes | | serpapi.com | From $50/mo** | From 5,000/mo** | Yes | | spaceserp.com | $59/lifetime | 15,000/mo | Yes | +| SearchApi.io | From $40/mo | From 10,000/mo | Yes | (*) Free upto a limit. If you are using ScrapingAnt you can lookup 10,000 times per month for free. (**) Free up to 100 per month. Paid from 5,000 to 10,000,000+ per month. diff --git a/components/settings/ScraperSettings.tsx b/components/settings/ScraperSettings.tsx index ca44ddf..36a48c9 100644 --- a/components/settings/ScraperSettings.tsx +++ b/components/settings/ScraperSettings.tsx @@ -54,7 +54,7 @@ const ScraperSettings = ({ settings, settingsError, updateSettings }:ScraperSett minWidth={270} /> - {['scrapingant', 'scrapingrobot', 'serply', 'serpapi', 'spaceSerp'].includes(settings.scraper_type) && ( + {['scrapingant', 'scrapingrobot', 'serply', 'serpapi', 'spaceSerp', 'searchapi'].includes(settings.scraper_type) && (
=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@types/jest/node_modules/pretty-format": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", + "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", + "dev": true, + "dependencies": { + "@jest/schemas": "^29.6.3", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@types/jest/node_modules/react-is": { + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz", + "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==", + "dev": true + }, "node_modules/@types/js-levenshtein": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@types/js-levenshtein/-/js-levenshtein-1.1.2.tgz", diff --git a/package.json b/package.json index 2c23398..f43bda8 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "@types/cookies": "^0.7.7", "@types/cryptr": "^4.0.1", "@types/isomorphic-fetch": "^0.0.36", + "@types/jest": "^29.5.8", "@types/jsonwebtoken": "^8.5.9", "@types/node": "18.11.0", "@types/nodemailer": "^6.4.6", diff --git a/scrapers/index.ts b/scrapers/index.ts index d65edb7..fcb4942 100644 --- a/scrapers/index.ts +++ b/scrapers/index.ts @@ -4,6 +4,7 @@ import serpapi from './services/serpapi'; import serply from './services/serply'; import spaceserp from './services/spaceserp'; import proxy from './services/proxy'; +import searchapi from './services/searchapi'; export default [ scrapingRobot, @@ -12,4 +13,5 @@ export default [ serply, spaceserp, proxy, + searchapi, ]; diff --git a/scrapers/services/searchapi.ts b/scrapers/services/searchapi.ts new file mode 100644 index 0000000..2b2f089 --- /dev/null +++ b/scrapers/services/searchapi.ts @@ -0,0 +1,38 @@ +const searchapi:ScraperSettings = { + id: 'searchapi', + name: 'SearchApi.io', + website: 'searchapi.io', + headers: (keyword, settings) => { + return { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${settings.scaping_api}`, + }; + }, + scrapeURL: (keyword, settings) => { + return `https://www.searchapi.io/api/v1/search?engine=google&q=${encodeURI(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}`; + }, + resultObjectKey: 'organic_results', + serpExtractor: (content) => { + const extractedResult = []; + const results: SearchApiResult[] = (typeof content === 'string') ? JSON.parse(content) : content as SearchApiResult[]; + + for (const { link, title, position } of results) { + if (title && link) { + extractedResult.push({ + title, + url: link, + position, + }); + } + } + return extractedResult; + }, +}; + +interface SearchApiResult { + title: string, + link: string, + position: number, +} + +export default searchapi; diff --git a/utils/refresh.ts b/utils/refresh.ts index 31253c4..8de08d5 100644 --- a/utils/refresh.ts +++ b/utils/refresh.ts @@ -17,11 +17,11 @@ const refreshAndUpdateKeywords = async (rawkeyword:Keyword[], settings:SettingsT const start = performance.now(); const updatedKeywords: KeywordType[] = []; - if (['scrapingant', 'serpapi'].includes(settings.scraper_type)) { + if (['scrapingant', 'serpapi', 'searchapi'].includes(settings.scraper_type)) { const refreshedResults = await refreshParallel(keywords, settings); if (refreshedResults.length > 0) { for (const keyword of rawkeyword) { - const refreshedkeywordData = refreshedResults.find((k) => k && k.ID === keyword.id); + const refreshedkeywordData = refreshedResults.find((k) => k && k.ID === keyword.ID); if (refreshedkeywordData) { const updatedkeyword = await updateKeywordPosition(keyword, refreshedkeywordData, settings); updatedKeywords.push(updatedkeyword);