diff --git a/README.md b/README.md index f8dddbe..e23ec5c 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ SerpBear is an Open Source Search Engine Position Tracking and Keyword Research #### How it Works -The App uses third party website scrapers like ScrapingAnt, ScrapingRobot, SearchApi, SerpApi or Your given Proxy ips to scrape google search results to see if your domain appears in the search result for the given keyword. +The App uses third party website scrapers like ScrapingAnt, ScrapingRobot, SearchApi, SerpApi, HasData or Your given Proxy ips to scrape google search results to see if your domain appears in the search result for the given keyword. The Keyword Research and keyword generation feature works by integrating your Google Ads test accounts into SerpBear. You can also view the added keyword's monthly search volume data once you [integrate Google Ads](https://docs.serpbear.com/miscellaneous/integrate-google-ads). @@ -52,6 +52,7 @@ If you don't want to use proxies, you can use third party Scraping services to s | SearchApi.io | From $40/mo | From 10,000/mo | Yes | | valueserp.com | Pay As You Go | $2.50/1000 req | No | | serper.dev | Pay As You Go | $1.00/1000 req | No | +| hasdata.com | From $29/mo | From 10,000/mo | Yes | **Tech Stack** diff --git a/scrapers/index.ts b/scrapers/index.ts index a2ea0d9..0cfb09c 100644 --- a/scrapers/index.ts +++ b/scrapers/index.ts @@ -7,6 +7,7 @@ import proxy from './services/proxy'; import searchapi from './services/searchapi'; import valueSerp from './services/valueserp'; import serper from './services/serper'; +import hasdata from './services/hasdata'; export default [ scrapingRobot, @@ -18,4 +19,5 @@ export default [ searchapi, valueSerp, serper, + hasdata, ]; diff --git a/scrapers/services/hasdata.ts b/scrapers/services/hasdata.ts new file mode 100644 index 0000000..353e718 --- /dev/null +++ b/scrapers/services/hasdata.ts @@ -0,0 +1,45 @@ +import countries from '../../utils/countries'; + +interface HasDataResult { + title: string, + link: string, + position: number, +} + +const hasdata:ScraperSettings = { + id: 'hasdata', + name: 'HasData', + website: 'hasdata.com', + allowsCity: true, + headers: (keyword, settings) => { + return { + 'Content-Type': 'application/json', + 'x-api-key': settings.scaping_api, + }; + }, + scrapeURL: (keyword, settings) => { + const country = keyword.country || 'US'; + const countryName = countries[country][0]; + const location = keyword.city && countryName ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : ''; + return `https://api.scrape-it.cloud/scrape/google/serp?q=${encodeURI(keyword.keyword)}${location}&num=100&gl=${country.toLowerCase()}&deviceType=${keyword.device}`; + }, + resultObjectKey: 'organicResults', + + serpExtractor: (content) => { + const extractedResult = []; + const results: HasDataResult[] = (typeof content === 'string') ? JSON.parse(content) : content as HasDataResult[]; + + for (const { link, title, position } of results) { + if (title && link) { + extractedResult.push({ + title, + url: link, + position, + }); + } + } + return extractedResult; + }, +}; + +export default hasdata;