Merge pull request #192 from valka465/main

Integrate HasData to SerpBear

README.md

@@ -22,7 +22,7 @@ SerpBear is an Open Source Search Engine Position Tracking and Keyword Research
 
 #### How it Works
 
-The App uses third party website scrapers like ScrapingAnt, ScrapingRobot, SearchApi, SerpApi or Your given Proxy ips to scrape google search results to see if your domain appears in the search result for the given keyword.
+The App uses third party website scrapers like ScrapingAnt, ScrapingRobot, SearchApi, SerpApi, HasData or Your given Proxy ips to scrape google search results to see if your domain appears in the search result for the given keyword.
 
 The Keyword Research and keyword generation feature works by integrating your Google Ads test accounts into SerpBear. You can also view the added keyword's monthly search volume data once you [integrate Google Ads](https://docs.serpbear.com/miscellaneous/integrate-google-ads).
 
@@ -52,6 +52,7 @@ If you don't want to use proxies, you can use third party Scraping services to s
 | SearchApi.io     | From $40/mo   | From 10,000/mo | Yes |
 | valueserp.com    | Pay As You Go | $2.50/1000 req | No  |
 | serper.dev       | Pay As You Go | $1.00/1000 req | No  |
+| hasdata.com      | From $29/mo   | From 10,000/mo | Yes |
 
 **Tech Stack**
 

scrapers/index.ts

@@ -7,6 +7,7 @@ import proxy from './services/proxy';
 import searchapi from './services/searchapi';
 import valueSerp from './services/valueserp';
 import serper from './services/serper';
+import hasdata from './services/hasdata';
 
 export default [
    scrapingRobot,
@@ -18,4 +19,5 @@ export default [
    searchapi,
    valueSerp,
    serper,
+   hasdata,
 ];

scrapers/services/hasdata.ts

@@ -0,0 +1,45 @@
+import countries from '../../utils/countries';
+
+interface HasDataResult {
+   title: string,
+   link: string,
+   position: number,
+}
+
+const hasdata:ScraperSettings = {
+   id: 'hasdata',
+   name: 'HasData',
+   website: 'hasdata.com',
+   allowsCity: true,
+   headers: (keyword, settings) => {
+      return {
+         'Content-Type': 'application/json',
+         'x-api-key': settings.scaping_api,
+      };
+   },
+   scrapeURL: (keyword, settings) => {
+      const country = keyword.country || 'US';
+      const countryName = countries[country][0];
+      const location = keyword.city && countryName ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+      return `https://api.scrape-it.cloud/scrape/google/serp?q=${encodeURI(keyword.keyword)}${location}&num=100&gl=${country.toLowerCase()}&deviceType=${keyword.device}`;
+   },
+   resultObjectKey: 'organicResults',
+   
+   serpExtractor: (content) => {
+      const extractedResult = [];
+      const results: HasDataResult[] = (typeof content === 'string') ? JSON.parse(content) : content as HasDataResult[];
+
+      for (const { link, title, position } of results) {
+         if (title && link) {
+            extractedResult.push({
+               title,
+               url: link,
+               position,
+            });
+         }
+      }
+      return extractedResult;
+   },
+};
+
+export default hasdata;