From b4ad69baaa0f865938f8b0eace6732a9e6b1b381 Mon Sep 17 00:00:00 2001
From: towfiqi <tislam100@gmail.com>
Date: Fri, 9 Feb 2024 00:43:28 +0600
Subject: [PATCH] feat: Adds Serper.dev integration

closes #138
---
 scrapers/index.ts           |  2 ++
 scrapers/services/serper.ts | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 scrapers/services/serper.ts

diff --git a/scrapers/index.ts b/scrapers/index.ts
index 2825010..a2ea0d9 100644
--- a/scrapers/index.ts
+++ b/scrapers/index.ts
@@ -6,6 +6,7 @@ import spaceserp from './services/spaceserp';
 import proxy from './services/proxy';
 import searchapi from './services/searchapi';
 import valueSerp from './services/valueserp';
+import serper from './services/serper';
 
 export default [
    scrapingRobot,
@@ -16,4 +17,5 @@ export default [
    proxy,
    searchapi,
    valueSerp,
+   serper,
 ];
diff --git a/scrapers/services/serper.ts b/scrapers/services/serper.ts
new file mode 100644
index 0000000..28dd721
--- /dev/null
+++ b/scrapers/services/serper.ts
@@ -0,0 +1,35 @@
+interface SerperResult {
+   title: string,
+   link: string,
+   position: number,
+}
+
+const serper:ScraperSettings = {
+   id: 'serper',
+   name: 'Serper.dev',
+   website: 'serper.dev',
+   allowsCity: true,
+   scrapeURL: (keyword, settings, countryData) => {
+      const country = keyword.country || 'US';
+      const lang = countryData[country][2];
+      return `https://google.serper.dev/search?q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`;
+   },
+   resultObjectKey: 'organic',
+   serpExtractor: (content) => {
+      const extractedResult = [];
+      const results: SerperResult[] = (typeof content === 'string') ? JSON.parse(content) : content as SerperResult[];
+
+      for (const { link, title, position } of results) {
+         if (title && link) {
+            extractedResult.push({
+               title,
+               url: link,
+               position,
+            });
+         }
+      }
+      return extractedResult;
+   },
+};
+
+export default serper;