From 3b6d034d6f7da0b4259070220fffff44184dd680 Mon Sep 17 00:00:00 2001
From: towfiqi <tislam100@gmail.com>
Date: Sun, 5 Mar 2023 12:28:21 +0600
Subject: [PATCH] feat: Adds Keyword Scraping Interval Settings.

fixes #81, #76
---
 components/settings/Settings.tsx | 23 ++++++++++
 cron.js                          | 74 +++++++++++++++++---------------
 types.d.ts                       |  3 +-
 3 files changed, 65 insertions(+), 35 deletions(-)

diff --git a/components/settings/Settings.tsx b/components/settings/Settings.tsx
index 57bcece..2bf11b9 100644
--- a/components/settings/Settings.tsx
+++ b/components/settings/Settings.tsx
@@ -97,6 +97,13 @@ const Settings = ({ closeSettings }:SettingsProps) => {
       { label: 'Monthly', value: 'monthly' },
       { label: 'Never', value: 'never' },
    ];
+   const scrapingOptions: SelectionOption[] = [
+      { label: 'Daily', value: 'daily' },
+      { label: 'Every Other Day', value: 'other_day' },
+      { label: 'Weekly', value: 'weekly' },
+      { label: 'Monthly', value: 'monthly' },
+      { label: 'Never', value: 'never' },
+   ];
    const allScrapers: SelectionOption[] = settings.available_scapers ? settings.available_scapers : [];
    const scraperOptions: SelectionOption[] = [{ label: 'None', value: 'none' }, ...allScrapers];
 
@@ -169,6 +176,22 @@ const Settings = ({ closeSettings }:SettingsProps) => {
                               />
                            </div>
                         )}
+                        {settings.scraper_type !== 'none' && (
+                           <div className="settings__section__input mb-5">
+                              <label className={labelStyle}>Scraping Frequency</label>
+                              <SelectField
+                                 multiple={false}
+                                 selected={[settings?.scrape_interval || 'daily']}
+                                 options={scrapingOptions}
+                                 defaultLabel={'Notification Settings'}
+                                 updateField={(updated:string[]) => updated[0] && updateSettings('scrape_interval', updated[0])}
+                                 rounded='rounded'
+                                 maxHeight={48}
+                                 minWidth={270}
+                              />
+                              <small className=' text-gray-500 pt-2 block'>This option requires Server/Docker Instance Restart to take Effect.</small>
+                           </div>
+                        )}
                      </div>
                   </div>
                )}
diff --git a/cron.js b/cron.js
index 833659e..335dfe1 100644
--- a/cron.js
+++ b/cron.js
@@ -49,6 +49,9 @@ const generateCronTime = (interval) => {
    if (interval === 'daily') {
       cronTime = '0 0 0 * * *';
    }
+   if (interval === 'other_day') {
+      cronTime = '0 0 2-30/2 * *';
+   }
    if (interval === 'daily_morning') {
       cronTime = '0 0 3 * * *';
    }
@@ -63,19 +66,43 @@ const generateCronTime = (interval) => {
 };
 
 const runAppCronJobs = () => {
-   // RUN SERP Scraping CRON (EveryDay at Midnight) 0 0 0 * *
-   const scrapeCronTime = generateCronTime('daily');
-   Cron(scrapeCronTime, () => {
-      // console.log('### Running Keyword Position Cron Job!');
-      const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
-      fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/cron`, fetchOpts)
-      .then((res) => res.json())
-      // .then((data) =>{ console.log(data)})
-      .catch((err) => {
-         console.log('ERROR Making Daily Scraper Cron Request..');
-         console.log(err);
-      });
-   }, { scheduled: true });
+   getAppSettings().then((settings) => {
+      // RUN SERP Scraping CRON (EveryDay at Midnight) 0 0 0 * *
+      const scrape_interval = settings.scrape_interval || 'daily';
+      if (scrape_interval !== 'never') {
+         const scrapeCronTime = generateCronTime(scrape_interval);
+         Cron(scrapeCronTime, () => {
+            // console.log('### Running Keyword Position Cron Job!');
+            const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
+            fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/cron`, fetchOpts)
+            .then((res) => res.json())
+            // .then((data) =>{ console.log(data)})
+            .catch((err) => {
+               console.log('ERROR Making SERP Scraper Cron Request..');
+               console.log(err);
+            });
+         }, { scheduled: true });
+      }
+
+      // RUN Email Notification CRON
+      const notif_interval = (!settings.notification_interval || settings.notification_interval === 'never') ? false : settings.notification_interval;
+      if (notif_interval) {
+         const cronTime = generateCronTime(notif_interval === 'daily' ? 'daily_morning' : notif_interval);
+         if (cronTime) {
+            Cron(cronTime, () => {
+               // console.log('### Sending Notification Email...');
+               const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
+               fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/notify`, fetchOpts)
+               .then((res) => res.json())
+               .then((data) => console.log(data))
+               .catch((err) => {
+                  console.log('ERROR Making Cron Email Notification Request..');
+                  console.log(err);
+               });
+            }, { scheduled: true });
+         }
+      }
+   });
 
    // Run Failed scraping CRON (Every Hour)
    const failedCronTime = generateCronTime('hourly');
@@ -115,27 +142,6 @@ const runAppCronJobs = () => {
          });
       }, { scheduled: true });
    }
-
-   // RUN Email Notification CRON
-   getAppSettings().then((settings) => {
-      const notif_interval = (!settings.notification_interval || settings.notification_interval === 'never') ? false : settings.notification_interval;
-      if (notif_interval) {
-         const cronTime = generateCronTime(notif_interval === 'daily' ? 'daily_morning' : notif_interval);
-         if (cronTime) {
-            Cron(cronTime, () => {
-               // console.log('### Sending Notification Email...');
-               const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
-               fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/notify`, fetchOpts)
-               .then((res) => res.json())
-               .then((data) => console.log(data))
-               .catch((err) => {
-                  console.log('ERROR Making Cron Email Notification Request..');
-                  console.log(err);
-               });
-            }, { scheduled: true });
-         }
-      }
-   });
 };
 
 runAppCronJobs();
diff --git a/types.d.ts b/types.d.ts
index d4161f0..db54a7c 100644
--- a/types.d.ts
+++ b/types.d.ts
@@ -78,7 +78,8 @@ type SettingsType = {
    smtp_username?: string,
    smtp_password?: string,
    search_console_integrated?: boolean,
-   available_scapers?: Array
+   available_scapers?: Array,
+   scrape_interval?: string
 }
 
 type KeywordSCDataChild = {