feat: Adds ability to disable/clear retry queue for failed keywords

components/settings/Settings.tsx

@@ -1,7 +1,7 @@
 import React, { useEffect, useState } from 'react';
 import { Toaster } from 'react-hot-toast';
 // import { useQuery } from 'react-query';
-import useUpdateSettings, { useFetchSettings } from '../../services/settings';
+import useUpdateSettings, { useClearFailedQueue, useFetchSettings } from '../../services/settings';
 import Icon from '../common/Icon';
 import SelectField, { SelectionOption } from '../common/SelectField';
 
@@ -18,6 +18,7 @@ type SettingsError = {
 const defaultSettings = {
    scraper_type: 'none',
    scrape_delay: 'none',
+   scrape_retry: false,
    notification_interval: 'daily',
    notification_email: '',
    smtp_server: '',
@@ -33,6 +34,7 @@ const Settings = ({ closeSettings }:SettingsProps) => {
    const [settingsError, setSettingsError] = useState<SettingsError|null>(null);
    const { mutate: updateMutate, isLoading: isUpdating } = useUpdateSettings(() => console.log(''));
    const { data: appSettings, isLoading } = useFetchSettings();
+   const { mutate: clearFailedMutate, isLoading: clearingQueue } = useClearFailedQueue(() => {});
 
    useEffect(() => {
       if (appSettings && appSettings.settings) {
@@ -59,7 +61,7 @@ const Settings = ({ closeSettings }:SettingsProps) => {
       if (e.target === e.currentTarget) { closeSettings(); }
    };
 
-   const updateSettings = (key: string, value:string|number) => {
+   const updateSettings = (key: string, value:string|number|boolean) => {
       setSettings({ ...settings, [key]: value });
    };
 
@@ -220,6 +222,37 @@ const Settings = ({ closeSettings }:SettingsProps) => {
                               />
                               <small className=' text-gray-500 pt-2 block'>This option requires Server/Docker Instance Restart to take Effect.</small>
                            </div>
+                           <div className="settings__section__input mb-5">
+                              <label className="relative inline-flex items-center cursor-pointer">
+                                 <span className="text-sm font-medium text-gray-900 dark:text-gray-300 w-56">Auto Retry Failed Keyword Scrape</span>
+                                 <input
+                                 type="checkbox"
+                                 value={settings?.scrape_retry ? 'true' : '' }
+                                 checked={settings.scrape_retry || false}
+                                 className="sr-only peer"
+                                 onChange={() => updateSettings('scrape_retry', !settings.scrape_retry)}
+                                 />
+                                 <div className="relative rounded-3xl w-11 h-6 bg-gray-200 peer-focus:outline-none peer-focus:ring-4
+                                 peer-focus:ring-blue-300 dark:peer-focus:ring-blue-800rounded-full peer dark:bg-gray-700
+                                 peer-checked:after:translate-x-full peer-checked:after:border-white after:content-['']
+                                 after:absolute after:top-[2px] after:left-[2px] after:bg-white  after:border-gray-300
+                                 after:border after:rounded-full after:h-5 after:w-5
+                                 after:transition-all dark:border-gray-600 peer-checked:bg-blue-600"></div>
+
+                              </label>
+                           </div>
+                           {settings?.scrape_retry && (settings.failed_queue?.length || 0) > 0 && (
+                              <div className="settings__section__input mb-5">
+                                 <label className={labelStyle}>Clear Failed Retry Queue</label>
+                                 <button
+                                 onClick={() => clearFailedMutate()}
+                                 className=' py-3 px-5 w-full rounded cursor-pointer bg-gray-100 text-gray-800
+                                 font-semibold text-sm hover:bg-gray-200'>
+                                    {clearingQueue && <Icon type="loading" size={14} />} Clear Failed Queue
+                                      ({settings.failed_queue?.length || 0} Keywords)
+                                 </button>
+                              </div>
+                           )}
                      </div>
                   </div>
                )}

pages/api/clearfailed.ts

@@ -0,0 +1,29 @@
+import { writeFile } from 'fs/promises';
+import type { NextApiRequest, NextApiResponse } from 'next';
+import verifyUser from '../../utils/verifyUser';
+
+type SettingsGetResponse = {
+   cleared?: boolean,
+   error?: string,
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+   const authorized = verifyUser(req, res);
+   if (authorized !== 'authorized') {
+      return res.status(401).json({ error: authorized });
+   }
+   if (req.method === 'PUT') {
+      return clearFailedQueue(req, res);
+   }
+   return res.status(502).json({ error: 'Unrecognized Route.' });
+}
+
+const clearFailedQueue = async (req: NextApiRequest, res: NextApiResponse<SettingsGetResponse>) => {
+   try {
+      await writeFile(`${process.cwd()}/data/failed_queue.json`, JSON.stringify([]), { encoding: 'utf-8' });
+      return res.status(200).json({ cleared: true });
+   } catch (error) {
+      console.log('[ERROR] Cleraring Failed Queue File.', error);
+      return res.status(200).json({ error: 'Error Cleraring Failed Queue!' });
+   }
+};

pages/api/settings.ts

@@ -57,6 +57,8 @@ const updateSettings = async (req: NextApiRequest, res: NextApiResponse<Settings
 export const getAppSettings = async () : Promise<SettingsType> => {
    try {
       const settingsRaw = await readFile(`${process.cwd()}/data/settings.json`, { encoding: 'utf-8' });
+      const failedQueueRaw = await readFile(`${process.cwd()}/data/failed_queue.json`, { encoding: 'utf-8' });
+      const failedQueue: string[] = failedQueueRaw ? JSON.parse(failedQueueRaw) : [];
       const settings: SettingsType = settingsRaw ? JSON.parse(settingsRaw) : {};
       let decryptedSettings = settings;
 
@@ -70,6 +72,7 @@ export const getAppSettings = async () : Promise<SettingsType> => {
             smtp_password,
             search_console_integrated: !!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL),
             available_scapers: allScrapers.map((scraper) => ({ label: scraper.name, value: scraper.id })),
+            failed_queue: failedQueue,
          };
       } catch (error) {
          console.log('Error Decrypting Settings API Keys!');
@@ -87,6 +90,7 @@ export const getAppSettings = async () : Promise<SettingsType> => {
          smtp_port: '',
          smtp_username: '',
          smtp_password: '',
+         scrape_retry: false,
       };
       await writeFile(`${process.cwd()}/data/settings.json`, JSON.stringify(settings), { encoding: 'utf-8' });
       return settings;

services/settings.ts

@@ -38,4 +38,27 @@ const useUpdateSettings = (onSuccess:Function|undefined) => {
    });
 };
 
+export function useClearFailedQueue(onSuccess:Function) {
+   const queryClient = useQueryClient();
+   return useMutation(async () => {
+      const headers = new Headers({ 'Content-Type': 'application/json', Accept: 'application/json' });
+      const fetchOpts = { method: 'PUT', headers };
+      const res = await fetch(`${window.location.origin}/api/clearfailed`, fetchOpts);
+      if (res.status >= 400 && res.status < 600) {
+         throw new Error('Bad response from server');
+      }
+      return res.json();
+   }, {
+      onSuccess: async () => {
+         onSuccess();
+         toast('Failed Queue Cleared', { icon: '✔️' });
+         queryClient.invalidateQueries(['settings']);
+      },
+      onError: () => {
+         console.log('Error Clearing Failed Queue!!!');
+         toast('Error Clearing Failed Queue.', { icon: '⚠️' });
+      },
+   });
+}
+
 export default useUpdateSettings;

types.d.ts

@@ -81,6 +81,8 @@ type SettingsType = {
    available_scapers?: Array,
    scrape_interval?: string,
    scrape_delay?: string,
+   scrape_retry?: boolean,
+   failed_queue?: string[]
    version?: string
 }
 

utils/refresh.ts

@@ -91,7 +91,7 @@ export const updateKeywordPosition = async (keywordRaw:Keyword, udpatedkeyword:
          };
 
          // If failed, Add to Retry Queue Cron
-         if (udpatedkeyword.error) {
+         if (udpatedkeyword.error && settings?.scrape_retry) {
             await retryScrape(keyword.ID);
          } else {
             await removeFromRetryQueue(keyword.ID);