UniqueSoft/serpbear
2
0
Fork 0
mirror of https://github.com/towfiqi/serpbear synced 2025-06-26 18:15:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: Add option to Delay Between scrapes.

Browse Source 
fixes #87
This commit is contained in:
towfiqi 2023-03-29 20:54:30 +06:00
parent d9505158c4
commit 0a83924ffe
3 changed files with 33 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
components/settings/Settings.tsx
View File

@ -17,6 +17,7 @@ type SettingsError = {
const defaultSettings = {
scraper_type: 'none',
scrape_delay: 'none',
notification_interval: 'daily',
notification_email: '',
smtp_server: '',
@ -105,6 +106,18 @@ const Settings = ({ closeSettings }:SettingsProps) => {
{ label: 'Monthly', value: 'monthly' },
{ label: 'Never', value: 'never' },
];
const delayOptions: SelectionOption[] = [
{ label: 'No Delay', value: '0' },
{ label: '5 Seconds', value: '5000' },
{ label: '10 Seconds', value: '10000' },
{ label: '30 Seconds', value: '30000' },
{ label: '1 Minutes', value: '60000' },
{ label: '2 Minutes', value: '120000' },
{ label: '5 Minutes', value: '300000' },
{ label: '10 Minutes', value: '600000' },
{ label: '15 Minutes', value: '900000' },
{ label: '30 Minutes', value: '1800000' },
];
const allScrapers: SelectionOption[] = settings.available_scapers ? settings.available_scapers : [];
const scraperOptions: SelectionOption[] = [{ label: 'None', value: 'none' }, ...allScrapers];
@ -193,6 +206,20 @@ const Settings = ({ closeSettings }:SettingsProps) => {
<small className=' text-gray-500 pt-2 block'>This option requires Server/Docker Instance Restart to take Effect.</small>
</div>
)}
<div className="settings__section__input mb-5">
<label className={labelStyle}>Delay Between Each keyword Scrape</label>
<SelectField
multiple={false}
selected={[settings?.scrape_delay || '0']}
options={delayOptions}
defaultLabel={'Delay Settings'}
updateField={(updated:string[]) => updated[0] && updateSettings('scrape_delay', updated[0])}
rounded='rounded'
maxHeight={48}
minWidth={270}
/>
<small className=' text-gray-500 pt-2 block'>This option requires Server/Docker Instance Restart to take Effect.</small>
</div>
</div>
</div>
)}

3
types.d.ts vendored
View File

@ -79,7 +79,8 @@ type SettingsType = {
smtp_password?: string,
search_console_integrated?: boolean,
available_scapers?: Array,
scrape_interval?: string
scrape_interval?: string,
scrape_delay?: string,
version?: string
}

4
utils/refresh.ts
View File

@ -1,4 +1,5 @@
import { performance } from 'perf_hooks';
import { setTimeout as sleep } from 'timers/promises';
import { RefreshResult, scrapeKeywordFromGoogle } from './scraper';
/**
@ -21,6 +22,9 @@ const refreshKeywords = async (keywords:KeywordType[], settings:SettingsType): P
console.log('START SCRAPE: ', keyword.keyword);
const refreshedkeywordData = await scrapeKeywordFromGoogle(keyword, settings);
refreshedResults.push(refreshedkeywordData);
if (keywords.length > 0 && settings.scrape_delay && settings.scrape_delay !== '0') {
await sleep(parseInt(settings.scrape_delay, 10));
}
}
}