UniqueSoft/serpbear
2
0
Fork 0
mirror of https://github.com/towfiqi/serpbear synced 2025-06-26 18:15:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: Adds Keyword Scraping Interval Settings.

Browse Source 
fixes #81, #76
This commit is contained in:
towfiqi
2023-03-05 12:28:21 +06:00
parent 5dd366b91e
commit 3b6d034d6f
3 changed files with 65 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
components/settings/Settings.tsx
View File

@@ -97,6 +97,13 @@ const Settings = ({ closeSettings }:SettingsProps) => {
{ label: 'Monthly', value: 'monthly' },
{ label: 'Never', value: 'never' },
];
const scrapingOptions: SelectionOption[] = [
{ label: 'Daily', value: 'daily' },
{ label: 'Every Other Day', value: 'other_day' },
{ label: 'Weekly', value: 'weekly' },
{ label: 'Monthly', value: 'monthly' },
{ label: 'Never', value: 'never' },
];
const allScrapers: SelectionOption[] = settings.available_scapers ? settings.available_scapers : [];
const scraperOptions: SelectionOption[] = [{ label: 'None', value: 'none' }, ...allScrapers];
@@ -169,6 +176,22 @@ const Settings = ({ closeSettings }:SettingsProps) => {
/>
</div>
)}
{settings.scraper_type !== 'none' && (
<div className="settings__section__input mb-5">
<label className={labelStyle}>Scraping Frequency</label>
<SelectField
multiple={false}
selected={[settings?.scrape_interval || 'daily']}
options={scrapingOptions}
defaultLabel={'Notification Settings'}
updateField={(updated:string[]) => updated[0] && updateSettings('scrape_interval', updated[0])}
rounded='rounded'
maxHeight={48}
minWidth={270}
/>
<small className=' text-gray-500 pt-2 block'>This option requires Server/Docker Instance Restart to take Effect.</small>
</div>
)}
</div>
</div>
)}

74
cron.js
View File

@@ -49,6 +49,9 @@ const generateCronTime = (interval) => {
if (interval === 'daily') {
cronTime = '0 0 0 * * *';
}
if (interval === 'other_day') {
cronTime = '0 0 2-30/2 * *';
}
if (interval === 'daily_morning') {
cronTime = '0 0 3 * * *';
}
@@ -63,19 +66,43 @@ const generateCronTime = (interval) => {
};
const runAppCronJobs = () => {
// RUN SERP Scraping CRON (EveryDay at Midnight) 0 0 0 * *
const scrapeCronTime = generateCronTime('daily');
Cron(scrapeCronTime, () => {
// console.log('### Running Keyword Position Cron Job!');
const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/cron`, fetchOpts)
.then((res) => res.json())
// .then((data) =>{ console.log(data)})
.catch((err) => {
console.log('ERROR Making Daily Scraper Cron Request..');
console.log(err);
});
}, { scheduled: true });
getAppSettings().then((settings) => {
// RUN SERP Scraping CRON (EveryDay at Midnight) 0 0 0 * *
const scrape_interval = settings.scrape_interval || 'daily';
if (scrape_interval !== 'never') {
const scrapeCronTime = generateCronTime(scrape_interval);
Cron(scrapeCronTime, () => {
// console.log('### Running Keyword Position Cron Job!');
const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/cron`, fetchOpts)
.then((res) => res.json())
// .then((data) =>{ console.log(data)})
.catch((err) => {
console.log('ERROR Making SERP Scraper Cron Request..');
console.log(err);
});
}, { scheduled: true });
}
// RUN Email Notification CRON
const notif_interval = (!settings.notification_interval || settings.notification_interval === 'never') ? false : settings.notification_interval;
if (notif_interval) {
const cronTime = generateCronTime(notif_interval === 'daily' ? 'daily_morning' : notif_interval);
if (cronTime) {
Cron(cronTime, () => {
// console.log('### Sending Notification Email...');
const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/notify`, fetchOpts)
.then((res) => res.json())
.then((data) => console.log(data))
.catch((err) => {
console.log('ERROR Making Cron Email Notification Request..');
console.log(err);
});
}, { scheduled: true });
}
}
});
// Run Failed scraping CRON (Every Hour)
const failedCronTime = generateCronTime('hourly');
@@ -115,27 +142,6 @@ const runAppCronJobs = () => {
});
}, { scheduled: true });
}
// RUN Email Notification CRON
getAppSettings().then((settings) => {
const notif_interval = (!settings.notification_interval || settings.notification_interval === 'never') ? false : settings.notification_interval;
if (notif_interval) {
const cronTime = generateCronTime(notif_interval === 'daily' ? 'daily_morning' : notif_interval);
if (cronTime) {
Cron(cronTime, () => {
// console.log('### Sending Notification Email...');
const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/notify`, fetchOpts)
.then((res) => res.json())
.then((data) => console.log(data))
.catch((err) => {
console.log('ERROR Making Cron Email Notification Request..');
console.log(err);
});
}, { scheduled: true });
}
}
});
};
runAppCronJobs();

3
types.d.ts vendored
View File

@@ -78,7 +78,8 @@ type SettingsType = {
smtp_username?: string,
smtp_password?: string,
search_console_integrated?: boolean,
available_scapers?: Array
available_scapers?: Array,
scrape_interval?: string
}
type KeywordSCDataChild = {