UniqueSoft/serpbear
2
0
Fork 0
mirror of https://github.com/towfiqi/serpbear synced 2025-06-26 18:15:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: Cron stopped on failing to parse failed queue

Browse Source 
closes #116
This commit is contained in:
towfiqi
2023-11-03 11:58:17 +06:00
parent 312d12f589
commit 8a949ce4c0
2 changed files with 18 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
cron.js
View File

@@ -111,16 +111,20 @@ const runAppCronJobs = () => {
readFile(`${process.cwd()}/data/failed_queue.json`, { encoding: 'utf-8' }, (err, data) => {
if (data) {
const keywordsToRetry = data ? JSON.parse(data) : [];
if (keywordsToRetry.length > 0) {
const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/refresh?id=${keywordsToRetry.join(',')}`, fetchOpts)
.then((res) => res.json())
.then((refreshedData) => console.log(refreshedData))
.catch((fetchErr) => {
console.log('ERROR Making failed_queue Cron Request..');
console.log(fetchErr);
});
try {
const keywordsToRetry = data ? JSON.parse(data) : [];
if (keywordsToRetry.length > 0) {
const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/refresh?id=${keywordsToRetry.join(',')}`, fetchOpts)
.then((res) => res.json())
.then((refreshedData) => console.log(refreshedData))
.catch((fetchErr) => {
console.log('ERROR Making failed_queue Cron Request..');
console.log(fetchErr);
});
}
} catch (error) {
console.log('ERROR Reading Failed Scrapes Queue File..', error);
}
} else {
console.log('ERROR Reading Failed Scrapes Queue File..', err);

8
utils/scraper.ts
View File

@@ -207,7 +207,7 @@ export const getSerp = (domain:string, result:SearchResult[]) : SERPObject => {
* @returns {void}
*/
export const retryScrape = async (keywordID: number) : Promise<void> => {
if (!keywordID) { return; }
if (!keywordID && !Number.isInteger(keywordID)) { return; }
let currentQueue: number[] = [];
const filePath = `${process.cwd()}/data/failed_queue.json`;
@@ -215,7 +215,7 @@ export const retryScrape = async (keywordID: number) : Promise<void> => {
currentQueue = currentQueueRaw ? JSON.parse(currentQueueRaw) : [];
if (!currentQueue.includes(keywordID)) {
currentQueue.push(keywordID);
currentQueue.push(Math.abs(keywordID));
}
await writeFile(filePath, JSON.stringify(currentQueue), { encoding: 'utf-8' }).catch((err) => { console.log(err); return '[]'; });
@@ -227,13 +227,13 @@ export const retryScrape = async (keywordID: number) : Promise<void> => {
* @returns {void}
*/
export const removeFromRetryQueue = async (keywordID: number) : Promise<void> => {
if (!keywordID) { return; }
if (!keywordID && !Number.isInteger(keywordID)) { return; }
let currentQueue: number[] = [];
const filePath = `${process.cwd()}/data/failed_queue.json`;
const currentQueueRaw = await readFile(filePath, { encoding: 'utf-8' }).catch((err) => { console.log(err); return '[]'; });
currentQueue = currentQueueRaw ? JSON.parse(currentQueueRaw) : [];
currentQueue = currentQueue.filter((item) => item !== keywordID);
currentQueue = currentQueue.filter((item) => item !== Math.abs(keywordID));
await writeFile(filePath, JSON.stringify(currentQueue), { encoding: 'utf-8' }).catch((err) => { console.log(err); return '[]'; });
};