UniqueSoft
/
firecrawl
mirror of https://github.com/mendableai/firecrawl
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update queue-worker.ts

This commit is contained in:
Nicolas 2024-08-17 03:09:31 -04:00
parent f797380112
commit 3fe82b4f12
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/api/src/services/queue-worker.ts
View File

@ -117,13 +117,13 @@ async function processJob(job: Job, token: string) {
// Check if the job URL is researchhub and block it immediately // Check if the job URL is researchhub and block it immediately
// TODO: remove this once solve the root issue // TODO: remove this once solve the root issue
if (job.data.url && job.data.url.includes("researchhub.com")) { if (job.data.url && (job.data.url.includes("researchhub.com") || job.data.url.includes("ebay.com"))) {
Logger.info(`🐂 Blocking job ${job.id} with URL ${job.data.url}`); Logger.info(`🐂 Blocking job ${job.id} with URL ${job.data.url}`);
const data = { const data = {
success: false, success: false,
docs: [], docs: [],
project_id: job.data.project_id, project_id: job.data.project_id,
error: "URL is blocked: researchhub.com", error: "URL is blocked. Please contact hello@firecrawl.com if you believe this is an error.",
}; };
await job.moveToCompleted(data.docs, token, false); await job.moveToCompleted(data.docs, token, false);
return data; return data;
@ -137,6 +137,7 @@ async function processJob(job: Job, token: string) {
current_url: "", current_url: "",
}); });
const start = Date.now(); const start = Date.now();
const { success, message, docs } = await startWebScraperPipeline({ const { success, message, docs } = await startWebScraperPipeline({
job, job,
token, token,