UniqueSoft
/
firecrawl
mirror of https://github.com/mendableai/firecrawl
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(fire-engine): propagate sentry trace

This commit is contained in:
Gergő Móricz 2024-08-22 23:37:23 +02:00
parent 1f0abacadf
commit 8d9ff90bcb
1 changed files with 29 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts
View File

@ -5,6 +5,7 @@ import { generateRequestParams } from "../single_url";
import { fetchAndProcessPdf } from "../utils/pdfProcessor"; import { fetchAndProcessPdf } from "../utils/pdfProcessor";
import { universalTimeout } from "../global"; import { universalTimeout } from "../global";
import { Logger } from "../../../lib/logger"; import { Logger } from "../../../lib/logger";
import * as Sentry from "@sentry/node";
/** /**
* Scrapes a URL with Fire-Engine * Scrapes a URL with Fire-Engine
@ -92,7 +93,10 @@ export async function scrapWithFireEngine({
}); });
const startTime = Date.now(); const startTime = Date.now();
const _response = await axiosInstance.post( const _response = await Sentry.startSpan({
name: "Call to fire-engine"
}, async span => {
return await axiosInstance.post(
process.env.FIRE_ENGINE_BETA_URL + endpoint, process.env.FIRE_ENGINE_BETA_URL + endpoint,
{ {
url: url, url: url,
@ -110,9 +114,14 @@ export async function scrapWithFireEngine({
{ {
headers: { headers: {
"Content-Type": "application/json", "Content-Type": "application/json",
...(Sentry.isInitialized() ? ({
"sentry-trace": Sentry.spanToTraceHeader(span),
"baggage": Sentry.spanToBaggageHeader(span),
}) : {}),
} }
} }
); );
});
let checkStatusResponse = await axiosInstance.get(`${process.env.FIRE_ENGINE_BETA_URL}/scrape/${_response.data.jobId}`); let checkStatusResponse = await axiosInstance.get(`${process.env.FIRE_ENGINE_BETA_URL}/scrape/${_response.data.jobId}`);
while (checkStatusResponse.data.processing && Date.now() - startTime < universalTimeout + waitParam) { while (checkStatusResponse.data.processing && Date.now() - startTime < universalTimeout + waitParam) {