UniqueSoft
/
firecrawl
mirror of https://github.com/mendableai/firecrawl
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(fire-engine): propagate sentry trace

This commit is contained in:
Gergő Móricz 2024-08-22 23:37:23 +02:00
parent 1f0abacadf
commit 8d9ff90bcb
1 changed files with 29 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts
View File

@ -5,6 +5,7 @@ import { generateRequestParams } from "../single_url";
import { fetchAndProcessPdf } from "../utils/pdfProcessor"; import { fetchAndProcessPdf } from "../utils/pdfProcessor";
import { universalTimeout } from "../global"; import { universalTimeout } from "../global";
import { Logger } from "../../../lib/logger"; import { Logger } from "../../../lib/logger";
import * as Sentry from "@sentry/node";
/** /**
* Scrapes a URL with Fire-Engine * Scrapes a URL with Fire-Engine
@ -92,27 +93,35 @@ export async function scrapWithFireEngine({
}); });
const startTime = Date.now(); const startTime = Date.now();
const _response = await axiosInstance.post( const _response = await Sentry.startSpan({
process.env.FIRE_ENGINE_BETA_URL + endpoint, name: "Call to fire-engine"
{ }, async span => {
url: url, return await axiosInstance.post(
wait: waitParam, process.env.FIRE_ENGINE_BETA_URL + endpoint,
screenshot: screenshotParam, {
fullPageScreenshot: fullPageScreenshotParam, url: url,
headers: headers, wait: waitParam,
pageOptions: pageOptions, screenshot: screenshotParam,
disableJsDom: pageOptions?.disableJsDom ?? false, fullPageScreenshot: fullPageScreenshotParam,
priority, headers: headers,
engine, pageOptions: pageOptions,
instantReturn: true, disableJsDom: pageOptions?.disableJsDom ?? false,
...fireEngineOptionsParam, priority,
}, engine,
{ instantReturn: true,
headers: { ...fireEngineOptionsParam,
"Content-Type": "application/json", },
{
headers: {
"Content-Type": "application/json",
...(Sentry.isInitialized() ? ({
"sentry-trace": Sentry.spanToTraceHeader(span),
"baggage": Sentry.spanToBaggageHeader(span),
}) : {}),
}
} }
} );
); });
let checkStatusResponse = await axiosInstance.get(`${process.env.FIRE_ENGINE_BETA_URL}/scrape/${_response.data.jobId}`); let checkStatusResponse = await axiosInstance.get(`${process.env.FIRE_ENGINE_BETA_URL}/scrape/${_response.data.jobId}`);
while (checkStatusResponse.data.processing && Date.now() - startTime < universalTimeout + waitParam) { while (checkStatusResponse.data.processing && Date.now() - startTime < universalTimeout + waitParam) {