UniqueSoft
/
firecrawl
mirror of https://github.com/mendableai/firecrawl
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(WebScraper): infinite regex leading to fly.io instance hangs

This commit is contained in:
Gergo Moricz 2024-07-18 19:13:43 +02:00
parent 6d1d46a987
commit 0b8047c7a0
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
apps/api/src/scraper/WebScraper/utils/replacePaths.ts
View File

@ -6,13 +6,13 @@ export const replacePathsWithAbsolutePaths = (documents: Document[]): Document[]
const baseUrl = new URL(document.metadata.sourceURL).origin;
const paths =
document.content.match(
/(!?\[.*?\])\(((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*)\)|href="([^"]+)"/g
/!?\[.*?\]\(.*?\)|href=".+?"/g
) || [];
paths.forEach((path: string) => {
try {
const isImage = path.startsWith("!");
let matchedUrl = path.match(/\(([^)]+)\)/) || path.match(/href="([^"]+)"/);
let matchedUrl = path.match(/\((.*?)\)/) || path.match(/href="([^"]+)"/);
let url = matchedUrl[1];
if (!url.startsWith("data:") && !url.startsWith("http")) {
@ -50,11 +50,11 @@ export const replaceImgPathsWithAbsolutePaths = (documents: Document[]): Documen
const baseUrl = new URL(document.metadata.sourceURL).origin;
const images =
document.content.match(
/!\[.*?\]\(((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*)\)/g
/!\[.*?\]\(.*?\)/g
) || [];
images.forEach((image: string) => {
let imageUrl = image.match(/\(([^)]+)\)/)[1];
let imageUrl = image.match(/\((.*?)\)/)[1];
let altText = image.match(/\[(.*?)\]/)[1];
if (!imageUrl.startsWith("data:image")) {