UniqueSoft
/
firecrawl
mirror of https://github.com/mendableai/firecrawl
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fixing document response

This commit is contained in:
rafaelsideguide 2024-04-18 14:12:39 -03:00
parent 704a059448
commit c4cc4b9262
1 changed files with 3 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
apps/api/src/scraper/WebScraper/index.ts
View File

@ -93,8 +93,7 @@ export class WebScraperDataProvider {
pdfDocuments.push({
content: pdfContent,
metadata: { sourceURL: pdfLink },
provider: "web",
type: "text",
provider: "web-scraper"
});
}
links = links.filter((link) => !link.endsWith(".pdf"));
@ -157,8 +156,7 @@ export class WebScraperDataProvider {
pdfDocuments.push({
content: pdfContent,
metadata: { sourceURL: pdfLink },
provider: "web",
type: "text",
provider: "web-scraper"
});
}
@ -188,8 +186,7 @@ export class WebScraperDataProvider {
pdfDocuments.push({
content: pdfContent,
metadata: { sourceURL: pdfLink },
provider: "web",
type: "text",
provider: "web-scraper"
});
}
links = links.filter((link) => !link.endsWith(".pdf"));