UniqueSoft
/
firecrawl
mirror of https://github.com/mendableai/firecrawl
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #17 from mendableai/feat/pdf-parser 

[Feat] Adding pdf parser
This commit is contained in:
Nicolas 2024-04-18 10:34:56 -07:00 committed by GitHub
parent 50cf97c709 6112cc1c2c
commit 0f6fdd7f63
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 221 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/.env.local
View File

@ -10,4 +10,4 @@ OPENAI_API_KEY=
BULL_AUTH_KEY=
LOGTAIL_KEY=
PLAYWRIGHT_MICROSERVICE_URL=
LLAMAPARSE_API_KEY=

2
apps/api/package.json
View File

@ -60,6 +60,7 @@
"date-fns": "^2.29.3",
"dotenv": "^16.3.1",
"express-rate-limit": "^6.7.0",
"form-data": "^4.0.0",
"glob": "^10.3.12",
"gpt3-tokenizer": "^1.1.5",
"ioredis": "^5.3.2",
@ -73,6 +74,7 @@
"mongoose": "^8.0.3",
"natural": "^6.3.0",
"openai": "^4.28.4",
"pdf-parse": "^1.1.1",
"pos": "^0.4.2",
"promptable": "^0.0.9",
"puppeteer": "^22.6.3",

26
apps/api/pnpm-lock.yaml
View File

@ -68,6 +68,9 @@ dependencies:
express-rate-limit:
specifier: ^6.7.0
version: 6.11.2(express@4.18.3)
form-data:
specifier: ^4.0.0
version: 4.0.0
glob:
specifier: ^10.3.12
version: 10.3.12
@ -82,7 +85,7 @@ dependencies:
version: 0.0.25
langchain:
specifier: ^0.1.25
version: 0.1.25(@supabase/supabase-js@2.39.7)(axios@1.6.7)(cheerio@1.0.0-rc.12)(ioredis@5.3.2)(puppeteer@22.6.3)(redis@4.6.13)(typesense@1.7.2)
version: 0.1.25(@supabase/supabase-js@2.39.7)(axios@1.6.7)(cheerio@1.0.0-rc.12)(ioredis@5.3.2)(pdf-parse@1.1.1)(puppeteer@22.6.3)(redis@4.6.13)(typesense@1.7.2)
languagedetect:
specifier: ^2.0.0
version: 2.0.0
@ -107,6 +110,9 @@ dependencies:
openai:
specifier: ^4.28.4
version: 4.28.4
pdf-parse:
specifier: ^1.1.1
version: 1.1.1
pos:
specifier: ^0.4.2
version: 0.4.2
@ -2498,7 +2504,6 @@ packages:
dependencies:
ms: 2.1.3
supports-color: 5.5.0
dev: true
/debug@4.3.4:
resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==}
@ -3997,7 +4002,7 @@ packages:
engines: {node: '>=6'}
dev: true
/langchain@0.1.25(@supabase/supabase-js@2.39.7)(axios@1.6.7)(cheerio@1.0.0-rc.12)(ioredis@5.3.2)(puppeteer@22.6.3)(redis@4.6.13)(typesense@1.7.2):
/langchain@0.1.25(@supabase/supabase-js@2.39.7)(axios@1.6.7)(cheerio@1.0.0-rc.12)(ioredis@5.3.2)(pdf-parse@1.1.1)(puppeteer@22.6.3)(redis@4.6.13)(typesense@1.7.2):
resolution: {integrity: sha512-sfEChvr4H2CklHdSByNBbytwBrFhgtA5kPOnwcBrxuXGg1iOaTzhVxQA0QcNcQucI3hZrsNbZjxGp+Can1ooZQ==}
engines: {node: '>=18'}
peerDependencies:
@ -4174,6 +4179,7 @@ packages:
ml-distance: 4.0.1
openapi-types: 12.1.3
p-retry: 4.6.2
pdf-parse: 1.1.1
puppeteer: 22.6.3(typescript@5.4.2)
redis: 4.6.13
uuid: 9.0.1
@ -4653,6 +4659,10 @@ packages:
resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
engines: {node: '>=10.5.0'}
/node-ensure@0.0.0:
resolution: {integrity: sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw==}
dev: false
/node-fetch@2.7.0:
resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==}
engines: {node: 4.x || >=6.0.0}
@ -4951,6 +4961,16 @@ packages:
/path-to-regexp@0.1.7:
resolution: {integrity: sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==}
/pdf-parse@1.1.1:
resolution: {integrity: sha512-v6ZJ/efsBpGrGGknjtq9J/oC8tZWq0KWL5vQrk2GlzLEQPUDB1ex+13Rmidl1neNN358Jn9EHZw5y07FFtaC7A==}
engines: {node: '>=6.8.1'}
dependencies:
debug: 3.2.7(supports-color@5.5.0)
node-ensure: 0.0.0
transitivePeerDependencies:
- supports-color
dev: false
/pend@1.2.0:
resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==}
dev: false

2
apps/api/src/index.ts
View File

@ -87,7 +87,7 @@ async function authenticateUser(req, res, mode?: string): Promise<{ success: boo
app.post("/v0/scrape", async (req, res) => {
try {
// make sure to authenticate user first, Bearer <token>
const { success, team_id, error, status } = await authenticateUser(req, res, "crawl");
const { success, team_id, error, status } = await authenticateUser(req, res, "scrape");
if (!success) {
return res.status(status).json({ error });
}

2
apps/api/src/lib/entities.ts
View File

@ -39,6 +39,7 @@ export class Document {
[key: string]: any;
};
childrenLinks?: string[];
provider?: string;
constructor(data: Partial<Document>) {
if (!data.content) {
@ -51,5 +52,6 @@ export class Document {
this.metadata = data.metadata || { sourceURL: "" };
this.markdown = data.markdown || "";
this.childrenLinks = data.childrenLinks || undefined;
this.provider = data.provider || undefined;
}
}

2
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -257,7 +257,7 @@ export class WebCrawler {
".js",
".ico",
".svg",
".pdf",
// ".pdf",
".zip",
".exe",
".dmg",

46
apps/api/src/scraper/WebScraper/index.ts
View File

@ -5,6 +5,7 @@ import { SitemapEntry, fetchSitemapData, getLinksFromSitemap } from "./sitemap";
import { WebCrawler } from "./crawler";
import { getValue, setValue } from "../../services/redis";
import { getImageDescription } from "./utils/gptVision";
import { fetchAndProcessPdf } from "./utils/pdfProcessor";
export class WebScraperDataProvider {
@ -75,7 +76,7 @@ export class WebScraperDataProvider {
limit: this.limit,
generateImgAltText: this.generateImgAltText,
});
const links = await crawler.start(inProgress, 5, this.limit);
let links = await crawler.start(inProgress, 5, this.limit);
if (this.returnOnlyUrls) {
return links.map((url) => ({
content: "",
@ -84,12 +85,26 @@ export class WebScraperDataProvider {
type: "text",
}));
}
let pdfLinks = links.filter((link) => link.endsWith(".pdf"));
let pdfDocuments: Document[] = [];
for (let pdfLink of pdfLinks) {
const pdfContent = await fetchAndProcessPdf(pdfLink);
pdfDocuments.push({
content: pdfContent,
metadata: { sourceURL: pdfLink },
provider: "web-scraper"
});
}
links = links.filter((link) => !link.endsWith(".pdf"));
let documents = await this.convertUrlsToDocuments(links, inProgress);
documents = await this.getSitemapData(this.urls[0], documents);
documents = this.replaceImgPathsWithAbsolutePaths(documents);
if (this.generateImgAltText) {
documents = await this.generatesImgAltText(documents);
}
documents = documents.concat(pdfDocuments);
// CACHING DOCUMENTS
// - parent document
@ -134,8 +149,19 @@ export class WebScraperDataProvider {
}
if (this.mode === "single_urls") {
let pdfLinks = this.urls.filter((link) => link.endsWith(".pdf"));
let pdfDocuments: Document[] = [];
for (let pdfLink of pdfLinks) {
const pdfContent = await fetchAndProcessPdf(pdfLink);
pdfDocuments.push({
content: pdfContent,
metadata: { sourceURL: pdfLink },
provider: "web-scraper"
});
}
let documents = await this.convertUrlsToDocuments(
this.urls,
this.urls.filter((link) => !link.endsWith(".pdf")),
inProgress
);
documents = this.replaceImgPathsWithAbsolutePaths(documents);
@ -144,6 +170,7 @@ export class WebScraperDataProvider {
}
const baseUrl = new URL(this.urls[0]).origin;
documents = await this.getSitemapData(baseUrl, documents);
documents = documents.concat(pdfDocuments);
await this.setCachedDocuments(documents);
documents = this.removeChildLinks(documents);
@ -151,7 +178,19 @@ export class WebScraperDataProvider {
return documents;
}
if (this.mode === "sitemap") {
const links = await getLinksFromSitemap(this.urls[0]);
let links = await getLinksFromSitemap(this.urls[0]);
let pdfLinks = links.filter((link) => link.endsWith(".pdf"));
let pdfDocuments: Document[] = [];
for (let pdfLink of pdfLinks) {
const pdfContent = await fetchAndProcessPdf(pdfLink);
pdfDocuments.push({
content: pdfContent,
metadata: { sourceURL: pdfLink },
provider: "web-scraper"
});
}
links = links.filter((link) => !link.endsWith(".pdf"));
let documents = await this.convertUrlsToDocuments(
links.slice(0, this.limit),
inProgress
@ -162,6 +201,7 @@ export class WebScraperDataProvider {
if (this.generateImgAltText) {
documents = await this.generatesImgAltText(documents);
}
documents = documents.concat(pdfDocuments);
await this.setCachedDocuments(documents);
documents = this.removeChildLinks(documents);

40
apps/api/src/scraper/WebScraper/utils/__tests__/pdfProcessor.test.ts Normal file
View File

@ -0,0 +1,40 @@
import * as pdfProcessor from '../pdfProcessor';
describe('PDF Processing Module - Integration Test', () => {
it('should download and read a simple PDF file by URL', async () => {
const pdfContent = await pdfProcessor.fetchAndProcessPdf('https://s3.us-east-1.amazonaws.com/storage.mendable.ai/rafa-testing/test%20%281%29.pdf');
expect(pdfContent).toEqual("Dummy PDF file");
});
it('should download and read a complex PDF file by URL', async () => {
const pdfContent = await pdfProcessor.fetchAndProcessPdf('https://arxiv.org/pdf/2307.06435.pdf');
const expectedContent = 'A Comprehensive Overview of Large Language Models\n' +
' a a,∗ b,∗ c,d,∗ e,f e,f g,i\n' +
' Humza Naveed , Asad Ullah Khan , Shi Qiu , Muhammad Saqib , Saeed Anwar , Muhammad Usman , Naveed Akhtar ,\n' +
' Nick Barnes h, Ajmal Mian i\n' +
' aUniversity of Engineering and Technology (UET), Lahore, Pakistan\n' +
' bThe Chinese University of Hong Kong (CUHK), HKSAR, China\n' +
' cUniversity of Technology Sydney (UTS), Sydney, Australia\n' +
' dCommonwealth Scientific and Industrial Research Organisation (CSIRO), Sydney, Australia\n' +
' eKing Fahd University of Petroleum and Minerals (KFUPM), Dhahran, Saudi Arabia\n' +
' fSDAIA-KFUPM Joint Research Center for Artificial Intelligence (JRCAI), Dhahran, Saudi Arabia\n' +
' gThe University of Melbourne (UoM), Melbourne, Australia\n' +
' hAustralian National University (ANU), Canberra, Australia\n' +
' iThe University of Western Australia (UWA), Perth, Australia\n' +
' Abstract\n' +
' Large Language Models (LLMs) have recently demonstrated remarkable capabilities in natural language processing tasks and\n' +
' beyond. This success of LLMs has led to a large influx of research contributions in this direction. These works encompass diverse\n' +
' topics such as architectural innovations, better training strategies, context length improvements, fine-tuning, multi-modal LLMs,\n' +
' robotics, datasets, benchmarking, efficiency, and more. With the rapid development of techniques and regular breakthroughs in\n' +
' LLM research, it has become considerably challenging to perceive the bigger picture of the advances in this direction. Considering\n' +
' the rapidly emerging plethora of literature on LLMs, it is imperative that the research community is able to benefit from a concise\n' +
' yet comprehensive overview of the recent developments in this field. This article provides an overview of the existing literature\n' +
' on a broad range of LLM-related concepts. Our self-contained comprehensive overview of LLMs discusses relevant background\n' +
' concepts along with covering the advanced topics at the frontier of research in LLMs. This review article is intended to not only\n' +
' provide a systematic survey but also a quick comprehensive reference for the researchers and practitioners to draw insights from\n' +
' extensive informative summaries of the existing works to advance the LLM research.\n'
expect(pdfContent).toContain(expectedContent);
}, 60000);
});

108
apps/api/src/scraper/WebScraper/utils/pdfProcessor.ts Normal file
View File

@ -0,0 +1,108 @@
import axios, { AxiosResponse } from "axios";
import fs from "fs";
import { createReadStream, createWriteStream } from "node:fs";
import FormData from "form-data";
import dotenv from "dotenv";
import pdf from "pdf-parse";
import path from "path";
import os from "os";
dotenv.config();
export async function fetchAndProcessPdf(url: string): Promise<string> {
const tempFilePath = await downloadPdf(url);
const content = await processPdfToText(tempFilePath);
fs.unlinkSync(tempFilePath); // Clean up the temporary file
return content;
}
async function downloadPdf(url: string): Promise<string> {
const response = await axios({
url,
method: 'GET',
responseType: 'stream',
});
const tempFilePath = path.join(os.tmpdir(), `tempPdf-${Date.now()}.pdf`);
const writer = createWriteStream(tempFilePath);
response.data.pipe(writer);
return new Promise((resolve, reject) => {
writer.on('finish', () => resolve(tempFilePath));
writer.on('error', reject);
});
}
export async function processPdfToText(filePath: string): Promise<string> {
let content = "";
if (process.env.LLAMAPARSE_API_KEY) {
const apiKey = process.env.LLAMAPARSE_API_KEY;
const headers = {
Authorization: `Bearer ${apiKey}`,
};
const base_url = "https://api.cloud.llamaindex.ai/api/parsing";
const fileType2 = "application/pdf";
try {
const formData = new FormData();
formData.append("file", createReadStream(filePath), {
filename: filePath,
contentType: fileType2,
});
const uploadUrl = `${base_url}/upload`;
const uploadResponse = await axios.post(uploadUrl, formData, {
headers: {
...headers,
...formData.getHeaders(),
},
});
const jobId = uploadResponse.data.id;
const resultType = "text";
const resultUrl = `${base_url}/job/${jobId}/result/${resultType}`;
let resultResponse: AxiosResponse;
let attempt = 0;
const maxAttempts = 10; // Maximum number of attempts
let resultAvailable = false;
while (attempt < maxAttempts && !resultAvailable) {
try {
resultResponse = await axios.get(resultUrl, { headers });
if (resultResponse.status === 200) {
resultAvailable = true; // Exit condition met
} else {
// If the status code is not 200, increment the attempt counter and wait
attempt++;
await new Promise((resolve) => setTimeout(resolve, 250)); // Wait for 2 seconds
}
} catch (error) {
console.error("Error fetching result:", error);
attempt++;
await new Promise((resolve) => setTimeout(resolve, 250)); // Wait for 2 seconds before retrying
// You may want to handle specific errors differently
}
}
if (!resultAvailable) {
content = await processPdf(filePath);
}
content = resultResponse.data[resultType];
} catch (error) {
console.error("Error processing document:", filePath, error);
content = await processPdf(filePath);
}
} else {
content = await processPdf(filePath);
}
return content;
}
async function processPdf(file: string){
const fileContent = fs.readFileSync(file);
const data = await pdf(fileContent);
return data.text;
}