UniqueSoft
/
firecrawl
mirror of https://github.com/mendableai/firecrawl
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update crawl.test.ts

This commit is contained in:
Nicolas 2024-05-15 12:42:14 -07:00
parent 1601e93d69
commit 4745d114be
1 changed files with 19 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
apps/test-suite/tests/crawl.test.ts
View File

@ -49,14 +49,29 @@ describe("Crawling Checkup (E2E)", () => {
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.send({ url: websiteData.website, pageOptions: { onlyMainContent: true }, crawlerOptions: { limit: 100 }}); .send({ url: websiteData.website, pageOptions: { onlyMainContent: true }, crawlerOptions: { limit: 100 }});
await new Promise(resolve => setTimeout(resolve, 300000)); // wait for 300 seconds const jobId = crawlResponse.body.jobId;
let completedResponse;
let isFinished = false;
const completedResponse = await request(TEST_URL) while (!isFinished) {
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`) completedResponse = await request(TEST_URL)
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`); .get(`/v0/crawl/status/${jobId}`)
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
isFinished = completedResponse.body.status === "completed";
if (!isFinished) {
await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
}
}
console.log('-------------------') console.log('-------------------')
console.log(websiteData.website); console.log(websiteData.website);
if(!completedResponse) {
// fail the test
console.log('No response');
return null;
}
if (!completedResponse.body.data) { if (!completedResponse.body.data) {
console.log(completedResponse.body.partial_data.length); console.log(completedResponse.body.partial_data.length);