UniqueSoft
/
firecrawl
mirror of https://github.com/mendableai/firecrawl
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

dependabot for security checks, fixed crawl test

This commit is contained in:
rafaelsideguide 2024-07-05 14:49:03 -03:00
parent 2b36de605a
commit a2cdc520e6
3 changed files with 52 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
.github/dependabot.yml vendored
View File

@ -5,11 +5,8 @@ updates:
directory: "/apps/playwright-service" directory: "/apps/playwright-service"
schedule: schedule:
interval: "weekly" interval: "weekly"
groups: open-pull-requests-limit: 0 # Disable version updates
prod-deps: security-updates: "all"
dependency-type: "production"
dev-deps:
dependency-type: "development"
commit-message: commit-message:
prefix: "apps/playwright-service" prefix: "apps/playwright-service"
include: "scope" include: "scope"
@ -19,11 +16,8 @@ updates:
directory: "/apps/python-sdk" directory: "/apps/python-sdk"
schedule: schedule:
interval: "weekly" interval: "weekly"
groups: open-pull-requests-limit: 0 # Disable version updates
prod-deps: security-updates: "all"
dependency-type: "production"
dev-deps:
dependency-type: "development"
commit-message: commit-message:
prefix: "apps/python-sdk" prefix: "apps/python-sdk"
include: "scope" include: "scope"
@ -33,11 +27,8 @@ updates:
directory: "/apps/api" directory: "/apps/api"
schedule: schedule:
interval: "weekly" interval: "weekly"
groups: open-pull-requests-limit: 0 # Disable version updates
prod-deps: security-updates: "all"
dependency-type: "production"
dev-deps:
dependency-type: "development"
commit-message: commit-message:
prefix: "apps/api" prefix: "apps/api"
include: "scope" include: "scope"
@ -47,11 +38,8 @@ updates:
directory: "/apps/test-suite" directory: "/apps/test-suite"
schedule: schedule:
interval: "weekly" interval: "weekly"
groups: open-pull-requests-limit: 0 # Disable version updates
prod-deps: security-updates: "all"
dependency-type: "production"
dev-deps:
dependency-type: "development"
commit-message: commit-message:
prefix: "apps/test-suite" prefix: "apps/test-suite"
include: "scope" include: "scope"

103
apps/test-suite/data/crawl.json
View File

@ -5,47 +5,46 @@
"expected_crawled_pages": ["https://www.vellum.ai/llm-leaderboard"] "expected_crawled_pages": ["https://www.vellum.ai/llm-leaderboard"]
}, },
{ {
"website": "https://openai.com/news", "website": "https://openai.com/news",
"expected_min_num_of_pages": 4, "expected_min_num_of_pages": 4,
"expected_crawled_pages": [ "expected_crawled_pages": [
"https://openai.com/news/company/", "https://openai.com/news/company/",
"https://openai.com/news/research/", "https://openai.com/news/research/",
"https://openai.com/news/safety-and-alignment/", "https://openai.com/news/safety-and-alignment/",
"https://openai.com/news/stories/" "https://openai.com/news/stories/"
] ]
}, },
{ {
"website": "https://www.framer.com/pricing", "website": "https://www.framer.com/pricing",
"expected_min_num_of_pages": 1, "expected_min_num_of_pages": 1,
"expected_not_crawled_pages": [ "expected_not_crawled_pages": [
"https://www.framer.com/features/navigation/", "https://www.framer.com/features/navigation/",
"https://www.framer.com/contact/", "https://www.framer.com/contact/",
"https://www.framer.com/add-ons/", "https://www.framer.com/add-ons/",
"https://www.framer.com/free-saas-ui-kit/", "https://www.framer.com/free-saas-ui-kit/",
"https://www.framer.com/help/", "https://www.framer.com/help/",
"https://www.framer.com/features/effects/", "https://www.framer.com/features/effects/",
"https://www.framer.com/enterprise/", "https://www.framer.com/enterprise/",
"https://www.framer.com/templates/" "https://www.framer.com/templates/"
] ]
}, },
{ {
"website": "https://mendable.ai/pricing", "website": "https://mendable.ai/pricing",
"expected_min_num_of_pages": 1, "expected_min_num_of_pages": 1,
"expected_not_crawled_pages": [ "expected_not_crawled_pages": [
"https://mendable.ai/", "https://mendable.ai/",
"https://mendable.ai/blog", "https://mendable.ai/blog",
"https://mendable.ai/signin", "https://mendable.ai/signin",
"https://mendable.ai/signup", "https://mendable.ai/signup",
"https://mendable.ai", "https://mendable.ai",
"https://mendable.ai/usecases/sales-enablement", "https://mendable.ai/usecases/sales-enablement",
"https://mendable.ai/usecases/documentation", "https://mendable.ai/usecases/documentation",
"https://mendable.ai/usecases/cs-enablement", "https://mendable.ai/usecases/cs-enablement",
"https://mendable.ai/usecases/productcopilot", "https://mendable.ai/usecases/productcopilot",
"https://mendable.ai/security" "https://mendable.ai/security"
], ],
"notes": "This one should not go backwards, but it does!" "notes": "This one should not go backwards, but it does!"
}, },
{ {
"website": "https://agentops.ai/blog", "website": "https://agentops.ai/blog",
"expected_min_num_of_pages": 6, "expected_min_num_of_pages": 6,
@ -99,12 +98,10 @@
"website": "https://firecrawl.dev", "website": "https://firecrawl.dev",
"expected_min_num_of_pages": 2, "expected_min_num_of_pages": 2,
"expected_crawled_pages": [ "expected_crawled_pages": [
"https://firecrawl.dev/", "https://www.firecrawl.dev/",
"https://firecrawl.dev/pricing" "https://www.firecrawl.dev/pricing"
] ]
}, },
{ {
"website": "https://fly.io/docs/gpus/gpu-quickstart", "website": "https://fly.io/docs/gpus/gpu-quickstart",
"expected_min_num_of_pages": 1, "expected_min_num_of_pages": 1,
@ -119,7 +116,6 @@
], ],
"notes": "This one should not go backwards, but it does!" "notes": "This one should not go backwards, but it does!"
}, },
{ {
"website": "https://www.instructables.com/circuits", "website": "https://www.instructables.com/circuits",
"expected_min_num_of_pages": 12, "expected_min_num_of_pages": 12,
@ -153,26 +149,5 @@
"https://richmondconfidential.org/2009/10/13/before-napa-there-was-winehaven/", "https://richmondconfidential.org/2009/10/13/before-napa-there-was-winehaven/",
"https://richmondconfidential.org/2009/10/13/family-calls-for-end-to-violence-at-memorial-for-slain-woman-friend/" "https://richmondconfidential.org/2009/10/13/family-calls-for-end-to-violence-at-memorial-for-slain-woman-friend/"
] ]
},
{
"website": "https://www.boardgamegeek.com",
"expected_min_num_of_pages": 15,
"expected_crawled_pages": [
"https://www.boardgamegeek.com/browse/boardgameartist",
"https://www.boardgamegeek.com/browse/boardgamehonor",
"https://www.boardgamegeek.com/browse/boardgamepublisher",
"https://www.boardgamegeek.com/browse/boardgamepodcast",
"https://www.boardgamegeek.com/wiki/page/Index",
"https://www.boardgamegeek.com/browse/boardgamecategory",
"https://www.boardgamegeek.com/boardgame/random",
"https://www.boardgamegeek.com/browse/boardgamemechanic",
"https://www.boardgamegeek.com/forums",
"https://www.boardgamegeek.com/gonecardboard",
"https://www.boardgamegeek.com/browse/boardgameaccessory",
"https://www.boardgamegeek.com/browse/boardgamedesigner",
"https://www.boardgamegeek.com/",
"https://www.boardgamegeek.com/previews",
"https://www.boardgamegeek.com/browse/boardgame"
]
} }
] ]

8
apps/test-suite/tests/crawl.test.ts
View File

@ -96,8 +96,8 @@ describe("Crawling Checkup (E2E)", () => {
website: websiteData.website, website: websiteData.website,
prompt: 'CRAWL', prompt: 'CRAWL',
expected_output: `SUCCESS: ${websiteData.expected_crawled_pages}`, expected_output: `SUCCESS: ${websiteData.expected_crawled_pages}`,
actual_output: `FAILURE: ${completedResponse.body.data}`, actual_output: `FAILURE: ${completedResponse.body.data.map((d: { url: string }) => d.url)}`,
error: `Expected crawled pages to contain ${websiteData.expected_crawled_pages}, but got ${completedResponse.body.data}` error: `Expected crawled pages to contain ${websiteData.expected_crawled_pages}, but got ${completedResponse.body.data.map((d: { url: string }) => d.url)}`
}); });
console.log('Error: ', errorLog); console.log('Error: ', errorLog);
continue; continue;
@ -109,8 +109,8 @@ describe("Crawling Checkup (E2E)", () => {
website: websiteData.website, website: websiteData.website,
prompt: 'CRAWL', prompt: 'CRAWL',
expected_output: `SUCCESS: ${websiteData.expected_not_crawled_pages}`, expected_output: `SUCCESS: ${websiteData.expected_not_crawled_pages}`,
actual_output: `FAILURE: ${completedResponse.body.data}`, actual_output: `FAILURE: ${completedResponse.body.data.map((d: { url: string }) => d.url)}`,
error: `Expected crawled pages to not contain ${websiteData.expected_not_crawled_pages}, but got ${completedResponse.body.data}` error: `Expected crawled pages to not contain ${websiteData.expected_not_crawled_pages}, but got ${completedResponse.body.data.map((d: { url: string }) => d.url)}`
}); });
console.log('Error: ', errorLog); console.log('Error: ', errorLog);
continue; continue;