Update index.ts
This commit is contained in:
parent
60a15d00eb
commit
22a5e85899
|
@ -1,5 +1,5 @@
|
||||||
import "dotenv/config";
|
import "dotenv/config";
|
||||||
import "./services/sentry"
|
import "./services/sentry";
|
||||||
import * as Sentry from "@sentry/node";
|
import * as Sentry from "@sentry/node";
|
||||||
import express, { NextFunction, Request, Response } from "express";
|
import express, { NextFunction, Request, Response } from "express";
|
||||||
import bodyParser from "body-parser";
|
import bodyParser from "body-parser";
|
||||||
|
@ -12,9 +12,9 @@ import os from "os";
|
||||||
import { Logger } from "./lib/logger";
|
import { Logger } from "./lib/logger";
|
||||||
import { adminRouter } from "./routes/admin";
|
import { adminRouter } from "./routes/admin";
|
||||||
import { ScrapeEvents } from "./lib/scrape-events";
|
import { ScrapeEvents } from "./lib/scrape-events";
|
||||||
import http from 'node:http';
|
import http from "node:http";
|
||||||
import https from 'node:https';
|
import https from "node:https";
|
||||||
import CacheableLookup from 'cacheable-lookup';
|
import CacheableLookup from "cacheable-lookup";
|
||||||
import { v1Router } from "./routes/v1";
|
import { v1Router } from "./routes/v1";
|
||||||
import expressWs from "express-ws";
|
import expressWs from "express-ws";
|
||||||
import { crawlStatusWSController } from "./controllers/v1/crawl-status-ws";
|
import { crawlStatusWSController } from "./controllers/v1/crawl-status-ws";
|
||||||
|
@ -31,11 +31,11 @@ Logger.info(`Number of CPUs: ${numCPUs} available`);
|
||||||
|
|
||||||
const cacheable = new CacheableLookup({
|
const cacheable = new CacheableLookup({
|
||||||
// this is important to avoid querying local hostnames see https://github.com/szmarczak/cacheable-lookup readme
|
// this is important to avoid querying local hostnames see https://github.com/szmarczak/cacheable-lookup readme
|
||||||
lookup:false
|
lookup: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
cacheable.install(http.globalAgent);
|
cacheable.install(http.globalAgent);
|
||||||
cacheable.install(https.globalAgent)
|
cacheable.install(https.globalAgent);
|
||||||
|
|
||||||
if (cluster.isMaster) {
|
if (cluster.isMaster) {
|
||||||
Logger.info(`Master ${process.pid} is running`);
|
Logger.info(`Master ${process.pid} is running`);
|
||||||
|
@ -115,9 +115,7 @@ if (cluster.isMaster) {
|
||||||
app.get(`/serverHealthCheck`, async (req, res) => {
|
app.get(`/serverHealthCheck`, async (req, res) => {
|
||||||
try {
|
try {
|
||||||
const scrapeQueue = getScrapeQueue();
|
const scrapeQueue = getScrapeQueue();
|
||||||
const [waitingJobs] = await Promise.all([
|
const [waitingJobs] = await Promise.all([scrapeQueue.getWaitingCount()]);
|
||||||
scrapeQueue.getWaitingCount(),
|
|
||||||
]);
|
|
||||||
|
|
||||||
const noWaitingJobs = waitingJobs === 0;
|
const noWaitingJobs = waitingJobs === 0;
|
||||||
// 200 if no active jobs, 503 if there are active jobs
|
// 200 if no active jobs, 503 if there are active jobs
|
||||||
|
@ -190,17 +188,43 @@ if (cluster.isMaster) {
|
||||||
res.send({ isProduction: global.isProduction });
|
res.send({ isProduction: global.isProduction });
|
||||||
});
|
});
|
||||||
|
|
||||||
app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: Response<ErrorResponse>, next: NextFunction) => {
|
app.use(
|
||||||
|
(
|
||||||
|
err: unknown,
|
||||||
|
req: Request<{}, ErrorResponse, undefined>,
|
||||||
|
res: Response<ErrorResponse>,
|
||||||
|
next: NextFunction
|
||||||
|
) => {
|
||||||
if (err instanceof ZodError) {
|
if (err instanceof ZodError) {
|
||||||
res.status(400).json({ success: false, error: "Bad Request", details: err.errors });
|
res
|
||||||
|
.status(400)
|
||||||
|
.json({ success: false, error: "Bad Request", details: err.errors });
|
||||||
} else {
|
} else {
|
||||||
next(err);
|
next(err);
|
||||||
}
|
}
|
||||||
});
|
}
|
||||||
|
);
|
||||||
|
|
||||||
Sentry.setupExpressErrorHandler(app);
|
Sentry.setupExpressErrorHandler(app);
|
||||||
|
|
||||||
app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: ResponseWithSentry<ErrorResponse>, next: NextFunction) => {
|
app.use(
|
||||||
|
(
|
||||||
|
err: unknown,
|
||||||
|
req: Request<{}, ErrorResponse, undefined>,
|
||||||
|
res: ResponseWithSentry<ErrorResponse>,
|
||||||
|
next: NextFunction
|
||||||
|
) => {
|
||||||
|
if (
|
||||||
|
err instanceof SyntaxError &&
|
||||||
|
"status" in err &&
|
||||||
|
err.status === 400 &&
|
||||||
|
"body" in err
|
||||||
|
) {
|
||||||
|
return res
|
||||||
|
.status(400)
|
||||||
|
.json({ success: false, error: "Bad request, malformed JSON" });
|
||||||
|
}
|
||||||
|
|
||||||
const id = res.sentry ?? uuidv4();
|
const id = res.sentry ?? uuidv4();
|
||||||
let verbose = JSON.stringify(err);
|
let verbose = JSON.stringify(err);
|
||||||
if (verbose === "{}") {
|
if (verbose === "{}") {
|
||||||
|
@ -213,15 +237,28 @@ if (cluster.isMaster) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Logger.error("Error occurred in request! (" + req.path + ") -- ID " + id + " -- " + verbose);
|
Logger.error(
|
||||||
res.status(500).json({ success: false, error: "An unexpected error occurred. Please contact hello@firecrawl.com for help. Your exception ID is " + id });
|
"Error occurred in request! (" +
|
||||||
|
req.path +
|
||||||
|
") -- ID " +
|
||||||
|
id +
|
||||||
|
" -- " +
|
||||||
|
verbose
|
||||||
|
);
|
||||||
|
res
|
||||||
|
.status(500)
|
||||||
|
.json({
|
||||||
|
success: false,
|
||||||
|
error:
|
||||||
|
"An unexpected error occurred. Please contact hello@firecrawl.com for help. Your exception ID is " +
|
||||||
|
id,
|
||||||
});
|
});
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
Logger.info(`Worker ${process.pid} started`);
|
Logger.info(`Worker ${process.pid} started`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// const sq = getScrapeQueue();
|
// const sq = getScrapeQueue();
|
||||||
|
|
||||||
// sq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting"));
|
// sq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting"));
|
||||||
|
@ -230,6 +267,3 @@ if (cluster.isMaster) {
|
||||||
// sq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused"));
|
// sq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused"));
|
||||||
// sq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed"));
|
// sq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed"));
|
||||||
// sq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed"));
|
// sq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed"));
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue