diff --git a/package-lock.json b/package-lock.json index 96c2d24..6a3988f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "axios": "^1.1.3", "axios-retry": "^3.3.1", "chart.js": "^3.9.1", - "cheerio": "^1.0.0-rc.12", + "cheerio": "^1.0.0", "concurrently": "^7.6.0", "cookies": "^0.8.0", "croner": "^5.3.5", @@ -53,8 +53,8 @@ "@types/jsonwebtoken": "^8.5.9", "@types/node": "18.11.0", "@types/nodemailer": "^6.4.6", - "@types/react": "18.0.21", - "@types/react-dom": "18.0.6", + "@types/react": "18.2.0", + "@types/react-dom": "18.2.0", "@types/react-timeago": "^4.1.3", "@types/react-window": "^1.8.5", "autoprefixer": "^10.4.12", @@ -2447,9 +2447,10 @@ "dev": true }, "node_modules/@types/react": { - "version": "18.0.21", - "resolved": "https://registry.npmjs.org/@types/react/-/react-18.0.21.tgz", - "integrity": "sha512-7QUCOxvFgnD5Jk8ZKlUAhVcRj7GuJRjnjjiY/IUBWKgOlnvDvTMLD4RTF7NPyVmbRhNrbomZiOepg7M/2Kj1mA==", + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.0.tgz", + "integrity": "sha512-0FLj93y5USLHdnhIhABk83rm8XEGA7kH3cr+YUlvxoUGp1xNt/DINUMvqPxLyOQMzLmZe8i4RTHbvb8MC7NmrA==", + "license": "MIT", "dependencies": { "@types/prop-types": "*", "@types/scheduler": "*", @@ -2457,10 +2458,11 @@ } }, "node_modules/@types/react-dom": { - "version": "18.0.6", - "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.0.6.tgz", - "integrity": "sha512-/5OFZgfIPSwy+YuIBP/FgJnQnsxhZhjjrnxudMddeblOouIodEQ75X14Rr4wGSG/bknL+Omy9iWlLo1u/9GzAA==", + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.0.tgz", + "integrity": "sha512-8yQrvS6sMpSwIovhPOwfyNf2Wz6v/B62LFSVYQ85+Rq3tLsBIG7rP5geMxaijTUxSkrO6RzN/IRuIAADYQsleA==", "dev": true, + "license": "MIT", "dependencies": { "@types/react": "*" } @@ -3395,7 +3397,8 @@ "node_modules/boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", - "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" }, "node_modules/brace-expansion": { "version": "1.1.11", @@ -3708,20 +3711,25 @@ "integrity": "sha512-Ro2JbLmvg83gXF5F4sniaQ+lTbSv18E+TIf2cOeiH1Iqd2PGFOtem+DUufMZsCJwFE7ywPOpfXFBwRTGq7dh6w==" }, "node_modules/cheerio": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", - "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0.tgz", + "integrity": "sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==", + "license": "MIT", "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", - "domutils": "^3.0.1", - "htmlparser2": "^8.0.1", - "parse5": "^7.0.0", - "parse5-htmlparser2-tree-adapter": "^7.0.0" + "domutils": "^3.1.0", + "encoding-sniffer": "^0.2.0", + "htmlparser2": "^9.1.0", + "parse5": "^7.1.2", + "parse5-htmlparser2-tree-adapter": "^7.0.0", + "parse5-parser-stream": "^7.1.2", + "undici": "^6.19.5", + "whatwg-mimetype": "^4.0.0" }, "engines": { - "node": ">= 6" + "node": ">=18.17" }, "funding": { "url": "https://github.com/cheeriojs/cheerio?sponsor=1" @@ -3731,6 +3739,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "license": "BSD-2-Clause", "dependencies": { "boolbase": "^1.0.0", "css-select": "^5.1.0", @@ -3743,6 +3752,15 @@ "url": "https://github.com/sponsors/fb55" } }, + "node_modules/cheerio/node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/chokidar": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", @@ -4490,6 +4508,7 @@ "version": "5.1.0", "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "license": "BSD-2-Clause", "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", @@ -4505,6 +4524,7 @@ "version": "6.1.0", "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "license": "BSD-2-Clause", "engines": { "node": ">= 6" }, @@ -4930,6 +4950,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", @@ -4948,7 +4969,8 @@ "type": "github", "url": "https://github.com/sponsors/fb55" } - ] + ], + "license": "BSD-2-Clause" }, "node_modules/domexception": { "version": "4.0.0", @@ -4966,6 +4988,7 @@ "version": "5.0.3", "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", "dependencies": { "domelementtype": "^2.3.0" }, @@ -4980,6 +5003,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "license": "BSD-2-Clause", "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", @@ -5221,6 +5245,43 @@ "iconv-lite": "^0.6.2" } }, + "node_modules/encoding-sniffer": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.0.tgz", + "integrity": "sha512-ju7Wq1kg04I3HtiYIOrUrdfdDvkyO9s5XM8QAj/bN61Yo/Vb4vgJxy5vi4Yxk01gWHbrofpPtpxM8bKger9jhg==", + "license": "MIT", + "dependencies": { + "iconv-lite": "^0.6.3", + "whatwg-encoding": "^3.1.1" + }, + "funding": { + "url": "https://github.com/fb55/encoding-sniffer?sponsor=1" + } + }, + "node_modules/encoding-sniffer/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/encoding-sniffer/node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/encoding/node_modules/iconv-lite": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", @@ -7145,9 +7206,9 @@ } }, "node_modules/htmlparser2": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", - "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", + "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==", "funding": [ "https://github.com/fb55/htmlparser2?sponsor=1", { @@ -7155,11 +7216,12 @@ "url": "https://github.com/sponsors/fb55" } ], + "license": "MIT", "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", - "domutils": "^3.0.1", - "entities": "^4.4.0" + "domutils": "^3.1.0", + "entities": "^4.5.0" } }, "node_modules/http-cache-semantics": { @@ -11559,6 +11621,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", "dependencies": { "boolbase": "^1.0.0" }, @@ -11927,11 +11990,24 @@ } }, "node_modules/parse5-htmlparser2-tree-adapter": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz", - "integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==", + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", + "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-parser-stream": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", + "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", + "license": "MIT", "dependencies": { - "domhandler": "^5.0.2", "parse5": "^7.0.0" }, "funding": { @@ -15174,6 +15250,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/undici": { + "version": "6.20.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.20.1.tgz", + "integrity": "sha512-AjQF1QsmqfJys+LXfGTNum+qw4S88CojRInG/6t31W/1fk6G59s92bnAvGz5Cmur+kQv2SURXEvvudLmbrE8QA==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, "node_modules/unified": { "version": "11.0.4", "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.4.tgz", diff --git a/package.json b/package.json index fdb0c2d..8042584 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "axios": "^1.1.3", "axios-retry": "^3.3.1", "chart.js": "^3.9.1", - "cheerio": "^1.0.0-rc.12", + "cheerio": "^1.0.0", "concurrently": "^7.6.0", "cookies": "^0.8.0", "croner": "^5.3.5", diff --git a/utils/scraper.ts b/utils/scraper.ts index 6b325f2..2527a10 100644 --- a/utils/scraper.ts +++ b/utils/scraper.ts @@ -1,5 +1,5 @@ import axios, { AxiosResponse, CreateAxiosDefaults } from 'axios'; -import cheerio from 'cheerio'; +import * as cheerio from 'cheerio'; import { readFile, writeFile } from 'fs/promises'; import HttpsProxyAgent from 'https-proxy-agent'; import countries from './countries'; @@ -124,7 +124,7 @@ export const scrapeKeywordFromGoogle = async (keyword:KeywordType, settings:Sett throw new Error(res); } } catch (error:any) { - refreshedResults.error = scraperError; + refreshedResults.error = scraperError || 'Unknown Error'; if (settings.scraper_type === 'proxy' && error && error.response && error.response.statusText) { refreshedResults.error = `[${error.response.status}] ${error.response.statusText}`; }