UniqueSoft/serpbear
2
0
Fork 0
mirror of https://github.com/towfiqi/serpbear synced 2025-06-26 18:15:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: Fixes broken scrape result issue for keywords with special characters.

Browse Source 
closes #221
This commit is contained in:
towfiqi
2024-11-12 09:18:40 +06:00
parent c5714c00ae
commit d9d7c6347e
7 changed files with 14 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
scrapers/services/hasdata.ts
View File

@@ -20,11 +20,10 @@ const hasdata:ScraperSettings = {
scrapeURL: (keyword, settings) => {
const country = keyword.country || 'US';
const countryName = countries[country][0];
const location = keyword.city && countryName ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
return `https://api.scrape-it.cloud/scrape/google/serp?q=${encodeURI(keyword.keyword)}${location}&num=100&gl=${country.toLowerCase()}&deviceType=${keyword.device}`;
const location = keyword.city && countryName ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
return `https://api.scrape-it.cloud/scrape/google/serp?q=${encodeURIComponent(keyword.keyword)}${location}&num=100&gl=${country.toLowerCase()}&deviceType=${keyword.device}`;
},
resultObjectKey: 'organicResults',
serpExtractor: (content) => {
const extractedResult = [];
const results: HasDataResult[] = (typeof content === 'string') ? JSON.parse(content) : content as HasDataResult[];

4
scrapers/services/searchapi.ts
View File

@@ -20,8 +20,8 @@ const searchapi:ScraperSettings = {
scrapeURL: (keyword) => {
const country = keyword.country || 'US';
const countryName = countries[country][0];
const location = keyword.city && countryName ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
return `https://www.searchapi.io/api/v1/search?engine=google&q=${encodeURI(keyword.keyword)}&num=100&gl=${country}&device=${keyword.device}${location}`;
const location = keyword.city && countryName ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
return `https://www.searchapi.io/api/v1/search?engine=google&q=${encodeURIComponent(keyword.keyword)}&num=100&gl=${country}&device=${keyword.device}${location}`;
},
resultObjectKey: 'organic_results',
serpExtractor: (content) => {

4
scrapers/services/serpapi.ts
View File

@@ -19,8 +19,8 @@ const serpapi:ScraperSettings = {
},
scrapeURL: (keyword, settings) => {
const countryName = countries[keyword.country || 'US'][0];
const location = keyword.city && keyword.country ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
return `https://serpapi.com/search?q=${encodeURI(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}${location}&api_key=${settings.scaping_api}`;
const location = keyword.city && keyword.country ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
return `https://serpapi.com/search?q=${encodeURIComponent(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}${location}&api_key=${settings.scaping_api}`;
},
resultObjectKey: 'organic_results',
serpExtractor: (content) => {

3
scrapers/services/serper.ts
View File

@@ -12,7 +12,8 @@ const serper:ScraperSettings = {
scrapeURL: (keyword, settings, countryData) => {
const country = keyword.country || 'US';
const lang = countryData[country][2];
return `https://google.serper.dev/search?q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`;
console.log('Serper URL :', `https://google.serper.dev/search?q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`);
return `https://google.serper.dev/search?q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`;
},
resultObjectKey: 'organic',
serpExtractor: (content) => {

2
scrapers/services/serply.ts
View File

@@ -20,7 +20,7 @@ const serply:ScraperSettings = {
},
scrapeURL: (keyword) => {
const country = scraperCountries.includes(keyword.country.toUpperCase()) ? keyword.country : 'US';
return `https://api.serply.io/v1/search/q=${encodeURI(keyword.keyword)}&num=100&hl=${country}`;
return `https://api.serply.io/v1/search/q=${encodeURIComponent(keyword.keyword)}&num=100&hl=${country}`;
},
resultObjectKey: 'result',
serpExtractor: (content) => {

4
scrapers/services/spaceserp.ts
View File

@@ -15,10 +15,10 @@ const spaceSerp:ScraperSettings = {
scrapeURL: (keyword, settings, countryData) => {
const country = keyword.country || 'US';
const countryName = countries[country][0];
const location = keyword.city ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
const location = keyword.city ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
const device = keyword.device === 'mobile' ? '&device=mobile' : '';
const lang = countryData[country][2];
return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${location}${device}&resultBlocks=`;
return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${location}${device}&resultBlocks=`;
},
resultObjectKey: 'organic_results',
serpExtractor: (content) => {

6
scrapers/services/valueserp.ts
View File

@@ -15,11 +15,11 @@ const valueSerp:ScraperSettings = {
scrapeURL: (keyword, settings, countryData) => {
const country = keyword.country || 'US';
const countryName = countries[country][0];
const location = keyword.city ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
const location = keyword.city ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
const device = keyword.device === 'mobile' ? '&device=mobile' : '';
const lang = countryData[country][2];
console.log(`https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`);
return `https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`;
console.log(`https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`);
return `https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`;
},
resultObjectKey: 'organic_results',
serpExtractor: (content) => {