UniqueSoft/serpbear
2
0
Fork 0
mirror of https://github.com/towfiqi/serpbear synced 2025-06-26 18:15:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

merge in main branch into add-serply

Browse Source
This commit is contained in:
googio 2022-12-01 13:28:43 -05:00
commit e6136db742
13 changed files with 121 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
CHANGELOG.md
View File

@ -2,6 +2,36 @@
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
### [0.1.4](https://github.com/towfiqi/serpbear/compare/v0.1.3...v0.1.4) (2022-12-01)
### Features
* Failed scrape now shows error details in UI. ([8c8064f](https://github.com/towfiqi/serpbear/commit/8c8064f222ea8177b26b6dd28866d1f421faca39))
### Bug Fixes
* Domains with www weren't loading keywords. ([3d1c690](https://github.com/towfiqi/serpbear/commit/3d1c690076a03598f0ac3f3663d905479d945897)), closes [#8](https://github.com/towfiqi/serpbear/issues/8)
* Emails were sending serps of previous day. ([6910558](https://github.com/towfiqi/serpbear/commit/691055811c2ae70ce1b878346300048c1e23f2eb))
* Fixes Broken ScrapingRobot Integration. ([1ed298f](https://github.com/towfiqi/serpbear/commit/1ed298f633a9ae5b402b431f1e50b35ffd44a6dc))
* scraper fails if matched domain has www ([38dc164](https://github.com/towfiqi/serpbear/commit/38dc164514b066b2007f2f3b2ae68005621963cc)), closes [#6](https://github.com/towfiqi/serpbear/issues/6) [#7](https://github.com/towfiqi/serpbear/issues/7)
* scraper fails when result has domain w/o www ([6d7cfec](https://github.com/towfiqi/serpbear/commit/6d7cfec95304fa7a61beaab07f7cd6af215255c3))
### [0.1.3](https://github.com/towfiqi/serpbear/compare/v0.1.2...v0.1.3) (2022-12-01)
### Features
* Adds a search field in Country select field. ([be4db26](https://github.com/towfiqi/serpbear/commit/be4db26316e7522f567a4ce6fc27e0a0f73f89f2)), closes [#2](https://github.com/towfiqi/serpbear/issues/2)
### Bug Fixes
* could not add 2 character domains. ([5acbe18](https://github.com/towfiqi/serpbear/commit/5acbe181ec978b50b588af378d17fb3070c241d1)), closes [#1](https://github.com/towfiqi/serpbear/issues/1)
* license location. ([a45237b](https://github.com/towfiqi/serpbear/commit/a45237b230a9830461cf7fccd4c717235112713b))
* No hint on how to add multiple keywords. ([9fa80cf](https://github.com/towfiqi/serpbear/commit/9fa80cf6098854d2a5bd5a8202aa0fd6886d1ba0)), closes [#3](https://github.com/towfiqi/serpbear/issues/3)
### 0.1.2 (2022-11-30)

29
components/common/SelectField.tsx
View File

@ -30,7 +30,9 @@ const SelectField = (props: SelectFieldProps) => {
flags = false,
emptyMsg = '' } = props;
const [showOptions, setShowOptions] = useState(false);
const [showOptions, setShowOptions] = useState<boolean>(false);
const [filterInput, setFilterInput] = useState<string>('');
const [filterdOptions, setFilterdOptions] = useState<SelectionOption[]>([]);
const selectedLabels = useMemo(() => {
return options.reduce((acc:string[], item:SelectionOption) :string[] => {
@ -51,6 +53,18 @@ const SelectField = (props: SelectFieldProps) => {
if (!multiple) { setShowOptions(false); }
};
const filterOptions = (event:React.FormEvent<HTMLInputElement>) => {
setFilterInput(event.currentTarget.value);
const filteredItems:SelectionOption[] = [];
const userVal = event.currentTarget.value.toLowerCase();
options.forEach((option:SelectionOption) => {
if (flags ? option.label.toLowerCase().startsWith(userVal) : option.label.toLowerCase().includes(userVal)) {
filteredItems.push(option);
}
});
setFilterdOptions(filteredItems);
};
return (
<div className="select font-semibold text-gray-500">
<div
@ -68,8 +82,19 @@ const SelectField = (props: SelectFieldProps) => {
<div
className={`select_list mt-1 border absolute min-w-[${minWidth}px]
${rounded === 'rounded-3xl' ? 'rounded-lg' : rounded} max-h-${maxHeight} bg-white z-50 overflow-y-auto styled-scrollbar`}>
{options.length > 20 && (
<div className=''>
<input
className=' border-b-[1px] p-3 w-full focus:outline-0 focus:border-blue-100'
type="text"
placeholder='Search..'
onChange={filterOptions}
value={filterInput}
/>
</div>
)}
<ul>
{options.map((opt) => {
{(options.length > 20 && filterdOptions.length > 0 && filterInput ? filterdOptions : options).map((opt) => {
const itemActive = selected.includes(opt.value);
return (
<li

2
components/domains/AddDomain.tsx
View File

@ -13,7 +13,7 @@ const AddDomain = ({ closeModal }: AddDomainProps) => {
const addDomain = () => {
// console.log('ADD NEW DOMAIN', newDomain);
if (/^[a-zA-Z0-9][a-zA-Z0-9-]{1,61}[a-zA-Z0-9](?:\.[a-zA-Z]{2,})+$/.test(newDomain)) {
if (/^[a-zA-Z0-9][a-zA-Z0-9-]{0,61}[a-zA-Z0-9](?:\.[a-zA-Z]{2,})+$/.test(newDomain)) {
setNewDomainError(false);
// TODO: Domain Action
addMutate(newDomain);

2
components/keywords/AddKeywords.tsx
View File

@ -49,7 +49,7 @@ const AddKeywords = ({ closeModal, domain, keywords }: AddKeywordsProps) => {
<div>
<textarea
className='w-full h-40 border rounded border-gray-200 p-4 outline-none focus:border-indigo-300'
placeholder='Type or Paste Keywords here...'
placeholder="Type or Paste Keywords here. Insert Each keyword in a New line."
value={newKeywordsData.keywords}
onChange={(e) => setNewKeywordsData({ ...newKeywordsData, keywords: e.target.value })}>
</textarea>

17
components/keywords/Keyword.tsx
View File

@ -21,7 +21,7 @@ type KeywordProps = {
const Keyword = (props: KeywordProps) => {
const { keywordData, refreshkeyword, favoriteKeyword, removeKeyword, selectKeyword, selected, showKeywordDetails, manageTags, lastItem } = props;
const {
keyword, domain, ID, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = 'false',
keyword, domain, ID, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = false,
} = keywordData;
const [showOptions, setShowOptions] = useState(false);
const [showPositionError, setPositionError] = useState(false);
@ -77,7 +77,7 @@ const Keyword = (props: KeywordProps) => {
<span className={`fflag fflag-${country} w-[18px] h-[12px] mr-2`} title={countries[country][0]} />{keyword}
</a>
{sticky && <button className='ml-2 relative top-[2px]' title='Favorite'><Icon type="star-filled" size={16} color="#fbd346" /></button>}
{lastUpdateError !== 'false'
{lastUpdateError && lastUpdateError.date
&& <button className='ml-2 relative top-[2px]' onClick={() => setPositionError(true)}>
<Icon type="error" size={18} color="#FF3672" />
</button>
@ -133,16 +133,19 @@ const Keyword = (props: KeywordProps) => {
</ul>
)}
</div>
{lastUpdateError !== 'false' && showPositionError
&& <div className=' absolute mt-[-70px] p-2 bg-white z-30 border border-red-200 rounded w-[220px] left-4 shadow-sm text-xs'>
{lastUpdateError && lastUpdateError.date && showPositionError && (
<div className=' absolute mt-[-70px] p-2 bg-white z-30 border border-red-200 rounded w-[220px] left-4 shadow-sm text-xs lg:bottom-12'>
Error Updating Keyword position (Tried <TimeAgo
title={dayjs(lastUpdateError).format('DD-MMM-YYYY, hh:mm:ss A')}
date={lastUpdateError} />)
title={dayjs(lastUpdateError.date).format('DD-MMM-YYYY, hh:mm:ss A')}
date={lastUpdateError.date} />)
<i className='absolute top-0 right-0 ml-2 p-2 font-semibold not-italic cursor-pointer' onClick={() => setPositionError(false)}>
<Icon type="close" size={16} color="#999" />
</i>
<div className=' border-t-[1px] border-red-100 mt-2 pt-1'>
{lastUpdateError.scraper && <strong className='capitalize'>{lastUpdateError.scraper}: </strong>}{lastUpdateError.error}
</div>
</div>
}
)}
</div>
);
};

5
cron.js
View File

@ -50,6 +50,9 @@ const generateCronTime = (interval) => {
if (interval === 'daily') {
cronTime = '0 0 0 * * *';
}
if (interval === 'daily_morning') {
cronTime = '0 0 0 7 * *';
}
if (interval === 'weekly') {
cronTime = '0 0 0 */7 * *';
}
@ -103,7 +106,7 @@ const runAppCronJobs = () => {
getAppSettings().then((settings) => {
const notif_interval = (!settings.notification_interval || settings.notification_interval === 'never') ? false : settings.notification_interval;
if (notif_interval) {
const cronTime = generateCronTime(notif_interval);
const cronTime = generateCronTime(notif_interval === 'daily' ? 'daily_morning' : notif_interval);
if (cronTime) {
cron.schedule(cronTime, () => {
// console.log('### Sending Notification Email...');

4
package-lock.json generated
View File

@ -1,12 +1,12 @@
{
"name": "serpbear",
"version": "0.1.2",
"version": "0.1.4",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "serpbear",
"version": "0.1.2",
"version": "0.1.4",
"dependencies": {
"@testing-library/react": "^13.4.0",
"@types/react-transition-group": "^4.4.5",

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "serpbear",
"version": "0.1.2",
"version": "0.1.4",
"private": true,
"scripts": {
"dev": "next dev",

3
pages/api/keywords.ts
View File

@ -44,7 +44,8 @@ const getKeywords = async (req: NextApiRequest, res: NextApiResponse<KeywordsGet
if (!req.query.domain && typeof req.query.domain !== 'string') {
return res.status(400).json({ error: 'Domain is Required!' });
}
const domain = (req.query.domain as string).replace('-', '.');
const domain = (req.query.domain as string).replaceAll('-', '.');
try {
const allKeywords:Keyword[] = await Keyword.findAll({ where: { domain } });
const keywords: KeywordType[] = parseKeywords(allKeywords.map((e) => e.get({ plain: true })));

13
pages/api/refresh.ts
View File

@ -79,18 +79,21 @@ export const refreshAndUpdateKeywords = async (initKeywords:Keyword[], settings:
const newPos = udpatedkeyword.position;
const newPosition = newPos !== false ? newPos : keyword.position;
const { history } = keyword;
const currentDate = new Date();
history[`${currentDate.getFullYear()}-${currentDate.getMonth() + 1}-${currentDate.getDate()}`] = newPosition;
const theDate = new Date();
history[`${theDate.getFullYear()}-${theDate.getMonth() + 1}-${theDate.getDate()}`] = newPosition;
const updatedVal = {
position: newPosition,
updating: false,
url: udpatedkeyword.url,
lastResult: udpatedkeyword.result,
history,
lastUpdated: udpatedkeyword.error ? keyword.lastUpdated : new Date().toJSON(),
lastUpdateError: udpatedkeyword.error ? new Date().toJSON() : 'false',
lastUpdated: udpatedkeyword.error ? keyword.lastUpdated : theDate.toJSON(),
lastUpdateError: udpatedkeyword.error
? JSON.stringify({ date: theDate.toJSON(), error: `${udpatedkeyword.error}`, scraper: settings.scraper_type })
: 'false',
};
updatedKeywords.push({ ...keyword, ...updatedVal });
updatedKeywords.push({ ...keyword, ...{ ...updatedVal, lastUpdateError: JSON.parse(updatedVal.lastUpdateError) } });
// If failed, Add to Retry Queue Cron
if (udpatedkeyword.error) {

2
types.d.ts vendored
View File

@ -31,7 +31,7 @@ type KeywordType = {
url: string,
tags: string[],
updating: boolean,
lastUpdateError: string
lastUpdateError: {date: string, error: string, scraper: string} | false
}
type KeywordLastResult = {

1
utils/parseKeywords.ts
View File

@ -11,6 +11,7 @@ const parseKeywords = (allKeywords: Keyword[]) : KeywordType[] => {
history: JSON.parse(keywrd.history),
tags: JSON.parse(keywrd.tags),
lastResult: JSON.parse(keywrd.lastResult),
lastUpdateError: keywrd.lastUpdateError !== 'false' && keywrd.lastUpdateError.includes('{') ? JSON.parse(keywrd.lastUpdateError) : false,
}));
return parsedItems;
};

61
utils/scraper.ts
View File

@ -20,10 +20,10 @@ type SERPObject = {
export type RefreshResult = false | {
ID: number,
keyword: string,
position:number|boolean,
position:number | boolean,
url: string,
result: SearchResult[],
error?: boolean
error?: boolean | string
}
interface SerplyResult {
@ -62,7 +62,7 @@ export const getScraperClient = (keyword:KeywordType, settings:SettingsType): Pr
if (settings && settings.scraper_type === 'scrapingrobot' && settings.scaping_api) {
const country = keyword.country || 'US';
const lang = countries[country][2];
apiURL = `https://api.scrapingrobot.com/?url=https%3A%2F%2Fwww.google.com%2Fsearch%3Fnum%3D100%26hl%3D${lang}%26q%3D${encodeURI(keyword.keyword)}&token=${settings.scaping_api}&proxyCountry=${country}&render=false${keyword.device === 'mobile' ? '&mobile=true' : ''}`;
apiURL = `https://api.scrapingrobot.com/?token=${settings.scaping_api}&proxyCountry=${country}&render=false${keyword.device === 'mobile' ? '&mobile=true' : ''}&url=https%3A%2F%2Fwww.google.com%2Fsearch%3Fnum%3D100%26hl%3D${lang}%26q%3D${encodeURI(keyword.keyword)}`;
}
// Serply.io docs https://docs.serply.io/api
@ -75,7 +75,7 @@ export const getScraperClient = (keyword:KeywordType, settings:SettingsType): Pr
headers['X-User-Agent'] = 'desktop';
}
headers['X-Proxy-Location'] = country;
headers['X-Api-Key'] = settings.scaping_api
headers['X-Api-Key'] = settings.scaping_api;
apiURL = `https://api.serply.io/v1/search/q=${encodeURI(keyword.keyword)}&num=100&hl=${country}`;
}
@ -96,8 +96,7 @@ export const getScraperClient = (keyword:KeywordType, settings:SettingsType): Pr
const axiosClient = axios.create(axiosConfig);
client = axiosClient.get(`https://www.google.com/search?num=100&q=${encodeURI(keyword.keyword)}`);
} else {
console.log(`calling ${apiURL}`);
client = fetch(apiURL, { method: 'GET', headers }).then((res) => res.json());
client = fetch(apiURL, { method: 'GET', headers });
}
return client;
@ -121,18 +120,26 @@ export const scrapeKeywordFromGoogle = async (keyword:KeywordType, settings:Sett
const scraperClient = getScraperClient(keyword, settings);
if (!scraperClient) { return false; }
let res:any = null; let scraperError:any = null;
try {
const res:any = await scraperClient;
if (res && (res.data || res.html || res.results)) {
// writeFile('result.txt', res.data, { encoding: 'utf-8' });
const extracted = extractScrapedResult(res.data || res.html || res.results, settings.scraper_type);
if (settings && settings.scraper_type === 'proxy' && settings.proxy) {
res = await scraperClient;
} else {
res = await scraperClient.then((result:any) => result.json());
}
if (res && (res.data || res.html || res.result || res.results)) {
const extracted = extractScrapedResult(res.data || res.html || res.result || res.results, settings.scraper_type);
const serp = getSerp(keyword.domain, extracted);
refreshedResults = { ID: keyword.ID, keyword: keyword.keyword, position: serp.postion, url: serp.url, result: extracted, error: false };
console.log('SERP: ', keyword.keyword, serp.postion, serp.url);
} else {
scraperError = res.detail || res.error || 'Unknown Error';
throw new Error(res);
}
} catch (error:any) {
console.log('#### SCRAPE ERROR: ', keyword.keyword, error?.code, error?.response?.status, error?.response?.data, error);
console.log('#### SCRAPE ERROR: ', keyword.keyword, '. Error: ', scraperError);
refreshedResults.error = scraperError;
}
return refreshedResults;
@ -147,21 +154,7 @@ export const scrapeKeywordFromGoogle = async (keyword:KeywordType, settings:Sett
export const extractScrapedResult = (content: string, scraper_type:string): SearchResult[] => {
const extractedResult = [];
if (scraper_type === 'serply') {
// results already in json
const results: SerplyResult[] = (typeof content === 'string') ? JSON.parse(content) : content as SerplyResult[];
for (const result of results) {
if (result.title && result.link) {
extractedResult.push({
title: result.title,
url: result.link,
position: result.realPosition,
});
}
}
return extractedResult;
}
const $ = cheerio.load(content);
const hasNumberofResult = $('body').find('#search > div > div');
const searchResult = hasNumberofResult.children();
@ -176,6 +169,18 @@ export const extractScrapedResult = (content: string, scraper_type:string): Sear
const cleanedURL = url ? url.replace('/url?q=', '').replace(/&sa=.*/, '') : '';
extractedResult.push({ title, url: cleanedURL, position: index });
}
} else if (scraper_type === 'serply') {
// results already in json
const results: SerplyResult[] = (typeof content === 'string') ? JSON.parse(content) : content as SerplyResult[];
for (const result of results) {
if (result.title && result.link) {
extractedResult.push({
title: result.title,
url: result.link,
position: result.realPosition,
});
}
}
} else {
for (let i = 1; i < searchResult.length; i += 1) {
if (searchResult[i]) {
@ -200,8 +205,8 @@ export const extractScrapedResult = (content: string, scraper_type:string): Sear
export const getSerp = (domain:string, result:SearchResult[]) : SERPObject => {
if (result.length === 0 || !domain) { return { postion: false, url: '' }; }
const foundItem = result.find((item) => {
const itemDomain = item.url.match(/^(?:https?:)?(?:\/\/)?([^/?]+)/i);
return itemDomain && itemDomain.includes(domain);
const itemDomain = item.url.replace('www.', '').match(/^(?:https?:)?(?:\/\/)?([^/?]+)/i);
return itemDomain && itemDomain.includes(domain.replace('www.', ''));
});
return { postion: foundItem ? foundItem.position : 0, url: foundItem && foundItem.url ? foundItem.url : '' };
};