mirror of
https://github.com/towfiqi/serpbear
synced 2025-06-26 18:15:54 +00:00
Compare commits
19 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e61dfb5b90 | ||
|
|
b9d58a721d | ||
|
|
b83df5f3db | ||
|
|
3b6d034d6f | ||
|
|
5dd366b91e | ||
|
|
5fc1779783 | ||
|
|
c5af94a146 | ||
|
|
c406588953 | ||
|
|
3c48d130b6 | ||
|
|
99dbbd1dd9 | ||
|
|
acc0b39d80 | ||
|
|
a1108d240e | ||
|
|
d9e0d0107a | ||
|
|
9e9dad7631 | ||
|
|
8139e399c1 | ||
|
|
cb24696a1f | ||
|
|
b50733defc | ||
|
|
0c8b457eee | ||
|
|
123ad81dae |
50
CHANGELOG.md
50
CHANGELOG.md
@@ -2,6 +2,56 @@
|
||||
|
||||
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
||||
|
||||
### [0.2.5](https://github.com/towfiqi/serpbear/compare/v0.2.4...v0.2.5) (2023-03-07)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* Adds current App version Number in Footer. ([b83df5f](https://github.com/towfiqi/serpbear/commit/b83df5f3dbd64db657d31f0526438e7165e1b475))
|
||||
* Adds Keyword Scraping Interval Settings. ([3b6d034](https://github.com/towfiqi/serpbear/commit/3b6d034d6f7da0b4259070220fffff44184dd680)), closes [#81](https://github.com/towfiqi/serpbear/issues/81) [#76](https://github.com/towfiqi/serpbear/issues/76)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Fixes Broken Image thumbnail loading issue. ([5dd366b](https://github.com/towfiqi/serpbear/commit/5dd366b91e2a94e658bf5250a8a0fa64c09e1c11))
|
||||
* Settings Update Toast was not showing up. ([b9d58a7](https://github.com/towfiqi/serpbear/commit/b9d58a721df12f3f34220a3ae5da6897e23c83ec))
|
||||
|
||||
### [0.2.4](https://github.com/towfiqi/serpbear/compare/v0.2.3...v0.2.4) (2023-02-15)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* Keyword ranking pages can now be clicked. ([c5af94a](https://github.com/towfiqi/serpbear/commit/c5af94a1469713ed4092253d26953ee0ed28c25d))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Fixes broken Login on windows ([c406588](https://github.com/towfiqi/serpbear/commit/c406588953035e4177a64011c13eb0e3aedffe89))
|
||||
* Fixes Node Cron memory leak issue. ([3c48d13](https://github.com/towfiqi/serpbear/commit/3c48d130b6f229a4ac27ec43ef1ea3a6640cecf6))
|
||||
|
||||
### [0.2.3](https://github.com/towfiqi/serpbear/compare/v0.2.2...v0.2.3) (2023-01-12)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* Ability to tag multiple keywords at once ([9e9dad7](https://github.com/towfiqi/serpbear/commit/9e9dad7631691b2a836fdd4c522b1f933b17e285)), closes [#54](https://github.com/towfiqi/serpbear/issues/54)
|
||||
* Set USERNAME as well as USER variable ([b50733d](https://github.com/towfiqi/serpbear/commit/b50733defc2c06e0f92ca3e88fd1f74684eee9c0))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Fixes Position and View Sort. ([8139e39](https://github.com/towfiqi/serpbear/commit/8139e399c13ab8be767facef9a19c67dec06ed64)), closes [#46](https://github.com/towfiqi/serpbear/issues/46)
|
||||
* Fixes wrong CTR value for Search Console Data ([cb24696](https://github.com/towfiqi/serpbear/commit/cb24696a1f47b02a11c68cd1c673ea8b1bacd144)), closes [#48](https://github.com/towfiqi/serpbear/issues/48)
|
||||
* Mobile Keyword Scraping not working. ([a1108d2](https://github.com/towfiqi/serpbear/commit/a1108d240ea38ab0886ef3722b0c937ec5a45591)), closes [#58](https://github.com/towfiqi/serpbear/issues/58)
|
||||
* ScrapingAnt Mobile Keyword Scrape not working ([acc0b39](https://github.com/towfiqi/serpbear/commit/acc0b39d80d4f9371967a0d425ed205c5d866eea))
|
||||
|
||||
### [0.2.2](https://github.com/towfiqi/serpbear/compare/v0.2.1...v0.2.2) (2022-12-25)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Fixes bug that prevents Saving API settings ([123ad81](https://github.com/towfiqi/serpbear/commit/123ad81dae10aa28848148d0f3da5cf1f7de7c57)), closes [#45](https://github.com/towfiqi/serpbear/issues/45)
|
||||
|
||||
### [0.2.1](https://github.com/towfiqi/serpbear/compare/v0.2.0...v0.2.1) (2022-12-24)
|
||||
|
||||
## [0.2.0](https://github.com/towfiqi/serpbear/compare/v0.1.7...v0.2.0) (2022-12-21)
|
||||
|
||||
@@ -29,7 +29,7 @@ COPY --from=builder --chown=nextjs:nodejs /app/cron.js ./
|
||||
COPY --from=builder --chown=nextjs:nodejs /app/email ./email
|
||||
RUN rm package.json
|
||||
RUN npm init -y
|
||||
RUN npm i cryptr dotenv node-cron @googleapis/searchconsole
|
||||
RUN npm i cryptr dotenv croner @googleapis/searchconsole
|
||||
RUN npm i -g concurrently
|
||||
|
||||
USER nextjs
|
||||
|
||||
@@ -9,10 +9,11 @@ import Icon from '../common/Icon';
|
||||
type DomainItemProps = {
|
||||
domain: DomainType,
|
||||
selected: boolean,
|
||||
isConsoleIntegrated: boolean
|
||||
isConsoleIntegrated: boolean,
|
||||
thumb: string,
|
||||
}
|
||||
|
||||
const DomainItem = ({ domain, selected, isConsoleIntegrated = false }: DomainItemProps) => {
|
||||
const DomainItem = ({ domain, selected, isConsoleIntegrated = false, thumb }: DomainItemProps) => {
|
||||
const { keywordsUpdated, slug, keywordCount = 0, avgPosition = 0, scVisits = 0, scImpressions = 0, scPosition = 0 } = domain;
|
||||
// const router = useRouter();
|
||||
return (
|
||||
@@ -21,10 +22,10 @@ const DomainItem = ({ domain, selected, isConsoleIntegrated = false }: DomainIte
|
||||
<a className='flex flex-col lg:flex-row'>
|
||||
<div className={`flex-1 p-6 flex ${!isConsoleIntegrated ? 'basis-1/3' : ''}`}>
|
||||
<div className="domain_thumb w-20 h-20 mr-6 bg-slate-100 rounded border border-gray-200 overflow-hidden">
|
||||
<img src={`https://image.thum.io/get/maxAge/96/width/200/https://${domain.domain}`} alt={domain.domain} />
|
||||
{thumb && <img src={thumb} alt={domain.domain} />}
|
||||
</div>
|
||||
<div className="domain_details flex-1">
|
||||
<h3 className='font-semibold text-base mb-2 capitalize'>{domain.domain}</h3>
|
||||
<h3 className='font-semibold text-base mb-2'>{domain.domain}</h3>
|
||||
{keywordsUpdated && (
|
||||
<span className=' text-gray-600 text-xs'>
|
||||
Updated <TimeAgo title={dayjs(keywordsUpdated).format('DD-MMM-YYYY, hh:mm:ss A')} date={keywordsUpdated} />
|
||||
|
||||
60
components/keywords/AddTags.tsx
Normal file
60
components/keywords/AddTags.tsx
Normal file
@@ -0,0 +1,60 @@
|
||||
import { useState } from 'react';
|
||||
import { useUpdateKeywordTags } from '../../services/keywords';
|
||||
import Icon from '../common/Icon';
|
||||
import Modal from '../common/Modal';
|
||||
|
||||
type AddTagsProps = {
|
||||
keywords: KeywordType[],
|
||||
closeModal: Function
|
||||
}
|
||||
|
||||
const AddTags = ({ keywords = [], closeModal }: AddTagsProps) => {
|
||||
const [tagInput, setTagInput] = useState('');
|
||||
const [inputError, setInputError] = useState('');
|
||||
const { mutate: updateMutate } = useUpdateKeywordTags(() => { setTagInput(''); });
|
||||
|
||||
const addTag = () => {
|
||||
if (keywords.length === 0) { return; }
|
||||
if (!tagInput) {
|
||||
setInputError('Please Insert a Tag!');
|
||||
setTimeout(() => { setInputError(''); }, 3000);
|
||||
return;
|
||||
}
|
||||
|
||||
const tagsArray = tagInput.split(',').map((t) => t.trim());
|
||||
const tagsPayload:any = {};
|
||||
keywords.forEach((keyword:KeywordType) => {
|
||||
tagsPayload[keyword.ID] = [...keyword.tags, ...tagsArray];
|
||||
});
|
||||
updateMutate({ tags: tagsPayload });
|
||||
};
|
||||
|
||||
return (
|
||||
<Modal closeModal={() => { closeModal(false); }} title={`Add New Tags to ${keywords.length} Selected Keyword`}>
|
||||
<div className="relative">
|
||||
{inputError && <span className="absolute top-[-24px] text-red-400 text-sm font-semibold">{inputError}</span>}
|
||||
<span className='absolute text-gray-400 top-3 left-2'><Icon type="tags" size={16} /></span>
|
||||
<input
|
||||
className='w-full border rounded border-gray-200 py-3 px-4 pl-8 outline-none focus:border-indigo-300'
|
||||
placeholder='Insert Tags. eg: tag1, tag2'
|
||||
value={tagInput}
|
||||
onChange={(e) => setTagInput(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.code === 'Enter') {
|
||||
e.preventDefault();
|
||||
addTag();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<button
|
||||
className=" absolute right-2 top-2 cursor-pointer rounded p-2 px-4 bg-indigo-600 text-white font-semibold text-sm"
|
||||
onClick={addTag}>
|
||||
Apply
|
||||
</button>
|
||||
</div>
|
||||
</Modal>
|
||||
|
||||
);
|
||||
};
|
||||
|
||||
export default AddTags;
|
||||
@@ -115,7 +115,10 @@ const Keyword = (props: KeywordProps) => {
|
||||
<div
|
||||
className={`keyword_url inline-block mt-4 mr-5 ml-5 lg:flex-1 text-gray-400 lg:m-0 max-w-[70px]
|
||||
overflow-hidden text-ellipsis whitespace-nowrap lg:max-w-none lg:pr-5`}>
|
||||
<span className='mr-3 lg:hidden'><Icon type="link-alt" size={14} color="#999" /></span>{turncatedURL || '-'}</div>
|
||||
<a href={url} target="_blank" rel="noreferrer"><span className='mr-3 lg:hidden'>
|
||||
<Icon type="link-alt" size={14} color="#999" /></span>{turncatedURL || '-'}
|
||||
</a>
|
||||
</div>
|
||||
<div
|
||||
className='inline-block mt-[4] top-[-5px] relative lg:flex-1 lg:m-0'>
|
||||
<span className='mr-2 lg:hidden'><Icon type="clock" size={14} color="#999" /></span>
|
||||
|
||||
@@ -85,7 +85,7 @@ const KeywordDetails = ({ keyword, closeDetails }:KeywordDetailsProps) => {
|
||||
<h3 className=' text-lg font-bold'>
|
||||
<span title={countries[keyword.country][0]}
|
||||
className={`fflag fflag-${keyword.country} w-[18px] h-[12px] mr-2`} /> {keyword.keyword}
|
||||
<span className='py-1 px-2 rounded bg-blue-50 text-blue-700 text-xs font-bold'>{keyword.position}</span>
|
||||
<span className='py-1 px-2 ml-2 rounded bg-blue-50 text-blue-700 text-xs font-bold'>{keyword.position}</span>
|
||||
</h3>
|
||||
<button
|
||||
className='absolute top-2 right-2 p-2 px-3 text-gray-400 hover:text-gray-700 transition-all hover:rotate-90'
|
||||
|
||||
@@ -82,10 +82,10 @@ const KeywordFilters = (props: KeywordFilterProps) => {
|
||||
{ value: 'alpha_desc', label: 'Alphabetically(Z-A)' },
|
||||
];
|
||||
if (integratedConsole) {
|
||||
sortOptionChoices.push({ value: 'imp_asc', label: `Most Viewed${isConsole ? ' (Default)' : ''}` });
|
||||
sortOptionChoices.push({ value: 'imp_desc', label: 'Least Viewed' });
|
||||
sortOptionChoices.push({ value: 'visits_asc', label: 'Most Visited' });
|
||||
sortOptionChoices.push({ value: 'visits_desc', label: 'Least Visited' });
|
||||
sortOptionChoices.push({ value: 'imp_desc', label: `Most Viewed${isConsole ? ' (Default)' : ''}` });
|
||||
sortOptionChoices.push({ value: 'imp_asc', label: 'Least Viewed' });
|
||||
sortOptionChoices.push({ value: 'visits_desc', label: 'Most Visited' });
|
||||
sortOptionChoices.push({ value: 'visits_asc', label: 'Least Visited' });
|
||||
}
|
||||
if (isConsole) {
|
||||
sortOptionChoices.splice(2, 2);
|
||||
|
||||
@@ -2,6 +2,7 @@ import { useState } from 'react';
|
||||
import { useUpdateKeywordTags } from '../../services/keywords';
|
||||
import Icon from '../common/Icon';
|
||||
import Modal from '../common/Modal';
|
||||
import AddTags from './AddTags';
|
||||
|
||||
type keywordTagManagerProps = {
|
||||
keyword: KeywordType|undefined,
|
||||
@@ -10,9 +11,8 @@ type keywordTagManagerProps = {
|
||||
}
|
||||
|
||||
const KeywordTagManager = ({ keyword, closeModal }: keywordTagManagerProps) => {
|
||||
const [tagInput, setTagInput] = useState('');
|
||||
const [inputError, setInputError] = useState('');
|
||||
const { mutate: updateMutate } = useUpdateKeywordTags(() => { setTagInput(''); });
|
||||
const [showAddTag, setShowAddTag] = useState<boolean>(false);
|
||||
const { mutate: updateMutate } = useUpdateKeywordTags(() => { });
|
||||
|
||||
const removeTag = (tag:String) => {
|
||||
if (!keyword) { return; }
|
||||
@@ -20,24 +20,6 @@ const KeywordTagManager = ({ keyword, closeModal }: keywordTagManagerProps) => {
|
||||
updateMutate({ tags: { [keyword.ID]: newTags } });
|
||||
};
|
||||
|
||||
const addTag = () => {
|
||||
if (!keyword) { return; }
|
||||
if (!tagInput) {
|
||||
setInputError('Please Insert a Tag!');
|
||||
setTimeout(() => { setInputError(''); }, 3000);
|
||||
return;
|
||||
}
|
||||
if (keyword.tags.includes(tagInput)) {
|
||||
setInputError('Tag Exist!');
|
||||
setTimeout(() => { setInputError(''); }, 3000);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('New Tag: ', tagInput);
|
||||
const newTags = [...keyword.tags, tagInput.trim()];
|
||||
updateMutate({ tags: { [keyword.ID]: newTags } });
|
||||
};
|
||||
|
||||
return (
|
||||
<Modal closeModal={() => { closeModal(false); }} title={`Tags for Keyword "${keyword && keyword.keyword}"`}>
|
||||
<div className="text-sm my-8 ">
|
||||
@@ -53,31 +35,27 @@ const KeywordTagManager = ({ keyword, closeModal }: keywordTagManagerProps) => {
|
||||
</button>
|
||||
</li>;
|
||||
})}
|
||||
<li className='inline-block py-1 px-1'>
|
||||
<button
|
||||
title='Add New Tag'
|
||||
className="cursor-pointer rounded p-1 px-3 bg-indigo-600 text-white font-semibold text-sm"
|
||||
onClick={() => setShowAddTag(true)}>+</button>
|
||||
</li>
|
||||
</ul>
|
||||
)}
|
||||
{keyword && keyword.tags.length === 0 && (
|
||||
<div className="text-center w-full text-gray-500">No Tags Added to this Keyword.</div>
|
||||
<div className="text-center w-full text-gray-500">
|
||||
No Tags Added to this Keyword. <button className=' text-indigo-600' onClick={() => setShowAddTag(true)}>+ Add Tag</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="relative">
|
||||
{inputError && <span className="absolute top-[-24px] text-red-400 text-sm font-semibold">{inputError}</span>}
|
||||
<span className='absolute text-gray-400 top-3 left-2'><Icon type="tags" size={16} /></span>
|
||||
<input
|
||||
className='w-full border rounded border-gray-200 py-3 px-4 pl-8 outline-none focus:border-indigo-300'
|
||||
placeholder='Insert Tags'
|
||||
value={tagInput}
|
||||
onChange={(e) => setTagInput(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.code === 'Enter') {
|
||||
e.preventDefault();
|
||||
addTag();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<button className=" absolute right-2 top-2 cursor-pointer rounded p-1 px-4 bg-blue-600 text-white font-bold" onClick={addTag}>+</button>
|
||||
</div>
|
||||
{showAddTag && keyword && (
|
||||
<AddTags
|
||||
keywords={[keyword]}
|
||||
closeModal={() => setShowAddTag(false)}
|
||||
/>
|
||||
)}
|
||||
</Modal>
|
||||
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import KeywordFilters from './KeywordFilter';
|
||||
import Modal from '../common/Modal';
|
||||
import { useDeleteKeywords, useFavKeywords, useRefreshKeywords } from '../../services/keywords';
|
||||
import KeywordTagManager from './KeywordTagManager';
|
||||
import AddTags from './AddTags';
|
||||
|
||||
type KeywordsTableProps = {
|
||||
domain: DomainType | null,
|
||||
@@ -28,6 +29,7 @@ const KeywordsTable = (props: KeywordsTableProps) => {
|
||||
const [showKeyDetails, setShowKeyDetails] = useState<KeywordType|null>(null);
|
||||
const [showRemoveModal, setShowRemoveModal] = useState<boolean>(false);
|
||||
const [showTagManager, setShowTagManager] = useState<null|number>(null);
|
||||
const [showAddTags, setShowAddTags] = useState<boolean>(false);
|
||||
const [filterParams, setFilterParams] = useState<KeywordFilters>({ countries: [], tags: [], search: '' });
|
||||
const [sortBy, setSortBy] = useState<string>('date_asc');
|
||||
const [scDataType, setScDataType] = useState<string>('threeDays');
|
||||
@@ -79,7 +81,7 @@ const KeywordsTable = (props: KeywordsTableProps) => {
|
||||
className='block px-2 py-2 cursor-pointer hover:text-indigo-600'
|
||||
onClick={() => { refreshMutate({ ids: selectedKeywords }); setSelectedKeywords([]); }}
|
||||
>
|
||||
<span className=' bg-indigo-100 text-blue-700 px-1 rounded'><Icon type="reload" size={11} /></span> Refresh Keyword
|
||||
<span className=' bg-indigo-100 text-blue-700 px-1 rounded'><Icon type="reload" size={11} /></span> Refresh Keywords
|
||||
</a>
|
||||
</li>
|
||||
<li className='inline-block mr-4'>
|
||||
@@ -87,7 +89,14 @@ const KeywordsTable = (props: KeywordsTableProps) => {
|
||||
className='block px-2 py-2 cursor-pointer hover:text-indigo-600'
|
||||
onClick={() => setShowRemoveModal(true)}
|
||||
>
|
||||
<span className=' bg-red-100 text-red-600 px-1 rounded'><Icon type="trash" size={14} /></span> Remove Keyword</a>
|
||||
<span className=' bg-red-100 text-red-600 px-1 rounded'><Icon type="trash" size={14} /></span> Remove Keywords</a>
|
||||
</li>
|
||||
<li className='inline-block mr-4'>
|
||||
<a
|
||||
className='block px-2 py-2 cursor-pointer hover:text-indigo-600'
|
||||
onClick={() => setShowAddTags(true)}
|
||||
>
|
||||
<span className=' bg-green-100 text-green-500 px-1 rounded'><Icon type="tags" size={14} /></span> Tag Keywords</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
@@ -222,6 +231,12 @@ const KeywordsTable = (props: KeywordsTableProps) => {
|
||||
closeModal={() => setShowTagManager(null)}
|
||||
/>
|
||||
)}
|
||||
{showAddTags && (
|
||||
<AddTags
|
||||
keywords={keywords.filter((k) => selectedKeywords.includes(k.ID))}
|
||||
closeModal={() => setShowAddTags(false)}
|
||||
/>
|
||||
)}
|
||||
<Toaster position='bottom-center' containerClassName="react_toaster" />
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -26,7 +26,7 @@ const SCKeywordsTable = ({ domain, keywords = [], isLoading = true, isConsoleInt
|
||||
const [device, setDevice] = useState<string>('desktop');
|
||||
const [selectedKeywords, setSelectedKeywords] = useState<string[]>([]);
|
||||
const [filterParams, setFilterParams] = useState<KeywordFilters>({ countries: [], tags: [], search: '' });
|
||||
const [sortBy, setSortBy] = useState<string>('imp_asc');
|
||||
const [sortBy, setSortBy] = useState<string>('imp_desc');
|
||||
const [isMobile, setIsMobile] = useState<boolean>(false);
|
||||
const [SCListHeight, setSCListHeight] = useState(500);
|
||||
const { keywordsData } = useFetchKeywords(router);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import React, { useEffect, useState } from 'react';
|
||||
import { Toaster } from 'react-hot-toast';
|
||||
// import { useQuery } from 'react-query';
|
||||
import useUpdateSettings, { useFetchSettings } from '../../services/settings';
|
||||
import Icon from '../common/Icon';
|
||||
@@ -63,21 +64,20 @@ const Settings = ({ closeSettings }:SettingsProps) => {
|
||||
|
||||
const performUpdate = () => {
|
||||
let error: null|SettingsError = null;
|
||||
if (settings.notification_interval !== 'never') {
|
||||
const { notification_interval, notification_email, notification_email_from, scraper_type, smtp_port, smtp_server, scaping_api } = settings;
|
||||
if (notification_interval !== 'never') {
|
||||
if (!settings.notification_email) {
|
||||
error = { type: 'no_email', msg: 'Insert a Valid Email address' };
|
||||
}
|
||||
if (settings.notification_email
|
||||
&& (!settings.smtp_port || !settings.smtp_server
|
||||
|| !settings.notification_email_from)) {
|
||||
if (notification_email && (!smtp_port || !smtp_server || !notification_email_from)) {
|
||||
let type = 'no_smtp_from';
|
||||
if (!settings.smtp_port) { type = 'no_smtp_port'; }
|
||||
if (!settings.smtp_server) { type = 'no_smtp_server'; }
|
||||
if (!smtp_port) { type = 'no_smtp_port'; }
|
||||
if (!smtp_server) { type = 'no_smtp_server'; }
|
||||
error = { type, msg: 'Insert SMTP Server details that will be used to send the emails.' };
|
||||
}
|
||||
}
|
||||
|
||||
if (settings?.scraper_type !== 'proxy' && settings?.scraper_type !== 'none') {
|
||||
if (scraper_type !== 'proxy' && scraper_type !== 'none' && !scaping_api) {
|
||||
error = { type: 'no_api_key', msg: 'Insert a Valid API Key or Token for the Scraper Service.' };
|
||||
}
|
||||
|
||||
@@ -98,6 +98,13 @@ const Settings = ({ closeSettings }:SettingsProps) => {
|
||||
{ label: 'Monthly', value: 'monthly' },
|
||||
{ label: 'Never', value: 'never' },
|
||||
];
|
||||
const scrapingOptions: SelectionOption[] = [
|
||||
{ label: 'Daily', value: 'daily' },
|
||||
{ label: 'Every Other Day', value: 'other_day' },
|
||||
{ label: 'Weekly', value: 'weekly' },
|
||||
{ label: 'Monthly', value: 'monthly' },
|
||||
{ label: 'Never', value: 'never' },
|
||||
];
|
||||
const allScrapers: SelectionOption[] = settings.available_scapers ? settings.available_scapers : [];
|
||||
const scraperOptions: SelectionOption[] = [{ label: 'None', value: 'none' }, ...allScrapers];
|
||||
|
||||
@@ -170,6 +177,22 @@ const Settings = ({ closeSettings }:SettingsProps) => {
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
{settings.scraper_type !== 'none' && (
|
||||
<div className="settings__section__input mb-5">
|
||||
<label className={labelStyle}>Scraping Frequency</label>
|
||||
<SelectField
|
||||
multiple={false}
|
||||
selected={[settings?.scrape_interval || 'daily']}
|
||||
options={scrapingOptions}
|
||||
defaultLabel={'Notification Settings'}
|
||||
updateField={(updated:string[]) => updated[0] && updateSettings('scrape_interval', updated[0])}
|
||||
rounded='rounded'
|
||||
maxHeight={48}
|
||||
minWidth={270}
|
||||
/>
|
||||
<small className=' text-gray-500 pt-2 block'>This option requires Server/Docker Instance Restart to take Effect.</small>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
@@ -271,6 +294,7 @@ const Settings = ({ closeSettings }:SettingsProps) => {
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<Toaster position='bottom-center' containerClassName="react_toaster" />
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
80
cron.js
80
cron.js
@@ -1,7 +1,7 @@
|
||||
const Cryptr = require('cryptr');
|
||||
const { promises } = require('fs');
|
||||
const { readFile } = require('fs');
|
||||
const cron = require('node-cron');
|
||||
const Cron = require('croner');
|
||||
require('dotenv').config({ path: './.env.local' });
|
||||
|
||||
const getAppSettings = async () => {
|
||||
@@ -49,6 +49,9 @@ const generateCronTime = (interval) => {
|
||||
if (interval === 'daily') {
|
||||
cronTime = '0 0 0 * * *';
|
||||
}
|
||||
if (interval === 'other_day') {
|
||||
cronTime = '0 0 2-30/2 * *';
|
||||
}
|
||||
if (interval === 'daily_morning') {
|
||||
cronTime = '0 0 3 * * *';
|
||||
}
|
||||
@@ -63,23 +66,47 @@ const generateCronTime = (interval) => {
|
||||
};
|
||||
|
||||
const runAppCronJobs = () => {
|
||||
// RUN SERP Scraping CRON (EveryDay at Midnight) 0 0 0 * *
|
||||
const scrapeCronTime = generateCronTime('daily');
|
||||
cron.schedule(scrapeCronTime, () => {
|
||||
// console.log('### Running Keyword Position Cron Job!');
|
||||
const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
|
||||
fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/cron`, fetchOpts)
|
||||
.then((res) => res.json())
|
||||
.then((data) => console.log(data))
|
||||
.catch((err) => {
|
||||
console.log('ERROR Making Daily Scraper Cron Request..');
|
||||
console.log(err);
|
||||
});
|
||||
}, { scheduled: true });
|
||||
getAppSettings().then((settings) => {
|
||||
// RUN SERP Scraping CRON (EveryDay at Midnight) 0 0 0 * *
|
||||
const scrape_interval = settings.scrape_interval || 'daily';
|
||||
if (scrape_interval !== 'never') {
|
||||
const scrapeCronTime = generateCronTime(scrape_interval);
|
||||
Cron(scrapeCronTime, () => {
|
||||
// console.log('### Running Keyword Position Cron Job!');
|
||||
const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
|
||||
fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/cron`, fetchOpts)
|
||||
.then((res) => res.json())
|
||||
// .then((data) =>{ console.log(data)})
|
||||
.catch((err) => {
|
||||
console.log('ERROR Making SERP Scraper Cron Request..');
|
||||
console.log(err);
|
||||
});
|
||||
}, { scheduled: true });
|
||||
}
|
||||
|
||||
// RUN Email Notification CRON
|
||||
const notif_interval = (!settings.notification_interval || settings.notification_interval === 'never') ? false : settings.notification_interval;
|
||||
if (notif_interval) {
|
||||
const cronTime = generateCronTime(notif_interval === 'daily' ? 'daily_morning' : notif_interval);
|
||||
if (cronTime) {
|
||||
Cron(cronTime, () => {
|
||||
// console.log('### Sending Notification Email...');
|
||||
const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
|
||||
fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/notify`, fetchOpts)
|
||||
.then((res) => res.json())
|
||||
.then((data) => console.log(data))
|
||||
.catch((err) => {
|
||||
console.log('ERROR Making Cron Email Notification Request..');
|
||||
console.log(err);
|
||||
});
|
||||
}, { scheduled: true });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Run Failed scraping CRON (Every Hour)
|
||||
const failedCronTime = generateCronTime('hourly');
|
||||
cron.schedule(failedCronTime, () => {
|
||||
Cron(failedCronTime, () => {
|
||||
// console.log('### Retrying Failed Scrapes...');
|
||||
|
||||
readFile(`${process.cwd()}/data/failed_queue.json`, { encoding: 'utf-8' }, (err, data) => {
|
||||
@@ -104,7 +131,7 @@ const runAppCronJobs = () => {
|
||||
// Run Google Search Console Scraper Daily
|
||||
if (process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL) {
|
||||
const searchConsoleCRONTime = generateCronTime('daily');
|
||||
cron.schedule(searchConsoleCRONTime, () => {
|
||||
Cron(searchConsoleCRONTime, () => {
|
||||
const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
|
||||
fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/searchconsole`, fetchOpts)
|
||||
.then((res) => res.json())
|
||||
@@ -115,27 +142,6 @@ const runAppCronJobs = () => {
|
||||
});
|
||||
}, { scheduled: true });
|
||||
}
|
||||
|
||||
// RUN Email Notification CRON
|
||||
getAppSettings().then((settings) => {
|
||||
const notif_interval = (!settings.notification_interval || settings.notification_interval === 'never') ? false : settings.notification_interval;
|
||||
if (notif_interval) {
|
||||
const cronTime = generateCronTime(notif_interval === 'daily' ? 'daily_morning' : notif_interval);
|
||||
if (cronTime) {
|
||||
cron.schedule(cronTime, () => {
|
||||
// console.log('### Sending Notification Email...');
|
||||
const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
|
||||
fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/notify`, fetchOpts)
|
||||
.then((res) => res.json())
|
||||
.then((data) => console.log(data))
|
||||
.catch((err) => {
|
||||
console.log('ERROR Making Cron Email Notification Request..');
|
||||
console.log(err);
|
||||
});
|
||||
}, { scheduled: true });
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
runAppCronJobs();
|
||||
|
||||
@@ -6,7 +6,7 @@ import Keyword from './models/keyword';
|
||||
const connection = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
host: '0.0.0.0',
|
||||
username: process.env.USER,
|
||||
username: process.env.USER_NAME ? process.env.USER_NAME : process.env.USER,
|
||||
password: process.env.PASSWORD,
|
||||
database: 'sequelize',
|
||||
dialectModule: sqlite3,
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
/** @type {import('next').NextConfig} */
|
||||
const { version } = require('./package.json');
|
||||
|
||||
const nextConfig = {
|
||||
reactStrictMode: true,
|
||||
swcMinify: false,
|
||||
output: 'standalone',
|
||||
publicRuntimeConfig: {
|
||||
version,
|
||||
},
|
||||
};
|
||||
|
||||
module.exports = nextConfig;
|
||||
|
||||
18
package-lock.json
generated
18
package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "serpbear",
|
||||
"version": "0.2.1",
|
||||
"version": "0.2.5",
|
||||
"lockfileVersion": 2,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "serpbear",
|
||||
"version": "0.2.1",
|
||||
"version": "0.2.5",
|
||||
"dependencies": {
|
||||
"@googleapis/searchconsole": "^1.0.0",
|
||||
"@testing-library/react": "^13.4.0",
|
||||
@@ -17,6 +17,7 @@
|
||||
"cheerio": "^1.0.0-rc.12",
|
||||
"concurrently": "^7.6.0",
|
||||
"cookies": "^0.8.0",
|
||||
"croner": "^5.3.5",
|
||||
"cryptr": "^6.0.3",
|
||||
"dayjs": "^1.11.5",
|
||||
"dotenv": "^16.0.3",
|
||||
@@ -3707,6 +3708,14 @@
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/croner": {
|
||||
"version": "5.3.5",
|
||||
"resolved": "https://registry.npmjs.org/croner/-/croner-5.3.5.tgz",
|
||||
"integrity": "sha512-VqaplJOVtaGuAxhsw2HM9GG0DLpVi3W9IsV7bKMAC12O7wMIOcZpCYHBw+xkFABzT3xp5MvUqTfbTewCgxgN+A==",
|
||||
"engines": {
|
||||
"node": ">=6.0"
|
||||
}
|
||||
},
|
||||
"node_modules/cross-spawn": {
|
||||
"version": "7.0.3",
|
||||
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
|
||||
@@ -15220,6 +15229,11 @@
|
||||
"yaml": "^1.10.0"
|
||||
}
|
||||
},
|
||||
"croner": {
|
||||
"version": "5.3.5",
|
||||
"resolved": "https://registry.npmjs.org/croner/-/croner-5.3.5.tgz",
|
||||
"integrity": "sha512-VqaplJOVtaGuAxhsw2HM9GG0DLpVi3W9IsV7bKMAC12O7wMIOcZpCYHBw+xkFABzT3xp5MvUqTfbTewCgxgN+A=="
|
||||
},
|
||||
"cross-spawn": {
|
||||
"version": "7.0.3",
|
||||
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "serpbear",
|
||||
"version": "0.2.1",
|
||||
"version": "0.2.5",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
@@ -25,6 +25,7 @@
|
||||
"cheerio": "^1.0.0-rc.12",
|
||||
"concurrently": "^7.6.0",
|
||||
"cookies": "^0.8.0",
|
||||
"croner": "^5.3.5",
|
||||
"cryptr": "^6.0.3",
|
||||
"dayjs": "^1.11.5",
|
||||
"dotenv": "^16.0.3",
|
||||
|
||||
@@ -153,10 +153,13 @@ const updateKeywords = async (req: NextApiRequest, res: NextApiResponse<Keywords
|
||||
}
|
||||
if (tags) {
|
||||
const tagsKeywordIDs = Object.keys(tags);
|
||||
const multipleKeywords = tagsKeywordIDs.length > 1;
|
||||
for (const keywordID of tagsKeywordIDs) {
|
||||
const response = await Keyword.findOne({ where: { ID: keywordID } });
|
||||
if (response) {
|
||||
await response.update({ tags: JSON.stringify(tags[keywordID]) });
|
||||
const selectedKeyword = await Keyword.findOne({ where: { ID: keywordID } });
|
||||
const currentTags = selectedKeyword && selectedKeyword.tags ? JSON.parse(selectedKeyword.tags) : [];
|
||||
const mergedTags = Array.from(new Set([...currentTags, ...tags[keywordID]]));
|
||||
if (selectedKeyword) {
|
||||
await selectedKeyword.update({ tags: JSON.stringify(multipleKeywords ? mergedTags : tags[keywordID]) });
|
||||
}
|
||||
}
|
||||
return res.status(200).json({ keywords });
|
||||
|
||||
@@ -18,10 +18,11 @@ const loginUser = async (req: NextApiRequest, res: NextApiResponse<loginResponse
|
||||
if (!req.body.username || !req.body.password) {
|
||||
return res.status(401).json({ error: 'Username Password Missing' });
|
||||
}
|
||||
const userName = process.env.USER_NAME ? process.env.USER_NAME : process.env.USER;
|
||||
|
||||
if (req.body.username === process.env.USER
|
||||
if (req.body.username === userName
|
||||
&& req.body.password === process.env.PASSWORD && process.env.SECRET) {
|
||||
const token = jwt.sign({ user: process.env.USER }, process.env.SECRET);
|
||||
const token = jwt.sign({ user: userName }, process.env.SECRET);
|
||||
const cookies = new Cookies(req, res);
|
||||
const expireDate = new Date();
|
||||
const sessDuration = process.env.SESSION_DURATION;
|
||||
@@ -30,7 +31,7 @@ const loginUser = async (req: NextApiRequest, res: NextApiResponse<loginResponse
|
||||
return res.status(200).json({ success: true, error: null });
|
||||
}
|
||||
|
||||
const error = req.body.username !== process.env.USER ? 'Incorrect Username' : 'Incorrect Password';
|
||||
const error = req.body.username !== userName ? 'Incorrect Username' : 'Incorrect Password';
|
||||
|
||||
return res.status(401).json({ success: false, error });
|
||||
};
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import Cryptr from 'cryptr';
|
||||
import getConfig from 'next/config';
|
||||
import { writeFile, readFile } from 'fs/promises';
|
||||
import verifyUser from '../../utils/verifyUser';
|
||||
import allScrapers from '../../scrapers/index';
|
||||
@@ -26,7 +27,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
|
||||
const getSettings = async (req: NextApiRequest, res: NextApiResponse<SettingsGetResponse>) => {
|
||||
const settings = await getAppSettings();
|
||||
if (settings) {
|
||||
return res.status(200).json({ settings });
|
||||
const { publicRuntimeConfig } = getConfig();
|
||||
const version = publicRuntimeConfig?.version;
|
||||
return res.status(200).json({ settings: { ...settings, version } });
|
||||
}
|
||||
return res.status(400).json({ error: 'Error Loading Settings!' });
|
||||
};
|
||||
|
||||
@@ -11,16 +11,43 @@ import { useFetchDomains } from '../../services/domains';
|
||||
import DomainItem from '../../components/domains/DomainItem';
|
||||
import Icon from '../../components/common/Icon';
|
||||
|
||||
type thumbImages = { [domain:string] : string }
|
||||
|
||||
const SingleDomain: NextPage = () => {
|
||||
const router = useRouter();
|
||||
const [noScrapprtError, setNoScrapprtError] = useState(false);
|
||||
const [showSettings, setShowSettings] = useState(false);
|
||||
const [showAddDomain, setShowAddDomain] = useState(false);
|
||||
const [domainThumbs, setDomainThumbs] = useState<thumbImages>({});
|
||||
const { data: appSettings } = useFetchSettings();
|
||||
const { data: domainsData, isLoading } = useFetchDomains(router, true);
|
||||
|
||||
useEffect(() => {
|
||||
console.log('Domains Data: ', domainsData);
|
||||
// console.log('Domains Data: ', domainsData);
|
||||
if (domainsData?.domains && domainsData.domains.length > 0) {
|
||||
const domainThumbsRaw = localStorage.getItem('domainThumbs');
|
||||
const domThumbs = domainThumbsRaw ? JSON.parse(domainThumbsRaw) : {};
|
||||
domainsData.domains.forEach(async (domain:DomainType) => {
|
||||
if (domain.domain) {
|
||||
if (!domThumbs[domain.domain]) {
|
||||
const domainImageBlob = await fetch(`https://image.thum.io/get/auth/66909-serpbear/maxAge/96/width/200/https://${domain.domain}`).then((res) => res.blob());
|
||||
if (domainImageBlob) {
|
||||
const reader = new FileReader();
|
||||
await new Promise((resolve, reject) => {
|
||||
reader.onload = resolve;
|
||||
reader.onerror = reject;
|
||||
reader.readAsDataURL(domainImageBlob);
|
||||
});
|
||||
const imageBase: string = reader.result && typeof reader.result === 'string' ? reader.result : '';
|
||||
localStorage.setItem('domainThumbs', JSON.stringify({ ...domThumbs, [domain.domain]: imageBase }));
|
||||
setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain.domain]: imageBase }));
|
||||
}
|
||||
} else {
|
||||
setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain.domain]: domThumbs[domain.domain] }));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}, [domainsData]);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -31,7 +58,7 @@ const SingleDomain: NextPage = () => {
|
||||
}, [appSettings]);
|
||||
|
||||
return (
|
||||
<div className="Domain ">
|
||||
<div className="Domain flex flex-col min-h-screen">
|
||||
{noScrapprtError && (
|
||||
<div className=' p-3 bg-red-600 text-white text-sm text-center'>
|
||||
A Scrapper/Proxy has not been set up Yet. Open Settings to set it up and start using the app.
|
||||
@@ -62,6 +89,7 @@ const SingleDomain: NextPage = () => {
|
||||
domain={domain}
|
||||
selected={false}
|
||||
isConsoleIntegrated={!!(appSettings && appSettings?.settings?.search_console_integrated) }
|
||||
thumb={domainThumbs[domain.domain]}
|
||||
// isConsoleIntegrated={false}
|
||||
/>;
|
||||
})}
|
||||
@@ -84,6 +112,9 @@ const SingleDomain: NextPage = () => {
|
||||
<CSSTransition in={showSettings} timeout={300} classNames="settings_anim" unmountOnExit mountOnEnter>
|
||||
<Settings closeSettings={() => setShowSettings(false)} />
|
||||
</CSSTransition>
|
||||
<footer className='text-center flex flex-1 justify-center pb-5 items-end'>
|
||||
<span className='text-gray-500 text-xs'><a href='https://github.com/towfiqi/serpbear' target="_blank" rel='noreferrer'>SerpBear v{appSettings?.settings?.version || '0.0.0'}</a></span>
|
||||
</footer>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -2,6 +2,11 @@ const scrapingAnt:ScraperSettings = {
|
||||
id: 'scrapingant',
|
||||
name: 'ScrapingAnt',
|
||||
website: 'scrapingant.com',
|
||||
headers: (keyword) => {
|
||||
// eslint-disable-next-line max-len
|
||||
const mobileAgent = 'Mozilla/5.0 (Linux; Android 10; SM-G996U Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Mobile Safari/537.36';
|
||||
return keyword && keyword.device === 'mobile' ? { 'Ant-User-Agent': mobileAgent } : {};
|
||||
},
|
||||
scrapeURL: (keyword, settings, countryData) => {
|
||||
const scraperCountries = ['AE', 'BR', 'CN', 'DE', 'ES', 'FR', 'GB', 'HK', 'PL', 'IN', 'IT', 'IL', 'JP', 'NL', 'RU', 'SA', 'US', 'CZ'];
|
||||
const country = scraperCountries.includes(keyword.country.toUpperCase()) ? keyword.country : 'US';
|
||||
|
||||
@@ -84,8 +84,8 @@ body {
|
||||
|
||||
.domKeywords_head--alpha_desc .domKeywords_head_keyword::after,
|
||||
.domKeywords_head--pos_desc .domKeywords_head_position::after,
|
||||
.domKeywords_head--imp_desc .domKeywords_head_imp::after,
|
||||
.domKeywords_head--visits_desc .domKeywords_head_visits::after,
|
||||
.domKeywords_head--imp_asc .domKeywords_head_imp::after,
|
||||
.domKeywords_head--visits_asc .domKeywords_head_visits::after,
|
||||
.domKeywords_head--ctr_desc .domKeywords_head_ctr::after {
|
||||
content: "↓";
|
||||
display: inline-block;
|
||||
@@ -98,8 +98,8 @@ body {
|
||||
|
||||
.domKeywords_head--alpha_asc .domKeywords_head_keyword::after,
|
||||
.domKeywords_head--pos_asc .domKeywords_head_position::after,
|
||||
.domKeywords_head--imp_asc .domKeywords_head_imp::after,
|
||||
.domKeywords_head--visits_asc .domKeywords_head_visits::after,
|
||||
.domKeywords_head--imp_desc .domKeywords_head_imp::after,
|
||||
.domKeywords_head--visits_desc .domKeywords_head_visits::after,
|
||||
.domKeywords_head--ctr_asc .domKeywords_head_ctr::after {
|
||||
content: "↑";
|
||||
display: inline-block;
|
||||
|
||||
4
types.d.ts
vendored
4
types.d.ts
vendored
@@ -78,7 +78,9 @@ type SettingsType = {
|
||||
smtp_username?: string,
|
||||
smtp_password?: string,
|
||||
search_console_integrated?: boolean,
|
||||
available_scapers?: Array
|
||||
available_scapers?: Array,
|
||||
scrape_interval?: string
|
||||
version?: string
|
||||
}
|
||||
|
||||
type KeywordSCDataChild = {
|
||||
|
||||
@@ -9,16 +9,16 @@ export const SCsortKeywords = (theKeywords:SCKeywordType[], sortBy:string) : SCK
|
||||
const keywords = theKeywords.map((k) => ({ ...k, position: k.position === 0 ? 111 : k.position }));
|
||||
switch (sortBy) {
|
||||
case 'imp_asc':
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => b.impressions - a.impressions);
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.impressions > b.impressions ? 1 : -1));
|
||||
break;
|
||||
case 'imp_desc':
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => a.impressions - b.impressions);
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.impressions > a.impressions ? 1 : -1));
|
||||
break;
|
||||
case 'visits_asc':
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => b.clicks - a.clicks);
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.clicks > b.clicks ? 1 : -1));
|
||||
break;
|
||||
case 'visits_desc':
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => a.clicks - b.clicks);
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.clicks > a.clicks ? 1 : -1));
|
||||
break;
|
||||
case 'ctr_asc':
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => b.ctr - a.ctr);
|
||||
@@ -27,17 +27,17 @@ export const SCsortKeywords = (theKeywords:SCKeywordType[], sortBy:string) : SCK
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => a.ctr - b.ctr);
|
||||
break;
|
||||
case 'pos_asc':
|
||||
sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.position > a.position ? 1 : -1));
|
||||
sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.position < a.position ? 1 : -1));
|
||||
sortedItems = sortedItems.map((k) => ({ ...k, position: k.position === 111 ? 0 : k.position }));
|
||||
break;
|
||||
case 'pos_desc':
|
||||
sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.position > b.position ? 1 : -1));
|
||||
sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.position < b.position ? 1 : -1));
|
||||
sortedItems = sortedItems.map((k) => ({ ...k, position: k.position === 111 ? 0 : k.position }));
|
||||
break;
|
||||
case 'alpha_asc':
|
||||
case 'alpha_desc':
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.keyword > a.keyword ? 1 : -1));
|
||||
break;
|
||||
case 'alpha_desc':
|
||||
case 'alpha_asc':
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.keyword > b.keyword ? 1 : -1));
|
||||
break;
|
||||
default:
|
||||
|
||||
@@ -49,7 +49,7 @@ export const getScraperClient = (keyword:KeywordType, settings:SettingsType, scr
|
||||
// Set Scraper Header
|
||||
const scrapeHeaders = scraper.headers ? scraper.headers(keyword, settings) : null;
|
||||
const scraperAPIURL = scraper.scrapeURL ? scraper.scrapeURL(keyword, settings, countries) : null;
|
||||
if (scrapeHeaders) {
|
||||
if (scrapeHeaders && Object.keys(scrapeHeaders).length > 0) {
|
||||
Object.keys(scrapeHeaders).forEach((headerItemKey:string) => {
|
||||
headers[headerItemKey] = scrapeHeaders[headerItemKey as keyof object];
|
||||
});
|
||||
@@ -114,7 +114,7 @@ export const scrapeKeywordFromGoogle = async (keyword:KeywordType, settings:Sett
|
||||
const scraperResult = scraperObj?.resultObjectKey && res[scraperObj.resultObjectKey] ? res[scraperObj.resultObjectKey] : '';
|
||||
const scrapeResult:string = (res.data || res.html || res.results || scraperResult || '');
|
||||
if (res && scrapeResult) {
|
||||
const extracted = scraperObj?.serpExtractor ? scraperObj.serpExtractor(scrapeResult) : extractScrapedResult(scrapeResult);
|
||||
const extracted = scraperObj?.serpExtractor ? scraperObj.serpExtractor(scrapeResult) : extractScrapedResult(scrapeResult, keyword.device);
|
||||
// await writeFile('result.txt', JSON.stringify(scrapeResult), { encoding: 'utf-8' }).catch((err) => { console.log(err); });
|
||||
const serp = getSerp(keyword.domain, extracted);
|
||||
refreshedResults = { ID: keyword.ID, keyword: keyword.keyword, position: serp.postion, url: serp.url, result: extracted, error: false };
|
||||
@@ -141,9 +141,10 @@ export const scrapeKeywordFromGoogle = async (keyword:KeywordType, settings:Sett
|
||||
/**
|
||||
* Extracts the Google Search result as object array from the Google Search's HTML content
|
||||
* @param {string} content - scraped google search page html data.
|
||||
* @param {string} device - The device of the keyword.
|
||||
* @returns {SearchResult[]}
|
||||
*/
|
||||
export const extractScrapedResult = (content: string): SearchResult[] => {
|
||||
export const extractScrapedResult = (content: string, device: string): SearchResult[] => {
|
||||
const extractedResult = [];
|
||||
|
||||
const $ = cheerio.load(content);
|
||||
@@ -163,6 +164,24 @@ export const extractScrapedResult = (content: string): SearchResult[] => {
|
||||
}
|
||||
}
|
||||
|
||||
// Mobile Scraper
|
||||
if (extractedResult.length === 0 && device === 'mobile') {
|
||||
const items = $('body').find('#rso > div');
|
||||
for (let i = 0; i < items.length; i += 1) {
|
||||
const item = $(items[i]);
|
||||
const linkDom = item.find('a[role="presentation"]');
|
||||
if (linkDom) {
|
||||
const url = linkDom.attr('href');
|
||||
const titleDom = linkDom.find('[role="link"]');
|
||||
const title = titleDom ? titleDom.text() : '';
|
||||
if (title && url) {
|
||||
lastPosition += 1;
|
||||
extractedResult.push({ title, url, position: lastPosition });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return extractedResult;
|
||||
};
|
||||
|
||||
@@ -193,7 +212,7 @@ export const retryScrape = async (keywordID: number) : Promise<void> => {
|
||||
|
||||
const filePath = `${process.cwd()}/data/failed_queue.json`;
|
||||
const currentQueueRaw = await readFile(filePath, { encoding: 'utf-8' }).catch((err) => { console.log(err); return '[]'; });
|
||||
currentQueue = JSON.parse(currentQueueRaw);
|
||||
currentQueue = currentQueueRaw ? JSON.parse(currentQueueRaw) : [];
|
||||
|
||||
if (!currentQueue.includes(keywordID)) {
|
||||
currentQueue.push(keywordID);
|
||||
@@ -213,7 +232,7 @@ export const removeFromRetryQueue = async (keywordID: number) : Promise<void> =>
|
||||
|
||||
const filePath = `${process.cwd()}/data/failed_queue.json`;
|
||||
const currentQueueRaw = await readFile(filePath, { encoding: 'utf-8' }).catch((err) => { console.log(err); return '[]'; });
|
||||
currentQueue = JSON.parse(currentQueueRaw);
|
||||
currentQueue = currentQueueRaw ? JSON.parse(currentQueueRaw) : [];
|
||||
currentQueue = currentQueue.filter((item) => item !== keywordID);
|
||||
|
||||
await writeFile(filePath, JSON.stringify(currentQueue), { encoding: 'utf-8' }).catch((err) => { console.log(err); return '[]'; });
|
||||
|
||||
@@ -54,7 +54,7 @@ const fetchSearchConsoleData = async (domainName:string, days:number, type?:stri
|
||||
date: row.keys[0],
|
||||
clicks: row.clicks,
|
||||
impressions: row.impressions,
|
||||
ctr: row.ctr,
|
||||
ctr: row.ctr * 100,
|
||||
position: row.position,
|
||||
});
|
||||
});
|
||||
@@ -101,7 +101,7 @@ export const parseSearchConsoleItem = (SCItem: SearchAnalyticsRawItem, domainNam
|
||||
const page = SCItem.keys[3] ? SCItem.keys[3].replace('https://', '').replace('http://', '').replace('www', '').replace(domainName, '') : '';
|
||||
const uid = `${country.toLowerCase()}:${device}:${keyword.replaceAll(' ', '_')}`;
|
||||
|
||||
return { keyword, uid, device, country, clicks, impressions, ctr, position, page };
|
||||
return { keyword, uid, device, country, clicks, impressions, ctr: ctr * 100, position, page };
|
||||
};
|
||||
|
||||
export const integrateKeywordSCData = (keyword: KeywordType, SCData:SCDomainDataType) : KeywordType => {
|
||||
|
||||
@@ -28,7 +28,7 @@ export const sortKeywords = (theKeywords:KeywordType[], sortBy:string, scDataTyp
|
||||
case 'alpha_desc':
|
||||
sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => (a.keyword > b.keyword ? 1 : -1));
|
||||
break;
|
||||
case 'imp_asc':
|
||||
case 'imp_desc':
|
||||
if (scDataType) {
|
||||
sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
|
||||
const bImpressionData = b.scData?.impressions[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
@@ -37,7 +37,7 @@ export const sortKeywords = (theKeywords:KeywordType[], sortBy:string, scDataTyp
|
||||
});
|
||||
}
|
||||
break;
|
||||
case 'imp_desc':
|
||||
case 'imp_asc':
|
||||
if (scDataType) {
|
||||
sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
|
||||
const bImpressionData = b.scData?.impressions[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
@@ -46,21 +46,21 @@ export const sortKeywords = (theKeywords:KeywordType[], sortBy:string, scDataTyp
|
||||
});
|
||||
}
|
||||
break;
|
||||
case 'visits_asc':
|
||||
if (scDataType) {
|
||||
sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
|
||||
const bImpressionData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
const aImpressionData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
return aImpressionData > bImpressionData ? 1 : -1;
|
||||
});
|
||||
}
|
||||
break;
|
||||
case 'visits_desc':
|
||||
if (scDataType) {
|
||||
sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
|
||||
const bImpressionData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
const aImpressionData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
return bImpressionData > aImpressionData ? 1 : -1;
|
||||
const bVisitsData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
const aVisitsData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
return aVisitsData > bVisitsData ? 1 : -1;
|
||||
});
|
||||
}
|
||||
break;
|
||||
case 'visits_asc':
|
||||
if (scDataType) {
|
||||
sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
|
||||
const bVisitsData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
const aVisitsData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
return bVisitsData > aVisitsData ? 1 : -1;
|
||||
});
|
||||
}
|
||||
break;
|
||||
|
||||
Reference in New Issue
Block a user