mirror of
https://github.com/towfiqi/serpbear
synced 2025-06-26 18:15:54 +00:00
Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
99dbbd1dd9 | ||
|
|
acc0b39d80 | ||
|
|
a1108d240e | ||
|
|
d9e0d0107a | ||
|
|
9e9dad7631 | ||
|
|
8139e399c1 | ||
|
|
cb24696a1f | ||
|
|
b50733defc |
16
CHANGELOG.md
16
CHANGELOG.md
@@ -2,6 +2,22 @@
|
||||
|
||||
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
||||
|
||||
### [0.2.3](https://github.com/towfiqi/serpbear/compare/v0.2.2...v0.2.3) (2023-01-12)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* Ability to tag multiple keywords at once ([9e9dad7](https://github.com/towfiqi/serpbear/commit/9e9dad7631691b2a836fdd4c522b1f933b17e285)), closes [#54](https://github.com/towfiqi/serpbear/issues/54)
|
||||
* Set USERNAME as well as USER variable ([b50733d](https://github.com/towfiqi/serpbear/commit/b50733defc2c06e0f92ca3e88fd1f74684eee9c0))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Fixes Position and View Sort. ([8139e39](https://github.com/towfiqi/serpbear/commit/8139e399c13ab8be767facef9a19c67dec06ed64)), closes [#46](https://github.com/towfiqi/serpbear/issues/46)
|
||||
* Fixes wrong CTR value for Search Console Data ([cb24696](https://github.com/towfiqi/serpbear/commit/cb24696a1f47b02a11c68cd1c673ea8b1bacd144)), closes [#48](https://github.com/towfiqi/serpbear/issues/48)
|
||||
* Mobile Keyword Scraping not working. ([a1108d2](https://github.com/towfiqi/serpbear/commit/a1108d240ea38ab0886ef3722b0c937ec5a45591)), closes [#58](https://github.com/towfiqi/serpbear/issues/58)
|
||||
* ScrapingAnt Mobile Keyword Scrape not working ([acc0b39](https://github.com/towfiqi/serpbear/commit/acc0b39d80d4f9371967a0d425ed205c5d866eea))
|
||||
|
||||
### [0.2.2](https://github.com/towfiqi/serpbear/compare/v0.2.1...v0.2.2) (2022-12-25)
|
||||
|
||||
|
||||
|
||||
60
components/keywords/AddTags.tsx
Normal file
60
components/keywords/AddTags.tsx
Normal file
@@ -0,0 +1,60 @@
|
||||
import { useState } from 'react';
|
||||
import { useUpdateKeywordTags } from '../../services/keywords';
|
||||
import Icon from '../common/Icon';
|
||||
import Modal from '../common/Modal';
|
||||
|
||||
type AddTagsProps = {
|
||||
keywords: KeywordType[],
|
||||
closeModal: Function
|
||||
}
|
||||
|
||||
const AddTags = ({ keywords = [], closeModal }: AddTagsProps) => {
|
||||
const [tagInput, setTagInput] = useState('');
|
||||
const [inputError, setInputError] = useState('');
|
||||
const { mutate: updateMutate } = useUpdateKeywordTags(() => { setTagInput(''); });
|
||||
|
||||
const addTag = () => {
|
||||
if (keywords.length === 0) { return; }
|
||||
if (!tagInput) {
|
||||
setInputError('Please Insert a Tag!');
|
||||
setTimeout(() => { setInputError(''); }, 3000);
|
||||
return;
|
||||
}
|
||||
|
||||
const tagsArray = tagInput.split(',').map((t) => t.trim());
|
||||
const tagsPayload:any = {};
|
||||
keywords.forEach((keyword:KeywordType) => {
|
||||
tagsPayload[keyword.ID] = [...keyword.tags, ...tagsArray];
|
||||
});
|
||||
updateMutate({ tags: tagsPayload });
|
||||
};
|
||||
|
||||
return (
|
||||
<Modal closeModal={() => { closeModal(false); }} title={`Add New Tags to ${keywords.length} Selected Keyword`}>
|
||||
<div className="relative">
|
||||
{inputError && <span className="absolute top-[-24px] text-red-400 text-sm font-semibold">{inputError}</span>}
|
||||
<span className='absolute text-gray-400 top-3 left-2'><Icon type="tags" size={16} /></span>
|
||||
<input
|
||||
className='w-full border rounded border-gray-200 py-3 px-4 pl-8 outline-none focus:border-indigo-300'
|
||||
placeholder='Insert Tags. eg: tag1, tag2'
|
||||
value={tagInput}
|
||||
onChange={(e) => setTagInput(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.code === 'Enter') {
|
||||
e.preventDefault();
|
||||
addTag();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<button
|
||||
className=" absolute right-2 top-2 cursor-pointer rounded p-2 px-4 bg-indigo-600 text-white font-semibold text-sm"
|
||||
onClick={addTag}>
|
||||
Apply
|
||||
</button>
|
||||
</div>
|
||||
</Modal>
|
||||
|
||||
);
|
||||
};
|
||||
|
||||
export default AddTags;
|
||||
@@ -85,7 +85,7 @@ const KeywordDetails = ({ keyword, closeDetails }:KeywordDetailsProps) => {
|
||||
<h3 className=' text-lg font-bold'>
|
||||
<span title={countries[keyword.country][0]}
|
||||
className={`fflag fflag-${keyword.country} w-[18px] h-[12px] mr-2`} /> {keyword.keyword}
|
||||
<span className='py-1 px-2 rounded bg-blue-50 text-blue-700 text-xs font-bold'>{keyword.position}</span>
|
||||
<span className='py-1 px-2 ml-2 rounded bg-blue-50 text-blue-700 text-xs font-bold'>{keyword.position}</span>
|
||||
</h3>
|
||||
<button
|
||||
className='absolute top-2 right-2 p-2 px-3 text-gray-400 hover:text-gray-700 transition-all hover:rotate-90'
|
||||
|
||||
@@ -82,10 +82,10 @@ const KeywordFilters = (props: KeywordFilterProps) => {
|
||||
{ value: 'alpha_desc', label: 'Alphabetically(Z-A)' },
|
||||
];
|
||||
if (integratedConsole) {
|
||||
sortOptionChoices.push({ value: 'imp_asc', label: `Most Viewed${isConsole ? ' (Default)' : ''}` });
|
||||
sortOptionChoices.push({ value: 'imp_desc', label: 'Least Viewed' });
|
||||
sortOptionChoices.push({ value: 'visits_asc', label: 'Most Visited' });
|
||||
sortOptionChoices.push({ value: 'visits_desc', label: 'Least Visited' });
|
||||
sortOptionChoices.push({ value: 'imp_desc', label: `Most Viewed${isConsole ? ' (Default)' : ''}` });
|
||||
sortOptionChoices.push({ value: 'imp_asc', label: 'Least Viewed' });
|
||||
sortOptionChoices.push({ value: 'visits_desc', label: 'Most Visited' });
|
||||
sortOptionChoices.push({ value: 'visits_asc', label: 'Least Visited' });
|
||||
}
|
||||
if (isConsole) {
|
||||
sortOptionChoices.splice(2, 2);
|
||||
|
||||
@@ -2,6 +2,7 @@ import { useState } from 'react';
|
||||
import { useUpdateKeywordTags } from '../../services/keywords';
|
||||
import Icon from '../common/Icon';
|
||||
import Modal from '../common/Modal';
|
||||
import AddTags from './AddTags';
|
||||
|
||||
type keywordTagManagerProps = {
|
||||
keyword: KeywordType|undefined,
|
||||
@@ -10,9 +11,8 @@ type keywordTagManagerProps = {
|
||||
}
|
||||
|
||||
const KeywordTagManager = ({ keyword, closeModal }: keywordTagManagerProps) => {
|
||||
const [tagInput, setTagInput] = useState('');
|
||||
const [inputError, setInputError] = useState('');
|
||||
const { mutate: updateMutate } = useUpdateKeywordTags(() => { setTagInput(''); });
|
||||
const [showAddTag, setShowAddTag] = useState<boolean>(false);
|
||||
const { mutate: updateMutate } = useUpdateKeywordTags(() => { });
|
||||
|
||||
const removeTag = (tag:String) => {
|
||||
if (!keyword) { return; }
|
||||
@@ -20,24 +20,6 @@ const KeywordTagManager = ({ keyword, closeModal }: keywordTagManagerProps) => {
|
||||
updateMutate({ tags: { [keyword.ID]: newTags } });
|
||||
};
|
||||
|
||||
const addTag = () => {
|
||||
if (!keyword) { return; }
|
||||
if (!tagInput) {
|
||||
setInputError('Please Insert a Tag!');
|
||||
setTimeout(() => { setInputError(''); }, 3000);
|
||||
return;
|
||||
}
|
||||
if (keyword.tags.includes(tagInput)) {
|
||||
setInputError('Tag Exist!');
|
||||
setTimeout(() => { setInputError(''); }, 3000);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('New Tag: ', tagInput);
|
||||
const newTags = [...keyword.tags, tagInput.trim()];
|
||||
updateMutate({ tags: { [keyword.ID]: newTags } });
|
||||
};
|
||||
|
||||
return (
|
||||
<Modal closeModal={() => { closeModal(false); }} title={`Tags for Keyword "${keyword && keyword.keyword}"`}>
|
||||
<div className="text-sm my-8 ">
|
||||
@@ -53,31 +35,27 @@ const KeywordTagManager = ({ keyword, closeModal }: keywordTagManagerProps) => {
|
||||
</button>
|
||||
</li>;
|
||||
})}
|
||||
<li className='inline-block py-1 px-1'>
|
||||
<button
|
||||
title='Add New Tag'
|
||||
className="cursor-pointer rounded p-1 px-3 bg-indigo-600 text-white font-semibold text-sm"
|
||||
onClick={() => setShowAddTag(true)}>+</button>
|
||||
</li>
|
||||
</ul>
|
||||
)}
|
||||
{keyword && keyword.tags.length === 0 && (
|
||||
<div className="text-center w-full text-gray-500">No Tags Added to this Keyword.</div>
|
||||
<div className="text-center w-full text-gray-500">
|
||||
No Tags Added to this Keyword. <button className=' text-indigo-600' onClick={() => setShowAddTag(true)}>+ Add Tag</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="relative">
|
||||
{inputError && <span className="absolute top-[-24px] text-red-400 text-sm font-semibold">{inputError}</span>}
|
||||
<span className='absolute text-gray-400 top-3 left-2'><Icon type="tags" size={16} /></span>
|
||||
<input
|
||||
className='w-full border rounded border-gray-200 py-3 px-4 pl-8 outline-none focus:border-indigo-300'
|
||||
placeholder='Insert Tags'
|
||||
value={tagInput}
|
||||
onChange={(e) => setTagInput(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.code === 'Enter') {
|
||||
e.preventDefault();
|
||||
addTag();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<button className=" absolute right-2 top-2 cursor-pointer rounded p-1 px-4 bg-blue-600 text-white font-bold" onClick={addTag}>+</button>
|
||||
</div>
|
||||
{showAddTag && keyword && (
|
||||
<AddTags
|
||||
keywords={[keyword]}
|
||||
closeModal={() => setShowAddTag(false)}
|
||||
/>
|
||||
)}
|
||||
</Modal>
|
||||
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import KeywordFilters from './KeywordFilter';
|
||||
import Modal from '../common/Modal';
|
||||
import { useDeleteKeywords, useFavKeywords, useRefreshKeywords } from '../../services/keywords';
|
||||
import KeywordTagManager from './KeywordTagManager';
|
||||
import AddTags from './AddTags';
|
||||
|
||||
type KeywordsTableProps = {
|
||||
domain: DomainType | null,
|
||||
@@ -28,6 +29,7 @@ const KeywordsTable = (props: KeywordsTableProps) => {
|
||||
const [showKeyDetails, setShowKeyDetails] = useState<KeywordType|null>(null);
|
||||
const [showRemoveModal, setShowRemoveModal] = useState<boolean>(false);
|
||||
const [showTagManager, setShowTagManager] = useState<null|number>(null);
|
||||
const [showAddTags, setShowAddTags] = useState<boolean>(false);
|
||||
const [filterParams, setFilterParams] = useState<KeywordFilters>({ countries: [], tags: [], search: '' });
|
||||
const [sortBy, setSortBy] = useState<string>('date_asc');
|
||||
const [scDataType, setScDataType] = useState<string>('threeDays');
|
||||
@@ -79,7 +81,7 @@ const KeywordsTable = (props: KeywordsTableProps) => {
|
||||
className='block px-2 py-2 cursor-pointer hover:text-indigo-600'
|
||||
onClick={() => { refreshMutate({ ids: selectedKeywords }); setSelectedKeywords([]); }}
|
||||
>
|
||||
<span className=' bg-indigo-100 text-blue-700 px-1 rounded'><Icon type="reload" size={11} /></span> Refresh Keyword
|
||||
<span className=' bg-indigo-100 text-blue-700 px-1 rounded'><Icon type="reload" size={11} /></span> Refresh Keywords
|
||||
</a>
|
||||
</li>
|
||||
<li className='inline-block mr-4'>
|
||||
@@ -87,7 +89,14 @@ const KeywordsTable = (props: KeywordsTableProps) => {
|
||||
className='block px-2 py-2 cursor-pointer hover:text-indigo-600'
|
||||
onClick={() => setShowRemoveModal(true)}
|
||||
>
|
||||
<span className=' bg-red-100 text-red-600 px-1 rounded'><Icon type="trash" size={14} /></span> Remove Keyword</a>
|
||||
<span className=' bg-red-100 text-red-600 px-1 rounded'><Icon type="trash" size={14} /></span> Remove Keywords</a>
|
||||
</li>
|
||||
<li className='inline-block mr-4'>
|
||||
<a
|
||||
className='block px-2 py-2 cursor-pointer hover:text-indigo-600'
|
||||
onClick={() => setShowAddTags(true)}
|
||||
>
|
||||
<span className=' bg-green-100 text-green-500 px-1 rounded'><Icon type="tags" size={14} /></span> Tag Keywords</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
@@ -222,6 +231,12 @@ const KeywordsTable = (props: KeywordsTableProps) => {
|
||||
closeModal={() => setShowTagManager(null)}
|
||||
/>
|
||||
)}
|
||||
{showAddTags && (
|
||||
<AddTags
|
||||
keywords={keywords.filter((k) => selectedKeywords.includes(k.ID))}
|
||||
closeModal={() => setShowAddTags(false)}
|
||||
/>
|
||||
)}
|
||||
<Toaster position='bottom-center' containerClassName="react_toaster" />
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -26,7 +26,7 @@ const SCKeywordsTable = ({ domain, keywords = [], isLoading = true, isConsoleInt
|
||||
const [device, setDevice] = useState<string>('desktop');
|
||||
const [selectedKeywords, setSelectedKeywords] = useState<string[]>([]);
|
||||
const [filterParams, setFilterParams] = useState<KeywordFilters>({ countries: [], tags: [], search: '' });
|
||||
const [sortBy, setSortBy] = useState<string>('imp_asc');
|
||||
const [sortBy, setSortBy] = useState<string>('imp_desc');
|
||||
const [isMobile, setIsMobile] = useState<boolean>(false);
|
||||
const [SCListHeight, setSCListHeight] = useState(500);
|
||||
const { keywordsData } = useFetchKeywords(router);
|
||||
|
||||
@@ -6,7 +6,7 @@ import Keyword from './models/keyword';
|
||||
const connection = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
host: '0.0.0.0',
|
||||
username: process.env.USER,
|
||||
username: process.env.USERNAME ? process.env.USERNAME : process.env.USER,
|
||||
password: process.env.PASSWORD,
|
||||
database: 'sequelize',
|
||||
dialectModule: sqlite3,
|
||||
|
||||
4
package-lock.json
generated
4
package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "serpbear",
|
||||
"version": "0.2.2",
|
||||
"version": "0.2.3",
|
||||
"lockfileVersion": 2,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "serpbear",
|
||||
"version": "0.2.2",
|
||||
"version": "0.2.3",
|
||||
"dependencies": {
|
||||
"@googleapis/searchconsole": "^1.0.0",
|
||||
"@testing-library/react": "^13.4.0",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "serpbear",
|
||||
"version": "0.2.2",
|
||||
"version": "0.2.3",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
|
||||
@@ -153,10 +153,13 @@ const updateKeywords = async (req: NextApiRequest, res: NextApiResponse<Keywords
|
||||
}
|
||||
if (tags) {
|
||||
const tagsKeywordIDs = Object.keys(tags);
|
||||
const multipleKeywords = tagsKeywordIDs.length > 1;
|
||||
for (const keywordID of tagsKeywordIDs) {
|
||||
const response = await Keyword.findOne({ where: { ID: keywordID } });
|
||||
if (response) {
|
||||
await response.update({ tags: JSON.stringify(tags[keywordID]) });
|
||||
const selectedKeyword = await Keyword.findOne({ where: { ID: keywordID } });
|
||||
const currentTags = selectedKeyword && selectedKeyword.tags ? JSON.parse(selectedKeyword.tags) : [];
|
||||
const mergedTags = Array.from(new Set([...currentTags, ...tags[keywordID]]));
|
||||
if (selectedKeyword) {
|
||||
await selectedKeyword.update({ tags: JSON.stringify(multipleKeywords ? mergedTags : tags[keywordID]) });
|
||||
}
|
||||
}
|
||||
return res.status(200).json({ keywords });
|
||||
|
||||
@@ -18,10 +18,10 @@ const loginUser = async (req: NextApiRequest, res: NextApiResponse<loginResponse
|
||||
if (!req.body.username || !req.body.password) {
|
||||
return res.status(401).json({ error: 'Username Password Missing' });
|
||||
}
|
||||
|
||||
if (req.body.username === process.env.USER
|
||||
const userName = process.env.USERNAME ? process.env.USERNAME : process.env.USER;
|
||||
if (req.body.username === userName
|
||||
&& req.body.password === process.env.PASSWORD && process.env.SECRET) {
|
||||
const token = jwt.sign({ user: process.env.USER }, process.env.SECRET);
|
||||
const token = jwt.sign({ user: userName }, process.env.SECRET);
|
||||
const cookies = new Cookies(req, res);
|
||||
const expireDate = new Date();
|
||||
const sessDuration = process.env.SESSION_DURATION;
|
||||
@@ -30,7 +30,7 @@ const loginUser = async (req: NextApiRequest, res: NextApiResponse<loginResponse
|
||||
return res.status(200).json({ success: true, error: null });
|
||||
}
|
||||
|
||||
const error = req.body.username !== process.env.USER ? 'Incorrect Username' : 'Incorrect Password';
|
||||
const error = req.body.username !== userName ? 'Incorrect Username' : 'Incorrect Password';
|
||||
|
||||
return res.status(401).json({ success: false, error });
|
||||
};
|
||||
|
||||
@@ -2,6 +2,11 @@ const scrapingAnt:ScraperSettings = {
|
||||
id: 'scrapingant',
|
||||
name: 'ScrapingAnt',
|
||||
website: 'scrapingant.com',
|
||||
headers: (keyword) => {
|
||||
// eslint-disable-next-line max-len
|
||||
const mobileAgent = 'Mozilla/5.0 (Linux; Android 10; SM-G996U Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Mobile Safari/537.36';
|
||||
return keyword && keyword.device === 'mobile' ? { 'Ant-User-Agent': mobileAgent } : {};
|
||||
},
|
||||
scrapeURL: (keyword, settings, countryData) => {
|
||||
const scraperCountries = ['AE', 'BR', 'CN', 'DE', 'ES', 'FR', 'GB', 'HK', 'PL', 'IN', 'IT', 'IL', 'JP', 'NL', 'RU', 'SA', 'US', 'CZ'];
|
||||
const country = scraperCountries.includes(keyword.country.toUpperCase()) ? keyword.country : 'US';
|
||||
|
||||
@@ -84,8 +84,8 @@ body {
|
||||
|
||||
.domKeywords_head--alpha_desc .domKeywords_head_keyword::after,
|
||||
.domKeywords_head--pos_desc .domKeywords_head_position::after,
|
||||
.domKeywords_head--imp_desc .domKeywords_head_imp::after,
|
||||
.domKeywords_head--visits_desc .domKeywords_head_visits::after,
|
||||
.domKeywords_head--imp_asc .domKeywords_head_imp::after,
|
||||
.domKeywords_head--visits_asc .domKeywords_head_visits::after,
|
||||
.domKeywords_head--ctr_desc .domKeywords_head_ctr::after {
|
||||
content: "↓";
|
||||
display: inline-block;
|
||||
@@ -98,8 +98,8 @@ body {
|
||||
|
||||
.domKeywords_head--alpha_asc .domKeywords_head_keyword::after,
|
||||
.domKeywords_head--pos_asc .domKeywords_head_position::after,
|
||||
.domKeywords_head--imp_asc .domKeywords_head_imp::after,
|
||||
.domKeywords_head--visits_asc .domKeywords_head_visits::after,
|
||||
.domKeywords_head--imp_desc .domKeywords_head_imp::after,
|
||||
.domKeywords_head--visits_desc .domKeywords_head_visits::after,
|
||||
.domKeywords_head--ctr_asc .domKeywords_head_ctr::after {
|
||||
content: "↑";
|
||||
display: inline-block;
|
||||
|
||||
@@ -9,16 +9,16 @@ export const SCsortKeywords = (theKeywords:SCKeywordType[], sortBy:string) : SCK
|
||||
const keywords = theKeywords.map((k) => ({ ...k, position: k.position === 0 ? 111 : k.position }));
|
||||
switch (sortBy) {
|
||||
case 'imp_asc':
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => b.impressions - a.impressions);
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.impressions > b.impressions ? 1 : -1));
|
||||
break;
|
||||
case 'imp_desc':
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => a.impressions - b.impressions);
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.impressions > a.impressions ? 1 : -1));
|
||||
break;
|
||||
case 'visits_asc':
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => b.clicks - a.clicks);
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.clicks > b.clicks ? 1 : -1));
|
||||
break;
|
||||
case 'visits_desc':
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => a.clicks - b.clicks);
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.clicks > a.clicks ? 1 : -1));
|
||||
break;
|
||||
case 'ctr_asc':
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => b.ctr - a.ctr);
|
||||
@@ -27,17 +27,17 @@ export const SCsortKeywords = (theKeywords:SCKeywordType[], sortBy:string) : SCK
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => a.ctr - b.ctr);
|
||||
break;
|
||||
case 'pos_asc':
|
||||
sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.position > a.position ? 1 : -1));
|
||||
sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.position < a.position ? 1 : -1));
|
||||
sortedItems = sortedItems.map((k) => ({ ...k, position: k.position === 111 ? 0 : k.position }));
|
||||
break;
|
||||
case 'pos_desc':
|
||||
sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.position > b.position ? 1 : -1));
|
||||
sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.position < b.position ? 1 : -1));
|
||||
sortedItems = sortedItems.map((k) => ({ ...k, position: k.position === 111 ? 0 : k.position }));
|
||||
break;
|
||||
case 'alpha_asc':
|
||||
case 'alpha_desc':
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.keyword > a.keyword ? 1 : -1));
|
||||
break;
|
||||
case 'alpha_desc':
|
||||
case 'alpha_asc':
|
||||
sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.keyword > b.keyword ? 1 : -1));
|
||||
break;
|
||||
default:
|
||||
|
||||
@@ -49,7 +49,7 @@ export const getScraperClient = (keyword:KeywordType, settings:SettingsType, scr
|
||||
// Set Scraper Header
|
||||
const scrapeHeaders = scraper.headers ? scraper.headers(keyword, settings) : null;
|
||||
const scraperAPIURL = scraper.scrapeURL ? scraper.scrapeURL(keyword, settings, countries) : null;
|
||||
if (scrapeHeaders) {
|
||||
if (scrapeHeaders && Object.keys(scrapeHeaders).length > 0) {
|
||||
Object.keys(scrapeHeaders).forEach((headerItemKey:string) => {
|
||||
headers[headerItemKey] = scrapeHeaders[headerItemKey as keyof object];
|
||||
});
|
||||
@@ -114,7 +114,7 @@ export const scrapeKeywordFromGoogle = async (keyword:KeywordType, settings:Sett
|
||||
const scraperResult = scraperObj?.resultObjectKey && res[scraperObj.resultObjectKey] ? res[scraperObj.resultObjectKey] : '';
|
||||
const scrapeResult:string = (res.data || res.html || res.results || scraperResult || '');
|
||||
if (res && scrapeResult) {
|
||||
const extracted = scraperObj?.serpExtractor ? scraperObj.serpExtractor(scrapeResult) : extractScrapedResult(scrapeResult);
|
||||
const extracted = scraperObj?.serpExtractor ? scraperObj.serpExtractor(scrapeResult) : extractScrapedResult(scrapeResult, keyword.device);
|
||||
// await writeFile('result.txt', JSON.stringify(scrapeResult), { encoding: 'utf-8' }).catch((err) => { console.log(err); });
|
||||
const serp = getSerp(keyword.domain, extracted);
|
||||
refreshedResults = { ID: keyword.ID, keyword: keyword.keyword, position: serp.postion, url: serp.url, result: extracted, error: false };
|
||||
@@ -141,9 +141,10 @@ export const scrapeKeywordFromGoogle = async (keyword:KeywordType, settings:Sett
|
||||
/**
|
||||
* Extracts the Google Search result as object array from the Google Search's HTML content
|
||||
* @param {string} content - scraped google search page html data.
|
||||
* @param {string} device - The device of the keyword.
|
||||
* @returns {SearchResult[]}
|
||||
*/
|
||||
export const extractScrapedResult = (content: string): SearchResult[] => {
|
||||
export const extractScrapedResult = (content: string, device: string): SearchResult[] => {
|
||||
const extractedResult = [];
|
||||
|
||||
const $ = cheerio.load(content);
|
||||
@@ -163,6 +164,24 @@ export const extractScrapedResult = (content: string): SearchResult[] => {
|
||||
}
|
||||
}
|
||||
|
||||
// Mobile Scraper
|
||||
if (extractedResult.length === 0 && device === 'mobile') {
|
||||
const items = $('body').find('#rso > div');
|
||||
for (let i = 0; i < items.length; i += 1) {
|
||||
const item = $(items[i]);
|
||||
const linkDom = item.find('a[role="presentation"]');
|
||||
if (linkDom) {
|
||||
const url = linkDom.attr('href');
|
||||
const titleDom = linkDom.find('[role="link"]');
|
||||
const title = titleDom ? titleDom.text() : '';
|
||||
if (title && url) {
|
||||
lastPosition += 1;
|
||||
extractedResult.push({ title, url, position: lastPosition });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return extractedResult;
|
||||
};
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ const fetchSearchConsoleData = async (domainName:string, days:number, type?:stri
|
||||
date: row.keys[0],
|
||||
clicks: row.clicks,
|
||||
impressions: row.impressions,
|
||||
ctr: row.ctr,
|
||||
ctr: row.ctr * 100,
|
||||
position: row.position,
|
||||
});
|
||||
});
|
||||
@@ -101,7 +101,7 @@ export const parseSearchConsoleItem = (SCItem: SearchAnalyticsRawItem, domainNam
|
||||
const page = SCItem.keys[3] ? SCItem.keys[3].replace('https://', '').replace('http://', '').replace('www', '').replace(domainName, '') : '';
|
||||
const uid = `${country.toLowerCase()}:${device}:${keyword.replaceAll(' ', '_')}`;
|
||||
|
||||
return { keyword, uid, device, country, clicks, impressions, ctr, position, page };
|
||||
return { keyword, uid, device, country, clicks, impressions, ctr: ctr * 100, position, page };
|
||||
};
|
||||
|
||||
export const integrateKeywordSCData = (keyword: KeywordType, SCData:SCDomainDataType) : KeywordType => {
|
||||
|
||||
@@ -28,7 +28,7 @@ export const sortKeywords = (theKeywords:KeywordType[], sortBy:string, scDataTyp
|
||||
case 'alpha_desc':
|
||||
sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => (a.keyword > b.keyword ? 1 : -1));
|
||||
break;
|
||||
case 'imp_asc':
|
||||
case 'imp_desc':
|
||||
if (scDataType) {
|
||||
sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
|
||||
const bImpressionData = b.scData?.impressions[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
@@ -37,7 +37,7 @@ export const sortKeywords = (theKeywords:KeywordType[], sortBy:string, scDataTyp
|
||||
});
|
||||
}
|
||||
break;
|
||||
case 'imp_desc':
|
||||
case 'imp_asc':
|
||||
if (scDataType) {
|
||||
sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
|
||||
const bImpressionData = b.scData?.impressions[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
@@ -46,21 +46,21 @@ export const sortKeywords = (theKeywords:KeywordType[], sortBy:string, scDataTyp
|
||||
});
|
||||
}
|
||||
break;
|
||||
case 'visits_asc':
|
||||
if (scDataType) {
|
||||
sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
|
||||
const bImpressionData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
const aImpressionData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
return aImpressionData > bImpressionData ? 1 : -1;
|
||||
});
|
||||
}
|
||||
break;
|
||||
case 'visits_desc':
|
||||
if (scDataType) {
|
||||
sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
|
||||
const bImpressionData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
const aImpressionData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
return bImpressionData > aImpressionData ? 1 : -1;
|
||||
const bVisitsData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
const aVisitsData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
return aVisitsData > bVisitsData ? 1 : -1;
|
||||
});
|
||||
}
|
||||
break;
|
||||
case 'visits_asc':
|
||||
if (scDataType) {
|
||||
sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
|
||||
const bVisitsData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
const aVisitsData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
|
||||
return bVisitsData > aVisitsData ? 1 : -1;
|
||||
});
|
||||
}
|
||||
break;
|
||||
|
||||
Reference in New Issue
Block a user