fix: Resolves Website Thumbnail missing issue

You can now set your own thum.io key to fetch screenshot. More in docs. closes #131
2025-06-26 18:15:54 +00:00 · 2023-11-11 11:17:44 +06:00
parent c870250fbd
commit 4a60271cac
5 changed files with 73 additions and 26 deletions
--- a/components/domains/DomainItem.tsx
+++ b/components/domains/DomainItem.tsx
@@ -11,9 +11,10 @@ type DomainItemProps = {
   selected: boolean,
   isConsoleIntegrated: boolean,
   thumb: string,
+   updateThumb: Function,
 }

-const DomainItem = ({ domain, selected, isConsoleIntegrated = false, thumb }: DomainItemProps) => {
+const DomainItem = ({ domain, selected, isConsoleIntegrated = false, thumb, updateThumb }: DomainItemProps) => {
   const { keywordsUpdated, slug, keywordCount = 0, avgPosition = 0, scVisits = 0, scImpressions = 0, scPosition = 0 } = domain;
   // const router = useRouter();
   return (
@@ -21,8 +22,20 @@ const DomainItem = ({ domain, selected, isConsoleIntegrated = false, thumb }: Do
         <Link href={`/domain/${slug}`} passHref={true}>
         <a className='flex flex-col lg:flex-row'>
            <div className={`flex-1 p-6 flex ${!isConsoleIntegrated ? 'basis-1/3' : ''}`}>
-               <div className="domain_thumb w-20 h-20 mr-6 bg-slate-100 rounded border border-gray-200 overflow-hidden">
-                  {thumb && <img src={thumb} alt={domain.domain} />}
+               <div className="group domain_thumb w-20 h-20 mr-6 bg-slate-100 rounded
+                  border border-gray-200 overflow-hidden flex justify-center relative">
+                  <button
+                     className=' absolute right-1 top-0 text-gray-400 p-1 transition-all
+                     invisible opacity-0 group-hover:visible group-hover:opacity-100 hover:text-gray-600 z-10'
+                     title='Reload Website Screenshot'
+                     onClick={(e) => { e.preventDefault(); e.stopPropagation(); updateThumb(domain.domain); }}
+                  >
+                     <Icon type="reload" size={12} />
+                  </button>
+                  <img
+                  className={`self-center ${!thumb ? 'max-w-[50px]' : ''}`}
+                  src={thumb || `https://www.google.com/s2/favicons?domain=${domain.domain}&sz=128`} alt={domain.domain}
+                  />
               </div>
               <div className="domain_details flex-1">
                  <h3 className='font-semibold text-base mb-2'>{domain.domain}</h3>
--- a/pages/api/settings.ts
+++ b/pages/api/settings.ts
@@ -55,6 +55,7 @@ const updateSettings = async (req: NextApiRequest, res: NextApiResponse<Settings
 };

 export const getAppSettings = async () : Promise<SettingsType> => {
+   const screenshotAPIKey = process.env.SCREENSHOT_API || '69408-serpbear';
   try {
      const settingsRaw = await readFile(`${process.cwd()}/data/settings.json`, { encoding: 'utf-8' });
      const failedQueueRaw = await readFile(`${process.cwd()}/data/failed_queue.json`, { encoding: 'utf-8' });
@@ -73,6 +74,7 @@ export const getAppSettings = async () : Promise<SettingsType> => {
            search_console_integrated: !!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL),
            available_scapers: allScrapers.map((scraper) => ({ label: scraper.name, value: scraper.id })),
            failed_queue: failedQueue,
+            screenshot_key: screenshotAPIKey,
         };
      } catch (error) {
         console.log('Error Decrypting Settings API Keys!');
@@ -81,7 +83,7 @@ export const getAppSettings = async () : Promise<SettingsType> => {
      return decryptedSettings;
   } catch (error) {
      console.log('[ERROR] Getting App Settings. ', error);
-      const settings = {
+      const settings: SettingsType = {
         scraper_type: 'none',
         notification_interval: 'never',
         notification_email: '',
@@ -91,6 +93,7 @@ export const getAppSettings = async () : Promise<SettingsType> => {
         smtp_username: '',
         smtp_password: '',
         scrape_retry: false,
+         screenshot_key: screenshotAPIKey,
      };
      const otherSettings = {
         available_scapers: allScrapers.map((scraper) => ({ label: scraper.name, value: scraper.id })),
--- a/pages/domains/index.tsx
+++ b/pages/domains/index.tsx
@@ -3,11 +3,12 @@ import type { NextPage } from 'next';
 import Head from 'next/head';
 import { useRouter } from 'next/router';
 import { CSSTransition } from 'react-transition-group';
+import toast, { Toaster } from 'react-hot-toast';
 import TopBar from '../../components/common/TopBar';
 import AddDomain from '../../components/domains/AddDomain';
 import Settings from '../../components/settings/Settings';
 import { useFetchSettings } from '../../services/settings';
-import { useFetchDomains } from '../../services/domains';
+import { fetchDomainScreenshot, useFetchDomains } from '../../services/domains';
 import DomainItem from '../../components/domains/DomainItem';
 import Icon from '../../components/common/Icon';

@@ -23,32 +24,17 @@ const SingleDomain: NextPage = () => {
   const { data: domainsData, isLoading } = useFetchDomains(router, true);

   useEffect(() => {
-      // console.log('Domains Data: ', domainsData);
-      if (domainsData?.domains && domainsData.domains.length > 0) {
-         const domainThumbsRaw = localStorage.getItem('domainThumbs');
-         const domThumbs = domainThumbsRaw ? JSON.parse(domainThumbsRaw) : {};
+      if (domainsData?.domains && domainsData.domains.length > 0 && appSettings?.settings?.screenshot_key) {
         domainsData.domains.forEach(async (domain:DomainType) => {
            if (domain.domain) {
-               if (!domThumbs[domain.domain]) {
-                  const domainImageBlob = await fetch(`https://image.thum.io/get/auth/66909-serpbear/maxAge/96/width/200/https://${domain.domain}`).then((res) => res.blob());
-                  if (domainImageBlob) {
-                     const reader = new FileReader();
-                     await new Promise((resolve, reject) => {
-                       reader.onload = resolve;
-                       reader.onerror = reject;
-                       reader.readAsDataURL(domainImageBlob);
-                     });
-                     const imageBase: string = reader.result && typeof reader.result === 'string' ? reader.result : '';
-                     localStorage.setItem('domainThumbs', JSON.stringify({ ...domThumbs, [domain.domain]: imageBase }));
-                     setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain.domain]: imageBase }));
-                  }
-               } else {
-                  setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain.domain]: domThumbs[domain.domain] }));
+               const domainThumb = await fetchDomainScreenshot(domain.domain, appSettings.settings.screenshot_key);
+               if (domainThumb) {
+                  setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain.domain]: domainThumb }));
               }
            }
         });
      }
-   }, [domainsData]);
+   }, [domainsData, appSettings]);

   useEffect(() => {
      // console.log('appSettings.settings: ', appSettings && appSettings.settings);
@@ -57,6 +43,18 @@ const SingleDomain: NextPage = () => {
      }
   }, [appSettings]);

+   const manuallyUpdateThumb = async (domain: string) => {
+      if (domain && appSettings?.settings?.screenshot_key) {
+         const domainThumb = await fetchDomainScreenshot(domain, appSettings.settings.screenshot_key, true);
+         if (domainThumb) {
+            toast(`${domain} Screenshot Updated Successfully!`, { icon: '✔️' });
+            setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain]: domainThumb }));
+         } else {
+            toast(`Failed to Fetch ${domain} Screenshot!`, { icon: '⚠️' });
+         }
+      }
+   };
+
   return (
      <div className="Domain flex flex-col min-h-screen">
         {noScrapprtError && (
@@ -90,6 +88,7 @@ const SingleDomain: NextPage = () => {
                           selected={false}
                           isConsoleIntegrated={!!(appSettings && appSettings?.settings?.search_console_integrated) }
                           thumb={domainThumbs[domain.domain]}
+                           updateThumb={manuallyUpdateThumb}
                           // isConsoleIntegrated={false}
                           />;
               })}
@@ -115,6 +114,7 @@ const SingleDomain: NextPage = () => {
         <footer className='text-center flex flex-1 justify-center pb-5 items-end'>
            <span className='text-gray-500 text-xs'><a href='https://github.com/towfiqi/serpbear' target="_blank" rel='noreferrer'>SerpBear v{appSettings?.settings?.version || '0.0.0'}</a></span>
         </footer>
+         <Toaster position='bottom-center' containerClassName="react_toaster" />
      </div>
   );
 };
--- a/services/domains.tsx
+++ b/services/domains.tsx
@@ -19,6 +19,36 @@ export async function fetchDomains(router: NextRouter, withStats:boolean) {
   return res.json();
 }

+export async function fetchDomainScreenshot(domain: string, screenshot_key:string, forceFetch = false): Promise<string | false> {
+   const domainThumbsRaw = localStorage.getItem('domainThumbs');
+   const domThumbs = domainThumbsRaw ? JSON.parse(domainThumbsRaw) : {};
+   if (!domThumbs[domain] || forceFetch) {
+      try {
+         const screenshotURL = `https://image.thum.io/get/auth/${screenshot_key}/maxAge/96/width/200/https://${domain}`;
+         const domainImageRes = await fetch(screenshotURL);
+         const domainImageBlob = domainImageRes.status === 200 ? await domainImageRes.blob() : false;
+         if (domainImageBlob) {
+            const reader = new FileReader();
+            await new Promise((resolve, reject) => {
+               reader.onload = resolve;
+               reader.onerror = reject;
+               reader.readAsDataURL(domainImageBlob);
+            });
+            const imageBase: string = reader.result && typeof reader.result === 'string' ? reader.result : '';
+            localStorage.setItem('domainThumbs', JSON.stringify({ ...domThumbs, [domain]: imageBase }));
+            return imageBase;
+         }
+         return false;
+      } catch (error) {
+         return false;
+      }
+   } else if (domThumbs[domain]) {
+         return domThumbs[domain];
+   }
+
+   return false;
+}
+
 export function useFetchDomains(router: NextRouter, withStats:boolean = false) {
   return useQuery('domains', () => fetchDomains(router, withStats));
 }
--- a/types.d.ts
+++ b/types.d.ts
@@ -83,7 +83,8 @@ type SettingsType = {
   scrape_delay?: string,
   scrape_retry?: boolean,
   failed_queue?: string[]
-   version?: string
+   version?: string,
+   screenshot_key?: string,
 }

 type KeywordSCDataChild = {