API-mega-list/fetch_apify_actors.py

"""
Script to fetch all Apify Actors from the API and compile them into a list
with affiliate links.
"""

import requests
import json
import time
from typing import List, Dict

# Apify API endpoint
API_BASE_URL = "https://api.apify.com/v2/store"
AFFILIATE_PARAM = "?fpr=p2hrc6"

def fetch_all_actors(limit: int = 100) -> List[Dict]:
    """
    Fetch all actors from Apify Store API with pagination.
    
    Args:
        limit: Number of actors to fetch per request (max 100)
    
    Returns:
        List of all actors with their details
    """
    all_actors = []
    offset = 0
    total_fetched = 0
    
    print("Starting to fetch Apify Actors...")
    
    while True:
        try:
            # Make API request with pagination
            params = {
                "limit": limit,
                "offset": offset
            }
            
            response = requests.get(API_BASE_URL, params=params)
            response.raise_for_status()
            
            data = response.json()
            
            # Extract actors from response
            actors = data.get("data", {}).get("items", [])
            
            if not actors:
                print(f"No more actors found at offset {offset}")
                break
            
            # Process each actor
            for actor in actors:
                actor_info = {
                    "name": actor.get("name", "Unknown"),
                    "username": actor.get("username", ""),
                    "title": actor.get("title", actor.get("name", "Unknown")),
                    "description": actor.get("description", ""),
                    "url": actor.get("url", ""),
                    "affiliate_url": "",
                    "stats": actor.get("stats", {}),
                    "categories": actor.get("categories", []),
                    "createdAt": actor.get("createdAt", ""),
                    "modifiedAt": actor.get("modifiedAt", "")
                }
                
                # Create affiliate URL
                if actor_info["url"]:
                    # Check if URL already has query parameters
                    separator = "&" if "?" in actor_info["url"] else "?"
                    actor_info["affiliate_url"] = f"{actor_info['url']}{separator}fpr=p2hrc6"
                else:
                    # Construct URL from username and name if URL is missing
                    if actor_info["username"] and actor_info["name"]:
                        actor_info["url"] = f"https://apify.com/{actor_info['username']}/{actor_info['name']}"
                        actor_info["affiliate_url"] = f"{actor_info['url']}?fpr=p2hrc6"
                
                all_actors.append(actor_info)
            
            total_fetched += len(actors)
            print(f"Fetched {total_fetched} actors so far... (offset: {offset})")
            
            # Check if we've reached the end
            total_count = data.get("data", {}).get("total", 0)
            if offset + len(actors) >= total_count:
                print(f"Reached end. Total actors: {total_count}")
                break
            
            offset += limit
            
            # Be respectful with API rate limits
            time.sleep(0.5)
            
        except requests.exceptions.RequestException as e:
            print(f"Error fetching actors at offset {offset}: {e}")
            print("Retrying in 5 seconds...")
            time.sleep(5)
            continue
        except Exception as e:
            print(f"Unexpected error: {e}")
            break
    
    print(f"\nTotal actors fetched: {len(all_actors)}")
    return all_actors

def save_to_json(actors: List[Dict], filename: str = "apify_actors.json"):
    """Save actors data to JSON file."""
    with open(filename, "w", encoding="utf-8") as f:
        json.dump(actors, f, indent=2, ensure_ascii=False)
    print(f"Saved {len(actors)} actors to {filename}")

def generate_markdown_list(actors: List[Dict], filename: str = "APIFY_ACTORS.md"):
    """Generate a markdown file with all actors and affiliate links."""
    with open(filename, "w", encoding="utf-8") as f:
        f.write("# Apify Actors List\n\n")
        f.write(f"Complete list of {len(actors)} Apify Actors (APIs) available on the Apify platform.\n\n")
        f.write("---\n\n")
        
        # Group by category if available
        actors_by_category = {}
        uncategorized = []
        
        for actor in actors:
            categories = actor.get("categories", [])
            if categories:
                for category in categories:
                    if category not in actors_by_category:
                        actors_by_category[category] = []
                    actors_by_category[category].append(actor)
            else:
                uncategorized.append(actor)
        
        # Write categorized actors
        if actors_by_category:
            for category in sorted(actors_by_category.keys()):
                f.write(f"## {category}\n\n")
                for actor in sorted(actors_by_category[category], key=lambda x: x.get("title", "")):
                    title = actor.get("title", actor.get("name", "Unknown"))
                    affiliate_url = actor.get("affiliate_url", actor.get("url", ""))
                    description = actor.get("description", "")
                    
                    if description:
                        f.write(f"- **[{title}]({affiliate_url})** - {description}\n")
                    else:
                        f.write(f"- **[{title}]({affiliate_url})**\n")
                f.write("\n")
        
        # Write uncategorized actors
        if uncategorized:
            f.write("## Uncategorized\n\n")
            for actor in sorted(uncategorized, key=lambda x: x.get("title", "")):
                title = actor.get("title", actor.get("name", "Unknown"))
                affiliate_url = actor.get("affiliate_url", actor.get("url", ""))
                description = actor.get("description", "")
                
                if description:
                    f.write(f"- **[{title}]({affiliate_url})** - {description}\n")
                else:
                    f.write(f"- **[{title}]({affiliate_url})**\n")
            f.write("\n")
        
        f.write("---\n\n")
        f.write(f"*Total: {len(actors)} Actors*\n")
        f.write(f"*Last updated: {time.strftime('%Y-%m-%d %H:%M:%S')}*\n")
    
    print(f"Generated markdown list: {filename}")

def generate_simple_list(actors: List[Dict], filename: str = "apify_actors_simple.txt"):
    """Generate a simple text file with just names and affiliate URLs."""
    with open(filename, "w", encoding="utf-8") as f:
        for actor in sorted(actors, key=lambda x: x.get("title", "")):
            title = actor.get("title", actor.get("name", "Unknown"))
            affiliate_url = actor.get("affiliate_url", actor.get("url", ""))
            f.write(f"{title}|{affiliate_url}\n")
    
    print(f"Generated simple list: {filename}")

if __name__ == "__main__":
    print("=" * 60)
    print("Apify Actors Fetcher")
    print("=" * 60)
    print()
    
    # Fetch all actors
    actors = fetch_all_actors(limit=100)
    
    if actors:
        # Save to JSON
        save_to_json(actors, "apify_actors.json")
        
        # Generate markdown list
        generate_markdown_list(actors, "APIFY_ACTORS.md")
        
        # Generate simple list
        generate_simple_list(actors, "apify_actors_simple.txt")
        
        print("\n" + "=" * 60)
        print("Done! All files have been generated.")
        print("=" * 60)
    else:
        print("No actors were fetched. Please check the API connection.")