GoClaw/server/orchestrator.ts

/**
 * GoClaw Orchestrator — Main AI Agent
 *
 * The orchestrator is the primary interface for the user. It has access to:
 * - All specialized agents (Browser, Tool Builder, Agent Compiler, custom)
 * - All tools (shell, file, HTTP, Docker, browser)
 * - All skills (registered capabilities)
 * - System management (create/install/run components)
 *
 * It uses a tool-use loop: LLM decides which tools to call, executes them,
 * feeds results back to LLM, and repeats until a final answer is ready.
 */

import { exec } from "child_process";
import { promisify } from "util";
import { readFile, writeFile, readdir, stat, mkdir } from "fs/promises";
import { existsSync } from "fs";
import { join, dirname } from "path";
import { invokeLLM } from "./_core/llm";
import { chatCompletion } from "./ollama";
import { getDb } from "./db";
import { agents, agentHistory } from "../drizzle/schema";
import { eq } from "drizzle-orm";

const execAsync = promisify(exec);

// ─── Tool Definitions for LLM ────────────────────────────────────────────────

export const ORCHESTRATOR_TOOLS = [
  {
    type: "function" as const,
    function: {
      name: "shell_exec",
      description:
        "Execute a shell command on the server. Use for: running scripts, installing packages, git operations, checking system status, compiling code.",
      parameters: {
        type: "object",
        properties: {
          command: { type: "string", description: "Shell command to execute" },
          cwd: { type: "string", description: "Working directory (default: project root)" },
          timeout: { type: "number", description: "Timeout in ms (default: 30000)" },
        },
        required: ["command"],
        additionalProperties: false,
      },
    },
  },
  {
    type: "function" as const,
    function: {
      name: "file_read",
      description: "Read a file from the filesystem. Returns file content as text.",
      parameters: {
        type: "object",
        properties: {
          path: { type: "string", description: "Absolute or relative file path" },
          encoding: { type: "string", description: "File encoding (default: utf-8)" },
        },
        required: ["path"],
        additionalProperties: false,
      },
    },
  },
  {
    type: "function" as const,
    function: {
      name: "file_write",
      description: "Write content to a file. Creates directories if needed.",
      parameters: {
        type: "object",
        properties: {
          path: { type: "string", description: "Absolute or relative file path" },
          content: { type: "string", description: "Content to write" },
          append: { type: "boolean", description: "Append to file instead of overwrite" },
        },
        required: ["path", "content"],
        additionalProperties: false,
      },
    },
  },
  {
    type: "function" as const,
    function: {
      name: "file_list",
      description: "List files in a directory.",
      parameters: {
        type: "object",
        properties: {
          path: { type: "string", description: "Directory path" },
          recursive: { type: "boolean", description: "List recursively" },
        },
        required: ["path"],
        additionalProperties: false,
      },
    },
  },
  {
    type: "function" as const,
    function: {
      name: "http_request",
      description: "Make an HTTP request to any URL. Supports GET, POST, PUT, DELETE.",
      parameters: {
        type: "object",
        properties: {
          url: { type: "string", description: "Target URL" },
          method: { type: "string", description: "HTTP method (default: GET)" },
          headers: { type: "object", description: "Request headers" },
          body: { type: "string", description: "Request body (for POST/PUT)" },
        },
        required: ["url"],
        additionalProperties: false,
      },
    },
  },
  {
    type: "function" as const,
    function: {
      name: "delegate_to_agent",
      description:
        "Delegate a task to a specialized agent. Use for: web browsing (Browser Agent), creating tools (Tool Builder), compiling agents (Agent Compiler), or any other specialized agent.",
      parameters: {
        type: "object",
        properties: {
          agentId: { type: "number", description: "Agent ID to delegate to" },
          message: { type: "string", description: "Task description for the agent" },
        },
        required: ["agentId", "message"],
        additionalProperties: false,
      },
    },
  },
  {
    type: "function" as const,
    function: {
      name: "list_agents",
      description: "List all available specialized agents with their capabilities.",
      parameters: {
        type: "object",
        properties: {},
        additionalProperties: false,
      },
    },
  },
  {
    type: "function" as const,
    function: {
      name: "list_skills",
      description: "List all installed skills (capabilities) available to agents.",
      parameters: {
        type: "object",
        properties: {},
        additionalProperties: false,
      },
    },
  },
  {
    type: "function" as const,
    function: {
      name: "install_skill",
      description:
        "Install a new skill into the system. A skill is a reusable capability module.",
      parameters: {
        type: "object",
        properties: {
          name: { type: "string", description: "Skill name (snake_case)" },
          description: { type: "string", description: "What this skill does" },
          code: { type: "string", description: "JavaScript implementation of the skill" },
          category: { type: "string", description: "Skill category (web, data, ai, system, etc.)" },
        },
        required: ["name", "description", "code"],
        additionalProperties: false,
      },
    },
  },
  {
    type: "function" as const,
    function: {
      name: "docker_exec",
      description: "Execute a Docker command (docker ps, docker logs, docker exec, etc.).",
      parameters: {
        type: "object",
        properties: {
          command: { type: "string", description: "Docker command (without 'docker' prefix)" },
        },
        required: ["command"],
        additionalProperties: false,
      },
    },
  },
];

// ─── Tool Execution ───────────────────────────────────────────────────────────

const PROJECT_ROOT = "/home/ubuntu/goclaw-control-center";

async function executeTool(
  toolName: string,
  args: Record<string, any>
): Promise<{ success: boolean; result?: any; error?: string }> {
  try {
    switch (toolName) {
      case "shell_exec": {
        const cwd = args.cwd || PROJECT_ROOT;
        const timeout = args.timeout || 30000;
        const { stdout, stderr } = await execAsync(args.command, { cwd, timeout });
        return {
          success: true,
          result: { stdout: stdout.trim(), stderr: stderr.trim(), command: args.command },
        };
      }

      case "file_read": {
        const filePath = args.path.startsWith("/") ? args.path : join(PROJECT_ROOT, args.path);
        if (!existsSync(filePath)) {
          return { success: false, error: `File not found: ${filePath}` };
        }
        const content = await readFile(filePath, (args.encoding as BufferEncoding) || "utf-8");
        const lines = (content as string).split("\n").length;
        return { success: true, result: { path: filePath, content, lines } };
      }

      case "file_write": {
        const filePath = args.path.startsWith("/") ? args.path : join(PROJECT_ROOT, args.path);
        await mkdir(dirname(filePath), { recursive: true });
        if (args.append) {
          const existing = existsSync(filePath)
            ? await readFile(filePath, "utf-8")
            : "";
          await writeFile(filePath, existing + args.content, "utf-8");
        } else {
          await writeFile(filePath, args.content, "utf-8");
        }
        return { success: true, result: { path: filePath, written: args.content.length } };
      }

      case "file_list": {
        const dirPath = args.path.startsWith("/") ? args.path : join(PROJECT_ROOT, args.path);
        if (!existsSync(dirPath)) {
          return { success: false, error: `Directory not found: ${dirPath}` };
        }
        const entries = await readdir(dirPath);
        const details = await Promise.all(
          entries.map(async (name) => {
            const fullPath = join(dirPath, name);
            const s = await stat(fullPath);
            return { name, type: s.isDirectory() ? "dir" : "file", size: s.size };
          })
        );
        return { success: true, result: { path: dirPath, entries: details } };
      }

      case "http_request": {
        const method = (args.method || "GET").toUpperCase();
        const response = await fetch(args.url, {
          method,
          headers: args.headers || {},
          body: args.body || undefined,
          signal: AbortSignal.timeout(15000),
        });
        const text = await response.text();
        let body: any = text;
        try {
          body = JSON.parse(text);
        } catch {
          // keep as text
        }
        return {
          success: true,
          result: {
            status: response.status,
            statusText: response.statusText,
            headers: Object.fromEntries(response.headers.entries()),
            body,
          },
        };
      }

      case "delegate_to_agent": {
        const db = await getDb();
        if (!db) return { success: false, error: "DB not available" };
        const [agent] = await db
          .select()
          .from(agents)
          .where(eq(agents.id, args.agentId))
          .limit(1);
        if (!agent) {
          return { success: false, error: `Agent ${args.agentId} not found` };
        }
        const messages: Array<{ role: "system" | "user" | "assistant"; content: string }> = [];
        if (agent.systemPrompt) {
          messages.push({ role: "system", content: agent.systemPrompt });
        }
        messages.push({ role: "user", content: args.message });
        const result = await chatCompletion(agent.model, messages, {
          temperature: agent.temperature ? parseFloat(agent.temperature as string) : 0.7,
          max_tokens: agent.maxTokens ?? 2048,
        });
        const response = result.choices[0]?.message?.content ?? "";
        // Save to agent history
        await db.insert(agentHistory).values({
          agentId: agent.id,
          userMessage: args.message,
          agentResponse: response,
          conversationId: `orchestrator-${Date.now()}`,
          status: "success",
        });
        return {
          success: true,
          result: {
            agentName: agent.name,
            agentRole: agent.role,
            response,
            model: result.model,
            usage: result.usage,
          },
        };
      }

      case "list_agents": {
        const db = await getDb();
        if (!db) return { success: false, error: "DB not available" };
        const allAgents = await db
          .select({
            id: agents.id,
            name: agents.name,
            role: agents.role,
            description: agents.description,
            model: agents.model,
            isActive: agents.isActive,
            allowedTools: agents.allowedTools,
            tags: agents.tags,
          })
          .from(agents)
          .where(eq(agents.isActive, true));
        return { success: true, result: { agents: allAgents, count: allAgents.length } };
      }

      case "list_skills": {
        const skillsPath = join(PROJECT_ROOT, "server/skills");
        if (!existsSync(skillsPath)) {
          return { success: true, result: { skills: [], count: 0 } };
        }
        const entries = await readdir(skillsPath);
        const skills = await Promise.all(
          entries.map(async (name) => {
            const metaPath = join(skillsPath, name, "skill.json");
            if (existsSync(metaPath)) {
              const meta = JSON.parse(await readFile(metaPath, "utf-8") as string);
              return { name, ...meta };
            }
            return { name, description: "No metadata" };
          })
        );
        return { success: true, result: { skills, count: skills.length } };
      }

      case "install_skill": {
        const skillsPath = join(PROJECT_ROOT, "server/skills", args.name);
        await mkdir(skillsPath, { recursive: true });
        // Write skill implementation
        await writeFile(join(skillsPath, "index.js"), args.code, "utf-8");
        // Write skill metadata
        const meta = {
          name: args.name,
          description: args.description,
          category: args.category || "custom",
          installedAt: new Date().toISOString(),
          version: "1.0.0",
        };
        await writeFile(join(skillsPath, "skill.json"), JSON.stringify(meta, null, 2), "utf-8");
        return { success: true, result: { installed: args.name, path: skillsPath } };
      }

      case "docker_exec": {
        const { stdout, stderr } = await execAsync(`docker ${args.command}`, {
          timeout: 15000,
        });
        return { success: true, result: { stdout: stdout.trim(), stderr: stderr.trim() } };
      }

      default:
        return { success: false, error: `Unknown tool: ${toolName}` };
    }
  } catch (err: any) {
    return { success: false, error: err.message || String(err) };
  }
}

// ─── Orchestrator Chat ────────────────────────────────────────────────────────

export interface OrchestratorMessage {
  role: "user" | "assistant" | "system";
  content: string;
}

export interface ToolCallStep {
  tool: string;
  args: Record<string, any>;
  result: any;
  success: boolean;
  durationMs: number;
}

export interface OrchestratorResult {
  success: boolean;
  response: string;
  toolCalls: ToolCallStep[];
  model?: string;
  usage?: { prompt_tokens: number; completion_tokens: number; total_tokens: number };
  error?: string;
}

const ORCHESTRATOR_SYSTEM_PROMPT = `You are GoClaw Orchestrator — the main AI agent managing the GoClaw distributed AI system.

You have full access to:
1. **Specialized Agents**: Browser Agent (web browsing), Tool Builder (create tools), Agent Compiler (create agents)
2. **System Tools**: shell_exec (run commands), file_read/write (manage files), http_request (web requests), docker_exec (Docker management)
3. **Skills Registry**: list_skills (see capabilities), install_skill (add new capabilities)

Your responsibilities:
- Answer user questions directly when possible
- Delegate complex web tasks to Browser Agent
- Delegate tool creation to Tool Builder agent
- Delegate agent creation to Agent Compiler
- Execute shell commands to manage the system, install packages, run scripts
- Read and write files to modify the codebase
- Monitor Docker containers and services

Decision making:
- For simple questions: answer directly without tools
- For web research: use delegate_to_agent with Browser Agent (id: 1)
- For creating tools: use delegate_to_agent with Tool Builder (id: 2)
- For creating agents: use delegate_to_agent with Agent Compiler (id: 3)
- For system tasks: use shell_exec, file_read/write
- For Docker: use docker_exec
- Always use list_agents first if you're unsure which agent to delegate to

Response style:
- Be concise and actionable
- Show what tools you used and their results
- If a task requires multiple steps, execute them in sequence
- Respond in the same language as the user

You are running on a Linux server with Node.js, Docker, and full internet access.`;

/**
 * Load orchestrator config from DB.
 * Returns { model, systemPrompt, allowedTools } from the agent with isOrchestrator=true.
 * Falls back to defaults if not found.
 */
export async function getOrchestratorConfig(): Promise<{
  id: number | null;
  name: string;
  model: string;
  systemPrompt: string;
  allowedTools: string[];
  temperature: number;
  maxTokens: number;
}> {
  try {
    const db = await getDb();
    if (!db) throw new Error("DB not available");
    const [orch] = await db
      .select()
      .from(agents)
      .where(eq(agents.isOrchestrator, true))
      .limit(1);
    if (orch) {
      return {
        id: orch.id,
        name: orch.name,
        model: orch.model,
        systemPrompt: orch.systemPrompt ?? ORCHESTRATOR_SYSTEM_PROMPT,
        allowedTools: (orch.allowedTools as string[]) ?? [],
        temperature: parseFloat(orch.temperature ?? "0.5"),
        maxTokens: orch.maxTokens ?? 8192,
      };
    }
  } catch (err) {
    console.error("[Orchestrator] Failed to load config from DB:", err);
  }
  // Fallback defaults
  return {
    id: null,
    name: "Orchestrator",
    model: "qwen2.5:7b",
    systemPrompt: ORCHESTRATOR_SYSTEM_PROMPT,
    allowedTools: [],
    temperature: 0.5,
    maxTokens: 8192,
  };
}

export async function orchestratorChat(
  messages: OrchestratorMessage[],
  model?: string,
  maxToolIterations: number = 10
): Promise<OrchestratorResult> {
  const toolCalls: ToolCallStep[] = [];

  // Load config from DB — model and systemPrompt are configurable
  const config = await getOrchestratorConfig();
  const activeModel = model ?? config.model;
  const activeSystemPrompt = config.systemPrompt;

  // Build conversation with system prompt
  const conversation: Array<{
    role: "system" | "user" | "assistant" | "tool" | "function";
    content: string | any;
    tool_call_id?: string;
    name?: string;
  }> = [
    { role: "system", content: activeSystemPrompt },
    ...messages.map((m) => ({ role: m.role, content: m.content })),
  ];

  let iterations = 0;
  let finalResponse = "";
  let lastUsage: any;
  let lastModel: string = activeModel;

  while (iterations < maxToolIterations) {
    iterations++;

    // Call LLM with tools using the model from DB config
    let llmResult: any;
    try {
      llmResult = await chatCompletion(activeModel, conversation as any, {
        temperature: config.temperature,
        max_tokens: config.maxTokens,
        tools: ORCHESTRATOR_TOOLS as any,
        tool_choice: "auto",
      });
    } catch (err: any) {
      // Fallback: try without tools if model doesn't support them
      try {
        const fallbackResult = await chatCompletion(activeModel, conversation as any, {
          temperature: config.temperature,
          max_tokens: config.maxTokens,
        });
        finalResponse = fallbackResult.choices[0]?.message?.content ?? "";
        lastUsage = fallbackResult.usage;
        lastModel = fallbackResult.model ?? activeModel;
        break;
      } catch (fallbackErr: any) {
        return {
          success: false,
          response: "",
          toolCalls,
          error: `LLM error (model: ${activeModel}): ${fallbackErr.message}`,
        };
      }
    }

    const choice = llmResult.choices?.[0];
    if (!choice) break;

    lastUsage = llmResult.usage;
    lastModel = llmResult.model || model;

    const message = choice.message;

    // Check if LLM wants to call tools
    if (choice.finish_reason === "tool_calls" && message.tool_calls?.length > 0) {
      // Add assistant message with tool calls to conversation
      conversation.push({
        role: "assistant",
        content: message.content || "",
        ...message,
      });

      // Execute each tool call
      for (const tc of message.tool_calls) {
        const toolName = tc.function?.name;
        let toolArgs: Record<string, any> = {};
        try {
          toolArgs = JSON.parse(tc.function?.arguments || "{}");
        } catch {
          toolArgs = {};
        }

        const startTime = Date.now();
        const toolResult = await executeTool(toolName, toolArgs);
        const durationMs = Date.now() - startTime;

        toolCalls.push({
          tool: toolName,
          args: toolArgs,
          result: toolResult.result,
          success: toolResult.success,
          durationMs,
        });

        // Add tool result to conversation
        conversation.push({
          role: "tool",
          content: JSON.stringify(
            toolResult.success
              ? toolResult.result
              : { error: toolResult.error }
          ),
          tool_call_id: tc.id,
          name: toolName,
        });
      }

      // Continue loop — LLM will process tool results
      continue;
    }

    // LLM finished — extract final response
    finalResponse = message.content || "";
    break;
  }

  return {
    success: true,
    response: finalResponse,
    toolCalls,
    model: lastModel,
    usage: lastUsage,
  };
}