/** * Minimal dependency-free HTML-to-readable-text conversion for `web_fetch`, not a full parser. It * removes non-content elements and tags, decodes common entities, collapses whitespace, and keeps * basic headings, lists, and links. A richer converter can replace it without changing the seam or * tool schema. * @module @deepseek-ai/dsh-tool-web/html */ /** Decode the handful of HTML entities common in textual content. */ function decodeEntities(text: string): string { return text .replace(/&(#[xX][0-9a-fA-F]+|#[0-9]+|[a-zA-Z]+);/g, (match, entity: string) => { if (entity.startsWith('#x') || entity.startsWith('#X')) { const code = Number.parseInt(entity.slice(2), 16) return safeFromCodePoint(code, match) } if (entity.startsWith('#')) { const code = Number.parseInt(entity.slice(1), 10) return safeFromCodePoint(code, match) } return NAMED_ENTITIES[entity] ?? match }) } const NAMED_ENTITIES: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', copy: '©', reg: '®', trade: '™', hellip: '…', mdash: '—', ndash: '–', } function safeFromCodePoint(code: number, fallback: string): string { try { return String.fromCodePoint(code) } catch { // An out-of-range code point (RangeError) is the only failure here; keep the // original entity text rather than throwing out of pure presentation. return fallback } } /** * Convert an HTML document to a readable markdown-ish text approximation. * Best-effort and lossy by design — fidelity is the job of a future heavier * converter, not this fallback. * * @param html - the raw HTML source. * @returns plain text with markdown headings, list bullets, and links; * whitespace collapsed to at most one blank line and trimmed. */ export function htmlToMarkdown(html: string): string { let text = html // Drop non-content elements entirely (including their contents). .replace(/]*>[\s\S]*?<\/script>/gi, '') .replace(/]*>[\s\S]*?<\/style>/gi, '') .replace(/]*>[\s\S]*?<\/noscript>/gi, '') .replace(//g, '') // Convert links to markdown before stripping tags. text = text.replace(/]*\bhref\s*=\s*["']([^"']*)["'][^>]*>([\s\S]*?)<\/a>/gi, (_match, href: string, label: string) => { const cleanLabel = label.replace(/<[^>]+>/g, '').trim() return cleanLabel.length > 0 ? `[${cleanLabel}](${href})` : href }) // Headings → markdown hashes. text = text.replace(/]*>([\s\S]*?)<\/h\1>/gi, (_match, level: string, body: string) => { const hashes = '#'.repeat(Number(level)) return `\n\n${hashes} ${body.replace(/<[^>]+>/g, '').trim()}\n\n` }) // List items → bullets. text = text.replace(/]*>([\s\S]*?)<\/li>/gi, (_match, body: string) => `\n- ${body.replace(/<[^>]+>/g, '').trim()}`) // Block-level breaks become paragraph breaks. text = text .replace(/<\/(p|div|section|article|header|footer|tr|table|ul|ol|blockquote)>/gi, '\n\n') .replace(//gi, '\n') // Drop all remaining tags, decode entities, collapse whitespace. text = text.replace(/<[^>]+>/g, '') text = decodeEntities(text) text = text .replace(/[ \t\f\v]+/g, ' ') .replace(/ *\n */g, '\n') .replace(/\n{3,}/g, '\n\n') .trim() return text }