docs: harden package README gates

packages/README.md

@@ -38,4 +38,4 @@ The inter-package dependency graph is generated: [docs/module-graph.md](../docs/
 
 The rule it must obey: **extension plugins depend on interfaces, never on the concrete loop.** `dsh-agent-loop` is swappable — UI/hook/tool plugins keep working against the `dsh-agent` vocabulary if the loop is replaced. The sanctioned exception is a **composition/bundle** package like `dsh-agent-core`, whose whole job is to assemble the concrete spine: it depends on `dsh-agent-loop` (and the other concrete spine plugins) on purpose. The rule constrains plugins that EXTEND the system, not the bundle that COMPOSES it. A swappable capability splits into interface / implementation / consumer packages (the bash trio is the template — see [capability seams](../docs/rfc/implemented/architecture/2026-06-13-capability-seams.md)).
 
-Each package has a `README.md` with purpose, service API, events, extension points, non-goals, a [Model Experience section](../docs/cookbook/adding-a-package.md#4-write-the-package-readme), and `## Known Limitations and Deferred Work` (or a justified allowlist entry).
+Package READMEs cover purpose, APIs, extension points, and [Model Experience](../docs/cookbook/adding-a-package.md#4-write-the-package-readme) unless on the model-agnostic [omission allowlist](../scripts/verify-package-readme-model-experience.ts). They also carry `## Known Limitations and Deferred Work` or use its [allowlist](../scripts/verify-package-readme-limitations.ts).

scripts/markdown.ts

@@ -5,7 +5,7 @@ import { gfmFromMarkdown } from 'mdast-util-gfm'
 import { gfm } from 'micromark-extension-gfm'
 import type { Nodes } from 'mdast'
 
-/** One authored Markdown line outside fenced code. */
+/** One authored Markdown line outside fenced code and rendered-away HTML comments. */
 export interface MarkdownProseLine {
   /** 1-based source line number. */
   index: number
@@ -13,6 +13,14 @@ export interface MarkdownProseLine {
   raw: string
 }
 
+/** One parsed Markdown heading, retaining its authored first line and rendered text. */
+export interface MarkdownHeadingLine extends MarkdownProseLine {
+  /** Parsed ATX or Setext heading depth. */
+  depth: 1 | 2 | 3 | 4 | 5 | 6
+  /** Rendered heading text, excluding raw HTML such as comments. */
+  text: string
+}
+
 /** Parse GitHub-flavored Markdown with the repository's standard extensions. */
 export function parseMarkdown(source: string): Nodes {
   return fromMarkdown(source, { extensions: [gfm()], mdastExtensions: [gfmFromMarkdown()] })
@@ -30,15 +38,93 @@ export function visitMarkdown(node: Nodes, visitor: (node: Nodes) => boolean | v
   }
 }
 
+/** Text a reader sees from one Markdown node; raw HTML itself contributes none. */
+function renderedText(node: Nodes): string {
+  if (node.type === 'text' || node.type === 'inlineCode') return node.value
+  if (node.type === 'image' || node.type === 'imageReference') return node.alt ?? ''
+  if (node.type === 'break') return ' '
+  if ('children' in node) return node.children.map(child => renderedText(child)).join('')
+  return ''
+}
+
+/** Return every parsed Markdown heading with its rendered text and source line. */
+export function markdownHeadingLines(source: string): MarkdownHeadingLine[] {
+  const rawLines = source.split('\n')
+  const headings: MarkdownHeadingLine[] = []
+  visitMarkdown(parseMarkdown(source), (node) => {
+    if (node.type !== 'heading' || node.position === undefined) return
+    headings.push({
+      depth: node.depth,
+      index: node.position.start.line,
+      raw: rawLines[node.position.start.line - 1] ?? '',
+      text: renderedText(node),
+    })
+  })
+  return headings
+}
+
+type ColumnRange = readonly [start: number, end: number]
+type OffsetRange = readonly [start: number, end: number]
+
+/** Source-column ranges occupied by parsed HTML comments, keyed by source line. */
+function htmlCommentRanges(source: string, rawLines: readonly string[]): Map<number, ColumnRange[]> {
+  const comments: OffsetRange[] = []
+  visitMarkdown(parseMarkdown(source), (node) => {
+    if (node.type !== 'html' || node.position?.start.offset === undefined) return
+    let cursor = 0
+    while (true) {
+      const start = node.value.indexOf('<!--', cursor)
+      if (start < 0) break
+      const close = node.value.indexOf('-->', start + '<!--'.length)
+      const end = close < 0 ? node.value.length : close + '-->'.length
+      comments.push([node.position.start.offset + start, node.position.start.offset + end])
+      cursor = end
+    }
+  })
+
+  const ranges = new Map<number, ColumnRange[]>()
+  let lineOffset = 0
+  rawLines.forEach((raw, index) => {
+    const lineEnd = lineOffset + raw.length
+    for (const [start, end] of comments) {
+      const from = Math.max(start, lineOffset)
+      const to = Math.min(end, lineEnd)
+      const coversEmptyLine = raw.length === 0 && start <= lineOffset && end > lineOffset
+      if (from < to || coversEmptyLine) {
+        const lineRanges = ranges.get(index + 1) ?? []
+        lineRanges.push([from - lineOffset, to - lineOffset])
+        ranges.set(index + 1, lineRanges)
+      }
+    }
+    lineOffset = lineEnd + 1
+  })
+  return ranges
+}
+
+/** Whether a source line retains non-whitespace text after HTML comments disappear. */
+function hasRenderedTextOutsideComments(raw: string, ranges: readonly ColumnRange[] | undefined): boolean {
+  if (ranges === undefined) return true
+  let cursor = 0
+  let visible = ''
+  for (const [start, end] of [...ranges].sort((left, right) => left[0] - right[0])) {
+    visible += raw.slice(cursor, start)
+    cursor = Math.max(cursor, end)
+  }
+  visible += raw.slice(cursor)
+  return visible.trim().length > 0
+}
+
 /**
- * Return source lines outside backtick or tilde fences.
+ * Return source lines outside backtick or tilde fences and HTML comments.
  * @param source - Markdown source whose prose should be retained verbatim.
  * @returns unfenced lines with their original 1-based locations.
  */
 export function markdownProseLines(source: string): MarkdownProseLine[] {
   let fence: { marker: '`' | '~'; length: number } | undefined
   const kept: MarkdownProseLine[] = []
-  source.split('\n').forEach((raw, i) => {
+  const rawLines = source.split('\n')
+  const comments = htmlCommentRanges(source, rawLines)
+  rawLines.forEach((raw, i) => {
     const token = /^ {0,3}(`{3,}|~{3,})/.exec(raw)?.[1]
     if (token !== undefined) {
       const marker = token[0] as '`' | '~'
@@ -49,7 +135,9 @@ export function markdownProseLines(source: string): MarkdownProseLine[] {
       }
       return
     }
-    if (fence === undefined) kept.push({ index: i + 1, raw })
+    if (fence === undefined && hasRenderedTextOutsideComments(raw, comments.get(i + 1))) {
+      kept.push({ index: i + 1, raw })
+    }
   })
   return kept
 }

scripts/verify-package-readme-limitations.ts

@@ -16,7 +16,7 @@
  *
  * The package set comes from `packages/<group>/<package>/package.json`, so a manifest with no
  * sibling README fails instead of escaping a README-only glob. Checks, per
- * package README (fenced code excluded):
+ * package README (fenced code and HTML comments excluded):
  * 1. Non-whitelisted: exactly one limitations-like heading, byte-equal to the
  *    canonical h2, with at least one top-level `- ` bullet before the next
  *    heading.
@@ -34,7 +34,7 @@
 
 import { existsSync, globSync, readFileSync } from 'node:fs'
 import { resolve } from 'node:path'
-import { markdownProseLines } from './markdown.ts'
+import { markdownHeadingLines, markdownProseLines } from './markdown.ts'
 
 const root = resolve(import.meta.dirname, '..')
 
@@ -61,8 +61,6 @@ function isLimitationsLike(headingText: string): boolean {
   )
 }
 
-const ATX_HEADING = /^ {0,3}#{1,6}[ \t]+/
-
 const packageJsons = globSync('packages/*/*/package.json', { cwd: root }).sort()
 const scannedPackages = new Set(packageJsons.map(path => path.slice(0, -'/package.json'.length)))
 const failures: string[] = []
@@ -82,9 +80,10 @@ for (const pkg of scannedPackages) {
     failures.push(`${readme}: package manifest has no sibling README with the \`${CANONICAL}\` section`)
     continue
   }
-  const lines = markdownProseLines(readFileSync(resolve(root, readme), 'utf8'))
-  const headings = lines.filter(line => ATX_HEADING.test(line.raw))
-  const limitations = headings.filter(line => isLimitationsLike(line.raw.replace(ATX_HEADING, '')))
+  const source = readFileSync(resolve(root, readme), 'utf8')
+  const lines = markdownProseLines(source)
+  const headings = markdownHeadingLines(source)
+  const limitations = headings.filter(heading => isLimitationsLike(heading.text))
 
   if (Object.hasOwn(NO_LIMITATIONS, pkg)) {
     for (const heading of limitations) {
@@ -102,13 +101,14 @@ for (const pkg of scannedPackages) {
     failures.push(`${readme}: ${limitations.length} limitations-like headings (lines ${limitations.map(line => line.index).join(', ')}) — keep exactly one \`${CANONICAL}\` section`)
     continue
   }
-  if (heading.raw.trimEnd() !== CANONICAL) {
+  if (heading.depth !== 2 || heading.raw.trimEnd() !== CANONICAL) {
     failures.push(`${readme}:${heading.index}: non-canonical heading ${JSON.stringify(heading.raw)} — use \`${CANONICAL}\``)
     continue
   }
-  const headingAt = lines.indexOf(heading)
+  const headingAt = lines.findIndex(line => line.index === heading.index)
   const body = lines.slice(headingAt + 1)
-  const end = body.findIndex(line => ATX_HEADING.test(line.raw))
+  const headingLines = new Set(headings.map(entry => entry.index))
+  const end = body.findIndex(line => headingLines.has(line.index))
   const section = end === -1 ? body : body.slice(0, end)
   if (!section.some(line => /^- /.test(line.raw))) {
     failures.push(`${readme}:${heading.index}: the \`${CANONICAL}\` section has no top-level \`- \` bullet — state the limitations, or whitelist the package if there are genuinely none`)

scripts/verify-package-readme-model-experience.ts

@@ -12,14 +12,13 @@
 
 import { existsSync, globSync, readFileSync } from 'node:fs'
 import { relative, resolve } from 'node:path'
-import { markdownProseLines, type MarkdownProseLine } from './markdown.ts'
+import { markdownHeadingLines, markdownProseLines, type MarkdownProseLine } from './markdown.ts'
 
 const root = resolve(import.meta.dirname, '..')
 const HEADING = '## Model Experience'
 const LIMITATIONS_HEADING = '## Known Limitations and Deferred Work'
 const MODEL_VIEW_LABEL = '**What the model sees**'
 const TOKEN_EFFECT_LABEL = '**Token effect**'
-const H2_HEADING = /^## .+$/
 
 type SentenceKind = 'none' | 'indirect'
 
@@ -195,27 +194,40 @@ for (const packageJson of packageJsons) {
   const text = readFileSync(abs, 'utf8')
   const rawLines = text.split('\n')
   const lines = markdownProseLines(text)
-  const h2Headings = lines.filter(line => H2_HEADING.test(line.raw))
-  const modelHeadings = h2Headings.filter(line => line.raw === HEADING)
+  const headings = markdownHeadingLines(text)
+  const h2Headings = headings.filter(heading => heading.depth === 2)
+  const modelExperienceHeadings = headings.filter(heading => heading.text
+    .trim().replaceAll(/\s+/g, ' ').toLowerCase() === 'model experience')
+  const modelHeadings = modelExperienceHeadings.filter(heading => heading.depth === 2 && heading.raw === HEADING)
   if (NO_MODEL_EXPERIENCE_SECTION[pkg] !== undefined) {
-    if (modelHeadings.length !== 0) {
-      failures.push({ path: readme, message: `audited model-agnostic package must omit ${HEADING}` })
+    if (modelExperienceHeadings.length !== 0) {
+      for (const heading of modelExperienceHeadings) {
+        failures.push({ path: readme, message: `line ${heading.index}: audited model-agnostic package must omit every Model Experience heading; found ${JSON.stringify(heading.raw)}` })
+      }
     } else {
       omittedSectionCount += 1
     }
     continue
   }
-  if (modelHeadings.length !== 1) {
+  const nonCanonicalModelHeading = modelExperienceHeadings.find(heading => heading.depth !== 2 || heading.raw !== HEADING)
+  if (nonCanonicalModelHeading !== undefined) {
+    failures.push({ path: readme, message: `line ${nonCanonicalModelHeading.index}: non-canonical Model Experience heading ${JSON.stringify(nonCanonicalModelHeading.raw)}; use exactly ${JSON.stringify(HEADING)}` })
+    continue
+  }
+  const modelHeading = modelHeadings.at(0)
+  if (modelHeading === undefined) {
     failures.push({
       path: readme,
-      message: modelHeadings.length === 0 ? `missing ${HEADING}` : `contains ${modelHeadings.length} copies of ${HEADING}`,
+      message: `missing ${HEADING}`,
     })
     continue
   }
-
-  const modelHeading = modelHeadings[0] as Line
+  if (modelHeadings.length !== 1) {
+    failures.push({ path: readme, message: `contains ${modelHeadings.length} copies of ${HEADING}` })
+    continue
+  }
   const modelH2Index = h2Headings.indexOf(modelHeading)
-  const limitationsH2Index = h2Headings.findIndex(heading => heading.raw === LIMITATIONS_HEADING)
+  const limitationsH2Index = h2Headings.findIndex(heading => heading.depth === 2 && heading.raw === LIMITATIONS_HEADING)
   if (limitationsH2Index >= 0) {
     if (modelH2Index !== h2Headings.length - 2 || limitationsH2Index !== h2Headings.length - 1) {
       failures.push({
@@ -229,8 +241,10 @@ for (const packageJson of packageJsons) {
     continue
   }
 
-  const body = lines.slice(lines.indexOf(modelHeading) + 1)
-  const nextH2 = body.findIndex(line => H2_HEADING.test(line.raw))
+  const modelHeadingAt = lines.findIndex(line => line.index === modelHeading.index)
+  const body = lines.slice(modelHeadingAt + 1)
+  const h2Lines = new Set(h2Headings.map(heading => heading.index))
+  const nextH2 = body.findIndex(line => h2Lines.has(line.index))
   const section = nextH2 < 0 ? body : body.slice(0, nextH2)
   const nextH2Line = nextH2 < 0 ? rawLines.length + 1 : (body[nextH2] as Line).index
   const rawSection = rawLines.slice(modelHeading.index, nextH2Line - 1)