Merge remote-tracking branch 'origin/master' into simpl-a1-drop-image

# Conflicts:
#	docs/architecture.md
#	packages/compact/compact-basic/README.md

packages/compact/compact-basic/src/index.ts

@@ -2,7 +2,8 @@
  * `BasicCompactService`: the first implementation of the
  * `@deepseek-ai/dsh-compact` seam. It owns the entire compaction strategy:
  *
- * - **Token estimation** — char/4 heuristic with per-block structural overhead.
+ * - **Token estimation** — chars/`charsPerToken` heuristic (config, default 4)
+ *   with per-block structural overhead.
  * - **Retention policy** — walk surface nodes tail→head, keep recent nodes up
  *   to a token budget, compact everything older. The cutoff is snapped forward
  *   to the next balanced tool-pairing boundary so a compacted region never
@@ -145,9 +146,11 @@ function finishError(finish: FinishReason): Error | undefined {
 }
 
 /**
- * Basic, dependency-light compaction backend. Defaults target a 128K context
- * window, compacting at 80% utilization and retaining ~20K tokens of recent
- * context.
+ * Basic, dependency-light compaction backend: estimates the surface's token
+ * footprint, summarizes the stale prefix through the model, and shadows it
+ * behind a durable checkpoint. Every threshold/budget knob is required config
+ * ({@link BasicCompactConfig}); the estimator's text density is the
+ * `charsPerToken` knob.
  */
 export class BasicCompactService extends CompactService {
   static inject = ['llm']
@@ -204,24 +207,27 @@ export class BasicCompactService extends CompactService {
 
   // ---- Token estimation (overridable hooks) ----
 
-  // TODO: char/4 is a coarse heuristic. Replace with an exact count — a real
-  // tokenizer, or the provider's post-response `usage` (input tokens) fed back
-  // as a correction — so threshold decisions match the model's actual budget.
+  // TODO: chars/charsPerToken is a coarse heuristic. Replace with an exact
+  // count — a real tokenizer, or the provider's post-response `usage` (input
+  // tokens) fed back as a correction — so threshold decisions match the
+  // model's actual budget.
   /**
-   * Estimate the token count of content blocks — char/4 with per-block
-   * overhead. Override in a subclass to plug in a real tokenizer.
+   * Estimate the token count of content blocks — chars divided by the
+   * `charsPerToken` config, with per-block overhead. Override in a subclass to
+   * plug in a real tokenizer.
    */
   estimateContentTokens(blocks: readonly ContentBlock[]): number {
+    const { charsPerToken } = this.config
     let tokens = 0
     for (const block of blocks) {
       switch (block.type) {
         case 'text':
         case 'reasoning':
-          tokens += Math.ceil(block.text.length / 4) + BLOCK_OVERHEAD
+          tokens += Math.ceil(block.text.length / charsPerToken) + BLOCK_OVERHEAD
           break
         case 'tool-call':
-          tokens += Math.ceil(block.name.length / 4)
-            + Math.ceil(block.arguments.length / 4)
+          tokens += Math.ceil(block.name.length / charsPerToken)
+            + Math.ceil(block.arguments.length / charsPerToken)
             + BLOCK_OVERHEAD
           break
         case 'tool-result':
@@ -230,7 +236,7 @@ export class BasicCompactService extends CompactService {
         default:
           // Unknown block types (merge-extensible ContentBlockMap):
           // estimate conservatively via JSON stringify.
-          tokens += BLOCK_OVERHEAD + Math.ceil(JSON.stringify(block).length / 4)
+          tokens += BLOCK_OVERHEAD + Math.ceil(JSON.stringify(block).length / charsPerToken)
       }
     }
     return tokens
@@ -260,7 +266,7 @@ export class BasicCompactService extends CompactService {
       total += this.estimateContentTokens(msg.content)
       total += ROLE_OVERHEAD
     }
-    if (systemPrompt) total += Math.ceil(systemPrompt.length / 4)
+    if (systemPrompt) total += Math.ceil(systemPrompt.length / this.config.charsPerToken)
     return total
   }
 

packages/compact/compact-basic/src/types.ts

@@ -10,10 +10,12 @@
  */
 
 /**
- * Backend configuration. Every knob is REQUIRED except `auto`: there is no
- * concrete data yet to justify default thresholds/budgets, so a consumer must
- * state each value explicitly rather than inherit a guessed default. `auto`
- * alone defaults to `true` (auto-compaction is the intended posture).
+ * Backend configuration. Every knob is REQUIRED except `auto` and
+ * `charsPerToken`: there is no concrete data yet to justify default
+ * thresholds/budgets, so a consumer must state each value explicitly rather
+ * than inherit a guessed default. `auto` alone defaults to `true`
+ * (auto-compaction is the intended posture), and `charsPerToken` defaults to
+ * the English-text heuristic its estimator was calibrated on.
  */
 export interface BasicCompactConfig {
   /** Context window size in tokens. */
@@ -30,13 +32,21 @@ export interface BasicCompactConfig {
   compactionRetries: number
   /** Enable automatic compaction on the `agent/pre-step` seam (default true). */
   auto?: boolean
+  /**
+   * Text density for the token estimator: estimated tokens = chars /
+   * `charsPerToken`. Defaults to 4 (typical English text). A CJK-heavy
+   * deployment should set ~1-2 — CJK runs at roughly 1-2 chars per token, so
+   * the default UNDERestimates several-fold and compaction fires far too late.
+   * May be fractional.
+   */
+  charsPerToken?: number
 }
 
-/** Resolved config with `auto` defaulted. */
+/** Resolved config with `auto` and `charsPerToken` defaulted. */
 export type ResolvedConfig = Required<BasicCompactConfig>
 
 /**
- * Default `auto` when unset and reject nonsensical numeric knobs.
+ * Default `auto`/`charsPerToken` when unset and reject nonsensical numeric knobs.
  *
  * Convergence is not a static config invariant: provider generation caps can be
  * spent on hidden or surfaced reasoning tokens, and the model may emit a summary
@@ -46,13 +56,14 @@ export type ResolvedConfig = Required<BasicCompactConfig>
  * throwing if the surface still exceeds the threshold.
  */
 export function resolveConfig(config: BasicCompactConfig): ResolvedConfig {
-  const resolved: ResolvedConfig = { auto: true, ...config }
+  const resolved: ResolvedConfig = { auto: true, charsPerToken: 4, ...config }
 
   assertPositiveInteger('contextWindow', resolved.contextWindow)
   assertRatio('thresholdRatio', resolved.thresholdRatio)
   assertNonNegativeInteger('retainTokens', resolved.retainTokens)
   assertPositiveInteger('maxTokens', resolved.maxTokens)
   assertNonNegativeInteger('compactionRetries', resolved.compactionRetries)
+  assertPositiveFinite('charsPerToken', resolved.charsPerToken)
   if (typeof resolved.summarizationModel !== 'string') {
     throw new Error('BasicCompactConfig: summarizationModel must be a string.')
   }
@@ -74,6 +85,12 @@ function assertNonNegativeInteger(name: string, value: number): void {
   }
 }
 
+function assertPositiveFinite(name: string, value: number): void {
+  if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) {
+    throw new Error(`BasicCompactConfig: ${name} (${value}) must be a positive finite number.`)
+  }
+}
+
 function assertRatio(name: string, value: number): void {
   if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0 || value > 1) {
     throw new Error(`BasicCompactConfig: ${name} (${value}) must be a number in (0, 1].`)