From 95da76069686d51f34e1d07cef003256687cb413 Mon Sep 17 00:00:00 2001 From: Chinesezjc Date: Wed, 5 Aug 2026 14:59:52 +0800 Subject: [PATCH] fix(tools): cap Python SDK list nesting at CPython's bracket limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A schema nesting arrays past ~200 levels rendered a `list[list[...]]` chain CPython's tokenizer rejects outright (`too many nested parentheses`), so the SDK block was not valid Python at all — the failure docstring escaping in the same file already guards against. The chain now degrades to `Any` at 180 levels; nesting restarts per TypedDict field, since a field annotation is its own logical line. Unions and nested objects are unaffected: neither accumulates open brackets. Also aligns the unreachable SDK_RENDERERS guard message with the two reachable ones, and corrects a test comment that still said class docstring. --- ...7-31-code-mode-language-dispatch.i18n.yaml | 4 +- .../2026-07-31-code-mode-language-dispatch.md | 2 +- ...26-07-31-code-mode-language-dispatch.zh.md | 2 +- packages/core/tools/src/index.ts | 2 +- packages/core/tools/src/py-types.ts | 50 ++++++++++++++++--- packages/core/tools/tests/py-types.spec.ts | 31 ++++++++++-- 6 files changed, 73 insertions(+), 18 deletions(-) diff --git a/.agents/notes/implemented/feature/2026-07-31-code-mode-language-dispatch.i18n.yaml b/.agents/notes/implemented/feature/2026-07-31-code-mode-language-dispatch.i18n.yaml index 3830e60848..3354a86d56 100644 --- a/.agents/notes/implemented/feature/2026-07-31-code-mode-language-dispatch.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-07-31-code-mode-language-dispatch.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-07-31-code-mode-language-dispatch.md -2026-07-31-code-mode-language-dispatch.md: d1fb598e22926eb017f7d3e2a3d1cb14870d4f4d -2026-07-31-code-mode-language-dispatch.zh.md: b5fc8b660c32b3ebdd8eef79439d4dedeb75b0c9 +2026-07-31-code-mode-language-dispatch.md: 6245891651aece73d5a51a6341bc4f76b98fad12 +2026-07-31-code-mode-language-dispatch.zh.md: 23dbd1c2a9d049d0648109c474b09feaae28886e diff --git a/.agents/notes/implemented/feature/2026-07-31-code-mode-language-dispatch.md b/.agents/notes/implemented/feature/2026-07-31-code-mode-language-dispatch.md index d1fb598e22..6245891651 100644 --- a/.agents/notes/implemented/feature/2026-07-31-code-mode-language-dispatch.md +++ b/.agents/notes/implemented/feature/2026-07-31-code-mode-language-dispatch.md @@ -23,7 +23,7 @@ Both tables are read with `Object.hasOwn` before use so a language named `toStri ### The Python SDK renderer -`py-types.ts` renders the same unified tool-schema vocabulary `jsonSchemaToTs` covers, targeting Python: `jsonSchemaToPy` emits a type expression per JSON-schema node, and `renderToolsSdkPy` assembles named `TypedDict`s for each visible tool's arguments and canonical output plus a `tools` object with usage instructions equivalent to the TypeScript flavor. Unsupported raw constructs degrade rather than throwing during assembly, matching the TypeScript renderer's contract. The output is deterministic — lexicographic tool order, byte-identical text for an unchanged tool set — so the prompt stays prefix-cache-friendly. Lexicographic means one ordered member stream: a tool whose name is not a legal attribute is listed as a `tools[name]` comment in its sorted position rather than partitioned to the end, matching how the TypeScript flavor quotes an exotic key in place. Two Python-specific placements follow from that: a description becomes the method's docstring emitted as the FIRST statement of its body (above the `async def` the first one would document the `Tools` class and the rest would be dead expressions, leaving every method undocumented), and because comment lines are not statements, a tool set with no method at all still needs an explicit `pass`. +`py-types.ts` renders the same unified tool-schema vocabulary `jsonSchemaToTs` covers, targeting Python: `jsonSchemaToPy` emits a type expression per JSON-schema node, and `renderToolsSdkPy` assembles named `TypedDict`s for each visible tool's arguments and canonical output plus a `tools` object with usage instructions equivalent to the TypeScript flavor. Unsupported raw constructs degrade rather than throwing during assembly, matching the TypeScript renderer's contract. The output is deterministic — lexicographic tool order, byte-identical text for an unchanged tool set — so the prompt stays prefix-cache-friendly. Lexicographic means one ordered member stream: a tool whose name is not a legal attribute is listed as a `tools[name]` comment in its sorted position rather than partitioned to the end, matching how the TypeScript flavor quotes an exotic key in place. That stream forces one thing directly — comment lines are not statements, so a tool set that emits no method at all still needs an explicit `pass`. Two further rules are Python-specific rather than consequences of the ordering. A description becomes the method's docstring emitted as the FIRST statement of its body: above the `async def` the first one would document the `Tools` class and the rest would be dead expressions, leaving every method undocumented. And a `list[…]` chain degrades to `Any` past `MAX_LIST_NESTING`, because CPython's tokenizer rejects a line with more than 200 open brackets and the block must stay parseable Python — the same reason `docLines` escapes quotes and backslashes. `ts-types` needs neither: TypeScript attaches a leading `/** … */` to the member that follows it and bounds nesting nowhere in its grammar. `renderType` validates the whole schema once (`assertSupportedJsonSchema`) and then trusts it, wrapping the walk in one `try/catch` that degrades to `Any` — the same trusted-after-validation stance the sibling `ts-types` renderer takes at this typed same-process seam ([Trust TypeScript at typed same-process seams](../../../../AGENTS.md)). It deliberately carries NO defenses against a schema whose accessors mutate between reads (post-validation cycles, TOCTOU on `const`/`enum`, self-referential functions): the input is a first-party registration (a `defineTool` literal or a raw registration) or a wire-derived plain JSON schema — the former is trusted per AGENTS.md, the latter is a `JSON.parse` product that physically cannot carry accessors, and `renderType` re-validates the whole tree on every call regardless — so such inputs are unreachable, and adding per-shape guards here would break symmetry with `ts-types` (which has none) for values the static interface forbids. `jsonSchemaToPy(schema: unknown)` accepts `unknown` and returns `Any` on a malformed schema — the Python counterpart of the TS flavor's `unknown` — but its contract is "degrade an unsupported schema", not "survive an adversarial mutating one". diff --git a/.agents/notes/implemented/feature/2026-07-31-code-mode-language-dispatch.zh.md b/.agents/notes/implemented/feature/2026-07-31-code-mode-language-dispatch.zh.md index b5fc8b660c..23dbd1c2a9 100644 --- a/.agents/notes/implemented/feature/2026-07-31-code-mode-language-dispatch.zh.md +++ b/.agents/notes/implemented/feature/2026-07-31-code-mode-language-dispatch.zh.md @@ -23,7 +23,7 @@ Code Mode 只生成一种 SDK 形态：TypeScript。`ToolRegistry` 为 `tools:sd ### Python SDK 渲染器 -`py-types.ts` 渲染 `jsonSchemaToTs` 所覆盖的同一套统一工具 schema 词汇，目标为 Python：`jsonSchemaToPy` 为每个 JSON-schema 节点发出一个类型表达式，`renderToolsSdkPy` 为每个可见工具的参数与规范输出装配具名 `TypedDict`，再加一个带用法说明的 `tools` 对象，与 TypeScript 形态等价。不支持的原始构造在装配时降级而非抛错，与 TypeScript 渲染器的契约一致。输出是确定性的——工具按字典序排列，工具集不变时文本逐字节相同——因此 prompt 保持 prefix-cache 友好。字典序意味着单一有序的成员流：名字不是合法属性的工具以 `tools[name]` 注释出现在它排序后的位置上，而不是被分拣到末尾，与 TypeScript 形态就地为异常键加引号的做法一致。由此带来两处 Python 特有的位置约定：描述会成为方法的 docstring，且必须作为方法体的**第一条语句**发出（放在 `async def` 之上，第一条会变成 `Tools` 的类文档、其余都是无效果表达式，导致每个方法都没有文档）；而注释行不是语句，所以一个没有任何方法的工具集仍需显式 `pass`。 +`py-types.ts` 渲染 `jsonSchemaToTs` 所覆盖的同一套统一工具 schema 词汇，目标为 Python：`jsonSchemaToPy` 为每个 JSON-schema 节点发出一个类型表达式，`renderToolsSdkPy` 为每个可见工具的参数与规范输出装配具名 `TypedDict`，再加一个带用法说明的 `tools` 对象，与 TypeScript 形态等价。不支持的原始构造在装配时降级而非抛错，与 TypeScript 渲染器的契约一致。输出是确定性的——工具按字典序排列，工具集不变时文本逐字节相同——因此 prompt 保持 prefix-cache 友好。字典序意味着单一有序的成员流：名字不是合法属性的工具以 `tools[name]` 注释出现在它排序后的位置上，而不是被分拣到末尾，与 TypeScript 形态就地为异常键加引号的做法一致。这个成员流直接决定了一件事：注释行不是语句，所以一个不发出任何方法的工具集仍需显式 `pass`。另有两条规则并非源自排序，而是 Python 特有。其一，描述会成为方法的 docstring，且必须作为方法体的**第一条语句**发出：放在 `async def` 之上，第一条会变成 `Tools` 的类文档、其余都是无效果表达式，导致每个方法都没有文档。其二，`list[…]` 链超过 `MAX_LIST_NESTING` 后降级为 `Any`，因为 CPython 的 tokenizer 拒绝一行中超过 200 个同时未闭合的括号，而这个块必须是可解析的 Python——与 `docLines` 转义引号和反斜杠是同一个理由。`ts-types` 两者都不需要：TypeScript 会把前置的 `/** … */` 附着到其后的成员上，其语法也不对嵌套设限。 `renderType` 先用 `assertSupportedJsonSchema` 整树校验一次、随后信任它，用单个 `try/catch` 把整个遍历兜住并降级为 `Any`——与姊妹渲染器 `ts-types` 在这个 typed 同进程 seam 上采取的「校验后信任」姿态一致（[Trust TypeScript at typed same-process seams](../../../../AGENTS.md)）。它有意不设任何针对「访问器在多次读取间变值」的防御（校验后成环、`const`/`enum` 的 TOCTOU、自引用函数）：输入是第一方注册（`defineTool` 字面量或 raw 注册）或从 wire 桥接而来的纯 JSON——前者按 AGENTS.md 受信任，后者是 `JSON.parse` 产物、物理上不可能携带访问器，且每次调用 `renderType` 都会整树重新校验——这类输入不可达，而在此加逐形态守卫会为静态接口所禁止的值破坏与 `ts-types`（没有这类守卫）的对称。`jsonSchemaToPy(schema: unknown)` 接受 `unknown` 并对畸形 schema 返回 `Any`——TypeScript 形态 `unknown` 的对应物——但它的契约是「降级不支持的 schema」，而非「扛住对抗性的可变 schema」。 diff --git a/packages/core/tools/src/index.ts b/packages/core/tools/src/index.ts index 7e3d1f5624..5c523af878 100644 --- a/packages/core/tools/src/index.ts +++ b/packages/core/tools/src/index.ts @@ -795,7 +795,7 @@ export class ToolRegistry extends Service { const render = SDK_RENDERERS[runtime.language] /* v8 ignore next 3 -- requireCodeRuntime rejects an unknown language before this ever runs. */ if (!Object.hasOwn(SDK_RENDERERS, runtime.language) || render === undefined) { - throw new Error(`dsh-tools: no SDK renderer registered for runtime language "${runtime.language}"`) + throw new Error(`dsh-tools: no SDK renderer registered for runtime language ${JSON.stringify(runtime.language)} (known: ${Object.keys(SDK_RENDERERS).map(name => JSON.stringify(name)).join(', ')})`) } return render(this.sdkSchemas(context.scope)) }, diff --git a/packages/core/tools/src/py-types.ts b/packages/core/tools/src/py-types.ts index 26deed174f..0472f06029 100644 --- a/packages/core/tools/src/py-types.ts +++ b/packages/core/tools/src/py-types.ts @@ -129,6 +129,24 @@ function camelCase(raw: string): string { /** Class-name base cap keeping each emitted name — and total text — linear in schema depth. */ const MAX_CLASS_NAME_BASE = 120 +/** + * Deepest `list[…]` nesting emitted into one annotation before the item type + * degrades to `Any`. CPython's tokenizer rejects a logical line holding more + * than 200 simultaneously-open brackets (`MAXLEVEL`, `SyntaxError: too many + * nested parentheses`), so an array chain deeper than that would render an SDK + * block that is not valid Python at all — the same failure the docstring + * escaping in {@link docLines} exists to prevent. 180 leaves headroom for the + * one bracket an annotation can add around the chain (`NotRequired[…]`). + * + * A CPython grammar limit, not a deployment choice, so it is fixed rather than + * configurable. The sibling `ts-types` renderer needs no counterpart: nothing + * in the TypeScript grammar bounds nesting, and its SDK block is never type- + * checked. Only bracket nesting counts — a `oneOf` renders as a flat `A | B` + * chain and nested objects render as separate `class` statements, so neither + * accumulates open brackets at any depth. + */ +const MAX_LIST_NESTING = 180 + /** Cap a class-name base at {@link MAX_CLASS_NAME_BASE} (see the callers for why capping keeps the render linear). */ function capClassNameBase(base: string): string { return base.length > MAX_CLASS_NAME_BASE ? base.slice(0, MAX_CLASS_NAME_BASE) : base @@ -238,14 +256,16 @@ function renderType(schema: unknown, className: string, state: RenderState): str phase: 'start' | 'children' kind?: 'oneOf' | 'array' | 'typeddict' node?: JsonSchemaNode - children: { schema: JsonSchemaNode; className: string }[] + /** Open `list[` brackets enclosing this node in the annotation being built ({@link MAX_LIST_NESTING}). */ + listDepth: number + children: { schema: JsonSchemaNode; className: string; listDepth: number }[] childIndex: number childTypes: string[] entries: [string, JsonSchemaNode][] allocated?: string } - const newFrame = (schema: JsonSchemaNode, className: string): Frame => - ({ schema, className, phase: 'start', children: [], childIndex: 0, childTypes: [], entries: [] }) + const newFrame = (schema: JsonSchemaNode, className: string, listDepth: number): Frame => + ({ schema, className, phase: 'start', listDepth, children: [], childIndex: 0, childTypes: [], entries: [] }) try { // Validate the WHOLE tree once, then trust it — the same contract the // sibling ts-types renderer follows at a typed same-process seam. Every @@ -254,7 +274,7 @@ function renderType(schema: unknown, className: string, state: RenderState): str // here (before anything is emitted) and degrades to `Any`, the Python // counterpart of the TS flavor's `unknown`. assertSupportedJsonSchema(schema) - const frames: Frame[] = [newFrame(schema, className)] + const frames: Frame[] = [newFrame(schema, className, 0)] let result: string | undefined /* jscpd:ignore-start -- the explicit-stack walk skeleton deliberately parallels ts-types.ts's renderSupportedSchema; the two sibling renderers keep symmetric shapes. */ @@ -276,7 +296,7 @@ function renderType(schema: unknown, className: string, state: RenderState): str /* v8 ignore next -- childIndex is bounded by children.length. */ if (child === undefined) throw new Error('missing python render child') frame.childIndex++ - frames.push(newFrame(child.schema, child.className)) + frames.push(newFrame(child.schema, child.className, child.listDepth)) continue } if (frame.kind === 'oneOf') { @@ -345,7 +365,9 @@ function renderType(schema: unknown, className: string, state: RenderState): str const node = frame.schema if (node.oneOf !== undefined) { frame.kind = 'oneOf' - frame.children = node.oneOf.map((branch, index) => ({ schema: branch, className: childClassName(frame.className, `${index + 1}`) })) + // A union renders as `A | B` — no brackets of its own, so the branches + // inherit the enclosing depth unchanged. + frame.children = node.oneOf.map((branch, index) => ({ schema: branch, className: childClassName(frame.className, `${index + 1}`), listDepth: frame.listDepth })) continue } if (node.type === undefined) { @@ -365,9 +387,18 @@ function renderType(schema: unknown, className: string, state: RenderState): str finish('list[Any]') break } + // Past MAX_LIST_NESTING another `list[` would push the annotation + // beyond CPython's open-bracket limit and make the whole SDK block + // unparseable, so the chain degrades here instead — an unusable + // annotation either way, and this one is valid Python. + if (frame.listDepth >= MAX_LIST_NESTING) { + state.typing.add('Any') + finish('Any') + break + } // An array of objects names its item type after the array field. frame.kind = 'array' - frame.children = [{ schema: node.items, className: frame.className }] + frame.children = [{ schema: node.items, className: frame.className, listDepth: frame.listDepth + 1 }] break } case 'object': { @@ -404,7 +435,10 @@ function renderType(schema: unknown, className: string, state: RenderState): str frame.entries = entries // frame.allocated was assigned two statements up; the ?? arm is for the type system only. /* v8 ignore next -- allocated is always set before children are built. */ - frame.children = entries.map(([field, child]) => ({ schema: child, className: childClassName(frame.allocated ?? '', camelCase(field)) })) + // A field annotation is its own logical line, so nesting restarts — + // at 1, reserving the bracket an optional field's `NotRequired[…]` + // wraps around it. + frame.children = entries.map(([field, child]) => ({ schema: child, className: childClassName(frame.allocated ?? '', camelCase(field)), listDepth: 1 })) break } /* v8 ignore next 4 -- assertSupportedJsonSchema narrowed this closed type union. */ diff --git a/packages/core/tools/tests/py-types.spec.ts b/packages/core/tools/tests/py-types.spec.ts index 5a4b7f625a..734debc088 100644 --- a/packages/core/tools/tests/py-types.spec.ts +++ b/packages/core/tools/tests/py-types.spec.ts @@ -500,16 +500,37 @@ describe('renderToolsSdkPy', () => { expect(text).toContain('closedEmpty: OpennessArgsClosedEmpty') }) - it('renders a deeply nested array schema without exhausting the call stack', () => { + it('renders a deeply nested array schema without exhausting the call stack, capped at CPython\'s bracket limit', () => { // The registry supports depth-unbounded schemas; the renderer must not - // reintroduce a recursion limit during prompt assembly. + // reintroduce a recursion limit during prompt assembly. It must also not + // emit more open brackets than CPython's tokenizer accepts (200), so the + // chain degrades to `Any` at MAX_LIST_NESTING instead of rendering an SDK + // block that is not valid Python. let deep: Record = { type: 'string' } for (let i = 0; i < 20000; i++) deep = { type: 'array', items: deep } const type = jsonSchemaToPy(deep) expect(type.startsWith('list[list[')).toBe(true) expect(type.endsWith(']]')).toBe(true) - expect(type).toContain('str') - expect(type.length).toBe('list['.length * 20000 + 'str'.length + ']'.repeat(20000).length) + // 180 `list[` levels around `Any`, not 20000 around `str`. + expect(type).toBe(`${'list['.repeat(180)}Any${']'.repeat(180)}`) + expect(type.split('[').length - 1).toBeLessThan(200) + }) + + it('keeps a chain just under the nesting cap exact, and restarts nesting per TypedDict field', () => { + // 179 levels still render the real item type: the cap degrades only what + // would not parse. + let under: Record = { type: 'string' } + for (let i = 0; i < 179; i++) under = { type: 'array', items: under } + expect(jsonSchemaToPy(under)).toBe(`${'list['.repeat(179)}str${']'.repeat(179)}`) + // A field annotation is a fresh logical line, so a 179-deep chain reached + // THROUGH an object field is unaffected by the depth spent on the object. + const tool: ToolSdkSchema = { + name: 'deep_field', + description: 'Deep array under a field.', + parameters: { type: 'object', additionalProperties: false, properties: { rows: under }, required: ['rows'] }, + output: { type: 'string' }, + } + expect(renderToolsSdkPy([tool])).toContain(` rows: ${'list['.repeat(179)}str${']'.repeat(179)}`) }) it('renders a deeply nested oneOf chain in linear time (no per-level re-materialization)', () => { @@ -664,7 +685,7 @@ describe('renderToolsSdkPy', () => { output: { type: 'string' }, }) const nul = renderToolsSdkPy([make('before\u0000after')]) - // Both emission sites: the class docstring and the `#` field comment. The + // Both emission sites: the method docstring and the `#` field comment. The // docstring's backslash is doubled by the same escaping that keeps a literal // backslash from escaping the closing triple quote, so Python parses it back // to the visible `\x00` the comment shows directly. Neither carries the byte.