mirror of
https://github.com/deepseek-ai/deepseek-harness
synced 2026-08-15 21:04:50 +00:00
225ed051b1
Nominal string types via a unique-symbol brand (zero runtime cost): an AgentId can no longer be passed where a CallId is expected. Each core package brands the IDs it owns — CallId in dsh-llm (tool-call correlation across blocks, chunks, session events, and execution results), SessionId in dsh-session, AgentId in dsh-agent. Construction goes through same-named factory functions; public string-in APIs (sessions.create, agentLoop.create) keep accepting plain strings and brand internally. Policy note in the brand module: brand IDs that cross package boundaries, not every string.
577 lines
18 KiB
TypeScript
577 lines
18 KiB
TypeScript
import { describe, expect, expectTypeOf, it } from 'vitest'
|
|
import { Context } from 'cordis'
|
|
import { CallId } from '@deepseek-ai/dsh-llm'
|
|
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
|
|
import ToolRegistry, {
|
|
defineTool, schemaSpecToJsonSchema,
|
|
type InferArgs, type SchemaSpec, type ToolExecutionResult,
|
|
} from '@deepseek-ai/dsh-tools'
|
|
|
|
async function setup() {
|
|
const ctx = new Context()
|
|
await ctx.plugin(SystemPrompt)
|
|
await ctx.plugin(ToolRegistry)
|
|
return ctx
|
|
}
|
|
|
|
const echoTool = defineTool({
|
|
name: 'echo',
|
|
description: 'echo arguments back',
|
|
parameters: { text: { type: 'string' } },
|
|
async execute(args) {
|
|
return [{ type: 'text' as const, text: args.text ?? '' }]
|
|
},
|
|
})
|
|
|
|
describe('ToolRegistry', () => {
|
|
it('registers tools, exposes schemas, and feeds the system-prompt assembly', async () => {
|
|
const ctx = await setup()
|
|
ctx.tools.register(echoTool)
|
|
|
|
expect(ctx.tools.schemas()).toEqual([{
|
|
name: 'echo',
|
|
description: 'echo arguments back',
|
|
parameters: { type: 'object', properties: { text: { type: 'string' } } },
|
|
}])
|
|
// schemas() result must not leak execute — ToolSchema deliberately has no
|
|
// 'execute' key, so widen through unknown to probe for the absent property
|
|
expect((ctx.tools.schemas()[0] as unknown as Record<string, unknown>).execute).toBeUndefined()
|
|
|
|
const assembly = await ctx.systemPrompt.assemble()
|
|
expect(assembly.tools.map(t => t.name)).toEqual(['echo'])
|
|
})
|
|
|
|
it('executes a tool and returns its content', async () => {
|
|
const ctx = await setup()
|
|
ctx.tools.register(echoTool)
|
|
const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
|
|
expect(result).toEqual({ callId: CallId('c1'), content: [{ type: 'text', text: 'hi' }], isError: false })
|
|
})
|
|
|
|
it('returns isError results for unknown tools and throwing tools', async () => {
|
|
const ctx = await setup()
|
|
ctx.tools.register({
|
|
...echoTool,
|
|
name: 'boom',
|
|
async execute() {
|
|
throw new Error('exploded')
|
|
},
|
|
})
|
|
|
|
const unknown = await ctx.tools.execute({ callId: CallId('c1'), name: 'nope', arguments: {} })
|
|
expect(unknown.isError).toBe(true)
|
|
|
|
const thrown = await ctx.tools.execute({ callId: CallId('c2'), name: 'boom', arguments: {} })
|
|
expect(thrown.isError).toBe(true)
|
|
expect(thrown.content[0]).toMatchObject({ text: 'Error: exploded' })
|
|
})
|
|
|
|
it('lets tools/execute waterfall listeners veto a call (permission pattern)', async () => {
|
|
const ctx = await setup()
|
|
ctx.tools.register(echoTool)
|
|
|
|
ctx.on('tools/execute', async (exec, next): Promise<ToolExecutionResult> => {
|
|
if (exec.name === 'echo') {
|
|
return {
|
|
callId: exec.callId,
|
|
content: [{ type: 'text', text: 'denied by policy' }],
|
|
isError: true,
|
|
}
|
|
}
|
|
return next()
|
|
})
|
|
|
|
const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
|
|
expect(result.isError).toBe(true)
|
|
expect(result.content[0]).toMatchObject({ text: 'denied by policy' })
|
|
})
|
|
|
|
it('composes multiple tools/execute listeners (sandbox-wrap pattern)', async () => {
|
|
const ctx = await setup()
|
|
ctx.tools.register(echoTool)
|
|
|
|
const order: string[] = []
|
|
ctx.on('tools/execute', async (_exec, next) => {
|
|
order.push('first:before')
|
|
const result = await next()
|
|
order.push('first:after')
|
|
return result
|
|
})
|
|
ctx.on('tools/execute', async (_exec, next) => {
|
|
order.push('second:before')
|
|
const result = await next()
|
|
order.push('second:after')
|
|
return result
|
|
})
|
|
|
|
const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'x' } })
|
|
expect(result.isError).toBe(false)
|
|
expect(order).toEqual(['first:before', 'second:before', 'second:after', 'first:after'])
|
|
})
|
|
|
|
it('rejects duplicate names and unregisters on fiber dispose (HMR safety)', async () => {
|
|
const ctx = await setup()
|
|
ctx.tools.register(echoTool)
|
|
expect(() => ctx.tools.register(echoTool)).toThrow('already registered')
|
|
|
|
const fiber = await ctx.plugin(Object.assign((inner: Context) => {
|
|
inner.tools.register({ ...echoTool, name: 'scoped' })
|
|
}, { inject: ['tools'] }))
|
|
expect(ctx.tools.schemas().map(t => t.name)).toEqual(['echo', 'scoped'])
|
|
|
|
await fiber.dispose()
|
|
expect(ctx.tools.schemas().map(t => t.name)).toEqual(['echo'])
|
|
})
|
|
|
|
it('returns a callable disposer from register() that unregisters the tool', async () => {
|
|
const ctx = await setup()
|
|
ctx.tools.register(echoTool)
|
|
|
|
// Register a second tool and call its returned disposer directly
|
|
const dispose = ctx.tools.register({ ...echoTool, name: 'disposable' })
|
|
expect(ctx.tools.schemas().map(t => t.name)).toEqual(['echo', 'disposable'])
|
|
|
|
dispose()
|
|
expect(ctx.tools.schemas().map(t => t.name)).toEqual(['echo'])
|
|
})
|
|
})
|
|
|
|
describe('defineTool / schema DSL', () => {
|
|
it('converts SchemaSpec to standard JSON Schema with required array', () => {
|
|
const spec = {
|
|
path: { type: 'string', required: true, description: 'Absolute path' },
|
|
offset: { type: 'number' },
|
|
limit: { type: 'number', description: 'Max lines' },
|
|
} satisfies SchemaSpec
|
|
const jsonSchema = schemaSpecToJsonSchema(spec)
|
|
expect(jsonSchema).toEqual({
|
|
type: 'object',
|
|
properties: {
|
|
path: { type: 'string', description: 'Absolute path' },
|
|
offset: { type: 'number' },
|
|
limit: { type: 'number', description: 'Max lines' },
|
|
},
|
|
required: ['path'],
|
|
})
|
|
})
|
|
|
|
it('handles empty spec (no properties, no required)', () => {
|
|
expect(schemaSpecToJsonSchema({})).toEqual({
|
|
type: 'object',
|
|
properties: {},
|
|
})
|
|
})
|
|
|
|
it('handles nested object spec', () => {
|
|
const spec = {
|
|
config: {
|
|
type: 'object',
|
|
required: true,
|
|
properties: {
|
|
host: { type: 'string', required: true },
|
|
port: { type: 'number' },
|
|
},
|
|
},
|
|
} satisfies SchemaSpec
|
|
const jsonSchema = schemaSpecToJsonSchema(spec)
|
|
expect(jsonSchema).toEqual({
|
|
type: 'object',
|
|
properties: {
|
|
config: {
|
|
type: 'object',
|
|
properties: {
|
|
host: { type: 'string' },
|
|
port: { type: 'number' },
|
|
},
|
|
required: ['host'],
|
|
},
|
|
},
|
|
required: ['config'],
|
|
})
|
|
})
|
|
|
|
it('defineTool returns a valid ToolDefinition with typed execute', async () => {
|
|
const ctx = await setup()
|
|
const tool = defineTool({
|
|
name: 'typed-echo',
|
|
description: 'A typed echo tool',
|
|
parameters: {
|
|
text: { type: 'string', required: true },
|
|
uppercase: { type: 'boolean' },
|
|
},
|
|
async execute(args) {
|
|
// args is typed: { text: string; uppercase?: boolean }
|
|
const result = args.uppercase ? args.text.toUpperCase() : args.text
|
|
return [{ type: 'text', text: result }]
|
|
},
|
|
})
|
|
|
|
ctx.tools.register(tool)
|
|
expect(ctx.tools.schemas()).toEqual([{
|
|
name: 'typed-echo',
|
|
description: 'A typed echo tool',
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
text: { type: 'string' },
|
|
uppercase: { type: 'boolean' },
|
|
},
|
|
required: ['text'],
|
|
},
|
|
}])
|
|
|
|
const result = await ctx.tools.execute({
|
|
callId: CallId('c1'),
|
|
name: 'typed-echo',
|
|
arguments: { text: 'hello', uppercase: true },
|
|
})
|
|
expect(result.isError).toBe(false)
|
|
expect(result.content).toEqual([{ type: 'text', text: 'HELLO' }])
|
|
})
|
|
|
|
it('type-level: InferArgs maps required properties to non-optional', () => {
|
|
// Compile-time check: if this compiles, InferArgs is correct.
|
|
// args.a is string (required), args.b is number|undefined (optional).
|
|
const tool = defineTool({
|
|
name: 'type-check',
|
|
description: '',
|
|
parameters: { a: { type: 'string' as const, required: true as const }, b: { type: 'number' as const } },
|
|
async execute(args) {
|
|
// Verify types at runtime via typeof
|
|
expect(typeof args.a).toBe('string')
|
|
// args.b should be undefined when not provided
|
|
void args
|
|
return [{ type: 'text', text: args.a }]
|
|
},
|
|
})
|
|
void tool
|
|
})
|
|
|
|
it('registry round-trips a defineTool definition (register→schemas→execute)', async () => {
|
|
const ctx = await setup()
|
|
ctx.tools.register(defineTool({
|
|
name: 'roundtrip',
|
|
description: 'Round-trip test',
|
|
parameters: {
|
|
req: { type: 'string', required: true },
|
|
opt: { type: 'number', description: 'Optional number' },
|
|
},
|
|
async execute(args) {
|
|
return [{ type: 'text', text: `${args.req}:${args.opt ?? 'none'}` }]
|
|
},
|
|
}))
|
|
|
|
// Schema round-trip: schemas() returns standard JSON Schema
|
|
const schemas = ctx.tools.schemas()
|
|
expect(schemas).toHaveLength(1)
|
|
expect(schemas[0]!.parameters).toEqual({
|
|
type: 'object',
|
|
properties: {
|
|
req: { type: 'string' },
|
|
opt: { type: 'number', description: 'Optional number' },
|
|
},
|
|
required: ['req'],
|
|
})
|
|
|
|
// Execution round-trip
|
|
const result = await ctx.tools.execute({
|
|
callId: CallId('c1'),
|
|
name: 'roundtrip',
|
|
arguments: { req: 'hello' },
|
|
})
|
|
expect(result.isError).toBe(false)
|
|
expect(result.content).toEqual([{ type: 'text', text: 'hello:none' }])
|
|
})
|
|
|
|
it('still accepts raw JSON-Schema ToolDefinition directly (MCP interop)', async () => {
|
|
const ctx = await setup()
|
|
ctx.tools.register({
|
|
name: 'raw-tool',
|
|
description: 'Raw JSON Schema tool (like an MCP adapter would register)',
|
|
parameters: {
|
|
type: 'object',
|
|
properties: { path: { type: 'string' } },
|
|
required: ['path'],
|
|
},
|
|
async execute(args: unknown) {
|
|
const p = args as { path: string }
|
|
return [{ type: 'text', text: p.path }]
|
|
},
|
|
})
|
|
|
|
const schemas = ctx.tools.schemas()
|
|
expect(schemas[0]!.parameters).toEqual({
|
|
type: 'object',
|
|
properties: { path: { type: 'string' } },
|
|
required: ['path'],
|
|
})
|
|
|
|
const result = await ctx.tools.execute({
|
|
callId: CallId('c1'),
|
|
name: 'raw-tool',
|
|
arguments: { path: '/tmp' },
|
|
})
|
|
expect(result.isError).toBe(false)
|
|
expect(result.content).toEqual([{ type: 'text', text: '/tmp' }])
|
|
})
|
|
})
|
|
|
|
describe('schema DSL edge cases', () => {
|
|
it('emits enum values in JSON Schema property', () => {
|
|
const spec = {
|
|
color: { type: 'string', enum: ['red', 'green', 'blue'], description: 'Color choice' },
|
|
} satisfies SchemaSpec
|
|
const jsonSchema = schemaSpecToJsonSchema(spec)
|
|
expect(jsonSchema.properties['color']).toMatchObject({
|
|
type: 'string',
|
|
enum: ['red', 'green', 'blue'],
|
|
description: 'Color choice',
|
|
})
|
|
})
|
|
|
|
it('emits default value in JSON Schema property', () => {
|
|
const spec = {
|
|
limit: { type: 'number', default: 25 },
|
|
} satisfies SchemaSpec
|
|
const jsonSchema = schemaSpecToJsonSchema(spec)
|
|
expect(jsonSchema.properties['limit']).toMatchObject({
|
|
type: 'number',
|
|
default: 25,
|
|
})
|
|
})
|
|
|
|
it('handles array items without nested properties (plain type array)', () => {
|
|
const spec = {
|
|
tags: { type: 'array', items: { type: 'string' } },
|
|
} satisfies SchemaSpec
|
|
const jsonSchema = schemaSpecToJsonSchema(spec)
|
|
expect(jsonSchema.properties['tags']).toEqual({
|
|
type: 'array',
|
|
items: { type: 'string' },
|
|
})
|
|
})
|
|
|
|
it('defineTool passes through strict flag when set to true', () => {
|
|
const tool = defineTool({
|
|
name: 'strict-tool',
|
|
description: 'A strict tool',
|
|
parameters: { input: { type: 'string' } },
|
|
strict: true,
|
|
async execute(args) {
|
|
return [{ type: 'text' as const, text: args.input ?? '' }]
|
|
},
|
|
})
|
|
expect(tool.strict).toBe(true)
|
|
})
|
|
|
|
it('defineTool omits strict when not provided', () => {
|
|
const tool = defineTool({
|
|
name: 'non-strict-tool',
|
|
description: 'A non-strict tool',
|
|
parameters: { input: { type: 'string' } },
|
|
async execute(args) {
|
|
return [{ type: 'text' as const, text: args.input ?? '' }]
|
|
},
|
|
})
|
|
expect('strict' in tool).toBe(false)
|
|
})
|
|
|
|
it('defineTool strict=false is included', () => {
|
|
const tool = defineTool({
|
|
name: 'explicitly-non-strict',
|
|
description: 'Explicitly non-strict',
|
|
parameters: { input: { type: 'string' } },
|
|
strict: false,
|
|
async execute(args) {
|
|
return [{ type: 'text' as const, text: args.input ?? '' }]
|
|
},
|
|
})
|
|
expect(tool.strict).toBe(false)
|
|
})
|
|
|
|
it('handles enum and default together in one property', () => {
|
|
const spec = {
|
|
level: { type: 'string', enum: ['low', 'high'], default: 'low' },
|
|
} satisfies SchemaSpec
|
|
const jsonSchema = schemaSpecToJsonSchema(spec)
|
|
expect(jsonSchema.properties['level']).toMatchObject({
|
|
type: 'string',
|
|
enum: ['low', 'high'],
|
|
default: 'low',
|
|
})
|
|
})
|
|
|
|
it('omits description, enum, default keys when not specified', () => {
|
|
const spec = {
|
|
bare: { type: 'string' },
|
|
} satisfies SchemaSpec
|
|
const jsonSchema = schemaSpecToJsonSchema(spec)
|
|
const prop = jsonSchema.properties['bare'] as Record<string, unknown>
|
|
expect(prop).toEqual({ type: 'string' })
|
|
expect('description' in prop).toBe(false)
|
|
expect('enum' in prop).toBe(false)
|
|
expect('default' in prop).toBe(false)
|
|
})
|
|
|
|
it('handles array with no items (items omitted)', () => {
|
|
const spec = {
|
|
raw: { type: 'array' },
|
|
} satisfies SchemaSpec
|
|
const jsonSchema = schemaSpecToJsonSchema(spec)
|
|
expect(jsonSchema.properties['raw']).toEqual({
|
|
type: 'array',
|
|
})
|
|
})
|
|
|
|
it('handles nested object with all-optional properties (no required array)', () => {
|
|
const spec = {
|
|
config: {
|
|
type: 'object',
|
|
properties: {
|
|
host: { type: 'string' },
|
|
port: { type: 'number' },
|
|
},
|
|
},
|
|
} satisfies SchemaSpec
|
|
const jsonSchema = schemaSpecToJsonSchema(spec)
|
|
expect(jsonSchema.properties['config']).toMatchObject({
|
|
type: 'object',
|
|
properties: {
|
|
host: { type: 'string' },
|
|
port: { type: 'number' },
|
|
},
|
|
})
|
|
// no 'required' key in the nested object because nothing is required
|
|
const config = jsonSchema.properties['config'] as Record<string, unknown>
|
|
expect('required' in config).toBe(false)
|
|
})
|
|
})
|
|
|
|
describe('schema DSL regressions (Codex review round 2)', () => {
|
|
it('InferArgs makes non-required keys genuinely optional (omittable)', () => {
|
|
type Args = InferArgs<{
|
|
path: { type: 'string'; required: true }
|
|
limit: { type: 'number' }
|
|
}>
|
|
expectTypeOf<Args>().toEqualTypeOf<{ path: string; limit?: number }>()
|
|
// omitting the optional key is assignable — the actual regression
|
|
const omitted: Args = { path: '/tmp' }
|
|
expect(omitted.limit).toBeUndefined()
|
|
})
|
|
|
|
it('InferArgs recurses into array items, including arrays of objects', () => {
|
|
type Args = InferArgs<{
|
|
names: { type: 'array'; required: true; items: { type: 'string' } }
|
|
servers: {
|
|
type: 'array'
|
|
items: {
|
|
type: 'object'
|
|
properties: {
|
|
host: { type: 'string'; required: true }
|
|
port: { type: 'number' }
|
|
}
|
|
}
|
|
}
|
|
}>
|
|
expectTypeOf<Args>().toEqualTypeOf<{
|
|
names: string[]
|
|
servers?: { host: string; port?: number }[]
|
|
}>()
|
|
})
|
|
|
|
it('runtime JSON Schema matches the array-of-objects inference', () => {
|
|
const spec = {
|
|
servers: {
|
|
type: 'array',
|
|
items: {
|
|
type: 'object',
|
|
properties: {
|
|
host: { type: 'string', required: true },
|
|
port: { type: 'number' },
|
|
},
|
|
},
|
|
},
|
|
} satisfies SchemaSpec
|
|
expect(schemaSpecToJsonSchema(spec)).toEqual({
|
|
type: 'object',
|
|
properties: {
|
|
servers: {
|
|
type: 'array',
|
|
items: {
|
|
type: 'object',
|
|
properties: {
|
|
host: { type: 'string' },
|
|
port: { type: 'number' },
|
|
},
|
|
required: ['host'],
|
|
},
|
|
},
|
|
},
|
|
})
|
|
})
|
|
|
|
it('reports messages from non-Error throws (throw { message })', async () => {
|
|
const ctx = await setup()
|
|
ctx.tools.register({
|
|
...echoTool,
|
|
name: 'object-thrower',
|
|
async execute() {
|
|
// testing non-Error throws on purpose
|
|
throw { message: 'denied by object' }
|
|
},
|
|
})
|
|
const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'object-thrower', arguments: {} })
|
|
expect(result.isError).toBe(true)
|
|
expect(result.content[0]).toMatchObject({ text: 'Error: denied by object' })
|
|
})
|
|
|
|
it('reports messages from throws of non-objects (throw "string")', async () => {
|
|
const ctx = await setup()
|
|
ctx.tools.register({
|
|
...echoTool,
|
|
name: 'string-thrower',
|
|
async execute() {
|
|
// testing primitive throws on purpose
|
|
throw 'kaboom'
|
|
},
|
|
})
|
|
const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'string-thrower', arguments: {} })
|
|
expect(result.isError).toBe(true)
|
|
expect(result.content[0]).toMatchObject({ text: 'Error: kaboom' })
|
|
})
|
|
|
|
it('reports messages from throws of objects without message property', async () => {
|
|
const ctx = await setup()
|
|
ctx.tools.register({
|
|
...echoTool,
|
|
name: 'object-no-message',
|
|
async execute() {
|
|
// testing object throw without .message
|
|
throw { code: 500 }
|
|
},
|
|
})
|
|
const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'object-no-message', arguments: {} })
|
|
expect(result.isError).toBe(true)
|
|
const firstContent = result.content[0]!
|
|
expect(firstContent.type).toBe('text')
|
|
if (firstContent.type === 'text') {
|
|
expect(firstContent.text).toBe('Error: [object Object]')
|
|
}
|
|
})
|
|
})
|
|
|
|
describe('ToolRegistry.get', () => {
|
|
it('get() returns the registered tool definition', async () => {
|
|
const ctx = await setup()
|
|
ctx.tools.register(echoTool)
|
|
const tool = ctx.tools.get('echo')
|
|
expect(tool).toBeDefined()
|
|
expect(tool!.name).toBe('echo')
|
|
})
|
|
|
|
it('get() returns undefined for unknown tool names', async () => {
|
|
const ctx = await setup()
|
|
expect(ctx.tools.get('nope')).toBeUndefined()
|
|
})
|
|
})
|