mirror of
https://github.com/deepseek-ai/deepseek-harness
synced 2026-08-15 21:04:50 +00:00
387 lines
16 KiB
TypeScript
387 lines
16 KiB
TypeScript
/**
|
|
* Web session model-directory and selection behavior: dynamic provider grouping,
|
|
* provider-local catalog failures, logged-target restoration, advisory unlisted
|
|
* models, and the prompt-assembly boundary for a running selection change.
|
|
*/
|
|
|
|
import { describe, expect, it } from 'vitest'
|
|
import { Context } from 'cordis'
|
|
import AgentRegistry, { agentEvents } from '@deepseek-ai/dsh-agent'
|
|
import type { Agent } from '@deepseek-ai/dsh-agent'
|
|
import LlmService, { LlmAdapter, ReasoningEffortId } from '@deepseek-ai/dsh-llm'
|
|
import type {
|
|
GenerateOptions, LlmCallConfig, LlmModelInfo, LlmModelReasoningInfo, LlmProviderInfo,
|
|
LlmResolvedModelInfo, StreamChunk,
|
|
} from '@deepseek-ai/dsh-llm'
|
|
import SessionStore from '@deepseek-ai/dsh-session'
|
|
import type { SessionId } from '@deepseek-ai/dsh-session'
|
|
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
|
|
import UserInteractionService from '@deepseek-ai/dsh-user-interaction'
|
|
import type { RpcRequest } from '@deepseek-ai/dsh-host-apiproxy/api/rpc'
|
|
import { RpcId } from '@deepseek-ai/dsh-host-apiproxy/api/rpc'
|
|
import { createApiProxy } from '../src/api-proxy.ts'
|
|
|
|
let nextRpc = 1
|
|
function request<P>(payload: P): RpcRequest<P> {
|
|
return { rpcId: RpcId(`models-${String(nextRpc++)}`), payload }
|
|
}
|
|
|
|
class CatalogAdapter extends LlmAdapter {
|
|
constructor(
|
|
private readonly name: string,
|
|
private readonly models: readonly LlmModelInfo[] | Error,
|
|
private readonly reasoning?: LlmModelReasoningInfo,
|
|
private readonly exactError?: Error,
|
|
) {
|
|
super()
|
|
}
|
|
|
|
override providerInfo(provider: string): LlmProviderInfo {
|
|
return { id: provider, name: this.name }
|
|
}
|
|
|
|
override listModels(): Promise<readonly LlmModelInfo[]> {
|
|
return this.models instanceof Error
|
|
? Promise.reject(this.models)
|
|
: Promise.resolve(this.models)
|
|
}
|
|
|
|
override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
|
|
if (this.exactError !== undefined) return Promise.reject(this.exactError)
|
|
return Promise.resolve({
|
|
provider,
|
|
id: model,
|
|
name: model,
|
|
...this.reasoning === undefined ? {} : { reasoning: this.reasoning },
|
|
})
|
|
}
|
|
|
|
override async *stream(_options: GenerateOptions): AsyncIterable<StreamChunk> {
|
|
// Catalog tests never enter provider streaming.
|
|
}
|
|
}
|
|
|
|
const REASONING: LlmModelReasoningInfo = {
|
|
efforts: [
|
|
{ id: ReasoningEffortId('off'), name: 'Off' },
|
|
{ id: ReasoningEffortId('high'), name: 'High' },
|
|
{ id: ReasoningEffortId('max'), name: 'Max' },
|
|
],
|
|
defaultEffort: ReasoningEffortId('high'),
|
|
}
|
|
|
|
async function harness(logged?: {
|
|
provider: string
|
|
model: string
|
|
reasoningEffort?: ReasoningEffortId
|
|
}): Promise<{
|
|
ctx: Context
|
|
agent: Agent
|
|
sessionId: SessionId
|
|
}> {
|
|
const ctx = new Context()
|
|
await ctx.plugin(SessionStore)
|
|
await ctx.plugin(SystemPrompt, { persona: '' })
|
|
await ctx.plugin(LlmService)
|
|
await ctx.plugin(UserInteractionService)
|
|
await ctx.plugin(AgentRegistry)
|
|
ctx.llm.registerAdapter(['deepseek'], new CatalogAdapter('DeepSeek', [
|
|
{ provider: 'deepseek', id: 'deepseek-chat', name: 'DeepSeek Chat' },
|
|
{ provider: 'deepseek', id: 'deepseek-reasoner', name: 'DeepSeek Reasoner', description: 'Reasoning model' },
|
|
], REASONING))
|
|
ctx.llm.registerAdapter(['broken'], new CatalogAdapter('Broken Provider', new Error('catalog offline')))
|
|
ctx.llm.registerAdapter(['metadata-broken'], new CatalogAdapter('Metadata Broken', [
|
|
{ provider: 'metadata-broken', id: 'listed', name: 'Listed' },
|
|
], undefined, new Error('reasoning metadata offline')))
|
|
ctx.llm.registerAdapter(['empty'], new CatalogAdapter('Empty Provider', []))
|
|
ctx.llm.registerAdapter(['duplicate'], new CatalogAdapter('Duplicate Provider', [
|
|
{ provider: 'duplicate', id: 'same', name: 'Same' },
|
|
{ provider: 'duplicate', id: 'same', name: 'Same Again' },
|
|
]))
|
|
const session = ctx.sessions.create()
|
|
if (logged !== undefined) {
|
|
session.append('request/header', { header: { config: logged }, reason: 'initial' })
|
|
}
|
|
const agent = {
|
|
id: session.id,
|
|
session,
|
|
status: 'running',
|
|
ctx,
|
|
} as Agent
|
|
ctx.agents.register(agent)
|
|
return { ctx, agent, sessionId: session.id }
|
|
}
|
|
|
|
function expectValue<T>(response: { result: { ok: true; value: T } | { ok: false } }): T {
|
|
if (!response.result.ok) throw new Error('expected successful response')
|
|
return response.result.value
|
|
}
|
|
|
|
describe('Web session model selection', () => {
|
|
it('rejects a second prompt image before attachment persistence', async () => {
|
|
const { ctx, sessionId } = await harness()
|
|
const api = createApiProxy(ctx, { provider: 'deepseek', model: 'deepseek-chat', cwd: '/tmp', workspaceRoot: '/tmp' })
|
|
const image = { type: 'image' as const, mediaType: 'image/png' as const, data: 'AA==' }
|
|
const response = await api.sessions.prompt(request({
|
|
sessionId,
|
|
mode: 'queue' as const,
|
|
content: [image, image],
|
|
}))
|
|
expect(response.result).toEqual({
|
|
ok: false,
|
|
error: {
|
|
code: 'attachment-error',
|
|
message: 'A prompt may contain at most one image.',
|
|
details: { reason: 'TOO_MANY_IMAGES' },
|
|
},
|
|
})
|
|
await ctx.fiber.dispose()
|
|
})
|
|
|
|
it('groups successful providers, isolates failures, and preserves an unlisted current model', async () => {
|
|
const { ctx, sessionId } = await harness({
|
|
provider: 'deepseek',
|
|
model: 'private-preview',
|
|
reasoningEffort: ReasoningEffortId('max'),
|
|
})
|
|
const api = createApiProxy(ctx, { provider: 'deepseek', model: 'deepseek-chat', cwd: '/tmp', workspaceRoot: '/tmp' })
|
|
|
|
const catalog = expectValue(await api.sessions.models(request({ sessionId })))
|
|
expect(catalog.current).toEqual({
|
|
provider: 'deepseek',
|
|
model: 'private-preview',
|
|
reasoningEffort: 'max',
|
|
})
|
|
expect(catalog.groups).toEqual([{
|
|
id: 'deepseek',
|
|
name: 'DeepSeek',
|
|
models: [
|
|
{ id: 'deepseek-chat', name: 'DeepSeek Chat', reasoning: REASONING },
|
|
{
|
|
id: 'deepseek-reasoner',
|
|
name: 'DeepSeek Reasoner',
|
|
description: 'Reasoning model',
|
|
reasoning: REASONING,
|
|
},
|
|
{
|
|
id: 'private-preview',
|
|
name: 'private-preview',
|
|
unlisted: true,
|
|
reasoning: REASONING,
|
|
},
|
|
],
|
|
}])
|
|
expect(catalog.failures).toEqual([
|
|
{ id: 'broken', name: 'Broken Provider', message: 'catalog offline' },
|
|
{ id: 'metadata-broken', name: 'Metadata Broken', message: 'reasoning metadata offline' },
|
|
{
|
|
id: 'duplicate',
|
|
name: 'Duplicate Provider',
|
|
message: 'adapter returned invalid or duplicate model metadata for provider "duplicate"',
|
|
},
|
|
])
|
|
await ctx.fiber.dispose()
|
|
})
|
|
|
|
it('accepts an advisory-unlisted model, rejects an unavailable provider, and switches only after the next assembly', async () => {
|
|
const { ctx, agent, sessionId } = await harness()
|
|
const api = createApiProxy(ctx, { provider: 'deepseek', model: 'deepseek-chat', cwd: '/tmp', workspaceRoot: '/tmp' })
|
|
const seed: LlmCallConfig = { provider: 'seed', model: 'seed', temperature: 0.2 }
|
|
const signal = new AbortController().signal
|
|
|
|
expect(expectValue(await api.sessions.models(request({ sessionId }))).current)
|
|
.toEqual({ provider: 'deepseek', model: 'deepseek-chat' })
|
|
expect((await ctx.systemPrompt.assemble()).variables)
|
|
.toMatchObject({ provider: 'deepseek', model: 'deepseek-chat' })
|
|
|
|
const selected = expectValue(await api.sessions.selectModel(request({
|
|
sessionId,
|
|
provider: 'deepseek',
|
|
model: 'private-preview',
|
|
reasoningEffort: 'max',
|
|
})))
|
|
expect(selected.selected).toEqual({
|
|
provider: 'deepseek',
|
|
model: 'private-preview',
|
|
reasoningEffort: 'max',
|
|
})
|
|
await expect(agentEvents(ctx, agent).waterfall(
|
|
'agent/request', 1, 0, signal, () => Promise.resolve(seed),
|
|
)).resolves.toMatchObject({ provider: 'deepseek', model: 'deepseek-chat' })
|
|
|
|
expect((await ctx.systemPrompt.assemble()).variables)
|
|
.toMatchObject({ provider: 'deepseek', model: 'private-preview' })
|
|
await expect(agentEvents(ctx, agent).waterfall(
|
|
'agent/request', 1, 1, signal, () => Promise.resolve(seed),
|
|
)).resolves.toMatchObject({
|
|
provider: 'deepseek',
|
|
model: 'private-preview',
|
|
reasoningEffort: 'max',
|
|
})
|
|
|
|
const unsupported = await api.sessions.selectModel(request({
|
|
sessionId,
|
|
provider: 'deepseek',
|
|
model: 'private-preview',
|
|
reasoningEffort: 'medium',
|
|
}))
|
|
expect(unsupported.result).toMatchObject({
|
|
ok: false,
|
|
error: {
|
|
code: 'model-unavailable',
|
|
message: 'provider "deepseek" model "private-preview" does not support reasoning effort "medium"',
|
|
},
|
|
})
|
|
|
|
const rejected = await api.sessions.selectModel(request({
|
|
sessionId,
|
|
provider: 'missing',
|
|
model: 'model',
|
|
}))
|
|
expect(rejected.result).toEqual({
|
|
ok: false,
|
|
error: {
|
|
code: 'model-unavailable',
|
|
message: 'no adapter registered for provider "missing"',
|
|
details: { provider: 'missing', model: 'model' },
|
|
},
|
|
})
|
|
expect(expectValue(await api.sessions.models(request({ sessionId }))).current)
|
|
.toEqual({ provider: 'deepseek', model: 'private-preview', reasoningEffort: 'max' })
|
|
await ctx.fiber.dispose()
|
|
})
|
|
|
|
it('refuses a text-only selection once the session log carries an image', async () => {
|
|
const { ctx, sessionId, agent } = await harness()
|
|
ctx.llm.registerAdapter(['text-only'], new class extends CatalogAdapter {
|
|
override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
|
|
return Promise.resolve({ provider, id: model, name: model, inputModalities: ['text'] })
|
|
}
|
|
}('Text Only', []))
|
|
ctx.llm.registerAdapter(['vision'], new class extends CatalogAdapter {
|
|
override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
|
|
return Promise.resolve({ provider, id: model, name: model, inputModalities: ['text', 'image'] })
|
|
}
|
|
}('Vision', []))
|
|
const api = createApiProxy(ctx, { provider: 'deepseek', model: 'deepseek-chat', cwd: '/tmp', workspaceRoot: '/tmp' })
|
|
|
|
// Before any image lands, a text-only selection is legitimate.
|
|
expect(expectValue(await api.sessions.selectModel(request({
|
|
sessionId, provider: 'text-only', model: 'plain',
|
|
}))).selected).toEqual({ provider: 'text-only', model: 'plain' })
|
|
|
|
agent.session.append('user/message', {
|
|
id: 'msg-image', role: 'user', source: { kind: 'user' },
|
|
content: [{ type: 'image', attachment: { attachmentId: 'att-1', mediaType: 'image/png', bytes: 8, width: 1, height: 1 } }],
|
|
} as never, { surfaceOp: 'append' })
|
|
|
|
// The log is immutable: a text-only route would fail every later turn.
|
|
const stranded = await api.sessions.selectModel(request({
|
|
sessionId, provider: 'text-only', model: 'plain',
|
|
}))
|
|
expect(stranded.result).toMatchObject({
|
|
ok: false,
|
|
error: { code: 'model-unavailable', message: expect.stringMatching(/history already contains images/) as unknown },
|
|
})
|
|
|
|
// Image-capable and modality-unknown routes stay selectable.
|
|
expect(expectValue(await api.sessions.selectModel(request({
|
|
sessionId, provider: 'vision', model: 'sees',
|
|
}))).selected).toEqual({ provider: 'vision', model: 'sees' })
|
|
expect(expectValue(await api.sessions.selectModel(request({
|
|
sessionId, provider: 'deepseek', model: 'deepseek-chat',
|
|
}))).selected).toEqual({ provider: 'deepseek', model: 'deepseek-chat', reasoningEffort: 'high' })
|
|
await ctx.fiber.dispose()
|
|
})
|
|
|
|
it('refuses a text-only selection while an image prompt is still queued (not yet logged)', async () => {
|
|
const { ctx, sessionId, agent } = await harness()
|
|
ctx.llm.registerAdapter(['text-only'], new class extends CatalogAdapter {
|
|
override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
|
|
return Promise.resolve({ provider, id: model, name: model, inputModalities: ['text'] })
|
|
}
|
|
}('Text Only', []))
|
|
const api = createApiProxy(ctx, { provider: 'deepseek', model: 'deepseek-chat', cwd: '/tmp', workspaceRoot: '/tmp' })
|
|
// The queued message enters the session log only when claimed — after a
|
|
// model switch would already have landed. The pending-inbox mirror must
|
|
// therefore gate the switch too.
|
|
ctx.emit('agent/inbox/enqueue', agent, {
|
|
id: 'q-1', role: 'user', source: { kind: 'user' },
|
|
content: [{ type: 'image', attachment: { attachmentId: 'att-q', mediaType: 'image/png', bytes: 8, width: 1, height: 1 } }],
|
|
} as never, 'queued')
|
|
const stranded = await api.sessions.selectModel(request({ sessionId, provider: 'text-only', model: 'plain' }))
|
|
expect(stranded.result.ok).toBe(false)
|
|
// Claiming the message drains the mirror; the log now owns the decision.
|
|
ctx.emit('agent/inbox/dequeue', agent, { id: 'q-1' } as never, 'queued')
|
|
expect(expectValue(await api.sessions.selectModel(request({
|
|
sessionId, provider: 'text-only', model: 'plain',
|
|
}))).selected).toEqual({ provider: 'text-only', model: 'plain' })
|
|
await ctx.fiber.dispose()
|
|
})
|
|
|
|
it('authorizes an attachment read referenced only from wrapped message content', async () => {
|
|
const { ctx, sessionId, agent } = await harness()
|
|
const ref = { attachmentId: 'att-w', mediaType: 'image/png' as const, bytes: 4, width: 1, height: 1 }
|
|
ctx.provide('attachments', {
|
|
readImage: () => Promise.resolve({ ref, data: new Uint8Array([1, 2, 3, 4]) }),
|
|
} as never)
|
|
const api = createApiProxy(ctx, { provider: 'deepseek', model: 'deepseek-chat', cwd: '/tmp', workspaceRoot: '/tmp' })
|
|
// The only reference lives inside an assistant/message wrapper — the same
|
|
// walk that gates model selection must authorize the read, or a real host
|
|
// denies galleries the fixture (with its own authorization mirror) serves.
|
|
agent.session.append('assistant/message', {
|
|
turn: 1, step: 0,
|
|
message: { id: 'a-1', role: 'assistant', source: { kind: 'model', provider: 'p', model: 'm' }, content: [{ type: 'image', attachment: ref }] },
|
|
} as never, { surfaceOp: 'append' })
|
|
const got = await api.sessions.attachment(request({ sessionId, attachmentId: 'att-w' as never }))
|
|
expect(got.result).toMatchObject({ ok: true, value: { attachment: ref } })
|
|
const denied = await api.sessions.attachment(request({ sessionId, attachmentId: 'att-other' as never }))
|
|
expect(denied.result).toMatchObject({ ok: false, error: { details: { reason: 'ATTACHMENT_NOT_REFERENCED' } } })
|
|
await ctx.fiber.dispose()
|
|
})
|
|
|
|
it('detects images on every replayed route: wrapped messages, streamed blocks, nested tool results', async () => {
|
|
const image = { type: 'image', attachment: { attachmentId: 'att-x', mediaType: 'image/png', bytes: 8, width: 1, height: 1 } }
|
|
const cases: { label: string; append: (agent: Agent) => void }[] = [
|
|
{
|
|
label: 'steering message wrapper',
|
|
append: (agent) => {
|
|
agent.session.append('steering/message', {
|
|
turn: 1, message: { id: 'st-1', role: 'user', source: { kind: 'user' }, content: [image] },
|
|
} as never, { surfaceOp: 'append' })
|
|
},
|
|
},
|
|
{
|
|
label: 'streamed assistant block',
|
|
append: (agent) => {
|
|
agent.session.append('assistant/chunk', {
|
|
turn: 1, step: 0, chunk: { type: 'block-end', index: 0, block: image },
|
|
} as never)
|
|
},
|
|
},
|
|
{
|
|
label: 'nested tool-result content',
|
|
append: (agent) => {
|
|
agent.session.append('user/message', {
|
|
id: 'tr-1', role: 'user', source: { kind: 'tool', callId: 'c1' },
|
|
content: [{ type: 'tool-result', toolCallId: 'c1', content: [image], isError: false }],
|
|
} as never, { surfaceOp: 'append' })
|
|
},
|
|
},
|
|
]
|
|
for (const { label, append } of cases) {
|
|
const { ctx, sessionId, agent } = await harness()
|
|
ctx.llm.registerAdapter(['text-only'], new class extends CatalogAdapter {
|
|
override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
|
|
return Promise.resolve({ provider, id: model, name: model, inputModalities: ['text'] })
|
|
}
|
|
}('Text Only', []))
|
|
const api = createApiProxy(ctx, { provider: 'deepseek', model: 'deepseek-chat', cwd: '/tmp', workspaceRoot: '/tmp' })
|
|
append(agent)
|
|
const stranded = await api.sessions.selectModel(request({ sessionId, provider: 'text-only', model: 'plain' }))
|
|
expect(stranded.result.ok, label).toBe(false)
|
|
await ctx.fiber.dispose()
|
|
}
|
|
})
|
|
})
|