mirror of
https://github.com/deepseek-ai/deepseek-harness
synced 2026-08-15 21:04:50 +00:00
Merge remote-tracking branch 'origin/master' into worktree/web-plugin-config
# Conflicts: # docs/event-producer-consumer.i18n.yaml # docs/event-producer-consumer.md # docs/event-producer-consumer.zh.md # docs/module-graph.i18n.yaml # docs/module-graph.md # docs/module-graph.zh.md # packages/client/ui-conversation/README.i18n.yaml # packages/client/ui-conversation/README.md # packages/client/ui-conversation/README.zh.md # packages/client/ui-conversation/package.json # pnpm-lock.yaml
This commit is contained in:
@@ -10,9 +10,11 @@ import type { Context } from 'cordis'
|
||||
import { installModelSelection } from '@deepseek-ai/dsh-agent'
|
||||
import type { Agent, ModelSelection, ModelSelectionRef, AgentOptions, AgentStatus } from '@deepseek-ai/dsh-agent'
|
||||
import { AGENT_DEFAULT_MODEL_SETTINGS_NAMESPACE } from '@deepseek-ai/dsh-agent-default-model'
|
||||
import { createUserMessage, freezeMessage, ReasoningEffortId } from '@deepseek-ai/dsh-llm'
|
||||
import { AttachmentError } from '@deepseek-ai/dsh-attachment'
|
||||
import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'
|
||||
import { contentHasImage, createUserMessage, freezeMessage, ReasoningEffortId } from '@deepseek-ai/dsh-llm'
|
||||
import { errorChain } from '@deepseek-ai/dsh-llm'
|
||||
import type { MessageSource } from '@deepseek-ai/dsh-llm'
|
||||
import type { ContentBlock, MessageSource } from '@deepseek-ai/dsh-llm'
|
||||
import { isAppendSurfaceEvent, lastActivityTime } from '@deepseek-ai/dsh-session'
|
||||
import type { Session, SessionEvent, SessionEventMap, SessionHeader, SessionId, UserMessage } from '@deepseek-ai/dsh-session'
|
||||
import type { SessionPersistence } from '@deepseek-ai/dsh-session-persistence'
|
||||
@@ -35,7 +37,7 @@ import type {} from '@deepseek-ai/dsh-tools'
|
||||
import type {
|
||||
ApiProxy, ConfigurableProviderView, CredentialView, GoalRef, HistoryEntry, HostFrame,
|
||||
ModelCatalogFailure, ModelProviderGroup,
|
||||
ModelReasoning, MuxFrame, QuestionResponsePayload, SessionProjectionsBlock, SessionSearchItem,
|
||||
ModelReasoning, MuxFrame, PromptContentPart, QuestionResponsePayload, SessionProjectionsBlock, SessionSearchItem,
|
||||
QueuedInboxItem, SessionSummary, SettingsNamespaceView, SubagentAddress, ToolEventView,
|
||||
WorkspaceId, WorkspaceView,
|
||||
} from './api/index.ts'
|
||||
@@ -102,6 +104,113 @@ const COLD_SUMMARY_BATCH_SIZE = 16
|
||||
/** Conversation message event types (the pagination counting unit). */
|
||||
const MESSAGE_TYPES = new Set(['user/message', 'assistant/message'])
|
||||
|
||||
/** Decode the browser payload while rejecting non-canonical base64 forms. */
|
||||
function decodeBase64(data: string): Uint8Array {
|
||||
const decoded = Buffer.from(data, 'base64')
|
||||
if (data.length === 0 || decoded.toString('base64') !== data) {
|
||||
throw new AttachmentError('Image upload is not canonical base64.', 'INVALID_IMAGE_BASE64')
|
||||
}
|
||||
return new Uint8Array(decoded)
|
||||
}
|
||||
|
||||
/** Validate one prompt as a batch before publishing any durable image object. */
|
||||
async function durablePromptContent(ctx: Context, content: readonly PromptContentPart[]): Promise<ContentBlock[]> {
|
||||
if (content.every(part => part.type === 'text')) {
|
||||
return content.map(part => ({ type: 'text', text: part.text }))
|
||||
}
|
||||
const limits = ctx.attachments.imageLimits
|
||||
if (content.filter(part => part.type === 'image').length > limits.maxImagesPerMessage) {
|
||||
throw new AttachmentError('Prompt exceeds the configured image-count limit.', 'TOO_MANY_IMAGES')
|
||||
}
|
||||
const prepared = content.map(part => part.type === 'text'
|
||||
? part
|
||||
: { part, data: decodeBase64(part.data) })
|
||||
const images = prepared.filter((part): part is Extract<typeof part, { data: Uint8Array }> => 'data' in part)
|
||||
const totalBytes = images.reduce((sum, image) => sum + image.data.byteLength, 0)
|
||||
if (totalBytes > limits.maxMessageImageBytes) {
|
||||
throw new AttachmentError('Prompt exceeds the configured aggregate image-byte limit.', 'IMAGES_TOO_LARGE')
|
||||
}
|
||||
for (const image of images) {
|
||||
await ctx.attachments.validateImage({
|
||||
data: image.data,
|
||||
mediaType: image.part.mediaType,
|
||||
...image.part.name === undefined ? {} : { name: image.part.name },
|
||||
})
|
||||
}
|
||||
const blocks: ContentBlock[] = []
|
||||
for (const item of prepared) {
|
||||
if (!('data' in item)) {
|
||||
blocks.push({ type: 'text', text: item.text })
|
||||
continue
|
||||
}
|
||||
const attachment = await ctx.attachments.saveImage({
|
||||
data: item.data,
|
||||
mediaType: item.part.mediaType,
|
||||
...item.part.name === undefined ? {} : { name: item.part.name },
|
||||
})
|
||||
blocks.push({ type: 'image', attachment })
|
||||
}
|
||||
return blocks
|
||||
}
|
||||
|
||||
/** Search durable content for an image reference, including nested tool results. */
|
||||
function imageBlockIn(content: unknown, match: (ref: ImageAttachmentRef) => boolean): ImageAttachmentRef | undefined {
|
||||
if (!Array.isArray(content)) return undefined
|
||||
for (const value of content) {
|
||||
if (typeof value !== 'object' || value === null || Array.isArray(value)) continue
|
||||
const block = value as { type?: unknown; attachment?: unknown; content?: unknown }
|
||||
if (block.type === 'image' && typeof block.attachment === 'object' && block.attachment !== null) {
|
||||
const ref = block.attachment as ImageAttachmentRef
|
||||
if (match(ref)) return ref
|
||||
}
|
||||
if (block.type === 'tool-result') {
|
||||
const nested = imageBlockIn(block.content, match)
|
||||
if (nested !== undefined) return nested
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** Search every durable event carrier that can own model-visible content. */
|
||||
function imageInEvent(event: SessionEvent, match: (ref: ImageAttachmentRef) => boolean): ImageAttachmentRef | undefined {
|
||||
const data = event.data as {
|
||||
content?: unknown
|
||||
message?: { content?: unknown }
|
||||
inserted?: Array<{ content?: unknown }>
|
||||
chunk?: { type?: unknown; block?: unknown }
|
||||
}
|
||||
const direct = imageBlockIn(data.content, match)
|
||||
if (direct !== undefined) return direct
|
||||
if (data.message !== undefined) {
|
||||
const wrapped = imageBlockIn(data.message.content, match)
|
||||
if (wrapped !== undefined) return wrapped
|
||||
}
|
||||
if (data.inserted !== undefined) {
|
||||
for (const message of data.inserted) {
|
||||
const inserted = imageBlockIn(message.content, match)
|
||||
if (inserted !== undefined) return inserted
|
||||
}
|
||||
}
|
||||
if (event.type === 'assistant/chunk' && data.chunk?.type === 'block-end') {
|
||||
return imageBlockIn([data.chunk.block], match)
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** True when the current model-visible surface contains an image. */
|
||||
function messagesHaveImage(messages: readonly { content: readonly ContentBlock[] }[]): boolean {
|
||||
return messages.some(message => contentHasImage(message.content))
|
||||
}
|
||||
|
||||
/** Resolve the first reference matching one opaque id. */
|
||||
function referencedImage(events: readonly SessionEvent[], attachmentId: string): ImageAttachmentRef | undefined {
|
||||
for (const event of events) {
|
||||
const found = imageInEvent(event, ref => String(ref.attachmentId) === attachmentId)
|
||||
if (found !== undefined) return found
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Product settings intentionally exposed beside model-provider namespaces.
|
||||
*
|
||||
@@ -869,6 +978,14 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
|
||||
const pendingQuestions = new Map<RpcId, PendingQuestion>()
|
||||
const pendingApprovals = new Map<RpcId, PendingApproval>()
|
||||
const muxQueues = new Set<FrameQueue<RpcRequest<MuxFrame>>>()
|
||||
const imageAdmissionChains = new WeakMap<Agent, Promise<void>>()
|
||||
|
||||
/** Serialize image admission with model selection for one agent. */
|
||||
function serializeImageAdmission<T>(agent: Agent, operation: () => Promise<T>): Promise<T> {
|
||||
const result = (imageAdmissionChains.get(agent) ?? Promise.resolve()).then(operation)
|
||||
imageAdmissionChains.set(agent, result.then(() => undefined, () => undefined))
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Install or return the session-local model selection that prompt assembly snapshots.
|
||||
@@ -1931,41 +2048,51 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
|
||||
const { sessionId, provider, model, reasoningEffort } = request.payload
|
||||
const found = await agentFor(sessionId)
|
||||
if ('error' in found) return err(request, found.error)
|
||||
try {
|
||||
const resolved = await ctx.llm.resolveCallConfig({
|
||||
provider,
|
||||
model,
|
||||
...reasoningEffort === undefined
|
||||
? {}
|
||||
: { reasoningEffort: ReasoningEffortId(reasoningEffort) },
|
||||
})
|
||||
const selected: ModelSelection = {
|
||||
provider: resolved.provider,
|
||||
model: resolved.model,
|
||||
...resolved.reasoningEffort === undefined
|
||||
? {}
|
||||
: { reasoningEffort: resolved.reasoningEffort },
|
||||
}
|
||||
selectionFor(found.agent).current = selected
|
||||
// A switch is also how this deployment's default is chosen: the next
|
||||
// session created without one of its own starts here. Sessions that
|
||||
// have already logged a selection are unaffected — they derive from
|
||||
// their own log (see selectionFor).
|
||||
return serializeImageAdmission(found.agent, async () => {
|
||||
try {
|
||||
await defaults.saveDefaultModelSelection?.(selected)
|
||||
const resolved = await ctx.llm.resolveCallConfig({
|
||||
provider,
|
||||
model,
|
||||
...reasoningEffort === undefined
|
||||
? {}
|
||||
: { reasoningEffort: ReasoningEffortId(reasoningEffort) },
|
||||
})
|
||||
const pendingImage = [...found.agent.inbox.nextTurn, ...found.agent.inbox.nextStep]
|
||||
.some(message => contentHasImage(message.content))
|
||||
if (pendingImage || messagesHaveImage(found.agent.session.deriveMessages())) {
|
||||
const info = await ctx.llm.resolveModelInfo(resolved.provider, resolved.model)
|
||||
if (info.inputModalities !== undefined && !info.inputModalities.includes('image')) {
|
||||
return err(request, {
|
||||
code: 'model-unavailable',
|
||||
message: `Model "${resolved.model}" does not accept image input, but this session already contains images; select an image-capable model.`,
|
||||
details: { provider, model },
|
||||
})
|
||||
}
|
||||
}
|
||||
const selected: ModelSelection = {
|
||||
provider: resolved.provider,
|
||||
model: resolved.model,
|
||||
...resolved.reasoningEffort === undefined
|
||||
? {}
|
||||
: { reasoningEffort: resolved.reasoningEffort },
|
||||
}
|
||||
selectionFor(found.agent).current = selected
|
||||
try {
|
||||
await defaults.saveDefaultModelSelection?.(selected)
|
||||
} catch (error: unknown) {
|
||||
ctx.logger.warn(
|
||||
`api-proxy: the model switch applies to this session but was not saved as the default: ${String(error)}`,
|
||||
)
|
||||
}
|
||||
return ok(request, { selected: { ...selected } })
|
||||
} catch (error: unknown) {
|
||||
ctx.logger.warn(
|
||||
`api-proxy: the model switch applies to this session but was not saved as the default: ${String(error)}`,
|
||||
)
|
||||
return err(request, {
|
||||
code: 'model-unavailable',
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
details: { provider, model },
|
||||
})
|
||||
}
|
||||
return ok(request, { selected: { ...selected } })
|
||||
} catch (error: unknown) {
|
||||
return err(request, {
|
||||
code: 'model-unavailable',
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
details: { provider, model },
|
||||
})
|
||||
}
|
||||
})
|
||||
},
|
||||
|
||||
async rename(request) {
|
||||
@@ -2103,19 +2230,101 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
|
||||
const agent = resolved.agent
|
||||
// The rpcId rides MessageSource into user/message (merge declaration in api/sessions.ts; provisional correlation).
|
||||
const source: MessageSource = { kind: 'user', rpcId: request.rpcId }
|
||||
try {
|
||||
const message: UserMessage = createUserMessage({ content, source })
|
||||
if (mode === 'steer') agent.steer(message)
|
||||
else agent.followup(message)
|
||||
} catch (error: unknown) {
|
||||
// A synchronous throw from steer/followup means disposed or invalid input; surface as agent-busy with the reason attached.
|
||||
return err(request, { code: 'agent-busy', message: 'prompt rejected', details: { reason: String(error) } })
|
||||
const hasImage = content.some(part => part.type === 'image')
|
||||
const admit = async (): Promise<RpcResponse<{ accepted: true }>> => {
|
||||
try {
|
||||
if (hasImage) {
|
||||
const current = selectionFor(agent).current
|
||||
const modelInfo = await ctx.llm.resolveModelInfo(current.provider, current.model)
|
||||
if (modelInfo.inputModalities !== undefined && !modelInfo.inputModalities.includes('image')) {
|
||||
return err(request, {
|
||||
code: 'attachment-error',
|
||||
message: `Model "${current.model}" does not support image input.`,
|
||||
details: { reason: 'MODEL_DOES_NOT_SUPPORT_IMAGES' },
|
||||
})
|
||||
}
|
||||
}
|
||||
const durable = await durablePromptContent(ctx, content)
|
||||
const message: UserMessage = createUserMessage({ content: durable, source })
|
||||
if (mode === 'steer') agent.steer(message)
|
||||
else agent.followup(message)
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof AttachmentError) {
|
||||
return err(request, {
|
||||
code: 'attachment-error',
|
||||
message: error.message,
|
||||
details: { reason: error.code },
|
||||
})
|
||||
}
|
||||
return err(request, {
|
||||
code: 'agent-busy',
|
||||
message: 'prompt rejected',
|
||||
details: { reason: String(error) },
|
||||
})
|
||||
}
|
||||
return ok(request, { accepted: true as const })
|
||||
}
|
||||
return hasImage ? serializeImageAdmission(agent, admit) : admit()
|
||||
},
|
||||
|
||||
async attachment(request) {
|
||||
const { sessionId, attachmentId } = request.payload
|
||||
let state: SessionReadState
|
||||
try {
|
||||
state = await readSessionState(sessionId)
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof SessionNotFound) {
|
||||
return err(request, {
|
||||
code: 'session-not-found',
|
||||
message: error.message,
|
||||
details: { sessionId },
|
||||
})
|
||||
}
|
||||
return err(request, {
|
||||
code: 'internal',
|
||||
message: `attachment authorization unavailable for session "${sessionId}": ${String(error)}`,
|
||||
details: {},
|
||||
})
|
||||
}
|
||||
const ref = referencedImage(state.events, String(attachmentId))
|
||||
if (ref === undefined) {
|
||||
return err(request, {
|
||||
code: 'attachment-error',
|
||||
message: 'Image is not referenced by this session.',
|
||||
details: { reason: 'ATTACHMENT_NOT_REFERENCED' },
|
||||
})
|
||||
}
|
||||
try {
|
||||
const stored = await ctx.attachments.readImage(ref)
|
||||
return ok(request, {
|
||||
attachment: stored.ref,
|
||||
data: Buffer.from(stored.data).toString('base64'),
|
||||
})
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof AttachmentError) {
|
||||
return err(request, {
|
||||
code: 'attachment-error',
|
||||
message: error.message,
|
||||
details: { reason: error.code },
|
||||
})
|
||||
}
|
||||
return err(request, {
|
||||
code: 'internal',
|
||||
message: 'Unable to read image attachment.',
|
||||
details: {},
|
||||
})
|
||||
}
|
||||
return ok(request, { accepted: true as const })
|
||||
},
|
||||
|
||||
updateQueue(request) {
|
||||
const { sessionId, itemId, action } = request.payload
|
||||
if (action.kind === 'edit' && action.content.some(block => block.type !== 'text')) {
|
||||
return Promise.resolve(err(request, {
|
||||
code: 'attachment-error',
|
||||
message: 'queue edits accept text content only',
|
||||
details: { reason: 'QUEUE_EDIT_NON_TEXT' },
|
||||
}))
|
||||
}
|
||||
const agent = ctx.agents.get(sessionId)
|
||||
if (agent !== undefined && hasSubagentOwner(agent.session, agent)) {
|
||||
return Promise.resolve(err(request, subagentOwnershipError(sessionId)))
|
||||
@@ -3164,6 +3373,17 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
|
||||
ctx.on('commands/change', () => {
|
||||
queue.push(frame({ type: 'host/commands-changed' }))
|
||||
}),
|
||||
// The recompose itself registers nothing (it re-parents the agent's
|
||||
// scope onto a standing mount that may already exist), so the
|
||||
// logged selection is the only commit point a client can follow.
|
||||
ctx.on('session/event', (session: Session, event: SessionEvent) => {
|
||||
if (event.type !== 'agent-preset/selected') return
|
||||
queue.push(frame({
|
||||
type: 'host/session-preset-changed',
|
||||
sessionId: session.id,
|
||||
agentPreset: event.data.agentPreset,
|
||||
}))
|
||||
}),
|
||||
ctx.on('settings/document-updated', (ns) => {
|
||||
// The RAW-section event, not the resolved one: a field going from
|
||||
// inherited to overridden leaves the resolved value equal, and a
|
||||
|
||||
Reference in New Issue
Block a user