feat(agent): add voice input and photo attachments to assistant

Wire mic through Whisper-compatible transcriptions on LLM_BASE_URL.

Photos upload to MinIO and reach the vision model as base64 image_url parts.
This commit is contained in:
ginnoir
2026-07-05 02:01:48 -05:00
parent 876a283d47
commit c8db5475d3
20 changed files with 636 additions and 65 deletions
+31 -15
View File
@@ -1,16 +1,14 @@
import { createLlmClient, type ChatMessage, type LlmClient } from "@/lib/llm";
import { buildVisionContentParts, textFromMessageContent } from "@/lib/llm/content";
import { AGENT_SYSTEM_PROMPT, AGENT_TOOLS } from "../tools";
import type { ClientChatMessage } from "../messages";
import { describeToolActivity } from "../tool-labels";
import { createApiToolExecutor, type ToolExecutor } from "../tool-executor";
import { resolveAssistantImageDataUrls } from "./resolve-images";
import { thinkingLabel, type AgentProgressEvent } from "./progress";
const MAX_TOOL_ROUNDS = 8;
export type ClientChatMessage = {
role: "user" | "assistant";
content: string;
};
export type AgentToolCallSummary = {
name: string;
status: number;
@@ -23,6 +21,26 @@ export type AgentChatResult = {
export type AgentProgressHandler = (event: AgentProgressEvent) => void;
async function toLlmUserMessage(message: ClientChatMessage): Promise<ChatMessage> {
if (message.role === "assistant") {
return { role: "assistant", content: message.content };
}
const imageUrls =
message.attachments?.filter((attachment) => attachment.type === "image").map((a) => a.url) ??
[];
if (imageUrls.length === 0) {
return { role: "user", content: message.content };
}
const dataUrls = await resolveAssistantImageDataUrls(imageUrls);
return {
role: "user",
content: buildVisionContentParts(message.content, dataUrls),
};
}
export async function runAgentChat(options: {
messages: ClientChatMessage[];
request: Request;
@@ -36,15 +54,11 @@ export async function runAgentChat(options: {
const onProgress = options.onProgress;
const systemPrompt = options.systemPrompt ?? AGENT_SYSTEM_PROMPT;
const transcript: ChatMessage[] = [
{ role: "system", content: systemPrompt },
...options.messages.map(
(message): ChatMessage => ({
role: message.role,
content: message.content,
}),
),
];
const userMessages = await Promise.all(
options.messages.map((message) => toLlmUserMessage(message)),
);
const transcript: ChatMessage[] = [{ role: "system", content: systemPrompt }, ...userMessages];
const toolCalls: AgentToolCallSummary[] = [];
@@ -64,7 +78,9 @@ export async function runAgentChat(options: {
return {
message: {
role: "assistant",
content: assistantMessage.content?.trim() || "I couldn't generate a response.",
content:
textFromMessageContent(assistantMessage.content).trim() ||
"I couldn't generate a response.",
},
toolCalls,
};