feat(agent): add voice input and photo attachments to assistant

Wire mic through Whisper-compatible transcriptions on LLM_BASE_URL.

Photos upload to MinIO and reach the vision model as base64 image_url parts.

src/modules/agent/server/resolve-images.ts

@@ -0,0 +1,35 @@
+import { minioClient, MINIO_BUCKET } from "@/lib/minio";
+
+const UPLOAD_PATH_PREFIX = "/api/uploads/";
+
+function uploadKeyFromUrl(url: string): string | null {
+  if (!url.startsWith(UPLOAD_PATH_PREFIX)) return null;
+  const key = url.slice(UPLOAD_PATH_PREFIX.length);
+  if (!key || key.includes("..")) return null;
+  return key;
+}
+
+export async function resolveAssistantImageDataUrl(url: string): Promise<string> {
+  const key = uploadKeyFromUrl(url);
+  if (!key) {
+    throw new Error("Unsupported image URL");
+  }
+
+  const stream = await minioClient.getObject(MINIO_BUCKET, key);
+  const chunks: Buffer[] = [];
+  for await (const chunk of stream) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array));
+  }
+  const buffer = Buffer.concat(chunks);
+
+  const stat = await minioClient.statObject(MINIO_BUCKET, key);
+  const metaData = stat.metaData as Record<string, string> | undefined;
+  const contentType =
+    metaData?.["content-type"] ?? metaData?.["Content-Type"] ?? "application/octet-stream";
+
+  return `data:${contentType};base64,${buffer.toString("base64")}`;
+}
+
+export async function resolveAssistantImageDataUrls(urls: string[]): Promise<string[]> {
+  return Promise.all(urls.map((url) => resolveAssistantImageDataUrl(url)));
+}

src/modules/agent/server/run.ts

@@ -1,16 +1,14 @@
 import { createLlmClient, type ChatMessage, type LlmClient } from "@/lib/llm";
+import { buildVisionContentParts, textFromMessageContent } from "@/lib/llm/content";
 import { AGENT_SYSTEM_PROMPT, AGENT_TOOLS } from "../tools";
+import type { ClientChatMessage } from "../messages";
 import { describeToolActivity } from "../tool-labels";
 import { createApiToolExecutor, type ToolExecutor } from "../tool-executor";
+import { resolveAssistantImageDataUrls } from "./resolve-images";
 import { thinkingLabel, type AgentProgressEvent } from "./progress";
 
 const MAX_TOOL_ROUNDS = 8;
 
-export type ClientChatMessage = {
-  role: "user" | "assistant";
-  content: string;
-};
-
 export type AgentToolCallSummary = {
   name: string;
   status: number;
@@ -23,6 +21,26 @@ export type AgentChatResult = {
 
 export type AgentProgressHandler = (event: AgentProgressEvent) => void;
 
+async function toLlmUserMessage(message: ClientChatMessage): Promise<ChatMessage> {
+  if (message.role === "assistant") {
+    return { role: "assistant", content: message.content };
+  }
+
+  const imageUrls =
+    message.attachments?.filter((attachment) => attachment.type === "image").map((a) => a.url) ??
+    [];
+
+  if (imageUrls.length === 0) {
+    return { role: "user", content: message.content };
+  }
+
+  const dataUrls = await resolveAssistantImageDataUrls(imageUrls);
+  return {
+    role: "user",
+    content: buildVisionContentParts(message.content, dataUrls),
+  };
+}
+
 export async function runAgentChat(options: {
   messages: ClientChatMessage[];
   request: Request;
@@ -36,15 +54,11 @@ export async function runAgentChat(options: {
   const onProgress = options.onProgress;
   const systemPrompt = options.systemPrompt ?? AGENT_SYSTEM_PROMPT;
 
-  const transcript: ChatMessage[] = [
-    { role: "system", content: systemPrompt },
-    ...options.messages.map(
-      (message): ChatMessage => ({
-        role: message.role,
-        content: message.content,
-      }),
-    ),
-  ];
+  const userMessages = await Promise.all(
+    options.messages.map((message) => toLlmUserMessage(message)),
+  );
+
+  const transcript: ChatMessage[] = [{ role: "system", content: systemPrompt }, ...userMessages];
 
   const toolCalls: AgentToolCallSummary[] = [];
 
@@ -64,7 +78,9 @@ export async function runAgentChat(options: {
       return {
         message: {
           role: "assistant",
-          content: assistantMessage.content?.trim() || "I couldn't generate a response.",
+          content:
+            textFromMessageContent(assistantMessage.content).trim() ||
+            "I couldn't generate a response.",
         },
         toolCalls,
       };