feat(agent): add voice input and photo attachments to assistant

Wire mic through Whisper-compatible transcriptions on LLM_BASE_URL.

Photos upload to MinIO and reach the vision model as base64 image_url parts.
This commit is contained in:
ginnoir
2026-07-05 02:01:48 -05:00
parent 876a283d47
commit c8db5475d3
20 changed files with 636 additions and 65 deletions
+20 -6
View File
@@ -1,9 +1,12 @@
import type { ClientChatMessage } from "./messages";
export type AssistantChatMessage = {
role: "user" | "assistant";
content: string;
imageUrl?: string;
};
const STORAGE_VERSION = "v1";
const STORAGE_VERSION = "v2";
const MAX_MESSAGES = 40;
function storageKey(userId: string) {
@@ -13,11 +16,22 @@ function storageKey(userId: string) {
function isValidMessage(value: unknown): value is AssistantChatMessage {
if (!value || typeof value !== "object") return false;
const row = value as Record<string, unknown>;
return (
(row.role === "user" || row.role === "assistant") &&
typeof row.content === "string" &&
row.content.trim().length > 0
);
if (row.role !== "user" && row.role !== "assistant") return false;
if (typeof row.content !== "string" || row.content.trim().length === 0) return false;
if (row.imageUrl !== undefined && typeof row.imageUrl !== "string") return false;
return true;
}
export function toClientChatMessage(message: AssistantChatMessage): ClientChatMessage {
if (!message.imageUrl) {
return { role: message.role, content: message.content };
}
return {
role: message.role,
content: message.content,
attachments: [{ type: "image", url: message.imageUrl }],
};
}
export function loadAssistantChat(userId: string): AssistantChatMessage[] {