feat(agent): add voice input and photo attachments to assistant

Wire mic through Whisper-compatible transcriptions on LLM_BASE_URL.

Photos upload to MinIO and reach the vision model as base64 image_url parts.
This commit is contained in:
ginnoir
2026-07-05 02:01:48 -05:00
parent 876a283d47
commit c8db5475d3
20 changed files with 636 additions and 65 deletions
+28
View File
@@ -0,0 +1,28 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import { clientChatInputSchema } from "../../src/modules/agent/messages";
describe("clientChatInputSchema", () => {
it("accepts messages with image attachments", () => {
const parsed = clientChatInputSchema.safeParse({
stream: true,
messages: [
{
role: "user",
content: "Add this appointment to the calendar",
attachments: [{ type: "image", url: "/api/uploads/assistant/house-1/photo.jpg" }],
},
],
});
assert.equal(parsed.success, true);
});
it("rejects empty message content", () => {
const parsed = clientChatInputSchema.safeParse({
messages: [{ role: "user", content: " " }],
});
assert.equal(parsed.success, false);
});
});
+23
View File
@@ -0,0 +1,23 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import { transcribeAudioFile } from "../../src/lib/llm/transcribe";
describe("transcribeAudioFile", () => {
it("returns mock text when llm base url is unset", async () => {
const original = process.env.LLM_BASE_URL;
const originalProvider = process.env.LLM_PROVIDER;
delete process.env.LLM_BASE_URL;
delete process.env.LLM_PROVIDER;
const text = await transcribeAudioFile(
new Blob(["audio"], { type: "audio/wav" }),
"recording.wav",
);
assert.match(text, /milk/i);
if (original === undefined) delete process.env.LLM_BASE_URL;
else process.env.LLM_BASE_URL = original;
if (originalProvider === undefined) delete process.env.LLM_PROVIDER;
else process.env.LLM_PROVIDER = originalProvider;
});
});
+12
View File
@@ -4,6 +4,7 @@ import {
clearAssistantChat,
loadAssistantChat,
saveAssistantChat,
toClientChatMessage,
} from "../../src/modules/agent/assistant-chat-storage";
const storage = new Map<string, string>();
@@ -49,4 +50,15 @@ describe("assistant chat storage", () => {
clearAssistantChat("user-a");
assert.deepEqual(loadAssistantChat("user-a"), []);
});
it("maps stored image messages to client attachments", () => {
const message = toClientChatMessage({
role: "user",
content: "read this",
imageUrl: "/api/uploads/assistant/home/photo.jpg",
});
assert.deepEqual(message.attachments, [
{ type: "image", url: "/api/uploads/assistant/home/photo.jpg" },
]);
});
});
+27
View File
@@ -0,0 +1,27 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import { buildVisionContentParts, textFromMessageContent } from "../../src/lib/llm/content";
describe("llm content helpers", () => {
it("reads plain string content", () => {
assert.equal(textFromMessageContent("hello"), "hello");
});
it("joins text parts from multimodal content", () => {
assert.equal(
textFromMessageContent([
{ type: "text", text: "first" },
{ type: "image_url", image_url: { url: "data:image/png;base64,abc" } },
{ type: "text", text: "second" },
]),
"first\nsecond",
);
});
it("builds vision parts with fallback prompt when text is empty", () => {
const parts = buildVisionContentParts("", ["data:image/jpeg;base64,abc"]);
assert.equal(parts.length, 2);
assert.equal(parts[0]?.type, "text");
assert.equal(parts[1]?.type, "image_url");
});
});