feat(agent): add voice input and photo attachments to assistant
Wire mic through Whisper-compatible transcriptions on LLM_BASE_URL. Photos upload to MinIO and reach the vision model as base64 image_url parts.
This commit is contained in:
@@ -0,0 +1,28 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import { clientChatInputSchema } from "../../src/modules/agent/messages";
|
||||
|
||||
describe("clientChatInputSchema", () => {
|
||||
it("accepts messages with image attachments", () => {
|
||||
const parsed = clientChatInputSchema.safeParse({
|
||||
stream: true,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: "Add this appointment to the calendar",
|
||||
attachments: [{ type: "image", url: "/api/uploads/assistant/house-1/photo.jpg" }],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
assert.equal(parsed.success, true);
|
||||
});
|
||||
|
||||
it("rejects empty message content", () => {
|
||||
const parsed = clientChatInputSchema.safeParse({
|
||||
messages: [{ role: "user", content: " " }],
|
||||
});
|
||||
|
||||
assert.equal(parsed.success, false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,23 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import { transcribeAudioFile } from "../../src/lib/llm/transcribe";
|
||||
|
||||
describe("transcribeAudioFile", () => {
|
||||
it("returns mock text when llm base url is unset", async () => {
|
||||
const original = process.env.LLM_BASE_URL;
|
||||
const originalProvider = process.env.LLM_PROVIDER;
|
||||
delete process.env.LLM_BASE_URL;
|
||||
delete process.env.LLM_PROVIDER;
|
||||
|
||||
const text = await transcribeAudioFile(
|
||||
new Blob(["audio"], { type: "audio/wav" }),
|
||||
"recording.wav",
|
||||
);
|
||||
assert.match(text, /milk/i);
|
||||
|
||||
if (original === undefined) delete process.env.LLM_BASE_URL;
|
||||
else process.env.LLM_BASE_URL = original;
|
||||
if (originalProvider === undefined) delete process.env.LLM_PROVIDER;
|
||||
else process.env.LLM_PROVIDER = originalProvider;
|
||||
});
|
||||
});
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
clearAssistantChat,
|
||||
loadAssistantChat,
|
||||
saveAssistantChat,
|
||||
toClientChatMessage,
|
||||
} from "../../src/modules/agent/assistant-chat-storage";
|
||||
|
||||
const storage = new Map<string, string>();
|
||||
@@ -49,4 +50,15 @@ describe("assistant chat storage", () => {
|
||||
clearAssistantChat("user-a");
|
||||
assert.deepEqual(loadAssistantChat("user-a"), []);
|
||||
});
|
||||
|
||||
it("maps stored image messages to client attachments", () => {
|
||||
const message = toClientChatMessage({
|
||||
role: "user",
|
||||
content: "read this",
|
||||
imageUrl: "/api/uploads/assistant/home/photo.jpg",
|
||||
});
|
||||
assert.deepEqual(message.attachments, [
|
||||
{ type: "image", url: "/api/uploads/assistant/home/photo.jpg" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import { buildVisionContentParts, textFromMessageContent } from "../../src/lib/llm/content";
|
||||
|
||||
describe("llm content helpers", () => {
|
||||
it("reads plain string content", () => {
|
||||
assert.equal(textFromMessageContent("hello"), "hello");
|
||||
});
|
||||
|
||||
it("joins text parts from multimodal content", () => {
|
||||
assert.equal(
|
||||
textFromMessageContent([
|
||||
{ type: "text", text: "first" },
|
||||
{ type: "image_url", image_url: { url: "data:image/png;base64,abc" } },
|
||||
{ type: "text", text: "second" },
|
||||
]),
|
||||
"first\nsecond",
|
||||
);
|
||||
});
|
||||
|
||||
it("builds vision parts with fallback prompt when text is empty", () => {
|
||||
const parts = buildVisionContentParts("", ["data:image/jpeg;base64,abc"]);
|
||||
assert.equal(parts.length, 2);
|
||||
assert.equal(parts[0]?.type, "text");
|
||||
assert.equal(parts[1]?.type, "image_url");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user