feat(agent): add voice input and photo attachments to assistant

Wire mic through Whisper-compatible transcriptions on LLM_BASE_URL.

Photos upload to MinIO and reach the vision model as base64 image_url parts.
This commit is contained in:
ginnoir
2026-07-05 02:01:48 -05:00
parent 876a283d47
commit c8db5475d3
20 changed files with 636 additions and 65 deletions
+51
View File
@@ -0,0 +1,51 @@
import { apiError, apiJson } from "@/lib/api-handler";
import { resolveApiAuth } from "@/lib/api-auth";
import { getAssistantPreferences } from "@/lib/assistant-preference";
import { transcribeAudioFile } from "@/lib/llm/transcribe";
export const runtime = "nodejs";
export const maxDuration = 120;
const MAX_AUDIO_BYTES = 25 * 1024 * 1024;
export async function POST(request: Request) {
const auth = await resolveApiAuth(request);
if (!auth?.userId) {
return apiError("Unauthorized", 401);
}
const assistant = await getAssistantPreferences(auth.userId);
if (!assistant.enabled) {
return apiError("Assistant not enabled", 403);
}
let formData: FormData;
try {
formData = await request.formData();
} catch {
return apiError("Invalid multipart body", 400);
}
const file = formData.get("file");
if (!(file instanceof File)) {
return apiError("No audio file in request", 400);
}
if (!file.type.startsWith("audio/") && file.type !== "video/webm") {
return apiError("Only audio recordings are allowed", 415);
}
if (file.size > MAX_AUDIO_BYTES) {
return apiError("Recording exceeds 25 MB limit", 413);
}
const filename = file.name.trim() || "recording.wav";
try {
const text = await transcribeAudioFile(file, filename);
return apiJson({ text });
} catch (err) {
const message = err instanceof Error ? err.message : "Transcription failed";
return apiError(message, 502);
}
}