feat(agent): add voice input and photo attachments to assistant
Wire mic through Whisper-compatible transcriptions on LLM_BASE_URL. Photos upload to MinIO and reach the vision model as base64 image_url parts.
This commit is contained in:
@@ -1,24 +1,11 @@
|
||||
import { z } from "zod";
|
||||
import { apiError, apiJson } from "@/lib/api-handler";
|
||||
import { resolveApiAuth } from "@/lib/api-auth";
|
||||
import { getAssistantPreferences, resolveAssistantSystemPrompt } from "@/lib/assistant-preference";
|
||||
import { isLlmConfigured } from "@/lib/llm";
|
||||
import { clientChatInputSchema } from "@/modules/agent/messages";
|
||||
import { encodeSseEvent } from "@/modules/agent/server/progress";
|
||||
import { runAgentChat } from "@/modules/agent/server/run";
|
||||
|
||||
const chatInput = z.object({
|
||||
stream: z.boolean().optional(),
|
||||
messages: z
|
||||
.array(
|
||||
z.object({
|
||||
role: z.enum(["user", "assistant"]),
|
||||
content: z.string().trim().min(1).max(8000),
|
||||
}),
|
||||
)
|
||||
.min(1)
|
||||
.max(40),
|
||||
});
|
||||
|
||||
export async function POST(request: Request) {
|
||||
const auth = await resolveApiAuth(request);
|
||||
if (!auth?.userId) {
|
||||
@@ -39,7 +26,7 @@ export async function POST(request: Request) {
|
||||
return apiError("Invalid JSON body", 400);
|
||||
}
|
||||
|
||||
const parsed = chatInput.safeParse(body);
|
||||
const parsed = clientChatInputSchema.safeParse(body);
|
||||
if (!parsed.success) {
|
||||
return apiError(parsed.error.issues[0]?.message ?? "Validation error", 400);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import { apiError, apiJson } from "@/lib/api-handler";
|
||||
import { resolveApiAuth } from "@/lib/api-auth";
|
||||
import { getAssistantPreferences } from "@/lib/assistant-preference";
|
||||
import { transcribeAudioFile } from "@/lib/llm/transcribe";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 120;
|
||||
|
||||
const MAX_AUDIO_BYTES = 25 * 1024 * 1024;
|
||||
|
||||
export async function POST(request: Request) {
|
||||
const auth = await resolveApiAuth(request);
|
||||
if (!auth?.userId) {
|
||||
return apiError("Unauthorized", 401);
|
||||
}
|
||||
|
||||
const assistant = await getAssistantPreferences(auth.userId);
|
||||
if (!assistant.enabled) {
|
||||
return apiError("Assistant not enabled", 403);
|
||||
}
|
||||
|
||||
let formData: FormData;
|
||||
try {
|
||||
formData = await request.formData();
|
||||
} catch {
|
||||
return apiError("Invalid multipart body", 400);
|
||||
}
|
||||
|
||||
const file = formData.get("file");
|
||||
if (!(file instanceof File)) {
|
||||
return apiError("No audio file in request", 400);
|
||||
}
|
||||
|
||||
if (!file.type.startsWith("audio/") && file.type !== "video/webm") {
|
||||
return apiError("Only audio recordings are allowed", 415);
|
||||
}
|
||||
|
||||
if (file.size > MAX_AUDIO_BYTES) {
|
||||
return apiError("Recording exceeds 25 MB limit", 413);
|
||||
}
|
||||
|
||||
const filename = file.name.trim() || "recording.wav";
|
||||
|
||||
try {
|
||||
const text = await transcribeAudioFile(file, filename);
|
||||
return apiJson({ text });
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "Transcription failed";
|
||||
return apiError(message, 502);
|
||||
}
|
||||
}
|
||||
@@ -33,7 +33,9 @@ export async function POST(request: Request) {
|
||||
}
|
||||
|
||||
const url = new URL(request.url);
|
||||
const scope = url.searchParams.get("scope") === "notes" ? "notes" : "garden";
|
||||
const scopeParam = url.searchParams.get("scope");
|
||||
const scope =
|
||||
scopeParam === "notes" ? "notes" : scopeParam === "assistant" ? "assistant" : "garden";
|
||||
|
||||
if (!file.type.startsWith("image/")) {
|
||||
return NextResponse.json({ error: "Only image files are allowed" }, { status: 415 });
|
||||
|
||||
Reference in New Issue
Block a user