feat(agent): add voice input and photo attachments to assistant
Wire mic through Whisper-compatible transcriptions on LLM_BASE_URL. Photos upload to MinIO and reach the vision model as base64 image_url parts.
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
import { getLlmConfig } from "./config";
|
||||
|
||||
const MAX_AUDIO_BYTES = 25 * 1024 * 1024;
|
||||
|
||||
export async function transcribeAudioFile(file: File | Blob, filename: string): Promise<string> {
|
||||
if (file.size > MAX_AUDIO_BYTES) {
|
||||
throw new Error("Recording exceeds 25 MB limit");
|
||||
}
|
||||
|
||||
const config = getLlmConfig();
|
||||
if (config.provider === "mock" || !config.baseUrl) {
|
||||
return "add milk to the shopping list";
|
||||
}
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("file", file, filename);
|
||||
formData.append("model", "whisper-1");
|
||||
|
||||
const url = `${config.baseUrl.replace(/\/$/, "")}/audio/transcriptions`;
|
||||
const headers: Record<string, string> = {};
|
||||
if (config.apiKey) {
|
||||
headers.Authorization = `Bearer ${config.apiKey}`;
|
||||
}
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: formData,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const detail = await response.text();
|
||||
throw new Error(`Transcription failed (${response.status}): ${detail.slice(0, 400)}`);
|
||||
}
|
||||
|
||||
const contentType = response.headers.get("content-type") ?? "";
|
||||
if (contentType.includes("application/json")) {
|
||||
const payload = (await response.json()) as { text?: string };
|
||||
const text = payload.text?.trim();
|
||||
if (!text) throw new Error("Transcription returned empty text");
|
||||
return text;
|
||||
}
|
||||
|
||||
const text = (await response.text()).trim();
|
||||
if (!text) throw new Error("Transcription returned empty text");
|
||||
return text;
|
||||
}
|
||||
Reference in New Issue
Block a user