feat(agent): add voice input and photo attachments to assistant
Wire mic through Whisper-compatible transcriptions on LLM_BASE_URL. Photos upload to MinIO and reach the vision model as base64 image_url parts.
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
import { minioClient, MINIO_BUCKET } from "@/lib/minio";
|
||||
|
||||
const UPLOAD_PATH_PREFIX = "/api/uploads/";
|
||||
|
||||
function uploadKeyFromUrl(url: string): string | null {
|
||||
if (!url.startsWith(UPLOAD_PATH_PREFIX)) return null;
|
||||
const key = url.slice(UPLOAD_PATH_PREFIX.length);
|
||||
if (!key || key.includes("..")) return null;
|
||||
return key;
|
||||
}
|
||||
|
||||
export async function resolveAssistantImageDataUrl(url: string): Promise<string> {
|
||||
const key = uploadKeyFromUrl(url);
|
||||
if (!key) {
|
||||
throw new Error("Unsupported image URL");
|
||||
}
|
||||
|
||||
const stream = await minioClient.getObject(MINIO_BUCKET, key);
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array));
|
||||
}
|
||||
const buffer = Buffer.concat(chunks);
|
||||
|
||||
const stat = await minioClient.statObject(MINIO_BUCKET, key);
|
||||
const metaData = stat.metaData as Record<string, string> | undefined;
|
||||
const contentType =
|
||||
metaData?.["content-type"] ?? metaData?.["Content-Type"] ?? "application/octet-stream";
|
||||
|
||||
return `data:${contentType};base64,${buffer.toString("base64")}`;
|
||||
}
|
||||
|
||||
export async function resolveAssistantImageDataUrls(urls: string[]): Promise<string[]> {
|
||||
return Promise.all(urls.map((url) => resolveAssistantImageDataUrl(url)));
|
||||
}
|
||||
Reference in New Issue
Block a user