feat(agent): add voice input and photo attachments to assistant

Wire mic through Whisper-compatible transcriptions on LLM_BASE_URL. Photos upload to MinIO and reach the vision model as base64 image_url parts.
2026-07-05 02:01:48 -05:00
parent 876a283d47
commit c8db5475d3
20 changed files with 636 additions and 65 deletions
@@ -1,7 +1,7 @@
 "use client";

 import { useEffect, useRef, useState } from "react";
-import { Loader2, Send } from "lucide-react";
+import { ImagePlus, Loader2, Mic, Send, Square } from "lucide-react";
 import { Button } from "@/components/ui/button";
 import { Input } from "@/components/ui/input";
 import { consumeAgentChatStream } from "../assistant-chat-stream";
@@ -9,8 +9,10 @@ import {
  clearAssistantChat,
  loadAssistantChat,
  saveAssistantChat,
+  toClientChatMessage,
  type AssistantChatMessage,
 } from "../assistant-chat-storage";
+import { useVoiceInput } from "./use-voice-input";

 type Props = {
  configured: boolean;
@@ -18,14 +20,42 @@ type Props = {
  assistantName: string;
 };

+type PendingImage = {
+  url: string;
+};
+
+async function uploadAssistantImage(file: File): Promise<string> {
+  const formData = new FormData();
+  formData.append("file", file);
+  const response = await fetch("/api/uploads?scope=assistant", { method: "POST", body: formData });
+  if (!response.ok) {
+    const payload = (await response.json().catch(() => null)) as { error?: string } | null;
+    throw new Error(payload?.error ?? "Image upload failed");
+  }
+  const payload = (await response.json()) as { url: string };
+  return payload.url;
+}
+
 export function AssistantPanel({ configured, userId, assistantName }: Props) {
  const [messages, setMessages] = useState<AssistantChatMessage[]>(() => loadAssistantChat(userId));
  const [input, setInput] = useState("");
+  const [pendingImage, setPendingImage] = useState<PendingImage | null>(null);
+  const [uploadingImage, setUploadingImage] = useState(false);
  const [error, setError] = useState<string | null>(null);
  const [isPending, setIsPending] = useState(false);
  const [activityLabel, setActivityLabel] = useState<string | null>(null);
  const listRef = useRef<HTMLDivElement>(null);
  const abortRef = useRef<AbortController | null>(null);
+  const imageInputRef = useRef<HTMLInputElement>(null);
+
+  const { state: voiceState, toggleRecording } = useVoiceInput({
+    disabled: isPending || uploadingImage,
+    onTranscript: (text) => {
+      setInput((current) => (current.trim() ? `${current.trim()} ${text}` : text));
+      setError(null);
+    },
+    onError: (message) => setError(message),
+  });

  useEffect(() => {
    saveAssistantChat(userId, messages);
@@ -47,22 +77,50 @@ export function AssistantPanel({ configured, userId, assistantName }: Props) {
  function clearChat() {
    abortRef.current?.abort();
    setMessages([]);
+    setInput("");
+    setPendingImage(null);
    setError(null);
    setActivityLabel(null);
    setIsPending(false);
    clearAssistantChat(userId);
  }

+  async function handleImageSelect(event: React.ChangeEvent<HTMLInputElement>) {
+    const file = event.target.files?.[0];
+    event.target.value = "";
+    if (!file || isPending || uploadingImage) return;
+
+    setUploadingImage(true);
+    setError(null);
+    try {
+      const url = await uploadAssistantImage(file);
+      setPendingImage({ url });
+    } catch (err) {
+      setError(err instanceof Error ? err.message : "Image upload failed");
+    } finally {
+      setUploadingImage(false);
+    }
+  }
+
  async function sendMessage() {
    const text = input.trim();
-    if (!text || isPending) return;
+    const hasImage = pendingImage !== null;
+    if ((!text && !hasImage) || isPending || voiceState !== "idle") return;

-    const nextMessages: AssistantChatMessage[] = [...messages, { role: "user", content: text }];
+    const content = text || "Help me with this image.";
+    const userMessage: AssistantChatMessage = {
+      role: "user",
+      content,
+      ...(pendingImage ? { imageUrl: pendingImage.url } : {}),
+    };
+
+    const nextMessages: AssistantChatMessage[] = [...messages, userMessage];
    setInput("");
+    setPendingImage(null);
    setError(null);
    setMessages(nextMessages);
    setIsPending(true);
-    setActivityLabel("Understanding your request…");
+    setActivityLabel(hasImage ? "Reading your photo…" : "Understanding your request…");
    scrollToBottom();

    abortRef.current?.abort();
@@ -73,7 +131,10 @@ export function AssistantPanel({ configured, userId, assistantName }: Props) {
      const response = await fetch("/api/agent/chat", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ messages: nextMessages, stream: true }),
+        body: JSON.stringify({
+          messages: nextMessages.map(toClientChatMessage),
+          stream: true,
+        }),
        signal: controller.signal,
      });

@@ -88,7 +149,10 @@ export function AssistantPanel({ configured, userId, assistantName }: Props) {
        throw new Error("Assistant returned an empty response");
      }

-      setMessages((current) => [...current, result.message]);
+      setMessages((current) => [
+        ...current,
+        { role: "assistant", content: result.message.content },
+      ]);
      scrollToBottom();
    } catch (err) {
      if (err instanceof Error && err.name === "AbortError") return;
@@ -101,13 +165,19 @@ export function AssistantPanel({ configured, userId, assistantName }: Props) {
  }

  const showEmptyState = messages.length === 0 && !isPending;
+  const inputDisabled = isPending || voiceState === "transcribing" || uploadingImage;
+  const canSend =
+    !isPending &&
+    voiceState === "idle" &&
+    !uploadingImage &&
+    (input.trim().length > 0 || pendingImage !== null);

  return (
    <div className="flex min-h-0 flex-1 flex-col gap-3">
      <div className="flex items-start justify-between gap-3">
        <p className="muted min-w-0 text-[12px] leading-relaxed">
          {configured
-            ? "Ask me to update lists, calendar, notes, or journal."
+            ? "Type, talk, or send a photo — I can update lists, calendar, notes, and more."
            : "Mock provider active — set LLM_BASE_URL for your homelab model."}
        </p>
        {messages.length > 0 ? (
@@ -129,8 +199,8 @@ export function AssistantPanel({ configured, userId, assistantName }: Props) {
      >
        {showEmptyState ? (
          <p className="muted text-[12px]">
-            Try &quot;add milk to the shopping list&quot; or &quot;what&apos;s on the calendar this
-            week?&quot;
+            Try &quot;add milk to the shopping list&quot;, tap the mic, or attach a photo of an
+            appointment card.
          </p>
        ) : (
          <div className="grid gap-2">
@@ -147,6 +217,14 @@ export function AssistantPanel({ configured, userId, assistantName }: Props) {
                <div className="eyebrow mb-0.5 text-[10px]">
                  {message.role === "user" ? "You" : assistantName}
                </div>
+                {message.imageUrl ? (
+                  // User-uploaded assistant attachment preview
+                  <img
+                    src={message.imageUrl}
+                    alt=""
+                    className="mb-2 max-h-40 w-full rounded-md object-contain"
+                  />
+                ) : null}
                {message.content}
              </div>
            ))}
@@ -169,24 +247,87 @@ export function AssistantPanel({ configured, userId, assistantName }: Props) {
        )}
      </div>

+      {pendingImage ? (
+        <div className="flex items-center gap-2 rounded-[var(--r-md)] border-[0.5px] bg-[var(--shade)] p-2">
+          <img
+            src={pendingImage.url}
+            alt=""
+            className="max-h-16 max-w-[40%] rounded-md object-contain"
+          />
+          <div className="min-w-0 flex-1 text-[11px] text-muted-foreground">Photo attached</div>
+          <button
+            type="button"
+            className="text-[11px] text-muted-foreground hover:text-foreground"
+            onClick={() => setPendingImage(null)}
+            disabled={inputDisabled}
+          >
+            Remove
+          </button>
+        </div>
+      ) : null}
+
      {error ? <p className="text-[12px] text-destructive">{error}</p> : null}

      <form
        className="flex gap-2"
        onSubmit={(event) => {
          event.preventDefault();
-          sendMessage();
+          void sendMessage();
        }}
      >
+        <input
+          ref={imageInputRef}
+          type="file"
+          accept="image/*"
+          className="hidden"
+          onChange={(event) => void handleImageSelect(event)}
+        />
+        <Button
+          type="button"
+          size="sm"
+          variant="outline"
+          disabled={inputDisabled}
+          aria-label="Attach photo"
+          onClick={() => imageInputRef.current?.click()}
+        >
+          {uploadingImage ? (
+            <Loader2 className="size-4 animate-spin" />
+          ) : (
+            <ImagePlus className="size-4" />
+          )}
+        </Button>
+        <Button
+          type="button"
+          size="sm"
+          variant={voiceState === "recording" ? "destructive" : "outline"}
+          disabled={inputDisabled}
+          aria-label={voiceState === "recording" ? "Stop recording" : "Record voice message"}
+          aria-pressed={voiceState === "recording"}
+          onClick={() => void toggleRecording()}
+        >
+          {voiceState === "transcribing" ? (
+            <Loader2 className="size-4 animate-spin" />
+          ) : voiceState === "recording" ? (
+            <Square className="size-4" />
+          ) : (
+            <Mic className="size-4" />
+          )}
+        </Button>
        <Input
          value={input}
          onChange={(event) => setInput(event.target.value)}
-          placeholder={`Ask ${assistantName}…`}
-          disabled={isPending}
+          placeholder={
+            voiceState === "recording"
+              ? "Listening… tap mic to stop"
+              : voiceState === "transcribing"
+                ? "Transcribing…"
+                : `Ask ${assistantName}…`
+          }
+          disabled={inputDisabled}
          aria-label={`Message for ${assistantName}`}
-          className="h-9"
+          className="h-9 min-w-0 flex-1"
        />
-        <Button type="submit" size="sm" disabled={isPending || !input.trim()} aria-label="Send">
+        <Button type="submit" size="sm" disabled={!canSend} aria-label="Send">
          {isPending ? <Loader2 className="size-4 animate-spin" /> : <Send className="size-4" />}
        </Button>
      </form>