Compare commits

..
23 Commits
Author SHA1 Message Date
ginnoir ea371b8085 chore: release v0.6.2
CI / checks (push) Has been skipped
Release Image / build-and-push (push) Successful in 9m6s
2026-07-08 20:46:44 -05:00
ginnoir c8a7160ef2 chore: ignore local worktrees in checks 2026-07-08 20:46:02 -05:00
ginnoir 49732b02f3 fix(agent): move route selector to settings 2026-07-08 19:58:48 -05:00
ginnoir ec3f96dab1 feat(agent): let users choose model route 2026-07-08 19:50:46 -05:00
ginnoir b279f16ba0 feat(agent): refresh assistant model catalog 2026-07-08 19:37:17 -05:00
ginnoir 4dc04b21d6 fix(agent): hide unrelated models for alias fallback 2026-07-08 19:07:57 -05:00
ginnoir bb679d02be fix(agent): improve model selector legibility 2026-07-08 18:59:41 -05:00
ginnoir 716bca0fcb fix(agent): use native model selector 2026-07-08 18:52:19 -05:00
ginnoir 1092941c45 test: stabilize assistant e2e 2026-07-08 17:48:06 -05:00
ginnoir 4c82d551ea feat(agent): add assistant model selector 2026-07-08 16:46:53 -05:00
ginnoir 27db599444 feat(agent): expose available assistant models 2026-07-08 16:43:22 -05:00
ginnoir 3983c30fe1 fix(agent): reject padded assistant model ids 2026-07-08 16:39:01 -05:00
ginnoir 9b7a04431c feat(agent): route chat through selected model 2026-07-08 16:34:22 -05:00
ginnoir bf7e07ead9 feat(agent): persist assistant model preference 2026-07-08 16:24:01 -05:00
ginnoir 876e72671a fix(agent): validate assistant model requests 2026-07-08 16:15:19 -05:00
ginnoir 0bf63cf8a8 feat(agent): add llm model discovery helper 2026-07-08 16:11:22 -05:00
ginnoir eb8e562565 docs: plan assistant model selector 2026-07-08 16:05:28 -05:00
ginnoir 1a58ef993e docs: specify assistant model selector 2026-07-08 15:58:48 -05:00
ginnoir 7c6b8d7c37 ci: install docker buildx plugin for gitea release build
CI / checks (push) Failing after 3s
Debian docker.io ships no buildx plugin, so the registry-cache

build failed when the runner CLI fell back to it. Install

Docker's official CLI plus buildx plugin instead.
2026-07-05 02:09:41 -05:00
ginnoir 3a5c6d056c chore: release v0.6.1
CI / checks (push) Has been skipped
Release Image / build-and-push (push) Failing after 45s
2026-07-05 02:02:07 -05:00
ginnoir c8db5475d3 feat(agent): add voice input and photo attachments to assistant
Wire mic through Whisper-compatible transcriptions on LLM_BASE_URL.

Photos upload to MinIO and reach the vision model as base64 image_url parts.
2026-07-05 02:01:48 -05:00
ginnoir 876a283d47 ci: faster pipelines with checks-only ci and registry build cache
CI / checks (push) Failing after 2m8s
Drop duplicate Next.js builds from CI, skip release commits, and cancel stale runs.

Cache pnpm on runners and use buildx registry cache for releases.
2026-07-04 23:54:53 -05:00
ginnoir c80070f5c3 ci: skip workflow on docs-only changes
CI / checks (push) Failing after 2m15s
CI / build (push) Failing after 12m59s
Ignore pushes and PRs that only touch markdown or docs/ so the runner queue stays clear.
2026-07-04 23:40:10 -05:00
47 changed files with 2827 additions and 151 deletions
+1
View File
@@ -1,5 +1,6 @@
node_modules
.next
.worktrees
.git
deploy
docs
+2
View File
@@ -49,6 +49,8 @@ OPENPLANTBOOK_CLIENT_SECRET=
# LLM assistant (OpenAI-compatible — Ollama, vLLM, LiteLLM, etc.)
# Leave LLM_BASE_URL unset to use the built-in mock provider (CI / local without a model).
# Voice input uses POST {LLM_BASE_URL}/audio/transcriptions (Whisper-compatible).
# Photo messages use vision via the same chat/completions endpoint.
LLM_PROVIDER=openai
LLM_BASE_URL=
LLM_API_KEY=
+13 -20
View File
@@ -3,10 +3,21 @@ name: CI
on:
push:
branches: [main]
paths-ignore:
- "**/*.md"
- "docs/**"
pull_request:
paths-ignore:
- "**/*.md"
- "docs/**"
concurrency:
group: ci-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
checks:
if: "github.event_name == 'pull_request' || !startsWith(github.event.head_commit.message, 'chore: release')"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
@@ -15,6 +26,8 @@ jobs:
run: |
corepack enable
corepack prepare pnpm@10.33.3 --activate
mkdir -p /pnpm-store
pnpm config set store-dir /pnpm-store
- name: Install dependencies
run: pnpm install --frozen-lockfile
@@ -27,23 +40,3 @@ jobs:
- name: Format check
run: pnpm format:check
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Enable pnpm
run: |
corepack enable
corepack prepare pnpm@10.33.3 --activate
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Build
run: pnpm build
env:
DATABASE_URL: postgres://ci_user:ci_password@localhost:5432/ci_database
AUTH_SECRET: ci-auth-secret-for-build
NEXT_PUBLIC_APP_URL: http://localhost:3000
+46 -12
View File
@@ -9,6 +9,10 @@ on:
- "v*"
workflow_dispatch:
concurrency:
group: release-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
jobs:
build-and-push:
runs-on: ubuntu-latest
@@ -27,8 +31,31 @@ jobs:
echo "image=registry.ginnoir.com/ginnoir/famapp"
} >> "$GITHUB_OUTPUT"
- name: Install docker
run: apt-get update -qq && apt-get install -y -qq docker.io
- name: Ensure docker CLI + buildx
run: |
# Debian's docker.io package ships no buildx plugin, which the
# registry-cache build below requires. Install Docker's official
# CLI + buildx plugin so the build works regardless of what the
# runner image happens to provide.
if docker buildx version >/dev/null 2>&1; then
echo "docker + buildx already available"
docker version
docker buildx version
exit 0
fi
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq
apt-get install -y -qq ca-certificates curl gnupg
install -m 0755 -d /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
chmod a+r /etc/apt/keyrings/docker.asc
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \
> /etc/apt/sources.list.d/docker.list
apt-get update -qq
apt-get install -y -qq docker-ce-cli docker-buildx-plugin
docker version
docker buildx version
- name: Login to registry
run: |
@@ -36,16 +63,23 @@ jobs:
--username "${{ secrets.REGISTRY_PUSH_USERNAME }}" \
--password-stdin
- name: Build image
- name: Set up buildx
run: |
docker build \
-t "${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.version }}" \
-t "${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.major_minor }}" \
-t "${{ steps.meta.outputs.image }}:latest" \
.
docker buildx create --name famapp-builder --use 2>/dev/null || docker buildx use famapp-builder
docker buildx inspect --bootstrap
- name: Push image
- name: Build and push image
env:
IMAGE: ${{ steps.meta.outputs.image }}
VERSION: ${{ steps.meta.outputs.version }}
MAJOR_MINOR: ${{ steps.meta.outputs.major_minor }}
CACHE: registry.ginnoir.com/ginnoir/famapp:buildcache
run: |
docker push "${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.version }}"
docker push "${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.major_minor }}"
docker push "${{ steps.meta.outputs.image }}:latest"
docker buildx build \
--push \
--tag "${IMAGE}:${VERSION}" \
--tag "${IMAGE}:${MAJOR_MINOR}" \
--tag "${IMAGE}:latest" \
--cache-from "type=registry,ref=${CACHE}" \
--cache-to "type=registry,ref=${CACHE},mode=max" \
.
+11 -30
View File
@@ -3,10 +3,21 @@ name: CI
on:
push:
branches: [main]
paths-ignore:
- "**/*.md"
- "docs/**"
pull_request:
paths-ignore:
- "**/*.md"
- "docs/**"
concurrency:
group: ci-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
checks:
if: "github.event_name == 'pull_request' || !startsWith(github.event.head_commit.message, 'chore: release')"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
@@ -27,33 +38,3 @@ jobs:
- run: pnpm lint
- run: pnpm format:check
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 10.33.3
- uses: actions/setup-node@v4
with:
node-version: 24
cache: pnpm
- run: pnpm install --frozen-lockfile
- uses: actions/cache@v4
with:
path: .next/cache
key: ${{ runner.os }}-nextjs-${{ hashFiles('pnpm-lock.yaml') }}-${{ hashFiles('src/**/*.ts', 'src/**/*.tsx', 'src/**/*.css') }}
restore-keys: |
${{ runner.os }}-nextjs-${{ hashFiles('pnpm-lock.yaml') }}-
${{ runner.os }}-nextjs-
- run: pnpm build
env:
DATABASE_URL: postgres://placeholder:placeholder@localhost:5432/placeholder
AUTH_SECRET: ci-placeholder-secret
NEXT_PUBLIC_APP_URL: http://localhost:3000
+32
View File
@@ -1,5 +1,37 @@
# Changelog
## [0.6.2](https://github.com/ginnoir/famapp/compare/v0.6.1...v0.6.2) (2026-07-09)
### Features
- **agent:** add assistant model selector ([4c82d55](https://github.com/ginnoir/famapp/commit/4c82d551ea1f0c8091a3ffc31935a8d22855bbc0))
- **agent:** add llm model discovery helper ([0bf63cf](https://github.com/ginnoir/famapp/commit/0bf63cf8a83a1adca58fa9e8bf42f16dac60687b))
- **agent:** expose available assistant models ([27db599](https://github.com/ginnoir/famapp/commit/27db599444fc0bb80d5a1650e2f5b5ba4f695c8e))
- **agent:** let users choose model route ([ec3f96d](https://github.com/ginnoir/famapp/commit/ec3f96dab170fb2333780438bdec3628cfc58116))
- **agent:** persist assistant model preference ([bf7e07e](https://github.com/ginnoir/famapp/commit/bf7e07ead91dd8c428f06551afb2113ce838cc82))
- **agent:** refresh assistant model catalog ([b279f16](https://github.com/ginnoir/famapp/commit/b279f16ba0e1537d2752d30bf5e565f4ede400af))
- **agent:** route chat through selected model ([9b7a044](https://github.com/ginnoir/famapp/commit/9b7a04431cb12609cbcd766e05fbc411599e0b34))
### Bug Fixes
- **agent:** hide unrelated models for alias fallback ([4dc04b2](https://github.com/ginnoir/famapp/commit/4dc04b21d63aaef7c773f238602f06edbbf03509))
- **agent:** improve model selector legibility ([bb679d0](https://github.com/ginnoir/famapp/commit/bb679d02be28440acbc860146cbcb3809224aa09))
- **agent:** move route selector to settings ([49732b0](https://github.com/ginnoir/famapp/commit/49732b02f31cbef57e6cd3b577bee6dbd998e029))
- **agent:** reject padded assistant model ids ([3983c30](https://github.com/ginnoir/famapp/commit/3983c30fe1b8a324f7d5826df0b24506fe054e50))
- **agent:** use native model selector ([716bca0](https://github.com/ginnoir/famapp/commit/716bca0fcba8f2026c814f668bf873cf61a0ad16))
- **agent:** validate assistant model requests ([876e726](https://github.com/ginnoir/famapp/commit/876e72671a0b82b579a9783eb86f452a4a026a52))
### Documentation
- plan assistant model selector ([eb8e562](https://github.com/ginnoir/famapp/commit/eb8e5625656a1e3fe97e19b0bb2514660cf0c5f6))
- specify assistant model selector ([1a58ef9](https://github.com/ginnoir/famapp/commit/1a58ef993e62ea2855d7d5aafc17446a0e91c33c))
## [0.6.1](https://github.com/ginnoir/famapp/compare/v0.6.0...v0.6.1) (2026-07-05)
### Features
- **agent:** add voice input and photo attachments to assistant ([c8db547](https://github.com/ginnoir/famapp/commit/c8db5475d314b239246e4786e362bb044992d660))
## [0.6.0](https://github.com/ginnoir/famapp/compare/v0.5.6...v0.6.0) (2026-07-05)
### Features
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,151 @@
# Assistant model selector design
Date: 2026-07-08
Status: approved for planning
## Context
The AI assistant chat currently uses one environment-configured model through `LLM_MODEL`.
The chat UI posts messages to `/api/agent/chat`, and the server creates the OpenAI-compatible
client without any request-time model choice.
ginnoir wants a model selector in the assistant chat. The selector should discover available
models from the configured OpenAI-compatible provider and save the selected model as the user's
default.
## Goals
- Show a compact model selector in the assistant chat panel.
- Discover models from the provider's `/models` endpoint server-side.
- Persist the selected model per user so it works across browser sessions and devices.
- Keep `LLM_MODEL` as the fallback when discovery fails, no model is saved, or the saved model
is no longer available.
- Preserve mock-provider behavior in CI and local setups without `LLM_BASE_URL`.
## Non-goals
- Model hosting, training, or fine-tuning.
- Multiple LLM providers in the same deployment.
- Per-message experimental settings beyond selecting the model ID.
- Exposing arbitrary browser-supplied model IDs to the provider.
## User experience
When the assistant bubble opens, the chat panel loads available model IDs from the server.
The selector appears near the existing assistant status and clear-chat controls. It should be
visible but compact enough not to reduce the message area materially.
Changing the selector immediately saves the user's default model. The next message uses that
model, and future assistant sessions start with the saved selection when it is still available.
If model discovery fails, the panel remains usable with the `LLM_MODEL` fallback and shows a
muted status that model discovery is unavailable. If the saved model has disappeared from the
provider, the server and UI fall back to `LLM_MODEL`.
## Architecture
### Configuration
`getLlmConfig()` remains the source for provider, base URL, API key, and fallback model.
No additional allowlist environment variable is required because model IDs come from the
provider's OpenAI-compatible `/models` endpoint.
### Persistence
Add nullable `assistant_model` storage to `users`.
The existing assistant preference loader should return:
- assistant enabled state
- assistant display name
- assistant system prompt
- saved assistant model ID
The value is nullable. `null` means "use the environment fallback model."
### Model discovery API
Add `GET /api/agent/models`.
Behavior:
- Require the same authenticated user/session or API auth shape as the chat endpoint.
- Require assistant access to be enabled for the user.
- If `LLM_BASE_URL` is missing or the provider is mock, return the fallback model as the only
available model.
- Fetch `${LLM_BASE_URL}/models` with `Authorization: Bearer ${LLM_API_KEY}` when configured.
- Accept OpenAI-style payloads with a top-level `data` array.
- Normalize each model to `{ id: string, label: string }`, using the ID as the label.
- Deduplicate, sort consistently, and include the fallback model if the provider omitted it.
- If discovery fails, return the fallback model plus a degraded status instead of failing the
chat UI.
The response should include enough metadata for the UI:
```json
{
"models": [{ "id": "llama3.2", "label": "llama3.2" }],
"selectedModel": "llama3.2",
"fallbackModel": "llama3.2",
"degraded": false
}
```
### Saving the default model
Add a server action for updating the user's assistant model, matching the existing assistant
settings actions. The update path must:
- Accept a model ID string or `null`.
- Validate length and basic shape before touching the database.
- Validate the requested model against the current discovered model list.
- Save `null` when the selected model matches the fallback so `LLM_MODEL` changes take effect for
users who have not chosen a non-default model.
- Revalidate assistant surfaces after saving.
### Chat request flow
Extend `clientChatInputSchema` with optional `model`.
The chat route should:
- Parse `model` from the request body.
- Resolve the effective model from request model, saved user default, and fallback model.
- Validate request model and saved user default against discovered models.
- Reject an invalid request model with `400`.
- Silently fall back when the saved user default is no longer available.
- Pass the effective model into `runAgentChat`.
`runAgentChat` should accept an optional model override. `createLlmClient` should support an
override object or equivalent path that replaces only the model while preserving the configured
provider, base URL, and API key.
## Error handling
- Missing auth: `401`.
- Assistant disabled: `403`.
- Invalid posted model: `400`.
- Provider `/models` failure: return fallback model from the model-discovery API with
`degraded: true`; do not block chat startup.
- LLM completion failure after a valid model is selected: keep the existing chat error behavior.
## Testing
Unit tests:
- Model discovery normalizes OpenAI-compatible `/models` responses.
- Discovery falls back to `LLM_MODEL` for mock or failed provider states.
- Chat input schema accepts an optional valid model string and rejects invalid shapes.
- Chat route rejects a model not returned by discovery.
- `runAgentChat` passes the effective model override into the LLM client path.
E2E smoke:
- After assistant opt-in, opening the assistant shows the model selector.
- Sending a message still renders the user message and assistant response with the mock provider.
## Rollout notes
This is additive. Existing deployments without a provider `/models` endpoint continue to use
`LLM_MODEL`. The database migration is nullable, so existing users keep current behavior until
they choose a model.
+1
View File
@@ -0,0 +1 @@
ALTER TABLE "users" ADD COLUMN "assistant_model" text;
+5
View File
@@ -0,0 +1,5 @@
ALTER TABLE "users" ADD COLUMN "assistant_model_route" text;--> statement-breakpoint
UPDATE "users"
SET "assistant_model_route" = "assistant_model",
"assistant_model" = NULL
WHERE "assistant_model" IN ('auto', 'uncensored');
+15 -1
View File
@@ -176,6 +176,20 @@
"when": 1780394000000,
"tag": "0024_user_assistant_customization",
"breakpoints": true
},
{
"idx": 25,
"version": "7",
"when": 1783560000000,
"tag": "0025_assistant_model",
"breakpoints": true
},
{
"idx": 26,
"version": "7",
"when": 1783561000000,
"tag": "0026_assistant_model_route",
"breakpoints": true
}
]
}
}
+1
View File
@@ -8,6 +8,7 @@ export default tseslint.config(
ignores: [
"node_modules/**",
".next/**",
".worktrees/**",
".claude/**",
".design-tmp/**",
"dist/**",
+8 -1
View File
@@ -178,7 +178,14 @@ const nextConfig: NextConfig = {
output: "standalone",
// Keep pino and pino-pretty as native Node.js requires so their worker-thread
// transport and stream internals work correctly inside the standalone bundle.
serverExternalPackages: ["pino", "pino-pretty", "drizzle-orm", "postgres"],
serverExternalPackages: [
"pino",
"pino-pretty",
"drizzle-orm",
"postgres",
"isomorphic-dompurify",
"jsdom",
],
};
export default nextConfig;
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "famapp",
"version": "0.6.0",
"version": "0.6.2",
"private": true,
"type": "module",
"packageManager": "pnpm@10.33.3",
+17 -15
View File
@@ -1,24 +1,12 @@
import { z } from "zod";
import { apiError, apiJson } from "@/lib/api-handler";
import { resolveApiAuth } from "@/lib/api-auth";
import { getAssistantPreferences, resolveAssistantSystemPrompt } from "@/lib/assistant-preference";
import { isLlmConfigured } from "@/lib/llm";
import { listLlmModels, resolveAssistantModel } from "@/lib/llm/models";
import { clientChatInputSchema } from "@/modules/agent/messages";
import { encodeSseEvent } from "@/modules/agent/server/progress";
import { runAgentChat } from "@/modules/agent/server/run";
const chatInput = z.object({
stream: z.boolean().optional(),
messages: z
.array(
z.object({
role: z.enum(["user", "assistant"]),
content: z.string().trim().min(1).max(8000),
}),
)
.min(1)
.max(40),
});
export async function POST(request: Request) {
const auth = await resolveApiAuth(request);
if (!auth?.userId) {
@@ -39,11 +27,23 @@ export async function POST(request: Request) {
return apiError("Invalid JSON body", 400);
}
const parsed = chatInput.safeParse(body);
const parsed = clientChatInputSchema.safeParse(body);
if (!parsed.success) {
return apiError(parsed.error.issues[0]?.message ?? "Validation error", 400);
}
const modelList = await listLlmModels({ route: assistant.modelRoute });
const modelResolution = resolveAssistantModel({
requestedModel: parsed.data.model,
savedModel: assistant.model,
fallbackModel: modelList.fallbackModel,
models: modelList.models,
});
if (!modelResolution.ok) {
return apiError(modelResolution.error, 400);
}
if (parsed.data.stream) {
const stream = new ReadableStream<Uint8Array>({
async start(controller) {
@@ -57,6 +57,7 @@ export async function POST(request: Request) {
messages: parsed.data.messages,
request,
systemPrompt,
model: modelResolution.model,
onProgress: send,
});
@@ -91,6 +92,7 @@ export async function POST(request: Request) {
messages: parsed.data.messages,
request,
systemPrompt,
model: modelResolution.model,
});
return apiJson({
+47
View File
@@ -0,0 +1,47 @@
import { apiError, apiJson } from "@/lib/api-handler";
import { resolveApiAuth } from "@/lib/api-auth";
import { getAssistantPreferences } from "@/lib/assistant-preference";
import { isValidAssistantModelRoute, listLlmModels, resolveAssistantModel } from "@/lib/llm/models";
export const dynamic = "force-dynamic";
function noStore(response: Response): Response {
response.headers.set("Cache-Control", "no-store");
return response;
}
export async function GET(request: Request) {
const auth = await resolveApiAuth(request);
if (!auth?.userId) {
return noStore(apiError("Unauthorized", 401));
}
const assistant = await getAssistantPreferences(auth.userId);
if (!assistant.enabled) {
return noStore(apiError("Assistant not enabled", 403));
}
const requestedRoute = new URL(request.url).searchParams.get("route");
if (requestedRoute !== null && !isValidAssistantModelRoute(requestedRoute)) {
return noStore(apiError("Invalid assistant model route", 400));
}
const modelRoute = requestedRoute ?? assistant.modelRoute;
const modelList = await listLlmModels({ route: modelRoute });
const resolved = resolveAssistantModel({
requestedModel: null,
savedModel: requestedRoute === null ? assistant.model : null,
fallbackModel: modelList.fallbackModel,
models: modelList.models,
});
return noStore(
apiJson({
models: modelList.models,
selectedModel: resolved.model,
fallbackModel: modelList.fallbackModel,
route: modelList.route,
degraded: modelList.degraded,
}),
);
}
+51
View File
@@ -0,0 +1,51 @@
import { apiError, apiJson } from "@/lib/api-handler";
import { resolveApiAuth } from "@/lib/api-auth";
import { getAssistantPreferences } from "@/lib/assistant-preference";
import { transcribeAudioFile } from "@/lib/llm/transcribe";
export const runtime = "nodejs";
export const maxDuration = 120;
const MAX_AUDIO_BYTES = 25 * 1024 * 1024;
export async function POST(request: Request) {
const auth = await resolveApiAuth(request);
if (!auth?.userId) {
return apiError("Unauthorized", 401);
}
const assistant = await getAssistantPreferences(auth.userId);
if (!assistant.enabled) {
return apiError("Assistant not enabled", 403);
}
let formData: FormData;
try {
formData = await request.formData();
} catch {
return apiError("Invalid multipart body", 400);
}
const file = formData.get("file");
if (!(file instanceof File)) {
return apiError("No audio file in request", 400);
}
if (!file.type.startsWith("audio/") && file.type !== "video/webm") {
return apiError("Only audio recordings are allowed", 415);
}
if (file.size > MAX_AUDIO_BYTES) {
return apiError("Recording exceeds 25 MB limit", 413);
}
const filename = file.name.trim() || "recording.wav";
try {
const text = await transcribeAudioFile(file, filename);
return apiJson({ text });
} catch (err) {
const message = err instanceof Error ? err.message : "Transcription failed";
return apiError(message, 502);
}
}
+3 -1
View File
@@ -33,7 +33,9 @@ export async function POST(request: Request) {
}
const url = new URL(request.url);
const scope = url.searchParams.get("scope") === "notes" ? "notes" : "garden";
const scopeParam = url.searchParams.get("scope");
const scope =
scopeParam === "notes" ? "notes" : scopeParam === "assistant" ? "assistant" : "garden";
if (!file.type.startsWith("image/")) {
return NextResponse.json({ error: "Only image files are allowed" }, { status: 415 });
+4
View File
@@ -104,6 +104,7 @@ export default async function RootLayout({ children }: { children: React.ReactNo
let signedIn = false;
let assistantEnabled = false;
let assistantName = DEFAULT_ASSISTANT_NAME;
let assistantModel: string | null = null;
const session = await auth();
if (session?.user?.id) {
@@ -117,6 +118,7 @@ export default async function RootLayout({ children }: { children: React.ReactNo
themeNavStyle: users.themeNavStyle,
assistantEnabled: users.assistantEnabled,
assistantName: users.assistantName,
assistantModel: users.assistantModel,
})
.from(users)
.where(eq(users.id, session.user.id))
@@ -129,6 +131,7 @@ export default async function RootLayout({ children }: { children: React.ReactNo
navStyle = row.themeNavStyle as NavStyle;
assistantEnabled = row.assistantEnabled;
assistantName = row.assistantName?.trim() || DEFAULT_ASSISTANT_NAME;
assistantModel = row.assistantModel?.trim() || null;
}
userDashboards = await db
.select({
@@ -186,6 +189,7 @@ export default async function RootLayout({ children }: { children: React.ReactNo
configured={isLlmConfigured()}
userId={session.user.id}
assistantName={assistantName}
assistantModel={assistantModel}
/>
) : null}
<AppToaster position="bottom-right" />
+40
View File
@@ -9,8 +9,15 @@ import {
MAX_ASSISTANT_NAME_LENGTH,
MAX_ASSISTANT_SYSTEM_PROMPT_LENGTH,
} from "@/lib/assistant-config";
import {
isValidAssistantModelRoute,
isValidLlmModelId,
listLlmModels,
type AssistantModelRoute,
} from "@/lib/llm/models";
import { users } from "@/modules/_core/schema";
import { getCurrentSession } from "@/lib/session";
import { getAssistantPreferences } from "@/lib/assistant-preference";
const assistantNameSchema = z
.string()
@@ -50,6 +57,39 @@ export async function setAssistantSystemPrompt(prompt: string | null): Promise<v
revalidateAssistantSurfaces();
}
export async function setAssistantModelRoute(route: AssistantModelRoute): Promise<void> {
if (!isValidAssistantModelRoute(route)) {
throw new Error("Invalid assistant model route");
}
const { user } = await getCurrentSession();
await db
.update(users)
.set({ assistantModelRoute: route, assistantModel: null })
.where(eq(users.id, user.id));
revalidateAssistantSurfaces();
}
export async function setAssistantModel(model: string | null): Promise<void> {
const { user } = await getCurrentSession();
const normalized = model?.trim() || null;
if (normalized !== null && !isValidLlmModelId(normalized)) {
throw new Error("Invalid assistant model");
}
const assistant = await getAssistantPreferences(user.id);
const available = await listLlmModels({ route: assistant.modelRoute });
const requested = normalized === available.fallbackModel ? null : normalized;
if (requested !== null && !available.models.some((option) => option.id === requested)) {
throw new Error("Invalid assistant model");
}
await db.update(users).set({ assistantModel: requested }).where(eq(users.id, user.id));
revalidateAssistantSurfaces();
}
export async function resetAssistantName(): Promise<void> {
await setAssistantName(DEFAULT_ASSISTANT_NAME);
}
+2
View File
@@ -324,6 +324,7 @@ function AppearanceSection({
themeNavStyle: string;
assistantEnabled: boolean;
assistantName: string;
assistantModelRoute: string | null;
assistantSystemPrompt: string | null;
};
}) {
@@ -361,6 +362,7 @@ function AppearanceSection({
<AssistantSettings
enabled={user.assistantEnabled}
name={user.assistantName}
modelRoute={user.assistantModelRoute}
systemPrompt={user.assistantSystemPrompt}
defaultSystemPrompt={AGENT_SYSTEM_PROMPT}
/>
+50 -1
View File
@@ -7,9 +7,11 @@ import {
resetAssistantSystemPrompt,
setAssistantEnabled,
setAssistantName,
setAssistantModelRoute,
setAssistantSystemPrompt,
} from "@/app/settings/assistant-actions";
import { DEFAULT_ASSISTANT_NAME, MAX_ASSISTANT_NAME_LENGTH } from "@/lib/assistant-config";
import type { AssistantModelRoute } from "@/lib/llm/models";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Switch } from "@/components/ui/switch";
@@ -17,15 +19,29 @@ import { Switch } from "@/components/ui/switch";
type Props = {
enabled: boolean;
name: string;
modelRoute: string | null;
systemPrompt: string | null;
defaultSystemPrompt: string;
};
export function AssistantSettings({ enabled, name, systemPrompt, defaultSystemPrompt }: Props) {
function normalizeAssistantModelRoute(value: string | null): AssistantModelRoute {
return value === "uncensored" ? "uncensored" : "auto";
}
export function AssistantSettings({
enabled,
name,
modelRoute,
systemPrompt,
defaultSystemPrompt,
}: Props) {
const [isPending, startTransition] = useTransition();
const router = useRouter();
const effectivePrompt = systemPrompt ?? defaultSystemPrompt;
const [selectedRoute, setSelectedRoute] = useState<AssistantModelRoute>(() =>
normalizeAssistantModelRoute(modelRoute),
);
const [savedName, setSavedName] = useState(name);
const [draftName, setDraftName] = useState(name);
const [savedPrompt, setSavedPrompt] = useState(systemPrompt);
@@ -45,6 +61,16 @@ export function AssistantSettings({ enabled, name, systemPrompt, defaultSystemPr
});
}
function changeModelRoute(nextRoute: string) {
const route = normalizeAssistantModelRoute(nextRoute);
setSelectedRoute(route);
startTransition(async () => {
await setAssistantModelRoute(route);
router.refresh();
});
}
function saveName() {
const next = draftName.trim();
if (!next) return;
@@ -105,6 +131,29 @@ export function AssistantSettings({ enabled, name, systemPrompt, defaultSystemPr
/>
</label>
<div className="flex flex-col gap-2">
<div className="min-w-0">
<label htmlFor="assistant-model-route" className="text-sm font-medium">
Model route
</label>
<p className="muted text-[12px] mt-0.5">
Choose the default router family for your assistant. Individual models can still be
picked from the chat panel.
</p>
</div>
<select
id="assistant-model-route"
aria-label="Assistant model route"
value={selectedRoute}
disabled={isPending}
onChange={(event) => changeModelRoute(event.target.value)}
className="input h-10"
>
<option value="auto">Auto</option>
<option value="uncensored">Uncensored</option>
</select>
</div>
<div className="flex flex-col gap-2">
<div className="flex items-end justify-between gap-3">
<div className="min-w-0 flex-1">
+10
View File
@@ -1,6 +1,7 @@
import { eq } from "drizzle-orm";
import { db } from "@/lib/db";
import { DEFAULT_ASSISTANT_NAME } from "@/lib/assistant-config";
import { isValidAssistantModelRoute, type AssistantModelRoute } from "@/lib/llm/models";
import { users } from "@/modules/_core/schema";
export {
@@ -14,6 +15,8 @@ export type AssistantPreferences = {
enabled: boolean;
name: string;
systemPrompt: string | null;
modelRoute: AssistantModelRoute | null;
model: string | null;
};
export async function getAssistantPreferences(userId: string): Promise<AssistantPreferences> {
@@ -22,6 +25,8 @@ export async function getAssistantPreferences(userId: string): Promise<Assistant
assistantEnabled: users.assistantEnabled,
assistantName: users.assistantName,
assistantSystemPrompt: users.assistantSystemPrompt,
assistantModelRoute: users.assistantModelRoute,
assistantModel: users.assistantModel,
})
.from(users)
.where(eq(users.id, userId))
@@ -31,6 +36,11 @@ export async function getAssistantPreferences(userId: string): Promise<Assistant
enabled: row?.assistantEnabled ?? false,
name: row?.assistantName?.trim() || DEFAULT_ASSISTANT_NAME,
systemPrompt: row?.assistantSystemPrompt ?? null,
modelRoute:
row?.assistantModelRoute && isValidAssistantModelRoute(row.assistantModelRoute)
? row.assistantModelRoute
: null,
model: row?.assistantModel?.trim() || null,
};
}
+28
View File
@@ -0,0 +1,28 @@
import type { ChatContentPart, ChatMessage } from "./types";
export function textFromMessageContent(content: ChatMessage["content"]): string {
if (!content) return "";
if (typeof content === "string") return content;
return content
.filter((part): part is Extract<ChatContentPart, { type: "text" }> => part.type === "text")
.map((part) => part.text)
.join("\n")
.trim();
}
export function buildVisionContentParts(text: string, imageDataUrls: string[]): ChatContentPart[] {
const parts: ChatContentPart[] = [];
const trimmed = text.trim();
if (trimmed) {
parts.push({ type: "text", text: trimmed });
} else if (imageDataUrls.length > 0) {
parts.push({
type: "text",
text: "Read this image and help with what the user needs.",
});
}
for (const url of imageDataUrls) {
parts.push({ type: "image_url", image_url: { url } });
}
return parts;
}
+3 -3
View File
@@ -13,8 +13,8 @@ export type {
export { getLlmConfig, isLlmConfigured } from "./config";
export { createMockLlmClient } from "./mock";
export function createLlmClient(override?: LlmClient): LlmClient {
if (override) return override;
export function createLlmClient(options?: { model?: string; override?: LlmClient }): LlmClient {
if (options?.override) return options.override;
const config = getLlmConfig();
if (config.provider === "mock" || !config.baseUrl) {
@@ -24,6 +24,6 @@ export function createLlmClient(override?: LlmClient): LlmClient {
return createOpenAiCompatibleClient({
baseUrl: config.baseUrl,
apiKey: config.apiKey,
model: config.model,
model: options?.model ?? config.model,
});
}
+4 -11
View File
@@ -1,14 +1,5 @@
import type { ChatCompletionRequest, ChatCompletionResult, LlmClient } from "./types";
function lastUserText(messages: ChatCompletionRequest["messages"]): string {
for (let i = messages.length - 1; i >= 0; i -= 1) {
const message = messages[i];
if (message?.role === "user" && message.content) {
return message.content.toLowerCase();
}
}
return "";
}
import { textFromMessageContent } from "./content";
function hadToolResults(messages: ChatCompletionRequest["messages"]): boolean {
return messages.some((message) => message.role === "tool");
@@ -17,7 +8,9 @@ function hadToolResults(messages: ChatCompletionRequest["messages"]): boolean {
export function createMockLlmClient(): LlmClient {
return {
async chatCompletion(request: ChatCompletionRequest): Promise<ChatCompletionResult> {
const userText = lastUserText(request.messages);
const userText = textFromMessageContent(
[...request.messages].reverse().find((message) => message.role === "user")?.content ?? "",
).toLowerCase();
if (hadToolResults(request.messages)) {
return {
+148
View File
@@ -0,0 +1,148 @@
import { getLlmConfig, type LlmConfig } from "./config";
export type LlmModelOption = {
id: string;
label: string;
};
export const ASSISTANT_MODEL_ROUTES = ["auto", "uncensored"] as const;
export type AssistantModelRoute = (typeof ASSISTANT_MODEL_ROUTES)[number];
export type LlmModelsResult = {
models: LlmModelOption[];
fallbackModel: string;
route: AssistantModelRoute;
degraded: boolean;
};
export type AssistantModelResolution =
| { ok: true; model: string }
| { ok: false; model: string; error: string };
const MODEL_ID_PATTERN = /^[A-Za-z0-9._:/-]+$/;
const MAX_MODEL_ID_LENGTH = 128;
export function isValidLlmModelId(value: string): boolean {
const trimmed = value.trim();
return (
trimmed.length > 0 &&
trimmed.length <= MAX_MODEL_ID_LENGTH &&
trimmed === value &&
MODEL_ID_PATTERN.test(trimmed)
);
}
export function isValidAssistantModelRoute(value: string): value is AssistantModelRoute {
return ASSISTANT_MODEL_ROUTES.some((route) => route === value);
}
function resolveModelRoute(
route: AssistantModelRoute | null | undefined,
config: LlmConfig,
): { route: AssistantModelRoute; fallbackModel: string } {
if (route) {
return { route, fallbackModel: route };
}
const defaultRoute: AssistantModelRoute = config.model === "uncensored" ? "uncensored" : "auto";
return { route: defaultRoute, fallbackModel: config.model };
}
export function normalizeLlmModelsPayload(payload: unknown): LlmModelOption[] {
const data =
typeof payload === "object" && payload !== null && "data" in payload
? (payload as { data?: unknown }).data
: null;
if (!Array.isArray(data)) return [];
const ids = new Set<string>();
for (const row of data) {
if (typeof row !== "object" || row === null || !("id" in row)) continue;
const id = (row as { id?: unknown }).id;
if (typeof id !== "string") continue;
if (!isValidLlmModelId(id)) continue;
ids.add(id);
}
return [...ids].sort((a, b) => a.localeCompare(b)).map((id) => ({ id, label: id }));
}
export async function listLlmModels(options?: {
config?: LlmConfig;
fetchImpl?: typeof fetch;
route?: AssistantModelRoute | null;
}): Promise<LlmModelsResult> {
const config = options?.config ?? getLlmConfig();
const fetchImpl = options?.fetchImpl ?? fetch;
const { route, fallbackModel } = resolveModelRoute(options?.route, config);
const fallbackOption = { id: fallbackModel, label: fallbackModel };
if (config.provider === "mock" || !config.baseUrl) {
return { models: [fallbackOption], fallbackModel, route, degraded: false };
}
try {
const headers: Record<string, string> = {};
if (config.apiKey) headers.Authorization = `Bearer ${config.apiKey}`;
const modelsUrl = new URL(`${config.baseUrl.replace(/\/$/, "")}/models`);
if (route === "uncensored") {
modelsUrl.searchParams.set("type", "uncensored");
}
const response = await fetchImpl(modelsUrl, {
method: "GET",
headers,
});
if (!response.ok) {
return { models: [fallbackOption], fallbackModel, route, degraded: true };
}
const models = normalizeLlmModelsPayload(await response.json());
if (models.length === 0) {
return { models: [fallbackOption], fallbackModel, route, degraded: true };
}
if (!models.some((model) => model.id === fallbackModel)) {
return { models: [fallbackOption], fallbackModel, route, degraded: false };
}
return {
models,
fallbackModel,
route,
degraded: false,
};
} catch {
return { models: [fallbackOption], fallbackModel, route, degraded: true };
}
}
export function resolveAssistantModel(options: {
requestedModel: string | null | undefined;
savedModel: string | null | undefined;
fallbackModel: string;
models: LlmModelOption[];
}): AssistantModelResolution {
const available = new Set(options.models.map((model) => model.id));
const fallback = available.has(options.fallbackModel)
? options.fallbackModel
: (options.models[0]?.id ?? options.fallbackModel);
if (options.requestedModel !== null && options.requestedModel !== undefined) {
if (!isValidLlmModelId(options.requestedModel) || !available.has(options.requestedModel)) {
return { ok: false, model: fallback, error: "Invalid assistant model" };
}
return { ok: true, model: options.requestedModel };
}
if (options.savedModel && available.has(options.savedModel)) {
return { ok: true, model: options.savedModel };
}
return { ok: true, model: fallback };
}
+2 -2
View File
@@ -1,8 +1,8 @@
import type { ChatCompletionRequest, ChatCompletionResult, LlmClient } from "./types";
import type { ChatCompletionRequest, ChatCompletionResult, ChatMessage, LlmClient } from "./types";
type OpenAiMessage = {
role: string;
content: string | null;
content: ChatMessage["content"];
tool_calls?: Array<{
id: string;
type: "function";
+47
View File
@@ -0,0 +1,47 @@
import { getLlmConfig } from "./config";
const MAX_AUDIO_BYTES = 25 * 1024 * 1024;
export async function transcribeAudioFile(file: File | Blob, filename: string): Promise<string> {
if (file.size > MAX_AUDIO_BYTES) {
throw new Error("Recording exceeds 25 MB limit");
}
const config = getLlmConfig();
if (config.provider === "mock" || !config.baseUrl) {
return "add milk to the shopping list";
}
const formData = new FormData();
formData.append("file", file, filename);
formData.append("model", "whisper-1");
const url = `${config.baseUrl.replace(/\/$/, "")}/audio/transcriptions`;
const headers: Record<string, string> = {};
if (config.apiKey) {
headers.Authorization = `Bearer ${config.apiKey}`;
}
const response = await fetch(url, {
method: "POST",
headers,
body: formData,
});
if (!response.ok) {
const detail = await response.text();
throw new Error(`Transcription failed (${response.status}): ${detail.slice(0, 400)}`);
}
const contentType = response.headers.get("content-type") ?? "";
if (contentType.includes("application/json")) {
const payload = (await response.json()) as { text?: string };
const text = payload.text?.trim();
if (!text) throw new Error("Transcription returned empty text");
return text;
}
const text = (await response.text()).trim();
if (!text) throw new Error("Transcription returned empty text");
return text;
}
+15 -1
View File
@@ -1,5 +1,19 @@
export type ChatRole = "system" | "user" | "assistant" | "tool";
export type ChatTextPart = {
type: "text";
text: string;
};
export type ChatImagePart = {
type: "image_url";
image_url: {
url: string;
};
};
export type ChatContentPart = ChatTextPart | ChatImagePart;
export type ChatToolCall = {
id: string;
type: "function";
@@ -11,7 +25,7 @@ export type ChatToolCall = {
export type ChatMessage = {
role: ChatRole;
content: string | null;
content: string | ChatContentPart[] | null;
tool_calls?: ChatToolCall[];
tool_call_id?: string;
name?: string;
+2
View File
@@ -38,6 +38,8 @@ export const users = pgTable("users", {
assistantEnabled: boolean("assistant_enabled").notNull().default(false),
assistantName: text("assistant_name").notNull().default("Assistant"),
assistantSystemPrompt: text("assistant_system_prompt"),
assistantModelRoute: text("assistant_model_route"),
assistantModel: text("assistant_model"),
defaultEventReminderOffsets: jsonb("default_event_reminder_offsets")
.notNull()
.$type<number[]>()
+20 -6
View File
@@ -1,9 +1,12 @@
import type { ClientChatMessage } from "./messages";
export type AssistantChatMessage = {
role: "user" | "assistant";
content: string;
imageUrl?: string;
};
const STORAGE_VERSION = "v1";
const STORAGE_VERSION = "v2";
const MAX_MESSAGES = 40;
function storageKey(userId: string) {
@@ -13,11 +16,22 @@ function storageKey(userId: string) {
function isValidMessage(value: unknown): value is AssistantChatMessage {
if (!value || typeof value !== "object") return false;
const row = value as Record<string, unknown>;
return (
(row.role === "user" || row.role === "assistant") &&
typeof row.content === "string" &&
row.content.trim().length > 0
);
if (row.role !== "user" && row.role !== "assistant") return false;
if (typeof row.content !== "string" || row.content.trim().length === 0) return false;
if (row.imageUrl !== undefined && typeof row.imageUrl !== "string") return false;
return true;
}
export function toClientChatMessage(message: AssistantChatMessage): ClientChatMessage {
if (!message.imageUrl) {
return { role: message.role, content: message.content };
}
return {
role: message.role,
content: message.content,
attachments: [{ type: "image", url: message.imageUrl }],
};
}
export function loadAssistantChat(userId: string): AssistantChatMessage[] {
@@ -8,9 +8,10 @@ type Props = {
configured: boolean;
userId: string;
assistantName: string;
assistantModel: string | null;
};
export function AssistantBubble({ configured, userId, assistantName }: Props) {
export function AssistantBubble({ configured, userId, assistantName, assistantModel }: Props) {
const [open, setOpen] = useState(false);
return (
@@ -41,6 +42,7 @@ export function AssistantBubble({ configured, userId, assistantName }: Props) {
configured={configured}
userId={userId}
assistantName={assistantName}
assistantModel={assistantModel}
/>
</div>
) : null}
+290 -28
View File
@@ -1,7 +1,8 @@
"use client";
import { useEffect, useRef, useState } from "react";
import { Loader2, Send } from "lucide-react";
import { useCallback, useEffect, useRef, useState, useTransition } from "react";
import { ChevronDown, ImagePlus, Loader2, Mic, RefreshCw, Send, Square } from "lucide-react";
import { setAssistantModel } from "@/app/settings/assistant-actions";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { consumeAgentChatStream } from "../assistant-chat-stream";
@@ -9,23 +10,72 @@ import {
clearAssistantChat,
loadAssistantChat,
saveAssistantChat,
toClientChatMessage,
type AssistantChatMessage,
} from "../assistant-chat-storage";
import { useVoiceInput } from "./use-voice-input";
type Props = {
configured: boolean;
userId: string;
assistantName: string;
assistantModel: string | null;
};
export function AssistantPanel({ configured, userId, assistantName }: Props) {
type PendingImage = {
url: string;
};
type LlmModelOption = {
id: string;
label: string;
};
type ModelsResponse = {
models: LlmModelOption[];
selectedModel: string;
fallbackModel: string;
degraded: boolean;
};
async function uploadAssistantImage(file: File): Promise<string> {
const formData = new FormData();
formData.append("file", file);
const response = await fetch("/api/uploads?scope=assistant", { method: "POST", body: formData });
if (!response.ok) {
const payload = (await response.json().catch(() => null)) as { error?: string } | null;
throw new Error(payload?.error ?? "Image upload failed");
}
const payload = (await response.json()) as { url: string };
return payload.url;
}
export function AssistantPanel({ configured, userId, assistantName, assistantModel }: Props) {
const [messages, setMessages] = useState<AssistantChatMessage[]>(() => loadAssistantChat(userId));
const [input, setInput] = useState("");
const [pendingImage, setPendingImage] = useState<PendingImage | null>(null);
const [uploadingImage, setUploadingImage] = useState(false);
const [error, setError] = useState<string | null>(null);
const [isPending, setIsPending] = useState(false);
const [activityLabel, setActivityLabel] = useState<string | null>(null);
const [modelOptions, setModelOptions] = useState<LlmModelOption[]>([]);
const [selectedModel, setSelectedModel] = useState(assistantModel ?? "");
const [fallbackModel, setFallbackModel] = useState("");
const [modelsDegraded, setModelsDegraded] = useState(false);
const [modelsLoading, setModelsLoading] = useState(true);
const [savingModel, startTransition] = useTransition();
const listRef = useRef<HTMLDivElement>(null);
const abortRef = useRef<AbortController | null>(null);
const imageInputRef = useRef<HTMLInputElement>(null);
const { state: voiceState, toggleRecording } = useVoiceInput({
disabled: isPending || uploadingImage,
onTranscript: (text) => {
setInput((current) => (current.trim() ? `${current.trim()} ${text}` : text));
setError(null);
},
onError: (message) => setError(message),
});
useEffect(() => {
saveAssistantChat(userId, messages);
@@ -37,6 +87,48 @@ export function AssistantPanel({ configured, userId, assistantName }: Props) {
};
}, []);
const loadModels = useCallback(async (options?: { refresh?: boolean; signal?: AbortSignal }) => {
if (options?.signal?.aborted) return;
setModelsLoading(true);
try {
const params = new URLSearchParams();
if (options?.refresh) params.set("refresh", "1");
const query = params.size > 0 ? `?${params.toString()}` : "";
const response = await fetch(`/api/agent/models${query}`, {
cache: "no-store",
signal: options?.signal,
});
if (!response.ok) throw new Error("Model discovery unavailable");
const payload = (await response.json()) as ModelsResponse;
if (options?.signal?.aborted) return;
setModelOptions(payload.models);
setSelectedModel(payload.selectedModel);
setFallbackModel(payload.fallbackModel);
setModelsDegraded(payload.degraded);
setError(null);
} catch (err) {
if (err instanceof Error && err.name === "AbortError") return;
setModelsDegraded(true);
setError("Model discovery unavailable");
} finally {
if (!options?.signal?.aborted) setModelsLoading(false);
}
}, []);
useEffect(() => {
const controller = new AbortController();
queueMicrotask(() => {
void loadModels({ signal: controller.signal });
});
return () => {
controller.abort();
};
}, [loadModels]);
function scrollToBottom() {
requestAnimationFrame(() => {
const node = listRef.current;
@@ -47,22 +139,65 @@ export function AssistantPanel({ configured, userId, assistantName }: Props) {
function clearChat() {
abortRef.current?.abort();
setMessages([]);
setInput("");
setPendingImage(null);
setError(null);
setActivityLabel(null);
setIsPending(false);
clearAssistantChat(userId);
}
async function handleImageSelect(event: React.ChangeEvent<HTMLInputElement>) {
const file = event.target.files?.[0];
event.target.value = "";
if (!file || isPending || uploadingImage) return;
setUploadingImage(true);
setError(null);
try {
const url = await uploadAssistantImage(file);
setPendingImage({ url });
} catch (err) {
setError(err instanceof Error ? err.message : "Image upload failed");
} finally {
setUploadingImage(false);
}
}
function changeModel(nextModel: string | null) {
if (!nextModel) return;
setSelectedModel(nextModel);
setError(null);
startTransition(async () => {
try {
await setAssistantModel(nextModel === fallbackModel ? null : nextModel);
} catch (err) {
setError(err instanceof Error ? err.message : "Could not save assistant model");
}
});
}
async function sendMessage() {
const text = input.trim();
if (!text || isPending) return;
const hasImage = pendingImage !== null;
if ((!text && !hasImage) || isPending || voiceState !== "idle") return;
const nextMessages: AssistantChatMessage[] = [...messages, { role: "user", content: text }];
const content = text || "Help me with this image.";
const userMessage: AssistantChatMessage = {
role: "user",
content,
...(pendingImage ? { imageUrl: pendingImage.url } : {}),
};
const nextMessages: AssistantChatMessage[] = [...messages, userMessage];
setInput("");
setPendingImage(null);
setError(null);
setMessages(nextMessages);
setIsPending(true);
setActivityLabel("Understanding your request…");
setActivityLabel(hasImage ? "Reading your photo…" : "Understanding your request…");
scrollToBottom();
abortRef.current?.abort();
@@ -73,7 +208,11 @@ export function AssistantPanel({ configured, userId, assistantName }: Props) {
const response = await fetch("/api/agent/chat", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ messages: nextMessages, stream: true }),
body: JSON.stringify({
messages: nextMessages.map(toClientChatMessage),
model: selectedModel || undefined,
stream: true,
}),
signal: controller.signal,
});
@@ -88,7 +227,10 @@ export function AssistantPanel({ configured, userId, assistantName }: Props) {
throw new Error("Assistant returned an empty response");
}
setMessages((current) => [...current, result.message]);
setMessages((current) => [
...current,
{ role: "assistant", content: result.message.content },
]);
scrollToBottom();
} catch (err) {
if (err instanceof Error && err.name === "AbortError") return;
@@ -101,25 +243,74 @@ export function AssistantPanel({ configured, userId, assistantName }: Props) {
}
const showEmptyState = messages.length === 0 && !isPending;
const inputDisabled = isPending || voiceState === "transcribing" || uploadingImage;
const canSend =
!isPending &&
!savingModel &&
voiceState === "idle" &&
!uploadingImage &&
(input.trim().length > 0 || pendingImage !== null);
return (
<div className="flex min-h-0 flex-1 flex-col gap-3">
<div className="flex items-start justify-between gap-3">
<p className="muted min-w-0 text-[12px] leading-relaxed">
{configured
? "Ask me to update lists, calendar, notes, or journal."
: "Mock provider active — set LLM_BASE_URL for your homelab model."}
</p>
{messages.length > 0 ? (
<button
<div className="min-w-0 flex-1">
<p className="muted text-[12px] leading-relaxed">
{configured
? "Type, talk, or send a photo — I can update lists, calendar, notes, and more."
: "Mock provider active — set LLM_BASE_URL for your homelab model."}
</p>
{modelsDegraded ? (
<p className="muted mt-1 text-[11px]">Model discovery unavailable; using fallback.</p>
) : null}
</div>
<div className="flex shrink-0 items-center gap-2">
{modelOptions.length > 0 ? (
<div className="relative max-w-36">
<select
aria-label="Assistant model"
value={selectedModel}
onChange={(event) => changeModel(event.target.value)}
disabled={modelsLoading || savingModel || isPending}
className="h-9 w-full max-w-36 appearance-none truncate rounded-[min(var(--radius-md),10px)] border border-input bg-[var(--card)] py-0 pr-8 pl-3 text-[13px] leading-9 text-[var(--ink)] outline-none transition-colors focus-visible:border-ring focus-visible:ring-3 focus-visible:ring-ring/50 disabled:cursor-not-allowed disabled:opacity-50"
>
{modelOptions.map((model) => (
<option key={model.id} value={model.id}>
{model.label}
</option>
))}
</select>
<ChevronDown
className="pointer-events-none absolute top-1/2 right-2 size-4 -translate-y-1/2 text-muted-foreground"
aria-hidden="true"
/>
</div>
) : null}
<Button
type="button"
onClick={clearChat}
disabled={isPending}
className="shrink-0 text-[11px] text-muted-foreground transition-colors hover:text-foreground disabled:opacity-50"
size="sm"
variant="outline"
aria-label="Refresh assistant models"
disabled={modelsLoading || savingModel || isPending}
onClick={() => void loadModels({ refresh: true })}
>
Clear
</button>
) : null}
{modelsLoading ? (
<Loader2 className="size-4 animate-spin" />
) : (
<RefreshCw className="size-4" />
)}
</Button>
{messages.length > 0 ? (
<button
type="button"
onClick={clearChat}
disabled={isPending}
className="shrink-0 text-[11px] text-muted-foreground transition-colors hover:text-foreground disabled:opacity-50"
>
Clear
</button>
) : null}
</div>
</div>
<div
@@ -129,8 +320,8 @@ export function AssistantPanel({ configured, userId, assistantName }: Props) {
>
{showEmptyState ? (
<p className="muted text-[12px]">
Try &quot;add milk to the shopping list&quot; or &quot;what&apos;s on the calendar this
week?&quot;
Try &quot;add milk to the shopping list&quot;, tap the mic, or attach a photo of an
appointment card.
</p>
) : (
<div className="grid gap-2">
@@ -147,6 +338,14 @@ export function AssistantPanel({ configured, userId, assistantName }: Props) {
<div className="eyebrow mb-0.5 text-[10px]">
{message.role === "user" ? "You" : assistantName}
</div>
{message.imageUrl ? (
// User-uploaded assistant attachment preview
<img
src={message.imageUrl}
alt=""
className="mb-2 max-h-40 w-full rounded-md object-contain"
/>
) : null}
{message.content}
</div>
))}
@@ -169,24 +368,87 @@ export function AssistantPanel({ configured, userId, assistantName }: Props) {
)}
</div>
{pendingImage ? (
<div className="flex items-center gap-2 rounded-[var(--r-md)] border-[0.5px] bg-[var(--shade)] p-2">
<img
src={pendingImage.url}
alt=""
className="max-h-16 max-w-[40%] rounded-md object-contain"
/>
<div className="min-w-0 flex-1 text-[11px] text-muted-foreground">Photo attached</div>
<button
type="button"
className="text-[11px] text-muted-foreground hover:text-foreground"
onClick={() => setPendingImage(null)}
disabled={inputDisabled}
>
Remove
</button>
</div>
) : null}
{error ? <p className="text-[12px] text-destructive">{error}</p> : null}
<form
className="flex gap-2"
onSubmit={(event) => {
event.preventDefault();
sendMessage();
void sendMessage();
}}
>
<input
ref={imageInputRef}
type="file"
accept="image/*"
className="hidden"
onChange={(event) => void handleImageSelect(event)}
/>
<Button
type="button"
size="sm"
variant="outline"
disabled={inputDisabled}
aria-label="Attach photo"
onClick={() => imageInputRef.current?.click()}
>
{uploadingImage ? (
<Loader2 className="size-4 animate-spin" />
) : (
<ImagePlus className="size-4" />
)}
</Button>
<Button
type="button"
size="sm"
variant={voiceState === "recording" ? "destructive" : "outline"}
disabled={inputDisabled}
aria-label={voiceState === "recording" ? "Stop recording" : "Record voice message"}
aria-pressed={voiceState === "recording"}
onClick={() => void toggleRecording()}
>
{voiceState === "transcribing" ? (
<Loader2 className="size-4 animate-spin" />
) : voiceState === "recording" ? (
<Square className="size-4" />
) : (
<Mic className="size-4" />
)}
</Button>
<Input
value={input}
onChange={(event) => setInput(event.target.value)}
placeholder={`Ask ${assistantName}`}
disabled={isPending}
placeholder={
voiceState === "recording"
? "Listening… tap mic to stop"
: voiceState === "transcribing"
? "Transcribing…"
: `Ask ${assistantName}`
}
disabled={inputDisabled}
aria-label={`Message for ${assistantName}`}
className="h-9"
className="h-9 min-w-0 flex-1"
/>
<Button type="submit" size="sm" disabled={isPending || !input.trim()} aria-label="Send">
<Button type="submit" size="sm" disabled={!canSend} aria-label="Send">
{isPending ? <Loader2 className="size-4 animate-spin" /> : <Send className="size-4" />}
</Button>
</form>
@@ -0,0 +1,129 @@
"use client";
import { useCallback, useEffect, useRef, useState } from "react";
export type VoiceInputState = "idle" | "recording" | "transcribing";
function pickRecorderFormat(): { mimeType: string; extension: string } {
const candidates = [
{ mimeType: "audio/wav", extension: "wav" },
{ mimeType: "audio/webm;codecs=opus", extension: "webm" },
{ mimeType: "audio/webm", extension: "webm" },
{ mimeType: "audio/mp4", extension: "m4a" },
];
for (const candidate of candidates) {
if (typeof MediaRecorder !== "undefined" && MediaRecorder.isTypeSupported(candidate.mimeType)) {
return candidate;
}
}
return { mimeType: "", extension: "webm" };
}
export function useVoiceInput(options: {
onTranscript: (text: string) => void;
onError: (message: string) => void;
disabled?: boolean;
}) {
const [state, setState] = useState<VoiceInputState>("idle");
const recorderRef = useRef<MediaRecorder | null>(null);
const chunksRef = useRef<Blob[]>([]);
const streamRef = useRef<MediaStream | null>(null);
const formatRef = useRef(pickRecorderFormat());
const onTranscriptRef = useRef(options.onTranscript);
const onErrorRef = useRef(options.onError);
useEffect(() => {
onTranscriptRef.current = options.onTranscript;
onErrorRef.current = options.onError;
}, [options.onTranscript, options.onError]);
const stopStream = useCallback(() => {
for (const track of streamRef.current?.getTracks() ?? []) {
track.stop();
}
streamRef.current = null;
}, []);
const stopRecording = useCallback(async () => {
const recorder = recorderRef.current;
if (!recorder || recorder.state === "inactive") return;
await new Promise<void>((resolve) => {
recorder.addEventListener("stop", () => resolve(), { once: true });
recorder.stop();
});
recorderRef.current = null;
stopStream();
const blob = new Blob(chunksRef.current, {
type: formatRef.current.mimeType || chunksRef.current[0]?.type || "audio/webm",
});
chunksRef.current = [];
if (blob.size === 0) {
setState("idle");
onErrorRef.current("No audio captured");
return;
}
setState("transcribing");
try {
const formData = new FormData();
formData.append("file", blob, `recording.${formatRef.current.extension}`);
const response = await fetch("/api/agent/transcribe", { method: "POST", body: formData });
if (!response.ok) {
const payload = (await response.json().catch(() => null)) as { error?: string } | null;
throw new Error(payload?.error ?? "Transcription failed");
}
const payload = (await response.json()) as { text: string };
onTranscriptRef.current(payload.text);
} catch (err) {
onErrorRef.current(err instanceof Error ? err.message : "Transcription failed");
} finally {
setState("idle");
}
}, [stopStream]);
const startRecording = useCallback(async () => {
if (options.disabled || state !== "idle") return;
if (typeof navigator === "undefined" || !navigator.mediaDevices?.getUserMedia) {
onErrorRef.current("Microphone not available in this browser");
return;
}
formatRef.current = pickRecorderFormat();
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
streamRef.current = stream;
const recorder = formatRef.current.mimeType
? new MediaRecorder(stream, { mimeType: formatRef.current.mimeType })
: new MediaRecorder(stream);
chunksRef.current = [];
recorder.addEventListener("dataavailable", (event) => {
if (event.data.size > 0) chunksRef.current.push(event.data);
});
recorder.start();
recorderRef.current = recorder;
setState("recording");
} catch {
stopStream();
onErrorRef.current("Microphone permission denied");
}
}, [options.disabled, state, stopStream]);
const toggleRecording = useCallback(async () => {
if (state === "recording") {
await stopRecording();
return;
}
if (state === "idle") {
await startRecording();
}
}, [startRecording, state, stopRecording]);
return { state, toggleRecording };
}
+26
View File
@@ -0,0 +1,26 @@
import { z } from "zod";
import { isValidLlmModelId } from "@/lib/llm/models";
export const clientChatAttachmentSchema = z.object({
type: z.literal("image"),
url: z.string().trim().min(1).max(2048),
});
export const clientChatMessageSchema = z.object({
role: z.enum(["user", "assistant"]),
content: z.string().trim().min(1).max(8000),
attachments: z.array(clientChatAttachmentSchema).max(3).optional(),
});
export const clientChatModelSchema = z
.string()
.refine((value) => isValidLlmModelId(value), "Invalid assistant model");
export const clientChatInputSchema = z.object({
stream: z.boolean().optional(),
model: clientChatModelSchema.optional(),
messages: z.array(clientChatMessageSchema).min(1).max(40),
});
export type ClientChatAttachment = z.infer<typeof clientChatAttachmentSchema>;
export type ClientChatMessage = z.infer<typeof clientChatMessageSchema>;
@@ -0,0 +1,35 @@
import { minioClient, MINIO_BUCKET } from "@/lib/minio";
const UPLOAD_PATH_PREFIX = "/api/uploads/";
function uploadKeyFromUrl(url: string): string | null {
if (!url.startsWith(UPLOAD_PATH_PREFIX)) return null;
const key = url.slice(UPLOAD_PATH_PREFIX.length);
if (!key || key.includes("..")) return null;
return key;
}
export async function resolveAssistantImageDataUrl(url: string): Promise<string> {
const key = uploadKeyFromUrl(url);
if (!key) {
throw new Error("Unsupported image URL");
}
const stream = await minioClient.getObject(MINIO_BUCKET, key);
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array));
}
const buffer = Buffer.concat(chunks);
const stat = await minioClient.statObject(MINIO_BUCKET, key);
const metaData = stat.metaData as Record<string, string> | undefined;
const contentType =
metaData?.["content-type"] ?? metaData?.["Content-Type"] ?? "application/octet-stream";
return `data:${contentType};base64,${buffer.toString("base64")}`;
}
export async function resolveAssistantImageDataUrls(urls: string[]): Promise<string[]> {
return Promise.all(urls.map((url) => resolveAssistantImageDataUrl(url)));
}
+33 -16
View File
@@ -1,16 +1,14 @@
import { createLlmClient, type ChatMessage, type LlmClient } from "@/lib/llm";
import { buildVisionContentParts, textFromMessageContent } from "@/lib/llm/content";
import { AGENT_SYSTEM_PROMPT, AGENT_TOOLS } from "../tools";
import type { ClientChatMessage } from "../messages";
import { describeToolActivity } from "../tool-labels";
import { createApiToolExecutor, type ToolExecutor } from "../tool-executor";
import { resolveAssistantImageDataUrls } from "./resolve-images";
import { thinkingLabel, type AgentProgressEvent } from "./progress";
const MAX_TOOL_ROUNDS = 8;
export type ClientChatMessage = {
role: "user" | "assistant";
content: string;
};
export type AgentToolCallSummary = {
name: string;
status: number;
@@ -23,28 +21,45 @@ export type AgentChatResult = {
export type AgentProgressHandler = (event: AgentProgressEvent) => void;
async function toLlmUserMessage(message: ClientChatMessage): Promise<ChatMessage> {
if (message.role === "assistant") {
return { role: "assistant", content: message.content };
}
const imageUrls =
message.attachments?.filter((attachment) => attachment.type === "image").map((a) => a.url) ??
[];
if (imageUrls.length === 0) {
return { role: "user", content: message.content };
}
const dataUrls = await resolveAssistantImageDataUrls(imageUrls);
return {
role: "user",
content: buildVisionContentParts(message.content, dataUrls),
};
}
export async function runAgentChat(options: {
messages: ClientChatMessage[];
request: Request;
systemPrompt?: string;
model?: string;
llm?: LlmClient;
executeTool?: ToolExecutor;
onProgress?: AgentProgressHandler;
}): Promise<AgentChatResult> {
const llm = options.llm ?? createLlmClient();
const llm = options.llm ?? createLlmClient({ model: options.model });
const executeTool = options.executeTool ?? createApiToolExecutor(options.request);
const onProgress = options.onProgress;
const systemPrompt = options.systemPrompt ?? AGENT_SYSTEM_PROMPT;
const transcript: ChatMessage[] = [
{ role: "system", content: systemPrompt },
...options.messages.map(
(message): ChatMessage => ({
role: message.role,
content: message.content,
}),
),
];
const userMessages = await Promise.all(
options.messages.map((message) => toLlmUserMessage(message)),
);
const transcript: ChatMessage[] = [{ role: "system", content: systemPrompt }, ...userMessages];
const toolCalls: AgentToolCallSummary[] = [];
@@ -64,7 +79,9 @@ export async function runAgentChat(options: {
return {
message: {
role: "assistant",
content: assistantMessage.content?.trim() || "I couldn't generate a response.",
content:
textFromMessageContent(assistantMessage.content).trim() ||
"I couldn't generate a response.",
},
toolCalls,
};
+2
View File
@@ -766,6 +766,8 @@ When no dedicated tool fits, or you are unsure how to do something:
1. Call get_api_docs with a relevant search term to find the right /api/v1/* endpoint.
2. Call call_api with the documented method, path, query, and body.
When the user sends a photo, read dates, times, locations, and action items from it, then use tools to act.
Lists: resolve list ids via list_lists. To complete items, list_list_items then update_list_item with done: true.
Journal: per-user private entries. Valid mood ids: ${JOURNAL_MOOD_IDS}. stress is 1-10. pillsTaken is boolean.
+30 -1
View File
@@ -1,4 +1,13 @@
import { expect, test } from "@playwright/test";
import { expect, test, type Page } from "@playwright/test";
async function ensureSignedIn(page: Page) {
await page.goto("/");
const devLogin = page.getByRole("button", { name: "Dev login" });
if (await devLogin.isVisible().catch(() => false)) {
await devLogin.click();
await page.waitForURL((url) => !url.pathname.startsWith("/login"));
}
}
test("assistant bubble is hidden until opted in", async ({ page }) => {
await page.goto("/");
@@ -6,15 +15,35 @@ test("assistant bubble is hidden until opted in", async ({ page }) => {
});
test("assistant chat smoke after opt-in", async ({ page }) => {
await ensureSignedIn(page);
await page.goto("/settings?s=appearance");
const assistantSwitch = page.getByRole("switch", { name: "AI assistant" });
if (!(await assistantSwitch.isChecked())) {
await assistantSwitch.click();
}
await expect(assistantSwitch).toBeChecked();
const routeSelector = page.getByRole("combobox", { name: "Assistant model route" });
await expect(routeSelector).toBeVisible();
await routeSelector.selectOption("auto");
await expect(routeSelector).toHaveValue("auto");
await expect(routeSelector).toBeEnabled();
await routeSelector.selectOption("uncensored");
await expect(routeSelector).toHaveValue("uncensored");
await expect(routeSelector).toBeEnabled();
await page.goto("/");
await page.getByRole("button", { name: "Open assistant" }).click();
await expect(page.getByRole("dialog", { name: "Assistant" })).toBeVisible();
await expect(page.getByRole("combobox", { name: "Assistant model route" })).toHaveCount(0);
const modelSelector = page.getByRole("combobox", { name: "Assistant model", exact: true });
await expect(modelSelector).toBeVisible();
const refreshModels = page.getByRole("button", { name: "Refresh assistant models" });
await expect(refreshModels).toBeVisible();
await refreshModels.click();
await expect.poll(() => modelSelector.evaluate((node) => node.tagName)).toBe("SELECT");
await expect
.poll(() => modelSelector.evaluate((node) => node.getBoundingClientRect().height))
.toBeGreaterThanOrEqual(36);
await page.getByLabel("Message for Assistant").fill("hello assistant");
await page.getByRole("button", { name: "Send" }).click();
+33
View File
@@ -84,3 +84,36 @@ describe("runAgentChat", () => {
assert.ok(result.message.content.length > 0);
});
});
it("passes a model override to the OpenAI-compatible client", async () => {
const originalBaseUrl = process.env.LLM_BASE_URL;
const originalModel = process.env.LLM_MODEL;
const originalProvider = process.env.LLM_PROVIDER;
const originalFetch = globalThis.fetch;
let requestBody: unknown = null;
process.env.LLM_BASE_URL = "https://llm.example.test/v1";
process.env.LLM_MODEL = "llama3.2";
delete process.env.LLM_PROVIDER;
globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => {
requestBody = JSON.parse(String(init?.body));
return Response.json({
choices: [{ message: { role: "assistant", content: "done" }, finish_reason: "stop" }],
});
}) as typeof fetch;
const { createLlmClient } = await import("../../src/lib/llm/index");
const client = createLlmClient({ model: "qwen2.5-coder" });
await client.chatCompletion({ messages: [{ role: "user", content: "hello" }] });
assert.equal((requestBody as { model?: string }).model, "qwen2.5-coder");
globalThis.fetch = originalFetch;
if (originalBaseUrl === undefined) delete process.env.LLM_BASE_URL;
else process.env.LLM_BASE_URL = originalBaseUrl;
if (originalModel === undefined) delete process.env.LLM_MODEL;
else process.env.LLM_MODEL = originalModel;
if (originalProvider === undefined) delete process.env.LLM_PROVIDER;
else process.env.LLM_PROVIDER = originalProvider;
});
+55
View File
@@ -0,0 +1,55 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import { clientChatInputSchema } from "../../src/modules/agent/messages";
describe("clientChatInputSchema", () => {
it("accepts messages with image attachments", () => {
const parsed = clientChatInputSchema.safeParse({
stream: true,
messages: [
{
role: "user",
content: "Add this appointment to the calendar",
attachments: [{ type: "image", url: "/api/uploads/assistant/house-1/photo.jpg" }],
},
],
});
assert.equal(parsed.success, true);
});
it("rejects empty message content", () => {
const parsed = clientChatInputSchema.safeParse({
messages: [{ role: "user", content: " " }],
});
assert.equal(parsed.success, false);
});
it("accepts an optional model ID", () => {
const parsed = clientChatInputSchema.safeParse({
model: "qwen2.5-coder",
messages: [{ role: "user", content: "hello" }],
});
assert.equal(parsed.success, true);
});
it("rejects invalid model IDs", () => {
const parsed = clientChatInputSchema.safeParse({
model: "bad model",
messages: [{ role: "user", content: "hello" }],
});
assert.equal(parsed.success, false);
});
it("rejects whitespace-padded model IDs", () => {
const parsed = clientChatInputSchema.safeParse({
model: " qwen2.5-coder ",
messages: [{ role: "user", content: "hello" }],
});
assert.equal(parsed.success, false);
});
});
+17
View File
@@ -0,0 +1,17 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
describe("GET /api/agent/models", () => {
it("is dynamic and returns no-store responses", async () => {
process.env.DATABASE_URL ??= "postgres://famapp:famapp@localhost:5432/famapp";
const { GET, dynamic } = await import("../../src/app/api/agent/models/route");
assert.equal(dynamic, "force-dynamic");
const response = await GET(new Request("http://localhost/api/agent/models?refresh=1"));
assert.equal(response.status, 401);
assert.equal(response.headers.get("Cache-Control"), "no-store");
});
});
+23
View File
@@ -0,0 +1,23 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import { transcribeAudioFile } from "../../src/lib/llm/transcribe";
describe("transcribeAudioFile", () => {
it("returns mock text when llm base url is unset", async () => {
const original = process.env.LLM_BASE_URL;
const originalProvider = process.env.LLM_PROVIDER;
delete process.env.LLM_BASE_URL;
delete process.env.LLM_PROVIDER;
const text = await transcribeAudioFile(
new Blob(["audio"], { type: "audio/wav" }),
"recording.wav",
);
assert.match(text, /milk/i);
if (original === undefined) delete process.env.LLM_BASE_URL;
else process.env.LLM_BASE_URL = original;
if (originalProvider === undefined) delete process.env.LLM_PROVIDER;
else process.env.LLM_PROVIDER = originalProvider;
});
});
+12
View File
@@ -4,6 +4,7 @@ import {
clearAssistantChat,
loadAssistantChat,
saveAssistantChat,
toClientChatMessage,
} from "../../src/modules/agent/assistant-chat-storage";
const storage = new Map<string, string>();
@@ -49,4 +50,15 @@ describe("assistant chat storage", () => {
clearAssistantChat("user-a");
assert.deepEqual(loadAssistantChat("user-a"), []);
});
it("maps stored image messages to client attachments", () => {
const message = toClientChatMessage({
role: "user",
content: "read this",
imageUrl: "/api/uploads/assistant/home/photo.jpg",
});
assert.deepEqual(message.attachments, [
{ type: "image", url: "/api/uploads/assistant/home/photo.jpg" },
]);
});
});
+27
View File
@@ -0,0 +1,27 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import { buildVisionContentParts, textFromMessageContent } from "../../src/lib/llm/content";
describe("llm content helpers", () => {
it("reads plain string content", () => {
assert.equal(textFromMessageContent("hello"), "hello");
});
it("joins text parts from multimodal content", () => {
assert.equal(
textFromMessageContent([
{ type: "text", text: "first" },
{ type: "image_url", image_url: { url: "data:image/png;base64,abc" } },
{ type: "text", text: "second" },
]),
"first\nsecond",
);
});
it("builds vision parts with fallback prompt when text is empty", () => {
const parts = buildVisionContentParts("", ["data:image/jpeg;base64,abc"]);
assert.equal(parts.length, 2);
assert.equal(parts[0]?.type, "text");
assert.equal(parts[1]?.type, "image_url");
});
});
+241
View File
@@ -0,0 +1,241 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import {
isValidAssistantModelRoute,
isValidLlmModelId,
listLlmModels,
normalizeLlmModelsPayload,
resolveAssistantModel,
} from "../../src/lib/llm/models";
import type { LlmConfig } from "../../src/lib/llm/config";
const openAiConfig: LlmConfig = {
provider: "openai",
baseUrl: "https://llm.example.test/v1",
apiKey: "secret",
model: "llama3.2",
};
describe("normalizeLlmModelsPayload", () => {
it("normalizes OpenAI-compatible data arrays", () => {
const models = normalizeLlmModelsPayload({
data: [{ id: "qwen2.5-coder" }, { id: "llama3.2" }, { id: "qwen2.5-coder" }],
});
assert.deepEqual(models, [
{ id: "llama3.2", label: "llama3.2" },
{ id: "qwen2.5-coder", label: "qwen2.5-coder" },
]);
});
it("ignores invalid or empty model rows", () => {
const models = normalizeLlmModelsPayload({
data: [
{ id: "" },
{ id: " " },
{ id: "bad model" },
{ id: " llama3.2 " },
{ object: "model" },
],
});
assert.deepEqual(models, []);
});
});
describe("isValidLlmModelId", () => {
it("accepts common provider model IDs", () => {
assert.equal(isValidLlmModelId("llama3.2"), true);
assert.equal(isValidLlmModelId("qwen2.5-coder:latest"), true);
assert.equal(isValidLlmModelId("hf.co/ginnoir/model-v1"), true);
});
it("rejects empty, whitespace, and overlong model IDs", () => {
assert.equal(isValidLlmModelId(""), false);
assert.equal(isValidLlmModelId("bad model"), false);
assert.equal(isValidLlmModelId("x".repeat(129)), false);
});
});
describe("isValidAssistantModelRoute", () => {
it("accepts the supported route families", () => {
assert.equal(isValidAssistantModelRoute("auto"), true);
assert.equal(isValidAssistantModelRoute("uncensored"), true);
});
it("rejects unsupported or padded route families", () => {
assert.equal(isValidAssistantModelRoute("bogus"), false);
assert.equal(isValidAssistantModelRoute(" uncensored "), false);
});
});
describe("listLlmModels", () => {
it("fetches provider models with API key auth when the fallback is advertised", async () => {
const requests: Request[] = [];
const result = await listLlmModels({
config: openAiConfig,
fetchImpl: async (input, init) => {
requests.push(new Request(input, init));
return Response.json({ data: [{ id: "qwen2.5-coder" }, { id: "llama3.2" }] });
},
});
assert.equal(requests[0]?.url, "https://llm.example.test/v1/models");
assert.equal(requests[0]?.headers.get("authorization"), "Bearer secret");
assert.deepEqual(result.models, [
{ id: "llama3.2", label: "llama3.2" },
{ id: "qwen2.5-coder", label: "qwen2.5-coder" },
]);
assert.equal(result.fallbackModel, "llama3.2");
assert.equal(result.degraded, false);
});
it("falls back to LLM_MODEL when provider discovery fails", async () => {
const result = await listLlmModels({
config: openAiConfig,
fetchImpl: async () => new Response("nope", { status: 500 }),
});
assert.deepEqual(result.models, [{ id: "llama3.2", label: "llama3.2" }]);
assert.equal(result.fallbackModel, "llama3.2");
assert.equal(result.degraded, true);
});
it("fetches the typed uncensored catalog when uncensored is the selected route", async () => {
const requests: Request[] = [];
const result = await listLlmModels({
config: { ...openAiConfig, model: "auto" },
route: "uncensored",
fetchImpl: async (input, init) => {
requests.push(new Request(input, init));
return Response.json({
data: [
{ id: "uncensored" },
{ id: "gemma4-uncensored:26b" },
{ id: "dolphin-mistral:latest" },
],
});
},
});
assert.equal(requests[0]?.url, "https://llm.example.test/v1/models?type=uncensored");
assert.deepEqual(result.models, [
{ id: "dolphin-mistral:latest", label: "dolphin-mistral:latest" },
{ id: "gemma4-uncensored:26b", label: "gemma4-uncensored:26b" },
{ id: "uncensored", label: "uncensored" },
]);
assert.equal(result.fallbackModel, "uncensored");
assert.equal(result.route, "uncensored");
assert.equal(result.degraded, false);
});
it("fetches the default catalog when auto is selected over an uncensored deployment default", async () => {
const requests: Request[] = [];
const result = await listLlmModels({
config: { ...openAiConfig, model: "uncensored" },
route: "auto",
fetchImpl: async (input, init) => {
requests.push(new Request(input, init));
return Response.json({
data: [{ id: "auto" }, { id: "qwen3:8b" }],
});
},
});
assert.equal(requests[0]?.url, "https://llm.example.test/v1/models");
assert.deepEqual(result.models, [
{ id: "auto", label: "auto" },
{ id: "qwen3:8b", label: "qwen3:8b" },
]);
assert.equal(result.fallbackModel, "auto");
assert.equal(result.route, "auto");
assert.equal(result.degraded, false);
});
it("uses fallback only for mock provider config", async () => {
const result = await listLlmModels({
config: { provider: "mock", baseUrl: null, apiKey: null, model: "llama3.2" },
fetchImpl: async () => {
throw new Error("fetch should not run for mock config");
},
});
assert.deepEqual(result.models, [{ id: "llama3.2", label: "llama3.2" }]);
assert.equal(result.degraded, false);
});
});
describe("resolveAssistantModel", () => {
it("uses a valid requested model before saved and fallback values", () => {
const resolved = resolveAssistantModel({
requestedModel: "qwen2.5-coder",
savedModel: "llama3.2",
fallbackModel: "llama3.2",
models: [
{ id: "llama3.2", label: "llama3.2" },
{ id: "qwen2.5-coder", label: "qwen2.5-coder" },
],
});
assert.deepEqual(resolved, { ok: true, model: "qwen2.5-coder" });
});
it("rejects invalid requested models", () => {
const resolved = resolveAssistantModel({
requestedModel: "bad model",
savedModel: null,
fallbackModel: "llama3.2",
models: [
{ id: "llama3.2", label: "llama3.2" },
{ id: "bad model", label: "bad model" },
],
});
assert.deepEqual(resolved, {
ok: false,
model: "llama3.2",
error: "Invalid assistant model",
});
});
it("rejects empty requested models", () => {
const resolved = resolveAssistantModel({
requestedModel: "",
savedModel: null,
fallbackModel: "llama3.2",
models: [{ id: "llama3.2", label: "llama3.2" }],
});
assert.deepEqual(resolved, {
ok: false,
model: "llama3.2",
error: "Invalid assistant model",
});
});
it("rejects unavailable requested models", () => {
const resolved = resolveAssistantModel({
requestedModel: "missing",
savedModel: null,
fallbackModel: "llama3.2",
models: [{ id: "llama3.2", label: "llama3.2" }],
});
assert.deepEqual(resolved, {
ok: false,
model: "llama3.2",
error: "Invalid assistant model",
});
});
it("silently falls back when a saved model is gone", () => {
const resolved = resolveAssistantModel({
requestedModel: null,
savedModel: "old-model",
fallbackModel: "llama3.2",
models: [{ id: "llama3.2", label: "llama3.2" }],
});
assert.deepEqual(resolved, { ok: true, model: "llama3.2" });
});
});