feat(llm): llama-swap multi-model menu (gpt-oss-20b default + gemma4 family)

Replace single llama-server with llama-swap so all benchmarked models are
selectable from Hermes' menu and hot-swapped on the one P100. Menu: gpt-oss-20b
(default, ~45s cold start), gemma-4-26b-a4b (MoE), gemma-4-12b, gemma-4-e4b.
qwen3-30b-a3b excluded (OOMs at 64k in 16GB). All 64k, q8/q8 KV, --parallel 1.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

stacks/llm/docker-compose.yml

@@ -1,61 +1,31 @@
-# llm stack — local LLM inference backend for the Hermes agent.
+# llm stack — model-swapping LLM backend for the Hermes agent.
 #
-# Single service: llama.cpp's OpenAI-compatible server (llama-server) serving
+# llama-swap fronts multiple GGUF models on the single Tesla P100 (16GB). Only one
-# Qwen2.5-14B-Instruct (Q4_K_M GGUF) on the host's Tesla P100-16GB via CDI.
+# model fits in VRAM at a time, so llama-swap presents all of them via /v1/models
-# Chosen over vLLM because the P100 (GP100, compute capability 6.0) lacks the
+# and hot-swaps on demand (selecting a different model = a few-second reload). The
-# DP4A INT8 instructions vLLM's AWQ/GPTQ kernels require — see
+# per-model llama-server commands + args live in llama-swap-config.yaml.
-# docs/superpowers/specs/2026-06-26-llm-backend-hermes-design.md.
 #
-# Pure env_file (LLAMA_API_KEY) — no Portainer UI env, no ${VAR} interpolation.
+# The bundled llama.cpp in llama-swap:cuda is build 9803 (5c7c22c3e) — the same
-# Image is infra-pinned out of Watchtower (manual tag bumps only).
+# build validated on this Pascal card for gemma4 + gpt-oss. Default model and the
+# selectable menu are driven from Hermes (~/.hermes/config.yaml: model.default =
+# gpt-oss-20b; provider valhalla-p100 models: list = the keys in the swap config).
 #
-# The server's OpenAI API is published on the host at 172.20.0.1:8090 (the edge
+# Endpoint published on 172.20.0.1:8090 (edge bridge gateway, a host IP) for the
-# bridge gateway, a local host IP). Host-side Hermes reaches it there directly;
+# host-side Hermes agent. Internal-only; no Caddy, no auth (LAN/host-only).
-# no Caddy block this round. Model weights live on the ZFS tier; the
+# Image is infra-pinned out of Watchtower.
-# /storage1/labdata/llm/models dir is pre-created with the GGUF before deploy.
 services:
-  llama-server:
+  llama-swap:
-    image: ghcr.io/ggml-org/llama.cpp:server-cuda
+    image: ghcr.io/mostlygeek/llama-swap:cuda
-    container_name: llama-server
+    container_name: llama-swap
     restart: unless-stopped
     labels:
       - "com.centurylabs.watchtower.enable=false"
     networks: [llm]
-    env_file:
-      - stack.env
     devices:
       - "nvidia.com/gpu=0"
     volumes:
       - /storage1/labdata/llm/models:/models
-    command:
+      - ./llama-swap-config.yaml:/app/config.yaml:ro
-      - "-m"
-      - "/models/Qwen2.5-14B-Instruct-Q4_K_M.gguf"
-      - "--alias"
-      - "qwen2.5-14b-instruct"
-      - "--parallel"
-      - "1"
-      - "-ngl"
-      - "99"
-      - "--ctx-size"
-      - "65536"
-      - "--rope-scaling"
-      - "yarn"
-      - "--rope-scale"
-      - "2"
-      - "--yarn-orig-ctx"
-      - "32768"
-      - "--override-kv"
-      - "qwen2.context_length=int:65536"
-      - "--flash-attn"
-      - "on"
-      - "--cache-type-k"
-      - "q8_0"
-      - "--cache-type-v"
-      - "q8_0"
-      - "--host"
-      - "0.0.0.0"
-      - "--port"
-      - "8080"
     ports:
       - "172.20.0.1:8090:8080"
     healthcheck:
@@ -63,7 +33,7 @@ services:
       interval: 30s
       timeout: 10s
       retries: 5
-      start_period: 180s
+      start_period: 30s
 
 networks:
   llm:

stacks/llm/llama-swap-config.yaml

@@ -0,0 +1,47 @@
+# llama-swap model menu for the Hermes backend (single P100, 16GB).
+# llama-swap presents every model below via /v1/models and hot-swaps on demand —
+# only one fits in VRAM at a time, so selecting a different model triggers a
+# brief reload. Default is chosen by Hermes (model.default = gpt-oss-20b).
+#
+# All four serve 64k context (Hermes' minimum). Args are the validated Pascal
+# config: q8_0 KV (q4_0 V-cache is pathological on GP100), flash-attn on,
+# --parallel 1 so one sequence gets the full 64k. gpt-oss/gemma4 are natively
+# >=128k so no YaRN/override-kv needed.
+#
+# Excluded: qwen3-30b-a3b-2507 (Q3) — OOMs at 64k in 16GB, so it can't meet
+# Hermes' 64k minimum on this GPU.
+
+healthCheckTimeout: 300
+logLevel: info
+
+macros:
+  common: "-ngl 99 --parallel 1 --ctx-size 65536 --flash-attn on --cache-type-k q8_0 --cache-type-v q8_0"
+
+models:
+  "gpt-oss-20b":
+    # MoE ~3.6B active. Fastest prefill (~365 tok/s) -> ~45s cold start on Hermes' 16k prompt.
+    cmd: >
+      /app/llama-server --port ${PORT} --host 0.0.0.0
+      -m /models/gpt-oss-20b-mxfp4.gguf --alias gpt-oss-20b
+      ${common}
+
+  "gemma-4-26b-a4b":
+    # gemma4 MoE, 4B active / 26B total (UD-Q3_K_M). Quality-leaning; ~147 tok/s prefill.
+    cmd: >
+      /app/llama-server --port ${PORT} --host 0.0.0.0
+      -m /models/gemma-4-26B-A4B-it-UD-Q3_K_M.gguf --alias gemma-4-26b-a4b
+      ${common}
+
+  "gemma-4-12b":
+    # Dense 12B (Q4_K_M). ~85 tok/s prefill.
+    cmd: >
+      /app/llama-server --port ${PORT} --host 0.0.0.0
+      -m /models/gemma-4-12b-it-Q4_K_M.gguf --alias gemma-4-12b
+      ${common}
+
+  "gemma-4-e4b":
+    # Small dense ~4B (Q4_K_M). Lots of VRAM headroom; ~173 tok/s prefill.
+    cmd: >
+      /app/llama-server --port ${PORT} --host 0.0.0.0
+      -m /models/gemma-4-E4B-it-Q4_K_M.gguf --alias gemma-4-e4b
+      ${common}