From 63fd3fd1ccdce98282b7eb031af6a0ef782dffe0 Mon Sep 17 00:00:00 2001 From: ginnoir Date: Sat, 27 Jun 2026 00:02:58 -0500 Subject: [PATCH] feat(llm): llama-swap multi-model menu (gpt-oss-20b default + gemma4 family) Replace single llama-server with llama-swap so all benchmarked models are selectable from Hermes' menu and hot-swapped on the one P100. Menu: gpt-oss-20b (default, ~45s cold start), gemma-4-26b-a4b (MoE), gemma-4-12b, gemma-4-e4b. qwen3-30b-a3b excluded (OOMs at 64k in 16GB). All 64k, q8/q8 KV, --parallel 1. Co-Authored-By: Claude Opus 4.8 --- stacks/llm/docker-compose.yml | 64 ++++++++----------------------- stacks/llm/llama-swap-config.yaml | 47 +++++++++++++++++++++++ 2 files changed, 64 insertions(+), 47 deletions(-) create mode 100644 stacks/llm/llama-swap-config.yaml diff --git a/stacks/llm/docker-compose.yml b/stacks/llm/docker-compose.yml index 012df4f..3df19dd 100644 --- a/stacks/llm/docker-compose.yml +++ b/stacks/llm/docker-compose.yml @@ -1,61 +1,31 @@ -# llm stack — local LLM inference backend for the Hermes agent. +# llm stack — model-swapping LLM backend for the Hermes agent. # -# Single service: llama.cpp's OpenAI-compatible server (llama-server) serving -# Qwen2.5-14B-Instruct (Q4_K_M GGUF) on the host's Tesla P100-16GB via CDI. -# Chosen over vLLM because the P100 (GP100, compute capability 6.0) lacks the -# DP4A INT8 instructions vLLM's AWQ/GPTQ kernels require — see -# docs/superpowers/specs/2026-06-26-llm-backend-hermes-design.md. +# llama-swap fronts multiple GGUF models on the single Tesla P100 (16GB). Only one +# model fits in VRAM at a time, so llama-swap presents all of them via /v1/models +# and hot-swaps on demand (selecting a different model = a few-second reload). The +# per-model llama-server commands + args live in llama-swap-config.yaml. # -# Pure env_file (LLAMA_API_KEY) — no Portainer UI env, no ${VAR} interpolation. -# Image is infra-pinned out of Watchtower (manual tag bumps only). +# The bundled llama.cpp in llama-swap:cuda is build 9803 (5c7c22c3e) — the same +# build validated on this Pascal card for gemma4 + gpt-oss. Default model and the +# selectable menu are driven from Hermes (~/.hermes/config.yaml: model.default = +# gpt-oss-20b; provider valhalla-p100 models: list = the keys in the swap config). # -# The server's OpenAI API is published on the host at 172.20.0.1:8090 (the edge -# bridge gateway, a local host IP). Host-side Hermes reaches it there directly; -# no Caddy block this round. Model weights live on the ZFS tier; the -# /storage1/labdata/llm/models dir is pre-created with the GGUF before deploy. +# Endpoint published on 172.20.0.1:8090 (edge bridge gateway, a host IP) for the +# host-side Hermes agent. Internal-only; no Caddy, no auth (LAN/host-only). +# Image is infra-pinned out of Watchtower. services: - llama-server: - image: ghcr.io/ggml-org/llama.cpp:server-cuda - container_name: llama-server + llama-swap: + image: ghcr.io/mostlygeek/llama-swap:cuda + container_name: llama-swap restart: unless-stopped labels: - "com.centurylabs.watchtower.enable=false" networks: [llm] - env_file: - - stack.env devices: - "nvidia.com/gpu=0" volumes: - /storage1/labdata/llm/models:/models - command: - - "-m" - - "/models/Qwen2.5-14B-Instruct-Q4_K_M.gguf" - - "--alias" - - "qwen2.5-14b-instruct" - - "--parallel" - - "1" - - "-ngl" - - "99" - - "--ctx-size" - - "65536" - - "--rope-scaling" - - "yarn" - - "--rope-scale" - - "2" - - "--yarn-orig-ctx" - - "32768" - - "--override-kv" - - "qwen2.context_length=int:65536" - - "--flash-attn" - - "on" - - "--cache-type-k" - - "q8_0" - - "--cache-type-v" - - "q8_0" - - "--host" - - "0.0.0.0" - - "--port" - - "8080" + - ./llama-swap-config.yaml:/app/config.yaml:ro ports: - "172.20.0.1:8090:8080" healthcheck: @@ -63,7 +33,7 @@ services: interval: 30s timeout: 10s retries: 5 - start_period: 180s + start_period: 30s networks: llm: diff --git a/stacks/llm/llama-swap-config.yaml b/stacks/llm/llama-swap-config.yaml new file mode 100644 index 0000000..d8caee7 --- /dev/null +++ b/stacks/llm/llama-swap-config.yaml @@ -0,0 +1,47 @@ +# llama-swap model menu for the Hermes backend (single P100, 16GB). +# llama-swap presents every model below via /v1/models and hot-swaps on demand — +# only one fits in VRAM at a time, so selecting a different model triggers a +# brief reload. Default is chosen by Hermes (model.default = gpt-oss-20b). +# +# All four serve 64k context (Hermes' minimum). Args are the validated Pascal +# config: q8_0 KV (q4_0 V-cache is pathological on GP100), flash-attn on, +# --parallel 1 so one sequence gets the full 64k. gpt-oss/gemma4 are natively +# >=128k so no YaRN/override-kv needed. +# +# Excluded: qwen3-30b-a3b-2507 (Q3) — OOMs at 64k in 16GB, so it can't meet +# Hermes' 64k minimum on this GPU. + +healthCheckTimeout: 300 +logLevel: info + +macros: + common: "-ngl 99 --parallel 1 --ctx-size 65536 --flash-attn on --cache-type-k q8_0 --cache-type-v q8_0" + +models: + "gpt-oss-20b": + # MoE ~3.6B active. Fastest prefill (~365 tok/s) -> ~45s cold start on Hermes' 16k prompt. + cmd: > + /app/llama-server --port ${PORT} --host 0.0.0.0 + -m /models/gpt-oss-20b-mxfp4.gguf --alias gpt-oss-20b + ${common} + + "gemma-4-26b-a4b": + # gemma4 MoE, 4B active / 26B total (UD-Q3_K_M). Quality-leaning; ~147 tok/s prefill. + cmd: > + /app/llama-server --port ${PORT} --host 0.0.0.0 + -m /models/gemma-4-26B-A4B-it-UD-Q3_K_M.gguf --alias gemma-4-26b-a4b + ${common} + + "gemma-4-12b": + # Dense 12B (Q4_K_M). ~85 tok/s prefill. + cmd: > + /app/llama-server --port ${PORT} --host 0.0.0.0 + -m /models/gemma-4-12b-it-Q4_K_M.gguf --alias gemma-4-12b + ${common} + + "gemma-4-e4b": + # Small dense ~4B (Q4_K_M). Lots of VRAM headroom; ~173 tok/s prefill. + cmd: > + /app/llama-server --port ${PORT} --host 0.0.0.0 + -m /models/gemma-4-E4B-it-Q4_K_M.gguf --alias gemma-4-e4b + ${common}