ginnoirandClaude Opus 4.8
63fd3fd1cc
Replace single llama-server with llama-swap so all benchmarked models are selectable from Hermes' menu and hot-swapped on the one P100. Menu: gpt-oss-20b (default, ~45s cold start), gemma-4-26b-a4b (MoE), gemma-4-12b, gemma-4-e4b. qwen3-30b-a3b excluded (OOMs at 64k in 16GB). All 64k, q8/q8 KV, --parallel 1. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
48 lines
1.8 KiB
YAML
48 lines
1.8 KiB
YAML
# llama-swap model menu for the Hermes backend (single P100, 16GB).
|
|
# llama-swap presents every model below via /v1/models and hot-swaps on demand —
|
|
# only one fits in VRAM at a time, so selecting a different model triggers a
|
|
# brief reload. Default is chosen by Hermes (model.default = gpt-oss-20b).
|
|
#
|
|
# All four serve 64k context (Hermes' minimum). Args are the validated Pascal
|
|
# config: q8_0 KV (q4_0 V-cache is pathological on GP100), flash-attn on,
|
|
# --parallel 1 so one sequence gets the full 64k. gpt-oss/gemma4 are natively
|
|
# >=128k so no YaRN/override-kv needed.
|
|
#
|
|
# Excluded: qwen3-30b-a3b-2507 (Q3) — OOMs at 64k in 16GB, so it can't meet
|
|
# Hermes' 64k minimum on this GPU.
|
|
|
|
healthCheckTimeout: 300
|
|
logLevel: info
|
|
|
|
macros:
|
|
common: "-ngl 99 --parallel 1 --ctx-size 65536 --flash-attn on --cache-type-k q8_0 --cache-type-v q8_0"
|
|
|
|
models:
|
|
"gpt-oss-20b":
|
|
# MoE ~3.6B active. Fastest prefill (~365 tok/s) -> ~45s cold start on Hermes' 16k prompt.
|
|
cmd: >
|
|
/app/llama-server --port ${PORT} --host 0.0.0.0
|
|
-m /models/gpt-oss-20b-mxfp4.gguf --alias gpt-oss-20b
|
|
${common}
|
|
|
|
"gemma-4-26b-a4b":
|
|
# gemma4 MoE, 4B active / 26B total (UD-Q3_K_M). Quality-leaning; ~147 tok/s prefill.
|
|
cmd: >
|
|
/app/llama-server --port ${PORT} --host 0.0.0.0
|
|
-m /models/gemma-4-26B-A4B-it-UD-Q3_K_M.gguf --alias gemma-4-26b-a4b
|
|
${common}
|
|
|
|
"gemma-4-12b":
|
|
# Dense 12B (Q4_K_M). ~85 tok/s prefill.
|
|
cmd: >
|
|
/app/llama-server --port ${PORT} --host 0.0.0.0
|
|
-m /models/gemma-4-12b-it-Q4_K_M.gguf --alias gemma-4-12b
|
|
${common}
|
|
|
|
"gemma-4-e4b":
|
|
# Small dense ~4B (Q4_K_M). Lots of VRAM headroom; ~173 tok/s prefill.
|
|
cmd: >
|
|
/app/llama-server --port ${PORT} --host 0.0.0.0
|
|
-m /models/gemma-4-E4B-it-Q4_K_M.gguf --alias gemma-4-e4b
|
|
${common}
|