# llama-swap model menu for the Hermes backend (single P100, 16GB).
# llama-swap presents every model below via /v1/models and hot-swaps on demand —
# only one fits in VRAM at a time, so selecting a different model triggers a
# brief reload. Default is chosen by Hermes (model.default = gpt-oss-20b).
#
# All serve 64k context (Hermes' minimum). Args are the validated Pascal
# config: q8_0 KV (q4_0 V-cache is pathological on GP100), flash-attn on,
# --parallel 1 so one sequence gets the full 64k. gpt-oss/gemma4/ornith are
# natively >=128k so no YaRN/override-kv needed.
#
# Excluded: qwen3-30b-a3b-2507 (Q3) — OOMs at 64k in 16GB, so it can't meet
# Hermes' 64k minimum on this GPU.

healthCheckTimeout: 300
logLevel: info

macros:
  # --jinja applies each model's embedded chat template (REQUIRED for gpt-oss'
  # harmony format, else content comes back empty; harmless/correct for gemma4).
  common: "-ngl 99 --parallel 1 --ctx-size 65536 --flash-attn on --cache-type-k q8_0 --cache-type-v q8_0 --jinja"

models:
  "gpt-oss-20b":
    # MoE ~3.6B active. Fastest prefill (~365 tok/s) -> ~45s cold start on Hermes' 16k prompt.
    cmd: >
      /app/llama-server --port ${PORT} --host 0.0.0.0
      -m /models/gpt-oss-20b-mxfp4.gguf --alias gpt-oss-20b
      ${common}

  "ornith-1.0-9b":
    # DeepReinforce Ornith-1.0, dense ~9B on Qwen 3.5 (Q5_K_M). MIT. Agentic-coding
    # tuned: <think> block (-> reasoning_content under --jinja) + Qwen3 XML tool calls.
    # Native 256k so no YaRN. Recommended sampling: temp 0.6 / top_p 0.95 / top_k 20.
    cmd: >
      /app/llama-server --port ${PORT} --host 0.0.0.0
      -m /models/ornith-1.0-9b-Q5_K_M.gguf --alias ornith-1.0-9b
      ${common}

  "gemma-4-26b-a4b":
    # gemma4 MoE, 4B active / 26B total (UD-Q3_K_M). Quality-leaning; ~147 tok/s prefill.
    cmd: >
      /app/llama-server --port ${PORT} --host 0.0.0.0
      -m /models/gemma-4-26B-A4B-it-UD-Q3_K_M.gguf --alias gemma-4-26b-a4b
      ${common}

  "gemma-4-12b":
    # Dense 12B (Q4_K_M). ~85 tok/s prefill.
    cmd: >
      /app/llama-server --port ${PORT} --host 0.0.0.0
      -m /models/gemma-4-12b-it-Q4_K_M.gguf --alias gemma-4-12b
      ${common}

  "gemma-4-e4b":
    # Small dense ~4B (Q4_K_M). Lots of VRAM headroom; ~173 tok/s prefill.
    cmd: >
      /app/llama-server --port ${PORT} --host 0.0.0.0
      -m /models/gemma-4-E4B-it-Q4_K_M.gguf --alias gemma-4-e4b
      ${common}