ginnoirandClaude Opus 4.8
13424fcf75
llama-server caps the slot to the GGUF training context (32768) and ignores the YaRN-extended size, leaving per-seq context at 32k. Raise qwen2.context_length metadata to 65536 so the full window is served per request. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
72 lines
2.0 KiB
YAML
72 lines
2.0 KiB
YAML
# llm stack — local LLM inference backend for the Hermes agent.
|
|
#
|
|
# Single service: llama.cpp's OpenAI-compatible server (llama-server) serving
|
|
# Qwen2.5-14B-Instruct (Q4_K_M GGUF) on the host's Tesla P100-16GB via CDI.
|
|
# Chosen over vLLM because the P100 (GP100, compute capability 6.0) lacks the
|
|
# DP4A INT8 instructions vLLM's AWQ/GPTQ kernels require — see
|
|
# docs/superpowers/specs/2026-06-26-llm-backend-hermes-design.md.
|
|
#
|
|
# Pure env_file (LLAMA_API_KEY) — no Portainer UI env, no ${VAR} interpolation.
|
|
# Image is infra-pinned out of Watchtower (manual tag bumps only).
|
|
#
|
|
# The server's OpenAI API is published on the host at 172.20.0.1:8090 (the edge
|
|
# bridge gateway, a local host IP). Host-side Hermes reaches it there directly;
|
|
# no Caddy block this round. Model weights live on the ZFS tier; the
|
|
# /storage1/labdata/llm/models dir is pre-created with the GGUF before deploy.
|
|
services:
|
|
llama-server:
|
|
image: ghcr.io/ggml-org/llama.cpp:server-cuda
|
|
container_name: llama-server
|
|
restart: unless-stopped
|
|
labels:
|
|
- "com.centurylabs.watchtower.enable=false"
|
|
networks: [llm]
|
|
env_file:
|
|
- stack.env
|
|
devices:
|
|
- "nvidia.com/gpu=0"
|
|
volumes:
|
|
- /storage1/labdata/llm/models:/models
|
|
command:
|
|
- "-m"
|
|
- "/models/Qwen2.5-14B-Instruct-Q4_K_M.gguf"
|
|
- "--alias"
|
|
- "qwen2.5-14b-instruct"
|
|
- "--parallel"
|
|
- "1"
|
|
- "-ngl"
|
|
- "99"
|
|
- "--ctx-size"
|
|
- "65536"
|
|
- "--rope-scaling"
|
|
- "yarn"
|
|
- "--rope-scale"
|
|
- "2"
|
|
- "--yarn-orig-ctx"
|
|
- "32768"
|
|
- "--override-kv"
|
|
- "qwen2.context_length=int:65536"
|
|
- "--flash-attn"
|
|
- "on"
|
|
- "--cache-type-k"
|
|
- "q8_0"
|
|
- "--cache-type-v"
|
|
- "q4_0"
|
|
- "--host"
|
|
- "0.0.0.0"
|
|
- "--port"
|
|
- "8080"
|
|
ports:
|
|
- "172.20.0.1:8090:8080"
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-fsS", "http://localhost:8080/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 5
|
|
start_period: 180s
|
|
|
|
networks:
|
|
llm:
|
|
name: llm
|
|
driver: bridge
|