# llm stack — local LLM inference backend for the Hermes agent.
#
# Single service: llama.cpp's OpenAI-compatible server (llama-server) serving
# Qwen2.5-14B-Instruct (Q4_K_M GGUF) on the host's Tesla P100-16GB via CDI.
# Chosen over vLLM because the P100 (GP100, compute capability 6.0) lacks the
# DP4A INT8 instructions vLLM's AWQ/GPTQ kernels require — see
# docs/superpowers/specs/2026-06-26-llm-backend-hermes-design.md.
#
# Pure env_file (LLAMA_API_KEY) — no Portainer UI env, no ${VAR} interpolation.
# Image is infra-pinned out of Watchtower (manual tag bumps only).
#
# The server's OpenAI API is published on the host at 172.20.0.1:8090 (the edge
# bridge gateway, a local host IP). Host-side Hermes reaches it there directly;
# no Caddy block this round. Model weights live on the ZFS tier; the
# /storage1/labdata/llm/models dir is pre-created with the GGUF before deploy.
services:
  llama-server:
    image: ghcr.io/ggml-org/llama.cpp:server-cuda
    container_name: llama-server
    restart: unless-stopped
    labels:
      - "com.centurylabs.watchtower.enable=false"
    networks: [llm]
    env_file:
      - stack.env
    devices:
      - "nvidia.com/gpu=0"
    volumes:
      - /storage1/labdata/llm/models:/models
    command:
      - "-m"
      - "/models/Qwen2.5-14B-Instruct-Q4_K_M.gguf"
      - "--alias"
      - "qwen2.5-14b-instruct"
      - "-ngl"
      - "99"
      - "--ctx-size"
      - "32768"
      - "--flash-attn"
      - "on"
      - "--cache-type-k"
      - "q8_0"
      - "--cache-type-v"
      - "q8_0"
      - "--host"
      - "0.0.0.0"
      - "--port"
      - "8080"
    ports:
      - "172.20.0.1:8090:8080"
    healthcheck:
      test: ["CMD", "curl", "-fsS", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 180s

networks:
  llm:
    name: llm
    driver: bridge