From 265b407e8dc5648a6ad2fa1c4cb4212201f74276 Mon Sep 17 00:00:00 2001
From: ginnoir <ginnoir@users.noreply.github.com>
Date: Fri, 26 Jun 2026 16:39:56 -0500
Subject: [PATCH] feat(llm): serve 64k context (YaRN) to meet Hermes' 64K
 minimum
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Hermes Agent rejects models with <64K context. Qwen2.5-14B is 32k native, so
enable YaRN rope-scaling (2x → 65536) and drop the V-cache to q4_0 for VRAM
headroom on the 16GB P100.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 stacks/llm/docker-compose.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/stacks/llm/docker-compose.yml b/stacks/llm/docker-compose.yml
index 6b17f6d..743e5b5 100644
--- a/stacks/llm/docker-compose.yml
+++ b/stacks/llm/docker-compose.yml
@@ -35,13 +35,19 @@ services:
       - "-ngl"
       - "99"
       - "--ctx-size"
+      - "65536"
+      - "--rope-scaling"
+      - "yarn"
+      - "--rope-scale"
+      - "2"
+      - "--yarn-orig-ctx"
       - "32768"
       - "--flash-attn"
       - "on"
       - "--cache-type-k"
       - "q8_0"
       - "--cache-type-v"
-      - "q8_0"
+      - "q4_0"
       - "--host"
       - "0.0.0.0"
       - "--port"