From 4aa2cd9468c4bde379d9c10c79b418d9ce0a1f03 Mon Sep 17 00:00:00 2001
From: ginnoir <ginnoir@users.noreply.github.com>
Date: Fri, 26 Jun 2026 17:04:17 -0500
Subject: [PATCH] fix(llm): q8_0 V-cache (q4_0 tanked generation to 1.3 tok/s
 on Pascal)

The q4_0 V-cache + flash-attention path is pathological on the GP100: 1.28
tok/s generation at 5-8% GPU util. q8_0 V-cache gives 9.2 tok/s and still fits
64k context in 16GB (15.3GB used, ~950MB free).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 stacks/llm/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stacks/llm/docker-compose.yml b/stacks/llm/docker-compose.yml
index 3721c3e..012df4f 100644
--- a/stacks/llm/docker-compose.yml
+++ b/stacks/llm/docker-compose.yml
@@ -51,7 +51,7 @@ services:
       - "--cache-type-k"
       - "q8_0"
       - "--cache-type-v"
-      - "q4_0"
+      - "q8_0"
       - "--host"
       - "0.0.0.0"
       - "--port"