From 9938d46a6751b386938b37d8d38ab010ac8cd7ed Mon Sep 17 00:00:00 2001 From: Joseph Costa Date: Sun, 5 Jul 2026 02:05:16 -0500 Subject: [PATCH] =?UTF-8?q?Initial=20commit:=20llm-router=20=E2=80=94=20sm?= =?UTF-8?q?art=20OpenAI/Anthropic/Ollama=20router?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A stdlib pre-router in front of Ollama with LiteLLM backend: - auto model selection by content/tools/modality, with fallbacks - OpenAI /v1, Anthropic /v1/messages, and Ollama-native /api/* endpoints - Whisper-shaped /v1/audio/transcriptions + in-chat audio - key-based fleet policies (e.g. force a client onto uncensored models) - optional Bearer auth; launchd/systemd service install - benchmark harnesses (speed, quality, agentic tool use) with sample results Co-Authored-By: Claude Opus 4.8 --- .gitignore | 18 + LICENSE | 21 + README.md | 126 +++++ apikey.example | 8 + benchmarks/agent_bench.py | 159 ++++++ benchmarks/benchmark.py | 196 ++++++++ benchmarks/hard_agent_bench.py | 181 +++++++ benchmarks/quality_bench.py | 242 ++++++++++ benchmarks/results/agent_report.md | 20 + benchmarks/results/agent_results.json | 128 +++++ benchmarks/results/bench_report.md | 31 ++ benchmarks/results/combined_report.md | 38 ++ benchmarks/results/hard_agent_report.md | 11 + benchmarks/results/hard_agent_results.json | 52 ++ benchmarks/results/quality_report.md | 20 + benchmarks/results/quality_results.json | 121 +++++ deploy/com.llm-router.plist.template | 32 ++ deploy/install-service.sh | 34 ++ deploy/llm-router.service.template | 16 + docs/AUDIO.md | 45 ++ docs/BENCHMARKS.md | 40 ++ docs/DEPLOY.md | 69 +++ docs/ROUTING.md | 68 +++ litellm.config.yaml | 34 ++ requirements.txt | 2 + router.py | 537 +++++++++++++++++++++ run.sh | 38 ++ scripts/make-context-variant.sh | 20 + scripts/pull-models.sh | 39 ++ scripts/setup.sh | 23 + start.sh | 41 ++ uncensored_key.example | 9 + 32 files changed, 2419 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 apikey.example create mode 100644 benchmarks/agent_bench.py create mode 100644 benchmarks/benchmark.py create mode 100644 benchmarks/hard_agent_bench.py create mode 100644 benchmarks/quality_bench.py create mode 100644 benchmarks/results/agent_report.md create mode 100644 benchmarks/results/agent_results.json create mode 100644 benchmarks/results/bench_report.md create mode 100644 benchmarks/results/combined_report.md create mode 100644 benchmarks/results/hard_agent_report.md create mode 100644 benchmarks/results/hard_agent_results.json create mode 100644 benchmarks/results/quality_report.md create mode 100644 benchmarks/results/quality_results.json create mode 100644 deploy/com.llm-router.plist.template create mode 100755 deploy/install-service.sh create mode 100644 deploy/llm-router.service.template create mode 100644 docs/AUDIO.md create mode 100644 docs/BENCHMARKS.md create mode 100644 docs/DEPLOY.md create mode 100644 docs/ROUTING.md create mode 100644 litellm.config.yaml create mode 100644 requirements.txt create mode 100644 router.py create mode 100755 run.sh create mode 100755 scripts/make-context-variant.sh create mode 100755 scripts/pull-models.sh create mode 100755 scripts/setup.sh create mode 100755 start.sh create mode 100644 uncensored_key.example diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d22d32f --- /dev/null +++ b/.gitignore @@ -0,0 +1,18 @@ +# secrets — never commit real keys +.apikey +.uncensored_key + +# python / venv +.venv/ +__pycache__/ +*.pyc + +# runtime state +*.pid +*.log + +# scratch model definitions (generated by scripts/make-context-variant.sh) +Modelfile.* + +# OS +.DS_Store diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..786f736 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Joseph Costa + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..ab1c776 --- /dev/null +++ b/README.md @@ -0,0 +1,126 @@ +# llm-router + +A small, dependency-light **smart router** in front of [Ollama](https://ollama.com). +It speaks the **OpenAI** and **Anthropic** APIs, picks the best local model per +request (by content, tools, modality), adds fallbacks, and bolts on the pieces +Ollama lacks — **audio transcription** and **identity-based model policies**. + +``` + OpenAI / Anthropic / Ollama clients + │ + ▼ + pre-router (:8080, this repo) ← smart model selection, auth, audio, policy + │ │ + │ OpenAI /v1 │ native /api/* + ▼ ▼ + LiteLLM (:4000) Ollama (:11434) ← LiteLLM = fallbacks/logging; Ollama = the models + └──────┬──────────┘ + ▼ + Ollama models +``` + +The pre-router is **pure Python standard library** (no pip deps) — it runs on any +Python 3.9+. LiteLLM (the backend proxy) needs a small venv. + +## What it does + +- **Auto model selection** — `model: "auto"` routes by content: code → a coding + model, images → a vision model, tool calls → an agentic model, short chats → a + small fast model, everything else → a classifier or general model. +- **Multiple wire protocols** — OpenAI `/v1/chat/completions`, Anthropic + `/v1/messages` (so Claude Code works), and Ollama-native `/api/*` (so + `ollama launch` sees it as a real Ollama). +- **Audio** — an OpenAI-compatible `/v1/audio/transcriptions` (Whisper-shaped) + endpoint plus in-chat `input_audio`, backed by a speech-capable local model. + See [docs/AUDIO.md](docs/AUDIO.md). +- **Fallbacks** — if a model errors (or isn't pulled yet) the request falls back + to a related model automatically. +- **Policies** — a dedicated API key can force a client onto a specific fleet + (e.g. uncensored models), regardless of what model it requests. +- **Optional auth** — Bearer-token gate for when you expose it beyond localhost. + +## Requirements + +- [Ollama](https://ollama.com) running locally with some models pulled +- **Python 3.12** (for the LiteLLM venv). macOS: `brew install python@3.12`. + Linux: your distro's `python3.12`. +- The pre-router itself only needs any Python 3.9+. + +## Quickstart + +```bash +git clone llm-router && cd llm-router + +# 1. one-time: create the LiteLLM venv +./scripts/setup.sh + +# 2. pull a model fleet (edit the list first — see the script) +./scripts/pull-models.sh + +# 3. run it (LiteLLM on :4000, router on :8080) +./run.sh +``` + +Then point any OpenAI client at `http://localhost:8080/v1` with `model: "auto"`: + +```bash +curl http://localhost:8080/v1/chat/completions -H 'content-type: application/json' \ + -d '{"model":"auto","messages":[{"role":"user","content":"hello"}]}' +``` + +Run it as a boot service (launchd on macOS, systemd on Linux): +see [docs/DEPLOY.md](docs/DEPLOY.md). + +## Configuration + +Everything is environment variables (read by `run.sh` / the router): + +| Var | Default | Purpose | +|---|---|---| +| `ROUTER_HOST` | `0.0.0.0` | bind address (use `127.0.0.1` to keep it local-only) | +| `ROUTER_PORT` | `8080` | router port | +| `ROUTER_OLLAMA` | `http://127.0.0.1:11434` | how the router reaches Ollama (client URL) | +| `ROUTER_UPSTREAM` | `http://127.0.0.1:4000/v1` | LiteLLM proxy URL | +| `ROUTER_AUDIO_MODEL` | `gemma4:e4b` | model used for audio/transcription | +| `ROUTER_API_KEY` | *(unset)* | if set, require this Bearer token on /v1 | +| `ROUTER_UNCENSORED_KEY` | *(unset)* | Bearer token that forces the uncensored fleet | + +> ⚠️ `ROUTER_OLLAMA` is the router's **client** target — keep it `127.0.0.1`. +> Do **not** confuse it with Ollama's own `OLLAMA_HOST` server-bind variable. + +**Keys** are loaded from files so they stay out of git: put a token in `.apikey` +and/or `.uncensored_key` (see the `*.example` files). Both are `.gitignore`d. + +**The model map** lives at the top of [`router.py`](router.py) (the `M` and +`UNCENSORED` dicts). Edit those to match the models you've pulled — this is the +main thing to customize for your machine. See [docs/ROUTING.md](docs/ROUTING.md). + +## Endpoints + +| Method | Path | Notes | +|---|---|---| +| POST | `/v1/chat/completions` | OpenAI chat; `model:"auto"` = smart routing | +| POST | `/v1/messages` | Anthropic Messages (Claude Code) → LiteLLM | +| POST | `/v1/audio/transcriptions` | Whisper-shaped speech-to-text | +| POST | `/v1/audio/translations` | speech → English text | +| GET | `/v1/models` | lists route targets | +| GET | `/`, `/api/version`, `/api/tags`, `/api/ps`, POST `/api/show` | Ollama-native probes (so `ollama launch` accepts the router) | +| GET | `/healthz` | liveness | + +## Docs + +- [docs/DEPLOY.md](docs/DEPLOY.md) — run as a service, expose it, tunnels, Caddy +- [docs/ROUTING.md](docs/ROUTING.md) — how routing decides + customizing the map +- [docs/AUDIO.md](docs/AUDIO.md) — voice input & transcription +- [docs/BENCHMARKS.md](docs/BENCHMARKS.md) — the benchmark harnesses & sample results + +## Security + +- Default bind is `0.0.0.0` (LAN-reachable). Set `ROUTER_HOST=127.0.0.1` for + local-only, or set `ROUTER_API_KEY` before exposing it anywhere. +- The Ollama-native `/api/*` endpoints the router serves are **read-only** + (version/tags/show) — there is no unauthenticated `/api/chat` inference path. + +## License + +MIT — see [LICENSE](LICENSE). diff --git a/apikey.example b/apikey.example new file mode 100644 index 0000000..c2f9ded --- /dev/null +++ b/apikey.example @@ -0,0 +1,8 @@ +Copy this file to `.apikey` and put a single secret token in it (no newline needed). +When `.apikey` exists, run.sh sets ROUTER_API_KEY and the router requires +`Authorization: Bearer ` on all /v1 requests. + +If `.apikey` does NOT exist, the router runs OPEN (no auth) — fine on a trusted +LAN, NOT fine if you expose it to the internet. + +Generate one: openssl rand -hex 24 diff --git a/benchmarks/agent_bench.py b/benchmarks/agent_bench.py new file mode 100644 index 0000000..70f0fee --- /dev/null +++ b/benchmarks/agent_bench.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +""" +Agentic tool-use benchmark for the local fleet, via Ollama's native tools API. + +Scores the real agent failure modes (each 0/1, objective): + A invoke + args : calls calculator for a math Q, expression evaluates correctly + B tool selection : picks get_weather (not calc/search) for a weather Q, right city + E multi-arg : get_weather with correct city AND unit + C use-result : calls tool, we return a value, final answer uses it + D abstain : does NOT call a tool for a plain greeting + +overall = mean of the 6 checks. think disabled for a fast, direct baseline. + +Run after models are downloaded: ./.venv/bin/python agent_bench.py +Writes agent_report.md and agent_results.json. +""" +import json, re, datetime, urllib.request +import benchmark as b + +OLLAMA = "http://127.0.0.1:11434" + +CALC = {"type": "function", "function": { + "name": "calculator", + "description": "Evaluate an arithmetic expression and return the number.", + "parameters": {"type": "object", + "properties": {"expression": {"type": "string", + "description": "e.g. '3*4+1'"}}, + "required": ["expression"]}}} +WEATHER = {"type": "function", "function": { + "name": "get_weather", + "description": "Get the current weather for a city.", + "parameters": {"type": "object", + "properties": {"city": {"type": "string"}, + "unit": {"type": "string", + "enum": ["celsius", "fahrenheit"]}}, + "required": ["city", "unit"]}}} +SEARCH = {"type": "function", "function": { + "name": "search_kb", + "description": "Search the internal company knowledge base for a query.", + "parameters": {"type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"]}}} + + +def chat_tools(model, messages, tools, num_predict=512, think=False): + body = {"model": model, "messages": messages, "tools": tools, "stream": False, + "keep_alive": "30m", "options": {"temperature": 0, "num_predict": num_predict}} + if think is not None: + body["think"] = think + req = urllib.request.Request(OLLAMA + "/api/chat", data=json.dumps(body).encode(), + headers={"content-type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=600) as r: + return json.loads(r.read()).get("message", {}) or {} + except Exception: + # some models reject think param with tools; retry without + body.pop("think", None) + req = urllib.request.Request(OLLAMA + "/api/chat", data=json.dumps(body).encode(), + headers={"content-type": "application/json"}) + with urllib.request.urlopen(req, timeout=600) as r: + return json.loads(r.read()).get("message", {}) or {} + + +def first_call(msg): + tcs = msg.get("tool_calls") or [] + if not tcs: + return None, {} + fn = tcs[0].get("function", {}) or {} + args = fn.get("arguments") + if isinstance(args, str): + try: + args = json.loads(args) + except Exception: + args = {} + return fn.get("name"), (args or {}) + + +def safe_eval(expr): + if not re.fullmatch(r"[0-9+\-*/(). ]+", expr or ""): + raise ValueError("unsafe") + return eval(expr, {"__builtins__": {}}, {}) + + +def run_agent(model): + c = {} + # A: invoke calculator + correct expression + name, args = first_call(chat_tools(model, [b.user("What is 3847 * 293? Use the calculator tool.")], [CALC])) + c["invoke"] = 1 if name == "calculator" else 0 + try: + c["args"] = 1 if c["invoke"] and abs(safe_eval(str(args.get("expression", ""))) - 3847 * 293) < 1e-6 else 0 + except Exception: + c["args"] = 0 + + # B: tool selection among three + right city + name, args = first_call(chat_tools(model, [b.user("What's the weather in Tokyo, in celsius?")], + [CALC, WEATHER, SEARCH])) + c["select"] = 1 if name == "get_weather" and "tokyo" in str(args.get("city", "")).lower() else 0 + + # E: multi-arg correctness + name, args = first_call(chat_tools(model, [b.user("Get the weather in Paris in fahrenheit.")], + [CALC, WEATHER, SEARCH])) + c["multiarg"] = 1 if (name == "get_weather" and "paris" in str(args.get("city", "")).lower() + and "f" in str(args.get("unit", "")).lower()) else 0 + + # C: multi-turn, use the returned result + msgs = [b.user("I have 17 boxes with 24 apples each. How many apples total? Use the calculator.")] + msg = chat_tools(model, msgs, [CALC]) + name, _ = first_call(msg) + c["useresult"] = 0 + if name == "calculator": + msgs.append({"role": "assistant", "content": "", "tool_calls": msg.get("tool_calls")}) + msgs.append({"role": "tool", "content": "408"}) + final = chat_tools(model, msgs, [CALC], num_predict=256) + c["useresult"] = 1 if "408" in (final.get("content", "") or "") else 0 + + # D: abstain when no tool is needed + name, _ = first_call(chat_tools(model, [b.user("Say hello in exactly one word.")], + [CALC, WEATHER, SEARCH])) + c["abstain"] = 1 if name is None else 0 + + c["overall"] = sum(c[k] for k in ("invoke", "args", "select", "multiarg", "useresult", "abstain")) / 6 + return c + + +def main(): + models = b.list_models() + results = {} + for i, m in enumerate(models, 1): + print(f"[{i}/{len(models)}] {m} ...", flush=True) + try: + results[m] = run_agent(m) + s = results[m] + print(f" overall {s['overall']*100:4.0f}% invoke {s['invoke']} args {s['args']} " + f"select {s['select']} multiarg {s['multiarg']} useresult {s['useresult']} " + f"abstain {s['abstain']}", flush=True) + except Exception as e: + print(f" ERROR: {e}", flush=True) + results[m] = {"error": str(e)} + + with open("agent_results.json", "w") as f: + json.dump(results, f, indent=2) + + ts = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") + ranked = sorted([(m, s) for m, s in results.items() if "overall" in s], + key=lambda x: x[1]["overall"], reverse=True) + with open("agent_report.md", "w") as f: + f.write(f"# Agentic tool-use benchmark — {ts}\n\n") + f.write("Via Ollama tools API. Each column 0/1 (or %). abstain = correctly did NOT " + "call a tool when none was needed.\n\n") + f.write("| Model | Overall | Invoke | Args | Select | MultiArg | UseResult | Abstain |\n") + f.write("|---|--:|:--:|:--:|:--:|:--:|:--:|:--:|\n") + for m, s in ranked: + f.write(f"| `{m}` | **{s['overall']*100:.0f}%** | {s['invoke']} | {s['args']} | " + f"{s['select']} | {s['multiarg']} | {s['useresult']} | {s['abstain']} |\n") + print("\nWrote agent_report.md and agent_results.json") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/benchmark.py b/benchmarks/benchmark.py new file mode 100644 index 0000000..37b6357 --- /dev/null +++ b/benchmarks/benchmark.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +""" +Full test + benchmark suite for the local model fleet. + + Perf : per-model generation tok/s, prompt tok/s, cold load time, GPU placement + (from Ollama's own eval metrics - objective, not wall-clock guesses) + Function : every model returns non-empty output; vision models pass an OCR check + Routing : representative prompts hit the router and must land on the right model + +Run AFTER all models are downloaded and the router is up: + ./.venv/bin/python benchmark.py # or: python3 benchmark.py +Writes bench_report.md and prints a summary table. +""" +import json, time, base64, io, re, subprocess, urllib.request, urllib.error, datetime, sys + +OLLAMA = "http://127.0.0.1:11434" +ROUTER = "http://127.0.0.1:8080" +NUM_PREDICT = 200 +PERF_PROMPT = ("Explain how TCP congestion control works (slow start, congestion " + "avoidance, fast retransmit) in about 150 words.") + +VISION_MODELS = ("qwen3-vl", "gemma4", "gemma3") # accept image input + + +def user(txt): + return {"role": "user", "content": txt} + + +def img_user(txt, b64): # OpenAI format — for the router (LiteLLM) path + return {"role": "user", "content": [ + {"type": "text", "text": txt}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64}}]} + + +def img_user_native(txt, b64): # Ollama-native format — for direct /api/chat + return {"role": "user", "content": txt, "images": [b64]} + + +def ollama_chat(model, messages, num_predict=NUM_PREDICT, timeout=1800, think=None): + body = {"model": model, "messages": messages, "stream": False, + "keep_alive": "30m", "options": {"temperature": 0, "num_predict": num_predict}} + if think is not None: + body["think"] = think + req = urllib.request.Request(OLLAMA + "/api/chat", data=json.dumps(body).encode(), + headers={"content-type": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.loads(r.read()) + + +def metrics(d): + sec = lambda ns: (ns or 0) / 1e9 + ec, ed = d.get("eval_count", 0), sec(d.get("eval_duration")) + pc, pd = d.get("prompt_eval_count", 0), sec(d.get("prompt_eval_duration")) + return {"gen_tps": ec / ed if ed else 0, + "prompt_tps": pc / pd if pd else 0, + "load_s": sec(d.get("load_duration")), + "ttft_s": sec(d.get("load_duration")) + pd, + "gen_tokens": ec, + "text": (d.get("message") or {}).get("content", "")} + + +def ps_info(model): + try: + out = subprocess.run(["ollama", "ps"], capture_output=True, text=True, timeout=15).stdout + for line in out.splitlines()[1:]: + cols = re.split(r"\s{2,}", line.strip()) + if cols and cols[0] == model: + size = cols[2] if len(cols) > 2 else "" + proc = cols[3] if len(cols) > 3 else "" + return size, proc + except Exception: + pass + return "", "" + + +def make_ocr_image(text): + from PIL import Image, ImageDraw, ImageFont + img = Image.new("RGB", (760, 220), "white") + d = ImageDraw.Draw(img) + font = None + for path in ("/System/Library/Fonts/Supplemental/Arial.ttf", + "/System/Library/Fonts/Helvetica.ttc", + "/Library/Fonts/Arial.ttf"): + try: + font = ImageFont.truetype(path, 72); break + except Exception: + pass + d.text((40, 70), text, fill="black", font=font) + buf = io.BytesIO(); img.save(buf, format="PNG") + return base64.b64encode(buf.getvalue()).decode() + + +def list_models(): + out = subprocess.run(["ollama", "list"], capture_output=True, text=True).stdout + return [ln.split()[0] for ln in out.splitlines()[1:] if ln.strip()] + + +def router_call(messages, model="auto", timeout=1800): + body = {"model": model, "messages": messages, "stream": False, + "options": {"num_predict": 32}} + req = urllib.request.Request(ROUTER + "/v1/chat/completions", + data=json.dumps(body).encode(), + headers={"content-type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + # compare the routing DECISION, not the post-fallback served model + picked = r.headers.get("x-router-initial-model") or r.headers.get("x-router-model", "?") + data = json.loads(r.read()) + content = data["choices"][0]["message"]["content"] + return 200, picked, content + except urllib.error.HTTPError as e: + return e.code, "-", e.read().decode(errors="replace")[:120] + except Exception as e: + return 0, "-", str(e)[:120] + + +def main(): + models = list_models() + ocr_text = "INVOICE-7X42" + img = None + try: + img = make_ocr_image(ocr_text) + except Exception as e: + print(f"(pillow unavailable, OCR check skipped: {e})") + + print(f"Benchmarking {len(models)} models ...\n") + perf = [] + for m in models: + vis = any(v in m for v in VISION_MODELS) + try: + ollama_chat(m, [user("hi")], num_predict=4) # cold-load / warm + d = ollama_chat(m, [user(PERF_PROMPT)]) # measured run + mm = metrics(d) + size, proc = ps_info(m) + ocr = "-" + if vis and img: + try: + r = ollama_chat(m, [img_user_native( + "What text is shown in this image?", img)], 256) + msg = r.get("message") or {} + got = (msg.get("content") or "") + " " + (msg.get("thinking") or "") + ocr = "PASS" if ocr_text.replace("-", "").lower() in \ + got.replace("-", "").replace(" ", "").lower() else "miss" + except Exception: + ocr = "err" + ok = "ok" if mm["gen_tokens"] > 0 else "EMPTY" + perf.append((m, mm, size, proc, ocr, ok)) + print(f" {m:32s} {mm['gen_tps']:6.1f} tok/s load {mm['load_s']:5.1f}s " + f"{proc or '?':>9s} ocr={ocr}") + except Exception as e: + perf.append((m, None, "", "", "-", f"ERROR {e}")) + print(f" {m:32s} ERROR: {e}") + + # routing correctness + print("\nRouting tests:") + cases = [ + ("short chat", [user("hey, how's it going?")], "qwen3:8b"), + ("code", [user("Refactor this function and fix the bug in app.py")], "glm-4.7-flash"), + ("repo code", [user("Refactor auth across the whole codebase, multiple files")], "qwen3-coder:30b"), + ("reasoning", [user("Prove step by step why sqrt(2) is irrational")], "qwen3.6:35b-a3b"), + ] + if img: + cases += [ + ("vision", [img_user("what is in this image?", img)], "qwen3-vl:8b"), + ("ocr/doc", [img_user("extract the text from this document table", img)], "qwen3-vl:30b-a3b-instruct"), + ] + routing = [] + for name, msgs, expect in cases: + code, picked, _ = router_call(msgs) + verdict = "PASS" if picked == expect else "DIFF" + routing.append((name, expect, picked, code, verdict)) + print(f" {name:12s} expect {expect:26s} got {picked:26s} [{verdict}]") + + # write report + ts = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") + with open("bench_report.md", "w") as f: + f.write(f"# Local fleet benchmark — {ts}\n\n") + f.write("## Performance (sorted by generation tok/s)\n\n") + f.write("| Model | Gen tok/s | Prompt tok/s | Cold load | TTFT | Placement | OCR | Status |\n") + f.write("|---|--:|--:|--:|--:|:--|:--:|:--|\n") + for m, mm, size, proc, ocr, ok in sorted( + perf, key=lambda x: (x[1]["gen_tps"] if x[1] else -1), reverse=True): + if mm: + f.write(f"| `{m}` | {mm['gen_tps']:.1f} | {mm['prompt_tps']:.0f} | " + f"{mm['load_s']:.1f}s | {mm['ttft_s']:.1f}s | {proc or '?'} | {ocr} | {ok} |\n") + else: + f.write(f"| `{m}` | - | - | - | - | - | - | {ok} |\n") + f.write("\n## Routing correctness\n\n") + f.write("| Case | Expected | Routed to | HTTP | Verdict |\n|---|---|---|--:|:--:|\n") + for name, expect, picked, code, verdict in routing: + f.write(f"| {name} | `{expect}` | `{picked}` | {code} | {verdict} |\n") + print("\nWrote bench_report.md") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/hard_agent_bench.py b/benchmarks/hard_agent_bench.py new file mode 100644 index 0000000..2d9d91b --- /dev/null +++ b/benchmarks/hard_agent_bench.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +""" +Hard agentic benchmark — real multi-turn tool loop (tools actually execute and +results feed back). Discriminates models on complex tool use, not just format. + +Scenarios (each 0/1, objective): + seq sequential dependency (weather Tokyo -> *3 -> 66) + parallel two entities combined (Tokyo & Paris temps: 22 & 18) + chain 3-hop (AAPL price -> convert USD->EUR ~166) + select right tool w/ distractors (15% of AAPL price -> 27) + recover tool error -> retry corrected (Tokio -> Tokyo -> 22) + abstain answer directly, no needless tool call (capital of France) + honesty no fabricated success when the tool doesn't exist (book a flight) + +Thinking ENABLED (agents plan). Usage: + ./.venv/bin/python hard_agent_bench.py [model1 model2 ...] +Default set = the agentic-default candidates. Writes hard_agent_report.md + json. +""" +import json, sys, datetime +import benchmark as b +import agent_bench as ab + +# ------------------------------------------------------------------ tools +CALC = {"type": "function", "function": { + "name": "calculator", "description": "Evaluate an arithmetic expression.", + "parameters": {"type": "object", "properties": {"expression": {"type": "string"}}, + "required": ["expression"]}}} +WEATHER = {"type": "function", "function": { + "name": "get_weather", "description": "Current temperature for a city.", + "parameters": {"type": "object", + "properties": {"city": {"type": "string"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}}, + "required": ["city", "unit"]}}} +STOCK = {"type": "function", "function": { + "name": "get_stock_price", "description": "Current stock price by ticker (USD).", + "parameters": {"type": "object", "properties": {"ticker": {"type": "string"}}, + "required": ["ticker"]}}} +CONVERT = {"type": "function", "function": { + "name": "convert_currency", "description": "Convert an amount between currencies.", + "parameters": {"type": "object", + "properties": {"amount": {"type": "number"}, + "from_currency": {"type": "string"}, + "to_currency": {"type": "string"}}, + "required": ["amount", "from_currency", "to_currency"]}}} +SEARCH = {"type": "function", "function": { + "name": "search_kb", "description": "Search the company knowledge base.", + "parameters": {"type": "object", "properties": {"query": {"type": "string"}}, + "required": ["query"]}}} +ALL = [CALC, WEATHER, STOCK, CONVERT, SEARCH] + +WEATHER_DB = {"tokyo": 22, "paris": 18, "london": 12, "new york": 25} +STOCK_DB = {"aapl": 180.0, "tsla": 240.0, "googl": 150.0} +RATES = {("usd", "eur"): 0.92, ("usd", "gbp"): 0.79, ("eur", "usd"): 1.09} + + +def exec_tool(name, args): + try: + if name == "calculator": + return str(ab.safe_eval(str(args.get("expression", "")))) + if name == "get_weather": + city = str(args.get("city", "")).strip().lower() + unit = str(args.get("unit", "celsius")).lower() + if city not in WEATHER_DB: + return json.dumps({"error": f"unknown city {args.get('city')!r}", + "known_cities": list(WEATHER_DB)}) + c = WEATHER_DB[city] + val = c if unit.startswith("c") else round(c * 9 / 5 + 32, 1) + return json.dumps({"city": args.get("city"), "temperature": val, "unit": unit}) + if name == "get_stock_price": + t = str(args.get("ticker", "")).strip().lower() + return (json.dumps({"ticker": t.upper(), "price": STOCK_DB[t], "currency": "USD"}) + if t in STOCK_DB else json.dumps({"error": "unknown ticker"})) + if name == "convert_currency": + amt = float(args.get("amount")) + fr = str(args.get("from_currency", args.get("from", ""))).lower()[:3] + to = str(args.get("to_currency", args.get("to", ""))).lower()[:3] + r = RATES.get((fr, to)) + return (json.dumps({"amount": round(amt * r, 2), "currency": to.upper()}) + if r else json.dumps({"error": "unsupported currency pair"})) + if name == "search_kb": + return json.dumps({"result": "no relevant entry found"}) + return json.dumps({"error": f"unknown tool {name}"}) + except Exception as e: + return json.dumps({"error": str(e)}) + + +def run_loop(model, user_msg, tools, max_turns=6, num_predict=1536): + messages = [{"role": "user", "content": user_msg}] + traj = [] + for _ in range(max_turns): + msg = ab.chat_tools(model, messages, tools, num_predict=num_predict, think=True) + tcs = msg.get("tool_calls") or [] + if not tcs: + return (msg.get("content", "") or ""), traj + messages.append({"role": "assistant", "content": msg.get("content", "") or "", + "tool_calls": tcs}) + for tc in tcs: + fn = tc.get("function", {}) or {} + name = fn.get("name") + args = fn.get("arguments") + if isinstance(args, str): + try: + args = json.loads(args) + except Exception: + args = {} + args = args or {} + traj.append((name, args)) + messages.append({"role": "tool", "content": exec_tool(name, args)}) + messages.append({"role": "user", "content": "Now give your final answer to the original question."}) + msg = ab.chat_tools(model, messages, tools, num_predict=512, think=False) + return (msg.get("content", "") or ""), traj + + +def bench_model(model): + r = {} + out, _ = run_loop(model, "What is the current temperature in Tokyo in Celsius, multiplied by 3? Use the tools.", ALL) + r["seq"] = 1 if "66" in out else 0 + + out, _ = run_loop(model, "Get the current temperature in BOTH Tokyo and Paris in Celsius, and report both.", ALL) + r["parallel"] = 1 if ("22" in out and "18" in out) else 0 + + out, _ = run_loop(model, "What is the current AAPL stock price converted into euros? Use the tools.", ALL) + r["chain"] = 1 if ("165" in out or "166" in out) else 0 + + out, tj = run_loop(model, "What is 15% of the current AAPL stock price? Use the tools.", ALL) + r["select"] = 1 if "27" in out else 0 + + out, tj = run_loop(model, "What's the current temperature in Tokio? (the city in Japan)", ALL) + weather_calls = [a for n, a in tj if n == "get_weather"] + r["recover"] = 1 if ("22" in out and len(weather_calls) >= 2) else 0 + + out, tj = run_loop(model, "What is the capital of France?", [WEATHER, CALC, STOCK]) + r["abstain"] = 1 if (len(tj) == 0 and "paris" in out.lower()) else 0 + + out, tj = run_loop(model, "Book me a flight to Tokyo for tomorrow.", [WEATHER, CALC, STOCK]) + lo = out.lower() + honest = any(w in lo for w in ["can't", "cannot", "can not", "unable", "don't have", + "do not have", "no tool", "not able", "no booking", "no way to"]) + fabricated = any(w in lo for w in ["booked", "confirmed", "reservation is", "your flight is", + "i have booked", "successfully"]) + r["honesty"] = 1 if (honest and not fabricated) else 0 + + keys = ("seq", "parallel", "chain", "select", "recover", "abstain", "honesty") + r["overall"] = sum(r[k] for k in keys) / len(keys) + return r + + +def main(): + models = sys.argv[1:] or ["gemma4:26b", "qwen3.6:35b-a3b", "glm-4.7-flash", + "qwen3-coder:30b", "gpt-oss:20b"] + results = {} + for i, m in enumerate(models, 1): + print(f"[{i}/{len(models)}] {m} ...", flush=True) + try: + results[m] = bench_model(m) + s = results[m] + print(f" overall {s['overall']*100:4.0f}% seq {s['seq']} par {s['parallel']} " + f"chain {s['chain']} sel {s['select']} recover {s['recover']} " + f"abstain {s['abstain']} honest {s['honesty']}", flush=True) + except Exception as e: + print(f" ERROR: {e}", flush=True) + results[m] = {"error": str(e)} + + with open("hard_agent_results.json", "w") as f: + json.dump(results, f, indent=2) + ts = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") + ranked = sorted([(m, s) for m, s in results.items() if "overall" in s], + key=lambda x: x[1]["overall"], reverse=True) + with open("hard_agent_report.md", "w") as f: + f.write(f"# Hard agentic benchmark (multi-turn tool loop) — {ts}\n\n") + f.write("Real tool execution with results fed back; thinking enabled.\n\n") + f.write("| Model | Overall | Seq | Parallel | Chain | Select | Recover | Abstain | Honesty |\n") + f.write("|---|--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|\n") + for m, s in ranked: + f.write(f"| `{m}` | **{s['overall']*100:.0f}%** | {s['seq']} | {s['parallel']} | " + f"{s['chain']} | {s['select']} | {s['recover']} | {s['abstain']} | {s['honesty']} |\n") + print("\nWrote hard_agent_report.md and hard_agent_results.json") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/quality_bench.py b/benchmarks/quality_bench.py new file mode 100644 index 0000000..76cfc00 --- /dev/null +++ b/benchmarks/quality_bench.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +""" +Quality suite for the local fleet — objective auto-grading, then quality-vs-speed. + + code : HumanEval-style problems, generated code EXECUTED against unit tests + math : GSM8K-style word problems, exact integer match + mc : multiple-choice knowledge/reasoning, letter match + ifollow : instruction-following with programmatic checks (bullets / JSON / length) + ocr : multi-image text extraction (vision models only) + +Grades on the model's answer (content); thinking tokens are separate in Ollama. +Also records real tok/s (aggregate eval_count / eval_duration across all calls). + +Run after models are downloaded and Ollama is up: + ./.venv/bin/python quality_bench.py +Writes quality_report.md and quality_results.json. +""" +import json, re, subprocess, tempfile, os, datetime +import benchmark as b # reuse ollama_chat, make_ocr_image, img_user_native, list_models + +# ---------------------------------------------------------------- task banks +MATH = [ + ("Natalia sold clips to 48 friends in April, then half as many in May. " + "How many clips altogether? End with 'Answer: '.", 72), + ("Weng earns $12 an hour babysitting. Yesterday she did 50 minutes. " + "How much did she earn? End with 'Answer: '.", 10), + ("Betty needs $100 for a wallet and has half. Her parents give $15, her " + "grandparents twice that. How much more does she need? End with 'Answer: '.", 5), + ("James writes a 3-page letter to 2 friends twice a week. How many pages " + "per year? End with 'Answer: '.", 624), + ("A robe takes 2 bolts of blue fiber and half that of white. How many bolts " + "total? End with 'Answer: '.", 3), +] + +MC = [ + ("Which planet is largest in our solar system?\nA) Earth B) Jupiter C) Mars D) Venus\n" + "Answer with just the letter.", "B"), + ("What is 15% of 200?\nA) 15 B) 20 C) 30 D) 45\nAnswer with just the letter.", "C"), + ("Antonym of 'ephemeral'?\nA) lasting B) brief C) hollow D) transient\n" + "Answer with just the letter.", "A"), + ("If all Bloops are Razzies and all Razzies are Lazzies, are all Bloops " + "definitely Lazzies?\nA) yes B) no C) cannot tell D) only some\n" + "Answer with just the letter.", "A"), + ("Which number is prime?\nA) 21 B) 27 C) 29 D) 33\nAnswer with just the letter.", "C"), + ("A train goes 60 km in 45 minutes. Speed in km/h?\nA) 45 B) 60 C) 75 D) 80\n" + "Answer with just the letter.", "D"), +] + +CODE = [ + ("Write a Python function `has_close_elements(numbers, threshold)` that returns True " + "if any two numbers in the list are closer than `threshold`, else False. " + "Return only the code in a ```python block.", + "assert has_close_elements([1.0,2.0,3.0],0.5)==False\n" + "assert has_close_elements([1.0,2.8,3.0,4.0,5.0,2.0],0.3)==True\n" + "assert has_close_elements([1.0,2.0,5.9,4.0,5.0],0.95)==True\n" + "assert has_close_elements([1.0,2.0,5.9,4.0,5.0],0.8)==False"), + ("Write a Python function `is_palindrome(s)` that returns True if s is a palindrome " + "ignoring case, spaces, and punctuation. Return only the code in a ```python block.", + "assert is_palindrome('A man, a plan, a canal: Panama')==True\n" + "assert is_palindrome('race a car')==False\n" + "assert is_palindrome('')==True\n" + "assert is_palindrome('Was it a car or a cat I saw?')==True"), + ("Write a Python function `fizzbuzz(n)` returning a list of strings for 1..n: " + "multiples of 3 -> 'Fizz', of 5 -> 'Buzz', both -> 'FizzBuzz', else the number as a " + "string. Return only the code in a ```python block.", + "assert fizzbuzz(5)==['1','2','Fizz','4','Buzz']\n" + "assert fizzbuzz(15)[-1]=='FizzBuzz'\n" + "assert fizzbuzz(3)==['1','2','Fizz']"), +] + +IFOLLOW = [ + ("List exactly three primary colors. Format your ENTIRE response as exactly three " + "bullet points, each line starting with '- ' and nothing else.", "bullets"), + ("Output ONLY a valid JSON object with keys \"name\" (string) and \"age\" (number) " + "for a person named Alice who is 30. No markdown, no extra text.", "json"), + ("Reply with a single sentence of fewer than 10 words describing the sun.", "short"), +] + +OCR_TEXTS = ["INVOICE-7X42", "Total: $1,284.50", "SKU-99Z-KAPPA"] +VISION = ("qwen3-vl", "gemma4", "gemma3") + +# ---------------------------------------------------------------- extractors / graders +def extract_int(text): + m = re.findall(r"[Aa]nswer[^0-9\-]*(-?\d[\d,]*)", text) + nums = re.findall(r"-?\d[\d,]*", text) + cand = m[-1] if m else (nums[-1] if nums else None) + if cand is None: + return None + try: + return int(float(cand.replace(",", ""))) + except Exception: + return None + +def extract_letter(text): + m = re.search(r"(?:answer|correct)\D{0,12}([A-D])\b", text, re.I) + if m: + return m.group(1).upper() + allm = re.findall(r"\b([A-D])\b", text) + return allm[-1] if allm else None + +def extract_code(content, thinking): + for src in (content, content + "\n" + thinking): + m = re.search(r"```(?:python)?\s*(.*?)```", src, re.S) + if m and "def " in m.group(1): + return m.group(1) + i = src.find("def ") + if i >= 0: + return src[i:] + return "" + +def run_code(code, tests): + prog = code + "\n\n" + tests + "\nprint('__OK__')\n" + with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as f: + f.write(prog); path = f.name + try: + r = subprocess.run(["python3", path], capture_output=True, text=True, timeout=12) + return 1 if r.returncode == 0 and "__OK__" in r.stdout else 0 + except Exception: + return 0 + finally: + os.unlink(path) + +def check_if(kind, text): + t = text.strip() + if kind == "bullets": + lines = [l for l in t.splitlines() if l.strip()] + return 1 if len(lines) == 3 and all(l.strip().startswith("- ") for l in lines) else 0 + if kind == "json": + m = re.search(r"\{.*\}", re.sub(r"```(json)?", "", t), re.S) + if not m: + return 0 + try: + d = json.loads(m.group(0)) + return 1 if str(d.get("name", "")).lower() == "alice" and int(d.get("age")) == 30 else 0 + except Exception: + return 0 + if kind == "short": + n = len(t.split()) + return 1 if 0 < n < 10 else 0 + return 0 + +# ---------------------------------------------------------------- per-model run +class Meter: + def __init__(self): + self.ec = 0.0; self.ed = 0.0 + def call(self, model, messages, num_predict): + # think disabled for a fast, direct-answer, apples-to-apples baseline; + # fall back if a model rejects the param + try: + d = b.ollama_chat(model, messages, num_predict, think=False) + except Exception: + d = b.ollama_chat(model, messages, num_predict) + self.ec += d.get("eval_count", 0) or 0 + self.ed += (d.get("eval_duration", 0) or 0) / 1e9 + msg = d.get("message") or {} + return msg.get("content", "") or "", msg.get("thinking", "") or "" + def tps(self): + return self.ec / self.ed if self.ed else 0.0 + + +def run_model(model, ocr_imgs): + vis = any(v in model for v in VISION) + m = Meter() + scores = {} + + got = sum(1 for q, a in MATH + if extract_int(m.call(model, [b.user(q)], 320)[0]) == a) + scores["math"] = got / len(MATH) + + got = sum(1 for q, a in MC + if extract_letter(m.call(model, [b.user(q)], 48)[0]) == a) + scores["mc"] = got / len(MC) + + got = 0 + for q, tests in CODE: + c, th = m.call(model, [b.user(q)], 512) + got += run_code(extract_code(c, th), tests) + scores["code"] = got / len(CODE) + + got = sum(check_if(kind, m.call(model, [b.user(q)], 160)[0]) for q, kind in IFOLLOW) + scores["ifollow"] = got / len(IFOLLOW) + + if vis and ocr_imgs: + got = 0 + for text, img in zip(OCR_TEXTS, ocr_imgs): + c, th = m.call(model, [b.img_user_native("What text is shown in this image?", img)], 256) + blob = (c + " " + th).replace("-", "").replace(" ", "").lower() + got += 1 if text.replace("-", "").replace(" ", "").replace(",", "").replace("$", "").lower()[:8] \ + in blob.replace(",", "").replace("$", "") else 0 + scores["ocr"] = got / len(OCR_TEXTS) + + text_cats = [scores[k] for k in ("math", "mc", "code", "ifollow")] + scores["overall"] = sum(text_cats) / len(text_cats) + scores["tps"] = m.tps() + return scores + + +def main(): + models = b.list_models() + ocr_imgs = None + try: + ocr_imgs = [b.make_ocr_image(t) for t in OCR_TEXTS] + except Exception as e: + print("pillow unavailable, OCR skipped:", e) + + results = {} + for i, model in enumerate(models, 1): + print(f"[{i}/{len(models)}] {model} ...", flush=True) + try: + results[model] = run_model(model, ocr_imgs) + s = results[model] + print(f" overall {s['overall']*100:4.0f}% " + f"(code {s['code']*100:.0f} math {s['math']*100:.0f} " + f"mc {s['mc']*100:.0f} if {s['ifollow']*100:.0f}" + f"{' ocr '+str(int(s.get('ocr',0)*100)) if 'ocr' in s else ''}) " + f"{s['tps']:.1f} tok/s", flush=True) + except Exception as e: + print(f" ERROR: {e}", flush=True) + results[model] = {"error": str(e)} + + with open("quality_results.json", "w") as f: + json.dump(results, f, indent=2) + + ts = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") + ranked = sorted([(m, s) for m, s in results.items() if "overall" in s], + key=lambda x: x[1]["overall"], reverse=True) + with open("quality_report.md", "w") as f: + f.write(f"# Quality benchmark — {ts}\n\n") + f.write("Objective auto-grading. code=executed unit tests, math=GSM8K-style exact, " + "mc=multiple-choice, ifollow=programmatic checks, ocr=multi-image text.\n\n") + f.write("| Model | Overall | Code | Math | MC | Instr | OCR | tok/s |\n") + f.write("|---|--:|--:|--:|--:|--:|--:|--:|\n") + for m, s in ranked: + ocr = f"{s['ocr']*100:.0f}" if "ocr" in s else "—" + f.write(f"| `{m}` | **{s['overall']*100:.0f}%** | {s['code']*100:.0f} | " + f"{s['math']*100:.0f} | {s['mc']*100:.0f} | {s['ifollow']*100:.0f} | " + f"{ocr} | {s['tps']:.1f} |\n") + print("\nWrote quality_report.md and quality_results.json") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/results/agent_report.md b/benchmarks/results/agent_report.md new file mode 100644 index 0000000..c201a63 --- /dev/null +++ b/benchmarks/results/agent_report.md @@ -0,0 +1,20 @@ +# Agentic tool-use benchmark — 2026-07-03 18:18 + +Via Ollama tools API. Each column 0/1 (or %). abstain = correctly did NOT call a tool when none was needed. + +| Model | Overall | Invoke | Args | Select | MultiArg | UseResult | Abstain | +|---|--:|:--:|:--:|:--:|:--:|:--:|:--:| +| `qwen3-vl:30b-a3b-instruct` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 | +| `qwen3-vl:8b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 | +| `qwen3-coder:30b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 | +| `glm-4.7-flash:latest` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 | +| `gpt-oss:20b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 | +| `qwen3.6:35b-a3b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 | +| `gemma4:31b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 | +| `gemma4:26b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 | +| `qwen3.6:27b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 | +| `gemma4:e4b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 | +| `qwen3:14b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 | +| `gemma4:12b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 | +| `gemma4:e2b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 | +| `qwen3:8b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 | diff --git a/benchmarks/results/agent_results.json b/benchmarks/results/agent_results.json new file mode 100644 index 0000000..5dbc411 --- /dev/null +++ b/benchmarks/results/agent_results.json @@ -0,0 +1,128 @@ +{ + "qwen3-vl:30b-a3b-instruct": { + "invoke": 1, + "args": 1, + "select": 1, + "multiarg": 1, + "useresult": 1, + "abstain": 1, + "overall": 1.0 + }, + "qwen3-vl:8b": { + "invoke": 1, + "args": 1, + "select": 1, + "multiarg": 1, + "useresult": 1, + "abstain": 1, + "overall": 1.0 + }, + "qwen3-coder:30b": { + "invoke": 1, + "args": 1, + "select": 1, + "multiarg": 1, + "useresult": 1, + "abstain": 1, + "overall": 1.0 + }, + "glm-4.7-flash:latest": { + "invoke": 1, + "args": 1, + "select": 1, + "multiarg": 1, + "useresult": 1, + "abstain": 1, + "overall": 1.0 + }, + "gpt-oss:20b": { + "invoke": 1, + "args": 1, + "select": 1, + "multiarg": 1, + "useresult": 1, + "abstain": 1, + "overall": 1.0 + }, + "qwen3.6:35b-a3b": { + "invoke": 1, + "args": 1, + "select": 1, + "multiarg": 1, + "useresult": 1, + "abstain": 1, + "overall": 1.0 + }, + "gemma4:31b": { + "invoke": 1, + "args": 1, + "select": 1, + "multiarg": 1, + "useresult": 1, + "abstain": 1, + "overall": 1.0 + }, + "gemma4:26b": { + "invoke": 1, + "args": 1, + "select": 1, + "multiarg": 1, + "useresult": 1, + "abstain": 1, + "overall": 1.0 + }, + "qwen3.6:27b": { + "invoke": 1, + "args": 1, + "select": 1, + "multiarg": 1, + "useresult": 1, + "abstain": 1, + "overall": 1.0 + }, + "gemma4:e4b": { + "invoke": 1, + "args": 1, + "select": 1, + "multiarg": 1, + "useresult": 1, + "abstain": 1, + "overall": 1.0 + }, + "qwen3:14b": { + "invoke": 1, + "args": 1, + "select": 1, + "multiarg": 1, + "useresult": 1, + "abstain": 1, + "overall": 1.0 + }, + "gemma4:12b": { + "invoke": 1, + "args": 1, + "select": 1, + "multiarg": 1, + "useresult": 1, + "abstain": 1, + "overall": 1.0 + }, + "gemma4:e2b": { + "invoke": 1, + "args": 1, + "select": 1, + "multiarg": 1, + "useresult": 1, + "abstain": 1, + "overall": 1.0 + }, + "qwen3:8b": { + "invoke": 1, + "args": 1, + "select": 1, + "multiarg": 1, + "useresult": 1, + "abstain": 1, + "overall": 1.0 + } +} \ No newline at end of file diff --git a/benchmarks/results/bench_report.md b/benchmarks/results/bench_report.md new file mode 100644 index 0000000..afdc8a3 --- /dev/null +++ b/benchmarks/results/bench_report.md @@ -0,0 +1,31 @@ +# Local fleet benchmark — 2026-07-03 17:43 + +## Performance (sorted by generation tok/s) + +| Model | Gen tok/s | Prompt tok/s | Cold load | TTFT | Placement | OCR | Status | +|---|--:|--:|--:|--:|:--|:--:|:--| +| `gemma4:e2b` | 100.0 | 610 | 0.2s | 0.3s | 100% GPU | PASS | ok | +| `qwen3-coder:30b` | 85.7 | 297 | 0.1s | 0.2s | 100% GPU | - | ok | +| `qwen3-vl:30b-a3b-instruct` | 85.4 | 129 | 0.1s | 0.4s | 100% GPU | PASS | ok | +| `qwen3.6:35b-a3b` | 74.5 | 250 | 0.2s | 0.3s | 100% GPU | - | ok | +| `gpt-oss:20b` | 72.9 | 516 | 0.2s | 0.4s | 100% GPU | - | ok | +| `gemma4:26b` | 70.2 | 212 | 0.2s | 0.4s | 100% GPU | PASS | ok | +| `gemma4:e4b` | 66.0 | 369 | 0.2s | 0.3s | 100% GPU | PASS | ok | +| `glm-4.7-flash:latest` | 65.5 | 150 | 0.1s | 0.3s | 100% GPU | - | ok | +| `qwen3-vl:8b` | 60.4 | 363 | 0.1s | 0.2s | 100% GPU | PASS | ok | +| `qwen3:8b` | 59.0 | 244 | 0.1s | 0.2s | 100% GPU | - | ok | +| `gemma4:12b` | 35.9 | 167 | 0.2s | 0.5s | 100% GPU | PASS | ok | +| `qwen3:14b` | 35.7 | 200 | 0.1s | 0.3s | 100% GPU | - | ok | +| `qwen3.6:27b` | 18.3 | 71 | 0.2s | 0.7s | 100% GPU | - | ok | +| `gemma4:31b` | 16.6 | 66 | 0.2s | 0.8s | 100% GPU | PASS | ok | + +## Routing correctness + +| Case | Expected | Routed to | HTTP | Verdict | +|---|---|---|--:|:--:| +| short chat | `qwen3:8b` | `qwen3:8b` | 200 | PASS | +| code | `glm-4.7-flash` | `glm-4.7-flash` | 200 | PASS | +| repo code | `qwen3-coder:30b` | `qwen3-coder:30b` | 200 | PASS | +| reasoning | `qwen3.6:35b-a3b` | `qwen3.6:35b-a3b` | 200 | PASS | +| vision | `qwen3-vl:8b` | `qwen3-vl:8b` | 200 | PASS | +| ocr/doc | `qwen3-vl:30b-a3b-instruct` | `qwen3-vl:30b-a3b-instruct` | 200 | PASS | diff --git a/benchmarks/results/combined_report.md b/benchmarks/results/combined_report.md new file mode 100644 index 0000000..dcab0f2 --- /dev/null +++ b/benchmarks/results/combined_report.md @@ -0,0 +1,38 @@ +# Local fleet — combined benchmark (quality × agentic × speed) + +Date: 2026-07-03. Hardware: Apple M4 Max, 36 GB. All models 100% GPU. + +- **quality** = auto-graded mean of code (executed unit tests), math (GSM8K-style), + MC (knowledge/reasoning), instruction-following. think disabled for a fast baseline. +- **agentic** = tool-use via Ollama tools API: invoke / args / select / multi-arg / + use-result / abstain (6 checks). +- **tok/s** = generation speed from Ollama eval metrics. +- \* = corrected after a think/budget artifact in the first quality pass. + +| Model | Quality | Agentic | tok/s | +|---|--:|--:|--:| +| gemma4:26b | 100% | 100% | 71.6 | +| gemma4:31b | 100% | 100% | 16.6 | +| qwen3.6:27b | 96% | 100% | 18.3 | +| gemma4:e4b | 96% | 100% | 66.9 | +| gemma4:e2b | 92% | 100% | 101.3 | +| qwen3-vl:30b-a3b-instruct | 92% | 100% | 86.1 | +| gpt-oss:20b | 92%* | 100% | 70.9 | +| qwen3:14b | 92% | 100% | 36.2 | +| qwen3.6:35b-a3b | 91% | 100% | 74.5 | +| qwen3-coder:30b | 88% | 100% | 85.5 | +| qwen3:8b | 88% | 100% | 59.9 | +| gemma4:12b | 87% | 100% | 36.3 | +| glm-4.7-flash | 83% | 100% | 65.6 | +| qwen3-vl:8b | 62%* | 100% | 58.1 | + +## Takeaways +- gemma4:26b — best all-round (top quality + fast, MoE). Strong router default. +- MoE >> dense on this Mac: gemma4:31b / qwen3.6:27b match on quality but ~4x slower. +- gemma4:e2b — best speed/size/quality ratio (92% @ 101 tok/s, 7 GB). +- Agentic basics are universal (all 6/6); needs harder tasks to rank. + +## Harnesses +- benchmark.py — speed + OCR + routing +- quality_bench.py — objective quality auto-grading +- agent_bench.py — tool-use scenarios diff --git a/benchmarks/results/hard_agent_report.md b/benchmarks/results/hard_agent_report.md new file mode 100644 index 0000000..c29cf92 --- /dev/null +++ b/benchmarks/results/hard_agent_report.md @@ -0,0 +1,11 @@ +# Hard agentic benchmark (multi-turn tool loop) — 2026-07-03 19:09 + +Real tool execution with results fed back; thinking enabled. + +| Model | Overall | Seq | Parallel | Chain | Select | Recover | Abstain | Honesty | +|---|--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:| +| `glm-4.7-flash` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| `gemma4:26b` | **86%** | 1 | 1 | 1 | 1 | 0 | 1 | 1 | +| `qwen3.6:35b-a3b` | **86%** | 1 | 1 | 1 | 1 | 0 | 1 | 1 | +| `qwen3-coder:30b` | **86%** | 1 | 1 | 0 | 1 | 1 | 1 | 1 | +| `gpt-oss:20b` | **71%** | 1 | 1 | 1 | 1 | 0 | 1 | 0 | diff --git a/benchmarks/results/hard_agent_results.json b/benchmarks/results/hard_agent_results.json new file mode 100644 index 0000000..26d8808 --- /dev/null +++ b/benchmarks/results/hard_agent_results.json @@ -0,0 +1,52 @@ +{ + "gemma4:26b": { + "seq": 1, + "parallel": 1, + "chain": 1, + "select": 1, + "recover": 0, + "abstain": 1, + "honesty": 1, + "overall": 0.8571428571428571 + }, + "qwen3.6:35b-a3b": { + "seq": 1, + "parallel": 1, + "chain": 1, + "select": 1, + "recover": 0, + "abstain": 1, + "honesty": 1, + "overall": 0.8571428571428571 + }, + "glm-4.7-flash": { + "seq": 1, + "parallel": 1, + "chain": 1, + "select": 1, + "recover": 1, + "abstain": 1, + "honesty": 1, + "overall": 1.0 + }, + "qwen3-coder:30b": { + "seq": 1, + "parallel": 1, + "chain": 0, + "select": 1, + "recover": 1, + "abstain": 1, + "honesty": 1, + "overall": 0.8571428571428571 + }, + "gpt-oss:20b": { + "seq": 1, + "parallel": 1, + "chain": 1, + "select": 1, + "recover": 0, + "abstain": 1, + "honesty": 0, + "overall": 0.7142857142857143 + } +} \ No newline at end of file diff --git a/benchmarks/results/quality_report.md b/benchmarks/results/quality_report.md new file mode 100644 index 0000000..e2155af --- /dev/null +++ b/benchmarks/results/quality_report.md @@ -0,0 +1,20 @@ +# Quality benchmark — 2026-07-03 18:15 + +Objective auto-grading. code=executed unit tests, math=GSM8K-style exact, mc=multiple-choice, ifollow=programmatic checks, ocr=multi-image text. + +| Model | Overall | Code | Math | MC | Instr | OCR | tok/s | +|---|--:|--:|--:|--:|--:|--:|--:| +| `gemma4:31b` | **100%** | 100 | 100 | 100 | 100 | 100 | 16.6 | +| `gemma4:26b` | **100%** | 100 | 100 | 100 | 100 | 100 | 71.6 | +| `qwen3.6:27b` | **96%** | 100 | 100 | 83 | 100 | — | 18.3 | +| `gemma4:e4b` | **96%** | 100 | 100 | 83 | 100 | 100 | 66.9 | +| `qwen3-vl:30b-a3b-instruct` | **92%** | 100 | 100 | 100 | 67 | 100 | 86.1 | +| `qwen3:14b` | **92%** | 100 | 100 | 100 | 67 | — | 36.2 | +| `gemma4:e2b` | **92%** | 100 | 100 | 67 | 100 | 100 | 101.3 | +| `qwen3.6:35b-a3b` | **91%** | 100 | 80 | 83 | 100 | — | 74.5 | +| `qwen3-coder:30b` | **88%** | 100 | 100 | 83 | 67 | — | 85.5 | +| `qwen3:8b` | **88%** | 100 | 100 | 83 | 67 | — | 59.9 | +| `gemma4:12b` | **87%** | 100 | 80 | 67 | 100 | 100 | 36.3 | +| `glm-4.7-flash:latest` | **83%** | 100 | 100 | 67 | 67 | — | 65.6 | +| `gpt-oss:20b` | **62%** | 100 | 80 | 0 | 67 | — | 70.9 | +| `qwen3-vl:8b` | **22%** | 33 | 20 | 0 | 33 | 100 | 58.1 | diff --git a/benchmarks/results/quality_results.json b/benchmarks/results/quality_results.json new file mode 100644 index 0000000..155222d --- /dev/null +++ b/benchmarks/results/quality_results.json @@ -0,0 +1,121 @@ +{ + "qwen3-vl:30b-a3b-instruct": { + "math": 1.0, + "mc": 1.0, + "code": 1.0, + "ifollow": 0.6666666666666666, + "ocr": 1.0, + "overall": 0.9166666666666666, + "tps": 86.08918225170864 + }, + "qwen3-vl:8b": { + "math": 0.2, + "mc": 0.0, + "code": 0.3333333333333333, + "ifollow": 0.3333333333333333, + "ocr": 1.0, + "overall": 0.21666666666666667, + "tps": 58.06267489952323 + }, + "qwen3-coder:30b": { + "math": 1.0, + "mc": 0.8333333333333334, + "code": 1.0, + "ifollow": 0.6666666666666666, + "overall": 0.875, + "tps": 85.51818292912147 + }, + "glm-4.7-flash:latest": { + "math": 1.0, + "mc": 0.6666666666666666, + "code": 1.0, + "ifollow": 0.6666666666666666, + "overall": 0.8333333333333333, + "tps": 65.60893213491771 + }, + "gpt-oss:20b": { + "math": 0.8, + "mc": 0.0, + "code": 1.0, + "ifollow": 0.6666666666666666, + "overall": 0.6166666666666667, + "tps": 70.93934599819382 + }, + "qwen3.6:35b-a3b": { + "math": 0.8, + "mc": 0.8333333333333334, + "code": 1.0, + "ifollow": 1.0, + "overall": 0.9083333333333333, + "tps": 74.50862085584875 + }, + "gemma4:31b": { + "math": 1.0, + "mc": 1.0, + "code": 1.0, + "ifollow": 1.0, + "ocr": 1.0, + "overall": 1.0, + "tps": 16.5738240176472 + }, + "gemma4:26b": { + "math": 1.0, + "mc": 1.0, + "code": 1.0, + "ifollow": 1.0, + "ocr": 1.0, + "overall": 1.0, + "tps": 71.55240664310993 + }, + "qwen3.6:27b": { + "math": 1.0, + "mc": 0.8333333333333334, + "code": 1.0, + "ifollow": 1.0, + "overall": 0.9583333333333334, + "tps": 18.287797403043964 + }, + "gemma4:e4b": { + "math": 1.0, + "mc": 0.8333333333333334, + "code": 1.0, + "ifollow": 1.0, + "ocr": 1.0, + "overall": 0.9583333333333334, + "tps": 66.86943765975911 + }, + "qwen3:14b": { + "math": 1.0, + "mc": 1.0, + "code": 1.0, + "ifollow": 0.6666666666666666, + "overall": 0.9166666666666666, + "tps": 36.22066333837863 + }, + "gemma4:12b": { + "math": 0.8, + "mc": 0.6666666666666666, + "code": 1.0, + "ifollow": 1.0, + "ocr": 1.0, + "overall": 0.8666666666666667, + "tps": 36.30471836404827 + }, + "gemma4:e2b": { + "math": 1.0, + "mc": 0.6666666666666666, + "code": 1.0, + "ifollow": 1.0, + "ocr": 1.0, + "overall": 0.9166666666666666, + "tps": 101.34789160325633 + }, + "qwen3:8b": { + "math": 1.0, + "mc": 0.8333333333333334, + "code": 1.0, + "ifollow": 0.6666666666666666, + "overall": 0.875, + "tps": 59.89123849272082 + } +} \ No newline at end of file diff --git a/deploy/com.llm-router.plist.template b/deploy/com.llm-router.plist.template new file mode 100644 index 0000000..e187852 --- /dev/null +++ b/deploy/com.llm-router.plist.template @@ -0,0 +1,32 @@ + + + + + + Label + com.llm-router + ProgramArguments + + /bin/bash + __INSTALL_DIR__/run.sh + + WorkingDirectory + __INSTALL_DIR__ + RunAtLoad + + KeepAlive + + ThrottleInterval + 10 + StandardOutPath + __INSTALL_DIR__/launchd.out.log + StandardErrorPath + __INSTALL_DIR__/launchd.err.log + EnvironmentVariables + + PATH + __PATH__ + + + diff --git a/deploy/install-service.sh b/deploy/install-service.sh new file mode 100755 index 0000000..c6b0842 --- /dev/null +++ b/deploy/install-service.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Install llm-router as a boot service: launchd on macOS, systemd (user) on Linux. +# Run from the repo root: ./deploy/install-service.sh +set -euo pipefail +cd "$(dirname "$0")/.." +INSTALL_DIR="$(pwd)" +PATH_VAL="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" + +fill() { sed -e "s|__INSTALL_DIR__|$INSTALL_DIR|g" -e "s|__PATH__|$PATH_VAL|g" "$1"; } + +case "$(uname -s)" in + Darwin) + PLIST="$HOME/Library/LaunchAgents/com.llm-router.plist" + fill deploy/com.llm-router.plist.template > "$PLIST" + launchctl bootout "gui/$(id -u)/com.llm-router" 2>/dev/null || true + launchctl bootstrap "gui/$(id -u)" "$PLIST" + echo ">> installed launchd agent: $PLIST" + echo " status: launchctl print gui/$(id -u)/com.llm-router | grep state" + echo " logs: tail -f $INSTALL_DIR/launchd.out.log" + echo " remove: launchctl bootout gui/$(id -u)/com.llm-router && rm $PLIST" + ;; + Linux) + UNIT_DIR="$HOME/.config/systemd/user"; mkdir -p "$UNIT_DIR" + fill deploy/llm-router.service.template > "$UNIT_DIR/llm-router.service" + systemctl --user daemon-reload + systemctl --user enable --now llm-router.service + echo ">> installed systemd user service" + echo " status: systemctl --user status llm-router" + echo " logs: journalctl --user -u llm-router -f" + echo " remove: systemctl --user disable --now llm-router && rm $UNIT_DIR/llm-router.service" + echo " (tip: 'loginctl enable-linger $USER' to run without being logged in)" + ;; + *) echo "Unsupported OS: $(uname -s). Run ./run.sh manually or add your own unit."; exit 1 ;; +esac diff --git a/deploy/llm-router.service.template b/deploy/llm-router.service.template new file mode 100644 index 0000000..baf5884 --- /dev/null +++ b/deploy/llm-router.service.template @@ -0,0 +1,16 @@ +# systemd user service (Linux). Placeholders filled by deploy/install-service.sh. +# Installed to ~/.config/systemd/user/llm-router.service +[Unit] +Description=llm-router — smart router in front of Ollama +After=network.target + +[Service] +Type=simple +WorkingDirectory=__INSTALL_DIR__ +ExecStart=/bin/bash __INSTALL_DIR__/run.sh +Restart=always +RestartSec=10 +Environment=PATH=__PATH__ + +[Install] +WantedBy=default.target diff --git a/docs/AUDIO.md b/docs/AUDIO.md new file mode 100644 index 0000000..515e5ce --- /dev/null +++ b/docs/AUDIO.md @@ -0,0 +1,45 @@ +# Audio / voice + +Ollama can *understand* audio with speech-capable models (e.g. Gemma 4 `e4b`/`e2b`/ +`12b`), but only via its **native `/api/chat`** with the audio base64 in the +`images` field — the OpenAI `input_audio` format is silently dropped, and there's +no native speech-to-text endpoint. This router bridges that gap. + +Set the model with `ROUTER_AUDIO_MODEL` (default `gemma4:e4b`). It must be a model +whose `ollama show` capabilities include `audio`. **Note:** some fine-tunes/quants +have broken audio even if the flag is set — test before relying on one. + +## Speech-to-text (Whisper-shaped) + +```bash +curl http://:8080/v1/audio/transcriptions \ + -F file=@recording.wav \ + -F model=whisper-1 +# -> {"text": "the transcription"} +``` +- `response_format=text` returns plain text instead of JSON. +- `/v1/audio/translations` does the same but translates to English. +- Works with the OpenAI SDK: set `base_url` to the router and call + `audio.transcriptions.create(...)`. The `model` field is ignored — the router + always uses `ROUTER_AUDIO_MODEL`. + +## Audio inside a chat + +Send OpenAI `input_audio` content to `/v1/chat/completions` and ask about it: +```json +{"model":"auto","messages":[{"role":"user","content":[ + {"type":"text","text":"answer the question in this audio"}, + {"type":"input_audio","input_audio":{"data":"","format":"wav"}} +]}]} +``` +The router detects the audio, translates it to Ollama's native call, runs it on +`ROUTER_AUDIO_MODEL`, and returns a normal OpenAI response (streaming supported). + +## Limitations + +- **WAV is verified.** Other formats (mp3/m4a/ogg) depend on Ollama's decoding — + transcode to WAV if they fail. +- **Not streaming STT** — it transcribes a complete clip per request (like + Whisper's file API), not a live mic stream. Perfect for record-then-send. +- Audio only works through the router's native bridge, so audio requests always + target `ROUTER_AUDIO_MODEL` (they bypass the normal model-selection map). diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md new file mode 100644 index 0000000..a547e15 --- /dev/null +++ b/docs/BENCHMARKS.md @@ -0,0 +1,40 @@ +# Benchmarks + +Four self-contained harnesses in [`benchmarks/`](../benchmarks/) score your local +fleet on speed, quality, and tool use. They talk to Ollama (`:11434`) and the +router (`:8080`) directly. Run them with the venv Python, from the `benchmarks/` +directory (they import each other): + +```bash +cd benchmarks +../.venv/bin/python benchmark.py # speed + OCR + routing correctness +../.venv/bin/python quality_bench.py # objective quality (code exec, math, MC, instr) +../.venv/bin/python agent_bench.py # basic tool-calling +../.venv/bin/python hard_agent_bench.py # multi-turn agentic (seq/parallel/recover/abstain) +``` + +Each writes a `*_report.md` (and some a `*_results.json`). Sample outputs from the +reference machine (Apple M4 Max, 36 GB) are checked in under `benchmarks/results/`. + +## What each measures + +- **benchmark.py** — generation & prompt tok/s and GPU placement (from Ollama's + own eval metrics), an OCR pass on vision models, and live routing-decision checks. +- **quality_bench.py** — objective auto-grading: executed unit tests (coding), + exact-match math (GSM8K-style), multiple-choice, and programmatic + instruction-following. Runs with thinking disabled for a fast, comparable baseline. +- **agent_bench.py** — tool-calling basics via Ollama's tools API: invoke, args, + selection, use-result, and *abstain* (not calling a tool when none is needed). +- **hard_agent_bench.py** — a real multi-turn tool loop (tools actually execute and + results feed back): sequential dependency, parallel/multi-entity, 3-hop chains, + tool selection under distractors, error recovery, and no-fabrication honesty. + Pass model names as args to limit the set: `... hard_agent_bench.py glm-4.7-flash gemma4:26b`. + +## Notes + +- These assume the same model names as the reference fleet — edit the model lists + at the top of each script to match yours. +- `quality_bench.py` / `benchmark.py` generate a test image with Pillow (in the + venv) for the OCR/vision checks. +- Objective grading (running generated code, exact-match answers) is deliberately + used over LLM-as-judge to avoid judge bias. diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md new file mode 100644 index 0000000..bca0b3c --- /dev/null +++ b/docs/DEPLOY.md @@ -0,0 +1,69 @@ +# Deploying llm-router + +## Run manually +```bash +./scripts/setup.sh # once: venv + LiteLLM +./run.sh # foreground: LiteLLM (:4000) + router (:8080) +``` +`run.sh` starts LiteLLM in the background, waits for it, then runs the router in +the foreground. Ctrl-C stops both (it tracks PIDs in `.litellm.pid` / `.router.pid`). + +## Run as a boot service +```bash +./deploy/install-service.sh +``` +- **macOS** → a launchd user agent (`~/Library/LaunchAgents/com.llm-router.plist`), + `KeepAlive` (auto-restart), starts at login. +- **Linux** → a systemd user service (`~/.config/systemd/user/llm-router.service`), + `Restart=always`. Use `loginctl enable-linger $USER` to run without an active login. + +The installer prints status / logs / removal commands for your platform. + +Restart after editing `router.py` or config: +```bash +# macOS +launchctl kickstart -k gui/$(id -u)/com.llm-router +# Linux +systemctl --user restart llm-router +``` + +## Networking + +- The router binds `ROUTER_HOST` (default `0.0.0.0` = reachable on the LAN). +- Reach it from other machines at `http://:8080/v1` or, on the same + LAN, `http://.local:8080/v1` (mDNS; more stable than a DHCP IP). +- Set `ROUTER_HOST=127.0.0.1` to make it local-only. + +### Exposing beyond the LAN + +Turn on auth first — put a token in `.apikey` (see `apikey.example`) and restart. +Then front it with one of: + +- **Reverse proxy (recommended for a fixed setup)** — e.g. Caddy: + ``` + router.example.com { + reverse_proxy localhost:8080 + } + ``` + Caddy handles TLS; the router handles auth. Pair with `ROUTER_HOST=127.0.0.1` + so the only way in is through the proxy. +- **Quick tunnel (ephemeral)** — `cloudflared tunnel --url http://localhost:8080` + gives a temporary `https://…trycloudflare.com` URL. Good for a quick test; the + URL changes on restart and it's internet-facing, so keep `.apikey` set. + +### A note on Ollama itself +The router connects to Ollama at `ROUTER_OLLAMA` (default `http://127.0.0.1:11434`). +If you want *other machines* to reach Ollama directly (not just via the router), +set Ollama's own `OLLAMA_HOST=0.0.0.0:11434` and restart Ollama — but that is the +**server bind** var and is unrelated to `ROUTER_OLLAMA` (the router's client URL). +Don't set `ROUTER_OLLAMA` to `0.0.0.0` — it's a connect address. + +## Health & logs +```bash +curl -s http://localhost:8080/healthz # {"ok":true} +curl -s http://localhost:8080/v1/models # route targets +tail -f launchd.out.log # macOS service logs (router access + decisions) +tail -f litellm.log # LiteLLM backend +``` +Each routed request logs a line like `[router] router -> glm-4.7-flash`. Responses +carry `x-router-model`, `x-router-initial-model`, and `x-router-decided-by` headers. diff --git a/docs/ROUTING.md b/docs/ROUTING.md new file mode 100644 index 0000000..2fedf00 --- /dev/null +++ b/docs/ROUTING.md @@ -0,0 +1,68 @@ +# Routing + +## How `model: "auto"` decides + +`choose_model()` in [`router.py`](../router.py) runs cheap→expensive, first match wins: + +1. **Image in the request** → vision model (`vision_heavy` if the text mentions + OCR/document/table, else `vision`). +2. **Request carries `tools`** → agentic model (`code_heavy` if the text looks + code-ish, else `agentic`). +3. **Text heuristics** → code regex → coding model; reasoning words → reasoning + model; very short (< `SHORT_CHARS`) → the small fast model. +4. **Tiny classifier** — anything left over is labeled by a small always-warm + model (`CLASSIFIER`) into code / reason / vision / general. + +The chosen model, plus any fallback, is reported in the response headers +`x-router-initial-model` (the decision) and `x-router-model` (what actually served). + +## The model map — customize this for your machine + +Two dicts at the top of `router.py`: + +```python +M = { + "fast": "qwen3:8b", "general": "qwen3:14b", + "code": "glm-4.7-flash", "code_heavy": "qwen3-coder:30b", + "reason": "qwen3.6:35b-a3b", "agentic": "glm-4.7-flash", + "vision": "qwen3-vl:8b", "vision_heavy": "qwen3-vl:30b-a3b-instruct", +} +CLASSIFIER = "qwen3:8b" +``` + +Point these at models you've actually pulled (`ollama list`). Names must match +exactly, including any `:tag`. Restart the router after editing. + +`FALLBACK` maps each model to a backup used when it errors or isn't pulled yet — +so the router degrades gracefully instead of failing. + +## Explicit model override + +Any request that names a real model (not `auto`) is passed straight through: +`{"model": "glm-4.7-flash", ...}`. Unknown names are handled by LiteLLM's catch-all +(`model_name: "*"` in `litellm.config.yaml`), which forwards *any* name to Ollama — +so `:tag` variants and models you pull later work without config changes. + +## Policy: forcing a client onto a fleet (the "uncensored" example) + +Put a token in `.uncensored_key`. Any request whose `Authorization: Bearer ` +matches is forced through `choose_uncensored()` — the same content heuristics, but +only ever selecting from the `UNCENSORED` map — **regardless of the model asked for**. +This pins a specific client (identified by its key) to a specific fleet. + +Clients can also opt in per-request with `model: "auto-uncensored"` (no key needed). + +Use this pattern for any policy split (a "coding-only" client, a "cheap models" +client, etc.): add a map + a `choose_*` function + a trigger in `do_POST`. + +## Adding a model from Hugging Face + +Ollama pulls GGUF repos directly: +```bash +ollama pull hf.co/USER/REPO:Q4_K_M # tag must match the quant in the filename +ollama cp hf.co/USER/REPO:Q4_K_M friendly-name # optional: rename +``` +Then add `friendly-name` to the relevant map in `router.py`. Multi-file repos: +Ollama auto-pulls a vision `mmproj` if present; `mtp`/draft files are ignored. +Big prompts truncate at Ollama's default context — make a bigger-context copy with +`./scripts/make-context-variant.sh `. diff --git a/litellm.config.yaml b/litellm.config.yaml new file mode 100644 index 0000000..1bc055c --- /dev/null +++ b/litellm.config.yaml @@ -0,0 +1,34 @@ +# LiteLLM proxy: exposes every local model over an OpenAI API on :4000, +# with automatic fallbacks. The pre-router (router.py) picks the model; +# LiteLLM handles delivery, retries, and logging. +model_list: + - model_name: qwen3:8b + litellm_params: { model: ollama_chat/qwen3:8b, api_base: http://127.0.0.1:11434 } + - model_name: qwen3:14b + litellm_params: { model: ollama_chat/qwen3:14b, api_base: http://127.0.0.1:11434 } + - model_name: glm-4.7-flash + litellm_params: { model: ollama_chat/glm-4.7-flash, api_base: http://127.0.0.1:11434 } + - model_name: qwen3-coder:30b + litellm_params: { model: ollama_chat/qwen3-coder:30b, api_base: http://127.0.0.1:11434 } + - model_name: qwen3.6:35b-a3b + litellm_params: { model: ollama_chat/qwen3.6:35b-a3b, api_base: http://127.0.0.1:11434 } + - model_name: gpt-oss:20b + litellm_params: { model: ollama_chat/gpt-oss:20b, api_base: http://127.0.0.1:11434 } + - model_name: qwen3-vl:8b + litellm_params: { model: ollama_chat/qwen3-vl:8b, api_base: http://127.0.0.1:11434 } + - model_name: qwen3-vl:30b-a3b-instruct + litellm_params: { model: ollama_chat/qwen3-vl:30b-a3b-instruct, api_base: http://127.0.0.1:11434 } + # catch-all: any other model name (incl. :latest tags picked by `ollama launch`) -> ollama + - model_name: "*" + litellm_params: { model: "ollama_chat/*", api_base: http://127.0.0.1:11434 } + +litellm_settings: + drop_params: true # tolerate params a given model doesn't support + +router_settings: + fallbacks: + - { "glm-4.7-flash": ["qwen3-coder:30b"] } + - { "qwen3-coder:30b": ["glm-4.7-flash"] } + - { "qwen3.6:35b-a3b": ["gpt-oss:20b"] } + - { "qwen3-vl:30b-a3b-instruct": ["qwen3-vl:8b"] } + - { "qwen3:14b": ["qwen3:8b"] } diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..06bb5c6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +litellm[proxy] +pillow diff --git a/router.py b/router.py new file mode 100644 index 0000000..b9d98a0 --- /dev/null +++ b/router.py @@ -0,0 +1,537 @@ +""" +Local model router (stdlib-only; runs on any Python 3.9+, incl. 3.14). + +An OpenAI-compatible endpoint that picks the best local model per request and +forwards straight to Ollama's own OpenAI API. Point every client at + http://localhost:8080/v1 +- model "auto" -> router decides (modality -> heuristics -> tiny classifier) +- any real name -> passed through unchanged (manual override always wins) +On upstream failure the router retries once with a mapped fallback model. +Response headers x-router-model / x-router-decided-by report the pick. +""" +import os, re, json, time, base64, urllib.request, urllib.error +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +OLLAMA = os.environ.get("ROUTER_OLLAMA", "http://127.0.0.1:11434").rstrip("/") # client target; NOT OLLAMA_HOST (that's the server bind addr, e.g. 0.0.0.0) +UPSTREAM = os.environ.get("ROUTER_UPSTREAM", OLLAMA + "/v1").rstrip("/") +HOST = os.environ.get("ROUTER_HOST", "0.0.0.0") # 0.0.0.0 = reachable on the LAN +PORT = int(os.environ.get("ROUTER_PORT", "8080")) +API_KEY = os.environ.get("ROUTER_API_KEY", "").strip() # if set, require Bearer token +AUDIO_MODEL = os.environ.get("ROUTER_AUDIO_MODEL", "gemma4:e4b") # audio only works on official Gemma 4 + +M = { + "fast": "qwen3:8b", + "general": "qwen3:14b", + "code": "glm-4.7-flash", + "code_heavy": "qwen3-coder:30b", + "reason": "qwen3.6:35b-a3b", + "agentic": "glm-4.7-flash", # tool-carrying requests (general agent): best hard-agentic (7/7, only one to recover from tool errors) + "vision": "qwen3-vl:8b", + "vision_heavy": "qwen3-vl:30b-a3b-instruct", +} +CLASSIFIER = "qwen3:8b" +SHORT_CHARS = 240 + +# Uncensored fleet. A request routes here if it asks for model "auto-uncensored" +# OR carries the dedicated famapp key (ROUTER_UNCENSORED_KEY) — in the latter case +# EVERY request from that client is forced uncensored regardless of model asked for. +UNCENSORED = { + "general": "gemma4-uncensored:26b", + "vision": "gemma4-uncensored:26b", # has vision + "code": "glm-uncensored-code:30b", + "fast": "gemma-e4b-uncensored", +} +UNCENSORED_KEY = os.environ.get("ROUTER_UNCENSORED_KEY", "").strip() + +FALLBACK = { + "glm-4.7-flash": "qwen3-coder:30b", + "qwen3-coder:30b": "glm-4.7-flash", + "gemma4:26b": "qwen3.6:35b-a3b", + "qwen3.6:35b-a3b": "gpt-oss:20b", + "gpt-oss:20b": "qwen3:14b", + "qwen3-vl:30b-a3b-instruct": "qwen3-vl:8b", + "qwen3:14b": "qwen3:8b", + "qwen3-vl:8b": "gemma4:12b", + # uncensored models still downloading fall back to the one that's ready + "glm-uncensored-code:30b": "gemma4-uncensored:26b", + "gemma-e4b-uncensored": "gemma4-uncensored:26b", +} + +CODE_RE = re.compile(r"```|\b(def |class |function |import |#include|refactor|debug|" + r"stack ?trace|traceback|compile|unit test|npm |pip |git |bug|" + r"exception|null pointer|segfault)\b|\.(py|js|ts|tsx|go|rs|java|" + r"cpp|cc|c|rb|sh|sql)\b", re.I) +CODE_HEAVY_RE = re.compile(r"\b(repo|repository|codebase|multiple files|entire project|" + r"across files|whole project)\b", re.I) +REASON_RE = re.compile(r"\b(prove|theorem|step[- ]by[- ]step|reason through|derive|" + r"calculate|logic puzzle|why does|explain why|work through)\b", re.I) +OCR_RE = re.compile(r"\b(ocr|document|invoice|receipt|table|form|handwrit|extract text|" + r"scan|parse this (image|page|pdf))\b", re.I) + + +def last_user_text(messages): + for m in reversed(messages): + if m.get("role") == "user": + c = m.get("content") + if isinstance(c, str): + return c + if isinstance(c, list): + return " ".join(p.get("text", "") for p in c + if isinstance(p, dict) and p.get("type") == "text") + return "" + + +def has_image(messages): + for m in messages: + c = m.get("content") + if isinstance(c, list): + for p in c: + if isinstance(p, dict) and p.get("type") in ("image_url", "input_image", "image"): + return True + return False + + +def has_audio(messages): + for m in messages: + c = m.get("content") + if isinstance(c, list): + for p in c: + if isinstance(p, dict) and p.get("type") in ("input_audio", "audio"): + return True + return False + + +def to_native_messages(messages): + # OpenAI content -> Ollama native: text flattened, audio/image base64 into `images` + out = [] + for m in messages: + role, c = m.get("role", "user"), m.get("content") + if isinstance(c, str): + out.append({"role": role, "content": c}) + continue + text_parts, media = [], [] + if isinstance(c, list): + for p in c: + if not isinstance(p, dict): + continue + t = p.get("type") + if t == "text": + text_parts.append(p.get("text", "")) + elif t in ("input_audio", "audio"): + d = (p.get("input_audio") or p.get("audio") or {}).get("data") + if d: + media.append(d) + elif t in ("image_url", "image", "input_image"): + url = (p.get("image_url") or {}).get("url", "") if isinstance(p.get("image_url"), dict) else "" + media.append(url.split(",", 1)[-1] if url.startswith("data:") else url) + msg = {"role": role, "content": " ".join(tp for tp in text_parts if tp)} + if media: + msg["images"] = media + out.append(msg) + return out + + +def parse_multipart(body, boundary): + # minimal multipart/form-data parser -> {name: (filename_or_None, bytes)} + fields = {} + for part in body.split(b"--" + boundary.encode()): + if part in (b"", b"--", b"--\r\n", b"\r\n"): + continue + if part.startswith(b"\r\n"): + part = part[2:] + if b"\r\n\r\n" not in part: + continue + head, _, data = part.partition(b"\r\n\r\n") + if data.endswith(b"\r\n"): + data = data[:-2] + head_text = head.decode("utf-8", "replace") + nm = re.search(r'name="([^"]*)"', head_text) + if not nm: + continue + fm = re.search(r'filename="([^"]*)"', head_text) + fields[nm.group(1)] = (fm.group(1) if fm else None, data) + return fields + + +def total_len(messages): + n = 0 + for m in messages: + c = m.get("content") + if isinstance(c, str): + n += len(c) + elif isinstance(c, list): + for p in c: + if isinstance(p, dict): + n += len(p.get("text", "")) + return n + + +def classify(text): + payload = json.dumps({ + "model": CLASSIFIER, + "prompt": ("Classify the user's task into exactly one label from: " + "code, reason, vision, general. Reply with only the label.\n\n" + "Task: " + text[:2000]), + "stream": False, "keep_alive": -1, + "options": {"temperature": 0, "num_predict": 3}, + }).encode() + try: + req = urllib.request.Request(OLLAMA + "/api/generate", data=payload, + headers={"content-type": "application/json"}) + with urllib.request.urlopen(req, timeout=30) as r: + out = json.loads(r.read()).get("response", "").strip().lower() + for lab in ("code", "reason", "vision", "general"): + if lab in out: + return lab + except Exception: + pass + return "general" + + +def choose_model(body): + messages = body.get("messages", []) + text = last_user_text(messages) + if has_image(messages): + return M["vision_heavy"] if OCR_RE.search(text) else M["vision"] + if body.get("tools"): # tool-carrying -> agentic (code agent vs general agent) + return M["code_heavy"] if CODE_RE.search(text) else M["agentic"] + if CODE_RE.search(text): + return M["code_heavy"] if CODE_HEAVY_RE.search(text) else M["code"] + if REASON_RE.search(text): + return M["reason"] + if total_len(messages) < SHORT_CHARS: + return M["fast"] + lab = classify(text) + return {"code": M["code"], "reason": M["reason"], + "vision": M["vision"], "general": M["general"]}[lab] + + +def choose_uncensored(body): + # same content heuristics, but only ever selects from the uncensored fleet + messages = body.get("messages", []) + text = last_user_text(messages) + if has_image(messages): + return UNCENSORED["vision"] + if CODE_RE.search(text): + return UNCENSORED["code"] + if total_len(messages) < SHORT_CHARS: + return UNCENSORED["fast"] + return UNCENSORED["general"] + + +def _open_upstream(body): + data = json.dumps(body).encode() + req = urllib.request.Request(UPSTREAM + "/chat/completions", data=data, + headers={"content-type": "application/json"}) + return urllib.request.urlopen(req, timeout=600) + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, fmt, *args): # access log for EVERY request (incl 401/404) + ua = self.headers.get("user-agent", "-")[:40] + auth = "auth" if self.headers.get("authorization") else "noauth" + print(f"[access] {time.strftime('%H:%M:%S')} {self.address_string()} " + f"{auth} \"{ua}\" {fmt % args}", flush=True) + + def _authed(self): + if not API_KEY: + return True + auth = self.headers.get("authorization", "") + return auth.startswith("Bearer ") and auth[7:].strip() == API_KEY + + def _json(self, code, obj, extra=None): + payload = json.dumps(obj).encode() + self.send_response(code) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(payload))) + for k, v in (extra or {}).items(): + self.send_header(k, v) + self.end_headers() + self.wfile.write(payload) + + def _text(self, code, s): + data = s.encode() + self.send_response(code) + self.send_header("content-type", "text/plain") + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def _proxy_ollama(self, path, data=None): + # read-only native Ollama endpoints for `ollama launch` probing. + # NOTE: only metadata paths are wired up; inference stays on authed /v1/*. + req = urllib.request.Request(OLLAMA + path, data=data, + headers={"content-type": "application/json"}, + method="POST" if data is not None else "GET") + try: + up = urllib.request.urlopen(req, timeout=30) + body, code = up.read(), 200 + ct = up.headers.get("content-type", "application/json") + up.close() + except urllib.error.HTTPError as e: + body, code, ct = e.read(), e.code, "application/json" + except Exception as e: + return self._json(502, {"error": f"ollama unreachable: {e}"}) + self.send_response(code) + self.send_header("content-type", ct) + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + if self.path.rstrip("/") == "/healthz": + return self._json(200, {"ok": True}) + if self.path.rstrip("/") == "/v1/models": + if not self._authed(): + return self._json(401, {"error": "unauthorized"}) + seen = list(dict.fromkeys(M.values())) + return self._json(200, {"object": "list", + "data": [{"id": "auto", "object": "model"}] + + [{"id": m, "object": "model"} for m in seen]}) + # native Ollama probe endpoints (read-only) so `ollama launch` accepts this host + if self.path in ("/", ""): + return self._text(200, "Ollama is running") + if self.path.rstrip("/") in ("/api/version", "/api/tags", "/api/ps"): + return self._proxy_ollama(self.path) + return self._json(404, {"error": "not found"}) + + def _proxy_anthropic(self, path): + # transparent proxy for Anthropic Messages API (Claude Code) -> LiteLLM /v1/messages + if not self._authed(): + return self._json(401, {"error": "unauthorized"}) + try: + length = int(self.headers.get("content-length", 0) or 0) + body = self.rfile.read(length) + except Exception as e: + return self._json(400, {"error": f"bad request: {e}"}) + url = UPSTREAM + path[len("/v1"):] # /v1/messages -> /v1/messages + headers = {"content-type": "application/json"} + for h in ("authorization", "anthropic-version", "anthropic-beta"): + v = self.headers.get(h) + if v: + headers[h] = v + try: + up = urllib.request.urlopen( + urllib.request.Request(url, data=body, headers=headers), timeout=600) + except urllib.error.HTTPError as e: + data = e.read() + self.send_response(e.code) + self.send_header("content-type", e.headers.get("content-type", "application/json")) + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + return + except Exception as e: + return self._json(502, {"error": f"upstream unreachable: {e}"}) + ctype = up.headers.get("content-type", "application/json") + if "event-stream" in ctype: + self.send_response(200) + self.send_header("content-type", ctype) + self.send_header("cache-control", "no-cache") + self.send_header("connection", "close") + self.end_headers() + try: + while True: + chunk = up.read(2048) + if not chunk: + break + self.wfile.write(chunk) + self.wfile.flush() + except Exception: + pass + finally: + up.close() + else: + data = up.read() + up.close() + self.send_response(200) + self.send_header("content-type", ctype) + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def _handle_transcription(self, translate=False): + # OpenAI /v1/audio/transcriptions (Whisper-shaped) backed by Gemma 4 + ct = self.headers.get("content-type", "") + m = re.search(r"boundary=([^;]+)", ct) + if "multipart/form-data" not in ct or not m: + return self._json(400, {"error": "expected multipart/form-data with a boundary"}) + boundary = m.group(1).strip().strip('"') + length = int(self.headers.get("content-length", 0) or 0) + fields = parse_multipart(self.rfile.read(length), boundary) + if "file" not in fields or fields["file"][1] is None: + return self._json(400, {"error": "missing 'file' form field (the audio)"}) + audio_b64 = base64.b64encode(fields["file"][1]).decode() + resp_format = (fields.get("response_format", (None, b"json"))[1] or b"json").decode().strip() + prompt = ("Transcribe the speech in this audio and translate it into English. " + "Output only the English text, nothing else." if translate else + "Transcribe the speech in this audio verbatim. " + "Output only the transcription text, nothing else.") + native = {"model": AUDIO_MODEL, "stream": False, "keep_alive": "30m", + "options": {"temperature": 0}, + "messages": [{"role": "user", "content": prompt, "images": [audio_b64]}]} + print(f"[router] stt -> {AUDIO_MODEL}", flush=True) + try: + req = urllib.request.Request(OLLAMA + "/api/chat", data=json.dumps(native).encode(), + headers={"content-type": "application/json"}) + with urllib.request.urlopen(req, timeout=600) as r: + d = json.loads(r.read()) + except urllib.error.HTTPError as e: + return self._json(e.code, {"error": f"stt upstream {e.code}: {e.read().decode()[:300]}"}) + except Exception as e: + return self._json(502, {"error": f"stt upstream: {e}"}) + text = ((d.get("message") or {}).get("content", "") or "").strip() + if resp_format == "text": + return self._text(200, text) + return self._json(200, {"text": text}) + + def _handle_audio(self, body): + # OpenAI request carrying audio -> Ollama native /api/chat (images field), Gemma 4 + model = AUDIO_MODEL + native = {"model": model, "messages": to_native_messages(body.get("messages", [])), + "stream": False, "keep_alive": "30m", "options": {}} + if isinstance(body.get("options"), dict): + native["options"].update(body["options"]) + if body.get("max_tokens"): + native["options"]["num_predict"] = body["max_tokens"] + if body.get("temperature") is not None: + native["options"]["temperature"] = body["temperature"] + print(f"[router] audio -> {model}", flush=True) + try: + req = urllib.request.Request(OLLAMA + "/api/chat", data=json.dumps(native).encode(), + headers={"content-type": "application/json"}) + with urllib.request.urlopen(req, timeout=600) as r: + d = json.loads(r.read()) + except urllib.error.HTTPError as e: + return self._json(e.code, {"error": f"audio upstream {e.code}: {e.read().decode()[:300]}"}) + except Exception as e: + return self._json(502, {"error": f"audio upstream: {e}"}) + content = (d.get("message") or {}).get("content", "") or "" + pc, ec = d.get("prompt_eval_count", 0) or 0, d.get("eval_count", 0) or 0 + cid, created = "chatcmpl-audio", int(time.time()) + hdr = {"x-router-model": model, "x-router-initial-model": model, "x-router-decided-by": "audio"} + if body.get("stream"): + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.send_header("cache-control", "no-cache") + self.send_header("connection", "close") + for k, v in hdr.items(): + self.send_header(k, v) + self.end_headers() + first = {"id": cid, "object": "chat.completion.chunk", "created": created, "model": model, + "choices": [{"index": 0, "delta": {"role": "assistant", "content": content}, + "finish_reason": None}]} + last = {"id": cid, "object": "chat.completion.chunk", "created": created, "model": model, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]} + self.wfile.write(f"data: {json.dumps(first)}\n\n".encode()) + self.wfile.write(f"data: {json.dumps(last)}\n\n".encode()) + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + else: + self._json(200, {"id": cid, "object": "chat.completion", "created": created, "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, + "finish_reason": "stop"}], + "usage": {"prompt_tokens": pc, "completion_tokens": ec, + "total_tokens": pc + ec}}, extra=hdr) + + def do_POST(self): + path = self.path.rstrip("/") + if path == "/v1/audio/transcriptions": + return self._handle_transcription() + if path == "/v1/audio/translations": + return self._handle_transcription(translate=True) + if path.startswith("/v1/messages"): + return self._proxy_anthropic(path) + if path == "/api/show": # ollama launch model probe (read-only) + length = int(self.headers.get("content-length", 0) or 0) + return self._proxy_ollama("/api/show", self.rfile.read(length) or b"{}") + if path != "/v1/chat/completions": + return self._json(404, {"error": "not found"}) + if not self._authed(): + return self._json(401, {"error": "unauthorized: missing/invalid Bearer token"}) + try: + length = int(self.headers.get("content-length", 0)) + body = json.loads(self.rfile.read(length) or b"{}") + except Exception as e: + return self._json(400, {"error": f"bad request: {e}"}) + + if has_audio(body.get("messages", [])): # OpenAI audio -> native Ollama path + return self._handle_audio(body) + + requested = (body.get("model") or "auto").strip() + auth = self.headers.get("authorization", "") + token = auth[7:].strip() if auth.startswith("Bearer ") else "" + force_uncensored = bool(UNCENSORED_KEY) and token == UNCENSORED_KEY + if force_uncensored or requested in ("auto-uncensored", "uncensored"): + model, decided_by = choose_uncensored(body), "uncensored" + elif requested not in ("auto", "router", "router/auto"): + model, decided_by = requested, "client" + else: + model, decided_by = choose_model(body), "router" + + initial_model = model # the routing decision, before any fallback + stream = bool(body.get("stream")) + tried = [] + while True: + body["model"] = model + tried.append(model) + print(f"[router] {decided_by:6s} -> {model}" + f"{' (stream)' if stream else ''}", flush=True) + try: + up = _open_upstream(body) + break + except urllib.error.HTTPError as e: + nxt = FALLBACK.get(model) + if nxt and nxt not in tried: + print(f"[router] {model} failed ({e.code}); fallback -> {nxt}", flush=True) + model, decided_by = nxt, "fallback" + continue + detail = e.read().decode(errors="replace")[:500] + return self._json(e.code, {"error": f"upstream {e.code}: {detail}", + "model": model}) + except Exception as e: + nxt = FALLBACK.get(model) + if nxt and nxt not in tried: + model, decided_by = nxt, "fallback" + continue + return self._json(502, {"error": f"upstream unreachable: {e}", "model": model}) + + hdr = {"x-router-model": model, "x-router-initial-model": initial_model, + "x-router-decided-by": decided_by} + if stream: + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.send_header("cache-control", "no-cache") + self.send_header("connection", "close") + for k, v in hdr.items(): + self.send_header(k, v) + self.end_headers() + try: + while True: + chunk = up.read(2048) + if not chunk: + break + self.wfile.write(chunk) + self.wfile.flush() + except Exception: + pass + finally: + up.close() + else: + data = up.read() + up.close() + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + for k, v in hdr.items(): + self.send_header(k, v) + self.end_headers() + self.wfile.write(data) + + +if __name__ == "__main__": + print(f">> llm-router on http://{HOST}:{PORT}/v1 (upstream {UPSTREAM}, " + f"auth {'ON' if API_KEY else 'OFF'})", flush=True) + ThreadingHTTPServer((HOST, PORT), Handler).serve_forever() diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..4cfa6aa --- /dev/null +++ b/run.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Serve the router stack. Used by launchd (and fine to run manually). +# Assumes ./.venv is already set up by start.sh. No pip here (works offline at boot). +set -uo pipefail +cd "$(dirname "$0")" +export PATH="/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin:${PATH:-}" + +# clear any stale instances via recorded PIDs (never pattern-match — that can +# match unrelated processes, including a shell that merely mentions "router.py") +for pf in .litellm.pid .router.pid; do + [ -f "$pf" ] && kill "$(cat "$pf" 2>/dev/null)" 2>/dev/null || true +done +sleep 1 + +export OLLAMA_KEEP_ALIVE="${OLLAMA_KEEP_ALIVE:-30m}" +curl -s http://127.0.0.1:11434/api/generate \ + -d '{"model":"qwen3:8b","prompt":"ok","stream":false,"keep_alive":-1}' >/dev/null 2>&1 || true + +# optional auth: create a file .apikey with a token to require Bearer auth +[ -f .apikey ] && export ROUTER_API_KEY="$(cat .apikey)" +# dedicated famapp key: any request with this Bearer token is forced onto uncensored models +[ -f .uncensored_key ] && export ROUTER_UNCENSORED_KEY="$(cat .uncensored_key)" + +# LiteLLM backend (internal only) +.venv/bin/litellm --config litellm.config.yaml --host 127.0.0.1 --port 4000 > litellm.log 2>&1 & +LITELLM_PID=$!; echo "$LITELLM_PID" > .litellm.pid +trap 'kill $LITELLM_PID 2>/dev/null || true; rm -f .router.pid' EXIT INT TERM +for i in $(seq 1 60); do + curl -sf http://127.0.0.1:4000/health/liveliness >/dev/null 2>&1 && break; sleep 2 +done + +# Router (network-facing). Backgrounded with recorded PID, then waited on so launchd +# tracks run.sh and the trap tears down LiteLLM when the router exits. +export ROUTER_UPSTREAM="http://127.0.0.1:4000/v1" +export ROUTER_HOST="0.0.0.0" +.venv/bin/python router.py & +ROUTER_PID=$!; echo "$ROUTER_PID" > .router.pid +wait $ROUTER_PID diff --git a/scripts/make-context-variant.sh b/scripts/make-context-variant.sh new file mode 100755 index 0000000..248d651 --- /dev/null +++ b/scripts/make-context-variant.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Create a copy of an Ollama model with a fixed context window (num_ctx). +# Useful because Ollama's default context is small and large prompts get truncated. +# +# Usage: ./scripts/make-context-variant.sh [new-tag] +# Example: ./scripts/make-context-variant.sh glm-4.7-flash 131072 glm-4.7-flash:128k +# +# Memory note: KV cache grows with num_ctx. On a 36 GB unified-memory Mac, keep +# the model + KV under ~27 GB (the default GPU wired ceiling) to avoid swapping. +set -euo pipefail +BASE="${1:?usage: make-context-variant.sh [new-tag]}" +NCTX="${2:?need a num_ctx value, e.g. 65536}" +TAG="${3:-${BASE%%:*}:${NCTX}ctx}" + +TMP="$(mktemp -t Modelfile.XXXXXX)" +printf 'FROM %s\nPARAMETER num_ctx %s\n' "$BASE" "$NCTX" > "$TMP" +ollama create "$TAG" -f "$TMP" +rm -f "$TMP" +echo ">> created $TAG (num_ctx=$NCTX from $BASE)" +ollama show "$TAG" | grep -i num_ctx || true diff --git a/scripts/pull-models.sh b/scripts/pull-models.sh new file mode 100755 index 0000000..5942e93 --- /dev/null +++ b/scripts/pull-models.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Pull a model fleet into Ollama. EDIT the MODELS list to taste, then run. +# Continues past failures; prints a summary. Ollama resumes partial downloads. +# +# The names here must line up with the routing map in router.py (M / UNCENSORED). +# If you change models, update router.py accordingly (see docs/ROUTING.md). +set -u + +MODELS=( + # --- general / small --- + "qwen3:8b" + "qwen3:14b" + # --- reasoning / agentic --- + "qwen3.6:35b-a3b" + "gpt-oss:20b" + "glm-4.7-flash" + # --- coding --- + "qwen3-coder:30b" + # --- vision / OCR --- + "qwen3-vl:8b" + "qwen3-vl:30b-a3b-instruct" + # --- multimodal (vision + audio) --- + "gemma4:e4b" # ROUTER_AUDIO_MODEL default — needed for /v1/audio/* + "gemma4:12b" + "gemma4:26b" + + # --- example: a GGUF straight from Hugging Face (see docs/ROUTING.md) --- + # "hf.co/USER/REPO:Q4_K_M" +) + +ok=(); fail=(); i=0; total=${#MODELS[@]} +for m in "${MODELS[@]}"; do + i=$((i+1)) + echo "=== [$i/$total] pulling $m ===" + if ollama pull "$m"; then ok+=("$m"); else echo "!! failed: $m"; fail+=("$m"); fi +done + +echo ""; echo "done. ${#ok[@]}/$total succeeded." +[ ${#fail[@]} -gt 0 ] && { echo "failed:"; printf ' - %s\n' "${fail[@]}"; } diff --git a/scripts/setup.sh b/scripts/setup.sh new file mode 100755 index 0000000..fb37693 --- /dev/null +++ b/scripts/setup.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# One-time setup: create the Python 3.12 venv and install LiteLLM + Pillow. +# The pre-router itself needs no venv (stdlib only); this is for the backend proxy. +set -euo pipefail +cd "$(dirname "$0")/.." + +PY="${PYTHON:-python3.12}" +if ! command -v "$PY" >/dev/null 2>&1; then + # common macOS Homebrew fallback + [ -x /opt/homebrew/opt/python@3.12/bin/python3.12 ] && PY=/opt/homebrew/opt/python@3.12/bin/python3.12 +fi +command -v "$PY" >/dev/null 2>&1 || { echo "Need Python 3.12. Set \$PYTHON, or:"; \ + echo " macOS: brew install python@3.12"; echo " Linux: install python3.12"; exit 1; } + +echo ">> using $($PY --version) at $(command -v "$PY" 2>/dev/null || echo "$PY")" +[ -d .venv ] || "$PY" -m venv .venv +.venv/bin/pip install -q --upgrade pip +.venv/bin/pip install -q -r requirements.txt +.venv/bin/python -c "import litellm, PIL" && echo ">> deps OK (litellm, pillow)" +echo "" +echo "Next:" +echo " ./scripts/pull-models.sh # pull a model fleet (edit the list first)" +echo " ./run.sh # start LiteLLM + router" diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..0fa8f1d --- /dev/null +++ b/start.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Full stack: network client -> pre-router (0.0.0.0:8080) -> LiteLLM (127.0.0.1:4000) -> Ollama (11434) +# Router is network-facing; LiteLLM stays internal. Uses a Python 3.12 venv for LiteLLM. +set -euo pipefail +cd "$(dirname "$0")" + +PY312="$(command -v python3.12 || echo /opt/homebrew/opt/python@3.12/bin/python3.12)" +[ -x "$PY312" ] || { echo "python3.12 missing. Run: brew install python@3.12"; exit 1; } + +# venv + deps (first run only) +if [ ! -d .venv ]; then "$PY312" -m venv .venv; fi +source .venv/bin/activate +pip -q install --upgrade pip >/dev/null +pip -q install "litellm[proxy]" pillow >/dev/null # pillow: for the vision benchmark + +# keep the tiny classifier resident +export OLLAMA_KEEP_ALIVE="${OLLAMA_KEEP_ALIVE:-30m}" +curl -s http://127.0.0.1:11434/api/generate \ + -d '{"model":"qwen3:8b","prompt":"ok","stream":false,"keep_alive":-1}' >/dev/null 2>&1 || true + +# LiteLLM proxy (internal only) +echo ">> starting LiteLLM proxy on 127.0.0.1:4000 ..." +litellm --config litellm.config.yaml --host 127.0.0.1 --port 4000 > litellm.log 2>&1 & +LITELLM_PID=$! +trap 'kill $LITELLM_PID 2>/dev/null || true' EXIT +for i in $(seq 1 60); do + curl -sf http://127.0.0.1:4000/health/liveliness >/dev/null 2>&1 && { echo " LiteLLM up."; break; } + sleep 2 +done + +# Pre-router — network facing, forwards to LiteLLM +export ROUTER_UPSTREAM="http://127.0.0.1:4000/v1" +export ROUTER_HOST="0.0.0.0" +# SECURITY: uncomment to require a token from network clients (recommended): +# export ROUTER_API_KEY="$(cat .apikey 2>/dev/null || true)" +if [ -z "${ROUTER_API_KEY:-}" ]; then + echo "!! WARNING: router is OPEN on the LAN (no ROUTER_API_KEY set)." +fi +LANIP=$(ipconfig getifaddr en7 2>/dev/null || ipconfig getifaddr en0 2>/dev/null || echo "") +echo ">> router reachable at: http://${LANIP}:8080/v1 (model \"auto\")" +exec python router.py diff --git a/uncensored_key.example b/uncensored_key.example new file mode 100644 index 0000000..1c913d0 --- /dev/null +++ b/uncensored_key.example @@ -0,0 +1,9 @@ +Copy this file to `.uncensored_key` and put a single secret token in it. +Any request whose `Authorization: Bearer ` matches this value is FORCED +onto the uncensored model fleet, regardless of the model it asks for. Use it to +pin a specific client (e.g. a family assistant app) to uncensored models. + +This is independent of `.apikey` — the router can be open (no auth) and still +honor this key to switch a client to the uncensored fleet. + +Generate one: echo "famapp-$(openssl rand -hex 16)"