Joseph CostaandClaude Opus 4.8
9938d46a67
A stdlib pre-router in front of Ollama with LiteLLM backend: - auto model selection by content/tools/modality, with fallbacks - OpenAI /v1, Anthropic /v1/messages, and Ollama-native /api/* endpoints - Whisper-shaped /v1/audio/transcriptions + in-chat audio - key-based fleet policies (e.g. force a client onto uncensored models) - optional Bearer auth; launchd/systemd service install - benchmark harnesses (speed, quality, agentic tool use) with sample results Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
538 lines
24 KiB
Python
538 lines
24 KiB
Python
"""
|
|
Local model router (stdlib-only; runs on any Python 3.9+, incl. 3.14).
|
|
|
|
An OpenAI-compatible endpoint that picks the best local model per request and
|
|
forwards straight to Ollama's own OpenAI API. Point every client at
|
|
http://localhost:8080/v1
|
|
- model "auto" -> router decides (modality -> heuristics -> tiny classifier)
|
|
- any real name -> passed through unchanged (manual override always wins)
|
|
On upstream failure the router retries once with a mapped fallback model.
|
|
Response headers x-router-model / x-router-decided-by report the pick.
|
|
"""
|
|
import os, re, json, time, base64, urllib.request, urllib.error
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
|
|
OLLAMA = os.environ.get("ROUTER_OLLAMA", "http://127.0.0.1:11434").rstrip("/") # client target; NOT OLLAMA_HOST (that's the server bind addr, e.g. 0.0.0.0)
|
|
UPSTREAM = os.environ.get("ROUTER_UPSTREAM", OLLAMA + "/v1").rstrip("/")
|
|
HOST = os.environ.get("ROUTER_HOST", "0.0.0.0") # 0.0.0.0 = reachable on the LAN
|
|
PORT = int(os.environ.get("ROUTER_PORT", "8080"))
|
|
API_KEY = os.environ.get("ROUTER_API_KEY", "").strip() # if set, require Bearer token
|
|
AUDIO_MODEL = os.environ.get("ROUTER_AUDIO_MODEL", "gemma4:e4b") # audio only works on official Gemma 4
|
|
|
|
M = {
|
|
"fast": "qwen3:8b",
|
|
"general": "qwen3:14b",
|
|
"code": "glm-4.7-flash",
|
|
"code_heavy": "qwen3-coder:30b",
|
|
"reason": "qwen3.6:35b-a3b",
|
|
"agentic": "glm-4.7-flash", # tool-carrying requests (general agent): best hard-agentic (7/7, only one to recover from tool errors)
|
|
"vision": "qwen3-vl:8b",
|
|
"vision_heavy": "qwen3-vl:30b-a3b-instruct",
|
|
}
|
|
CLASSIFIER = "qwen3:8b"
|
|
SHORT_CHARS = 240
|
|
|
|
# Uncensored fleet. A request routes here if it asks for model "auto-uncensored"
|
|
# OR carries the dedicated famapp key (ROUTER_UNCENSORED_KEY) — in the latter case
|
|
# EVERY request from that client is forced uncensored regardless of model asked for.
|
|
UNCENSORED = {
|
|
"general": "gemma4-uncensored:26b",
|
|
"vision": "gemma4-uncensored:26b", # has vision
|
|
"code": "glm-uncensored-code:30b",
|
|
"fast": "gemma-e4b-uncensored",
|
|
}
|
|
UNCENSORED_KEY = os.environ.get("ROUTER_UNCENSORED_KEY", "").strip()
|
|
|
|
FALLBACK = {
|
|
"glm-4.7-flash": "qwen3-coder:30b",
|
|
"qwen3-coder:30b": "glm-4.7-flash",
|
|
"gemma4:26b": "qwen3.6:35b-a3b",
|
|
"qwen3.6:35b-a3b": "gpt-oss:20b",
|
|
"gpt-oss:20b": "qwen3:14b",
|
|
"qwen3-vl:30b-a3b-instruct": "qwen3-vl:8b",
|
|
"qwen3:14b": "qwen3:8b",
|
|
"qwen3-vl:8b": "gemma4:12b",
|
|
# uncensored models still downloading fall back to the one that's ready
|
|
"glm-uncensored-code:30b": "gemma4-uncensored:26b",
|
|
"gemma-e4b-uncensored": "gemma4-uncensored:26b",
|
|
}
|
|
|
|
CODE_RE = re.compile(r"```|\b(def |class |function |import |#include|refactor|debug|"
|
|
r"stack ?trace|traceback|compile|unit test|npm |pip |git |bug|"
|
|
r"exception|null pointer|segfault)\b|\.(py|js|ts|tsx|go|rs|java|"
|
|
r"cpp|cc|c|rb|sh|sql)\b", re.I)
|
|
CODE_HEAVY_RE = re.compile(r"\b(repo|repository|codebase|multiple files|entire project|"
|
|
r"across files|whole project)\b", re.I)
|
|
REASON_RE = re.compile(r"\b(prove|theorem|step[- ]by[- ]step|reason through|derive|"
|
|
r"calculate|logic puzzle|why does|explain why|work through)\b", re.I)
|
|
OCR_RE = re.compile(r"\b(ocr|document|invoice|receipt|table|form|handwrit|extract text|"
|
|
r"scan|parse this (image|page|pdf))\b", re.I)
|
|
|
|
|
|
def last_user_text(messages):
|
|
for m in reversed(messages):
|
|
if m.get("role") == "user":
|
|
c = m.get("content")
|
|
if isinstance(c, str):
|
|
return c
|
|
if isinstance(c, list):
|
|
return " ".join(p.get("text", "") for p in c
|
|
if isinstance(p, dict) and p.get("type") == "text")
|
|
return ""
|
|
|
|
|
|
def has_image(messages):
|
|
for m in messages:
|
|
c = m.get("content")
|
|
if isinstance(c, list):
|
|
for p in c:
|
|
if isinstance(p, dict) and p.get("type") in ("image_url", "input_image", "image"):
|
|
return True
|
|
return False
|
|
|
|
|
|
def has_audio(messages):
|
|
for m in messages:
|
|
c = m.get("content")
|
|
if isinstance(c, list):
|
|
for p in c:
|
|
if isinstance(p, dict) and p.get("type") in ("input_audio", "audio"):
|
|
return True
|
|
return False
|
|
|
|
|
|
def to_native_messages(messages):
|
|
# OpenAI content -> Ollama native: text flattened, audio/image base64 into `images`
|
|
out = []
|
|
for m in messages:
|
|
role, c = m.get("role", "user"), m.get("content")
|
|
if isinstance(c, str):
|
|
out.append({"role": role, "content": c})
|
|
continue
|
|
text_parts, media = [], []
|
|
if isinstance(c, list):
|
|
for p in c:
|
|
if not isinstance(p, dict):
|
|
continue
|
|
t = p.get("type")
|
|
if t == "text":
|
|
text_parts.append(p.get("text", ""))
|
|
elif t in ("input_audio", "audio"):
|
|
d = (p.get("input_audio") or p.get("audio") or {}).get("data")
|
|
if d:
|
|
media.append(d)
|
|
elif t in ("image_url", "image", "input_image"):
|
|
url = (p.get("image_url") or {}).get("url", "") if isinstance(p.get("image_url"), dict) else ""
|
|
media.append(url.split(",", 1)[-1] if url.startswith("data:") else url)
|
|
msg = {"role": role, "content": " ".join(tp for tp in text_parts if tp)}
|
|
if media:
|
|
msg["images"] = media
|
|
out.append(msg)
|
|
return out
|
|
|
|
|
|
def parse_multipart(body, boundary):
|
|
# minimal multipart/form-data parser -> {name: (filename_or_None, bytes)}
|
|
fields = {}
|
|
for part in body.split(b"--" + boundary.encode()):
|
|
if part in (b"", b"--", b"--\r\n", b"\r\n"):
|
|
continue
|
|
if part.startswith(b"\r\n"):
|
|
part = part[2:]
|
|
if b"\r\n\r\n" not in part:
|
|
continue
|
|
head, _, data = part.partition(b"\r\n\r\n")
|
|
if data.endswith(b"\r\n"):
|
|
data = data[:-2]
|
|
head_text = head.decode("utf-8", "replace")
|
|
nm = re.search(r'name="([^"]*)"', head_text)
|
|
if not nm:
|
|
continue
|
|
fm = re.search(r'filename="([^"]*)"', head_text)
|
|
fields[nm.group(1)] = (fm.group(1) if fm else None, data)
|
|
return fields
|
|
|
|
|
|
def total_len(messages):
|
|
n = 0
|
|
for m in messages:
|
|
c = m.get("content")
|
|
if isinstance(c, str):
|
|
n += len(c)
|
|
elif isinstance(c, list):
|
|
for p in c:
|
|
if isinstance(p, dict):
|
|
n += len(p.get("text", ""))
|
|
return n
|
|
|
|
|
|
def classify(text):
|
|
payload = json.dumps({
|
|
"model": CLASSIFIER,
|
|
"prompt": ("Classify the user's task into exactly one label from: "
|
|
"code, reason, vision, general. Reply with only the label.\n\n"
|
|
"Task: " + text[:2000]),
|
|
"stream": False, "keep_alive": -1,
|
|
"options": {"temperature": 0, "num_predict": 3},
|
|
}).encode()
|
|
try:
|
|
req = urllib.request.Request(OLLAMA + "/api/generate", data=payload,
|
|
headers={"content-type": "application/json"})
|
|
with urllib.request.urlopen(req, timeout=30) as r:
|
|
out = json.loads(r.read()).get("response", "").strip().lower()
|
|
for lab in ("code", "reason", "vision", "general"):
|
|
if lab in out:
|
|
return lab
|
|
except Exception:
|
|
pass
|
|
return "general"
|
|
|
|
|
|
def choose_model(body):
|
|
messages = body.get("messages", [])
|
|
text = last_user_text(messages)
|
|
if has_image(messages):
|
|
return M["vision_heavy"] if OCR_RE.search(text) else M["vision"]
|
|
if body.get("tools"): # tool-carrying -> agentic (code agent vs general agent)
|
|
return M["code_heavy"] if CODE_RE.search(text) else M["agentic"]
|
|
if CODE_RE.search(text):
|
|
return M["code_heavy"] if CODE_HEAVY_RE.search(text) else M["code"]
|
|
if REASON_RE.search(text):
|
|
return M["reason"]
|
|
if total_len(messages) < SHORT_CHARS:
|
|
return M["fast"]
|
|
lab = classify(text)
|
|
return {"code": M["code"], "reason": M["reason"],
|
|
"vision": M["vision"], "general": M["general"]}[lab]
|
|
|
|
|
|
def choose_uncensored(body):
|
|
# same content heuristics, but only ever selects from the uncensored fleet
|
|
messages = body.get("messages", [])
|
|
text = last_user_text(messages)
|
|
if has_image(messages):
|
|
return UNCENSORED["vision"]
|
|
if CODE_RE.search(text):
|
|
return UNCENSORED["code"]
|
|
if total_len(messages) < SHORT_CHARS:
|
|
return UNCENSORED["fast"]
|
|
return UNCENSORED["general"]
|
|
|
|
|
|
def _open_upstream(body):
|
|
data = json.dumps(body).encode()
|
|
req = urllib.request.Request(UPSTREAM + "/chat/completions", data=data,
|
|
headers={"content-type": "application/json"})
|
|
return urllib.request.urlopen(req, timeout=600)
|
|
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
protocol_version = "HTTP/1.1"
|
|
|
|
def log_message(self, fmt, *args): # access log for EVERY request (incl 401/404)
|
|
ua = self.headers.get("user-agent", "-")[:40]
|
|
auth = "auth" if self.headers.get("authorization") else "noauth"
|
|
print(f"[access] {time.strftime('%H:%M:%S')} {self.address_string()} "
|
|
f"{auth} \"{ua}\" {fmt % args}", flush=True)
|
|
|
|
def _authed(self):
|
|
if not API_KEY:
|
|
return True
|
|
auth = self.headers.get("authorization", "")
|
|
return auth.startswith("Bearer ") and auth[7:].strip() == API_KEY
|
|
|
|
def _json(self, code, obj, extra=None):
|
|
payload = json.dumps(obj).encode()
|
|
self.send_response(code)
|
|
self.send_header("content-type", "application/json")
|
|
self.send_header("content-length", str(len(payload)))
|
|
for k, v in (extra or {}).items():
|
|
self.send_header(k, v)
|
|
self.end_headers()
|
|
self.wfile.write(payload)
|
|
|
|
def _text(self, code, s):
|
|
data = s.encode()
|
|
self.send_response(code)
|
|
self.send_header("content-type", "text/plain")
|
|
self.send_header("content-length", str(len(data)))
|
|
self.end_headers()
|
|
self.wfile.write(data)
|
|
|
|
def _proxy_ollama(self, path, data=None):
|
|
# read-only native Ollama endpoints for `ollama launch` probing.
|
|
# NOTE: only metadata paths are wired up; inference stays on authed /v1/*.
|
|
req = urllib.request.Request(OLLAMA + path, data=data,
|
|
headers={"content-type": "application/json"},
|
|
method="POST" if data is not None else "GET")
|
|
try:
|
|
up = urllib.request.urlopen(req, timeout=30)
|
|
body, code = up.read(), 200
|
|
ct = up.headers.get("content-type", "application/json")
|
|
up.close()
|
|
except urllib.error.HTTPError as e:
|
|
body, code, ct = e.read(), e.code, "application/json"
|
|
except Exception as e:
|
|
return self._json(502, {"error": f"ollama unreachable: {e}"})
|
|
self.send_response(code)
|
|
self.send_header("content-type", ct)
|
|
self.send_header("content-length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def do_GET(self):
|
|
if self.path.rstrip("/") == "/healthz":
|
|
return self._json(200, {"ok": True})
|
|
if self.path.rstrip("/") == "/v1/models":
|
|
if not self._authed():
|
|
return self._json(401, {"error": "unauthorized"})
|
|
seen = list(dict.fromkeys(M.values()))
|
|
return self._json(200, {"object": "list",
|
|
"data": [{"id": "auto", "object": "model"}] +
|
|
[{"id": m, "object": "model"} for m in seen]})
|
|
# native Ollama probe endpoints (read-only) so `ollama launch` accepts this host
|
|
if self.path in ("/", ""):
|
|
return self._text(200, "Ollama is running")
|
|
if self.path.rstrip("/") in ("/api/version", "/api/tags", "/api/ps"):
|
|
return self._proxy_ollama(self.path)
|
|
return self._json(404, {"error": "not found"})
|
|
|
|
def _proxy_anthropic(self, path):
|
|
# transparent proxy for Anthropic Messages API (Claude Code) -> LiteLLM /v1/messages
|
|
if not self._authed():
|
|
return self._json(401, {"error": "unauthorized"})
|
|
try:
|
|
length = int(self.headers.get("content-length", 0) or 0)
|
|
body = self.rfile.read(length)
|
|
except Exception as e:
|
|
return self._json(400, {"error": f"bad request: {e}"})
|
|
url = UPSTREAM + path[len("/v1"):] # /v1/messages -> <litellm>/v1/messages
|
|
headers = {"content-type": "application/json"}
|
|
for h in ("authorization", "anthropic-version", "anthropic-beta"):
|
|
v = self.headers.get(h)
|
|
if v:
|
|
headers[h] = v
|
|
try:
|
|
up = urllib.request.urlopen(
|
|
urllib.request.Request(url, data=body, headers=headers), timeout=600)
|
|
except urllib.error.HTTPError as e:
|
|
data = e.read()
|
|
self.send_response(e.code)
|
|
self.send_header("content-type", e.headers.get("content-type", "application/json"))
|
|
self.send_header("content-length", str(len(data)))
|
|
self.end_headers()
|
|
self.wfile.write(data)
|
|
return
|
|
except Exception as e:
|
|
return self._json(502, {"error": f"upstream unreachable: {e}"})
|
|
ctype = up.headers.get("content-type", "application/json")
|
|
if "event-stream" in ctype:
|
|
self.send_response(200)
|
|
self.send_header("content-type", ctype)
|
|
self.send_header("cache-control", "no-cache")
|
|
self.send_header("connection", "close")
|
|
self.end_headers()
|
|
try:
|
|
while True:
|
|
chunk = up.read(2048)
|
|
if not chunk:
|
|
break
|
|
self.wfile.write(chunk)
|
|
self.wfile.flush()
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
up.close()
|
|
else:
|
|
data = up.read()
|
|
up.close()
|
|
self.send_response(200)
|
|
self.send_header("content-type", ctype)
|
|
self.send_header("content-length", str(len(data)))
|
|
self.end_headers()
|
|
self.wfile.write(data)
|
|
|
|
def _handle_transcription(self, translate=False):
|
|
# OpenAI /v1/audio/transcriptions (Whisper-shaped) backed by Gemma 4
|
|
ct = self.headers.get("content-type", "")
|
|
m = re.search(r"boundary=([^;]+)", ct)
|
|
if "multipart/form-data" not in ct or not m:
|
|
return self._json(400, {"error": "expected multipart/form-data with a boundary"})
|
|
boundary = m.group(1).strip().strip('"')
|
|
length = int(self.headers.get("content-length", 0) or 0)
|
|
fields = parse_multipart(self.rfile.read(length), boundary)
|
|
if "file" not in fields or fields["file"][1] is None:
|
|
return self._json(400, {"error": "missing 'file' form field (the audio)"})
|
|
audio_b64 = base64.b64encode(fields["file"][1]).decode()
|
|
resp_format = (fields.get("response_format", (None, b"json"))[1] or b"json").decode().strip()
|
|
prompt = ("Transcribe the speech in this audio and translate it into English. "
|
|
"Output only the English text, nothing else." if translate else
|
|
"Transcribe the speech in this audio verbatim. "
|
|
"Output only the transcription text, nothing else.")
|
|
native = {"model": AUDIO_MODEL, "stream": False, "keep_alive": "30m",
|
|
"options": {"temperature": 0},
|
|
"messages": [{"role": "user", "content": prompt, "images": [audio_b64]}]}
|
|
print(f"[router] stt -> {AUDIO_MODEL}", flush=True)
|
|
try:
|
|
req = urllib.request.Request(OLLAMA + "/api/chat", data=json.dumps(native).encode(),
|
|
headers={"content-type": "application/json"})
|
|
with urllib.request.urlopen(req, timeout=600) as r:
|
|
d = json.loads(r.read())
|
|
except urllib.error.HTTPError as e:
|
|
return self._json(e.code, {"error": f"stt upstream {e.code}: {e.read().decode()[:300]}"})
|
|
except Exception as e:
|
|
return self._json(502, {"error": f"stt upstream: {e}"})
|
|
text = ((d.get("message") or {}).get("content", "") or "").strip()
|
|
if resp_format == "text":
|
|
return self._text(200, text)
|
|
return self._json(200, {"text": text})
|
|
|
|
def _handle_audio(self, body):
|
|
# OpenAI request carrying audio -> Ollama native /api/chat (images field), Gemma 4
|
|
model = AUDIO_MODEL
|
|
native = {"model": model, "messages": to_native_messages(body.get("messages", [])),
|
|
"stream": False, "keep_alive": "30m", "options": {}}
|
|
if isinstance(body.get("options"), dict):
|
|
native["options"].update(body["options"])
|
|
if body.get("max_tokens"):
|
|
native["options"]["num_predict"] = body["max_tokens"]
|
|
if body.get("temperature") is not None:
|
|
native["options"]["temperature"] = body["temperature"]
|
|
print(f"[router] audio -> {model}", flush=True)
|
|
try:
|
|
req = urllib.request.Request(OLLAMA + "/api/chat", data=json.dumps(native).encode(),
|
|
headers={"content-type": "application/json"})
|
|
with urllib.request.urlopen(req, timeout=600) as r:
|
|
d = json.loads(r.read())
|
|
except urllib.error.HTTPError as e:
|
|
return self._json(e.code, {"error": f"audio upstream {e.code}: {e.read().decode()[:300]}"})
|
|
except Exception as e:
|
|
return self._json(502, {"error": f"audio upstream: {e}"})
|
|
content = (d.get("message") or {}).get("content", "") or ""
|
|
pc, ec = d.get("prompt_eval_count", 0) or 0, d.get("eval_count", 0) or 0
|
|
cid, created = "chatcmpl-audio", int(time.time())
|
|
hdr = {"x-router-model": model, "x-router-initial-model": model, "x-router-decided-by": "audio"}
|
|
if body.get("stream"):
|
|
self.send_response(200)
|
|
self.send_header("content-type", "text/event-stream")
|
|
self.send_header("cache-control", "no-cache")
|
|
self.send_header("connection", "close")
|
|
for k, v in hdr.items():
|
|
self.send_header(k, v)
|
|
self.end_headers()
|
|
first = {"id": cid, "object": "chat.completion.chunk", "created": created, "model": model,
|
|
"choices": [{"index": 0, "delta": {"role": "assistant", "content": content},
|
|
"finish_reason": None}]}
|
|
last = {"id": cid, "object": "chat.completion.chunk", "created": created, "model": model,
|
|
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}
|
|
self.wfile.write(f"data: {json.dumps(first)}\n\n".encode())
|
|
self.wfile.write(f"data: {json.dumps(last)}\n\n".encode())
|
|
self.wfile.write(b"data: [DONE]\n\n")
|
|
self.wfile.flush()
|
|
else:
|
|
self._json(200, {"id": cid, "object": "chat.completion", "created": created, "model": model,
|
|
"choices": [{"index": 0, "message": {"role": "assistant", "content": content},
|
|
"finish_reason": "stop"}],
|
|
"usage": {"prompt_tokens": pc, "completion_tokens": ec,
|
|
"total_tokens": pc + ec}}, extra=hdr)
|
|
|
|
def do_POST(self):
|
|
path = self.path.rstrip("/")
|
|
if path == "/v1/audio/transcriptions":
|
|
return self._handle_transcription()
|
|
if path == "/v1/audio/translations":
|
|
return self._handle_transcription(translate=True)
|
|
if path.startswith("/v1/messages"):
|
|
return self._proxy_anthropic(path)
|
|
if path == "/api/show": # ollama launch model probe (read-only)
|
|
length = int(self.headers.get("content-length", 0) or 0)
|
|
return self._proxy_ollama("/api/show", self.rfile.read(length) or b"{}")
|
|
if path != "/v1/chat/completions":
|
|
return self._json(404, {"error": "not found"})
|
|
if not self._authed():
|
|
return self._json(401, {"error": "unauthorized: missing/invalid Bearer token"})
|
|
try:
|
|
length = int(self.headers.get("content-length", 0))
|
|
body = json.loads(self.rfile.read(length) or b"{}")
|
|
except Exception as e:
|
|
return self._json(400, {"error": f"bad request: {e}"})
|
|
|
|
if has_audio(body.get("messages", [])): # OpenAI audio -> native Ollama path
|
|
return self._handle_audio(body)
|
|
|
|
requested = (body.get("model") or "auto").strip()
|
|
auth = self.headers.get("authorization", "")
|
|
token = auth[7:].strip() if auth.startswith("Bearer ") else ""
|
|
force_uncensored = bool(UNCENSORED_KEY) and token == UNCENSORED_KEY
|
|
if force_uncensored or requested in ("auto-uncensored", "uncensored"):
|
|
model, decided_by = choose_uncensored(body), "uncensored"
|
|
elif requested not in ("auto", "router", "router/auto"):
|
|
model, decided_by = requested, "client"
|
|
else:
|
|
model, decided_by = choose_model(body), "router"
|
|
|
|
initial_model = model # the routing decision, before any fallback
|
|
stream = bool(body.get("stream"))
|
|
tried = []
|
|
while True:
|
|
body["model"] = model
|
|
tried.append(model)
|
|
print(f"[router] {decided_by:6s} -> {model}"
|
|
f"{' (stream)' if stream else ''}", flush=True)
|
|
try:
|
|
up = _open_upstream(body)
|
|
break
|
|
except urllib.error.HTTPError as e:
|
|
nxt = FALLBACK.get(model)
|
|
if nxt and nxt not in tried:
|
|
print(f"[router] {model} failed ({e.code}); fallback -> {nxt}", flush=True)
|
|
model, decided_by = nxt, "fallback"
|
|
continue
|
|
detail = e.read().decode(errors="replace")[:500]
|
|
return self._json(e.code, {"error": f"upstream {e.code}: {detail}",
|
|
"model": model})
|
|
except Exception as e:
|
|
nxt = FALLBACK.get(model)
|
|
if nxt and nxt not in tried:
|
|
model, decided_by = nxt, "fallback"
|
|
continue
|
|
return self._json(502, {"error": f"upstream unreachable: {e}", "model": model})
|
|
|
|
hdr = {"x-router-model": model, "x-router-initial-model": initial_model,
|
|
"x-router-decided-by": decided_by}
|
|
if stream:
|
|
self.send_response(200)
|
|
self.send_header("content-type", "text/event-stream")
|
|
self.send_header("cache-control", "no-cache")
|
|
self.send_header("connection", "close")
|
|
for k, v in hdr.items():
|
|
self.send_header(k, v)
|
|
self.end_headers()
|
|
try:
|
|
while True:
|
|
chunk = up.read(2048)
|
|
if not chunk:
|
|
break
|
|
self.wfile.write(chunk)
|
|
self.wfile.flush()
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
up.close()
|
|
else:
|
|
data = up.read()
|
|
up.close()
|
|
self.send_response(200)
|
|
self.send_header("content-type", "application/json")
|
|
self.send_header("content-length", str(len(data)))
|
|
for k, v in hdr.items():
|
|
self.send_header(k, v)
|
|
self.end_headers()
|
|
self.wfile.write(data)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
print(f">> llm-router on http://{HOST}:{PORT}/v1 (upstream {UPSTREAM}, "
|
|
f"auth {'ON' if API_KEY else 'OFF'})", flush=True)
|
|
ThreadingHTTPServer((HOST, PORT), Handler).serve_forever()
|