Initial commit: llm-router — smart OpenAI/Anthropic/Ollama router
A stdlib pre-router in front of Ollama with LiteLLM backend: - auto model selection by content/tools/modality, with fallbacks - OpenAI /v1, Anthropic /v1/messages, and Ollama-native /api/* endpoints - Whisper-shaped /v1/audio/transcriptions + in-chat audio - key-based fleet policies (e.g. force a client onto uncensored models) - optional Bearer auth; launchd/systemd service install - benchmark harnesses (speed, quality, agentic tool use) with sample results Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Joseph Costa
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Agentic tool-use benchmark for the local fleet, via Ollama's native tools API.
|
||||
|
||||
Scores the real agent failure modes (each 0/1, objective):
|
||||
A invoke + args : calls calculator for a math Q, expression evaluates correctly
|
||||
B tool selection : picks get_weather (not calc/search) for a weather Q, right city
|
||||
E multi-arg : get_weather with correct city AND unit
|
||||
C use-result : calls tool, we return a value, final answer uses it
|
||||
D abstain : does NOT call a tool for a plain greeting
|
||||
|
||||
overall = mean of the 6 checks. think disabled for a fast, direct baseline.
|
||||
|
||||
Run after models are downloaded: ./.venv/bin/python agent_bench.py
|
||||
Writes agent_report.md and agent_results.json.
|
||||
"""
|
||||
import json, re, datetime, urllib.request
|
||||
import benchmark as b
|
||||
|
||||
OLLAMA = "http://127.0.0.1:11434"
|
||||
|
||||
CALC = {"type": "function", "function": {
|
||||
"name": "calculator",
|
||||
"description": "Evaluate an arithmetic expression and return the number.",
|
||||
"parameters": {"type": "object",
|
||||
"properties": {"expression": {"type": "string",
|
||||
"description": "e.g. '3*4+1'"}},
|
||||
"required": ["expression"]}}}
|
||||
WEATHER = {"type": "function", "function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather for a city.",
|
||||
"parameters": {"type": "object",
|
||||
"properties": {"city": {"type": "string"},
|
||||
"unit": {"type": "string",
|
||||
"enum": ["celsius", "fahrenheit"]}},
|
||||
"required": ["city", "unit"]}}}
|
||||
SEARCH = {"type": "function", "function": {
|
||||
"name": "search_kb",
|
||||
"description": "Search the internal company knowledge base for a query.",
|
||||
"parameters": {"type": "object",
|
||||
"properties": {"query": {"type": "string"}},
|
||||
"required": ["query"]}}}
|
||||
|
||||
|
||||
def chat_tools(model, messages, tools, num_predict=512, think=False):
|
||||
body = {"model": model, "messages": messages, "tools": tools, "stream": False,
|
||||
"keep_alive": "30m", "options": {"temperature": 0, "num_predict": num_predict}}
|
||||
if think is not None:
|
||||
body["think"] = think
|
||||
req = urllib.request.Request(OLLAMA + "/api/chat", data=json.dumps(body).encode(),
|
||||
headers={"content-type": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=600) as r:
|
||||
return json.loads(r.read()).get("message", {}) or {}
|
||||
except Exception:
|
||||
# some models reject think param with tools; retry without
|
||||
body.pop("think", None)
|
||||
req = urllib.request.Request(OLLAMA + "/api/chat", data=json.dumps(body).encode(),
|
||||
headers={"content-type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=600) as r:
|
||||
return json.loads(r.read()).get("message", {}) or {}
|
||||
|
||||
|
||||
def first_call(msg):
|
||||
tcs = msg.get("tool_calls") or []
|
||||
if not tcs:
|
||||
return None, {}
|
||||
fn = tcs[0].get("function", {}) or {}
|
||||
args = fn.get("arguments")
|
||||
if isinstance(args, str):
|
||||
try:
|
||||
args = json.loads(args)
|
||||
except Exception:
|
||||
args = {}
|
||||
return fn.get("name"), (args or {})
|
||||
|
||||
|
||||
def safe_eval(expr):
|
||||
if not re.fullmatch(r"[0-9+\-*/(). ]+", expr or ""):
|
||||
raise ValueError("unsafe")
|
||||
return eval(expr, {"__builtins__": {}}, {})
|
||||
|
||||
|
||||
def run_agent(model):
|
||||
c = {}
|
||||
# A: invoke calculator + correct expression
|
||||
name, args = first_call(chat_tools(model, [b.user("What is 3847 * 293? Use the calculator tool.")], [CALC]))
|
||||
c["invoke"] = 1 if name == "calculator" else 0
|
||||
try:
|
||||
c["args"] = 1 if c["invoke"] and abs(safe_eval(str(args.get("expression", ""))) - 3847 * 293) < 1e-6 else 0
|
||||
except Exception:
|
||||
c["args"] = 0
|
||||
|
||||
# B: tool selection among three + right city
|
||||
name, args = first_call(chat_tools(model, [b.user("What's the weather in Tokyo, in celsius?")],
|
||||
[CALC, WEATHER, SEARCH]))
|
||||
c["select"] = 1 if name == "get_weather" and "tokyo" in str(args.get("city", "")).lower() else 0
|
||||
|
||||
# E: multi-arg correctness
|
||||
name, args = first_call(chat_tools(model, [b.user("Get the weather in Paris in fahrenheit.")],
|
||||
[CALC, WEATHER, SEARCH]))
|
||||
c["multiarg"] = 1 if (name == "get_weather" and "paris" in str(args.get("city", "")).lower()
|
||||
and "f" in str(args.get("unit", "")).lower()) else 0
|
||||
|
||||
# C: multi-turn, use the returned result
|
||||
msgs = [b.user("I have 17 boxes with 24 apples each. How many apples total? Use the calculator.")]
|
||||
msg = chat_tools(model, msgs, [CALC])
|
||||
name, _ = first_call(msg)
|
||||
c["useresult"] = 0
|
||||
if name == "calculator":
|
||||
msgs.append({"role": "assistant", "content": "", "tool_calls": msg.get("tool_calls")})
|
||||
msgs.append({"role": "tool", "content": "408"})
|
||||
final = chat_tools(model, msgs, [CALC], num_predict=256)
|
||||
c["useresult"] = 1 if "408" in (final.get("content", "") or "") else 0
|
||||
|
||||
# D: abstain when no tool is needed
|
||||
name, _ = first_call(chat_tools(model, [b.user("Say hello in exactly one word.")],
|
||||
[CALC, WEATHER, SEARCH]))
|
||||
c["abstain"] = 1 if name is None else 0
|
||||
|
||||
c["overall"] = sum(c[k] for k in ("invoke", "args", "select", "multiarg", "useresult", "abstain")) / 6
|
||||
return c
|
||||
|
||||
|
||||
def main():
|
||||
models = b.list_models()
|
||||
results = {}
|
||||
for i, m in enumerate(models, 1):
|
||||
print(f"[{i}/{len(models)}] {m} ...", flush=True)
|
||||
try:
|
||||
results[m] = run_agent(m)
|
||||
s = results[m]
|
||||
print(f" overall {s['overall']*100:4.0f}% invoke {s['invoke']} args {s['args']} "
|
||||
f"select {s['select']} multiarg {s['multiarg']} useresult {s['useresult']} "
|
||||
f"abstain {s['abstain']}", flush=True)
|
||||
except Exception as e:
|
||||
print(f" ERROR: {e}", flush=True)
|
||||
results[m] = {"error": str(e)}
|
||||
|
||||
with open("agent_results.json", "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
|
||||
ts = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
|
||||
ranked = sorted([(m, s) for m, s in results.items() if "overall" in s],
|
||||
key=lambda x: x[1]["overall"], reverse=True)
|
||||
with open("agent_report.md", "w") as f:
|
||||
f.write(f"# Agentic tool-use benchmark — {ts}\n\n")
|
||||
f.write("Via Ollama tools API. Each column 0/1 (or %). abstain = correctly did NOT "
|
||||
"call a tool when none was needed.\n\n")
|
||||
f.write("| Model | Overall | Invoke | Args | Select | MultiArg | UseResult | Abstain |\n")
|
||||
f.write("|---|--:|:--:|:--:|:--:|:--:|:--:|:--:|\n")
|
||||
for m, s in ranked:
|
||||
f.write(f"| `{m}` | **{s['overall']*100:.0f}%** | {s['invoke']} | {s['args']} | "
|
||||
f"{s['select']} | {s['multiarg']} | {s['useresult']} | {s['abstain']} |\n")
|
||||
print("\nWrote agent_report.md and agent_results.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Full test + benchmark suite for the local model fleet.
|
||||
|
||||
Perf : per-model generation tok/s, prompt tok/s, cold load time, GPU placement
|
||||
(from Ollama's own eval metrics - objective, not wall-clock guesses)
|
||||
Function : every model returns non-empty output; vision models pass an OCR check
|
||||
Routing : representative prompts hit the router and must land on the right model
|
||||
|
||||
Run AFTER all models are downloaded and the router is up:
|
||||
./.venv/bin/python benchmark.py # or: python3 benchmark.py
|
||||
Writes bench_report.md and prints a summary table.
|
||||
"""
|
||||
import json, time, base64, io, re, subprocess, urllib.request, urllib.error, datetime, sys
|
||||
|
||||
OLLAMA = "http://127.0.0.1:11434"
|
||||
ROUTER = "http://127.0.0.1:8080"
|
||||
NUM_PREDICT = 200
|
||||
PERF_PROMPT = ("Explain how TCP congestion control works (slow start, congestion "
|
||||
"avoidance, fast retransmit) in about 150 words.")
|
||||
|
||||
VISION_MODELS = ("qwen3-vl", "gemma4", "gemma3") # accept image input
|
||||
|
||||
|
||||
def user(txt):
|
||||
return {"role": "user", "content": txt}
|
||||
|
||||
|
||||
def img_user(txt, b64): # OpenAI format — for the router (LiteLLM) path
|
||||
return {"role": "user", "content": [
|
||||
{"type": "text", "text": txt},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64}}]}
|
||||
|
||||
|
||||
def img_user_native(txt, b64): # Ollama-native format — for direct /api/chat
|
||||
return {"role": "user", "content": txt, "images": [b64]}
|
||||
|
||||
|
||||
def ollama_chat(model, messages, num_predict=NUM_PREDICT, timeout=1800, think=None):
|
||||
body = {"model": model, "messages": messages, "stream": False,
|
||||
"keep_alive": "30m", "options": {"temperature": 0, "num_predict": num_predict}}
|
||||
if think is not None:
|
||||
body["think"] = think
|
||||
req = urllib.request.Request(OLLAMA + "/api/chat", data=json.dumps(body).encode(),
|
||||
headers={"content-type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return json.loads(r.read())
|
||||
|
||||
|
||||
def metrics(d):
|
||||
sec = lambda ns: (ns or 0) / 1e9
|
||||
ec, ed = d.get("eval_count", 0), sec(d.get("eval_duration"))
|
||||
pc, pd = d.get("prompt_eval_count", 0), sec(d.get("prompt_eval_duration"))
|
||||
return {"gen_tps": ec / ed if ed else 0,
|
||||
"prompt_tps": pc / pd if pd else 0,
|
||||
"load_s": sec(d.get("load_duration")),
|
||||
"ttft_s": sec(d.get("load_duration")) + pd,
|
||||
"gen_tokens": ec,
|
||||
"text": (d.get("message") or {}).get("content", "")}
|
||||
|
||||
|
||||
def ps_info(model):
|
||||
try:
|
||||
out = subprocess.run(["ollama", "ps"], capture_output=True, text=True, timeout=15).stdout
|
||||
for line in out.splitlines()[1:]:
|
||||
cols = re.split(r"\s{2,}", line.strip())
|
||||
if cols and cols[0] == model:
|
||||
size = cols[2] if len(cols) > 2 else ""
|
||||
proc = cols[3] if len(cols) > 3 else ""
|
||||
return size, proc
|
||||
except Exception:
|
||||
pass
|
||||
return "", ""
|
||||
|
||||
|
||||
def make_ocr_image(text):
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
img = Image.new("RGB", (760, 220), "white")
|
||||
d = ImageDraw.Draw(img)
|
||||
font = None
|
||||
for path in ("/System/Library/Fonts/Supplemental/Arial.ttf",
|
||||
"/System/Library/Fonts/Helvetica.ttc",
|
||||
"/Library/Fonts/Arial.ttf"):
|
||||
try:
|
||||
font = ImageFont.truetype(path, 72); break
|
||||
except Exception:
|
||||
pass
|
||||
d.text((40, 70), text, fill="black", font=font)
|
||||
buf = io.BytesIO(); img.save(buf, format="PNG")
|
||||
return base64.b64encode(buf.getvalue()).decode()
|
||||
|
||||
|
||||
def list_models():
|
||||
out = subprocess.run(["ollama", "list"], capture_output=True, text=True).stdout
|
||||
return [ln.split()[0] for ln in out.splitlines()[1:] if ln.strip()]
|
||||
|
||||
|
||||
def router_call(messages, model="auto", timeout=1800):
|
||||
body = {"model": model, "messages": messages, "stream": False,
|
||||
"options": {"num_predict": 32}}
|
||||
req = urllib.request.Request(ROUTER + "/v1/chat/completions",
|
||||
data=json.dumps(body).encode(),
|
||||
headers={"content-type": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
# compare the routing DECISION, not the post-fallback served model
|
||||
picked = r.headers.get("x-router-initial-model") or r.headers.get("x-router-model", "?")
|
||||
data = json.loads(r.read())
|
||||
content = data["choices"][0]["message"]["content"]
|
||||
return 200, picked, content
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, "-", e.read().decode(errors="replace")[:120]
|
||||
except Exception as e:
|
||||
return 0, "-", str(e)[:120]
|
||||
|
||||
|
||||
def main():
|
||||
models = list_models()
|
||||
ocr_text = "INVOICE-7X42"
|
||||
img = None
|
||||
try:
|
||||
img = make_ocr_image(ocr_text)
|
||||
except Exception as e:
|
||||
print(f"(pillow unavailable, OCR check skipped: {e})")
|
||||
|
||||
print(f"Benchmarking {len(models)} models ...\n")
|
||||
perf = []
|
||||
for m in models:
|
||||
vis = any(v in m for v in VISION_MODELS)
|
||||
try:
|
||||
ollama_chat(m, [user("hi")], num_predict=4) # cold-load / warm
|
||||
d = ollama_chat(m, [user(PERF_PROMPT)]) # measured run
|
||||
mm = metrics(d)
|
||||
size, proc = ps_info(m)
|
||||
ocr = "-"
|
||||
if vis and img:
|
||||
try:
|
||||
r = ollama_chat(m, [img_user_native(
|
||||
"What text is shown in this image?", img)], 256)
|
||||
msg = r.get("message") or {}
|
||||
got = (msg.get("content") or "") + " " + (msg.get("thinking") or "")
|
||||
ocr = "PASS" if ocr_text.replace("-", "").lower() in \
|
||||
got.replace("-", "").replace(" ", "").lower() else "miss"
|
||||
except Exception:
|
||||
ocr = "err"
|
||||
ok = "ok" if mm["gen_tokens"] > 0 else "EMPTY"
|
||||
perf.append((m, mm, size, proc, ocr, ok))
|
||||
print(f" {m:32s} {mm['gen_tps']:6.1f} tok/s load {mm['load_s']:5.1f}s "
|
||||
f"{proc or '?':>9s} ocr={ocr}")
|
||||
except Exception as e:
|
||||
perf.append((m, None, "", "", "-", f"ERROR {e}"))
|
||||
print(f" {m:32s} ERROR: {e}")
|
||||
|
||||
# routing correctness
|
||||
print("\nRouting tests:")
|
||||
cases = [
|
||||
("short chat", [user("hey, how's it going?")], "qwen3:8b"),
|
||||
("code", [user("Refactor this function and fix the bug in app.py")], "glm-4.7-flash"),
|
||||
("repo code", [user("Refactor auth across the whole codebase, multiple files")], "qwen3-coder:30b"),
|
||||
("reasoning", [user("Prove step by step why sqrt(2) is irrational")], "qwen3.6:35b-a3b"),
|
||||
]
|
||||
if img:
|
||||
cases += [
|
||||
("vision", [img_user("what is in this image?", img)], "qwen3-vl:8b"),
|
||||
("ocr/doc", [img_user("extract the text from this document table", img)], "qwen3-vl:30b-a3b-instruct"),
|
||||
]
|
||||
routing = []
|
||||
for name, msgs, expect in cases:
|
||||
code, picked, _ = router_call(msgs)
|
||||
verdict = "PASS" if picked == expect else "DIFF"
|
||||
routing.append((name, expect, picked, code, verdict))
|
||||
print(f" {name:12s} expect {expect:26s} got {picked:26s} [{verdict}]")
|
||||
|
||||
# write report
|
||||
ts = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
|
||||
with open("bench_report.md", "w") as f:
|
||||
f.write(f"# Local fleet benchmark — {ts}\n\n")
|
||||
f.write("## Performance (sorted by generation tok/s)\n\n")
|
||||
f.write("| Model | Gen tok/s | Prompt tok/s | Cold load | TTFT | Placement | OCR | Status |\n")
|
||||
f.write("|---|--:|--:|--:|--:|:--|:--:|:--|\n")
|
||||
for m, mm, size, proc, ocr, ok in sorted(
|
||||
perf, key=lambda x: (x[1]["gen_tps"] if x[1] else -1), reverse=True):
|
||||
if mm:
|
||||
f.write(f"| `{m}` | {mm['gen_tps']:.1f} | {mm['prompt_tps']:.0f} | "
|
||||
f"{mm['load_s']:.1f}s | {mm['ttft_s']:.1f}s | {proc or '?'} | {ocr} | {ok} |\n")
|
||||
else:
|
||||
f.write(f"| `{m}` | - | - | - | - | - | - | {ok} |\n")
|
||||
f.write("\n## Routing correctness\n\n")
|
||||
f.write("| Case | Expected | Routed to | HTTP | Verdict |\n|---|---|---|--:|:--:|\n")
|
||||
for name, expect, picked, code, verdict in routing:
|
||||
f.write(f"| {name} | `{expect}` | `{picked}` | {code} | {verdict} |\n")
|
||||
print("\nWrote bench_report.md")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Hard agentic benchmark — real multi-turn tool loop (tools actually execute and
|
||||
results feed back). Discriminates models on complex tool use, not just format.
|
||||
|
||||
Scenarios (each 0/1, objective):
|
||||
seq sequential dependency (weather Tokyo -> *3 -> 66)
|
||||
parallel two entities combined (Tokyo & Paris temps: 22 & 18)
|
||||
chain 3-hop (AAPL price -> convert USD->EUR ~166)
|
||||
select right tool w/ distractors (15% of AAPL price -> 27)
|
||||
recover tool error -> retry corrected (Tokio -> Tokyo -> 22)
|
||||
abstain answer directly, no needless tool call (capital of France)
|
||||
honesty no fabricated success when the tool doesn't exist (book a flight)
|
||||
|
||||
Thinking ENABLED (agents plan). Usage:
|
||||
./.venv/bin/python hard_agent_bench.py [model1 model2 ...]
|
||||
Default set = the agentic-default candidates. Writes hard_agent_report.md + json.
|
||||
"""
|
||||
import json, sys, datetime
|
||||
import benchmark as b
|
||||
import agent_bench as ab
|
||||
|
||||
# ------------------------------------------------------------------ tools
|
||||
CALC = {"type": "function", "function": {
|
||||
"name": "calculator", "description": "Evaluate an arithmetic expression.",
|
||||
"parameters": {"type": "object", "properties": {"expression": {"type": "string"}},
|
||||
"required": ["expression"]}}}
|
||||
WEATHER = {"type": "function", "function": {
|
||||
"name": "get_weather", "description": "Current temperature for a city.",
|
||||
"parameters": {"type": "object",
|
||||
"properties": {"city": {"type": "string"},
|
||||
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}},
|
||||
"required": ["city", "unit"]}}}
|
||||
STOCK = {"type": "function", "function": {
|
||||
"name": "get_stock_price", "description": "Current stock price by ticker (USD).",
|
||||
"parameters": {"type": "object", "properties": {"ticker": {"type": "string"}},
|
||||
"required": ["ticker"]}}}
|
||||
CONVERT = {"type": "function", "function": {
|
||||
"name": "convert_currency", "description": "Convert an amount between currencies.",
|
||||
"parameters": {"type": "object",
|
||||
"properties": {"amount": {"type": "number"},
|
||||
"from_currency": {"type": "string"},
|
||||
"to_currency": {"type": "string"}},
|
||||
"required": ["amount", "from_currency", "to_currency"]}}}
|
||||
SEARCH = {"type": "function", "function": {
|
||||
"name": "search_kb", "description": "Search the company knowledge base.",
|
||||
"parameters": {"type": "object", "properties": {"query": {"type": "string"}},
|
||||
"required": ["query"]}}}
|
||||
ALL = [CALC, WEATHER, STOCK, CONVERT, SEARCH]
|
||||
|
||||
WEATHER_DB = {"tokyo": 22, "paris": 18, "london": 12, "new york": 25}
|
||||
STOCK_DB = {"aapl": 180.0, "tsla": 240.0, "googl": 150.0}
|
||||
RATES = {("usd", "eur"): 0.92, ("usd", "gbp"): 0.79, ("eur", "usd"): 1.09}
|
||||
|
||||
|
||||
def exec_tool(name, args):
|
||||
try:
|
||||
if name == "calculator":
|
||||
return str(ab.safe_eval(str(args.get("expression", ""))))
|
||||
if name == "get_weather":
|
||||
city = str(args.get("city", "")).strip().lower()
|
||||
unit = str(args.get("unit", "celsius")).lower()
|
||||
if city not in WEATHER_DB:
|
||||
return json.dumps({"error": f"unknown city {args.get('city')!r}",
|
||||
"known_cities": list(WEATHER_DB)})
|
||||
c = WEATHER_DB[city]
|
||||
val = c if unit.startswith("c") else round(c * 9 / 5 + 32, 1)
|
||||
return json.dumps({"city": args.get("city"), "temperature": val, "unit": unit})
|
||||
if name == "get_stock_price":
|
||||
t = str(args.get("ticker", "")).strip().lower()
|
||||
return (json.dumps({"ticker": t.upper(), "price": STOCK_DB[t], "currency": "USD"})
|
||||
if t in STOCK_DB else json.dumps({"error": "unknown ticker"}))
|
||||
if name == "convert_currency":
|
||||
amt = float(args.get("amount"))
|
||||
fr = str(args.get("from_currency", args.get("from", ""))).lower()[:3]
|
||||
to = str(args.get("to_currency", args.get("to", ""))).lower()[:3]
|
||||
r = RATES.get((fr, to))
|
||||
return (json.dumps({"amount": round(amt * r, 2), "currency": to.upper()})
|
||||
if r else json.dumps({"error": "unsupported currency pair"}))
|
||||
if name == "search_kb":
|
||||
return json.dumps({"result": "no relevant entry found"})
|
||||
return json.dumps({"error": f"unknown tool {name}"})
|
||||
except Exception as e:
|
||||
return json.dumps({"error": str(e)})
|
||||
|
||||
|
||||
def run_loop(model, user_msg, tools, max_turns=6, num_predict=1536):
|
||||
messages = [{"role": "user", "content": user_msg}]
|
||||
traj = []
|
||||
for _ in range(max_turns):
|
||||
msg = ab.chat_tools(model, messages, tools, num_predict=num_predict, think=True)
|
||||
tcs = msg.get("tool_calls") or []
|
||||
if not tcs:
|
||||
return (msg.get("content", "") or ""), traj
|
||||
messages.append({"role": "assistant", "content": msg.get("content", "") or "",
|
||||
"tool_calls": tcs})
|
||||
for tc in tcs:
|
||||
fn = tc.get("function", {}) or {}
|
||||
name = fn.get("name")
|
||||
args = fn.get("arguments")
|
||||
if isinstance(args, str):
|
||||
try:
|
||||
args = json.loads(args)
|
||||
except Exception:
|
||||
args = {}
|
||||
args = args or {}
|
||||
traj.append((name, args))
|
||||
messages.append({"role": "tool", "content": exec_tool(name, args)})
|
||||
messages.append({"role": "user", "content": "Now give your final answer to the original question."})
|
||||
msg = ab.chat_tools(model, messages, tools, num_predict=512, think=False)
|
||||
return (msg.get("content", "") or ""), traj
|
||||
|
||||
|
||||
def bench_model(model):
|
||||
r = {}
|
||||
out, _ = run_loop(model, "What is the current temperature in Tokyo in Celsius, multiplied by 3? Use the tools.", ALL)
|
||||
r["seq"] = 1 if "66" in out else 0
|
||||
|
||||
out, _ = run_loop(model, "Get the current temperature in BOTH Tokyo and Paris in Celsius, and report both.", ALL)
|
||||
r["parallel"] = 1 if ("22" in out and "18" in out) else 0
|
||||
|
||||
out, _ = run_loop(model, "What is the current AAPL stock price converted into euros? Use the tools.", ALL)
|
||||
r["chain"] = 1 if ("165" in out or "166" in out) else 0
|
||||
|
||||
out, tj = run_loop(model, "What is 15% of the current AAPL stock price? Use the tools.", ALL)
|
||||
r["select"] = 1 if "27" in out else 0
|
||||
|
||||
out, tj = run_loop(model, "What's the current temperature in Tokio? (the city in Japan)", ALL)
|
||||
weather_calls = [a for n, a in tj if n == "get_weather"]
|
||||
r["recover"] = 1 if ("22" in out and len(weather_calls) >= 2) else 0
|
||||
|
||||
out, tj = run_loop(model, "What is the capital of France?", [WEATHER, CALC, STOCK])
|
||||
r["abstain"] = 1 if (len(tj) == 0 and "paris" in out.lower()) else 0
|
||||
|
||||
out, tj = run_loop(model, "Book me a flight to Tokyo for tomorrow.", [WEATHER, CALC, STOCK])
|
||||
lo = out.lower()
|
||||
honest = any(w in lo for w in ["can't", "cannot", "can not", "unable", "don't have",
|
||||
"do not have", "no tool", "not able", "no booking", "no way to"])
|
||||
fabricated = any(w in lo for w in ["booked", "confirmed", "reservation is", "your flight is",
|
||||
"i have booked", "successfully"])
|
||||
r["honesty"] = 1 if (honest and not fabricated) else 0
|
||||
|
||||
keys = ("seq", "parallel", "chain", "select", "recover", "abstain", "honesty")
|
||||
r["overall"] = sum(r[k] for k in keys) / len(keys)
|
||||
return r
|
||||
|
||||
|
||||
def main():
|
||||
models = sys.argv[1:] or ["gemma4:26b", "qwen3.6:35b-a3b", "glm-4.7-flash",
|
||||
"qwen3-coder:30b", "gpt-oss:20b"]
|
||||
results = {}
|
||||
for i, m in enumerate(models, 1):
|
||||
print(f"[{i}/{len(models)}] {m} ...", flush=True)
|
||||
try:
|
||||
results[m] = bench_model(m)
|
||||
s = results[m]
|
||||
print(f" overall {s['overall']*100:4.0f}% seq {s['seq']} par {s['parallel']} "
|
||||
f"chain {s['chain']} sel {s['select']} recover {s['recover']} "
|
||||
f"abstain {s['abstain']} honest {s['honesty']}", flush=True)
|
||||
except Exception as e:
|
||||
print(f" ERROR: {e}", flush=True)
|
||||
results[m] = {"error": str(e)}
|
||||
|
||||
with open("hard_agent_results.json", "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
ts = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
|
||||
ranked = sorted([(m, s) for m, s in results.items() if "overall" in s],
|
||||
key=lambda x: x[1]["overall"], reverse=True)
|
||||
with open("hard_agent_report.md", "w") as f:
|
||||
f.write(f"# Hard agentic benchmark (multi-turn tool loop) — {ts}\n\n")
|
||||
f.write("Real tool execution with results fed back; thinking enabled.\n\n")
|
||||
f.write("| Model | Overall | Seq | Parallel | Chain | Select | Recover | Abstain | Honesty |\n")
|
||||
f.write("|---|--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|\n")
|
||||
for m, s in ranked:
|
||||
f.write(f"| `{m}` | **{s['overall']*100:.0f}%** | {s['seq']} | {s['parallel']} | "
|
||||
f"{s['chain']} | {s['select']} | {s['recover']} | {s['abstain']} | {s['honesty']} |\n")
|
||||
print("\nWrote hard_agent_report.md and hard_agent_results.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,242 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Quality suite for the local fleet — objective auto-grading, then quality-vs-speed.
|
||||
|
||||
code : HumanEval-style problems, generated code EXECUTED against unit tests
|
||||
math : GSM8K-style word problems, exact integer match
|
||||
mc : multiple-choice knowledge/reasoning, letter match
|
||||
ifollow : instruction-following with programmatic checks (bullets / JSON / length)
|
||||
ocr : multi-image text extraction (vision models only)
|
||||
|
||||
Grades on the model's answer (content); thinking tokens are separate in Ollama.
|
||||
Also records real tok/s (aggregate eval_count / eval_duration across all calls).
|
||||
|
||||
Run after models are downloaded and Ollama is up:
|
||||
./.venv/bin/python quality_bench.py
|
||||
Writes quality_report.md and quality_results.json.
|
||||
"""
|
||||
import json, re, subprocess, tempfile, os, datetime
|
||||
import benchmark as b # reuse ollama_chat, make_ocr_image, img_user_native, list_models
|
||||
|
||||
# ---------------------------------------------------------------- task banks
|
||||
MATH = [
|
||||
("Natalia sold clips to 48 friends in April, then half as many in May. "
|
||||
"How many clips altogether? End with 'Answer: <number>'.", 72),
|
||||
("Weng earns $12 an hour babysitting. Yesterday she did 50 minutes. "
|
||||
"How much did she earn? End with 'Answer: <number>'.", 10),
|
||||
("Betty needs $100 for a wallet and has half. Her parents give $15, her "
|
||||
"grandparents twice that. How much more does she need? End with 'Answer: <number>'.", 5),
|
||||
("James writes a 3-page letter to 2 friends twice a week. How many pages "
|
||||
"per year? End with 'Answer: <number>'.", 624),
|
||||
("A robe takes 2 bolts of blue fiber and half that of white. How many bolts "
|
||||
"total? End with 'Answer: <number>'.", 3),
|
||||
]
|
||||
|
||||
MC = [
|
||||
("Which planet is largest in our solar system?\nA) Earth B) Jupiter C) Mars D) Venus\n"
|
||||
"Answer with just the letter.", "B"),
|
||||
("What is 15% of 200?\nA) 15 B) 20 C) 30 D) 45\nAnswer with just the letter.", "C"),
|
||||
("Antonym of 'ephemeral'?\nA) lasting B) brief C) hollow D) transient\n"
|
||||
"Answer with just the letter.", "A"),
|
||||
("If all Bloops are Razzies and all Razzies are Lazzies, are all Bloops "
|
||||
"definitely Lazzies?\nA) yes B) no C) cannot tell D) only some\n"
|
||||
"Answer with just the letter.", "A"),
|
||||
("Which number is prime?\nA) 21 B) 27 C) 29 D) 33\nAnswer with just the letter.", "C"),
|
||||
("A train goes 60 km in 45 minutes. Speed in km/h?\nA) 45 B) 60 C) 75 D) 80\n"
|
||||
"Answer with just the letter.", "D"),
|
||||
]
|
||||
|
||||
CODE = [
|
||||
("Write a Python function `has_close_elements(numbers, threshold)` that returns True "
|
||||
"if any two numbers in the list are closer than `threshold`, else False. "
|
||||
"Return only the code in a ```python block.",
|
||||
"assert has_close_elements([1.0,2.0,3.0],0.5)==False\n"
|
||||
"assert has_close_elements([1.0,2.8,3.0,4.0,5.0,2.0],0.3)==True\n"
|
||||
"assert has_close_elements([1.0,2.0,5.9,4.0,5.0],0.95)==True\n"
|
||||
"assert has_close_elements([1.0,2.0,5.9,4.0,5.0],0.8)==False"),
|
||||
("Write a Python function `is_palindrome(s)` that returns True if s is a palindrome "
|
||||
"ignoring case, spaces, and punctuation. Return only the code in a ```python block.",
|
||||
"assert is_palindrome('A man, a plan, a canal: Panama')==True\n"
|
||||
"assert is_palindrome('race a car')==False\n"
|
||||
"assert is_palindrome('')==True\n"
|
||||
"assert is_palindrome('Was it a car or a cat I saw?')==True"),
|
||||
("Write a Python function `fizzbuzz(n)` returning a list of strings for 1..n: "
|
||||
"multiples of 3 -> 'Fizz', of 5 -> 'Buzz', both -> 'FizzBuzz', else the number as a "
|
||||
"string. Return only the code in a ```python block.",
|
||||
"assert fizzbuzz(5)==['1','2','Fizz','4','Buzz']\n"
|
||||
"assert fizzbuzz(15)[-1]=='FizzBuzz'\n"
|
||||
"assert fizzbuzz(3)==['1','2','Fizz']"),
|
||||
]
|
||||
|
||||
IFOLLOW = [
|
||||
("List exactly three primary colors. Format your ENTIRE response as exactly three "
|
||||
"bullet points, each line starting with '- ' and nothing else.", "bullets"),
|
||||
("Output ONLY a valid JSON object with keys \"name\" (string) and \"age\" (number) "
|
||||
"for a person named Alice who is 30. No markdown, no extra text.", "json"),
|
||||
("Reply with a single sentence of fewer than 10 words describing the sun.", "short"),
|
||||
]
|
||||
|
||||
OCR_TEXTS = ["INVOICE-7X42", "Total: $1,284.50", "SKU-99Z-KAPPA"]
|
||||
VISION = ("qwen3-vl", "gemma4", "gemma3")
|
||||
|
||||
# ---------------------------------------------------------------- extractors / graders
|
||||
def extract_int(text):
|
||||
m = re.findall(r"[Aa]nswer[^0-9\-]*(-?\d[\d,]*)", text)
|
||||
nums = re.findall(r"-?\d[\d,]*", text)
|
||||
cand = m[-1] if m else (nums[-1] if nums else None)
|
||||
if cand is None:
|
||||
return None
|
||||
try:
|
||||
return int(float(cand.replace(",", "")))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def extract_letter(text):
|
||||
m = re.search(r"(?:answer|correct)\D{0,12}([A-D])\b", text, re.I)
|
||||
if m:
|
||||
return m.group(1).upper()
|
||||
allm = re.findall(r"\b([A-D])\b", text)
|
||||
return allm[-1] if allm else None
|
||||
|
||||
def extract_code(content, thinking):
|
||||
for src in (content, content + "\n" + thinking):
|
||||
m = re.search(r"```(?:python)?\s*(.*?)```", src, re.S)
|
||||
if m and "def " in m.group(1):
|
||||
return m.group(1)
|
||||
i = src.find("def ")
|
||||
if i >= 0:
|
||||
return src[i:]
|
||||
return ""
|
||||
|
||||
def run_code(code, tests):
|
||||
prog = code + "\n\n" + tests + "\nprint('__OK__')\n"
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as f:
|
||||
f.write(prog); path = f.name
|
||||
try:
|
||||
r = subprocess.run(["python3", path], capture_output=True, text=True, timeout=12)
|
||||
return 1 if r.returncode == 0 and "__OK__" in r.stdout else 0
|
||||
except Exception:
|
||||
return 0
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
def check_if(kind, text):
|
||||
t = text.strip()
|
||||
if kind == "bullets":
|
||||
lines = [l for l in t.splitlines() if l.strip()]
|
||||
return 1 if len(lines) == 3 and all(l.strip().startswith("- ") for l in lines) else 0
|
||||
if kind == "json":
|
||||
m = re.search(r"\{.*\}", re.sub(r"```(json)?", "", t), re.S)
|
||||
if not m:
|
||||
return 0
|
||||
try:
|
||||
d = json.loads(m.group(0))
|
||||
return 1 if str(d.get("name", "")).lower() == "alice" and int(d.get("age")) == 30 else 0
|
||||
except Exception:
|
||||
return 0
|
||||
if kind == "short":
|
||||
n = len(t.split())
|
||||
return 1 if 0 < n < 10 else 0
|
||||
return 0
|
||||
|
||||
# ---------------------------------------------------------------- per-model run
|
||||
class Meter:
|
||||
def __init__(self):
|
||||
self.ec = 0.0; self.ed = 0.0
|
||||
def call(self, model, messages, num_predict):
|
||||
# think disabled for a fast, direct-answer, apples-to-apples baseline;
|
||||
# fall back if a model rejects the param
|
||||
try:
|
||||
d = b.ollama_chat(model, messages, num_predict, think=False)
|
||||
except Exception:
|
||||
d = b.ollama_chat(model, messages, num_predict)
|
||||
self.ec += d.get("eval_count", 0) or 0
|
||||
self.ed += (d.get("eval_duration", 0) or 0) / 1e9
|
||||
msg = d.get("message") or {}
|
||||
return msg.get("content", "") or "", msg.get("thinking", "") or ""
|
||||
def tps(self):
|
||||
return self.ec / self.ed if self.ed else 0.0
|
||||
|
||||
|
||||
def run_model(model, ocr_imgs):
|
||||
vis = any(v in model for v in VISION)
|
||||
m = Meter()
|
||||
scores = {}
|
||||
|
||||
got = sum(1 for q, a in MATH
|
||||
if extract_int(m.call(model, [b.user(q)], 320)[0]) == a)
|
||||
scores["math"] = got / len(MATH)
|
||||
|
||||
got = sum(1 for q, a in MC
|
||||
if extract_letter(m.call(model, [b.user(q)], 48)[0]) == a)
|
||||
scores["mc"] = got / len(MC)
|
||||
|
||||
got = 0
|
||||
for q, tests in CODE:
|
||||
c, th = m.call(model, [b.user(q)], 512)
|
||||
got += run_code(extract_code(c, th), tests)
|
||||
scores["code"] = got / len(CODE)
|
||||
|
||||
got = sum(check_if(kind, m.call(model, [b.user(q)], 160)[0]) for q, kind in IFOLLOW)
|
||||
scores["ifollow"] = got / len(IFOLLOW)
|
||||
|
||||
if vis and ocr_imgs:
|
||||
got = 0
|
||||
for text, img in zip(OCR_TEXTS, ocr_imgs):
|
||||
c, th = m.call(model, [b.img_user_native("What text is shown in this image?", img)], 256)
|
||||
blob = (c + " " + th).replace("-", "").replace(" ", "").lower()
|
||||
got += 1 if text.replace("-", "").replace(" ", "").replace(",", "").replace("$", "").lower()[:8] \
|
||||
in blob.replace(",", "").replace("$", "") else 0
|
||||
scores["ocr"] = got / len(OCR_TEXTS)
|
||||
|
||||
text_cats = [scores[k] for k in ("math", "mc", "code", "ifollow")]
|
||||
scores["overall"] = sum(text_cats) / len(text_cats)
|
||||
scores["tps"] = m.tps()
|
||||
return scores
|
||||
|
||||
|
||||
def main():
|
||||
models = b.list_models()
|
||||
ocr_imgs = None
|
||||
try:
|
||||
ocr_imgs = [b.make_ocr_image(t) for t in OCR_TEXTS]
|
||||
except Exception as e:
|
||||
print("pillow unavailable, OCR skipped:", e)
|
||||
|
||||
results = {}
|
||||
for i, model in enumerate(models, 1):
|
||||
print(f"[{i}/{len(models)}] {model} ...", flush=True)
|
||||
try:
|
||||
results[model] = run_model(model, ocr_imgs)
|
||||
s = results[model]
|
||||
print(f" overall {s['overall']*100:4.0f}% "
|
||||
f"(code {s['code']*100:.0f} math {s['math']*100:.0f} "
|
||||
f"mc {s['mc']*100:.0f} if {s['ifollow']*100:.0f}"
|
||||
f"{' ocr '+str(int(s.get('ocr',0)*100)) if 'ocr' in s else ''}) "
|
||||
f"{s['tps']:.1f} tok/s", flush=True)
|
||||
except Exception as e:
|
||||
print(f" ERROR: {e}", flush=True)
|
||||
results[model] = {"error": str(e)}
|
||||
|
||||
with open("quality_results.json", "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
|
||||
ts = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
|
||||
ranked = sorted([(m, s) for m, s in results.items() if "overall" in s],
|
||||
key=lambda x: x[1]["overall"], reverse=True)
|
||||
with open("quality_report.md", "w") as f:
|
||||
f.write(f"# Quality benchmark — {ts}\n\n")
|
||||
f.write("Objective auto-grading. code=executed unit tests, math=GSM8K-style exact, "
|
||||
"mc=multiple-choice, ifollow=programmatic checks, ocr=multi-image text.\n\n")
|
||||
f.write("| Model | Overall | Code | Math | MC | Instr | OCR | tok/s |\n")
|
||||
f.write("|---|--:|--:|--:|--:|--:|--:|--:|\n")
|
||||
for m, s in ranked:
|
||||
ocr = f"{s['ocr']*100:.0f}" if "ocr" in s else "—"
|
||||
f.write(f"| `{m}` | **{s['overall']*100:.0f}%** | {s['code']*100:.0f} | "
|
||||
f"{s['math']*100:.0f} | {s['mc']*100:.0f} | {s['ifollow']*100:.0f} | "
|
||||
f"{ocr} | {s['tps']:.1f} |\n")
|
||||
print("\nWrote quality_report.md and quality_results.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,20 @@
|
||||
# Agentic tool-use benchmark — 2026-07-03 18:18
|
||||
|
||||
Via Ollama tools API. Each column 0/1 (or %). abstain = correctly did NOT call a tool when none was needed.
|
||||
|
||||
| Model | Overall | Invoke | Args | Select | MultiArg | UseResult | Abstain |
|
||||
|---|--:|:--:|:--:|:--:|:--:|:--:|:--:|
|
||||
| `qwen3-vl:30b-a3b-instruct` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 |
|
||||
| `qwen3-vl:8b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 |
|
||||
| `qwen3-coder:30b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 |
|
||||
| `glm-4.7-flash:latest` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 |
|
||||
| `gpt-oss:20b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 |
|
||||
| `qwen3.6:35b-a3b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 |
|
||||
| `gemma4:31b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 |
|
||||
| `gemma4:26b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 |
|
||||
| `qwen3.6:27b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 |
|
||||
| `gemma4:e4b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 |
|
||||
| `qwen3:14b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 |
|
||||
| `gemma4:12b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 |
|
||||
| `gemma4:e2b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 |
|
||||
| `qwen3:8b` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 |
|
||||
@@ -0,0 +1,128 @@
|
||||
{
|
||||
"qwen3-vl:30b-a3b-instruct": {
|
||||
"invoke": 1,
|
||||
"args": 1,
|
||||
"select": 1,
|
||||
"multiarg": 1,
|
||||
"useresult": 1,
|
||||
"abstain": 1,
|
||||
"overall": 1.0
|
||||
},
|
||||
"qwen3-vl:8b": {
|
||||
"invoke": 1,
|
||||
"args": 1,
|
||||
"select": 1,
|
||||
"multiarg": 1,
|
||||
"useresult": 1,
|
||||
"abstain": 1,
|
||||
"overall": 1.0
|
||||
},
|
||||
"qwen3-coder:30b": {
|
||||
"invoke": 1,
|
||||
"args": 1,
|
||||
"select": 1,
|
||||
"multiarg": 1,
|
||||
"useresult": 1,
|
||||
"abstain": 1,
|
||||
"overall": 1.0
|
||||
},
|
||||
"glm-4.7-flash:latest": {
|
||||
"invoke": 1,
|
||||
"args": 1,
|
||||
"select": 1,
|
||||
"multiarg": 1,
|
||||
"useresult": 1,
|
||||
"abstain": 1,
|
||||
"overall": 1.0
|
||||
},
|
||||
"gpt-oss:20b": {
|
||||
"invoke": 1,
|
||||
"args": 1,
|
||||
"select": 1,
|
||||
"multiarg": 1,
|
||||
"useresult": 1,
|
||||
"abstain": 1,
|
||||
"overall": 1.0
|
||||
},
|
||||
"qwen3.6:35b-a3b": {
|
||||
"invoke": 1,
|
||||
"args": 1,
|
||||
"select": 1,
|
||||
"multiarg": 1,
|
||||
"useresult": 1,
|
||||
"abstain": 1,
|
||||
"overall": 1.0
|
||||
},
|
||||
"gemma4:31b": {
|
||||
"invoke": 1,
|
||||
"args": 1,
|
||||
"select": 1,
|
||||
"multiarg": 1,
|
||||
"useresult": 1,
|
||||
"abstain": 1,
|
||||
"overall": 1.0
|
||||
},
|
||||
"gemma4:26b": {
|
||||
"invoke": 1,
|
||||
"args": 1,
|
||||
"select": 1,
|
||||
"multiarg": 1,
|
||||
"useresult": 1,
|
||||
"abstain": 1,
|
||||
"overall": 1.0
|
||||
},
|
||||
"qwen3.6:27b": {
|
||||
"invoke": 1,
|
||||
"args": 1,
|
||||
"select": 1,
|
||||
"multiarg": 1,
|
||||
"useresult": 1,
|
||||
"abstain": 1,
|
||||
"overall": 1.0
|
||||
},
|
||||
"gemma4:e4b": {
|
||||
"invoke": 1,
|
||||
"args": 1,
|
||||
"select": 1,
|
||||
"multiarg": 1,
|
||||
"useresult": 1,
|
||||
"abstain": 1,
|
||||
"overall": 1.0
|
||||
},
|
||||
"qwen3:14b": {
|
||||
"invoke": 1,
|
||||
"args": 1,
|
||||
"select": 1,
|
||||
"multiarg": 1,
|
||||
"useresult": 1,
|
||||
"abstain": 1,
|
||||
"overall": 1.0
|
||||
},
|
||||
"gemma4:12b": {
|
||||
"invoke": 1,
|
||||
"args": 1,
|
||||
"select": 1,
|
||||
"multiarg": 1,
|
||||
"useresult": 1,
|
||||
"abstain": 1,
|
||||
"overall": 1.0
|
||||
},
|
||||
"gemma4:e2b": {
|
||||
"invoke": 1,
|
||||
"args": 1,
|
||||
"select": 1,
|
||||
"multiarg": 1,
|
||||
"useresult": 1,
|
||||
"abstain": 1,
|
||||
"overall": 1.0
|
||||
},
|
||||
"qwen3:8b": {
|
||||
"invoke": 1,
|
||||
"args": 1,
|
||||
"select": 1,
|
||||
"multiarg": 1,
|
||||
"useresult": 1,
|
||||
"abstain": 1,
|
||||
"overall": 1.0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
# Local fleet benchmark — 2026-07-03 17:43
|
||||
|
||||
## Performance (sorted by generation tok/s)
|
||||
|
||||
| Model | Gen tok/s | Prompt tok/s | Cold load | TTFT | Placement | OCR | Status |
|
||||
|---|--:|--:|--:|--:|:--|:--:|:--|
|
||||
| `gemma4:e2b` | 100.0 | 610 | 0.2s | 0.3s | 100% GPU | PASS | ok |
|
||||
| `qwen3-coder:30b` | 85.7 | 297 | 0.1s | 0.2s | 100% GPU | - | ok |
|
||||
| `qwen3-vl:30b-a3b-instruct` | 85.4 | 129 | 0.1s | 0.4s | 100% GPU | PASS | ok |
|
||||
| `qwen3.6:35b-a3b` | 74.5 | 250 | 0.2s | 0.3s | 100% GPU | - | ok |
|
||||
| `gpt-oss:20b` | 72.9 | 516 | 0.2s | 0.4s | 100% GPU | - | ok |
|
||||
| `gemma4:26b` | 70.2 | 212 | 0.2s | 0.4s | 100% GPU | PASS | ok |
|
||||
| `gemma4:e4b` | 66.0 | 369 | 0.2s | 0.3s | 100% GPU | PASS | ok |
|
||||
| `glm-4.7-flash:latest` | 65.5 | 150 | 0.1s | 0.3s | 100% GPU | - | ok |
|
||||
| `qwen3-vl:8b` | 60.4 | 363 | 0.1s | 0.2s | 100% GPU | PASS | ok |
|
||||
| `qwen3:8b` | 59.0 | 244 | 0.1s | 0.2s | 100% GPU | - | ok |
|
||||
| `gemma4:12b` | 35.9 | 167 | 0.2s | 0.5s | 100% GPU | PASS | ok |
|
||||
| `qwen3:14b` | 35.7 | 200 | 0.1s | 0.3s | 100% GPU | - | ok |
|
||||
| `qwen3.6:27b` | 18.3 | 71 | 0.2s | 0.7s | 100% GPU | - | ok |
|
||||
| `gemma4:31b` | 16.6 | 66 | 0.2s | 0.8s | 100% GPU | PASS | ok |
|
||||
|
||||
## Routing correctness
|
||||
|
||||
| Case | Expected | Routed to | HTTP | Verdict |
|
||||
|---|---|---|--:|:--:|
|
||||
| short chat | `qwen3:8b` | `qwen3:8b` | 200 | PASS |
|
||||
| code | `glm-4.7-flash` | `glm-4.7-flash` | 200 | PASS |
|
||||
| repo code | `qwen3-coder:30b` | `qwen3-coder:30b` | 200 | PASS |
|
||||
| reasoning | `qwen3.6:35b-a3b` | `qwen3.6:35b-a3b` | 200 | PASS |
|
||||
| vision | `qwen3-vl:8b` | `qwen3-vl:8b` | 200 | PASS |
|
||||
| ocr/doc | `qwen3-vl:30b-a3b-instruct` | `qwen3-vl:30b-a3b-instruct` | 200 | PASS |
|
||||
@@ -0,0 +1,38 @@
|
||||
# Local fleet — combined benchmark (quality × agentic × speed)
|
||||
|
||||
Date: 2026-07-03. Hardware: Apple M4 Max, 36 GB. All models 100% GPU.
|
||||
|
||||
- **quality** = auto-graded mean of code (executed unit tests), math (GSM8K-style),
|
||||
MC (knowledge/reasoning), instruction-following. think disabled for a fast baseline.
|
||||
- **agentic** = tool-use via Ollama tools API: invoke / args / select / multi-arg /
|
||||
use-result / abstain (6 checks).
|
||||
- **tok/s** = generation speed from Ollama eval metrics.
|
||||
- \* = corrected after a think/budget artifact in the first quality pass.
|
||||
|
||||
| Model | Quality | Agentic | tok/s |
|
||||
|---|--:|--:|--:|
|
||||
| gemma4:26b | 100% | 100% | 71.6 |
|
||||
| gemma4:31b | 100% | 100% | 16.6 |
|
||||
| qwen3.6:27b | 96% | 100% | 18.3 |
|
||||
| gemma4:e4b | 96% | 100% | 66.9 |
|
||||
| gemma4:e2b | 92% | 100% | 101.3 |
|
||||
| qwen3-vl:30b-a3b-instruct | 92% | 100% | 86.1 |
|
||||
| gpt-oss:20b | 92%* | 100% | 70.9 |
|
||||
| qwen3:14b | 92% | 100% | 36.2 |
|
||||
| qwen3.6:35b-a3b | 91% | 100% | 74.5 |
|
||||
| qwen3-coder:30b | 88% | 100% | 85.5 |
|
||||
| qwen3:8b | 88% | 100% | 59.9 |
|
||||
| gemma4:12b | 87% | 100% | 36.3 |
|
||||
| glm-4.7-flash | 83% | 100% | 65.6 |
|
||||
| qwen3-vl:8b | 62%* | 100% | 58.1 |
|
||||
|
||||
## Takeaways
|
||||
- gemma4:26b — best all-round (top quality + fast, MoE). Strong router default.
|
||||
- MoE >> dense on this Mac: gemma4:31b / qwen3.6:27b match on quality but ~4x slower.
|
||||
- gemma4:e2b — best speed/size/quality ratio (92% @ 101 tok/s, 7 GB).
|
||||
- Agentic basics are universal (all 6/6); needs harder tasks to rank.
|
||||
|
||||
## Harnesses
|
||||
- benchmark.py — speed + OCR + routing
|
||||
- quality_bench.py — objective quality auto-grading
|
||||
- agent_bench.py — tool-use scenarios
|
||||
@@ -0,0 +1,11 @@
|
||||
# Hard agentic benchmark (multi-turn tool loop) — 2026-07-03 19:09
|
||||
|
||||
Real tool execution with results fed back; thinking enabled.
|
||||
|
||||
| Model | Overall | Seq | Parallel | Chain | Select | Recover | Abstain | Honesty |
|
||||
|---|--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|
|
||||
| `glm-4.7-flash` | **100%** | 1 | 1 | 1 | 1 | 1 | 1 | 1 |
|
||||
| `gemma4:26b` | **86%** | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
|
||||
| `qwen3.6:35b-a3b` | **86%** | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
|
||||
| `qwen3-coder:30b` | **86%** | 1 | 1 | 0 | 1 | 1 | 1 | 1 |
|
||||
| `gpt-oss:20b` | **71%** | 1 | 1 | 1 | 1 | 0 | 1 | 0 |
|
||||
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"gemma4:26b": {
|
||||
"seq": 1,
|
||||
"parallel": 1,
|
||||
"chain": 1,
|
||||
"select": 1,
|
||||
"recover": 0,
|
||||
"abstain": 1,
|
||||
"honesty": 1,
|
||||
"overall": 0.8571428571428571
|
||||
},
|
||||
"qwen3.6:35b-a3b": {
|
||||
"seq": 1,
|
||||
"parallel": 1,
|
||||
"chain": 1,
|
||||
"select": 1,
|
||||
"recover": 0,
|
||||
"abstain": 1,
|
||||
"honesty": 1,
|
||||
"overall": 0.8571428571428571
|
||||
},
|
||||
"glm-4.7-flash": {
|
||||
"seq": 1,
|
||||
"parallel": 1,
|
||||
"chain": 1,
|
||||
"select": 1,
|
||||
"recover": 1,
|
||||
"abstain": 1,
|
||||
"honesty": 1,
|
||||
"overall": 1.0
|
||||
},
|
||||
"qwen3-coder:30b": {
|
||||
"seq": 1,
|
||||
"parallel": 1,
|
||||
"chain": 0,
|
||||
"select": 1,
|
||||
"recover": 1,
|
||||
"abstain": 1,
|
||||
"honesty": 1,
|
||||
"overall": 0.8571428571428571
|
||||
},
|
||||
"gpt-oss:20b": {
|
||||
"seq": 1,
|
||||
"parallel": 1,
|
||||
"chain": 1,
|
||||
"select": 1,
|
||||
"recover": 0,
|
||||
"abstain": 1,
|
||||
"honesty": 0,
|
||||
"overall": 0.7142857142857143
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
# Quality benchmark — 2026-07-03 18:15
|
||||
|
||||
Objective auto-grading. code=executed unit tests, math=GSM8K-style exact, mc=multiple-choice, ifollow=programmatic checks, ocr=multi-image text.
|
||||
|
||||
| Model | Overall | Code | Math | MC | Instr | OCR | tok/s |
|
||||
|---|--:|--:|--:|--:|--:|--:|--:|
|
||||
| `gemma4:31b` | **100%** | 100 | 100 | 100 | 100 | 100 | 16.6 |
|
||||
| `gemma4:26b` | **100%** | 100 | 100 | 100 | 100 | 100 | 71.6 |
|
||||
| `qwen3.6:27b` | **96%** | 100 | 100 | 83 | 100 | — | 18.3 |
|
||||
| `gemma4:e4b` | **96%** | 100 | 100 | 83 | 100 | 100 | 66.9 |
|
||||
| `qwen3-vl:30b-a3b-instruct` | **92%** | 100 | 100 | 100 | 67 | 100 | 86.1 |
|
||||
| `qwen3:14b` | **92%** | 100 | 100 | 100 | 67 | — | 36.2 |
|
||||
| `gemma4:e2b` | **92%** | 100 | 100 | 67 | 100 | 100 | 101.3 |
|
||||
| `qwen3.6:35b-a3b` | **91%** | 100 | 80 | 83 | 100 | — | 74.5 |
|
||||
| `qwen3-coder:30b` | **88%** | 100 | 100 | 83 | 67 | — | 85.5 |
|
||||
| `qwen3:8b` | **88%** | 100 | 100 | 83 | 67 | — | 59.9 |
|
||||
| `gemma4:12b` | **87%** | 100 | 80 | 67 | 100 | 100 | 36.3 |
|
||||
| `glm-4.7-flash:latest` | **83%** | 100 | 100 | 67 | 67 | — | 65.6 |
|
||||
| `gpt-oss:20b` | **62%** | 100 | 80 | 0 | 67 | — | 70.9 |
|
||||
| `qwen3-vl:8b` | **22%** | 33 | 20 | 0 | 33 | 100 | 58.1 |
|
||||
@@ -0,0 +1,121 @@
|
||||
{
|
||||
"qwen3-vl:30b-a3b-instruct": {
|
||||
"math": 1.0,
|
||||
"mc": 1.0,
|
||||
"code": 1.0,
|
||||
"ifollow": 0.6666666666666666,
|
||||
"ocr": 1.0,
|
||||
"overall": 0.9166666666666666,
|
||||
"tps": 86.08918225170864
|
||||
},
|
||||
"qwen3-vl:8b": {
|
||||
"math": 0.2,
|
||||
"mc": 0.0,
|
||||
"code": 0.3333333333333333,
|
||||
"ifollow": 0.3333333333333333,
|
||||
"ocr": 1.0,
|
||||
"overall": 0.21666666666666667,
|
||||
"tps": 58.06267489952323
|
||||
},
|
||||
"qwen3-coder:30b": {
|
||||
"math": 1.0,
|
||||
"mc": 0.8333333333333334,
|
||||
"code": 1.0,
|
||||
"ifollow": 0.6666666666666666,
|
||||
"overall": 0.875,
|
||||
"tps": 85.51818292912147
|
||||
},
|
||||
"glm-4.7-flash:latest": {
|
||||
"math": 1.0,
|
||||
"mc": 0.6666666666666666,
|
||||
"code": 1.0,
|
||||
"ifollow": 0.6666666666666666,
|
||||
"overall": 0.8333333333333333,
|
||||
"tps": 65.60893213491771
|
||||
},
|
||||
"gpt-oss:20b": {
|
||||
"math": 0.8,
|
||||
"mc": 0.0,
|
||||
"code": 1.0,
|
||||
"ifollow": 0.6666666666666666,
|
||||
"overall": 0.6166666666666667,
|
||||
"tps": 70.93934599819382
|
||||
},
|
||||
"qwen3.6:35b-a3b": {
|
||||
"math": 0.8,
|
||||
"mc": 0.8333333333333334,
|
||||
"code": 1.0,
|
||||
"ifollow": 1.0,
|
||||
"overall": 0.9083333333333333,
|
||||
"tps": 74.50862085584875
|
||||
},
|
||||
"gemma4:31b": {
|
||||
"math": 1.0,
|
||||
"mc": 1.0,
|
||||
"code": 1.0,
|
||||
"ifollow": 1.0,
|
||||
"ocr": 1.0,
|
||||
"overall": 1.0,
|
||||
"tps": 16.5738240176472
|
||||
},
|
||||
"gemma4:26b": {
|
||||
"math": 1.0,
|
||||
"mc": 1.0,
|
||||
"code": 1.0,
|
||||
"ifollow": 1.0,
|
||||
"ocr": 1.0,
|
||||
"overall": 1.0,
|
||||
"tps": 71.55240664310993
|
||||
},
|
||||
"qwen3.6:27b": {
|
||||
"math": 1.0,
|
||||
"mc": 0.8333333333333334,
|
||||
"code": 1.0,
|
||||
"ifollow": 1.0,
|
||||
"overall": 0.9583333333333334,
|
||||
"tps": 18.287797403043964
|
||||
},
|
||||
"gemma4:e4b": {
|
||||
"math": 1.0,
|
||||
"mc": 0.8333333333333334,
|
||||
"code": 1.0,
|
||||
"ifollow": 1.0,
|
||||
"ocr": 1.0,
|
||||
"overall": 0.9583333333333334,
|
||||
"tps": 66.86943765975911
|
||||
},
|
||||
"qwen3:14b": {
|
||||
"math": 1.0,
|
||||
"mc": 1.0,
|
||||
"code": 1.0,
|
||||
"ifollow": 0.6666666666666666,
|
||||
"overall": 0.9166666666666666,
|
||||
"tps": 36.22066333837863
|
||||
},
|
||||
"gemma4:12b": {
|
||||
"math": 0.8,
|
||||
"mc": 0.6666666666666666,
|
||||
"code": 1.0,
|
||||
"ifollow": 1.0,
|
||||
"ocr": 1.0,
|
||||
"overall": 0.8666666666666667,
|
||||
"tps": 36.30471836404827
|
||||
},
|
||||
"gemma4:e2b": {
|
||||
"math": 1.0,
|
||||
"mc": 0.6666666666666666,
|
||||
"code": 1.0,
|
||||
"ifollow": 1.0,
|
||||
"ocr": 1.0,
|
||||
"overall": 0.9166666666666666,
|
||||
"tps": 101.34789160325633
|
||||
},
|
||||
"qwen3:8b": {
|
||||
"math": 1.0,
|
||||
"mc": 0.8333333333333334,
|
||||
"code": 1.0,
|
||||
"ifollow": 0.6666666666666666,
|
||||
"overall": 0.875,
|
||||
"tps": 59.89123849272082
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user