Joseph CostaandClaude Opus 4.8
9938d46a67
A stdlib pre-router in front of Ollama with LiteLLM backend: - auto model selection by content/tools/modality, with fallbacks - OpenAI /v1, Anthropic /v1/messages, and Ollama-native /api/* endpoints - Whisper-shaped /v1/audio/transcriptions + in-chat audio - key-based fleet policies (e.g. force a client onto uncensored models) - optional Bearer auth; launchd/systemd service install - benchmark harnesses (speed, quality, agentic tool use) with sample results Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
42 lines
1.8 KiB
Bash
Executable File
42 lines
1.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Full stack: network client -> pre-router (0.0.0.0:8080) -> LiteLLM (127.0.0.1:4000) -> Ollama (11434)
|
|
# Router is network-facing; LiteLLM stays internal. Uses a Python 3.12 venv for LiteLLM.
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")"
|
|
|
|
PY312="$(command -v python3.12 || echo /opt/homebrew/opt/python@3.12/bin/python3.12)"
|
|
[ -x "$PY312" ] || { echo "python3.12 missing. Run: brew install python@3.12"; exit 1; }
|
|
|
|
# venv + deps (first run only)
|
|
if [ ! -d .venv ]; then "$PY312" -m venv .venv; fi
|
|
source .venv/bin/activate
|
|
pip -q install --upgrade pip >/dev/null
|
|
pip -q install "litellm[proxy]" pillow >/dev/null # pillow: for the vision benchmark
|
|
|
|
# keep the tiny classifier resident
|
|
export OLLAMA_KEEP_ALIVE="${OLLAMA_KEEP_ALIVE:-30m}"
|
|
curl -s http://127.0.0.1:11434/api/generate \
|
|
-d '{"model":"qwen3:8b","prompt":"ok","stream":false,"keep_alive":-1}' >/dev/null 2>&1 || true
|
|
|
|
# LiteLLM proxy (internal only)
|
|
echo ">> starting LiteLLM proxy on 127.0.0.1:4000 ..."
|
|
litellm --config litellm.config.yaml --host 127.0.0.1 --port 4000 > litellm.log 2>&1 &
|
|
LITELLM_PID=$!
|
|
trap 'kill $LITELLM_PID 2>/dev/null || true' EXIT
|
|
for i in $(seq 1 60); do
|
|
curl -sf http://127.0.0.1:4000/health/liveliness >/dev/null 2>&1 && { echo " LiteLLM up."; break; }
|
|
sleep 2
|
|
done
|
|
|
|
# Pre-router — network facing, forwards to LiteLLM
|
|
export ROUTER_UPSTREAM="http://127.0.0.1:4000/v1"
|
|
export ROUTER_HOST="0.0.0.0"
|
|
# SECURITY: uncomment to require a token from network clients (recommended):
|
|
# export ROUTER_API_KEY="$(cat .apikey 2>/dev/null || true)"
|
|
if [ -z "${ROUTER_API_KEY:-}" ]; then
|
|
echo "!! WARNING: router is OPEN on the LAN (no ROUTER_API_KEY set)."
|
|
fi
|
|
LANIP=$(ipconfig getifaddr en7 2>/dev/null || ipconfig getifaddr en0 2>/dev/null || echo "<lan-ip>")
|
|
echo ">> router reachable at: http://${LANIP}:8080/v1 (model \"auto\")"
|
|
exec python router.py
|