#!/usr/bin/env bash # Full stack: network client -> pre-router (0.0.0.0:8080) -> LiteLLM (127.0.0.1:4000) -> Ollama (11434) # Router is network-facing; LiteLLM stays internal. Uses a Python 3.12 venv for LiteLLM. set -euo pipefail cd "$(dirname "$0")" PY312="$(command -v python3.12 || echo /opt/homebrew/opt/python@3.12/bin/python3.12)" [ -x "$PY312" ] || { echo "python3.12 missing. Run: brew install python@3.12"; exit 1; } # venv + deps (first run only) if [ ! -d .venv ]; then "$PY312" -m venv .venv; fi source .venv/bin/activate pip -q install --upgrade pip >/dev/null pip -q install "litellm[proxy]" pillow >/dev/null # pillow: for the vision benchmark # keep the tiny classifier resident export OLLAMA_KEEP_ALIVE="${OLLAMA_KEEP_ALIVE:-30m}" curl -s http://127.0.0.1:11434/api/generate \ -d '{"model":"qwen3:8b","prompt":"ok","stream":false,"keep_alive":-1}' >/dev/null 2>&1 || true # LiteLLM proxy (internal only) echo ">> starting LiteLLM proxy on 127.0.0.1:4000 ..." litellm --config litellm.config.yaml --host 127.0.0.1 --port 4000 > litellm.log 2>&1 & LITELLM_PID=$! trap 'kill $LITELLM_PID 2>/dev/null || true' EXIT for i in $(seq 1 60); do curl -sf http://127.0.0.1:4000/health/liveliness >/dev/null 2>&1 && { echo " LiteLLM up."; break; } sleep 2 done # Pre-router — network facing, forwards to LiteLLM export ROUTER_UPSTREAM="http://127.0.0.1:4000/v1" export ROUTER_HOST="0.0.0.0" # SECURITY: uncomment to require a token from network clients (recommended): # export ROUTER_API_KEY="$(cat .apikey 2>/dev/null || true)" if [ -z "${ROUTER_API_KEY:-}" ]; then echo "!! WARNING: router is OPEN on the LAN (no ROUTER_API_KEY set)." fi LANIP=$(ipconfig getifaddr en7 2>/dev/null || ipconfig getifaddr en0 2>/dev/null || echo "") echo ">> router reachable at: http://${LANIP}:8080/v1 (model \"auto\")" exec python router.py