Joseph CostaandClaude Opus 4.8
9938d46a67
A stdlib pre-router in front of Ollama with LiteLLM backend: - auto model selection by content/tools/modality, with fallbacks - OpenAI /v1, Anthropic /v1/messages, and Ollama-native /api/* endpoints - Whisper-shaped /v1/audio/transcriptions + in-chat audio - key-based fleet policies (e.g. force a client onto uncensored models) - optional Bearer auth; launchd/systemd service install - benchmark harnesses (speed, quality, agentic tool use) with sample results Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
35 lines
1.7 KiB
YAML
35 lines
1.7 KiB
YAML
# LiteLLM proxy: exposes every local model over an OpenAI API on :4000,
|
|
# with automatic fallbacks. The pre-router (router.py) picks the model;
|
|
# LiteLLM handles delivery, retries, and logging.
|
|
model_list:
|
|
- model_name: qwen3:8b
|
|
litellm_params: { model: ollama_chat/qwen3:8b, api_base: http://127.0.0.1:11434 }
|
|
- model_name: qwen3:14b
|
|
litellm_params: { model: ollama_chat/qwen3:14b, api_base: http://127.0.0.1:11434 }
|
|
- model_name: glm-4.7-flash
|
|
litellm_params: { model: ollama_chat/glm-4.7-flash, api_base: http://127.0.0.1:11434 }
|
|
- model_name: qwen3-coder:30b
|
|
litellm_params: { model: ollama_chat/qwen3-coder:30b, api_base: http://127.0.0.1:11434 }
|
|
- model_name: qwen3.6:35b-a3b
|
|
litellm_params: { model: ollama_chat/qwen3.6:35b-a3b, api_base: http://127.0.0.1:11434 }
|
|
- model_name: gpt-oss:20b
|
|
litellm_params: { model: ollama_chat/gpt-oss:20b, api_base: http://127.0.0.1:11434 }
|
|
- model_name: qwen3-vl:8b
|
|
litellm_params: { model: ollama_chat/qwen3-vl:8b, api_base: http://127.0.0.1:11434 }
|
|
- model_name: qwen3-vl:30b-a3b-instruct
|
|
litellm_params: { model: ollama_chat/qwen3-vl:30b-a3b-instruct, api_base: http://127.0.0.1:11434 }
|
|
# catch-all: any other model name (incl. :latest tags picked by `ollama launch`) -> ollama
|
|
- model_name: "*"
|
|
litellm_params: { model: "ollama_chat/*", api_base: http://127.0.0.1:11434 }
|
|
|
|
litellm_settings:
|
|
drop_params: true # tolerate params a given model doesn't support
|
|
|
|
router_settings:
|
|
fallbacks:
|
|
- { "glm-4.7-flash": ["qwen3-coder:30b"] }
|
|
- { "qwen3-coder:30b": ["glm-4.7-flash"] }
|
|
- { "qwen3.6:35b-a3b": ["gpt-oss:20b"] }
|
|
- { "qwen3-vl:30b-a3b-instruct": ["qwen3-vl:8b"] }
|
|
- { "qwen3:14b": ["qwen3:8b"] }
|