# Serve on Windows: LiteLLM (:4000) + router (:8080). Windows equivalent of run.sh. # Run: powershell -ExecutionPolicy Bypass -File .\run.ps1 $ErrorActionPreference = "Stop" Set-Location $PSScriptRoot # optional local config (host-specific env vars, kept out of git — see .env.example) if (Test-Path ".env") { Get-Content ".env" | Where-Object { $_ -match '^\s*[^#=]+=' } | ForEach-Object { $k, $v = $_ -split '=', 2 [Environment]::SetEnvironmentVariable($k.Trim(), $v.Trim(), 'Process') } } # load keys from files into env (kept out of git) if (Test-Path ".apikey") { $env:ROUTER_API_KEY = (Get-Content ".apikey" -Raw).Trim() } if (Test-Path ".uncensored_key") { $env:ROUTER_UNCENSORED_KEY = (Get-Content ".uncensored_key" -Raw).Trim() } if (-not $env:OLLAMA_KEEP_ALIVE) { $env:OLLAMA_KEEP_ALIVE = "30m" } # warm the small classifier so ambiguous routing has no cold start (best effort) try { Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:11434/api/generate" -ContentType "application/json" ` -TimeoutSec 10 -Body '{"model":"qwen3:8b","prompt":"ok","stream":false,"keep_alive":-1}' | Out-Null } catch {} # LiteLLM backend (localhost only) Write-Host ">> starting LiteLLM on 127.0.0.1:4000 ..." $litellm = Start-Process -FilePath ".\.venv\Scripts\litellm.exe" ` -ArgumentList "--config", "litellm.config.yaml", "--host", "127.0.0.1", "--port", "4000" ` -RedirectStandardOutput "litellm.log" -RedirectStandardError "litellm.err.log" -NoNewWindow -PassThru $litellm.Id | Set-Content ".litellm.pid" for ($i = 0; $i -lt 60; $i++) { try { Invoke-WebRequest "http://127.0.0.1:4000/health/liveliness" -TimeoutSec 2 -UseBasicParsing | Out-Null; break } catch { Start-Sleep -Seconds 2 } } # Router (network-facing). Trap tears down LiteLLM on exit. $env:ROUTER_UPSTREAM = "http://127.0.0.1:4000/v1" if (-not $env:ROUTER_HOST) { $env:ROUTER_HOST = "0.0.0.0" } Write-Host ">> router on http://$($env:ROUTER_HOST):8080/v1 (use model 'auto')" try { & ".\.venv\Scripts\python.exe" router.py } finally { if ($litellm -and -not $litellm.HasExited) { Stop-Process -Id $litellm.Id -Force -ErrorAction SilentlyContinue } Remove-Item ".router.pid" -ErrorAction SilentlyContinue }