Joseph CostaandClaude Opus 4.8
ab0cd4fe85
- run.sh / run.ps1: load an optional .env for host-specific env vars - .env.example + docs: ComfyUI can run on another host (ROUTER_COMFYUI) - .gitignore: .env Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
48 lines
2.2 KiB
PowerShell
48 lines
2.2 KiB
PowerShell
# Serve on Windows: LiteLLM (:4000) + router (:8080). Windows equivalent of run.sh.
|
|
# Run: powershell -ExecutionPolicy Bypass -File .\run.ps1
|
|
$ErrorActionPreference = "Stop"
|
|
Set-Location $PSScriptRoot
|
|
|
|
# optional local config (host-specific env vars, kept out of git — see .env.example)
|
|
if (Test-Path ".env") {
|
|
Get-Content ".env" | Where-Object { $_ -match '^\s*[^#=]+=' } | ForEach-Object {
|
|
$k, $v = $_ -split '=', 2
|
|
[Environment]::SetEnvironmentVariable($k.Trim(), $v.Trim(), 'Process')
|
|
}
|
|
}
|
|
|
|
# load keys from files into env (kept out of git)
|
|
if (Test-Path ".apikey") { $env:ROUTER_API_KEY = (Get-Content ".apikey" -Raw).Trim() }
|
|
if (Test-Path ".uncensored_key") { $env:ROUTER_UNCENSORED_KEY = (Get-Content ".uncensored_key" -Raw).Trim() }
|
|
if (-not $env:OLLAMA_KEEP_ALIVE) { $env:OLLAMA_KEEP_ALIVE = "30m" }
|
|
|
|
# warm the small classifier so ambiguous routing has no cold start (best effort)
|
|
try {
|
|
Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:11434/api/generate" -ContentType "application/json" `
|
|
-TimeoutSec 10 -Body '{"model":"qwen3:8b","prompt":"ok","stream":false,"keep_alive":-1}' | Out-Null
|
|
} catch {}
|
|
|
|
# LiteLLM backend (localhost only)
|
|
Write-Host ">> starting LiteLLM on 127.0.0.1:4000 ..."
|
|
$litellm = Start-Process -FilePath ".\.venv\Scripts\litellm.exe" `
|
|
-ArgumentList "--config", "litellm.config.yaml", "--host", "127.0.0.1", "--port", "4000" `
|
|
-RedirectStandardOutput "litellm.log" -RedirectStandardError "litellm.err.log" -NoNewWindow -PassThru
|
|
$litellm.Id | Set-Content ".litellm.pid"
|
|
|
|
for ($i = 0; $i -lt 60; $i++) {
|
|
try { Invoke-WebRequest "http://127.0.0.1:4000/health/liveliness" -TimeoutSec 2 -UseBasicParsing | Out-Null; break }
|
|
catch { Start-Sleep -Seconds 2 }
|
|
}
|
|
|
|
# Router (network-facing). Trap tears down LiteLLM on exit.
|
|
$env:ROUTER_UPSTREAM = "http://127.0.0.1:4000/v1"
|
|
if (-not $env:ROUTER_HOST) { $env:ROUTER_HOST = "0.0.0.0" }
|
|
Write-Host ">> router on http://$($env:ROUTER_HOST):8080/v1 (use model 'auto')"
|
|
try {
|
|
& ".\.venv\Scripts\python.exe" router.py
|
|
}
|
|
finally {
|
|
if ($litellm -and -not $litellm.HasExited) { Stop-Process -Id $litellm.Id -Force -ErrorAction SilentlyContinue }
|
|
Remove-Item ".router.pid" -ErrorAction SilentlyContinue
|
|
}
|