feat: multi-backend proxy with auto rate-limit jailing
Refactor from monolithic proxy to multi-backend architecture (Backend interface: OpenCode Zen + Kilo gateway). - Add auto rate-limit jailing: models returning HTTP 429 are hidden from /v1/models immediately and re-probed every 30 min - Backend interface supports model aliases, custom headers, and per-backend routing - Full Anthropic Messages API support with OpenAI format conversion - 9 free models across two backends with Claude name aliases
This commit is contained in:
Renato
+229
-129
@@ -1,98 +1,16 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ZenBaseURL is the upstream OpenCode Zen API base URL.
|
||||
const ZenBaseURL = "https://opencode.ai/zen/v1"
|
||||
|
||||
// Free models available via OpenCode Zen (no API key required).
|
||||
// These are the 5 models currently offered for free.
|
||||
var zenModels = []ModelInfo{
|
||||
{
|
||||
ID: "big-pickle",
|
||||
Name: "Big Pickle (Free)",
|
||||
OwnedBy: "opencode-zen",
|
||||
ContextWindow: 200_000,
|
||||
MaxOutputTokens: 32_000,
|
||||
Description: "Permanentemente gratis. Código legacy, refactors, mantenimiento.",
|
||||
SupportsVision: false,
|
||||
},
|
||||
{
|
||||
ID: "deepseek-v4-flash-free",
|
||||
Name: "DeepSeek V4 Flash (Free)",
|
||||
OwnedBy: "deepseek",
|
||||
ContextWindow: 1_000_000,
|
||||
MaxOutputTokens: 384_000,
|
||||
Description: "El todoterreno. 1M contexto, 284B params (13B activos), razonamiento.",
|
||||
SupportsVision: false,
|
||||
},
|
||||
{
|
||||
ID: "mimo-v2.5-free",
|
||||
Name: "MiMo V2.5 (Free)",
|
||||
OwnedBy: "xiaomi",
|
||||
ContextWindow: 1_000_000,
|
||||
MaxOutputTokens: 32_000,
|
||||
Description: "Multimodal de Xiaomi. 1M contexto, imágenes → código. MIT license.",
|
||||
SupportsVision: true,
|
||||
},
|
||||
{
|
||||
ID: "north-mini-code-free",
|
||||
Name: "North Mini Code (Free)",
|
||||
OwnedBy: "cohere",
|
||||
ContextWindow: 256_000,
|
||||
MaxOutputTokens: 64_000,
|
||||
Description: "Cohere. 30B total / 3B activos (MoE). Respuesta rápida, scripts.",
|
||||
SupportsVision: false,
|
||||
},
|
||||
{
|
||||
ID: "nemotron-3-ultra-free",
|
||||
Name: "Nemotron 3 Ultra (Free)",
|
||||
OwnedBy: "nvidia",
|
||||
ContextWindow: 1_000_000,
|
||||
MaxOutputTokens: 16_384,
|
||||
Description: "NVIDIA. 550B params (55B activos). Mamba-2 + MoE híbrido. Razonamiento.",
|
||||
SupportsVision: false,
|
||||
},
|
||||
}
|
||||
|
||||
// Model aliases: short name → full model ID.
|
||||
var modelAliases = map[string]string{
|
||||
// Short aliases
|
||||
"pickle": "big-pickle",
|
||||
"big-pickle": "big-pickle",
|
||||
"deepseek": "deepseek-v4-flash-free",
|
||||
"deepseek-v4": "deepseek-v4-flash-free",
|
||||
"ds": "deepseek-v4-flash-free",
|
||||
"mimo": "mimo-v2.5-free",
|
||||
"mimo-v2.5": "mimo-v2.5-free",
|
||||
"xiaomi": "mimo-v2.5-free",
|
||||
"north": "north-mini-code-free",
|
||||
"north-mini": "north-mini-code-free",
|
||||
"cohere": "north-mini-code-free",
|
||||
"nemotron": "nemotron-3-ultra-free",
|
||||
"nemotron-3": "nemotron-3-ultra-free",
|
||||
"nvidia": "nemotron-3-ultra-free",
|
||||
|
||||
// Claude model name aliases (for Claude Code compatibility)
|
||||
// Claude Code validates model names client-side; set ANTHROPIC_MODEL to a Claude name.
|
||||
"claude-sonnet-4-6": "deepseek-v4-flash-free",
|
||||
"claude-sonnet-4-5": "deepseek-v4-flash-free",
|
||||
"claude-sonnet-4": "deepseek-v4-flash-free",
|
||||
"claude-opus-4-8": "deepseek-v4-flash-free",
|
||||
"claude-opus-4-5": "deepseek-v4-flash-free",
|
||||
"claude-opus-4": "deepseek-v4-flash-free",
|
||||
"claude-haiku-4-5": "north-mini-code-free",
|
||||
"claude-haiku-4": "north-mini-code-free",
|
||||
"claude-3.5-sonnet": "north-mini-code-free",
|
||||
"claude-3.5-haiku": "big-pickle",
|
||||
}
|
||||
|
||||
// ModelInfo describes a single model.
|
||||
// ModelInfo describes a single model exposed by a backend.
|
||||
type ModelInfo struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
@@ -103,11 +21,31 @@ type ModelInfo struct {
|
||||
SupportsVision bool `json:"supports_vision"`
|
||||
}
|
||||
|
||||
// Proxy handles forwarding OpenAI-format requests to OpenCode Zen.
|
||||
// healthCheckInterval is how often caged models get re-probed.
|
||||
const healthCheckInterval = 30 * time.Minute
|
||||
|
||||
// healthPingTimeout caps each health probe. Set high enough for slow reasoning
|
||||
// models (nvidia/nemotron can take ~80s to first token).
|
||||
const healthPingTimeout = 90 * time.Second
|
||||
|
||||
// modelHealth tracks the rate-limit state of a single model.
|
||||
type modelHealth struct {
|
||||
status string // "active" | "rate_limited"
|
||||
lastCheck time.Time
|
||||
}
|
||||
|
||||
// Proxy routes OpenAI-format requests to the right Backend and manages session IDs.
|
||||
type Proxy struct {
|
||||
apiKey string
|
||||
sessions map[string]*session // keyed by client-provided API key
|
||||
apiKey string
|
||||
|
||||
backends []Backend
|
||||
defaultBackend Backend
|
||||
|
||||
sessions map[string]*session
|
||||
mu sync.Mutex
|
||||
|
||||
health map[string]*modelHealth
|
||||
healthMu sync.RWMutex
|
||||
}
|
||||
|
||||
type session struct {
|
||||
@@ -115,63 +53,74 @@ type session struct {
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
// NewProxy creates a new Proxy instance.
|
||||
// NewProxy creates a Proxy with the default set of free-model backends.
|
||||
//
|
||||
// Order matters: more specific backends come first so their model IDs win over
|
||||
// the catch-all opencode aliases. The default backend is the fallback for
|
||||
// unknown models (pass-through as-is).
|
||||
func NewProxy(apiKey string) *Proxy {
|
||||
return &Proxy{
|
||||
apiKey: apiKey,
|
||||
sessions: make(map[string]*session),
|
||||
oc := OpenCodeBackend{}
|
||||
kl := KiloBackend{}
|
||||
p := &Proxy{
|
||||
apiKey: apiKey,
|
||||
backends: []Backend{kl, oc},
|
||||
defaultBackend: oc,
|
||||
sessions: make(map[string]*session),
|
||||
health: make(map[string]*modelHealth),
|
||||
}
|
||||
go p.healthLoop()
|
||||
return p
|
||||
}
|
||||
|
||||
// APIKey returns the configured API key (empty = no auth).
|
||||
// APIKey returns the configured proxy API key (empty = no auth).
|
||||
func (p *Proxy) APIKey() string { return p.apiKey }
|
||||
|
||||
// Models returns the list of available free models.
|
||||
func (p *Proxy) Models() []ModelInfo { return zenModels }
|
||||
|
||||
// ModelByID returns the ModelInfo entry for a given model ID, or nil if not found.
|
||||
func ModelByID(id string) *ModelInfo {
|
||||
for _, m := range zenModels {
|
||||
if m.ID == id {
|
||||
return &m
|
||||
// Models returns the aggregated free-model catalogue across all backends,
|
||||
// excluding any model currently caged for rate limiting (HTTP 429).
|
||||
func (p *Proxy) Models() []ModelInfo {
|
||||
var all []ModelInfo
|
||||
for _, b := range p.backends {
|
||||
for _, m := range b.Models() {
|
||||
if !p.isRateLimited(m.ID) {
|
||||
all = append(all, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return all
|
||||
}
|
||||
|
||||
// ResolveModel maps a requested model name to a Zen model ID.
|
||||
// Returns the resolved model ID and whether it was found.
|
||||
// BackendFor returns the backend that recognises the model, or the default
|
||||
// backend (pass-through) when no backend claims it.
|
||||
func (p *Proxy) BackendFor(model string) Backend {
|
||||
b, _, _ := p.resolveModel(model)
|
||||
return b
|
||||
}
|
||||
|
||||
// resolveModel walks the backends in order and returns the first match.
|
||||
// When nothing matches it returns the default backend with the model unchanged
|
||||
// and ok=false, so callers can forward it as-is (pass-through).
|
||||
func (p *Proxy) resolveModel(model string) (backend Backend, resolved string, ok bool) {
|
||||
for _, b := range p.backends {
|
||||
if r, matched := b.Resolve(model); matched {
|
||||
return b, r, true
|
||||
}
|
||||
}
|
||||
return p.defaultBackend, model, false
|
||||
}
|
||||
|
||||
// ResolveModel maps a requested model name to its upstream ID (backend-aware).
|
||||
// Kept for backwards compatibility with existing call sites.
|
||||
func (p *Proxy) ResolveModel(requested string) (string, bool) {
|
||||
// Check aliases first (covers both short names and exact IDs)
|
||||
if mapped, ok := modelAliases[requested]; ok {
|
||||
return mapped, true
|
||||
}
|
||||
// Check if it's already a valid model ID (belt and suspenders)
|
||||
for _, m := range zenModels {
|
||||
if m.ID == requested {
|
||||
return requested, true
|
||||
}
|
||||
}
|
||||
// Pass through unknown models (might be newly added upstream)
|
||||
return requested, false
|
||||
_, r, ok := p.resolveModel(requested)
|
||||
return r, ok
|
||||
}
|
||||
|
||||
// ZenChatURL returns the full upstream URL for chat completions.
|
||||
func ZenChatURL() string {
|
||||
return ZenBaseURL + "/chat/completions"
|
||||
}
|
||||
|
||||
// ZenModelsURL returns the full upstream URL for model listing.
|
||||
func ZenModelsURL() string {
|
||||
return ZenBaseURL + "/models"
|
||||
}
|
||||
|
||||
// SessionID returns a session ID for the given user key, rotating every 30 minutes.
|
||||
// SessionID returns a session ID for the given user key, rotating every 30 min.
|
||||
func (p *Proxy) SessionID(userKey string) string {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
if s, ok := p.sessions[userKey]; ok && time.Now().Before(s.expiresAt) {
|
||||
if s, exists := p.sessions[userKey]; exists && time.Now().Before(s.expiresAt) {
|
||||
return s.id
|
||||
}
|
||||
|
||||
@@ -192,3 +141,154 @@ func randomHex(n int) string {
|
||||
rand.Read(b)
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
// ---------- Model health / rate-limit tracking ----------
|
||||
|
||||
// markRateLimited cages a model so it disappears from /v1/models until it recovers.
|
||||
// Only logs the transition to avoid spam when many 429s arrive in a row.
|
||||
func (p *Proxy) markRateLimited(modelID string) {
|
||||
p.healthMu.Lock()
|
||||
defer p.healthMu.Unlock()
|
||||
h := p.health[modelID]
|
||||
if h == nil {
|
||||
h = &modelHealth{}
|
||||
p.health[modelID] = h
|
||||
}
|
||||
if h.status != "rate_limited" {
|
||||
fmt.Printf("[health] caging model '%s' (rate limited)\n", modelID)
|
||||
}
|
||||
h.status = "rate_limited"
|
||||
h.lastCheck = time.Now()
|
||||
}
|
||||
|
||||
// markActive restores a caged model so it reappears in /v1/models.
|
||||
func (p *Proxy) markActive(modelID string) {
|
||||
p.healthMu.Lock()
|
||||
defer p.healthMu.Unlock()
|
||||
h := p.health[modelID]
|
||||
if h == nil {
|
||||
h = &modelHealth{}
|
||||
p.health[modelID] = h
|
||||
}
|
||||
h.status = "active"
|
||||
h.lastCheck = time.Now()
|
||||
}
|
||||
|
||||
// isRateLimited reports whether a model is currently caged for rate limiting.
|
||||
func (p *Proxy) isRateLimited(modelID string) bool {
|
||||
p.healthMu.RLock()
|
||||
defer p.healthMu.RUnlock()
|
||||
h, ok := p.health[modelID]
|
||||
return ok && h.status == "rate_limited"
|
||||
}
|
||||
|
||||
// healthLoop runs an initial full scan on startup, then re-probes caged models
|
||||
// every healthCheckInterval (30 min). Released models reappear in /v1/models.
|
||||
func (p *Proxy) healthLoop() {
|
||||
p.scanAllModels()
|
||||
ticker := time.NewTicker(healthCheckInterval)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
p.recheckCagedModels()
|
||||
}
|
||||
}
|
||||
|
||||
// scanAllModels probes every known model once. Models returning non-200 (429)
|
||||
// are caged; the rest stay active. Used for the startup scan.
|
||||
func (p *Proxy) scanAllModels() {
|
||||
var ids []string
|
||||
for _, b := range p.backends {
|
||||
for _, m := range b.Models() {
|
||||
ids = append(ids, m.ID)
|
||||
}
|
||||
}
|
||||
fmt.Printf("[health] initial scan: probing %d models\n", len(ids))
|
||||
for _, id := range ids {
|
||||
ok := p.pingModel(id)
|
||||
p.healthMu.Lock()
|
||||
h := p.health[id]
|
||||
if h == nil {
|
||||
h = &modelHealth{}
|
||||
p.health[id] = h
|
||||
}
|
||||
h.lastCheck = time.Now()
|
||||
if ok {
|
||||
h.status = "active"
|
||||
} else {
|
||||
h.status = "rate_limited"
|
||||
fmt.Printf("[health] caging model '%s' (rate limited on initial scan)\n", id)
|
||||
}
|
||||
p.healthMu.Unlock()
|
||||
}
|
||||
active := 0
|
||||
p.healthMu.RLock()
|
||||
for _, h := range p.health {
|
||||
if h.status == "active" {
|
||||
active++
|
||||
}
|
||||
}
|
||||
p.healthMu.RUnlock()
|
||||
fmt.Printf("[health] initial scan complete: %d active, %d caged\n", active, len(ids)-active)
|
||||
}
|
||||
|
||||
// recheckCagedModels re-probes only the models currently caged. A 200 releases
|
||||
// them; a 429 or timeout keeps them caged for another interval.
|
||||
func (p *Proxy) recheckCagedModels() {
|
||||
p.healthMu.RLock()
|
||||
var caged []string
|
||||
for id, h := range p.health {
|
||||
if h.status == "rate_limited" {
|
||||
caged = append(caged, id)
|
||||
}
|
||||
}
|
||||
p.healthMu.RUnlock()
|
||||
|
||||
if len(caged) == 0 {
|
||||
return
|
||||
}
|
||||
fmt.Printf("[health] re-checking %d caged models\n", len(caged))
|
||||
for _, id := range caged {
|
||||
ok := p.pingModel(id)
|
||||
p.healthMu.Lock()
|
||||
h := p.health[id]
|
||||
if h == nil {
|
||||
h = &modelHealth{}
|
||||
p.health[id] = h
|
||||
}
|
||||
h.lastCheck = time.Now()
|
||||
if ok {
|
||||
h.status = "active"
|
||||
fmt.Printf("[health] releasing model '%s' (recovered)\n", id)
|
||||
} else {
|
||||
fmt.Printf("[health] model '%s' still rate limited, retry in 30m\n", id)
|
||||
}
|
||||
p.healthMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// pingModel sends a minimal chat completion to the model's backend and reports
|
||||
// whether the upstream answered HTTP 200. Timeouts and non-200 count as
|
||||
// unhealthy so the model stays caged (conservative).
|
||||
func (p *Proxy) pingModel(modelID string) bool {
|
||||
backend, resolved, known := p.resolveModel(modelID)
|
||||
if !known {
|
||||
resolved = modelID
|
||||
}
|
||||
|
||||
body := fmt.Sprintf(`{"model":%q,"messages":[{"role":"user","content":"OK"}],"max_tokens":5,"temperature":0}`, resolved)
|
||||
req, err := http.NewRequest("POST", backend.ChatURL(), bytes.NewReader([]byte(body)))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for k, v := range backend.Headers(RequestID(), "ses_health") {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: healthPingTimeout}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
resp.Body.Close()
|
||||
return resp.StatusCode == 200
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user