6krrt/config.yaml

# Local LLM router config
# All weights, thresholds, and provider settings live here so they can be
# tuned without touching code. Loaded/validated by config.py.

objective:
  # Quality is the objective. Cost is a constraint and a tiebreak. Eco is
  # logged per request but is NOT optimized here — that judgement is made
  # outside this router.
  #
  # This replaced a weighted blend (cost 0.4 / eco 0.2 / proficiency 0.4).
  # Measurement killed it: turning the cost weight from 0.4 to ZERO changed
  # the winner in only 2 of 6 categories, so the blend was never steering on
  # quality — while 60% of every decision adjudicated fractions of a cent
  # (all real traffic to date totals $0.07).

  # Proficiency differences smaller than this are treated as equal and the
  # cheaper model wins. This is measurement noise, not preference: scores
  # currently rest on 2-3 samples per category, so a 0.05 gap is
  # indistinguishable from sampling variation and paying for it buys noise.
  # Narrow it as samples accumulate.
  quality_tolerance: 0.10

  # Cost is priced per-request from catalog prices, NOT from a benchmark
  # sweep. A fixed 400-token reference task ranked glm-5.2-fast 3.2x cheaper
  # than deepseek-v4-flash; on a realistic 70k-token prompt deepseek is 5.0x
  # cheaper. Attribution inverts with prompt size, so a fixed-shape benchmark
  # cannot rank models for a workload of another shape. Catalog prices scaled
  # to the actual request agree with the live measurement, cost nothing, and
  # need no sweep.

  # Share of prompt tokens served from the provider's prefix cache. Agent
  # clients resend the whole conversation each turn, so most of it is a hit.
  #
  # Measured token-weighted across 50 sessions and 40.7M tokens (2026-08-23):
  # 91.7% overall, 92.6% on the sessions above 400k tokens, which are the ones
  # that carry the cost. The previous 0.84 came from 2.2M tokens of much
  # earlier traffic.
  #
  # Raising it changed NO winner in any of the nine categories at 60k context
  # — checked before editing. It is here because the number should be true,
  # not because the routing needed it.
  assumed_cache_rate: 0.917

  # Completion length assumed when pricing a request. Real sessions here median
  # around 200-400 completion tokens against enormous prompts.
  assumed_completion_tokens: 500

  # Per-request ceiling on measured ENERGY, in kWh. null disables it.
  #
  # Denominated in kWh rather than dollars because the plan is a subscription
  # with a 6.25 kWh quota, not pay-per-request. Dollars accrue; a quota is a
  # wall you hit mid-task. So this is the cost mandate stated as a guarantee.
  # For scale: the reference task runs ~5e-06 kWh on the cheapest model and
  # ~2.2e-04 on the most expensive.
  max_energy_per_request: null

  # The subscription's kWh allowance per billing period, for reporting burn in
  # /health. Set to match your plan; null disables the report. NeuralWatt also
  # returns allowance_remaining_usd per request, which is logged, but that is a
  # dollar figure while the plan is denominated in energy.
  plan_kwh_per_period: 6.25

context:
  safety_factor: 0.75          # fraction of advertised context treated as usable
  default_output_reserve_tokens: 4096
  # Per-model exceptions to the two settings above, for a row whose real
  # limits you have measured. The global factor has to hold for the whole
  # catalog, so it is deliberately pessimistic; a model you have actually
  # pushed to its limit deserves its own number.
  #
  # Both keys are optional and each falls back to the global on its own.
  # An override of 0 reserve tokens means zero, not "unset".
  #
  #   per_model_overrides:
  #     qwen3.6-35b:
  #       safety_factor: 0.85
  #       output_reserve_tokens: 8192
  #
  # Takes effect on the next `python poller.py` -- effective_context_window is
  # computed at poll time, not per request.
  per_model_overrides: {}

tiers:
  # Maps a tier number to a human label, purely for logging/dashboards.
  1: "cheap / simple"
  2: "mid / general"
  3: "frontier / high-stakes"

tiering:
  # Auto-tiering pass knobs. cheap_completion_max is the completion-cost
  # (per 1M tokens) ceiling below which a model is eligible for tier 1.
  # model_tiers overrides the heuristic per model_id and applies to ALL
  # providers (limitation vs a (model_id, provider) key).
  cheap_completion_max: 1.00

  # Advertised context_window at or above which a model is NOT eligible for
  # tier 1, whatever it costs. Tier is a capability FLOOR (routing drops any
  # row with tier < required_tier), so tier 1 means "simple work only" — and
  # deciding that on price alone excluded deepseek-v4-flash from every tier-2
  # request purely for being $0.28/1M, despite a 1M window and 1.00 on all
  # three coding categories.
  #
  # 512000 sits in the empty band between the catalog's 256K class (262128)
  # and its 1M class (1048560) — a 2x margin either side, so it is not fitted
  # to any one model. gemma-4-31b (256K) stays tier 1; the 1M rows do not.
  tier1_context_max: 512000

  model_tiers: {}

proficiency:
  # Blending rule: leaderboard vs self-eval, once self-eval sample size
  # crosses the threshold below. Below threshold, leaderboard score alone
  # is used so thin self-eval data doesn't dominate.
  self_eval_min_samples: 10
  leaderboard_weight: 0.3
  self_eval_weight: 0.7
  categories:
    - coding_general
    - coding_refactor
    - debugging
    - docs_writing
    - summarization
    - translation
    - reasoning_math
    - tool_use_agentic
    - general_chat

escalation:
  enabled: true
  max_tier: 3

  # Bump the tier when the classifier is unsure of its own call. DEFAULT OFF:
  # this pays frontier prices on a hunch, before anything has gone wrong. The
  # iteration budget below spends after a check has actually failed, which is
  # strictly better on both mandates — the cheap attempt usually succeeds and
  # costs nothing extra, and when it fails you have evidence.
  preemptive_on_low_confidence: false
  min_confidence_before_bump: 0.6

iteration:
  # A tier is not only a capability floor, it is a budget for getting the
  # answer right. These are corrective attempts AFTER a verification failure,
  # not speculative retries.
  #
  # Retries are matched to the failure: a truncated answer gets a bigger token
  # budget on the SAME model (a different one would also run out), while a
  # malformed answer escalates to the next-best candidate (more tokens will
  # not make unparseable output parse).
  enabled: true
  attempts_by_tier:
    1: 0    # cheap/simple — one shot; iterating costs more than it is worth
    2: 1
    3: 2

  # Interactive requests are capped below their tier's budget regardless of
  # tier: every retry doubles time-to-answer, and in interactive use latency
  # IS a quality loss. Batch work does not care.
  max_attempts_interactive: 1

pinch:
  # Relevance-based context pruning (ported from the MIT-licensed llmrouter's
  # "pinch"). This is an OPTIONAL, pre-dispatch stage: when a conversation
  # exceeds budget_tokens, the provider-bound messages are trimmed BEFORE any
  # paid token is sent upstream. User/assistant/system messages are always
  # kept verbatim; only old TOOL RESULTS are shortened or dropped, because
  # they carry the bulk of a long agent session's tokens and are least needed
  # in full by the time the next turn is answered.
  #
  # It does NOT touch the classifier's input, and it defaults off — enable it
  # only if long sessions are shipping more prompt tokens than you want to pay
  # for. Tool results can only be dropped safely because tool outputs are
  # idempotent enough for a placeholder; a wrong guess here loses context, so
  # start conservative (large budget, small reduction).
  enabled: false
  budget_tokens: 50000
  keep_last_turns: 4
  max_summarize_chars: 4000

routing:
  # Access gating is prose-only in the NeuralWatt catalog ("Private preview
  # (grant-gated)", "(Canary)"), so the poller parses it into access_level and
  # routing excludes anything not listed here. Add 'preview'/'canary' only if
  # the account actually holds the grant — otherwise dispatch earns a 403.
  allowed_access_levels:
    - public

  # '-flex' rows are held server-side during peak until a capacity gap opens.
  # That's correct for overnight/batch agent work and wrong for anything
  # interactive, so a request has to opt in via latency_tolerance.
  default_latency_tolerance: interactive   # 'interactive' | 'batch'

  # Minimum tool_use_agentic proficiency required of a model when the REQUEST
  # carries tool definitions. A filter, not a weight, because it is a
  # capability requirement rather than a preference.
  #
  # It does NOT ask whether the task is agentic — it asks whether the model
  # can be trusted with tools that are on the table. The measured failure is
  # the second one: deepseek-v4-flash scores 0.33 here, and the recorded case
  # is a NON-agentic prompt ("it is 1:20pm, my meeting is at 3pm, how many
  # minutes?") where it called two tools instead of subtracting. A model that
  # over-reaches is a hazard wherever tools exist, not only where a classifier
  # would say "agentic" — which it cannot do anyway: asked to identify six
  # unambiguous tool-use prompts, the local models managed 2/6 and 1/6.
  # Whether tools are present is stated in the request body. Read it.
  #
  # 0.5 sits in the empty band between the only two values the catalog
  # currently holds (0.33 and 1.00), so it is not fitted to either. A model
  # with NO measured tool score is unproven rather than proven bad, and is not
  # dropped.
  #
  # CURRENTLY DISABLED (null) pending experimentation. The trade being
  # measured: opencode sends `tools` on essentially every request, so with the
  # filter on, deepseek-v4-flash is excluded from ordinary agent traffic and
  # the ~7x cost advantage on coding routes goes unused. With it off, that
  # advantage applies — and a model measured at 0.33 on tool use handles
  # requests where tools are on the table.
  #
  # What would settle it is outcome data, not another benchmark: run with it
  # off, let POST /outcome report real pass/fail, and compare
  # tool_use_agentic proficiency for deepseek before and after. That is the
  # one signal here that knows whether the work actually worked.
  min_tool_proficiency: null
  tool_use_category: tool_use_agentic

  # Request-side capability gates. These read the request body (image parts,
  # response_format) and hard-restrict to models whose catalog row declares the
  # capability. Unlike min_tool_proficiency these are NOT proficiency gates —
  # a wrong guess is a guaranteed provider 400, so they gate by default and
  # fail closed when the catalog flag is unknown.
  require_vision: true
  require_json_mode: true

  # There is deliberately no flex discount knob. Whether a flex row is usable
  # at all is a hard filter above (latency_tolerance), not a price adjustment
  # -- being held server-side during peak is a latency property, and the
  # catalog advertises flex and standard at the same token price anyway.

local_vision:
  # Local Ollama vision fallback. Used ONLY when a request carries image
  # parts and routing finds NO cloud model that supports_vision — the cloud
  # catalog currently excludes the cost leader (deepseek) on vision, so a
  # fallback is what keeps image requests working instead of 422ing.
  # It speaks the OpenAI-compatible /v1 surface, so this is an Ollama endpoint
  # and the model must be pulled (`ollama pull qwen3-vl:4b`) on that host.
  enabled: true
  base_url: "http://localhost:11434/v1"
  api_key_env: null
  model: "qwen3-vl:4b"
  timeout_seconds: 60
  max_images: 4
  max_image_bytes: 9437184

verification:
  # Structural checks (parse the code, never run it) are free, pure Python and
  # always on — they need no model and run anywhere, down to an RPi.
  # This section governs the LOCAL LLM check, which is not free.
  #
  # Set local_llm_enabled: false on a host with no usable local inference.
  # Structural checking and POST /outcome both keep working; only the
  # refusal/incoherence class of failure stops being caught.
  local_llm_enabled: true

  # This check speaks Ollama's NATIVE API (/api/chat with think=False), which
  # no cloud provider offers, so it is configured SEPARATELY from the
  # classifier rather than derived from it. Deriving it meant that pointing
  # classification anywhere else sent these requests to <that host>/api/chat.
  #
  # Point it at the SAME Ollama as the classifier when that is across a VPN;
  # leave the model null in that case, since the hosts then match.
  base_url: "http://localhost:11434"
  # null means "whatever classifier.model is", correct only while both run on
  # the same local Ollama — config load REFUSES the null once the hosts differ,
  # because the fallback would name a model this Ollama has never heard of and
  # the verifier would fail silently.
  model: "mistral-nemo:12b"

  # Only check answers this large. Measured on real traffic: a local check
  # costs ~15% of a median 193-token answer, so it would only pay if such
  # answers failed more than ~15% of the time. At 1,500 completion tokens the
  # break-even failure rate drops to ~1.9%, which is plausible. Below the
  # threshold a check costs more than the risk it removes.
  min_completion_tokens: 600

  # The check runs AFTER the response has gone back to the client, so it never
  # adds its ~6s to anyone's latency. It exists to learn which models fail on
  # real work, not to gate answers.
  timeout_seconds: 60
  max_output_tokens: 1024

  # How far back /outcome looks when a report carries no request_id and no
  # matching directory. A test run follows the completion that caused it within
  # seconds, so this is deliberately short: a wide window sweeps in sessions
  # that finished long ago and makes every report look ambiguous. If more than
  # one conversation was active inside it, the report is refused rather than
  # guessed at.
  outcome_attribution_window_seconds: 120

freshness:
  stale_after_days: 3
  # Router refuses to route to a model whose row is stale/deprecated,
  # regardless of how good its score would otherwise be.
  exclude_stale: true
  exclude_deprecated: true

database:
  path: "router.db"

classifier:
  # The local LLM that classifies each task before a cloud model answers it.
  # "Local" means your hardware, not necessarily this machine — the box with
  # the GPU is usually not the laptop you are typing on. It speaks plain
  # OpenAI-compatible chat completions, so point it wherever Ollama lives:
  #
  #   same machine    base_url: http://localhost:11434/v1
  #   over a VPN      base_url: http://<vpn-ip>:11434/v1
  #                   (serving host needs deploy/ollama-over-vpn.conf; Ollama
  #                    binds loopback-only by default and will refuse)
  #
  # Any OpenAI-compatible endpoint works, so a cloud model can classify too —
  # set api_key_env for one that checks a key. Worth knowing before assuming
  # local is the cheap option: on five prompts NeuralWatt's deepseek-v4-flash
  # classified in 1.02s mean against qwen3.5's 11.58s on an RTX 6000, agreed
  # with the label 5/5 against 2/4, and cost $0.093 per thousand calls. Local
  # inference is not free, it is unbilled.
  provider: "ollama"
  base_url: "http://localhost:11434/v1"
  # Unset means unauthenticated, which is the Ollama case. Name the env var
  # holding the key when the endpoint actually checks one.
  api_key_env: null
  model: "mistral-nemo:12b"   # must match a model `ollama list` reports
  # A cold Ollama took 43s to answer the first classification, which blew the
  # old 30s ceiling and returned 503 to the client. Warm it is ~5s. The
  # ceiling is for a cold model load, not the steady state.
  timeout_seconds: 120
  # Classification must be reproducible: at the default temperature the same
  # task was classified tier 2 then tier 1 on consecutive calls, which routed
  # it to two different models. Routing that changes under an identical
  # prompt is untraceable.
  temperature: 0
  # Hard cap on the classifier's generation, to bound a failure mode that
  # cascades on REASONING models: they emit a long chain of thought, blow past
  # timeout_seconds, and Ollama keeps generating after the client gives up AND
  # serializes per model, so one runaway request queues every later one behind
  # it. Measured on qwen3.5, which failed this way on 4 of 19 calls — each one
  # a 15s wait ending in a silent fallback.
  #
  # The current default (mistral-nemo) does not reason, so this does not bind
  # for it. Kept anyway: it costs nothing when unused and is the only thing
  # standing between a swapped-in reasoning model and that cascade. If you do
  # swap one in, note 256 was too tight — the trace consumed the whole budget
  # and the model was truncated before emitting any JSON.
  max_output_tokens: 1024

  # Where routing lands when the classifier times out, errors, or returns
  # something unparseable. A local model being slow should degrade routing,
  # not refuse the request — the caller is a coding agent that would rather
  # have a mid-tier answer than a 502.
  # Ceiling on the text handed to the classifier (head + tail, middle elided).
  # It decides a category and a tier; it does not need the document, and
  # feeding it one is harmful rather than merely wasteful. Measured on a ~20k
  # token prompt: qwen3.5 spent 28.7s and returned empty (budget consumed by
  # its reasoning trace), mistral-nemo spent 41.8s echoing the input back
  # inside its JSON. Both land on source: "fallback" — the same answer an
  # instant failure gives, after 30-40s of local inference.
  #
  # Nothing is lost: chat_completions measures the real conversation with
  # estimate_prompt_tokens and takes the larger value, so required_context
  # never depends on what the classifier saw. 0 disables clamping.
  max_input_chars: 8000

  # When the chat path supplies the previous turn as context (see the pinch /
  # context notes), frame the classifier input as "Context: <prev> / Message:
  # <current>" so a short follow-up inherits the prior turn's complexity
  # instead of being classified in isolation as trivial.
  context_framing: true

  fallback_tier: 2
  fallback_category: general_chat
  response_format: "json"       # ask Ollama to constrain output to valid JSON
  # The dispatcher appends the authoritative category list from
  # proficiency.categories to this prompt at call time. Do not enumerate the
  # categories here as well — a hand-copied list drifts, and a category the
  # model invents joins against nothing in the proficiency table.
  system_prompt: |
    You are a task router. Given a task description and any attached context,
    respond with ONLY a JSON object with these fields:
    {
      "task_category": one of the allowed categories listed below,
      "task_tier": integer 1-3, where 1 is cheap/simple, 2 is mid/general,
                    and 3 is frontier/high-stakes,
      "required_context_tokens": integer estimate of prompt+context token count,
      "confidence": float 0-1
    }

dispatch_providers:
  # NeuralWatt is the only provider. The dict shape and the models table's
  # (model_id, provider) key are kept so a second one can be added without a
  # migration.
  neuralwatt:
    base_url: "https://api.neuralwatt.com/v1"
    api_key_env: "NEURALWATT_API_KEY"

logging:
  # Whether to write a row to energy_observations for every completion. Off
  # means no cost accounting, no reference sweep and no /outcome attribution,
  # so leave it on unless you are debugging.
  #
  # There is no log_path. Nothing ever wrote a file -- the dispatcher prints to
  # stderr and the systemd unit hands that to the journal (`journalctl --user
  # -u llm-router -f`), so the setting named a destination that did not exist.
  log_energy_observations: true

  # Whether to write a row to route_decisions for every routing decision
  # (route | dispatch | chat | passthrough | local-vision). Off leaves the
  # monitoring TUI's decision history empty; it does not change what routes.
  log_route_decisions: true

  # debug | info | warning | error.
  #
  # info gives one line per request: what it was classified as, which model won,
  # what it cost, how long each stage took. debug adds why -- every candidate
  # that was dropped and by which filter, the ranking with scores, the
  # classifier's raw reply. It logs no conversation text at any level; prompts
  # here run 60k-150k tokens and the journal is on disk.
  #
  # LLM_ROUTER_LOG_LEVEL overrides this, so a running service can be turned up
  # without editing a tracked file:
  #
  #   systemctl --user edit llm-router      # Environment="LLM_ROUTER_LOG_LEVEL=debug"
  #   systemctl --user restart llm-router
  level: info