327 lines
12 KiB
Python
327 lines
12 KiB
Python
"""Gaming mode: ``local_compute.enabled`` as an outer gate over local hardware.
|
|
|
|
The operator stops Ollama to give the GPU back to a game. Without a flag the
|
|
router finds that out one ``classifier.timeout_seconds`` at a time, at every
|
|
local call site independently.
|
|
|
|
Two properties are asserted throughout, and they are different claims:
|
|
|
|
* **skip, not fail** — with the flag off a local call must not be attempted
|
|
and its error handled; the attempt is the expensive part. So the tests
|
|
assert on whether a client was *constructed*, not on the returned value.
|
|
* **refuse, not warn** — turning the mode on without a cloud classifier is
|
|
rejected, because the degradation it would cause (every request classified
|
|
by a static guess, recorded as the fully scored ``general_chat``) is
|
|
invisible afterwards.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
from fastapi import HTTPException
|
|
from fastapi.testclient import TestClient
|
|
from pydantic import ValidationError
|
|
|
|
import admin
|
|
import config as config_mod
|
|
import dispatcher
|
|
import session_cache
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
SCHEMA_SQL = (ROOT / "config" / "schema.sql").read_text()
|
|
|
|
|
|
def _now_iso() -> str:
|
|
from datetime import datetime, timezone
|
|
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def _cloud_fallback() -> SimpleNamespace:
|
|
return SimpleNamespace(
|
|
base_url="https://cloud.example/v1",
|
|
model="cloud-classifier",
|
|
timeout_seconds=2,
|
|
api_key_env=None,
|
|
max_output_tokens=1024,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def gaming_mode(monkeypatch):
|
|
"""Local compute off, with the cloud classifier the mode requires."""
|
|
session_cache.clear()
|
|
monkeypatch.setattr(dispatcher, "_last_classifier_failure", 0.0)
|
|
dispatcher._provider_refusal_since.clear()
|
|
monkeypatch.setattr(dispatcher.cfg.local_compute, "enabled", False)
|
|
monkeypatch.setattr(
|
|
dispatcher.cfg.classifier, "cloud_fallback", _cloud_fallback()
|
|
)
|
|
yield
|
|
dispatcher._provider_refusal_since.clear()
|
|
session_cache.clear()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The classifier: skipped without an attempt, and the cloud step takes over.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_classifier_is_skipped_without_being_dialled(gaming_mode, monkeypatch):
|
|
built: list[int] = []
|
|
monkeypatch.setattr(
|
|
dispatcher, "_classifier_client", lambda: built.append(1) or object()
|
|
)
|
|
monkeypatch.setattr(dispatcher, "_cloud_classifier_client", lambda cf: None)
|
|
|
|
got = dispatcher.classify("do a thing", None)
|
|
|
|
assert built == [], "local classifier dialled with local compute disabled"
|
|
assert got.source == "fallback"
|
|
|
|
|
|
def test_the_cloud_classifier_becomes_the_classifier(gaming_mode, monkeypatch):
|
|
"""And its source stays attributable, so proficiency keeps training.
|
|
|
|
``classifier_cloud`` is in ``report_outcome``'s attributable set, unlike
|
|
``fallback`` / ``session_stale`` / ``session_history``. If gaming mode
|
|
produced one of those instead, every outcome reported while the GPU was
|
|
busy would be recorded and then excluded from folding.
|
|
"""
|
|
monkeypatch.setattr(
|
|
dispatcher,
|
|
"_classifier_client",
|
|
lambda: pytest.fail("local classifier must not be constructed"),
|
|
)
|
|
monkeypatch.setattr(dispatcher, "_cloud_classifier_client", lambda cf: object())
|
|
monkeypatch.setattr(
|
|
session_cache,
|
|
"classify_one",
|
|
lambda *a, **k: {
|
|
"task_category": "reasoning_math",
|
|
"task_tier": 3,
|
|
"required_context_tokens": 1234,
|
|
"confidence": 0.8,
|
|
},
|
|
)
|
|
|
|
got = dispatcher.classify("do a thing", None)
|
|
|
|
assert got.source == "classifier_cloud"
|
|
assert got.task_category == "reasoning_math"
|
|
|
|
|
|
def test_free_cascade_steps_still_run_ahead_of_the_cloud_call(
|
|
gaming_mode, monkeypatch
|
|
):
|
|
"""A cached real classification of THIS session is not worth paying to redo.
|
|
|
|
"Force cloud" means the cloud replaces the local *model*, not that it
|
|
replaces free correct answers.
|
|
"""
|
|
cloud_calls: list[int] = []
|
|
monkeypatch.setattr(
|
|
dispatcher,
|
|
"_cloud_classifier_client",
|
|
lambda cf: cloud_calls.append(1) or object(),
|
|
)
|
|
monkeypatch.setattr(
|
|
dispatcher, "_classifier_client", lambda: pytest.fail("dialled local")
|
|
)
|
|
session_cache.put("sess-gaming", task_category="coding_refactor", task_tier=3)
|
|
token = dispatcher._current_session_key.set("sess-gaming")
|
|
try:
|
|
got = dispatcher.classify("do a thing", None)
|
|
finally:
|
|
dispatcher._current_session_key.reset(token)
|
|
|
|
assert got.source == "session_stale"
|
|
assert cloud_calls == [], "paid for a classification the session already held"
|
|
|
|
|
|
def test_gaming_mode_does_not_open_the_failure_circuit(gaming_mode, monkeypatch):
|
|
"""Nothing failed, so nothing should be backing off.
|
|
|
|
If gaming mode stamped the failure clock, turning the mode back off would
|
|
leave the local classifier suppressed for another cooldown window on top.
|
|
"""
|
|
monkeypatch.setattr(dispatcher, "_cloud_classifier_client", lambda cf: None)
|
|
monkeypatch.setattr(
|
|
dispatcher, "_classifier_client", lambda: pytest.fail("dialled local")
|
|
)
|
|
|
|
dispatcher.classify("do a thing", None)
|
|
|
|
assert dispatcher._last_classifier_failure == 0.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The other local call sites.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_local_dispatch_entries_are_not_dispatchable(gaming_mode, monkeypatch):
|
|
"""The config entry still exists; it is just not usable right now."""
|
|
entry = SimpleNamespace(
|
|
model_id="qwen2.5-coder-router:14b",
|
|
eligible_categories=["file_summarization"],
|
|
)
|
|
monkeypatch.setattr(dispatcher.cfg, "local_dispatch_models", [entry])
|
|
|
|
assert dispatcher._local_dispatch_entry_configured(entry.model_id) is entry
|
|
assert dispatcher._local_dispatch_config_for(entry.model_id) is None
|
|
assert dispatcher._local_dispatch_fallback_entry("file_summarization") is None
|
|
|
|
|
|
def test_local_dispatch_entries_are_dispatchable_when_enabled(monkeypatch):
|
|
"""The gate is the only thing suppressing them — a control for the above."""
|
|
entry = SimpleNamespace(
|
|
model_id="qwen2.5-coder-router:14b",
|
|
eligible_categories=["file_summarization"],
|
|
)
|
|
monkeypatch.setattr(dispatcher.cfg, "local_dispatch_models", [entry])
|
|
monkeypatch.setattr(dispatcher.cfg.local_compute, "enabled", True)
|
|
|
|
assert dispatcher._local_dispatch_config_for(entry.model_id) is entry
|
|
assert dispatcher._local_dispatch_fallback_entry("file_summarization") is entry
|
|
|
|
|
|
@pytest.fixture
|
|
def seeded_db(tmp_path, monkeypatch):
|
|
"""A catalog holding one cloud row and one active ollama-local row."""
|
|
path = tmp_path / "gaming.db"
|
|
conn = sqlite3.connect(path)
|
|
conn.row_factory = sqlite3.Row
|
|
conn.executescript(SCHEMA_SQL)
|
|
conn.executemany(
|
|
"""
|
|
INSERT INTO models
|
|
(model_id, provider, availability, access_level, tier,
|
|
context_window, effective_context_window, latency_class,
|
|
eligible_categories, last_updated)
|
|
VALUES (?, ?, 'active', 'public', 1, 128000, 128000, 'standard', ?, ?)
|
|
""",
|
|
[
|
|
# Timezone-aware, because scoring_coverage compares last_updated
|
|
# against an aware "now" and SQLite's datetime('now') is naive.
|
|
("gemma-4-31b", "neuralwatt", None, _now_iso()),
|
|
(
|
|
"qwen2.5-coder-router:14b",
|
|
"ollama-local",
|
|
"file_summarization,diff_checking",
|
|
_now_iso(),
|
|
),
|
|
],
|
|
)
|
|
conn.commit()
|
|
|
|
def _open():
|
|
c = sqlite3.connect(path)
|
|
c.row_factory = sqlite3.Row
|
|
return c
|
|
|
|
monkeypatch.setattr(dispatcher, "_db", _open)
|
|
yield conn
|
|
conn.close()
|
|
|
|
|
|
def test_local_rows_leave_the_candidate_set(gaming_mode, seeded_db):
|
|
"""Excluded at load, not selected-then-503'd at dispatch."""
|
|
ids = {
|
|
r["model_id"]
|
|
for r in dispatcher.load_candidates(seeded_db, "file_summarization")
|
|
}
|
|
assert ids == {"gemma-4-31b"}
|
|
|
|
|
|
def test_local_rows_are_candidates_when_local_compute_is_on(
|
|
seeded_db, monkeypatch
|
|
):
|
|
monkeypatch.setattr(dispatcher.cfg.local_compute, "enabled", True)
|
|
ids = {
|
|
r["model_id"]
|
|
for r in dispatcher.load_candidates(seeded_db, "file_summarization")
|
|
}
|
|
assert ids == {"gemma-4-31b", "qwen2.5-coder-router:14b"}
|
|
|
|
|
|
def test_v1_models_stops_advertising_local_rows(gaming_mode, seeded_db):
|
|
"""Listing a model that every pin against it will be refused for is a lie."""
|
|
client = TestClient(dispatcher.app)
|
|
ids = {m["id"] for m in client.get("/v1/models").json()["data"]}
|
|
assert "gemma-4-31b" in ids
|
|
assert "qwen2.5-coder-router:14b" not in ids
|
|
|
|
|
|
def test_health_reports_not_probed_rather_than_unreachable(gaming_mode, seeded_db):
|
|
"""False would mean "we asked and it did not answer". We did not ask."""
|
|
client = TestClient(dispatcher.app)
|
|
body = client.get("/health").json()
|
|
assert body["local_compute_enabled"] is False
|
|
assert body["classifier_reachable"] is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The refusal.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_config_load_refuses_gaming_mode_without_a_cloud_classifier():
|
|
"""A hand-edited config.yaml gets the same answer as the toggle."""
|
|
with pytest.raises(ValidationError) as exc:
|
|
config_mod.RouterConfig(
|
|
**_router_config_kwargs(local_compute_enabled=False, cloud=False)
|
|
)
|
|
assert "classifier.cloud_fallback" in str(exc.value)
|
|
|
|
|
|
def test_config_load_accepts_gaming_mode_with_a_cloud_classifier():
|
|
cfg = config_mod.RouterConfig(
|
|
**_router_config_kwargs(local_compute_enabled=False, cloud=True)
|
|
)
|
|
assert cfg.local_compute.enabled is False
|
|
|
|
|
|
def test_runtime_knob_refuses_gaming_mode_without_a_cloud_classifier(monkeypatch):
|
|
cfg = SimpleNamespace(classifier=SimpleNamespace(cloud_fallback=None))
|
|
with pytest.raises(HTTPException) as exc:
|
|
admin._require_cloud_classifier_for_gaming_mode(cfg)
|
|
assert exc.value.status_code == 409
|
|
assert "classifier.cloud_fallback" in exc.value.detail
|
|
|
|
|
|
def test_runtime_knob_allows_gaming_mode_with_a_cloud_classifier():
|
|
cfg = SimpleNamespace(classifier=SimpleNamespace(cloud_fallback=_cloud_fallback()))
|
|
admin._require_cloud_classifier_for_gaming_mode(cfg) # must not raise
|
|
|
|
|
|
def test_turning_local_compute_back_on_is_never_refused():
|
|
"""The refusal guards the degraded direction only."""
|
|
assert admin._BOOL_KNOBS["local_compute_enabled"] == ("local_compute", "enabled")
|
|
|
|
|
|
def _router_config_kwargs(*, local_compute_enabled: bool, cloud: bool) -> dict:
|
|
"""A minimal-but-valid RouterConfig payload, built from the shipped file.
|
|
|
|
Reads config/config.yaml rather than hand-rolling 21 sections, so this
|
|
stays honest when a required section is added — and mutates only the two
|
|
keys under test.
|
|
"""
|
|
import yaml
|
|
|
|
store = dict(yaml.safe_load((ROOT / "config" / "config.yaml").read_text()))
|
|
store["local_compute"] = {"enabled": local_compute_enabled}
|
|
classifier = dict(store["classifier"])
|
|
if cloud:
|
|
classifier["cloud_fallback"] = {
|
|
"base_url": "https://cloud.example/v1",
|
|
"model": "cloud-classifier",
|
|
}
|
|
else:
|
|
classifier.pop("cloud_fallback", None)
|
|
store["classifier"] = classifier
|
|
return store
|