91c739c2af
Add tests/test_allowlist.py and tests/test_admin_allowlist.py covering config validation, poller filter/prune behavior, admin CRUD, and integration scenarios A/B/C plus NeuralWatt regression. Also set conn.row_factory = sqlite3.Row in poller.main() so the new allowlist SELECT returns Row objects, matching the rest of the codebase.
288 lines
9.7 KiB
Python
288 lines
9.7 KiB
Python
"""Tests for the admin allowlist CRUD and catalog-preview endpoints.
|
|
|
|
Uses ``build_router(cfg, _db_callable, base_dir)`` with a temp DB and a copy
|
|
of the real config.yaml so the merged-provider lookups see openrouter with
|
|
``require_allowlist: true``. No real provider calls are made.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import shutil
|
|
import sqlite3
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
from fastapi import FastAPI
|
|
from starlette.testclient import TestClient
|
|
|
|
import admin
|
|
from admin import build_router
|
|
from config import load_config
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
SCHEMA_SQL = (ROOT / "config" / "schema.sql").read_text()
|
|
|
|
|
|
def _make_db(tmp_path: Path) -> sqlite3.Connection:
|
|
conn = sqlite3.connect(str(tmp_path / "test.db"))
|
|
conn.row_factory = sqlite3.Row
|
|
return conn
|
|
|
|
|
|
@pytest.fixture
|
|
def client(tmp_path, monkeypatch):
|
|
"""A TestClient for an isolated admin router over a temp config copy."""
|
|
(tmp_path / "config").mkdir(parents=True, exist_ok=True)
|
|
config_yaml = tmp_path / "config" / "config.yaml"
|
|
shutil.copyfile(ROOT / "config" / "config.yaml", config_yaml)
|
|
|
|
schema_sql = (ROOT / "config" / "schema.sql").read_text()
|
|
conn = _make_db(tmp_path)
|
|
conn.executescript(schema_sql)
|
|
conn.close()
|
|
|
|
cfg = load_config(str(config_yaml))
|
|
|
|
def _db_factory() -> sqlite3.Connection:
|
|
return _make_db(tmp_path)
|
|
|
|
router = build_router(cfg, _db_factory, base_dir=str(tmp_path))
|
|
app = FastAPI()
|
|
app.include_router(router, prefix="/admin")
|
|
|
|
monkeypatch.setattr(
|
|
admin.poller,
|
|
"fetch_openrouter",
|
|
lambda name: [
|
|
MagicMock(
|
|
model_id="google/test-model",
|
|
display_name="Test Model",
|
|
cost_per_1m_prompt=0.5,
|
|
cost_per_1m_completion=1.0,
|
|
context_window=131072,
|
|
supports_tools=True,
|
|
supports_json_mode=True,
|
|
supports_vision=False,
|
|
supports_reasoning=False,
|
|
access_level="public",
|
|
pricing_tbd=False,
|
|
)
|
|
],
|
|
)
|
|
|
|
with TestClient(app) as test_client:
|
|
yield test_client
|
|
|
|
|
|
@pytest.fixture
|
|
def list_client(tmp_path, monkeypatch):
|
|
"""Client seeded with two allowlist rows for openrouter."""
|
|
(tmp_path / "config").mkdir(parents=True, exist_ok=True)
|
|
config_yaml = tmp_path / "config" / "config.yaml"
|
|
shutil.copyfile(ROOT / "config" / "config.yaml", config_yaml)
|
|
|
|
conn = _make_db(tmp_path)
|
|
conn.executescript(SCHEMA_SQL)
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
for mid in ("mistralai/codestral-2508", "qwen/qwen3-coder"):
|
|
conn.execute(
|
|
"INSERT INTO provider_model_allowlist (provider, model_id, added_at, note) "
|
|
"VALUES (?, ?, ?, ?)",
|
|
("openrouter", mid, now, "seed"),
|
|
)
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
cfg = load_config(str(config_yaml))
|
|
|
|
def _db_factory() -> sqlite3.Connection:
|
|
return _make_db(tmp_path)
|
|
|
|
router = build_router(cfg, _db_factory, base_dir=str(tmp_path))
|
|
app = FastAPI()
|
|
app.include_router(router, prefix="/admin")
|
|
|
|
monkeypatch.setattr(
|
|
admin.poller,
|
|
"fetch_openrouter",
|
|
lambda name: [
|
|
MagicMock(
|
|
model_id="google/test-model",
|
|
display_name="Test Model",
|
|
cost_per_1m_prompt=0.5,
|
|
cost_per_1m_completion=1.0,
|
|
context_window=131072,
|
|
supports_tools=True,
|
|
supports_json_mode=True,
|
|
supports_vision=False,
|
|
supports_reasoning=False,
|
|
access_level="public",
|
|
pricing_tbd=False,
|
|
)
|
|
],
|
|
)
|
|
|
|
with TestClient(app) as test_client:
|
|
yield test_client
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# GET /admin/api/providers/{name}/allowlist
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_allowlist_get_empty(client):
|
|
"""GET returns an empty list when the allowlist table has no rows."""
|
|
resp = client.get("/admin/api/providers/openrouter/allowlist")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == []
|
|
|
|
|
|
def test_allowlist_get_populated(list_client):
|
|
"""GET returns seeded allowlist rows sorted by model_id."""
|
|
resp = list_client.get("/admin/api/providers/openrouter/allowlist")
|
|
assert resp.status_code == 200
|
|
rows = resp.json()
|
|
assert len(rows) == 2
|
|
assert rows[0]["model_id"] == "mistralai/codestral-2508"
|
|
assert rows[1]["model_id"] == "qwen/qwen3-coder"
|
|
for row in rows:
|
|
assert "added_at" in row
|
|
assert "note" in row
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# POST /admin/api/providers/{name}/allowlist
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_allowlist_post_success(client):
|
|
"""POST adds a model to the openrouter allowlist."""
|
|
resp = client.post(
|
|
"/admin/api/providers/openrouter/allowlist",
|
|
json={"model_id": "z-ai/glm-5.2", "note": "cost workhorse"},
|
|
)
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
assert body["model_id"] == "z-ai/glm-5.2"
|
|
assert body["provider"] == "openrouter"
|
|
assert body["note"] == "cost workhorse"
|
|
assert "added_at" in body
|
|
|
|
# Idempotent visible through GET
|
|
resp = client.get("/admin/api/providers/openrouter/allowlist")
|
|
assert resp.status_code == 200
|
|
assert [r["model_id"] for r in resp.json()] == ["z-ai/glm-5.2"]
|
|
|
|
|
|
def test_allowlist_post_provider_without_allowlist_returns_409(client):
|
|
"""POST to neuralwatt (require_allowlist=false) returns 409."""
|
|
resp = client.post(
|
|
"/admin/api/providers/neuralwatt/allowlist",
|
|
json={"model_id": "some/model"},
|
|
)
|
|
assert resp.status_code == 409
|
|
assert "allowlist is not required" in resp.json()["detail"]
|
|
|
|
|
|
def test_allowlist_post_unknown_provider_returns_404(client):
|
|
"""POST to an unknown provider returns 404 before any 409."""
|
|
resp = client.post(
|
|
"/admin/api/providers/nonexistent/allowlist",
|
|
json={"model_id": "some/model"},
|
|
)
|
|
assert resp.status_code == 404
|
|
assert "provider not found" in resp.json()["detail"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# DELETE /admin/api/providers/{name}/allowlist/{model_id}
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_allowlist_delete_removes_row(list_client):
|
|
"""DELETE removes an existing allowlist row."""
|
|
resp = list_client.delete(
|
|
"/admin/api/providers/openrouter/allowlist/qwen/qwen3-coder"
|
|
)
|
|
assert resp.status_code == 200
|
|
assert resp.json() == {"deleted": "qwen/qwen3-coder", "provider": "openrouter"}
|
|
|
|
rows = list_client.get("/admin/api/providers/openrouter/allowlist").json()
|
|
assert [r["model_id"] for r in rows] == ["mistralai/codestral-2508"]
|
|
|
|
|
|
def test_allowlist_delete_idempotent(list_client):
|
|
"""DELETE on an already-absent row still returns 200."""
|
|
resp = list_client.delete(
|
|
"/admin/api/providers/openrouter/allowlist/qwen/qwen3-coder"
|
|
)
|
|
assert resp.status_code == 200
|
|
|
|
resp = list_client.delete(
|
|
"/admin/api/providers/openrouter/allowlist/qwen/qwen3-coder"
|
|
)
|
|
assert resp.status_code == 200
|
|
assert resp.json() == {"deleted": "qwen/qwen3-coder", "provider": "openrouter"}
|
|
|
|
rows = list_client.get("/admin/api/providers/openrouter/allowlist").json()
|
|
assert [r["model_id"] for r in rows] == ["mistralai/codestral-2508"]
|
|
|
|
|
|
def test_allowlist_delete_slash_and_colon_id(list_client):
|
|
"""DELETE works for vendor/model ids and for ids containing a colon."""
|
|
list_client.post(
|
|
"/admin/api/providers/openrouter/allowlist",
|
|
json={"model_id": "z-ai/glm-5.2:free", "note": "free tier"},
|
|
)
|
|
resp = list_client.delete(
|
|
"/admin/api/providers/openrouter/allowlist/z-ai/glm-5.2:free"
|
|
)
|
|
assert resp.status_code == 200
|
|
assert resp.json() == {"deleted": "z-ai/glm-5.2:free", "provider": "openrouter"}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# GET /admin/api/providers/{name}/catalog-preview
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_catalog_preview_returns_json_array(client):
|
|
"""catalog-preview returns a JSON array of ModelRow-derived dicts."""
|
|
resp = client.get("/admin/api/providers/openrouter/catalog-preview")
|
|
assert resp.status_code == 200
|
|
rows = resp.json()
|
|
assert isinstance(rows, list)
|
|
assert len(rows) == 1
|
|
row = rows[0]
|
|
assert row["model_id"] == "google/test-model"
|
|
assert row["display_name"] == "Test Model"
|
|
assert row["cost_per_1m_prompt"] == 0.5
|
|
assert row["supports_tools"] is True
|
|
|
|
|
|
def test_catalog_preview_non_allowlist_provider_returns_409(client):
|
|
"""A provider without require_allowlist rejects preview with 409."""
|
|
resp = client.get("/admin/api/providers/neuralwatt/catalog-preview")
|
|
assert resp.status_code == 409
|
|
assert "allowlist is not required" in resp.json()["detail"]
|
|
|
|
|
|
def test_catalog_preview_allowlisted_non_openrouter_returns_400(client, tmp_path):
|
|
"""A provider that requires allowlist but is not openrouter gets 400."""
|
|
local_yaml = tmp_path / "config" / "config.local.yaml"
|
|
local_yaml.write_text(
|
|
"dispatch_providers:\n"
|
|
" otherprovider:\n"
|
|
" base_url: https://example.com/v1\n"
|
|
" api_key_env: OTHER_API_KEY\n"
|
|
" enabled: true\n"
|
|
" require_allowlist: true\n"
|
|
)
|
|
|
|
resp = client.get("/admin/api/providers/otherprovider/catalog-preview")
|
|
assert resp.status_code == 400
|
|
assert "only supported for openrouter" in resp.json()["detail"]
|