fix(classifier): validate confidence_threshold range, take percent in the UI #47

Merged
alee merged 1 commits from fix/confidence-threshold-percent-ui into main 2026-09-06 23:36:48 +00:00
4 changed files with 74 additions and 6 deletions

View File

@@ -877,9 +877,12 @@ function classifierModeFieldsHtml(mode, data) {
</select>
</div>
<div class="col-md-3">
<input class="form-control form-control-sm" type="number" step="0.05" min="0" max="1"
id="classifier-encoder-threshold" placeholder="confidence_threshold"
value="${enc.confidence_threshold != null ? enc.confidence_threshold : ''}">
<div class="input-group input-group-sm">
<input class="form-control form-control-sm" type="number" step="1" min="0" max="100"
id="classifier-encoder-threshold" placeholder="confidence % (0-100)"
value="${enc.confidence_threshold != null ? Math.round(enc.confidence_threshold * 100) : ''}">
<span class="input-group-text">%</span>
</div>
</div>
</div>`;
}
@@ -967,8 +970,10 @@ function collectClassifierConfigBody() {
const model = document.getElementById('classifier-encoder-model').value.trim();
if (model) encoder.model = model;
encoder.device = document.getElementById('classifier-encoder-device').value;
const threshold = document.getElementById('classifier-encoder-threshold').value;
if (threshold !== '') encoder.confidence_threshold = parseFloat(threshold);
const thresholdPct = document.getElementById('classifier-encoder-threshold').value;
// The field is 0-100 for a human to type ("80" meaning 80%); the backend
// wants the 0.0-1.0 probability classify_zero_shot actually returns.
if (thresholdPct !== '') encoder.confidence_threshold = parseFloat(thresholdPct) / 100;
body.encoder = encoder;
}
return body;

View File

@@ -796,9 +796,27 @@ class LocalEncoderConfig(StrictModel):
# Below this, the classification is treated as a FAILURE, not a low-
# confidence answer -- the caller cascades exactly as it would for a
# local-LLM parse failure, rather than confidently mis-routing on a
# guess the encoder itself was unsure about.
# guess the encoder itself was unsure about. classify_zero_shot returns
# a 0.0-1.0 probability, so this must be too -- a percent-style value
# (e.g. 80 meaning "80%") silently makes every real confidence score
# read as below-threshold, since no probability exceeds 1.0. Caught live
# 2026-09-06: the admin UI took a raw number with no conversion or
# bound, so typing the intuitive "80" broke classification on every
# request. The UI now converts 0-100 to 0.0-1.0 before saving; this
# validator is the fail-closed backstop for any other caller.
confidence_threshold: float = 0.5
@field_validator("confidence_threshold")
@classmethod
def confidence_threshold_in_range(cls, v: float) -> float:
if not (0.0 <= v <= 1.0):
raise ValueError(
f"classifier.encoder.confidence_threshold must be in [0.0, 1.0], "
f"got {v!r} -- classify_zero_shot returns a 0-1 probability, "
f"not a percent"
)
return v
class ClassifierConfig(StrictModel):
provider: str

View File

@@ -133,6 +133,21 @@ def test_admin_controls_has_a_dedicated_classifier_card(admin_client):
assert "(baseUrl && model) ? { base_url: baseUrl, model } : null" in text
def test_admin_controls_confidence_threshold_field_is_percent_with_conversion(admin_client):
"""The encoder confidence_threshold field takes 0-100 (a human types "80"
meaning 80%) and converts to the 0.0-1.0 probability classify_zero_shot
actually returns -- typing the intuitive percent value used to be stored
literally, so no real confidence score could ever clear the threshold
and every classification silently failed (live 2026-09-06)."""
resp = admin_client.get("/admin/controls")
text = resp.text
assert 'id="classifier-encoder-threshold"' in text
assert 'max="100"' in text
assert "parseFloat(thresholdPct) / 100" in text
# Loading back a stored 0.0-1.0 value must display it as a percent.
assert "Math.round(enc.confidence_threshold * 100)" in text
def test_admin_models_returns_html_with_availability_marker(admin_client):
"""GET /admin/models returns 200, text/html, and contains the Model
Availability card title."""

View File

@@ -139,6 +139,36 @@ def test_local_encoder_rejects_unknown_device(raw):
RouterConfig(**cfg)
def test_local_encoder_rejects_percent_style_confidence_threshold(raw):
"""classify_zero_shot returns a 0.0-1.0 probability, so a percent-style
value (e.g. 80 meaning "80%") must be rejected -- otherwise no real
confidence score can ever clear the threshold and every classification
silently fails. Caught live 2026-09-06 via the admin UI taking a raw
number with no conversion."""
cfg = copy.deepcopy(raw)
cfg["classifier"]["mode"] = "local_encoder"
cfg["classifier"]["encoder"] = {"confidence_threshold": 80}
with pytest.raises(ValueError, match=r"must be in \[0.0, 1.0\]"):
RouterConfig(**cfg)
def test_local_encoder_rejects_negative_confidence_threshold(raw):
cfg = copy.deepcopy(raw)
cfg["classifier"]["mode"] = "local_encoder"
cfg["classifier"]["encoder"] = {"confidence_threshold": -0.1}
with pytest.raises(ValueError, match=r"must be in \[0.0, 1.0\]"):
RouterConfig(**cfg)
def test_local_encoder_accepts_confidence_threshold_at_bounds(raw):
cfg = copy.deepcopy(raw)
cfg["classifier"]["mode"] = "local_encoder"
cfg["classifier"]["encoder"] = {"confidence_threshold": 0.0}
assert RouterConfig(**cfg).classifier.encoder.confidence_threshold == 0.0
cfg["classifier"]["encoder"] = {"confidence_threshold": 1.0}
assert RouterConfig(**cfg).classifier.encoder.confidence_threshold == 1.0
def test_local_encoder_unaffected_by_the_cloud_llm_validator(raw):
"""A local_encoder config leaving cloud_primary/auto both unset must not
trip the cloud_llm validator -- it's scoped to mode == cloud_llm."""