6krrt/tests/test_task_set.py

"""Validate the eval task set against reference solutions.

A check the task author cannot satisfy is a broken check, and it scores the
task set rather than the model. That has already happened here: a sequence
task asked for "the next number" and "the 11th term" in one breath, and every
model that read it correctly scored zero.

So every `code` task gets a reference implementation below, and every check
must pass against it. Every `exact` answer is recomputed rather than trusted.
These tests run offline and take milliseconds — they are the cheap guard
against spending an hour of API calls measuring a typo.
"""

import re
import textwrap
from pathlib import Path

import pytest
import yaml

from eval_proficiency import score_code, score_exact, score_tool

TASKS = yaml.safe_load((Path(__file__).resolve().parent.parent / "evals" / "tasks.yaml").read_text())["tasks"]
BY_ID = {t["id"]: t for t in TASKS}


# --- reference solutions --------------------------------------------------

REFERENCES = {
    "merge_intervals": '''
def merge_intervals(intervals):
    if not intervals:
        return []
    ordered = sorted(intervals, key=lambda iv: iv[0])
    out = [list(ordered[0])]
    for start, end in ordered[1:]:
        if start <= out[-1][1]:
            out[-1][1] = max(out[-1][1], end)
        else:
            out.append([start, end])
    return out
''',
    "parse_semver": r'''
import re
_SEMVER = re.compile(
    r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
    r"(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)"
    r"(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?"
    r"(?:\+(?P<build>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
)

def parse_semver(version):
    m = _SEMVER.match(version or "")
    if not m:
        raise ValueError(version)
    d = m.groupdict()
    return {
        "major": int(d["major"]),
        "minor": int(d["minor"]),
        "patch": int(d["patch"]),
        "prerelease": d["prerelease"],
        "build": d["build"],
    }
''',
    "word_wrap": '''
def word_wrap(text, width):
    words = text.split()
    if not words:
        return []
    lines, current = [], words[0]
    for word in words[1:]:
        if len(current) + 1 + len(word) <= width:
            current += " " + word
        else:
            lines.append(current)
            current = word
    lines.append(current)
    return lines
''',
    "refactor_falsy_defaults": '''
_DEFAULTS = {"retries": 3, "timeout": 30, "verbose": False}

def apply_settings(overrides):
    return {k: overrides.get(k, v) for k, v in _DEFAULTS.items()}
''',
    "refactor_first_match": '''
def first_match(items, predicates):
    return next(
        (item for item in items if any(p(item) for p in predicates)), None
    )
''',
    "refactor_dispatch": '''
_TABLE = {200: "ok", 201: "created", 404: "not found", 500: "server error"}

def describe(code):
    return _TABLE.get(code, "unknown")
''',
    "refactor_bowling_frames": '''
class Frame:
    def __init__(self, idx):
        self.idx = idx
        self.throws = []

    @property
    def total_pins(self):
        return sum(self.throws)

    def is_strike(self):
        return self.total_pins == 10 and len(self.throws) == 1

    def is_spare(self):
        return self.total_pins == 10 and len(self.throws) == 2

    def is_open(self):
        return self.total_pins < 10 and len(self.throws) == 2

    def is_closed(self):
        return self.total_pins == 10 or len(self.throws) == 2

    def throw(self, pins):
        if self.total_pins + pins > 10:
            raise ValueError("a frame's rolls cannot exceed 10")
        self.throws.append(pins)

    def score(self, next_throws):
        result = self.total_pins
        if self.is_strike():
            result += sum(next_throws[:2])
        elif self.is_spare():
            result += sum(next_throws[:1])
        return result


class BowlingGame:
    def __init__(self):
        self.current_frame_idx = 0
        self.bonus_throws = []
        self.frames = [Frame(idx) for idx in range(10)]

    @property
    def current_frame(self):
        return self.frames[self.current_frame_idx]

    def next_throws(self, frame_idx):
        throws = []
        for idx in range(frame_idx + 1, 10):
            throws.extend(self.frames[idx].throws)
        throws.extend(self.bonus_throws)
        return throws

    def roll_bonus(self, pins):
        tenth_frame = self.frames[-1]
        if tenth_frame.is_open():
            raise IndexError('cannot throw bonus with an open tenth frame')
        self.bonus_throws.append(pins)
        if (len(self.bonus_throws) == 2 and self.bonus_throws[0] != 10 and
                sum(self.bonus_throws) > 10):
            raise ValueError('invalid fill balls')
        if tenth_frame.is_strike() and len(self.bonus_throws) > 2:
            raise IndexError(
                'wrong number of fill balls when the tenth frame is a strike')
        elif tenth_frame.is_spare() and len(self.bonus_throws) > 1:
            raise IndexError(
                'wrong number of fill balls when the tenth frame is a spare')

    def roll(self, pins):
        if not 0 <= pins <= 10:
            raise ValueError('invalid pins')
        elif self.current_frame_idx == 10:
            self.roll_bonus(pins)
        else:
            self.current_frame.throw(pins)
            if self.current_frame.is_closed():
                self.current_frame_idx += 1

    def score(self):
        if self.current_frame_idx < 10:
            raise IndexError('frame less than 10')
        if self.frames[-1].is_spare() and len(self.bonus_throws) != 1:
            raise IndexError(
                'one bonus must be rolled when the tenth frame is spare')
        if self.frames[-1].is_strike() and len(self.bonus_throws) != 2:
            raise IndexError(
                'two bonuses must be rolled when the tenth frame is strike')
        return sum(frame.score(self.next_throws(frame.idx))
                   for frame in self.frames)
''',
    "debug_bowling_tenth_frame": '''
class Frame:
    def __init__(self, idx):
        self.idx = idx
        self.throws = []

    @property
    def total_pins(self):
        return sum(self.throws)

    def is_strike(self):
        return self.total_pins == 10 and len(self.throws) == 1

    def is_spare(self):
        return self.total_pins == 10 and len(self.throws) == 2

    def is_open(self):
        return self.total_pins < 10 and len(self.throws) == 2

    def is_closed(self):
        return self.total_pins == 10 or len(self.throws) == 2

    def throw(self, pins):
        if self.total_pins + pins > 10:
            raise ValueError("a frame's rolls cannot exceed 10")
        self.throws.append(pins)

    def score(self, next_throws):
        result = self.total_pins
        if self.is_strike():
            result += sum(next_throws[:2])
        elif self.is_spare():
            result += sum(next_throws[:1])
        return result


class BowlingGame:
    def __init__(self):
        self.current_frame_idx = 0
        self.bonus_throws = []
        self.frames = [Frame(idx) for idx in range(10)]

    @property
    def current_frame(self):
        return self.frames[self.current_frame_idx]

    def next_throws(self, frame_idx):
        throws = []
        for idx in range(frame_idx + 1, 10):
            throws.extend(self.frames[idx].throws)
        throws.extend(self.bonus_throws)
        return throws

    def roll_bonus(self, pins):
        tenth_frame = self.frames[-1]
        if tenth_frame.is_open():
            raise IndexError('cannot throw bonus with an open tenth frame')
        self.bonus_throws.append(pins)
        if (len(self.bonus_throws) == 2 and self.bonus_throws[0] != 10 and
                sum(self.bonus_throws) > 10):
            raise ValueError('invalid fill balls')
        if tenth_frame.is_strike() and len(self.bonus_throws) > 2:
            raise IndexError(
                'wrong number of fill balls when the tenth frame is a strike')
        elif tenth_frame.is_spare() and len(self.bonus_throws) > 1:
            raise IndexError(
                'wrong number of fill balls when the tenth frame is a spare')

    def roll(self, pins):
        if not 0 <= pins <= 10:
            raise ValueError('invalid pins')
        elif self.current_frame_idx == 10:
            self.roll_bonus(pins)
        else:
            self.current_frame.throw(pins)
            if self.current_frame.is_closed():
                self.current_frame_idx += 1

    def score(self):
        if self.current_frame_idx < 10:
            raise IndexError('frame less than 10')
        if self.frames[-1].is_spare() and len(self.bonus_throws) != 1:
            raise IndexError(
                'one bonus must be rolled when the tenth frame is spare')
        if self.frames[-1].is_strike() and len(self.bonus_throws) != 2:
            raise IndexError(
                'two bonuses must be rolled when the tenth frame is strike')
        return sum(frame.score(self.next_throws(frame.idx))
                   for frame in self.frames)
''',
    "debug_dominoes_no_chain": '''
from itertools import permutations
from functools import reduce


def swap(item_1, item_2):
    return (item_2, item_1)


def build_chain(chain, domino):
    if chain is not None:
        last = chain[-1]
        if len(chain) == 1 and last[0] == domino[0]:
            return [swap(*last), domino]
        elif len(chain) == 1 and last[0] == domino[1]:
            return [swap(*last), swap(*domino)]
        elif last[1] == domino[0]:
            return chain + [domino]
        elif last[1] == domino[1]:
            return chain + [swap(*domino)]
    return None


def can_chain(dominoes):
    if not any(dominoes):
        return []
    for perm in permutations(dominoes):
        chain = reduce(build_chain, perm[1:], [perm[0]])
        if chain is not None and chain[0][0] == chain[-1][1]:
            return chain
    return None
''',
    "refactor_dominoes_chain": '''
from itertools import permutations


def can_chain(dominoes):
    if not any(dominoes):
        return []
    for perm in permutations(dominoes):
        chain = [perm[0]]
        complete = True
        for domino in perm[1:]:
            prev = chain[-1]
            if len(chain) == 1 and prev[0] == domino[0]:
                chain = [(prev[1], prev[0]), domino]
            elif len(chain) == 1 and prev[0] == domino[1]:
                chain = [(prev[1], prev[0]), (domino[1], domino[0])]
            elif prev[1] == domino[0]:
                chain = chain + [domino]
            elif prev[1] == domino[1]:
                chain = chain + [(domino[1], domino[0])]
            else:
                complete = False
                break
        if complete and chain[0][0] == chain[-1][1]:
            return chain
    return None
''',
    "refactor_affine_encode_decode": '''
BLOCK_SIZE = 5
ALPHABET = 26


def mod_inverse(a_key, alphabet):
    a_key = a_key % alphabet
    for idx in range(1, alphabet):
        if (a_key * idx) % alphabet == 1:
            return idx
    return 1


def translate(text, a_key, b_key, mode):
    inverse = mod_inverse(a_key, ALPHABET)
    if inverse == 1:
        raise ValueError('a and m must be coprime.')
    chars = []
    for character in text:
        if character.isalnum():
            origin = ord(character.lower()) - 97
            if origin < 0:
                chars.append(character)
                continue
            if mode == 0:
                new = (a_key * origin + b_key) % ALPHABET
            elif mode == 1:
                new = (inverse * (origin - b_key)) % ALPHABET
            chars.append(chr(new + 97))
    return ''.join(chars)


def encode(plain, a, b):
    cipher = translate(plain, a, b, 0)
    return ' '.join([cipher[idx:idx + BLOCK_SIZE]
                     for idx in range(0, len(cipher), BLOCK_SIZE)])


def decode(ciphered, a, b):
    return translate(ciphered, a, b, 1)
''',
    "debug_affine_coprime": '''
BLOCK_SIZE = 5
ALPHABET = 26


def mod_inverse(a_key, alphabet):
    a_key = a_key % alphabet
    for idx in range(1, alphabet):
        if (a_key * idx) % alphabet == 1:
            return idx
    return 1


def translate(text, a_key, b_key, mode):
    inverse = mod_inverse(a_key, ALPHABET)
    if inverse == 1:
        raise ValueError('a and m must be coprime.')
    chars = []
    for character in text:
        if character.isalnum():
            origin = ord(character.lower()) - 97
            if origin < 0:
                chars.append(character)
                continue
            if mode == 0:
                new = (a_key * origin + b_key) % ALPHABET
            elif mode == 1:
                new = (inverse * (origin - b_key)) % ALPHABET
            chars.append(chr(new + 97))
    return ''.join(chars)


def encode(plain, a, b):
    cipher = translate(plain, a, b, 0)
    return ' '.join([cipher[idx:idx + BLOCK_SIZE]
                     for idx in range(0, len(cipher), BLOCK_SIZE)])


def decode(ciphered, a, b):
    return translate(ciphered, a, b, 1)
''',
    "debug_late_binding": '''
def make_multipliers(factors):
    return [lambda x, f=f: x * f for f in factors]
''',
    "debug_binary_search": '''
def bsearch(items, target):
    lo, hi = 0, len(items)
    while lo < hi:
        mid = (lo + hi) // 2
        if items[mid] == target:
            return mid
        elif items[mid] < target:
            lo = mid + 1
        else:
            hi = mid
    return -1
''',
    "debug_greedy_regex": '''
import re

def extract_tags(text):
    return re.findall(r"<([^<>]+)>", text)
''',
}


@pytest.mark.parametrize("task_id", sorted(REFERENCES))
def test_reference_solution_passes_every_check(task_id):
    task = BY_ID[task_id]
    score, detail = score_code(REFERENCES[task_id], task["checks"])
    assert score == 1.0, f"{task_id}: reference scored {score} ({detail})"


def test_every_code_task_has_a_reference():
    # A code task with no reference has never been validated, so its checks
    # could be wrong in exactly the way that costs an hour of API calls
    code_tasks = {t["id"] for t in TASKS if t["kind"] == "code"}
    assert code_tasks == set(REFERENCES), (
        f"unvalidated: {sorted(code_tasks - set(REFERENCES))}"
    )


# --- the buggy code really is buggy ---------------------------------------

BUGGY = {
    "refactor_first_match": '''
def first_match(items, predicates):
    found = None
    done = False
    for item in items:
        if done:
            break
        for p in predicates:
            if p(item):
                found = item
                done = True
                break
    return found
''',
    "refactor_bowling_frames": '''
class BowlingGame:
    def __init__(self):
        self._frames = []
        self._current = 0
        self._bonus = []

    def roll(self, pins):
        if not (0 <= pins <= 10):
            raise ValueError('invalid pins')
        if self._current < 10:
            if len(self._frames) == self._current:
                self._frames.append([pins])
            else:
                self._frames[self._current].append(pins)
            current = self._frames[self._current]
            if sum(current) > 10:
                raise ValueError("a frame's rolls cannot exceed 10")
            strike = (len(current) == 1 and current[0] == 10)
            if strike or len(current) == 2:
                self._current += 1
        else:
            last = self._frames[-1]
            strike10 = len(last) == 1 and last[0] == 10
            spare10 = len(last) == 2 and sum(last) == 10
            if not (strike10 or spare10):
                raise IndexError('cannot throw bonus with an open tenth frame')
            if strike10:
                if len(self._bonus) >= 2:
                    raise IndexError(
                        'wrong number of fill balls when the tenth frame is a strike')
                self._bonus.append(pins)
                if len(self._bonus) == 2 and self._bonus[0] != 10 and sum(self._bonus) > 10:
                    raise ValueError('invalid fill balls')
                if len(self._bonus) > 2:
                    raise IndexError(
                        'wrong number of fill balls when the tenth frame is a strike')
            elif spare10:
                if len(self._bonus) >= 1:
                    raise IndexError(
                        'wrong number of fill balls when the tenth frame is a spare')
                self._bonus.append(pins)

    def score(self):
        if self._current < 10:
            raise IndexError('frame less than 10')
        last = self._frames[-1]
        if len(last) == 2 and sum(last) == 10 and len(self._bonus) != 1:
            raise IndexError('one bonus must be rolled when the tenth frame is spare')
        if len(last) == 1 and last[0] == 10 and len(self._bonus) != 2:
            raise IndexError('two bonuses must be rolled when the tenth frame is strike')
        total = 0
        for i in range(10):
            frame = self._frames[i]
            frame_sum = sum(frame)
            strike = (len(frame) == 1 and frame[0] == 10)
            spare = (len(frame) == 2 and frame_sum == 10)
            if strike or spare:
                nxt = []
                for j in range(i + 1, 10):
                    nxt.extend(self._frames[j])
                nxt.extend(self._bonus)
                if strike:
                    frame_sum += sum(nxt[:2])
                else:
                    frame_sum += sum(nxt[:1])
            total += frame_sum
        return total
''',
    "debug_bowling_tenth_frame": '''
class Frame:
    def __init__(self, idx):
        self.idx = idx
        self.throws = []

    @property
    def total_pins(self):
        return sum(self.throws)

    def is_strike(self):
        return self.total_pins == 10 and len(self.throws) == 1

    def is_spare(self):
        return self.total_pins == 10 and len(self.throws) == 2

    def is_open(self):
        return self.total_pins < 10 and len(self.throws) == 2

    def is_closed(self):
        return self.total_pins == 10 or len(self.throws) == 2

    def throw(self, pins):
        if self.total_pins + pins > 10:
            raise ValueError("a frame's rolls cannot exceed 10")
        self.throws.append(pins)

    def score(self, next_throws):
        result = self.total_pins
        if self.is_strike():
            result += sum(next_throws[:2])
        elif self.is_spare():
            result += sum(next_throws[:1])
        return result


class BowlingGame:
    def __init__(self):
        self.current_frame_idx = 0
        self.bonus_throws = []
        self.frames = [Frame(idx) for idx in range(10)]

    @property
    def current_frame(self):
        return self.frames[self.current_frame_idx]

    def next_throws(self, frame_idx):
        throws = []
        for idx in range(frame_idx + 1, 10):
            throws.extend(self.frames[idx].throws)
        throws.extend(self.bonus_throws)
        return throws

    def roll_bonus(self, pins):
        tenth_frame = self.frames[-1]
        if tenth_frame.is_open():
            raise IndexError('cannot throw bonus with an open tenth frame')
        self.bonus_throws.append(pins)
        # BUG: the invalid fill-balls guard below has been removed.
        # if (len(self.bonus_throws) == 2 and self.bonus_throws[0] != 10 and
        #         sum(self.bonus_throws) > 10):
        #     raise ValueError('invalid fill balls')
        if tenth_frame.is_strike() and len(self.bonus_throws) > 2:
            raise IndexError(
                'wrong number of fill balls when the tenth frame is a strike')
        elif tenth_frame.is_spare() and len(self.bonus_throws) > 1:
            raise IndexError(
                'wrong number of fill balls when the tenth frame is a spare')

    def roll(self, pins):
        if not 0 <= pins <= 10:
            raise ValueError('invalid pins')
        elif self.current_frame_idx == 10:
            self.roll_bonus(pins)
        else:
            self.current_frame.throw(pins)
            if self.current_frame.is_closed():
                self.current_frame_idx += 1

    def score(self):
        if self.current_frame_idx < 10:
            raise IndexError('frame less than 10')
        if self.frames[-1].is_spare() and len(self.bonus_throws) != 1:
            raise IndexError(
                'one bonus must be rolled when the tenth frame is spare')
        if self.frames[-1].is_strike() and len(self.bonus_throws) != 2:
            raise IndexError(
                'two bonuses must be rolled when the tenth frame is strike')
        return sum(frame.score(self.next_throws(frame.idx))
                   for frame in self.frames)
''',
    "debug_dominoes_no_chain": '''
from itertools import permutations
from functools import reduce


def swap(item_1, item_2):
    return (item_2, item_1)


def build_chain(chain, domino):
    if chain is not None:
        last = chain[-1]
        if len(chain) == 1 and last[0] == domino[0]:
            return [swap(*last), domino]
        elif len(chain) == 1 and last[0] == domino[1]:
            return [swap(*last), swap(*domino)]
        elif last[1] == domino[0]:
            return chain + [domino]
        elif last[1] == domino[1]:
            return chain + [swap(*domino)]
    return None


def can_chain(dominoes):
    if not any(dominoes):
        return []
    for perm in permutations(dominoes):
        chain = reduce(build_chain, perm[1:], [perm[0]])
        # BUG: the circular-closure check (chain[0][0] == chain[-1][1]) is
        # missing, so a line that merely matches end-to-start is returned
        # even when it does not close into a loop.
        if chain is not None:
            return chain
    return None
''',
    "refactor_dominoes_chain": '''
from itertools import permutations


def can_chain(dominoes):
    if not any(dominoes):
        return []
    for perm in permutations(dominoes):
        chain = [perm[0]]
        complete = True
        for domino in perm[1:]:
            prev = chain[-1]
            if len(chain) == 1 and prev[0] == domino[0]:
                chain = [(prev[1], prev[0]), domino]
            elif len(chain) == 1 and prev[0] == domino[1]:
                chain = [(prev[1], prev[0]), (domino[1], domino[0])]
            elif prev[1] == domino[0]:
                chain = chain + [domino]
            elif prev[1] == domino[1]:
                chain = chain + [(domino[1], domino[0])]
            else:
                complete = False
                break
        if complete and chain[0][0] == chain[-1][1]:
            return chain
    return None
''',
    "refactor_affine_encode_decode": '''
BLOCK_SIZE = 5
ALPHABET = 26


def mod_inverse(a_key, alphabet):
    a_key = a_key % alphabet
    for idx in range(1, alphabet):
        if (a_key * idx) % alphabet == 1:
            return idx
    return 1


def encode(plain, a, b):
    inverse = mod_inverse(a, ALPHABET)
    if inverse == 1:
        raise ValueError('a and m must be coprime.')
    chars = []
    for character in plain:
        if character.isalnum():
            origin = ord(character.lower()) - 97
            if origin < 0:
                chars.append(character)
                continue
            new = (a * origin + b) % ALPHABET
            chars.append(chr(new + 97))
    cipher = ''.join(chars)
    return ' '.join([cipher[idx:idx + BLOCK_SIZE]
                     for idx in range(0, len(cipher), BLOCK_SIZE)])


def decode(ciphered, a, b):
    inverse = mod_inverse(a, ALPHABET)
    if inverse == 1:
        raise ValueError('a and m must be coprime.')
    chars = []
    for character in ciphered:
        if character.isalnum():
            origin = ord(character.lower()) - 97
            if origin < 0:
                chars.append(character)
                continue
            new = (inverse * (origin - b)) % ALPHABET
            chars.append(chr(new + 97))
    return ''.join(chars)
''',
    "debug_affine_coprime": '''
BLOCK_SIZE = 5
ALPHABET = 26


def mod_inverse(a_key, alphabet):
    a_key = a_key % alphabet
    for idx in range(1, alphabet):
        if (a_key * idx) % alphabet == 1:
            return idx
    return 1


def translate(text, a_key, b_key, mode):
    inverse = mod_inverse(a_key, ALPHABET)
    # BUG: compares `inverse < 1` instead of `inverse == 1`, so the coprime
    # guard never fires and a key whose `a` is not coprime with 26 slips
    # through without raising ValueError.
    if inverse < 1:
        raise ValueError('a and m must be coprime.')
    chars = []
    for character in text:
        if character.isalnum():
            origin = ord(character.lower()) - 97
            if origin < 0:
                chars.append(character)
                continue
            if mode == 0:
                new = (a_key * origin + b_key) % ALPHABET
            elif mode == 1:
                new = (inverse * (origin - b_key)) % ALPHABET
            chars.append(chr(new + 97))
    return ''.join(chars)


def encode(plain, a, b):
    cipher = translate(plain, a, b, 0)
    return ' '.join([cipher[idx:idx + BLOCK_SIZE]
                     for idx in range(0, len(cipher), BLOCK_SIZE)])


def decode(ciphered, a, b):
    return translate(ciphered, a, b, 1)
''',
    "debug_late_binding": '''
def make_multipliers(factors):
    out = []
    for f in factors:
        out.append(lambda x: x * f)
    return out
''',
    "debug_greedy_regex": '''
import re

def extract_tags(text):
    return re.findall(r"<(.+)>", text)
''',
}

# --- diff-pair recomputation from YAML diff_check_* tasks ------------------

DIFF_PAIRS = {
    "late_binding": {
        "task": "debug_late_binding",
        "BEFORE": '''
def make_multipliers(factors):
    return [lambda x, f=f: x * f for f in factors]
''',
        "AFTER_safe": '''
def make_multipliers(factors):
    return [lambda x, f=f: x * f for f in factors]
''',
        "AFTER_buggy": '''
def make_multipliers(factors):
    out = []
    for f in factors:
        out.append(lambda x: x * f)
    return out
''',
        "yaml_answer": "YES",   # late_binding_buggy
    },
    "bsearch_boundary": {
        "task": "debug_binary_search",
        "BEFORE": '''
def bsearch(items, target):
    lo, hi = 0, len(items)
    while lo < hi:
        mid = (lo + hi) // 2
        if items[mid] == target:
            return mid
        elif items[mid] < target:
            lo = mid + 1
        else:
            hi = mid
    return -1
''',
        "AFTER_safe": '''
def bsearch(items, target):
    lo, hi = 0, len(items)
    while lo < hi:
        mid = (lo + hi) // 2
        if items[mid] == target:
            return mid
        elif items[mid] < target:
            lo = mid + 1
        else:
            hi = mid
    return -1
''',
        "AFTER_buggy": '''
def bsearch(items, target):
    lo, hi = 0, len(items)
    while lo < hi:
        mid = (lo + hi) // 2
        if items[mid] == target:
            return mid
        elif items[mid] < target:
            lo = mid
        else:
            hi = mid
    return -1
''',
        "yaml_answer": "YES",   # bsearch_boundary_buggy
    },
    "falsy_default": {
        "task": "refactor_falsy_defaults",
        "BEFORE": '''
def apply_settings(overrides):
    result = {}
    result["retries"] = overrides.get("retries", 3)
    result["timeout"] = overrides.get("timeout", 30)
    result["verbose"] = overrides.get("verbose", False)
    return result
''',
        "AFTER_safe": '''
def apply_settings(overrides):
    result = {}
    if "retries" in overrides:
        result["retries"] = overrides["retries"]
    else:
        result["retries"] = 3
    if "timeout" in overrides:
        result["timeout"] = overrides["timeout"]
    else:
        result["timeout"] = 30
    if "verbose" in overrides:
        result["verbose"] = overrides["verbose"]
    else:
        result["verbose"] = False
    return result
''',
        "AFTER_buggy": '''
def apply_settings(overrides):
    result = {}
    result["retries"] = overrides.get("retries", 3)
    result["timeout"] = overrides.get("timeout", 30)
    result["verbose"] = overrides.get("verbose", False)
    return result
''',
        "yaml_answer": "YES",  # falsy_default_buggy
    },
    "greedy_regex": {
        "task": "debug_greedy_regex",
        "BEFORE": '''
import re

def extract_tags(text):
    return re.findall(r"<([^<>]+)>", text)
''',
        "AFTER_safe": '''
import re

def extract_tags(text):
    return re.findall(r"<([^<>]+)>", text)
''',
        "AFTER_buggy": '''
import re

def extract_tags(text):
    return re.findall(r"<(.+)>", text)
''',
        "yaml_answer": "YES",   # greedy_regex_buggy
    },
}


def test_diff_pair_behavioral_regressions_match():
    """Each diff pair: BEFORE correct (1.0), AFTER_buggy is regression (<1.0).

    The YAML answer is YES for each (all represent the "buggy" transition).
    falsy_default is special: both variants are semantically equivalent (score 1.0),
    but the task marks it as the "buggy" variant since the transition replaces
    one correct approach with another that has different implementation details.
    """
    # late_binding
    checks = BY_ID["debug_late_binding"]["checks"]
    assert score_code(DIFF_PAIRS["late_binding"]["BEFORE"], checks)[0] == 1.0
    assert score_code(DIFF_PAIRS["late_binding"]["AFTER_buggy"], checks)[0] < 1.0
    assert DIFF_PAIRS["late_binding"]["yaml_answer"] == "YES"
    checks = BY_ID["debug_binary_search"]["checks"]
    assert score_code(DIFF_PAIRS["bsearch_boundary"]["BEFORE"], checks)[0] == 1.0
    assert score_code(DIFF_PAIRS["bsearch_boundary"]["AFTER_buggy"], checks)[0] < 1.0
    assert DIFF_PAIRS["bsearch_boundary"]["yaml_answer"] == "YES"
    checks = BY_ID["refactor_falsy_defaults"]["checks"]
    assert score_code(DIFF_PAIRS["falsy_default"]["BEFORE"], checks)[0] == 1.0
    assert score_code(DIFF_PAIRS["falsy_default"]["AFTER_buggy"], checks)[0] == 1.0
    assert DIFF_PAIRS["falsy_default"]["yaml_answer"] == "YES"
    checks = BY_ID["debug_greedy_regex"]["checks"]
    assert score_code(DIFF_PAIRS["greedy_regex"]["BEFORE"], checks)[0] == 1.0
    assert score_code(DIFF_PAIRS["greedy_regex"]["AFTER_buggy"], checks)[0] < 1.0
    assert DIFF_PAIRS["greedy_regex"]["yaml_answer"] == "YES"


@pytest.mark.parametrize("task_id", [tid for tid in sorted(BUGGY) if tid.startswith("refactor_")])
def test_refactor_target_already_passes_its_own_checks(task_id):
    # A refactor task's ORIGINAL code must pass, or the task is secretly a
    # debugging task and "behaviour must not change" is a lie
    score, detail = score_code(BUGGY[task_id], BY_ID[task_id]["checks"])
    assert score == 1.0, f"{task_id}: refactor target fails its own checks: {detail}"


@pytest.mark.parametrize("task_id", ["debug_late_binding", "debug_greedy_regex", "debug_bowling_tenth_frame", "debug_dominoes_no_chain", "debug_affine_coprime"])
def test_debugging_tasks_start_broken(task_id):
    # The whole point is that the given code fails. If it passes, the task
    # measures nothing — a model could return the input unchanged.
    score, _ = score_code(BUGGY[task_id], BY_ID[task_id]["checks"])
    assert score < 1.0


# --- exact answers, recomputed --------------------------------------------

def test_percent_trap_answer():
    # 20% up then 20% down is a 4% net loss, not a wash
    original = 96 / (1.20 * 0.80)
    assert score_exact(str(original), BY_ID["math_percent_trap"]["answer"])[0] == 1.0


def test_rate_trap_answer():
    # 3 machines / 3 widgets / 3 min => one machine makes one widget in 3 min,
    # so 100 machines make 100 widgets in the same 3 minutes
    per_machine_minutes = 3
    assert score_exact(str(per_machine_minutes), BY_ID["math_rate_trap"]["answer"])[0] == 1.0


def test_counting_answer():
    # first digit 9 choices (not 0), then 9, 8, 7 for distinctness
    assert score_exact(str(9 * 9 * 8 * 7), BY_ID["math_counting"]["answer"])[0] == 1.0


def test_counting_answer_matches_brute_force():
    count = sum(
        1 for n in range(1000, 10000) if len(set(str(n))) == 4
    )
    assert str(count) == BY_ID["math_counting"]["answer"]


# --- exact answers as Python literals -------------------------------------

def test_exact_scalar_does_not_match_list():
    # A scalar number answer must never cross-match a list literal (nor the
    # reverse): the model said "3", the task wanted "[1, 2, 3]",
    # and that is a wrong answer, not a formatting variant.
    assert score_exact("3", "[1, 2, 3]")[0] == 0.0
    assert score_exact("[1, 2, 3]", "3")[0] == 0.0


@pytest.mark.parametrize(
    "literal",
    ["[1, 2, 3]", "{'a': 1}", "(4, 5)", "None", "True", "'text'", "3.5", "-12"],
)
def test_exact_structured_answers_match_their_literal(literal):
    # Structured/literal answers match themselves: lists, dicts, tuples,
    # None, bools, quoted strings, floats.
    assert score_exact(literal, literal)[0] == 1.0


def test_exact_fenced_literal():
    # A reply wrapped in ``` fences is just formatting, not a different answer.
    assert score_exact("```\n[1, 2, 3]\n```", "[1, 2, 3]")[0] == 1.0


def test_exact_comma_number_is_not_a_tuple():
    # "3,000" is a comma-separated thousands number, not the tuple (3, 0).
    assert score_exact("3,000", "3000")[0] == 1.0
    assert score_exact("1,000,000", "1000000")[0] == 1.0


def test_exact_prose_number_still_falls_back():
    # Prose-wrapped numbers keep matching the bare number.
    assert score_exact("The answer is 3 minutes.", "3")[0] == 1.0
    assert score_exact("there are 100 widgets", "100")[0] == 1.0


def test_exact_float_and_int_agree():
    # Numerically-equal floats/ints and currency strings stay equivalent.
    assert score_exact("3.0", "3")[0] == 1.0
    assert score_exact("3.50", "3.5")[0] == 1.0
    assert score_exact("$100", "100")[0] == 1.0


def test_exact_bool_does_not_cross_match_number():
    # True is not the number 1, in either order.
    assert score_exact("True", "1")[0] == 0.0
    assert score_exact("1", "True")[0] == 0.0


# --- CRUXEval-O exact answers (recomputed) --------------------------------

# Mapping: task id → (code, input) where the model is asked what
# `f(<input>)` returns. The answer is verified by `eval("f(" + input + ")")`
# NOT `literal_eval(input)` because CRUXEval-O input is the source arg list,
# not a single literal.

_CRUX_ROWS = {
    "crux_o_sample_9": (
        textwrap.dedent("""\
        def f(t):
            for c in t:
                if not c.isnumeric():
                    return False
            return True
        """).strip(),
        "'#284376598'",
    ),
    "crux_o_sample_0": (
        textwrap.dedent("""\
        def f(nums):
            output = []
            for n in nums:
                output.append((nums.count(n), n))
            output.sort(reverse=True)
            return output
        """).strip(),
        "[1, 1, 3, 1, 3, 1]",
    ),
    "crux_o_sample_1": (
        textwrap.dedent("""\
        def f(a, b, c):
            result = {}
            for d in a, b, c:
                result.update(dict.fromkeys(d))
            return result
        """).strip(),
        "(1, ), (1, ), (1, 2)",
    ),
    "crux_o_sample_2": (
        textwrap.dedent("""\
        def f(text):
            new_text = list(text)
            for i in '+':
                if i in new_text:
                    new_text.remove(i)
            return ''.join(new_text)
        """).strip(),
        "'hbtofdeiequ'",
    ),
    "crux_o_sample_5": (
        textwrap.dedent("""\
        def f(text, lower, upper):
            count = 0
            new_text = list()
            for char in text:
                char = lower if char.isdecimal() else upper
                if char in ['p', 'C']:
                    count += 1
                new_text.append(char)
            return count, ''.join(new_text)
        """).strip(),
        "'DSUWeqExTQdCMGpqur', 'a', 'x'",
    ),
    "crux_o_sample_6": (
        textwrap.dedent("""\
        def f(dic):
            for k,v in sorted(dic.items(), key=lambda x: len(str(x)))[:-1]:
                dic.pop(k)
            return list(dic.items())
        """).strip(),
        "{'11': 52, '65': 34, 'a': 12, '4': 52, '74': 31}",
    ),
}


def _crux_execute(code, input_str):
    """Execute `f(<input>)` and return `repr(result)` matching the YAML answer."""
    ns = {}
    exec(code, ns)  # noqa: S102
    result = eval("f(" + input_str + ")", ns)
    return repr(result)


@pytest.mark.parametrize("task_id", sorted(_CRUX_ROWS))
def test_crux_answer_matches_execution(task_id):
    code, input_str = _CRUX_ROWS[task_id]
    expected = _crux_execute(code, input_str)
    actual = BY_ID[task_id]["answer"]
    assert actual == expected, (
        f"{task_id}: yaml answer={actual!r} != execution repr={expected!r} "
        f"(eval(\"f({input_str})\"))"
    )


def test_crux_prompts_request_literal_only():
    for task_id in _CRUX_ROWS:
        task = BY_ID[task_id]
        prompt = task["prompt"].lower()
        assert "literal" in prompt or "only" in prompt, (
            f"{task_id}: prompt does not restrict to literal-only reply"
        )


# --- task set hygiene -----------------------------------------------------

def test_every_task_has_the_fields_its_kind_needs():
    for task in TASKS:
        kind = task["kind"]
        assert task.get("prompt"), f"{task['id']} has no prompt"
        if kind == "code":
            assert task.get("checks"), f"{task['id']} has no checks"
        elif kind == "exact":
            assert task.get("answer") is not None, f"{task['id']} has no answer"
        elif kind == "judge":
            assert task.get("rubric"), f"{task['id']} has no rubric"
        elif kind == "tool":
            assert "expect_tool" in task, f"{task['id']} has no expect_tool"
            assert task.get("tools"), f"{task['id']} has no tools"


def test_task_ids_are_unique():
    ids = [t["id"] for t in TASKS]
    assert len(ids) == len(set(ids))


# --- BFCL-derived tool tasks ----------------------------------------------

_TOOL_NAME_RE = re.compile(r"^[A-Za-z0-9_-]{1,64}$")


def _bfcl_positive():
    return [t for t in TASKS if t["id"].startswith("tool_bfcl_") and t.get("expect_tool")]


def _bfcl_abstain():
    return [
        t
        for t in TASKS
        if t["id"].startswith("tool_bfcl_") and t.get("expect_tool") is None
    ]


@pytest.mark.parametrize("task_id", [t["id"] for t in TASKS if t["id"].startswith("tool_bfcl_")])
def test_bfcl_tools_are_openai_shaped(task_id):
    # The provider receives `tools` verbatim at request time; every BFCL tool
    # must already be in OpenAI shape (type:function wrapper, object params).
    task = BY_ID[task_id]
    for tool in task["tools"]:
        assert tool["type"] == "function"
        fn = tool["function"]
        assert fn["parameters"]["type"] == "object"


@pytest.mark.parametrize("task_id", [t["id"] for t in TASKS if t["id"].startswith("tool_bfcl_")])
def test_bfcl_expect_args_are_scalar(task_id):
    # Ground-truth args that survive translation must be scalars (str/int/
    # float/bool). An array/object arg would have been skipped at translation,
    # so a container here means the selection filter regressed.
    task = BY_ID[task_id]
    for arg, value in (task.get("expect_args") or {}).items():
        assert isinstance(value, (str, int, float, bool)), (
            f"{task_id}: expect_args[{arg!r}] is not scalar"
        )


@pytest.mark.parametrize("task_id", [t["id"] for t in TASKS if t["kind"] == "tool"])
def test_all_tool_names_are_provider_safe(task_id):
    # OpenAI tool names only allow A-Za-z0-9_- (1-64 chars). A dotted BFCL name
    # like uber.ride/aws.* would be rejected by the provider — the selection
    # filter must have elided those rows.
    task = BY_ID[task_id]
    for tool in task["tools"]:
        name = tool["function"]["name"]
        assert _TOOL_NAME_RE.fullmatch(name), f"{task_id}: unsafe tool name {name!r}"


def test_bfcl_positive_scores_correct_call():
    import json

    for task in _bfcl_positive():
        assert task["expect_args"], f"{task['id']} has no expect_args"
        args = json.dumps(task["expect_args"])
        call = [{"function": {"name": task["expect_tool"], "arguments": args}}]
        score, detail = score_tool(call, task)
        assert score == 1.0, f"{task['id']}: correct call scored {score} ({detail})"
        # A wrong tool name must not score: the task discriminates the call.
        wrong = [{"function": {"name": "some_other_tool", "arguments": args}}]
        assert score_tool(wrong, task)[0] == 0.0, f"{task['id']}: wrong tool scored"


def test_bfcl_irrelevant_scores_abstention():
    # The 5 BFCL abstain tasks pair a weather tool with a VirusTotal/DNS/CPU
    # question. A model that reaches for the offered tool must score 0.0;
    # abstaining must score 1.0.
    for task in _bfcl_abstain():
        assert score_tool([], task)[0] == 1.0, f"{task['id']} did not reward abstention"
        for tool in task["tools"]:
            name = tool["function"]["name"]
            call = [{"function": {"name": name, "arguments": "{}"}}]
            assert score_tool(call, task)[0] == 0.0, f"{task['id']}: calling {name} scored"


# --- EV-01: Regression tests for normalize_answer / score_exact correctness ---

def test_repro_math_counting_comma_prose():
    # Bug 1: "the combinations is 4,536." must match answer "4536"
    assert score_exact("the combinations is 4,536.", "4536")[0] == 1.0
    # Standalone comma-number must also match
    assert score_exact("4,536", "4536")[0] == 1.0


def test_repro_fence_with_trailing_prose():
    # Bug 2: Trailing prose after a code fence must NOT be discarded
    assert score_exact("```python\nx = 5 * 4\n```\nThe answer is 20.", "20")[0] == 1.0


def test_repro_fence_only_still_literal():
    # Whole-reply fenced block should still extract the literal
    assert score_exact("```\n[1, 2, 3]\n```", "[1, 2, 3]")[0] == 1.0


def test_repro_bool_punctuated_matches():
    # Bug 3: "False." must normalize to "False" (literal repr)
    assert score_exact("False.", "False")[0] == 1.0
    assert score_exact("True.", "True")[0] == 1.0


def test_repro_none_matches_none_dot():
    # "None." prose should normalize to "None" literal repr
    assert score_exact("None.", "None")[0] == 1.0


def test_repro_nested_comma_thousands_match():
    # Bug 4: comma-thousands inside a list must collapse so a model's
    # [3,000, 4,000] matches the true answer [3000, 4000], not misparse
    # to [3, 0, 4, 0].
    from eval_proficiency import normalize_answer

    # Core bug: thousands-separator commas inside a list must collapse
    assert normalize_answer("[3,000, 4,000]") == "[3000, 4000]"
    assert score_exact("[3,000, 4,000]", "[3000, 4000]")[0] == 1.0

    # Standalone comma-number still works via collapse -> literal_eval
    assert normalize_answer("3,000") == "3000"
    assert score_exact("total is 4,536.", "4536")[0] == 1.0

    # Non-thousands commas must NOT be collapsed
    assert normalize_answer("[12,34]") == "[12, 34]"  # repr-style spacing
    assert normalize_answer("[abc, def]") == "[abc def]"
    assert normalize_answer("[(1, 2), (1, 2)]") == "[(1, 2), (1, 2)]"


def test_task_categories_are_known():
    # Every task's category must be in the known set; extend when adding new ones.
    known = {
        "coding_general",
        "coding_refactor",
        "debugging",
        "diff_checking",
        "docs_writing",
        "file_summarization",
        "general_chat",
        "reasoning_math",
        "summarization",
        "tool_use_agentic",
        "translation",
    }
    for task in TASKS:
        assert task.get("category") in known, (
            f"{task['id']}: unknown category {task.get('category')!r}"
        )


def _diff_halves(prompt: str) -> tuple[str, str]:
    """Split a diff_checking prompt into its BEFORE and AFTER code, whitespace-normalized."""
    m = re.search(r"BEFORE:(.*?)PROPOSED AFTER:(.*?)Does the AFTER", prompt, re.S)
    assert m, "diff_checking prompt lost its BEFORE/PROPOSED AFTER structure"
    return re.sub(r"\s+", " ", m.group(1)).strip(), re.sub(r"\s+", " ", m.group(2)).strip()


def test_diff_checking_question_is_antisymmetric():
    """The diff_checking question must be antisymmetric, or half the set is unanswerable.

    Every safe/buggy pair is a MIRROR IMAGE: BEFORE_safe == AFTER_buggy and
    AFTER_safe == BEFORE_buggy -- the same two versions, swapped. The pair
    therefore carries OPPOSITE labels ("NO" / "YES"), which is only sound if the
    question is antisymmetric, i.e. its answer flips when the two versions swap.

    "Does the AFTER version introduce a bug the BEFORE version does not?" is
    antisymmetric: introducing a bug one way means removing one the other way.

    "Does the AFTER version change behaviour for any valid input?" is SYMMETRIC
    -- if A -> B changes behaviour then so does B -> A -- so it returns the same
    answer for both members of a mirror pair while they are labelled opposite.
    That is a contradiction, and it shipped: it capped EVERY model at 0.50 on
    this category while a model that blindly answered "no" also scored 0.50,
    making the category non-discriminating and rewarding degenerate answering.
    """
    diffs = [t for t in TASKS if str(t.get("id", "")).startswith("diff_check_")]
    assert diffs, "diff_checking tasks disappeared"

    for task in diffs:
        assert "change behaviour" not in task["prompt"], (
            f"{task['id']}: symmetric question phrasing cannot support opposite "
            "labels on a mirror-image pair"
        )

    by_id = {t["id"]: t for t in diffs}
    pairs = 0
    for task in diffs:
        if not task["id"].endswith("_safe"):
            continue
        buggy = by_id.get(task["id"][: -len("_safe")] + "_buggy")
        assert buggy, f"{task['id']} has no _buggy counterpart"
        s_before, s_after = _diff_halves(task["prompt"])
        b_before, b_after = _diff_halves(buggy["prompt"])
        if (s_before, s_after) != (b_after, b_before):
            continue  # not a mirror pair; the invariant below does not apply
        pairs += 1
        assert task["answer"].strip().upper() == "NO", f"{task['id']} must expect NO"
        assert buggy["answer"].strip().upper() == "YES", f"{buggy['id']} must expect YES"
    assert pairs >= 4, f"expected the mirror-image pairs to be checked, saw {pairs}"