6krrt/evals/tasks.yaml

# Self-eval task set. One score per task per model; scores accumulate into
# proficiency.self_eval_score and self_eval_samples.
#
# Four kinds, chosen so each category is scored the most objective way it
# admits:
#
#   code   model writes Python; each check is eval'd against it in a
#          subprocess. Score = fraction of checks passing. Fully objective.
#   exact  model replies with one value; compared after normalization.
#   tool   model is given a tool schema; scored on whether it calls the right
#          tool with the right arguments. Structural, no judge needed.
#   judge  a strong model scores the output against a rubric. Only for the
#          prose categories, where nothing checkable exists.
#
# DIFFICULTY: the happy path is not worth testing. Every current model passes
# "reverse a list", and a category where everyone scores 1.00 discriminates no
# better than the constant 0.5 it replaced. Each task here carries at least one
# edge case a plausible-looking solution gets wrong: touching vs overlapping
# intervals, present-but-falsy values, late-binding closures, greedy regexes,
# empty input, or a trap in the arithmetic.
#
# Every `code` task is validated against a reference solution by
# tests/test_task_set.py. A check my own reference cannot pass is a broken
# check, and would score the task set rather than the model — which has
# already happened here once.
#
# Keep prompts tight and self-contained. An ambiguous task scores the prompt.

tasks:
  # --- coding_general -----------------------------------------------------
  - id: merge_intervals
    category: coding_general
    kind: code
    entrypoint: merge_intervals
    prompt: |
      Write a Python function `merge_intervals(intervals)` where intervals is
      a list of [start, end] lists. Merge all overlapping intervals and return
      a new list of [start, end] lists sorted by start. Intervals that merely
      touch (one ends exactly where the next begins) must be merged. Input may
      be unsorted and may contain intervals fully nested inside others.
      Reply with ONLY the function definition — no explanation, no fences.
    checks:
      - 'merge_intervals([[1,3],[2,6],[8,10]]) == [[1,6],[8,10]]'
      - 'merge_intervals([]) == []'
      - 'merge_intervals([[1,4],[4,5]]) == [[1,5]]'
      - 'merge_intervals([[1,10],[2,3]]) == [[1,10]]'
      - 'merge_intervals([[5,6],[1,2]]) == [[1,2],[5,6]]'
      - 'merge_intervals([[1,2]]) == [[1,2]]'

  - id: parse_semver
    category: coding_general
    kind: code
    entrypoint: parse_semver
    prompt: |
      Write a Python function `parse_semver(version)` that parses a semantic
      version string into a dict with keys: major, minor, patch (ints), and
      prerelease, build (strings, or None when absent). Valid examples:
      "1.2.3", "1.2.3-alpha.1", "1.2.3+build.5", "1.2.3-rc.1+exp.sha.5114f85".
      Raise ValueError if the string is not a valid semantic version, for
      example "1.2" or "1.2.x".
      Reply with ONLY the function definition — no explanation, no fences.
    checks:
      - 'parse_semver("1.2.3") == {"major":1,"minor":2,"patch":3,"prerelease":None,"build":None}'
      - 'parse_semver("1.2.3-alpha.1")["prerelease"] == "alpha.1"'
      - 'parse_semver("1.2.3+build.5")["build"] == "build.5"'
      - 'parse_semver("1.2.3-rc.1+exp.sha.5114f85")["prerelease"] == "rc.1"'
      - 'parse_semver("1.2.3-rc.1+exp.sha.5114f85")["build"] == "exp.sha.5114f85"'
      - 'parse_semver("0.0.0")["major"] == 0'
      - 'raises(ValueError, parse_semver, "1.2")'
      - 'raises(ValueError, parse_semver, "1.2.x")'

  - id: word_wrap
    category: coding_general
    kind: code
    entrypoint: word_wrap
    prompt: |
      Write a Python function `word_wrap(text, width)` returning a list of
      lines. Split on whitespace and pack as many words per line as fit within
      `width` characters, joining words with a single space. Never split a
      word: a word longer than `width` gets its own line. Runs of whitespace
      collapse. Empty or whitespace-only text returns an empty list.
      Reply with ONLY the function definition — no explanation, no fences.
    checks:
      - 'word_wrap("the quick brown fox", 10) == ["the quick", "brown fox"]'
      - 'word_wrap("", 5) == []'
      - 'word_wrap("   ", 5) == []'
      - 'word_wrap("supercalifragilistic", 5) == ["supercalifragilistic"]'
      - 'word_wrap("a  b   c", 3) == ["a b", "c"]'
      - 'word_wrap("aa bb cc", 5) == ["aa bb", "cc"]'

  # --- coding_refactor ----------------------------------------------------
  - id: refactor_falsy_defaults
    category: coding_refactor
    kind: code
    entrypoint: apply_settings
    prompt: |
      Refactor this function to remove the repetition. Behaviour must be
      preserved EXACTLY, including for values that are present but falsy.
      Reply with ONLY the rewritten function — no explanation, no fences.

      def apply_settings(overrides):
          result = {}
          if "retries" in overrides:
              result["retries"] = overrides["retries"]
          else:
              result["retries"] = 3
          if "timeout" in overrides:
              result["timeout"] = overrides["timeout"]
          else:
              result["timeout"] = 30
          if "verbose" in overrides:
              result["verbose"] = overrides["verbose"]
          else:
              result["verbose"] = False
          return result
    checks:
      - 'apply_settings({}) == {"retries":3,"timeout":30,"verbose":False}'
      - 'apply_settings({"retries":0})["retries"] == 0'
      - 'apply_settings({"timeout":0})["timeout"] == 0'
      - 'apply_settings({"verbose":True})["verbose"] is True'
      - 'apply_settings({"retries":5}) == {"retries":5,"timeout":30,"verbose":False}'

  - id: refactor_first_match
    category: coding_refactor
    kind: code
    entrypoint: first_match
    prompt: |
      Refactor this to remove the nested loops and the flag variable.
      Behaviour must be preserved exactly, including which item wins when
      several match. Reply with ONLY the rewritten function — no explanation,
      no fences.

      def first_match(items, predicates):
          found = None
          done = False
          for item in items:
              if done:
                  break
              for p in predicates:
                  if p(item):
                      found = item
                      done = True
                      break
          return found
    checks:
      - 'first_match([1,2,3,4], [lambda x: x > 2]) == 3'
      - 'first_match([1,2,3], [lambda x: x > 10]) is None'
      - 'first_match([], [lambda x: True]) is None'
      - 'first_match([1,2,3], []) is None'
      - 'first_match([5,2,9], [lambda x: x > 8, lambda x: x < 3]) == 2'
      - 'first_match([0,1], [lambda x: x == 0]) == 0'

  - id: refactor_dispatch
    category: coding_refactor
    kind: code
    entrypoint: describe
    prompt: |
      Refactor this if/elif chain into a table-driven lookup. Behaviour must be
      preserved exactly for every input, including inputs that match no case.
      Reply with ONLY the rewritten code — no explanation, no fences.

      def describe(code):
          if code == 200:
              return "ok"
          elif code == 201:
              return "created"
          elif code == 404:
              return "not found"
          elif code == 500:
              return "server error"
          else:
              return "unknown"
    checks:
      - 'describe(200) == "ok"'
      - 'describe(404) == "not found"'
      - 'describe(500) == "server error"'
      - 'describe(418) == "unknown"'
      - 'describe(0) == "unknown"'
      - 'describe(None) == "unknown"'

  # --- debugging ----------------------------------------------------------
  - id: debug_late_binding
    category: debugging
    kind: code
    entrypoint: make_multipliers
    prompt: |
      This should return one multiplier function per factor, but every
      returned function behaves the same. Fix it. Reply with ONLY the
      corrected function — no explanation, no fences.

      def make_multipliers(factors):
          out = []
          for f in factors:
              out.append(lambda x: x * f)
          return out
    checks:
      - '[m(2) for m in make_multipliers([1,2,3])] == [2,4,6]'
      - '[m(10) for m in make_multipliers([0,1])] == [0,10]'
      - 'make_multipliers([]) == []'
      - 'make_multipliers([7])[0](3) == 21'

  - id: debug_binary_search
    category: debugging
    kind: code
    entrypoint: bsearch
    prompt: |
      This binary search should return the index of target in a sorted list,
      or -1 if absent. It is wrong for some inputs — one case loops forever.
      Fix it. Reply with ONLY the corrected function — no explanation, no
      fences.

      def bsearch(items, target):
          lo, hi = 0, len(items)
          while lo < hi:
              mid = (lo + hi) // 2
              if items[mid] == target:
                  return mid
              elif items[mid] < target:
                  lo = mid
              else:
                  hi = mid
          return -1
    checks:
      - 'bsearch([1,3,5,7], 7) == 3'
      - 'bsearch([1,3,5,7], 1) == 0'
      - 'bsearch([], 1) == -1'
      - 'bsearch([1], 1) == 0'
      - 'bsearch([1,3], 2) == -1'
      - 'bsearch([1,2,3,4,5,6], 6) == 5'

  - id: debug_greedy_regex
    category: debugging
    kind: code
    entrypoint: extract_tags
    prompt: |
      This should return the name inside each angle-bracket tag, in order, but
      it returns the wrong thing when there is more than one tag. Fix it.
      Reply with ONLY the corrected function — no explanation, no fences.

      import re

      def extract_tags(text):
          return re.findall(r"<(.+)>", text)
    checks:
      - 'extract_tags("<a><b>") == ["a","b"]'
      - 'extract_tags("<one>") == ["one"]'
      - 'extract_tags("") == []'
      - 'extract_tags("no tags here") == []'
      - 'extract_tags("x <a> y <bc> z") == ["a","bc"]'

  # --- reasoning_math -----------------------------------------------------
  - id: math_percent_trap
    category: reasoning_math
    kind: exact
    answer: "100"
    prompt: |
      A price rises by 20%, then falls by 20% of its new value. The final
      price is 96. What was the original price? Reply with ONLY the number.

  - id: math_rate_trap
    category: reasoning_math
    kind: exact
    answer: "3"
    prompt: |
      Three machines take 3 minutes to make 3 widgets, each machine working
      independently at the same constant rate. How many minutes do 100
      machines take to make 100 widgets? Reply with ONLY the number.

  - id: math_counting
    category: reasoning_math
    kind: exact
    answer: "4536"
    prompt: |
      How many 4-digit whole numbers have four distinct digits and do not
      begin with 0? Reply with ONLY the number.

  # --- tool_use_agentic ---------------------------------------------------
  - id: tool_multi_arg
    category: tool_use_agentic
    kind: tool
    prompt: Convert 250 US dollars into Japanese yen.
    expect_tool: convert_currency
    expect_args:
      amount: 250
      from_currency: USD
      to_currency: JPY
    tools:
      - type: function
        function:
          name: get_weather
          description: Get the current weather for a location.
          parameters:
            type: object
            properties:
              location: {type: string}
            required: [location]
      - type: function
        function:
          name: convert_currency
          description: Convert an amount between two currencies.
          parameters:
            type: object
            properties:
              amount: {type: number}
              from_currency: {type: string, description: ISO 4217 code}
              to_currency: {type: string, description: ISO 4217 code}
            required: [amount, from_currency, to_currency]

  - id: tool_no_tool_needed
    category: tool_use_agentic
    kind: tool
    prompt: |
      It is 1:20pm and my meeting starts at 3pm. How many minutes away is it?
    expect_tool: null   # plain arithmetic; both times are already given
    tools:
      - type: function
        function:
          name: get_calendar_event
          description: Look up a calendar event by title.
          parameters:
            type: object
            properties:
              title: {type: string}
            required: [title]
      - type: function
        function:
          name: get_current_time
          description: Get the current wall-clock time.
          parameters:
            type: object
            properties: {}

  - id: tool_abstain_creative
    category: tool_use_agentic
    kind: tool
    prompt: Write me a haiku about winter.
    expect_tool: null
    tools:
      - type: function
        function:
          name: get_weather
          description: Get the current weather for a location.
          parameters:
            type: object
            properties:
              location: {type: string}
            required: [location]

  # --- docs_writing -------------------------------------------------------
  - id: docs_function
    category: docs_writing
    kind: judge
    prompt: |
      Write a docstring for this function. Reply with ONLY the docstring text.

      def retry(fn, attempts=3, backoff=2.0):
          delay = 1.0
          for i in range(attempts):
              try:
                  return fn()
              except Exception:
                  if i == attempts - 1:
                      raise
                  time.sleep(delay)
                  delay *= backoff
    rubric: |
      Score 0-1. Award 1.0 ONLY if it states what the function does, documents
      every parameter including defaults, states the return value, AND states
      that the last exception is re-raised when all attempts fail. Deduct 0.3
      for any invented behaviour the code does not have. Deduct 0.2 if it omits
      that the delay grows by `backoff` between attempts.

  - id: docs_gotcha
    category: docs_writing
    kind: judge
    prompt: |
      Write a docstring for this function. Reply with ONLY the docstring text.

      def dedupe(items, key=None):
          seen = set()
          out = []
          for item in items:
              k = key(item) if key else item
              if k in seen:
                  continue
              seen.add(k)
              out.append(item)
          return out
    rubric: |
      Score 0-1. Award 1.0 ONLY if it states that ORDER IS PRESERVED and that
      the FIRST occurrence is kept, documents `key`, and notes that elements
      (or their keys) must be hashable. Deduct 0.4 if it omits the
      order-preservation guarantee — that is the whole reason to use this over
      set(). Deduct 0.3 if it omits the hashability requirement.

  # --- summarization ------------------------------------------------------
  - id: summarize_incident
    category: summarization
    kind: judge
    prompt: |
      Summarize in at most two sentences:

      At 02:14 UTC the checkout service began returning 502s. The on-call
      engineer found the connection pool exhausted. A deploy at 01:58 had
      lowered the pool size from 50 to 5 through a bad template variable. The
      deploy was rolled back at 02:31 and errors stopped by 02:34. Roughly
      12,000 requests failed. No data was lost.
    rubric: |
      Score 0-1. Award 1.0 ONLY if it names the ROOT CAUSE specifically (a bad
      template variable in the 01:58 deploy cut the pool from 50 to 5), the
      resolution (rollback), and the impact (~12k failed requests, no data
      lost), in two sentences or fewer. Deduct 0.4 for saying only "the
      connection pool was exhausted" — that is the symptom, not the cause.
      Deduct 0.5 for any invented detail.

  - id: summarize_buried_lede
    category: summarization
    kind: judge
    prompt: |
      Summarize the single most important point in one sentence:

      The migration ran for six hours. Throughput averaged 4,200 rows per
      second, peaking at 6,100. The team used a rolling window of 5,000 rows
      per batch. Disk usage on the replica grew steadily. Partway through, a
      unique constraint on the accounts table silently rejected 812 rows,
      which were logged to a dead-letter file that nobody has yet processed.
      CPU stayed under 40% throughout.
    rubric: |
      Score 0-1. The important point is that 812 rows were silently dropped and
      remain unprocessed — everything else is routine telemetry. Award 1.0 only
      if the summary leads with that. Score 0.3 or below if it summarizes the
      throughput statistics instead. Must be one sentence.

  # --- translation --------------------------------------------------------
  - id: translate_technical
    category: translation
    kind: judge
    prompt: |
      Translate into French. Reply with ONLY the translation.

      "The connection pool was exhausted because a recent deploy reduced its
      size. Roll back the deploy and the errors should stop within a few
      minutes."
    rubric: |
      Score 0-1 on accuracy and fluency. Award 1.0 only for correct technical
      register on "connection pool", "deploy" and "roll back", AND natural
      French rather than a word-for-word calque. Deduct 0.3 per omission or
      untranslated fragment.

  - id: translate_register
    category: translation
    kind: judge
    prompt: |
      Translate into Spanish, preserving the hedging and the informal tone.
      Reply with ONLY the translation.

      "I'm not totally sure this is the right call, but I'd lean towards
      shipping it and seeing what breaks — we can always roll it back."
    rubric: |
      Score 0-1. Award 1.0 only if the HEDGING is preserved ("not totally
      sure", "I'd lean towards") rather than flattened into a confident
      statement, the register stays informal, and "roll it back" is rendered
      idiomatically. Deduct 0.4 if the hedging is lost.

  # --- general_chat -------------------------------------------------------
  - id: chat_explain
    category: general_chat
    kind: judge
    prompt: |
      Explain to a non-programmer, in under 100 words, why a program can be
      correct and still be too slow to use.
    rubric: |
      Score 0-1. Award 1.0 only if it distinguishes correctness from
      performance, gives at least one concrete relatable example, stays under
      100 words, and leaves no jargon unexplained. Deduct 0.3 if over 100
      words.

  - id: chat_pushback
    category: general_chat
    kind: judge
    prompt: |
      A colleague says "we should rewrite the whole service in Rust, it'll be
      faster." Reply in under 80 words, taking the suggestion seriously but
      identifying what you would want to know first.
    rubric: |
      Score 0-1. Award 1.0 only if it avoids both pure agreement and pure
      dismissal, names at least two specific things worth establishing first
      (for example where time is actually spent, migration cost, team
      familiarity), and stays under 80 words. Score 0.3 or below for a reply
      that simply agrees or simply refuses.