962 lines
34 KiB
Python
962 lines
34 KiB
Python
"""Tests for scripts/verify_commit.py.
|
|
|
|
All tests are offline and use a temporary git repository built by each test.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
# Path to the script under test
|
|
_SCRIPT_DIR = Path(__file__).resolve().parent.parent / "scripts"
|
|
VERIFY_COMMIT_PY = _SCRIPT_DIR / "verify_commit.py"
|
|
KNOWN_FAILURES_TXT = _SCRIPT_DIR / "verify_known_failures.txt"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _git(repo: Path, *args: str) -> subprocess.CompletedProcess:
|
|
"""Run a git command in *repo* and return the CompletedProcess."""
|
|
return subprocess.run(
|
|
["git", "-C", str(repo)] + list(args),
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30,
|
|
check=False,
|
|
)
|
|
|
|
|
|
def _init_repo(repo: Path) -> None:
|
|
"""Initialise an empty git repository at *repo*."""
|
|
_git(repo, "init")
|
|
_git(repo, "config", "user.email", "test@test.com")
|
|
_git(repo, "config", "user.name", "Test")
|
|
|
|
|
|
def _add_commit(repo: Path, msg: str = "commit") -> None:
|
|
"""Stage all changes and commit."""
|
|
_git(repo, "add", ".")
|
|
_git(repo, "commit", "-m", msg)
|
|
|
|
|
|
def _run_verify(
|
|
repo: Path,
|
|
*extra_args: str,
|
|
) -> subprocess.CompletedProcess:
|
|
"""Run ``verify_commit.py`` against *repo* and return the result.
|
|
|
|
``HEAD`` is passed as the positional SHA. Callers that want to verify
|
|
a different SHA must include it explicitly after the options.
|
|
"""
|
|
cmd = [
|
|
sys.executable,
|
|
str(VERIFY_COMMIT_PY),
|
|
"--repo", str(repo),
|
|
"--no-lint",
|
|
]
|
|
if extra_args:
|
|
cmd.extend(extra_args)
|
|
cmd.append("HEAD")
|
|
return subprocess.run(cmd, capture_output=True, text=True, timeout=60,
|
|
check=False)
|
|
|
|
|
|
def _on_rm_error(func, path, exc_info):
|
|
"""Handle permission errors on git's .git/ files during cleanup."""
|
|
os.chmod(path, stat.S_IWRITE)
|
|
func(path)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# select_tests unit tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestSelectTests:
|
|
"""Pure-function tests for ``select_tests``."""
|
|
|
|
def _read(self, _path: str) -> str:
|
|
"""Default read stub — returns ``""``."""
|
|
return ""
|
|
|
|
def test_changed_test_file_is_selected(self) -> None:
|
|
"""A changed test file selects itself."""
|
|
from scripts.verify_commit import select_tests
|
|
|
|
changed = ["tests/test_bar.py"]
|
|
test_files = ["tests/test_bar.py", "tests/test_baz.py"]
|
|
result = select_tests(changed, test_files, self._read)
|
|
assert result == ["tests/test_bar.py"]
|
|
|
|
def test_bare_import_selects_test(self) -> None:
|
|
"""``src/foo.py`` selects a test that ``import foo``."""
|
|
from scripts.verify_commit import select_tests
|
|
|
|
changed = ["src/foo.py"]
|
|
|
|
def read(path: str) -> str:
|
|
return "import foo\n\ndef test_something(): pass\n"
|
|
|
|
result = select_tests(changed, ["tests/test_foo.py"], read)
|
|
assert result == ["tests/test_foo.py"]
|
|
|
|
def test_from_src_import_selects_test(self) -> None:
|
|
"""``src/foo.py`` selects a test that ``from src import foo``."""
|
|
from scripts.verify_commit import select_tests
|
|
|
|
changed = ["src/foo.py"]
|
|
|
|
def read(path: str) -> str:
|
|
return "from src import foo\n\ndef test_something(): pass\n"
|
|
|
|
result = select_tests(changed, ["tests/test_foo.py"], read)
|
|
assert result == ["tests/test_foo.py"]
|
|
|
|
def test_from_src_dot_import_selects_test(self) -> None:
|
|
"""``src/foo.py`` selects a test that ``from src.foo import …``."""
|
|
from scripts.verify_commit import select_tests
|
|
|
|
changed = ["src/foo.py"]
|
|
|
|
def read(path: str) -> str:
|
|
return "from src.foo import something\n\ndef test_something(): pass\n"
|
|
|
|
result = select_tests(changed, ["tests/test_foo.py"], read)
|
|
assert result == ["tests/test_foo.py"]
|
|
|
|
def test_unrelated_file_selects_nothing(self) -> None:
|
|
"""An unrelated file selects no tests."""
|
|
from scripts.verify_commit import select_tests
|
|
|
|
changed = ["config/settings.yaml", "README.md"]
|
|
test_files = ["tests/test_foo.py"]
|
|
|
|
def read(path: str) -> str:
|
|
return "import foo\n"
|
|
|
|
result = select_tests(changed, test_files, read)
|
|
assert result == []
|
|
|
|
def test_bare_import_does_not_prefix_match(self) -> None:
|
|
"""``import foobar`` does not match ``src/foo.py``."""
|
|
from scripts.verify_commit import select_tests
|
|
|
|
changed = ["src/foo.py"]
|
|
|
|
def read(path: str) -> str:
|
|
return "import foobar\n"
|
|
|
|
result = select_tests(changed, ["tests/test_bar.py"], read)
|
|
assert result == []
|
|
|
|
def test_multiple_changed_sources(self) -> None:
|
|
"""Multiple changed sources each select their own tests."""
|
|
from scripts.verify_commit import select_tests
|
|
|
|
changed = ["src/bar.py", "src/baz.py"]
|
|
|
|
def read(path: str) -> str:
|
|
tbl = {
|
|
"tests/test_bar.py": "import bar\n",
|
|
"tests/test_baz.py": "import baz\n",
|
|
"tests/test_other.py": "import something\n",
|
|
}
|
|
return tbl.get(path, "")
|
|
|
|
test_files = ["tests/test_bar.py", "tests/test_baz.py",
|
|
"tests/test_other.py"]
|
|
result = select_tests(changed, test_files, read)
|
|
# Ordering is preserved from iteration
|
|
assert set(result) == {"tests/test_bar.py", "tests/test_baz.py"}
|
|
|
|
def test_dedup_same_test_called_twice(self) -> None:
|
|
"""A test asserting about two modules is listed only once."""
|
|
from scripts.verify_commit import select_tests
|
|
|
|
changed = ["src/bar.py", "src/baz.py"]
|
|
|
|
def read(path: str) -> str:
|
|
return "import bar\nimport baz\n"
|
|
|
|
test_files = ["tests/test_both.py"]
|
|
result = select_tests(changed, test_files, read)
|
|
assert result == ["tests/test_both.py"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Integration tests — temp git repo
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestVerifyCommitIntegration:
|
|
"""End-to-end tests using a temporary git repository."""
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _setup(self, tmp_path: Path) -> None:
|
|
"""Build a temp repo with one good commit."""
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
_init_repo(repo)
|
|
|
|
# Build initial content: src/foo.py + tests/test_foo.py
|
|
src = repo / "src"
|
|
tests = repo / "tests"
|
|
src.mkdir()
|
|
tests.mkdir()
|
|
|
|
(src / "foo.py").write_text("def foo():\n return 42\n")
|
|
(tests / "test_foo.py").write_text(
|
|
"from src import foo\n\n"
|
|
"def test_foo():\n"
|
|
" assert foo.foo() == 42\n"
|
|
)
|
|
|
|
_add_commit(repo, "good: test passes")
|
|
self.repo = repo
|
|
|
|
# -- Good commit -------------------------------------------------------
|
|
|
|
def test_good_commit_exit_0_and_pass_tests(self) -> None:
|
|
"""Good commit exits 0 and prints PASS tests."""
|
|
r = _run_verify(self.repo)
|
|
assert r.returncode == 0, f"Expected 0, got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
|
assert "PASS tests" in r.stdout, f"Missing PASS tests in:\n{r.stdout}"
|
|
|
|
# -- Bad commit --------------------------------------------------------
|
|
|
|
def test_bad_commit_exit_1_and_fail_tests(self) -> None:
|
|
"""Bad commit (test fails) exits 1 and prints FAIL tests."""
|
|
# Break the test by changing foo.py
|
|
src = self.repo / "src"
|
|
(src / "foo.py").write_text("def foo():\n return 0\n")
|
|
_add_commit(self.repo, "bad: test fails")
|
|
|
|
r = _run_verify(self.repo)
|
|
assert r.returncode == 1, f"Expected 1, got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
|
assert "FAIL tests" in r.stdout, f"Missing FAIL tests in:\n{r.stdout}"
|
|
|
|
def test_failure_with_warnings_shows_summary_not_warnings(self) -> None:
|
|
"""Failing test with warnings shows short summary, not the warnings tail."""
|
|
tests = self.repo / "tests"
|
|
(tests / "test_warn_fail.py").write_text(
|
|
"import warnings\n"
|
|
"from src import foo\n\n"
|
|
"def test_warn_fail():\n"
|
|
" warnings.warn('this is a noisy warning')\n"
|
|
" assert foo.foo() == 999\n"
|
|
)
|
|
_add_commit(self.repo, "warn and fail")
|
|
|
|
r = _run_verify(self.repo)
|
|
assert r.returncode == 1
|
|
assert "FAIL tests" in r.stdout
|
|
assert "FAILED" in r.stdout
|
|
assert "failed" in r.stdout.lower()
|
|
assert "this is a noisy warning" not in r.stdout, f"Warning leaked into output:\n{r.stdout}"
|
|
|
|
# -- require-clean: modified tracked file ------------------------------
|
|
|
|
def test_require_clean_fails_on_modified_tracked(self) -> None:
|
|
"""``--require-clean`` fails when a tracked file is modified."""
|
|
self.repo.joinpath("src/foo.py").write_text("def foo():\n return 99\n")
|
|
|
|
r = _run_verify(self.repo, "--require-clean")
|
|
assert r.returncode == 1, f"Expected 1, got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
|
assert "FAIL clean" in r.stdout, f"Missing FAIL clean in:\n{r.stdout}"
|
|
|
|
def test_require_clean_passes_with_untracked_only(self) -> None:
|
|
"""``--require-clean`` passes when only untracked files exist."""
|
|
# Create an untracked file
|
|
self.repo.joinpath("scratch.py").write_text("# untracked\n")
|
|
|
|
r = _run_verify(self.repo, "--require-clean")
|
|
assert r.returncode == 0, f"Expected 0, got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
|
assert "PASS clean" in r.stdout, f"Missing PASS clean in:\n{r.stdout}"
|
|
|
|
# -- known_failures check ----------------------------------------------
|
|
|
|
def test_known_failures_passes(self) -> None:
|
|
"""known_failures check PASSes when the file is empty."""
|
|
r = _run_verify(self.repo)
|
|
assert "PASS known_failures" in r.stdout, (
|
|
f"Missing PASS known_failures in:\n{r.stdout}"
|
|
)
|
|
|
|
# -- commit check ------------------------------------------------------
|
|
|
|
def test_commit_check_passes(self) -> None:
|
|
"""commit check PASSes with a valid HEAD."""
|
|
r = _run_verify(self.repo)
|
|
assert "PASS commit" in r.stdout, f"Missing PASS commit in:\n{r.stdout}"
|
|
|
|
# -- output budget -----------------------------------------------------
|
|
|
|
def test_output_max_40_lines(self) -> None:
|
|
"""Output stays within 40 lines."""
|
|
r = _run_verify(self.repo)
|
|
n = len(r.stdout.splitlines())
|
|
assert n <= 40, f"Output has {n} lines, max 40:\n{r.stdout}"
|
|
|
|
# -- failure detail cap ------------------------------------------------
|
|
|
|
def test_failure_detail_capped_at_20_lines(self) -> None:
|
|
"""25 failing tests produce exactly 20 lines: 18 FAILED + 1 truncation + 1 count."""
|
|
tests = self.repo / "tests"
|
|
lines = []
|
|
for i in range(25):
|
|
lines.append(f"def test_fail_{i:02d}():\n assert False\n")
|
|
(tests / "test_many_fail.py").write_text("\n".join(lines))
|
|
_add_commit(self.repo, "25 failures")
|
|
|
|
r = _run_verify(self.repo)
|
|
assert r.returncode == 1
|
|
assert "FAIL tests" in r.stdout
|
|
stripped = [l.strip() for l in r.stdout.splitlines()]
|
|
detail_lines = [l for l in stripped
|
|
if l.startswith(("FAILED ", "ERROR ", "... and"))
|
|
or ("failed" in l.lower() and "passed" in l.lower())]
|
|
assert len(detail_lines) == 20, (
|
|
f"Expected 20 detail lines, got {len(detail_lines)}:\n{r.stdout}"
|
|
)
|
|
assert any("... and " in l for l in detail_lines)
|
|
|
|
def test_error_prefix_matches_collection_errors(self) -> None:
|
|
"""A pytest collection ERROR line (no trailing S) is captured."""
|
|
tests = self.repo / "tests"
|
|
# Syntax error triggers a collection ERROR (not FAILED)
|
|
(tests / "test_bad_syntax.py").write_text("def this is not valid python:\n")
|
|
_add_commit(self.repo, "syntax error")
|
|
|
|
r = _run_verify(self.repo)
|
|
assert r.returncode == 1
|
|
assert "FAIL tests" in r.stdout
|
|
stripped = [l.strip() for l in r.stdout.splitlines()]
|
|
assert any("ERROR " in l for l in stripped), (
|
|
f"Expected 'ERROR ' in output:\n{r.stdout}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CLI argument parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestArgParse:
|
|
"""Unit tests for argument parsing (no side effects)."""
|
|
|
|
def test_defaults(self) -> None:
|
|
"""Default values are sensible."""
|
|
from scripts.verify_commit import parse_args
|
|
|
|
args = parse_args(["HEAD"])
|
|
assert args.repo == "."
|
|
assert args.base is None # resolved to SHA^ in main()
|
|
assert args.full is False
|
|
assert args.require_clean is False
|
|
assert args.no_lint is False
|
|
assert args.python == sys.executable
|
|
assert args.ruff_cmd == "uvx ruff@0.16.9"
|
|
assert args.sha == "HEAD"
|
|
|
|
def test_all_flags(self) -> None:
|
|
"""All flags can be overridden."""
|
|
from scripts.verify_commit import parse_args
|
|
|
|
args = parse_args([
|
|
"--repo", "/tmp/x",
|
|
"--base", "main",
|
|
"--full",
|
|
"--require-clean",
|
|
"--no-lint",
|
|
"--python", "/usr/bin/python3",
|
|
"--ruff-cmd", "/home/me/.local/bin/ruff",
|
|
"HEAD",
|
|
])
|
|
assert args.repo == "/tmp/x"
|
|
assert args.base == "main"
|
|
assert args.full is True
|
|
assert args.require_clean is True
|
|
assert args.no_lint is True
|
|
assert args.python == "/usr/bin/python3"
|
|
assert args.ruff_cmd == "/home/me/.local/bin/ruff"
|
|
assert args.sha == "HEAD"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Lint check: _strip_coords & new_findings
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestStripCoords:
|
|
"""Pure-function tests for ``_strip_coords``."""
|
|
|
|
def test_removes_line_and_column(self) -> None:
|
|
from scripts.verify_commit import _strip_coords
|
|
result = _strip_coords("foo.py:10:5: F401 `os` imported but unused")
|
|
assert result == "foo.py: F401 `os` imported but unused"
|
|
|
|
def test_removes_only_line(self) -> None:
|
|
from scripts.verify_commit import _strip_coords
|
|
result = _strip_coords("bar.py:3:1: E302 expected 2 blank lines, found 1")
|
|
assert result == "bar.py: E302 expected 2 blank lines, found 1"
|
|
|
|
def test_returns_line_unmodified_if_no_coords(self) -> None:
|
|
from scripts.verify_commit import _strip_coords
|
|
result = _strip_coords("some random output")
|
|
assert result == "some random output"
|
|
|
|
def test_handles_colon_in_message(self) -> None:
|
|
from scripts.verify_commit import _strip_coords
|
|
result = _strip_coords("x.py:1:1: WPS111 Found module with too many imports: 42")
|
|
assert result == "x.py: WPS111 Found module with too many imports: 42"
|
|
|
|
def test_handles_concise_format(self) -> None:
|
|
"""_strip_coords handles concise format."""
|
|
from scripts.verify_commit import _strip_coords
|
|
result = _strip_coords("scripts/foo.py:10:5: F401 unused import")
|
|
assert result == "scripts/foo.py: F401 unused import"
|
|
|
|
def test_returns_short_lines_unchanged(self) -> None:
|
|
"""_strip_coords returns short lines unchanged (e.g. ruff full output coords)."""
|
|
from scripts.verify_commit import _strip_coords
|
|
result = _strip_coords(" --> scripts/foo.py:190:14")
|
|
assert result == "--> scripts/foo.py:190:14"
|
|
|
|
def test_returns_plain_lines_unchanged(self) -> None:
|
|
"""_strip_coords returns plain lines unchanged."""
|
|
from scripts.verify_commit import _strip_coords
|
|
result = _strip_coords("Found 1 error.")
|
|
assert result == "Found 1 error."
|
|
|
|
|
|
class TestNewFindings:
|
|
"""Pure-function tests for ``new_findings``."""
|
|
|
|
def test_no_new_findings(self) -> None:
|
|
from scripts.verify_commit import new_findings
|
|
result = new_findings(
|
|
["foo.py:10:5: F401 `os` imported but unused"],
|
|
["foo.py:10:5: F401 `os` imported but unused"],
|
|
)
|
|
assert result == []
|
|
|
|
def test_ignores_line_col_shifts(self) -> None:
|
|
from scripts.verify_commit import new_findings
|
|
result = new_findings(
|
|
["foo.py:1:1: F401 `os` imported but unused"],
|
|
["foo.py:42:7: F401 `os` imported but unused"],
|
|
)
|
|
assert result == []
|
|
|
|
def test_new_finding_detected(self) -> None:
|
|
from scripts.verify_commit import new_findings
|
|
result = new_findings(
|
|
["foo.py:10:5: F401 `os` imported but unused"],
|
|
["foo.py:10:5: F401 `os` imported but unused",
|
|
"foo.py:20:3: E302 expected 2 blank lines, found 1"],
|
|
)
|
|
assert result == ["foo.py: E302 expected 2 blank lines, found 1"]
|
|
|
|
def test_issue_removed_in_after_is_not_new(self) -> None:
|
|
from scripts.verify_commit import new_findings
|
|
result = new_findings(
|
|
["foo.py:10:5: F401 `os` imported but unused"],
|
|
[],
|
|
)
|
|
assert result == []
|
|
|
|
def test_multiple_new_findings_sorted(self) -> None:
|
|
from scripts.verify_commit import new_findings
|
|
result = new_findings(
|
|
[],
|
|
["b.py:1:1: E302 expected 2 blank lines",
|
|
"a.py:1:1: F401 `os` imported but unused"],
|
|
)
|
|
assert result == [
|
|
"a.py: F401 `os` imported but unused",
|
|
"b.py: E302 expected 2 blank lines",
|
|
]
|
|
|
|
def test_ignores_empty_lines(self) -> None:
|
|
from scripts.verify_commit import new_findings
|
|
result = new_findings(
|
|
[],
|
|
["", "a.py:1:1: F401 unused import", " "],
|
|
)
|
|
assert result == ["a.py: F401 unused import"]
|
|
|
|
|
|
def test_lint_dirty_to_clean_passes(self, tmp_path: Path) -> None:
|
|
"""Dirty -> Clean = PASS."""
|
|
from scripts.verify_commit import check_lint
|
|
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
_init_repo(repo)
|
|
src = repo / "src"
|
|
src.mkdir()
|
|
f = src / "foo.py"
|
|
f.write_text("import os\n")
|
|
_add_commit(repo, "initial")
|
|
|
|
# Mock ruff output: before has issues, after is clean
|
|
def ruff_stub(stdin, path):
|
|
if "initial" in stdin: # this is a simplification for the stub
|
|
return "foo.py:1:1: F401 unused import\nFound 1 error.\n", 1
|
|
return "All checks passed!\n", 0
|
|
|
|
# We need to inject the stub into check_lint.
|
|
# check_lint uses subprocess.run(shlex.split(ruff_cmd)...)
|
|
# To test this properly without modifying scripts/verify_commit.py,
|
|
# we create a real executable script that returns different things based on stdin.
|
|
|
|
ruff_bin = tmp_path / "ruff_stub"
|
|
ruff_bin.write_text("""#!/usr/bin/env python3
|
|
import sys
|
|
stdin = sys.stdin.read()
|
|
if "import os" in stdin:
|
|
print("src/foo.py:1:1: F401 unused import")
|
|
print("Found 1 error.")
|
|
sys.exit(1)
|
|
print("All checks passed!")
|
|
sys.exit(0)
|
|
""")
|
|
ruff_bin.chmod(0o755)
|
|
|
|
# For 'before' (baseline), we provide 'import os'
|
|
# For 'after' (current), we provide something else.
|
|
# But check_lint reads from git show.
|
|
|
|
# Let's just use the real check_lint logic but control the files.
|
|
# Commit 1: dirty
|
|
f.write_text("import os\n")
|
|
_add_commit(repo, "dirty")
|
|
# Commit 2: clean
|
|
f.write_text("print('hello')\n")
|
|
_add_commit(repo, "clean")
|
|
|
|
# We need the ruff_bin to actually act as ruff.
|
|
# Since check_lint uses subprocess.run, we pass ruff_bin as ruff_cmd.
|
|
|
|
status, _detail = check_lint(
|
|
["src/foo.py"], str(ruff_bin), str(repo), "HEAD~1", "HEAD"
|
|
)
|
|
assert status == "PASS"
|
|
|
|
def test_lint_summary_lines_ignored(self, tmp_path: Path) -> None:
|
|
"""3 findings -> 1 finding (ignoring summary lines) = PASS if the finding is the same."""
|
|
from scripts.verify_commit import check_lint
|
|
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
_init_repo(repo)
|
|
src = repo / "src"
|
|
src.mkdir()
|
|
f = src / "foo.py"
|
|
f.write_text("import os\n")
|
|
_add_commit(repo, "initial")
|
|
|
|
ruff_bin = tmp_path / "ruff_summary"
|
|
# This stub returns 3 lines (1 real, 2 summary) for baseline
|
|
# and 1 line (1 real, 0 summary) for current.
|
|
# Or rather, it should just return the findings.
|
|
ruff_bin.write_text("""#!/usr/bin/env python3
|
|
import sys
|
|
stdin = sys.stdin.read()
|
|
if "baseline" in stdin:
|
|
print("src/foo.py:1:1: F401 unused")
|
|
print("Found 3 errors.")
|
|
print("[*] 1 fixable")
|
|
sys.exit(1)
|
|
if "current" in stdin:
|
|
print("src/foo.py:1:1: F401 unused")
|
|
print("Found 1 error.")
|
|
sys.exit(1)
|
|
sys.exit(0)
|
|
""")
|
|
ruff_bin.chmod(0o755)
|
|
|
|
# We need to trick git show.
|
|
# Since check_lint uses `_git("show", f"{base}:{path}", repo=repo)`,
|
|
# we can't easily mock the output of git show without mocking _git.
|
|
# However, we can mock the content by adding a marker to the files.
|
|
|
|
f.write_text("baseline\n")
|
|
_add_commit(repo, "baseline")
|
|
f.write_text("current\n")
|
|
_add_commit(repo, "current")
|
|
|
|
status, _detail = check_lint(
|
|
["src/foo.py"], str(ruff_bin), str(repo), "HEAD~1", "HEAD"
|
|
)
|
|
assert status == "PASS"
|
|
|
|
def test_lint_new_finding_fails(self, tmp_path: Path) -> None:
|
|
"""Clean -> One new finding = FAIL and detail names it."""
|
|
from scripts.verify_commit import check_lint
|
|
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
_init_repo(repo)
|
|
src = repo / "src"
|
|
src.mkdir()
|
|
f = src / "foo.py"
|
|
f.write_text("print('clean')\n")
|
|
_add_commit(repo, "initial")
|
|
|
|
ruff_bin = tmp_path / "ruff_new"
|
|
ruff_bin.write_text("""#!/usr/bin/env python3
|
|
import sys
|
|
stdin = sys.stdin.read()
|
|
if "clean" in stdin:
|
|
print("All checks passed!")
|
|
sys.exit(0)
|
|
if "dirty" in stdin:
|
|
print("src/foo.py:5:1: E302 expected 2 blank lines")
|
|
print("Found 1 error.")
|
|
sys.exit(1)
|
|
sys.exit(0)
|
|
""")
|
|
ruff_bin.chmod(0o755)
|
|
|
|
f.write_text("dirty\n")
|
|
_add_commit(repo, "make dirty")
|
|
|
|
status, _detail = check_lint(
|
|
["src/foo.py"], str(ruff_bin), str(repo), "HEAD~1", "HEAD"
|
|
)
|
|
assert status == "FAIL"
|
|
assert any("E302" in d for d in _detail)
|
|
|
|
def test_lint_shifted_line_passes(self, tmp_path: Path) -> None:
|
|
"""Same finding on a shifted line = PASS."""
|
|
from scripts.verify_commit import check_lint
|
|
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
_init_repo(repo)
|
|
src = repo / "src"
|
|
src.mkdir()
|
|
f = src / "foo.py"
|
|
f.write_text("baseline\n")
|
|
_add_commit(repo, "initial")
|
|
|
|
ruff_bin = tmp_path / "ruff_shift"
|
|
ruff_bin.write_text("""#!/usr/bin/env python3
|
|
import sys
|
|
stdin = sys.stdin.read()
|
|
if "baseline" in stdin:
|
|
print("src/foo.py:1:1: F401 unused")
|
|
sys.exit(1)
|
|
if "current" in stdin:
|
|
print("src/foo.py:10:1: F401 unused")
|
|
sys.exit(1)
|
|
sys.exit(0)
|
|
""")
|
|
ruff_bin.chmod(0o755)
|
|
|
|
f.write_text("current\n")
|
|
_add_commit(repo, "shift")
|
|
|
|
status, _detail = check_lint(
|
|
["src/foo.py"], str(ruff_bin), str(repo), "HEAD~1", "HEAD"
|
|
)
|
|
assert status == "PASS"
|
|
|
|
|
|
class TestCheckLint:
|
|
"""Tests for ``check_lint`` with a fake ruff binary."""
|
|
|
|
def test_no_changed_files(self) -> None:
|
|
from scripts.verify_commit import check_lint
|
|
status, detail = check_lint([], "ruff", "/tmp", "HEAD~1", "HEAD")
|
|
assert status == "PASS"
|
|
assert "Nothing to lint" in detail[0]
|
|
|
|
def test_no_python_files(self) -> None:
|
|
from scripts.verify_commit import check_lint
|
|
status, detail = check_lint(["README.md"], "ruff", "/tmp", "HEAD~1", "HEAD")
|
|
assert status == "PASS"
|
|
assert "No Python files to lint" in detail[0]
|
|
|
|
def test_skip_when_ruff_not_found(self, tmp_path: Path) -> None:
|
|
"""When ruff is not on PATH, check_lint returns SKIP."""
|
|
from scripts.verify_commit import check_lint
|
|
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
_init_repo(repo)
|
|
(repo / "src" / "foo.py").parent.mkdir(exist_ok=True)
|
|
(repo / "src" / "foo.py").write_text("import os\n")
|
|
_add_commit(repo, "initial")
|
|
|
|
status, detail = check_lint(
|
|
["src/foo.py"], "nonexistent-rufff", str(repo), "HEAD~1", "HEAD"
|
|
)
|
|
assert status == "SKIP"
|
|
assert "not found" in detail[0].lower()
|
|
|
|
def test_skip_on_exit_127(self, tmp_path: Path) -> None:
|
|
"""Simulate a ruff that exits 127 (e.g. ruff@version not installed)."""
|
|
from scripts.verify_commit import check_lint
|
|
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
_init_repo(repo)
|
|
(repo / "src").mkdir(exist_ok=True)
|
|
(repo / "src" / "foo.py").write_text("import os\n")
|
|
_add_commit(repo, "initial")
|
|
|
|
fake_ruff = tmp_path / "fake_ruff_127"
|
|
fake_ruff.write_text("#!/bin/sh\nexit 127\n")
|
|
fake_ruff.chmod(0o755)
|
|
|
|
status, detail = check_lint(
|
|
["src/foo.py"], str(fake_ruff), str(repo), "HEAD~1", "HEAD"
|
|
)
|
|
assert status == "SKIP"
|
|
assert "127" in detail[0]
|
|
|
|
def test_integration_with_fake_ruff(self, tmp_path: Path) -> None:
|
|
"""Use a fake ruff script to verify check_lint reports new findings."""
|
|
from scripts.verify_commit import check_lint
|
|
|
|
# Set up a small git repo with one commit
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
_init_repo(repo)
|
|
|
|
src = repo / "src"
|
|
src.mkdir()
|
|
a_py = src / "a.py"
|
|
a_py.write_text("import os\nimport sys\n")
|
|
_add_commit(repo, "initial")
|
|
|
|
# Now modify to add a new issue
|
|
a_py.write_text("import os\nimport sys\nimport pathlib\n")
|
|
_add_commit(repo, "add import")
|
|
|
|
# Create a fake ruff that reports F401 on pathlib
|
|
ruff_bin = tmp_path / "ruff"
|
|
ruff_bin.write_text("""\
|
|
#!/usr/bin/env python3
|
|
import sys
|
|
stdin = sys.stdin.read() if not sys.stdin.isatty() else ""
|
|
args = sys.argv[1:] # ["check", ...]
|
|
if "--stdin-filename" in args:
|
|
idx = args.index("--stdin-filename")
|
|
path = args[idx + 1]
|
|
if "a.py" in path and "pathlib" in stdin:
|
|
print(f"{path}:3:1: F401 `pathlib` imported but unused")
|
|
sys.exit(1 if "pathlib" in stdin else 0)
|
|
else:
|
|
# ruff check <file> — skip subcommand
|
|
py_files = [a for a in args if a.endswith(".py")]
|
|
path = py_files[0] if py_files else ""
|
|
if "a.py" in path:
|
|
print(f"{path}:3:1: F401 `pathlib` imported but unused")
|
|
sys.exit(1)
|
|
""")
|
|
ruff_bin.chmod(0o755)
|
|
|
|
status, detail = check_lint(
|
|
["src/a.py"], str(ruff_bin), str(repo), "HEAD~1", "HEAD"
|
|
)
|
|
assert status == "FAIL", f"Expected FAIL, got {status}: {detail}"
|
|
assert any("F401" in d for d in detail), f"Expected F401 in detail: {detail}"
|
|
|
|
def test_fake_ruff_passes_no_new_issues(self, tmp_path: Path) -> None:
|
|
"""When baseline and current both have same issues, check_lint PASSes."""
|
|
from scripts.verify_commit import check_lint
|
|
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
_init_repo(repo)
|
|
|
|
src = repo / "src"
|
|
src.mkdir()
|
|
# Write code that already has import os
|
|
a_py = src / "a.py"
|
|
a_py.write_text("import os\nimport sys\n")
|
|
_add_commit(repo, "initial")
|
|
|
|
# Same content, just cosmetic change
|
|
a_py.write_text("import os\nimport sys\n# comment\n")
|
|
_add_commit(repo, "add comment")
|
|
|
|
# Create a fake ruff that reports F401 for both baseline and current
|
|
ruff_bin = tmp_path / "noop-ruff"
|
|
ruff_bin.write_text("""\
|
|
#!/usr/bin/env python3
|
|
import sys
|
|
stdin = sys.stdin.read() if not sys.stdin.isatty() else ""
|
|
if stdin:
|
|
if "import os" in stdin:
|
|
sys.exit(0)
|
|
if len(sys.argv) > 1:
|
|
path = [a for a in sys.argv[1:] if not a.startswith("-")][0]
|
|
if "a.py" in path:
|
|
sys.exit(0)
|
|
sys.exit(0)
|
|
""")
|
|
ruff_bin.chmod(0o755)
|
|
|
|
status, detail = check_lint(
|
|
["src/a.py"], str(ruff_bin), str(repo), "HEAD~1", "HEAD"
|
|
)
|
|
assert status == "PASS", f"Expected PASS, got {status}: {detail}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SHA positional argument — exit 2, older-SHA verification, --base/--full
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestSHAPositional:
|
|
"""Tests for the SHA positional argument and git-archive-based checks."""
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _setup(self, tmp_path: Path) -> None:
|
|
"""Build a temp repo with one good commit."""
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
_init_repo(repo)
|
|
|
|
src = repo / "src"
|
|
tests = repo / "tests"
|
|
src.mkdir()
|
|
tests.mkdir()
|
|
|
|
(src / "good.py").write_text("def good():\n return True\n")
|
|
(tests / "test_good.py").write_text(
|
|
"from src.good import good\n\ndef test_good():\n"
|
|
" assert good() is True\n"
|
|
)
|
|
_add_commit(repo, "good: passes")
|
|
self.repo = repo
|
|
|
|
def test_sha_positional_HEAD_passes(self) -> None:
|
|
"""``verify_commit.py --repo <repo> HEAD`` exits 0 on a good repo."""
|
|
r = subprocess.run(
|
|
[
|
|
sys.executable, str(VERIFY_COMMIT_PY),
|
|
"--repo", str(self.repo),
|
|
"--no-lint",
|
|
"HEAD",
|
|
],
|
|
capture_output=True, text=True, timeout=60, check=False,
|
|
)
|
|
assert r.returncode == 0, f"Expected 0, got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
|
assert "PASS tests" in r.stdout
|
|
|
|
def test_unresolvable_sha_exits_2(self) -> None:
|
|
"""A non-existent SHA exits 2 with a usage-style message."""
|
|
r = subprocess.run(
|
|
[
|
|
sys.executable, str(VERIFY_COMMIT_PY),
|
|
"--repo", str(self.repo),
|
|
"--no-lint",
|
|
"deadbeef",
|
|
],
|
|
capture_output=True, text=True, timeout=60, check=False,
|
|
)
|
|
assert r.returncode == 2, f"Expected 2, got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
|
|
|
def test_older_sha_while_head_broken(self) -> None:
|
|
"""Verifying an OLDER SHA exits 0 even when HEAD is broken."""
|
|
# Create a second commit with a failing test
|
|
(self.repo / "src" / "good.py").write_text("def good():\n return False\n")
|
|
_add_commit(self.repo, "bad: test fails")
|
|
|
|
# Get the SHA of the first (good) commit
|
|
r = _git(self.repo, "rev-parse", "HEAD~1")
|
|
first_sha = r.stdout.strip()
|
|
|
|
# Verify the older SHA — should pass even though HEAD is broken
|
|
r = subprocess.run(
|
|
[
|
|
sys.executable, str(VERIFY_COMMIT_PY),
|
|
"--repo", str(self.repo),
|
|
"--no-lint",
|
|
first_sha,
|
|
],
|
|
capture_output=True, text=True, timeout=60, check=False,
|
|
)
|
|
assert r.returncode == 0, (
|
|
f"Expected 0 (older SHA is good), got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
|
)
|
|
assert "PASS tests" in r.stdout
|
|
|
|
def test_require_clean_with_explicit_sha(self) -> None:
|
|
"""``--require-clean HEAD`` works with the positional SHA."""
|
|
# Clean state: all good, exit 0
|
|
r = subprocess.run(
|
|
[
|
|
sys.executable, str(VERIFY_COMMIT_PY),
|
|
"--repo", str(self.repo),
|
|
"--require-clean",
|
|
"--no-lint",
|
|
"HEAD",
|
|
],
|
|
capture_output=True, text=True, timeout=60, check=False,
|
|
)
|
|
assert r.returncode == 0, f"Expected 0, got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
|
|
|
# Dirty state: modify a tracked file without committing
|
|
(self.repo / "src" / "good.py").write_text("def good():\n return True # edited\n")
|
|
r = subprocess.run(
|
|
[
|
|
sys.executable, str(VERIFY_COMMIT_PY),
|
|
"--repo", str(self.repo),
|
|
"--require-clean",
|
|
"--no-lint",
|
|
"HEAD",
|
|
],
|
|
capture_output=True, text=True, timeout=60, check=False,
|
|
)
|
|
assert r.returncode == 1, f"Expected 1, got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
|
|
|
def test_full_with_base_and_sha(self) -> None:
|
|
"""``--base <older> --full HEAD`` runs full suite against HEAD."""
|
|
# Modify code so the test still passes
|
|
(self.repo / "src" / "good.py").write_text("def good():\n return True\n")
|
|
_add_commit(self.repo, "refactor: still passes")
|
|
|
|
_head_sha = _git(self.repo, "rev-parse", "HEAD").stdout.strip()
|
|
|
|
r = subprocess.run(
|
|
[
|
|
sys.executable, str(VERIFY_COMMIT_PY),
|
|
"--repo", str(self.repo),
|
|
"--base", "HEAD~1",
|
|
"--full",
|
|
"--no-lint",
|
|
"HEAD",
|
|
],
|
|
capture_output=True, text=True, timeout=60, check=False,
|
|
)
|
|
assert r.returncode == 0, f"Expected 0, got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
|
|
|
def test_base_defaults_to_parent(self) -> None:
|
|
"""When --base is omitted, the default is SHA^."""
|
|
# First commit is good, second commit modifies code but tests still pass
|
|
(self.repo / "src" / "good.py").write_text("def good():\n return True\n")
|
|
_add_commit(self.repo, "good2")
|
|
|
|
# Run without --base — should default to HEAD~1
|
|
r = subprocess.run(
|
|
[
|
|
sys.executable, str(VERIFY_COMMIT_PY),
|
|
"--repo", str(self.repo),
|
|
"--no-lint",
|
|
"HEAD",
|
|
],
|
|
capture_output=True, text=True, timeout=60, check=False,
|
|
)
|
|
assert r.returncode == 0, f"Expected 0, got {r.returncode}\n{r.stdout}\n{r.stderr}"
|
|
assert "PASS tests" in r.stdout
|
|
|