llm-wiki/harness/tests/test_release_gate.py

from __future__ import annotations

from pathlib import Path
import subprocess
import sys
import tempfile
import unittest


RUNTIME = Path(__file__).resolve().parents[1] / "runtime"
sys.path.insert(0, str(RUNTIME))
import release_gate  # noqa: E402


class ReleaseGateTest(unittest.TestCase):
    def test_r2_default_commands_cover_runtime_and_corpus_connections(self) -> None:
        root = Path(__file__).resolve().parents[2]
        commands = {command.name: command for command in release_gate.default_commands(root)}
        expected = {
            "typed_contract_gate",
            "projection_gate",
            "semantic_surface_gate",
            "hub_semantic_gate",
            "semantic_certificate_gate",
            "local_semantic_gate",
            "semantic_regression_gate",
            "workflow-dispatch-contract",
            "workflow-source-connections",
            "consistency-contract",
            "full-links",
            "active-structure",
            "hook-tests",
        }
        self.assertTrue(expected <= set(commands))
        self.assertEqual(commands["typed_contract_gate"].release, 1)
        self.assertEqual(commands["projection_gate"].release, 1)
        self.assertEqual(commands["semantic_surface_gate"].release, 1)
        self.assertEqual(commands["hub_semantic_gate"].release, 1)
        self.assertEqual(commands["semantic_certificate_gate"].release, 1)
        self.assertEqual(commands["local_semantic_gate"].release, 2)
        self.assertEqual(commands["semantic_regression_gate"].release, 2)
        self.assertEqual(commands["hub_semantic_gate"].argv[-2:], ("--mode", "hub"))
        self.assertEqual(commands["local_semantic_gate"].argv[-2:], ("--mode", "local"))
        self.assertIn(
            "harness/runtime/semantic_surface_extractor.py",
            {part for command in commands.values() for part in command.argv},
        )
        self.assertNotIn(
            "semantic_candidate_builder.py",
            {part for command in commands.values() for part in command.argv},
        )

    def test_levels_are_cumulative_and_commands_are_injectable(self) -> None:
        with tempfile.TemporaryDirectory() as directory:
            root = Path(directory)
            observed: list[str] = []

            def runner(argv, **_kwargs):
                observed.append(argv[0])
                return subprocess.CompletedProcess(argv, 0, "ok", "")

            commands = [
                release_gate.GateCommand("r1", ("one",), release=1),
                release_gate.GateCommand("r2", ("two",), release=2),
                release_gate.GateCommand("r3", ("three",), release=3),
            ]
            result = release_gate.run_gate(root, "r2", commands=commands, runner=runner, source_paths=[])
            self.assertEqual(result["status"], "PASS")
            self.assertEqual(observed, ["one", "two"])
            self.assertTrue(result["cumulative"])
            self.assertEqual(result["schema_version"], "harness-release-gate/v2")
            self.assertIs(result["gates"], result["checks"])

    def test_missing_python_gate_command_is_a_quality_failure(self) -> None:
        with tempfile.TemporaryDirectory() as directory:
            root = Path(directory)
            command = release_gate.GateCommand(
                "local_semantic_gate",
                (sys.executable, "harness/runtime/missing.py", "--check"),
            )
            result = release_gate.run_gate(root, "r1", commands=[command], source_paths=[])
            self.assertEqual(result["status"], "FAIL")
            self.assertEqual(result["gates"][1]["findings"][0]["code"], "COMMAND_MISSING")

    def test_exit_one_is_quality_failure_and_exit_two_is_environment_error(self) -> None:
        with tempfile.TemporaryDirectory() as directory:
            root = Path(directory)
            commands = [release_gate.GateCommand("check", ("command",))]

            def failure(argv, **_kwargs):
                return subprocess.CompletedProcess(argv, 1, "drift", "")

            def error(argv, **_kwargs):
                return subprocess.CompletedProcess(argv, 2, "", "schema error")

            self.assertEqual(
                release_gate.run_gate(root, "r1", commands=commands, runner=failure, source_paths=[])["status"],
                "FAIL",
            )
            self.assertEqual(
                release_gate.run_gate(root, "r1", commands=commands, runner=error, source_paths=[])["status"],
                "ERROR",
            )

    def test_legacy_checker_exit_two_with_explicit_finding_is_normalized_to_failure(self) -> None:
        with tempfile.TemporaryDirectory() as directory:
            root = Path(directory)
            commands = [release_gate.GateCommand("moc", ("command",))]

            def runner(argv, **_kwargs):
                return subprocess.CompletedProcess(
                    argv,
                    2,
                    '{"status":"FAIL","error":{"code":"MISSING_PARENT_HUB"}}',
                    "",
                )

            result = release_gate.run_gate(root, "r1", commands=commands, runner=runner, source_paths=[])
            self.assertEqual(result["status"], "FAIL")

    def test_structured_child_findings_are_exposed_on_gate(self) -> None:
        with tempfile.TemporaryDirectory() as directory:
            root = Path(directory)
            commands = [release_gate.GateCommand("typed_contract_gate", ("command",))]

            def runner(argv, **_kwargs):
                return subprocess.CompletedProcess(
                    argv,
                    1,
                    '{"status":"FAIL","findings":[{"code":"STALE_CONTRACT_REVISION"}]}',
                    "",
                )

            result = release_gate.run_gate(root, "r1", commands=commands, runner=runner, source_paths=[])
            self.assertEqual(
                result["gates"][1]["findings"],
                [{"code": "STALE_CONTRACT_REVISION"}],
            )

    def test_r3_requires_canonical_vault_authority_even_when_layout_checker_passes(self) -> None:
        with tempfile.TemporaryDirectory() as directory:
            root = Path(directory)
            command = release_gate.GateCommand(
                "vault-layout",
                ("layout",),
                release=3,
                json_requirements=(("mode", "canonical"), ("authority", "vault")),
            )

            def compatibility(argv, **_kwargs):
                return subprocess.CompletedProcess(
                    argv, 0, '{"status":"PASS","mode":"compatibility","authority":"legacy"}', ""
                )

            result = release_gate.run_gate(
                root, "r3", commands=[command], runner=compatibility, source_paths=[]
            )
            self.assertEqual(result["status"], "FAIL")
            self.assertEqual(result["checks"][1]["findings"][0]["code"], "RELEASE_CONTRACT_MISMATCH")


if __name__ == "__main__":
    unittest.main()