llm-wiki/harness/tests/test_execution_profile.py

from __future__ import annotations

from pathlib import Path
import json
import sys
import tempfile
import unittest


RUNTIME = Path(__file__).resolve().parents[1] / "runtime"
sys.path.insert(0, str(RUNTIME))
import execution_profile  # noqa: E402


class ExecutionProfileTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls) -> None:
        cls.document = execution_profile.load_profiles(execution_profile.DEFAULT_PROFILES)

    def resolve(self, profile: str, risk: str, findings: int = 0, public: bool = False):
        return execution_profile.resolve_policy(self.document, profile, risk, findings, public)

    def test_capture_keeps_deterministic_checks_without_expensive_reviews_at_low_risk(self) -> None:
        result = self.resolve("capture", "low")
        self.assertTrue(result["always_checks"])
        self.assertFalse(result["semantic_review"]["required"])
        self.assertFalse(result["adversarial_review"]["required"])
        self.assertEqual(result["output_mode"], "failures-only")
        self.assertEqual(result["dispatch"], {
            "semantic_review": "skip",
            "adversarial_review": "skip",
        })

    def test_design_scales_reviews_with_risk(self) -> None:
        medium = self.resolve("design", "medium")
        high = self.resolve("design", "high")
        self.assertTrue(medium["semantic_review"]["required"])
        self.assertEqual(medium["dispatch"]["semantic_review"], "dispatch")
        self.assertFalse(medium["adversarial_review"]["required"])
        self.assertTrue(high["adversarial_review"]["required"])
        self.assertEqual(medium["output_mode"], "decision-risk-summary")

    def test_audit_and_publish_conditions_are_evaluated(self) -> None:
        audit = self.resolve("audit", "low", findings=5)
        publish = self.resolve("publish", "low", public=True)
        self.assertTrue(audit["semantic_review"]["required"])
        self.assertEqual(audit["adversarial_review"]["matched_conditions"], ["finding_count>=5"])
        self.assertTrue(publish["adversarial_review"]["required"])
        self.assertEqual(audit["output_mode"], "detailed-artifact")
        self.assertEqual(publish["output_mode"], "public-claim-verification")

    def test_workflow_resolution_uses_nested_default_and_allows_risk_override(self) -> None:
        default = execution_profile.resolve_workflow_policy(
            self.document,
            "branch-spec",
            execution_profile.DEFAULT_WORKFLOW_DIR,
            None,
            0,
            False,
        )
        overridden = execution_profile.resolve_workflow_policy(
            self.document,
            "branch-spec",
            execution_profile.DEFAULT_WORKFLOW_DIR,
            "high",
            0,
            False,
        )
        self.assertEqual(default["profile"], "design")
        self.assertEqual(default["context"]["risk"], "medium")
        self.assertEqual(default["workflow"], "branch-spec")
        self.assertEqual(overridden["context"]["risk"], "high")
        self.assertEqual(overridden["dispatch"]["adversarial_review"], "dispatch")

    def test_design_bearing_mandatory_gates_cannot_be_disabled_by_low_risk(self) -> None:
        result = self.resolve("capture", "low")
        self.assertEqual(result["mandatory_gates"]["semantic_coherence"], "skip")
        design = execution_profile.resolve_policy(
            self.document,
            "capture",
            "low",
            0,
            False,
            True,
            False,
        )
        self.assertEqual(design["mandatory_gates"]["typed_contract"], "required")
        self.assertEqual(design["mandatory_gates"]["semantic_coherence"], "required")
        self.assertTrue(design["semantic_review"]["required"])

    def test_claims_require_proof_and_risk_only_increases_review_intensity(self) -> None:
        low = execution_profile.resolve_policy(self.document, "design", "low", 0, False, True, True)
        high = execution_profile.resolve_policy(self.document, "design", "high", 0, False, True, True)
        self.assertEqual(low["mandatory_gates"]["proof_manifest"], "required")
        self.assertGreater(high["review_intensity"]["semantic_passes"], low["review_intensity"]["semantic_passes"])
        self.assertEqual(low["mandatory_gates"], high["mandatory_gates"])

    def test_every_neutral_workflow_resolves_to_dispatch_and_output_contracts(self) -> None:
        paths = sorted(execution_profile.DEFAULT_WORKFLOW_DIR.glob("*.json"))
        self.assertEqual(len(paths), 24)
        for path in paths:
            workflow = path.stem
            result = execution_profile.resolve_workflow_policy(
                self.document,
                workflow,
                execution_profile.DEFAULT_WORKFLOW_DIR,
                None,
                0,
                False,
            )
            self.assertIn(result["dispatch"]["semantic_review"], {"dispatch", "skip"})
            self.assertIn(result["dispatch"]["adversarial_review"], {"dispatch", "skip"})
            self.assertIn(result["output_contract"]["mode"], execution_profile.OUTPUT_MODES)

    def test_workflow_contract_rejects_root_level_profile(self) -> None:
        with tempfile.TemporaryDirectory() as directory:
            workflow_dir = Path(directory)
            (workflow_dir / "bad.json").write_text(
                json.dumps({
                    "schema_version": 1,
                    "source_kind": "workflow",
                    "id": "bad",
                    "profile": "capture",
                    "execution_contract": {
                        "kind": "orchestrated",
                        "profile": "capture",
                        "default_risk": "low",
                    },
                }),
                encoding="utf-8",
            )
            with self.assertRaises(execution_profile.ProfileError):
                execution_profile.load_workflow_contract("bad", workflow_dir)

    def test_unknown_condition_fails_closed(self) -> None:
        document = dict(self.document)
        document["profiles"] = {
            "bad": {
                "semantic_review": {"mode": "required"},
                "adversarial_review": {"mode": "required_when", "conditions": ["maybe"]},
            }
        }
        with self.assertRaises(execution_profile.ProfileError):
            execution_profile.resolve_policy(document, "bad", "low", 0, False)


if __name__ == "__main__":
    unittest.main()