import { mkdir, readFile, rm, writeFile, } from "node:fs/promises"; import path from "node:path"; import { describe, expect, it } from "vitest"; import { localEvidenceAssessmentArtifactSchema } from "../../scripts/contracts/release-artifacts.ts"; import { verifyArchivedLocalEvidence } from "../../scripts/lib/local-release-evidence.ts"; import { LOCAL_EVIDENCE_ASSESSMENT_PATH, LOCAL_EVIDENCE_VERIFIER_SOURCE_PATHS, } from "../../scripts/lib/release-candidate.ts"; import { supplyChainDigest } from "../../scripts/lib/supply-chain.ts"; import { createArchivedAssessmentFixture, digest, digestBytes, passingAssessment, } from "./security-followup-fixture.ts"; describe("security local evidence contracts", () => { it("rejects a PASS local assessment with a failed check or failure diagnostic", () => { const failedCheck = passingAssessment(); failedCheck.checks.secretScan = "FAIL"; const failureDiagnostic = passingAssessment(); failureDiagnostic.failures.push("secret scan failed"); expect(localEvidenceAssessmentArtifactSchema.safeParse(failedCheck).success).toBe(false); expect(localEvidenceAssessmentArtifactSchema.safeParse(failureDiagnostic).success).toBe(false); expect(localEvidenceAssessmentArtifactSchema.parse(passingAssessment()).status).toBe("PASS"); }); it("rejects archived verification when independently required policy bytes are absent", async () => { const fixture = await createArchivedAssessmentFixture(); try { const result = await verifyArchivedLocalEvidence({ extractionRoot: fixture.root, expectedManifest: fixture.manifest, }); expect(result.status).toBe("FAIL"); expect(result.failures.join("\n")).toMatch(/archived local check|policy|missing/u); } finally { await rm(fixture.root, { recursive: true, force: true }); } }); it("binds the executable secret-scan rule source and rejects missing or changed archived bytes", async () => { expect(LOCAL_EVIDENCE_VERIFIER_SOURCE_PATHS).toContain("scripts/lib/secret-scan.ts"); const fixture = await createArchivedAssessmentFixture(); const ruleSourcePath = path.join(fixture.root, "scripts/lib/secret-scan.ts"); try { const missing = await verifyArchivedLocalEvidence({ extractionRoot: fixture.root, expectedManifest: fixture.manifest, }); expect(missing.failures).toContain( "archived policy input is missing or invalid: scripts/lib/secret-scan.ts", ); await mkdir(path.dirname(ruleSourcePath), { recursive: true }); await writeFile(ruleSourcePath, "export const secretScanRules = () => [];\n"); const changed = await verifyArchivedLocalEvidence({ extractionRoot: fixture.root, expectedManifest: fixture.manifest, }); expect(changed.failures).toContain( "archived policy input binding mismatch: scripts/lib/secret-scan.ts", ); } finally { await rm(fixture.root, { recursive: true, force: true }); } }); it("keeps every direct runtime import of the archived local verifier in its source binding", async () => { const verifierPath = path.join(process.cwd(), "scripts/lib/local-release-evidence.ts"); const source = await readFile(verifierPath, "utf8"); const directRuntimeSources = [ ...source.matchAll(/\bfrom\s+"(\.{1,2}\/[^"\n]+\.ts)"/gu), ].map((match) => path .relative(process.cwd(), path.resolve(path.dirname(verifierPath), match[1]!)) .replaceAll(path.sep, "/"), ); expect(LOCAL_EVIDENCE_VERIFIER_SOURCE_PATHS).toEqual( expect.arrayContaining(directRuntimeSources), ); }); it("rejects a rehashed PASS assessment over contradictory archived subordinate FAIL evidence", async () => { const fixture = await createArchivedAssessmentFixture(); try { const supplyPath = "artifacts/security/supply-chain-verification.json"; const coherencePath = "artifacts/security/supply-chain-coherence.json"; const supply = JSON.parse( await readFile(path.join(fixture.root, supplyPath), "utf8"), ) as Record; const coherence = { schemaVersion: 1, status: "FAIL", dependencyCount: 0, lockfileSha256: fixture.manifest.lockfileSha256, distSha256: fixture.manifest.distSha256, sbomSha256: digestBytes( await readFile(path.join(fixture.root, "artifacts/release/sbom.cdx.json")), ), failures: ["fixture subordinate failure"], }; const changed = new Map([ [ supplyPath, Buffer.from( `${JSON.stringify({ ...supply, localStatus: "FAIL", failures: ["fixture subordinate failure"], })}\n`, ), ], [coherencePath, Buffer.from(`${JSON.stringify(coherence)}\n`)], ]); for (const [memberPath, bytes] of changed) { await mkdir(path.dirname(path.join(fixture.root, memberPath)), { recursive: true }); await writeFile(path.join(fixture.root, memberPath), bytes); } const assessmentPath = path.join(fixture.root, LOCAL_EVIDENCE_ASSESSMENT_PATH); const assessment = localEvidenceAssessmentArtifactSchema.parse( JSON.parse(await readFile(assessmentPath, "utf8")) as unknown, ); assessment.evidenceInputs = assessment.evidenceInputs .filter(({ path: memberPath }) => memberPath !== coherencePath) .map((row) => { const bytes = changed.get(row.path); return bytes ? { path: row.path, bytes: bytes.byteLength, sha256: digestBytes(bytes) } : row; }); const coherenceBytes = changed.get(coherencePath)!; assessment.evidenceInputs.push({ path: coherencePath, bytes: coherenceBytes.byteLength, sha256: digestBytes(coherenceBytes), }); assessment.evidenceInputs.sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0, ); const assessmentBytes = Buffer.from(`${JSON.stringify(assessment)}\n`); await writeFile(assessmentPath, assessmentBytes); const changedWithAssessment = new Map(changed); changedWithAssessment.set(LOCAL_EVIDENCE_ASSESSMENT_PATH, assessmentBytes); const files = fixture.manifest.files .filter(({ path: memberPath }) => memberPath !== coherencePath) .map((row) => { const bytes = changedWithAssessment.get(row.path); return bytes ? { path: row.path, bytes: bytes.byteLength, sha256: digestBytes(bytes) } : row; }); files.push({ path: coherencePath, bytes: coherenceBytes.byteLength, sha256: digestBytes(coherenceBytes), }); files.sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0, ); const manifest = { ...fixture.manifest, files, bundleSha256: supplyChainDigest(files), }; await writeFile( path.join(fixture.root, "artifacts/release/release-candidate.json"), `${JSON.stringify(manifest)}\n`, ); const result = await verifyArchivedLocalEvidence({ extractionRoot: fixture.root, expectedManifest: manifest, }); expect(result.status).toBe("FAIL"); expect(result.failures.join("\n")).toMatch(/supply-chain.*not.*PASS|subordinate/u); } finally { await rm(fixture.root, { recursive: true, force: true }); } }); it("rejects archived verification when the assessment is absent", async () => { const fixture = await createArchivedAssessmentFixture(); try { await rm(path.join(fixture.root, LOCAL_EVIDENCE_ASSESSMENT_PATH)); const result = await verifyArchivedLocalEvidence({ extractionRoot: fixture.root, expectedManifest: fixture.manifest, }); expect(result.status).toBe("FAIL"); expect(result.failures).toContain("local evidence assessment is missing or invalid"); } finally { await rm(fixture.root, { recursive: true, force: true }); } }); it("rejects digest-bound raw identity evidence that is not valid under its strict producer schema", async () => { const fixture = await createArchivedAssessmentFixture(); try { const provenancePath = "artifacts/release/provenance.json"; const malformed = Buffer.from( `${JSON.stringify({ predicate: { materials: { sourceSetSha256: digest("source set") } } })}\n`, ); await writeFile(path.join(fixture.root, provenancePath), malformed); const assessmentPath = path.join(fixture.root, LOCAL_EVIDENCE_ASSESSMENT_PATH); const assessment = localEvidenceAssessmentArtifactSchema.parse( JSON.parse(await readFile(assessmentPath, "utf8")) as unknown, ); assessment.evidenceInputs = assessment.evidenceInputs.map((row) => row.path === provenancePath ? { path: provenancePath, bytes: malformed.byteLength, sha256: digestBytes(malformed) } : row, ); const assessmentBytes = Buffer.from(`${JSON.stringify(assessment)}\n`); await writeFile(assessmentPath, assessmentBytes); const files = fixture.manifest.files.map((row) => { if (row.path === provenancePath) { return { path: provenancePath, bytes: malformed.byteLength, sha256: digestBytes(malformed) }; } if (row.path === LOCAL_EVIDENCE_ASSESSMENT_PATH) { return { path: LOCAL_EVIDENCE_ASSESSMENT_PATH, bytes: assessmentBytes.byteLength, sha256: digestBytes(assessmentBytes), }; } return row; }); const manifest = { ...fixture.manifest, files, bundleSha256: supplyChainDigest(files) }; await writeFile( path.join(fixture.root, "artifacts/release/release-candidate.json"), `${JSON.stringify(manifest)}\n`, ); const result = await verifyArchivedLocalEvidence({ extractionRoot: fixture.root, expectedManifest: manifest, }); expect(result.status).toBe("FAIL"); expect(result.failures).toContain( "archived source/build/provenance identities are missing or invalid", ); } finally { await rm(fixture.root, { recursive: true, force: true }); } }); });