refactor: validate generated evidence artifacts
This commit is contained in:
DongHyeonka
@@ -1,10 +1,18 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
dependencyInventoryArtifactSchema,
|
||||
fieldWebVitalsArtifactSchema,
|
||||
jsonSchemaDocumentArtifactSchema,
|
||||
labPerformanceArtifactSchema,
|
||||
parseBuildManifestArtifact,
|
||||
parseReleaseArtifact,
|
||||
parseRuntimeConfigArtifact,
|
||||
projectReleaseTokens,
|
||||
registrySnapshotArtifactSchema,
|
||||
supplyChainVerificationArtifactSchema,
|
||||
} from "../../scripts/contracts/release-artifacts.ts";
|
||||
|
||||
const EMPTY_CONTRACT_SET_DIGEST =
|
||||
@@ -126,4 +134,241 @@ describe("release artifact contracts", () => {
|
||||
}),
|
||||
).not.toHaveProperty("API_CONTRACT_VERSION");
|
||||
});
|
||||
|
||||
it("enforces dependency inventory counts at the executable writer boundary", () => {
|
||||
const inventory = {
|
||||
schemaVersion: 2,
|
||||
packageManager: "pnpm@11.17.0",
|
||||
lockfileSha256: "a".repeat(64),
|
||||
dependencyCount: 1,
|
||||
directDependencyCount: 1,
|
||||
dependencies: [
|
||||
{
|
||||
name: "zod",
|
||||
version: "4.4.3",
|
||||
direct: true,
|
||||
scope: "production",
|
||||
optional: false,
|
||||
license: "MIT",
|
||||
integrity: `sha512-${"a".repeat(86)}`,
|
||||
dependencies: [],
|
||||
},
|
||||
],
|
||||
} as const;
|
||||
|
||||
expect(dependencyInventoryArtifactSchema.parse(inventory)).toEqual(
|
||||
inventory,
|
||||
);
|
||||
expect(() =>
|
||||
dependencyInventoryArtifactSchema.parse({
|
||||
...inventory,
|
||||
dependencyCount: 2,
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("rejects undeclared supply-chain verification evidence", () => {
|
||||
const verification = {
|
||||
schemaVersion: 1,
|
||||
localStatus: "PASS",
|
||||
promotionStatus: "FAIL_UNVERIFIED",
|
||||
lockfileSha256: "a".repeat(64),
|
||||
sourceSetSha256: "b".repeat(64),
|
||||
distSha256: "c".repeat(64),
|
||||
sbomSha256: "d".repeat(64),
|
||||
dependencyDiff: {
|
||||
added: [],
|
||||
removed: [],
|
||||
changed: [],
|
||||
upgrades: [],
|
||||
},
|
||||
highRiskReview: [],
|
||||
vulnerabilityStatus: "FAIL_UNVERIFIED",
|
||||
provenanceAttestationStatus: "FAIL_UNVERIFIED",
|
||||
failures: [],
|
||||
} as const;
|
||||
|
||||
expect(supplyChainVerificationArtifactSchema.parse(verification)).toEqual(
|
||||
verification,
|
||||
);
|
||||
expect(() =>
|
||||
supplyChainVerificationArtifactSchema.parse({
|
||||
...verification,
|
||||
undocumented: true,
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("preserves no-baseline and failure registry evidence", () => {
|
||||
const failureEvidence = {
|
||||
schemaVersion: 2,
|
||||
generatedAt: "2026-08-01T00:00:00.000Z",
|
||||
baselineDigest: null,
|
||||
currentDigest: "a".repeat(64),
|
||||
compatibility: { impact: "not-evaluated", changes: [] },
|
||||
failures: ["missing registry source"],
|
||||
registries: [],
|
||||
} as const;
|
||||
|
||||
expect(registrySnapshotArtifactSchema.parse(failureEvidence)).toEqual(
|
||||
failureEvidence,
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves unverified field evidence when no samples are eligible", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
generatedAt: "2026-08-01T00:00:00.000Z",
|
||||
window: {
|
||||
days: 28,
|
||||
start: "2026-07-04T00:00:00.000Z",
|
||||
end: "2026-08-01T00:00:00.000Z",
|
||||
},
|
||||
context: {
|
||||
source: "config/performance/field-input.example.json",
|
||||
sourceSystem: null,
|
||||
exportId: null,
|
||||
network: "production-real-user",
|
||||
routeAggregation: "route-id-only",
|
||||
releaseId: null,
|
||||
privacyApprovalRef: null,
|
||||
thresholdDecisionRef: null,
|
||||
validationFailures: ["input: invalid evidence"],
|
||||
},
|
||||
metrics: { p75LcpMs: null, p75Cls: null, p75InpMs: null },
|
||||
thresholds: {
|
||||
p75LcpMs: 2_500,
|
||||
p75Cls: 0.1,
|
||||
p75InpMs: 200,
|
||||
minimumEligibleSamples: null,
|
||||
},
|
||||
eligibility: {
|
||||
consentRequired: true,
|
||||
totalSamples: 0,
|
||||
eligibleSamples: 0,
|
||||
minimumEligibleSamples: null,
|
||||
routeSamples: {},
|
||||
},
|
||||
status: "FAIL_UNVERIFIED",
|
||||
passed: false,
|
||||
} as const;
|
||||
|
||||
expect(fieldWebVitalsArtifactSchema.parse(report)).toEqual(report);
|
||||
});
|
||||
|
||||
it("preserves verified field evidence that fails an approved threshold", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
generatedAt: "2026-08-01T00:00:00.000Z",
|
||||
window: {
|
||||
days: 28,
|
||||
start: "2026-07-04T00:00:00.000Z",
|
||||
end: "2026-08-01T00:00:00.000Z",
|
||||
},
|
||||
context: {
|
||||
source: "provider.json",
|
||||
sourceSystem: "provider",
|
||||
exportId: "export-1",
|
||||
network: "production-real-user",
|
||||
routeAggregation: "route-id-only",
|
||||
releaseId: "release-1",
|
||||
privacyApprovalRef: "privacy-1",
|
||||
thresholdDecisionRef: "decision-1",
|
||||
validationFailures: [],
|
||||
},
|
||||
metrics: { p75LcpMs: 2_501, p75Cls: 0.1, p75InpMs: 200 },
|
||||
thresholds: {
|
||||
p75LcpMs: 2_500,
|
||||
p75Cls: 0.1,
|
||||
p75InpMs: 200,
|
||||
minimumEligibleSamples: 1,
|
||||
},
|
||||
eligibility: {
|
||||
consentRequired: true,
|
||||
totalSamples: 1,
|
||||
eligibleSamples: 1,
|
||||
minimumEligibleSamples: 1,
|
||||
routeSamples: { APP_HOME: 1 },
|
||||
},
|
||||
status: "FAIL_THRESHOLD",
|
||||
passed: false,
|
||||
} as const;
|
||||
|
||||
expect(fieldWebVitalsArtifactSchema.parse(report)).toEqual(report);
|
||||
});
|
||||
|
||||
it("accepts the lab writer's nested browser context as JSON evidence", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
generatedAt: "2026-08-01T00:00:00.000Z",
|
||||
context: {
|
||||
runner: { platform: "linux", architecture: "x64", nodeVersion: "v24" },
|
||||
browser: { name: "chromium", version: "140" },
|
||||
viewport: { width: 1280, height: 720 },
|
||||
network: {
|
||||
profile: "contract-fast-4g",
|
||||
latencyMs: 40,
|
||||
downloadBytesPerSecond: 200_000,
|
||||
uploadBytesPerSecond: 93_750,
|
||||
},
|
||||
cpu: { throttlingRate: 4 },
|
||||
cache: { state: "cold", isolation: "new-browser-context" },
|
||||
build: { buildId: "build-1", releaseId: "release-1" },
|
||||
},
|
||||
metrics: { lcpMs: 1_000, cls: 0.01, namedInteractionMs: 100 },
|
||||
thresholds: { lcpMs: 2_500, cls: 0.1, namedInteractionMs: 200 },
|
||||
fixtures: [
|
||||
{ name: "missing-context", passed: true },
|
||||
{ name: "lcp-over-threshold", passed: true },
|
||||
],
|
||||
passed: true,
|
||||
} as const;
|
||||
|
||||
expect(labPerformanceArtifactSchema.parse(report)).toEqual(report);
|
||||
});
|
||||
|
||||
it("rejects non-JSON values in generated schema documents", () => {
|
||||
expect(() =>
|
||||
jsonSchemaDocumentArtifactSchema.parse({
|
||||
$schema: "https://json-schema.org/draft/2020-12/schema",
|
||||
invalid: () => undefined,
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("keeps machine-readable evidence publishers on the validated writer", async () => {
|
||||
const writerFiles = [
|
||||
"scripts/generate-build-manifest.ts",
|
||||
"scripts/generate-supply-chain.ts",
|
||||
"scripts/collect-web-vitals-evidence.ts",
|
||||
"scripts/test-performance.ts",
|
||||
"scripts/verify-release.ts",
|
||||
"scripts/drill-runbook.ts",
|
||||
"scripts/check-registries.ts",
|
||||
] as const;
|
||||
const sources = await Promise.all(
|
||||
writerFiles.map(async (file) => ({
|
||||
file,
|
||||
source: await readFile(file, "utf8"),
|
||||
})),
|
||||
);
|
||||
|
||||
for (const { file, source } of sources) {
|
||||
const directWrites = source.match(/\bwriteFile\s*\(/gu) ?? [];
|
||||
if (file === "scripts/generate-supply-chain.ts") {
|
||||
expect(directWrites, file).toHaveLength(1);
|
||||
expect(source, file).toMatch(
|
||||
/writeFile\(\s*["']artifacts\/release\/checksums\.txt["']/u,
|
||||
);
|
||||
} else {
|
||||
expect(directWrites, file).toHaveLength(0);
|
||||
}
|
||||
}
|
||||
|
||||
expect(
|
||||
sources.flatMap(({ source }) =>
|
||||
source.match(/\bwriteValidatedJsonArtifact\s*\(/gu) ?? [],
|
||||
),
|
||||
).toHaveLength(19);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user