refactor: validate generated evidence artifacts
This commit is contained in:
DongHyeonka
@@ -1,9 +1,28 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { generateArtifactSchemas } from "../../scripts/generate-artifact-schemas.ts";
|
||||
import { assertMatchesJsonSchema } from "../../scripts/lib/json-schema.ts";
|
||||
|
||||
const temporaryDirectories: string[] = [];
|
||||
|
||||
async function temporaryDirectory(): Promise<string> {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), "artifact-schemas-"));
|
||||
temporaryDirectories.push(directory);
|
||||
return directory;
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporaryDirectories.splice(0).map((directory) =>
|
||||
rm(directory, { force: true, recursive: true }),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
async function json(path: string): Promise<unknown> {
|
||||
return JSON.parse(await readFile(path, "utf8")) as unknown;
|
||||
}
|
||||
@@ -55,4 +74,130 @@ describe("checked-in JSON Schema execution", () => {
|
||||
assertMatchesJsonSchema(schema, catalog, "optional recipe catalog"),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("generates deterministic draft 2020-12 artifact schemas and detects drift", async () => {
|
||||
const root = await temporaryDirectory();
|
||||
await generateArtifactSchemas({ root });
|
||||
const generatedPath = path.join(
|
||||
root,
|
||||
"schemas/artifacts/dependency-inventory.schema.json",
|
||||
);
|
||||
const generated = await readFile(generatedPath, "utf8");
|
||||
expect(generated.endsWith("\n")).toBe(true);
|
||||
expect(JSON.parse(generated)).toMatchObject({
|
||||
$schema: "https://json-schema.org/draft/2020-12/schema",
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
});
|
||||
|
||||
await generateArtifactSchemas({ root, check: true });
|
||||
await writeFile(generatedPath, `${generated.trimEnd()} \n`, "utf8");
|
||||
|
||||
await expect(
|
||||
generateArtifactSchemas({ root, check: true }),
|
||||
).rejects.toThrow(/dependency-inventory\.schema\.json/u);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "dependency inventory",
|
||||
path: "schemas/artifacts/dependency-inventory.schema.json",
|
||||
value: {
|
||||
schemaVersion: 2,
|
||||
packageManager: "pnpm@11.17.0",
|
||||
lockfileSha256: "a".repeat(64),
|
||||
dependencyCount: 1,
|
||||
directDependencyCount: 1,
|
||||
dependencies: [
|
||||
{
|
||||
name: "zod",
|
||||
version: "4.4.3",
|
||||
direct: true,
|
||||
scope: "production",
|
||||
optional: false,
|
||||
license: "MIT",
|
||||
integrity: `sha512-${"a".repeat(86)}`,
|
||||
dependencies: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "registry snapshot",
|
||||
path: "schemas/artifacts/registry-snapshot.schema.json",
|
||||
value: {
|
||||
schemaVersion: 2,
|
||||
generatedAt: "2026-08-01T00:00:00.000Z",
|
||||
baselineDigest: "b".repeat(64),
|
||||
currentDigest: "c".repeat(64),
|
||||
compatibility: { impact: "none", changes: [] },
|
||||
failures: [],
|
||||
registries: Array.from({ length: 11 }, (_, index) => ({
|
||||
registryId: `registry-${index}`,
|
||||
owner: "platform",
|
||||
source: `src/registry-${index}.ts`,
|
||||
rowCount: 0,
|
||||
contract: {},
|
||||
rows: {},
|
||||
})),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "supply-chain verification",
|
||||
path: "schemas/artifacts/supply-chain-verification.schema.json",
|
||||
value: {
|
||||
schemaVersion: 1,
|
||||
localStatus: "PASS",
|
||||
promotionStatus: "FAIL_UNVERIFIED",
|
||||
lockfileSha256: "d".repeat(64),
|
||||
sourceSetSha256: "e".repeat(64),
|
||||
distSha256: "f".repeat(64),
|
||||
sbomSha256: "0".repeat(64),
|
||||
dependencyDiff: {
|
||||
added: [],
|
||||
removed: [],
|
||||
changed: [],
|
||||
upgrades: [],
|
||||
},
|
||||
highRiskReview: [],
|
||||
vulnerabilityStatus: "FAIL_UNVERIFIED",
|
||||
provenanceAttestationStatus: "FAIL_UNVERIFIED",
|
||||
failures: [],
|
||||
},
|
||||
},
|
||||
])("accepts $name and rejects undeclared top-level fields", async (fixture) => {
|
||||
const schema = await json(fixture.path);
|
||||
|
||||
expect(() =>
|
||||
assertMatchesJsonSchema(schema, fixture.value, fixture.name),
|
||||
).not.toThrow();
|
||||
expect(() =>
|
||||
assertMatchesJsonSchema(
|
||||
schema,
|
||||
{ ...fixture.value, undocumented: true },
|
||||
fixture.name,
|
||||
),
|
||||
).toThrow(/checked-in JSON Schema/u);
|
||||
if (fixture.name === "dependency inventory") {
|
||||
const dependency = fixture.value.dependencies?.[0];
|
||||
if (dependency === undefined) {
|
||||
throw new TypeError("dependency inventory fixture is incomplete");
|
||||
}
|
||||
expect(() =>
|
||||
assertMatchesJsonSchema(
|
||||
schema,
|
||||
{
|
||||
...fixture.value,
|
||||
dependencies: [
|
||||
{
|
||||
...dependency,
|
||||
undocumented: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
fixture.name,
|
||||
),
|
||||
).toThrow(/checked-in JSON Schema/u);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,10 +1,18 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
dependencyInventoryArtifactSchema,
|
||||
fieldWebVitalsArtifactSchema,
|
||||
jsonSchemaDocumentArtifactSchema,
|
||||
labPerformanceArtifactSchema,
|
||||
parseBuildManifestArtifact,
|
||||
parseReleaseArtifact,
|
||||
parseRuntimeConfigArtifact,
|
||||
projectReleaseTokens,
|
||||
registrySnapshotArtifactSchema,
|
||||
supplyChainVerificationArtifactSchema,
|
||||
} from "../../scripts/contracts/release-artifacts.ts";
|
||||
|
||||
const EMPTY_CONTRACT_SET_DIGEST =
|
||||
@@ -126,4 +134,241 @@ describe("release artifact contracts", () => {
|
||||
}),
|
||||
).not.toHaveProperty("API_CONTRACT_VERSION");
|
||||
});
|
||||
|
||||
it("enforces dependency inventory counts at the executable writer boundary", () => {
|
||||
const inventory = {
|
||||
schemaVersion: 2,
|
||||
packageManager: "pnpm@11.17.0",
|
||||
lockfileSha256: "a".repeat(64),
|
||||
dependencyCount: 1,
|
||||
directDependencyCount: 1,
|
||||
dependencies: [
|
||||
{
|
||||
name: "zod",
|
||||
version: "4.4.3",
|
||||
direct: true,
|
||||
scope: "production",
|
||||
optional: false,
|
||||
license: "MIT",
|
||||
integrity: `sha512-${"a".repeat(86)}`,
|
||||
dependencies: [],
|
||||
},
|
||||
],
|
||||
} as const;
|
||||
|
||||
expect(dependencyInventoryArtifactSchema.parse(inventory)).toEqual(
|
||||
inventory,
|
||||
);
|
||||
expect(() =>
|
||||
dependencyInventoryArtifactSchema.parse({
|
||||
...inventory,
|
||||
dependencyCount: 2,
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("rejects undeclared supply-chain verification evidence", () => {
|
||||
const verification = {
|
||||
schemaVersion: 1,
|
||||
localStatus: "PASS",
|
||||
promotionStatus: "FAIL_UNVERIFIED",
|
||||
lockfileSha256: "a".repeat(64),
|
||||
sourceSetSha256: "b".repeat(64),
|
||||
distSha256: "c".repeat(64),
|
||||
sbomSha256: "d".repeat(64),
|
||||
dependencyDiff: {
|
||||
added: [],
|
||||
removed: [],
|
||||
changed: [],
|
||||
upgrades: [],
|
||||
},
|
||||
highRiskReview: [],
|
||||
vulnerabilityStatus: "FAIL_UNVERIFIED",
|
||||
provenanceAttestationStatus: "FAIL_UNVERIFIED",
|
||||
failures: [],
|
||||
} as const;
|
||||
|
||||
expect(supplyChainVerificationArtifactSchema.parse(verification)).toEqual(
|
||||
verification,
|
||||
);
|
||||
expect(() =>
|
||||
supplyChainVerificationArtifactSchema.parse({
|
||||
...verification,
|
||||
undocumented: true,
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("preserves no-baseline and failure registry evidence", () => {
|
||||
const failureEvidence = {
|
||||
schemaVersion: 2,
|
||||
generatedAt: "2026-08-01T00:00:00.000Z",
|
||||
baselineDigest: null,
|
||||
currentDigest: "a".repeat(64),
|
||||
compatibility: { impact: "not-evaluated", changes: [] },
|
||||
failures: ["missing registry source"],
|
||||
registries: [],
|
||||
} as const;
|
||||
|
||||
expect(registrySnapshotArtifactSchema.parse(failureEvidence)).toEqual(
|
||||
failureEvidence,
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves unverified field evidence when no samples are eligible", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
generatedAt: "2026-08-01T00:00:00.000Z",
|
||||
window: {
|
||||
days: 28,
|
||||
start: "2026-07-04T00:00:00.000Z",
|
||||
end: "2026-08-01T00:00:00.000Z",
|
||||
},
|
||||
context: {
|
||||
source: "config/performance/field-input.example.json",
|
||||
sourceSystem: null,
|
||||
exportId: null,
|
||||
network: "production-real-user",
|
||||
routeAggregation: "route-id-only",
|
||||
releaseId: null,
|
||||
privacyApprovalRef: null,
|
||||
thresholdDecisionRef: null,
|
||||
validationFailures: ["input: invalid evidence"],
|
||||
},
|
||||
metrics: { p75LcpMs: null, p75Cls: null, p75InpMs: null },
|
||||
thresholds: {
|
||||
p75LcpMs: 2_500,
|
||||
p75Cls: 0.1,
|
||||
p75InpMs: 200,
|
||||
minimumEligibleSamples: null,
|
||||
},
|
||||
eligibility: {
|
||||
consentRequired: true,
|
||||
totalSamples: 0,
|
||||
eligibleSamples: 0,
|
||||
minimumEligibleSamples: null,
|
||||
routeSamples: {},
|
||||
},
|
||||
status: "FAIL_UNVERIFIED",
|
||||
passed: false,
|
||||
} as const;
|
||||
|
||||
expect(fieldWebVitalsArtifactSchema.parse(report)).toEqual(report);
|
||||
});
|
||||
|
||||
it("preserves verified field evidence that fails an approved threshold", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
generatedAt: "2026-08-01T00:00:00.000Z",
|
||||
window: {
|
||||
days: 28,
|
||||
start: "2026-07-04T00:00:00.000Z",
|
||||
end: "2026-08-01T00:00:00.000Z",
|
||||
},
|
||||
context: {
|
||||
source: "provider.json",
|
||||
sourceSystem: "provider",
|
||||
exportId: "export-1",
|
||||
network: "production-real-user",
|
||||
routeAggregation: "route-id-only",
|
||||
releaseId: "release-1",
|
||||
privacyApprovalRef: "privacy-1",
|
||||
thresholdDecisionRef: "decision-1",
|
||||
validationFailures: [],
|
||||
},
|
||||
metrics: { p75LcpMs: 2_501, p75Cls: 0.1, p75InpMs: 200 },
|
||||
thresholds: {
|
||||
p75LcpMs: 2_500,
|
||||
p75Cls: 0.1,
|
||||
p75InpMs: 200,
|
||||
minimumEligibleSamples: 1,
|
||||
},
|
||||
eligibility: {
|
||||
consentRequired: true,
|
||||
totalSamples: 1,
|
||||
eligibleSamples: 1,
|
||||
minimumEligibleSamples: 1,
|
||||
routeSamples: { APP_HOME: 1 },
|
||||
},
|
||||
status: "FAIL_THRESHOLD",
|
||||
passed: false,
|
||||
} as const;
|
||||
|
||||
expect(fieldWebVitalsArtifactSchema.parse(report)).toEqual(report);
|
||||
});
|
||||
|
||||
it("accepts the lab writer's nested browser context as JSON evidence", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
generatedAt: "2026-08-01T00:00:00.000Z",
|
||||
context: {
|
||||
runner: { platform: "linux", architecture: "x64", nodeVersion: "v24" },
|
||||
browser: { name: "chromium", version: "140" },
|
||||
viewport: { width: 1280, height: 720 },
|
||||
network: {
|
||||
profile: "contract-fast-4g",
|
||||
latencyMs: 40,
|
||||
downloadBytesPerSecond: 200_000,
|
||||
uploadBytesPerSecond: 93_750,
|
||||
},
|
||||
cpu: { throttlingRate: 4 },
|
||||
cache: { state: "cold", isolation: "new-browser-context" },
|
||||
build: { buildId: "build-1", releaseId: "release-1" },
|
||||
},
|
||||
metrics: { lcpMs: 1_000, cls: 0.01, namedInteractionMs: 100 },
|
||||
thresholds: { lcpMs: 2_500, cls: 0.1, namedInteractionMs: 200 },
|
||||
fixtures: [
|
||||
{ name: "missing-context", passed: true },
|
||||
{ name: "lcp-over-threshold", passed: true },
|
||||
],
|
||||
passed: true,
|
||||
} as const;
|
||||
|
||||
expect(labPerformanceArtifactSchema.parse(report)).toEqual(report);
|
||||
});
|
||||
|
||||
it("rejects non-JSON values in generated schema documents", () => {
|
||||
expect(() =>
|
||||
jsonSchemaDocumentArtifactSchema.parse({
|
||||
$schema: "https://json-schema.org/draft/2020-12/schema",
|
||||
invalid: () => undefined,
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("keeps machine-readable evidence publishers on the validated writer", async () => {
|
||||
const writerFiles = [
|
||||
"scripts/generate-build-manifest.ts",
|
||||
"scripts/generate-supply-chain.ts",
|
||||
"scripts/collect-web-vitals-evidence.ts",
|
||||
"scripts/test-performance.ts",
|
||||
"scripts/verify-release.ts",
|
||||
"scripts/drill-runbook.ts",
|
||||
"scripts/check-registries.ts",
|
||||
] as const;
|
||||
const sources = await Promise.all(
|
||||
writerFiles.map(async (file) => ({
|
||||
file,
|
||||
source: await readFile(file, "utf8"),
|
||||
})),
|
||||
);
|
||||
|
||||
for (const { file, source } of sources) {
|
||||
const directWrites = source.match(/\bwriteFile\s*\(/gu) ?? [];
|
||||
if (file === "scripts/generate-supply-chain.ts") {
|
||||
expect(directWrites, file).toHaveLength(1);
|
||||
expect(source, file).toMatch(
|
||||
/writeFile\(\s*["']artifacts\/release\/checksums\.txt["']/u,
|
||||
);
|
||||
} else {
|
||||
expect(directWrites, file).toHaveLength(0);
|
||||
}
|
||||
}
|
||||
|
||||
expect(
|
||||
sources.flatMap(({ source }) =>
|
||||
source.match(/\bwriteValidatedJsonArtifact\s*\(/gu) ?? [],
|
||||
),
|
||||
).toHaveLength(19);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
import {
|
||||
mkdtemp,
|
||||
open,
|
||||
readFile,
|
||||
readdir,
|
||||
rename,
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
|
||||
import {
|
||||
createValidatedJsonArtifactWriter,
|
||||
writeValidatedJsonArtifact,
|
||||
} from "../../scripts/lib/validated-json-artifact.ts";
|
||||
|
||||
const artifactSchema = z
|
||||
.object({ schemaVersion: z.literal(1), name: z.string().min(1) })
|
||||
.strict();
|
||||
|
||||
const temporaryDirectories: string[] = [];
|
||||
|
||||
async function temporaryDirectory(): Promise<string> {
|
||||
const directory = await mkdtemp(
|
||||
path.join(tmpdir(), "validated-json-artifact-"),
|
||||
);
|
||||
temporaryDirectories.push(directory);
|
||||
return directory;
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporaryDirectories.splice(0).map((directory) =>
|
||||
rm(directory, { force: true, recursive: true }),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
describe("validated JSON artifact writer", () => {
|
||||
it.each(["existing", "missing"] as const)(
|
||||
"rejects invalid %s artifacts before changing destination state",
|
||||
async (destinationState) => {
|
||||
const directory = await temporaryDirectory();
|
||||
const destination = path.join(directory, "artifact.json");
|
||||
if (destinationState === "existing") {
|
||||
await writeFile(destination, "previous-bytes\n", "utf8");
|
||||
}
|
||||
const entriesBefore = await readdir(directory);
|
||||
|
||||
await expect(
|
||||
writeValidatedJsonArtifact({
|
||||
path: destination,
|
||||
schema: artifactSchema,
|
||||
value: { schemaVersion: 1, name: "" },
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
|
||||
expect(await readdir(directory)).toEqual(entriesBefore);
|
||||
if (destinationState === "existing") {
|
||||
await expect(readFile(destination, "utf8")).resolves.toBe(
|
||||
"previous-bytes\n",
|
||||
);
|
||||
} else {
|
||||
await expect(readFile(destination, "utf8")).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("publishes complete formatted bytes through a sibling rename", async () => {
|
||||
const directory = await temporaryDirectory();
|
||||
const destination = path.join(directory, "artifact.json");
|
||||
|
||||
await writeValidatedJsonArtifact({
|
||||
path: destination,
|
||||
schema: artifactSchema,
|
||||
value: { schemaVersion: 1, name: "valid" },
|
||||
});
|
||||
|
||||
await expect(readFile(destination, "utf8")).resolves.toBe(
|
||||
'{\n "schemaVersion": 1,\n "name": "valid"\n}\n',
|
||||
);
|
||||
expect(await readdir(directory)).toEqual(["artifact.json"]);
|
||||
});
|
||||
|
||||
it.each(["write", "rename"] as const)(
|
||||
"cleans only its owned sibling temp when %s fails",
|
||||
async (failurePoint) => {
|
||||
const directory = await temporaryDirectory();
|
||||
const destination = path.join(directory, "artifact.json");
|
||||
const unrelatedTemp = path.join(directory, ".artifact.json.unrelated.tmp");
|
||||
const ownedTemp = path.join(directory, ".artifact.json.owned.tmp");
|
||||
await writeFile(destination, "previous-bytes\n", "utf8");
|
||||
await writeFile(unrelatedTemp, "unrelated\n", "utf8");
|
||||
const writer = createValidatedJsonArtifactWriter({
|
||||
createNonce: () => "owned",
|
||||
fileSystem: {
|
||||
open: async (target, flags) => {
|
||||
const handle = await open(target, flags);
|
||||
return {
|
||||
writeFile: async (data, encoding) => {
|
||||
await handle.writeFile(data, encoding);
|
||||
if (failurePoint === "write") {
|
||||
throw new Error("injected write failure");
|
||||
}
|
||||
},
|
||||
close: async () => handle.close(),
|
||||
};
|
||||
},
|
||||
rename: async (source, target) => {
|
||||
if (failurePoint === "rename") {
|
||||
throw new Error("injected rename failure");
|
||||
}
|
||||
await rename(source, target);
|
||||
},
|
||||
rm,
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
writer({
|
||||
path: destination,
|
||||
schema: artifactSchema,
|
||||
value: { schemaVersion: 1, name: "valid" },
|
||||
}),
|
||||
).rejects.toThrow(`injected ${failurePoint} failure`);
|
||||
|
||||
await expect(readFile(destination, "utf8")).resolves.toBe(
|
||||
"previous-bytes\n",
|
||||
);
|
||||
await expect(readFile(unrelatedTemp, "utf8")).resolves.toBe(
|
||||
"unrelated\n",
|
||||
);
|
||||
await expect(readFile(ownedTemp, "utf8")).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("does not delete a pre-existing colliding sibling temp", async () => {
|
||||
const directory = await temporaryDirectory();
|
||||
const destination = path.join(directory, "artifact.json");
|
||||
const collidingTemp = path.join(directory, ".artifact.json.collision.tmp");
|
||||
await writeFile(collidingTemp, "another-writer\n", "utf8");
|
||||
const writer = createValidatedJsonArtifactWriter({
|
||||
createNonce: () => "collision",
|
||||
});
|
||||
|
||||
await expect(
|
||||
writer({
|
||||
path: destination,
|
||||
schema: artifactSchema,
|
||||
value: { schemaVersion: 1, name: "valid" },
|
||||
}),
|
||||
).rejects.toMatchObject({ code: "EEXIST" });
|
||||
|
||||
await expect(readFile(collidingTemp, "utf8")).resolves.toBe(
|
||||
"another-writer\n",
|
||||
);
|
||||
await expect(readFile(destination, "utf8")).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user