diff --git a/config/performance/field-input.example.json b/config/performance/field-input.example.json index 9223dae..73b7ac0 100644 --- a/config/performance/field-input.example.json +++ b/config/performance/field-input.example.json @@ -1,5 +1,25 @@ { "schemaVersion": 1, "releaseId": "local-release", + "environment": "replace-with-production", + "source": { + "system": "", + "exportId": "" + }, + "privacy": { + "approved": false, + "approvalRef": "" + }, + "window": { + "start": "2026-06-01T00:00:00Z", + "end": "2026-06-29T00:00:00Z" + }, + "thresholdDecision": { + "status": "pending", + "minimumEligibleSamples": null, + "owner": "", + "reviewedAt": "", + "evidenceRef": "" + }, "samples": [] } diff --git a/config/schemas/field-web-vitals.schema.json b/config/schemas/field-web-vitals.schema.json index d63048c..b49b2b8 100644 --- a/config/schemas/field-web-vitals.schema.json +++ b/config/schemas/field-web-vitals.schema.json @@ -8,6 +8,7 @@ "window", "context", "metrics", + "thresholds", "eligibility", "status", "passed" @@ -18,12 +19,55 @@ "window": { "type": "object", "required": ["days", "start", "end"] }, "context": { "type": "object", - "required": ["source", "network", "routeAggregation", "releaseId"] + "required": [ + "source", + "sourceSystem", + "exportId", + "network", + "routeAggregation", + "releaseId", + "privacyApprovalRef", + "thresholdDecisionRef", + "validationFailures" + ], + "properties": { + "source": { "type": "string" }, + "sourceSystem": { "type": ["string", "null"] }, + "exportId": { "type": ["string", "null"] }, + "network": { "const": "production-real-user" }, + "routeAggregation": { "const": "route-id-only" }, + "releaseId": { "type": ["string", "null"] }, + "privacyApprovalRef": { "type": ["string", "null"] }, + "thresholdDecisionRef": { "type": ["string", "null"] }, + "validationFailures": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": false + }, + "thresholds": { + "type": "object", + "required": [ + "p75LcpMs", + "p75Cls", + "p75InpMs", + "minimumEligibleSamples" + ] + }, + "metrics": { + "type": "object", + "required": ["p75LcpMs", "p75Cls", "p75InpMs"] }, - "metrics": { "type": "object" }, "eligibility": { "type": "object", - "required": ["consentRequired", "eligibleSamples", "minimumEligibleSamples"] + "required": [ + "consentRequired", + "totalSamples", + "eligibleSamples", + "minimumEligibleSamples", + "routeSamples" + ] }, "status": { "enum": ["PASS", "FAIL_THRESHOLD", "FAIL_UNVERIFIED"] diff --git a/docs/operations/performance-evidence.md b/docs/operations/performance-evidence.md index 29864ce..04f6a5b 100644 --- a/docs/operations/performance-evidence.md +++ b/docs/operations/performance-evidence.md @@ -13,5 +13,10 @@ Performance evidence is deliberately split by measurement context: The field minimum eligible-sample threshold is intentionally unresolved until a privacy-approved telemetry baseline exists. Therefore the field command fails closed with `FAIL_UNVERIFIED` when run against the example input. Provide -`FIELD_WEB_VITALS_INPUT` and a reviewed `MIN_ELIGIBLE_SAMPLES` only after that -decision is recorded. +`FIELD_WEB_VITALS_INPUT` and `MIN_ELIGIBLE_SAMPLES` only after that decision is +recorded. The external input must identify a production release and an exact +28-day export window, name the source/export, carry privacy-approval and +threshold-decision references, and contain only non-negative route-ID samples. +The environment threshold must be a positive integer equal to the approved +decision embedded in the input. Invalid metadata fails as `FAIL_UNVERIFIED`; +the example can never serve as production evidence. diff --git a/scripts/collect-web-vitals-evidence.mjs b/scripts/collect-web-vitals-evidence.mjs index 4eac58c..6abd763 100644 --- a/scripts/collect-web-vitals-evidence.mjs +++ b/scripts/collect-web-vitals-evidence.mjs @@ -4,23 +4,19 @@ import { evaluateFieldBudget, percentile75, } from "../src/application/policies/performance-budgets.js"; +import { validateFieldEvidenceInput } from "./lib/field-vitals-evidence.mjs"; const inputPath = process.env.FIELD_WEB_VITALS_INPUT ?? "config/performance/field-input.example.json"; -const input = - /** @type {{ - * releaseId: string, - * samples: Array<{ - * timestamp: string, - * consent: boolean, - * releaseId: string, - * routeId: string, - * lcpMs: number, - * cls: number, - * inpMs: number - * }> - * }} */ (JSON.parse(await readFile(inputPath, "utf8"))); +const rawInput = JSON.parse(await readFile(inputPath, "utf8")); +const now = new Date(); +const validation = validateFieldEvidenceInput( + rawInput, + process.env.MIN_ELIGIBLE_SAMPLES, + now, +); +const input = validation.data; const configured = /** @type {{ * p75LcpMs: number, @@ -30,17 +26,17 @@ const configured = * }} */ ( JSON.parse(await readFile("config/performance/budgets.json", "utf8")).field ); -const minimumEligibleSamples = process.env.MIN_ELIGIBLE_SAMPLES - ? Number(process.env.MIN_ELIGIBLE_SAMPLES) - : configured.minimumEligibleSamples; -const end = new Date(); -const start = new Date(end); -start.setUTCDate(start.getUTCDate() - 28); -const eligible = input.samples.filter((sample) => { +const minimumEligibleSamples = validation.minimumEligibleSamples; +const fallbackEnd = now; +const fallbackStart = new Date(fallbackEnd); +fallbackStart.setUTCDate(fallbackStart.getUTCDate() - 28); +const start = input ? new Date(input.window.start) : fallbackStart; +const end = input ? new Date(input.window.end) : fallbackEnd; +const eligible = (input?.samples ?? []).filter((sample) => { const timestamp = new Date(sample.timestamp); return ( sample.consent === true && - sample.releaseId === input.releaseId && + sample.releaseId === input?.releaseId && timestamp >= start && timestamp <= end ); @@ -55,6 +51,8 @@ const result = evaluateFieldBudget( { metrics, eligibleSamples: eligible.length }, thresholds, ); +const passed = validation.passed && result.passed; +const status = validation.passed ? result.status : "FAIL_UNVERIFIED"; const routeSamples = Object.fromEntries( Object.entries( eligible.reduce( @@ -68,24 +66,30 @@ const routeSamples = Object.fromEntries( ); const report = { schemaVersion: 1, - generatedAt: end.toISOString(), + generatedAt: now.toISOString(), window: { days: 28, start: start.toISOString(), end: end.toISOString() }, context: { source: inputPath, + sourceSystem: input?.source.system ?? null, + exportId: input?.source.exportId ?? null, network: "production-real-user", routeAggregation: "route-id-only", - releaseId: input.releaseId, + releaseId: input?.releaseId ?? null, + privacyApprovalRef: input?.privacy.approvalRef ?? null, + thresholdDecisionRef: input?.thresholdDecision.evidenceRef ?? null, + validationFailures: validation.failures, }, metrics, thresholds, eligibility: { consentRequired: true, + totalSamples: input?.samples.length ?? 0, eligibleSamples: eligible.length, minimumEligibleSamples, routeSamples, }, - status: result.status, - passed: result.passed, + status, + passed, }; await mkdir("artifacts/performance", { recursive: true }); @@ -93,9 +97,9 @@ await writeFile( "artifacts/performance/field-web-vitals.json", `${JSON.stringify(report, null, 2)}\n`, ); -if (!result.passed) { +if (!passed) { process.stderr.write( - `Field Web Vitals: ${result.status} (minimum eligible sample threshold and 28-day production data are required)\n`, + `Field Web Vitals: ${status} (approved threshold decision and valid 28-day production evidence are required)\n`, ); process.exit(1); } diff --git a/scripts/lib/field-vitals-evidence.mjs b/scripts/lib/field-vitals-evidence.mjs new file mode 100644 index 0000000..c0dffb3 --- /dev/null +++ b/scripts/lib/field-vitals-evidence.mjs @@ -0,0 +1,122 @@ +import { z } from "zod"; + +const WINDOW_MILLISECONDS = 28 * 24 * 60 * 60 * 1000; +const nonEmptyString = z.string().trim().min(1); +const timestamp = nonEmptyString.refine( + (value) => Number.isFinite(Date.parse(value)), + "must be an RFC 3339 timestamp", +); +const sampleSchema = z + .object({ + timestamp, + consent: z.boolean(), + releaseId: nonEmptyString, + routeId: nonEmptyString.regex(/^[A-Z][A-Z0-9_]*$/), + lcpMs: z.number().finite().nonnegative(), + cls: z.number().finite().nonnegative(), + inpMs: z.number().finite().nonnegative(), + }) + .strict(); + +const fieldEvidenceInputSchema = z + .object({ + schemaVersion: z.literal(1), + environment: z.literal("production"), + releaseId: nonEmptyString.refine( + (value) => value !== "local-release", + "must identify an immutable production release", + ), + source: z + .object({ + system: nonEmptyString, + exportId: nonEmptyString, + }) + .strict(), + privacy: z + .object({ + approved: z.literal(true), + approvalRef: nonEmptyString, + }) + .strict(), + window: z + .object({ + start: timestamp, + end: timestamp, + }) + .strict(), + thresholdDecision: z + .object({ + status: z.literal("approved"), + minimumEligibleSamples: z.number().int().positive(), + owner: nonEmptyString, + reviewedAt: timestamp, + evidenceRef: nonEmptyString, + }) + .strict(), + samples: z.array(sampleSchema), + }) + .strict() + .superRefine((input, context) => { + const start = Date.parse(input.window.start); + const end = Date.parse(input.window.end); + if (end - start !== WINDOW_MILLISECONDS) { + context.addIssue({ + code: "custom", + path: ["window"], + message: "must cover exactly 28 days", + }); + } + }); + +/** + * @param {unknown} input + * @param {string | undefined} configuredMinimum + * @param {Date} [now] + */ +export function validateFieldEvidenceInput( + input, + configuredMinimum, + now = new Date(), +) { + const parsed = fieldEvidenceInputSchema.safeParse(input); + const failures = parsed.success + ? [] + : parsed.error.issues.map( + (issue) => `${issue.path.join(".") || "input"}: ${issue.message}`, + ); + const minimumEligibleSamples = Number(configuredMinimum); + if ( + configuredMinimum === undefined || + !Number.isInteger(minimumEligibleSamples) || + minimumEligibleSamples <= 0 + ) { + failures.push("MIN_ELIGIBLE_SAMPLES: must be a positive integer"); + } + + if (parsed.success) { + if ( + parsed.data.thresholdDecision.minimumEligibleSamples !== + minimumEligibleSamples + ) { + failures.push( + "MIN_ELIGIBLE_SAMPLES: does not match the approved threshold decision", + ); + } + if (Date.parse(parsed.data.window.end) > now.getTime()) { + failures.push("window.end: must not be in the future"); + } + if (Date.parse(parsed.data.thresholdDecision.reviewedAt) > now.getTime()) { + failures.push("thresholdDecision.reviewedAt: must not be in the future"); + } + } + + return Object.freeze({ + data: parsed.success ? parsed.data : null, + failures: Object.freeze(failures), + minimumEligibleSamples: + Number.isInteger(minimumEligibleSamples) && minimumEligibleSamples > 0 + ? minimumEligibleSamples + : null, + passed: parsed.success && failures.length === 0, + }); +} diff --git a/tests/unit/field-vitals-evidence.test.js b/tests/unit/field-vitals-evidence.test.js new file mode 100644 index 0000000..c0ca57c --- /dev/null +++ b/tests/unit/field-vitals-evidence.test.js @@ -0,0 +1,94 @@ +import { describe, expect, it } from "vitest"; + +import { validateFieldEvidenceInput } from "../../scripts/lib/field-vitals-evidence.mjs"; + +const input = { + schemaVersion: 1, + environment: "production", + releaseId: "release-2026-06-29", + source: { + system: "privacy-approved-rum-export", + exportId: "export-2026-06-29", + }, + privacy: { + approved: true, + approvalRef: "PRIVACY-42", + }, + window: { + start: "2026-06-01T00:00:00Z", + end: "2026-06-29T00:00:00Z", + }, + thresholdDecision: { + status: "approved", + minimumEligibleSamples: 25, + owner: "performance-owner", + reviewedAt: "2026-06-30T00:00:00Z", + evidenceRef: "PERF-BASELINE-7", + }, + samples: [ + { + timestamp: "2026-06-20T00:00:00Z", + consent: true, + releaseId: "release-2026-06-29", + routeId: "APP_HOME", + lcpMs: 1200, + cls: 0.01, + inpMs: 80, + }, + ], +}; + +describe("field Web Vitals evidence input", () => { + it("accepts reviewed, coherent 28-day production metadata", () => { + expect( + validateFieldEvidenceInput( + input, + "25", + new Date("2026-07-01T00:00:00Z"), + ), + ).toMatchObject({ + failures: [], + minimumEligibleSamples: 25, + passed: true, + }); + }); + + it("rejects a threshold that does not match the owner decision", () => { + expect( + validateFieldEvidenceInput( + input, + "10", + new Date("2026-07-01T00:00:00Z"), + ), + ).toMatchObject({ + failures: [ + "MIN_ELIGIBLE_SAMPLES: does not match the approved threshold decision", + ], + passed: false, + }); + }); + + it("rejects local, unapproved, malformed, or impossible measurements", () => { + const invalid = { + ...input, + environment: "local", + releaseId: "local-release", + privacy: { approved: false, approvalRef: "" }, + window: { ...input.window, end: "2026-06-28T00:00:00Z" }, + samples: [{ ...input.samples[0], lcpMs: -1 }], + }; + const validation = validateFieldEvidenceInput( + invalid, + "-1", + new Date("2026-07-01T00:00:00Z"), + ); + expect(validation.passed).toBe(false); + expect(validation.failures.join("\n")).toContain("environment"); + expect(validation.failures.join("\n")).toContain("releaseId"); + expect(validation.failures.join("\n")).toContain("privacy"); + expect(validation.failures.join("\n")).toContain("lcpMs"); + expect(validation.failures.join("\n")).toContain( + "MIN_ELIGIBLE_SAMPLES: must be a positive integer", + ); + }); +});