merge: harden field performance evidence

scripts/collect-web-vitals-evidence.mjs

@@ -4,23 +4,19 @@ import {
   evaluateFieldBudget,
   percentile75,
 } from "../src/application/policies/performance-budgets.js";
+import { validateFieldEvidenceInput } from "./lib/field-vitals-evidence.mjs";
 
 const inputPath =
   process.env.FIELD_WEB_VITALS_INPUT ||
   "config/performance/field-input.example.json";
-const input =
-  /** @type {{
-   *   releaseId: string,
-   *   samples: Array<{
-   *     timestamp: string,
-   *     consent: boolean,
-   *     releaseId: string,
-   *     routeId: string,
-   *     lcpMs: number,
-   *     cls: number,
-   *     inpMs: number
-   *   }>
-   * }} */ (JSON.parse(await readFile(inputPath, "utf8")));
+const rawInput = JSON.parse(await readFile(inputPath, "utf8"));
+const now = new Date();
+const validation = validateFieldEvidenceInput(
+  rawInput,
+  process.env.MIN_ELIGIBLE_SAMPLES,
+  now,
+);
+const input = validation.data;
 const configured =
   /** @type {{
    *   p75LcpMs: number,
@@ -30,17 +26,17 @@ const configured =
    * }} */ (
     JSON.parse(await readFile("config/performance/budgets.json", "utf8")).field
   );
-const minimumEligibleSamples = process.env.MIN_ELIGIBLE_SAMPLES
-  ? Number(process.env.MIN_ELIGIBLE_SAMPLES)
-  : configured.minimumEligibleSamples;
-const end = new Date();
-const start = new Date(end);
-start.setUTCDate(start.getUTCDate() - 28);
-const eligible = input.samples.filter((sample) => {
+const minimumEligibleSamples = validation.minimumEligibleSamples;
+const fallbackEnd = now;
+const fallbackStart = new Date(fallbackEnd);
+fallbackStart.setUTCDate(fallbackStart.getUTCDate() - 28);
+const start = input ? new Date(input.window.start) : fallbackStart;
+const end = input ? new Date(input.window.end) : fallbackEnd;
+const eligible = (input?.samples ?? []).filter((sample) => {
   const timestamp = new Date(sample.timestamp);
   return (
     sample.consent === true &&
-    sample.releaseId === input.releaseId &&
+    sample.releaseId === input?.releaseId &&
     timestamp >= start &&
     timestamp <= end
   );
@@ -55,6 +51,8 @@ const result = evaluateFieldBudget(
   { metrics, eligibleSamples: eligible.length },
   thresholds,
 );
+const passed = validation.passed && result.passed;
+const status = validation.passed ? result.status : "FAIL_UNVERIFIED";
 const routeSamples = Object.fromEntries(
   Object.entries(
     eligible.reduce(
@@ -68,24 +66,30 @@ const routeSamples = Object.fromEntries(
 );
 const report = {
   schemaVersion: 1,
-  generatedAt: end.toISOString(),
+  generatedAt: now.toISOString(),
   window: { days: 28, start: start.toISOString(), end: end.toISOString() },
   context: {
     source: inputPath,
+    sourceSystem: input?.source.system ?? null,
+    exportId: input?.source.exportId ?? null,
     network: "production-real-user",
     routeAggregation: "route-id-only",
-    releaseId: input.releaseId,
+    releaseId: input?.releaseId ?? null,
+    privacyApprovalRef: input?.privacy.approvalRef ?? null,
+    thresholdDecisionRef: input?.thresholdDecision.evidenceRef ?? null,
+    validationFailures: validation.failures,
   },
   metrics,
   thresholds,
   eligibility: {
     consentRequired: true,
+    totalSamples: input?.samples.length ?? 0,
     eligibleSamples: eligible.length,
     minimumEligibleSamples,
     routeSamples,
   },
-  status: result.status,
-  passed: result.passed,
+  status,
+  passed,
 };
 
 await mkdir("artifacts/performance", { recursive: true });
@@ -93,9 +97,9 @@ await writeFile(
   "artifacts/performance/field-web-vitals.json",
   `${JSON.stringify(report, null, 2)}\n`,
 );
-if (!result.passed) {
+if (!passed) {
   process.stderr.write(
-    `Field Web Vitals: ${result.status} (minimum eligible sample threshold and 28-day production data are required)\n`,
+    `Field Web Vitals: ${status} (approved threshold decision and valid 28-day production evidence are required)\n`,
   );
   process.exit(1);
 }

scripts/lib/field-vitals-evidence.mjs

@@ -0,0 +1,122 @@
+import { z } from "zod";
+
+const WINDOW_MILLISECONDS = 28 * 24 * 60 * 60 * 1000;
+const nonEmptyString = z.string().trim().min(1);
+const timestamp = nonEmptyString.refine(
+  (value) => Number.isFinite(Date.parse(value)),
+  "must be an RFC 3339 timestamp",
+);
+const sampleSchema = z
+  .object({
+    timestamp,
+    consent: z.boolean(),
+    releaseId: nonEmptyString,
+    routeId: nonEmptyString.regex(/^[A-Z][A-Z0-9_]*$/),
+    lcpMs: z.number().finite().nonnegative(),
+    cls: z.number().finite().nonnegative(),
+    inpMs: z.number().finite().nonnegative(),
+  })
+  .strict();
+
+const fieldEvidenceInputSchema = z
+  .object({
+    schemaVersion: z.literal(1),
+    environment: z.literal("production"),
+    releaseId: nonEmptyString.refine(
+      (value) => value !== "local-release",
+      "must identify an immutable production release",
+    ),
+    source: z
+      .object({
+        system: nonEmptyString,
+        exportId: nonEmptyString,
+      })
+      .strict(),
+    privacy: z
+      .object({
+        approved: z.literal(true),
+        approvalRef: nonEmptyString,
+      })
+      .strict(),
+    window: z
+      .object({
+        start: timestamp,
+        end: timestamp,
+      })
+      .strict(),
+    thresholdDecision: z
+      .object({
+        status: z.literal("approved"),
+        minimumEligibleSamples: z.number().int().positive(),
+        owner: nonEmptyString,
+        reviewedAt: timestamp,
+        evidenceRef: nonEmptyString,
+      })
+      .strict(),
+    samples: z.array(sampleSchema),
+  })
+  .strict()
+  .superRefine((input, context) => {
+    const start = Date.parse(input.window.start);
+    const end = Date.parse(input.window.end);
+    if (end - start !== WINDOW_MILLISECONDS) {
+      context.addIssue({
+        code: "custom",
+        path: ["window"],
+        message: "must cover exactly 28 days",
+      });
+    }
+  });
+
+/**
+ * @param {unknown} input
+ * @param {string | undefined} configuredMinimum
+ * @param {Date} [now]
+ */
+export function validateFieldEvidenceInput(
+  input,
+  configuredMinimum,
+  now = new Date(),
+) {
+  const parsed = fieldEvidenceInputSchema.safeParse(input);
+  const failures = parsed.success
+    ? []
+    : parsed.error.issues.map(
+        (issue) => `${issue.path.join(".") || "input"}: ${issue.message}`,
+      );
+  const minimumEligibleSamples = Number(configuredMinimum);
+  if (
+    configuredMinimum === undefined ||
+    !Number.isInteger(minimumEligibleSamples) ||
+    minimumEligibleSamples <= 0
+  ) {
+    failures.push("MIN_ELIGIBLE_SAMPLES: must be a positive integer");
+  }
+
+  if (parsed.success) {
+    if (
+      parsed.data.thresholdDecision.minimumEligibleSamples !==
+      minimumEligibleSamples
+    ) {
+      failures.push(
+        "MIN_ELIGIBLE_SAMPLES: does not match the approved threshold decision",
+      );
+    }
+    if (Date.parse(parsed.data.window.end) > now.getTime()) {
+      failures.push("window.end: must not be in the future");
+    }
+    if (Date.parse(parsed.data.thresholdDecision.reviewedAt) > now.getTime()) {
+      failures.push("thresholdDecision.reviewedAt: must not be in the future");
+    }
+  }
+
+  return Object.freeze({
+    data: parsed.success ? parsed.data : null,
+    failures: Object.freeze(failures),
+    minimumEligibleSamples:
+      Number.isInteger(minimumEligibleSamples) && minimumEligibleSamples > 0
+        ? minimumEligibleSamples
+        : null,
+    passed: parsed.success && failures.length === 0,
+  });
+}