feat: harden test and registry evidence
This commit is contained in:
donghyeon-ka
+243
-56
@@ -1,28 +1,65 @@
|
||||
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import {
|
||||
access,
|
||||
mkdir,
|
||||
readFile,
|
||||
readdir,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
/** @param {string} name @param {string} fallback */
|
||||
import {
|
||||
canonicalizeRegistryValue,
|
||||
diffRegistrySnapshots,
|
||||
registrySnapshotDigest,
|
||||
validateBreakingEvidence,
|
||||
verifyRegistryBaselineApproval,
|
||||
} from "./lib/registry-compatibility.mjs";
|
||||
|
||||
/** @param {string} name @param {string | undefined} fallback */
|
||||
function argumentValue(name, fallback) {
|
||||
const index = process.argv.indexOf(name);
|
||||
return index >= 0 && process.argv[index + 1] ? process.argv[index + 1] : fallback;
|
||||
return index >= 0 && process.argv[index + 1]
|
||||
? process.argv[index + 1]
|
||||
: fallback;
|
||||
}
|
||||
|
||||
const governancePath = argumentValue(
|
||||
"--governance",
|
||||
"config/contracts/registry-governance.json",
|
||||
const defaultGovernancePath = "config/contracts/registry-governance.json";
|
||||
const governancePath =
|
||||
/** @type {string} */ (
|
||||
argumentValue("--governance", defaultGovernancePath)
|
||||
);
|
||||
const artifactPath =
|
||||
/** @type {string} */ (
|
||||
argumentValue("--artifact", "artifacts/quality/registries.json")
|
||||
);
|
||||
const usesRepositoryBaseline =
|
||||
governancePath === defaultGovernancePath &&
|
||||
!process.argv.includes("--no-baseline");
|
||||
const baselinePath = argumentValue(
|
||||
"--baseline",
|
||||
usesRepositoryBaseline
|
||||
? "config/contracts/registry-baseline.json"
|
||||
: undefined,
|
||||
);
|
||||
const artifactPath = argumentValue(
|
||||
"--artifact",
|
||||
"artifacts/quality/registries.json",
|
||||
const approvalPath = argumentValue(
|
||||
"--approval",
|
||||
usesRepositoryBaseline
|
||||
? "config/contracts/registry-baseline.approval.json"
|
||||
: undefined,
|
||||
);
|
||||
const governance = JSON.parse(
|
||||
await readFile(governancePath, "utf8"),
|
||||
const evidencePath = argumentValue(
|
||||
"--compatibility-evidence",
|
||||
usesRepositoryBaseline
|
||||
? "config/contracts/registry-change-evidence.json"
|
||||
: undefined,
|
||||
);
|
||||
const governance = JSON.parse(await readFile(governancePath, "utf8"));
|
||||
const failures = [];
|
||||
const owners = new Map();
|
||||
const snapshots = [];
|
||||
const rowsByRegistry = new Map();
|
||||
const sourcesByRegistry = new Map();
|
||||
const registryExtensions = [".js", ".jsx", ".mjs", ".ts", ".tsx", ".mts"];
|
||||
|
||||
/** @param {string} declaredPath */
|
||||
@@ -38,7 +75,7 @@ async function resolveRegistrySource(declaredPath) {
|
||||
await access(candidate);
|
||||
candidates.push(candidate);
|
||||
} catch {
|
||||
// A migration may legitimately replace the declared extension.
|
||||
// A TypeScript migration may replace the declared extension.
|
||||
}
|
||||
}
|
||||
if (candidates.length > 1) {
|
||||
@@ -50,6 +87,44 @@ async function resolveRegistrySource(declaredPath) {
|
||||
return candidates[0] ?? null;
|
||||
}
|
||||
|
||||
/** @param {unknown} value */
|
||||
function runtimeType(value) {
|
||||
if (value === null) return "null";
|
||||
if (Array.isArray(value)) return "array";
|
||||
if (Number.isInteger(value)) return "integer";
|
||||
return typeof value;
|
||||
}
|
||||
|
||||
/** @param {unknown} value @param {string} declaration */
|
||||
function matchesDeclaredType(value, declaration) {
|
||||
const actual = runtimeType(value);
|
||||
return declaration
|
||||
.split("|")
|
||||
.some(
|
||||
(candidate) =>
|
||||
candidate === actual ||
|
||||
(candidate === "number" && actual === "integer"),
|
||||
);
|
||||
}
|
||||
|
||||
/** @param {string} directory @returns {Promise<string[]>} */
|
||||
async function filesBelow(directory) {
|
||||
try {
|
||||
const entries = await readdir(directory, { withFileTypes: true });
|
||||
const groups = await Promise.all(
|
||||
entries.map((entry) => {
|
||||
const target = path.join(directory, entry.name);
|
||||
return entry.isDirectory() ? filesBelow(target) : [target];
|
||||
}),
|
||||
);
|
||||
return groups.flat().filter((file) =>
|
||||
/\.(?:js|jsx|mjs|ts|tsx|mts)$/.test(file),
|
||||
);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
for (const specification of governance.registries) {
|
||||
if (owners.has(specification.registryId)) {
|
||||
failures.push(`duplicate owner for ${specification.registryId}`);
|
||||
@@ -74,6 +149,10 @@ for (const specification of governance.registries) {
|
||||
}
|
||||
|
||||
rowsByRegistry.set(specification.registryId, rows);
|
||||
sourcesByRegistry.set(
|
||||
specification.registryId,
|
||||
sourcePath ?? specification.path,
|
||||
);
|
||||
|
||||
for (const [rowName, row] of Object.entries(rows)) {
|
||||
if (!row || typeof row !== "object" || Array.isArray(row)) {
|
||||
@@ -85,6 +164,26 @@ for (const specification of governance.registries) {
|
||||
failures.push(`${specification.registryId}.${rowName} missing ${field}`);
|
||||
}
|
||||
}
|
||||
for (const [field, declaredType] of Object.entries(
|
||||
specification.fieldTypes ?? {},
|
||||
)) {
|
||||
if (
|
||||
field in row &&
|
||||
!matchesDeclaredType(row[field], String(declaredType))
|
||||
) {
|
||||
failures.push(
|
||||
`${specification.registryId}.${rowName}.${field} expected ${declaredType}, received ${runtimeType(row[field])}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (
|
||||
specification.keyField &&
|
||||
row[specification.keyField] !== rowName
|
||||
) {
|
||||
failures.push(
|
||||
`${specification.registryId}.${rowName}.${specification.keyField} must match its registry key`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (const field of specification.uniqueFields ?? []) {
|
||||
@@ -93,12 +192,13 @@ for (const specification of governance.registries) {
|
||||
if (!row || typeof row !== "object" || Array.isArray(row)) continue;
|
||||
const value = row[field];
|
||||
if (value === undefined) continue;
|
||||
if (values.has(value)) {
|
||||
const identity = JSON.stringify(canonicalizeRegistryValue(value));
|
||||
if (values.has(identity)) {
|
||||
failures.push(
|
||||
`${specification.registryId}.${rowName} duplicates ${field}=${String(value)} from ${values.get(value)}`,
|
||||
`${specification.registryId}.${rowName} duplicates ${field}=${String(value)} from ${values.get(identity)}`,
|
||||
);
|
||||
} else {
|
||||
values.set(value, rowName);
|
||||
values.set(identity, rowName);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -121,12 +221,22 @@ for (const specification of governance.registries) {
|
||||
}
|
||||
}
|
||||
|
||||
const contract = Object.freeze({
|
||||
requiredFields: specification.requiredFields,
|
||||
fieldTypes: specification.fieldTypes ?? {},
|
||||
uniqueFields: specification.uniqueFields ?? [],
|
||||
allowedValues: specification.allowedValues ?? {},
|
||||
references: specification.references ?? [],
|
||||
keyField: specification.keyField ?? null,
|
||||
breakingFields: specification.breakingFields ?? [],
|
||||
});
|
||||
snapshots.push({
|
||||
registryId: specification.registryId,
|
||||
owner: specification.owner,
|
||||
source: sourcePath ?? specification.path,
|
||||
rowCount: Object.keys(rows).length,
|
||||
rows,
|
||||
contract,
|
||||
rows: canonicalizeRegistryValue(rows),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -145,18 +255,74 @@ for (const specification of governance.registries) {
|
||||
Object.values(targetRows)
|
||||
.filter((row) => row && typeof row === "object" && !Array.isArray(row))
|
||||
.map((row) => row[reference.targetField])
|
||||
.filter((value) => value !== undefined),
|
||||
.filter((value) => value !== undefined && value !== null),
|
||||
);
|
||||
for (const [rowName, row] of Object.entries(rows)) {
|
||||
if (!row || typeof row !== "object" || Array.isArray(row)) continue;
|
||||
const value = row[reference.field];
|
||||
if (value !== undefined && !targetValues.has(value)) {
|
||||
if (
|
||||
value !== undefined &&
|
||||
value !== null &&
|
||||
!targetValues.has(value)
|
||||
) {
|
||||
failures.push(
|
||||
`${specification.registryId}.${rowName}.${reference.field} references unknown ${reference.registryId}.${reference.targetField}=${String(value)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const consumer of specification.consumers ?? []) {
|
||||
try {
|
||||
const source = await readFile(consumer.path, "utf8");
|
||||
if (!source.includes(consumer.token)) {
|
||||
failures.push(
|
||||
`${specification.registryId} consumer ${consumer.path} is missing ${consumer.token}`,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
failures.push(
|
||||
`${specification.registryId} consumer source is missing: ${consumer.path}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (specification.consumerIdentityField) {
|
||||
const consumerFiles = (
|
||||
await Promise.all(
|
||||
(specification.consumerDirectories ?? []).map(filesBelow),
|
||||
)
|
||||
).flat();
|
||||
const sourcePath = sourcesByRegistry.get(specification.registryId);
|
||||
const consumerText = (
|
||||
await Promise.all(
|
||||
consumerFiles
|
||||
.filter((file) => file !== sourcePath)
|
||||
.map((file) => readFile(file, "utf8")),
|
||||
)
|
||||
).join("\n");
|
||||
const exemptions = new Set(specification.orphanExemptRows ?? []);
|
||||
for (const [rowName, row] of Object.entries(rows)) {
|
||||
if (
|
||||
!row ||
|
||||
typeof row !== "object" ||
|
||||
Array.isArray(row) ||
|
||||
exemptions.has(rowName)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const identity = row[specification.consumerIdentityField];
|
||||
if (
|
||||
(typeof identity !== "string" &&
|
||||
typeof identity !== "number") ||
|
||||
!consumerText.includes(String(identity))
|
||||
) {
|
||||
failures.push(
|
||||
`${specification.registryId}.${rowName} has no executable consumer for ${specification.consumerIdentityField}=${String(identity)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const sourceFiles = governance.sourceDirectories ?? [
|
||||
@@ -166,59 +332,80 @@ const sourceFiles = governance.sourceDirectories ?? [
|
||||
];
|
||||
const adHocPatterns = [
|
||||
{ name: "direct fetch", expression: /\bfetch\s*\(/ },
|
||||
{ name: "direct localStorage", expression: /\blocalStorage\.(?:get|set|remove)Item/ },
|
||||
{
|
||||
name: "direct localStorage",
|
||||
expression: /\blocalStorage\.(?:get|set|remove)Item/,
|
||||
},
|
||||
{ name: "direct import.meta.env", expression: /\bimport\.meta\.env\./ },
|
||||
{ name: "raw API path", expression: /["']\/api\// },
|
||||
];
|
||||
|
||||
/** @param {string} directory */
|
||||
async function scanDirectory(directory) {
|
||||
try {
|
||||
await access(directory);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const entries = await import("node:fs/promises").then(({ readdir }) =>
|
||||
readdir(directory, { withFileTypes: true }),
|
||||
);
|
||||
for (const entry of entries) {
|
||||
const target = path.join(directory, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await scanDirectory(target);
|
||||
continue;
|
||||
}
|
||||
if (!/\.(js|jsx|mjs|ts|tsx|mts)$/.test(entry.name)) continue;
|
||||
const content = await readFile(target, "utf8");
|
||||
for (const sourceDirectory of sourceFiles) {
|
||||
for (const file of await filesBelow(sourceDirectory)) {
|
||||
const content = await readFile(file, "utf8");
|
||||
for (const pattern of adHocPatterns) {
|
||||
if (pattern.expression.test(content)) {
|
||||
failures.push(`ad-hoc ${pattern.name} in ${target}`);
|
||||
failures.push(`ad-hoc ${pattern.name} in ${file}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const sourceDirectory of sourceFiles) {
|
||||
await scanDirectory(sourceDirectory);
|
||||
const currentSnapshot =
|
||||
/** @type {Readonly<Record<string, unknown>>} */ (
|
||||
canonicalizeRegistryValue({
|
||||
schemaVersion: 2,
|
||||
registries: snapshots,
|
||||
})
|
||||
);
|
||||
let baselineDigest = null;
|
||||
let currentDigest = registrySnapshotDigest(currentSnapshot);
|
||||
let compatibility =
|
||||
/** @type {{impact: string, changes: readonly Record<string, unknown>[]}} */ ({
|
||||
impact: "not-evaluated",
|
||||
changes: [],
|
||||
});
|
||||
|
||||
if (baselinePath && approvalPath && evidencePath) {
|
||||
try {
|
||||
const baseline = JSON.parse(await readFile(baselinePath, "utf8"));
|
||||
const approval = JSON.parse(await readFile(approvalPath, "utf8"));
|
||||
const approvalResult = verifyRegistryBaselineApproval(baseline, approval);
|
||||
baselineDigest = approvalResult.actualDigest;
|
||||
if (!approvalResult.passed) {
|
||||
failures.push(
|
||||
`registry baseline approval digest mismatch: approved=${approvalResult.approvedDigest} actual=${approvalResult.actualDigest}`,
|
||||
);
|
||||
}
|
||||
compatibility = diffRegistrySnapshots(baseline, currentSnapshot);
|
||||
const evidence = JSON.parse(await readFile(evidencePath, "utf8"));
|
||||
const evidenceResult = validateBreakingEvidence(compatibility, evidence);
|
||||
failures.push(...evidenceResult.failures);
|
||||
} catch (error) {
|
||||
failures.push(
|
||||
`registry compatibility evidence unavailable: ${
|
||||
error instanceof Error ? error.name : "unknown"
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await mkdir("artifacts/quality", { recursive: true });
|
||||
await writeFile(
|
||||
artifactPath,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
schemaVersion: 1,
|
||||
generatedAt: new Date().toISOString(),
|
||||
compatibilityImpact: governance.compatibilityImpact.current,
|
||||
failures,
|
||||
registries: snapshots,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
const report = {
|
||||
schemaVersion: 2,
|
||||
generatedAt: new Date().toISOString(),
|
||||
baselineDigest,
|
||||
currentDigest,
|
||||
compatibility,
|
||||
failures,
|
||||
registries: snapshots,
|
||||
};
|
||||
await mkdir(path.dirname(artifactPath), { recursive: true });
|
||||
await writeFile(artifactPath, `${JSON.stringify(report, null, 2)}\n`);
|
||||
|
||||
if (failures.length > 0) {
|
||||
process.stderr.write(`Registry governance failed:\n${failures.join("\n")}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
process.stdout.write(`Registry governance: ${snapshots.length} registries PASS\n`);
|
||||
process.stdout.write(
|
||||
`Registry governance: ${snapshots.length} registries PASS; compatibility=${compatibility.impact}\n`,
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user