Merge lane/J: test system (10-02, 10-04~10-07, 11-04~11-08, F-2, F-4, F-12, 00-03)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
donghyeon-ka
2026-09-27 13:28:55 +09:00
co-authored by Claude Opus 5.5
43 changed files with 1908 additions and 172 deletions
+103
View File
@@ -0,0 +1,103 @@
import { readFile } from "node:fs/promises";
import { describe, expect, it } from "vitest";
import {
FEATURE_SHAPE_BASELINE_PATH,
renameReferenceFeature,
} from "../../scripts/scale/measure-feature-shape-growth.ts";
/**
* 11-04 (D-18). The scale evidence used to come from stub features that only
* differ by name, so the cost of one app growing towards its declared ceiling
* (20 contexts x 200 features) was never measured. The committed baseline is a
* measurement with copies of the real reference feature.
*/
const context = { id: "shape-context-0", routeRoot: "shape-0" } as const;
type Run = Readonly<{
addedFeatures: number;
routes: number;
tsc: Readonly<{ passed: boolean; instantiations: number; wallMs: number }>;
build: Readonly<{
wallMs: number;
initialJavascriptGzipBytes: number;
lazyJavascriptFiles: number;
}>;
loadBefore: readonly number[];
}>;
describe("feature-shape growth measurement (11-04)", () => {
it("renames every feature-owned identifier of a reference copy", () => {
const source = [
'featureId: "reference-feature"',
'path: "/examples/reference-resources/:id"',
"REFERENCE_RESOURCE_LIST",
"ReferenceResourcePage",
'contextId: "reference-context"',
'topic: "qinv.01k10f7m3w9p6r2c8v5n4x"',
"NoRequest",
].join("\n");
const renamed = renameReferenceFeature(source, "s007", context);
expect(renamed).not.toMatch(/reference/iu);
expect(renamed).toContain('featureId: "s007-feature"');
expect(renamed).toContain('path: "/shape-0/s007-resources/:id"');
expect(renamed).toContain("S007_RESOURCE_LIST");
expect(renamed).toContain("S007ResourcePage");
expect(renamed).toContain('contextId: "shape-context-0"');
expect(renamed).toContain('topic: "qinv.shape.s007"');
expect(renamed).toContain("S007NoRequest");
});
it("keeps platform-owned names that only contain the word", () => {
const source =
"REFERENCE_EXTERNAL_BEARER REFERENCE_FEATURE_V1 supportReference";
expect(renameReferenceFeature(source, "s001", context)).toBe(source);
});
it("commits a baseline measured at 25 and 100 real-shape features, twice each", async () => {
const baseline = JSON.parse(
await readFile(FEATURE_SHAPE_BASELINE_PATH, "utf8"),
) as Readonly<{
environment: Readonly<Record<string, unknown>>;
runs: readonly Run[];
}>;
expect(baseline.environment).toMatchObject({
cpuModel: expect.any(String),
logicalCpus: expect.any(Number),
});
const bySize = (size: number) =>
baseline.runs.filter((run) => run.addedFeatures === size);
const [origin] = bySize(0);
expect(origin?.tsc.passed).toBe(true);
for (const size of [25, 100]) {
const runs = bySize(size);
// A failed typecheck is the result and is not repeated.
expect(runs.length).toBeGreaterThanOrEqual(runs[0]?.tsc.passed ? 2 : 1);
for (const run of runs) {
// Every copy contributes the reference feature's own routes.
expect((run.routes - origin!.routes) % size).toBe(0);
expect(run.routes).toBeGreaterThan(origin!.routes);
expect(run.build.lazyJavascriptFiles).toBeGreaterThan(
origin!.build.lazyJavascriptFiles,
);
expect(run.loadBefore).toHaveLength(3);
}
// Compiler counts are deterministic; only times vary with the host.
expect(new Set(runs.map((run) => run.tsc.instantiations)).size).toBe(1);
}
});
it("states in the acceptance matrix what the ceilings rest on", async () => {
const acceptance = await readFile(
"docs/reviews/2026-09-23-extreme-scale-acceptance.md",
"utf8",
);
expect(acceptance).toContain(FEATURE_SHAPE_BASELINE_PATH);
expect(acceptance).toMatch(/\| Growth axis \| Verified \| Condition \| Not verified \|/u);
});
});
+142 -3
View File
@@ -1,10 +1,24 @@
import { lstat, mkdtemp, mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
import { spawnSync } from "node:child_process";
import {
copyFile,
lstat,
mkdtemp,
mkdir,
readdir,
readFile,
rm,
stat,
writeFile,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { afterEach, describe, expect, it } from "vitest";
import { linkFixtureNodeModules } from "../../scripts/lib/fixture-node-modules.ts";
import {
FIXTURE_PNPM_ENVIRONMENT,
linkFixtureNodeModules,
} from "../../scripts/lib/fixture-node-modules.ts";
/**
* A removal or provider fixture runs `pnpm` inside a throwaway copy of the
@@ -146,3 +160,128 @@ describe("fixture node_modules linking", () => {
}
});
});
/**
* F-4. pnpm 11 verifies the install before it runs a script and, finding the
* fixture's copied state unfamiliar, installs. `.pnpm` and `.bin` are
* directory links into the repository's own node_modules, so that install
* rewrote the shared `.pnpm/lock.yaml` and bin shims. A fixture launched
* through `pnpm <script>` inherited `pnpm_config_verify_deps_before_run=false`
* from pnpm and was safe; the same fixture started directly
* (`node scripts/run-vitest.ts run <file>`) was not.
*/
describe("pnpm inside a linked fixture (F-4)", () => {
const scratch: string[] = [];
afterEach(async () => {
await Promise.all(
scratch.splice(0).map((root) => rm(root, { recursive: true, force: true })),
);
});
/** A tiny installed workspace standing in for the repository, offline. */
async function installedSource() {
const root = await mkdtemp(path.join(tmpdir(), "fixture-pnpm-"));
scratch.push(root);
const tool = path.join(root, "tool");
const source = path.join(root, "source");
await mkdir(tool, { recursive: true });
await mkdir(source, { recursive: true });
await writeFile(
path.join(tool, "package.json"),
JSON.stringify({ name: "tool", version: "1.0.0", bin: { tool: "cli.js" } }),
);
await writeFile(path.join(tool, "cli.js"), "#!/usr/bin/env node\n");
await writeFile(
path.join(source, "package.json"),
JSON.stringify({
name: "fixture-source",
version: "1.0.0",
private: true,
packageManager: "pnpm@11.17.0",
scripts: { hello: "echo hello-from-fixture" },
dependencies: { tool: "file:../tool" },
}),
);
await writeFile(path.join(source, "pnpm-workspace.yaml"), "allowBuilds: {}\n");
const environment: NodeJS.ProcessEnv = {
...pnpmFreeEnvironment(),
pnpm_config_store_dir: path.join(root, "store"),
};
// Installed on a developer machine (no CI); the fixtures then run pnpm
// with CI=true, which pnpm reads as different settings and reinstalls.
const { CI: _ci, ...developer } = environment;
const install = spawnSync("corepack", ["pnpm", "install", "--offline"], {
cwd: source,
encoding: "utf8",
env: developer,
});
expect(install.status, install.stdout + install.stderr).toBe(0);
return { root, source, environment };
}
// The test runner itself may run under pnpm, which exports the setting.
function pnpmFreeEnvironment(): NodeJS.ProcessEnv {
const environment: NodeJS.ProcessEnv = { ...process.env, CI: "true" };
delete environment.pnpm_config_verify_deps_before_run;
return environment;
}
async function sharedState(source: string) {
const lock = await stat(path.join(source, "node_modules/.pnpm/lock.yaml"));
const bin = await stat(path.join(source, "node_modules/.bin/tool"));
return [lock.mtimeMs, lock.ino, bin.mtimeMs, bin.ino];
}
async function runInFixture(
root: string,
source: string,
environment: NodeJS.ProcessEnv,
) {
const fixture = path.join(root, `fixture-${scratch.length}-${Date.now()}`);
await mkdir(fixture);
for (const file of ["package.json", "pnpm-lock.yaml", "pnpm-workspace.yaml"]) {
await copyFile(path.join(source, file), path.join(fixture, file));
}
await linkFixtureNodeModules(fixture, source);
const before = await sharedState(source);
const result = spawnSync("corepack", ["pnpm", "hello"], {
cwd: fixture,
encoding: "utf8",
env: environment,
});
return { result, changed: (await sharedState(source)).join() !== before.join() };
}
it("installs through the linked directories unless told not to", async () => {
const { root, source, environment } = await installedSource();
// The hazard: the default re-verification installs into the shared tree.
expect((await runInFixture(root, source, environment)).changed).toBe(true);
const guarded = await runInFixture(root, source, {
...environment,
...FIXTURE_PNPM_ENVIRONMENT,
});
expect(guarded.result.status, guarded.result.stderr).toBe(0);
expect(guarded.result.stdout).toContain("hello-from-fixture");
expect(guarded.changed).toBe(false);
}, 60_000);
it("is used by every fixture that links node_modules and runs pnpm", async () => {
const listed = spawnSync(
"git",
["grep", "-l", "linkFixtureNodeModules(", "--", "scripts", "tests"],
{ encoding: "utf8" },
);
expect(listed.status, listed.stderr).toBe(0);
const offenders: string[] = [];
for (const file of listed.stdout.trim().split("\n")) {
if (file === "scripts/lib/fixture-node-modules.ts") continue;
if (file === "tests/unit/fixture-node-modules.test.ts") continue;
const source = await readFile(file, "utf8");
const runsPnpm = /\["pnpm"|\[pnpmCli\b/u.test(source);
if (runsPnpm && !source.includes("FIXTURE_PNPM_ENVIRONMENT")) offenders.push(file);
}
expect(offenders).toEqual([]);
});
});
+106
View File
@@ -0,0 +1,106 @@
import { spawn } from "node:child_process";
import { createServer, type Server } from "node:http";
import type { AddressInfo } from "node:net";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import {
assertPortUnused,
assertServedRelease,
previewArguments,
summarizeSamples,
} from "../../scripts/lib/lab-performance.ts";
/**
* 11-07. The lab performance gate measured once, and measured whatever server
* held its port: `vite preview` without `--strictPort` moved elsewhere and the
* script never checked which build answered.
*/
const servers: Server[] = [];
afterEach(async () => {
await Promise.all(
servers.splice(0).map(
(server) => new Promise<void>((resolve) => server.close(() => resolve())),
),
);
});
async function foreignServer(manifest: unknown): Promise<string> {
const server = createServer((_request, response) => {
response
.writeHead(200, { "content-type": "application/json" })
.end(JSON.stringify(manifest));
});
servers.push(server);
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", () => resolve()));
return `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
}
describe("lab performance gate (11-07)", () => {
it("starts the preview server on exactly the requested port", () => {
expect(previewArguments(4173)).toEqual([
"pnpm",
"preview",
"--host",
"127.0.0.1",
"--port",
"4173",
"--strictPort",
]);
});
it("summarizes repeated runs by their median and keeps every sample", () => {
expect(summarizeSamples([326, 1103, 410, 390, 505])).toEqual({
median: 410,
min: 326,
max: 1103,
samples: [326, 1103, 410, 390, 505],
});
expect(summarizeSamples([1, 4, 2, 3]).median).toBe(2.5);
expect(() => summarizeSamples([])).toThrow(RangeError);
});
it("refuses a port that another server already answers on", async () => {
const baseUrl = await foreignServer({});
await expect(assertPortUnused(baseUrl)).rejects.toThrow(/another server/u);
await expect(assertPortUnused("http://127.0.0.1:9")).resolves.toBeUndefined();
});
it("refuses a server that serves a different build", async () => {
const baseUrl = await foreignServer({ buildId: "other", releaseId: "r1" });
await expect(
assertServedRelease(baseUrl, { buildId: "measured", releaseId: "r1" }),
).rejects.toThrow(/serves build other\/r1, not the measured measured\/r1/u);
await expect(
assertServedRelease(baseUrl, { buildId: "other", releaseId: "r1" }),
).resolves.toBeUndefined();
});
it("fails the gate before measuring when its port is taken", async () => {
const baseUrl = await foreignServer({ buildId: "other", releaseId: "r1" });
const child = spawn(
process.execPath,
[path.resolve("scripts/test-performance.ts")],
{
env: { ...process.env, LAB_PERFORMANCE_PORT: new URL(baseUrl).port },
stdio: ["ignore", "ignore", "pipe"],
},
);
let stderr = "";
child.stderr.on("data", (chunk: Buffer) => {
stderr += chunk.toString("utf8");
});
const code = await new Promise<number | null>((resolve) => {
const timer = setTimeout(() => child.kill("SIGKILL"), 20_000);
child.on("exit", (exitCode) => {
clearTimeout(timer);
resolve(exitCode);
});
});
expect(code).not.toBe(0);
expect(stderr).toMatch(/another server holds the port/u);
}, 30_000);
});
+14 -2
View File
@@ -1,8 +1,8 @@
import { mkdtemp, mkdir, symlink, writeFile } from "node:fs/promises";
import { mkdtemp, mkdir, rm, symlink, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { afterEach, describe, expect, it } from "vitest";
import {
buildRepositoryFileInventory,
@@ -23,8 +23,19 @@ function gitResult(
};
}
// 00-03. Every temp root is removed after its test; they used to pile up in
// /tmp by the thousand.
const temporaryRoots: string[] = [];
afterEach(async () => {
await Promise.all(
temporaryRoots.splice(0).map((root) => rm(root, { recursive: true, force: true })),
);
});
async function repositoryFixture() {
const root = await mkdtemp(path.join(tmpdir(), "repository-inventory-"));
temporaryRoots.push(root);
await mkdir(path.join(root, "src"));
await mkdir(path.join(root, "docs"));
await writeFile(path.join(root, "src", "tracked.ts"), "tracked\n");
@@ -203,6 +214,7 @@ describe("repository file inventory", () => {
it("rejects a tracked symlink whose real path escapes the repository", async () => {
const repositoryRoot = await repositoryFixture();
const outside = await mkdtemp(path.join(tmpdir(), "inventory-outside-"));
temporaryRoots.push(outside);
await writeFile(path.join(outside, "secret.ts"), "secret\n");
await symlink(
path.join(outside, "secret.ts"),
@@ -1,6 +1,6 @@
import { describe, expect, it, vi } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";
import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
@@ -80,10 +80,19 @@ describe("SW-RR-03 one authoritative cacheable asset table", () => {
* had just produced, failing the release build at install time.
*/
describe("SW-02 the generator and the decoder share one path grammar", () => {
// 00-03. The dist fixtures used to stay in /tmp.
const temporaryRoots: string[] = [];
afterEach(async () => {
await Promise.all(
temporaryRoots.splice(0).map((root) => rm(root, { recursive: true, force: true })),
);
});
async function distWith(
files: Readonly<Record<string, string>>,
): Promise<string> {
const root = await mkdtemp(path.join(tmpdir(), "sw-assets-"));
temporaryRoots.push(root);
for (const [relative, content] of Object.entries(files)) {
const absolute = path.join(root, relative);
await mkdir(path.dirname(absolute), { recursive: true });
@@ -0,0 +1,70 @@
import { spawnSync } from "node:child_process";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { afterAll, describe, expect, it } from "vitest";
const FORBIDDEN = "tests/fixtures/test-evidence/forbidden";
const roots: string[] = [];
afterAll(async () => {
await Promise.all(roots.map((root) => rm(root, { recursive: true, force: true })));
});
function checkSource(sourceRoot: string) {
const result = spawnSync(
process.execPath,
[
"scripts/check-test-evidence.ts",
"--source-only",
"--source-root",
sourceRoot,
"--artifact",
"artifacts/quality/test-evidence-source-rules.json",
],
{ encoding: "utf8" },
);
return { status: result.status, stderr: result.stderr };
}
// 11-08. Each forbidden fixture is checked alone, so a rule that stops firing
// is not hidden by another fixture in the same directory still failing.
describe("test evidence source rules", () => {
it("rejects a focused test", () => {
const result = checkSource(`${FORBIDDEN}/focused-test.fixture`);
expect(result.status).toBe(1);
expect(result.stderr).toContain("focused test");
}, 30_000);
it("rejects a quarantine whose expiry date has passed", () => {
const result = checkSource(`${FORBIDDEN}/expired-quarantine.fixture`);
expect(result.status).toBe(1);
expect(result.stderr).toContain("quarantine expired on 2020-01-01");
}, 30_000);
it("rejects a skip without a quarantine", () => {
const result = checkSource(`${FORBIDDEN}/unowned-skip.fixture`);
expect(result.status).toBe(1);
expect(result.stderr).toContain("lacks owned expiring quarantine");
}, 30_000);
it("accepts a skip under an unexpired quarantine", async () => {
const root = await mkdtemp(path.join(tmpdir(), "test-evidence-quarantine-"));
roots.push(root);
const file = path.join(root, "quarantined.fixture");
// Built from parts so this file is not itself a skip the checker scans.
const skip = ["test", "skip"].join(".");
await writeFile(
file,
[
`// ${"quarantine"}(owner=frontend-platform, defect=open-fixture, expires=2999-12-31)`,
`${skip}("quarantined", async () => {});`,
"",
].join("\n"),
);
const result = checkSource(file);
expect(result.stderr).toBe("");
expect(result.status).toBe(0);
}, 30_000);
});
+14
View File
@@ -0,0 +1,14 @@
import { describe, expect, it, vi } from "vitest";
// tests/setup.ts restores real timers after every test, so a test that fails
// before its own vi.useRealTimers() does not hand fake timers to the next one.
describe("global test setup", () => {
it("may end with fake timers still installed", () => {
vi.useFakeTimers();
expect(vi.isFakeTimers()).toBe(true);
});
it("starts the next test on real timers", () => {
expect(vi.isFakeTimers()).toBe(false);
});
});