From f5ba042dfa6d11348c8794066b1fa96b8ac6d987 Mon Sep 17 00:00:00 2001 From: jarvis Date: Tue, 18 Aug 2026 05:57:06 +0000 Subject: [PATCH] test(ri-050): RI-1-002 publish-gate negative controls (#1275) (#1305) --- ...ri-1-002-publish-gate-negative-controls.md | 45 +++ scripts/publish-gate-structure.test.mjs | 310 ++++++++++++++++++ scripts/verify-release.test.mjs | 36 +- 3 files changed, 386 insertions(+), 5 deletions(-) create mode 100644 scratchpads/ri-1-002-publish-gate-negative-controls.md create mode 100644 scripts/publish-gate-structure.test.mjs diff --git a/scratchpads/ri-1-002-publish-gate-negative-controls.md b/scratchpads/ri-1-002-publish-gate-negative-controls.md new file mode 100644 index 00000000..cba2d328 --- /dev/null +++ b/scratchpads/ri-1-002-publish-gate-negative-controls.md @@ -0,0 +1,45 @@ +# RI-1-002 — Publish-gate negative controls (SDLC-D-034 second half) + +- Task: RI-1-002 (docs/release-integrity workstream, PRD item RI-N1), issue ref #1275 +- Branch: `test/ri-050-publish-gate-negative` (base `origin/next` @ d8e0aec9 = PR #1277, RI-1-001) +- Budget: worker estimate ~45K tokens; keep scoped to the two test files + scratchpad. + +## Objective + +Checked-in negative-control tests that PROVE the publish gate fails when it must: + +1. Broken mandatory check blocks every publish step (structural DAG proof from `.woodpecker/publish.yml`). +2. Bypass shapes fail the checker: missing edge, hidden effect (non-`publish` name), detached verify, always-pass verify (`failure: ignore` / `success` override), conditional verify (`when`). +3. Exact-commit identity: no HEAD-moving step between verify and publish effects; legitimate re-checkout requires verify to re-run after it. +4. `verify-release.mjs` composition control: a SUBSET stage list fails the composition check. + +## Plan + +- NEW `scripts/publish-gate-structure.test.mjs` — self-contained structural checker (`assertPublishGateBlocksOnVerify`) + positive control on the real pipeline + one negative-control test per bypass shape (S1–S6, documented in file header) + positive control for the legitimate re-checkout shape. +- EXTEND `scripts/verify-release.test.mjs` — refactor the stage-mirror test body into `assertStagesMirrorCi(stages, ci)`; add negative control dropping each stage one at a time (subset must throw). + +## Conventions confirmed + +- Root `test:checkout` = `node --test scripts/*.test.mjs` → new file auto-joins `pnpm test`. +- Test-enumeration guard population is `*test*.sh` under `packages/mosaic/framework/tools/` only → unaffected. +- Root eslint covers only `**/*.{ts,tsx}` → .mjs files need Prettier style only (printWidth 100, singleQuote, semi, trailingComma all). +- Do NOT touch docs/TASKS.md, docs/release-integrity/TASKS.md, docs/scratchpads/. + +## Progress log + +- [x] Base verified: publish.yml `verify` step + verify-release.mjs present; HEAD contains origin/next. +- [x] Wrote scripts/publish-gate-structure.test.mjs +- [x] Extended scripts/verify-release.test.mjs (mirror fn + subset negative control) +- [x] Gates: node --test scripts (31 tests pass), prettier clean on touched files, pnpm typecheck PASS, pnpm lint PASS, pnpm format:check PASS +- [x] Committed ff585b88 + pushed, PR #1305 → next (no conflicts). Stopped before merge per task instruction. + +## Evidence + +- `node --test scripts/verify-release.test.mjs scripts/publish-gate-structure.test.mjs` → 31 tests, 0 fail. +- Mutation sanity: temporarily removing the `verify` edge from build-gateway in publish.yml → structure test goes red (verified manually during dev, then reverted). +- Gates run from repo root on this worktree; results in Progress log. + +## Risks / notes + +- Effect detection (`isPublishCommand`) is deliberately over-broad (any npm/pnpm/yarn command mentioning `publish`, any kaniko/docker-push/`--destination`) — fail-closed: a false positive forces justification, a false negative is the actual hazard. +- `git fetch` flagged as HEAD-moving even though fetch alone doesn't move HEAD — fail-closed on the classic `fetch && reset` pair. diff --git a/scripts/publish-gate-structure.test.mjs b/scripts/publish-gate-structure.test.mjs new file mode 100644 index 00000000..68a30f80 --- /dev/null +++ b/scripts/publish-gate-structure.test.mjs @@ -0,0 +1,310 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import test from 'node:test'; + +// RI-1-002 / RI-N1 publish-gate NEGATIVE CONTROLS (SDLC-D-034). +// +// scripts/verify-release.test.mjs pins the POSITIVE structure of the publish +// gate: every publish effect declares a direct `depends_on: verify` edge and +// the verify step asserts commit identity + runs the canonical command. This +// suite is the negative-control set: each test feeds a structural gate +// checker a pipeline in which the gate is bypassed by ONE specific shape and +// asserts the checker goes RED. The controls prove from the pipeline FILE — +// never by executing Woodpecker — that a verify step that FAILS (nonzero +// exit) blocks every publish effect. +// +// Woodpecker semantics these controls rely on: +// - A step that exits nonzero FAILS, and every step that transitively +// depends on a failed step is SKIPPED — never run. That skip is the only +// thing standing between a failed mandatory check and a publish effect. +// - `detach: true` removes the step from the wait graph: the pipeline does +// not wait for detached steps, so their failure can never block anything. +// - `failure: ignore` reports a failed step as success to the DAG. +// - `success: [codes...]` overrides which exit codes count as success; +// admitting any nonzero code launders a failed verification into green. +// - `when` on the verify step would skip verification entirely on some +// event/path classes while publish effects still run. +// +// Bypass shapes covered (one negative-control test each): +// S1 Missing edge — a publish effect whose dependency closure does not +// contain `verify` (a refactor drops the depends_on entry). +// S2 Hidden effect — a step whose NAME does not start with `publish` but +// whose COMMANDS publish npm packages or push images. Effects are +// classified by commands, so renaming a step cannot un-gate it. +// S3 Detached verify — `verify: { detach: true }`: publish steps no longer +// wait for verify, so the depends_on edge is decorative. +// S4 Always-pass verify — `failure: ignore`, or a `success` override +// admitting nonzero exit codes: verify fails, the DAG sees success. +// S5 Conditional verify — a `when`/path filter on verify itself. +// S6 Exact-commit drift — a HEAD-moving step (git checkout/switch/reset/ +// clean/pull/clone/fetch) ordered between `verify` and a publish +// effect: the verified commit would not be the published commit. A +// LEGITIMATE re-checkout is allowed only when `verify` itself runs +// after it — positive control included. +// S7 Gate removal — the verify step deleted or renamed away entirely. + +// Reuse the monorepo's existing YAML parser (@mosaicstack/mosaic's direct +// dependency) instead of adding a root dependency or vendoring a parser. +const mosaicRequire = createRequire( + path.resolve(process.cwd(), 'packages', 'mosaic', 'package.json'), +); +const { parse: parseYaml } = mosaicRequire('yaml'); + +const publishYmlPath = path.join(process.cwd(), '.woodpecker', 'publish.yml'); + +async function readPublishPipeline() { + return parseYaml(await readFile(publishYmlPath, 'utf8')); +} + +// A command has a publish EFFECT when it publishes npm packages (`publish` +// anywhere after a package-manager token — `pnpm --filter "@x/*" publish` +// puts flags and quoted filters between the binary and the subcommand) or +// pushes an image (kaniko, docker push, or a registry --destination). +// Deliberately over-broad: a false positive forces justification, a false +// negative is the actual hazard. +function isPublishCommand(command) { + return ( + /(^|\s)\/kaniko\/executor\b/.test(command) || + /(^|\s)docker\s+push\b/.test(command) || + /(^|\s)--destination(\s|=)/.test(command) || + (/\bpublish\b/.test(command) && /(^|\s)(npm|pnpm|yarn)(\s|$)/.test(command)) + ); +} + +function hasPublishEffect(step) { + return (step.commands ?? []).some(isPublishCommand); +} + +// A step is a publish effect when its name says so OR (S2) when any of its +// commands does — classification must not depend on the name alone. +function publishEffectSteps(pipeline) { + return Object.entries(pipeline.steps ?? {}) + .filter(([name, step]) => name.startsWith('publish') || hasPublishEffect(step)) + .map(([name]) => name); +} + +// Transitive closure of a step's depends_on graph. +function dependencyClosure(pipeline, stepName, seen = new Set()) { + const dependencies = pipeline.steps?.[stepName]?.depends_on ?? []; + for (const dependency of dependencies) { + if (seen.has(dependency)) continue; + seen.add(dependency); + dependencyClosure(pipeline, dependency, seen); + } + return seen; +} + +// Deliberately over-broad: `git fetch` alone does not move HEAD, but the +// classic re-checkout pair is `git fetch && git reset --hard `; a +// fetch step sitting between verify and a publish effect deserves scrutiny, +// so the gate fails closed on it. +function movesHead(step) { + return (step.commands ?? []).some((command) => + /(^|\s)git\s+(checkout|switch|reset|clean|pull|clone|fetch)\b/.test(command), + ); +} + +// The structural gate checker: green only when a failed (nonzero-exit) +// verify provably blocks every publish effect on the same commit. +function assertPublishGateBlocksOnVerify(pipeline) { + assert.ok(pipeline.steps, 'publish pipeline must define steps'); + const verify = pipeline.steps.verify; + assert.ok(verify, 'publish pipeline must define a `verify` step (S7)'); + + // S5: a skipped verification authorizes publishes exactly as much as a + // failed one — verify must be unconditional. + assert.equal(verify.when, undefined, '`verify` must not carry a when/path filter (S5)'); + + // S3/S4: the depends_on edges are only meaningful if verify's own failure + // is both awaited and terminal for the DAG. + assert.equal(verify.detach, undefined, '`verify` must not be detached (S3)'); + assert.equal( + verify.failure, + undefined, + '`verify` must not tolerate its own failure (S4: failure: ignore launders a failed gate into success)', + ); + assert.equal( + verify.success, + undefined, + '`verify` must not override success exit codes (S4: nonzero codes would make failed verification pass)', + ); + + const effects = publishEffectSteps(pipeline); + assert.ok(effects.length > 0, 'publish pipeline must contain publish effect steps to guard'); + + const verifyClosure = dependencyClosure(pipeline, 'verify'); + for (const stepName of effects) { + // S1: only the failure-skip semantics of the DAG stand between a failed + // verify and this effect — the verify edge in its closure is the proof. + const closure = dependencyClosure(pipeline, stepName); + assert.ok( + closure.has('verify'), + `publish effect '${stepName}' must transitively depend on verify (S1) — a failed verify must skip it`, + ); + + // S6: any step ordered after verify (outside its closure) but inside the + // effect's chain must not be able to move HEAD. If the pipeline + // legitimately re-checks-out, verify must run after the re-checkout. + for (const chainStep of closure) { + if (chainStep === 'verify' || verifyClosure.has(chainStep)) continue; + assert.ok( + !movesHead(pipeline.steps[chainStep]), + `step '${chainStep}' sits between verify and publish effect '${stepName}' and can move HEAD (S6)` + + ' — verify must re-run after any re-checkout', + ); + } + } + return effects; +} + +// A minimal but healthy gate used as the base for every negative-control +// mutation: verify (identity + canonical command) → build → publish-npm, +// with the publish effect blocked by verify both directly and through build. +const HEALTHY_GATE_YAML = ` +steps: + verify: + image: node:24-alpine + commands: + - | + if [ -z "$CI_COMMIT_SHA" ] || [ "$CI_COMMIT_SHA" != "$(git rev-parse HEAD)" ]; then + echo "identity mismatch" >&2 + exit 1 + fi + - pnpm verify:release + build: + image: node:24-alpine + commands: + - pnpm build + depends_on: + - verify + publish-npm: + image: node:24-alpine + commands: + - npm publish + depends_on: + - build + - verify +`; + +// Fresh parse per call so every negative control mutates its own object. +function healthyPipeline() { + return parseYaml(HEALTHY_GATE_YAML); +} + +test('the real publish pipeline: a failed verify provably blocks every publish effect', async () => { + const pipeline = await readPublishPipeline(); + const effects = assertPublishGateBlocksOnVerify(pipeline); + assert.deepEqual(effects.sort(), [ + 'build-appservice', + 'build-gateway', + 'build-web', + 'publish-next-npm', + 'publish-npm', + ]); +}); + +test('fixture sanity: the healthy gate base passes the checker unmutated', () => { + assertPublishGateBlocksOnVerify(healthyPipeline()); +}); + +test('S1 negative control: a publish effect with no verify edge fails the checker', () => { + const pipeline = healthyPipeline(); + pipeline.steps['publish-npm'].depends_on = ['build']; + pipeline.steps.build.depends_on = []; + assert.throws( + () => assertPublishGateBlocksOnVerify(pipeline), + /publish-npm.*must transitively depend on verify/s, + ); +}); + +test('S2 negative control: an npm publish hidden behind a non-publish step name fails the checker', () => { + const pipeline = healthyPipeline(); + delete pipeline.steps['publish-npm']; + pipeline.steps.build.depends_on = []; + pipeline.steps.deploy = { + image: 'node:24-alpine', + commands: ['npm publish'], + depends_on: ['build'], + }; + // Detection must be by COMMAND: the name says "deploy", the commands say + // publish — an un-gated effect under either reading. + assert.throws( + () => assertPublishGateBlocksOnVerify(pipeline), + /deploy.*must transitively depend on verify/s, + ); +}); + +test('S2 negative control: a kaniko image push under a build-* name fails the checker when ungated', () => { + const pipeline = healthyPipeline(); + delete pipeline.steps['publish-npm']; + pipeline.steps.build.depends_on = []; + pipeline.steps['push-platform-image'] = { + image: 'gcr.io/kaniko-project/executor:debug', + commands: ['/kaniko/executor --context . --destination reg.example/img:latest'], + depends_on: ['build'], + }; + assert.throws( + () => assertPublishGateBlocksOnVerify(pipeline), + /push-platform-image.*must transitively depend on verify/s, + ); +}); + +test('S3 negative control: a detached verify fails the checker', () => { + const pipeline = healthyPipeline(); + pipeline.steps.verify.detach = true; + assert.throws(() => assertPublishGateBlocksOnVerify(pipeline), /detached \(S3\)/); +}); + +test('S4 negative control: failure: ignore on verify fails the checker', () => { + const pipeline = healthyPipeline(); + pipeline.steps.verify.failure = 'ignore'; + assert.throws(() => assertPublishGateBlocksOnVerify(pipeline), /tolerate its own failure/); +}); + +test('S4 negative control: a success override admitting nonzero exit codes fails the checker', () => { + const pipeline = healthyPipeline(); + pipeline.steps.verify.success = [0, 1]; + assert.throws(() => assertPublishGateBlocksOnVerify(pipeline), /success exit codes/); +}); + +test('S5 negative control: a when filter on verify fails the checker', () => { + const pipeline = healthyPipeline(); + pipeline.steps.verify.when = [{ event: 'push' }]; + assert.throws(() => assertPublishGateBlocksOnVerify(pipeline), /when\/path filter \(S5\)/); +}); + +test('S6 negative control: a HEAD-moving step between verify and publish fails the checker', () => { + const pipeline = healthyPipeline(); + pipeline.steps.resync = { + image: 'node:24-alpine', + commands: ['git fetch origin', 'git reset --hard origin/main'], + depends_on: [], + }; + pipeline.steps.build.depends_on = ['verify', 'resync']; + // resync sits AFTER verify in the publish chain (verify does not depend on + // it), so the verified commit could be replaced before publishing. + assert.throws(() => assertPublishGateBlocksOnVerify(pipeline), /resync.*can move HEAD/s); +}); + +test('S6 positive control: a legitimate re-checkout passes when verify re-runs after it', () => { + const pipeline = healthyPipeline(); + pipeline.steps.resync = { + image: 'node:24-alpine', + commands: ['git fetch origin', 'git reset --hard origin/main'], + depends_on: [], + }; + pipeline.steps.verify.depends_on = ['resync']; + pipeline.steps.build.depends_on = ['verify']; + // resync precedes verify in the chain, so verification covers the + // re-checked-out HEAD — the exact-commit contract holds. + assertPublishGateBlocksOnVerify(pipeline); +}); + +test('S7 negative control: deleting the verify step entirely fails the checker', () => { + const pipeline = healthyPipeline(); + delete pipeline.steps.verify; + pipeline.steps['publish-npm'].depends_on = ['build']; + assert.throws(() => assertPublishGateBlocksOnVerify(pipeline), /`verify` step/); +}); diff --git a/scripts/verify-release.test.mjs b/scripts/verify-release.test.mjs index d83d1c18..49e9ff5b 100644 --- a/scripts/verify-release.test.mjs +++ b/scripts/verify-release.test.mjs @@ -9,7 +9,10 @@ import { STAGES } from './verify-release.mjs'; // SDLC-D-034 checkout invariant: publication in .woodpecker/publish.yml is // bound to exact-commit terminal verification. This suite parses the real // pipeline files and fails red when the gate is bypassed, weakened, or drifts -// out of sync with the canonical `pnpm verify:release` command. +// out of sync with the canonical `pnpm verify:release` command. The negative +// controls for pipeline DAG/bypass shapes live in +// scripts/publish-gate-structure.test.mjs (RI-1-002); this file owns the +// canonical-command composition controls. // Reuse the monorepo's existing YAML parser (@mosaicstack/mosaic's direct // dependency) instead of adding a root dependency or vendoring a parser. @@ -219,13 +222,16 @@ steps: assert.throws(() => assertPublishGate(parseYaml(noIdentityPipeline)), /CI_COMMIT_SHA/); }); -test('the canonical verify:release stages mirror the PR CI pipeline one-for-one', async () => { - const ci = parseYaml(await readFile(ciYmlPath, 'utf8')); - const canonical = Object.fromEntries(STAGES.map((stage) => [stage.name, stage.commands])); +// The composition check: the canonical stage table must mirror the PR CI +// pipeline's complete mandatory set. Parameterized by the stage list so the +// subset negative control below can prove a dropped stage goes red (RI-1-002: +// the canonical command cannot silently lose a check). +function assertStagesMirrorCi(stages, ci) { + const canonical = Object.fromEntries(stages.map((stage) => [stage.name, stage.commands])); // The complete mandatory set, in gate order. assert.deepEqual( - STAGES.map((stage) => stage.name), + stages.map((stage) => stage.name), ['sanitization', 'upgrade-guard', 'typecheck', 'lint', 'format', 'test', 'build'], ); @@ -269,6 +275,26 @@ test('the canonical verify:release stages mirror the PR CI pipeline one-for-one' `ci.yml test step must keep its pipeline-level prerequisite '${fragment}'`, ); } +} + +test('the canonical verify:release stages mirror the PR CI pipeline one-for-one', async () => { + const ci = parseYaml(await readFile(ciYmlPath, 'utf8')); + assertStagesMirrorCi(STAGES, ci); +}); + +test('a subset stage list fails the composition check — a dropped stage cannot pass silently', async () => { + const ci = parseYaml(await readFile(ciYmlPath, 'utf8')); + // Drop each stage one at a time: every stage is load-bearing, so every drop + // must go red. If any drop went green, a refactor could silently delete a + // mandatory check from the canonical command. + for (const stage of STAGES) { + const subset = STAGES.filter((entry) => entry.name !== stage.name); + assert.throws( + () => assertStagesMirrorCi(subset, ci), + Error, + `composition check must fail when the '${stage.name}' stage is dropped from the table`, + ); + } }); test('the root package.json exposes verify:release as the canonical command', async () => {