test(ri-050): RI-1-002 publish-gate negative controls (#1275) (#1305)

2026-08-18 05:57:06 +00:00
parent 7c7dab3898
commit f5ba042dfa
3 changed files with 386 additions and 5 deletions
@@ -0,0 +1,310 @@
+import assert from 'node:assert/strict';
+import { readFile } from 'node:fs/promises';
+import { createRequire } from 'node:module';
+import path from 'node:path';
+import test from 'node:test';
+
+// RI-1-002 / RI-N1 publish-gate NEGATIVE CONTROLS (SDLC-D-034).
+//
+// scripts/verify-release.test.mjs pins the POSITIVE structure of the publish
+// gate: every publish effect declares a direct `depends_on: verify` edge and
+// the verify step asserts commit identity + runs the canonical command. This
+// suite is the negative-control set: each test feeds a structural gate
+// checker a pipeline in which the gate is bypassed by ONE specific shape and
+// asserts the checker goes RED. The controls prove from the pipeline FILE —
+// never by executing Woodpecker — that a verify step that FAILS (nonzero
+// exit) blocks every publish effect.
+//
+// Woodpecker semantics these controls rely on:
+//   - A step that exits nonzero FAILS, and every step that transitively
+//     depends on a failed step is SKIPPED — never run. That skip is the only
+//     thing standing between a failed mandatory check and a publish effect.
+//   - `detach: true` removes the step from the wait graph: the pipeline does
+//     not wait for detached steps, so their failure can never block anything.
+//   - `failure: ignore` reports a failed step as success to the DAG.
+//   - `success: [codes...]` overrides which exit codes count as success;
+//     admitting any nonzero code launders a failed verification into green.
+//   - `when` on the verify step would skip verification entirely on some
+//     event/path classes while publish effects still run.
+//
+// Bypass shapes covered (one negative-control test each):
+//   S1  Missing edge — a publish effect whose dependency closure does not
+//       contain `verify` (a refactor drops the depends_on entry).
+//   S2  Hidden effect — a step whose NAME does not start with `publish` but
+//       whose COMMANDS publish npm packages or push images. Effects are
+//       classified by commands, so renaming a step cannot un-gate it.
+//   S3  Detached verify — `verify: { detach: true }`: publish steps no longer
+//       wait for verify, so the depends_on edge is decorative.
+//   S4  Always-pass verify — `failure: ignore`, or a `success` override
+//       admitting nonzero exit codes: verify fails, the DAG sees success.
+//   S5  Conditional verify — a `when`/path filter on verify itself.
+//   S6  Exact-commit drift — a HEAD-moving step (git checkout/switch/reset/
+//       clean/pull/clone/fetch) ordered between `verify` and a publish
+//       effect: the verified commit would not be the published commit. A
+//       LEGITIMATE re-checkout is allowed only when `verify` itself runs
+//       after it — positive control included.
+//   S7  Gate removal — the verify step deleted or renamed away entirely.
+
+// Reuse the monorepo's existing YAML parser (@mosaicstack/mosaic's direct
+// dependency) instead of adding a root dependency or vendoring a parser.
+const mosaicRequire = createRequire(
+  path.resolve(process.cwd(), 'packages', 'mosaic', 'package.json'),
+);
+const { parse: parseYaml } = mosaicRequire('yaml');
+
+const publishYmlPath = path.join(process.cwd(), '.woodpecker', 'publish.yml');
+
+async function readPublishPipeline() {
+  return parseYaml(await readFile(publishYmlPath, 'utf8'));
+}
+
+// A command has a publish EFFECT when it publishes npm packages (`publish`
+// anywhere after a package-manager token — `pnpm --filter "@x/*" publish`
+// puts flags and quoted filters between the binary and the subcommand) or
+// pushes an image (kaniko, docker push, or a registry --destination).
+// Deliberately over-broad: a false positive forces justification, a false
+// negative is the actual hazard.
+function isPublishCommand(command) {
+  return (
+    /(^|\s)\/kaniko\/executor\b/.test(command) ||
+    /(^|\s)docker\s+push\b/.test(command) ||
+    /(^|\s)--destination(\s|=)/.test(command) ||
+    (/\bpublish\b/.test(command) && /(^|\s)(npm|pnpm|yarn)(\s|$)/.test(command))
+  );
+}
+
+function hasPublishEffect(step) {
+  return (step.commands ?? []).some(isPublishCommand);
+}
+
+// A step is a publish effect when its name says so OR (S2) when any of its
+// commands does — classification must not depend on the name alone.
+function publishEffectSteps(pipeline) {
+  return Object.entries(pipeline.steps ?? {})
+    .filter(([name, step]) => name.startsWith('publish') || hasPublishEffect(step))
+    .map(([name]) => name);
+}
+
+// Transitive closure of a step's depends_on graph.
+function dependencyClosure(pipeline, stepName, seen = new Set()) {
+  const dependencies = pipeline.steps?.[stepName]?.depends_on ?? [];
+  for (const dependency of dependencies) {
+    if (seen.has(dependency)) continue;
+    seen.add(dependency);
+    dependencyClosure(pipeline, dependency, seen);
+  }
+  return seen;
+}
+
+// Deliberately over-broad: `git fetch` alone does not move HEAD, but the
+// classic re-checkout pair is `git fetch && git reset --hard <remote>`; a
+// fetch step sitting between verify and a publish effect deserves scrutiny,
+// so the gate fails closed on it.
+function movesHead(step) {
+  return (step.commands ?? []).some((command) =>
+    /(^|\s)git\s+(checkout|switch|reset|clean|pull|clone|fetch)\b/.test(command),
+  );
+}
+
+// The structural gate checker: green only when a failed (nonzero-exit)
+// verify provably blocks every publish effect on the same commit.
+function assertPublishGateBlocksOnVerify(pipeline) {
+  assert.ok(pipeline.steps, 'publish pipeline must define steps');
+  const verify = pipeline.steps.verify;
+  assert.ok(verify, 'publish pipeline must define a `verify` step (S7)');
+
+  // S5: a skipped verification authorizes publishes exactly as much as a
+  // failed one — verify must be unconditional.
+  assert.equal(verify.when, undefined, '`verify` must not carry a when/path filter (S5)');
+
+  // S3/S4: the depends_on edges are only meaningful if verify's own failure
+  // is both awaited and terminal for the DAG.
+  assert.equal(verify.detach, undefined, '`verify` must not be detached (S3)');
+  assert.equal(
+    verify.failure,
+    undefined,
+    '`verify` must not tolerate its own failure (S4: failure: ignore launders a failed gate into success)',
+  );
+  assert.equal(
+    verify.success,
+    undefined,
+    '`verify` must not override success exit codes (S4: nonzero codes would make failed verification pass)',
+  );
+
+  const effects = publishEffectSteps(pipeline);
+  assert.ok(effects.length > 0, 'publish pipeline must contain publish effect steps to guard');
+
+  const verifyClosure = dependencyClosure(pipeline, 'verify');
+  for (const stepName of effects) {
+    // S1: only the failure-skip semantics of the DAG stand between a failed
+    // verify and this effect — the verify edge in its closure is the proof.
+    const closure = dependencyClosure(pipeline, stepName);
+    assert.ok(
+      closure.has('verify'),
+      `publish effect '${stepName}' must transitively depend on verify (S1) — a failed verify must skip it`,
+    );
+
+    // S6: any step ordered after verify (outside its closure) but inside the
+    // effect's chain must not be able to move HEAD. If the pipeline
+    // legitimately re-checks-out, verify must run after the re-checkout.
+    for (const chainStep of closure) {
+      if (chainStep === 'verify' || verifyClosure.has(chainStep)) continue;
+      assert.ok(
+        !movesHead(pipeline.steps[chainStep]),
+        `step '${chainStep}' sits between verify and publish effect '${stepName}' and can move HEAD (S6)` +
+          ' — verify must re-run after any re-checkout',
+      );
+    }
+  }
+  return effects;
+}
+
+// A minimal but healthy gate used as the base for every negative-control
+// mutation: verify (identity + canonical command) → build → publish-npm,
+// with the publish effect blocked by verify both directly and through build.
+const HEALTHY_GATE_YAML = `
+steps:
+  verify:
+    image: node:24-alpine
+    commands:
+      - |
+        if [ -z "$CI_COMMIT_SHA" ] || [ "$CI_COMMIT_SHA" != "$(git rev-parse HEAD)" ]; then
+          echo "identity mismatch" >&2
+          exit 1
+        fi
+      - pnpm verify:release
+  build:
+    image: node:24-alpine
+    commands:
+      - pnpm build
+    depends_on:
+      - verify
+  publish-npm:
+    image: node:24-alpine
+    commands:
+      - npm publish
+    depends_on:
+      - build
+      - verify
+`;
+
+// Fresh parse per call so every negative control mutates its own object.
+function healthyPipeline() {
+  return parseYaml(HEALTHY_GATE_YAML);
+}
+
+test('the real publish pipeline: a failed verify provably blocks every publish effect', async () => {
+  const pipeline = await readPublishPipeline();
+  const effects = assertPublishGateBlocksOnVerify(pipeline);
+  assert.deepEqual(effects.sort(), [
+    'build-appservice',
+    'build-gateway',
+    'build-web',
+    'publish-next-npm',
+    'publish-npm',
+  ]);
+});
+
+test('fixture sanity: the healthy gate base passes the checker unmutated', () => {
+  assertPublishGateBlocksOnVerify(healthyPipeline());
+});
+
+test('S1 negative control: a publish effect with no verify edge fails the checker', () => {
+  const pipeline = healthyPipeline();
+  pipeline.steps['publish-npm'].depends_on = ['build'];
+  pipeline.steps.build.depends_on = [];
+  assert.throws(
+    () => assertPublishGateBlocksOnVerify(pipeline),
+    /publish-npm.*must transitively depend on verify/s,
+  );
+});
+
+test('S2 negative control: an npm publish hidden behind a non-publish step name fails the checker', () => {
+  const pipeline = healthyPipeline();
+  delete pipeline.steps['publish-npm'];
+  pipeline.steps.build.depends_on = [];
+  pipeline.steps.deploy = {
+    image: 'node:24-alpine',
+    commands: ['npm publish'],
+    depends_on: ['build'],
+  };
+  // Detection must be by COMMAND: the name says "deploy", the commands say
+  // publish — an un-gated effect under either reading.
+  assert.throws(
+    () => assertPublishGateBlocksOnVerify(pipeline),
+    /deploy.*must transitively depend on verify/s,
+  );
+});
+
+test('S2 negative control: a kaniko image push under a build-* name fails the checker when ungated', () => {
+  const pipeline = healthyPipeline();
+  delete pipeline.steps['publish-npm'];
+  pipeline.steps.build.depends_on = [];
+  pipeline.steps['push-platform-image'] = {
+    image: 'gcr.io/kaniko-project/executor:debug',
+    commands: ['/kaniko/executor --context . --destination reg.example/img:latest'],
+    depends_on: ['build'],
+  };
+  assert.throws(
+    () => assertPublishGateBlocksOnVerify(pipeline),
+    /push-platform-image.*must transitively depend on verify/s,
+  );
+});
+
+test('S3 negative control: a detached verify fails the checker', () => {
+  const pipeline = healthyPipeline();
+  pipeline.steps.verify.detach = true;
+  assert.throws(() => assertPublishGateBlocksOnVerify(pipeline), /detached \(S3\)/);
+});
+
+test('S4 negative control: failure: ignore on verify fails the checker', () => {
+  const pipeline = healthyPipeline();
+  pipeline.steps.verify.failure = 'ignore';
+  assert.throws(() => assertPublishGateBlocksOnVerify(pipeline), /tolerate its own failure/);
+});
+
+test('S4 negative control: a success override admitting nonzero exit codes fails the checker', () => {
+  const pipeline = healthyPipeline();
+  pipeline.steps.verify.success = [0, 1];
+  assert.throws(() => assertPublishGateBlocksOnVerify(pipeline), /success exit codes/);
+});
+
+test('S5 negative control: a when filter on verify fails the checker', () => {
+  const pipeline = healthyPipeline();
+  pipeline.steps.verify.when = [{ event: 'push' }];
+  assert.throws(() => assertPublishGateBlocksOnVerify(pipeline), /when\/path filter \(S5\)/);
+});
+
+test('S6 negative control: a HEAD-moving step between verify and publish fails the checker', () => {
+  const pipeline = healthyPipeline();
+  pipeline.steps.resync = {
+    image: 'node:24-alpine',
+    commands: ['git fetch origin', 'git reset --hard origin/main'],
+    depends_on: [],
+  };
+  pipeline.steps.build.depends_on = ['verify', 'resync'];
+  // resync sits AFTER verify in the publish chain (verify does not depend on
+  // it), so the verified commit could be replaced before publishing.
+  assert.throws(() => assertPublishGateBlocksOnVerify(pipeline), /resync.*can move HEAD/s);
+});
+
+test('S6 positive control: a legitimate re-checkout passes when verify re-runs after it', () => {
+  const pipeline = healthyPipeline();
+  pipeline.steps.resync = {
+    image: 'node:24-alpine',
+    commands: ['git fetch origin', 'git reset --hard origin/main'],
+    depends_on: [],
+  };
+  pipeline.steps.verify.depends_on = ['resync'];
+  pipeline.steps.build.depends_on = ['verify'];
+  // resync precedes verify in the chain, so verification covers the
+  // re-checked-out HEAD — the exact-commit contract holds.
+  assertPublishGateBlocksOnVerify(pipeline);
+});
+
+test('S7 negative control: deleting the verify step entirely fails the checker', () => {
+  const pipeline = healthyPipeline();
+  delete pipeline.steps.verify;
+  pipeline.steps['publish-npm'].depends_on = ['build'];
+  assert.throws(() => assertPublishGateBlocksOnVerify(pipeline), /`verify` step/);
+});