Add reviewed-binary push check and scratch regression suite (#1825)
This commit is contained in:
jason.woltje committed
1 parent
b617b80f68
commit
cceea407e2
8 files changed
+453
No files matched your search
Executable
+110
@@ -0,0 +1,110 @@
|
||||
#!/bin/bash
|
||||
# Check an explicit committed push range. Fetch is opt-in; never pushes.
|
||||
command -v node >/dev/null 2>&1 || { echo 'STOP node unavailable'; exit 2; }
|
||||
exec node --input-type=module - "$@" <<'JS'
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { join } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
const stop = (reason, code = 1) => { console.log(`STOP ${reason}`); process.exit(code); };
|
||||
const git = (...args) => {
|
||||
const r = spawnSync('git', args, { maxBuffer: 256 * 1024 * 1024 });
|
||||
if (r.error || r.status !== 0) throw new Error('git');
|
||||
return r.stdout;
|
||||
};
|
||||
// Git paths must round-trip as UTF-8 before comparing them with queue paths.
|
||||
const pathsText = bytes => new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(bytes);
|
||||
const shownPath = path => /[\r\n]/.test(path) ? JSON.stringify(path) : path;
|
||||
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
const text = (...args) => git(...args).toString('utf8').trimEnd();
|
||||
const resolve = ref => text('rev-parse', '--verify', '--end-of-options', `${ref}^{commit}`);
|
||||
const argv = process.argv.slice(2);
|
||||
if (argv.length < 2) stop('usage: push-check.sh SHA EXPECTED_LOG [--row N]... [--base-ref REF] [--fetch]', 2);
|
||||
const [ref, expectedFile] = argv.splice(0, 2);
|
||||
let baseRef = 'refs/remotes/origin/refactor', fetch = false;
|
||||
const rowIds = new Set();
|
||||
while (argv.length) {
|
||||
const flag = argv.shift();
|
||||
if (flag === '--fetch' && !fetch) fetch = true;
|
||||
else if (flag === '--row' && /^[1-9][0-9]*$/.test(argv[0] ?? '') && Number.isSafeInteger(Number(argv[0]))) rowIds.add(Number(argv.shift()));
|
||||
else if (flag === '--base-ref' && argv[0] && !argv[0].startsWith('-')) baseRef = argv.shift();
|
||||
else stop('usage', 2);
|
||||
}
|
||||
let sha, base, expected;
|
||||
try { sha = resolve(ref); expected = readFileSync(expectedFile); }
|
||||
catch { stop('sha or expected log unavailable', 2); }
|
||||
if (fetch) { try { git('fetch', '-q', 'origin'); } catch { stop('fetch', 2); } }
|
||||
try { base = resolve(baseRef); } catch { stop('base ref unavailable', 2); }
|
||||
try {
|
||||
if (!git('log', '--oneline', `${base}..${sha}`).equals(expected)) stop('log');
|
||||
console.log('LOG_OK');
|
||||
try { git('merge-base', '--is-ancestor', base, sha); } catch { stop('ff'); }
|
||||
console.log('FF_OK');
|
||||
|
||||
// --no-renames keeps -z numstat to exactly one path per record. Never parse
|
||||
// quoted path output or consult working-tree bytes.
|
||||
const binaries = pathsText(git('diff', '--no-color', '--no-ext-diff', '--no-textconv', '--no-renames', '--numstat', '-z', base, sha))
|
||||
.split('\0').filter(Boolean).filter(s => s.startsWith('-\t-\t')).map(s => s.slice(4));
|
||||
const tree = commit => {
|
||||
const entries = new Map();
|
||||
for (const entry of pathsText(git('ls-tree', '-r', '-z', commit)).split('\0').filter(Boolean)) {
|
||||
const tab = entry.indexOf('\t');
|
||||
const [mode, type, oid] = entry.slice(0, tab).split(' ');
|
||||
entries.set(entry.slice(tab + 1), { mode, type, oid });
|
||||
}
|
||||
return entries;
|
||||
};
|
||||
const have = tree(sha);
|
||||
const present = binaries.filter(path => have.has(path));
|
||||
let approved = [];
|
||||
if (present.length && rowIds.size) {
|
||||
let rows;
|
||||
try {
|
||||
const root = text('rev-parse', '--show-toplevel');
|
||||
const { rows: readRows } = await import(pathToFileURL(join(root, 'packages/queue/src/store.mjs')).href);
|
||||
const result = readRows({ cwd: process.cwd(), env: process.env });
|
||||
if (result.code !== 0) throw new Error('queue');
|
||||
rows = result.rows;
|
||||
for (const id of rowIds) if (!rows.some(r => r.id === id)) throw new Error('row');
|
||||
} catch { stop('queue unavailable or invalid', 2); }
|
||||
for (const row of rows.filter(r => rowIds.has(r.id))) {
|
||||
const round = row.review?.rounds.at(-1);
|
||||
if (!round || round.receipts?.some(r => r.verdict === 'changes')) continue;
|
||||
if (!round.receipts?.some(r => r.verdict === 'approve' && r.candidate === round.candidate.digest)) continue;
|
||||
approved.push(round.candidate);
|
||||
}
|
||||
}
|
||||
const candidates = new Map();
|
||||
for (const c of approved) {
|
||||
if (c.kind !== 'commit') continue;
|
||||
const candidateTree = tree(c.digest);
|
||||
let parent;
|
||||
try { parent = text('rev-parse', '--verify', `${c.digest}^1`); }
|
||||
catch { parent = text('hash-object', '-t', 'tree', '/dev/null'); }
|
||||
const changed = new Set(pathsText(git('diff-tree', '-r', '--no-renames', '--name-only', '-z', parent, c.digest)).split('\0').filter(Boolean));
|
||||
candidates.set(c.digest, { candidateTree, changed });
|
||||
}
|
||||
for (const path of present) {
|
||||
const h = have.get(path);
|
||||
const digest = hash(git('cat-file', 'blob', h.oid));
|
||||
const matches = approved.some(c => {
|
||||
if (c.kind === 'manifest') return c.text.split('\n').includes(`${digest} ${path}`);
|
||||
const { candidateTree, changed } = candidates.get(c.digest);
|
||||
const entry = candidateTree.get(path);
|
||||
return changed.has(path) && entry?.type === 'blob' && entry.mode === h.mode && hash(git('cat-file', 'blob', entry.oid)) === digest;
|
||||
});
|
||||
if (!matches) stop(`binary ${shownPath(path)} ${digest} not in a reviewed candidate`);
|
||||
}
|
||||
console.log(present.length ? `BINARY_REVIEWED ${present.length}` : 'NO_BINARY');
|
||||
|
||||
const diff = git('diff', '--no-color', '--no-ext-diff', '--no-textconv', base, sha).toString('utf8');
|
||||
const secret = /BEGIN [A-Z ]*PRIVATE KEY|ghp_[A-Za-z0-9]{20}|sk-ant-[A-Za-z0-9-]{10}|xox[bp]-|AKIA[0-9A-Z]{16}|(token|secret|password)["' ]*[:=] *["'][A-Za-z0-9_\-]{24,}/i;
|
||||
if (diff.split('\n').some(line => line.startsWith('+') && secret.test(line))) stop('secrets');
|
||||
console.log('SECRETS_CLEAN');
|
||||
if (/co-authored-by|generated with/i.test(text('log', '--format=%B', `${base}..${sha}`))) stop('attribution');
|
||||
console.log('NO_ATTRIBUTION');
|
||||
console.log(`PUSH_READY ${base}..${sha}`);
|
||||
} catch { stop('git or environment error', 2); }
|
||||
JS
|
||||
Executable
+203
@@ -0,0 +1,203 @@
|
||||
#!/bin/bash
|
||||
# All repositories, queue writes, remotes and mutants are disposable scratch.
|
||||
set -euo pipefail
|
||||
ROOT=$(cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
exec node --input-type=module - "$ROOT" "$@" <<'JS'
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { readFileSync, writeFileSync, mkdtempSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { createHash } from 'node:crypto';
|
||||
const root = process.argv[2];
|
||||
const { scratchRepo, genesisCommitted, cli } = await import(pathToFileURL(join(root, 'packages/queue/tests/helpers.mjs')));
|
||||
const target = process.argv[3] ?? join(root, 'scripts/push-check.sh');
|
||||
const sha256 = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
const binary = Buffer.from([0, 1, 2, 3]);
|
||||
const digest = sha256(binary);
|
||||
const cleanups = [];
|
||||
let count = 0;
|
||||
function fixture() {
|
||||
const repo = scratchRepo({ after: fn => cleanups.push(fn) });
|
||||
genesisCommitted(repo);
|
||||
const run = (args, by = 'darkwing', allow = false) => {
|
||||
const r = cli(repo, args, { by });
|
||||
assert.ok(r.code === 0 || allow, r.err);
|
||||
return r;
|
||||
};
|
||||
run(['move', '6', 'blocked', '--reason', 'paused', '--op', 'block-six-0001']);
|
||||
run(['move', '9', 'in-progress', '--op', 'start-nine-0001']);
|
||||
run(['set', '9', 'reviewers', 'dewey,filbert', '--op', 'set-reviewers-0001'], 'sage');
|
||||
// A real local bare remote and clone, with no network or origin access.
|
||||
const bare = join(repo.base, 'remote.git');
|
||||
repo.g('init', '--bare', '-q', bare);
|
||||
repo.g('clone', '-q', bare, join(repo.base, 'clone'));
|
||||
repo.g('remote', 'add', 'origin', bare);
|
||||
const base = repo.g('rev-parse', 'HEAD').trim();
|
||||
repo.g('update-ref', 'refs/remotes/origin/refactor', base);
|
||||
const commit = (path, bytes, message = 'content') => {
|
||||
if (bytes === null) repo.g('rm', '--', path);
|
||||
else { writeFileSync(join(repo.root, path), bytes); repo.g('add', '--', path); }
|
||||
repo.g('commit', '-q', '-m', message);
|
||||
return repo.g('rev-parse', 'HEAD').trim();
|
||||
};
|
||||
const check = (sha, extra = [], expectedCode = 0, reason = null, script = target) => {
|
||||
const log = join(repo.base, 'expected.log');
|
||||
writeFileSync(log, repo.g('log', '--oneline', `${base}..${sha}`));
|
||||
const r = spawnSync('bash', [script, sha, log, ...extra], { cwd: repo.root, env: repo.env, encoding: 'utf8' });
|
||||
assert.equal(r.status, expectedCode, `${r.stdout}\n${r.stderr}`);
|
||||
const lines = r.stdout.trim().split('\n');
|
||||
if (expectedCode === 0) assert.match(lines.at(-1), /^PUSH_READY /);
|
||||
else {
|
||||
assert.equal(lines.filter(s => s.startsWith('STOP ')).length, 1, r.stdout);
|
||||
assert.match(lines.at(-1), reason ?? /^STOP /);
|
||||
assert.ok(!r.stdout.includes('PUSH_READY'), r.stdout);
|
||||
}
|
||||
count++;
|
||||
return r.stdout;
|
||||
};
|
||||
let n = 0;
|
||||
const round = (candidate, verdict = 'approve') => {
|
||||
n++;
|
||||
const r = run(['move', '9', 'in-review', '--candidate', candidate, '--op', `review-nine-000${n}`], 'darkwing', true);
|
||||
// Missing dummy transport credentials fails the request, after opening a
|
||||
// valid round. Recording a receipt is cooperative and needs no credential.
|
||||
assert.equal(r.code, 1, r.out + r.err);
|
||||
const row = JSON.parse(run(['show', '9']).out);
|
||||
const c = row.review.rounds.at(-1).candidate.digest;
|
||||
run(['review', 'record', '9', '--verdict', verdict, '--candidate', c, '--comment', String(n), '--op', `verdict-nine-00${n}`], 'filbert');
|
||||
};
|
||||
const manifest = (d = digest, path = 'asset.bin') => {
|
||||
const file = join(repo.base, 'manifest.sha256');
|
||||
writeFileSync(file, `${d} ${path}\n`);
|
||||
return file;
|
||||
};
|
||||
return { repo, run, base, commit, check, round, manifest };
|
||||
}
|
||||
try {
|
||||
{
|
||||
const f = fixture();
|
||||
const sha = f.commit('text.txt', 'hello\n');
|
||||
assert.match(f.check(sha), /NO_BINARY\nSECRETS_CLEAN\nNO_ATTRIBUTION\nPUSH_READY /);
|
||||
f.check(sha, ['--base-ref', f.base]);
|
||||
f.check(sha, ['--bad-option'], 2, /^STOP usage/);
|
||||
// Explicit fetch goes only to the scratch remote.
|
||||
f.check(sha, ['--fetch']);
|
||||
const wrong = join(f.repo.base, 'wrong.log'); writeFileSync(wrong, 'wrong\n');
|
||||
const r = spawnSync('bash', [target, sha, wrong], { cwd: f.repo.root, env: f.repo.env, encoding: 'utf8' });
|
||||
assert.equal(r.status, 1); assert.equal(r.stdout, 'STOP log\n'); count++;
|
||||
f.repo.g('update-ref', 'refs/remotes/origin/refactor', sha);
|
||||
const log = join(f.repo.base, 'empty.log'); writeFileSync(log, '');
|
||||
const ff = spawnSync('bash', [target, f.base, log], { cwd: f.repo.root, env: f.repo.env, encoding: 'utf8' });
|
||||
assert.equal(ff.status, 1); assert.equal(ff.stdout, 'LOG_OK\nSTOP ff\n'); count++;
|
||||
}
|
||||
{
|
||||
const f = fixture(), sha = f.commit('asset.bin', binary);
|
||||
f.check(sha, [], 1, /^STOP binary asset.bin /);
|
||||
f.round(f.manifest('0'.repeat(64)));
|
||||
f.check(sha, ['--row', '9'], 1, /^STOP binary asset.bin /);
|
||||
}
|
||||
{
|
||||
const f = fixture(), sha = f.commit('asset.bin', binary);
|
||||
f.round(f.manifest());
|
||||
writeFileSync(join(f.repo.root, 'asset.bin'), 'working tree differs');
|
||||
assert.match(f.check(sha, ['--row', '9']), /BINARY_REVIEWED 1\n/);
|
||||
f.check(sha, [], 1, /^STOP binary asset.bin /);
|
||||
const other = f.commit('asset.bin', Buffer.from([0, 8, 9]));
|
||||
// The working tree is the approved version; the pushed blob isn't.
|
||||
writeFileSync(join(f.repo.root, 'asset.bin'), binary);
|
||||
f.check(other, ['--row', '9'], 1, /^STOP binary asset.bin /);
|
||||
}
|
||||
{
|
||||
const f = fixture(), sha = f.commit('asset.bin', binary);
|
||||
f.round(sha);
|
||||
assert.match(f.check(sha, ['--row', '9']), /BINARY_REVIEWED 1\n/);
|
||||
const other = f.commit('asset.bin', Buffer.from([0, 7]));
|
||||
f.check(other, ['--row', '9'], 1, /^STOP binary asset.bin /);
|
||||
}
|
||||
{
|
||||
const f = fixture(), sha = f.commit('space tab\tasset.bin', binary);
|
||||
f.round(sha);
|
||||
assert.match(f.check(sha, ['--row', '9']), /BINARY_REVIEWED 1\n/);
|
||||
}
|
||||
{
|
||||
const f = fixture(), sha = f.commit('line\nbreak.bin', binary);
|
||||
f.check(sha, [], 1, /^STOP binary "line\\nbreak.bin" /);
|
||||
writeFileSync(Buffer.concat([Buffer.from(f.repo.root + '/'), Buffer.from([255]), Buffer.from('.bin')]), binary);
|
||||
f.repo.g('add', '-A'); f.repo.g('commit', '-q', '-m', 'invalid UTF-8 path');
|
||||
f.check(f.repo.g('rev-parse', 'HEAD').trim(), [], 2, /^STOP git or environment error$/);
|
||||
}
|
||||
{
|
||||
const f = fixture();
|
||||
f.commit('asset.bin', binary);
|
||||
const unrelated = f.commit('text.txt', 'unrelated\n');
|
||||
f.round(unrelated);
|
||||
f.check(unrelated, ['--row', '9'], 1, /^STOP binary asset.bin /);
|
||||
}
|
||||
{
|
||||
const f = fixture(), sha = f.commit('asset.bin', binary);
|
||||
f.round(f.manifest(), 'changes');
|
||||
const row = JSON.parse(f.run(['show', '9']).out);
|
||||
f.run(['review', 'record', '9', '--verdict', 'approve', '--candidate', row.review.rounds.at(-1).candidate.digest, '--comment', '2', '--op', 'approve-dewey-0001'], 'dewey');
|
||||
f.check(sha, ['--row', '9'], 1, /^STOP binary asset.bin /);
|
||||
}
|
||||
{
|
||||
const f = fixture(), sha = f.commit('asset.bin', binary);
|
||||
f.round(f.manifest());
|
||||
f.run(['move', '9', 'in-progress', '--op', 'revise-nine-0001']);
|
||||
f.round(f.manifest('0'.repeat(64)));
|
||||
f.check(sha, ['--row', '9'], 1, /^STOP binary asset.bin /);
|
||||
}
|
||||
{
|
||||
const f = fixture(), sha = f.commit('asset.bin', binary);
|
||||
f.round(f.manifest());
|
||||
// This shape cannot be recorded by the queue. Its read API must reject
|
||||
// tampered receipts before the guard can treat them as authority.
|
||||
const path = f.repo.queuePath;
|
||||
const doc = JSON.parse(readFileSync(path));
|
||||
doc.rows.find(r => r.id === 9).review.rounds.at(-1).receipts[0].candidate = '0'.repeat(64);
|
||||
writeFileSync(path, JSON.stringify(doc));
|
||||
f.check(sha, ['--row', '9'], 2, /^STOP queue unavailable or invalid$/);
|
||||
}
|
||||
{
|
||||
const f = fixture(), added = f.commit('asset.bin', binary);
|
||||
f.repo.g('update-ref', 'refs/remotes/origin/refactor', added);
|
||||
const deleted = f.commit('asset.bin', null);
|
||||
// Explicit base retains the check helper's original expected log range.
|
||||
f.repo.g('update-ref', 'refs/remotes/origin/refactor', f.base);
|
||||
assert.match(f.check(deleted, ['--base-ref', added], 1, /^STOP log$/), /STOP log/);
|
||||
const log = join(f.repo.base, 'delete.log'); writeFileSync(log, f.repo.g('log', '--oneline', `${added}..${deleted}`));
|
||||
const r = spawnSync('bash', [target, deleted, log, '--base-ref', added], { cwd: f.repo.root, env: f.repo.env, encoding: 'utf8' });
|
||||
assert.equal(r.status, 0, r.stdout); assert.match(r.stdout, /NO_BINARY\n/); assert.match(r.stdout.trim().split('\n').at(-1), /^PUSH_READY /); count++;
|
||||
}
|
||||
for (const [message, content, reason, prefix] of [
|
||||
['content', 'password="' + 'a'.repeat(30) + '"\n', 'secrets', 'LOG_OK\nFF_OK\nNO_BINARY\n'],
|
||||
['Generated with example', 'hello\n', 'attribution', 'LOG_OK\nFF_OK\nNO_BINARY\nSECRETS_CLEAN\n'],
|
||||
]) {
|
||||
const f = fixture(), sha = f.commit('text.txt', content, message);
|
||||
assert.equal(f.check(sha, [], 1), `${prefix}STOP ${reason}\n`);
|
||||
}
|
||||
console.log(`PASS push-check ${count} checks`);
|
||||
if (!process.argv[3]) {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'push-check-mutants-'));
|
||||
cleanups.push(() => rmSync(dir, { recursive: true, force: true }));
|
||||
const source = readFileSync(target, 'utf8');
|
||||
const mutants = [
|
||||
['skip-hash', "c.text.split('\\n').includes(`${digest} ${path}`)", "c.text.split('\\n').some(line => line.endsWith(` ${path}`))"],
|
||||
['accept-changes', "round.receipts?.some(r => r.verdict === 'changes')", 'false'],
|
||||
['earlier-round', 'row.review?.rounds.at(-1)', 'row.review?.rounds.at(0)'],
|
||||
['working-tree', "hash(git('cat-file', 'blob', h.oid))", 'hash(readFileSync(path))'],
|
||||
['no-row', 'if (!matches) stop(', 'if (!matches && rowIds.size) stop('],
|
||||
];
|
||||
for (const [name, before, after] of mutants) {
|
||||
assert.ok(source.includes(before), name);
|
||||
const file = join(dir, `${name}.sh`); writeFileSync(file, source.replace(before, after));
|
||||
const r = spawnSync('timeout', ['-k', '10', '120', 'bash', join(root, 'scripts/test-push-check.sh'), file], { encoding: 'utf8', maxBuffer: 4 << 20 });
|
||||
assert.equal(r.status, 1, `${name} survived or infrastructure failed: ${r.status}\n${r.stdout}\n${r.stderr}`);
|
||||
assert.match(r.stderr, /AssertionError/, r.stderr);
|
||||
console.log(`KILLED ${name}`);
|
||||
}
|
||||
}
|
||||
} finally { for (const fn of cleanups.reverse()) fn(); }
|
||||
JS
|
||||
Reference in new issue
Block a user