Compare commits

..

2 Commits

Author SHA1 Message Date
Jarvis
7210b7391a fix(ci): gitignore vite/vitest *.timestamp-*.mjs to stop turbo traversal race
All checks were successful
ci/woodpecker/pr/ci Pipeline was successful
ci/woodpecker/push/ci Pipeline was successful
The push/ci lint step intermittently failed with:

  x Package traversal error: .../packages/macp/vitest.config.ts.timestamp-
    <n>.mjs: IO error ... No such file or directory (os error 2)

vite/vitest/esbuild write a transient *.timestamp-*.mjs next to a TS
config while loading it, then unlink it. The files were untracked but not
ignored, so turbo's package traversal hashed them and raced the unlink.
Ignoring them excludes them from turbo's input set and removes the race.

Same class of fix as the pglite timeout/OOM change in this PR: transient
test tooling artifacts destabilising CI.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-23 23:37:33 -05:00
Jarvis
80570f7040 fix(db): stop pglite migration tests flaking CI on timeout + WASM OOM
Some checks failed
ci/woodpecker/push/ci Pipeline failed
ci/woodpecker/pr/ci Pipeline was successful
packages/db's migrate.test.ts spins up a real PGlite (WASM Postgres)
instance per test and applies the full drizzle migration set. Each case
takes ~3-5s locally and longer on CI, where turbo runs ~20 packages'
suites concurrently. Two failure modes resulted, bouncing between the
push/ci and pr/ci pipelines on identical SHAs:

  FAIL src/migrate.test.ts > runPgliteMigrations > ...
    Error: Test timed out in 5000ms.
    -> memory access out of bounds  (wasm:/wasm/...)

1. The 5s vitest default timeout expires mid-migration -> phantom
   'Test timed out in 5000ms'. Raise testTimeout/hookTimeout to 120s so
   legitimately-slow migrations finish.
2. Each PGlite WASM heap is multi-hundred-MB (RSS ~705MB for this file
   alone); parallel forks multiply the peak and tip the runner into the
   WASM OOM. Pin the package to a single fork so only one instance is
   resident at a time.

Also register packages/db/vitest.config.ts in eslint's allowDefaultProject
(alongside the gateway/storage vitest configs) so the typed lint can parse
the now-non-trivial config.

Verified: full db suite green 3x locally with the new config; each run
~13s, no timeouts, no OOM. eslint clean on both files.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-23 23:29:41 -05:00
6 changed files with 66 additions and 203 deletions

7
.gitignore vendored
View File

@@ -15,3 +15,10 @@ infra/step-ca/dev-password
# Scratch dirs created by the framework git-wrapper shell test harnesses
.mosaic-test-work/
# Transient config files vite/vitest/esbuild write next to a *.config.ts while
# loading it, then unlink. They are untracked but were not ignored, so turbo's
# package traversal hashed them and intermittently failed CI with "Package
# traversal error: ... .timestamp-*.mjs: No such file or directory" when the
# file vanished mid-scan. Ignoring them removes the race.
*.timestamp-*.mjs

View File

@@ -28,6 +28,7 @@ export default tseslint.config(
'apps/web/e2e/helpers/*.ts',
'apps/web/playwright.config.ts',
'apps/gateway/vitest.config.ts',
'packages/db/vitest.config.ts',
'packages/storage/vitest.config.ts',
'packages/mosaic/__tests__/*.ts',
'tools/federation-harness/*.ts',

View File

@@ -4,5 +4,22 @@ export default defineConfig({
test: {
globals: true,
environment: 'node',
// The migration suite spins up a real PGlite (WASM Postgres) instance per
// test and applies the full drizzle migration set. Each case legitimately
// takes ~5s locally and considerably longer on CI, where turbo runs many
// packages' test suites concurrently. The 5s vitest default then expires
// mid-migration and the run fails as a phantom "Test timed out in 5000ms"
// (often surfacing the underlying WASM `memory access out of bounds` when
// the heap is starved). Give migrations real headroom.
testTimeout: 120_000,
hookTimeout: 120_000,
// Each PGlite instance carries a multi-hundred-MB WASM heap. Running test
// files in parallel forks multiplies that peak and is what tips the CI
// runner into the WASM OOM. A single fork keeps only one instance resident
// at a time — slightly slower, but deterministic.
pool: 'forks',
poolOptions: {
forks: { singleFork: true },
},
},
});

View File

@@ -30,7 +30,6 @@ import {
refreshActiveFleetUnits,
readRosterAgentNames,
buildRelaunchCommands,
checkFrameworkDrift,
FRAMEWORK_RESEED_PACKAGE,
} from './runtime/update-checker.js';
import { runWizard } from './wizard.js';
@@ -419,48 +418,6 @@ program
// checkForAllUpdates imported statically above
const { execSync } = await import('node:child_process');
// Re-seed the framework from the freshly-installed package, propagate shipped
// systemd unit fixes to the active units, and (opt-in) relaunch durable
// agents. Shared by the "packages updated" and the "framework drift" paths.
const reseedFramework = (reason: string): void => {
console.log(reason);
const reseed = runFrameworkReseed();
if (!reseed.ok) {
console.error(
`\n⚠ Framework re-seed skipped: ${reseed.reason ?? 'unknown'}.\n` +
' Activate manually: bash "$(npm root -g)/@mosaicstack/mosaic/framework/install.sh" ' +
'(MOSAIC_SYNC_ONLY=1 MOSAIC_INSTALL_MODE=keep)',
);
return;
}
console.log('✔ Framework re-seeded.');
// Propagate shipped systemd unit fixes to the ACTIVE units (re-seed only
// touches ~/.config/mosaic/systemd/user; systemd runs ~/.config/systemd/user).
const units = refreshActiveFleetUnits();
if (units.refreshed.length > 0) {
console.log(`✔ Refreshed ${units.refreshed.length} active systemd unit(s).`);
}
const agents = readRosterAgentNames();
if (agents.length === 0) return;
if (opts.relaunch) {
console.log(`\nRelaunching ${agents.length} fleet agent(s) to pick up the new runtime…`);
for (const restart of buildRelaunchCommands(agents)) {
try {
execSync(restart.join(' '), { stdio: 'inherit', timeout: 30_000 });
} catch {
console.error(` ⚠ failed to restart agent — run: ${restart.join(' ')}`);
}
}
console.log('✔ Agents relaunched.');
} else {
console.log(
`\n ${agents.length} fleet agent(s) are still running the previous runtime. ` +
'Restart them to activate the update:\n mosaic update --relaunch ' +
'(or: mosaic fleet restart <agent>)',
);
}
};
console.log('Checking for updates…');
const results = checkForAllUpdates({ skipCache: true });
@@ -475,18 +432,6 @@ program
process.exit(1);
}
console.log('\n✔ All packages up to date.');
// #642: the CLI may have been upgraded outside `mosaic update` (e.g. a
// direct `npm i -g`), leaving the framework files stale even though no
// package is reported outdated. Detect that via the framework version and
// re-seed so shipped launcher/runtime fixes still activate.
const drift = checkFrameworkDrift();
if (drift.drifted && opts.reseed !== false) {
reseedFramework(
`\nFramework drift detected (on-disk v${drift.installed} < bundled v${drift.bundled}) — ` +
'the CLI was updated outside `mosaic update`. Re-seeding framework files into ' +
'~/.config/mosaic (data-safe; keeps your edits)…',
);
}
return;
}
@@ -511,17 +456,52 @@ program
// F3-m3 / R13: the CLI is updated, but the framework files in
// ~/.config/mosaic/ are still the previous version. Re-seed them from the
// freshly-installed package so shipped launcher/runtime changes ACTIVATE.
// Re-seed when the framework-bearing package itself updated OR the on-disk
// framework is older than the freshly-installed one (#642 — e.g. only
// sibling packages were outdated but the CLI was already ahead).
// Only when the framework-bearing package itself updated.
const mosaicUpdated = outdated.some(
(r: { package: string }) => r.package === FRAMEWORK_RESEED_PACKAGE,
);
const drift = checkFrameworkDrift();
if ((mosaicUpdated || drift.drifted) && opts.reseed !== false) {
reseedFramework(
if (mosaicUpdated && opts.reseed !== false) {
console.log(
'\nRe-seeding framework files into ~/.config/mosaic (data-safe; keeps your edits)…',
);
const reseed = runFrameworkReseed();
if (reseed.ok) {
console.log('✔ Framework re-seeded.');
// Propagate shipped systemd unit fixes to the ACTIVE units (re-seed only
// touches ~/.config/mosaic/systemd/user; systemd runs ~/.config/systemd/user).
const units = refreshActiveFleetUnits();
if (units.refreshed.length > 0) {
console.log(`✔ Refreshed ${units.refreshed.length} active systemd unit(s).`);
}
const agents = readRosterAgentNames();
if (agents.length > 0) {
if (opts.relaunch) {
console.log(
`\nRelaunching ${agents.length} fleet agent(s) to pick up the new runtime…`,
);
for (const restart of buildRelaunchCommands(agents)) {
try {
execSync(restart.join(' '), { stdio: 'inherit', timeout: 30_000 });
} catch {
console.error(` ⚠ failed to restart agent — run: ${restart.join(' ')}`);
}
}
console.log('✔ Agents relaunched.');
} else {
console.log(
`\n ${agents.length} fleet agent(s) are still running the previous runtime. ` +
'Restart them to activate the update:\n mosaic update --relaunch ' +
'(or: mosaic fleet restart <agent>)',
);
}
}
} else {
console.error(
`\n⚠ Framework re-seed skipped: ${reseed.reason ?? 'unknown'}.\n` +
' Activate manually: bash "$(npm root -g)/@mosaicstack/mosaic/framework/install.sh" ' +
'(MOSAIC_SYNC_ONLY=1 MOSAIC_INSTALL_MODE=keep)',
);
}
}
});

View File

@@ -8,9 +8,6 @@ import {
readRosterAgentNames,
runFrameworkReseed,
refreshActiveFleetUnits,
readInstalledFrameworkVersion,
readBundledFrameworkVersion,
checkFrameworkDrift,
} from './update-checker.js';
import { existsSync, readFileSync } from 'node:fs';
@@ -126,73 +123,3 @@ describe('refreshActiveFleetUnits', () => {
expect(existsSync(join(configHome, 'systemd', 'user', 'mosaic-agent@.service'))).toBe(false);
});
});
/**
* #642: re-seed when the on-disk framework is older than the bundled one even
* if no package is reported outdated (CLI upgraded outside `mosaic update`).
*/
describe('framework drift detection', () => {
let home: string; // stand-in for ~/.config/mosaic
let fw: string; // stand-in for the bundled framework root
beforeEach(() => {
const root = mkdtempSync(join(tmpdir(), 'mosaic-drift-'));
home = join(root, 'mosaic');
fw = join(root, 'framework');
mkdirSync(home, { recursive: true });
mkdirSync(fw, { recursive: true });
});
afterEach(() => {
rmSync(join(home, '..'), { recursive: true, force: true });
});
const writeInstalled = (v: string) => writeFileSync(join(home, '.framework-version'), v);
const writeBundled = (v: string) =>
writeFileSync(join(fw, 'install.sh'), `#!/usr/bin/env bash\nFRAMEWORK_VERSION=${v}\n`);
describe('readInstalledFrameworkVersion', () => {
it('returns undefined when the version file is absent', () => {
expect(readInstalledFrameworkVersion(home)).toBeUndefined();
});
it('parses the integer (tolerating surrounding whitespace)', () => {
writeInstalled(' 3\n');
expect(readInstalledFrameworkVersion(home)).toBe(3);
});
it('returns undefined for non-numeric content', () => {
writeInstalled('not-a-number\n');
expect(readInstalledFrameworkVersion(home)).toBeUndefined();
});
});
describe('readBundledFrameworkVersion', () => {
it('returns undefined when install.sh is absent', () => {
expect(readBundledFrameworkVersion(fw)).toBeUndefined();
});
it('parses FRAMEWORK_VERSION=<n> from install.sh', () => {
writeBundled('4');
expect(readBundledFrameworkVersion(fw)).toBe(4);
});
});
describe('checkFrameworkDrift', () => {
it('reports drift when on-disk is older than bundled', () => {
writeInstalled('3');
writeBundled('4');
expect(checkFrameworkDrift(home, fw)).toEqual({ drifted: true, installed: 3, bundled: 4 });
});
it('no drift when versions match', () => {
writeInstalled('4');
writeBundled('4');
expect(checkFrameworkDrift(home, fw)).toMatchObject({ drifted: false });
});
it('no drift when on-disk is newer than bundled', () => {
writeInstalled('5');
writeBundled('4');
expect(checkFrameworkDrift(home, fw)).toMatchObject({ drifted: false });
});
it('no drift (conservative) when a version cannot be read', () => {
writeBundled('4'); // installed version file missing
expect(checkFrameworkDrift(home, fw)).toMatchObject({ drifted: false, bundled: 4 });
});
});
});

View File

@@ -521,75 +521,6 @@ export function runFrameworkReseed(
}
}
// ─── Framework drift detection (#642) ────────────────────────────────────────
//
// `mosaic update` only re-seeds the framework when the @mosaicstack/mosaic
// package itself is upgraded *within that command*. When the CLI is upgraded
// some OTHER way — a direct `npm i -g @mosaicstack/mosaic`, or an upgrade run
// where only sibling packages were outdated — the framework files in
// ~/.config/mosaic stay stale and shipped launcher/runtime fixes never
// activate. Comparing the on-disk framework schema version against the version
// bundled in the installed package detects exactly that situation.
/** Read the framework schema version recorded on disk (~/.config/mosaic/.framework-version). */
export function readInstalledFrameworkVersion(
mosaicHome = join(homedir(), '.config', 'mosaic'),
): number | undefined {
const vf = join(mosaicHome, '.framework-version');
if (!existsSync(vf)) return undefined;
try {
const n = parseInt(readFileSync(vf, 'utf-8').trim(), 10);
return Number.isFinite(n) ? n : undefined;
} catch {
return undefined;
}
}
/**
* Read the framework schema version shipped in the installed package by parsing
* `FRAMEWORK_VERSION=<n>` out of the bundled install.sh (the authoritative
* source the installer writes to .framework-version).
*/
export function readBundledFrameworkVersion(
frameworkRoot = resolveBundledFrameworkRoot(),
): number | undefined {
const installer = join(frameworkRoot, 'install.sh');
if (!existsSync(installer)) return undefined;
try {
const m = readFileSync(installer, 'utf-8').match(/^\s*FRAMEWORK_VERSION=(\d+)/m);
const raw = m?.[1];
if (!raw) return undefined;
const n = parseInt(raw, 10);
return Number.isFinite(n) ? n : undefined;
} catch {
return undefined;
}
}
export interface FrameworkDrift {
/** True only when both versions are known AND the on-disk one is older. */
drifted: boolean;
installed?: number;
bundled?: number;
}
/**
* Detect whether the on-disk framework is older than the framework bundled in
* the installed CLI (#642). Conservative: if either version can't be read the
* result is no-drift, so a missing/unreadable version file never triggers an
* unexpected re-seed.
*/
export function checkFrameworkDrift(
mosaicHome = join(homedir(), '.config', 'mosaic'),
frameworkRoot = resolveBundledFrameworkRoot(),
): FrameworkDrift {
const installed = readInstalledFrameworkVersion(mosaicHome);
const bundled = readBundledFrameworkVersion(frameworkRoot);
const drifted =
typeof installed === 'number' && typeof bundled === 'number' && installed < bundled;
return { drifted, installed, bundled };
}
/**
* Best-effort parse of the fleet roster for agent names (used to relaunch
* durable agents after a re-seed). Returns [] when no roster exists.