chore: consolidate new foundation and archive v1 (#1495)

v1/packages/queue/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@mosaicstack/queue",
+  "version": "0.0.4",
+  "repository": {
+    "type": "git",
+    "url": "https://git.mosaicstack.dev/mosaicstack/stack.git",
+    "directory": "packages/queue"
+  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc",
+    "lint": "eslint src",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run --passWithNoTests"
+  },
+  "dependencies": {
+    "@mosaicstack/types": "workspace:*",
+    "commander": "^13.0.0",
+    "ioredis": "^5.10.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.8.0",
+    "vitest": "^2.0.0"
+  },
+  "publishConfig": {
+    "registry": "https://git.mosaicstack.dev/api/packages/mosaicstack/npm/",
+    "access": "public"
+  },
+  "files": [
+    "dist"
+  ]
+}

v1/packages/queue/src/adapters/bullmq.ts

@@ -0,0 +1,50 @@
+import Redis from 'ioredis';
+
+import type { QueueAdapter, QueueConfig, TaskPayload } from '../types.js';
+
+const DEFAULT_VALKEY_URL = 'redis://localhost:6380';
+
+export function createBullMQAdapter(config: QueueConfig): QueueAdapter {
+  if (config.type !== 'bullmq') {
+    throw new Error(`Expected config type "bullmq", got "${config.type}"`);
+  }
+
+  const url = config.url ?? process.env['VALKEY_URL'] ?? DEFAULT_VALKEY_URL;
+  const redis = new Redis(url, { maxRetriesPerRequest: 3 });
+
+  return {
+    name: 'bullmq',
+
+    async enqueue(queueName: string, payload: TaskPayload): Promise<void> {
+      await redis.lpush(queueName, JSON.stringify(payload));
+    },
+
+    async dequeue(queueName: string): Promise<TaskPayload | null> {
+      const item = await redis.rpop(queueName);
+      if (!item) return null;
+      return JSON.parse(item) as TaskPayload;
+    },
+
+    async length(queueName: string): Promise<number> {
+      return redis.llen(queueName);
+    },
+
+    async publish(channel: string, message: string): Promise<void> {
+      await redis.publish(channel, message);
+    },
+
+    subscribe(channel: string, handler: (message: string) => void): () => void {
+      const sub = redis.duplicate();
+      sub.subscribe(channel).catch(() => {});
+      sub.on('message', (_ch: string, msg: string) => handler(msg));
+      return () => {
+        sub.unsubscribe(channel).catch(() => {});
+        sub.disconnect();
+      };
+    },
+
+    async close(): Promise<void> {
+      await redis.quit();
+    },
+  };
+}

v1/packages/queue/src/adapters/local.test.ts

@@ -0,0 +1,81 @@
+import { mkdtempSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+
+import type { TaskPayload } from '../types.js';
+import { createLocalAdapter } from './local.js';
+
+function makePayload(id: string): TaskPayload {
+  return { id, type: 'test', data: { value: id }, createdAt: new Date().toISOString() };
+}
+
+describe('LocalAdapter', () => {
+  let dataDir: string;
+
+  beforeEach(() => {
+    dataDir = mkdtempSync(join(tmpdir(), 'mosaic-queue-test-'));
+  });
+
+  afterEach(() => {
+    rmSync(dataDir, { recursive: true, force: true });
+  });
+
+  it('enqueue + dequeue in FIFO order', async () => {
+    const adapter = createLocalAdapter({ type: 'local', dataDir });
+    const a = makePayload('a');
+    const b = makePayload('b');
+    const c = makePayload('c');
+
+    await adapter.enqueue('tasks', a);
+    await adapter.enqueue('tasks', b);
+    await adapter.enqueue('tasks', c);
+
+    expect(await adapter.dequeue('tasks')).toEqual(a);
+    expect(await adapter.dequeue('tasks')).toEqual(b);
+    expect(await adapter.dequeue('tasks')).toEqual(c);
+    expect(await adapter.dequeue('tasks')).toBeNull();
+  });
+
+  it('length accuracy', async () => {
+    const adapter = createLocalAdapter({ type: 'local', dataDir });
+
+    expect(await adapter.length('q')).toBe(0);
+    await adapter.enqueue('q', makePayload('1'));
+    await adapter.enqueue('q', makePayload('2'));
+    expect(await adapter.length('q')).toBe(2);
+    await adapter.dequeue('q');
+    expect(await adapter.length('q')).toBe(1);
+  });
+
+  it('publish + subscribe delivery', async () => {
+    const adapter = createLocalAdapter({ type: 'local', dataDir });
+    const received: string[] = [];
+
+    const unsub = adapter.subscribe('chan', (msg) => received.push(msg));
+    await adapter.publish('chan', 'hello');
+    await adapter.publish('chan', 'world');
+
+    expect(received).toEqual(['hello', 'world']);
+
+    unsub();
+    await adapter.publish('chan', 'after-unsub');
+    expect(received).toEqual(['hello', 'world']);
+  });
+
+  it('persistence survives close and re-create', async () => {
+    const p1 = makePayload('x');
+    const p2 = makePayload('y');
+
+    const adapter1 = createLocalAdapter({ type: 'local', dataDir });
+    await adapter1.enqueue('persist-q', p1);
+    await adapter1.enqueue('persist-q', p2);
+    await adapter1.close();
+
+    const adapter2 = createLocalAdapter({ type: 'local', dataDir });
+    expect(await adapter2.length('persist-q')).toBe(2);
+    expect(await adapter2.dequeue('persist-q')).toEqual(p1);
+    expect(await adapter2.dequeue('persist-q')).toEqual(p2);
+    await adapter2.close();
+  });
+});

v1/packages/queue/src/adapters/local.ts

@@ -0,0 +1,87 @@
+import { mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { EventEmitter } from 'node:events';
+
+import type { QueueAdapter, QueueConfig, TaskPayload } from '../types.js';
+
+const DEFAULT_DATA_DIR = '.mosaic/queue';
+
+export function createLocalAdapter(config: QueueConfig): QueueAdapter {
+  if (config.type !== 'local') {
+    throw new Error(`Expected config type "local", got "${config.type}"`);
+  }
+
+  const dataDir = config.dataDir ?? DEFAULT_DATA_DIR;
+  const queues = new Map<string, TaskPayload[]>();
+  const emitter = new EventEmitter();
+
+  mkdirSync(dataDir, { recursive: true });
+
+  // Load existing JSON files on startup
+  for (const file of readdirSync(dataDir)) {
+    if (!file.endsWith('.json')) continue;
+    const queueName = file.slice(0, -5);
+    try {
+      const raw = readFileSync(join(dataDir, file), 'utf-8');
+      const items = JSON.parse(raw) as TaskPayload[];
+      if (Array.isArray(items)) {
+        queues.set(queueName, items);
+      }
+    } catch {
+      // Ignore corrupt files
+    }
+  }
+
+  function persist(queueName: string): void {
+    const items = queues.get(queueName) ?? [];
+    writeFileSync(join(dataDir, `${queueName}.json`), JSON.stringify(items), 'utf-8');
+  }
+
+  function getQueue(queueName: string): TaskPayload[] {
+    let q = queues.get(queueName);
+    if (!q) {
+      q = [];
+      queues.set(queueName, q);
+    }
+    return q;
+  }
+
+  return {
+    name: 'local',
+
+    async enqueue(queueName: string, payload: TaskPayload): Promise<void> {
+      getQueue(queueName).push(payload);
+      persist(queueName);
+    },
+
+    async dequeue(queueName: string): Promise<TaskPayload | null> {
+      const q = getQueue(queueName);
+      const item = q.shift() ?? null;
+      persist(queueName);
+      return item;
+    },
+
+    async length(queueName: string): Promise<number> {
+      return getQueue(queueName).length;
+    },
+
+    async publish(channel: string, message: string): Promise<void> {
+      emitter.emit(channel, message);
+    },
+
+    subscribe(channel: string, handler: (message: string) => void): () => void {
+      emitter.on(channel, handler);
+      return () => {
+        emitter.off(channel, handler);
+      };
+    },
+
+    async close(): Promise<void> {
+      for (const queueName of queues.keys()) {
+        persist(queueName);
+      }
+      queues.clear();
+      emitter.removeAllListeners();
+    },
+  };
+}

v1/packages/queue/src/cli.spec.ts

@@ -0,0 +1,62 @@
+import { describe, it, expect } from 'vitest';
+import { Command } from 'commander';
+import { registerQueueCommand } from './cli.js';
+
+describe('registerQueueCommand', () => {
+  function buildProgram(): Command {
+    const program = new Command('mosaic');
+    registerQueueCommand(program);
+    return program;
+  }
+
+  it('registers a "queue" subcommand', () => {
+    const program = buildProgram();
+    const queueCmd = program.commands.find((c) => c.name() === 'queue');
+    expect(queueCmd).toBeDefined();
+  });
+
+  it('queue has list, stats, pause, resume, jobs, drain subcommands', () => {
+    const program = buildProgram();
+    const queueCmd = program.commands.find((c) => c.name() === 'queue');
+    expect(queueCmd).toBeDefined();
+
+    const names = queueCmd!.commands.map((c) => c.name());
+    expect(names).toContain('list');
+    expect(names).toContain('stats');
+    expect(names).toContain('pause');
+    expect(names).toContain('resume');
+    expect(names).toContain('jobs');
+    expect(names).toContain('drain');
+  });
+
+  it('jobs subcommand has a "tail" subcommand', () => {
+    const program = buildProgram();
+    const queueCmd = program.commands.find((c) => c.name() === 'queue');
+    const jobsCmd = queueCmd!.commands.find((c) => c.name() === 'jobs');
+    expect(jobsCmd).toBeDefined();
+
+    const tailCmd = jobsCmd!.commands.find((c) => c.name() === 'tail');
+    expect(tailCmd).toBeDefined();
+  });
+
+  it('drain has a --yes option', () => {
+    const program = buildProgram();
+    const queueCmd = program.commands.find((c) => c.name() === 'queue');
+    const drainCmd = queueCmd!.commands.find((c) => c.name() === 'drain');
+    expect(drainCmd).toBeDefined();
+
+    const optionNames = drainCmd!.options.map((o) => o.long);
+    expect(optionNames).toContain('--yes');
+  });
+
+  it('stats accepts an optional [name] argument', () => {
+    const program = buildProgram();
+    const queueCmd = program.commands.find((c) => c.name() === 'queue');
+    const statsCmd = queueCmd!.commands.find((c) => c.name() === 'stats');
+    expect(statsCmd).toBeDefined();
+    // Should not throw when called without argument
+    const args = statsCmd!.registeredArguments;
+    expect(args.length).toBe(1);
+    expect(args[0]!.required).toBe(false);
+  });
+});

v1/packages/queue/src/cli.ts

@@ -0,0 +1,248 @@
+import type { Command } from 'commander';
+
+import { createLocalAdapter } from './adapters/local.js';
+import type { QueueConfig } from './types.js';
+
+/** Resolve adapter type from env; defaults to 'local'. */
+function resolveAdapterType(): 'bullmq' | 'local' {
+  const t = process.env['QUEUE_ADAPTER'] ?? 'local';
+  return t === 'bullmq' ? 'bullmq' : 'local';
+}
+
+function resolveConfig(): QueueConfig {
+  const type = resolveAdapterType();
+  if (type === 'bullmq') {
+    return { type: 'bullmq', url: process.env['VALKEY_URL'] };
+  }
+  return { type: 'local', dataDir: process.env['QUEUE_DATA_DIR'] };
+}
+
+const BULLMQ_ONLY_MSG =
+  'not supported by local adapter — use the bullmq tier for this (set QUEUE_ADAPTER=bullmq)';
+
+/**
+ * Register queue subcommands on an existing Commander program.
+ * Follows the same pattern as registerQualityRails in @mosaicstack/quality-rails.
+ */
+export function registerQueueCommand(parent: Command): void {
+  buildQueueCommand(parent.command('queue').description('Manage Mosaic job queues'));
+}
+
+function buildQueueCommand(queue: Command): void {
+  // ─── list ──────────────────────────────────────────────────────────────
+  queue
+    .command('list')
+    .description('List all queues known to the configured adapter')
+    .action(async () => {
+      const config = resolveConfig();
+
+      if (config.type === 'local') {
+        const adapter = createLocalAdapter(config);
+        // Local adapter tracks queues in its internal Map; we expose them by
+        // listing JSON files in the data dir.
+        const { readdirSync } = await import('node:fs');
+        const { existsSync } = await import('node:fs');
+        const dataDir = config.dataDir ?? '.mosaic/queue';
+        if (!existsSync(dataDir)) {
+          console.log('No queues found (data dir does not exist yet).');
+          await adapter.close();
+          return;
+        }
+        const files = readdirSync(dataDir).filter((f: string) => f.endsWith('.json'));
+        if (files.length === 0) {
+          console.log('No queues found.');
+        } else {
+          console.log('Queues (local adapter):');
+          for (const f of files) {
+            console.log(`  - ${f.slice(0, -5)}`);
+          }
+        }
+        await adapter.close();
+        return;
+      }
+
+      // bullmq — not enough info to enumerate queues without a BullMQ Board
+      console.log(BULLMQ_ONLY_MSG);
+      process.exit(0);
+    });
+
+  // ─── stats ─────────────────────────────────────────────────────────────
+  queue
+    .command('stats [name]')
+    .description('Show stats for a queue (or all queues)')
+    .action(async (name?: string) => {
+      const config = resolveConfig();
+
+      if (config.type === 'local') {
+        const adapter = createLocalAdapter(config);
+        const { readdirSync } = await import('node:fs');
+        const { existsSync } = await import('node:fs');
+        const dataDir = config.dataDir ?? '.mosaic/queue';
+
+        let names: string[] = [];
+        if (name) {
+          names = [name];
+        } else {
+          if (existsSync(dataDir)) {
+            names = readdirSync(dataDir)
+              .filter((f: string) => f.endsWith('.json'))
+              .map((f: string) => f.slice(0, -5));
+          }
+        }
+
+        if (names.length === 0) {
+          console.log('No queues found.');
+          await adapter.close();
+          return;
+        }
+
+        for (const queueName of names) {
+          const len = await adapter.length(queueName);
+          console.log(`Queue: ${queueName}`);
+          console.log(`  waiting:   ${len}`);
+          console.log(`  active:    0 (local adapter — no active tracking)`);
+          console.log(`  completed: 0 (local adapter — no completed tracking)`);
+          console.log(`  failed:    0 (local adapter — no failed tracking)`);
+          console.log(`  delayed:   0 (local adapter — no delayed tracking)`);
+        }
+        await adapter.close();
+        return;
+      }
+
+      // bullmq
+      console.log(BULLMQ_ONLY_MSG);
+      process.exit(0);
+    });
+
+  // ─── pause ─────────────────────────────────────────────────────────────
+  queue
+    .command('pause <name>')
+    .description('Pause job processing for a queue')
+    .action(async (_name: string) => {
+      const config = resolveConfig();
+
+      if (config.type === 'local') {
+        console.log(BULLMQ_ONLY_MSG);
+        process.exit(0);
+        return;
+      }
+
+      console.log(BULLMQ_ONLY_MSG);
+      process.exit(0);
+    });
+
+  // ─── resume ────────────────────────────────────────────────────────────
+  queue
+    .command('resume <name>')
+    .description('Resume job processing for a queue')
+    .action(async (_name: string) => {
+      const config = resolveConfig();
+
+      if (config.type === 'local') {
+        console.log(BULLMQ_ONLY_MSG);
+        process.exit(0);
+        return;
+      }
+
+      console.log(BULLMQ_ONLY_MSG);
+      process.exit(0);
+    });
+
+  // ─── jobs tail ─────────────────────────────────────────────────────────
+  const jobs = queue.command('jobs').description('Job-level operations');
+
+  jobs
+    .command('tail [name]')
+    .description('Stream new jobs as they arrive (poll-based)')
+    .option('--interval <ms>', 'Poll interval in ms', '2000')
+    .action(async (name: string | undefined, opts: { interval: string }) => {
+      const config = resolveConfig();
+      const pollMs = parseInt(opts.interval, 10);
+
+      if (config.type === 'local') {
+        const adapter = createLocalAdapter(config);
+        const { existsSync, readdirSync } = await import('node:fs');
+        const dataDir = config.dataDir ?? '.mosaic/queue';
+
+        let names: string[] = [];
+        if (name) {
+          names = [name];
+        } else {
+          if (existsSync(dataDir)) {
+            names = readdirSync(dataDir)
+              .filter((f: string) => f.endsWith('.json'))
+              .map((f: string) => f.slice(0, -5));
+          }
+        }
+
+        if (names.length === 0) {
+          console.log('No queues to tail.');
+          await adapter.close();
+          return;
+        }
+
+        console.log(`Tailing queues: ${names.join(', ')} (Ctrl-C to stop)`);
+        const lastLen = new Map<string, number>();
+        for (const qn of names) {
+          lastLen.set(qn, await adapter.length(qn));
+        }
+
+        const timer = setInterval(async () => {
+          for (const qn of names) {
+            const len = await adapter.length(qn);
+            const prev = lastLen.get(qn) ?? 0;
+            if (len > prev) {
+              console.log(
+                `[${new Date().toISOString()}] ${qn}: ${len - prev} new job(s) (total: ${len})`,
+              );
+            }
+            lastLen.set(qn, len);
+          }
+        }, pollMs);
+
+        process.on('SIGINT', async () => {
+          clearInterval(timer);
+          await adapter.close();
+          process.exit(0);
+        });
+
+        return;
+      }
+
+      // bullmq — use subscribe on the channel
+      console.log(BULLMQ_ONLY_MSG);
+      process.exit(0);
+    });
+
+  // ─── drain ─────────────────────────────────────────────────────────────
+  queue
+    .command('drain <name>')
+    .description('Drain all pending jobs from a queue')
+    .option('--yes', 'Skip confirmation prompt')
+    .action(async (name: string, opts: { yes?: boolean }) => {
+      if (!opts.yes) {
+        console.error(
+          `WARNING: This will remove all pending jobs from queue "${name}". Re-run with --yes to confirm.`,
+        );
+        process.exit(1);
+        return;
+      }
+
+      const config = resolveConfig();
+
+      if (config.type === 'local') {
+        const adapter = createLocalAdapter(config);
+        let removed = 0;
+        while ((await adapter.length(name)) > 0) {
+          await adapter.dequeue(name);
+          removed++;
+        }
+        console.log(`Drained ${removed} job(s) from queue "${name}".`);
+        await adapter.close();
+        return;
+      }
+
+      console.log(BULLMQ_ONLY_MSG);
+      process.exit(0);
+    });
+}

v1/packages/queue/src/factory.ts

@@ -0,0 +1,18 @@
+import type { QueueAdapter, QueueConfig } from './types.js';
+
+type QueueType = QueueConfig['type'];
+
+const registry = new Map<QueueType, (config: QueueConfig) => QueueAdapter>();
+
+export function registerQueueAdapter(
+  type: QueueType,
+  factory: (config: QueueConfig) => QueueAdapter,
+): void {
+  registry.set(type, factory);
+}
+
+export function createQueueAdapter(config: QueueConfig): QueueAdapter {
+  const factory = registry.get(config.type);
+  if (!factory) throw new Error(`No adapter registered for type: ${config.type}`);
+  return factory(config);
+}

v1/packages/queue/src/index.ts

@@ -0,0 +1,21 @@
+export {
+  createQueue,
+  createQueueClient,
+  type QueueConfig,
+  type QueueHandle,
+  type QueueClient,
+  type TaskPayload,
+} from './queue.js';
+
+export { type QueueAdapter, type QueueConfig as QueueAdapterConfig } from './types.js';
+export { createQueueAdapter, registerQueueAdapter } from './factory.js';
+export { createBullMQAdapter } from './adapters/bullmq.js';
+export { createLocalAdapter } from './adapters/local.js';
+export { registerQueueCommand } from './cli.js';
+
+import { registerQueueAdapter } from './factory.js';
+import { createBullMQAdapter } from './adapters/bullmq.js';
+import { createLocalAdapter } from './adapters/local.js';
+
+registerQueueAdapter('bullmq', createBullMQAdapter);
+registerQueueAdapter('local', createLocalAdapter);

v1/packages/queue/src/queue.ts

@@ -0,0 +1,67 @@
+import Redis from 'ioredis';
+
+const DEFAULT_VALKEY_URL = 'redis://localhost:6380';
+
+export interface QueueConfig {
+  url?: string;
+}
+
+export interface QueueHandle {
+  redis: Redis;
+  close: () => Promise<void>;
+}
+
+export interface TaskPayload {
+  id: string;
+  type: string;
+  data: Record<string, unknown>;
+  createdAt: string;
+}
+
+export function createQueue(config?: QueueConfig): QueueHandle {
+  const url = config?.url ?? process.env['VALKEY_URL'] ?? DEFAULT_VALKEY_URL;
+  const redis = new Redis(url, { maxRetriesPerRequest: 3 });
+
+  return {
+    redis,
+    close: async () => {
+      await redis.quit();
+    },
+  };
+}
+
+export function createQueueClient(handle: QueueHandle) {
+  const { redis } = handle;
+
+  return {
+    async enqueue(queueName: string, payload: TaskPayload): Promise<void> {
+      await redis.lpush(queueName, JSON.stringify(payload));
+    },
+
+    async dequeue(queueName: string): Promise<TaskPayload | null> {
+      const item = await redis.rpop(queueName);
+      if (!item) return null;
+      return JSON.parse(item) as TaskPayload;
+    },
+
+    async length(queueName: string): Promise<number> {
+      return redis.llen(queueName);
+    },
+
+    async publish(channel: string, message: string): Promise<void> {
+      await redis.publish(channel, message);
+    },
+
+    subscribe(channel: string, handler: (message: string) => void): () => void {
+      const sub = redis.duplicate();
+      sub.subscribe(channel).catch(() => {});
+      sub.on('message', (_ch: string, msg: string) => handler(msg));
+      return () => {
+        sub.unsubscribe(channel).catch(() => {});
+        sub.disconnect();
+      };
+    },
+  };
+}
+
+export type QueueClient = ReturnType<typeof createQueueClient>;

v1/packages/queue/src/types.ts

@@ -0,0 +1,18 @@
+export interface TaskPayload {
+  id: string;
+  type: string;
+  data: Record<string, unknown>;
+  createdAt: string;
+}
+
+export interface QueueAdapter {
+  readonly name: string;
+  enqueue(queueName: string, payload: TaskPayload): Promise<void>;
+  dequeue(queueName: string): Promise<TaskPayload | null>;
+  length(queueName: string): Promise<number>;
+  publish(channel: string, message: string): Promise<void>;
+  subscribe(channel: string, handler: (message: string) => void): () => void;
+  close(): Promise<void>;
+}
+
+export type QueueConfig = { type: 'bullmq'; url?: string } | { type: 'local'; dataDir?: string };

v1/packages/queue/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "dist",
+    "rootDir": "src"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

v1/packages/queue/vitest.config.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    globals: true,
+    environment: 'node',
+  },
+});