chore: consolidate new foundation and archive v1 (#1495)

v1/packages/log/src/agent-logs.ts

@@ -0,0 +1,136 @@
+import { eq, and, desc, lt, sql, type Db, agentLogs } from '@mosaicstack/db';
+
+export type AgentLog = typeof agentLogs.$inferSelect;
+export type NewAgentLog = typeof agentLogs.$inferInsert;
+
+export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
+export type LogCategory = 'decision' | 'tool_use' | 'learning' | 'error' | 'general';
+export type LogTier = 'hot' | 'warm' | 'cold';
+
+export interface LogQuery {
+  userId?: string;
+  sessionId?: string;
+  level?: LogLevel;
+  category?: LogCategory;
+  tier?: LogTier;
+  since?: Date;
+  until?: Date;
+  limit?: number;
+  offset?: number;
+}
+
+export function createAgentLogsRepo(db: Db) {
+  return {
+    async ingest(entry: NewAgentLog): Promise<AgentLog> {
+      const rows = await db.insert(agentLogs).values(entry).returning();
+      return rows[0]!;
+    },
+
+    async ingestBatch(entries: NewAgentLog[]): Promise<AgentLog[]> {
+      if (entries.length === 0) return [];
+      return db.insert(agentLogs).values(entries).returning();
+    },
+
+    async query(params: LogQuery): Promise<AgentLog[]> {
+      const conditions = [];
+
+      if (params.userId) conditions.push(eq(agentLogs.userId, params.userId));
+      if (params.sessionId) conditions.push(eq(agentLogs.sessionId, params.sessionId));
+      if (params.level) conditions.push(eq(agentLogs.level, params.level));
+      if (params.category) conditions.push(eq(agentLogs.category, params.category));
+      if (params.tier) conditions.push(eq(agentLogs.tier, params.tier));
+      if (params.since) conditions.push(sql`${agentLogs.createdAt} >= ${params.since}`);
+      if (params.until) conditions.push(sql`${agentLogs.createdAt} <= ${params.until}`);
+
+      const where = conditions.length > 0 ? and(...conditions) : undefined;
+
+      return db
+        .select()
+        .from(agentLogs)
+        .where(where)
+        .orderBy(desc(agentLogs.createdAt))
+        .limit(params.limit ?? 100)
+        .offset(params.offset ?? 0);
+    },
+
+    async findById(id: string): Promise<AgentLog | undefined> {
+      const rows = await db.select().from(agentLogs).where(eq(agentLogs.id, id));
+      return rows[0];
+    },
+
+    /**
+     * Transition hot logs for one session to warm tier. Session retention is
+     * default-deny: no other session's logs can be changed by this operation.
+     */
+    async promoteSessionToWarm(sessionId: string, olderThan: Date): Promise<number> {
+      const result = await db
+        .update(agentLogs)
+        .set({ tier: 'warm', summarizedAt: new Date() })
+        .where(
+          and(
+            eq(agentLogs.sessionId, sessionId),
+            eq(agentLogs.tier, 'hot'),
+            lt(agentLogs.createdAt, olderThan),
+          ),
+        )
+        .returning();
+      return result.length;
+    },
+
+    /**
+     * Transition hot logs older than the cutoff to warm tier.
+     * Reserved for a separately authorized global retention job.
+     */
+    async promoteToWarm(olderThan: Date): Promise<number> {
+      const result = await db
+        .update(agentLogs)
+        .set({ tier: 'warm', summarizedAt: new Date() })
+        .where(and(eq(agentLogs.tier, 'hot'), lt(agentLogs.createdAt, olderThan)))
+        .returning();
+      return result.length;
+    },
+
+    /**
+     * Transition warm logs older than the cutoff to cold tier.
+     */
+    async promoteToCold(olderThan: Date): Promise<number> {
+      const result = await db
+        .update(agentLogs)
+        .set({ tier: 'cold', archivedAt: new Date() })
+        .where(and(eq(agentLogs.tier, 'warm'), lt(agentLogs.createdAt, olderThan)))
+        .returning();
+      return result.length;
+    },
+
+    /**
+     * Delete cold logs older than the retention period.
+     */
+    async purge(olderThan: Date): Promise<number> {
+      const result = await db
+        .delete(agentLogs)
+        .where(and(eq(agentLogs.tier, 'cold'), lt(agentLogs.createdAt, olderThan)))
+        .returning();
+      return result.length;
+    },
+
+    /**
+     * Get hot logs ready for summarization (decisions + learnings).
+     */
+    async getLogsForSummarization(olderThan: Date, limit = 100): Promise<AgentLog[]> {
+      return db
+        .select()
+        .from(agentLogs)
+        .where(
+          and(
+            eq(agentLogs.tier, 'hot'),
+            lt(agentLogs.createdAt, olderThan),
+            sql`${agentLogs.category} IN ('decision', 'learning', 'tool_use')`,
+          ),
+        )
+        .orderBy(agentLogs.createdAt)
+        .limit(limit);
+    },
+  };
+}
+
+export type AgentLogsRepo = ReturnType<typeof createAgentLogsRepo>;

v1/packages/log/src/cli.spec.ts

@@ -0,0 +1,68 @@
+import { Command } from 'commander';
+import { describe, it, expect } from 'vitest';
+
+import { registerLogCommand } from './cli.js';
+
+function buildTestProgram(): Command {
+  const program = new Command('mosaic');
+  program.exitOverride(); // prevent process.exit in tests
+  registerLogCommand(program);
+  return program;
+}
+
+describe('registerLogCommand', () => {
+  it('registers a "log" subcommand on the parent', () => {
+    const program = buildTestProgram();
+    const names = program.commands.map((c) => c.name());
+    expect(names).toContain('log');
+  });
+
+  it('log command has tail, search, export, and level subcommands', () => {
+    const program = buildTestProgram();
+    const logCmd = program.commands.find((c) => c.name() === 'log');
+    expect(logCmd).toBeDefined();
+    const subNames = logCmd!.commands.map((c) => c.name());
+    expect(subNames).toContain('tail');
+    expect(subNames).toContain('search');
+    expect(subNames).toContain('export');
+    expect(subNames).toContain('level');
+  });
+
+  it('tail subcommand has expected options', () => {
+    const program = buildTestProgram();
+    const logCmd = program.commands.find((c) => c.name() === 'log')!;
+    const tailCmd = logCmd.commands.find((c) => c.name() === 'tail')!;
+    const optionNames = tailCmd.options.map((o) => o.long);
+    expect(optionNames).toContain('--agent');
+    expect(optionNames).toContain('--level');
+    expect(optionNames).toContain('--category');
+    expect(optionNames).toContain('--tier');
+    expect(optionNames).toContain('--limit');
+    expect(optionNames).toContain('--db');
+  });
+
+  it('search subcommand accepts a positional query argument', () => {
+    const program = buildTestProgram();
+    const logCmd = program.commands.find((c) => c.name() === 'log')!;
+    const searchCmd = logCmd.commands.find((c) => c.name() === 'search')!;
+    // Commander stores positional args in _args
+    const argNames = searchCmd.registeredArguments.map((a) => a.name());
+    expect(argNames).toContain('query');
+  });
+
+  it('export subcommand accepts a positional path argument', () => {
+    const program = buildTestProgram();
+    const logCmd = program.commands.find((c) => c.name() === 'log')!;
+    const exportCmd = logCmd.commands.find((c) => c.name() === 'export')!;
+    const argNames = exportCmd.registeredArguments.map((a) => a.name());
+    expect(argNames).toContain('path');
+  });
+
+  it('level subcommand accepts a positional level argument', () => {
+    const program = buildTestProgram();
+    const logCmd = program.commands.find((c) => c.name() === 'log')!;
+    const levelCmd = logCmd.commands.find((c) => c.name() === 'level')!;
+    const argNames = levelCmd.registeredArguments.map((a) => a.name());
+    expect(argNames).toContain('level');
+  });
+});

v1/packages/log/src/cli.ts

@@ -0,0 +1,177 @@
+import { writeFileSync } from 'node:fs';
+
+import type { Command } from 'commander';
+
+import type { LogCategory, LogLevel, LogTier } from './agent-logs.js';
+
+interface FilterOptions {
+  agent?: string;
+  level?: string;
+  category?: string;
+  tier?: string;
+  limit?: string;
+  db?: string;
+}
+
+function parseLimit(raw: string | undefined, defaultVal = 50): number {
+  if (!raw) return defaultVal;
+  const n = parseInt(raw, 10);
+  return Number.isFinite(n) && n > 0 ? n : defaultVal;
+}
+
+function buildQuery(opts: FilterOptions) {
+  return {
+    ...(opts.agent ? { sessionId: opts.agent } : {}),
+    ...(opts.level ? { level: opts.level as LogLevel } : {}),
+    ...(opts.category ? { category: opts.category as LogCategory } : {}),
+    ...(opts.tier ? { tier: opts.tier as LogTier } : {}),
+    limit: parseLimit(opts.limit),
+  };
+}
+
+async function openDb(connectionString: string) {
+  const { createDb } = await import('@mosaicstack/db');
+  return createDb(connectionString);
+}
+
+function resolveConnectionString(opts: FilterOptions): string | undefined {
+  return opts.db ?? process.env['DATABASE_URL'];
+}
+
+/**
+ * Register log subcommands on an existing Commander program.
+ * This avoids cross-package Commander version mismatches by using the
+ * caller's Command instance directly.
+ */
+export function registerLogCommand(parent: Command): void {
+  const log = parent.command('log').description('Query and manage agent logs');
+
+  // ─── tail ───────────────────────────────────────────────────────────────
+
+  log
+    .command('tail')
+    .description('Tail recent agent logs')
+    .option('--agent <id>', 'Filter by agent/session ID')
+    .option('--level <level>', 'Filter by log level (debug|info|warn|error)')
+    .option('--category <cat>', 'Filter by category (decision|tool_use|learning|error|general)')
+    .option('--tier <tier>', 'Filter by tier (hot|warm|cold)')
+    .option('--limit <n>', 'Number of logs to return (default 50)', '50')
+    .option('--db <connection-string>', 'Database connection string (or set DATABASE_URL)')
+    .action(async (opts: FilterOptions) => {
+      const connStr = resolveConnectionString(opts);
+      if (!connStr) {
+        console.error('Database connection required: use --db or set DATABASE_URL');
+        process.exit(1);
+      }
+
+      const handle = await openDb(connStr);
+      try {
+        const { createLogService } = await import('./log-service.js');
+        const svc = createLogService(handle.db);
+        const query = buildQuery(opts);
+
+        const logs = await svc.logs.query(query);
+        if (logs.length === 0) {
+          console.log('No logs found.');
+          return;
+        }
+        for (const entry of logs) {
+          const ts = new Date(entry.createdAt).toISOString();
+          console.log(`[${ts}] [${entry.level}] [${entry.category}] ${entry.content}`);
+        }
+      } finally {
+        await handle.close();
+      }
+    });
+
+  // ─── search ─────────────────────────────────────────────────────────────
+
+  log
+    .command('search <query>')
+    .description('Full-text search over agent logs')
+    .option('--agent <id>', 'Filter by agent/session ID')
+    .option('--level <level>', 'Filter by log level (debug|info|warn|error)')
+    .option('--category <cat>', 'Filter by category (decision|tool_use|learning|error|general)')
+    .option('--tier <tier>', 'Filter by tier (hot|warm|cold)')
+    .option('--limit <n>', 'Number of logs to return (default 50)', '50')
+    .option('--db <connection-string>', 'Database connection string (or set DATABASE_URL)')
+    .action(async (query: string, opts: FilterOptions) => {
+      const connStr = resolveConnectionString(opts);
+      if (!connStr) {
+        console.error('Database connection required: use --db or set DATABASE_URL');
+        process.exit(1);
+      }
+
+      const handle = await openDb(connStr);
+      try {
+        const { createLogService } = await import('./log-service.js');
+        const svc = createLogService(handle.db);
+        const baseQuery = buildQuery(opts);
+
+        const logs = await svc.logs.query(baseQuery);
+        const lowerQ = query.toLowerCase();
+        const matched = logs.filter(
+          (e) =>
+            e.content.toLowerCase().includes(lowerQ) ||
+            (e.metadata != null && JSON.stringify(e.metadata).toLowerCase().includes(lowerQ)),
+        );
+
+        if (matched.length === 0) {
+          console.log('No matching logs found.');
+          return;
+        }
+        for (const entry of matched) {
+          const ts = new Date(entry.createdAt).toISOString();
+          console.log(`[${ts}] [${entry.level}] [${entry.category}] ${entry.content}`);
+        }
+      } finally {
+        await handle.close();
+      }
+    });
+
+  // ─── export ─────────────────────────────────────────────────────────────
+
+  log
+    .command('export <path>')
+    .description('Export matching logs to an NDJSON file')
+    .option('--agent <id>', 'Filter by agent/session ID')
+    .option('--level <level>', 'Filter by log level (debug|info|warn|error)')
+    .option('--category <cat>', 'Filter by category (decision|tool_use|learning|error|general)')
+    .option('--tier <tier>', 'Filter by tier (hot|warm|cold)')
+    .option('--limit <n>', 'Number of logs to export (default 50)', '50')
+    .option('--db <connection-string>', 'Database connection string (or set DATABASE_URL)')
+    .action(async (outputPath: string, opts: FilterOptions) => {
+      const connStr = resolveConnectionString(opts);
+      if (!connStr) {
+        console.error('Database connection required: use --db or set DATABASE_URL');
+        process.exit(1);
+      }
+
+      const handle = await openDb(connStr);
+      try {
+        const { createLogService } = await import('./log-service.js');
+        const svc = createLogService(handle.db);
+        const query = buildQuery(opts);
+
+        const logs = await svc.logs.query(query);
+        const ndjson = logs.map((e) => JSON.stringify(e)).join('\n');
+        writeFileSync(outputPath, ndjson, 'utf8');
+        console.log(`Exported ${logs.length} log(s) to ${outputPath}`);
+      } finally {
+        await handle.close();
+      }
+    });
+
+  // ─── level ──────────────────────────────────────────────────────────────
+
+  log
+    .command('level <level>')
+    .description('Set runtime log level for the connected log service')
+    .action((level: string) => {
+      void level;
+      console.log(
+        'Runtime log level adjustment is not supported in current mode (DB-backed log service).',
+      );
+      process.exitCode = 0;
+    });
+}

v1/packages/log/src/index.ts

@@ -0,0 +1,24 @@
+export { createLogService, type LogService } from './log-service.js';
+export {
+  createAgentLogsRepo,
+  type AgentLogsRepo,
+  type AgentLog,
+  type NewAgentLog,
+  type LogLevel,
+  type LogCategory,
+  type LogTier,
+  type LogQuery,
+} from './agent-logs.js';
+export { registerLogCommand } from './cli.js';
+export {
+  redactSensitiveContent,
+  type RedactionResult,
+  type SensitiveClassification,
+} from './redaction.js';
+export {
+  createRuntimeAuditLogEntry,
+  type RuntimeAuditEvent,
+  type RuntimeAuditErrorCode,
+  type RuntimeAuditOperation,
+  type RuntimeAuditOutcome,
+} from './runtime-audit.js';

v1/packages/log/src/log-service.ts

@@ -0,0 +1,12 @@
+import type { Db } from '@mosaicstack/db';
+import { createAgentLogsRepo, type AgentLogsRepo } from './agent-logs.js';
+
+export interface LogService {
+  logs: AgentLogsRepo;
+}
+
+export function createLogService(db: Db): LogService {
+  return {
+    logs: createAgentLogsRepo(db),
+  };
+}

v1/packages/log/src/redaction.spec.ts

@@ -0,0 +1,25 @@
+import { describe, expect, it } from 'vitest';
+import { redactSensitiveContent } from './redaction.js';
+
+describe('redactSensitiveContent', (): void => {
+  it('redacts seeded secret and PII canaries before persistence or egress', (): void => {
+    const result = redactSensitiveContent(
+      'email [email protected] token=sk_CANARY12345678 phone +1 555 555 1212',
+    );
+    expect(result.content).not.toContain('[email protected]');
+    expect(result.content).not.toContain('sk_CANARY12345678');
+    expect(result.content).not.toContain('+1 555 555 1212');
+    expect(result.classifications).toEqual(['secret', 'pii']);
+  });
+
+  it('redacts common provider credential formats', (): void => {
+    const result = redactSensitiveContent(
+      'Authorization: Bearer canary.bearer.token jwt eyJcanary.eyJpayload.eyJsignature aws AKIACANARY1234567890',
+    );
+
+    expect(result.content).not.toContain('canary.bearer.token');
+    expect(result.content).not.toContain('eyJcanary.eyJpayload.eyJsignature');
+    expect(result.content).not.toContain('AKIACANARY1234567890');
+    expect(result.classifications).toEqual(['secret']);
+  });
+});

v1/packages/log/src/redaction.ts

@@ -0,0 +1,40 @@
+export type SensitiveClassification = 'secret' | 'pii';
+
+export interface RedactionResult {
+  content: string;
+  classifications: SensitiveClassification[];
+}
+
+const SECRET_PATTERNS: RegExp[] = [
+  /\b(?:sk|ghp|gitea)_[A-Za-z0-9_-]{8,}\b/g,
+  /\b(?:api[_-]?key|token|password|secret)\s*[:=]\s*[^\s,;]+/gi,
+  /\b(?:authorization\s*:\s*)?bearer\s+[A-Za-z0-9._~+/-]+=*/gi,
+  /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g,
+  /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g,
+  /-----BEGIN(?: [A-Z]+)* KEY-----[\s\S]*?-----END(?: [A-Z]+)* KEY-----/g,
+  /https?:\/\/[^\s?#]+[^\s]*[?&](?:token|key|secret|signature|sig)=[^\s&#]+/gi,
+];
+const PII_PATTERNS: RegExp[] = [
+  /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,
+  /\b\+?\d[\d(). -]{7,}\d\b/g,
+];
+
+export function redactSensitiveContent(content: string): RedactionResult {
+  let redacted = content;
+  const classifications: SensitiveClassification[] = [];
+  for (const pattern of SECRET_PATTERNS) {
+    if (pattern.test(redacted)) {
+      classifications.push('secret');
+      redacted = redacted.replace(pattern, '[REDACTED_SECRET]');
+    }
+    pattern.lastIndex = 0;
+  }
+  for (const pattern of PII_PATTERNS) {
+    if (pattern.test(redacted)) {
+      classifications.push('pii');
+      redacted = redacted.replace(pattern, '[REDACTED_PII]');
+    }
+    pattern.lastIndex = 0;
+  }
+  return { content: redacted, classifications: [...new Set(classifications)] };
+}

v1/packages/log/src/runtime-audit.test.ts

@@ -0,0 +1,52 @@
+import { describe, expect, it } from 'vitest';
+import { createRuntimeAuditLogEntry } from './runtime-audit.js';
+
+describe('createRuntimeAuditLogEntry', (): void => {
+  it('serializes only allowlisted runtime audit metadata', (): void => {
+    const entry = createRuntimeAuditLogEntry({
+      providerId: 'fleet',
+      operation: 'session.send',
+      outcome: 'succeeded',
+      actorId: 'actor-1',
+      tenantId: 'tenant-1',
+      channelId: 'discord',
+      correlationId: 'correlation-1',
+      resourceId: 'session-1',
+      durationMs: 12,
+    });
+
+    expect(entry).toMatchObject({
+      sessionId: 'runtime:fleet',
+      userId: 'actor-1',
+      level: 'info',
+      category: 'tool_use',
+      content: 'runtime.provider.audit',
+      metadata: {
+        providerId: 'fleet',
+        operation: 'session.send',
+        outcome: 'succeeded',
+        correlationId: 'correlation-1',
+        resourceId: expect.stringMatching(/^sha256:/),
+        durationMs: 12,
+      },
+    });
+    expect(JSON.stringify(entry)).not.toContain('approvalRef');
+  });
+
+  it('hashes every resource ID without blocking a runtime audit or persisting its raw value', (): void => {
+    const entry = createRuntimeAuditLogEntry({
+      providerId: 'fleet',
+      operation: 'session.send',
+      outcome: 'succeeded',
+      actorId: 'actor-1',
+      tenantId: 'tenant-1',
+      channelId: 'discord',
+      correlationId: 'correlation-1',
+      resourceId: 'credential-canary:secret-value',
+      durationMs: 12,
+    });
+
+    expect(entry.metadata).toMatchObject({ resourceId: expect.stringMatching(/^sha256:/) });
+    expect(JSON.stringify(entry)).not.toContain('secret-value');
+  });
+});

v1/packages/log/src/runtime-audit.ts

@@ -0,0 +1,87 @@
+import { createHash } from 'node:crypto';
+import type { NewAgentLog } from './agent-logs.js';
+
+export type RuntimeAuditOperation =
+  | 'session.list'
+  | 'session.tree'
+  | 'session.stream'
+  | 'session.send'
+  | 'session.attach'
+  | 'session.terminate'
+  | 'runtime.capabilities'
+  | 'runtime.health'
+  | 'runtime.transitional-capabilities';
+
+export type RuntimeAuditOutcome = 'requested' | 'succeeded' | 'denied' | 'failed';
+export type RuntimeAuditErrorCode = 'policy_denied' | 'provider_error';
+
+/**
+ * Deliberately metadata-only runtime audit record. It has no fields for message
+ * content, credentials, approval references, tool arguments, or tool output.
+ */
+export interface RuntimeAuditEvent {
+  providerId: string;
+  operation: RuntimeAuditOperation;
+  outcome: RuntimeAuditOutcome;
+  actorId: string;
+  tenantId: string;
+  channelId: string;
+  correlationId: string;
+  resourceId?: string;
+  durationMs?: number;
+  errorCode?: RuntimeAuditErrorCode;
+}
+
+const SAFE_IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+
+function safeIdentifier(value: string): string {
+  if (SAFE_IDENTIFIER.test(value)) return value;
+  return hashIdentifier(value);
+}
+
+function hashIdentifier(value: string): string {
+  return `sha256:${createHash('sha256').update(value).digest('hex')}`;
+}
+
+/**
+ * Converts a typed audit event into a durable log entry using an explicit
+ * allowlist. Values that could carry credentials or untrusted content are
+ * rejected before persistence or structured log emission.
+ */
+export function createRuntimeAuditLogEntry(event: RuntimeAuditEvent): NewAgentLog {
+  const providerId = safeIdentifier(event.providerId);
+  const actorId = safeIdentifier(event.actorId);
+  const tenantId = safeIdentifier(event.tenantId);
+  const channelId = safeIdentifier(event.channelId);
+  const correlationId = safeIdentifier(event.correlationId);
+  // Provider resource identifiers may be opaque or user-derived, so never persist them raw.
+  const resourceId = event.resourceId ? hashIdentifier(event.resourceId) : undefined;
+  const persistedUserId = SAFE_IDENTIFIER.test(event.actorId) ? event.actorId : null;
+
+  if (
+    event.durationMs !== undefined &&
+    (!Number.isInteger(event.durationMs) || event.durationMs < 0)
+  ) {
+    throw new Error('Runtime audit duration must be a non-negative integer');
+  }
+
+  return {
+    sessionId: `runtime:${providerId}`,
+    userId: persistedUserId,
+    level: event.outcome === 'failed' ? 'error' : event.outcome === 'denied' ? 'warn' : 'info',
+    category: event.operation.startsWith('session.') ? 'tool_use' : 'general',
+    content: 'runtime.provider.audit',
+    metadata: {
+      providerId,
+      operation: event.operation,
+      outcome: event.outcome,
+      actorId,
+      tenantId,
+      channelId,
+      correlationId,
+      ...(resourceId ? { resourceId } : {}),
+      ...(event.durationMs !== undefined ? { durationMs: event.durationMs } : {}),
+      ...(event.errorCode ? { errorCode: event.errorCode } : {}),
+    },
+  };
+}