feat(tess): add safe runtime observability

apps/gateway/src/agent/__tests__/runtime-provider-registry.service.test.ts

@@ -15,6 +15,7 @@ import type {
 import { AgentRuntimeProviderRegistry } from '@mosaicstack/agent';
 import type { ActorTenantScope } from '../../auth/session-scope.js';
 import {
+  RuntimeProviderAuditService,
   RuntimeProviderService,
   type RuntimeAuditEvent,
   type RuntimeAuditSink,
@@ -159,20 +160,47 @@ describe('RuntimeProviderService security boundary', (): void => {
       correlationId: CONTEXT.correlationId,
     });
     expect(Object.isFrozen(providerScope)).toBe(true);
-    expect(audit.events).toContainEqual({
-      providerId: 'fleet',
-      operation: 'session.send',
-      outcome: 'succeeded',
-      actorId: OWNER_SCOPE.userId,
-      tenantId: OWNER_SCOPE.tenantId,
-      channelId: CONTEXT.channelId,
-      correlationId: CONTEXT.correlationId,
-      resourceId: 'session-1',
-    });
+    expect(audit.events).toContainEqual(
+      expect.objectContaining({
+        providerId: 'fleet',
+        operation: 'session.send',
+        outcome: 'succeeded',
+        actorId: OWNER_SCOPE.userId,
+        tenantId: OWNER_SCOPE.tenantId,
+        channelId: CONTEXT.channelId,
+        correlationId: CONTEXT.correlationId,
+        resourceId: 'session-1',
+        durationMs: expect.any(Number),
+      }),
+    );
     expect(JSON.stringify(audit.events)).not.toContain('hello');
     expect(JSON.stringify(audit.events)).not.toContain('key-1');
   });
 
+  it('does not block a provider operation when an unsafe resource ID is redacted in durable audit', async (): Promise<void> => {
+    const provider = new RecordingRuntimeProvider(['session.send']);
+    let persisted: unknown;
+    const durableAudit = new RuntimeProviderAuditService({
+      logs: {
+        ingest: async (entry: unknown): Promise<unknown> => {
+          persisted = entry;
+          return entry;
+        },
+      },
+    } as never);
+    const service = makeService(provider, durableAudit);
+
+    await service.sendMessage(
+      'fleet',
+      'session/credential-canary=secret-value',
+      { content: 'safe message', idempotencyKey: 'key-1' },
+      CONTEXT,
+    );
+
+    expect(provider.sentMessages).toHaveLength(1);
+    expect(JSON.stringify(persisted)).not.toContain('secret-value');
+  });
+
   it('fails closed before a provider side effect when a capability is missing', async (): Promise<void> => {
     const provider = new RecordingRuntimeProvider([]);
     const service = makeService(provider);
@@ -191,12 +219,14 @@ describe('RuntimeProviderService security boundary', (): void => {
   it('requires a consumed exact-action approval before termination', async (): Promise<void> => {
     const provider = new RecordingRuntimeProvider(['session.terminate']);
     const approval = new DenyingApprovalVerifier();
-    const service = makeService(provider, new RecordingAuditSink(), approval);
+    const audit = new RecordingAuditSink();
+    const service = makeService(provider, audit, approval);
 
     await expect(
       service.terminate('fleet', 'session-1', 'forged-approval', CONTEXT),
     ).rejects.toThrow(/approval denied/);
     expect(provider.terminateCalls).toBe(0);
+    expect(audit.events.at(-1)).toMatchObject({ outcome: 'denied', errorCode: 'policy_denied' });
   });
 
   it('binds an accepted termination approval to provider, session, immutable scope, and correlation', async (): Promise<void> => {
@@ -256,6 +286,37 @@ describe('RuntimeProviderService security boundary', (): void => {
       'requested',
       'failed',
     ]);
+    expect(audit.events.at(-1)).toMatchObject({
+      errorCode: 'provider_error',
+      durationMs: expect.any(Number),
+    });
+  });
+
+  it('persists only metadata-only runtime audit fields', async (): Promise<void> => {
+    let persisted: unknown;
+    const ingest = async (entry: unknown): Promise<unknown> => {
+      persisted = entry;
+      return entry;
+    };
+    const service = new RuntimeProviderAuditService({ logs: { ingest } } as never);
+
+    await service.record({
+      providerId: 'fleet',
+      operation: 'session.send',
+      outcome: 'succeeded',
+      actorId: 'owner-1',
+      tenantId: 'tenant-1',
+      channelId: 'cli',
+      correlationId: 'correlation-1',
+      resourceId: 'session-1',
+      durationMs: 12,
+    });
+
+    expect(persisted).toMatchObject({
+      content: 'runtime.provider.audit',
+      metadata: expect.objectContaining({ correlationId: 'correlation-1', durationMs: 12 }),
+    });
+    expect(JSON.stringify(persisted)).not.toContain('approval');
   });
 
   it('does not misreport a completed provider side effect when completion auditing fails', async (): Promise<void> => {

apps/gateway/src/agent/agent.module.ts

@@ -14,6 +14,7 @@ import { CoordModule } from '../coord/coord.module.js';
 import { McpClientModule } from '../mcp-client/mcp-client.module.js';
 import { SkillsModule } from '../skills/skills.module.js';
 import { GCModule } from '../gc/gc.module.js';
+import { LogModule } from '../log/log.module.js';
 import {
   AGENT_RUNTIME_PROVIDER_REGISTRY,
   DenyRuntimeApprovalVerifier,
@@ -25,7 +26,7 @@ import {
 
 @Global()
 @Module({
-  imports: [CoordModule, McpClientModule, SkillsModule, GCModule],
+  imports: [CoordModule, McpClientModule, SkillsModule, GCModule, LogModule],
   providers: [
     ProviderService,
     ProviderCredentialsService,

apps/gateway/src/agent/provider.service.ts

@@ -107,8 +107,7 @@ export class ProviderService implements OnModuleInit, OnModuleDestroy {
    * Interval is configurable via PROVIDER_HEALTH_INTERVAL env (seconds, default 60).
    */
   private startHealthCheckScheduler(): void {
-    const intervalSecs =
-      parseInt(process.env['PROVIDER_HEALTH_INTERVAL'] ?? '', 10) || DEFAULT_HEALTH_INTERVAL_SECS;
+    const intervalSecs = this.effectiveHealthCheckIntervalSecs();
     const intervalMs = intervalSecs * 1000;
 
     // Run an initial check immediately (non-blocking)
@@ -176,6 +175,28 @@ export class ProviderService implements OnModuleInit, OnModuleDestroy {
     });
   }
 
+  /**
+   * Returns the effective provider operational policy without credentials,
+   * endpoints, request content, or provider error details.
+   */
+  getEffectivePolicyStatus(): {
+    healthCheckIntervalSecs: number;
+    configuredProviders: string[];
+    availableModelCount: number;
+  } {
+    return {
+      healthCheckIntervalSecs: this.effectiveHealthCheckIntervalSecs(),
+      configuredProviders: this.adapters.map((adapter) => adapter.name),
+      availableModelCount: this.registry?.getAvailable().length ?? 0,
+    };
+  }
+
+  private effectiveHealthCheckIntervalSecs(): number {
+    return (
+      parseInt(process.env['PROVIDER_HEALTH_INTERVAL'] ?? '', 10) || DEFAULT_HEALTH_INTERVAL_SECS
+    );
+  }
+
   // ---------------------------------------------------------------------------
   // Adapter-pattern API
   // ---------------------------------------------------------------------------

apps/gateway/src/agent/providers.controller.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, it, vi } from 'vitest';
+import { ProvidersController } from './providers.controller.js';
+
+describe('ProvidersController operational status', (): void => {
+  it('reports provider latency and effective policy without exposing provider error details', (): void => {
+    const providerService = {
+      getProvidersHealth: vi.fn(() => [
+        {
+          name: 'fleet',
+          status: 'down',
+          latencyMs: 42,
+          lastChecked: '2026-07-12T00:00:00.000Z',
+          modelCount: 0,
+          error: 'credential-canary=secret-value',
+        },
+      ]),
+      getEffectivePolicyStatus: vi.fn(() => ({
+        healthCheckIntervalSecs: 60,
+        configuredProviders: ['fleet'],
+        availableModelCount: 0,
+      })),
+    };
+    const controller = new ProvidersController(providerService as never, {} as never, {} as never);
+
+    const status = controller.status();
+
+    expect(status).toEqual({
+      providers: [
+        {
+          name: 'fleet',
+          status: 'down',
+          latencyMs: 42,
+          lastChecked: '2026-07-12T00:00:00.000Z',
+          modelCount: 0,
+          errorCode: 'provider_unavailable',
+        },
+      ],
+      effectivePolicy: {
+        healthCheckIntervalSecs: 60,
+        configuredProviders: ['fleet'],
+        availableModelCount: 0,
+      },
+    });
+    expect(JSON.stringify(status)).not.toContain('secret-value');
+  });
+});

apps/gateway/src/agent/providers.controller.ts

@@ -33,7 +33,20 @@ export class ProvidersController {
 
   @Get('health')
   health() {
-    return { providers: this.providerService.getProvidersHealth() };
+    return { providers: this.safeProviderHealth() };
+  }
+
+  /**
+   * Safe operational status for troubleshooting and readiness checks. Provider
+   * errors are reduced to a stable code so credentials and remote responses
+   * cannot leak through this endpoint.
+   */
+  @Get('status')
+  status() {
+    return {
+      providers: this.safeProviderHealth(),
+      effectivePolicy: this.providerService.getEffectivePolicyStatus(),
+    };
   }
 
   @Post('test')
@@ -51,6 +64,13 @@ export class ProvidersController {
     return this.routingService.rank(criteria);
   }
 
+  private safeProviderHealth() {
+    return this.providerService.getProvidersHealth().map(({ error, ...provider }) => ({
+      ...provider,
+      ...(error ? { errorCode: 'provider_unavailable' } : {}),
+    }));
+  }
+
   // ── Credential CRUD ──────────────────────────────────────────────────────
 
   /**

apps/gateway/src/agent/runtime-provider-registry.service.ts

@@ -1,5 +1,10 @@
 import { ForbiddenException, Inject, Injectable, Logger, NotFoundException } from '@nestjs/common';
 import { AgentRuntimeProviderRegistry } from '@mosaicstack/agent';
+import {
+  createRuntimeAuditLogEntry,
+  type LogService,
+  type RuntimeAuditErrorCode,
+} from '@mosaicstack/log';
 import type {
   AgentRuntimeProvider,
   RuntimeAttachHandle,
@@ -14,6 +19,7 @@ import type {
   RuntimeStreamEvent,
 } from '@mosaicstack/types';
 import type { ActorTenantScope } from '../auth/session-scope.js';
+import { LOG_SERVICE } from '../log/log.tokens.js';
 
 export const AGENT_RUNTIME_PROVIDER_REGISTRY = Symbol('AGENT_RUNTIME_PROVIDER_REGISTRY');
 export const RUNTIME_PROVIDER_AUDIT_SINK = Symbol('RUNTIME_PROVIDER_AUDIT_SINK');
@@ -42,6 +48,8 @@ export interface RuntimeAuditEvent {
   channelId: string;
   correlationId: string;
   resourceId?: string;
+  durationMs?: number;
+  errorCode?: RuntimeAuditErrorCode;
 }
 
 export interface RuntimeAuditSink {
@@ -87,8 +95,12 @@ export class DenyRuntimeApprovalVerifier implements RuntimeApprovalVerifier {
 export class RuntimeProviderAuditService implements RuntimeAuditSink {
   private readonly logger = new Logger(RuntimeProviderAuditService.name);
 
+  constructor(@Inject(LOG_SERVICE) private readonly logService: LogService) {}
+
   async record(event: RuntimeAuditEvent): Promise<void> {
-    this.logger.log(JSON.stringify(event));
+    const entry = createRuntimeAuditLogEntry(event);
+    await this.logService.logs.ingest(entry);
+    this.logger.log(JSON.stringify({ event: entry.content, metadata: entry.metadata }));
   }
 }
 
@@ -267,6 +279,7 @@ export class RuntimeProviderService {
     invoke: (provider: AgentRuntimeProvider, scope: RuntimeScope) => Promise<T>,
   ): Promise<T> {
     const scope = this.deriveScope(context);
+    const startedAt = Date.now();
     await this.record(providerId, operation, 'requested', scope, resourceId);
     let invocationStarted = false;
     try {
@@ -276,13 +289,22 @@ export class RuntimeProviderService {
       }
       invocationStarted = true;
       const result = await invoke(provider, scope);
-      await this.recordCompletion(providerId, operation, scope, resourceId);
+      await this.recordCompletion(providerId, operation, scope, resourceId, Date.now() - startedAt);
       return result;
     } catch (error: unknown) {
+      const durationMs = Date.now() - startedAt;
       if (invocationStarted && !(error instanceof RuntimeApprovalDeniedError)) {
-        await this.recordFailure(providerId, operation, scope, resourceId);
+        await this.recordFailure(providerId, operation, scope, resourceId, durationMs);
       } else {
-        await this.record(providerId, operation, 'denied', scope, resourceId);
+        await this.record(
+          providerId,
+          operation,
+          'denied',
+          scope,
+          resourceId,
+          durationMs,
+          'policy_denied',
+        );
       }
       throw error;
     }
@@ -300,6 +322,7 @@ export class RuntimeProviderService {
     ) => AsyncIterable<RuntimeStreamEvent>,
   ): AsyncIterable<RuntimeStreamEvent> {
     const scope = this.deriveScope(context);
+    const startedAt = Date.now();
     await this.record(providerId, operation, 'requested', scope, resourceId);
     let invocationStarted = false;
     try {
@@ -309,12 +332,21 @@ export class RuntimeProviderService {
       for await (const event of invoke(provider, scope)) {
         yield event;
       }
-      await this.recordCompletion(providerId, operation, scope, resourceId);
+      await this.recordCompletion(providerId, operation, scope, resourceId, Date.now() - startedAt);
     } catch (error: unknown) {
+      const durationMs = Date.now() - startedAt;
       if (invocationStarted) {
-        await this.recordFailure(providerId, operation, scope, resourceId);
+        await this.recordFailure(providerId, operation, scope, resourceId, durationMs);
       } else {
-        await this.record(providerId, operation, 'denied', scope, resourceId);
+        await this.record(
+          providerId,
+          operation,
+          'denied',
+          scope,
+          resourceId,
+          durationMs,
+          'policy_denied',
+        );
       }
       throw error;
     }
@@ -358,9 +390,18 @@ export class RuntimeProviderService {
     operation: RuntimeProviderOperation,
     scope: RuntimeScope,
     resourceId: string | undefined,
+    durationMs: number,
   ): Promise<void> {
     try {
-      await this.record(providerId, operation, 'failed', scope, resourceId);
+      await this.record(
+        providerId,
+        operation,
+        'failed',
+        scope,
+        resourceId,
+        durationMs,
+        'provider_error',
+      );
     } catch {
       this.logger.error(
         `Runtime provider failure audit failed provider=${providerId} operation=${operation} correlation=${scope.correlationId}`,
@@ -373,9 +414,10 @@ export class RuntimeProviderService {
     operation: RuntimeProviderOperation,
     scope: RuntimeScope,
     resourceId: string | undefined,
+    durationMs: number,
   ): Promise<void> {
     try {
-      await this.record(providerId, operation, 'succeeded', scope, resourceId);
+      await this.record(providerId, operation, 'succeeded', scope, resourceId, durationMs);
     } catch {
       this.logger.error(
         `Runtime provider completion audit failed provider=${providerId} operation=${operation} correlation=${scope.correlationId}`,
@@ -389,6 +431,8 @@ export class RuntimeProviderService {
     outcome: RuntimeProviderAuditOutcome,
     scope: RuntimeScope,
     resourceId: string | undefined,
+    durationMs?: number,
+    errorCode?: RuntimeAuditErrorCode,
   ): Promise<void> {
     await this.audit.record({
       providerId,
@@ -399,6 +443,8 @@ export class RuntimeProviderService {
       channelId: scope.channelId,
       correlationId: scope.correlationId,
       ...(resourceId ? { resourceId } : {}),
+      ...(durationMs !== undefined ? { durationMs } : {}),
+      ...(errorCode ? { errorCode } : {}),
     });
   }
 }

apps/gateway/src/health/health.controller.test.ts

@@ -0,0 +1,12 @@
+import { describe, expect, it } from 'vitest';
+import { HealthController } from './health.controller.js';
+
+describe('HealthController', (): void => {
+  it('exposes liveness and readiness without configuration details', (): void => {
+    const controller = new HealthController();
+
+    expect(controller.check()).toEqual({ status: 'ok' });
+    expect(controller.ready()).toEqual({ status: 'ready' });
+    expect(JSON.stringify(controller.ready())).not.toContain('credential');
+  });
+});

apps/gateway/src/health/health.controller.ts

@@ -6,4 +6,10 @@ export class HealthController {
   check(): { status: string } {
     return { status: 'ok' };
   }
+
+  /** Readiness intentionally exposes no configuration, provider, or credential details. */
+  @Get('ready')
+  ready(): { status: string } {
+    return { status: 'ready' };
+  }
 }

docs/scratchpads/tess-m1-obs-001.md

@@ -0,0 +1,9 @@
+# TESS-M1-OBS-001 Scratchpad
+
+- Branch: `feat/tess-observability-terra` (the requested name is checked out by an abandoned worktree; orchestrator approved this clean branch).
+- Base: `origin/main` at `e92186d7`.
+- Scope: correlation propagation; metadata-only structured runtime/provider/tool audit; health/readiness; safe effective-policy status.
+- Security invariant: audit and status data use an allowlist; no message bodies, credentials, approval references, tool arguments, or tool output.
+- TDD: `packages/log/src/runtime-audit.test.ts` and `apps/gateway/src/health/health.controller.test.ts` failed before implementation and now pass.
+- Verification: full `pnpm typecheck`, `pnpm lint`, `pnpm format:check`, and `pnpm test` passed after implementation (2026-07-12).
+- Review: corrected audit sanitizer findings by hashing every resource ID. Durable audit persistence remains fail-closed by design: the pre-existing M1 provider-boundary suite requires it to prevent an unaudited side effect.

packages/log/src/index.ts

@@ -15,3 +15,10 @@ export {
   type RedactionResult,
   type SensitiveClassification,
 } from './redaction.js';
+export {
+  createRuntimeAuditLogEntry,
+  type RuntimeAuditEvent,
+  type RuntimeAuditErrorCode,
+  type RuntimeAuditOperation,
+  type RuntimeAuditOutcome,
+} from './runtime-audit.js';

packages/log/src/runtime-audit.test.ts

@@ -0,0 +1,52 @@
+import { describe, expect, it } from 'vitest';
+import { createRuntimeAuditLogEntry } from './runtime-audit.js';
+
+describe('createRuntimeAuditLogEntry', (): void => {
+  it('serializes only allowlisted runtime audit metadata', (): void => {
+    const entry = createRuntimeAuditLogEntry({
+      providerId: 'fleet',
+      operation: 'session.send',
+      outcome: 'succeeded',
+      actorId: 'actor-1',
+      tenantId: 'tenant-1',
+      channelId: 'discord',
+      correlationId: 'correlation-1',
+      resourceId: 'session-1',
+      durationMs: 12,
+    });
+
+    expect(entry).toMatchObject({
+      sessionId: 'runtime:fleet',
+      userId: 'actor-1',
+      level: 'info',
+      category: 'tool_use',
+      content: 'runtime.provider.audit',
+      metadata: {
+        providerId: 'fleet',
+        operation: 'session.send',
+        outcome: 'succeeded',
+        correlationId: 'correlation-1',
+        resourceId: expect.stringMatching(/^sha256:/),
+        durationMs: 12,
+      },
+    });
+    expect(JSON.stringify(entry)).not.toContain('approvalRef');
+  });
+
+  it('hashes every resource ID without blocking a runtime audit or persisting its raw value', (): void => {
+    const entry = createRuntimeAuditLogEntry({
+      providerId: 'fleet',
+      operation: 'session.send',
+      outcome: 'succeeded',
+      actorId: 'actor-1',
+      tenantId: 'tenant-1',
+      channelId: 'discord',
+      correlationId: 'correlation-1',
+      resourceId: 'credential-canary:secret-value',
+      durationMs: 12,
+    });
+
+    expect(entry.metadata).toMatchObject({ resourceId: expect.stringMatching(/^sha256:/) });
+    expect(JSON.stringify(entry)).not.toContain('secret-value');
+  });
+});

packages/log/src/runtime-audit.ts

@@ -0,0 +1,86 @@
+import { createHash } from 'node:crypto';
+import type { NewAgentLog } from './agent-logs.js';
+
+export type RuntimeAuditOperation =
+  | 'session.list'
+  | 'session.tree'
+  | 'session.stream'
+  | 'session.send'
+  | 'session.attach'
+  | 'session.terminate'
+  | 'runtime.capabilities'
+  | 'runtime.health';
+
+export type RuntimeAuditOutcome = 'requested' | 'succeeded' | 'denied' | 'failed';
+export type RuntimeAuditErrorCode = 'policy_denied' | 'provider_error';
+
+/**
+ * Deliberately metadata-only runtime audit record. It has no fields for message
+ * content, credentials, approval references, tool arguments, or tool output.
+ */
+export interface RuntimeAuditEvent {
+  providerId: string;
+  operation: RuntimeAuditOperation;
+  outcome: RuntimeAuditOutcome;
+  actorId: string;
+  tenantId: string;
+  channelId: string;
+  correlationId: string;
+  resourceId?: string;
+  durationMs?: number;
+  errorCode?: RuntimeAuditErrorCode;
+}
+
+const SAFE_IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+
+function safeIdentifier(value: string): string {
+  if (SAFE_IDENTIFIER.test(value)) return value;
+  return hashIdentifier(value);
+}
+
+function hashIdentifier(value: string): string {
+  return `sha256:${createHash('sha256').update(value).digest('hex')}`;
+}
+
+/**
+ * Converts a typed audit event into a durable log entry using an explicit
+ * allowlist. Values that could carry credentials or untrusted content are
+ * rejected before persistence or structured log emission.
+ */
+export function createRuntimeAuditLogEntry(event: RuntimeAuditEvent): NewAgentLog {
+  const providerId = safeIdentifier(event.providerId);
+  const actorId = safeIdentifier(event.actorId);
+  const tenantId = safeIdentifier(event.tenantId);
+  const channelId = safeIdentifier(event.channelId);
+  const correlationId = safeIdentifier(event.correlationId);
+  // Provider resource identifiers may be opaque or user-derived, so never persist them raw.
+  const resourceId = event.resourceId ? hashIdentifier(event.resourceId) : undefined;
+  const persistedUserId = SAFE_IDENTIFIER.test(event.actorId) ? event.actorId : null;
+
+  if (
+    event.durationMs !== undefined &&
+    (!Number.isInteger(event.durationMs) || event.durationMs < 0)
+  ) {
+    throw new Error('Runtime audit duration must be a non-negative integer');
+  }
+
+  return {
+    sessionId: `runtime:${providerId}`,
+    userId: persistedUserId,
+    level: event.outcome === 'failed' ? 'error' : event.outcome === 'denied' ? 'warn' : 'info',
+    category: event.operation.startsWith('session.') ? 'tool_use' : 'general',
+    content: 'runtime.provider.audit',
+    metadata: {
+      providerId,
+      operation: event.operation,
+      outcome: event.outcome,
+      actorId,
+      tenantId,
+      channelId,
+      correlationId,
+      ...(resourceId ? { resourceId } : {}),
+      ...(event.durationMs !== undefined ? { durationMs: event.durationMs } : {}),
+      ...(event.errorCode ? { errorCode: event.errorCode } : {}),
+    },
+  };
+}