feat(tess): persist durable session state
Some checks failed
ci/woodpecker/pr/ci Pipeline failed

This commit is contained in:
Jarvis
2026-07-12 23:10:21 -05:00
parent e3b5113be2
commit 102a7b606b
22 changed files with 14785 additions and 13 deletions

View File

@@ -10,14 +10,17 @@ import { ProvidersController } from './providers.controller.js';
import { SessionsController } from './sessions.controller.js'; import { SessionsController } from './sessions.controller.js';
import { AgentConfigsController } from './agent-configs.controller.js'; import { AgentConfigsController } from './agent-configs.controller.js';
import { RoutingController } from './routing/routing.controller.js'; import { RoutingController } from './routing/routing.controller.js';
import { TessDurableSessionRepository } from './tess-durable-session.repository.js';
import { TessDurableSessionService } from './tess-durable-session.service.js';
import { CoordModule } from '../coord/coord.module.js'; import { CoordModule } from '../coord/coord.module.js';
import { McpClientModule } from '../mcp-client/mcp-client.module.js'; import { McpClientModule } from '../mcp-client/mcp-client.module.js';
import { SkillsModule } from '../skills/skills.module.js'; import { SkillsModule } from '../skills/skills.module.js';
import { GCModule } from '../gc/gc.module.js'; import { GCModule } from '../gc/gc.module.js';
import { LogModule } from '../log/log.module.js'; import { LogModule } from '../log/log.module.js';
import { CommandsModule } from '../commands/commands.module.js';
import { CommandRuntimeApprovalVerifier } from '../commands/runtime-approval-verifier.js';
import { import {
AGENT_RUNTIME_PROVIDER_REGISTRY, AGENT_RUNTIME_PROVIDER_REGISTRY,
DenyRuntimeApprovalVerifier,
RUNTIME_APPROVAL_VERIFIER, RUNTIME_APPROVAL_VERIFIER,
RUNTIME_PROVIDER_AUDIT_SINK, RUNTIME_PROVIDER_AUDIT_SINK,
RuntimeProviderAuditService, RuntimeProviderAuditService,
@@ -26,13 +29,15 @@ import {
@Global() @Global()
@Module({ @Module({
imports: [CoordModule, McpClientModule, SkillsModule, GCModule, LogModule], imports: [CoordModule, McpClientModule, SkillsModule, GCModule, LogModule, CommandsModule],
providers: [ providers: [
ProviderService, ProviderService,
ProviderCredentialsService, ProviderCredentialsService,
RoutingService, RoutingService,
RoutingEngineService, RoutingEngineService,
SkillLoaderService, SkillLoaderService,
TessDurableSessionRepository,
TessDurableSessionService,
{ {
provide: AGENT_RUNTIME_PROVIDER_REGISTRY, provide: AGENT_RUNTIME_PROVIDER_REGISTRY,
useFactory: (): AgentRuntimeProviderRegistry => new AgentRuntimeProviderRegistry(), useFactory: (): AgentRuntimeProviderRegistry => new AgentRuntimeProviderRegistry(),
@@ -42,10 +47,9 @@ import {
provide: RUNTIME_PROVIDER_AUDIT_SINK, provide: RUNTIME_PROVIDER_AUDIT_SINK,
useExisting: RuntimeProviderAuditService, useExisting: RuntimeProviderAuditService,
}, },
DenyRuntimeApprovalVerifier,
{ {
provide: RUNTIME_APPROVAL_VERIFIER, provide: RUNTIME_APPROVAL_VERIFIER,
useExisting: DenyRuntimeApprovalVerifier, useExisting: CommandRuntimeApprovalVerifier,
}, },
RuntimeProviderService, RuntimeProviderService,
AgentService, AgentService,
@@ -58,6 +62,7 @@ import {
RoutingService, RoutingService,
RoutingEngineService, RoutingEngineService,
SkillLoaderService, SkillLoaderService,
TessDurableSessionService,
RuntimeProviderService, RuntimeProviderService,
AGENT_RUNTIME_PROVIDER_REGISTRY, AGENT_RUNTIME_PROVIDER_REGISTRY,
], ],

View File

@@ -0,0 +1,10 @@
import type { RuntimeProviderRequestContext } from './runtime-provider-registry.service.js';
/** Server-side request for a replay-safe provider message. */
export interface TessProviderOutboxDto {
sessionId: string;
idempotencyKey: string;
correlationId: string;
content: string;
context: RuntimeProviderRequestContext;
}

View File

@@ -0,0 +1,336 @@
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { eq, sql, tessInbox } from '@mosaicstack/db';
import { DurableSessionCoordinator, type DurableSessionIdentity } from '@mosaicstack/agent';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { createPgliteDb, runPgliteMigrations, type DbHandle } from '@mosaicstack/db';
import { TessDurableSessionRepository } from './tess-durable-session.repository.js';
import { TessDurableSessionService } from './tess-durable-session.service.js';
const IDENTITY: DurableSessionIdentity = {
sessionId: 'tess-pglite-session',
tenantId: 'tenant-pglite',
ownerId: 'tess-owner',
providerId: 'fleet',
runtimeSessionId: 'nova',
};
describe('TessDurableSessionRepository', () => {
let dataDir: string | undefined;
let handle: DbHandle | undefined;
let previousAuthSecret: string | undefined;
beforeEach((): void => {
previousAuthSecret = process.env['BETTER_AUTH_SECRET'];
process.env['BETTER_AUTH_SECRET'] = 'tess-durable-state-test-sealing-key';
});
afterEach(async (): Promise<void> => {
await handle?.close();
if (dataDir) rmSync(dataDir, { recursive: true, force: true });
if (previousAuthSecret === undefined) delete process.env['BETTER_AUTH_SECRET'];
else process.env['BETTER_AUTH_SECRET'] = previousAuthSecret;
});
it('survives a full PGlite close/reopen mid-session without duplicate inbox or outbox side effects', async () => {
dataDir = mkdtempSync(join(tmpdir(), 'tess-durable-recovery-'));
handle = createPgliteDb(dataDir);
await runPgliteMigrations(handle);
await seedOwner(handle);
const beforeRestart = new DurableSessionCoordinator(
new TessDurableSessionRepository(handle.db),
);
await beforeRestart.create(IDENTITY);
await beforeRestart.receive({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'inbox-before-kill',
correlationId: 'correlation-before-kill',
content: 'resume after a kill',
});
await beforeRestart.enqueueOutbox({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'outbox-before-kill',
correlationId: 'correlation-before-kill',
channelId: 'cli',
kind: 'provider.send',
content: 'one response only',
});
await beforeRestart.checkpoint({
sessionId: IDENTITY.sessionId,
checkpointId: 'checkpoint-before-kill',
cursor: 'cursor-before-kill',
summary: 'restart-safe state',
compactionEpoch: 1,
});
await beforeRestart.handoff({
sessionId: IDENTITY.sessionId,
handoffId: 'handoff-before-kill',
destination: 'mos',
correlationId: 'correlation-before-kill',
checkpointId: 'checkpoint-before-kill',
status: 'pending',
});
await beforeRestart.checkpoint({
sessionId: IDENTITY.sessionId,
checkpointId: 'checkpoint-after-handoff',
cursor: 'cursor-after-handoff',
summary: 'newer state cannot strand the portable handoff',
compactionEpoch: 2,
});
await handle.close();
handle = createPgliteDb(dataDir);
const afterRestart = new DurableSessionCoordinator(new TessDurableSessionRepository(handle.db));
const recovered = await afterRestart.recover(IDENTITY.sessionId);
const resumedHandoff = await afterRestart.resumeHandoff('handoff-before-kill');
const handled: string[] = [];
const effects: string[] = [];
await afterRestart.drainInbox(IDENTITY.sessionId, async (entry): Promise<void> => {
handled.push(entry.idempotencyKey);
});
await afterRestart.dispatchOutbox(IDENTITY.sessionId, async (entry): Promise<void> => {
effects.push(entry.idempotencyKey);
});
await afterRestart.drainInbox(IDENTITY.sessionId, async (entry): Promise<void> => {
handled.push(entry.idempotencyKey);
});
await afterRestart.dispatchOutbox(IDENTITY.sessionId, async (entry): Promise<void> => {
effects.push(entry.idempotencyKey);
});
expect(recovered.identity).toEqual(IDENTITY);
expect(recovered.checkpoint).toMatchObject({ checkpointId: 'checkpoint-after-handoff' });
expect(recovered.handoffs).toMatchObject([{ handoffId: 'handoff-before-kill' }]);
expect(resumedHandoff.checkpoint).toMatchObject({ checkpointId: 'checkpoint-before-kill' });
expect(handled).toEqual(['inbox-before-kill']);
expect(effects).toEqual(['outbox-before-kill']);
}, 30_000);
it('redacts sensitive durable payloads before persistence', async () => {
dataDir = mkdtempSync(join(tmpdir(), 'tess-durable-redaction-'));
handle = createPgliteDb(dataDir);
await runPgliteMigrations(handle);
await seedOwner(handle);
const coordinator = new DurableSessionCoordinator(new TessDurableSessionRepository(handle.db));
await coordinator.create(IDENTITY);
await coordinator.receive({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'redacted-inbox',
correlationId: 'correlation-redaction',
content: 'api_key=super-secret-canary',
});
await coordinator.enqueueOutbox({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'redacted-outbox',
correlationId: 'correlation-redaction',
channelId: 'cli',
kind: 'provider.send',
content: 'email operator@example.test api_key=super-secret-canary',
});
await coordinator.checkpoint({
sessionId: IDENTITY.sessionId,
checkpointId: 'redacted-checkpoint',
cursor: 'bearer super-secret-canary',
summary: 'email operator@example.test',
compactionEpoch: 0,
});
const snapshot = await coordinator.snapshot(IDENTITY.sessionId);
const [persisted] = await handle.db
.select({ content: tessInbox.content })
.from(tessInbox)
.where(eq(tessInbox.idempotencyKey, 'redacted-inbox'));
expect(JSON.stringify(snapshot)).not.toContain('super-secret-canary');
expect(JSON.stringify(snapshot)).not.toContain('operator@example.test');
expect(persisted?.content).not.toContain('super-secret-canary');
expect(persisted?.content).not.toContain('[REDACTED]');
}, 30_000);
it('rejects database inbox and outbox idempotency-key conflicts', async () => {
dataDir = mkdtempSync(join(tmpdir(), 'tess-durable-idempotency-conflict-'));
handle = createPgliteDb(dataDir);
await runPgliteMigrations(handle);
await seedOwner(handle);
const coordinator = new DurableSessionCoordinator(new TessDurableSessionRepository(handle.db));
await coordinator.create(IDENTITY);
await coordinator.receive({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'inbox-conflict',
correlationId: 'correlation-inbox',
content: 'original inbox',
});
await coordinator.enqueueOutbox({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'outbox-conflict',
correlationId: 'correlation-outbox',
channelId: 'cli',
kind: 'provider.send',
content: 'original outbox',
});
await expect(
coordinator.receive({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'inbox-conflict',
correlationId: 'forged-correlation',
content: 'original inbox',
}),
).rejects.toThrow(/idempotency conflict/);
await expect(
coordinator.enqueueOutbox({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'outbox-conflict',
correlationId: 'correlation-outbox',
channelId: 'forged-channel',
kind: 'provider.send',
content: 'original outbox',
}),
).rejects.toThrow(/idempotency conflict/);
}, 30_000);
it('does not requeue a live outbox claim during a normal scoped dispatch', async () => {
dataDir = mkdtempSync(join(tmpdir(), 'tess-durable-live-claim-'));
handle = createPgliteDb(dataDir);
await runPgliteMigrations(handle);
await seedOwner(handle);
const repository = new TessDurableSessionRepository(handle.db);
const coordinator = new DurableSessionCoordinator(repository);
const runtimeProviders = { sendMessage: vi.fn().mockResolvedValue(undefined) };
const service = new TessDurableSessionService(repository, runtimeProviders as never);
const input = {
sessionId: IDENTITY.sessionId,
idempotencyKey: 'live-effect',
correlationId: 'correlation-live',
content: 'must not duplicate',
context: {
actorScope: { userId: IDENTITY.ownerId, tenantId: IDENTITY.tenantId },
channelId: 'cli',
correlationId: 'correlation-live',
},
};
await coordinator.create(IDENTITY);
await service.queueProviderSend(input);
expect(await repository.claimOutbox(IDENTITY.sessionId)).toMatchObject({
status: 'processing',
});
await service.dispatchProviderOutbox(IDENTITY.sessionId, input);
await expect(
service.recoverProviderSession(IDENTITY.sessionId, {
...input,
context: {
...input.context,
actorScope: { userId: 'intruder', tenantId: 'tenant-pglite' },
},
}),
).rejects.toThrow(/scope or correlation mismatch/);
expect(runtimeProviders.sendMessage).not.toHaveBeenCalled();
expect(await coordinator.snapshot(IDENTITY.sessionId)).toMatchObject({
outbox: [{ idempotencyKey: 'live-effect', status: 'processing' }],
});
}, 30_000);
it('rejects an outbox correlation mismatch before claiming the pending effect', async () => {
dataDir = mkdtempSync(join(tmpdir(), 'tess-durable-outbox-mismatch-'));
handle = createPgliteDb(dataDir);
await runPgliteMigrations(handle);
await seedOwner(handle);
const repository = new TessDurableSessionRepository(handle.db);
const coordinator = new DurableSessionCoordinator(repository);
const runtimeProviders = { sendMessage: vi.fn().mockResolvedValue(undefined) };
const service = new TessDurableSessionService(repository, runtimeProviders as never);
const input = {
sessionId: IDENTITY.sessionId,
idempotencyKey: 'mismatch-effect',
correlationId: 'correlation-expected',
content: 'must remain pending',
context: {
actorScope: { userId: IDENTITY.ownerId, tenantId: IDENTITY.tenantId },
channelId: 'cli',
correlationId: 'correlation-expected',
},
};
await coordinator.create(IDENTITY);
await service.queueProviderSend(input);
await expect(
service.dispatchProviderOutbox(IDENTITY.sessionId, {
...input,
correlationId: 'correlation-forged',
context: { ...input.context, correlationId: 'correlation-forged' },
}),
).rejects.toThrow(/scope or correlation mismatch/);
expect(runtimeProviders.sendMessage).not.toHaveBeenCalled();
expect(await coordinator.snapshot(IDENTITY.sessionId)).toMatchObject({
outbox: [{ idempotencyKey: 'mismatch-effect', status: 'pending' }],
});
}, 30_000);
it('dispatches only the outbox record bound to the supplied correlation and channel', async () => {
dataDir = mkdtempSync(join(tmpdir(), 'tess-durable-outbox-scope-'));
handle = createPgliteDb(dataDir);
await runPgliteMigrations(handle);
await seedOwner(handle);
const repository = new TessDurableSessionRepository(handle.db);
const coordinator = new DurableSessionCoordinator(repository);
const runtimeProviders = { sendMessage: vi.fn().mockResolvedValue(undefined) };
const service = new TessDurableSessionService(repository, runtimeProviders as never);
const first = {
sessionId: IDENTITY.sessionId,
idempotencyKey: 'scoped-effect-one',
correlationId: 'correlation-one',
content: 'first result',
context: {
actorScope: { userId: IDENTITY.ownerId, tenantId: IDENTITY.tenantId },
channelId: 'cli',
correlationId: 'correlation-one',
},
};
const second = {
...first,
idempotencyKey: 'scoped-effect-two',
correlationId: 'correlation-two',
content: 'second result',
context: { ...first.context, correlationId: 'correlation-two' },
};
await coordinator.create(IDENTITY);
await service.queueProviderSend(first);
await service.queueProviderSend(second);
await service.dispatchProviderOutbox(IDENTITY.sessionId, first);
expect(runtimeProviders.sendMessage).toHaveBeenCalledTimes(1);
expect(runtimeProviders.sendMessage).toHaveBeenCalledWith(
IDENTITY.providerId,
IDENTITY.runtimeSessionId,
{ content: 'first result', idempotencyKey: 'scoped-effect-one' },
first.context,
);
expect(await coordinator.snapshot(IDENTITY.sessionId)).toMatchObject({
outbox: [
{ idempotencyKey: 'scoped-effect-one', status: 'delivered' },
{ idempotencyKey: 'scoped-effect-two', status: 'pending' },
],
});
}, 30_000);
});
async function seedOwner(handle: DbHandle): Promise<void> {
await handle.db.execute(sql`
INSERT INTO users (id, name, email, email_verified, created_at, updated_at)
VALUES ('tess-owner', 'Tess Owner', 'tess-owner@example.test', false, now(), now())
`);
}

View File

@@ -0,0 +1,454 @@
import { Inject, Injectable } from '@nestjs/common';
import {
and,
asc,
desc,
eq,
tessCheckpoints,
tessHandoffs,
tessInbox,
tessOutbox,
tessSessions,
type Db,
} from '@mosaicstack/db';
import { seal, unseal } from '@mosaicstack/auth';
import { redactSensitiveContent } from '@mosaicstack/log';
import type {
DurableCheckpoint,
DurableCheckpointInput,
DurableEnqueueResult,
DurableHandoff,
DurableHandoffInput,
DurableInboxEntry,
DurableInboxInput,
DurableInboxStatus,
DurableOutboxEntry,
DurableOutboxInput,
DurableOutboxStatus,
DurableSessionIdentity,
DurableSessionSnapshot,
DurableSessionStore,
} from '@mosaicstack/agent';
import { DB } from '../database/database.module.js';
@Injectable()
export class TessDurableSessionRepository implements DurableSessionStore {
constructor(@Inject(DB) private readonly db: Db) {}
async create(identity: DurableSessionIdentity): Promise<void> {
await this.db
.insert(tessSessions)
.values({
id: identity.sessionId,
tenantId: identity.tenantId,
ownerId: identity.ownerId,
providerId: identity.providerId,
runtimeSessionId: identity.runtimeSessionId,
})
.onConflictDoNothing();
const existing = await this.session(identity.sessionId);
if (!existing || !sameIdentity(existing, identity)) {
throw new Error(`Durable Tess session identity conflict: ${identity.sessionId}`);
}
}
async snapshot(sessionId: string): Promise<DurableSessionSnapshot | null> {
const identity = await this.session(sessionId);
if (!identity) return null;
const [inbox, outbox, checkpoints, handoffs] = await Promise.all([
this.db
.select()
.from(tessInbox)
.where(eq(tessInbox.sessionId, sessionId))
.orderBy(asc(tessInbox.createdAt)),
this.db
.select()
.from(tessOutbox)
.where(eq(tessOutbox.sessionId, sessionId))
.orderBy(asc(tessOutbox.createdAt)),
this.db
.select()
.from(tessCheckpoints)
.where(eq(tessCheckpoints.sessionId, sessionId))
.orderBy(desc(tessCheckpoints.compactionEpoch), desc(tessCheckpoints.createdAt))
.limit(1),
this.db
.select()
.from(tessHandoffs)
.where(eq(tessHandoffs.sessionId, sessionId))
.orderBy(asc(tessHandoffs.createdAt)),
]);
const checkpoint = checkpoints[0];
return {
identity,
inbox: inbox.map(toInbox),
outbox: outbox.map(toOutbox),
...(checkpoint ? { checkpoint: toCheckpoint(checkpoint) } : {}),
handoffs: handoffs.map(toHandoff),
};
}
async enqueueInbox(input: DurableInboxInput): Promise<DurableEnqueueResult<DurableInboxStatus>> {
const record: DurableInboxInput = {
...input,
content: redactSensitiveContent(input.content).content,
};
const inserted = await this.db
.insert(tessInbox)
.values({ ...record, content: seal(record.content), status: 'pending' })
.onConflictDoNothing()
.returning({ status: tessInbox.status });
if (inserted[0]) return { accepted: true, status: inserted[0].status };
const existing = await this.db
.select()
.from(tessInbox)
.where(
and(
eq(tessInbox.sessionId, input.sessionId),
eq(tessInbox.idempotencyKey, input.idempotencyKey),
),
)
.limit(1);
if (!existing[0]) throw new Error(`Durable Tess inbox enqueue failed: ${input.idempotencyKey}`);
const entry = toInbox(existing[0]);
if (!sameInbox(entry, record)) {
throw new Error(`Durable Tess inbox idempotency conflict: ${input.idempotencyKey}`);
}
return { accepted: false, status: entry.status };
}
async claimInbox(sessionId: string): Promise<DurableInboxEntry | null> {
for (let attempt = 0; attempt < 3; attempt += 1) {
const candidate = await this.db
.select()
.from(tessInbox)
.where(and(eq(tessInbox.sessionId, sessionId), eq(tessInbox.status, 'pending')))
.orderBy(asc(tessInbox.createdAt))
.limit(1);
const entry = candidate[0];
if (!entry) return null;
const claimed = await this.db
.update(tessInbox)
.set({ status: 'processing', updatedAt: new Date() })
.where(and(eq(tessInbox.id, entry.id), eq(tessInbox.status, 'pending')))
.returning();
if (claimed[0]) return toInbox(claimed[0]);
}
return null;
}
async completeInbox(sessionId: string, idempotencyKey: string): Promise<void> {
await this.db
.update(tessInbox)
.set({ status: 'processed', updatedAt: new Date() })
.where(
and(
eq(tessInbox.sessionId, sessionId),
eq(tessInbox.idempotencyKey, idempotencyKey),
eq(tessInbox.status, 'processing'),
),
);
}
async releaseInbox(sessionId: string, idempotencyKey: string): Promise<void> {
await this.db
.update(tessInbox)
.set({ status: 'pending', updatedAt: new Date() })
.where(
and(
eq(tessInbox.sessionId, sessionId),
eq(tessInbox.idempotencyKey, idempotencyKey),
eq(tessInbox.status, 'processing'),
),
);
}
async enqueueOutbox(
input: DurableOutboxInput,
): Promise<DurableEnqueueResult<DurableOutboxStatus>> {
const record: DurableOutboxInput = {
...input,
content: redactSensitiveContent(input.content).content,
};
const inserted = await this.db
.insert(tessOutbox)
.values({ ...record, content: seal(record.content), status: 'pending' })
.onConflictDoNothing()
.returning({ status: tessOutbox.status });
if (inserted[0]) return { accepted: true, status: inserted[0].status };
const existing = await this.db
.select()
.from(tessOutbox)
.where(
and(
eq(tessOutbox.sessionId, input.sessionId),
eq(tessOutbox.idempotencyKey, input.idempotencyKey),
),
)
.limit(1);
if (!existing[0])
throw new Error(`Durable Tess outbox enqueue failed: ${input.idempotencyKey}`);
const entry = toOutbox(existing[0]);
if (!sameOutbox(entry, record)) {
throw new Error(`Durable Tess outbox idempotency conflict: ${input.idempotencyKey}`);
}
return { accepted: false, status: entry.status };
}
async claimOutbox(sessionId: string): Promise<DurableOutboxEntry | null> {
for (let attempt = 0; attempt < 3; attempt += 1) {
const candidate = await this.db
.select()
.from(tessOutbox)
.where(and(eq(tessOutbox.sessionId, sessionId), eq(tessOutbox.status, 'pending')))
.orderBy(asc(tessOutbox.createdAt))
.limit(1);
const entry = candidate[0];
if (!entry) return null;
const claimed = await this.db
.update(tessOutbox)
.set({ status: 'processing', updatedAt: new Date() })
.where(and(eq(tessOutbox.id, entry.id), eq(tessOutbox.status, 'pending')))
.returning();
if (claimed[0]) return toOutbox(claimed[0]);
}
return null;
}
async claimOutboxByKey(
sessionId: string,
idempotencyKey: string,
): Promise<DurableOutboxEntry | null> {
const claimed = await this.db
.update(tessOutbox)
.set({ status: 'processing', updatedAt: new Date() })
.where(
and(
eq(tessOutbox.sessionId, sessionId),
eq(tessOutbox.idempotencyKey, idempotencyKey),
eq(tessOutbox.status, 'pending'),
),
)
.returning();
return claimed[0] ? toOutbox(claimed[0]) : null;
}
async completeOutbox(sessionId: string, idempotencyKey: string): Promise<void> {
await this.db
.update(tessOutbox)
.set({ status: 'delivered', updatedAt: new Date() })
.where(
and(
eq(tessOutbox.sessionId, sessionId),
eq(tessOutbox.idempotencyKey, idempotencyKey),
eq(tessOutbox.status, 'processing'),
),
);
}
async releaseOutbox(sessionId: string, idempotencyKey: string): Promise<void> {
await this.db
.update(tessOutbox)
.set({ status: 'pending', updatedAt: new Date() })
.where(
and(
eq(tessOutbox.sessionId, sessionId),
eq(tessOutbox.idempotencyKey, idempotencyKey),
eq(tessOutbox.status, 'processing'),
),
);
}
async checkpoint(input: DurableCheckpointInput): Promise<void> {
const checkpoint: DurableCheckpointInput = {
...input,
cursor: redactSensitiveContent(input.cursor).content,
summary: redactSensitiveContent(input.summary).content,
};
const inserted = await this.db
.insert(tessCheckpoints)
.values({
...checkpoint,
cursor: seal(checkpoint.cursor),
summary: seal(checkpoint.summary),
})
.onConflictDoNothing()
.returning({ checkpointId: tessCheckpoints.checkpointId });
if (inserted[0]) return;
const existing = await this.findCheckpoint(input.sessionId, input.checkpointId);
if (!existing || !sameCheckpoint(existing, checkpoint)) {
throw new Error(`Durable Tess checkpoint identity conflict: ${input.checkpointId}`);
}
}
async findCheckpoint(sessionId: string, checkpointId: string): Promise<DurableCheckpoint | null> {
const checkpoints = await this.db
.select()
.from(tessCheckpoints)
.where(
and(
eq(tessCheckpoints.sessionId, sessionId),
eq(tessCheckpoints.checkpointId, checkpointId),
),
)
.limit(1);
const checkpoint = checkpoints[0];
return checkpoint ? toCheckpoint(checkpoint) : null;
}
async handoff(input: DurableHandoffInput): Promise<void> {
const checkpoint = await this.findCheckpoint(input.sessionId, input.checkpointId);
if (!checkpoint) {
throw new Error(`Durable Tess handoff checkpoint is unavailable: ${input.checkpointId}`);
}
const inserted = await this.db
.insert(tessHandoffs)
.values({ ...input })
.onConflictDoNothing()
.returning({ handoffId: tessHandoffs.handoffId });
if (inserted[0]) return;
const existing = await this.findHandoff(input.handoffId);
if (!existing || !sameHandoff(existing, input)) {
throw new Error(`Durable Tess handoff identity conflict: ${input.handoffId}`);
}
}
async findHandoff(handoffId: string): Promise<DurableHandoff | null> {
const handoffs = await this.db
.select()
.from(tessHandoffs)
.where(eq(tessHandoffs.handoffId, handoffId))
.limit(1);
const handoff = handoffs[0];
return handoff ? toHandoff(handoff) : null;
}
async requeueInFlight(sessionId: string): Promise<void> {
// Inbox handlers are process-local work. A provider outbox claim may have
// reached an external target before a crash, so it is deliberately not
// replayed by generic recovery.
await this.db
.update(tessInbox)
.set({ status: 'pending', updatedAt: new Date() })
.where(and(eq(tessInbox.sessionId, sessionId), eq(tessInbox.status, 'processing')));
}
private async session(sessionId: string): Promise<DurableSessionIdentity | null> {
const sessions = await this.db
.select()
.from(tessSessions)
.where(eq(tessSessions.id, sessionId))
.limit(1);
const session = sessions[0];
return session
? {
sessionId: session.id,
tenantId: session.tenantId,
ownerId: session.ownerId,
providerId: session.providerId,
runtimeSessionId: session.runtimeSessionId,
}
: null;
}
}
function sameIdentity(left: DurableSessionIdentity, right: DurableSessionIdentity): boolean {
return (
left.sessionId === right.sessionId &&
left.tenantId === right.tenantId &&
left.ownerId === right.ownerId &&
left.providerId === right.providerId &&
left.runtimeSessionId === right.runtimeSessionId
);
}
function sameInbox(left: DurableInboxEntry, right: DurableInboxInput): boolean {
return (
left.sessionId === right.sessionId &&
left.idempotencyKey === right.idempotencyKey &&
left.correlationId === right.correlationId &&
left.content === right.content
);
}
function sameOutbox(left: DurableOutboxEntry, right: DurableOutboxInput): boolean {
return (
left.sessionId === right.sessionId &&
left.idempotencyKey === right.idempotencyKey &&
left.correlationId === right.correlationId &&
left.channelId === right.channelId &&
left.kind === right.kind &&
left.content === right.content
);
}
function sameCheckpoint(left: DurableCheckpoint, right: DurableCheckpointInput): boolean {
return (
left.sessionId === right.sessionId &&
left.checkpointId === right.checkpointId &&
left.cursor === right.cursor &&
left.summary === right.summary &&
left.compactionEpoch === right.compactionEpoch
);
}
function sameHandoff(left: DurableHandoff, right: DurableHandoffInput): boolean {
return (
left.sessionId === right.sessionId &&
left.handoffId === right.handoffId &&
left.destination === right.destination &&
left.correlationId === right.correlationId &&
left.checkpointId === right.checkpointId &&
left.status === right.status
);
}
function toInbox(row: typeof tessInbox.$inferSelect): DurableInboxEntry {
return {
sessionId: row.sessionId,
idempotencyKey: row.idempotencyKey,
correlationId: row.correlationId,
content: unseal(row.content),
status: row.status,
};
}
function toOutbox(row: typeof tessOutbox.$inferSelect): DurableOutboxEntry {
return {
sessionId: row.sessionId,
idempotencyKey: row.idempotencyKey,
correlationId: row.correlationId,
channelId: row.channelId,
kind: row.kind,
content: unseal(row.content),
status: row.status,
};
}
function toCheckpoint(row: typeof tessCheckpoints.$inferSelect): DurableCheckpoint {
return {
sessionId: row.sessionId,
checkpointId: row.checkpointId,
cursor: unseal(row.cursor),
summary: unseal(row.summary),
compactionEpoch: row.compactionEpoch,
};
}
function toHandoff(row: typeof tessHandoffs.$inferSelect): DurableHandoff {
return {
sessionId: row.sessionId,
handoffId: row.handoffId,
destination: row.destination,
correlationId: row.correlationId,
checkpointId: row.checkpointId,
status: row.status,
};
}

View File

@@ -0,0 +1,93 @@
import { ForbiddenException, Inject, Injectable } from '@nestjs/common';
import { DurableSessionCoordinator } from '@mosaicstack/agent';
import type { TessProviderOutboxDto } from './tess-durable-session.dto.js';
import { TessDurableSessionRepository } from './tess-durable-session.repository.js';
import { RuntimeProviderService } from './runtime-provider-registry.service.js';
/**
* Scoped gateway boundary for the canonical Tess state machine. It deliberately
* uses composition: raw state methods cannot be injected into channel, CLI, or
* MCP adapters without a server-derived actor/tenant/correlation context.
*/
@Injectable()
export class TessDurableSessionService {
private readonly coordinator: DurableSessionCoordinator;
constructor(
@Inject(TessDurableSessionRepository) repository: TessDurableSessionRepository,
@Inject(RuntimeProviderService) private readonly runtimeProviders: RuntimeProviderService,
) {
this.coordinator = new DurableSessionCoordinator(repository);
}
async queueProviderSend(input: TessProviderOutboxDto): Promise<void> {
const snapshot = await this.coordinator.snapshot(input.sessionId);
this.assertScope(snapshot.identity.ownerId, snapshot.identity.tenantId, input);
await this.coordinator.enqueueOutbox({
sessionId: input.sessionId,
idempotencyKey: input.idempotencyKey,
correlationId: input.correlationId,
channelId: input.context.channelId,
kind: 'provider.send',
content: input.content,
});
}
async dispatchProviderOutbox(sessionId: string, input: TessProviderOutboxDto): Promise<void> {
if (sessionId !== input.sessionId) {
throw new ForbiddenException('Durable Tess outbox session mismatch');
}
const snapshot = await this.coordinator.snapshot(sessionId);
this.assertScope(snapshot.identity.ownerId, snapshot.identity.tenantId, input);
const pendingEntry = snapshot.outbox.find(
(entry): boolean => entry.idempotencyKey === input.idempotencyKey,
);
if (!pendingEntry) return;
// Validate immutable routing before claiming. A caller with a mismatched
// correlation/channel must not strand a pending external side effect.
this.assertOutboxScope(pendingEntry, input);
await this.coordinator.dispatchOutboxEntry(
sessionId,
input.idempotencyKey,
async (entry): Promise<void> => {
this.assertOutboxScope(entry, input);
await this.runtimeProviders.sendMessage(
snapshot.identity.providerId,
snapshot.identity.runtimeSessionId,
{ content: entry.content, idempotencyKey: entry.idempotencyKey },
input.context,
);
},
);
}
/** Startup/recovery-only path; normal queue/dispatch methods never requeue live work. */
async recoverProviderSession(sessionId: string, input: TessProviderOutboxDto): Promise<void> {
const snapshot = await this.coordinator.snapshot(sessionId);
this.assertScope(snapshot.identity.ownerId, snapshot.identity.tenantId, input);
await this.coordinator.recover(sessionId);
}
private assertOutboxScope(
entry: { kind: string; correlationId: string; channelId: string },
input: TessProviderOutboxDto,
): void {
if (
entry.kind !== 'provider.send' ||
entry.correlationId !== input.correlationId ||
entry.channelId !== input.context.channelId
) {
throw new ForbiddenException('Durable Tess outbox scope or correlation mismatch');
}
}
private assertScope(ownerId: string, tenantId: string, input: TessProviderOutboxDto): void {
if (
input.context.actorScope.userId !== ownerId ||
input.context.actorScope.tenantId !== tenantId ||
input.context.correlationId !== input.correlationId
) {
throw new ForbiddenException('Durable Tess session scope or correlation mismatch');
}
}
}

View File

@@ -12,8 +12,10 @@ const adminCommand: CommandDef = {
}; };
const payload: SlashCommandPayload = { command: 'gc', conversationId: 'conversation-1' }; const payload: SlashCommandPayload = { command: 'gc', conversationId: 'conversation-1' };
function createService(role: string): CommandAuthorizationService { function createService(
const entries = new Map<string, string>(); role: string,
entries: Map<string, string> = new Map<string, string>(),
): CommandAuthorizationService {
const db = { const db = {
select: () => ({ from: () => ({ where: () => ({ limit: async () => [{ role }] }) }) }), select: () => ({ from: () => ({ where: () => ({ limit: async () => [{ role }] }) }) }),
}; };
@@ -58,4 +60,55 @@ describe('CommandAuthorizationService', () => {
(await service.authorize(adminCommand, payload, 'member-1', 'forged-approval-id')).allowed, (await service.authorize(adminCommand, payload, 'member-1', 'forged-approval-id')).allowed,
).toBe(false); ).toBe(false);
}); });
it('denies a malformed durable approval expiry instead of treating it as unexpired', async (): Promise<void> => {
const entries = new Map<string, string>();
const action = {
providerId: 'fleet',
sessionId: 'nova',
actorId: 'admin-1',
tenantId: 'tenant-1',
channelId: 'discord:operator',
correlationId: 'correlation-malformed-expiry',
};
const service = createService('admin', entries);
const approval = await service.createRuntimeTerminationApproval(action);
expect(approval).not.toBeNull();
const key = `tess:command-approval:${approval!.approvalId}`;
const stored = entries.get(key);
expect(stored).toBeDefined();
entries.set(key, JSON.stringify({ ...JSON.parse(stored!), expiresAt: 'not-a-date' }));
expect(await service.consumeRuntimeTerminationApproval(approval!.approvalId, action)).toBe(
false,
);
});
it('persists and consumes one exact runtime termination approval across a service restart', async (): Promise<void> => {
const entries = new Map<string, string>();
const action = {
providerId: 'fleet',
sessionId: 'nova',
actorId: 'admin-1',
tenantId: 'tenant-1',
channelId: 'discord:operator',
correlationId: 'correlation-1',
};
const beforeRestart = createService('admin', entries);
const approval = await beforeRestart.createRuntimeTerminationApproval(action);
const afterRestart = createService('admin', entries);
expect(
await afterRestart.consumeRuntimeTerminationApproval(approval!.approvalId, {
...action,
sessionId: 'forged-session',
}),
).toBe(false);
expect(await afterRestart.consumeRuntimeTerminationApproval(approval!.approvalId, action)).toBe(
true,
);
expect(await afterRestart.consumeRuntimeTerminationApproval(approval!.approvalId, action)).toBe(
false,
);
});
}); });

View File

@@ -15,6 +15,22 @@ export interface CommandApproval {
expiresAt: string; expiresAt: string;
} }
/** Exact immutable binding for a privileged runtime termination. */
export interface RuntimeTerminationApprovalAction {
providerId: string;
sessionId: string;
actorId: string;
tenantId: string;
channelId: string;
correlationId: string;
}
export interface RuntimeTerminationApproval extends RuntimeTerminationApprovalAction {
approvalId: string;
actionDigest: string;
expiresAt: string;
}
export interface CommandAuthorizationResult { export interface CommandAuthorizationResult {
allowed: boolean; allowed: boolean;
reason?: string; reason?: string;
@@ -75,6 +91,52 @@ export class CommandAuthorizationService {
return approval; return approval;
} }
/**
* Uses the same `tess:command-approval:*` store and one-time deletion rule as
* command approvals. This deliberately avoids a parallel approval database.
*/
async createRuntimeTerminationApproval(
action: RuntimeTerminationApprovalAction,
): Promise<RuntimeTerminationApproval | null> {
if (!this.hasRuntimeTerminationAction(action)) return null;
const role = await this.resolveRole(action.actorId);
if (role !== 'admin') return null;
const approval: RuntimeTerminationApproval = {
approvalId: randomUUID(),
actionDigest: this.runtimeActionDigest(action),
...action,
expiresAt: new Date(Date.now() + 5 * 60_000).toISOString(),
};
await this.redis.set(this.key(approval.approvalId), JSON.stringify(approval), 'EX', '300');
return approval;
}
async consumeRuntimeTerminationApproval(
approvalId: string,
action: RuntimeTerminationApprovalAction,
): Promise<boolean> {
const encoded = await this.redis.get(this.key(approvalId));
if (!encoded) return false;
let approval: unknown;
try {
approval = JSON.parse(encoded);
} catch {
return false;
}
if (
!this.isRuntimeTerminationApproval(approval) ||
approval.actionDigest !== this.runtimeActionDigest(action) ||
approval.actorId !== action.actorId ||
approval.tenantId !== action.tenantId ||
!this.isUnexpired(approval.expiresAt)
) {
return false;
}
if ((await this.resolveRole(approval.actorId)) !== 'admin') return false;
return (await this.redis.del(this.key(approvalId))) === 1;
}
private async resolveRole(actorId: string): Promise<CommandRole | null> { private async resolveRole(actorId: string): Promise<CommandRole | null> {
const [user] = await this.db const [user] = await this.db
.select({ role: usersTable.role }) .select({ role: usersTable.role })
@@ -98,12 +160,17 @@ export class CommandAuthorizationService {
const key = this.key(approvalId); const key = this.key(approvalId);
const encoded = await this.redis.get(key); const encoded = await this.redis.get(key);
if (!encoded) return false; if (!encoded) return false;
const parsed: unknown = JSON.parse(encoded); let parsed: unknown;
try {
parsed = JSON.parse(encoded);
} catch {
return false;
}
if ( if (
!this.isApproval(parsed) || !this.isCommandApproval(parsed) ||
parsed.actorId !== actorId || parsed.actorId !== actorId ||
parsed.actionDigest !== actionDigest || parsed.actionDigest !== actionDigest ||
Date.parse(parsed.expiresAt) <= Date.now() !this.isUnexpired(parsed.expiresAt)
) )
return false; return false;
return (await this.redis.del(key)) === 1; return (await this.redis.del(key)) === 1;
@@ -121,13 +188,62 @@ export class CommandAuthorizationService {
.digest('hex'); .digest('hex');
} }
private isApproval(value: unknown): value is CommandApproval { private hasRuntimeTerminationAction(action: RuntimeTerminationApprovalAction): boolean {
return [
action.providerId,
action.sessionId,
action.actorId,
action.tenantId,
action.channelId,
action.correlationId,
].every((value: string): boolean => value.trim().length > 0);
}
private runtimeActionDigest(action: RuntimeTerminationApprovalAction): string {
return createHash('sha256')
.update(
JSON.stringify({
providerId: action.providerId,
sessionId: action.sessionId,
actorId: action.actorId,
tenantId: action.tenantId,
channelId: action.channelId,
correlationId: action.correlationId,
}),
)
.digest('hex');
}
private isUnexpired(expiresAt: unknown): expiresAt is string {
if (typeof expiresAt !== 'string') return false;
const expiresAtMs = Date.parse(expiresAt);
return Number.isFinite(expiresAtMs) && expiresAtMs > Date.now();
}
private isCommandApproval(value: unknown): value is CommandApproval {
return ( return (
typeof value === 'object' && typeof value === 'object' &&
value !== null && value !== null &&
'approvalId' in value && 'approvalId' in value &&
'actionDigest' in value && 'actionDigest' in value &&
'actorId' in value && 'actorId' in value &&
'expiresAt' in value &&
'command' in value
);
}
private isRuntimeTerminationApproval(value: unknown): value is RuntimeTerminationApproval {
return (
typeof value === 'object' &&
value !== null &&
'approvalId' in value &&
'actionDigest' in value &&
'actorId' in value &&
'tenantId' in value &&
'providerId' in value &&
'sessionId' in value &&
'channelId' in value &&
'correlationId' in value &&
'expiresAt' in value 'expiresAt' in value
); );
} }

View File

@@ -6,6 +6,7 @@ import { ReloadModule } from '../reload/reload.module.js';
import { CommandAuthorizationService } from './command-authorization.service.js'; import { CommandAuthorizationService } from './command-authorization.service.js';
import { CommandExecutorService } from './command-executor.service.js'; import { CommandExecutorService } from './command-executor.service.js';
import { CommandRegistryService } from './command-registry.service.js'; import { CommandRegistryService } from './command-registry.service.js';
import { CommandRuntimeApprovalVerifier } from './runtime-approval-verifier.js';
import { COMMANDS_REDIS } from './commands.tokens.js'; import { COMMANDS_REDIS } from './commands.tokens.js';
const COMMANDS_QUEUE_HANDLE = 'COMMANDS_QUEUE_HANDLE'; const COMMANDS_QUEUE_HANDLE = 'COMMANDS_QUEUE_HANDLE';
@@ -26,9 +27,15 @@ const COMMANDS_QUEUE_HANDLE = 'COMMANDS_QUEUE_HANDLE';
}, },
CommandRegistryService, CommandRegistryService,
CommandAuthorizationService, CommandAuthorizationService,
CommandRuntimeApprovalVerifier,
CommandExecutorService,
],
exports: [
CommandRegistryService,
CommandAuthorizationService,
CommandRuntimeApprovalVerifier,
CommandExecutorService, CommandExecutorService,
], ],
exports: [CommandRegistryService, CommandExecutorService],
}) })
export class CommandsModule implements OnApplicationShutdown { export class CommandsModule implements OnApplicationShutdown {
constructor(@Inject(COMMANDS_QUEUE_HANDLE) private readonly handle: QueueHandle) {} constructor(@Inject(COMMANDS_QUEUE_HANDLE) private readonly handle: QueueHandle) {}

View File

@@ -0,0 +1,23 @@
import { Inject, Injectable } from '@nestjs/common';
import type {
RuntimeApprovalVerifier,
RuntimeTerminationAction,
} from '../agent/runtime-provider-registry.service.js';
import { CommandAuthorizationService } from './command-authorization.service.js';
/**
* Adapter from the provider registry's exact termination action to the shared,
* Redis-backed `tess:command-approval:*` store. It has no separate approval
* persistence or replay semantics.
*/
@Injectable()
export class CommandRuntimeApprovalVerifier implements RuntimeApprovalVerifier {
constructor(
@Inject(CommandAuthorizationService)
private readonly authorization: CommandAuthorizationService,
) {}
async consume(approvalRef: string, action: RuntimeTerminationAction): Promise<boolean> {
return this.authorization.consumeRuntimeTerminationApproval(approvalRef, action);
}
}

View File

@@ -0,0 +1,63 @@
# TESS-M2-002 — Durable Tess State
- **Issue:** #708
- **Task:** `TESS-M2-002` / `TESS-STA-001`, `TESS-SEC-007..008`
- **Branch:** `feat/tess-durable-state`
- **Base:** fresh `origin/main` at `e3b5113be21e51d015fa1ae54572929b2a4acd9f`
- **Budget assumption:** 38K task estimate; no explicit cap. Use focused TDD plus workspace validation.
## Objective
Persist a Tess session's immutable identity, inbox/outbox idempotency state, checkpoints,
handoffs, and approval bindings so a new service instance can recover it after a process
restart or context compaction without replaying a completed message or applied side effect.
## Plan
1. Write recovery/idempotency tests first in `packages/agent/src/tess-durable-session.test.ts`.
2. Add transport-neutral durable-state contracts/state machine in `packages/agent`.
3. Add canonical PostgreSQL schema/migration and a gateway Drizzle repository adapter.
4. Wire gateway service/module and reuse `tess:command-approval:*` durable approval semantics
for exact, actor/tenant/action-bound approval consumption.
5. Test PGlite restart recovery with separate service instances sharing the same durable DB.
6. Document the recovery/compaction operation and update Tess architecture evidence.
7. Run focused, cold-cache, workspace, migration, review, commit, push, and open PR to `main`.
## Required Evidence
| Requirement | Primary evidence |
| --- | --- |
| Restart recovery | Test creates a second coordinator over unchanged durable store after simulated process death. |
| No duplicate side effects | Duplicate ingress and post-restart dispatch assert one handler/effect invocation. |
| Compaction survival | Checkpoint/handoff/reconstructed state preserve the same session identity and pending records. |
| Durable approvals | Existing `tess:command-approval` record is consumed only once and survives a new authorization service instance. |
| Handoff | Stored handoff is portable and reconstructed without live process state. |
## Progress
- Intake complete: PRD AC-TESS-06, threat TM-07/TM-08, and verification matrix reviewed.
- Affected surfaces: `packages/agent`, `apps/gateway`, `packages/db`; auth/authorization and DB migration tests required.
- TDD is required (security authorization and critical state mutation).
## Risks
- An external provider action cannot be atomically committed with the database. The outbox
gives the receiver a stable idempotency key; generic recovery never replays an ambiguous
`processing` effect, and completed effects are never redispatched.
- PostgreSQL is canonical; PGlite is the local/restart test implementation.
## Verification
- TDD red: `pnpm --filter @mosaicstack/agent test src/tess-durable-session.test.ts`
initially failed because the durable-state module did not exist.
- Focused green: 7 agent state-machine tests; 6 PGlite repository tests (including
close/reopen recovery and encrypted-at-rest redaction); 5 durable-approval tests; DB migration tests.
- Full cold-cache green: `pnpm turbo run typecheck lint test --force` completed
88 tasks with zero cache hits; `pnpm format:check` and `git diff --check` passed.
- Fresh worktree dependency install passed with
`pnpm install --frozen-lockfile --store-dir /home/jarvis/.local/share/pnpm/store`.
The default pnpm store path was inaccessible to this harness, so the explicit
user-owned store path was required.
- Codex review identified plaintext durable payload risk; resolved by AES-256-GCM sealing
after redaction, with an at-rest ciphertext assertion in the PGlite suite.
- Pending final clean review, commit, and PR.

View File

@@ -41,7 +41,7 @@ Every call receives an immutable, server-derived actor/tenant/channel scope and
`@mosaicstack/agent` owns the explicit `AgentRuntimeProviderRegistry`; duplicate provider IDs are rejected rather than replaced. Gateway owns `RuntimeProviderService`, which creates a frozen `RuntimeScope` from authenticated `ActorTenantScope` and trusted ingress channel/correlation metadata before every provider call. The service checks the declared provider capability before invoking a side effect and records metadata-only audit events (`providerId`, operation, outcome, actor/tenant/channel, correlation, and resource ID). It never records message bodies, idempotency keys, or approval references. `@mosaicstack/agent` owns the explicit `AgentRuntimeProviderRegistry`; duplicate provider IDs are rejected rather than replaced. Gateway owns `RuntimeProviderService`, which creates a frozen `RuntimeScope` from authenticated `ActorTenantScope` and trusted ingress channel/correlation metadata before every provider call. The service checks the declared provider capability before invoking a side effect and records metadata-only audit events (`providerId`, operation, outcome, actor/tenant/channel, correlation, and resource ID). It never records message bodies, idempotency keys, or approval references.
Termination is fail-closed: a runtime approval verifier must consume a one-time, exact action binding for the provider, session, actor, tenant, channel, and correlation ID before `terminate` reaches a provider. Until the durable verifier is wired, the default verifier denies termination. This internal service introduces no HTTP endpoint; later Discord, CLI, MCP, and provider adapters consume the same gateway boundary. Termination is fail-closed: a runtime approval verifier consumes a one-time, exact action binding for the provider, session, actor, tenant, channel, and correlation ID before `terminate` reaches a provider. The verifier reuses the Redis-backed `tess:command-approval:*` store and its expiry/delete-on-consume semantics; it has no parallel approval store. This internal service introduces no HTTP endpoint; later Discord, CLI, MCP, and provider adapters consume the same gateway boundary.
## Authority Model ## Authority Model
@@ -56,7 +56,30 @@ Termination is fail-closed: a runtime approval verifier must consume a one-time,
A Tess session has stable `sessionId`, `tenantId`, `ownerId`, provider/runtime identity, ingress bindings, cursor, checkpoint, inbox/outbox, and idempotency records. Discord and CLI bind to the same authorized session. Ownership is verified server-side on every list/read/attach/send/terminate operation. A Tess session has stable `sessionId`, `tenantId`, `ownerId`, provider/runtime identity, ingress bindings, cursor, checkpoint, inbox/outbox, and idempotency records. Discord and CLI bind to the same authorized session. Ownership is verified server-side on every list/read/attach/send/terminate operation.
Valkey may hold ephemeral coordination state; PostgreSQL is canonical for durable session bindings, approvals, audit, checkpoints, inbox/outbox, and idempotency. Pi session files are replay sources, not cross-agent truth. Valkey holds the existing short-lived, one-time command-approval records; PostgreSQL is canonical for durable session bindings, checkpoints, inbox/outbox, and idempotency. Pi session files are replay sources, not cross-agent truth.
### M2 Durable Recovery
`@mosaicstack/agent` owns a transport-neutral state machine and `apps/gateway` provides its
PostgreSQL adapter. `tess_sessions` holds immutable identity; inbox/outbox records use a
per-session unique idempotency key and transition `pending → processing → processed|delivered`.
Checkpoints are immutable history scoped by session and checkpoint ID: the latest checkpoint
supports compaction recovery, while a handoff always resolves the exact checkpoint it references.
Recovery requeues only interrupted inbox work; an ambiguous `processing` outbox record is preserved
until separately authorized reconciliation can establish its external delivery state.
Provider sends travel through the existing `RuntimeProviderService` with the persisted outbox
idempotency key. A normal dispatch claims exactly one outbox record and verifies its stored
correlation and channel against the server-derived request scope; it never requeues or drains
another live record. Inbox/outbox payloads and checkpoint cursor/summary pass through the existing
secret/PII redactor and AES-256-GCM sealing before persistence; decryption occurs only in the
scoped gateway repository path, and runtime audit remains metadata-only.
An external effect cannot share a database transaction. If a process dies after an effect begins
but before its terminal outbox transition, automatic recovery does not replay that ambiguous claim.
It remains `processing` until separately authorized reconciliation can establish delivery state;
completed effects are never redispatched. Operators can therefore restart the gateway/Pi service,
reconstruct the session, and resume pending inbox work without relying on process-local state.
## Transport Strategy ## Transport Strategy

View File

@@ -2,3 +2,4 @@ export const VERSION = '0.0.0';
export * from './runtime-provider-registry.js'; export * from './runtime-provider-registry.js';
export * from './tmux-fleet-runtime-provider.js'; export * from './tmux-fleet-runtime-provider.js';
export * from './tess-durable-session.js';

View File

@@ -0,0 +1,287 @@
import { describe, expect, it } from 'vitest';
import {
DurableSessionCoordinator,
InMemoryDurableSessionStore,
type DurableSessionIdentity,
} from './tess-durable-session.js';
const IDENTITY: DurableSessionIdentity = {
sessionId: 'tess-session-1',
tenantId: 'tenant-1',
ownerId: 'owner-1',
providerId: 'fleet',
runtimeSessionId: 'nova',
};
describe('DurableSessionCoordinator', () => {
it('reconstructs an exact session identity, pending inbox/outbox, checkpoint, and handoff after a simulated process restart', async () => {
const store = new InMemoryDurableSessionStore();
const beforeRestart = new DurableSessionCoordinator(store);
await beforeRestart.create(IDENTITY);
await beforeRestart.receive({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'ingress-1',
correlationId: 'correlation-1',
content: 'continue the session',
});
await beforeRestart.enqueueOutbox({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'outbox-1',
correlationId: 'correlation-1',
channelId: 'cli',
kind: 'provider.send',
content: 'resumable response',
});
await beforeRestart.checkpoint({
sessionId: IDENTITY.sessionId,
checkpointId: 'checkpoint-1',
cursor: 'cursor-42',
summary: 'operator asked for recovery proof',
compactionEpoch: 0,
});
await beforeRestart.handoff({
sessionId: IDENTITY.sessionId,
handoffId: 'handoff-1',
destination: 'mos',
correlationId: 'correlation-1',
checkpointId: 'checkpoint-1',
status: 'pending',
});
// Simulate an ungraceful process death: no in-memory coordinator state survives.
const afterRestart = new DurableSessionCoordinator(store);
const recovered = await afterRestart.recover(IDENTITY.sessionId);
expect(recovered.identity).toEqual(IDENTITY);
expect(recovered.inbox).toMatchObject([{ idempotencyKey: 'ingress-1', status: 'pending' }]);
expect(recovered.outbox).toMatchObject([{ idempotencyKey: 'outbox-1', status: 'pending' }]);
expect(recovered.checkpoint).toMatchObject({
checkpointId: 'checkpoint-1',
cursor: 'cursor-42',
});
expect(recovered.handoffs).toMatchObject([{ handoffId: 'handoff-1', status: 'pending' }]);
});
it('deduplicates duplicate ingress and never reprocesses an inbox record after restart or compaction', async () => {
const store = new InMemoryDurableSessionStore();
const firstProcess = new DurableSessionCoordinator(store);
const handled: string[] = [];
await firstProcess.create(IDENTITY);
await expect(
firstProcess.receive({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'ingress-duplicate',
correlationId: 'correlation-2',
content: 'only process me once',
}),
).resolves.toMatchObject({ accepted: true });
await expect(
firstProcess.receive({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'ingress-duplicate',
correlationId: 'correlation-2',
content: 'only process me once',
}),
).resolves.toMatchObject({ accepted: false, status: 'pending' });
await expect(
firstProcess.receive({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'ingress-duplicate',
correlationId: 'forged-correlation',
content: 'only process me once',
}),
).rejects.toThrow(/idempotency conflict/);
await firstProcess.drainInbox(IDENTITY.sessionId, async (entry) => {
handled.push(entry.idempotencyKey);
});
await firstProcess.checkpoint({
sessionId: IDENTITY.sessionId,
checkpointId: 'checkpoint-after-inbox',
cursor: 'cursor-43',
summary: 'safe to compact',
compactionEpoch: 1,
});
const afterRestartAndCompaction = new DurableSessionCoordinator(store);
await afterRestartAndCompaction.recover(IDENTITY.sessionId);
await afterRestartAndCompaction.drainInbox(IDENTITY.sessionId, async (entry) => {
handled.push(entry.idempotencyKey);
});
expect(handled).toEqual(['ingress-duplicate']);
});
it('does not redispatch an already applied outbox side effect after replay, restart, or compaction', async () => {
const store = new InMemoryDurableSessionStore();
const beforeRestart = new DurableSessionCoordinator(store);
const appliedEffects: string[] = [];
await beforeRestart.create(IDENTITY);
await beforeRestart.enqueueOutbox({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'effect-1',
correlationId: 'correlation-3',
channelId: 'cli',
kind: 'provider.send',
content: 'send exactly once',
});
await expect(
beforeRestart.enqueueOutbox({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'effect-1',
correlationId: 'correlation-3',
channelId: 'cli',
kind: 'provider.send',
content: 'send exactly once',
}),
).resolves.toMatchObject({ accepted: false, status: 'pending' });
await beforeRestart.dispatchOutbox(IDENTITY.sessionId, async (entry) => {
appliedEffects.push(entry.idempotencyKey);
});
await beforeRestart.checkpoint({
sessionId: IDENTITY.sessionId,
checkpointId: 'checkpoint-after-effect',
cursor: 'cursor-44',
summary: 'effect persisted before compaction',
compactionEpoch: 1,
});
const afterRestartAndCompaction = new DurableSessionCoordinator(store);
await afterRestartAndCompaction.recover(IDENTITY.sessionId);
await afterRestartAndCompaction.dispatchOutbox(IDENTITY.sessionId, async (entry) => {
appliedEffects.push(entry.idempotencyKey);
});
expect(appliedEffects).toEqual(['effect-1']);
});
it('rejects outbox idempotency-key reuse when immutable effect data differs', async () => {
const store = new InMemoryDurableSessionStore();
const coordinator = new DurableSessionCoordinator(store);
await coordinator.create(IDENTITY);
await coordinator.enqueueOutbox({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'outbox-conflict',
correlationId: 'correlation-outbox',
channelId: 'cli',
kind: 'provider.send',
content: 'original effect',
});
await expect(
coordinator.enqueueOutbox({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'outbox-conflict',
correlationId: 'correlation-outbox',
channelId: 'forged-channel',
kind: 'provider.send',
content: 'original effect',
}),
).rejects.toThrow(/idempotency conflict/);
});
it('retains an ambiguous failed provider effect as processing until explicit recovery', async () => {
const store = new InMemoryDurableSessionStore();
const beforeRestart = new DurableSessionCoordinator(store);
await beforeRestart.create(IDENTITY);
await beforeRestart.enqueueOutbox({
sessionId: IDENTITY.sessionId,
idempotencyKey: 'ambiguous-effect',
correlationId: 'correlation-ambiguous',
channelId: 'cli',
kind: 'provider.send',
content: 'preserve this effect claim',
});
await expect(
beforeRestart.dispatchOutbox(IDENTITY.sessionId, async (): Promise<void> => {
throw new Error('provider connection dropped after submit');
}),
).rejects.toThrow(/connection dropped/);
const afterRestart = new DurableSessionCoordinator(store);
await afterRestart.recover(IDENTITY.sessionId);
const calls: string[] = [];
await afterRestart.dispatchOutbox(IDENTITY.sessionId, async (entry): Promise<void> => {
calls.push(entry.idempotencyKey);
});
expect(calls).toEqual([]);
expect(await afterRestart.snapshot(IDENTITY.sessionId)).toMatchObject({
outbox: [{ idempotencyKey: 'ambiguous-effect', status: 'processing' }],
});
});
it('rejects a handoff-id replay with different immutable state', async () => {
const store = new InMemoryDurableSessionStore();
const coordinator = new DurableSessionCoordinator(store);
await coordinator.create(IDENTITY);
await coordinator.checkpoint({
sessionId: IDENTITY.sessionId,
checkpointId: 'checkpoint-conflict',
cursor: 'cursor-conflict',
summary: 'handoff conflict proof',
compactionEpoch: 0,
});
await coordinator.handoff({
sessionId: IDENTITY.sessionId,
handoffId: 'handoff-conflict',
destination: 'mos',
correlationId: 'correlation-conflict',
checkpointId: 'checkpoint-conflict',
status: 'pending',
});
await expect(
coordinator.handoff({
sessionId: IDENTITY.sessionId,
handoffId: 'handoff-conflict',
destination: 'forged-destination',
correlationId: 'correlation-conflict',
checkpointId: 'checkpoint-conflict',
status: 'pending',
}),
).rejects.toThrow(/identity conflict/);
});
it('keeps a handoff portable and resumes it from its durable checkpoint without process-local references', async () => {
const store = new InMemoryDurableSessionStore();
const source = new DurableSessionCoordinator(store);
await source.create(IDENTITY);
await source.checkpoint({
sessionId: IDENTITY.sessionId,
checkpointId: 'checkpoint-handoff',
cursor: 'cursor-45',
summary: 'portable state',
compactionEpoch: 2,
});
await source.handoff({
sessionId: IDENTITY.sessionId,
handoffId: 'handoff-portable',
destination: 'mos',
correlationId: 'correlation-4',
checkpointId: 'checkpoint-handoff',
status: 'pending',
});
await source.checkpoint({
sessionId: IDENTITY.sessionId,
checkpointId: 'checkpoint-later',
cursor: 'cursor-46',
summary: 'newer compacted state must not strand the handoff',
compactionEpoch: 3,
});
const receivingProcess = new DurableSessionCoordinator(store);
const handoff = await receivingProcess.resumeHandoff('handoff-portable');
expect(handoff.identity).toEqual(IDENTITY);
expect(handoff.checkpoint).toMatchObject({ checkpointId: 'checkpoint-handoff' });
expect(handoff.handoff).toMatchObject({ handoffId: 'handoff-portable', destination: 'mos' });
});
});

View File

@@ -0,0 +1,489 @@
export interface DurableSessionIdentity {
sessionId: string;
tenantId: string;
ownerId: string;
providerId: string;
runtimeSessionId: string;
}
export type DurableInboxStatus = 'pending' | 'processing' | 'processed';
export type DurableOutboxStatus = 'pending' | 'processing' | 'delivered';
export interface DurableInboxInput {
sessionId: string;
idempotencyKey: string;
correlationId: string;
content: string;
}
export interface DurableInboxEntry extends DurableInboxInput {
status: DurableInboxStatus;
}
export interface DurableOutboxInput {
sessionId: string;
idempotencyKey: string;
correlationId: string;
channelId: string;
kind: string;
content: string;
}
export interface DurableOutboxEntry extends DurableOutboxInput {
status: DurableOutboxStatus;
}
export interface DurableCheckpointInput {
sessionId: string;
checkpointId: string;
cursor: string;
summary: string;
compactionEpoch: number;
}
export interface DurableCheckpoint extends DurableCheckpointInput {}
export interface DurableHandoffInput {
sessionId: string;
handoffId: string;
destination: string;
correlationId: string;
checkpointId: string;
status: 'pending' | 'accepted';
}
export interface DurableHandoff extends DurableHandoffInput {}
export interface DurableSessionSnapshot {
identity: DurableSessionIdentity;
inbox: DurableInboxEntry[];
outbox: DurableOutboxEntry[];
checkpoint?: DurableCheckpoint;
handoffs: DurableHandoff[];
}
export interface DurableHandoffRecovery {
identity: DurableSessionIdentity;
checkpoint: DurableCheckpoint;
handoff: DurableHandoff;
}
export interface DurableEnqueueResult<TStatus extends string> {
accepted: boolean;
status: TStatus;
}
/**
* A durable-state port. Implementations must atomically claim and complete work
* records. Recovery may requeue interrupted inbox work, but never an
* externally visible outbox effect: an ambiguous provider result remains
* claimed until a separately authorized reconciliation proves it safe.
*/
export interface DurableSessionStore {
create(identity: DurableSessionIdentity): Promise<void>;
snapshot(sessionId: string): Promise<DurableSessionSnapshot | null>;
enqueueInbox(input: DurableInboxInput): Promise<DurableEnqueueResult<DurableInboxStatus>>;
claimInbox(sessionId: string): Promise<DurableInboxEntry | null>;
completeInbox(sessionId: string, idempotencyKey: string): Promise<void>;
releaseInbox(sessionId: string, idempotencyKey: string): Promise<void>;
enqueueOutbox(input: DurableOutboxInput): Promise<DurableEnqueueResult<DurableOutboxStatus>>;
claimOutbox(sessionId: string): Promise<DurableOutboxEntry | null>;
claimOutboxByKey(sessionId: string, idempotencyKey: string): Promise<DurableOutboxEntry | null>;
completeOutbox(sessionId: string, idempotencyKey: string): Promise<void>;
releaseOutbox(sessionId: string, idempotencyKey: string): Promise<void>;
checkpoint(input: DurableCheckpointInput): Promise<void>;
findCheckpoint(sessionId: string, checkpointId: string): Promise<DurableCheckpoint | null>;
handoff(input: DurableHandoffInput): Promise<void>;
findHandoff(handoffId: string): Promise<DurableHandoff | null>;
requeueInFlight(sessionId: string): Promise<void>;
}
export class DurableSessionNotFoundError extends Error {
constructor(sessionId: string) {
super(`Durable Tess session not found: ${sessionId}`);
this.name = 'DurableSessionNotFoundError';
}
}
export class DurableSessionCoordinator {
constructor(private readonly store: DurableSessionStore) {}
async create(identity: DurableSessionIdentity): Promise<void> {
this.assertIdentity(identity);
await this.store.create(identity);
}
async receive(input: DurableInboxInput): Promise<DurableEnqueueResult<DurableInboxStatus>> {
this.assertRecord(input.sessionId, input.idempotencyKey, input.correlationId, input.content);
return this.store.enqueueInbox(input);
}
async enqueueOutbox(
input: DurableOutboxInput,
): Promise<DurableEnqueueResult<DurableOutboxStatus>> {
this.assertRecord(
input.sessionId,
input.idempotencyKey,
input.correlationId,
input.channelId,
input.content,
);
if (input.kind.trim().length === 0) throw new Error('Durable outbox kind is required');
return this.store.enqueueOutbox(input);
}
async checkpoint(input: DurableCheckpointInput): Promise<void> {
this.assertRecord(input.sessionId, input.checkpointId, input.cursor, input.summary);
if (!Number.isInteger(input.compactionEpoch) || input.compactionEpoch < 0) {
throw new Error('Durable checkpoint compaction epoch must be a non-negative integer');
}
await this.store.checkpoint(input);
}
async handoff(input: DurableHandoffInput): Promise<void> {
this.assertRecord(input.sessionId, input.handoffId, input.destination, input.correlationId);
if (input.checkpointId.trim().length === 0)
throw new Error('Durable handoff checkpoint is required');
await this.store.handoff(input);
}
/** Read durable state without changing claim status; safe during normal operation. */
async snapshot(sessionId: string): Promise<DurableSessionSnapshot> {
const snapshot = await this.store.snapshot(sessionId);
if (!snapshot) throw new DurableSessionNotFoundError(sessionId);
return snapshot;
}
/** Requeue interrupted inbox work during recovery; preserve ambiguous outbox claims. */
async recover(sessionId: string): Promise<DurableSessionSnapshot> {
await this.store.requeueInFlight(sessionId);
return this.snapshot(sessionId);
}
async drainInbox(
sessionId: string,
handler: (entry: DurableInboxEntry) => Promise<void>,
): Promise<void> {
for (;;) {
const entry = await this.store.claimInbox(sessionId);
if (!entry) return;
try {
await handler(entry);
} catch (error: unknown) {
await this.store.releaseInbox(sessionId, entry.idempotencyKey);
throw error;
}
// If this write fails after the handler succeeded, leave the record
// processing. A recovery path can retry it with its stable idempotency key.
await this.store.completeInbox(sessionId, entry.idempotencyKey);
}
}
async dispatchOutbox(
sessionId: string,
dispatcher: (entry: DurableOutboxEntry) => Promise<void>,
): Promise<void> {
for (;;) {
const entry = await this.store.claimOutbox(sessionId);
if (!entry) return;
// A provider failure can be ambiguous: it may occur after the receiver
// accepted the idempotency key. Preserve the claim for reconciliation.
await dispatcher(entry);
// Do not requeue an effect after it has been applied but before its
// terminal state could be persisted; recovery preserves the claim.
await this.store.completeOutbox(sessionId, entry.idempotencyKey);
}
}
async dispatchOutboxEntry(
sessionId: string,
idempotencyKey: string,
dispatcher: (entry: DurableOutboxEntry) => Promise<void>,
): Promise<void> {
const entry = await this.store.claimOutboxByKey(sessionId, idempotencyKey);
if (!entry) return;
// A provider failure can be ambiguous, so this stays processing until
// separately authorized reconciliation proves it safe to resolve.
await dispatcher(entry);
await this.store.completeOutbox(sessionId, entry.idempotencyKey);
}
async resumeHandoff(handoffId: string): Promise<DurableHandoffRecovery> {
const handoff = await this.store.findHandoff(handoffId);
if (!handoff) throw new Error(`Durable Tess handoff not found: ${handoffId}`);
const snapshot = await this.snapshot(handoff.sessionId);
const checkpoint = await this.store.findCheckpoint(handoff.sessionId, handoff.checkpointId);
if (!checkpoint) {
throw new Error(`Durable Tess handoff checkpoint is unavailable: ${handoff.checkpointId}`);
}
return { identity: snapshot.identity, checkpoint, handoff };
}
private assertIdentity(identity: DurableSessionIdentity): void {
this.assertRecord(identity.sessionId, identity.tenantId, identity.ownerId, identity.providerId);
if (identity.runtimeSessionId.trim().length === 0) {
throw new Error('Durable runtime session identity is required');
}
}
private assertRecord(...values: string[]): void {
if (values.some((value: string): boolean => value.trim().length === 0)) {
throw new Error('Durable session records require non-empty fields');
}
}
}
interface InMemorySessionState {
identity: DurableSessionIdentity;
inbox: Map<string, DurableInboxEntry>;
outbox: Map<string, DurableOutboxEntry>;
checkpoints: Map<string, DurableCheckpoint>;
handoffs: Map<string, DurableHandoff>;
}
/** Reference store for deterministic domain tests; production uses the gateway DB adapter. */
export class InMemoryDurableSessionStore implements DurableSessionStore {
private readonly sessions = new Map<string, InMemorySessionState>();
async create(identity: DurableSessionIdentity): Promise<void> {
const existing = this.sessions.get(identity.sessionId);
if (existing) {
if (!identitiesEqual(existing.identity, identity)) {
throw new Error(`Durable Tess session identity conflict: ${identity.sessionId}`);
}
return;
}
this.sessions.set(identity.sessionId, {
identity: copyIdentity(identity),
inbox: new Map(),
outbox: new Map(),
checkpoints: new Map(),
handoffs: new Map(),
});
}
async snapshot(sessionId: string): Promise<DurableSessionSnapshot | null> {
const state = this.sessions.get(sessionId);
if (!state) return null;
const checkpoint = latestCheckpoint(state.checkpoints);
return {
identity: copyIdentity(state.identity),
inbox: [...state.inbox.values()].map(copyInbox),
outbox: [...state.outbox.values()].map(copyOutbox),
...(checkpoint ? { checkpoint: copyCheckpoint(checkpoint) } : {}),
handoffs: [...state.handoffs.values()].map(copyHandoff),
};
}
async enqueueInbox(input: DurableInboxInput): Promise<DurableEnqueueResult<DurableInboxStatus>> {
const state = this.require(input.sessionId);
const existing = state.inbox.get(input.idempotencyKey);
if (existing) {
if (!sameInbox(existing, input)) {
throw new Error(`Durable Tess inbox idempotency conflict: ${input.idempotencyKey}`);
}
return { accepted: false, status: existing.status };
}
state.inbox.set(input.idempotencyKey, { ...input, status: 'pending' });
return { accepted: true, status: 'pending' };
}
async claimInbox(sessionId: string): Promise<DurableInboxEntry | null> {
const state = this.require(sessionId);
const entry = [...state.inbox.values()].find(
(candidate: DurableInboxEntry): boolean => candidate.status === 'pending',
);
if (!entry) return null;
entry.status = 'processing';
return copyInbox(entry);
}
async completeInbox(sessionId: string, idempotencyKey: string): Promise<void> {
this.requireEntry(this.require(sessionId).inbox, idempotencyKey, 'inbox').status = 'processed';
}
async releaseInbox(sessionId: string, idempotencyKey: string): Promise<void> {
this.requireEntry(this.require(sessionId).inbox, idempotencyKey, 'inbox').status = 'pending';
}
async enqueueOutbox(
input: DurableOutboxInput,
): Promise<DurableEnqueueResult<DurableOutboxStatus>> {
const state = this.require(input.sessionId);
const existing = state.outbox.get(input.idempotencyKey);
if (existing) {
if (!sameOutbox(existing, input)) {
throw new Error(`Durable Tess outbox idempotency conflict: ${input.idempotencyKey}`);
}
return { accepted: false, status: existing.status };
}
state.outbox.set(input.idempotencyKey, { ...input, status: 'pending' });
return { accepted: true, status: 'pending' };
}
async claimOutbox(sessionId: string): Promise<DurableOutboxEntry | null> {
const state = this.require(sessionId);
const entry = [...state.outbox.values()].find(
(candidate: DurableOutboxEntry): boolean => candidate.status === 'pending',
);
if (!entry) return null;
entry.status = 'processing';
return copyOutbox(entry);
}
async claimOutboxByKey(
sessionId: string,
idempotencyKey: string,
): Promise<DurableOutboxEntry | null> {
const entry = this.require(sessionId).outbox.get(idempotencyKey);
if (!entry || entry.status !== 'pending') return null;
entry.status = 'processing';
return copyOutbox(entry);
}
async completeOutbox(sessionId: string, idempotencyKey: string): Promise<void> {
this.requireEntry(this.require(sessionId).outbox, idempotencyKey, 'outbox').status =
'delivered';
}
async releaseOutbox(sessionId: string, idempotencyKey: string): Promise<void> {
this.requireEntry(this.require(sessionId).outbox, idempotencyKey, 'outbox').status = 'pending';
}
async checkpoint(input: DurableCheckpointInput): Promise<void> {
const checkpoints = this.require(input.sessionId).checkpoints;
const existing = checkpoints.get(input.checkpointId);
if (existing && !sameCheckpoint(existing, input)) {
throw new Error(`Durable Tess checkpoint identity conflict: ${input.checkpointId}`);
}
if (!existing) checkpoints.set(input.checkpointId, { ...input });
}
async findCheckpoint(sessionId: string, checkpointId: string): Promise<DurableCheckpoint | null> {
const checkpoint = this.require(sessionId).checkpoints.get(checkpointId);
return checkpoint ? copyCheckpoint(checkpoint) : null;
}
async handoff(input: DurableHandoffInput): Promise<void> {
const state = this.require(input.sessionId);
if (!state.checkpoints.has(input.checkpointId)) {
throw new Error(`Durable Tess handoff checkpoint is unavailable: ${input.checkpointId}`);
}
const existing = state.handoffs.get(input.handoffId);
if (existing && !sameHandoff(existing, input)) {
throw new Error(`Durable Tess handoff identity conflict: ${input.handoffId}`);
}
if (!existing) state.handoffs.set(input.handoffId, { ...input });
}
async findHandoff(handoffId: string): Promise<DurableHandoff | null> {
for (const state of this.sessions.values()) {
const handoff = state.handoffs.get(handoffId);
if (handoff) return copyHandoff(handoff);
}
return null;
}
async requeueInFlight(sessionId: string): Promise<void> {
const state = this.require(sessionId);
for (const entry of state.inbox.values()) {
if (entry.status === 'processing') entry.status = 'pending';
}
}
private require(sessionId: string): InMemorySessionState {
const state = this.sessions.get(sessionId);
if (!state) throw new DurableSessionNotFoundError(sessionId);
return state;
}
private requireEntry<T extends { status: string }>(
records: Map<string, T>,
idempotencyKey: string,
kind: string,
): T {
const entry = records.get(idempotencyKey);
if (!entry) throw new Error(`Durable Tess ${kind} entry not found: ${idempotencyKey}`);
return entry;
}
}
function identitiesEqual(left: DurableSessionIdentity, right: DurableSessionIdentity): boolean {
return (
left.sessionId === right.sessionId &&
left.tenantId === right.tenantId &&
left.ownerId === right.ownerId &&
left.providerId === right.providerId &&
left.runtimeSessionId === right.runtimeSessionId
);
}
function sameInbox(left: DurableInboxEntry, right: DurableInboxInput): boolean {
return (
left.sessionId === right.sessionId &&
left.idempotencyKey === right.idempotencyKey &&
left.correlationId === right.correlationId &&
left.content === right.content
);
}
function sameOutbox(left: DurableOutboxEntry, right: DurableOutboxInput): boolean {
return (
left.sessionId === right.sessionId &&
left.idempotencyKey === right.idempotencyKey &&
left.correlationId === right.correlationId &&
left.channelId === right.channelId &&
left.kind === right.kind &&
left.content === right.content
);
}
function sameCheckpoint(left: DurableCheckpoint, right: DurableCheckpointInput): boolean {
return (
left.sessionId === right.sessionId &&
left.checkpointId === right.checkpointId &&
left.cursor === right.cursor &&
left.summary === right.summary &&
left.compactionEpoch === right.compactionEpoch
);
}
function latestCheckpoint(
checkpoints: Map<string, DurableCheckpoint>,
): DurableCheckpoint | undefined {
return [...checkpoints.values()].sort(
(left: DurableCheckpoint, right: DurableCheckpoint): number =>
right.compactionEpoch - left.compactionEpoch,
)[0];
}
function sameHandoff(left: DurableHandoff, right: DurableHandoffInput): boolean {
return (
left.sessionId === right.sessionId &&
left.handoffId === right.handoffId &&
left.destination === right.destination &&
left.correlationId === right.correlationId &&
left.checkpointId === right.checkpointId &&
left.status === right.status
);
}
function copyIdentity(identity: DurableSessionIdentity): DurableSessionIdentity {
return { ...identity };
}
function copyInbox(entry: DurableInboxEntry): DurableInboxEntry {
return { ...entry };
}
function copyOutbox(entry: DurableOutboxEntry): DurableOutboxEntry {
return { ...entry };
}
function copyCheckpoint(checkpoint: DurableCheckpoint): DurableCheckpoint {
return { ...checkpoint };
}
function copyHandoff(handoff: DurableHandoff): DurableHandoff {
return { ...handoff };
}

View File

@@ -0,0 +1,68 @@
CREATE TYPE "public"."tess_handoff_status" AS ENUM('pending', 'accepted');--> statement-breakpoint
CREATE TYPE "public"."tess_inbox_status" AS ENUM('pending', 'processing', 'processed');--> statement-breakpoint
CREATE TYPE "public"."tess_outbox_status" AS ENUM('pending', 'processing', 'delivered');--> statement-breakpoint
CREATE TABLE "tess_checkpoints" (
"session_id" text PRIMARY KEY NOT NULL,
"checkpoint_id" text NOT NULL,
"cursor" text NOT NULL,
"summary" text NOT NULL,
"compaction_epoch" integer NOT NULL,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
CONSTRAINT "tess_checkpoints_checkpoint_id_unique" UNIQUE("checkpoint_id")
);
--> statement-breakpoint
CREATE TABLE "tess_handoffs" (
"handoff_id" text PRIMARY KEY NOT NULL,
"session_id" text NOT NULL,
"destination" text NOT NULL,
"correlation_id" text NOT NULL,
"checkpoint_id" text NOT NULL,
"status" "tess_handoff_status" DEFAULT 'pending' NOT NULL,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
);
--> statement-breakpoint
CREATE TABLE "tess_inbox" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"session_id" text NOT NULL,
"idempotency_key" text NOT NULL,
"correlation_id" text NOT NULL,
"content" text NOT NULL,
"status" "tess_inbox_status" DEFAULT 'pending' NOT NULL,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
);
--> statement-breakpoint
CREATE TABLE "tess_outbox" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"session_id" text NOT NULL,
"idempotency_key" text NOT NULL,
"correlation_id" text NOT NULL,
"kind" text NOT NULL,
"content" text NOT NULL,
"status" "tess_outbox_status" DEFAULT 'pending' NOT NULL,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
);
--> statement-breakpoint
CREATE TABLE "tess_sessions" (
"id" text PRIMARY KEY NOT NULL,
"tenant_id" text NOT NULL,
"owner_id" text NOT NULL,
"provider_id" text NOT NULL,
"runtime_session_id" text NOT NULL,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
);
--> statement-breakpoint
ALTER TABLE "tess_checkpoints" ADD CONSTRAINT "tess_checkpoints_session_id_tess_sessions_id_fk" FOREIGN KEY ("session_id") REFERENCES "public"."tess_sessions"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "tess_handoffs" ADD CONSTRAINT "tess_handoffs_session_id_tess_sessions_id_fk" FOREIGN KEY ("session_id") REFERENCES "public"."tess_sessions"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "tess_inbox" ADD CONSTRAINT "tess_inbox_session_id_tess_sessions_id_fk" FOREIGN KEY ("session_id") REFERENCES "public"."tess_sessions"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "tess_outbox" ADD CONSTRAINT "tess_outbox_session_id_tess_sessions_id_fk" FOREIGN KEY ("session_id") REFERENCES "public"."tess_sessions"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "tess_sessions" ADD CONSTRAINT "tess_sessions_owner_id_users_id_fk" FOREIGN KEY ("owner_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
CREATE INDEX "tess_handoffs_session_status_idx" ON "tess_handoffs" USING btree ("session_id","status");--> statement-breakpoint
CREATE UNIQUE INDEX "tess_inbox_session_idempotency_idx" ON "tess_inbox" USING btree ("session_id","idempotency_key");--> statement-breakpoint
CREATE INDEX "tess_inbox_session_status_created_idx" ON "tess_inbox" USING btree ("session_id","status","created_at");--> statement-breakpoint
CREATE UNIQUE INDEX "tess_outbox_session_idempotency_idx" ON "tess_outbox" USING btree ("session_id","idempotency_key");--> statement-breakpoint
CREATE INDEX "tess_outbox_session_status_created_idx" ON "tess_outbox" USING btree ("session_id","status","created_at");

View File

@@ -0,0 +1,5 @@
ALTER TABLE "tess_checkpoints" DROP CONSTRAINT "tess_checkpoints_checkpoint_id_unique";--> statement-breakpoint
ALTER TABLE "tess_checkpoints" DROP CONSTRAINT "tess_checkpoints_pkey";--> statement-breakpoint
ALTER TABLE "tess_checkpoints" ADD COLUMN "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL;--> statement-breakpoint
CREATE UNIQUE INDEX "tess_checkpoints_session_idempotency_idx" ON "tess_checkpoints" USING btree ("session_id","checkpoint_id");--> statement-breakpoint
CREATE INDEX "tess_checkpoints_session_epoch_idx" ON "tess_checkpoints" USING btree ("session_id","compaction_epoch");

View File

@@ -0,0 +1,5 @@
ALTER TABLE "tess_outbox" ADD COLUMN "channel_id" text;
--> statement-breakpoint
UPDATE "tess_outbox" SET "channel_id" = 'legacy:unknown' WHERE "channel_id" IS NULL;
--> statement-breakpoint
ALTER TABLE "tess_outbox" ALTER COLUMN "channel_id" SET NOT NULL;

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -85,6 +85,27 @@
"when": 1782310438919, "when": 1782310438919,
"tag": "0011_bitter_gateway", "tag": "0011_bitter_gateway",
"breakpoints": true "breakpoints": true
},
{
"idx": 12,
"version": "7",
"when": 1783911983447,
"tag": "0012_tess_durable_state",
"breakpoints": true
},
{
"idx": 13,
"version": "7",
"when": 1783913232578,
"tag": "0013_tess_checkpoint_history",
"breakpoints": true
},
{
"idx": 14,
"version": "7",
"when": 1783913398006,
"tag": "0014_tess_outbox_channel_scope",
"breakpoints": true
} }
] ]
} }

View File

@@ -487,6 +487,113 @@ export const agentLogs = pgTable(
], ],
); );
// ─── Tess durable session state ─────────────────────────────────────────────
// PostgreSQL is canonical for restart-safe Tess session recovery. The state
// machine lives in @mosaicstack/agent; these records are its durable adapter.
export const tessInboxStatusEnum = pgEnum('tess_inbox_status', [
'pending',
'processing',
'processed',
]);
export const tessOutboxStatusEnum = pgEnum('tess_outbox_status', [
'pending',
'processing',
'delivered',
]);
export const tessHandoffStatusEnum = pgEnum('tess_handoff_status', ['pending', 'accepted']);
export const tessSessions = pgTable('tess_sessions', {
id: text('id').primaryKey(),
tenantId: text('tenant_id').notNull(),
ownerId: text('owner_id')
.notNull()
.references(() => users.id, { onDelete: 'cascade' }),
providerId: text('provider_id').notNull(),
runtimeSessionId: text('runtime_session_id').notNull(),
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
});
export const tessInbox = pgTable(
'tess_inbox',
{
id: uuid('id').primaryKey().defaultRandom(),
sessionId: text('session_id')
.notNull()
.references(() => tessSessions.id, { onDelete: 'cascade' }),
idempotencyKey: text('idempotency_key').notNull(),
correlationId: text('correlation_id').notNull(),
content: text('content').notNull(),
status: tessInboxStatusEnum('status').notNull().default('pending'),
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
},
(t) => [
uniqueIndex('tess_inbox_session_idempotency_idx').on(t.sessionId, t.idempotencyKey),
index('tess_inbox_session_status_created_idx').on(t.sessionId, t.status, t.createdAt),
],
);
export const tessOutbox = pgTable(
'tess_outbox',
{
id: uuid('id').primaryKey().defaultRandom(),
sessionId: text('session_id')
.notNull()
.references(() => tessSessions.id, { onDelete: 'cascade' }),
idempotencyKey: text('idempotency_key').notNull(),
correlationId: text('correlation_id').notNull(),
channelId: text('channel_id').notNull(),
kind: text('kind').notNull(),
content: text('content').notNull(),
status: tessOutboxStatusEnum('status').notNull().default('pending'),
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
},
(t) => [
uniqueIndex('tess_outbox_session_idempotency_idx').on(t.sessionId, t.idempotencyKey),
index('tess_outbox_session_status_created_idx').on(t.sessionId, t.status, t.createdAt),
],
);
export const tessCheckpoints = pgTable(
'tess_checkpoints',
{
id: uuid('id').primaryKey().defaultRandom(),
sessionId: text('session_id')
.notNull()
.references(() => tessSessions.id, { onDelete: 'cascade' }),
checkpointId: text('checkpoint_id').notNull(),
cursor: text('cursor').notNull(),
summary: text('summary').notNull(),
compactionEpoch: integer('compaction_epoch').notNull(),
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
},
(t) => [
uniqueIndex('tess_checkpoints_session_idempotency_idx').on(t.sessionId, t.checkpointId),
index('tess_checkpoints_session_epoch_idx').on(t.sessionId, t.compactionEpoch),
],
);
export const tessHandoffs = pgTable(
'tess_handoffs',
{
handoffId: text('handoff_id').primaryKey(),
sessionId: text('session_id')
.notNull()
.references(() => tessSessions.id, { onDelete: 'cascade' }),
destination: text('destination').notNull(),
correlationId: text('correlation_id').notNull(),
checkpointId: text('checkpoint_id').notNull(),
status: tessHandoffStatusEnum('status').notNull().default('pending'),
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
},
(t) => [index('tess_handoffs_session_status_idx').on(t.sessionId, t.status)],
);
// ─── Skills ───────────────────────────────────────────────────────────────── // ─── Skills ─────────────────────────────────────────────────────────────────
export const skills = pgTable( export const skills = pgTable(