forked from mosaicstack/stack
feat(tess): persist durable session state
This commit is contained in:
@@ -10,14 +10,17 @@ import { ProvidersController } from './providers.controller.js';
|
||||
import { SessionsController } from './sessions.controller.js';
|
||||
import { AgentConfigsController } from './agent-configs.controller.js';
|
||||
import { RoutingController } from './routing/routing.controller.js';
|
||||
import { TessDurableSessionRepository } from './tess-durable-session.repository.js';
|
||||
import { TessDurableSessionService } from './tess-durable-session.service.js';
|
||||
import { CoordModule } from '../coord/coord.module.js';
|
||||
import { McpClientModule } from '../mcp-client/mcp-client.module.js';
|
||||
import { SkillsModule } from '../skills/skills.module.js';
|
||||
import { GCModule } from '../gc/gc.module.js';
|
||||
import { LogModule } from '../log/log.module.js';
|
||||
import { CommandsModule } from '../commands/commands.module.js';
|
||||
import { CommandRuntimeApprovalVerifier } from '../commands/runtime-approval-verifier.js';
|
||||
import {
|
||||
AGENT_RUNTIME_PROVIDER_REGISTRY,
|
||||
DenyRuntimeApprovalVerifier,
|
||||
RUNTIME_APPROVAL_VERIFIER,
|
||||
RUNTIME_PROVIDER_AUDIT_SINK,
|
||||
RuntimeProviderAuditService,
|
||||
@@ -26,13 +29,15 @@ import {
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
imports: [CoordModule, McpClientModule, SkillsModule, GCModule, LogModule],
|
||||
imports: [CoordModule, McpClientModule, SkillsModule, GCModule, LogModule, CommandsModule],
|
||||
providers: [
|
||||
ProviderService,
|
||||
ProviderCredentialsService,
|
||||
RoutingService,
|
||||
RoutingEngineService,
|
||||
SkillLoaderService,
|
||||
TessDurableSessionRepository,
|
||||
TessDurableSessionService,
|
||||
{
|
||||
provide: AGENT_RUNTIME_PROVIDER_REGISTRY,
|
||||
useFactory: (): AgentRuntimeProviderRegistry => new AgentRuntimeProviderRegistry(),
|
||||
@@ -42,10 +47,9 @@ import {
|
||||
provide: RUNTIME_PROVIDER_AUDIT_SINK,
|
||||
useExisting: RuntimeProviderAuditService,
|
||||
},
|
||||
DenyRuntimeApprovalVerifier,
|
||||
{
|
||||
provide: RUNTIME_APPROVAL_VERIFIER,
|
||||
useExisting: DenyRuntimeApprovalVerifier,
|
||||
useExisting: CommandRuntimeApprovalVerifier,
|
||||
},
|
||||
RuntimeProviderService,
|
||||
AgentService,
|
||||
@@ -58,6 +62,7 @@ import {
|
||||
RoutingService,
|
||||
RoutingEngineService,
|
||||
SkillLoaderService,
|
||||
TessDurableSessionService,
|
||||
RuntimeProviderService,
|
||||
AGENT_RUNTIME_PROVIDER_REGISTRY,
|
||||
],
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
import type { RuntimeProviderRequestContext } from './runtime-provider-registry.service.js';
|
||||
|
||||
/** Server-side request for a replay-safe provider message. */
|
||||
export interface TessProviderOutboxDto {
|
||||
sessionId: string;
|
||||
idempotencyKey: string;
|
||||
correlationId: string;
|
||||
content: string;
|
||||
context: RuntimeProviderRequestContext;
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { eq, sql, tessInbox } from '@mosaicstack/db';
|
||||
import { DurableSessionCoordinator, type DurableSessionIdentity } from '@mosaicstack/agent';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { createPgliteDb, runPgliteMigrations, type DbHandle } from '@mosaicstack/db';
|
||||
import { TessDurableSessionRepository } from './tess-durable-session.repository.js';
|
||||
import { TessDurableSessionService } from './tess-durable-session.service.js';
|
||||
|
||||
const IDENTITY: DurableSessionIdentity = {
|
||||
sessionId: 'tess-pglite-session',
|
||||
tenantId: 'tenant-pglite',
|
||||
ownerId: 'tess-owner',
|
||||
providerId: 'fleet',
|
||||
runtimeSessionId: 'nova',
|
||||
};
|
||||
|
||||
describe('TessDurableSessionRepository', () => {
|
||||
let dataDir: string | undefined;
|
||||
let handle: DbHandle | undefined;
|
||||
let previousAuthSecret: string | undefined;
|
||||
|
||||
beforeEach((): void => {
|
||||
previousAuthSecret = process.env['BETTER_AUTH_SECRET'];
|
||||
process.env['BETTER_AUTH_SECRET'] = 'tess-durable-state-test-sealing-key';
|
||||
});
|
||||
|
||||
afterEach(async (): Promise<void> => {
|
||||
await handle?.close();
|
||||
if (dataDir) rmSync(dataDir, { recursive: true, force: true });
|
||||
if (previousAuthSecret === undefined) delete process.env['BETTER_AUTH_SECRET'];
|
||||
else process.env['BETTER_AUTH_SECRET'] = previousAuthSecret;
|
||||
});
|
||||
|
||||
it('survives a full PGlite close/reopen mid-session without duplicate inbox or outbox side effects', async () => {
|
||||
dataDir = mkdtempSync(join(tmpdir(), 'tess-durable-recovery-'));
|
||||
handle = createPgliteDb(dataDir);
|
||||
await runPgliteMigrations(handle);
|
||||
await seedOwner(handle);
|
||||
|
||||
const beforeRestart = new DurableSessionCoordinator(
|
||||
new TessDurableSessionRepository(handle.db),
|
||||
);
|
||||
await beforeRestart.create(IDENTITY);
|
||||
await beforeRestart.receive({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'inbox-before-kill',
|
||||
correlationId: 'correlation-before-kill',
|
||||
content: 'resume after a kill',
|
||||
});
|
||||
await beforeRestart.enqueueOutbox({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'outbox-before-kill',
|
||||
correlationId: 'correlation-before-kill',
|
||||
channelId: 'cli',
|
||||
kind: 'provider.send',
|
||||
content: 'one response only',
|
||||
});
|
||||
await beforeRestart.checkpoint({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
checkpointId: 'checkpoint-before-kill',
|
||||
cursor: 'cursor-before-kill',
|
||||
summary: 'restart-safe state',
|
||||
compactionEpoch: 1,
|
||||
});
|
||||
await beforeRestart.handoff({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
handoffId: 'handoff-before-kill',
|
||||
destination: 'mos',
|
||||
correlationId: 'correlation-before-kill',
|
||||
checkpointId: 'checkpoint-before-kill',
|
||||
status: 'pending',
|
||||
});
|
||||
await beforeRestart.checkpoint({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
checkpointId: 'checkpoint-after-handoff',
|
||||
cursor: 'cursor-after-handoff',
|
||||
summary: 'newer state cannot strand the portable handoff',
|
||||
compactionEpoch: 2,
|
||||
});
|
||||
|
||||
await handle.close();
|
||||
handle = createPgliteDb(dataDir);
|
||||
|
||||
const afterRestart = new DurableSessionCoordinator(new TessDurableSessionRepository(handle.db));
|
||||
const recovered = await afterRestart.recover(IDENTITY.sessionId);
|
||||
const resumedHandoff = await afterRestart.resumeHandoff('handoff-before-kill');
|
||||
const handled: string[] = [];
|
||||
const effects: string[] = [];
|
||||
|
||||
await afterRestart.drainInbox(IDENTITY.sessionId, async (entry): Promise<void> => {
|
||||
handled.push(entry.idempotencyKey);
|
||||
});
|
||||
await afterRestart.dispatchOutbox(IDENTITY.sessionId, async (entry): Promise<void> => {
|
||||
effects.push(entry.idempotencyKey);
|
||||
});
|
||||
await afterRestart.drainInbox(IDENTITY.sessionId, async (entry): Promise<void> => {
|
||||
handled.push(entry.idempotencyKey);
|
||||
});
|
||||
await afterRestart.dispatchOutbox(IDENTITY.sessionId, async (entry): Promise<void> => {
|
||||
effects.push(entry.idempotencyKey);
|
||||
});
|
||||
|
||||
expect(recovered.identity).toEqual(IDENTITY);
|
||||
expect(recovered.checkpoint).toMatchObject({ checkpointId: 'checkpoint-after-handoff' });
|
||||
expect(recovered.handoffs).toMatchObject([{ handoffId: 'handoff-before-kill' }]);
|
||||
expect(resumedHandoff.checkpoint).toMatchObject({ checkpointId: 'checkpoint-before-kill' });
|
||||
expect(handled).toEqual(['inbox-before-kill']);
|
||||
expect(effects).toEqual(['outbox-before-kill']);
|
||||
}, 30_000);
|
||||
|
||||
it('redacts sensitive durable payloads before persistence', async () => {
|
||||
dataDir = mkdtempSync(join(tmpdir(), 'tess-durable-redaction-'));
|
||||
handle = createPgliteDb(dataDir);
|
||||
await runPgliteMigrations(handle);
|
||||
await seedOwner(handle);
|
||||
|
||||
const coordinator = new DurableSessionCoordinator(new TessDurableSessionRepository(handle.db));
|
||||
await coordinator.create(IDENTITY);
|
||||
await coordinator.receive({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'redacted-inbox',
|
||||
correlationId: 'correlation-redaction',
|
||||
content: 'api_key=super-secret-canary',
|
||||
});
|
||||
await coordinator.enqueueOutbox({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'redacted-outbox',
|
||||
correlationId: 'correlation-redaction',
|
||||
channelId: 'cli',
|
||||
kind: 'provider.send',
|
||||
content: 'email [email protected] api_key=super-secret-canary',
|
||||
});
|
||||
await coordinator.checkpoint({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
checkpointId: 'redacted-checkpoint',
|
||||
cursor: 'bearer super-secret-canary',
|
||||
summary: 'email [email protected]',
|
||||
compactionEpoch: 0,
|
||||
});
|
||||
|
||||
const snapshot = await coordinator.snapshot(IDENTITY.sessionId);
|
||||
const [persisted] = await handle.db
|
||||
.select({ content: tessInbox.content })
|
||||
.from(tessInbox)
|
||||
.where(eq(tessInbox.idempotencyKey, 'redacted-inbox'));
|
||||
|
||||
expect(JSON.stringify(snapshot)).not.toContain('super-secret-canary');
|
||||
expect(JSON.stringify(snapshot)).not.toContain('[email protected]');
|
||||
expect(persisted?.content).not.toContain('super-secret-canary');
|
||||
expect(persisted?.content).not.toContain('[REDACTED]');
|
||||
}, 30_000);
|
||||
|
||||
it('rejects database inbox and outbox idempotency-key conflicts', async () => {
|
||||
dataDir = mkdtempSync(join(tmpdir(), 'tess-durable-idempotency-conflict-'));
|
||||
handle = createPgliteDb(dataDir);
|
||||
await runPgliteMigrations(handle);
|
||||
await seedOwner(handle);
|
||||
|
||||
const coordinator = new DurableSessionCoordinator(new TessDurableSessionRepository(handle.db));
|
||||
await coordinator.create(IDENTITY);
|
||||
await coordinator.receive({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'inbox-conflict',
|
||||
correlationId: 'correlation-inbox',
|
||||
content: 'original inbox',
|
||||
});
|
||||
await coordinator.enqueueOutbox({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'outbox-conflict',
|
||||
correlationId: 'correlation-outbox',
|
||||
channelId: 'cli',
|
||||
kind: 'provider.send',
|
||||
content: 'original outbox',
|
||||
});
|
||||
|
||||
await expect(
|
||||
coordinator.receive({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'inbox-conflict',
|
||||
correlationId: 'forged-correlation',
|
||||
content: 'original inbox',
|
||||
}),
|
||||
).rejects.toThrow(/idempotency conflict/);
|
||||
await expect(
|
||||
coordinator.enqueueOutbox({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'outbox-conflict',
|
||||
correlationId: 'correlation-outbox',
|
||||
channelId: 'forged-channel',
|
||||
kind: 'provider.send',
|
||||
content: 'original outbox',
|
||||
}),
|
||||
).rejects.toThrow(/idempotency conflict/);
|
||||
}, 30_000);
|
||||
|
||||
it('does not requeue a live outbox claim during a normal scoped dispatch', async () => {
|
||||
dataDir = mkdtempSync(join(tmpdir(), 'tess-durable-live-claim-'));
|
||||
handle = createPgliteDb(dataDir);
|
||||
await runPgliteMigrations(handle);
|
||||
await seedOwner(handle);
|
||||
|
||||
const repository = new TessDurableSessionRepository(handle.db);
|
||||
const coordinator = new DurableSessionCoordinator(repository);
|
||||
const runtimeProviders = { sendMessage: vi.fn().mockResolvedValue(undefined) };
|
||||
const service = new TessDurableSessionService(repository, runtimeProviders as never);
|
||||
const input = {
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'live-effect',
|
||||
correlationId: 'correlation-live',
|
||||
content: 'must not duplicate',
|
||||
context: {
|
||||
actorScope: { userId: IDENTITY.ownerId, tenantId: IDENTITY.tenantId },
|
||||
channelId: 'cli',
|
||||
correlationId: 'correlation-live',
|
||||
},
|
||||
};
|
||||
|
||||
await coordinator.create(IDENTITY);
|
||||
await service.queueProviderSend(input);
|
||||
expect(await repository.claimOutbox(IDENTITY.sessionId)).toMatchObject({
|
||||
status: 'processing',
|
||||
});
|
||||
|
||||
await service.dispatchProviderOutbox(IDENTITY.sessionId, input);
|
||||
await expect(
|
||||
service.recoverProviderSession(IDENTITY.sessionId, {
|
||||
...input,
|
||||
context: {
|
||||
...input.context,
|
||||
actorScope: { userId: 'intruder', tenantId: 'tenant-pglite' },
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow(/scope or correlation mismatch/);
|
||||
|
||||
expect(runtimeProviders.sendMessage).not.toHaveBeenCalled();
|
||||
expect(await coordinator.snapshot(IDENTITY.sessionId)).toMatchObject({
|
||||
outbox: [{ idempotencyKey: 'live-effect', status: 'processing' }],
|
||||
});
|
||||
}, 30_000);
|
||||
|
||||
it('rejects an outbox correlation mismatch before claiming the pending effect', async () => {
|
||||
dataDir = mkdtempSync(join(tmpdir(), 'tess-durable-outbox-mismatch-'));
|
||||
handle = createPgliteDb(dataDir);
|
||||
await runPgliteMigrations(handle);
|
||||
await seedOwner(handle);
|
||||
|
||||
const repository = new TessDurableSessionRepository(handle.db);
|
||||
const coordinator = new DurableSessionCoordinator(repository);
|
||||
const runtimeProviders = { sendMessage: vi.fn().mockResolvedValue(undefined) };
|
||||
const service = new TessDurableSessionService(repository, runtimeProviders as never);
|
||||
const input = {
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'mismatch-effect',
|
||||
correlationId: 'correlation-expected',
|
||||
content: 'must remain pending',
|
||||
context: {
|
||||
actorScope: { userId: IDENTITY.ownerId, tenantId: IDENTITY.tenantId },
|
||||
channelId: 'cli',
|
||||
correlationId: 'correlation-expected',
|
||||
},
|
||||
};
|
||||
|
||||
await coordinator.create(IDENTITY);
|
||||
await service.queueProviderSend(input);
|
||||
await expect(
|
||||
service.dispatchProviderOutbox(IDENTITY.sessionId, {
|
||||
...input,
|
||||
correlationId: 'correlation-forged',
|
||||
context: { ...input.context, correlationId: 'correlation-forged' },
|
||||
}),
|
||||
).rejects.toThrow(/scope or correlation mismatch/);
|
||||
|
||||
expect(runtimeProviders.sendMessage).not.toHaveBeenCalled();
|
||||
expect(await coordinator.snapshot(IDENTITY.sessionId)).toMatchObject({
|
||||
outbox: [{ idempotencyKey: 'mismatch-effect', status: 'pending' }],
|
||||
});
|
||||
}, 30_000);
|
||||
|
||||
it('dispatches only the outbox record bound to the supplied correlation and channel', async () => {
|
||||
dataDir = mkdtempSync(join(tmpdir(), 'tess-durable-outbox-scope-'));
|
||||
handle = createPgliteDb(dataDir);
|
||||
await runPgliteMigrations(handle);
|
||||
await seedOwner(handle);
|
||||
|
||||
const repository = new TessDurableSessionRepository(handle.db);
|
||||
const coordinator = new DurableSessionCoordinator(repository);
|
||||
const runtimeProviders = { sendMessage: vi.fn().mockResolvedValue(undefined) };
|
||||
const service = new TessDurableSessionService(repository, runtimeProviders as never);
|
||||
const first = {
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'scoped-effect-one',
|
||||
correlationId: 'correlation-one',
|
||||
content: 'first result',
|
||||
context: {
|
||||
actorScope: { userId: IDENTITY.ownerId, tenantId: IDENTITY.tenantId },
|
||||
channelId: 'cli',
|
||||
correlationId: 'correlation-one',
|
||||
},
|
||||
};
|
||||
const second = {
|
||||
...first,
|
||||
idempotencyKey: 'scoped-effect-two',
|
||||
correlationId: 'correlation-two',
|
||||
content: 'second result',
|
||||
context: { ...first.context, correlationId: 'correlation-two' },
|
||||
};
|
||||
|
||||
await coordinator.create(IDENTITY);
|
||||
await service.queueProviderSend(first);
|
||||
await service.queueProviderSend(second);
|
||||
await service.dispatchProviderOutbox(IDENTITY.sessionId, first);
|
||||
|
||||
expect(runtimeProviders.sendMessage).toHaveBeenCalledTimes(1);
|
||||
expect(runtimeProviders.sendMessage).toHaveBeenCalledWith(
|
||||
IDENTITY.providerId,
|
||||
IDENTITY.runtimeSessionId,
|
||||
{ content: 'first result', idempotencyKey: 'scoped-effect-one' },
|
||||
first.context,
|
||||
);
|
||||
expect(await coordinator.snapshot(IDENTITY.sessionId)).toMatchObject({
|
||||
outbox: [
|
||||
{ idempotencyKey: 'scoped-effect-one', status: 'delivered' },
|
||||
{ idempotencyKey: 'scoped-effect-two', status: 'pending' },
|
||||
],
|
||||
});
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
async function seedOwner(handle: DbHandle): Promise<void> {
|
||||
await handle.db.execute(sql`
|
||||
INSERT INTO users (id, name, email, email_verified, created_at, updated_at)
|
||||
VALUES ('tess-owner', 'Tess Owner', '[email protected]', false, now(), now())
|
||||
`);
|
||||
}
|
||||
@@ -0,0 +1,454 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import {
|
||||
and,
|
||||
asc,
|
||||
desc,
|
||||
eq,
|
||||
tessCheckpoints,
|
||||
tessHandoffs,
|
||||
tessInbox,
|
||||
tessOutbox,
|
||||
tessSessions,
|
||||
type Db,
|
||||
} from '@mosaicstack/db';
|
||||
import { seal, unseal } from '@mosaicstack/auth';
|
||||
import { redactSensitiveContent } from '@mosaicstack/log';
|
||||
import type {
|
||||
DurableCheckpoint,
|
||||
DurableCheckpointInput,
|
||||
DurableEnqueueResult,
|
||||
DurableHandoff,
|
||||
DurableHandoffInput,
|
||||
DurableInboxEntry,
|
||||
DurableInboxInput,
|
||||
DurableInboxStatus,
|
||||
DurableOutboxEntry,
|
||||
DurableOutboxInput,
|
||||
DurableOutboxStatus,
|
||||
DurableSessionIdentity,
|
||||
DurableSessionSnapshot,
|
||||
DurableSessionStore,
|
||||
} from '@mosaicstack/agent';
|
||||
import { DB } from '../database/database.module.js';
|
||||
|
||||
@Injectable()
|
||||
export class TessDurableSessionRepository implements DurableSessionStore {
|
||||
constructor(@Inject(DB) private readonly db: Db) {}
|
||||
|
||||
async create(identity: DurableSessionIdentity): Promise<void> {
|
||||
await this.db
|
||||
.insert(tessSessions)
|
||||
.values({
|
||||
id: identity.sessionId,
|
||||
tenantId: identity.tenantId,
|
||||
ownerId: identity.ownerId,
|
||||
providerId: identity.providerId,
|
||||
runtimeSessionId: identity.runtimeSessionId,
|
||||
})
|
||||
.onConflictDoNothing();
|
||||
|
||||
const existing = await this.session(identity.sessionId);
|
||||
if (!existing || !sameIdentity(existing, identity)) {
|
||||
throw new Error(`Durable Tess session identity conflict: ${identity.sessionId}`);
|
||||
}
|
||||
}
|
||||
|
||||
async snapshot(sessionId: string): Promise<DurableSessionSnapshot | null> {
|
||||
const identity = await this.session(sessionId);
|
||||
if (!identity) return null;
|
||||
|
||||
const [inbox, outbox, checkpoints, handoffs] = await Promise.all([
|
||||
this.db
|
||||
.select()
|
||||
.from(tessInbox)
|
||||
.where(eq(tessInbox.sessionId, sessionId))
|
||||
.orderBy(asc(tessInbox.createdAt)),
|
||||
this.db
|
||||
.select()
|
||||
.from(tessOutbox)
|
||||
.where(eq(tessOutbox.sessionId, sessionId))
|
||||
.orderBy(asc(tessOutbox.createdAt)),
|
||||
this.db
|
||||
.select()
|
||||
.from(tessCheckpoints)
|
||||
.where(eq(tessCheckpoints.sessionId, sessionId))
|
||||
.orderBy(desc(tessCheckpoints.compactionEpoch), desc(tessCheckpoints.createdAt))
|
||||
.limit(1),
|
||||
this.db
|
||||
.select()
|
||||
.from(tessHandoffs)
|
||||
.where(eq(tessHandoffs.sessionId, sessionId))
|
||||
.orderBy(asc(tessHandoffs.createdAt)),
|
||||
]);
|
||||
|
||||
const checkpoint = checkpoints[0];
|
||||
return {
|
||||
identity,
|
||||
inbox: inbox.map(toInbox),
|
||||
outbox: outbox.map(toOutbox),
|
||||
...(checkpoint ? { checkpoint: toCheckpoint(checkpoint) } : {}),
|
||||
handoffs: handoffs.map(toHandoff),
|
||||
};
|
||||
}
|
||||
|
||||
async enqueueInbox(input: DurableInboxInput): Promise<DurableEnqueueResult<DurableInboxStatus>> {
|
||||
const record: DurableInboxInput = {
|
||||
...input,
|
||||
content: redactSensitiveContent(input.content).content,
|
||||
};
|
||||
const inserted = await this.db
|
||||
.insert(tessInbox)
|
||||
.values({ ...record, content: seal(record.content), status: 'pending' })
|
||||
.onConflictDoNothing()
|
||||
.returning({ status: tessInbox.status });
|
||||
if (inserted[0]) return { accepted: true, status: inserted[0].status };
|
||||
|
||||
const existing = await this.db
|
||||
.select()
|
||||
.from(tessInbox)
|
||||
.where(
|
||||
and(
|
||||
eq(tessInbox.sessionId, input.sessionId),
|
||||
eq(tessInbox.idempotencyKey, input.idempotencyKey),
|
||||
),
|
||||
)
|
||||
.limit(1);
|
||||
if (!existing[0]) throw new Error(`Durable Tess inbox enqueue failed: ${input.idempotencyKey}`);
|
||||
const entry = toInbox(existing[0]);
|
||||
if (!sameInbox(entry, record)) {
|
||||
throw new Error(`Durable Tess inbox idempotency conflict: ${input.idempotencyKey}`);
|
||||
}
|
||||
return { accepted: false, status: entry.status };
|
||||
}
|
||||
|
||||
async claimInbox(sessionId: string): Promise<DurableInboxEntry | null> {
|
||||
for (let attempt = 0; attempt < 3; attempt += 1) {
|
||||
const candidate = await this.db
|
||||
.select()
|
||||
.from(tessInbox)
|
||||
.where(and(eq(tessInbox.sessionId, sessionId), eq(tessInbox.status, 'pending')))
|
||||
.orderBy(asc(tessInbox.createdAt))
|
||||
.limit(1);
|
||||
const entry = candidate[0];
|
||||
if (!entry) return null;
|
||||
const claimed = await this.db
|
||||
.update(tessInbox)
|
||||
.set({ status: 'processing', updatedAt: new Date() })
|
||||
.where(and(eq(tessInbox.id, entry.id), eq(tessInbox.status, 'pending')))
|
||||
.returning();
|
||||
if (claimed[0]) return toInbox(claimed[0]);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async completeInbox(sessionId: string, idempotencyKey: string): Promise<void> {
|
||||
await this.db
|
||||
.update(tessInbox)
|
||||
.set({ status: 'processed', updatedAt: new Date() })
|
||||
.where(
|
||||
and(
|
||||
eq(tessInbox.sessionId, sessionId),
|
||||
eq(tessInbox.idempotencyKey, idempotencyKey),
|
||||
eq(tessInbox.status, 'processing'),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async releaseInbox(sessionId: string, idempotencyKey: string): Promise<void> {
|
||||
await this.db
|
||||
.update(tessInbox)
|
||||
.set({ status: 'pending', updatedAt: new Date() })
|
||||
.where(
|
||||
and(
|
||||
eq(tessInbox.sessionId, sessionId),
|
||||
eq(tessInbox.idempotencyKey, idempotencyKey),
|
||||
eq(tessInbox.status, 'processing'),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async enqueueOutbox(
|
||||
input: DurableOutboxInput,
|
||||
): Promise<DurableEnqueueResult<DurableOutboxStatus>> {
|
||||
const record: DurableOutboxInput = {
|
||||
...input,
|
||||
content: redactSensitiveContent(input.content).content,
|
||||
};
|
||||
const inserted = await this.db
|
||||
.insert(tessOutbox)
|
||||
.values({ ...record, content: seal(record.content), status: 'pending' })
|
||||
.onConflictDoNothing()
|
||||
.returning({ status: tessOutbox.status });
|
||||
if (inserted[0]) return { accepted: true, status: inserted[0].status };
|
||||
|
||||
const existing = await this.db
|
||||
.select()
|
||||
.from(tessOutbox)
|
||||
.where(
|
||||
and(
|
||||
eq(tessOutbox.sessionId, input.sessionId),
|
||||
eq(tessOutbox.idempotencyKey, input.idempotencyKey),
|
||||
),
|
||||
)
|
||||
.limit(1);
|
||||
if (!existing[0])
|
||||
throw new Error(`Durable Tess outbox enqueue failed: ${input.idempotencyKey}`);
|
||||
const entry = toOutbox(existing[0]);
|
||||
if (!sameOutbox(entry, record)) {
|
||||
throw new Error(`Durable Tess outbox idempotency conflict: ${input.idempotencyKey}`);
|
||||
}
|
||||
return { accepted: false, status: entry.status };
|
||||
}
|
||||
|
||||
async claimOutbox(sessionId: string): Promise<DurableOutboxEntry | null> {
|
||||
for (let attempt = 0; attempt < 3; attempt += 1) {
|
||||
const candidate = await this.db
|
||||
.select()
|
||||
.from(tessOutbox)
|
||||
.where(and(eq(tessOutbox.sessionId, sessionId), eq(tessOutbox.status, 'pending')))
|
||||
.orderBy(asc(tessOutbox.createdAt))
|
||||
.limit(1);
|
||||
const entry = candidate[0];
|
||||
if (!entry) return null;
|
||||
const claimed = await this.db
|
||||
.update(tessOutbox)
|
||||
.set({ status: 'processing', updatedAt: new Date() })
|
||||
.where(and(eq(tessOutbox.id, entry.id), eq(tessOutbox.status, 'pending')))
|
||||
.returning();
|
||||
if (claimed[0]) return toOutbox(claimed[0]);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async claimOutboxByKey(
|
||||
sessionId: string,
|
||||
idempotencyKey: string,
|
||||
): Promise<DurableOutboxEntry | null> {
|
||||
const claimed = await this.db
|
||||
.update(tessOutbox)
|
||||
.set({ status: 'processing', updatedAt: new Date() })
|
||||
.where(
|
||||
and(
|
||||
eq(tessOutbox.sessionId, sessionId),
|
||||
eq(tessOutbox.idempotencyKey, idempotencyKey),
|
||||
eq(tessOutbox.status, 'pending'),
|
||||
),
|
||||
)
|
||||
.returning();
|
||||
return claimed[0] ? toOutbox(claimed[0]) : null;
|
||||
}
|
||||
|
||||
async completeOutbox(sessionId: string, idempotencyKey: string): Promise<void> {
|
||||
await this.db
|
||||
.update(tessOutbox)
|
||||
.set({ status: 'delivered', updatedAt: new Date() })
|
||||
.where(
|
||||
and(
|
||||
eq(tessOutbox.sessionId, sessionId),
|
||||
eq(tessOutbox.idempotencyKey, idempotencyKey),
|
||||
eq(tessOutbox.status, 'processing'),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async releaseOutbox(sessionId: string, idempotencyKey: string): Promise<void> {
|
||||
await this.db
|
||||
.update(tessOutbox)
|
||||
.set({ status: 'pending', updatedAt: new Date() })
|
||||
.where(
|
||||
and(
|
||||
eq(tessOutbox.sessionId, sessionId),
|
||||
eq(tessOutbox.idempotencyKey, idempotencyKey),
|
||||
eq(tessOutbox.status, 'processing'),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async checkpoint(input: DurableCheckpointInput): Promise<void> {
|
||||
const checkpoint: DurableCheckpointInput = {
|
||||
...input,
|
||||
cursor: redactSensitiveContent(input.cursor).content,
|
||||
summary: redactSensitiveContent(input.summary).content,
|
||||
};
|
||||
const inserted = await this.db
|
||||
.insert(tessCheckpoints)
|
||||
.values({
|
||||
...checkpoint,
|
||||
cursor: seal(checkpoint.cursor),
|
||||
summary: seal(checkpoint.summary),
|
||||
})
|
||||
.onConflictDoNothing()
|
||||
.returning({ checkpointId: tessCheckpoints.checkpointId });
|
||||
if (inserted[0]) return;
|
||||
|
||||
const existing = await this.findCheckpoint(input.sessionId, input.checkpointId);
|
||||
if (!existing || !sameCheckpoint(existing, checkpoint)) {
|
||||
throw new Error(`Durable Tess checkpoint identity conflict: ${input.checkpointId}`);
|
||||
}
|
||||
}
|
||||
|
||||
async findCheckpoint(sessionId: string, checkpointId: string): Promise<DurableCheckpoint | null> {
|
||||
const checkpoints = await this.db
|
||||
.select()
|
||||
.from(tessCheckpoints)
|
||||
.where(
|
||||
and(
|
||||
eq(tessCheckpoints.sessionId, sessionId),
|
||||
eq(tessCheckpoints.checkpointId, checkpointId),
|
||||
),
|
||||
)
|
||||
.limit(1);
|
||||
const checkpoint = checkpoints[0];
|
||||
return checkpoint ? toCheckpoint(checkpoint) : null;
|
||||
}
|
||||
|
||||
async handoff(input: DurableHandoffInput): Promise<void> {
|
||||
const checkpoint = await this.findCheckpoint(input.sessionId, input.checkpointId);
|
||||
if (!checkpoint) {
|
||||
throw new Error(`Durable Tess handoff checkpoint is unavailable: ${input.checkpointId}`);
|
||||
}
|
||||
const inserted = await this.db
|
||||
.insert(tessHandoffs)
|
||||
.values({ ...input })
|
||||
.onConflictDoNothing()
|
||||
.returning({ handoffId: tessHandoffs.handoffId });
|
||||
if (inserted[0]) return;
|
||||
|
||||
const existing = await this.findHandoff(input.handoffId);
|
||||
if (!existing || !sameHandoff(existing, input)) {
|
||||
throw new Error(`Durable Tess handoff identity conflict: ${input.handoffId}`);
|
||||
}
|
||||
}
|
||||
|
||||
async findHandoff(handoffId: string): Promise<DurableHandoff | null> {
|
||||
const handoffs = await this.db
|
||||
.select()
|
||||
.from(tessHandoffs)
|
||||
.where(eq(tessHandoffs.handoffId, handoffId))
|
||||
.limit(1);
|
||||
const handoff = handoffs[0];
|
||||
return handoff ? toHandoff(handoff) : null;
|
||||
}
|
||||
|
||||
async requeueInFlight(sessionId: string): Promise<void> {
|
||||
// Inbox handlers are process-local work. A provider outbox claim may have
|
||||
// reached an external target before a crash, so it is deliberately not
|
||||
// replayed by generic recovery.
|
||||
await this.db
|
||||
.update(tessInbox)
|
||||
.set({ status: 'pending', updatedAt: new Date() })
|
||||
.where(and(eq(tessInbox.sessionId, sessionId), eq(tessInbox.status, 'processing')));
|
||||
}
|
||||
|
||||
private async session(sessionId: string): Promise<DurableSessionIdentity | null> {
|
||||
const sessions = await this.db
|
||||
.select()
|
||||
.from(tessSessions)
|
||||
.where(eq(tessSessions.id, sessionId))
|
||||
.limit(1);
|
||||
const session = sessions[0];
|
||||
return session
|
||||
? {
|
||||
sessionId: session.id,
|
||||
tenantId: session.tenantId,
|
||||
ownerId: session.ownerId,
|
||||
providerId: session.providerId,
|
||||
runtimeSessionId: session.runtimeSessionId,
|
||||
}
|
||||
: null;
|
||||
}
|
||||
}
|
||||
|
||||
function sameIdentity(left: DurableSessionIdentity, right: DurableSessionIdentity): boolean {
|
||||
return (
|
||||
left.sessionId === right.sessionId &&
|
||||
left.tenantId === right.tenantId &&
|
||||
left.ownerId === right.ownerId &&
|
||||
left.providerId === right.providerId &&
|
||||
left.runtimeSessionId === right.runtimeSessionId
|
||||
);
|
||||
}
|
||||
|
||||
function sameInbox(left: DurableInboxEntry, right: DurableInboxInput): boolean {
|
||||
return (
|
||||
left.sessionId === right.sessionId &&
|
||||
left.idempotencyKey === right.idempotencyKey &&
|
||||
left.correlationId === right.correlationId &&
|
||||
left.content === right.content
|
||||
);
|
||||
}
|
||||
|
||||
function sameOutbox(left: DurableOutboxEntry, right: DurableOutboxInput): boolean {
|
||||
return (
|
||||
left.sessionId === right.sessionId &&
|
||||
left.idempotencyKey === right.idempotencyKey &&
|
||||
left.correlationId === right.correlationId &&
|
||||
left.channelId === right.channelId &&
|
||||
left.kind === right.kind &&
|
||||
left.content === right.content
|
||||
);
|
||||
}
|
||||
|
||||
function sameCheckpoint(left: DurableCheckpoint, right: DurableCheckpointInput): boolean {
|
||||
return (
|
||||
left.sessionId === right.sessionId &&
|
||||
left.checkpointId === right.checkpointId &&
|
||||
left.cursor === right.cursor &&
|
||||
left.summary === right.summary &&
|
||||
left.compactionEpoch === right.compactionEpoch
|
||||
);
|
||||
}
|
||||
|
||||
function sameHandoff(left: DurableHandoff, right: DurableHandoffInput): boolean {
|
||||
return (
|
||||
left.sessionId === right.sessionId &&
|
||||
left.handoffId === right.handoffId &&
|
||||
left.destination === right.destination &&
|
||||
left.correlationId === right.correlationId &&
|
||||
left.checkpointId === right.checkpointId &&
|
||||
left.status === right.status
|
||||
);
|
||||
}
|
||||
|
||||
function toInbox(row: typeof tessInbox.$inferSelect): DurableInboxEntry {
|
||||
return {
|
||||
sessionId: row.sessionId,
|
||||
idempotencyKey: row.idempotencyKey,
|
||||
correlationId: row.correlationId,
|
||||
content: unseal(row.content),
|
||||
status: row.status,
|
||||
};
|
||||
}
|
||||
|
||||
function toOutbox(row: typeof tessOutbox.$inferSelect): DurableOutboxEntry {
|
||||
return {
|
||||
sessionId: row.sessionId,
|
||||
idempotencyKey: row.idempotencyKey,
|
||||
correlationId: row.correlationId,
|
||||
channelId: row.channelId,
|
||||
kind: row.kind,
|
||||
content: unseal(row.content),
|
||||
status: row.status,
|
||||
};
|
||||
}
|
||||
|
||||
function toCheckpoint(row: typeof tessCheckpoints.$inferSelect): DurableCheckpoint {
|
||||
return {
|
||||
sessionId: row.sessionId,
|
||||
checkpointId: row.checkpointId,
|
||||
cursor: unseal(row.cursor),
|
||||
summary: unseal(row.summary),
|
||||
compactionEpoch: row.compactionEpoch,
|
||||
};
|
||||
}
|
||||
|
||||
function toHandoff(row: typeof tessHandoffs.$inferSelect): DurableHandoff {
|
||||
return {
|
||||
sessionId: row.sessionId,
|
||||
handoffId: row.handoffId,
|
||||
destination: row.destination,
|
||||
correlationId: row.correlationId,
|
||||
checkpointId: row.checkpointId,
|
||||
status: row.status,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { ForbiddenException, Inject, Injectable } from '@nestjs/common';
|
||||
import { DurableSessionCoordinator } from '@mosaicstack/agent';
|
||||
import type { TessProviderOutboxDto } from './tess-durable-session.dto.js';
|
||||
import { TessDurableSessionRepository } from './tess-durable-session.repository.js';
|
||||
import { RuntimeProviderService } from './runtime-provider-registry.service.js';
|
||||
|
||||
/**
|
||||
* Scoped gateway boundary for the canonical Tess state machine. It deliberately
|
||||
* uses composition: raw state methods cannot be injected into channel, CLI, or
|
||||
* MCP adapters without a server-derived actor/tenant/correlation context.
|
||||
*/
|
||||
@Injectable()
|
||||
export class TessDurableSessionService {
|
||||
private readonly coordinator: DurableSessionCoordinator;
|
||||
|
||||
constructor(
|
||||
@Inject(TessDurableSessionRepository) repository: TessDurableSessionRepository,
|
||||
@Inject(RuntimeProviderService) private readonly runtimeProviders: RuntimeProviderService,
|
||||
) {
|
||||
this.coordinator = new DurableSessionCoordinator(repository);
|
||||
}
|
||||
|
||||
async queueProviderSend(input: TessProviderOutboxDto): Promise<void> {
|
||||
const snapshot = await this.coordinator.snapshot(input.sessionId);
|
||||
this.assertScope(snapshot.identity.ownerId, snapshot.identity.tenantId, input);
|
||||
await this.coordinator.enqueueOutbox({
|
||||
sessionId: input.sessionId,
|
||||
idempotencyKey: input.idempotencyKey,
|
||||
correlationId: input.correlationId,
|
||||
channelId: input.context.channelId,
|
||||
kind: 'provider.send',
|
||||
content: input.content,
|
||||
});
|
||||
}
|
||||
|
||||
async dispatchProviderOutbox(sessionId: string, input: TessProviderOutboxDto): Promise<void> {
|
||||
if (sessionId !== input.sessionId) {
|
||||
throw new ForbiddenException('Durable Tess outbox session mismatch');
|
||||
}
|
||||
const snapshot = await this.coordinator.snapshot(sessionId);
|
||||
this.assertScope(snapshot.identity.ownerId, snapshot.identity.tenantId, input);
|
||||
const pendingEntry = snapshot.outbox.find(
|
||||
(entry): boolean => entry.idempotencyKey === input.idempotencyKey,
|
||||
);
|
||||
if (!pendingEntry) return;
|
||||
// Validate immutable routing before claiming. A caller with a mismatched
|
||||
// correlation/channel must not strand a pending external side effect.
|
||||
this.assertOutboxScope(pendingEntry, input);
|
||||
await this.coordinator.dispatchOutboxEntry(
|
||||
sessionId,
|
||||
input.idempotencyKey,
|
||||
async (entry): Promise<void> => {
|
||||
this.assertOutboxScope(entry, input);
|
||||
await this.runtimeProviders.sendMessage(
|
||||
snapshot.identity.providerId,
|
||||
snapshot.identity.runtimeSessionId,
|
||||
{ content: entry.content, idempotencyKey: entry.idempotencyKey },
|
||||
input.context,
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/** Startup/recovery-only path; normal queue/dispatch methods never requeue live work. */
|
||||
async recoverProviderSession(sessionId: string, input: TessProviderOutboxDto): Promise<void> {
|
||||
const snapshot = await this.coordinator.snapshot(sessionId);
|
||||
this.assertScope(snapshot.identity.ownerId, snapshot.identity.tenantId, input);
|
||||
await this.coordinator.recover(sessionId);
|
||||
}
|
||||
|
||||
private assertOutboxScope(
|
||||
entry: { kind: string; correlationId: string; channelId: string },
|
||||
input: TessProviderOutboxDto,
|
||||
): void {
|
||||
if (
|
||||
entry.kind !== 'provider.send' ||
|
||||
entry.correlationId !== input.correlationId ||
|
||||
entry.channelId !== input.context.channelId
|
||||
) {
|
||||
throw new ForbiddenException('Durable Tess outbox scope or correlation mismatch');
|
||||
}
|
||||
}
|
||||
|
||||
private assertScope(ownerId: string, tenantId: string, input: TessProviderOutboxDto): void {
|
||||
if (
|
||||
input.context.actorScope.userId !== ownerId ||
|
||||
input.context.actorScope.tenantId !== tenantId ||
|
||||
input.context.correlationId !== input.correlationId
|
||||
) {
|
||||
throw new ForbiddenException('Durable Tess session scope or correlation mismatch');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -12,8 +12,10 @@ const adminCommand: CommandDef = {
|
||||
};
|
||||
const payload: SlashCommandPayload = { command: 'gc', conversationId: 'conversation-1' };
|
||||
|
||||
function createService(role: string): CommandAuthorizationService {
|
||||
const entries = new Map<string, string>();
|
||||
function createService(
|
||||
role: string,
|
||||
entries: Map<string, string> = new Map<string, string>(),
|
||||
): CommandAuthorizationService {
|
||||
const db = {
|
||||
select: () => ({ from: () => ({ where: () => ({ limit: async () => [{ role }] }) }) }),
|
||||
};
|
||||
@@ -58,4 +60,55 @@ describe('CommandAuthorizationService', () => {
|
||||
(await service.authorize(adminCommand, payload, 'member-1', 'forged-approval-id')).allowed,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('denies a malformed durable approval expiry instead of treating it as unexpired', async (): Promise<void> => {
|
||||
const entries = new Map<string, string>();
|
||||
const action = {
|
||||
providerId: 'fleet',
|
||||
sessionId: 'nova',
|
||||
actorId: 'admin-1',
|
||||
tenantId: 'tenant-1',
|
||||
channelId: 'discord:operator',
|
||||
correlationId: 'correlation-malformed-expiry',
|
||||
};
|
||||
const service = createService('admin', entries);
|
||||
const approval = await service.createRuntimeTerminationApproval(action);
|
||||
expect(approval).not.toBeNull();
|
||||
const key = `tess:command-approval:${approval!.approvalId}`;
|
||||
const stored = entries.get(key);
|
||||
expect(stored).toBeDefined();
|
||||
entries.set(key, JSON.stringify({ ...JSON.parse(stored!), expiresAt: 'not-a-date' }));
|
||||
|
||||
expect(await service.consumeRuntimeTerminationApproval(approval!.approvalId, action)).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('persists and consumes one exact runtime termination approval across a service restart', async (): Promise<void> => {
|
||||
const entries = new Map<string, string>();
|
||||
const action = {
|
||||
providerId: 'fleet',
|
||||
sessionId: 'nova',
|
||||
actorId: 'admin-1',
|
||||
tenantId: 'tenant-1',
|
||||
channelId: 'discord:operator',
|
||||
correlationId: 'correlation-1',
|
||||
};
|
||||
const beforeRestart = createService('admin', entries);
|
||||
const approval = await beforeRestart.createRuntimeTerminationApproval(action);
|
||||
|
||||
const afterRestart = createService('admin', entries);
|
||||
expect(
|
||||
await afterRestart.consumeRuntimeTerminationApproval(approval!.approvalId, {
|
||||
...action,
|
||||
sessionId: 'forged-session',
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(await afterRestart.consumeRuntimeTerminationApproval(approval!.approvalId, action)).toBe(
|
||||
true,
|
||||
);
|
||||
expect(await afterRestart.consumeRuntimeTerminationApproval(approval!.approvalId, action)).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,6 +15,22 @@ export interface CommandApproval {
|
||||
expiresAt: string;
|
||||
}
|
||||
|
||||
/** Exact immutable binding for a privileged runtime termination. */
|
||||
export interface RuntimeTerminationApprovalAction {
|
||||
providerId: string;
|
||||
sessionId: string;
|
||||
actorId: string;
|
||||
tenantId: string;
|
||||
channelId: string;
|
||||
correlationId: string;
|
||||
}
|
||||
|
||||
export interface RuntimeTerminationApproval extends RuntimeTerminationApprovalAction {
|
||||
approvalId: string;
|
||||
actionDigest: string;
|
||||
expiresAt: string;
|
||||
}
|
||||
|
||||
export interface CommandAuthorizationResult {
|
||||
allowed: boolean;
|
||||
reason?: string;
|
||||
@@ -75,6 +91,52 @@ export class CommandAuthorizationService {
|
||||
return approval;
|
||||
}
|
||||
|
||||
/**
|
||||
* Uses the same `tess:command-approval:*` store and one-time deletion rule as
|
||||
* command approvals. This deliberately avoids a parallel approval database.
|
||||
*/
|
||||
async createRuntimeTerminationApproval(
|
||||
action: RuntimeTerminationApprovalAction,
|
||||
): Promise<RuntimeTerminationApproval | null> {
|
||||
if (!this.hasRuntimeTerminationAction(action)) return null;
|
||||
const role = await this.resolveRole(action.actorId);
|
||||
if (role !== 'admin') return null;
|
||||
|
||||
const approval: RuntimeTerminationApproval = {
|
||||
approvalId: randomUUID(),
|
||||
actionDigest: this.runtimeActionDigest(action),
|
||||
...action,
|
||||
expiresAt: new Date(Date.now() + 5 * 60_000).toISOString(),
|
||||
};
|
||||
await this.redis.set(this.key(approval.approvalId), JSON.stringify(approval), 'EX', '300');
|
||||
return approval;
|
||||
}
|
||||
|
||||
async consumeRuntimeTerminationApproval(
|
||||
approvalId: string,
|
||||
action: RuntimeTerminationApprovalAction,
|
||||
): Promise<boolean> {
|
||||
const encoded = await this.redis.get(this.key(approvalId));
|
||||
if (!encoded) return false;
|
||||
let approval: unknown;
|
||||
try {
|
||||
approval = JSON.parse(encoded);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
!this.isRuntimeTerminationApproval(approval) ||
|
||||
approval.actionDigest !== this.runtimeActionDigest(action) ||
|
||||
approval.actorId !== action.actorId ||
|
||||
approval.tenantId !== action.tenantId ||
|
||||
!this.isUnexpired(approval.expiresAt)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if ((await this.resolveRole(approval.actorId)) !== 'admin') return false;
|
||||
return (await this.redis.del(this.key(approvalId))) === 1;
|
||||
}
|
||||
|
||||
private async resolveRole(actorId: string): Promise<CommandRole | null> {
|
||||
const [user] = await this.db
|
||||
.select({ role: usersTable.role })
|
||||
@@ -98,12 +160,17 @@ export class CommandAuthorizationService {
|
||||
const key = this.key(approvalId);
|
||||
const encoded = await this.redis.get(key);
|
||||
if (!encoded) return false;
|
||||
const parsed: unknown = JSON.parse(encoded);
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(encoded);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
!this.isApproval(parsed) ||
|
||||
!this.isCommandApproval(parsed) ||
|
||||
parsed.actorId !== actorId ||
|
||||
parsed.actionDigest !== actionDigest ||
|
||||
Date.parse(parsed.expiresAt) <= Date.now()
|
||||
!this.isUnexpired(parsed.expiresAt)
|
||||
)
|
||||
return false;
|
||||
return (await this.redis.del(key)) === 1;
|
||||
@@ -121,13 +188,62 @@ export class CommandAuthorizationService {
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
private isApproval(value: unknown): value is CommandApproval {
|
||||
private hasRuntimeTerminationAction(action: RuntimeTerminationApprovalAction): boolean {
|
||||
return [
|
||||
action.providerId,
|
||||
action.sessionId,
|
||||
action.actorId,
|
||||
action.tenantId,
|
||||
action.channelId,
|
||||
action.correlationId,
|
||||
].every((value: string): boolean => value.trim().length > 0);
|
||||
}
|
||||
|
||||
private runtimeActionDigest(action: RuntimeTerminationApprovalAction): string {
|
||||
return createHash('sha256')
|
||||
.update(
|
||||
JSON.stringify({
|
||||
providerId: action.providerId,
|
||||
sessionId: action.sessionId,
|
||||
actorId: action.actorId,
|
||||
tenantId: action.tenantId,
|
||||
channelId: action.channelId,
|
||||
correlationId: action.correlationId,
|
||||
}),
|
||||
)
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
private isUnexpired(expiresAt: unknown): expiresAt is string {
|
||||
if (typeof expiresAt !== 'string') return false;
|
||||
const expiresAtMs = Date.parse(expiresAt);
|
||||
return Number.isFinite(expiresAtMs) && expiresAtMs > Date.now();
|
||||
}
|
||||
|
||||
private isCommandApproval(value: unknown): value is CommandApproval {
|
||||
return (
|
||||
typeof value === 'object' &&
|
||||
value !== null &&
|
||||
'approvalId' in value &&
|
||||
'actionDigest' in value &&
|
||||
'actorId' in value &&
|
||||
'expiresAt' in value &&
|
||||
'command' in value
|
||||
);
|
||||
}
|
||||
|
||||
private isRuntimeTerminationApproval(value: unknown): value is RuntimeTerminationApproval {
|
||||
return (
|
||||
typeof value === 'object' &&
|
||||
value !== null &&
|
||||
'approvalId' in value &&
|
||||
'actionDigest' in value &&
|
||||
'actorId' in value &&
|
||||
'tenantId' in value &&
|
||||
'providerId' in value &&
|
||||
'sessionId' in value &&
|
||||
'channelId' in value &&
|
||||
'correlationId' in value &&
|
||||
'expiresAt' in value
|
||||
);
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import { ReloadModule } from '../reload/reload.module.js';
|
||||
import { CommandAuthorizationService } from './command-authorization.service.js';
|
||||
import { CommandExecutorService } from './command-executor.service.js';
|
||||
import { CommandRegistryService } from './command-registry.service.js';
|
||||
import { CommandRuntimeApprovalVerifier } from './runtime-approval-verifier.js';
|
||||
import { COMMANDS_REDIS } from './commands.tokens.js';
|
||||
|
||||
const COMMANDS_QUEUE_HANDLE = 'COMMANDS_QUEUE_HANDLE';
|
||||
@@ -26,9 +27,15 @@ const COMMANDS_QUEUE_HANDLE = 'COMMANDS_QUEUE_HANDLE';
|
||||
},
|
||||
CommandRegistryService,
|
||||
CommandAuthorizationService,
|
||||
CommandRuntimeApprovalVerifier,
|
||||
CommandExecutorService,
|
||||
],
|
||||
exports: [
|
||||
CommandRegistryService,
|
||||
CommandAuthorizationService,
|
||||
CommandRuntimeApprovalVerifier,
|
||||
CommandExecutorService,
|
||||
],
|
||||
exports: [CommandRegistryService, CommandExecutorService],
|
||||
})
|
||||
export class CommandsModule implements OnApplicationShutdown {
|
||||
constructor(@Inject(COMMANDS_QUEUE_HANDLE) private readonly handle: QueueHandle) {}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import type {
|
||||
RuntimeApprovalVerifier,
|
||||
RuntimeTerminationAction,
|
||||
} from '../agent/runtime-provider-registry.service.js';
|
||||
import { CommandAuthorizationService } from './command-authorization.service.js';
|
||||
|
||||
/**
|
||||
* Adapter from the provider registry's exact termination action to the shared,
|
||||
* Redis-backed `tess:command-approval:*` store. It has no separate approval
|
||||
* persistence or replay semantics.
|
||||
*/
|
||||
@Injectable()
|
||||
export class CommandRuntimeApprovalVerifier implements RuntimeApprovalVerifier {
|
||||
constructor(
|
||||
@Inject(CommandAuthorizationService)
|
||||
private readonly authorization: CommandAuthorizationService,
|
||||
) {}
|
||||
|
||||
async consume(approvalRef: string, action: RuntimeTerminationAction): Promise<boolean> {
|
||||
return this.authorization.consumeRuntimeTerminationApproval(approvalRef, action);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
# TESS-M2-002 — Durable Tess State
|
||||
|
||||
- **Issue:** #708
|
||||
- **Task:** `TESS-M2-002` / `TESS-STA-001`, `TESS-SEC-007..008`
|
||||
- **Branch:** `feat/tess-durable-state`
|
||||
- **Base:** fresh `origin/main` at `e3b5113be21e51d015fa1ae54572929b2a4acd9f`
|
||||
- **Budget assumption:** 38K task estimate; no explicit cap. Use focused TDD plus workspace validation.
|
||||
|
||||
## Objective
|
||||
|
||||
Persist a Tess session's immutable identity, inbox/outbox idempotency state, checkpoints,
|
||||
handoffs, and approval bindings so a new service instance can recover it after a process
|
||||
restart or context compaction without replaying a completed message or applied side effect.
|
||||
|
||||
## Plan
|
||||
|
||||
1. Write recovery/idempotency tests first in `packages/agent/src/tess-durable-session.test.ts`.
|
||||
2. Add transport-neutral durable-state contracts/state machine in `packages/agent`.
|
||||
3. Add canonical PostgreSQL schema/migration and a gateway Drizzle repository adapter.
|
||||
4. Wire gateway service/module and reuse `tess:command-approval:*` durable approval semantics
|
||||
for exact, actor/tenant/action-bound approval consumption.
|
||||
5. Test PGlite restart recovery with separate service instances sharing the same durable DB.
|
||||
6. Document the recovery/compaction operation and update Tess architecture evidence.
|
||||
7. Run focused, cold-cache, workspace, migration, review, commit, push, and open PR to `main`.
|
||||
|
||||
## Required Evidence
|
||||
|
||||
| Requirement | Primary evidence |
|
||||
| --- | --- |
|
||||
| Restart recovery | Test creates a second coordinator over unchanged durable store after simulated process death. |
|
||||
| No duplicate side effects | Duplicate ingress and post-restart dispatch assert one handler/effect invocation. |
|
||||
| Compaction survival | Checkpoint/handoff/reconstructed state preserve the same session identity and pending records. |
|
||||
| Durable approvals | Existing `tess:command-approval` record is consumed only once and survives a new authorization service instance. |
|
||||
| Handoff | Stored handoff is portable and reconstructed without live process state. |
|
||||
|
||||
## Progress
|
||||
|
||||
- Intake complete: PRD AC-TESS-06, threat TM-07/TM-08, and verification matrix reviewed.
|
||||
- Affected surfaces: `packages/agent`, `apps/gateway`, `packages/db`; auth/authorization and DB migration tests required.
|
||||
- TDD is required (security authorization and critical state mutation).
|
||||
|
||||
## Risks
|
||||
|
||||
- An external provider action cannot be atomically committed with the database. The outbox
|
||||
gives the receiver a stable idempotency key; generic recovery never replays an ambiguous
|
||||
`processing` effect, and completed effects are never redispatched.
|
||||
- PostgreSQL is canonical; PGlite is the local/restart test implementation.
|
||||
|
||||
## Verification
|
||||
|
||||
- TDD red: `pnpm --filter @mosaicstack/agent test src/tess-durable-session.test.ts`
|
||||
initially failed because the durable-state module did not exist.
|
||||
- Focused green: 7 agent state-machine tests; 6 PGlite repository tests (including
|
||||
close/reopen recovery and encrypted-at-rest redaction); 5 durable-approval tests; DB migration tests.
|
||||
- Full cold-cache green: `pnpm turbo run typecheck lint test --force` completed
|
||||
88 tasks with zero cache hits; `pnpm format:check` and `git diff --check` passed.
|
||||
- Fresh worktree dependency install passed with
|
||||
`pnpm install --frozen-lockfile --store-dir /home/jarvis/.local/share/pnpm/store`.
|
||||
The default pnpm store path was inaccessible to this harness, so the explicit
|
||||
user-owned store path was required.
|
||||
- Codex review identified plaintext durable payload risk; resolved by AES-256-GCM sealing
|
||||
after redaction, with an at-rest ciphertext assertion in the PGlite suite.
|
||||
- Pending final clean review, commit, and PR.
|
||||
@@ -41,7 +41,7 @@ Every call receives an immutable, server-derived actor/tenant/channel scope and
|
||||
|
||||
`@mosaicstack/agent` owns the explicit `AgentRuntimeProviderRegistry`; duplicate provider IDs are rejected rather than replaced. Gateway owns `RuntimeProviderService`, which creates a frozen `RuntimeScope` from authenticated `ActorTenantScope` and trusted ingress channel/correlation metadata before every provider call. The service checks the declared provider capability before invoking a side effect and records metadata-only audit events (`providerId`, operation, outcome, actor/tenant/channel, correlation, and resource ID). It never records message bodies, idempotency keys, or approval references.
|
||||
|
||||
Termination is fail-closed: a runtime approval verifier must consume a one-time, exact action binding for the provider, session, actor, tenant, channel, and correlation ID before `terminate` reaches a provider. Until the durable verifier is wired, the default verifier denies termination. This internal service introduces no HTTP endpoint; later Discord, CLI, MCP, and provider adapters consume the same gateway boundary.
|
||||
Termination is fail-closed: a runtime approval verifier consumes a one-time, exact action binding for the provider, session, actor, tenant, channel, and correlation ID before `terminate` reaches a provider. The verifier reuses the Redis-backed `tess:command-approval:*` store and its expiry/delete-on-consume semantics; it has no parallel approval store. This internal service introduces no HTTP endpoint; later Discord, CLI, MCP, and provider adapters consume the same gateway boundary.
|
||||
|
||||
## Authority Model
|
||||
|
||||
@@ -56,7 +56,30 @@ Termination is fail-closed: a runtime approval verifier must consume a one-time,
|
||||
|
||||
A Tess session has stable `sessionId`, `tenantId`, `ownerId`, provider/runtime identity, ingress bindings, cursor, checkpoint, inbox/outbox, and idempotency records. Discord and CLI bind to the same authorized session. Ownership is verified server-side on every list/read/attach/send/terminate operation.
|
||||
|
||||
Valkey may hold ephemeral coordination state; PostgreSQL is canonical for durable session bindings, approvals, audit, checkpoints, inbox/outbox, and idempotency. Pi session files are replay sources, not cross-agent truth.
|
||||
Valkey holds the existing short-lived, one-time command-approval records; PostgreSQL is canonical for durable session bindings, checkpoints, inbox/outbox, and idempotency. Pi session files are replay sources, not cross-agent truth.
|
||||
|
||||
### M2 Durable Recovery
|
||||
|
||||
`@mosaicstack/agent` owns a transport-neutral state machine and `apps/gateway` provides its
|
||||
PostgreSQL adapter. `tess_sessions` holds immutable identity; inbox/outbox records use a
|
||||
per-session unique idempotency key and transition `pending → processing → processed|delivered`.
|
||||
Checkpoints are immutable history scoped by session and checkpoint ID: the latest checkpoint
|
||||
supports compaction recovery, while a handoff always resolves the exact checkpoint it references.
|
||||
Recovery requeues only interrupted inbox work; an ambiguous `processing` outbox record is preserved
|
||||
until separately authorized reconciliation can establish its external delivery state.
|
||||
|
||||
Provider sends travel through the existing `RuntimeProviderService` with the persisted outbox
|
||||
idempotency key. A normal dispatch claims exactly one outbox record and verifies its stored
|
||||
correlation and channel against the server-derived request scope; it never requeues or drains
|
||||
another live record. Inbox/outbox payloads and checkpoint cursor/summary pass through the existing
|
||||
secret/PII redactor and AES-256-GCM sealing before persistence; decryption occurs only in the
|
||||
scoped gateway repository path, and runtime audit remains metadata-only.
|
||||
|
||||
An external effect cannot share a database transaction. If a process dies after an effect begins
|
||||
but before its terminal outbox transition, automatic recovery does not replay that ambiguous claim.
|
||||
It remains `processing` until separately authorized reconciliation can establish delivery state;
|
||||
completed effects are never redispatched. Operators can therefore restart the gateway/Pi service,
|
||||
reconstruct the session, and resume pending inbox work without relying on process-local state.
|
||||
|
||||
## Transport Strategy
|
||||
|
||||
|
||||
@@ -2,3 +2,4 @@ export const VERSION = '0.0.0';
|
||||
|
||||
export * from './runtime-provider-registry.js';
|
||||
export * from './tmux-fleet-runtime-provider.js';
|
||||
export * from './tess-durable-session.js';
|
||||
|
||||
@@ -0,0 +1,287 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
DurableSessionCoordinator,
|
||||
InMemoryDurableSessionStore,
|
||||
type DurableSessionIdentity,
|
||||
} from './tess-durable-session.js';
|
||||
|
||||
const IDENTITY: DurableSessionIdentity = {
|
||||
sessionId: 'tess-session-1',
|
||||
tenantId: 'tenant-1',
|
||||
ownerId: 'owner-1',
|
||||
providerId: 'fleet',
|
||||
runtimeSessionId: 'nova',
|
||||
};
|
||||
|
||||
describe('DurableSessionCoordinator', () => {
|
||||
it('reconstructs an exact session identity, pending inbox/outbox, checkpoint, and handoff after a simulated process restart', async () => {
|
||||
const store = new InMemoryDurableSessionStore();
|
||||
const beforeRestart = new DurableSessionCoordinator(store);
|
||||
|
||||
await beforeRestart.create(IDENTITY);
|
||||
await beforeRestart.receive({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'ingress-1',
|
||||
correlationId: 'correlation-1',
|
||||
content: 'continue the session',
|
||||
});
|
||||
await beforeRestart.enqueueOutbox({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'outbox-1',
|
||||
correlationId: 'correlation-1',
|
||||
channelId: 'cli',
|
||||
kind: 'provider.send',
|
||||
content: 'resumable response',
|
||||
});
|
||||
await beforeRestart.checkpoint({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
checkpointId: 'checkpoint-1',
|
||||
cursor: 'cursor-42',
|
||||
summary: 'operator asked for recovery proof',
|
||||
compactionEpoch: 0,
|
||||
});
|
||||
await beforeRestart.handoff({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
handoffId: 'handoff-1',
|
||||
destination: 'mos',
|
||||
correlationId: 'correlation-1',
|
||||
checkpointId: 'checkpoint-1',
|
||||
status: 'pending',
|
||||
});
|
||||
|
||||
// Simulate an ungraceful process death: no in-memory coordinator state survives.
|
||||
const afterRestart = new DurableSessionCoordinator(store);
|
||||
const recovered = await afterRestart.recover(IDENTITY.sessionId);
|
||||
|
||||
expect(recovered.identity).toEqual(IDENTITY);
|
||||
expect(recovered.inbox).toMatchObject([{ idempotencyKey: 'ingress-1', status: 'pending' }]);
|
||||
expect(recovered.outbox).toMatchObject([{ idempotencyKey: 'outbox-1', status: 'pending' }]);
|
||||
expect(recovered.checkpoint).toMatchObject({
|
||||
checkpointId: 'checkpoint-1',
|
||||
cursor: 'cursor-42',
|
||||
});
|
||||
expect(recovered.handoffs).toMatchObject([{ handoffId: 'handoff-1', status: 'pending' }]);
|
||||
});
|
||||
|
||||
it('deduplicates duplicate ingress and never reprocesses an inbox record after restart or compaction', async () => {
|
||||
const store = new InMemoryDurableSessionStore();
|
||||
const firstProcess = new DurableSessionCoordinator(store);
|
||||
const handled: string[] = [];
|
||||
|
||||
await firstProcess.create(IDENTITY);
|
||||
await expect(
|
||||
firstProcess.receive({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'ingress-duplicate',
|
||||
correlationId: 'correlation-2',
|
||||
content: 'only process me once',
|
||||
}),
|
||||
).resolves.toMatchObject({ accepted: true });
|
||||
await expect(
|
||||
firstProcess.receive({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'ingress-duplicate',
|
||||
correlationId: 'correlation-2',
|
||||
content: 'only process me once',
|
||||
}),
|
||||
).resolves.toMatchObject({ accepted: false, status: 'pending' });
|
||||
await expect(
|
||||
firstProcess.receive({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'ingress-duplicate',
|
||||
correlationId: 'forged-correlation',
|
||||
content: 'only process me once',
|
||||
}),
|
||||
).rejects.toThrow(/idempotency conflict/);
|
||||
|
||||
await firstProcess.drainInbox(IDENTITY.sessionId, async (entry) => {
|
||||
handled.push(entry.idempotencyKey);
|
||||
});
|
||||
await firstProcess.checkpoint({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
checkpointId: 'checkpoint-after-inbox',
|
||||
cursor: 'cursor-43',
|
||||
summary: 'safe to compact',
|
||||
compactionEpoch: 1,
|
||||
});
|
||||
|
||||
const afterRestartAndCompaction = new DurableSessionCoordinator(store);
|
||||
await afterRestartAndCompaction.recover(IDENTITY.sessionId);
|
||||
await afterRestartAndCompaction.drainInbox(IDENTITY.sessionId, async (entry) => {
|
||||
handled.push(entry.idempotencyKey);
|
||||
});
|
||||
|
||||
expect(handled).toEqual(['ingress-duplicate']);
|
||||
});
|
||||
|
||||
it('does not redispatch an already applied outbox side effect after replay, restart, or compaction', async () => {
|
||||
const store = new InMemoryDurableSessionStore();
|
||||
const beforeRestart = new DurableSessionCoordinator(store);
|
||||
const appliedEffects: string[] = [];
|
||||
|
||||
await beforeRestart.create(IDENTITY);
|
||||
await beforeRestart.enqueueOutbox({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'effect-1',
|
||||
correlationId: 'correlation-3',
|
||||
channelId: 'cli',
|
||||
kind: 'provider.send',
|
||||
content: 'send exactly once',
|
||||
});
|
||||
await expect(
|
||||
beforeRestart.enqueueOutbox({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'effect-1',
|
||||
correlationId: 'correlation-3',
|
||||
channelId: 'cli',
|
||||
kind: 'provider.send',
|
||||
content: 'send exactly once',
|
||||
}),
|
||||
).resolves.toMatchObject({ accepted: false, status: 'pending' });
|
||||
|
||||
await beforeRestart.dispatchOutbox(IDENTITY.sessionId, async (entry) => {
|
||||
appliedEffects.push(entry.idempotencyKey);
|
||||
});
|
||||
await beforeRestart.checkpoint({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
checkpointId: 'checkpoint-after-effect',
|
||||
cursor: 'cursor-44',
|
||||
summary: 'effect persisted before compaction',
|
||||
compactionEpoch: 1,
|
||||
});
|
||||
|
||||
const afterRestartAndCompaction = new DurableSessionCoordinator(store);
|
||||
await afterRestartAndCompaction.recover(IDENTITY.sessionId);
|
||||
await afterRestartAndCompaction.dispatchOutbox(IDENTITY.sessionId, async (entry) => {
|
||||
appliedEffects.push(entry.idempotencyKey);
|
||||
});
|
||||
|
||||
expect(appliedEffects).toEqual(['effect-1']);
|
||||
});
|
||||
|
||||
it('rejects outbox idempotency-key reuse when immutable effect data differs', async () => {
|
||||
const store = new InMemoryDurableSessionStore();
|
||||
const coordinator = new DurableSessionCoordinator(store);
|
||||
|
||||
await coordinator.create(IDENTITY);
|
||||
await coordinator.enqueueOutbox({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'outbox-conflict',
|
||||
correlationId: 'correlation-outbox',
|
||||
channelId: 'cli',
|
||||
kind: 'provider.send',
|
||||
content: 'original effect',
|
||||
});
|
||||
await expect(
|
||||
coordinator.enqueueOutbox({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'outbox-conflict',
|
||||
correlationId: 'correlation-outbox',
|
||||
channelId: 'forged-channel',
|
||||
kind: 'provider.send',
|
||||
content: 'original effect',
|
||||
}),
|
||||
).rejects.toThrow(/idempotency conflict/);
|
||||
});
|
||||
|
||||
it('retains an ambiguous failed provider effect as processing until explicit recovery', async () => {
|
||||
const store = new InMemoryDurableSessionStore();
|
||||
const beforeRestart = new DurableSessionCoordinator(store);
|
||||
|
||||
await beforeRestart.create(IDENTITY);
|
||||
await beforeRestart.enqueueOutbox({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
idempotencyKey: 'ambiguous-effect',
|
||||
correlationId: 'correlation-ambiguous',
|
||||
channelId: 'cli',
|
||||
kind: 'provider.send',
|
||||
content: 'preserve this effect claim',
|
||||
});
|
||||
await expect(
|
||||
beforeRestart.dispatchOutbox(IDENTITY.sessionId, async (): Promise<void> => {
|
||||
throw new Error('provider connection dropped after submit');
|
||||
}),
|
||||
).rejects.toThrow(/connection dropped/);
|
||||
|
||||
const afterRestart = new DurableSessionCoordinator(store);
|
||||
await afterRestart.recover(IDENTITY.sessionId);
|
||||
const calls: string[] = [];
|
||||
await afterRestart.dispatchOutbox(IDENTITY.sessionId, async (entry): Promise<void> => {
|
||||
calls.push(entry.idempotencyKey);
|
||||
});
|
||||
|
||||
expect(calls).toEqual([]);
|
||||
expect(await afterRestart.snapshot(IDENTITY.sessionId)).toMatchObject({
|
||||
outbox: [{ idempotencyKey: 'ambiguous-effect', status: 'processing' }],
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects a handoff-id replay with different immutable state', async () => {
|
||||
const store = new InMemoryDurableSessionStore();
|
||||
const coordinator = new DurableSessionCoordinator(store);
|
||||
|
||||
await coordinator.create(IDENTITY);
|
||||
await coordinator.checkpoint({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
checkpointId: 'checkpoint-conflict',
|
||||
cursor: 'cursor-conflict',
|
||||
summary: 'handoff conflict proof',
|
||||
compactionEpoch: 0,
|
||||
});
|
||||
await coordinator.handoff({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
handoffId: 'handoff-conflict',
|
||||
destination: 'mos',
|
||||
correlationId: 'correlation-conflict',
|
||||
checkpointId: 'checkpoint-conflict',
|
||||
status: 'pending',
|
||||
});
|
||||
|
||||
await expect(
|
||||
coordinator.handoff({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
handoffId: 'handoff-conflict',
|
||||
destination: 'forged-destination',
|
||||
correlationId: 'correlation-conflict',
|
||||
checkpointId: 'checkpoint-conflict',
|
||||
status: 'pending',
|
||||
}),
|
||||
).rejects.toThrow(/identity conflict/);
|
||||
});
|
||||
|
||||
it('keeps a handoff portable and resumes it from its durable checkpoint without process-local references', async () => {
|
||||
const store = new InMemoryDurableSessionStore();
|
||||
const source = new DurableSessionCoordinator(store);
|
||||
|
||||
await source.create(IDENTITY);
|
||||
await source.checkpoint({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
checkpointId: 'checkpoint-handoff',
|
||||
cursor: 'cursor-45',
|
||||
summary: 'portable state',
|
||||
compactionEpoch: 2,
|
||||
});
|
||||
await source.handoff({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
handoffId: 'handoff-portable',
|
||||
destination: 'mos',
|
||||
correlationId: 'correlation-4',
|
||||
checkpointId: 'checkpoint-handoff',
|
||||
status: 'pending',
|
||||
});
|
||||
await source.checkpoint({
|
||||
sessionId: IDENTITY.sessionId,
|
||||
checkpointId: 'checkpoint-later',
|
||||
cursor: 'cursor-46',
|
||||
summary: 'newer compacted state must not strand the handoff',
|
||||
compactionEpoch: 3,
|
||||
});
|
||||
|
||||
const receivingProcess = new DurableSessionCoordinator(store);
|
||||
const handoff = await receivingProcess.resumeHandoff('handoff-portable');
|
||||
|
||||
expect(handoff.identity).toEqual(IDENTITY);
|
||||
expect(handoff.checkpoint).toMatchObject({ checkpointId: 'checkpoint-handoff' });
|
||||
expect(handoff.handoff).toMatchObject({ handoffId: 'handoff-portable', destination: 'mos' });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,489 @@
|
||||
export interface DurableSessionIdentity {
|
||||
sessionId: string;
|
||||
tenantId: string;
|
||||
ownerId: string;
|
||||
providerId: string;
|
||||
runtimeSessionId: string;
|
||||
}
|
||||
|
||||
export type DurableInboxStatus = 'pending' | 'processing' | 'processed';
|
||||
export type DurableOutboxStatus = 'pending' | 'processing' | 'delivered';
|
||||
|
||||
export interface DurableInboxInput {
|
||||
sessionId: string;
|
||||
idempotencyKey: string;
|
||||
correlationId: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface DurableInboxEntry extends DurableInboxInput {
|
||||
status: DurableInboxStatus;
|
||||
}
|
||||
|
||||
export interface DurableOutboxInput {
|
||||
sessionId: string;
|
||||
idempotencyKey: string;
|
||||
correlationId: string;
|
||||
channelId: string;
|
||||
kind: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface DurableOutboxEntry extends DurableOutboxInput {
|
||||
status: DurableOutboxStatus;
|
||||
}
|
||||
|
||||
export interface DurableCheckpointInput {
|
||||
sessionId: string;
|
||||
checkpointId: string;
|
||||
cursor: string;
|
||||
summary: string;
|
||||
compactionEpoch: number;
|
||||
}
|
||||
|
||||
export interface DurableCheckpoint extends DurableCheckpointInput {}
|
||||
|
||||
export interface DurableHandoffInput {
|
||||
sessionId: string;
|
||||
handoffId: string;
|
||||
destination: string;
|
||||
correlationId: string;
|
||||
checkpointId: string;
|
||||
status: 'pending' | 'accepted';
|
||||
}
|
||||
|
||||
export interface DurableHandoff extends DurableHandoffInput {}
|
||||
|
||||
export interface DurableSessionSnapshot {
|
||||
identity: DurableSessionIdentity;
|
||||
inbox: DurableInboxEntry[];
|
||||
outbox: DurableOutboxEntry[];
|
||||
checkpoint?: DurableCheckpoint;
|
||||
handoffs: DurableHandoff[];
|
||||
}
|
||||
|
||||
export interface DurableHandoffRecovery {
|
||||
identity: DurableSessionIdentity;
|
||||
checkpoint: DurableCheckpoint;
|
||||
handoff: DurableHandoff;
|
||||
}
|
||||
|
||||
export interface DurableEnqueueResult<TStatus extends string> {
|
||||
accepted: boolean;
|
||||
status: TStatus;
|
||||
}
|
||||
|
||||
/**
|
||||
* A durable-state port. Implementations must atomically claim and complete work
|
||||
* records. Recovery may requeue interrupted inbox work, but never an
|
||||
* externally visible outbox effect: an ambiguous provider result remains
|
||||
* claimed until a separately authorized reconciliation proves it safe.
|
||||
*/
|
||||
export interface DurableSessionStore {
|
||||
create(identity: DurableSessionIdentity): Promise<void>;
|
||||
snapshot(sessionId: string): Promise<DurableSessionSnapshot | null>;
|
||||
enqueueInbox(input: DurableInboxInput): Promise<DurableEnqueueResult<DurableInboxStatus>>;
|
||||
claimInbox(sessionId: string): Promise<DurableInboxEntry | null>;
|
||||
completeInbox(sessionId: string, idempotencyKey: string): Promise<void>;
|
||||
releaseInbox(sessionId: string, idempotencyKey: string): Promise<void>;
|
||||
enqueueOutbox(input: DurableOutboxInput): Promise<DurableEnqueueResult<DurableOutboxStatus>>;
|
||||
claimOutbox(sessionId: string): Promise<DurableOutboxEntry | null>;
|
||||
claimOutboxByKey(sessionId: string, idempotencyKey: string): Promise<DurableOutboxEntry | null>;
|
||||
completeOutbox(sessionId: string, idempotencyKey: string): Promise<void>;
|
||||
releaseOutbox(sessionId: string, idempotencyKey: string): Promise<void>;
|
||||
checkpoint(input: DurableCheckpointInput): Promise<void>;
|
||||
findCheckpoint(sessionId: string, checkpointId: string): Promise<DurableCheckpoint | null>;
|
||||
handoff(input: DurableHandoffInput): Promise<void>;
|
||||
findHandoff(handoffId: string): Promise<DurableHandoff | null>;
|
||||
requeueInFlight(sessionId: string): Promise<void>;
|
||||
}
|
||||
|
||||
export class DurableSessionNotFoundError extends Error {
|
||||
constructor(sessionId: string) {
|
||||
super(`Durable Tess session not found: ${sessionId}`);
|
||||
this.name = 'DurableSessionNotFoundError';
|
||||
}
|
||||
}
|
||||
|
||||
export class DurableSessionCoordinator {
|
||||
constructor(private readonly store: DurableSessionStore) {}
|
||||
|
||||
async create(identity: DurableSessionIdentity): Promise<void> {
|
||||
this.assertIdentity(identity);
|
||||
await this.store.create(identity);
|
||||
}
|
||||
|
||||
async receive(input: DurableInboxInput): Promise<DurableEnqueueResult<DurableInboxStatus>> {
|
||||
this.assertRecord(input.sessionId, input.idempotencyKey, input.correlationId, input.content);
|
||||
return this.store.enqueueInbox(input);
|
||||
}
|
||||
|
||||
async enqueueOutbox(
|
||||
input: DurableOutboxInput,
|
||||
): Promise<DurableEnqueueResult<DurableOutboxStatus>> {
|
||||
this.assertRecord(
|
||||
input.sessionId,
|
||||
input.idempotencyKey,
|
||||
input.correlationId,
|
||||
input.channelId,
|
||||
input.content,
|
||||
);
|
||||
if (input.kind.trim().length === 0) throw new Error('Durable outbox kind is required');
|
||||
return this.store.enqueueOutbox(input);
|
||||
}
|
||||
|
||||
async checkpoint(input: DurableCheckpointInput): Promise<void> {
|
||||
this.assertRecord(input.sessionId, input.checkpointId, input.cursor, input.summary);
|
||||
if (!Number.isInteger(input.compactionEpoch) || input.compactionEpoch < 0) {
|
||||
throw new Error('Durable checkpoint compaction epoch must be a non-negative integer');
|
||||
}
|
||||
await this.store.checkpoint(input);
|
||||
}
|
||||
|
||||
async handoff(input: DurableHandoffInput): Promise<void> {
|
||||
this.assertRecord(input.sessionId, input.handoffId, input.destination, input.correlationId);
|
||||
if (input.checkpointId.trim().length === 0)
|
||||
throw new Error('Durable handoff checkpoint is required');
|
||||
await this.store.handoff(input);
|
||||
}
|
||||
|
||||
/** Read durable state without changing claim status; safe during normal operation. */
|
||||
async snapshot(sessionId: string): Promise<DurableSessionSnapshot> {
|
||||
const snapshot = await this.store.snapshot(sessionId);
|
||||
if (!snapshot) throw new DurableSessionNotFoundError(sessionId);
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
/** Requeue interrupted inbox work during recovery; preserve ambiguous outbox claims. */
|
||||
async recover(sessionId: string): Promise<DurableSessionSnapshot> {
|
||||
await this.store.requeueInFlight(sessionId);
|
||||
return this.snapshot(sessionId);
|
||||
}
|
||||
|
||||
async drainInbox(
|
||||
sessionId: string,
|
||||
handler: (entry: DurableInboxEntry) => Promise<void>,
|
||||
): Promise<void> {
|
||||
for (;;) {
|
||||
const entry = await this.store.claimInbox(sessionId);
|
||||
if (!entry) return;
|
||||
try {
|
||||
await handler(entry);
|
||||
} catch (error: unknown) {
|
||||
await this.store.releaseInbox(sessionId, entry.idempotencyKey);
|
||||
throw error;
|
||||
}
|
||||
// If this write fails after the handler succeeded, leave the record
|
||||
// processing. A recovery path can retry it with its stable idempotency key.
|
||||
await this.store.completeInbox(sessionId, entry.idempotencyKey);
|
||||
}
|
||||
}
|
||||
|
||||
async dispatchOutbox(
|
||||
sessionId: string,
|
||||
dispatcher: (entry: DurableOutboxEntry) => Promise<void>,
|
||||
): Promise<void> {
|
||||
for (;;) {
|
||||
const entry = await this.store.claimOutbox(sessionId);
|
||||
if (!entry) return;
|
||||
// A provider failure can be ambiguous: it may occur after the receiver
|
||||
// accepted the idempotency key. Preserve the claim for reconciliation.
|
||||
await dispatcher(entry);
|
||||
// Do not requeue an effect after it has been applied but before its
|
||||
// terminal state could be persisted; recovery preserves the claim.
|
||||
await this.store.completeOutbox(sessionId, entry.idempotencyKey);
|
||||
}
|
||||
}
|
||||
|
||||
async dispatchOutboxEntry(
|
||||
sessionId: string,
|
||||
idempotencyKey: string,
|
||||
dispatcher: (entry: DurableOutboxEntry) => Promise<void>,
|
||||
): Promise<void> {
|
||||
const entry = await this.store.claimOutboxByKey(sessionId, idempotencyKey);
|
||||
if (!entry) return;
|
||||
// A provider failure can be ambiguous, so this stays processing until
|
||||
// separately authorized reconciliation proves it safe to resolve.
|
||||
await dispatcher(entry);
|
||||
await this.store.completeOutbox(sessionId, entry.idempotencyKey);
|
||||
}
|
||||
|
||||
async resumeHandoff(handoffId: string): Promise<DurableHandoffRecovery> {
|
||||
const handoff = await this.store.findHandoff(handoffId);
|
||||
if (!handoff) throw new Error(`Durable Tess handoff not found: ${handoffId}`);
|
||||
const snapshot = await this.snapshot(handoff.sessionId);
|
||||
const checkpoint = await this.store.findCheckpoint(handoff.sessionId, handoff.checkpointId);
|
||||
if (!checkpoint) {
|
||||
throw new Error(`Durable Tess handoff checkpoint is unavailable: ${handoff.checkpointId}`);
|
||||
}
|
||||
return { identity: snapshot.identity, checkpoint, handoff };
|
||||
}
|
||||
|
||||
private assertIdentity(identity: DurableSessionIdentity): void {
|
||||
this.assertRecord(identity.sessionId, identity.tenantId, identity.ownerId, identity.providerId);
|
||||
if (identity.runtimeSessionId.trim().length === 0) {
|
||||
throw new Error('Durable runtime session identity is required');
|
||||
}
|
||||
}
|
||||
|
||||
private assertRecord(...values: string[]): void {
|
||||
if (values.some((value: string): boolean => value.trim().length === 0)) {
|
||||
throw new Error('Durable session records require non-empty fields');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface InMemorySessionState {
|
||||
identity: DurableSessionIdentity;
|
||||
inbox: Map<string, DurableInboxEntry>;
|
||||
outbox: Map<string, DurableOutboxEntry>;
|
||||
checkpoints: Map<string, DurableCheckpoint>;
|
||||
handoffs: Map<string, DurableHandoff>;
|
||||
}
|
||||
|
||||
/** Reference store for deterministic domain tests; production uses the gateway DB adapter. */
|
||||
export class InMemoryDurableSessionStore implements DurableSessionStore {
|
||||
private readonly sessions = new Map<string, InMemorySessionState>();
|
||||
|
||||
async create(identity: DurableSessionIdentity): Promise<void> {
|
||||
const existing = this.sessions.get(identity.sessionId);
|
||||
if (existing) {
|
||||
if (!identitiesEqual(existing.identity, identity)) {
|
||||
throw new Error(`Durable Tess session identity conflict: ${identity.sessionId}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
this.sessions.set(identity.sessionId, {
|
||||
identity: copyIdentity(identity),
|
||||
inbox: new Map(),
|
||||
outbox: new Map(),
|
||||
checkpoints: new Map(),
|
||||
handoffs: new Map(),
|
||||
});
|
||||
}
|
||||
|
||||
async snapshot(sessionId: string): Promise<DurableSessionSnapshot | null> {
|
||||
const state = this.sessions.get(sessionId);
|
||||
if (!state) return null;
|
||||
const checkpoint = latestCheckpoint(state.checkpoints);
|
||||
return {
|
||||
identity: copyIdentity(state.identity),
|
||||
inbox: [...state.inbox.values()].map(copyInbox),
|
||||
outbox: [...state.outbox.values()].map(copyOutbox),
|
||||
...(checkpoint ? { checkpoint: copyCheckpoint(checkpoint) } : {}),
|
||||
handoffs: [...state.handoffs.values()].map(copyHandoff),
|
||||
};
|
||||
}
|
||||
|
||||
async enqueueInbox(input: DurableInboxInput): Promise<DurableEnqueueResult<DurableInboxStatus>> {
|
||||
const state = this.require(input.sessionId);
|
||||
const existing = state.inbox.get(input.idempotencyKey);
|
||||
if (existing) {
|
||||
if (!sameInbox(existing, input)) {
|
||||
throw new Error(`Durable Tess inbox idempotency conflict: ${input.idempotencyKey}`);
|
||||
}
|
||||
return { accepted: false, status: existing.status };
|
||||
}
|
||||
state.inbox.set(input.idempotencyKey, { ...input, status: 'pending' });
|
||||
return { accepted: true, status: 'pending' };
|
||||
}
|
||||
|
||||
async claimInbox(sessionId: string): Promise<DurableInboxEntry | null> {
|
||||
const state = this.require(sessionId);
|
||||
const entry = [...state.inbox.values()].find(
|
||||
(candidate: DurableInboxEntry): boolean => candidate.status === 'pending',
|
||||
);
|
||||
if (!entry) return null;
|
||||
entry.status = 'processing';
|
||||
return copyInbox(entry);
|
||||
}
|
||||
|
||||
async completeInbox(sessionId: string, idempotencyKey: string): Promise<void> {
|
||||
this.requireEntry(this.require(sessionId).inbox, idempotencyKey, 'inbox').status = 'processed';
|
||||
}
|
||||
|
||||
async releaseInbox(sessionId: string, idempotencyKey: string): Promise<void> {
|
||||
this.requireEntry(this.require(sessionId).inbox, idempotencyKey, 'inbox').status = 'pending';
|
||||
}
|
||||
|
||||
async enqueueOutbox(
|
||||
input: DurableOutboxInput,
|
||||
): Promise<DurableEnqueueResult<DurableOutboxStatus>> {
|
||||
const state = this.require(input.sessionId);
|
||||
const existing = state.outbox.get(input.idempotencyKey);
|
||||
if (existing) {
|
||||
if (!sameOutbox(existing, input)) {
|
||||
throw new Error(`Durable Tess outbox idempotency conflict: ${input.idempotencyKey}`);
|
||||
}
|
||||
return { accepted: false, status: existing.status };
|
||||
}
|
||||
state.outbox.set(input.idempotencyKey, { ...input, status: 'pending' });
|
||||
return { accepted: true, status: 'pending' };
|
||||
}
|
||||
|
||||
async claimOutbox(sessionId: string): Promise<DurableOutboxEntry | null> {
|
||||
const state = this.require(sessionId);
|
||||
const entry = [...state.outbox.values()].find(
|
||||
(candidate: DurableOutboxEntry): boolean => candidate.status === 'pending',
|
||||
);
|
||||
if (!entry) return null;
|
||||
entry.status = 'processing';
|
||||
return copyOutbox(entry);
|
||||
}
|
||||
|
||||
async claimOutboxByKey(
|
||||
sessionId: string,
|
||||
idempotencyKey: string,
|
||||
): Promise<DurableOutboxEntry | null> {
|
||||
const entry = this.require(sessionId).outbox.get(idempotencyKey);
|
||||
if (!entry || entry.status !== 'pending') return null;
|
||||
entry.status = 'processing';
|
||||
return copyOutbox(entry);
|
||||
}
|
||||
|
||||
async completeOutbox(sessionId: string, idempotencyKey: string): Promise<void> {
|
||||
this.requireEntry(this.require(sessionId).outbox, idempotencyKey, 'outbox').status =
|
||||
'delivered';
|
||||
}
|
||||
|
||||
async releaseOutbox(sessionId: string, idempotencyKey: string): Promise<void> {
|
||||
this.requireEntry(this.require(sessionId).outbox, idempotencyKey, 'outbox').status = 'pending';
|
||||
}
|
||||
|
||||
async checkpoint(input: DurableCheckpointInput): Promise<void> {
|
||||
const checkpoints = this.require(input.sessionId).checkpoints;
|
||||
const existing = checkpoints.get(input.checkpointId);
|
||||
if (existing && !sameCheckpoint(existing, input)) {
|
||||
throw new Error(`Durable Tess checkpoint identity conflict: ${input.checkpointId}`);
|
||||
}
|
||||
if (!existing) checkpoints.set(input.checkpointId, { ...input });
|
||||
}
|
||||
|
||||
async findCheckpoint(sessionId: string, checkpointId: string): Promise<DurableCheckpoint | null> {
|
||||
const checkpoint = this.require(sessionId).checkpoints.get(checkpointId);
|
||||
return checkpoint ? copyCheckpoint(checkpoint) : null;
|
||||
}
|
||||
|
||||
async handoff(input: DurableHandoffInput): Promise<void> {
|
||||
const state = this.require(input.sessionId);
|
||||
if (!state.checkpoints.has(input.checkpointId)) {
|
||||
throw new Error(`Durable Tess handoff checkpoint is unavailable: ${input.checkpointId}`);
|
||||
}
|
||||
const existing = state.handoffs.get(input.handoffId);
|
||||
if (existing && !sameHandoff(existing, input)) {
|
||||
throw new Error(`Durable Tess handoff identity conflict: ${input.handoffId}`);
|
||||
}
|
||||
if (!existing) state.handoffs.set(input.handoffId, { ...input });
|
||||
}
|
||||
|
||||
async findHandoff(handoffId: string): Promise<DurableHandoff | null> {
|
||||
for (const state of this.sessions.values()) {
|
||||
const handoff = state.handoffs.get(handoffId);
|
||||
if (handoff) return copyHandoff(handoff);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async requeueInFlight(sessionId: string): Promise<void> {
|
||||
const state = this.require(sessionId);
|
||||
for (const entry of state.inbox.values()) {
|
||||
if (entry.status === 'processing') entry.status = 'pending';
|
||||
}
|
||||
}
|
||||
|
||||
private require(sessionId: string): InMemorySessionState {
|
||||
const state = this.sessions.get(sessionId);
|
||||
if (!state) throw new DurableSessionNotFoundError(sessionId);
|
||||
return state;
|
||||
}
|
||||
|
||||
private requireEntry<T extends { status: string }>(
|
||||
records: Map<string, T>,
|
||||
idempotencyKey: string,
|
||||
kind: string,
|
||||
): T {
|
||||
const entry = records.get(idempotencyKey);
|
||||
if (!entry) throw new Error(`Durable Tess ${kind} entry not found: ${idempotencyKey}`);
|
||||
return entry;
|
||||
}
|
||||
}
|
||||
|
||||
function identitiesEqual(left: DurableSessionIdentity, right: DurableSessionIdentity): boolean {
|
||||
return (
|
||||
left.sessionId === right.sessionId &&
|
||||
left.tenantId === right.tenantId &&
|
||||
left.ownerId === right.ownerId &&
|
||||
left.providerId === right.providerId &&
|
||||
left.runtimeSessionId === right.runtimeSessionId
|
||||
);
|
||||
}
|
||||
|
||||
function sameInbox(left: DurableInboxEntry, right: DurableInboxInput): boolean {
|
||||
return (
|
||||
left.sessionId === right.sessionId &&
|
||||
left.idempotencyKey === right.idempotencyKey &&
|
||||
left.correlationId === right.correlationId &&
|
||||
left.content === right.content
|
||||
);
|
||||
}
|
||||
|
||||
function sameOutbox(left: DurableOutboxEntry, right: DurableOutboxInput): boolean {
|
||||
return (
|
||||
left.sessionId === right.sessionId &&
|
||||
left.idempotencyKey === right.idempotencyKey &&
|
||||
left.correlationId === right.correlationId &&
|
||||
left.channelId === right.channelId &&
|
||||
left.kind === right.kind &&
|
||||
left.content === right.content
|
||||
);
|
||||
}
|
||||
|
||||
function sameCheckpoint(left: DurableCheckpoint, right: DurableCheckpointInput): boolean {
|
||||
return (
|
||||
left.sessionId === right.sessionId &&
|
||||
left.checkpointId === right.checkpointId &&
|
||||
left.cursor === right.cursor &&
|
||||
left.summary === right.summary &&
|
||||
left.compactionEpoch === right.compactionEpoch
|
||||
);
|
||||
}
|
||||
|
||||
function latestCheckpoint(
|
||||
checkpoints: Map<string, DurableCheckpoint>,
|
||||
): DurableCheckpoint | undefined {
|
||||
return [...checkpoints.values()].sort(
|
||||
(left: DurableCheckpoint, right: DurableCheckpoint): number =>
|
||||
right.compactionEpoch - left.compactionEpoch,
|
||||
)[0];
|
||||
}
|
||||
|
||||
function sameHandoff(left: DurableHandoff, right: DurableHandoffInput): boolean {
|
||||
return (
|
||||
left.sessionId === right.sessionId &&
|
||||
left.handoffId === right.handoffId &&
|
||||
left.destination === right.destination &&
|
||||
left.correlationId === right.correlationId &&
|
||||
left.checkpointId === right.checkpointId &&
|
||||
left.status === right.status
|
||||
);
|
||||
}
|
||||
|
||||
function copyIdentity(identity: DurableSessionIdentity): DurableSessionIdentity {
|
||||
return { ...identity };
|
||||
}
|
||||
|
||||
function copyInbox(entry: DurableInboxEntry): DurableInboxEntry {
|
||||
return { ...entry };
|
||||
}
|
||||
|
||||
function copyOutbox(entry: DurableOutboxEntry): DurableOutboxEntry {
|
||||
return { ...entry };
|
||||
}
|
||||
|
||||
function copyCheckpoint(checkpoint: DurableCheckpoint): DurableCheckpoint {
|
||||
return { ...checkpoint };
|
||||
}
|
||||
|
||||
function copyHandoff(handoff: DurableHandoff): DurableHandoff {
|
||||
return { ...handoff };
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
CREATE TYPE "public"."tess_handoff_status" AS ENUM('pending', 'accepted');--> statement-breakpoint
|
||||
CREATE TYPE "public"."tess_inbox_status" AS ENUM('pending', 'processing', 'processed');--> statement-breakpoint
|
||||
CREATE TYPE "public"."tess_outbox_status" AS ENUM('pending', 'processing', 'delivered');--> statement-breakpoint
|
||||
CREATE TABLE "tess_checkpoints" (
|
||||
"session_id" text PRIMARY KEY NOT NULL,
|
||||
"checkpoint_id" text NOT NULL,
|
||||
"cursor" text NOT NULL,
|
||||
"summary" text NOT NULL,
|
||||
"compaction_epoch" integer NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "tess_checkpoints_checkpoint_id_unique" UNIQUE("checkpoint_id")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "tess_handoffs" (
|
||||
"handoff_id" text PRIMARY KEY NOT NULL,
|
||||
"session_id" text NOT NULL,
|
||||
"destination" text NOT NULL,
|
||||
"correlation_id" text NOT NULL,
|
||||
"checkpoint_id" text NOT NULL,
|
||||
"status" "tess_handoff_status" DEFAULT 'pending' NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "tess_inbox" (
|
||||
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
|
||||
"session_id" text NOT NULL,
|
||||
"idempotency_key" text NOT NULL,
|
||||
"correlation_id" text NOT NULL,
|
||||
"content" text NOT NULL,
|
||||
"status" "tess_inbox_status" DEFAULT 'pending' NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "tess_outbox" (
|
||||
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
|
||||
"session_id" text NOT NULL,
|
||||
"idempotency_key" text NOT NULL,
|
||||
"correlation_id" text NOT NULL,
|
||||
"kind" text NOT NULL,
|
||||
"content" text NOT NULL,
|
||||
"status" "tess_outbox_status" DEFAULT 'pending' NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "tess_sessions" (
|
||||
"id" text PRIMARY KEY NOT NULL,
|
||||
"tenant_id" text NOT NULL,
|
||||
"owner_id" text NOT NULL,
|
||||
"provider_id" text NOT NULL,
|
||||
"runtime_session_id" text NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
ALTER TABLE "tess_checkpoints" ADD CONSTRAINT "tess_checkpoints_session_id_tess_sessions_id_fk" FOREIGN KEY ("session_id") REFERENCES "public"."tess_sessions"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "tess_handoffs" ADD CONSTRAINT "tess_handoffs_session_id_tess_sessions_id_fk" FOREIGN KEY ("session_id") REFERENCES "public"."tess_sessions"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "tess_inbox" ADD CONSTRAINT "tess_inbox_session_id_tess_sessions_id_fk" FOREIGN KEY ("session_id") REFERENCES "public"."tess_sessions"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "tess_outbox" ADD CONSTRAINT "tess_outbox_session_id_tess_sessions_id_fk" FOREIGN KEY ("session_id") REFERENCES "public"."tess_sessions"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "tess_sessions" ADD CONSTRAINT "tess_sessions_owner_id_users_id_fk" FOREIGN KEY ("owner_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
CREATE INDEX "tess_handoffs_session_status_idx" ON "tess_handoffs" USING btree ("session_id","status");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "tess_inbox_session_idempotency_idx" ON "tess_inbox" USING btree ("session_id","idempotency_key");--> statement-breakpoint
|
||||
CREATE INDEX "tess_inbox_session_status_created_idx" ON "tess_inbox" USING btree ("session_id","status","created_at");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "tess_outbox_session_idempotency_idx" ON "tess_outbox" USING btree ("session_id","idempotency_key");--> statement-breakpoint
|
||||
CREATE INDEX "tess_outbox_session_status_created_idx" ON "tess_outbox" USING btree ("session_id","status","created_at");
|
||||
@@ -0,0 +1,5 @@
|
||||
ALTER TABLE "tess_checkpoints" DROP CONSTRAINT "tess_checkpoints_checkpoint_id_unique";--> statement-breakpoint
|
||||
ALTER TABLE "tess_checkpoints" DROP CONSTRAINT "tess_checkpoints_pkey";--> statement-breakpoint
|
||||
ALTER TABLE "tess_checkpoints" ADD COLUMN "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL;--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "tess_checkpoints_session_idempotency_idx" ON "tess_checkpoints" USING btree ("session_id","checkpoint_id");--> statement-breakpoint
|
||||
CREATE INDEX "tess_checkpoints_session_epoch_idx" ON "tess_checkpoints" USING btree ("session_id","compaction_epoch");
|
||||
@@ -0,0 +1,5 @@
|
||||
ALTER TABLE "tess_outbox" ADD COLUMN "channel_id" text;
|
||||
--> statement-breakpoint
|
||||
UPDATE "tess_outbox" SET "channel_id" = 'legacy:unknown' WHERE "channel_id" IS NULL;
|
||||
--> statement-breakpoint
|
||||
ALTER TABLE "tess_outbox" ALTER COLUMN "channel_id" SET NOT NULL;
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -85,6 +85,27 @@
|
||||
"when": 1782310438919,
|
||||
"tag": "0011_bitter_gateway",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 12,
|
||||
"version": "7",
|
||||
"when": 1783911983447,
|
||||
"tag": "0012_tess_durable_state",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 13,
|
||||
"version": "7",
|
||||
"when": 1783913232578,
|
||||
"tag": "0013_tess_checkpoint_history",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 14,
|
||||
"version": "7",
|
||||
"when": 1783913398006,
|
||||
"tag": "0014_tess_outbox_channel_scope",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -487,6 +487,113 @@ export const agentLogs = pgTable(
|
||||
],
|
||||
);
|
||||
|
||||
// ─── Tess durable session state ─────────────────────────────────────────────
|
||||
// PostgreSQL is canonical for restart-safe Tess session recovery. The state
|
||||
// machine lives in @mosaicstack/agent; these records are its durable adapter.
|
||||
|
||||
export const tessInboxStatusEnum = pgEnum('tess_inbox_status', [
|
||||
'pending',
|
||||
'processing',
|
||||
'processed',
|
||||
]);
|
||||
export const tessOutboxStatusEnum = pgEnum('tess_outbox_status', [
|
||||
'pending',
|
||||
'processing',
|
||||
'delivered',
|
||||
]);
|
||||
export const tessHandoffStatusEnum = pgEnum('tess_handoff_status', ['pending', 'accepted']);
|
||||
|
||||
export const tessSessions = pgTable('tess_sessions', {
|
||||
id: text('id').primaryKey(),
|
||||
tenantId: text('tenant_id').notNull(),
|
||||
ownerId: text('owner_id')
|
||||
.notNull()
|
||||
.references(() => users.id, { onDelete: 'cascade' }),
|
||||
providerId: text('provider_id').notNull(),
|
||||
runtimeSessionId: text('runtime_session_id').notNull(),
|
||||
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
|
||||
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
|
||||
});
|
||||
|
||||
export const tessInbox = pgTable(
|
||||
'tess_inbox',
|
||||
{
|
||||
id: uuid('id').primaryKey().defaultRandom(),
|
||||
sessionId: text('session_id')
|
||||
.notNull()
|
||||
.references(() => tessSessions.id, { onDelete: 'cascade' }),
|
||||
idempotencyKey: text('idempotency_key').notNull(),
|
||||
correlationId: text('correlation_id').notNull(),
|
||||
content: text('content').notNull(),
|
||||
status: tessInboxStatusEnum('status').notNull().default('pending'),
|
||||
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
|
||||
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
|
||||
},
|
||||
(t) => [
|
||||
uniqueIndex('tess_inbox_session_idempotency_idx').on(t.sessionId, t.idempotencyKey),
|
||||
index('tess_inbox_session_status_created_idx').on(t.sessionId, t.status, t.createdAt),
|
||||
],
|
||||
);
|
||||
|
||||
export const tessOutbox = pgTable(
|
||||
'tess_outbox',
|
||||
{
|
||||
id: uuid('id').primaryKey().defaultRandom(),
|
||||
sessionId: text('session_id')
|
||||
.notNull()
|
||||
.references(() => tessSessions.id, { onDelete: 'cascade' }),
|
||||
idempotencyKey: text('idempotency_key').notNull(),
|
||||
correlationId: text('correlation_id').notNull(),
|
||||
channelId: text('channel_id').notNull(),
|
||||
kind: text('kind').notNull(),
|
||||
content: text('content').notNull(),
|
||||
status: tessOutboxStatusEnum('status').notNull().default('pending'),
|
||||
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
|
||||
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
|
||||
},
|
||||
(t) => [
|
||||
uniqueIndex('tess_outbox_session_idempotency_idx').on(t.sessionId, t.idempotencyKey),
|
||||
index('tess_outbox_session_status_created_idx').on(t.sessionId, t.status, t.createdAt),
|
||||
],
|
||||
);
|
||||
|
||||
export const tessCheckpoints = pgTable(
|
||||
'tess_checkpoints',
|
||||
{
|
||||
id: uuid('id').primaryKey().defaultRandom(),
|
||||
sessionId: text('session_id')
|
||||
.notNull()
|
||||
.references(() => tessSessions.id, { onDelete: 'cascade' }),
|
||||
checkpointId: text('checkpoint_id').notNull(),
|
||||
cursor: text('cursor').notNull(),
|
||||
summary: text('summary').notNull(),
|
||||
compactionEpoch: integer('compaction_epoch').notNull(),
|
||||
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
|
||||
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
|
||||
},
|
||||
(t) => [
|
||||
uniqueIndex('tess_checkpoints_session_idempotency_idx').on(t.sessionId, t.checkpointId),
|
||||
index('tess_checkpoints_session_epoch_idx').on(t.sessionId, t.compactionEpoch),
|
||||
],
|
||||
);
|
||||
|
||||
export const tessHandoffs = pgTable(
|
||||
'tess_handoffs',
|
||||
{
|
||||
handoffId: text('handoff_id').primaryKey(),
|
||||
sessionId: text('session_id')
|
||||
.notNull()
|
||||
.references(() => tessSessions.id, { onDelete: 'cascade' }),
|
||||
destination: text('destination').notNull(),
|
||||
correlationId: text('correlation_id').notNull(),
|
||||
checkpointId: text('checkpoint_id').notNull(),
|
||||
status: tessHandoffStatusEnum('status').notNull().default('pending'),
|
||||
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
|
||||
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
|
||||
},
|
||||
(t) => [index('tess_handoffs_session_status_idx').on(t.sessionId, t.status)],
|
||||
);
|
||||
|
||||
// ─── Skills ─────────────────────────────────────────────────────────────────
|
||||
|
||||
export const skills = pgTable(
|
||||
|
||||
Reference in New Issue
Block a user