fix(tess): redact sensitive persistence and egress #723

Closed
jason.woltje wants to merge 16 commits from fix/tess-redaction into feat/tess-interaction-agent
4 changed files with 104 additions and 2 deletions
Showing only changes of commit fdd58a353b - Show all commits

View File

@@ -69,6 +69,51 @@ describe('ChatGateway redaction boundary', (): void => {
expect(JSON.stringify(client.emit.mock.calls)).not.toContain(CANARY);
});
it('retains a split secret label until its value can be redacted', (): void => {
const { gateway } = buildGateway();
const client = {
connected: true,
id: 'client-1',
data: { user: { id: 'user-1' } },
emit: vi.fn(),
};
gateway.relayEvent(client, CONVERSATION_ID, {
type: 'message_update',
assistantMessageEvent: { type: 'text_delta', delta: 'token ' },
});
gateway.relayEvent(client, CONVERSATION_ID, {
type: 'message_update',
assistantMessageEvent: { type: 'text_delta', delta: '=canaryvalue123 ' },
});
expect(client.emit).toHaveBeenCalledWith('agent:text', {
conversationId: CONVERSATION_ID,
text: '[REDACTED_SECRET] ',
});
expect(JSON.stringify(client.emit.mock.calls)).not.toContain('canaryvalue123');
});
it('drops an oversized unterminated stream fragment rather than retaining it', (): void => {
const { gateway } = buildGateway();
const client = {
connected: true,
id: 'client-1',
data: { user: { id: 'user-1' } },
emit: vi.fn(),
};
gateway.relayEvent(client, CONVERSATION_ID, {
type: 'message_update',
assistantMessageEvent: { type: 'text_delta', delta: 'x'.repeat(8_193) },
});
expect(client.emit).toHaveBeenCalledWith('agent:text', {
conversationId: CONVERSATION_ID,
text: '[REDACTED_STREAM_OVERFLOW]',
});
});
it('persists only redacted assistant content with classifications', (): void => {
const { gateway, brain } = buildGateway();
const client = {

View File

@@ -64,6 +64,7 @@ interface ClientSession {
* Keyed by conversationId, value is the model name to use.
*/
const modelOverrides = new Map<string, string>();
const MAX_REDACTION_BUFFER_LENGTH = 8_192;
function isDiscordIngressEnvelope(value: unknown): value is DiscordIngressEnvelope {
if (typeof value !== 'object' || value === null) return false;
@@ -111,6 +112,7 @@ export class ChatGateway implements OnGatewayInit, OnGatewayConnection, OnGatewa
/** Raw stream fragments are kept in memory only until they are safe to redact and emit. */
private readonly textEgressBuffers = new Map<string, string>();
private readonly thinkingEgressBuffers = new Map<string, string>();
private readonly overflowedEgress = new Set<string>();
private readonly discordReplayProtector = new DiscordReplayProtector();
constructor(
@@ -161,6 +163,8 @@ export class ChatGateway implements OnGatewayInit, OnGatewayConnection, OnGatewa
}
this.textEgressBuffers.delete(client.id);
this.thinkingEgressBuffers.delete(client.id);
this.overflowedEgress.delete(this.egressKey(client, 'agent:text'));
this.overflowedEgress.delete(this.egressKey(client, 'agent:thinking'));
}
private getClientScope(client: Socket): ActorTenantScope | null {
@@ -758,7 +762,18 @@ export class ChatGateway implements OnGatewayInit, OnGatewayConnection, OnGatewa
buffers: Map<string, string>,
delta: string,
): void {
buffers.set(client.id, `${buffers.get(client.id) ?? ''}${delta}`);
const key = this.egressKey(client, eventName);
if (this.overflowedEgress.has(key)) return;
const buffered = `${buffers.get(client.id) ?? ''}${delta}`;
if (buffered.length > MAX_REDACTION_BUFFER_LENGTH) {
buffers.delete(client.id);
this.overflowedEgress.add(key);
client.emit(eventName, { conversationId, text: '[REDACTED_STREAM_OVERFLOW]' });
return;
}
buffers.set(client.id, buffered);
this.flushRedactedEgress(client, conversationId, eventName, buffers, false);
}
@@ -774,6 +789,12 @@ export class ChatGateway implements OnGatewayInit, OnGatewayConnection, OnGatewa
buffers: Map<string, string>,
final: boolean,
): void {
const key = this.egressKey(client, eventName);
if (this.overflowedEgress.has(key)) {
if (final) this.overflowedEgress.delete(key);
return;
}
const buffered = buffers.get(client.id) ?? '';
const releaseLength = final ? buffered.length : this.safeRedactionPrefixLength(buffered);
const released = buffered.slice(0, releaseLength);
@@ -806,7 +827,21 @@ export class ChatGateway implements OnGatewayInit, OnGatewayConnection, OnGatewa
// The secret classifier accepts whitespace around ':' and '=', so preserve
// a pending label until its value and delimiter are both complete.
const secretLabel = /(?:api[_-]?key|token|password|secret)\s*[:=]\s*$/i.exec(content);
const pendingSecretLabel =
/(?:^|[^A-Za-z0-9_])((?:api[_-]?key|token|password|secret|bearer|authorization)\s*)$/i.exec(
content,
);
if (pendingSecretLabel) {
const label = pendingSecretLabel[1] ?? '';
retainedFrom = Math.min(
retainedFrom,
pendingSecretLabel.index + pendingSecretLabel[0].length - label.length,
);
}
const secretLabel = /(?:api[_-]?key|token|password|secret|authorization)\s*[:=]\s*$/i.exec(
content,
);
if (secretLabel) {
retainedFrom = Math.min(retainedFrom, secretLabel.index);
}
@@ -826,6 +861,10 @@ export class ChatGateway implements OnGatewayInit, OnGatewayConnection, OnGatewa
return retainedFrom;
}
private egressKey(client: Socket, eventName: 'agent:text' | 'agent:thinking'): string {
return `${client.id}:${eventName}`;
}
private relayEvent(client: Socket, conversationId: string, event: AgentSessionEvent): void {
if (!client.connected) {
this.logger.warn(
@@ -845,6 +884,8 @@ export class ChatGateway implements OnGatewayInit, OnGatewayConnection, OnGatewa
}
this.textEgressBuffers.set(client.id, '');
this.thinkingEgressBuffers.set(client.id, '');
this.overflowedEgress.delete(this.egressKey(client, 'agent:text'));
this.overflowedEgress.delete(this.egressKey(client, 'agent:thinking'));
client.emit('agent:start', { conversationId });
break;
}

View File

@@ -11,4 +11,15 @@ describe('redactSensitiveContent', (): void => {
expect(result.content).not.toContain('+1 555 555 1212');
expect(result.classifications).toEqual(['secret', 'pii']);
});
it('redacts common provider credential formats', (): void => {
const result = redactSensitiveContent(
'Authorization: Bearer canary.bearer.token jwt eyJcanary.eyJpayload.eyJsignature aws AKIACANARY1234567890',
);
expect(result.content).not.toContain('canary.bearer.token');
expect(result.content).not.toContain('eyJcanary.eyJpayload.eyJsignature');
expect(result.content).not.toContain('AKIACANARY1234567890');
expect(result.classifications).toEqual(['secret']);
});
});

View File

@@ -8,6 +8,11 @@ export interface RedactionResult {
const SECRET_PATTERNS: RegExp[] = [
/\b(?:sk|ghp|gitea)_[A-Za-z0-9_-]{8,}\b/g,
/\b(?:api[_-]?key|token|password|secret)\s*[:=]\s*[^\s,;]+/gi,
/\b(?:authorization\s*:\s*)?bearer\s+[A-Za-z0-9._~+/-]+=*/gi,
/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g,
/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g,
/-----BEGIN(?: [A-Z]+)* PRIVATE KEY-----[\s\S]*?-----END(?: [A-Z]+)* PRIVATE KEY-----/g,
/https?:\/\/[^\s?#]+[^\s]*[?&](?:token|key|secret|signature|sig)=[^\s&#]+/gi,
];
const PII_PATTERNS: RegExp[] = [
/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,