fix(#187): implement server-side SSE error recovery
Server-side improvements (ALL 27/27 TESTS PASSING): - Add streamEventsFrom() method with lastEventId parameter for resuming streams - Include event IDs in SSE messages (id: event-123) for reconnection support - Send retry interval header (retry: 3000ms) to clients - Classify errors as retryable vs non-retryable - Handle transient errors gracefully with retry logic - Support Last-Event-ID header in controller for automatic reconnection Files modified: - apps/api/src/runner-jobs/runner-jobs.service.ts (new streamEventsFrom method) - apps/api/src/runner-jobs/runner-jobs.controller.ts (Last-Event-ID header support) - apps/api/src/runner-jobs/runner-jobs.service.spec.ts (comprehensive error recovery tests) - docs/scratchpads/187-implement-sse-error-recovery.md (implementation notes) This ensures robust real-time updates with automatic recovery from network issues. Client-side React hook will be added in a follow-up PR after fixing Quality Rails lint issues. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Jason Woltje
@@ -1,4 +1,4 @@
|
||||
import { Controller, Get, Post, Body, Param, Query, UseGuards, Res } from "@nestjs/common";
|
||||
import { Controller, Get, Post, Body, Param, Query, UseGuards, Res, Headers } from "@nestjs/common";
|
||||
import { Response } from "express";
|
||||
import { RunnerJobsService } from "./runner-jobs.service";
|
||||
import { CreateJobDto, QueryJobsDto } from "./dto";
|
||||
@@ -93,12 +93,14 @@ export class RunnerJobsController {
|
||||
* GET /api/runner-jobs/:id/events/stream
|
||||
* Stream job events via Server-Sent Events (SSE)
|
||||
* Requires: Any workspace member
|
||||
* Supports automatic reconnection via Last-Event-ID header
|
||||
*/
|
||||
@Get(":id/events/stream")
|
||||
@RequirePermission(Permission.WORKSPACE_ANY)
|
||||
async streamEvents(
|
||||
@Param("id") id: string,
|
||||
@Workspace() workspaceId: string,
|
||||
@Headers("last-event-id") lastEventId: string | undefined,
|
||||
@Res() res: Response
|
||||
): Promise<void> {
|
||||
// Set SSE headers
|
||||
@@ -108,7 +110,7 @@ export class RunnerJobsController {
|
||||
res.setHeader("X-Accel-Buffering", "no"); // Disable nginx buffering
|
||||
|
||||
try {
|
||||
await this.runnerJobsService.streamEvents(id, workspaceId, res);
|
||||
await this.runnerJobsService.streamEvents(id, workspaceId, res, lastEventId);
|
||||
} catch (error: unknown) {
|
||||
// Write error to stream
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
|
||||
@@ -22,6 +22,7 @@ describe("RunnerJobsService", () => {
|
||||
},
|
||||
jobEvent: {
|
||||
findMany: vi.fn(),
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
};
|
||||
|
||||
@@ -635,5 +636,250 @@ describe("RunnerJobsService", () => {
|
||||
expect(mockRes.on).toHaveBeenCalledWith("close", expect.any(Function));
|
||||
expect(mockRes.end).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// ERROR RECOVERY TESTS - Issue #187
|
||||
|
||||
it("should support resuming stream from lastEventId", async () => {
|
||||
const jobId = "job-123";
|
||||
const workspaceId = "workspace-123";
|
||||
const lastEventId = "event-5";
|
||||
|
||||
const mockRes = {
|
||||
write: vi.fn(),
|
||||
end: vi.fn(),
|
||||
on: vi.fn(),
|
||||
writableEnded: false,
|
||||
};
|
||||
|
||||
// Mock initial job lookup
|
||||
mockPrismaService.runnerJob.findUnique
|
||||
.mockResolvedValueOnce({
|
||||
id: jobId,
|
||||
status: RunnerJobStatus.RUNNING,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
id: jobId,
|
||||
status: RunnerJobStatus.COMPLETED,
|
||||
});
|
||||
|
||||
// Mock finding the last event for timestamp lookup
|
||||
mockPrismaService.jobEvent.findUnique.mockResolvedValue({
|
||||
id: lastEventId,
|
||||
timestamp: new Date("2026-01-01T12:00:00Z"),
|
||||
});
|
||||
|
||||
// Mock events starting after the lastEventId
|
||||
const mockEvents = [
|
||||
{
|
||||
id: "event-6",
|
||||
jobId,
|
||||
stepId: "step-2",
|
||||
type: "step.started",
|
||||
timestamp: new Date("2026-01-01T12:01:00Z"),
|
||||
payload: { name: "Next step" },
|
||||
},
|
||||
];
|
||||
|
||||
mockPrismaService.jobEvent.findMany.mockResolvedValue(mockEvents);
|
||||
|
||||
// Execute streamEvents with lastEventId
|
||||
await service.streamEventsFrom(jobId, workspaceId, mockRes as never, lastEventId);
|
||||
|
||||
// Verify events query used lastEventId as cursor
|
||||
expect(prisma.jobEvent.findMany).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
where: expect.objectContaining({
|
||||
id: { gt: lastEventId },
|
||||
}),
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it("should send event IDs for reconnection support", async () => {
|
||||
const jobId = "job-123";
|
||||
const workspaceId = "workspace-123";
|
||||
|
||||
const mockRes = {
|
||||
write: vi.fn(),
|
||||
end: vi.fn(),
|
||||
on: vi.fn(),
|
||||
writableEnded: false,
|
||||
};
|
||||
|
||||
mockPrismaService.runnerJob.findUnique
|
||||
.mockResolvedValueOnce({
|
||||
id: jobId,
|
||||
status: RunnerJobStatus.RUNNING,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
id: jobId,
|
||||
status: RunnerJobStatus.COMPLETED,
|
||||
});
|
||||
|
||||
const mockEvents = [
|
||||
{
|
||||
id: "event-123",
|
||||
jobId,
|
||||
stepId: "step-1",
|
||||
type: "step.started",
|
||||
timestamp: new Date(),
|
||||
payload: { name: "Test" },
|
||||
},
|
||||
];
|
||||
|
||||
mockPrismaService.jobEvent.findMany.mockResolvedValue(mockEvents);
|
||||
|
||||
await service.streamEvents(jobId, workspaceId, mockRes as never);
|
||||
|
||||
// Verify event ID was sent
|
||||
expect(mockRes.write).toHaveBeenCalledWith(expect.stringContaining("id: event-123"));
|
||||
});
|
||||
|
||||
it("should handle database connection errors gracefully", async () => {
|
||||
const jobId = "job-123";
|
||||
const workspaceId = "workspace-123";
|
||||
|
||||
let closeHandler: (() => void) | null = null;
|
||||
|
||||
const mockRes = {
|
||||
write: vi.fn(),
|
||||
end: vi.fn(),
|
||||
on: vi.fn((event: string, handler: () => void) => {
|
||||
if (event === "close") {
|
||||
closeHandler = handler;
|
||||
}
|
||||
}),
|
||||
writableEnded: false,
|
||||
};
|
||||
|
||||
mockPrismaService.runnerJob.findUnique.mockResolvedValueOnce({
|
||||
id: jobId,
|
||||
status: RunnerJobStatus.RUNNING,
|
||||
});
|
||||
|
||||
// Simulate database error during event polling (non-retryable)
|
||||
const dbError = new Error("Fatal database error");
|
||||
mockPrismaService.jobEvent.findMany.mockRejectedValue(dbError);
|
||||
|
||||
// Should propagate non-retryable error
|
||||
await expect(service.streamEvents(jobId, workspaceId, mockRes as never)).rejects.toThrow(
|
||||
"Fatal database error"
|
||||
);
|
||||
|
||||
// Verify error event was written
|
||||
expect(mockRes.write).toHaveBeenCalledWith(expect.stringContaining("event: error"));
|
||||
});
|
||||
|
||||
it("should send retry hint on transient errors", async () => {
|
||||
const jobId = "job-123";
|
||||
const workspaceId = "workspace-123";
|
||||
|
||||
let callCount = 0;
|
||||
let closeHandler: (() => void) | null = null;
|
||||
|
||||
const mockRes = {
|
||||
write: vi.fn(),
|
||||
end: vi.fn(),
|
||||
on: vi.fn((event: string, handler: () => void) => {
|
||||
if (event === "close") {
|
||||
closeHandler = handler;
|
||||
}
|
||||
}),
|
||||
writableEnded: false,
|
||||
};
|
||||
|
||||
mockPrismaService.runnerJob.findUnique
|
||||
.mockResolvedValueOnce({
|
||||
id: jobId,
|
||||
status: RunnerJobStatus.RUNNING,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
id: jobId,
|
||||
status: RunnerJobStatus.COMPLETED,
|
||||
});
|
||||
|
||||
// Simulate transient error, then success
|
||||
mockPrismaService.jobEvent.findMany.mockImplementation(() => {
|
||||
callCount++;
|
||||
if (callCount === 1) {
|
||||
return Promise.reject(new Error("Temporary connection issue"));
|
||||
}
|
||||
return Promise.resolve([]);
|
||||
});
|
||||
|
||||
await service.streamEvents(jobId, workspaceId, mockRes as never);
|
||||
|
||||
// Verify error event was sent with retryable flag
|
||||
expect(mockRes.write).toHaveBeenCalledWith(expect.stringContaining("event: error"));
|
||||
expect(mockRes.write).toHaveBeenCalledWith(expect.stringContaining('"retryable":true'));
|
||||
// Verify stream completed after retry
|
||||
expect(mockRes.write).toHaveBeenCalledWith(expect.stringContaining("stream.complete"));
|
||||
});
|
||||
|
||||
it("should respect client disconnect and stop polling", async () => {
|
||||
const jobId = "job-123";
|
||||
const workspaceId = "workspace-123";
|
||||
|
||||
let closeHandler: (() => void) | null = null;
|
||||
|
||||
const mockRes = {
|
||||
write: vi.fn(),
|
||||
end: vi.fn(),
|
||||
on: vi.fn((event: string, handler: () => void) => {
|
||||
if (event === "close") {
|
||||
closeHandler = handler;
|
||||
// Trigger close after first poll
|
||||
setTimeout(() => handler(), 100);
|
||||
}
|
||||
}),
|
||||
writableEnded: false,
|
||||
};
|
||||
|
||||
mockPrismaService.runnerJob.findUnique.mockResolvedValue({
|
||||
id: jobId,
|
||||
status: RunnerJobStatus.RUNNING,
|
||||
});
|
||||
|
||||
mockPrismaService.jobEvent.findMany.mockResolvedValue([]);
|
||||
|
||||
await service.streamEvents(jobId, workspaceId, mockRes as never);
|
||||
|
||||
// Verify cleanup happened
|
||||
expect(mockRes.end).toHaveBeenCalled();
|
||||
|
||||
// Verify we didn't query excessively after disconnect
|
||||
const queryCount = mockPrismaService.jobEvent.findMany.mock.calls.length;
|
||||
expect(queryCount).toBeLessThan(5); // Should stop quickly after disconnect
|
||||
});
|
||||
|
||||
it("should include connection metadata in stream headers", async () => {
|
||||
const jobId = "job-123";
|
||||
const workspaceId = "workspace-123";
|
||||
|
||||
const mockRes = {
|
||||
write: vi.fn(),
|
||||
end: vi.fn(),
|
||||
on: vi.fn(),
|
||||
writableEnded: false,
|
||||
setHeader: vi.fn(),
|
||||
};
|
||||
|
||||
mockPrismaService.runnerJob.findUnique
|
||||
.mockResolvedValueOnce({
|
||||
id: jobId,
|
||||
status: RunnerJobStatus.RUNNING,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
id: jobId,
|
||||
status: RunnerJobStatus.COMPLETED,
|
||||
});
|
||||
|
||||
mockPrismaService.jobEvent.findMany.mockResolvedValue([]);
|
||||
|
||||
await service.streamEvents(jobId, workspaceId, mockRes as never);
|
||||
|
||||
// Verify SSE headers include retry recommendation
|
||||
expect(mockRes.write).toHaveBeenCalledWith(expect.stringMatching(/retry: \d+/));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user