fix(#196): fix race condition in job status updates

Implemented optimistic locking with version field and SELECT FOR UPDATE
transactions to prevent data corruption from concurrent job status updates.

Changes:
- Added version field to RunnerJob schema for optimistic locking
- Created migration 20260202_add_runner_job_version_for_concurrency
- Implemented ConcurrentUpdateException for conflict detection
- Updated RunnerJobsService methods with optimistic locking:
  * updateStatus() - with version checking and retry logic
  * updateProgress() - with version checking and retry logic
  * cancel() - with version checking and retry logic
- Updated CoordinatorIntegrationService with SELECT FOR UPDATE:
  * updateJobStatus() - transaction with row locking
  * completeJob() - transaction with row locking
  * failJob() - transaction with row locking
  * updateJobProgress() - optimistic locking
- Added retry mechanism (3 attempts) with exponential backoff
- Added comprehensive concurrency tests (10 tests, all passing)
- Updated existing test mocks to support updateMany

Test Results:
- All 10 concurrency tests passing ✓
- Tests cover concurrent status updates, progress updates, completions,
  cancellations, retry logic, and exponential backoff

This fix prevents race conditions that could cause:
- Lost job results (double completion)
- Lost progress updates
- Invalid status transitions
- Data corruption under concurrent access

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Jason Woltje
2026-02-02 12:51:17 -06:00
parent a3b48dd631
commit ef25167c24
251 changed files with 7045 additions and 261 deletions

View File

@@ -1,10 +1,20 @@
import { Injectable, NotFoundException } from "@nestjs/common";
import { Prisma } from "@prisma/client";
import { Prisma, Idea } from "@prisma/client";
import { PrismaService } from "../prisma/prisma.service";
import { ActivityService } from "../activity/activity.service";
import { IdeaStatus } from "@prisma/client";
import type { CreateIdeaDto, CaptureIdeaDto, UpdateIdeaDto, QueryIdeasDto } from "./dto";
type IdeaWithRelations = Idea & {
creator: { id: string; name: string; email: string };
domain: { id: string; name: string; color: string | null } | null;
project: { id: string; name: string; color: string | null } | null;
};
type IdeaCaptured = Idea & {
creator: { id: string; name: string; email: string };
};
/**
* Service for managing ideas
*/
@@ -18,7 +28,11 @@ export class IdeasService {
/**
* Create a new idea
*/
async create(workspaceId: string, userId: string, createIdeaDto: CreateIdeaDto) {
async create(
workspaceId: string,
userId: string,
createIdeaDto: CreateIdeaDto
): Promise<IdeaWithRelations> {
const domainConnection = createIdeaDto.domainId
? { connect: { id: createIdeaDto.domainId } }
: undefined;
@@ -70,7 +84,11 @@ export class IdeasService {
* Quick capture - create an idea with minimal fields
* Optimized for rapid idea capture from the front-end
*/
async capture(workspaceId: string, userId: string, captureIdeaDto: CaptureIdeaDto) {
async capture(
workspaceId: string,
userId: string,
captureIdeaDto: CaptureIdeaDto
): Promise<IdeaCaptured> {
const data: Prisma.IdeaCreateInput = {
workspace: { connect: { id: workspaceId } },
creator: { connect: { id: userId } },
@@ -103,7 +121,15 @@ export class IdeasService {
/**
* Get paginated ideas with filters
*/
async findAll(query: QueryIdeasDto) {
async findAll(query: QueryIdeasDto): Promise<{
data: IdeaWithRelations[];
meta: {
total: number;
page: number;
limit: number;
totalPages: number;
};
}> {
const page = query.page ?? 1;
const limit = query.limit ?? 50;
const skip = (page - 1) * limit;
@@ -177,7 +203,7 @@ export class IdeasService {
/**
* Get a single idea by ID
*/
async findOne(id: string, workspaceId: string) {
async findOne(id: string, workspaceId: string): Promise<IdeaWithRelations> {
const idea = await this.prisma.idea.findUnique({
where: {
id,
@@ -206,7 +232,12 @@ export class IdeasService {
/**
* Update an idea
*/
async update(id: string, workspaceId: string, userId: string, updateIdeaDto: UpdateIdeaDto) {
async update(
id: string,
workspaceId: string,
userId: string,
updateIdeaDto: UpdateIdeaDto
): Promise<IdeaWithRelations> {
// Verify idea exists
const existingIdea = await this.prisma.idea.findUnique({
where: { id, workspaceId },
@@ -265,7 +296,7 @@ export class IdeasService {
/**
* Delete an idea
*/
async remove(id: string, workspaceId: string, userId: string) {
async remove(id: string, workspaceId: string, userId: string): Promise<void> {
// Verify idea exists
const idea = await this.prisma.idea.findUnique({
where: { id, workspaceId },