fix(#196): fix race condition in job status updates

Implemented optimistic locking with version field and SELECT FOR UPDATE
transactions to prevent data corruption from concurrent job status updates.

Changes:
- Added version field to RunnerJob schema for optimistic locking
- Created migration 20260202_add_runner_job_version_for_concurrency
- Implemented ConcurrentUpdateException for conflict detection
- Updated RunnerJobsService methods with optimistic locking:
  * updateStatus() - with version checking and retry logic
  * updateProgress() - with version checking and retry logic
  * cancel() - with version checking and retry logic
- Updated CoordinatorIntegrationService with SELECT FOR UPDATE:
  * updateJobStatus() - transaction with row locking
  * completeJob() - transaction with row locking
  * failJob() - transaction with row locking
  * updateJobProgress() - optimistic locking
- Added retry mechanism (3 attempts) with exponential backoff
- Added comprehensive concurrency tests (10 tests, all passing)
- Updated existing test mocks to support updateMany

Test Results:
- All 10 concurrency tests passing ✓
- Tests cover concurrent status updates, progress updates, completions,
  cancellations, retry logic, and exponential backoff

This fix prevents race conditions that could cause:
- Lost job results (double completion)
- Lost progress updates
- Invalid status transitions
- Data corruption under concurrent access

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Jason Woltje
2026-02-02 12:51:17 -06:00
parent a3b48dd631
commit ef25167c24
251 changed files with 7045 additions and 261 deletions

View File

@@ -1,6 +1,6 @@
import { Injectable, Logger } from "@nestjs/common";
import { PrismaService } from "../prisma/prisma.service";
import { ActivityAction, EntityType, Prisma } from "@prisma/client";
import { ActivityAction, EntityType, Prisma, ActivityLog } from "@prisma/client";
import type {
CreateActivityLogInput,
PaginatedActivityLogs,
@@ -20,7 +20,7 @@ export class ActivityService {
/**
* Create a new activity log entry
*/
async logActivity(input: CreateActivityLogInput) {
async logActivity(input: CreateActivityLogInput): Promise<ActivityLog> {
try {
return await this.prisma.activityLog.create({
data: input as unknown as Prisma.ActivityLogCreateInput,
@@ -167,7 +167,7 @@ export class ActivityService {
userId: string,
taskId: string,
details?: Prisma.JsonValue
) {
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -186,7 +186,7 @@ export class ActivityService {
userId: string,
taskId: string,
details?: Prisma.JsonValue
) {
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -205,7 +205,7 @@ export class ActivityService {
userId: string,
taskId: string,
details?: Prisma.JsonValue
) {
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -224,7 +224,7 @@ export class ActivityService {
userId: string,
taskId: string,
details?: Prisma.JsonValue
) {
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -238,7 +238,12 @@ export class ActivityService {
/**
* Log task assignment
*/
async logTaskAssigned(workspaceId: string, userId: string, taskId: string, assigneeId: string) {
async logTaskAssigned(
workspaceId: string,
userId: string,
taskId: string,
assigneeId: string
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -257,7 +262,7 @@ export class ActivityService {
userId: string,
eventId: string,
details?: Prisma.JsonValue
) {
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -276,7 +281,7 @@ export class ActivityService {
userId: string,
eventId: string,
details?: Prisma.JsonValue
) {
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -295,7 +300,7 @@ export class ActivityService {
userId: string,
eventId: string,
details?: Prisma.JsonValue
) {
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -314,7 +319,7 @@ export class ActivityService {
userId: string,
projectId: string,
details?: Prisma.JsonValue
) {
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -333,7 +338,7 @@ export class ActivityService {
userId: string,
projectId: string,
details?: Prisma.JsonValue
) {
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -352,7 +357,7 @@ export class ActivityService {
userId: string,
projectId: string,
details?: Prisma.JsonValue
) {
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -366,7 +371,11 @@ export class ActivityService {
/**
* Log workspace creation
*/
async logWorkspaceCreated(workspaceId: string, userId: string, details?: Prisma.JsonValue) {
async logWorkspaceCreated(
workspaceId: string,
userId: string,
details?: Prisma.JsonValue
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -380,7 +389,11 @@ export class ActivityService {
/**
* Log workspace update
*/
async logWorkspaceUpdated(workspaceId: string, userId: string, details?: Prisma.JsonValue) {
async logWorkspaceUpdated(
workspaceId: string,
userId: string,
details?: Prisma.JsonValue
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -399,7 +412,7 @@ export class ActivityService {
userId: string,
memberId: string,
role: string
) {
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -413,7 +426,11 @@ export class ActivityService {
/**
* Log workspace member removed
*/
async logWorkspaceMemberRemoved(workspaceId: string, userId: string, memberId: string) {
async logWorkspaceMemberRemoved(
workspaceId: string,
userId: string,
memberId: string
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -427,7 +444,11 @@ export class ActivityService {
/**
* Log user profile update
*/
async logUserUpdated(workspaceId: string, userId: string, details?: Prisma.JsonValue) {
async logUserUpdated(
workspaceId: string,
userId: string,
details?: Prisma.JsonValue
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -446,7 +467,7 @@ export class ActivityService {
userId: string,
domainId: string,
details?: Prisma.JsonValue
) {
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -465,7 +486,7 @@ export class ActivityService {
userId: string,
domainId: string,
details?: Prisma.JsonValue
) {
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -484,7 +505,7 @@ export class ActivityService {
userId: string,
domainId: string,
details?: Prisma.JsonValue
) {
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -503,7 +524,7 @@ export class ActivityService {
userId: string,
ideaId: string,
details?: Prisma.JsonValue
) {
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -522,7 +543,7 @@ export class ActivityService {
userId: string,
ideaId: string,
details?: Prisma.JsonValue
) {
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,
@@ -541,7 +562,7 @@ export class ActivityService {
userId: string,
ideaId: string,
details?: Prisma.JsonValue
) {
): Promise<ActivityLog> {
return this.logActivity({
workspaceId,
userId,