From d7328dbceb160b5cd31f31fe17ba37fc5c4a6cb9 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Sun, 1 Feb 2026 20:56:45 -0600 Subject: [PATCH] feat(#163): Add BullMQ dependencies Added bullmq@^5.67.2 and @nestjs/bullmq@^11.0.4 to support job queue management for the M4.2 Infrastructure milestone. BullMQ provides job progress tracking, automatic retry, rate limiting, and job dependencies over plain Valkey, complementing the existing ioredis setup. Verified: - pnpm install succeeds with no conflicts - pnpm build completes successfully - All packages resolve correctly in pnpm-lock.yaml Co-Authored-By: Claude Opus 4.5 --- apps/api/package.json | 2 + docs/scratchpads/163-bullmq-dependencies.md | 45 +++++++ pnpm-lock.yaml | 142 ++++++++++++++++++++ 3 files changed, 189 insertions(+) create mode 100644 docs/scratchpads/163-bullmq-dependencies.md diff --git a/apps/api/package.json b/apps/api/package.json index 01f1627..7a2dc7c 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -26,6 +26,7 @@ "dependencies": { "@anthropic-ai/sdk": "^0.72.1", "@mosaic/shared": "workspace:*", + "@nestjs/bullmq": "^11.0.4", "@nestjs/common": "^11.1.12", "@nestjs/core": "^11.1.12", "@nestjs/mapped-types": "^2.1.0", @@ -45,6 +46,7 @@ "adm-zip": "^0.5.16", "archiver": "^7.0.1", "better-auth": "^1.4.17", + "bullmq": "^5.67.2", "class-transformer": "^0.5.1", "class-validator": "^0.14.3", "gray-matter": "^4.0.3", diff --git a/docs/scratchpads/163-bullmq-dependencies.md b/docs/scratchpads/163-bullmq-dependencies.md new file mode 100644 index 0000000..0507723 --- /dev/null +++ b/docs/scratchpads/163-bullmq-dependencies.md @@ -0,0 +1,45 @@ +# Issue #163: Add BullMQ Dependencies + +## Objective + +Add BullMQ and @nestjs/bullmq packages to the API for job queue management required by the M4.2 Infrastructure milestone. These packages enable job progress tracking, automatic retry with exponential backoff, rate limiting, and job dependencies for the mosaic-stitcher wrapper architecture. + +## Context + +- **Architecture**: Job queue for mosaic-stitcher (control layer wrapper over OpenClaw) +- **Why BullMQ**: Provides advanced queue features over plain Valkey (progress tracking, retries, rate limiting, dependencies) +- **Existing Setup**: API already uses ioredis with Valkey +- **Target Version**: Compatible with existing NestJS 11.1.12 ecosystem + +## Approach + +1. Add bullmq package (latest stable) +2. Add @nestjs/bullmq adapter (latest stable, compatible with NestJS 11) +3. Verify compatibility with existing ioredis/Valkey setup +4. Run quality gates (pnpm install, pnpm build) +5. Commit with message: `feat(#163): Add BullMQ dependencies` + +## Progress + +- [x] Add dependencies to apps/api/package.json +- [x] Run pnpm install +- [x] Run pnpm build +- [x] Commit changes + +## Testing Plan + +1. Run `pnpm install` from workspace root - verify no conflicts +2. Run `pnpm build` in apps/api - verify TypeScript compilation succeeds +3. Check that bullmq and @nestjs/bullmq are in node_modules + +## Acceptance Criteria + +- [x] Packages installed and `pnpm install` succeeds +- [x] No conflicts with existing dependencies +- [x] `pnpm build` succeeds + +## Notes + +- Using workspace root for pnpm commands to respect monorepo configuration +- BullMQ works with any Redis-compatible client (ioredis already configured) +- No code changes needed in this task - dependency addition only diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6d1b562..9127000 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -60,6 +60,9 @@ importers: '@mosaic/shared': specifier: workspace:* version: link:../../packages/shared + '@nestjs/bullmq': + specifier: ^11.0.4 + version: 11.0.4(@nestjs/common@11.1.12(class-transformer@0.5.1)(class-validator@0.14.3)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.12)(bullmq@5.67.2) '@nestjs/common': specifier: ^11.1.12 version: 11.1.12(class-transformer@0.5.1)(class-validator@0.14.3)(reflect-metadata@0.2.2)(rxjs@7.8.2) @@ -117,6 +120,9 @@ importers: better-auth: specifier: ^1.4.17 version: 1.4.17(@prisma/client@6.19.2(prisma@6.19.2(magicast@0.3.5)(typescript@5.9.3))(typescript@5.9.3))(better-sqlite3@12.6.2)(drizzle-orm@0.41.0(@opentelemetry/api@1.9.0)(@prisma/client@5.22.0(prisma@6.19.2(magicast@0.3.5)(typescript@5.9.3)))(@types/pg@8.16.0)(better-sqlite3@12.6.2)(kysely@0.28.10)(pg@8.17.2)(prisma@6.19.2(magicast@0.3.5)(typescript@5.9.3)))(next@16.1.6(@babel/core@7.28.6)(@opentelemetry/api@1.9.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(pg@8.17.2)(prisma@6.19.2(magicast@0.3.5)(typescript@5.9.3))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(vitest@4.0.18(@opentelemetry/api@1.9.0)(@types/node@22.19.7)(jiti@2.6.1)(jsdom@26.1.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)) + bullmq: + specifier: ^5.67.2 + version: 5.67.2 class-transformer: specifier: ^0.5.1 version: 0.5.1 @@ -1305,6 +1311,49 @@ packages: resolution: {integrity: sha512-XyroGQXcHrZdvmrGJvsA9KNeOOgGMg1Vg9OlheUsBOSKznLMDl+YChxbkboRHvtFYJEMRYmlV3uoo/njCw05iw==} engines: {node: '>=16'} + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==} + cpu: [arm64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + resolution: {integrity: sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==} + cpu: [x64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + resolution: {integrity: sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==} + cpu: [arm64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + resolution: {integrity: sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==} + cpu: [arm] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + resolution: {integrity: sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==} + cpu: [x64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + resolution: {integrity: sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==} + cpu: [x64] + os: [win32] + + '@nestjs/bull-shared@11.0.4': + resolution: {integrity: sha512-VBJcDHSAzxQnpcDfA0kt9MTGUD1XZzfByV70su0W0eDCQ9aqIEBlzWRW21tv9FG9dIut22ysgDidshdjlnczLw==} + peerDependencies: + '@nestjs/common': ^10.0.0 || ^11.0.0 + '@nestjs/core': ^10.0.0 || ^11.0.0 + + '@nestjs/bullmq@11.0.4': + resolution: {integrity: sha512-wBzK9raAVG0/6NTMdvLGM4/FQ1lsB35/pYS8L6a0SDgkTiLpd7mAjQ8R692oMx5s7IjvgntaZOuTUrKYLNfIkA==} + peerDependencies: + '@nestjs/common': ^10.0.0 || ^11.0.0 + '@nestjs/core': ^10.0.0 || ^11.0.0 + bullmq: ^3.0.0 || ^4.0.0 || ^5.0.0 + '@nestjs/cli@11.0.16': resolution: {integrity: sha512-P0H+Vcjki6P5160E5QnMt3Q0X5FTg4PZkP99Ig4lm/4JWqfw32j3EXv3YBTJ2DmxLwOQ/IS9F7dzKpMAgzKTGg==} engines: {node: '>= 20.11'} @@ -3115,6 +3164,9 @@ packages: buffer@6.0.3: resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + bullmq@5.67.2: + resolution: {integrity: sha512-3KYqNqQptKcgksACO1li4YW9/jxEh6XWa1lUg4OFrHa80Pf0C7H9zeb6ssbQQDfQab/K3QCXopbZ40vrvcyrLw==} + bundle-name@4.1.0: resolution: {integrity: sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==} engines: {node: '>=18'} @@ -3380,6 +3432,10 @@ packages: resolution: {integrity: sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==} engines: {node: '>= 14'} + cron-parser@4.9.0: + resolution: {integrity: sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==} + engines: {node: '>=12.0.0'} + cross-spawn@7.0.6: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} @@ -4565,6 +4621,10 @@ packages: peerDependencies: react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 + luxon@3.7.2: + resolution: {integrity: sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==} + engines: {node: '>=12'} + lz-string@1.5.0: resolution: {integrity: sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==} hasBin: true @@ -4705,6 +4765,13 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + msgpackr-extract@3.0.3: + resolution: {integrity: sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==} + hasBin: true + + msgpackr@1.11.5: + resolution: {integrity: sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==} + multer@2.0.2: resolution: {integrity: sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==} engines: {node: '>= 10.16.0'} @@ -4786,6 +4853,10 @@ packages: encoding: optional: true + node-gyp-build-optional-packages@5.2.2: + resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==} + hasBin: true + node-releases@2.0.27: resolution: {integrity: sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==} @@ -7091,6 +7162,38 @@ snapshots: chevrotain: 10.5.0 lilconfig: 2.1.0 + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + optional: true + + '@nestjs/bull-shared@11.0.4(@nestjs/common@11.1.12(class-transformer@0.5.1)(class-validator@0.14.3)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.12)': + dependencies: + '@nestjs/common': 11.1.12(class-transformer@0.5.1)(class-validator@0.14.3)(reflect-metadata@0.2.2)(rxjs@7.8.2) + '@nestjs/core': 11.1.12(@nestjs/common@11.1.12(class-transformer@0.5.1)(class-validator@0.14.3)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/platform-express@11.1.12)(@nestjs/websockets@11.1.12)(reflect-metadata@0.2.2)(rxjs@7.8.2) + tslib: 2.8.1 + + '@nestjs/bullmq@11.0.4(@nestjs/common@11.1.12(class-transformer@0.5.1)(class-validator@0.14.3)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.12)(bullmq@5.67.2)': + dependencies: + '@nestjs/bull-shared': 11.0.4(@nestjs/common@11.1.12(class-transformer@0.5.1)(class-validator@0.14.3)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.12) + '@nestjs/common': 11.1.12(class-transformer@0.5.1)(class-validator@0.14.3)(reflect-metadata@0.2.2)(rxjs@7.8.2) + '@nestjs/core': 11.1.12(@nestjs/common@11.1.12(class-transformer@0.5.1)(class-validator@0.14.3)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/platform-express@11.1.12)(@nestjs/websockets@11.1.12)(reflect-metadata@0.2.2)(rxjs@7.8.2) + bullmq: 5.67.2 + tslib: 2.8.1 + '@nestjs/cli@11.0.16(@swc/core@1.15.11)(@types/node@22.19.7)': dependencies: '@angular-devkit/core': 19.2.19(chokidar@4.0.3) @@ -9285,6 +9388,18 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 + bullmq@5.67.2: + dependencies: + cron-parser: 4.9.0 + ioredis: 5.9.2 + msgpackr: 1.11.5 + node-abort-controller: 3.1.1 + semver: 7.7.3 + tslib: 2.8.1 + uuid: 11.1.0 + transitivePeerDependencies: + - supports-color + bundle-name@4.1.0: dependencies: run-applescript: 7.1.0 @@ -9546,6 +9661,10 @@ snapshots: crc-32: 1.2.2 readable-stream: 4.7.0 + cron-parser@4.9.0: + dependencies: + luxon: 3.7.2 + cross-spawn@7.0.6: dependencies: path-key: 3.1.1 @@ -10716,6 +10835,8 @@ snapshots: dependencies: react: 19.2.4 + luxon@3.7.2: {} + lz-string@1.5.0: {} magic-string@0.30.17: @@ -10854,6 +10975,22 @@ snapshots: ms@2.1.3: {} + msgpackr-extract@3.0.3: + dependencies: + node-gyp-build-optional-packages: 5.2.2 + optionalDependencies: + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.3 + optional: true + + msgpackr@1.11.5: + optionalDependencies: + msgpackr-extract: 3.0.3 + multer@2.0.2: dependencies: append-field: 1.0.0 @@ -10923,6 +11060,11 @@ snapshots: dependencies: whatwg-url: 5.0.0 + node-gyp-build-optional-packages@5.2.2: + dependencies: + detect-libc: 2.1.2 + optional: true + node-releases@2.0.27: {} normalize-path@3.0.0: {}