Compare commits
5 Commits
de37e7be90
...
feat/ms22-
| Author | SHA1 | Date | |
|---|---|---|---|
| 7c7a821b0f | |||
| c640d22394 | |||
| 5e7346adc7 | |||
| d07a840f25 | |||
| 4b2e48af9c |
@@ -1,232 +0,0 @@
|
||||
# API Pipeline - Mosaic Stack
|
||||
# Quality gates, build, and Docker publish for @mosaic/api
|
||||
#
|
||||
# Triggers on: apps/api/**, packages/**, root configs
|
||||
# Security chain: source audit + Trivy container scan
|
||||
|
||||
when:
|
||||
- event: [push, pull_request, manual]
|
||||
path:
|
||||
include:
|
||||
- "apps/api/**"
|
||||
- "packages/**"
|
||||
- "pnpm-lock.yaml"
|
||||
- "pnpm-workspace.yaml"
|
||||
- "turbo.json"
|
||||
- "package.json"
|
||||
- ".woodpecker/api.yml"
|
||||
- ".trivyignore"
|
||||
|
||||
variables:
|
||||
- &node_image "node:24-alpine"
|
||||
- &install_deps |
|
||||
corepack enable
|
||||
pnpm install --frozen-lockfile
|
||||
- &use_deps |
|
||||
corepack enable
|
||||
- &turbo_env
|
||||
TURBO_API:
|
||||
from_secret: turbo_api
|
||||
TURBO_TOKEN:
|
||||
from_secret: turbo_token
|
||||
TURBO_TEAM:
|
||||
from_secret: turbo_team
|
||||
- &kaniko_setup |
|
||||
mkdir -p /kaniko/.docker
|
||||
echo "{\"auths\":{\"git.mosaicstack.dev\":{\"username\":\"$GITEA_USER\",\"password\":\"$GITEA_TOKEN\"}}}" > /kaniko/.docker/config.json
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.7-alpine3.22
|
||||
environment:
|
||||
POSTGRES_DB: test_db
|
||||
POSTGRES_USER: test_user
|
||||
POSTGRES_PASSWORD: test_password
|
||||
|
||||
steps:
|
||||
# === Quality Gates ===
|
||||
|
||||
install:
|
||||
image: *node_image
|
||||
commands:
|
||||
- *install_deps
|
||||
|
||||
security-audit:
|
||||
image: *node_image
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm audit --audit-level=high
|
||||
depends_on:
|
||||
- install
|
||||
|
||||
prisma-generate:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm --filter "@mosaic/api" prisma:generate
|
||||
depends_on:
|
||||
- install
|
||||
|
||||
lint:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
<<: *turbo_env
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm turbo lint --filter=@mosaic/api
|
||||
depends_on:
|
||||
- prisma-generate
|
||||
|
||||
typecheck:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
<<: *turbo_env
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm turbo typecheck --filter=@mosaic/api
|
||||
depends_on:
|
||||
- prisma-generate
|
||||
|
||||
prisma-migrate:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
DATABASE_URL: "postgresql://test_user:test_password@postgres:5432/test_db?schema=public"
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm --filter "@mosaic/api" prisma migrate deploy
|
||||
depends_on:
|
||||
- prisma-generate
|
||||
|
||||
test:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
DATABASE_URL: "postgresql://test_user:test_password@postgres:5432/test_db?schema=public"
|
||||
ENCRYPTION_KEY: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm --filter "@mosaic/api" exec vitest run --exclude 'src/auth/auth-rls.integration.spec.ts' --exclude 'src/credentials/user-credential.model.spec.ts' --exclude 'src/job-events/job-events.performance.spec.ts' --exclude 'src/knowledge/services/fulltext-search.spec.ts' --exclude 'src/mosaic-telemetry/mosaic-telemetry.module.spec.ts'
|
||||
depends_on:
|
||||
- prisma-migrate
|
||||
|
||||
# === Build ===
|
||||
|
||||
build:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
NODE_ENV: "production"
|
||||
<<: *turbo_env
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm turbo build --filter=@mosaic/api
|
||||
depends_on:
|
||||
- lint
|
||||
- typecheck
|
||||
- test
|
||||
- security-audit
|
||||
|
||||
# === Docker Build & Push ===
|
||||
|
||||
docker-build-api:
|
||||
image: gcr.io/kaniko-project/executor:debug
|
||||
environment:
|
||||
GITEA_USER:
|
||||
from_secret: gitea_username
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
CI_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
|
||||
CI_COMMIT_TAG: ${CI_COMMIT_TAG}
|
||||
commands:
|
||||
- *kaniko_setup
|
||||
- |
|
||||
DESTINATIONS=""
|
||||
if [ -n "$CI_COMMIT_TAG" ]; then
|
||||
DESTINATIONS="--destination git.mosaicstack.dev/mosaic/stack-api:$CI_COMMIT_TAG"
|
||||
elif [ "$CI_COMMIT_BRANCH" = "main" ]; then
|
||||
DESTINATIONS="--destination git.mosaicstack.dev/mosaic/stack-api:latest"
|
||||
fi
|
||||
/kaniko/executor --context . --dockerfile apps/api/Dockerfile --snapshot-mode=redo $DESTINATIONS
|
||||
when:
|
||||
- branch: [main]
|
||||
event: [push, manual, tag]
|
||||
depends_on:
|
||||
- build
|
||||
|
||||
# === Container Security Scan ===
|
||||
|
||||
security-trivy-api:
|
||||
image: aquasec/trivy:latest
|
||||
environment:
|
||||
GITEA_USER:
|
||||
from_secret: gitea_username
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
CI_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
|
||||
CI_COMMIT_TAG: ${CI_COMMIT_TAG}
|
||||
commands:
|
||||
- |
|
||||
if [ -n "$$CI_COMMIT_TAG" ]; then
|
||||
SCAN_TAG="$$CI_COMMIT_TAG"
|
||||
elif [ "$$CI_COMMIT_BRANCH" = "main" ]; then
|
||||
SCAN_TAG="latest"
|
||||
else
|
||||
SCAN_TAG="latest"
|
||||
fi
|
||||
mkdir -p ~/.docker
|
||||
echo "{\"auths\":{\"git.mosaicstack.dev\":{\"username\":\"$$GITEA_USER\",\"password\":\"$$GITEA_TOKEN\"}}}" > ~/.docker/config.json
|
||||
trivy image --exit-code 1 --severity HIGH,CRITICAL --ignore-unfixed \
|
||||
--ignorefile .trivyignore \
|
||||
git.mosaicstack.dev/mosaic/stack-api:$$SCAN_TAG
|
||||
when:
|
||||
- branch: [main]
|
||||
event: [push, manual, tag]
|
||||
depends_on:
|
||||
- docker-build-api
|
||||
|
||||
# === Package Linking ===
|
||||
|
||||
link-packages:
|
||||
image: alpine:3
|
||||
environment:
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
commands:
|
||||
- apk add --no-cache curl
|
||||
- sleep 10
|
||||
- |
|
||||
set -e
|
||||
link_package() {
|
||||
PKG="$$1"
|
||||
echo "Linking $$PKG..."
|
||||
for attempt in 1 2 3; do
|
||||
STATUS=$$(curl -s -o /tmp/link-response.txt -w "%{http_code}" -X POST \
|
||||
-H "Authorization: token $$GITEA_TOKEN" \
|
||||
"https://git.mosaicstack.dev/api/v1/packages/mosaic/container/$$PKG/-/link/stack")
|
||||
if [ "$$STATUS" = "201" ] || [ "$$STATUS" = "204" ]; then
|
||||
echo " Linked $$PKG"
|
||||
return 0
|
||||
elif [ "$$STATUS" = "400" ]; then
|
||||
echo " $$PKG already linked"
|
||||
return 0
|
||||
elif [ "$$STATUS" = "404" ] && [ $$attempt -lt 3 ]; then
|
||||
echo " $$PKG not found yet, retrying in 5s (attempt $$attempt/3)..."
|
||||
sleep 5
|
||||
else
|
||||
echo " FAILED: $$PKG status $$STATUS"
|
||||
cat /tmp/link-response.txt
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
link_package "stack-api"
|
||||
when:
|
||||
- branch: [main]
|
||||
event: [push, manual, tag]
|
||||
depends_on:
|
||||
- security-trivy-api
|
||||
337
.woodpecker/ci.yml
Normal file
337
.woodpecker/ci.yml
Normal file
@@ -0,0 +1,337 @@
|
||||
# Unified CI Pipeline - Mosaic Stack
|
||||
# Single install, parallel quality gates, sequential deploy
|
||||
#
|
||||
# Replaces: api.yml, orchestrator.yml, web.yml
|
||||
# Keeps: coordinator.yml (Python), infra.yml (separate concerns)
|
||||
#
|
||||
# Flow:
|
||||
# install → security-audit
|
||||
# → prisma-generate → lint + typecheck (parallel)
|
||||
# → prisma-migrate → test
|
||||
# → build (after all gates pass)
|
||||
# → docker builds (main only, parallel)
|
||||
# → trivy scans (main only, parallel)
|
||||
# → package linking (main only)
|
||||
|
||||
when:
|
||||
- event: [push, pull_request, manual]
|
||||
path:
|
||||
include:
|
||||
- "apps/api/**"
|
||||
- "apps/orchestrator/**"
|
||||
- "apps/web/**"
|
||||
- "packages/**"
|
||||
- "pnpm-lock.yaml"
|
||||
- "pnpm-workspace.yaml"
|
||||
- "turbo.json"
|
||||
- "package.json"
|
||||
- ".woodpecker/ci.yml"
|
||||
- ".trivyignore"
|
||||
|
||||
variables:
|
||||
- &node_image "node:24-alpine"
|
||||
- &install_deps |
|
||||
corepack enable
|
||||
pnpm install --frozen-lockfile
|
||||
- &use_deps |
|
||||
corepack enable
|
||||
- &turbo_env
|
||||
TURBO_API:
|
||||
from_secret: turbo_api
|
||||
TURBO_TOKEN:
|
||||
from_secret: turbo_token
|
||||
TURBO_TEAM:
|
||||
from_secret: turbo_team
|
||||
- &kaniko_setup |
|
||||
mkdir -p /kaniko/.docker
|
||||
echo "{\"auths\":{\"git.mosaicstack.dev\":{\"username\":\"$GITEA_USER\",\"password\":\"$GITEA_TOKEN\"}}}" > /kaniko/.docker/config.json
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.7-alpine3.22
|
||||
environment:
|
||||
POSTGRES_DB: test_db
|
||||
POSTGRES_USER: test_user
|
||||
POSTGRES_PASSWORD: test_password
|
||||
|
||||
steps:
|
||||
# ─── Install (once) ─────────────────────────────────────────
|
||||
install:
|
||||
image: *node_image
|
||||
commands:
|
||||
- *install_deps
|
||||
|
||||
# ─── Security Audit (once) ──────────────────────────────────
|
||||
security-audit:
|
||||
image: *node_image
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm audit --audit-level=high
|
||||
depends_on:
|
||||
- install
|
||||
|
||||
# ─── Prisma Generate ────────────────────────────────────────
|
||||
prisma-generate:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm --filter "@mosaic/api" prisma:generate
|
||||
depends_on:
|
||||
- install
|
||||
|
||||
# ─── Lint (all packages) ────────────────────────────────────
|
||||
lint:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
<<: *turbo_env
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm turbo lint
|
||||
depends_on:
|
||||
- prisma-generate
|
||||
|
||||
# ─── Typecheck (all packages, parallel with lint) ───────────
|
||||
typecheck:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
<<: *turbo_env
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm turbo typecheck
|
||||
depends_on:
|
||||
- prisma-generate
|
||||
|
||||
# ─── Prisma Migrate (test DB) ──────────────────────────────
|
||||
prisma-migrate:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
DATABASE_URL: "postgresql://test_user:test_password@postgres:5432/test_db?schema=public"
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm --filter "@mosaic/api" prisma migrate deploy
|
||||
depends_on:
|
||||
- prisma-generate
|
||||
|
||||
# ─── Test (all packages) ───────────────────────────────────
|
||||
test:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
DATABASE_URL: "postgresql://test_user:test_password@postgres:5432/test_db?schema=public"
|
||||
ENCRYPTION_KEY: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
<<: *turbo_env
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm --filter "@mosaic/api" exec vitest run --exclude 'src/auth/auth-rls.integration.spec.ts' --exclude 'src/credentials/user-credential.model.spec.ts' --exclude 'src/job-events/job-events.performance.spec.ts' --exclude 'src/knowledge/services/fulltext-search.spec.ts' --exclude 'src/mosaic-telemetry/mosaic-telemetry.module.spec.ts'
|
||||
- pnpm turbo test --filter=@mosaic/orchestrator --filter=@mosaic/web
|
||||
depends_on:
|
||||
- prisma-migrate
|
||||
|
||||
# ─── Build (all packages) ──────────────────────────────────
|
||||
build:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
NODE_ENV: "production"
|
||||
<<: *turbo_env
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm turbo build
|
||||
depends_on:
|
||||
- lint
|
||||
- typecheck
|
||||
- test
|
||||
- security-audit
|
||||
|
||||
# ─── Docker Builds (main only, parallel) ───────────────────
|
||||
|
||||
docker-build-api:
|
||||
image: gcr.io/kaniko-project/executor:debug
|
||||
environment:
|
||||
GITEA_USER:
|
||||
from_secret: gitea_username
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
CI_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
|
||||
CI_COMMIT_TAG: ${CI_COMMIT_TAG}
|
||||
commands:
|
||||
- *kaniko_setup
|
||||
- |
|
||||
DESTINATIONS=""
|
||||
if [ -n "$CI_COMMIT_TAG" ]; then
|
||||
DESTINATIONS="--destination git.mosaicstack.dev/mosaic/stack-api:$CI_COMMIT_TAG"
|
||||
elif [ "$CI_COMMIT_BRANCH" = "main" ]; then
|
||||
DESTINATIONS="--destination git.mosaicstack.dev/mosaic/stack-api:latest"
|
||||
fi
|
||||
/kaniko/executor --context . --dockerfile apps/api/Dockerfile --snapshot-mode=redo $DESTINATIONS
|
||||
when:
|
||||
- branch: [main]
|
||||
event: [push, manual, tag]
|
||||
depends_on:
|
||||
- build
|
||||
|
||||
docker-build-orchestrator:
|
||||
image: gcr.io/kaniko-project/executor:debug
|
||||
environment:
|
||||
GITEA_USER:
|
||||
from_secret: gitea_username
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
CI_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
|
||||
CI_COMMIT_TAG: ${CI_COMMIT_TAG}
|
||||
commands:
|
||||
- *kaniko_setup
|
||||
- |
|
||||
DESTINATIONS=""
|
||||
if [ -n "$CI_COMMIT_TAG" ]; then
|
||||
DESTINATIONS="--destination git.mosaicstack.dev/mosaic/stack-orchestrator:$CI_COMMIT_TAG"
|
||||
elif [ "$CI_COMMIT_BRANCH" = "main" ]; then
|
||||
DESTINATIONS="--destination git.mosaicstack.dev/mosaic/stack-orchestrator:latest"
|
||||
fi
|
||||
/kaniko/executor --context . --dockerfile apps/orchestrator/Dockerfile --snapshot-mode=redo $DESTINATIONS
|
||||
when:
|
||||
- branch: [main]
|
||||
event: [push, manual, tag]
|
||||
depends_on:
|
||||
- build
|
||||
|
||||
docker-build-web:
|
||||
image: gcr.io/kaniko-project/executor:debug
|
||||
environment:
|
||||
GITEA_USER:
|
||||
from_secret: gitea_username
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
CI_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
|
||||
CI_COMMIT_TAG: ${CI_COMMIT_TAG}
|
||||
commands:
|
||||
- *kaniko_setup
|
||||
- |
|
||||
DESTINATIONS=""
|
||||
if [ -n "$CI_COMMIT_TAG" ]; then
|
||||
DESTINATIONS="--destination git.mosaicstack.dev/mosaic/stack-web:$CI_COMMIT_TAG"
|
||||
elif [ "$CI_COMMIT_BRANCH" = "main" ]; then
|
||||
DESTINATIONS="--destination git.mosaicstack.dev/mosaic/stack-web:latest"
|
||||
fi
|
||||
/kaniko/executor --context . --dockerfile apps/web/Dockerfile --snapshot-mode=redo --build-arg NEXT_PUBLIC_API_URL=https://api.mosaicstack.dev $DESTINATIONS
|
||||
when:
|
||||
- branch: [main]
|
||||
event: [push, manual, tag]
|
||||
depends_on:
|
||||
- build
|
||||
|
||||
# ─── Container Security Scans (main only) ──────────────────
|
||||
|
||||
security-trivy-api:
|
||||
image: aquasec/trivy:latest
|
||||
environment:
|
||||
GITEA_USER:
|
||||
from_secret: gitea_username
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
CI_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
|
||||
CI_COMMIT_TAG: ${CI_COMMIT_TAG}
|
||||
commands:
|
||||
- |
|
||||
if [ -n "$$CI_COMMIT_TAG" ]; then SCAN_TAG="$$CI_COMMIT_TAG"; else SCAN_TAG="latest"; fi
|
||||
mkdir -p ~/.docker
|
||||
echo "{\"auths\":{\"git.mosaicstack.dev\":{\"username\":\"$$GITEA_USER\",\"password\":\"$$GITEA_TOKEN\"}}}" > ~/.docker/config.json
|
||||
trivy image --exit-code 1 --severity HIGH,CRITICAL --ignore-unfixed --ignorefile .trivyignore git.mosaicstack.dev/mosaic/stack-api:$$SCAN_TAG
|
||||
when:
|
||||
- branch: [main]
|
||||
event: [push, manual, tag]
|
||||
depends_on:
|
||||
- docker-build-api
|
||||
|
||||
security-trivy-orchestrator:
|
||||
image: aquasec/trivy:latest
|
||||
environment:
|
||||
GITEA_USER:
|
||||
from_secret: gitea_username
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
CI_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
|
||||
CI_COMMIT_TAG: ${CI_COMMIT_TAG}
|
||||
commands:
|
||||
- |
|
||||
if [ -n "$$CI_COMMIT_TAG" ]; then SCAN_TAG="$$CI_COMMIT_TAG"; else SCAN_TAG="latest"; fi
|
||||
mkdir -p ~/.docker
|
||||
echo "{\"auths\":{\"git.mosaicstack.dev\":{\"username\":\"$$GITEA_USER\",\"password\":\"$$GITEA_TOKEN\"}}}" > ~/.docker/config.json
|
||||
trivy image --exit-code 1 --severity HIGH,CRITICAL --ignore-unfixed --ignorefile .trivyignore git.mosaicstack.dev/mosaic/stack-orchestrator:$$SCAN_TAG
|
||||
when:
|
||||
- branch: [main]
|
||||
event: [push, manual, tag]
|
||||
depends_on:
|
||||
- docker-build-orchestrator
|
||||
|
||||
security-trivy-web:
|
||||
image: aquasec/trivy:latest
|
||||
environment:
|
||||
GITEA_USER:
|
||||
from_secret: gitea_username
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
CI_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
|
||||
CI_COMMIT_TAG: ${CI_COMMIT_TAG}
|
||||
commands:
|
||||
- |
|
||||
if [ -n "$$CI_COMMIT_TAG" ]; then SCAN_TAG="$$CI_COMMIT_TAG"; else SCAN_TAG="latest"; fi
|
||||
mkdir -p ~/.docker
|
||||
echo "{\"auths\":{\"git.mosaicstack.dev\":{\"username\":\"$$GITEA_USER\",\"password\":\"$$GITEA_TOKEN\"}}}" > ~/.docker/config.json
|
||||
trivy image --exit-code 1 --severity HIGH,CRITICAL --ignore-unfixed --ignorefile .trivyignore git.mosaicstack.dev/mosaic/stack-web:$$SCAN_TAG
|
||||
when:
|
||||
- branch: [main]
|
||||
event: [push, manual, tag]
|
||||
depends_on:
|
||||
- docker-build-web
|
||||
|
||||
# ─── Package Linking (main only, once) ─────────────────────
|
||||
|
||||
link-packages:
|
||||
image: alpine:3
|
||||
environment:
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
commands:
|
||||
- apk add --no-cache curl
|
||||
- sleep 10
|
||||
- |
|
||||
set -e
|
||||
link_package() {
|
||||
PKG="$$1"
|
||||
echo "Linking $$PKG..."
|
||||
for attempt in 1 2 3; do
|
||||
STATUS=$$(curl -s -o /tmp/link-response.txt -w "%{http_code}" -X POST \
|
||||
-H "Authorization: token $$GITEA_TOKEN" \
|
||||
"https://git.mosaicstack.dev/api/v1/packages/mosaic/container/$$PKG/-/link/stack")
|
||||
if [ "$$STATUS" = "201" ] || [ "$$STATUS" = "204" ]; then
|
||||
echo " Linked $$PKG"
|
||||
return 0
|
||||
elif [ "$$STATUS" = "400" ]; then
|
||||
echo " $$PKG already linked"
|
||||
return 0
|
||||
elif [ "$$STATUS" = "404" ] && [ $$attempt -lt 3 ]; then
|
||||
echo " $$PKG not found yet, retrying in 5s (attempt $$attempt/3)..."
|
||||
sleep 5
|
||||
else
|
||||
echo " FAILED: $$PKG status $$STATUS"
|
||||
cat /tmp/link-response.txt
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
link_package "stack-api"
|
||||
link_package "stack-orchestrator"
|
||||
link_package "stack-web"
|
||||
when:
|
||||
- branch: [main]
|
||||
event: [push, manual, tag]
|
||||
depends_on:
|
||||
- security-trivy-api
|
||||
- security-trivy-orchestrator
|
||||
- security-trivy-web
|
||||
@@ -1,202 +0,0 @@
|
||||
# Orchestrator Pipeline - Mosaic Stack
|
||||
# Quality gates, build, and Docker publish for @mosaic/orchestrator
|
||||
#
|
||||
# Triggers on: apps/orchestrator/**, packages/**, root configs
|
||||
# Security chain: source audit + Trivy container scan
|
||||
|
||||
when:
|
||||
- event: [push, pull_request, manual]
|
||||
path:
|
||||
include:
|
||||
- "apps/orchestrator/**"
|
||||
- "packages/**"
|
||||
- "pnpm-lock.yaml"
|
||||
- "pnpm-workspace.yaml"
|
||||
- "turbo.json"
|
||||
- "package.json"
|
||||
- ".woodpecker/orchestrator.yml"
|
||||
- ".trivyignore"
|
||||
|
||||
variables:
|
||||
- &node_image "node:24-alpine"
|
||||
- &install_deps |
|
||||
corepack enable
|
||||
pnpm install --frozen-lockfile
|
||||
- &use_deps |
|
||||
corepack enable
|
||||
- &turbo_env
|
||||
TURBO_API:
|
||||
from_secret: turbo_api
|
||||
TURBO_TOKEN:
|
||||
from_secret: turbo_token
|
||||
TURBO_TEAM:
|
||||
from_secret: turbo_team
|
||||
- &kaniko_setup |
|
||||
mkdir -p /kaniko/.docker
|
||||
echo "{\"auths\":{\"git.mosaicstack.dev\":{\"username\":\"$GITEA_USER\",\"password\":\"$GITEA_TOKEN\"}}}" > /kaniko/.docker/config.json
|
||||
|
||||
steps:
|
||||
# === Quality Gates ===
|
||||
|
||||
install:
|
||||
image: *node_image
|
||||
commands:
|
||||
- *install_deps
|
||||
|
||||
security-audit:
|
||||
image: *node_image
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm audit --audit-level=high
|
||||
depends_on:
|
||||
- install
|
||||
|
||||
lint:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
<<: *turbo_env
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm turbo lint --filter=@mosaic/orchestrator
|
||||
depends_on:
|
||||
- install
|
||||
|
||||
typecheck:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
<<: *turbo_env
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm turbo typecheck --filter=@mosaic/orchestrator
|
||||
depends_on:
|
||||
- install
|
||||
|
||||
test:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
<<: *turbo_env
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm turbo test --filter=@mosaic/orchestrator
|
||||
depends_on:
|
||||
- install
|
||||
|
||||
# === Build ===
|
||||
|
||||
build:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
NODE_ENV: "production"
|
||||
<<: *turbo_env
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm turbo build --filter=@mosaic/orchestrator
|
||||
depends_on:
|
||||
- lint
|
||||
- typecheck
|
||||
- test
|
||||
- security-audit
|
||||
|
||||
# === Docker Build & Push ===
|
||||
|
||||
docker-build-orchestrator:
|
||||
image: gcr.io/kaniko-project/executor:debug
|
||||
environment:
|
||||
GITEA_USER:
|
||||
from_secret: gitea_username
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
CI_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
|
||||
CI_COMMIT_TAG: ${CI_COMMIT_TAG}
|
||||
commands:
|
||||
- *kaniko_setup
|
||||
- |
|
||||
DESTINATIONS=""
|
||||
if [ -n "$CI_COMMIT_TAG" ]; then
|
||||
DESTINATIONS="--destination git.mosaicstack.dev/mosaic/stack-orchestrator:$CI_COMMIT_TAG"
|
||||
elif [ "$CI_COMMIT_BRANCH" = "main" ]; then
|
||||
DESTINATIONS="--destination git.mosaicstack.dev/mosaic/stack-orchestrator:latest"
|
||||
fi
|
||||
/kaniko/executor --context . --dockerfile apps/orchestrator/Dockerfile --snapshot-mode=redo $DESTINATIONS
|
||||
when:
|
||||
- branch: [main]
|
||||
event: [push, manual, tag]
|
||||
depends_on:
|
||||
- build
|
||||
|
||||
# === Container Security Scan ===
|
||||
|
||||
security-trivy-orchestrator:
|
||||
image: aquasec/trivy:latest
|
||||
environment:
|
||||
GITEA_USER:
|
||||
from_secret: gitea_username
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
CI_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
|
||||
CI_COMMIT_TAG: ${CI_COMMIT_TAG}
|
||||
commands:
|
||||
- |
|
||||
if [ -n "$$CI_COMMIT_TAG" ]; then
|
||||
SCAN_TAG="$$CI_COMMIT_TAG"
|
||||
elif [ "$$CI_COMMIT_BRANCH" = "main" ]; then
|
||||
SCAN_TAG="latest"
|
||||
else
|
||||
SCAN_TAG="latest"
|
||||
fi
|
||||
mkdir -p ~/.docker
|
||||
echo "{\"auths\":{\"git.mosaicstack.dev\":{\"username\":\"$$GITEA_USER\",\"password\":\"$$GITEA_TOKEN\"}}}" > ~/.docker/config.json
|
||||
trivy image --exit-code 1 --severity HIGH,CRITICAL --ignore-unfixed \
|
||||
--ignorefile .trivyignore \
|
||||
git.mosaicstack.dev/mosaic/stack-orchestrator:$$SCAN_TAG
|
||||
when:
|
||||
- branch: [main]
|
||||
event: [push, manual, tag]
|
||||
depends_on:
|
||||
- docker-build-orchestrator
|
||||
|
||||
# === Package Linking ===
|
||||
|
||||
link-packages:
|
||||
image: alpine:3
|
||||
environment:
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
commands:
|
||||
- apk add --no-cache curl
|
||||
- sleep 10
|
||||
- |
|
||||
set -e
|
||||
link_package() {
|
||||
PKG="$$1"
|
||||
echo "Linking $$PKG..."
|
||||
for attempt in 1 2 3; do
|
||||
STATUS=$$(curl -s -o /tmp/link-response.txt -w "%{http_code}" -X POST \
|
||||
-H "Authorization: token $$GITEA_TOKEN" \
|
||||
"https://git.mosaicstack.dev/api/v1/packages/mosaic/container/$$PKG/-/link/stack")
|
||||
if [ "$$STATUS" = "201" ] || [ "$$STATUS" = "204" ]; then
|
||||
echo " Linked $$PKG"
|
||||
return 0
|
||||
elif [ "$$STATUS" = "400" ]; then
|
||||
echo " $$PKG already linked"
|
||||
return 0
|
||||
elif [ "$$STATUS" = "404" ] && [ $$attempt -lt 3 ]; then
|
||||
echo " $$PKG not found yet, retrying in 5s (attempt $$attempt/3)..."
|
||||
sleep 5
|
||||
else
|
||||
echo " FAILED: $$PKG status $$STATUS"
|
||||
cat /tmp/link-response.txt
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
link_package "stack-orchestrator"
|
||||
when:
|
||||
- branch: [main]
|
||||
event: [push, manual, tag]
|
||||
depends_on:
|
||||
- security-trivy-orchestrator
|
||||
@@ -1,202 +0,0 @@
|
||||
# Web Pipeline - Mosaic Stack
|
||||
# Quality gates, build, and Docker publish for @mosaic/web
|
||||
#
|
||||
# Triggers on: apps/web/**, packages/**, root configs
|
||||
# Security chain: source audit + Trivy container scan
|
||||
|
||||
when:
|
||||
- event: [push, pull_request, manual]
|
||||
path:
|
||||
include:
|
||||
- "apps/web/**"
|
||||
- "packages/**"
|
||||
- "pnpm-lock.yaml"
|
||||
- "pnpm-workspace.yaml"
|
||||
- "turbo.json"
|
||||
- "package.json"
|
||||
- ".woodpecker/web.yml"
|
||||
- ".trivyignore"
|
||||
|
||||
variables:
|
||||
- &node_image "node:24-alpine"
|
||||
- &install_deps |
|
||||
corepack enable
|
||||
pnpm install --frozen-lockfile
|
||||
- &use_deps |
|
||||
corepack enable
|
||||
- &turbo_env
|
||||
TURBO_API:
|
||||
from_secret: turbo_api
|
||||
TURBO_TOKEN:
|
||||
from_secret: turbo_token
|
||||
TURBO_TEAM:
|
||||
from_secret: turbo_team
|
||||
- &kaniko_setup |
|
||||
mkdir -p /kaniko/.docker
|
||||
echo "{\"auths\":{\"git.mosaicstack.dev\":{\"username\":\"$GITEA_USER\",\"password\":\"$GITEA_TOKEN\"}}}" > /kaniko/.docker/config.json
|
||||
|
||||
steps:
|
||||
# === Quality Gates ===
|
||||
|
||||
install:
|
||||
image: *node_image
|
||||
commands:
|
||||
- *install_deps
|
||||
|
||||
security-audit:
|
||||
image: *node_image
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm audit --audit-level=high
|
||||
depends_on:
|
||||
- install
|
||||
|
||||
lint:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
<<: *turbo_env
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm turbo lint --filter=@mosaic/web
|
||||
depends_on:
|
||||
- install
|
||||
|
||||
typecheck:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
<<: *turbo_env
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm turbo typecheck --filter=@mosaic/web
|
||||
depends_on:
|
||||
- install
|
||||
|
||||
test:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
<<: *turbo_env
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm turbo test --filter=@mosaic/web
|
||||
depends_on:
|
||||
- install
|
||||
|
||||
# === Build ===
|
||||
|
||||
build:
|
||||
image: *node_image
|
||||
environment:
|
||||
SKIP_ENV_VALIDATION: "true"
|
||||
NODE_ENV: "production"
|
||||
<<: *turbo_env
|
||||
commands:
|
||||
- *use_deps
|
||||
- pnpm turbo build --filter=@mosaic/web
|
||||
depends_on:
|
||||
- lint
|
||||
- typecheck
|
||||
- test
|
||||
- security-audit
|
||||
|
||||
# === Docker Build & Push ===
|
||||
|
||||
docker-build-web:
|
||||
image: gcr.io/kaniko-project/executor:debug
|
||||
environment:
|
||||
GITEA_USER:
|
||||
from_secret: gitea_username
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
CI_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
|
||||
CI_COMMIT_TAG: ${CI_COMMIT_TAG}
|
||||
commands:
|
||||
- *kaniko_setup
|
||||
- |
|
||||
DESTINATIONS=""
|
||||
if [ -n "$CI_COMMIT_TAG" ]; then
|
||||
DESTINATIONS="--destination git.mosaicstack.dev/mosaic/stack-web:$CI_COMMIT_TAG"
|
||||
elif [ "$CI_COMMIT_BRANCH" = "main" ]; then
|
||||
DESTINATIONS="--destination git.mosaicstack.dev/mosaic/stack-web:latest"
|
||||
fi
|
||||
/kaniko/executor --context . --dockerfile apps/web/Dockerfile --snapshot-mode=redo --build-arg NEXT_PUBLIC_API_URL=https://api.mosaicstack.dev $DESTINATIONS
|
||||
when:
|
||||
- branch: [main]
|
||||
event: [push, manual, tag]
|
||||
depends_on:
|
||||
- build
|
||||
|
||||
# === Container Security Scan ===
|
||||
|
||||
security-trivy-web:
|
||||
image: aquasec/trivy:latest
|
||||
environment:
|
||||
GITEA_USER:
|
||||
from_secret: gitea_username
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
CI_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
|
||||
CI_COMMIT_TAG: ${CI_COMMIT_TAG}
|
||||
commands:
|
||||
- |
|
||||
if [ -n "$$CI_COMMIT_TAG" ]; then
|
||||
SCAN_TAG="$$CI_COMMIT_TAG"
|
||||
elif [ "$$CI_COMMIT_BRANCH" = "main" ]; then
|
||||
SCAN_TAG="latest"
|
||||
else
|
||||
SCAN_TAG="latest"
|
||||
fi
|
||||
mkdir -p ~/.docker
|
||||
echo "{\"auths\":{\"git.mosaicstack.dev\":{\"username\":\"$$GITEA_USER\",\"password\":\"$$GITEA_TOKEN\"}}}" > ~/.docker/config.json
|
||||
trivy image --exit-code 1 --severity HIGH,CRITICAL --ignore-unfixed \
|
||||
--ignorefile .trivyignore \
|
||||
git.mosaicstack.dev/mosaic/stack-web:$$SCAN_TAG
|
||||
when:
|
||||
- branch: [main]
|
||||
event: [push, manual, tag]
|
||||
depends_on:
|
||||
- docker-build-web
|
||||
|
||||
# === Package Linking ===
|
||||
|
||||
link-packages:
|
||||
image: alpine:3
|
||||
environment:
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
commands:
|
||||
- apk add --no-cache curl
|
||||
- sleep 10
|
||||
- |
|
||||
set -e
|
||||
link_package() {
|
||||
PKG="$$1"
|
||||
echo "Linking $$PKG..."
|
||||
for attempt in 1 2 3; do
|
||||
STATUS=$$(curl -s -o /tmp/link-response.txt -w "%{http_code}" -X POST \
|
||||
-H "Authorization: token $$GITEA_TOKEN" \
|
||||
"https://git.mosaicstack.dev/api/v1/packages/mosaic/container/$$PKG/-/link/stack")
|
||||
if [ "$$STATUS" = "201" ] || [ "$$STATUS" = "204" ]; then
|
||||
echo " Linked $$PKG"
|
||||
return 0
|
||||
elif [ "$$STATUS" = "400" ]; then
|
||||
echo " $$PKG already linked"
|
||||
return 0
|
||||
elif [ "$$STATUS" = "404" ] && [ $$attempt -lt 3 ]; then
|
||||
echo " $$PKG not found yet, retrying in 5s (attempt $$attempt/3)..."
|
||||
sleep 5
|
||||
else
|
||||
echo " FAILED: $$PKG status $$STATUS"
|
||||
cat /tmp/link-response.txt
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
link_package "stack-web"
|
||||
when:
|
||||
- branch: [main]
|
||||
event: [push, manual, tag]
|
||||
depends_on:
|
||||
- security-trivy-web
|
||||
@@ -0,0 +1,24 @@
|
||||
-- CreateTable
|
||||
CREATE TABLE "agent_memories" (
|
||||
"id" UUID NOT NULL,
|
||||
"workspace_id" UUID NOT NULL,
|
||||
"agent_id" TEXT NOT NULL,
|
||||
"key" TEXT NOT NULL,
|
||||
"value" JSONB NOT NULL,
|
||||
"created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updated_at" TIMESTAMPTZ NOT NULL,
|
||||
|
||||
CONSTRAINT "agent_memories_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "agent_memories_workspace_id_agent_id_key_key" ON "agent_memories"("workspace_id", "agent_id", "key");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "agent_memories_workspace_id_idx" ON "agent_memories"("workspace_id");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "agent_memories_agent_id_idx" ON "agent_memories"("agent_id");
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "agent_memories" ADD CONSTRAINT "agent_memories_workspace_id_fkey" FOREIGN KEY ("workspace_id") REFERENCES "workspaces"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
@@ -299,6 +299,7 @@ model Workspace {
|
||||
agentSessions AgentSession[]
|
||||
agentTasks AgentTask[]
|
||||
findings Finding[]
|
||||
agentMemories AgentMemory[]
|
||||
userLayouts UserLayout[]
|
||||
knowledgeEntries KnowledgeEntry[]
|
||||
knowledgeTags KnowledgeTag[]
|
||||
@@ -765,6 +766,23 @@ model AgentSession {
|
||||
@@map("agent_sessions")
|
||||
}
|
||||
|
||||
model AgentMemory {
|
||||
id String @id @default(uuid()) @db.Uuid
|
||||
workspaceId String @map("workspace_id") @db.Uuid
|
||||
agentId String @map("agent_id")
|
||||
key String
|
||||
value Json
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz
|
||||
updatedAt DateTime @updatedAt @map("updated_at") @db.Timestamptz
|
||||
|
||||
workspace Workspace @relation(fields: [workspaceId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@unique([workspaceId, agentId, key])
|
||||
@@index([workspaceId])
|
||||
@@index([agentId])
|
||||
@@map("agent_memories")
|
||||
}
|
||||
|
||||
model WidgetDefinition {
|
||||
id String @id @default(uuid()) @db.Uuid
|
||||
|
||||
|
||||
102
apps/api/src/agent-memory/agent-memory.controller.spec.ts
Normal file
102
apps/api/src/agent-memory/agent-memory.controller.spec.ts
Normal file
@@ -0,0 +1,102 @@
|
||||
import { Test, TestingModule } from "@nestjs/testing";
|
||||
import { AgentMemoryController } from "./agent-memory.controller";
|
||||
import { AgentMemoryService } from "./agent-memory.service";
|
||||
import { AuthGuard } from "../auth/guards/auth.guard";
|
||||
import { WorkspaceGuard, PermissionGuard } from "../common/guards";
|
||||
import { describe, it, expect, beforeEach, vi } from "vitest";
|
||||
|
||||
describe("AgentMemoryController", () => {
|
||||
let controller: AgentMemoryController;
|
||||
|
||||
const mockAgentMemoryService = {
|
||||
upsert: vi.fn(),
|
||||
findAll: vi.fn(),
|
||||
findOne: vi.fn(),
|
||||
remove: vi.fn(),
|
||||
};
|
||||
|
||||
const mockGuard = { canActivate: vi.fn(() => true) };
|
||||
|
||||
beforeEach(async () => {
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
controllers: [AgentMemoryController],
|
||||
providers: [
|
||||
{
|
||||
provide: AgentMemoryService,
|
||||
useValue: mockAgentMemoryService,
|
||||
},
|
||||
],
|
||||
})
|
||||
.overrideGuard(AuthGuard)
|
||||
.useValue(mockGuard)
|
||||
.overrideGuard(WorkspaceGuard)
|
||||
.useValue(mockGuard)
|
||||
.overrideGuard(PermissionGuard)
|
||||
.useValue(mockGuard)
|
||||
.compile();
|
||||
|
||||
controller = module.get<AgentMemoryController>(AgentMemoryController);
|
||||
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
const workspaceId = "workspace-1";
|
||||
const agentId = "agent-1";
|
||||
const key = "context";
|
||||
|
||||
describe("upsert", () => {
|
||||
it("should upsert a memory entry", async () => {
|
||||
const dto = { value: { foo: "bar" } };
|
||||
const mockEntry = { id: "mem-1", workspaceId, agentId, key, value: dto.value };
|
||||
|
||||
mockAgentMemoryService.upsert.mockResolvedValue(mockEntry);
|
||||
|
||||
const result = await controller.upsert(agentId, key, dto, workspaceId);
|
||||
|
||||
expect(mockAgentMemoryService.upsert).toHaveBeenCalledWith(workspaceId, agentId, key, dto);
|
||||
expect(result).toEqual(mockEntry);
|
||||
});
|
||||
});
|
||||
|
||||
describe("findAll", () => {
|
||||
it("should list all memory entries for an agent", async () => {
|
||||
const mockEntries = [
|
||||
{ id: "mem-1", key: "a", value: 1 },
|
||||
{ id: "mem-2", key: "b", value: 2 },
|
||||
];
|
||||
|
||||
mockAgentMemoryService.findAll.mockResolvedValue(mockEntries);
|
||||
|
||||
const result = await controller.findAll(agentId, workspaceId);
|
||||
|
||||
expect(mockAgentMemoryService.findAll).toHaveBeenCalledWith(workspaceId, agentId);
|
||||
expect(result).toEqual(mockEntries);
|
||||
});
|
||||
});
|
||||
|
||||
describe("findOne", () => {
|
||||
it("should get a single memory entry", async () => {
|
||||
const mockEntry = { id: "mem-1", key, value: "v" };
|
||||
|
||||
mockAgentMemoryService.findOne.mockResolvedValue(mockEntry);
|
||||
|
||||
const result = await controller.findOne(agentId, key, workspaceId);
|
||||
|
||||
expect(mockAgentMemoryService.findOne).toHaveBeenCalledWith(workspaceId, agentId, key);
|
||||
expect(result).toEqual(mockEntry);
|
||||
});
|
||||
});
|
||||
|
||||
describe("remove", () => {
|
||||
it("should delete a memory entry", async () => {
|
||||
const mockResponse = { message: "Memory entry deleted successfully" };
|
||||
|
||||
mockAgentMemoryService.remove.mockResolvedValue(mockResponse);
|
||||
|
||||
const result = await controller.remove(agentId, key, workspaceId);
|
||||
|
||||
expect(mockAgentMemoryService.remove).toHaveBeenCalledWith(workspaceId, agentId, key);
|
||||
expect(result).toEqual(mockResponse);
|
||||
});
|
||||
});
|
||||
});
|
||||
89
apps/api/src/agent-memory/agent-memory.controller.ts
Normal file
89
apps/api/src/agent-memory/agent-memory.controller.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Put,
|
||||
Delete,
|
||||
Body,
|
||||
Param,
|
||||
UseGuards,
|
||||
HttpCode,
|
||||
HttpStatus,
|
||||
} from "@nestjs/common";
|
||||
import { AgentMemoryService } from "./agent-memory.service";
|
||||
import { UpsertAgentMemoryDto } from "./dto";
|
||||
import { AuthGuard } from "../auth/guards/auth.guard";
|
||||
import { WorkspaceGuard, PermissionGuard } from "../common/guards";
|
||||
import { Workspace, Permission, RequirePermission } from "../common/decorators";
|
||||
|
||||
/**
|
||||
* Controller for per-agent key/value memory endpoints.
|
||||
* All endpoints require authentication and workspace context.
|
||||
*
|
||||
* Guards are applied in order:
|
||||
* 1. AuthGuard - Verifies user authentication
|
||||
* 2. WorkspaceGuard - Validates workspace access
|
||||
* 3. PermissionGuard - Checks role-based permissions
|
||||
*/
|
||||
@Controller("agents/:agentId/memory")
|
||||
@UseGuards(AuthGuard, WorkspaceGuard, PermissionGuard)
|
||||
export class AgentMemoryController {
|
||||
constructor(private readonly agentMemoryService: AgentMemoryService) {}
|
||||
|
||||
/**
|
||||
* PUT /api/agents/:agentId/memory/:key
|
||||
* Upsert a memory entry for an agent
|
||||
* Requires: MEMBER role or higher
|
||||
*/
|
||||
@Put(":key")
|
||||
@RequirePermission(Permission.WORKSPACE_MEMBER)
|
||||
async upsert(
|
||||
@Param("agentId") agentId: string,
|
||||
@Param("key") key: string,
|
||||
@Body() dto: UpsertAgentMemoryDto,
|
||||
@Workspace() workspaceId: string
|
||||
) {
|
||||
return this.agentMemoryService.upsert(workspaceId, agentId, key, dto);
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/agents/:agentId/memory
|
||||
* List all memory entries for an agent
|
||||
* Requires: Any workspace member (including GUEST)
|
||||
*/
|
||||
@Get()
|
||||
@RequirePermission(Permission.WORKSPACE_ANY)
|
||||
async findAll(@Param("agentId") agentId: string, @Workspace() workspaceId: string) {
|
||||
return this.agentMemoryService.findAll(workspaceId, agentId);
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/agents/:agentId/memory/:key
|
||||
* Get a single memory entry by key
|
||||
* Requires: Any workspace member (including GUEST)
|
||||
*/
|
||||
@Get(":key")
|
||||
@RequirePermission(Permission.WORKSPACE_ANY)
|
||||
async findOne(
|
||||
@Param("agentId") agentId: string,
|
||||
@Param("key") key: string,
|
||||
@Workspace() workspaceId: string
|
||||
) {
|
||||
return this.agentMemoryService.findOne(workspaceId, agentId, key);
|
||||
}
|
||||
|
||||
/**
|
||||
* DELETE /api/agents/:agentId/memory/:key
|
||||
* Remove a memory entry
|
||||
* Requires: MEMBER role or higher
|
||||
*/
|
||||
@Delete(":key")
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@RequirePermission(Permission.WORKSPACE_MEMBER)
|
||||
async remove(
|
||||
@Param("agentId") agentId: string,
|
||||
@Param("key") key: string,
|
||||
@Workspace() workspaceId: string
|
||||
) {
|
||||
return this.agentMemoryService.remove(workspaceId, agentId, key);
|
||||
}
|
||||
}
|
||||
13
apps/api/src/agent-memory/agent-memory.module.ts
Normal file
13
apps/api/src/agent-memory/agent-memory.module.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
import { Module } from "@nestjs/common";
|
||||
import { AgentMemoryController } from "./agent-memory.controller";
|
||||
import { AgentMemoryService } from "./agent-memory.service";
|
||||
import { PrismaModule } from "../prisma/prisma.module";
|
||||
import { AuthModule } from "../auth/auth.module";
|
||||
|
||||
@Module({
|
||||
imports: [PrismaModule, AuthModule],
|
||||
controllers: [AgentMemoryController],
|
||||
providers: [AgentMemoryService],
|
||||
exports: [AgentMemoryService],
|
||||
})
|
||||
export class AgentMemoryModule {}
|
||||
126
apps/api/src/agent-memory/agent-memory.service.spec.ts
Normal file
126
apps/api/src/agent-memory/agent-memory.service.spec.ts
Normal file
@@ -0,0 +1,126 @@
|
||||
import { Test, TestingModule } from "@nestjs/testing";
|
||||
import { AgentMemoryService } from "./agent-memory.service";
|
||||
import { PrismaService } from "../prisma/prisma.service";
|
||||
import { NotFoundException } from "@nestjs/common";
|
||||
import { describe, it, expect, beforeEach, vi } from "vitest";
|
||||
|
||||
describe("AgentMemoryService", () => {
|
||||
let service: AgentMemoryService;
|
||||
|
||||
const mockPrismaService = {
|
||||
agentMemory: {
|
||||
upsert: vi.fn(),
|
||||
findMany: vi.fn(),
|
||||
findUnique: vi.fn(),
|
||||
delete: vi.fn(),
|
||||
},
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
AgentMemoryService,
|
||||
{
|
||||
provide: PrismaService,
|
||||
useValue: mockPrismaService,
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<AgentMemoryService>(AgentMemoryService);
|
||||
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
const workspaceId = "workspace-1";
|
||||
const agentId = "agent-1";
|
||||
const key = "session-context";
|
||||
|
||||
describe("upsert", () => {
|
||||
it("should upsert a memory entry", async () => {
|
||||
const dto = { value: { data: "some context" } };
|
||||
const mockEntry = {
|
||||
id: "mem-1",
|
||||
workspaceId,
|
||||
agentId,
|
||||
key,
|
||||
value: dto.value,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
};
|
||||
|
||||
mockPrismaService.agentMemory.upsert.mockResolvedValue(mockEntry);
|
||||
|
||||
const result = await service.upsert(workspaceId, agentId, key, dto);
|
||||
|
||||
expect(mockPrismaService.agentMemory.upsert).toHaveBeenCalledWith({
|
||||
where: { workspaceId_agentId_key: { workspaceId, agentId, key } },
|
||||
create: { workspaceId, agentId, key, value: dto.value },
|
||||
update: { value: dto.value },
|
||||
});
|
||||
expect(result).toEqual(mockEntry);
|
||||
});
|
||||
});
|
||||
|
||||
describe("findAll", () => {
|
||||
it("should return all memory entries for an agent", async () => {
|
||||
const mockEntries = [
|
||||
{ id: "mem-1", key: "a", value: 1 },
|
||||
{ id: "mem-2", key: "b", value: 2 },
|
||||
];
|
||||
|
||||
mockPrismaService.agentMemory.findMany.mockResolvedValue(mockEntries);
|
||||
|
||||
const result = await service.findAll(workspaceId, agentId);
|
||||
|
||||
expect(mockPrismaService.agentMemory.findMany).toHaveBeenCalledWith({
|
||||
where: { workspaceId, agentId },
|
||||
orderBy: { key: "asc" },
|
||||
});
|
||||
expect(result).toEqual(mockEntries);
|
||||
});
|
||||
});
|
||||
|
||||
describe("findOne", () => {
|
||||
it("should return a memory entry by key", async () => {
|
||||
const mockEntry = { id: "mem-1", workspaceId, agentId, key, value: "ctx" };
|
||||
|
||||
mockPrismaService.agentMemory.findUnique.mockResolvedValue(mockEntry);
|
||||
|
||||
const result = await service.findOne(workspaceId, agentId, key);
|
||||
|
||||
expect(mockPrismaService.agentMemory.findUnique).toHaveBeenCalledWith({
|
||||
where: { workspaceId_agentId_key: { workspaceId, agentId, key } },
|
||||
});
|
||||
expect(result).toEqual(mockEntry);
|
||||
});
|
||||
|
||||
it("should throw NotFoundException when key not found", async () => {
|
||||
mockPrismaService.agentMemory.findUnique.mockResolvedValue(null);
|
||||
|
||||
await expect(service.findOne(workspaceId, agentId, key)).rejects.toThrow(NotFoundException);
|
||||
});
|
||||
});
|
||||
|
||||
describe("remove", () => {
|
||||
it("should delete a memory entry", async () => {
|
||||
const mockEntry = { id: "mem-1", workspaceId, agentId, key, value: "x" };
|
||||
|
||||
mockPrismaService.agentMemory.findUnique.mockResolvedValue(mockEntry);
|
||||
mockPrismaService.agentMemory.delete.mockResolvedValue(mockEntry);
|
||||
|
||||
const result = await service.remove(workspaceId, agentId, key);
|
||||
|
||||
expect(mockPrismaService.agentMemory.delete).toHaveBeenCalledWith({
|
||||
where: { workspaceId_agentId_key: { workspaceId, agentId, key } },
|
||||
});
|
||||
expect(result).toEqual({ message: "Memory entry deleted successfully" });
|
||||
});
|
||||
|
||||
it("should throw NotFoundException when key not found", async () => {
|
||||
mockPrismaService.agentMemory.findUnique.mockResolvedValue(null);
|
||||
|
||||
await expect(service.remove(workspaceId, agentId, key)).rejects.toThrow(NotFoundException);
|
||||
});
|
||||
});
|
||||
});
|
||||
79
apps/api/src/agent-memory/agent-memory.service.ts
Normal file
79
apps/api/src/agent-memory/agent-memory.service.ts
Normal file
@@ -0,0 +1,79 @@
|
||||
import { Injectable, NotFoundException } from "@nestjs/common";
|
||||
import { PrismaService } from "../prisma/prisma.service";
|
||||
import { Prisma } from "@prisma/client";
|
||||
import type { UpsertAgentMemoryDto } from "./dto";
|
||||
|
||||
@Injectable()
|
||||
export class AgentMemoryService {
|
||||
constructor(private readonly prisma: PrismaService) {}
|
||||
|
||||
/**
|
||||
* Upsert a memory entry for an agent.
|
||||
*/
|
||||
async upsert(workspaceId: string, agentId: string, key: string, dto: UpsertAgentMemoryDto) {
|
||||
return this.prisma.agentMemory.upsert({
|
||||
where: {
|
||||
workspaceId_agentId_key: { workspaceId, agentId, key },
|
||||
},
|
||||
create: {
|
||||
workspaceId,
|
||||
agentId,
|
||||
key,
|
||||
value: dto.value as Prisma.InputJsonValue,
|
||||
},
|
||||
update: {
|
||||
value: dto.value as Prisma.InputJsonValue,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* List all memory entries for an agent in a workspace.
|
||||
*/
|
||||
async findAll(workspaceId: string, agentId: string) {
|
||||
return this.prisma.agentMemory.findMany({
|
||||
where: { workspaceId, agentId },
|
||||
orderBy: { key: "asc" },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a single memory entry by key.
|
||||
*/
|
||||
async findOne(workspaceId: string, agentId: string, key: string) {
|
||||
const entry = await this.prisma.agentMemory.findUnique({
|
||||
where: {
|
||||
workspaceId_agentId_key: { workspaceId, agentId, key },
|
||||
},
|
||||
});
|
||||
|
||||
if (!entry) {
|
||||
throw new NotFoundException(`Memory key "${key}" not found for agent "${agentId}"`);
|
||||
}
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a memory entry by key.
|
||||
*/
|
||||
async remove(workspaceId: string, agentId: string, key: string) {
|
||||
const entry = await this.prisma.agentMemory.findUnique({
|
||||
where: {
|
||||
workspaceId_agentId_key: { workspaceId, agentId, key },
|
||||
},
|
||||
});
|
||||
|
||||
if (!entry) {
|
||||
throw new NotFoundException(`Memory key "${key}" not found for agent "${agentId}"`);
|
||||
}
|
||||
|
||||
await this.prisma.agentMemory.delete({
|
||||
where: {
|
||||
workspaceId_agentId_key: { workspaceId, agentId, key },
|
||||
},
|
||||
});
|
||||
|
||||
return { message: "Memory entry deleted successfully" };
|
||||
}
|
||||
}
|
||||
1
apps/api/src/agent-memory/dto/index.ts
Normal file
1
apps/api/src/agent-memory/dto/index.ts
Normal file
@@ -0,0 +1 @@
|
||||
export * from "./upsert-agent-memory.dto";
|
||||
10
apps/api/src/agent-memory/dto/upsert-agent-memory.dto.ts
Normal file
10
apps/api/src/agent-memory/dto/upsert-agent-memory.dto.ts
Normal file
@@ -0,0 +1,10 @@
|
||||
import { IsNotEmpty } from "class-validator";
|
||||
|
||||
/**
|
||||
* DTO for upserting an agent memory entry.
|
||||
* The value accepts any JSON-serializable data.
|
||||
*/
|
||||
export class UpsertAgentMemoryDto {
|
||||
@IsNotEmpty({ message: "value must not be empty" })
|
||||
value!: unknown;
|
||||
}
|
||||
@@ -28,6 +28,7 @@ import { BrainModule } from "./brain/brain.module";
|
||||
import { CronModule } from "./cron/cron.module";
|
||||
import { AgentTasksModule } from "./agent-tasks/agent-tasks.module";
|
||||
import { FindingsModule } from "./findings/findings.module";
|
||||
import { AgentMemoryModule } from "./agent-memory/agent-memory.module";
|
||||
import { ValkeyModule } from "./valkey/valkey.module";
|
||||
import { BullMqModule } from "./bullmq/bullmq.module";
|
||||
import { StitcherModule } from "./stitcher/stitcher.module";
|
||||
@@ -103,6 +104,7 @@ import { RlsContextInterceptor } from "./common/interceptors/rls-context.interce
|
||||
CronModule,
|
||||
AgentTasksModule,
|
||||
FindingsModule,
|
||||
AgentMemoryModule,
|
||||
RunnerJobsModule,
|
||||
JobEventsModule,
|
||||
JobStepsModule,
|
||||
|
||||
@@ -52,3 +52,22 @@
|
||||
| **Total** | **31** | **15** | **~371K** | **~175K** |
|
||||
|
||||
Remaining estimate: ~143K tokens (Codex budget).
|
||||
|
||||
## MS22 — Fleet Evolution (Phase 0: Knowledge Layer)
|
||||
|
||||
| id | status | milestone | description | issue | repo | branch | depends_on | blocks | agent | started_at | completed_at | estimate | used | notes |
|
||||
| --------------- | ----------- | ------------ | ------------------------------------------------------------ | -------- | ----- | ------------------------------ | --------------------------------------------------------- | ------------- | ------------ | ---------- | ------------ | -------- | ---- | --------------------------------------------- |
|
||||
| MS22-PLAN-001 | done | p0-knowledge | PRD + mission bootstrap + TASKS.md | TASKS:P0 | stack | feat/ms22-knowledge-schema | — | MS22-DB-001 | orchestrator | 2026-02-28 | 2026-02-28 | 10K | 8K | PRD-MS22.md, mission fleet-evolution-20260228 |
|
||||
| MS22-DB-001 | done | p0-knowledge | Findings module (pgvector, CRUD, similarity search) | TASKS:P0 | api | feat/ms22-findings | MS22-PLAN-001 | — | codex | 2026-02-28 | 2026-02-28 | 20K | ~22K | PR #585 merged, CI green |
|
||||
| MS22-API-001 | done | p0-knowledge | Findings API endpoints | TASKS:P0 | api | feat/ms22-findings | MS22-DB-001 | — | codex | 2026-02-28 | 2026-02-28 | — | — | Combined with DB-001 |
|
||||
| MS22-DB-002 | done | p0-knowledge | AgentMemory module (key/value store, upsert) | TASKS:P0 | api | feat/ms22-agent-memory | MS22-DB-001 | — | codex | 2026-02-28 | 2026-02-28 | 15K | ~16K | PR #586 merged, CI green |
|
||||
| MS22-API-002 | done | p0-knowledge | AgentMemory API endpoints | TASKS:P0 | api | feat/ms22-agent-memory | MS22-DB-002 | — | codex | 2026-02-28 | 2026-02-28 | — | — | Combined with DB-002 |
|
||||
| MS22-DB-004 | done | p0-knowledge | ConversationArchive module (pgvector, ingest, search) | TASKS:P0 | api | feat/ms22-conversation-archive | MS22-DB-001 | — | codex | 2026-02-28 | 2026-02-28 | 20K | ~18K | PR #587 merged, CI green |
|
||||
| MS22-API-004 | done | p0-knowledge | ConversationArchive API endpoints | TASKS:P0 | api | feat/ms22-conversation-archive | MS22-DB-004 | — | codex | 2026-02-28 | 2026-02-28 | — | — | Combined with DB-004 |
|
||||
| MS22-API-005 | done | p0-knowledge | EmbeddingService (reuse existing KnowledgeModule) | TASKS:P0 | api | — | — | — | orchestrator | 2026-02-28 | 2026-02-28 | 0 | 0 | Already existed; no work needed |
|
||||
| MS22-DB-003 | not-started | p0-knowledge | Task model: add assigned_agent field + migration | TASKS:P0 | api | feat/ms22-task-agent | MS22-DB-001 | MS22-API-003 | — | — | — | 8K | — | Small schema + migration only |
|
||||
| MS22-API-003 | not-started | p0-knowledge | Task API: expose assigned_agent in CRUD | TASKS:P0 | api | feat/ms22-task-agent | MS22-DB-003 | MS22-TEST-001 | — | — | — | 8K | — | Extend existing TaskModule |
|
||||
| MS22-TEST-001 | not-started | p0-knowledge | Integration tests: Findings + AgentMemory + ConvArchive | TASKS:P0 | api | test/ms22-integration | MS22-API-001,MS22-API-002,MS22-API-004 | MS22-VER-P0 | — | — | — | 20K | — | E2E with live postgres |
|
||||
| MS22-SKILL-001 | not-started | p0-knowledge | OpenClaw mosaic skill (agents read/write findings/memory) | TASKS:P0 | stack | feat/ms22-openclaw-skill | MS22-API-001,MS22-API-002 | MS22-VER-P0 | — | — | — | 15K | — | Skill in ~/.agents/skills/mosaic/ |
|
||||
| MS22-INGEST-001 | not-started | p0-knowledge | Session log ingestion pipeline (OpenClaw logs → ConvArchive) | TASKS:P0 | stack | feat/ms22-ingest | MS22-API-004 | MS22-VER-P0 | — | — | — | 20K | — | Script to batch-ingest existing logs |
|
||||
| MS22-VER-P0 | not-started | p0-knowledge | Phase 0 verification: all modules deployed + smoke tested | TASKS:P0 | stack | — | MS22-TEST-001,MS22-SKILL-001,MS22-INGEST-001,MS22-API-003 | — | — | — | — | 5K | — | |
|
||||
|
||||
64
docs/scratchpads/ms22-agent-memory.md
Normal file
64
docs/scratchpads/ms22-agent-memory.md
Normal file
@@ -0,0 +1,64 @@
|
||||
# MS22 Agent Memory Module
|
||||
|
||||
## Objective
|
||||
|
||||
Add per-agent key/value store: AgentMemory model + NestJS module with CRUD endpoints.
|
||||
|
||||
## Issues
|
||||
|
||||
- MS22-DB-002: Add AgentMemory schema model
|
||||
- MS22-API-002: Add agent-memory NestJS module
|
||||
|
||||
## Plan
|
||||
|
||||
1. AgentMemory model → schema.prisma (after AgentSession, line 736)
|
||||
2. Add `agentMemories AgentMemory[]` relation to Workspace model
|
||||
3. Create apps/api/src/agent-memory/ with service, controller, DTOs, specs
|
||||
4. Register in app.module.ts
|
||||
5. Migrate: `prisma migrate dev --name ms22_agent_memory`
|
||||
6. lint + build
|
||||
7. Commit
|
||||
|
||||
## Endpoints
|
||||
|
||||
- PUT /api/agents/:agentId/memory/:key (upsert)
|
||||
- GET /api/agents/:agentId/memory (list all)
|
||||
- GET /api/agents/:agentId/memory/:key (get one)
|
||||
- DELETE /api/agents/:agentId/memory/:key (remove)
|
||||
|
||||
## Auth
|
||||
|
||||
- @UseGuards(AuthGuard, WorkspaceGuard, PermissionGuard)
|
||||
- @Workspace() decorator for workspaceId
|
||||
- Permission.WORKSPACE_MEMBER for write ops
|
||||
- Permission.WORKSPACE_ANY for read ops
|
||||
|
||||
## Schema
|
||||
|
||||
```prisma
|
||||
model AgentMemory {
|
||||
id String @id @default(uuid()) @db.Uuid
|
||||
workspaceId String @map("workspace_id") @db.Uuid
|
||||
agentId String @map("agent_id")
|
||||
key String
|
||||
value Json
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz
|
||||
updatedAt DateTime @updatedAt @map("updated_at") @db.Timestamptz
|
||||
|
||||
workspace Workspace @relation(fields: [workspaceId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@unique([workspaceId, agentId, key])
|
||||
@@index([workspaceId])
|
||||
@@index([agentId])
|
||||
@@map("agent_memories")
|
||||
}
|
||||
```
|
||||
|
||||
## Progress
|
||||
|
||||
- [ ] Schema
|
||||
- [ ] Module files
|
||||
- [ ] app.module.ts
|
||||
- [ ] Migration
|
||||
- [ ] lint/build
|
||||
- [ ] Commit
|
||||
@@ -35,7 +35,8 @@
|
||||
"docker:ps": "docker compose ps",
|
||||
"docker:build": "docker compose build",
|
||||
"docker:restart": "docker compose restart",
|
||||
"prepare": "husky || true"
|
||||
"prepare": "husky || true",
|
||||
"ingest:sessions": "tsx scripts/ingest-openclaw-sessions.ts"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@typescript-eslint/eslint-plugin": "^8.26.0",
|
||||
|
||||
621
scripts/ingest-openclaw-sessions.ts
Normal file
621
scripts/ingest-openclaw-sessions.ts
Normal file
@@ -0,0 +1,621 @@
|
||||
import { createReadStream, constants as fsConstants } from "node:fs";
|
||||
import { access, readdir, stat } from "node:fs/promises";
|
||||
import { homedir } from "node:os";
|
||||
import * as path from "node:path";
|
||||
import * as process from "node:process";
|
||||
import { createInterface } from "node:readline";
|
||||
|
||||
const DEFAULT_ENDPOINT = "https://mosaic-api.woltje.com/conversation-archive/ingest";
|
||||
|
||||
type IngestRole = "user" | "assistant";
|
||||
|
||||
interface IngestMessage {
|
||||
role: IngestRole;
|
||||
content: string;
|
||||
timestamp?: string;
|
||||
}
|
||||
|
||||
interface IngestPayload {
|
||||
sessionId: string;
|
||||
workspaceId: string;
|
||||
title: string;
|
||||
messages: IngestMessage[];
|
||||
agentId?: string;
|
||||
}
|
||||
|
||||
interface CliOptions {
|
||||
workspaceId: string;
|
||||
agentId?: string;
|
||||
since?: Date;
|
||||
sessionsDir?: string;
|
||||
endpoint: string;
|
||||
}
|
||||
|
||||
interface ParsedSession {
|
||||
sessionId: string;
|
||||
title: string;
|
||||
messages: IngestMessage[];
|
||||
startedAt?: string;
|
||||
endedAt?: string;
|
||||
parseErrors: number;
|
||||
inferredAgentId?: string;
|
||||
}
|
||||
|
||||
interface SendResult {
|
||||
ok: boolean;
|
||||
status: number;
|
||||
body: string;
|
||||
}
|
||||
|
||||
interface IngestSummary {
|
||||
discovered: number;
|
||||
processed: number;
|
||||
ingested: number;
|
||||
skippedSince: number;
|
||||
skippedEmpty: number;
|
||||
skippedDuplicate: number;
|
||||
failed: number;
|
||||
}
|
||||
|
||||
function printUsage(): void {
|
||||
console.log(
|
||||
[
|
||||
"Usage:",
|
||||
" pnpm ingest:sessions --workspace-id <id> [--agent-id <id>] [--since <ISO date>] [--sessions-dir <path>] [--endpoint <url>]",
|
||||
"",
|
||||
"Required:",
|
||||
" --workspace-id Target Mosaic workspace ID",
|
||||
"",
|
||||
"Optional:",
|
||||
" --agent-id Agent ID to include in each ingest payload",
|
||||
" --since Skip sessions before this date/time (ISO8601 or YYYY-MM-DD)",
|
||||
" --sessions-dir Override session directory path",
|
||||
` --endpoint Ingest endpoint (default: ${DEFAULT_ENDPOINT})`,
|
||||
].join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
function expandHomePath(inputPath: string): string {
|
||||
if (inputPath === "~") {
|
||||
return homedir();
|
||||
}
|
||||
if (inputPath.startsWith("~/")) {
|
||||
return path.join(homedir(), inputPath.slice(2));
|
||||
}
|
||||
return inputPath;
|
||||
}
|
||||
|
||||
function parseSinceDate(rawDate: string): Date {
|
||||
const parsed = new Date(rawDate);
|
||||
if (Number.isNaN(parsed.getTime())) {
|
||||
throw new Error(`Invalid --since date: "${rawDate}". Use ISO8601 or YYYY-MM-DD.`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseCliArgs(args: string[]): CliOptions {
|
||||
let workspaceId: string | null = null;
|
||||
let agentId: string | undefined;
|
||||
let since: Date | undefined;
|
||||
let sessionsDir: string | undefined;
|
||||
let endpoint = DEFAULT_ENDPOINT;
|
||||
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index];
|
||||
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
printUsage();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (arg.startsWith("--workspace-id=")) {
|
||||
workspaceId = arg.slice("--workspace-id=".length);
|
||||
continue;
|
||||
}
|
||||
if (arg === "--workspace-id") {
|
||||
const value = args[index + 1];
|
||||
if (!value) {
|
||||
throw new Error("Missing value for --workspace-id");
|
||||
}
|
||||
workspaceId = value;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg.startsWith("--agent-id=")) {
|
||||
agentId = arg.slice("--agent-id=".length);
|
||||
continue;
|
||||
}
|
||||
if (arg === "--agent-id") {
|
||||
const value = args[index + 1];
|
||||
if (!value) {
|
||||
throw new Error("Missing value for --agent-id");
|
||||
}
|
||||
agentId = value;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg.startsWith("--since=")) {
|
||||
since = parseSinceDate(arg.slice("--since=".length));
|
||||
continue;
|
||||
}
|
||||
if (arg === "--since") {
|
||||
const value = args[index + 1];
|
||||
if (!value) {
|
||||
throw new Error("Missing value for --since");
|
||||
}
|
||||
since = parseSinceDate(value);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg.startsWith("--sessions-dir=")) {
|
||||
sessionsDir = arg.slice("--sessions-dir=".length);
|
||||
continue;
|
||||
}
|
||||
if (arg === "--sessions-dir") {
|
||||
const value = args[index + 1];
|
||||
if (!value) {
|
||||
throw new Error("Missing value for --sessions-dir");
|
||||
}
|
||||
sessionsDir = value;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg.startsWith("--endpoint=")) {
|
||||
endpoint = arg.slice("--endpoint=".length);
|
||||
continue;
|
||||
}
|
||||
if (arg === "--endpoint") {
|
||||
const value = args[index + 1];
|
||||
if (!value) {
|
||||
throw new Error("Missing value for --endpoint");
|
||||
}
|
||||
endpoint = value;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new Error(`Unknown flag: ${arg}`);
|
||||
}
|
||||
|
||||
if (!workspaceId || workspaceId.trim().length === 0) {
|
||||
throw new Error("--workspace-id is required");
|
||||
}
|
||||
|
||||
return {
|
||||
workspaceId: workspaceId.trim(),
|
||||
agentId: agentId?.trim(),
|
||||
since,
|
||||
sessionsDir: sessionsDir ? path.resolve(expandHomePath(sessionsDir)) : undefined,
|
||||
endpoint,
|
||||
};
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null;
|
||||
}
|
||||
|
||||
function asString(value: unknown): string | null {
|
||||
return typeof value === "string" ? value : null;
|
||||
}
|
||||
|
||||
function normalizeIsoTimestamp(value: unknown): string | null {
|
||||
if (typeof value === "string") {
|
||||
const parsed = new Date(value);
|
||||
if (!Number.isNaN(parsed.getTime())) {
|
||||
return parsed.toISOString();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
const millis = value >= 1_000_000_000_000 ? value : value * 1000;
|
||||
const parsed = new Date(millis);
|
||||
if (!Number.isNaN(parsed.getTime())) {
|
||||
return parsed.toISOString();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function truncate(value: string, maxLength: number): string {
|
||||
if (value.length <= maxLength) {
|
||||
return value;
|
||||
}
|
||||
return `${value.slice(0, maxLength - 3)}...`;
|
||||
}
|
||||
|
||||
function deriveTitle(content: string, fallbackSessionId: string): string {
|
||||
const firstLine = content
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => line.trim())
|
||||
.find((line) => line.length > 0);
|
||||
|
||||
if (!firstLine) {
|
||||
return `OpenClaw session ${fallbackSessionId}`;
|
||||
}
|
||||
|
||||
const normalized = firstLine.replace(/\s+/gu, " ").trim();
|
||||
return truncate(normalized, 140);
|
||||
}
|
||||
|
||||
function extractTextContent(content: unknown): string {
|
||||
if (typeof content === "string") {
|
||||
return content.trim();
|
||||
}
|
||||
|
||||
if (Array.isArray(content)) {
|
||||
const parts: string[] = [];
|
||||
for (const item of content) {
|
||||
if (typeof item === "string") {
|
||||
const trimmed = item.trim();
|
||||
if (trimmed.length > 0) {
|
||||
parts.push(trimmed);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!isRecord(item)) {
|
||||
continue;
|
||||
}
|
||||
const itemType = asString(item.type);
|
||||
if (itemType !== null && itemType !== "text") {
|
||||
continue;
|
||||
}
|
||||
const textValue = asString(item.text);
|
||||
if (textValue && textValue.trim().length > 0) {
|
||||
parts.push(textValue.trim());
|
||||
}
|
||||
}
|
||||
return parts.join("\n\n").trim();
|
||||
}
|
||||
|
||||
if (isRecord(content)) {
|
||||
const textValue = asString(content.text);
|
||||
if (textValue) {
|
||||
return textValue.trim();
|
||||
}
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
function inferAgentIdFromPath(filePath: string): string | null {
|
||||
const pathParts = filePath.split(path.sep);
|
||||
const agentsIndex = pathParts.lastIndexOf("agents");
|
||||
if (agentsIndex < 0) {
|
||||
return null;
|
||||
}
|
||||
const candidate = pathParts[agentsIndex + 1];
|
||||
return candidate && candidate.trim().length > 0 ? candidate : null;
|
||||
}
|
||||
|
||||
async function parseSessionFile(filePath: string): Promise<ParsedSession> {
|
||||
const fallbackSessionId = path.basename(filePath, path.extname(filePath));
|
||||
const inferredAgentId = inferAgentIdFromPath(filePath) ?? undefined;
|
||||
|
||||
let sessionId = fallbackSessionId;
|
||||
let title: string | null = null;
|
||||
let startedAt: string | undefined;
|
||||
let endedAt: string | undefined;
|
||||
let parseErrors = 0;
|
||||
const messages: IngestMessage[] = [];
|
||||
|
||||
const readStream = createReadStream(filePath, { encoding: "utf8" });
|
||||
const lineReader = createInterface({
|
||||
input: readStream,
|
||||
crlfDelay: Number.POSITIVE_INFINITY,
|
||||
});
|
||||
|
||||
for await (const rawLine of lineReader) {
|
||||
const line = rawLine.trim();
|
||||
if (line.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let parsedLine: unknown;
|
||||
try {
|
||||
parsedLine = JSON.parse(line) as unknown;
|
||||
} catch {
|
||||
parseErrors += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!isRecord(parsedLine)) {
|
||||
parseErrors += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const eventType = asString(parsedLine.type);
|
||||
if (eventType === "session") {
|
||||
const rawSessionId = asString(parsedLine.id);
|
||||
if (rawSessionId && rawSessionId.trim().length > 0) {
|
||||
sessionId = rawSessionId;
|
||||
}
|
||||
|
||||
const sessionTimestamp = normalizeIsoTimestamp(parsedLine.timestamp);
|
||||
if (sessionTimestamp) {
|
||||
startedAt ??= sessionTimestamp;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (eventType !== "message") {
|
||||
continue;
|
||||
}
|
||||
|
||||
const messageRecord = parsedLine.message;
|
||||
if (!isRecord(messageRecord)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const role = asString(messageRecord.role);
|
||||
if (role !== "user" && role !== "assistant") {
|
||||
continue;
|
||||
}
|
||||
|
||||
const content = extractTextContent(messageRecord.content);
|
||||
if (content.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const timestamp =
|
||||
normalizeIsoTimestamp(messageRecord.timestamp) ?? normalizeIsoTimestamp(parsedLine.timestamp);
|
||||
|
||||
const message: IngestMessage = {
|
||||
role,
|
||||
content,
|
||||
timestamp: timestamp ?? undefined,
|
||||
};
|
||||
messages.push(message);
|
||||
|
||||
if (!title && role === "user") {
|
||||
title = deriveTitle(content, sessionId);
|
||||
}
|
||||
if (!startedAt && timestamp) {
|
||||
startedAt = timestamp;
|
||||
}
|
||||
if (timestamp) {
|
||||
endedAt = timestamp;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
sessionId,
|
||||
title: title ?? `OpenClaw session ${sessionId}`,
|
||||
messages,
|
||||
startedAt,
|
||||
endedAt,
|
||||
parseErrors,
|
||||
inferredAgentId,
|
||||
};
|
||||
}
|
||||
|
||||
async function pathExists(candidatePath: string): Promise<boolean> {
|
||||
try {
|
||||
await access(candidatePath, fsConstants.F_OK);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function discoverSessionDirectories(overrideDir?: string): Promise<string[]> {
|
||||
if (overrideDir) {
|
||||
if (!(await pathExists(overrideDir))) {
|
||||
throw new Error(`Provided --sessions-dir does not exist: ${overrideDir}`);
|
||||
}
|
||||
return [overrideDir];
|
||||
}
|
||||
|
||||
const defaultDir = path.join(homedir(), ".openclaw", "sessions");
|
||||
if (await pathExists(defaultDir)) {
|
||||
return [defaultDir];
|
||||
}
|
||||
|
||||
const agentsRoot = path.join(homedir(), ".openclaw", "agents");
|
||||
if (!(await pathExists(agentsRoot))) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const agentEntries = await readdir(agentsRoot, { withFileTypes: true });
|
||||
const directories: string[] = [];
|
||||
for (const entry of agentEntries) {
|
||||
if (!entry.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
const sessionsDir = path.join(agentsRoot, entry.name, "sessions");
|
||||
if (await pathExists(sessionsDir)) {
|
||||
directories.push(sessionsDir);
|
||||
}
|
||||
}
|
||||
|
||||
return directories.sort((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
async function discoverSessionFiles(overrideDir?: string): Promise<string[]> {
|
||||
const directories = await discoverSessionDirectories(overrideDir);
|
||||
const files: string[] = [];
|
||||
|
||||
for (const directory of directories) {
|
||||
const entries = await readdir(directory, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile() || !entry.name.endsWith(".jsonl")) {
|
||||
continue;
|
||||
}
|
||||
files.push(path.join(directory, entry.name));
|
||||
}
|
||||
}
|
||||
|
||||
return files.sort((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
async function resolveSessionTimestamp(session: ParsedSession, filePath: string): Promise<Date> {
|
||||
const sessionTimestamp = session.startedAt ?? session.endedAt;
|
||||
if (sessionTimestamp) {
|
||||
const parsed = new Date(sessionTimestamp);
|
||||
if (!Number.isNaN(parsed.getTime())) {
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
|
||||
const fileStat = await stat(filePath);
|
||||
return fileStat.mtime;
|
||||
}
|
||||
|
||||
function buildPayload(
|
||||
options: CliOptions,
|
||||
session: ParsedSession,
|
||||
fallbackAgentId: string | undefined
|
||||
): IngestPayload {
|
||||
const payload: IngestPayload = {
|
||||
sessionId: session.sessionId,
|
||||
workspaceId: options.workspaceId,
|
||||
title: session.title,
|
||||
messages: session.messages,
|
||||
};
|
||||
|
||||
const selectedAgentId = options.agentId ?? fallbackAgentId;
|
||||
if (selectedAgentId && selectedAgentId.trim().length > 0) {
|
||||
payload.agentId = selectedAgentId.trim();
|
||||
}
|
||||
|
||||
return payload;
|
||||
}
|
||||
|
||||
async function sendIngestRequest(
|
||||
endpoint: string,
|
||||
token: string,
|
||||
payload: IngestPayload
|
||||
): Promise<SendResult> {
|
||||
const response = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
|
||||
const body = await response.text();
|
||||
return {
|
||||
ok: response.ok,
|
||||
status: response.status,
|
||||
body,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeFailureBody(body: string): string {
|
||||
const compact = body.replace(/\s+/gu, " ").trim();
|
||||
if (compact.length === 0) {
|
||||
return "(empty response body)";
|
||||
}
|
||||
return truncate(compact, 220);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const options = parseCliArgs(process.argv.slice(2));
|
||||
const token = process.env.MOSAIC_API_TOKEN;
|
||||
if (!token || token.trim().length === 0) {
|
||||
throw new Error("MOSAIC_API_TOKEN environment variable is required.");
|
||||
}
|
||||
|
||||
const sessionFiles = await discoverSessionFiles(options.sessionsDir);
|
||||
if (sessionFiles.length === 0) {
|
||||
console.log("No OpenClaw session files found.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Discovered ${sessionFiles.length} session file(s).`);
|
||||
if (options.since) {
|
||||
console.log(`Applying --since filter at ${options.since.toISOString()}.`);
|
||||
}
|
||||
|
||||
const summary: IngestSummary = {
|
||||
discovered: sessionFiles.length,
|
||||
processed: 0,
|
||||
ingested: 0,
|
||||
skippedSince: 0,
|
||||
skippedEmpty: 0,
|
||||
skippedDuplicate: 0,
|
||||
failed: 0,
|
||||
};
|
||||
|
||||
for (const [index, filePath] of sessionFiles.entries()) {
|
||||
const position = `${index + 1}/${sessionFiles.length}`;
|
||||
const parsedSession = await parseSessionFile(filePath);
|
||||
summary.processed += 1;
|
||||
|
||||
if (parsedSession.messages.length === 0) {
|
||||
summary.skippedEmpty += 1;
|
||||
console.log(
|
||||
`[${position}] Skipped ${parsedSession.sessionId}: no user/assistant text messages.`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const sessionDate = await resolveSessionTimestamp(parsedSession, filePath);
|
||||
if (options.since && sessionDate.getTime() < options.since.getTime()) {
|
||||
summary.skippedSince += 1;
|
||||
console.log(
|
||||
`[${position}] Skipped ${parsedSession.sessionId}: session is before --since (${sessionDate.toISOString()}).`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const payload = buildPayload(options, parsedSession, parsedSession.inferredAgentId);
|
||||
let result: SendResult;
|
||||
|
||||
try {
|
||||
result = await sendIngestRequest(options.endpoint, token, payload);
|
||||
} catch (error) {
|
||||
summary.failed += 1;
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(`[${position}] Failed ${parsedSession.sessionId}: request error: ${message}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (result.ok) {
|
||||
summary.ingested += 1;
|
||||
const note =
|
||||
parsedSession.parseErrors > 0 ? ` (parse warnings: ${parsedSession.parseErrors})` : "";
|
||||
console.log(
|
||||
`[${position}] Ingested ${parsedSession.sessionId} (${parsedSession.messages.length} messages)${note}.`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (result.status === 409) {
|
||||
summary.skippedDuplicate += 1;
|
||||
console.log(`[${position}] Skipped ${parsedSession.sessionId}: already exists (409).`);
|
||||
continue;
|
||||
}
|
||||
|
||||
summary.failed += 1;
|
||||
console.error(
|
||||
`[${position}] Failed ${parsedSession.sessionId}: HTTP ${result.status} ${summarizeFailureBody(result.body)}`
|
||||
);
|
||||
}
|
||||
|
||||
console.log("\nIngestion summary:");
|
||||
console.log(` Discovered: ${summary.discovered}`);
|
||||
console.log(` Processed: ${summary.processed}`);
|
||||
console.log(` Ingested: ${summary.ingested}`);
|
||||
console.log(` Skipped (--since): ${summary.skippedSince}`);
|
||||
console.log(` Skipped (empty): ${summary.skippedEmpty}`);
|
||||
console.log(` Skipped (duplicate): ${summary.skippedDuplicate}`);
|
||||
console.log(` Failed: ${summary.failed}`);
|
||||
|
||||
if (summary.failed > 0) {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error: unknown) => {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(`Fatal error: ${message}`);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user