Compare commits

..

7 Commits

Author SHA1 Message Date
1f63727582 chore(orchestrator): add MS22 PRD, mark P1a+P1b done 2026-03-01 09:05:16 -06:00
ab2b68c93c Merge pull request 'feat(api): agent fleet DB schema + migration (MS22-P1a)' (#607) from feat/ms22-p1a-schema into main
All checks were successful
ci/woodpecker/push/ci Pipeline was successful
Reviewed-on: #607
2026-03-01 15:03:23 +00:00
c1ec0ad7ef Merge pull request 'feat(api): CryptoService for API key encryption (MS22-P1b)' (#606) from feat/ms22-p1b-crypto into main
All checks were successful
ci/woodpecker/push/ci Pipeline was successful
Reviewed-on: #606
2026-03-01 15:02:50 +00:00
e5b772f7cb Merge pull request 'chore(orchestrator): MS22 Phase 1 task breakdown' (#605) from chore/ms22-p1-tasks into main
Reviewed-on: #605
2026-03-01 15:02:27 +00:00
7a46c81897 feat(api): add agent fleet Prisma schema (MS22-P1a)
All checks were successful
ci/woodpecker/push/ci Pipeline was successful
2026-03-01 08:42:10 -06:00
3688f89c37 feat(api): add CryptoService for secret encryption (MS22-P1b)
All checks were successful
ci/woodpecker/push/ci Pipeline was successful
2026-03-01 08:41:28 -06:00
e59e517d5c feat(api): add CryptoService for secret encryption (MS22-P1b) 2026-03-01 08:40:40 -06:00
8 changed files with 468 additions and 2 deletions

View File

@@ -0,0 +1,109 @@
-- CreateTable
CREATE TABLE "SystemConfig" (
"id" TEXT NOT NULL,
"key" TEXT NOT NULL,
"value" TEXT NOT NULL,
"encrypted" BOOLEAN NOT NULL DEFAULT false,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "SystemConfig_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "BreakglassUser" (
"id" TEXT NOT NULL,
"username" TEXT NOT NULL,
"passwordHash" TEXT NOT NULL,
"isActive" BOOLEAN NOT NULL DEFAULT true,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "BreakglassUser_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "LlmProvider" (
"id" TEXT NOT NULL,
"userId" TEXT NOT NULL,
"name" TEXT NOT NULL,
"displayName" TEXT NOT NULL,
"type" TEXT NOT NULL,
"baseUrl" TEXT,
"apiKey" TEXT,
"apiType" TEXT NOT NULL DEFAULT 'openai-completions',
"models" JSONB NOT NULL DEFAULT '[]',
"isActive" BOOLEAN NOT NULL DEFAULT true,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "LlmProvider_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "UserContainer" (
"id" TEXT NOT NULL,
"userId" TEXT NOT NULL,
"containerId" TEXT,
"containerName" TEXT NOT NULL,
"gatewayPort" INTEGER,
"gatewayToken" TEXT NOT NULL,
"status" TEXT NOT NULL DEFAULT 'stopped',
"lastActiveAt" TIMESTAMP(3),
"idleTimeoutMin" INTEGER NOT NULL DEFAULT 30,
"config" JSONB NOT NULL DEFAULT '{}',
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "UserContainer_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "SystemContainer" (
"id" TEXT NOT NULL,
"name" TEXT NOT NULL,
"role" TEXT NOT NULL,
"containerId" TEXT,
"gatewayPort" INTEGER,
"gatewayToken" TEXT NOT NULL,
"status" TEXT NOT NULL DEFAULT 'stopped',
"primaryModel" TEXT NOT NULL,
"isActive" BOOLEAN NOT NULL DEFAULT true,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "SystemContainer_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "UserAgentConfig" (
"id" TEXT NOT NULL,
"userId" TEXT NOT NULL,
"primaryModel" TEXT,
"fallbackModels" JSONB NOT NULL DEFAULT '[]',
"personality" TEXT,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "UserAgentConfig_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE UNIQUE INDEX "SystemConfig_key_key" ON "SystemConfig"("key");
-- CreateIndex
CREATE UNIQUE INDEX "BreakglassUser_username_key" ON "BreakglassUser"("username");
-- CreateIndex
CREATE INDEX "LlmProvider_userId_idx" ON "LlmProvider"("userId");
-- CreateIndex
CREATE UNIQUE INDEX "LlmProvider_userId_name_key" ON "LlmProvider"("userId", "name");
-- CreateIndex
CREATE UNIQUE INDEX "UserContainer_userId_key" ON "UserContainer"("userId");
-- CreateIndex
CREATE UNIQUE INDEX "SystemContainer_name_key" ON "SystemContainer"("name");
-- CreateIndex
CREATE UNIQUE INDEX "UserAgentConfig_userId_key" ON "UserAgentConfig"("userId");

View File

@@ -1625,3 +1625,81 @@ model ConversationArchive {
@@index([startedAt])
@@map("conversation_archives")
}
// ============================================
// AGENT FLEET MODULE
// ============================================
model SystemConfig {
id String @id @default(cuid())
key String @unique
value String
encrypted Boolean @default(false)
updatedAt DateTime @updatedAt
}
model BreakglassUser {
id String @id @default(cuid())
username String @unique
passwordHash String
isActive Boolean @default(true)
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
}
model LlmProvider {
id String @id @default(cuid())
userId String
name String
displayName String
type String
baseUrl String?
apiKey String?
apiType String @default("openai-completions")
models Json @default("[]")
isActive Boolean @default(true)
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
@@unique([userId, name])
@@index([userId])
}
model UserContainer {
id String @id @default(cuid())
userId String @unique
containerId String?
containerName String
gatewayPort Int?
gatewayToken String
status String @default("stopped")
lastActiveAt DateTime?
idleTimeoutMin Int @default(30)
config Json @default("{}")
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
}
model SystemContainer {
id String @id @default(cuid())
name String @unique
role String
containerId String?
gatewayPort Int?
gatewayToken String
status String @default("stopped")
primaryModel String
isActive Boolean @default(true)
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
}
model UserAgentConfig {
id String @id @default(cuid())
userId String @unique
primaryModel String?
fallbackModels Json @default("[]")
personality String?
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
}

View File

@@ -39,6 +39,7 @@ import { JobStepsModule } from "./job-steps/job-steps.module";
import { CoordinatorIntegrationModule } from "./coordinator-integration/coordinator-integration.module";
import { FederationModule } from "./federation/federation.module";
import { CredentialsModule } from "./credentials/credentials.module";
import { CryptoModule } from "./crypto/crypto.module";
import { MosaicTelemetryModule } from "./mosaic-telemetry";
import { SpeechModule } from "./speech/speech.module";
import { DashboardModule } from "./dashboard/dashboard.module";
@@ -111,6 +112,7 @@ import { RlsContextInterceptor } from "./common/interceptors/rls-context.interce
CoordinatorIntegrationModule,
FederationModule,
CredentialsModule,
CryptoModule,
MosaicTelemetryModule,
SpeechModule,
DashboardModule,

View File

@@ -0,0 +1,10 @@
import { Module } from "@nestjs/common";
import { ConfigModule } from "@nestjs/config";
import { CryptoService } from "./crypto.service";
@Module({
imports: [ConfigModule],
providers: [CryptoService],
exports: [CryptoService],
})
export class CryptoModule {}

View File

@@ -0,0 +1,71 @@
import { describe, it, expect, beforeEach } from "vitest";
import { ConfigService } from "@nestjs/config";
import { CryptoService } from "./crypto.service";
function createConfigService(secret?: string): ConfigService {
return {
get: (key: string) => {
if (key === "MOSAIC_SECRET_KEY") {
return secret;
}
return undefined;
},
} as unknown as ConfigService;
}
describe("CryptoService", () => {
let service: CryptoService;
beforeEach(() => {
service = new CryptoService(createConfigService("this-is-a-test-secret-key-with-32+chars"));
});
it("encrypt -> decrypt roundtrip", () => {
const plaintext = "my-secret-api-key";
const encrypted = service.encrypt(plaintext);
const decrypted = service.decrypt(encrypted);
expect(encrypted.startsWith("enc:")).toBe(true);
expect(decrypted).toBe(plaintext);
});
it("decrypt rejects tampered ciphertext", () => {
const encrypted = service.encrypt("sensitive-token");
const payload = encrypted.slice(4);
const bytes = Buffer.from(payload, "base64");
bytes[bytes.length - 1] = bytes[bytes.length - 1]! ^ 0xff;
const tampered = `enc:${bytes.toString("base64")}`;
expect(() => service.decrypt(tampered)).toThrow();
});
it("decrypt rejects non-encrypted string", () => {
expect(() => service.decrypt("plain-text-value")).toThrow();
});
it("isEncrypted detects prefix correctly", () => {
expect(service.isEncrypted("enc:abc")).toBe(true);
expect(service.isEncrypted("ENC:abc")).toBe(false);
expect(service.isEncrypted("plain-text")).toBe(false);
});
it("generateToken returns 64-char hex string", () => {
const token = service.generateToken();
expect(token).toMatch(/^[0-9a-f]{64}$/);
});
it("different plaintexts produce different ciphertexts (random IV)", () => {
const encryptedA = service.encrypt("value-a");
const encryptedB = service.encrypt("value-b");
expect(encryptedA).not.toBe(encryptedB);
});
it("missing MOSAIC_SECRET_KEY throws on construction", () => {
expect(() => new CryptoService(createConfigService(undefined))).toThrow();
});
});

View File

@@ -0,0 +1,82 @@
import { Injectable } from "@nestjs/common";
import { ConfigService } from "@nestjs/config";
import { createCipheriv, createDecipheriv, hkdfSync, randomBytes } from "crypto";
const ALGORITHM = "aes-256-gcm";
const ENCRYPTED_PREFIX = "enc:";
const IV_LENGTH = 12;
const AUTH_TAG_LENGTH = 16;
const DERIVED_KEY_LENGTH = 32;
const HKDF_SALT = "mosaic.crypto.v1";
const HKDF_INFO = "mosaic-db-secret-encryption";
@Injectable()
export class CryptoService {
private readonly key: Buffer;
constructor(private readonly config: ConfigService) {
const secret = this.config.get<string>("MOSAIC_SECRET_KEY");
if (!secret) {
throw new Error("MOSAIC_SECRET_KEY environment variable is required");
}
if (secret.length < 32) {
throw new Error("MOSAIC_SECRET_KEY must be at least 32 characters");
}
this.key = Buffer.from(
hkdfSync(
"sha256",
Buffer.from(secret, "utf8"),
Buffer.from(HKDF_SALT, "utf8"),
Buffer.from(HKDF_INFO, "utf8"),
DERIVED_KEY_LENGTH
)
);
}
encrypt(plaintext: string): string {
const iv = randomBytes(IV_LENGTH);
const cipher = createCipheriv(ALGORITHM, this.key, iv);
const ciphertext = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]);
const authTag = cipher.getAuthTag();
const payload = Buffer.concat([iv, ciphertext, authTag]).toString("base64");
return `${ENCRYPTED_PREFIX}${payload}`;
}
decrypt(encrypted: string): string {
if (!this.isEncrypted(encrypted)) {
throw new Error("Value is not encrypted");
}
const payloadBase64 = encrypted.slice(ENCRYPTED_PREFIX.length);
try {
const payload = Buffer.from(payloadBase64, "base64");
if (payload.length < IV_LENGTH + AUTH_TAG_LENGTH) {
throw new Error("Encrypted payload is too short");
}
const iv = payload.subarray(0, IV_LENGTH);
const authTag = payload.subarray(payload.length - AUTH_TAG_LENGTH);
const ciphertext = payload.subarray(IV_LENGTH, payload.length - AUTH_TAG_LENGTH);
const decipher = createDecipheriv(ALGORITHM, this.key, iv);
decipher.setAuthTag(authTag);
return Buffer.concat([decipher.update(ciphertext), decipher.final()]).toString("utf8");
} catch {
throw new Error("Failed to decrypt value");
}
}
isEncrypted(value: string): boolean {
return value.startsWith(ENCRYPTED_PREFIX);
}
generateToken(): string {
return randomBytes(32).toString("hex");
}
}

114
docs/PRD-MS22.md Normal file
View File

@@ -0,0 +1,114 @@
# PRD: MS22 — Fleet Evolution (DB-Centric Agent Architecture)
## Metadata
- Owner: Jason Woltje
- Date: 2026-03-01
- Status: in-progress
- Design Doc: `docs/design/MS22-DB-CENTRIC-ARCHITECTURE.md`
## Problem Statement
Mosaic Stack needs a multi-user agent fleet where each user gets their own isolated OpenClaw instance with their own LLM provider credentials and agent config. The system must be Docker-first with minimal environment variables and all configuration managed through the WebUI.
## Objectives
1. **Minimal bootstrap** — 2 env vars (`DATABASE_URL`, `MOSAIC_SECRET_KEY`) to start the entire stack
2. **DB-centric config** — All runtime config in Postgres, managed via WebUI
3. **Per-user isolation** — Each user gets their own OpenClaw container with own API keys, memory, sessions
4. **Onboarding wizard** — First-boot experience: breakglass admin → OIDC → LLM provider → agent config
5. **Settings UI** — Runtime management of providers, agents, and auth config
6. **Mosaic as gatekeeper** — Users never talk to OpenClaw directly; Mosaic proxies all requests
7. **Zero cross-user access** — Full container, volume, and DB isolation between users
## Security Requirements
- User A cannot access User B's API keys, chat history, or agent memory
- All API keys stored encrypted (AES-256-GCM) in database
- Breakglass admin always works as OIDC fallback
- OIDC config stored in DB (not env vars) — configured via settings UI
- Container-to-container communication blocked by default
- Admin cannot decrypt other users' API keys
## Phase 0: Knowledge Layer — COMPLETE
- Findings API (pgvector, CRUD, similarity search)
- AgentMemory API (key/value store)
- ConversationArchive API (pgvector, ingest, search)
- OpenClaw mosaic skill
- Session log ingestion pipeline
## Phase 1: DB-Centric Agent Fleet
### Phase 1a: DB Schema — COMPLETE
- SystemConfig, BreakglassUser, LlmProvider, UserContainer, SystemContainer, UserAgentConfig tables
### Phase 1b: Encryption Service — COMPLETE
- CryptoService (AES-256-GCM using MOSAIC_SECRET_KEY)
### Phase 1c: Internal Config API
- `GET /api/internal/agent-config/:id` — assembles openclaw.json from DB
- Auth: bearer token (container's own gateway token)
- Returns complete openclaw.json with decrypted provider credentials
### Phase 1d: Container Lifecycle Manager
- Docker API integration via `dockerode` npm package
- Start/stop/health-check/reap user containers
- Auto-generate gateway tokens, assign ports
- Docker socket access required (`/var/run/docker.sock`)
### Phase 1e: Onboarding API
- First-boot detection (`SystemConfig.onboarding.completed`)
- `POST /api/onboarding/breakglass` — create admin user
- `POST /api/onboarding/oidc` — save OIDC provider config
- `POST /api/onboarding/provider` — add LLM provider + test connection
- `POST /api/onboarding/complete` — mark done
### Phase 1f: Onboarding Wizard UI
- Multi-step wizard component
- Skip-able OIDC step
- LLM provider connection test
### Phase 1g: Settings API
- CRUD: LLM providers (per-user scoped)
- CRUD: Agent config (model assignments, personalities)
- CRUD: OIDC config (admin only)
- Breakglass password reset (admin only)
### Phase 1h: Settings UI
- Settings/Providers page
- Settings/Agent Config page
- Settings/Auth page (OIDC + breakglass)
### Phase 1i: Chat Proxy
- Route WebUI chat to user's OpenClaw container
- SSE streaming pass-through
- Ensure container is running before proxying (auto-start)
### Phase 1j: Docker Compose + Entrypoint
- Simplified compose (core services only — user containers are dynamic)
- Entrypoint: fetch config from API, write openclaw.json, start gateway
- Health check integration
### Phase 1k: Idle Reaper
- Cron job to stop inactive user containers
- Configurable idle timeout (default 30min)
- Preserve state volumes
## Future Phases (out of scope)
- Phase 2: Agent fleet standup (predefined agent roles)
- Phase 3: WebUI chat + task management integration
- Phase 4: Multi-LLM provider management UI (advanced)
- Team workspaces (shared agent contexts) — explicitly out of scope

View File

@@ -78,8 +78,8 @@ Design doc: `docs/design/MS22-DB-CENTRIC-ARCHITECTURE.md`
| Task ID | Status | Phase | Description | Issue | Scope | Branch | Depends On | Blocks | Assigned Worker | Started | Completed | Est Tokens | Act Tokens | Notes |
| -------- | ----------- | -------- | --------------------------------------------------------------------------------------------------------------------- | ----- | ------- | ---------------------------- | ---------- | --------------- | --------------- | ------- | --------- | ---------- | ---------- | ----- |
| MS22-P1a | not-started | phase-1a | Prisma schema: SystemConfig, BreakglassUser, LlmProvider, UserContainer, SystemContainer, UserAgentConfig + migration | — | api | feat/ms22-p1a-schema | — | P1b,P1c,P1d,P1e | — | — | — | 20K | — | |
| MS22-P1b | not-started | phase-1b | Encryption service (AES-256-GCM) for API keys and tokens | — | api | feat/ms22-p1b-crypto | — | P1c,P1e,P1g | — | — | — | 15K | — | |
| MS22-P1a | done | phase-1a | Prisma schema: SystemConfig, BreakglassUser, LlmProvider, UserContainer, SystemContainer, UserAgentConfig + migration | — | api | feat/ms22-p1a-schema | — | P1b,P1c,P1d,P1e | — | — | — | 20K | — | |
| MS22-P1b | done | phase-1b | Encryption service (AES-256-GCM) for API keys and tokens | — | api | feat/ms22-p1b-crypto | — | P1c,P1e,P1g | — | — | — | 15K | — | |
| MS22-P1c | not-started | phase-1c | Internal config endpoint: assemble openclaw.json from DB | — | api | feat/ms22-p1c-config-api | P1a,P1b | P1i,P1j | — | — | — | 20K | — | |
| MS22-P1d | not-started | phase-1d | ContainerLifecycleService: Docker API (dockerode) start/stop/health/reap | — | api | feat/ms22-p1d-container-mgr | P1a | P1i,P1k | — | — | — | 25K | — | |
| MS22-P1e | not-started | phase-1e | Onboarding API: breakglass, OIDC, provider, agents, complete | — | api | feat/ms22-p1e-onboarding-api | P1a,P1b | P1f | — | — | — | 20K | — | |