diff --git a/.env.example b/.env.example index 2e36dd6..f77dacc 100644 --- a/.env.example +++ b/.env.example @@ -101,6 +101,14 @@ JWT_EXPIRATION=24h # SECURITY: Store production keys in a secure secrets manager (see docs/design/credential-security.md) ENCRYPTION_KEY=REPLACE_WITH_64_CHAR_HEX_STRING_GENERATE_WITH_OPENSSL_RAND_HEX_32 +# ====================== +# OpenBao Secrets Management +# ====================== +# OpenBao provides Transit encryption for sensitive credentials +# Auto-initialized on first run via openbao-init sidecar +OPENBAO_ADDR=http://openbao:8200 +OPENBAO_PORT=8200 + # ====================== # Ollama (Optional AI Service) # ====================== diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 6a3e2bd..1128d88 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -68,11 +68,65 @@ services: networks: - mosaic-network + openbao: + image: quay.io/openbao/openbao:2 + container_name: mosaic-openbao + restart: unless-stopped + user: root + ports: + - "127.0.0.1:${OPENBAO_PORT:-8200}:8200" + volumes: + - openbao_data:/openbao/data + - openbao_init:/openbao/init + - ./openbao/config.hcl:/openbao/config/config.hcl:ro + environment: + VAULT_ADDR: http://0.0.0.0:8200 + SKIP_SETCAP: "true" + entrypoint: ["/bin/sh", "-c"] + command: ["bao server -config=/openbao/config/config.hcl"] + cap_add: + - IPC_LOCK + healthcheck: + test: ["CMD-SHELL", "nc -z 127.0.0.1 8200 || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + networks: + - mosaic-network + labels: + com.mosaic.service: "secrets" + com.mosaic.description: "OpenBao secrets management" + + openbao-init: + image: quay.io/openbao/openbao:2 + container_name: mosaic-openbao-init + restart: unless-stopped + user: root + volumes: + - openbao_init:/openbao/init + - ./openbao/init.sh:/init.sh:ro + environment: + VAULT_ADDR: http://openbao:8200 + command: /init.sh + depends_on: + openbao: + condition: service_healthy + networks: + - mosaic-network + labels: + com.mosaic.service: "secrets-init" + com.mosaic.description: "OpenBao auto-initialization sidecar" + volumes: postgres_data: name: mosaic-postgres-data valkey_data: name: mosaic-valkey-data + openbao_data: + name: mosaic-openbao-data + openbao_init: + name: mosaic-openbao-init networks: mosaic-network: diff --git a/docker/openbao/config.hcl b/docker/openbao/config.hcl new file mode 100644 index 0000000..d9f578c --- /dev/null +++ b/docker/openbao/config.hcl @@ -0,0 +1,24 @@ +# OpenBao Server Configuration +# File storage backend for turnkey deployment + +storage "file" { + path = "/openbao/data" +} + +# HTTP API listener +listener "tcp" { + address = "0.0.0.0:8200" + tls_disable = 1 +} + +# Disable memory locking for Docker compatibility +disable_mlock = true + +# API address for cluster communication +api_addr = "http://0.0.0.0:8200" + +# UI enabled for debugging (disable in production) +ui = true + +# Log level +log_level = "info" diff --git a/docker/openbao/init.sh b/docker/openbao/init.sh new file mode 100755 index 0000000..d3f5f53 --- /dev/null +++ b/docker/openbao/init.sh @@ -0,0 +1,294 @@ +#!/bin/sh +set -e + +# OpenBao Auto-Init Script +# Auto-initializes, unseals, and configures OpenBao on first run +# Idempotent - safe to run multiple times + +INIT_DIR="/openbao/init" +UNSEAL_KEY_FILE="${INIT_DIR}/unseal-key" +ROOT_TOKEN_FILE="${INIT_DIR}/root-token" +APPROLE_CREDS_FILE="${INIT_DIR}/approle-credentials" + +VAULT_ADDR="http://openbao:8200" +export VAULT_ADDR + +# Ensure init directory exists +mkdir -p "${INIT_DIR}" + +echo "=== OpenBao Auto-Init Script ===" +echo "Vault Address: ${VAULT_ADDR}" + +# Wait for OpenBao to be ready +echo "Waiting for OpenBao API to be available..." +MAX_RETRIES=30 +RETRY_COUNT=0 +while ! wget -q -O- "${VAULT_ADDR}/v1/sys/init" >/dev/null 2>&1; do + RETRY_COUNT=$((RETRY_COUNT + 1)) + if [ ${RETRY_COUNT} -ge ${MAX_RETRIES} ]; then + echo "ERROR: OpenBao API not available after ${MAX_RETRIES} attempts" + exit 1 + fi + echo "Waiting for OpenBao... (${RETRY_COUNT}/${MAX_RETRIES})" + sleep 2 +done +echo "OpenBao API is available" + +# Check initialization status +INIT_STATUS=$(wget -qO- "${VAULT_ADDR}/v1/sys/init" 2>/dev/null || echo '{"initialized":false}') +IS_INITIALIZED=$(echo "${INIT_STATUS}" | grep -o '"initialized":[^,}]*' | cut -d':' -f2) + +if [ "${IS_INITIALIZED}" = "true" ]; then + echo "OpenBao is already initialized" + + # Check if sealed + SEAL_STATUS=$(wget -qO- "${VAULT_ADDR}/v1/sys/seal-status" 2>/dev/null) + IS_SEALED=$(echo "${SEAL_STATUS}" | grep -o '"sealed":[^,}]*' | cut -d':' -f2) + + if [ "${IS_SEALED}" = "true" ]; then + echo "OpenBao is sealed - unsealing..." + + if [ ! -f "${UNSEAL_KEY_FILE}" ]; then + echo "ERROR: Unseal key not found at ${UNSEAL_KEY_FILE}" + exit 1 + fi + + UNSEAL_KEY=$(cat "${UNSEAL_KEY_FILE}") + + # Unseal with retry logic + MAX_UNSEAL_RETRIES=3 + UNSEAL_RETRY=0 + UNSEAL_SUCCESS=false + + while [ ${UNSEAL_RETRY} -lt ${MAX_UNSEAL_RETRIES} ]; do + UNSEAL_RESPONSE=$(wget -qO- --header="Content-Type: application/json" --post-data="{\"key\":\"${UNSEAL_KEY}\"}" "${VAULT_ADDR}/v1/sys/unseal" 2>&1) + + # Verify unseal was successful by checking sealed status + sleep 1 + VERIFY_STATUS=$(wget -qO- "${VAULT_ADDR}/v1/sys/seal-status" 2>/dev/null || echo '{"sealed":true}') + VERIFY_SEALED=$(echo "${VERIFY_STATUS}" | grep -o '"sealed":[^,}]*' | cut -d':' -f2) + + if [ "${VERIFY_SEALED}" = "false" ]; then + UNSEAL_SUCCESS=true + echo "OpenBao unsealed successfully" + break + fi + + UNSEAL_RETRY=$((UNSEAL_RETRY + 1)) + echo "Unseal attempt ${UNSEAL_RETRY} failed, retrying..." + sleep 2 + done + + if [ "${UNSEAL_SUCCESS}" = "false" ]; then + echo "ERROR: Failed to unseal OpenBao after ${MAX_UNSEAL_RETRIES} attempts" + exit 1 + fi + else + echo "OpenBao is already unsealed" + fi + + # Verify Transit engine and AppRole are configured + if [ -f "${ROOT_TOKEN_FILE}" ]; then + export VAULT_TOKEN=$(cat "${ROOT_TOKEN_FILE}") + + # Check Transit engine + if ! bao secrets list | grep -q "transit/"; then + echo "Transit engine not found - configuring..." + bao secrets enable transit + echo "Transit secrets engine enabled" + else + echo "Transit secrets engine already enabled" + fi + + # Check AppRole + if ! bao auth list | grep -q "approle/"; then + echo "AppRole not found - configuring..." + bao auth enable approle + echo "AppRole auth method enabled" + else + echo "AppRole auth method already enabled" + fi + fi + + echo "Initialization check complete - OpenBao is ready" + exit 0 +fi + +echo "OpenBao is not initialized - initializing with 1-of-1 key shares..." + +# Initialize with 1 key share, threshold 1 (turnkey mode) +INIT_OUTPUT=$(bao operator init -key-shares=1 -key-threshold=1 -format=json) + +# Extract unseal key and root token from JSON output +# First collapse multi-line JSON to single line, then parse +INIT_JSON=$(echo "${INIT_OUTPUT}" | tr -d '\n' | tr -d ' ') +UNSEAL_KEY=$(echo "${INIT_JSON}" | grep -o '"unseal_keys_b64":\["[^"]*"' | cut -d'"' -f4) +ROOT_TOKEN=$(echo "${INIT_JSON}" | grep -o '"root_token":"[^"]*"' | cut -d'"' -f4) + +# Save to files +echo "${UNSEAL_KEY}" > "${UNSEAL_KEY_FILE}" +echo "${ROOT_TOKEN}" > "${ROOT_TOKEN_FILE}" +chmod 600 "${UNSEAL_KEY_FILE}" "${ROOT_TOKEN_FILE}" + +echo "Initialization complete" +echo "Unseal key saved to ${UNSEAL_KEY_FILE}" +echo "Root token saved to ${ROOT_TOKEN_FILE}" + +# Unseal with retry logic +echo "Unsealing OpenBao..." +MAX_UNSEAL_RETRIES=3 +UNSEAL_RETRY=0 +UNSEAL_SUCCESS=false + +while [ ${UNSEAL_RETRY} -lt ${MAX_UNSEAL_RETRIES} ]; do + UNSEAL_RESPONSE=$(wget -qO- --header="Content-Type: application/json" --post-data="{\"key\":\"${UNSEAL_KEY}\"}" "${VAULT_ADDR}/v1/sys/unseal" 2>&1) + + # Verify unseal was successful by checking sealed status + sleep 1 + VERIFY_STATUS=$(wget -qO- "${VAULT_ADDR}/v1/sys/seal-status" 2>/dev/null || echo '{"sealed":true}') + VERIFY_SEALED=$(echo "${VERIFY_STATUS}" | grep -o '"sealed":[^,}]*' | cut -d':' -f2) + + if [ "${VERIFY_SEALED}" = "false" ]; then + UNSEAL_SUCCESS=true + echo "OpenBao unsealed successfully" + break + fi + + UNSEAL_RETRY=$((UNSEAL_RETRY + 1)) + echo "Unseal attempt ${UNSEAL_RETRY} failed, retrying..." + sleep 2 +done + +if [ "${UNSEAL_SUCCESS}" = "false" ]; then + echo "ERROR: Failed to unseal OpenBao after ${MAX_UNSEAL_RETRIES} attempts" + exit 1 +fi + +# Configure with root token +export VAULT_TOKEN="${ROOT_TOKEN}" + +# Enable Transit secrets engine +echo "Enabling Transit secrets engine..." +bao secrets enable transit +echo "Transit secrets engine enabled" + +# Create Transit encryption keys +echo "Creating Transit encryption keys..." + +bao write -f transit/keys/mosaic-credentials type=aes256-gcm96 +echo "Created key: mosaic-credentials" + +bao write -f transit/keys/mosaic-account-tokens type=aes256-gcm96 +echo "Created key: mosaic-account-tokens" + +bao write -f transit/keys/mosaic-federation type=aes256-gcm96 +echo "Created key: mosaic-federation" + +bao write -f transit/keys/mosaic-llm-config type=aes256-gcm96 +echo "Created key: mosaic-llm-config" + +echo "All Transit keys created" + +# Enable AppRole auth method +echo "Enabling AppRole auth method..." +bao auth enable approle +echo "AppRole auth method enabled" + +# Create Transit-only policy +echo "Creating Transit-only policy..." +cat > /tmp/transit-policy.hcl < "${APPROLE_CREDS_FILE}" </dev/null || echo '{"sealed":false}') + IS_SEALED=$(echo "${SEAL_STATUS}" | grep -o '"sealed":[^,}]*' | cut -d':' -f2) + + if [ "${IS_SEALED}" = "true" ]; then + echo "OpenBao is sealed - unsealing..." + if [ -f "${UNSEAL_KEY_FILE}" ]; then + UNSEAL_KEY=$(cat "${UNSEAL_KEY_FILE}") + + # Try to unseal with verification + UNSEAL_RESPONSE=$(wget -qO- --header="Content-Type: application/json" --post-data="{\"key\":\"${UNSEAL_KEY}\"}" "${VAULT_ADDR}/v1/sys/unseal" 2>&1) + + # Verify unseal was successful + sleep 1 + VERIFY_STATUS=$(wget -qO- "${VAULT_ADDR}/v1/sys/seal-status" 2>/dev/null || echo '{"sealed":true}') + VERIFY_SEALED=$(echo "${VERIFY_STATUS}" | grep -o '"sealed":[^,}]*' | cut -d':' -f2) + + if [ "${VERIFY_SEALED}" = "false" ]; then + echo "OpenBao unsealed successfully" + else + echo "WARNING: Unseal operation completed but OpenBao is still sealed" + fi + else + echo "WARNING: Unseal key not found, cannot auto-unseal" + fi + fi +done diff --git a/tests/integration/docker-stack.test.ts b/tests/integration/docker-stack.test.ts index eaaa58b..23245f5 100644 --- a/tests/integration/docker-stack.test.ts +++ b/tests/integration/docker-stack.test.ts @@ -1,6 +1,6 @@ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import { exec } from 'child_process'; -import { promisify } from 'util'; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { exec } from "child_process"; +import { promisify } from "util"; const execAsync = promisify(exec); @@ -8,7 +8,7 @@ const execAsync = promisify(exec); * Docker Stack Integration Tests * Tests the full Docker Compose stack deployment */ -describe('Docker Stack Integration Tests', () => { +describe("Docker Stack Integration Tests", () => { const TIMEOUT = 120000; // 2 minutes for Docker operations const HEALTH_CHECK_RETRIES = 30; const HEALTH_CHECK_INTERVAL = 2000; @@ -28,19 +28,14 @@ describe('Docker Stack Integration Tests', () => { */ async function waitForService( serviceName: string, - retries = HEALTH_CHECK_RETRIES, + retries = HEALTH_CHECK_RETRIES ): Promise { for (let i = 0; i < retries; i++) { try { - const { stdout } = await execAsync( - `docker compose ps --format json ${serviceName}`, - ); + const { stdout } = await execAsync(`docker compose ps --format json ${serviceName}`); const serviceInfo = JSON.parse(stdout); - if ( - serviceInfo.Health === 'healthy' || - serviceInfo.State === 'running' - ) { + if (serviceInfo.Health === "healthy" || serviceInfo.State === "running") { return true; } } catch (error) { @@ -64,190 +59,186 @@ describe('Docker Stack Integration Tests', () => { } } - describe('Core Services', () => { + describe("Core Services", () => { beforeAll(async () => { // Ensure clean state - await dockerCompose('down -v').catch(() => {}); + await dockerCompose("down -v").catch(() => {}); }, TIMEOUT); afterAll(async () => { // Cleanup - await dockerCompose('down -v').catch(() => {}); + await dockerCompose("down -v").catch(() => {}); }, TIMEOUT); it( - 'should start PostgreSQL with health check', + "should start PostgreSQL with health check", async () => { // Start only PostgreSQL - await dockerCompose('up -d postgres'); + await dockerCompose("up -d postgres"); // Wait for service to be healthy - const isHealthy = await waitForService('postgres'); + const isHealthy = await waitForService("postgres"); expect(isHealthy).toBe(true); // Verify container is running - const ps = await dockerCompose('ps postgres'); - expect(ps).toContain('mosaic-postgres'); - expect(ps).toContain('Up'); + const ps = await dockerCompose("ps postgres"); + expect(ps).toContain("mosaic-postgres"); + expect(ps).toContain("Up"); }, - TIMEOUT, + TIMEOUT ); it( - 'should start Valkey with health check', + "should start Valkey with health check", async () => { // Start Valkey - await dockerCompose('up -d valkey'); + await dockerCompose("up -d valkey"); // Wait for service to be healthy - const isHealthy = await waitForService('valkey'); + const isHealthy = await waitForService("valkey"); expect(isHealthy).toBe(true); // Verify container is running - const ps = await dockerCompose('ps valkey'); - expect(ps).toContain('mosaic-valkey'); - expect(ps).toContain('Up'); + const ps = await dockerCompose("ps valkey"); + expect(ps).toContain("mosaic-valkey"); + expect(ps).toContain("Up"); }, - TIMEOUT, + TIMEOUT ); it( - 'should have proper network configuration', + "should have proper network configuration", async () => { // Check if mosaic-internal network exists - const { stdout } = await execAsync('docker network ls'); - expect(stdout).toContain('mosaic-internal'); - expect(stdout).toContain('mosaic-public'); + const { stdout } = await execAsync("docker network ls"); + expect(stdout).toContain("mosaic-internal"); + expect(stdout).toContain("mosaic-public"); }, - TIMEOUT, + TIMEOUT ); it( - 'should have proper volume configuration', + "should have proper volume configuration", async () => { // Check if volumes are created - const { stdout } = await execAsync('docker volume ls'); - expect(stdout).toContain('mosaic-postgres-data'); - expect(stdout).toContain('mosaic-valkey-data'); + const { stdout } = await execAsync("docker volume ls"); + expect(stdout).toContain("mosaic-postgres-data"); + expect(stdout).toContain("mosaic-valkey-data"); }, - TIMEOUT, + TIMEOUT ); }); - describe('Application Services', () => { + describe("Application Services", () => { beforeAll(async () => { // Start core services first - await dockerCompose('up -d postgres valkey'); - await waitForService('postgres'); - await waitForService('valkey'); + await dockerCompose("up -d postgres valkey"); + await waitForService("postgres"); + await waitForService("valkey"); }, TIMEOUT); afterAll(async () => { - await dockerCompose('down -v').catch(() => {}); + await dockerCompose("down -v").catch(() => {}); }, TIMEOUT); it( - 'should start API service with dependencies', + "should start API service with dependencies", async () => { // Start API - await dockerCompose('up -d api'); + await dockerCompose("up -d api"); // Wait for API to be healthy - const isHealthy = await waitForService('api'); + const isHealthy = await waitForService("api"); expect(isHealthy).toBe(true); // Verify API is accessible - const apiHealthy = await checkHttpEndpoint('http://localhost:3001/health'); + const apiHealthy = await checkHttpEndpoint("http://localhost:3001/health"); expect(apiHealthy).toBe(true); }, - TIMEOUT, + TIMEOUT ); it( - 'should start Web service with dependencies', + "should start Web service with dependencies", async () => { // Ensure API is running - await dockerCompose('up -d api'); - await waitForService('api'); + await dockerCompose("up -d api"); + await waitForService("api"); // Start Web - await dockerCompose('up -d web'); + await dockerCompose("up -d web"); // Wait for Web to be healthy - const isHealthy = await waitForService('web'); + const isHealthy = await waitForService("web"); expect(isHealthy).toBe(true); // Verify Web is accessible - const webHealthy = await checkHttpEndpoint('http://localhost:3000'); + const webHealthy = await checkHttpEndpoint("http://localhost:3000"); expect(webHealthy).toBe(true); }, - TIMEOUT, + TIMEOUT ); }); - describe('Optional Services (Profiles)', () => { + describe("Optional Services (Profiles)", () => { afterAll(async () => { - await dockerCompose('down -v').catch(() => {}); + await dockerCompose("down -v").catch(() => {}); }, TIMEOUT); it( - 'should start Authentik services with profile', + "should start Authentik services with profile", async () => { // Start Authentik services using profile - await dockerCompose('--profile authentik up -d'); + await dockerCompose("--profile authentik up -d"); // Wait for Authentik dependencies - await waitForService('authentik-postgres'); - await waitForService('authentik-redis'); + await waitForService("authentik-postgres"); + await waitForService("authentik-redis"); // Verify Authentik server starts - const isHealthy = await waitForService('authentik-server'); + const isHealthy = await waitForService("authentik-server"); expect(isHealthy).toBe(true); // Verify Authentik is accessible - const authentikHealthy = await checkHttpEndpoint( - 'http://localhost:9000/-/health/live/', - ); + const authentikHealthy = await checkHttpEndpoint("http://localhost:9000/-/health/live/"); expect(authentikHealthy).toBe(true); }, - TIMEOUT, + TIMEOUT ); it( - 'should start Ollama service with profile', + "should start Ollama service with profile", async () => { // Start Ollama using profile - await dockerCompose('--profile ollama up -d ollama'); + await dockerCompose("--profile ollama up -d ollama"); // Wait for Ollama to start - const isHealthy = await waitForService('ollama'); + const isHealthy = await waitForService("ollama"); expect(isHealthy).toBe(true); // Verify Ollama is accessible - const ollamaHealthy = await checkHttpEndpoint( - 'http://localhost:11434/api/tags', - ); + const ollamaHealthy = await checkHttpEndpoint("http://localhost:11434/api/tags"); expect(ollamaHealthy).toBe(true); }, - TIMEOUT, + TIMEOUT ); }); - describe('Full Stack', () => { + describe("Full Stack", () => { it( - 'should start entire stack with all services', + "should start entire stack with all services", async () => { // Clean slate - await dockerCompose('down -v').catch(() => {}); + await dockerCompose("down -v").catch(() => {}); // Start all core services (without profiles) - await dockerCompose('up -d'); + await dockerCompose("up -d"); // Wait for all core services - const postgresHealthy = await waitForService('postgres'); - const valkeyHealthy = await waitForService('valkey'); - const apiHealthy = await waitForService('api'); - const webHealthy = await waitForService('web'); + const postgresHealthy = await waitForService("postgres"); + const valkeyHealthy = await waitForService("valkey"); + const apiHealthy = await waitForService("api"); + const webHealthy = await waitForService("web"); expect(postgresHealthy).toBe(true); expect(valkeyHealthy).toBe(true); @@ -255,255 +246,246 @@ describe('Docker Stack Integration Tests', () => { expect(webHealthy).toBe(true); // Verify all services are running - const ps = await dockerCompose('ps'); - expect(ps).toContain('mosaic-postgres'); - expect(ps).toContain('mosaic-valkey'); - expect(ps).toContain('mosaic-api'); - expect(ps).toContain('mosaic-web'); + const ps = await dockerCompose("ps"); + expect(ps).toContain("mosaic-postgres"); + expect(ps).toContain("mosaic-valkey"); + expect(ps).toContain("mosaic-api"); + expect(ps).toContain("mosaic-web"); }, - TIMEOUT * 2, + TIMEOUT * 2 ); }); - describe('Service Dependencies', () => { + describe("Service Dependencies", () => { beforeAll(async () => { - await dockerCompose('down -v').catch(() => {}); + await dockerCompose("down -v").catch(() => {}); }, TIMEOUT); afterAll(async () => { - await dockerCompose('down -v').catch(() => {}); + await dockerCompose("down -v").catch(() => {}); }, TIMEOUT); it( - 'should enforce dependency order', + "should enforce dependency order", async () => { // Start web service (should auto-start dependencies) - await dockerCompose('up -d web'); + await dockerCompose("up -d web"); // Wait a bit for services to start await new Promise((resolve) => setTimeout(resolve, 10000)); // Verify all dependencies are running - const ps = await dockerCompose('ps'); - expect(ps).toContain('mosaic-postgres'); - expect(ps).toContain('mosaic-valkey'); - expect(ps).toContain('mosaic-api'); - expect(ps).toContain('mosaic-web'); + const ps = await dockerCompose("ps"); + expect(ps).toContain("mosaic-postgres"); + expect(ps).toContain("mosaic-valkey"); + expect(ps).toContain("mosaic-api"); + expect(ps).toContain("mosaic-web"); }, - TIMEOUT, + TIMEOUT ); }); - describe('Container Labels', () => { + describe("Container Labels", () => { beforeAll(async () => { - await dockerCompose('up -d postgres valkey'); + await dockerCompose("up -d postgres valkey"); }, TIMEOUT); afterAll(async () => { - await dockerCompose('down -v').catch(() => {}); + await dockerCompose("down -v").catch(() => {}); }, TIMEOUT); - it('should have proper labels on containers', async () => { + it("should have proper labels on containers", async () => { // Check PostgreSQL labels const { stdout: pgLabels } = await execAsync( - 'docker inspect mosaic-postgres --format "{{json .Config.Labels}}"', + 'docker inspect mosaic-postgres --format "{{json .Config.Labels}}"' ); const pgLabelsObj = JSON.parse(pgLabels); - expect(pgLabelsObj['com.mosaic.service']).toBe('database'); - expect(pgLabelsObj['com.mosaic.description']).toBeDefined(); + expect(pgLabelsObj["com.mosaic.service"]).toBe("database"); + expect(pgLabelsObj["com.mosaic.description"]).toBeDefined(); // Check Valkey labels const { stdout: valkeyLabels } = await execAsync( - 'docker inspect mosaic-valkey --format "{{json .Config.Labels}}"', + 'docker inspect mosaic-valkey --format "{{json .Config.Labels}}"' ); const valkeyLabelsObj = JSON.parse(valkeyLabels); - expect(valkeyLabelsObj['com.mosaic.service']).toBe('cache'); - expect(valkeyLabelsObj['com.mosaic.description']).toBeDefined(); + expect(valkeyLabelsObj["com.mosaic.service"]).toBe("cache"); + expect(valkeyLabelsObj["com.mosaic.description"]).toBeDefined(); }); + }); - describe('Failure Scenarios', () => { + describe("Failure Scenarios", () => { const TIMEOUT = 120000; beforeAll(async () => { - await dockerCompose('down -v').catch(() => {}); + await dockerCompose("down -v").catch(() => {}); }, TIMEOUT); afterAll(async () => { - await dockerCompose('down -v').catch(() => {}); + await dockerCompose("down -v").catch(() => {}); }, TIMEOUT); it( - 'should handle service restart after crash', + "should handle service restart after crash", async () => { - await dockerCompose('up -d postgres'); - await waitForService('postgres'); + await dockerCompose("up -d postgres"); + await waitForService("postgres"); - const { stdout: containerName } = await execAsync( - 'docker compose ps -q postgres', - ); + const { stdout: containerName } = await execAsync("docker compose ps -q postgres"); const trimmedName = containerName.trim(); await execAsync(`docker kill ${trimmedName}`); await new Promise((resolve) => setTimeout(resolve, 3000)); - await dockerCompose('up -d postgres'); + await dockerCompose("up -d postgres"); - const isHealthy = await waitForService('postgres'); + const isHealthy = await waitForService("postgres"); expect(isHealthy).toBe(true); }, - TIMEOUT, + TIMEOUT ); it( - 'should handle port conflict gracefully', + "should handle port conflict gracefully", async () => { try { const { stdout } = await execAsync( - 'docker run -d -p 5432:5432 --name port-blocker postgres:17-alpine', + "docker run -d -p 5432:5432 --name port-blocker postgres:17-alpine" ); - await dockerCompose('up -d postgres').catch((error) => { - expect(error.message).toContain('port is already allocated'); + await dockerCompose("up -d postgres").catch((error) => { + expect(error.message).toContain("port is already allocated"); }); } finally { - await execAsync('docker rm -f port-blocker').catch(() => {}); + await execAsync("docker rm -f port-blocker").catch(() => {}); } }, - TIMEOUT, + TIMEOUT ); it( - 'should handle invalid volume mount paths', + "should handle invalid volume mount paths", async () => { try { await execAsync( - 'docker run -d --name invalid-mount -v /nonexistent/path:/data postgres:17-alpine', + "docker run -d --name invalid-mount -v /nonexistent/path:/data postgres:17-alpine" ); const { stdout } = await execAsync( - 'docker ps -a -f name=invalid-mount --format "{{.Status}}"', + 'docker ps -a -f name=invalid-mount --format "{{.Status}}"' ); expect(stdout).toBeDefined(); } catch (error) { expect(error).toBeDefined(); } finally { - await execAsync('docker rm -f invalid-mount').catch(() => {}); + await execAsync("docker rm -f invalid-mount").catch(() => {}); } }, - TIMEOUT, + TIMEOUT ); it( - 'should handle network partition scenarios', + "should handle network partition scenarios", async () => { - await dockerCompose('up -d postgres valkey'); - await waitForService('postgres'); - await waitForService('valkey'); + await dockerCompose("up -d postgres valkey"); + await waitForService("postgres"); + await waitForService("valkey"); - const { stdout: postgresContainer } = await execAsync( - 'docker compose ps -q postgres', - ); + const { stdout: postgresContainer } = await execAsync("docker compose ps -q postgres"); const trimmedPostgres = postgresContainer.trim(); - await execAsync( - `docker network disconnect mosaic-internal ${trimmedPostgres}`, - ); + await execAsync(`docker network disconnect mosaic-internal ${trimmedPostgres}`); await new Promise((resolve) => setTimeout(resolve, 2000)); - await execAsync( - `docker network connect mosaic-internal ${trimmedPostgres}`, - ); + await execAsync(`docker network connect mosaic-internal ${trimmedPostgres}`); - const isHealthy = await waitForService('postgres'); + const isHealthy = await waitForService("postgres"); expect(isHealthy).toBe(true); }, - TIMEOUT, + TIMEOUT ); it( - 'should handle container out of memory scenario', + "should handle container out of memory scenario", async () => { try { const { stdout } = await execAsync( - 'docker run -d --name mem-limited --memory="10m" postgres:17-alpine', + 'docker run -d --name mem-limited --memory="10m" postgres:17-alpine' ); await new Promise((resolve) => setTimeout(resolve, 5000)); const { stdout: status } = await execAsync( - 'docker inspect mem-limited --format "{{.State.Status}}"', + 'docker inspect mem-limited --format "{{.State.Status}}"' ); - expect(['exited', 'running']).toContain(status.trim()); + expect(["exited", "running"]).toContain(status.trim()); } finally { - await execAsync('docker rm -f mem-limited').catch(() => {}); + await execAsync("docker rm -f mem-limited").catch(() => {}); } }, - TIMEOUT, + TIMEOUT ); it( - 'should handle disk space issues gracefully', + "should handle disk space issues gracefully", async () => { - await dockerCompose('up -d postgres'); - await waitForService('postgres'); + await dockerCompose("up -d postgres"); + await waitForService("postgres"); - const { stdout } = await execAsync('docker system df'); - expect(stdout).toContain('Images'); - expect(stdout).toContain('Containers'); - expect(stdout).toContain('Volumes'); + const { stdout } = await execAsync("docker system df"); + expect(stdout).toContain("Images"); + expect(stdout).toContain("Containers"); + expect(stdout).toContain("Volumes"); }, - TIMEOUT, + TIMEOUT ); it( - 'should handle service dependency failures', + "should handle service dependency failures", async () => { - await dockerCompose('up -d postgres').catch(() => {}); + await dockerCompose("up -d postgres").catch(() => {}); - const { stdout: postgresContainer } = await execAsync( - 'docker compose ps -q postgres', - ); + const { stdout: postgresContainer } = await execAsync("docker compose ps -q postgres"); const trimmedPostgres = postgresContainer.trim(); if (trimmedPostgres) { await execAsync(`docker stop ${trimmedPostgres}`); try { - await dockerCompose('up -d api'); + await dockerCompose("up -d api"); } catch (error) { expect(error).toBeDefined(); } - await dockerCompose('start postgres').catch(() => {}); - await waitForService('postgres'); + await dockerCompose("start postgres").catch(() => {}); + await waitForService("postgres"); - await dockerCompose('up -d api'); - const apiHealthy = await waitForService('api'); + await dockerCompose("up -d api"); + const apiHealthy = await waitForService("api"); expect(apiHealthy).toBe(true); } }, - TIMEOUT, + TIMEOUT ); it( - 'should recover from corrupted volume data', + "should recover from corrupted volume data", async () => { - await dockerCompose('up -d postgres'); - await waitForService('postgres'); + await dockerCompose("up -d postgres"); + await waitForService("postgres"); - await dockerCompose('down'); + await dockerCompose("down"); - await execAsync('docker volume rm mosaic-postgres-data').catch(() => {}); + await execAsync("docker volume rm mosaic-postgres-data").catch(() => {}); - await dockerCompose('up -d postgres'); - const isHealthy = await waitForService('postgres'); + await dockerCompose("up -d postgres"); + const isHealthy = await waitForService("postgres"); expect(isHealthy).toBe(true); }, - TIMEOUT, + TIMEOUT ); }); }); diff --git a/tests/integration/openbao.test.ts b/tests/integration/openbao.test.ts new file mode 100644 index 0000000..3d3dc4b --- /dev/null +++ b/tests/integration/openbao.test.ts @@ -0,0 +1,600 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { exec } from "child_process"; +import { promisify } from "util"; + +const execAsync = promisify(exec); + +/** + * OpenBao Integration Tests + * Tests the OpenBao deployment with auto-init and Transit engine + */ +describe("OpenBao Integration Tests", () => { + const TIMEOUT = 120000; // 2 minutes for Docker operations + const HEALTH_CHECK_RETRIES = 30; + const HEALTH_CHECK_INTERVAL = 2000; + + // Top-level setup: Start services once for all tests + beforeAll(async () => { + // Ensure clean state + await execAsync("docker compose down -v", { + cwd: `${process.cwd()}/docker`, + }).catch(() => {}); + + // Start OpenBao and init container + await execAsync("docker compose up -d openbao openbao-init", { + cwd: `${process.cwd()}/docker`, + }); + + // Wait for OpenBao to be healthy + const openbaoHealthy = await waitForService("openbao"); + if (!openbaoHealthy) { + throw new Error("OpenBao failed to become healthy"); + } + + // Wait for initialization to complete (init container running) + const initRunning = await waitForService("openbao-init"); + if (!initRunning) { + throw new Error("OpenBao init container failed to start"); + } + + // Wait for initialization to complete (give it time to configure) + await new Promise((resolve) => setTimeout(resolve, 30000)); + }, 180000); // 3 minutes for initial setup + + // Top-level teardown: Clean up after all tests + afterAll(async () => { + await execAsync("docker compose down -v", { + cwd: `${process.cwd()}/docker`, + }).catch(() => {}); + }, TIMEOUT); + + /** + * Helper function to execute Docker Compose commands + */ + async function dockerCompose(command: string): Promise { + const { stdout } = await execAsync(`docker compose ${command}`, { + cwd: `${process.cwd()}/docker`, + }); + return stdout; + } + + /** + * Helper function to check if a service is healthy + */ + async function waitForService( + serviceName: string, + retries = HEALTH_CHECK_RETRIES + ): Promise { + for (let i = 0; i < retries; i++) { + try { + const { stdout } = await execAsync(`docker compose ps --format json ${serviceName}`, { + cwd: `${process.cwd()}/docker`, + }); + const serviceInfo = JSON.parse(stdout); + + if (serviceInfo.Health === "healthy" || serviceInfo.State === "running") { + return true; + } + } catch (error) { + // Service not ready yet + } + + await new Promise((resolve) => setTimeout(resolve, HEALTH_CHECK_INTERVAL)); + } + return false; + } + + /** + * Helper function to check HTTP endpoint + * Returns true for any non-5xx status (including sealed/uninitialized states) + */ + async function checkHttpEndpoint(url: string): Promise { + try { + const response = await fetch(url); + // OpenBao returns 501 when uninitialized, 503 when sealed + // Both are valid "healthy" states for these tests + return response.status < 500; + } catch (error) { + return false; + } + } + + /** + * Helper to execute commands inside OpenBao container + */ + async function execInBao(command: string): Promise { + const { stdout } = await execAsync(`docker compose exec -T openbao ${command}`, { + cwd: `${process.cwd()}/docker`, + }); + return stdout; + } + + /** + * Helper to read secret files from OpenBao init volume + * Uses docker run to mount volume and read file safely + * Sanitizes error messages to prevent secret leakage + */ + async function readSecretFile(fileName: string): Promise { + try { + const { stdout } = await execAsync( + `docker run --rm -v mosaic-openbao-init:/data alpine cat /data/${fileName}` + ); + return stdout.trim(); + } catch (error) { + // Sanitize error message to prevent secret leakage + const sanitizedError = new Error( + `Failed to read secret file: ${fileName} (file may not exist or volume not mounted)` + ); + throw sanitizedError; + } + } + + /** + * Helper to read and parse JSON secret file + */ + async function readSecretJSON(fileName: string): Promise { + try { + const content = await readSecretFile(fileName); + return JSON.parse(content); + } catch (error) { + // Sanitize error to prevent leaking partial secret data + const sanitizedError = new Error(`Failed to parse secret JSON from: ${fileName}`); + throw sanitizedError; + } + } + + describe("OpenBao Service Startup", () => { + it( + "should start OpenBao server with health check", + async () => { + // Start OpenBao service + await dockerCompose("up -d openbao"); + + // Wait for service to be healthy + const isHealthy = await waitForService("openbao"); + expect(isHealthy).toBe(true); + + // Verify container is running + const ps = await dockerCompose("ps openbao"); + expect(ps).toContain("mosaic-openbao"); + expect(ps).toContain("Up"); + }, + TIMEOUT + ); + + it( + "should have OpenBao API accessible on port 8200", + async () => { + // Ensure OpenBao is running + await dockerCompose("up -d openbao"); + await waitForService("openbao"); + + // Check health endpoint with flags to accept sealed/uninitialized states + const isHealthy = await checkHttpEndpoint( + "http://localhost:8200/v1/sys/health?standbyok=true&sealedok=true&uninitok=true" + ); + expect(isHealthy).toBe(true); + }, + TIMEOUT + ); + + it( + "should have proper volume configuration", + async () => { + // Start OpenBao + await dockerCompose("up -d openbao"); + await waitForService("openbao"); + + // Check if volumes are created + const { stdout } = await execAsync("docker volume ls"); + expect(stdout).toContain("mosaic-openbao-data"); + expect(stdout).toContain("mosaic-openbao-config"); + expect(stdout).toContain("mosaic-openbao-init"); + }, + TIMEOUT + ); + }); + + describe("OpenBao Auto-Initialization", () => { + it( + "should initialize OpenBao on first run", + async () => { + // Wait for init container to complete + await new Promise((resolve) => setTimeout(resolve, 10000)); + + // Check initialization status + const status = await execInBao("bao status -format=json"); + const statusObj = JSON.parse(status); + + expect(statusObj.initialized).toBe(true); + expect(statusObj.sealed).toBe(false); + }, + TIMEOUT + ); + + it( + "should create unseal key in init volume", + async () => { + // Wait for init to complete + await new Promise((resolve) => setTimeout(resolve, 10000)); + + // Check if unseal key file exists + const { stdout } = await execAsync( + "docker run --rm -v mosaic-openbao-init:/data alpine ls -la /data/" + ); + expect(stdout).toContain("unseal-key"); + }, + TIMEOUT + ); + + it( + "should be idempotent on restart", + async () => { + // Wait for first init + await new Promise((resolve) => setTimeout(resolve, 10000)); + + // Restart init container + await dockerCompose("restart openbao-init"); + await new Promise((resolve) => setTimeout(resolve, 5000)); + + // Should still be initialized and unsealed + const status = await execInBao("bao status -format=json"); + const statusObj = JSON.parse(status); + + expect(statusObj.initialized).toBe(true); + expect(statusObj.sealed).toBe(false); + }, + TIMEOUT + ); + }); + + describe("OpenBao Transit Engine", () => { + it( + "should enable Transit secrets engine", + async () => { + // Get root token from init volume + const { stdout: token } = await execAsync( + "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token" + ); + + // Check if Transit is enabled + const { stdout: mounts } = await execAsync( + `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao secrets list -format=json`, + { cwd: `${process.cwd()}/docker` } + ); + const mountsObj = JSON.parse(mounts); + + expect(mountsObj["transit/"]).toBeDefined(); + expect(mountsObj["transit/"].type).toBe("transit"); + }, + TIMEOUT + ); + + it( + "should create mosaic-credentials Transit key", + async () => { + const { stdout: token } = await execAsync( + "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token" + ); + + // List Transit keys + const { stdout: keys } = await execAsync( + `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao list -format=json transit/keys`, + { cwd: `${process.cwd()}/docker` } + ); + const keysArray = JSON.parse(keys); + + expect(keysArray).toContain("mosaic-credentials"); + }, + TIMEOUT + ); + + it( + "should create mosaic-account-tokens Transit key", + async () => { + const { stdout: token } = await execAsync( + "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token" + ); + + const { stdout: keys } = await execAsync( + `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao list -format=json transit/keys`, + { cwd: `${process.cwd()}/docker` } + ); + const keysArray = JSON.parse(keys); + + expect(keysArray).toContain("mosaic-account-tokens"); + }, + TIMEOUT + ); + + it( + "should create mosaic-federation Transit key", + async () => { + const { stdout: token } = await execAsync( + "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token" + ); + + const { stdout: keys } = await execAsync( + `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao list -format=json transit/keys`, + { cwd: `${process.cwd()}/docker` } + ); + const keysArray = JSON.parse(keys); + + expect(keysArray).toContain("mosaic-federation"); + }, + TIMEOUT + ); + + it( + "should create mosaic-llm-config Transit key", + async () => { + const { stdout: token } = await execAsync( + "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token" + ); + + const { stdout: keys } = await execAsync( + `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao list -format=json transit/keys`, + { cwd: `${process.cwd()}/docker` } + ); + const keysArray = JSON.parse(keys); + + expect(keysArray).toContain("mosaic-llm-config"); + }, + TIMEOUT + ); + + it( + "should verify Transit keys use aes256-gcm96", + async () => { + const { stdout: token } = await execAsync( + "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token" + ); + + // Check key type for mosaic-credentials + const { stdout: keyInfo } = await execAsync( + `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao read -format=json transit/keys/mosaic-credentials`, + { cwd: `${process.cwd()}/docker` } + ); + const keyInfoObj = JSON.parse(keyInfo); + + expect(keyInfoObj.data.type).toBe("aes256-gcm96"); + }, + TIMEOUT + ); + }); + + describe("OpenBao AppRole Configuration", () => { + it( + "should enable AppRole auth method", + async () => { + const { stdout: token } = await execAsync( + "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token" + ); + + // Check if AppRole is enabled + const { stdout: auths } = await execAsync( + `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao auth list -format=json`, + { cwd: `${process.cwd()}/docker` } + ); + const authsObj = JSON.parse(auths); + + expect(authsObj["approle/"]).toBeDefined(); + expect(authsObj["approle/"].type).toBe("approle"); + }, + TIMEOUT + ); + + it( + "should create mosaic-transit AppRole", + async () => { + const { stdout: token } = await execAsync( + "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token" + ); + + // List AppRoles + const { stdout: roles } = await execAsync( + `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao list -format=json auth/approle/role`, + { cwd: `${process.cwd()}/docker` } + ); + const rolesArray = JSON.parse(roles); + + expect(rolesArray).toContain("mosaic-transit"); + }, + TIMEOUT + ); + + it( + "should create AppRole credentials file", + async () => { + // Check if credentials file exists + const { stdout } = await execAsync( + "docker run --rm -v mosaic-openbao-init:/data alpine ls -la /data/" + ); + expect(stdout).toContain("approle-credentials"); + }, + TIMEOUT + ); + + it( + "should have valid AppRole credentials", + async () => { + // Read credentials file + const { stdout: credentials } = await execAsync( + "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/approle-credentials" + ); + const credsObj = JSON.parse(credentials); + + expect(credsObj.role_id).toBeDefined(); + expect(credsObj.secret_id).toBeDefined(); + expect(typeof credsObj.role_id).toBe("string"); + expect(typeof credsObj.secret_id).toBe("string"); + }, + TIMEOUT + ); + }); + + describe("OpenBao Auto-Unseal on Restart", () => { + it( + "should auto-unseal after container restart", + async () => { + // Verify initially unsealed + let status = await execInBao("bao status -format=json"); + let statusObj = JSON.parse(status); + expect(statusObj.sealed).toBe(false); + + // Restart OpenBao container + await dockerCompose("restart openbao"); + await waitForService("openbao"); + + // Restart init container to trigger unseal + await dockerCompose("restart openbao-init"); + await new Promise((resolve) => setTimeout(resolve, 10000)); + + // Verify auto-unsealed + status = await execInBao("bao status -format=json"); + statusObj = JSON.parse(status); + expect(statusObj.sealed).toBe(false); + }, + TIMEOUT + ); + + it( + "should preserve Transit keys after restart", + async () => { + // Restart OpenBao + await dockerCompose("restart openbao openbao-init"); + await waitForService("openbao"); + await new Promise((resolve) => setTimeout(resolve, 10000)); + + const { stdout: token } = await execAsync( + "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token" + ); + + // Verify Transit keys still exist + const { stdout: keys } = await execAsync( + `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao list -format=json transit/keys`, + { cwd: `${process.cwd()}/docker` } + ); + const keysArray = JSON.parse(keys); + + expect(keysArray).toContain("mosaic-credentials"); + expect(keysArray).toContain("mosaic-account-tokens"); + expect(keysArray).toContain("mosaic-federation"); + expect(keysArray).toContain("mosaic-llm-config"); + }, + TIMEOUT + ); + }); + + describe("OpenBao Security and Policies", () => { + it( + "should have Transit-only policy for AppRole", + async () => { + const { stdout: token } = await execAsync( + "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token" + ); + + // Read policy + const { stdout: policy } = await execAsync( + `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao policy read mosaic-transit-policy`, + { cwd: `${process.cwd()}/docker` } + ); + + // Verify policy allows encrypt/decrypt + expect(policy).toContain("transit/encrypt/*"); + expect(policy).toContain("transit/decrypt/*"); + }, + TIMEOUT + ); + + it( + "should test AppRole can encrypt data", + async () => { + // Get AppRole credentials + const { stdout: credentials } = await execAsync( + "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/approle-credentials" + ); + const credsObj = JSON.parse(credentials); + + // Login with AppRole + const { stdout: loginResponse } = await execAsync( + `docker compose exec -T openbao bao write -format=json auth/approle/login role_id=${credsObj.role_id} secret_id=${credsObj.secret_id}`, + { cwd: `${process.cwd()}/docker` } + ); + const loginObj = JSON.parse(loginResponse); + const appRoleToken = loginObj.auth.client_token; + + // Try to encrypt + const testData = Buffer.from("test-data").toString("base64"); + const { stdout: encryptResponse } = await execAsync( + `docker compose exec -T -e VAULT_TOKEN=${appRoleToken} openbao bao write -format=json transit/encrypt/mosaic-credentials plaintext=${testData}`, + { cwd: `${process.cwd()}/docker` } + ); + const encryptObj = JSON.parse(encryptResponse); + + expect(encryptObj.data.ciphertext).toBeDefined(); + expect(encryptObj.data.ciphertext).toMatch(/^vault:v\d+:/); + }, + TIMEOUT + ); + + it( + "should test AppRole can decrypt data", + async () => { + // Get AppRole credentials and login + const { stdout: credentials } = await execAsync( + "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/approle-credentials" + ); + const credsObj = JSON.parse(credentials); + + const { stdout: loginResponse } = await execAsync( + `docker compose exec -T openbao bao write -format=json auth/approle/login role_id=${credsObj.role_id} secret_id=${credsObj.secret_id}`, + { cwd: `${process.cwd()}/docker` } + ); + const loginObj = JSON.parse(loginResponse); + const appRoleToken = loginObj.auth.client_token; + + // Encrypt data first + const testData = Buffer.from("test-decrypt").toString("base64"); + const { stdout: encryptResponse } = await execAsync( + `docker compose exec -T -e VAULT_TOKEN=${appRoleToken} openbao bao write -format=json transit/encrypt/mosaic-credentials plaintext=${testData}`, + { cwd: `${process.cwd()}/docker` } + ); + const encryptObj = JSON.parse(encryptResponse); + const ciphertext = encryptObj.data.ciphertext; + + // Decrypt + const { stdout: decryptResponse } = await execAsync( + `docker compose exec -T -e VAULT_TOKEN=${appRoleToken} openbao bao write -format=json transit/decrypt/mosaic-credentials ciphertext=${ciphertext}`, + { cwd: `${process.cwd()}/docker` } + ); + const decryptObj = JSON.parse(decryptResponse); + + const plaintext = Buffer.from(decryptObj.data.plaintext, "base64").toString(); + expect(plaintext).toBe("test-decrypt"); + }, + TIMEOUT + ); + }); + + describe("OpenBao Service Dependencies", () => { + it( + "should start openbao-init only after openbao is healthy", + async () => { + // Start both services + await dockerCompose("up -d openbao openbao-init"); + + // Wait for OpenBao to be healthy + const openbaoHealthy = await waitForService("openbao"); + expect(openbaoHealthy).toBe(true); + + // Init should start after OpenBao is healthy + await new Promise((resolve) => setTimeout(resolve, 5000)); + + const ps = await dockerCompose("ps openbao-init"); + expect(ps).toContain("mosaic-openbao-init"); + }, + TIMEOUT + ); + }); +});