From c195b8c8fd8565f3aa2dfb589c3453c217cb3b04 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Sun, 8 Feb 2026 17:30:30 -0600 Subject: [PATCH] feat(openbao): add standalone deployment for swarm compatibility - Create docker-compose.openbao.yml for standalone OpenBao deployment - Includes openbao and openbao-init services - Auto-initialization on first run - Connects to swarm's mosaic_internal network - Binds to localhost:8200 for security - Update docker-compose.swarm.yml - Comment out OpenBao service (cannot run in swarm) - Add clear note about standalone requirement - Update volumes section - Update header with current config - Create docs/OPENBAO-DEPLOYMENT.md - Comprehensive deployment guide - 4 deployment options: standalone, bundled, external, fallback - Clear explanation why OpenBao can't run in swarm - Deployment workflows for each scenario - Troubleshooting section - Update docs/SWARM-DEPLOYMENT.md - Add Step 1: Deploy OpenBao standalone FIRST - Remove manual initialization (now automatic) - Update expected services list - Reference OpenBao deployment guide - Update README.md - Clarify OpenBao standalone requirement for swarm - Update deployment steps - Highlight critical requirement at top of notes Key changes: - OpenBao MUST be deployed standalone when using swarm - Automatic initialization via openbao-init sidecar - Clear documentation for all deployment options - Swarm stack no longer includes OpenBao Co-Authored-By: Claude Opus 4.6 --- README.md | 16 +- docker-compose.openbao.yml | 93 +++++++++++ docker-compose.swarm.yml | 70 ++++---- docs/OPENBAO-DEPLOYMENT.md | 318 +++++++++++++++++++++++++++++++++++++ docs/SWARM-DEPLOYMENT.md | 105 ++++++------ 5 files changed, 504 insertions(+), 98 deletions(-) create mode 100644 docker-compose.openbao.yml create mode 100644 docs/OPENBAO-DEPLOYMENT.md diff --git a/README.md b/README.md index ccfb9b9..f08d2e1 100644 --- a/README.md +++ b/README.md @@ -173,16 +173,18 @@ docker network create --driver=overlay traefik-public cp .env.swarm.example .env nano .env # Configure domains, passwords, API keys -# 4. Deploy stack -./scripts/deploy-swarm.sh mosaic +# 4. CRITICAL: Deploy OpenBao standalone FIRST +# OpenBao cannot run in swarm mode - deploy as standalone container +docker compose -f docker-compose.openbao.yml up -d +sleep 30 # Wait for auto-initialization -# 5. Check deployment status +# 5. Deploy swarm stack +IMAGE_TAG=dev ./scripts/deploy-swarm.sh mosaic + +# 6. Check deployment status docker stack services mosaic docker stack ps mosaic -# 6. CRITICAL: Initialize OpenBao manually (see docs) -# Unlike docker-compose, swarm requires manual OpenBao initialization - # Access services via Traefik # Web: http://mosaic.mosaicstack.dev # API: http://api.mosaicstack.dev @@ -200,9 +202,9 @@ docker stack ps mosaic **Important Notes:** +- **OpenBao Requirement:** OpenBao MUST be deployed as standalone container (not in swarm). Use `docker-compose.openbao.yml` or external Vault. - Swarm does NOT support docker-compose profiles - To use external services (PostgreSQL, Authentik, etc.), manually comment them out in `docker-compose.swarm.yml` -- OpenBao requires manual initialization (no auto-init sidecar in swarm mode) See [Docker Swarm Deployment Guide](docs/SWARM-DEPLOYMENT.md) and [Quick Reference](docs/SWARM-QUICKREF.md) for complete documentation. diff --git a/docker-compose.openbao.yml b/docker-compose.openbao.yml new file mode 100644 index 0000000..316c007 --- /dev/null +++ b/docker-compose.openbao.yml @@ -0,0 +1,93 @@ +# ============================================== +# OpenBao Standalone Deployment +# ============================================== +# +# IMPORTANT: This file deploys OpenBao as a STANDALONE container. +# Do NOT include this in docker stack deploy - it will fail due to port binding conflicts. +# +# Usage: +# docker compose -f docker-compose.openbao.yml up -d +# +# This is required when: +# - Using Docker Swarm (stateful services don't work well in swarm) +# - You want OpenBao isolated from the main stack +# +# Alternative: Use external HashiCorp Vault or managed secrets service +# ============================================== + +services: + # ====================== + # OpenBao Secrets Vault + # ====================== + openbao: + image: git.mosaicstack.dev/mosaic/stack-openbao:${IMAGE_TAG:-dev} + container_name: mosaic-openbao + env_file: .env + environment: + OPENBAO_ADDR: http://0.0.0.0:8200 + OPENBAO_DEV_ROOT_TOKEN_ID: ${OPENBAO_DEV_ROOT_TOKEN_ID:-root} + ports: + - "127.0.0.1:${OPENBAO_PORT:-8200}:8200" # Localhost only for security + volumes: + - openbao_data:/openbao/data + - openbao_logs:/openbao/logs + - openbao_init:/openbao/init + cap_add: + - IPC_LOCK + healthcheck: + test: + - CMD + - wget + - --spider + - --quiet + - http://localhost:8200/v1/sys/health?standbyok=true + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + restart: unless-stopped + networks: + - mosaic_internal + + # ====================== + # OpenBao Init Sidecar + # ====================== + # Auto-initializes and unseals OpenBao on first run + openbao-init: + image: git.mosaicstack.dev/mosaic/stack-openbao:${IMAGE_TAG:-dev} + container_name: mosaic-openbao-init + env_file: .env + command: /openbao/init.sh + environment: + OPENBAO_ADDR: http://openbao:8200 + volumes: + - openbao_init:/openbao/init + depends_on: + openbao: + condition: service_healthy + restart: "no" + networks: + - mosaic_internal + +# ====================== +# Volumes +# ====================== +volumes: + openbao_data: + name: mosaic-openbao-data + driver: local + openbao_logs: + name: mosaic-openbao-logs + driver: local + openbao_init: + name: mosaic-openbao-init + driver: local + +# ====================== +# Networks +# ====================== +# Connect to the swarm stack's internal network +networks: + mosaic_internal: + external: true + name: mosaic_internal diff --git a/docker-compose.swarm.yml b/docker-compose.swarm.yml index 60babf3..a19c0bd 100644 --- a/docker-compose.swarm.yml +++ b/docker-compose.swarm.yml @@ -8,9 +8,9 @@ # Current Configuration: # - PostgreSQL: ENABLED (internal) # - Valkey: ENABLED (internal) -# - OpenBao: ENABLED (internal) +# - OpenBao: DISABLED (must use standalone - see docker-compose.openbao.yml) # - Authentik: DISABLED (commented out - using external OIDC) -# - Ollama: ENABLED (internal) +# - Ollama: DISABLED (commented out - using external Ollama) # # For detailed deployment instructions, see: # docs/SWARM-DEPLOYMENT.md @@ -78,32 +78,38 @@ services: condition: on-failure # ====================== - # OpenBao Secrets Vault + # OpenBao Secrets Vault - COMMENTED OUT # ====================== - openbao: - image: git.mosaicstack.dev/mosaic/stack-openbao:${IMAGE_TAG:-latest} - env_file: .env - environment: - OPENBAO_ADDR: ${OPENBAO_ADDR:-http://0.0.0.0:8200} - OPENBAO_DEV_ROOT_TOKEN_ID: ${OPENBAO_DEV_ROOT_TOKEN_ID:-root} - volumes: - - openbao_data:/openbao/data - - openbao_logs:/openbao/logs - - openbao_init:/openbao/init - cap_add: - - IPC_LOCK - healthcheck: - test: - ["CMD", "wget", "--spider", "--quiet", "http://localhost:8200/v1/sys/health?standbyok=true"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 30s - networks: - - internal - deploy: - restart_policy: - condition: on-failure + # IMPORTANT: OpenBao CANNOT run in swarm mode due to port binding conflicts. + # Deploy OpenBao as a standalone container instead: + # docker compose -f docker-compose.openbao.yml up -d + # + # Alternative: Use external HashiCorp Vault or managed secrets service + # + # openbao: + # image: git.mosaicstack.dev/mosaic/stack-openbao:${IMAGE_TAG:-latest} + # env_file: .env + # environment: + # OPENBAO_ADDR: ${OPENBAO_ADDR:-http://0.0.0.0:8200} + # OPENBAO_DEV_ROOT_TOKEN_ID: ${OPENBAO_DEV_ROOT_TOKEN_ID:-root} + # volumes: + # - openbao_data:/openbao/data + # - openbao_logs:/openbao/logs + # - openbao_init:/openbao/init + # cap_add: + # - IPC_LOCK + # healthcheck: + # test: + # ["CMD", "wget", "--spider", "--quiet", "http://localhost:8200/v1/sys/health?standbyok=true"] + # interval: 10s + # timeout: 5s + # retries: 5 + # start_period: 30s + # networks: + # - internal + # deploy: + # restart_policy: + # condition: on-failure # ====================== # Authentik - COMMENTED OUT (Using External Authentik) @@ -361,16 +367,18 @@ services: volumes: postgres_data: valkey_data: - openbao_data: - openbao_logs: - openbao_init: + # OpenBao volumes - commented out (using standalone deployment) + # openbao_data: + # openbao_logs: + # openbao_init: # Authentik volumes - commented out (using external Authentik) # authentik_postgres_data: # authentik_redis_data: # authentik_media: # authentik_certs: # authentik_templates: - ollama_data: + # Ollama volume - commented out (using external Ollama) + # ollama_data: orchestrator_workspace: # ====================== diff --git a/docs/OPENBAO-DEPLOYMENT.md b/docs/OPENBAO-DEPLOYMENT.md new file mode 100644 index 0000000..930abd6 --- /dev/null +++ b/docs/OPENBAO-DEPLOYMENT.md @@ -0,0 +1,318 @@ +# OpenBao Deployment Guide + +## Overview + +OpenBao provides Transit encryption for sensitive credentials in Mosaic Stack. Due to the stateful nature of secrets management and port binding requirements, OpenBao has specific deployment constraints. + +## Deployment Options + +### Option 1: Standalone Container (Recommended for Swarm) + +**When to use:** + +- Docker Swarm deployments +- You want OpenBao isolated from the main stack +- You need guaranteed port availability + +**How to deploy:** + +```bash +# Deploy OpenBao as standalone container +docker compose -f docker-compose.openbao.yml up -d + +# Check status +docker ps | grep openbao + +# View logs +docker logs mosaic-openbao +docker logs mosaic-openbao-init + +# The init container auto-initializes OpenBao on first run +# Check initialization status +docker logs mosaic-openbao-init +``` + +**Configuration:** + +The standalone deployment: + +- Binds to `127.0.0.1:8200` (localhost only for security) +- Auto-initializes via `openbao-init` sidecar +- Connects to swarm stack via `mosaic_internal` network +- Uses named volumes for persistence + +**File:** `docker-compose.openbao.yml` + +### Option 2: Bundled (Standalone Docker Compose Only) + +**When to use:** + +- Standalone docker-compose deployment (NOT swarm) +- Development/testing environments +- All-in-one turnkey setup + +**How to deploy:** + +```bash +# Use docker-compose.yml with full profile +export COMPOSE_PROFILES=full +docker compose up -d +``` + +**Configuration:** + +OpenBao runs as part of the main stack with: + +- `openbao` and `openbao-init` services +- Automatic initialization on first startup +- Integrated with other services + +**File:** `docker-compose.yml` (with `COMPOSE_PROFILES=full` or `COMPOSE_PROFILES=openbao`) + +### Option 3: External HashiCorp Vault + +**When to use:** + +- Production environments with existing Vault infrastructure +- Managed secrets service (HashiCorp Cloud Platform, AWS Secrets Manager + Vault) +- Multi-region deployments +- High availability requirements + +**How to configure:** + +1. Set environment variables: + + ```bash + OPENBAO_ADDR=https://vault.example.com:8200 + OPENBAO_ROLE_ID=your-approle-role-id + OPENBAO_SECRET_ID=your-approle-secret-id + ``` + +2. Ensure external Vault has: + - Transit secrets engine enabled + - Encryption key created: `mosaic-credentials` + - AppRole configured for Mosaic API authentication + +3. Comment out OpenBao in all compose files + +4. API will automatically connect to external Vault + +### Option 4: Fallback Mode (No Vault) + +**When to use:** + +- Development/testing without secrets infrastructure +- Simplified deployments +- Gradual migration to Vault + +**How to configure:** + +1. Comment out or don't deploy OpenBao +2. Set `ENCRYPTION_KEY` in `.env` (required!) +3. API automatically falls back to AES-256-GCM encryption + +**Configuration:** + +```bash +# Generate encryption key +ENCRYPTION_KEY=$(openssl rand -hex 32) +echo "ENCRYPTION_KEY=$ENCRYPTION_KEY" >> .env +``` + +**Note:** This provides graceful degradation but lacks the key management features of OpenBao/Vault. + +## Why OpenBao Can't Run in Swarm + +OpenBao is a **stateful service** that binds to a specific port (`8200`). In Docker Swarm: + +1. **Port binding conflicts:** Swarm services use overlay networks and can't reliably bind to host ports +2. **State management:** OpenBao maintains unsealed state and encryption keys that don't work with Swarm's task model +3. **Multiple replicas:** Swarm tries to create multiple replicas, causing port conflicts + +**Solution:** Deploy OpenBao as a standalone container that connects to the swarm network. + +## Deployment Workflows + +### Swarm + Standalone OpenBao + +```bash +# 1. Deploy OpenBao standalone +docker compose -f docker-compose.openbao.yml up -d + +# 2. Wait for initialization +sleep 30 +docker logs mosaic-openbao-init + +# 3. Deploy swarm stack +IMAGE_TAG=dev ./scripts/deploy-swarm.sh mosaic + +# 4. Verify API connects to OpenBao +docker service logs mosaic_api | grep -i openbao +``` + +### Standalone Compose + Bundled OpenBao + +```bash +# 1. Set profile +export COMPOSE_PROFILES=full + +# 2. Deploy everything +docker compose up -d + +# 3. Verify OpenBao initialization +docker logs mosaic-openbao-init +``` + +### External Vault + +```bash +# 1. Configure .env with external Vault URL and credentials +# OPENBAO_ADDR=https://vault.example.com:8200 +# OPENBAO_ROLE_ID=... +# OPENBAO_SECRET_ID=... + +# 2. Deploy stack (no OpenBao) +IMAGE_TAG=dev ./scripts/deploy-swarm.sh mosaic + +# 3. Verify API connects to external Vault +docker service logs mosaic_api | grep -i vault +``` + +## Network Configuration + +### Standalone OpenBao with Swarm Stack + +The standalone OpenBao container connects to the swarm stack's internal network: + +```yaml +networks: + mosaic_internal: + external: true + name: mosaic_internal +``` + +This allows services in the swarm stack to reach OpenBao at `http://openbao:8200`. + +### Port Exposure + +- **Standalone:** `127.0.0.1:8200` (localhost only) +- **Bundled:** No port exposure (internal network only) +- **External:** Your Vault URL (typically HTTPS on 8200) + +**Security Note:** Never expose OpenBao to the public internet without proper TLS and authentication. + +## Initialization and Unsealing + +### Auto-Initialization (Standalone & Bundled) + +The `openbao-init` sidecar automatically: + +1. Initializes OpenBao on first run +2. Creates unseal keys and root token +3. Unseals OpenBao +4. Enables Transit secrets engine +5. Creates `mosaic-credentials` encryption key +6. Sets up AppRole authentication + +**Credentials location:** `/openbao/init/approle-credentials` (mounted volume) + +### Manual Initialization (External Vault) + +For external Vault, you must manually: + +```bash +# 1. Enable transit engine +vault secrets enable transit + +# 2. Create encryption key +vault write -f transit/keys/mosaic-credentials + +# 3. Enable AppRole +vault auth enable approle + +# 4. Create role for Mosaic API +vault write auth/approle/role/mosaic-api \ + token_policies="default" \ + token_ttl=1h \ + token_max_ttl=4h + +# 5. Get credentials +vault read auth/approle/role/mosaic-api/role-id +vault write -f auth/approle/role/mosaic-api/secret-id + +# 6. Set in .env +# OPENBAO_ROLE_ID= +# OPENBAO_SECRET_ID= +``` + +## Troubleshooting + +### OpenBao Won't Start in Swarm + +**Symptom:** `Error initializing listener: bind: address already in use` + +**Cause:** OpenBao cannot run in swarm mode + +**Fix:** Deploy as standalone container: + +```bash +docker compose -f docker-compose.openbao.yml up -d +``` + +### API Can't Connect to OpenBao + +**Symptom:** API logs show connection errors to OpenBao + +**Check:** + +1. OpenBao is running: `docker ps | grep openbao` +2. Network connectivity: `docker exec mosaic_api curl http://openbao:8200/v1/sys/health` +3. Environment variable: `OPENBAO_ADDR=http://openbao:8200` + +**Swarm specific:** Ensure standalone OpenBao is connected to `mosaic_internal` network + +### Initialization Failed + +**Symptom:** `openbao-init` container exits with errors + +**Check:** + +1. OpenBao is healthy: `docker logs mosaic-openbao` +2. Init logs: `docker logs mosaic-openbao-init` +3. Volume permissions: `docker volume inspect mosaic-openbao-init` + +**Fix:** Remove volumes and redeploy: + +```bash +docker compose -f docker-compose.openbao.yml down -v +docker compose -f docker-compose.openbao.yml up -d +``` + +### Fallback Mode Active (Unintended) + +**Symptom:** API logs show "Using fallback encryption" + +**Cause:** OpenBao not reachable + +**Fix:** + +1. Verify OpenBao deployment +2. Check `OPENBAO_ADDR` in `.env` +3. Test connectivity from API container + +## Security Considerations + +1. **Never commit unseal keys or root tokens** to version control +2. **Store credentials securely** - Use a password manager or secrets management system +3. **Rotate AppRole secrets regularly** - At least every 90 days +4. **Use TLS for external Vault** - Never use HTTP in production +5. **Restrict port exposure** - OpenBao should only be accessible from API/services +6. **Monitor access logs** - Review Vault audit logs regularly +7. **Backup regularly** - Backup OpenBao volumes and external Vault state + +## See Also + +- [OpenBao Documentation](OPENBAO.md) - Detailed OpenBao configuration and usage +- [Swarm Deployment Guide](SWARM-DEPLOYMENT.md) - Complete swarm deployment instructions +- [Configuration Guide](CONFIGURATION.md) - All environment variables +- [Credential Security Design](design/credential-security.md) - Architecture and security model diff --git a/docs/SWARM-DEPLOYMENT.md b/docs/SWARM-DEPLOYMENT.md index d028840..86ba0dc 100644 --- a/docs/SWARM-DEPLOYMENT.md +++ b/docs/SWARM-DEPLOYMENT.md @@ -98,13 +98,40 @@ docker login git.mosaicstack.dev ## Deployment -### Deploy the Stack +### Step 1: Deploy OpenBao (Standalone) + +**CRITICAL:** OpenBao CANNOT run in swarm mode due to port binding conflicts. Deploy it as a standalone container first. + +```bash +cd /opt/mosaic/stack + +# Deploy OpenBao standalone +docker compose -f docker-compose.openbao.yml up -d + +# Verify OpenBao is running +docker ps | grep openbao + +# Check initialization (should complete in ~30 seconds) +docker logs mosaic-openbao-init --follow + +# Wait for OpenBao to be ready +sleep 30 +``` + +**Alternative Options:** + +- **External Vault:** Skip this step and configure `OPENBAO_ADDR` to point to your external Vault instance +- **Fallback Mode:** Skip this step - API will use AES-256-GCM encryption with `ENCRYPTION_KEY` + +See [OpenBao Deployment Guide](OPENBAO-DEPLOYMENT.md) for detailed options. + +### Step 2: Deploy the Swarm Stack ```bash cd /opt/mosaic/stack # Using the deploy script (recommended) -./scripts/deploy-swarm.sh mosaic +IMAGE_TAG=dev ./scripts/deploy-swarm.sh mosaic # Or manually IMAGE_TAG=dev docker stack deploy \ @@ -122,10 +149,11 @@ docker stack services mosaic # ID NAME MODE REPLICAS IMAGE # abc123 mosaic_postgres replicated 1/1 git.mosaicstack.dev/mosaic/stack-postgres:dev # def456 mosaic_valkey replicated 1/1 valkey/valkey:8-alpine -# ghi789 mosaic_openbao replicated 1/1 git.mosaicstack.dev/mosaic/stack-openbao:dev # jkl012 mosaic_api replicated 1/1 git.mosaicstack.dev/mosaic/stack-api:dev # mno345 mosaic_web replicated 1/1 git.mosaicstack.dev/mosaic/stack-web:dev # pqr678 mosaic_orchestrator replicated 1/1 git.mosaicstack.dev/mosaic/stack-orchestrator:dev +# +# Note: OpenBao runs as standalone container, not in swarm stack # Check detailed task status docker stack ps mosaic --no-trunc @@ -137,71 +165,28 @@ docker service logs mosaic_web --follow ## Post-Deployment Configuration -### 1. Initialize OpenBao (CRITICAL) +### 1. Verify OpenBao Initialization -**Important:** Unlike docker-compose, the swarm file does NOT include the `openbao-init` sidecar. You must manually initialize OpenBao: +OpenBao was deployed as a standalone container with automatic initialization. Verify it completed: ```bash -# Wait for OpenBao to be healthy -sleep 30 +# Check OpenBao container is running +docker ps | grep openbao -# Get the OpenBao container ID -OPENBAO_CONTAINER=$(docker ps -q -f label=com.docker.swarm.service.name=mosaic_openbao) +# Verify initialization completed +docker logs mosaic-openbao-init -# Initialize OpenBao -docker exec $OPENBAO_CONTAINER bao operator init -key-shares=1 -key-threshold=1 +# Expected output: +# ✓ OpenBao initialized successfully +# ✓ Transit engine enabled +# ✓ Encryption key created: mosaic-credentials +# ✓ AppRole authentication configured -# Save the output - you'll need the Unseal Key and Root Token! -# Example output: -# Unseal Key 1: abc123... -# Initial Root Token: xyz789... - -# Unseal OpenBao -docker exec $OPENBAO_CONTAINER bao operator unseal - -# Enable Transit engine -docker exec -e BAO_TOKEN= $OPENBAO_CONTAINER \ - bao secrets enable transit - -# Create encryption key -docker exec -e BAO_TOKEN= $OPENBAO_CONTAINER \ - bao write -f transit/keys/mosaic-credentials - -# Create AppRole for API -docker exec -e BAO_TOKEN= $OPENBAO_CONTAINER \ - bao auth enable approle - -docker exec -e BAO_TOKEN= $OPENBAO_CONTAINER \ - bao write auth/approle/role/mosaic-api \ - token_policies="default" \ - token_ttl=1h \ - token_max_ttl=4h - -# Get Role ID and Secret ID -docker exec -e BAO_TOKEN= $OPENBAO_CONTAINER \ - bao read auth/approle/role/mosaic-api/role-id - -docker exec -e BAO_TOKEN= $OPENBAO_CONTAINER \ - bao write -f auth/approle/role/mosaic-api/secret-id - -# Update API service with AppRole credentials (optional - or store in volume) -docker service update mosaic_api \ - --env-add OPENBAO_ROLE_ID= \ - --env-add OPENBAO_SECRET_ID= +# Test OpenBao connectivity from host +curl http://localhost:8200/v1/sys/health ``` -**Automated Alternative:** Create an init script and run it as a Docker container: - -```bash -# See docker/openbao/init.sh for reference -# Create a one-time task that runs initialization -docker service create --name openbao-init \ - --network mosaic_internal \ - --restart-condition=none \ - --mount type=volume,source=mosaic_openbao_init,target=/openbao/init \ - git.mosaicstack.dev/mosaic/stack-openbao:dev \ - /openbao/init.sh -``` +**If using external Vault:** Skip this step - verify your external Vault is configured per [OpenBao Deployment Guide](OPENBAO-DEPLOYMENT.md#option-3-external-hashicorp-vault). ### 2. Run Database Migrations