stack/SWARM-QUICKREF.md

Docker Swarm Quick Reference

Initial Setup

# 1. Configure environment
cp .env.swarm.example .env
nano .env  # Set passwords, API keys, domains

# 2. Create Traefik network (if needed)
docker network create --driver=overlay traefik-public

# 3. Deploy stack
./deploy-swarm.sh mosaic

Common Commands

Stack Management

# Deploy/update stack
docker stack deploy -c docker-compose.swarm.yml mosaic

# List all stacks
docker stack ls

# Remove stack
docker stack rm mosaic

# List services in stack
docker stack services mosaic

# List tasks in stack
docker stack ps mosaic

Service Management

# List all services
docker service ls

# Inspect service
docker service inspect mosaic_api

# View service logs
docker service logs mosaic_api --tail 100 --follow

# Scale service
docker service scale mosaic_web=3

# Update service (force redeploy)
docker service update --force mosaic_api

# Update service image
docker service update --image mosaic-stack-api:latest mosaic_api

# Rollback service
docker service rollback mosaic_api

Monitoring

# Watch service status
watch -n 2 'docker service ls'

# Service resource usage
docker stats $(docker ps --filter label=com.docker.swarm.service.name=mosaic_api -q)

# Check service placement
docker service ps mosaic_api --format "table {{.Name}}\t{{.Node}}\t{{.CurrentState}}"

Debugging

# Check why service failed
docker service ps mosaic_api --no-trunc

# View recent logs with timestamps
docker service logs mosaic_api --timestamps --tail 50

# Follow logs in real-time
docker service logs mosaic_api --follow

# Exec into running container
docker exec -it $(docker ps -q -f label=com.docker.swarm.service.name=mosaic_api) sh

Network Management

# List networks
docker network ls

# Inspect traefik-public network
docker network inspect traefik-public

# List containers on traefik-public
docker network inspect traefik-public --format '{{range .Containers}}{{.Name}} {{end}}'

Volume Management

# List volumes
docker volume ls --filter label=com.docker.stack.namespace=mosaic

# Inspect volume
docker volume inspect mosaic_postgres_data

# Backup volume
docker run --rm -v mosaic_postgres_data:/data -v $(pwd):/backup alpine \
  tar czf /backup/postgres-$(date +%Y%m%d-%H%M%S).tar.gz -C /data .

# Restore volume
docker run --rm -v mosaic_postgres_data:/data -v $(pwd):/backup alpine \
  tar xzf /backup/postgres-20260208-143022.tar.gz -C /data

Service-Specific Commands

Database (PostgreSQL)

# Connect to database
docker exec -it $(docker ps -q -f label=com.docker.swarm.service.name=mosaic_postgres) \
  psql -U mosaic -d mosaic

# Run migrations (from API container)
docker exec $(docker ps -q -f label=com.docker.swarm.service.name=mosaic_api) \
  pnpm prisma migrate deploy

# View database logs
docker service logs mosaic_postgres --tail 100

API Service

# View API logs
docker service logs mosaic_api --follow

# Check API health
curl http://api.mosaicstack.dev/health

# Force API redeploy
docker service update --force mosaic_api

Web Service

# View web logs
docker service logs mosaic_web --follow

# Scale web to 3 replicas
docker service scale mosaic_web=3

# Check web health
curl http://mosaic.mosaicstack.dev

Authentik

# View Authentik logs
docker service logs mosaic_authentik-server --follow
docker service logs mosaic_authentik-worker --follow

# Access Authentik UI
open http://auth.mosaicstack.dev

Troubleshooting

Service Won't Start

# 1. Check service tasks
docker service ps mosaic_api --no-trunc

# 2. View service logs
docker service logs mosaic_api --tail 100

# 3. Check if image exists
docker images | grep mosaic-stack-api

# 4. Rebuild and update
docker compose -f docker-compose.swarm.yml build api
docker service update --image mosaic-stack-api:latest mosaic_api

Traefik Not Routing

# 1. Verify service is on traefik-public network
docker service inspect mosaic_web | grep -A 10 Networks

# 2. Check Traefik labels
docker service inspect mosaic_web --format '{{json .Spec.Labels}}' | jq

# 3. Verify DNS resolution
ping mosaic.mosaicstack.dev

# 4. Check Traefik logs (if Traefik is a service)
docker service logs traefik --tail 50

Database Connection Failed

# 1. Check postgres is running
docker service ls | grep postgres

# 2. Check postgres health
docker service ps mosaic_postgres

# 3. View postgres logs
docker service logs mosaic_postgres --tail 50

# 4. Test connection from API container
docker exec $(docker ps -q -f label=com.docker.swarm.service.name=mosaic_api) \
  sh -c 'nc -zv postgres 5432'

Out of Memory / Resources

# Check node resources
docker node ls
docker node inspect self --format '{{json .Description.Resources}}' | jq

# Check service resource limits
docker service inspect mosaic_api --format '{{json .Spec.TaskTemplate.Resources}}' | jq

# Update resource limits
docker service update --limit-memory 1g --reserve-memory 512m mosaic_api

Useful Aliases

Add to ~/.bashrc or ~/.zshrc:

# Stack shortcuts
alias dss='docker stack services'
alias dsp='docker stack ps'
alias dsl='docker service logs'
alias dsi='docker service inspect'
alias dsu='docker service update'

# Mosaic-specific
alias mosaic-logs='docker service logs mosaic_api --follow'
alias mosaic-status='docker stack services mosaic'
alias mosaic-ps='docker stack ps mosaic'
alias mosaic-deploy='./deploy-swarm.sh mosaic'

Emergency Procedures

Complete Stack Restart

# 1. Remove stack (keeps volumes)
docker stack rm mosaic

# 2. Wait for cleanup (30 seconds)
sleep 30

# 3. Redeploy
./deploy-swarm.sh mosaic

Database Recovery

# 1. Stop API to prevent writes
docker service scale mosaic_api=0

# 2. Backup current database
docker run --rm -v mosaic_postgres_data:/data -v $(pwd):/backup alpine \
  tar czf /backup/postgres-emergency-$(date +%Y%m%d-%H%M%S).tar.gz -C /data .

# 3. Stop postgres
docker service scale mosaic_postgres=0

# 4. Restore from backup
docker run --rm -v mosaic_postgres_data:/data -v $(pwd):/backup alpine \
  sh -c 'rm -rf /data/* && tar xzf /backup/postgres-20260208.tar.gz -C /data'

# 5. Restart postgres
docker service scale mosaic_postgres=1

# 6. Wait for postgres healthy
sleep 10

# 7. Restart API
docker service scale mosaic_api=1

Health Checks

# API
curl http://api.mosaicstack.dev/health

# Web
curl http://mosaic.mosaicstack.dev

# Authentik
curl http://auth.mosaicstack.dev/-/health/live/

# Postgres (from API container)
docker exec $(docker ps -q -f label=com.docker.swarm.service.name=mosaic_api) \
  sh -c 'nc -zv postgres 5432'

# Valkey (from API container)
docker exec $(docker ps -q -f label=com.docker.swarm.service.name=mosaic_api) \
  sh -c 'nc -zv valkey 6379'