From 11136e2f233c42d952485194d2b24d3044b602aa Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Sun, 1 Mar 2026 07:54:28 -0600 Subject: [PATCH] =?UTF-8?q?fix(docker):=20use=20envsubst=20template=20patt?= =?UTF-8?q?ern=20=E2=80=94=20no=20hardcoded=20URLs=20or=20keys=20(MS22-P1a?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker/migrations/002_agent_fleet.sql | 154 ++++++++++++++++++ docker/openclaw-compose.yml | 16 +- docker/openclaw-instances/entrypoint.sh | 23 +++ docker/openclaw-instances/jarvis-main.env | 1 + ...is-main.json => jarvis-main.json.template} | 2 +- .../openclaw-instances/jarvis-operations.env | 1 + ...s.json => jarvis-operations.json.template} | 2 +- docker/openclaw-instances/jarvis-projects.env | 1 + ...cts.json => jarvis-projects.json.template} | 2 +- docker/openclaw-instances/jarvis-research.env | 1 + ...rch.json => jarvis-research.json.template} | 2 +- 11 files changed, 193 insertions(+), 12 deletions(-) create mode 100644 docker/migrations/002_agent_fleet.sql create mode 100755 docker/openclaw-instances/entrypoint.sh rename docker/openclaw-instances/{jarvis-main.json => jarvis-main.json.template} (95%) rename docker/openclaw-instances/{jarvis-operations.json => jarvis-operations.json.template} (94%) rename docker/openclaw-instances/{jarvis-projects.json => jarvis-projects.json.template} (94%) rename docker/openclaw-instances/{jarvis-research.json => jarvis-research.json.template} (94%) diff --git a/docker/migrations/002_agent_fleet.sql b/docker/migrations/002_agent_fleet.sql new file mode 100644 index 0000000..f27ce5b --- /dev/null +++ b/docker/migrations/002_agent_fleet.sql @@ -0,0 +1,154 @@ +-- Migration: Agent Fleet Infrastructure +-- Adds tables for multi-instance OpenClaw orchestration +-- Run after base schema exists + +-- ============================================ +-- INSTANCE REGISTRY +-- ============================================ +CREATE TABLE IF NOT EXISTS instances ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name VARCHAR(255) NOT NULL UNIQUE, + role VARCHAR(50) NOT NULL CHECK (role IN ('gateway', 'department', 'task_worker', 'user_session')), + parent_id UUID REFERENCES instances(id) ON DELETE SET NULL, + status VARCHAR(20) NOT NULL DEFAULT 'starting' CHECK (status IN ('starting', 'running', 'paused', 'stopped', 'error')), + config JSONB NOT NULL DEFAULT '{}', + capabilities JSONB NOT NULL DEFAULT '[]', + handles JSONB NOT NULL DEFAULT '[]', -- Projects/channels this instance handles + max_children INT DEFAULT 5, + max_workers INT DEFAULT 10, + idle_timeout_minutes INT DEFAULT 30, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_heartbeat_at TIMESTAMPTZ, + metadata JSONB DEFAULT '{}' +); + +CREATE INDEX idx_instances_role ON instances(role); +CREATE INDEX idx_instances_parent ON instances(parent_id); +CREATE INDEX idx_instances_status ON instances(status); + +-- ============================================ +-- SESSIONS (User/Project Sessions) +-- ============================================ +CREATE TABLE IF NOT EXISTS sessions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + instance_id UUID NOT NULL REFERENCES instances(id) ON DELETE CASCADE, + session_key VARCHAR(255) NOT NULL UNIQUE, + user_id VARCHAR(255), -- Discord user ID, etc. + channel_id VARCHAR(255), -- Discord channel ID + project_context VARCHAR(255), -- Which project this session is for + status VARCHAR(20) NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'idle', 'paused', 'closed')), + context_window JSONB DEFAULT '[]', -- Conversation history + metadata JSONB DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + closed_at TIMESTAMPTZ +); + +CREATE INDEX idx_sessions_instance ON sessions(instance_id); +CREATE INDEX idx_sessions_user ON sessions(user_id); +CREATE INDEX idx_sessions_channel ON sessions(channel_id); +CREATE INDEX idx_sessions_status ON sessions(status); + +-- ============================================ +-- SESSION SUMMARIES (For RAG Context) +-- ============================================ +CREATE TABLE IF NOT EXISTS session_summaries ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + session_id UUID REFERENCES sessions(id) ON DELETE CASCADE, + instance_id UUID NOT NULL REFERENCES instances(id) ON DELETE CASCADE, + summary_type VARCHAR(50) NOT NULL CHECK (summary_type IN ('decision', 'key_info', 'action_item', 'lesson', 'milestone')), + content TEXT NOT NULL, + embedding VECTOR(1024), -- bge-m3 dimension + metadata JSONB DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_summaries_instance ON session_summaries(instance_id); +CREATE INDEX idx_summaries_type ON session_summaries(summary_type); +CREATE INDEX idx_summaries_session ON session_summaries(session_id); + +-- pgvector index for semantic search +CREATE INDEX IF NOT EXISTS idx_summaries_embedding ON session_summaries +USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100); + +-- ============================================ +-- EVENT LOG (Audit Trail) +-- ============================================ +CREATE TABLE IF NOT EXISTS event_log ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + instance_id UUID REFERENCES instances(id) ON DELETE SET NULL, + session_id UUID REFERENCES sessions(id) ON DELETE SET NULL, + event_type VARCHAR(100) NOT NULL, + event_data JSONB DEFAULT '{}', + severity VARCHAR(20) NOT NULL DEFAULT 'info' CHECK (severity IN ('debug', 'info', 'warn', 'error')), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_event_log_instance ON event_log(instance_id); +CREATE INDEX idx_event_log_session ON event_log(session_id); +CREATE INDEX idx_event_log_type ON event_log(event_type); +CREATE INDEX idx_event_log_severity ON event_log(severity); +CREATE INDEX idx_event_log_created ON event_log(created_at); + +-- ============================================ +-- CHANNEL MAPPINGS (Routing) +-- ============================================ +CREATE TABLE IF NOT EXISTS channel_mappings ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + channel_id VARCHAR(255) NOT NULL UNIQUE, + channel_type VARCHAR(50) NOT NULL, -- discord, telegram, etc. + instance_id UUID NOT NULL REFERENCES instances(id) ON DELETE CASCADE, + project_context VARCHAR(255), + skills JSONB DEFAULT '[]', -- Allowed skills for this channel + metadata JSONB DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_channel_mappings_instance ON channel_mappings(instance_id); +CREATE INDEX idx_channel_mappings_channel ON channel_mappings(channel_id); + +-- ============================================ +-- TASK QUEUE (For Worker Spawning) +-- ============================================ +CREATE TABLE IF NOT EXISTS task_queue ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + task_type VARCHAR(50) NOT NULL, + parent_instance_id UUID REFERENCES instances(id) ON DELETE SET NULL, + status VARCHAR(20) NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'claimed', 'running', 'completed', 'failed', 'cancelled')), + task_data JSONB NOT NULL DEFAULT '{}', + result_data JSONB, + assigned_worker_id UUID REFERENCES instances(id) ON DELETE SET NULL, + priority INT DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ +); + +CREATE INDEX idx_task_queue_status ON task_queue(status); +CREATE INDEX idx_task_queue_parent ON task_queue(parent_instance_id); +CREATE INDEX idx_task_queue_priority ON task_queue(priority DESC, created_at); + +-- ============================================ +-- DEFAULT INSTANCES (Seed Data) +-- ============================================ +INSERT INTO instances (name, role, status, config, capabilities, handles, max_children, max_workers) +VALUES + ('jarvis-main', 'gateway', 'running', + '{"skills": ["orchestration", "mosaic-bootstrap", "all-projects-access"]}', + '["spawn", "kill", "route", "delegate"]', + '[]', 5, 10), + ('jarvis-projects', 'department', 'running', + '{"skills": ["coding-agent", "next-best-practices", "nestjs-best-practices", "fastapi"]}', + '["spawn-workers", "code-review", "ci-cd"]', + '["mosaic-stack", "sagephr", "lark-path"]', 3, 5), + ('jarvis-research', 'department', 'running', + '{"skills": ["web-search", "web-fetch", "brainstorming"]}', + '["research", "discovery", "analysis"]', + '[]', 2, 3), + ('jarvis-operations', 'department', 'running', + '{"skills": ["monitoring", "healthcheck", "maintenance"]}', + '["monitoring", "maintenance", "alerts"]', + '[]', 2, 3) +ON CONFLICT (name) DO NOTHING; diff --git a/docker/openclaw-compose.yml b/docker/openclaw-compose.yml index 7fb2686..4bd306b 100644 --- a/docker/openclaw-compose.yml +++ b/docker/openclaw-compose.yml @@ -1,13 +1,13 @@ services: jarvis-main: image: alpine/openclaw:latest - command: ["gateway", "run", "--bind", "lan", "--auth", "token"] + command: ["/config/entrypoint.sh"] env_file: - ./openclaw-instances/jarvis-main.env environment: OPENCLAW_CONFIG_PATH: /config/openclaw.json volumes: - - jarvis-main-config:/config/openclaw.json:ro + - jarvis-main-config:/config:ro - jarvis-main-state:/home/node/.openclaw networks: - mosaic-stack_internal @@ -34,13 +34,13 @@ services: jarvis-projects: image: alpine/openclaw:latest - command: ["gateway", "run", "--bind", "lan", "--auth", "token"] + command: ["/config/entrypoint.sh"] env_file: - ./openclaw-instances/jarvis-projects.env environment: OPENCLAW_CONFIG_PATH: /config/openclaw.json volumes: - - jarvis-projects-config:/config/openclaw.json:ro + - jarvis-projects-config:/config:ro - jarvis-projects-state:/home/node/.openclaw networks: - mosaic-stack_internal @@ -67,13 +67,13 @@ services: jarvis-research: image: alpine/openclaw:latest - command: ["gateway", "run", "--bind", "lan", "--auth", "token"] + command: ["/config/entrypoint.sh"] env_file: - ./openclaw-instances/jarvis-research.env environment: OPENCLAW_CONFIG_PATH: /config/openclaw.json volumes: - - jarvis-research-config:/config/openclaw.json:ro + - jarvis-research-config:/config:ro - jarvis-research-state:/home/node/.openclaw networks: - mosaic-stack_internal @@ -100,13 +100,13 @@ services: jarvis-operations: image: alpine/openclaw:latest - command: ["gateway", "run", "--bind", "lan", "--auth", "token"] + command: ["/config/entrypoint.sh"] env_file: - ./openclaw-instances/jarvis-operations.env environment: OPENCLAW_CONFIG_PATH: /config/openclaw.json volumes: - - jarvis-operations-config:/config/openclaw.json:ro + - jarvis-operations-config:/config:ro - jarvis-operations-state:/home/node/.openclaw networks: - mosaic-stack_internal diff --git a/docker/openclaw-instances/entrypoint.sh b/docker/openclaw-instances/entrypoint.sh new file mode 100755 index 0000000..3b2fde6 --- /dev/null +++ b/docker/openclaw-instances/entrypoint.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# OpenClaw container entrypoint — renders config template via envsubst then starts gateway +set -e + +TEMPLATE="/config/openclaw.json.template" +CONFIG="/tmp/openclaw.json" + +if [ ! -f "$TEMPLATE" ]; then + echo "ERROR: Config template not found at $TEMPLATE" + exit 1 +fi + +# Validate required env vars +: "${ZAI_API_KEY:?ZAI_API_KEY is required}" +: "${OPENCLAW_GATEWAY_TOKEN:?OPENCLAW_GATEWAY_TOKEN is required}" +: "${OLLAMA_BASE_URL:?OLLAMA_BASE_URL is required (e.g. http://10.1.1.42:11434)}" + +# Render template -> final config (no hardcoded values in image or volumes) +envsubst < "$TEMPLATE" > "$CONFIG" + +export OPENCLAW_CONFIG_PATH="$CONFIG" + +exec openclaw gateway run --bind lan --auth token "$@" diff --git a/docker/openclaw-instances/jarvis-main.env b/docker/openclaw-instances/jarvis-main.env index 719d4bc..893d930 100644 --- a/docker/openclaw-instances/jarvis-main.env +++ b/docker/openclaw-instances/jarvis-main.env @@ -1,3 +1,4 @@ OPENCLAW_CONFIG_PATH=/config/openclaw.json ZAI_API_KEY=REPLACE_WITH_ZAI_API_KEY OPENCLAW_GATEWAY_TOKEN=REPLACE_WITH_UNIQUE_GATEWAY_TOKEN +OLLAMA_BASE_URL=REPLACE_WITH_OLLAMA_BASE_URL diff --git a/docker/openclaw-instances/jarvis-main.json b/docker/openclaw-instances/jarvis-main.json.template similarity index 95% rename from docker/openclaw-instances/jarvis-main.json rename to docker/openclaw-instances/jarvis-main.json.template index 5feeae9..4752124 100644 --- a/docker/openclaw-instances/jarvis-main.json +++ b/docker/openclaw-instances/jarvis-main.json.template @@ -22,7 +22,7 @@ "mode": "merge", "providers": { "ollama": { - "baseUrl": "http://10.1.1.42:11434/v1", + "baseUrl": "${OLLAMA_BASE_URL}/v1", "api": "openai-completions", "models": [ { diff --git a/docker/openclaw-instances/jarvis-operations.env b/docker/openclaw-instances/jarvis-operations.env index 719d4bc..893d930 100644 --- a/docker/openclaw-instances/jarvis-operations.env +++ b/docker/openclaw-instances/jarvis-operations.env @@ -1,3 +1,4 @@ OPENCLAW_CONFIG_PATH=/config/openclaw.json ZAI_API_KEY=REPLACE_WITH_ZAI_API_KEY OPENCLAW_GATEWAY_TOKEN=REPLACE_WITH_UNIQUE_GATEWAY_TOKEN +OLLAMA_BASE_URL=REPLACE_WITH_OLLAMA_BASE_URL diff --git a/docker/openclaw-instances/jarvis-operations.json b/docker/openclaw-instances/jarvis-operations.json.template similarity index 94% rename from docker/openclaw-instances/jarvis-operations.json rename to docker/openclaw-instances/jarvis-operations.json.template index 87ce1af..afef855 100644 --- a/docker/openclaw-instances/jarvis-operations.json +++ b/docker/openclaw-instances/jarvis-operations.json.template @@ -21,7 +21,7 @@ "mode": "merge", "providers": { "ollama": { - "baseUrl": "http://10.1.1.42:11434/v1", + "baseUrl": "${OLLAMA_BASE_URL}/v1", "api": "openai-completions", "models": [ { diff --git a/docker/openclaw-instances/jarvis-projects.env b/docker/openclaw-instances/jarvis-projects.env index 719d4bc..893d930 100644 --- a/docker/openclaw-instances/jarvis-projects.env +++ b/docker/openclaw-instances/jarvis-projects.env @@ -1,3 +1,4 @@ OPENCLAW_CONFIG_PATH=/config/openclaw.json ZAI_API_KEY=REPLACE_WITH_ZAI_API_KEY OPENCLAW_GATEWAY_TOKEN=REPLACE_WITH_UNIQUE_GATEWAY_TOKEN +OLLAMA_BASE_URL=REPLACE_WITH_OLLAMA_BASE_URL diff --git a/docker/openclaw-instances/jarvis-projects.json b/docker/openclaw-instances/jarvis-projects.json.template similarity index 94% rename from docker/openclaw-instances/jarvis-projects.json rename to docker/openclaw-instances/jarvis-projects.json.template index 71c8c00..e12decf 100644 --- a/docker/openclaw-instances/jarvis-projects.json +++ b/docker/openclaw-instances/jarvis-projects.json.template @@ -20,7 +20,7 @@ "mode": "merge", "providers": { "ollama": { - "baseUrl": "http://10.1.1.42:11434/v1", + "baseUrl": "${OLLAMA_BASE_URL}/v1", "api": "openai-completions", "models": [ { diff --git a/docker/openclaw-instances/jarvis-research.env b/docker/openclaw-instances/jarvis-research.env index 719d4bc..893d930 100644 --- a/docker/openclaw-instances/jarvis-research.env +++ b/docker/openclaw-instances/jarvis-research.env @@ -1,3 +1,4 @@ OPENCLAW_CONFIG_PATH=/config/openclaw.json ZAI_API_KEY=REPLACE_WITH_ZAI_API_KEY OPENCLAW_GATEWAY_TOKEN=REPLACE_WITH_UNIQUE_GATEWAY_TOKEN +OLLAMA_BASE_URL=REPLACE_WITH_OLLAMA_BASE_URL diff --git a/docker/openclaw-instances/jarvis-research.json b/docker/openclaw-instances/jarvis-research.json.template similarity index 94% rename from docker/openclaw-instances/jarvis-research.json rename to docker/openclaw-instances/jarvis-research.json.template index 71c8c00..e12decf 100644 --- a/docker/openclaw-instances/jarvis-research.json +++ b/docker/openclaw-instances/jarvis-research.json.template @@ -20,7 +20,7 @@ "mode": "merge", "providers": { "ollama": { - "baseUrl": "http://10.1.1.42:11434/v1", + "baseUrl": "${OLLAMA_BASE_URL}/v1", "api": "openai-completions", "models": [ {