stack/tests/integration/openbao.test.ts

import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { exec } from "child_process";
import { promisify } from "util";

const execAsync = promisify(exec);

/**
 * OpenBao Integration Tests
 * Tests the OpenBao deployment with auto-init and Transit engine
 */
describe("OpenBao Integration Tests", () => {
  const TIMEOUT = 120000; // 2 minutes for Docker operations
  const HEALTH_CHECK_RETRIES = 30;
  const HEALTH_CHECK_INTERVAL = 2000;

  // Top-level setup: Start services once for all tests
  beforeAll(async () => {
    // Ensure clean state
    await execAsync("docker compose down -v", {
      cwd: `${process.cwd()}/docker`,
    }).catch(() => {});

    // Start OpenBao and init container
    await execAsync("docker compose up -d openbao openbao-init", {
      cwd: `${process.cwd()}/docker`,
    });

    // Wait for OpenBao to be healthy
    const openbaoHealthy = await waitForService("openbao");
    if (!openbaoHealthy) {
      throw new Error("OpenBao failed to become healthy");
    }

    // Wait for initialization to complete (init container running)
    const initRunning = await waitForService("openbao-init");
    if (!initRunning) {
      throw new Error("OpenBao init container failed to start");
    }

    // Wait for initialization to complete (give it time to configure)
    await new Promise((resolve) => setTimeout(resolve, 30000));
  }, 180000); // 3 minutes for initial setup

  // Top-level teardown: Clean up after all tests
  afterAll(async () => {
    await execAsync("docker compose down -v", {
      cwd: `${process.cwd()}/docker`,
    }).catch(() => {});
  }, TIMEOUT);

  /**
   * Helper function to execute Docker Compose commands
   */
  async function dockerCompose(command: string): Promise<string> {
    const { stdout } = await execAsync(`docker compose ${command}`, {
      cwd: `${process.cwd()}/docker`,
    });
    return stdout;
  }

  /**
   * Helper function to check if a service is healthy
   */
  async function waitForService(
    serviceName: string,
    retries = HEALTH_CHECK_RETRIES
  ): Promise<boolean> {
    for (let i = 0; i < retries; i++) {
      try {
        const { stdout } = await execAsync(`docker compose ps --format json ${serviceName}`, {
          cwd: `${process.cwd()}/docker`,
        });
        const serviceInfo = JSON.parse(stdout);

        if (serviceInfo.Health === "healthy" || serviceInfo.State === "running") {
          return true;
        }
      } catch (error) {
        // Service not ready yet
      }

      await new Promise((resolve) => setTimeout(resolve, HEALTH_CHECK_INTERVAL));
    }
    return false;
  }

  /**
   * Helper function to check HTTP endpoint
   * Returns true for any non-5xx status (including sealed/uninitialized states)
   */
  async function checkHttpEndpoint(url: string): Promise<boolean> {
    try {
      const response = await fetch(url);
      // OpenBao returns 501 when uninitialized, 503 when sealed
      // Both are valid "healthy" states for these tests
      return response.status < 500;
    } catch (error) {
      return false;
    }
  }

  /**
   * Helper to execute commands inside OpenBao container
   */
  async function execInBao(command: string): Promise<string> {
    const { stdout } = await execAsync(`docker compose exec -T openbao ${command}`, {
      cwd: `${process.cwd()}/docker`,
    });
    return stdout;
  }

  /**
   * Helper to read secret files from OpenBao init volume
   * Uses docker run to mount volume and read file safely
   * Sanitizes error messages to prevent secret leakage
   */
  async function readSecretFile(fileName: string): Promise<string> {
    try {
      const { stdout } = await execAsync(
        `docker run --rm -v mosaic-openbao-init:/data alpine cat /data/${fileName}`
      );
      return stdout.trim();
    } catch (error) {
      // Sanitize error message to prevent secret leakage
      const sanitizedError = new Error(
        `Failed to read secret file: ${fileName} (file may not exist or volume not mounted)`
      );
      throw sanitizedError;
    }
  }

  /**
   * Helper to read and parse JSON secret file
   */
  async function readSecretJSON(fileName: string): Promise<any> {
    try {
      const content = await readSecretFile(fileName);
      return JSON.parse(content);
    } catch (error) {
      // Sanitize error to prevent leaking partial secret data
      const sanitizedError = new Error(`Failed to parse secret JSON from: ${fileName}`);
      throw sanitizedError;
    }
  }

  describe("OpenBao Service Startup", () => {
    it(
      "should start OpenBao server with health check",
      async () => {
        // Start OpenBao service
        await dockerCompose("up -d openbao");

        // Wait for service to be healthy
        const isHealthy = await waitForService("openbao");
        expect(isHealthy).toBe(true);

        // Verify container is running
        const ps = await dockerCompose("ps openbao");
        expect(ps).toContain("mosaic-openbao");
        expect(ps).toContain("Up");
      },
      TIMEOUT
    );

    it(
      "should have OpenBao API accessible on port 8200",
      async () => {
        // Ensure OpenBao is running
        await dockerCompose("up -d openbao");
        await waitForService("openbao");

        // Check health endpoint with flags to accept sealed/uninitialized states
        const isHealthy = await checkHttpEndpoint(
          "http://localhost:8200/v1/sys/health?standbyok=true&sealedok=true&uninitok=true"
        );
        expect(isHealthy).toBe(true);
      },
      TIMEOUT
    );

    it(
      "should have proper volume configuration",
      async () => {
        // Start OpenBao
        await dockerCompose("up -d openbao");
        await waitForService("openbao");

        // Check if volumes are created
        const { stdout } = await execAsync("docker volume ls");
        expect(stdout).toContain("mosaic-openbao-data");
        expect(stdout).toContain("mosaic-openbao-config");
        expect(stdout).toContain("mosaic-openbao-init");
      },
      TIMEOUT
    );
  });

  describe("OpenBao Auto-Initialization", () => {
    it(
      "should initialize OpenBao on first run",
      async () => {
        // Wait for init container to complete
        await new Promise((resolve) => setTimeout(resolve, 10000));

        // Check initialization status
        const status = await execInBao("bao status -format=json");
        const statusObj = JSON.parse(status);

        expect(statusObj.initialized).toBe(true);
        expect(statusObj.sealed).toBe(false);
      },
      TIMEOUT
    );

    it(
      "should create unseal key in init volume",
      async () => {
        // Wait for init to complete
        await new Promise((resolve) => setTimeout(resolve, 10000));

        // Check if unseal key file exists
        const { stdout } = await execAsync(
          "docker run --rm -v mosaic-openbao-init:/data alpine ls -la /data/"
        );
        expect(stdout).toContain("unseal-key");
      },
      TIMEOUT
    );

    it(
      "should be idempotent on restart",
      async () => {
        // Wait for first init
        await new Promise((resolve) => setTimeout(resolve, 10000));

        // Restart init container
        await dockerCompose("restart openbao-init");
        await new Promise((resolve) => setTimeout(resolve, 5000));

        // Should still be initialized and unsealed
        const status = await execInBao("bao status -format=json");
        const statusObj = JSON.parse(status);

        expect(statusObj.initialized).toBe(true);
        expect(statusObj.sealed).toBe(false);
      },
      TIMEOUT
    );
  });

  describe("OpenBao Transit Engine", () => {
    it(
      "should enable Transit secrets engine",
      async () => {
        // Get root token from init volume
        const { stdout: token } = await execAsync(
          "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token"
        );

        // Check if Transit is enabled
        const { stdout: mounts } = await execAsync(
          `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao secrets list -format=json`,
          { cwd: `${process.cwd()}/docker` }
        );
        const mountsObj = JSON.parse(mounts);

        expect(mountsObj["transit/"]).toBeDefined();
        expect(mountsObj["transit/"].type).toBe("transit");
      },
      TIMEOUT
    );

    it(
      "should create mosaic-credentials Transit key",
      async () => {
        const { stdout: token } = await execAsync(
          "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token"
        );

        // List Transit keys
        const { stdout: keys } = await execAsync(
          `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao list -format=json transit/keys`,
          { cwd: `${process.cwd()}/docker` }
        );
        const keysArray = JSON.parse(keys);

        expect(keysArray).toContain("mosaic-credentials");
      },
      TIMEOUT
    );

    it(
      "should create mosaic-account-tokens Transit key",
      async () => {
        const { stdout: token } = await execAsync(
          "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token"
        );

        const { stdout: keys } = await execAsync(
          `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao list -format=json transit/keys`,
          { cwd: `${process.cwd()}/docker` }
        );
        const keysArray = JSON.parse(keys);

        expect(keysArray).toContain("mosaic-account-tokens");
      },
      TIMEOUT
    );

    it(
      "should create mosaic-federation Transit key",
      async () => {
        const { stdout: token } = await execAsync(
          "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token"
        );

        const { stdout: keys } = await execAsync(
          `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao list -format=json transit/keys`,
          { cwd: `${process.cwd()}/docker` }
        );
        const keysArray = JSON.parse(keys);

        expect(keysArray).toContain("mosaic-federation");
      },
      TIMEOUT
    );

    it(
      "should create mosaic-llm-config Transit key",
      async () => {
        const { stdout: token } = await execAsync(
          "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token"
        );

        const { stdout: keys } = await execAsync(
          `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao list -format=json transit/keys`,
          { cwd: `${process.cwd()}/docker` }
        );
        const keysArray = JSON.parse(keys);

        expect(keysArray).toContain("mosaic-llm-config");
      },
      TIMEOUT
    );

    it(
      "should verify Transit keys use aes256-gcm96",
      async () => {
        const { stdout: token } = await execAsync(
          "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token"
        );

        // Check key type for mosaic-credentials
        const { stdout: keyInfo } = await execAsync(
          `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao read -format=json transit/keys/mosaic-credentials`,
          { cwd: `${process.cwd()}/docker` }
        );
        const keyInfoObj = JSON.parse(keyInfo);

        expect(keyInfoObj.data.type).toBe("aes256-gcm96");
      },
      TIMEOUT
    );
  });

  describe("OpenBao AppRole Configuration", () => {
    it(
      "should enable AppRole auth method",
      async () => {
        const { stdout: token } = await execAsync(
          "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token"
        );

        // Check if AppRole is enabled
        const { stdout: auths } = await execAsync(
          `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao auth list -format=json`,
          { cwd: `${process.cwd()}/docker` }
        );
        const authsObj = JSON.parse(auths);

        expect(authsObj["approle/"]).toBeDefined();
        expect(authsObj["approle/"].type).toBe("approle");
      },
      TIMEOUT
    );

    it(
      "should create mosaic-transit AppRole",
      async () => {
        const { stdout: token } = await execAsync(
          "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token"
        );

        // List AppRoles
        const { stdout: roles } = await execAsync(
          `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao list -format=json auth/approle/role`,
          { cwd: `${process.cwd()}/docker` }
        );
        const rolesArray = JSON.parse(roles);

        expect(rolesArray).toContain("mosaic-transit");
      },
      TIMEOUT
    );

    it(
      "should create AppRole credentials file",
      async () => {
        // Check if credentials file exists
        const { stdout } = await execAsync(
          "docker run --rm -v mosaic-openbao-init:/data alpine ls -la /data/"
        );
        expect(stdout).toContain("approle-credentials");
      },
      TIMEOUT
    );

    it(
      "should have valid AppRole credentials",
      async () => {
        // Read credentials file
        const { stdout: credentials } = await execAsync(
          "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/approle-credentials"
        );
        const credsObj = JSON.parse(credentials);

        expect(credsObj.role_id).toBeDefined();
        expect(credsObj.secret_id).toBeDefined();
        expect(typeof credsObj.role_id).toBe("string");
        expect(typeof credsObj.secret_id).toBe("string");
      },
      TIMEOUT
    );
  });

  describe("OpenBao Auto-Unseal on Restart", () => {
    it(
      "should auto-unseal after container restart",
      async () => {
        // Verify initially unsealed
        let status = await execInBao("bao status -format=json");
        let statusObj = JSON.parse(status);
        expect(statusObj.sealed).toBe(false);

        // Restart OpenBao container
        await dockerCompose("restart openbao");
        await waitForService("openbao");

        // Restart init container to trigger unseal
        await dockerCompose("restart openbao-init");
        await new Promise((resolve) => setTimeout(resolve, 10000));

        // Verify auto-unsealed
        status = await execInBao("bao status -format=json");
        statusObj = JSON.parse(status);
        expect(statusObj.sealed).toBe(false);
      },
      TIMEOUT
    );

    it(
      "should preserve Transit keys after restart",
      async () => {
        // Restart OpenBao
        await dockerCompose("restart openbao openbao-init");
        await waitForService("openbao");
        await new Promise((resolve) => setTimeout(resolve, 10000));

        const { stdout: token } = await execAsync(
          "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token"
        );

        // Verify Transit keys still exist
        const { stdout: keys } = await execAsync(
          `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao list -format=json transit/keys`,
          { cwd: `${process.cwd()}/docker` }
        );
        const keysArray = JSON.parse(keys);

        expect(keysArray).toContain("mosaic-credentials");
        expect(keysArray).toContain("mosaic-account-tokens");
        expect(keysArray).toContain("mosaic-federation");
        expect(keysArray).toContain("mosaic-llm-config");
      },
      TIMEOUT
    );
  });

  describe("OpenBao Security and Policies", () => {
    it(
      "should have Transit-only policy for AppRole",
      async () => {
        const { stdout: token } = await execAsync(
          "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/root-token"
        );

        // Read policy
        const { stdout: policy } = await execAsync(
          `docker compose exec -T -e VAULT_TOKEN=${token.trim()} openbao bao policy read mosaic-transit-policy`,
          { cwd: `${process.cwd()}/docker` }
        );

        // Verify policy allows encrypt/decrypt
        expect(policy).toContain("transit/encrypt/*");
        expect(policy).toContain("transit/decrypt/*");
      },
      TIMEOUT
    );

    it(
      "should test AppRole can encrypt data",
      async () => {
        // Get AppRole credentials
        const { stdout: credentials } = await execAsync(
          "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/approle-credentials"
        );
        const credsObj = JSON.parse(credentials);

        // Login with AppRole
        const { stdout: loginResponse } = await execAsync(
          `docker compose exec -T openbao bao write -format=json auth/approle/login role_id=${credsObj.role_id} secret_id=${credsObj.secret_id}`,
          { cwd: `${process.cwd()}/docker` }
        );
        const loginObj = JSON.parse(loginResponse);
        const appRoleToken = loginObj.auth.client_token;

        // Try to encrypt
        const testData = Buffer.from("test-data").toString("base64");
        const { stdout: encryptResponse } = await execAsync(
          `docker compose exec -T -e VAULT_TOKEN=${appRoleToken} openbao bao write -format=json transit/encrypt/mosaic-credentials plaintext=${testData}`,
          { cwd: `${process.cwd()}/docker` }
        );
        const encryptObj = JSON.parse(encryptResponse);

        expect(encryptObj.data.ciphertext).toBeDefined();
        expect(encryptObj.data.ciphertext).toMatch(/^vault:v\d+:/);
      },
      TIMEOUT
    );

    it(
      "should test AppRole can decrypt data",
      async () => {
        // Get AppRole credentials and login
        const { stdout: credentials } = await execAsync(
          "docker run --rm -v mosaic-openbao-init:/data alpine cat /data/approle-credentials"
        );
        const credsObj = JSON.parse(credentials);

        const { stdout: loginResponse } = await execAsync(
          `docker compose exec -T openbao bao write -format=json auth/approle/login role_id=${credsObj.role_id} secret_id=${credsObj.secret_id}`,
          { cwd: `${process.cwd()}/docker` }
        );
        const loginObj = JSON.parse(loginResponse);
        const appRoleToken = loginObj.auth.client_token;

        // Encrypt data first
        const testData = Buffer.from("test-decrypt").toString("base64");
        const { stdout: encryptResponse } = await execAsync(
          `docker compose exec -T -e VAULT_TOKEN=${appRoleToken} openbao bao write -format=json transit/encrypt/mosaic-credentials plaintext=${testData}`,
          { cwd: `${process.cwd()}/docker` }
        );
        const encryptObj = JSON.parse(encryptResponse);
        const ciphertext = encryptObj.data.ciphertext;

        // Decrypt
        const { stdout: decryptResponse } = await execAsync(
          `docker compose exec -T -e VAULT_TOKEN=${appRoleToken} openbao bao write -format=json transit/decrypt/mosaic-credentials ciphertext=${ciphertext}`,
          { cwd: `${process.cwd()}/docker` }
        );
        const decryptObj = JSON.parse(decryptResponse);

        const plaintext = Buffer.from(decryptObj.data.plaintext, "base64").toString();
        expect(plaintext).toBe("test-decrypt");
      },
      TIMEOUT
    );
  });

  describe("OpenBao Service Dependencies", () => {
    it(
      "should start openbao-init only after openbao is healthy",
      async () => {
        // Start both services
        await dockerCompose("up -d openbao openbao-init");

        // Wait for OpenBao to be healthy
        const openbaoHealthy = await waitForService("openbao");
        expect(openbaoHealthy).toBe(true);

        // Init should start after OpenBao is healthy
        await new Promise((resolve) => setTimeout(resolve, 5000));

        const ps = await dockerCompose("ps openbao-init");
        expect(ps).toContain("mosaic-openbao-init");
      },
      TIMEOUT
    );
  });
});