From 7feb686d73a6eb5cc9f9ac2b17a6b62afcd52110 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Sat, 7 Feb 2026 11:21:38 -0600 Subject: [PATCH] feat(#344): Add CI operations service to orchestrator - Add CIOperationsService for Woodpecker CI integration - Add types for pipeline status, failure diagnosis - Add waitForPipeline with auto-diagnosis on failure - Add getPipelineLogs for log retrieval - Integrate CIModule into orchestrator app Co-Authored-By: Claude Sonnet 4.5 --- apps/orchestrator/src/app.module.ts | 2 + .../src/ci/ci-operations.service.ts | 423 ++++++++++++++++++ apps/orchestrator/src/ci/ci.module.ts | 8 + apps/orchestrator/src/ci/index.ts | 3 + .../src/ci/types/ci-operations.types.ts | 85 ++++ apps/orchestrator/src/ci/types/index.ts | 1 + 6 files changed, 522 insertions(+) create mode 100644 apps/orchestrator/src/ci/ci-operations.service.ts create mode 100644 apps/orchestrator/src/ci/ci.module.ts create mode 100644 apps/orchestrator/src/ci/index.ts create mode 100644 apps/orchestrator/src/ci/types/ci-operations.types.ts create mode 100644 apps/orchestrator/src/ci/types/index.ts diff --git a/apps/orchestrator/src/app.module.ts b/apps/orchestrator/src/app.module.ts index 5ff056a..bb84567 100644 --- a/apps/orchestrator/src/app.module.ts +++ b/apps/orchestrator/src/app.module.ts @@ -6,6 +6,7 @@ import { HealthModule } from "./api/health/health.module"; import { AgentsModule } from "./api/agents/agents.module"; import { CoordinatorModule } from "./coordinator/coordinator.module"; import { BudgetModule } from "./budget/budget.module"; +import { CIModule } from "./ci"; import { orchestratorConfig } from "./config/orchestrator.config"; /** @@ -47,6 +48,7 @@ import { orchestratorConfig } from "./config/orchestrator.config"; AgentsModule, CoordinatorModule, BudgetModule, + CIModule, ], }) export class AppModule {} diff --git a/apps/orchestrator/src/ci/ci-operations.service.ts b/apps/orchestrator/src/ci/ci-operations.service.ts new file mode 100644 index 0000000..4c8e555 --- /dev/null +++ b/apps/orchestrator/src/ci/ci-operations.service.ts @@ -0,0 +1,423 @@ +import { Injectable, Logger } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; +import { exec } from "node:child_process"; +import { promisify } from "node:util"; +import { + CIOperationError, + FailureCategory, + FailureDiagnosis, + PipelineInfo, + PipelineStatus, + PipelineWaitOptions, + PipelineWaitResult, +} from "./types"; + +const execAsync = promisify(exec); + +/** + * Service for managing Woodpecker CI operations + */ +@Injectable() +export class CIOperationsService { + private readonly logger = new Logger(CIOperationsService.name); + private readonly woodpeckerServer: string; + private readonly woodpeckerToken: string; + private readonly defaultTimeout: number; + private readonly defaultPollInterval: number; + + constructor(private readonly configService: ConfigService) { + this.woodpeckerServer = + this.configService.get("orchestrator.ci.woodpeckerServer") ?? + process.env.WOODPECKER_SERVER ?? + ""; + this.woodpeckerToken = + this.configService.get("orchestrator.ci.woodpeckerToken") ?? + process.env.WOODPECKER_TOKEN ?? + ""; + this.defaultTimeout = this.configService.get("orchestrator.ci.timeout", 1800); + this.defaultPollInterval = this.configService.get("orchestrator.ci.pollInterval", 10); + + if (!this.woodpeckerServer || !this.woodpeckerToken) { + this.logger.warn( + "Woodpecker CI not configured. Set WOODPECKER_SERVER and WOODPECKER_TOKEN environment variables." + ); + } + } + + /** + * Check if Woodpecker CI is configured + */ + isConfigured(): boolean { + return Boolean(this.woodpeckerServer && this.woodpeckerToken); + } + + /** + * Get the latest pipeline for a repository + */ + async getLatestPipeline(repo: string): Promise { + if (!this.isConfigured()) { + throw new CIOperationError("Woodpecker CI is not configured", "getLatestPipeline"); + } + + try { + this.logger.debug(`Getting latest pipeline for ${repo}`); + + const { stdout } = await this.execWoodpecker(["pipeline", "ls", repo, "--limit", "1"]); + + const pipelines = this.parsePipelineList(stdout); + if (pipelines.length === 0) { + this.logger.debug(`No pipeline found for ${repo}`); + return null; + } + + return pipelines[0]; + } catch (error) { + this.logger.error(`Failed to get latest pipeline: ${String(error)}`); + throw new CIOperationError( + `Failed to get latest pipeline for ${repo}`, + "getLatestPipeline", + error as Error + ); + } + } + + /** + * Get specific pipeline by number + */ + async getPipeline(repo: string, pipelineNumber: number): Promise { + if (!this.isConfigured()) { + throw new CIOperationError("Woodpecker CI is not configured", "getPipeline"); + } + + try { + this.logger.debug(`Getting pipeline #${pipelineNumber.toString()} for ${repo}`); + + const { stdout } = await this.execWoodpecker([ + "pipeline", + "info", + repo, + pipelineNumber.toString(), + ]); + + return this.parsePipelineInfo(stdout, pipelineNumber); + } catch (error) { + this.logger.error(`Failed to get pipeline: ${String(error)}`); + throw new CIOperationError( + `Failed to get pipeline #${pipelineNumber.toString()} for ${repo}`, + "getPipeline", + error as Error + ); + } + } + + /** + * Wait for pipeline to complete + */ + async waitForPipeline( + repo: string, + pipelineNumber: number, + options: PipelineWaitOptions = {} + ): Promise { + if (!this.isConfigured()) { + throw new CIOperationError("Woodpecker CI is not configured", "waitForPipeline"); + } + + const timeout = options.timeout ?? this.defaultTimeout; + const pollInterval = options.pollInterval ?? this.defaultPollInterval; + const fetchLogsOnFailure = options.fetchLogsOnFailure ?? true; + + this.logger.log( + `Waiting for pipeline #${pipelineNumber.toString()} in ${repo} (timeout: ${timeout.toString()}s)` + ); + + const startTime = Date.now(); + const timeoutMs = timeout * 1000; + + while (Date.now() - startTime < timeoutMs) { + const pipeline = await this.getPipeline(repo, pipelineNumber); + + if ( + pipeline.status === PipelineStatus.SUCCESS || + pipeline.status === PipelineStatus.FAILURE || + pipeline.status === PipelineStatus.KILLED || + pipeline.status === PipelineStatus.ERROR + ) { + // Pipeline completed + const success = pipeline.status === PipelineStatus.SUCCESS; + const result: PipelineWaitResult = { + success, + pipeline, + }; + + if (!success && fetchLogsOnFailure) { + this.logger.log( + `Pipeline #${pipelineNumber.toString()} failed, fetching logs for diagnosis` + ); + result.logs = await this.getPipelineLogs(repo, pipelineNumber); + result.diagnosis = this.diagnoseFail(result.logs); + } + + if (success) { + this.logger.log(`✓ Pipeline #${pipelineNumber.toString()} succeeded`); + } else { + this.logger.warn( + `✗ Pipeline #${pipelineNumber.toString()} failed with status: ${pipeline.status}` + ); + } + + return result; + } + + // Still running or pending + await this.delay(pollInterval * 1000); + } + + // Timeout + const pipeline = await this.getPipeline(repo, pipelineNumber); + throw new CIOperationError( + `Pipeline #${pipelineNumber.toString()} did not complete within ${timeout.toString()}s (status: ${pipeline.status})`, + "waitForPipeline" + ); + } + + /** + * Get pipeline logs + */ + async getPipelineLogs(repo: string, pipelineNumber: number, step?: string): Promise { + if (!this.isConfigured()) { + throw new CIOperationError("Woodpecker CI is not configured", "getPipelineLogs"); + } + + try { + this.logger.debug(`Getting logs for pipeline #${pipelineNumber.toString()}`); + + const args = ["log", "show", repo, pipelineNumber.toString()]; + if (step) { + args.push(step); + } + + const { stdout } = await this.execWoodpecker(args); + return stdout; + } catch (error) { + this.logger.error(`Failed to get pipeline logs: ${String(error)}`); + throw new CIOperationError( + `Failed to get logs for pipeline #${pipelineNumber.toString()}`, + "getPipelineLogs", + error as Error + ); + } + } + + /** + * Diagnose pipeline failure from logs + */ + private diagnoseFail(logs: string): FailureDiagnosis { + // Check for common failure patterns + if (/eslint|lint.*error/i.test(logs)) { + return { + category: FailureCategory.LINT, + message: "Linting errors detected", + suggestion: "Run 'pnpm lint' locally to identify and fix linting issues", + details: this.extractErrors(logs, /error.*$/gim), + }; + } + + if (/type.*error|typescript.*error|tsc.*error/i.test(logs)) { + return { + category: FailureCategory.TYPE_CHECK, + message: "TypeScript type errors detected", + suggestion: "Run 'pnpm typecheck' locally to identify type errors", + details: this.extractErrors(logs, /error TS\d+:.*$/gim), + }; + } + + if (/test.*fail|tests.*fail|vitest.*fail/i.test(logs)) { + return { + category: FailureCategory.TEST, + message: "Test failures detected", + suggestion: "Run 'pnpm test' locally to identify failing tests", + details: this.extractErrors(logs, /FAIL.*$/gim), + }; + } + + if (/build.*fail|compilation.*fail/i.test(logs)) { + return { + category: FailureCategory.BUILD, + message: "Build errors detected", + suggestion: "Run 'pnpm build' locally to identify build issues", + details: this.extractErrors(logs, /error.*$/gim), + }; + } + + if (/secret|security|vulnerability/i.test(logs)) { + return { + category: FailureCategory.SECURITY, + message: "Security check failed", + suggestion: "Review security scan results and remove any hardcoded secrets", + details: this.extractErrors(logs, /.*secret.*|.*security.*/gim), + }; + } + + return { + category: FailureCategory.UNKNOWN, + message: "Pipeline failed with unknown error", + suggestion: "Review full pipeline logs to identify the issue", + }; + } + + /** + * Extract error messages from logs using regex + */ + private extractErrors(logs: string, pattern: RegExp): string { + const matches = logs.match(pattern); + if (!matches || matches.length === 0) { + return ""; + } + + // Return first 10 matches to avoid overwhelming output + return matches.slice(0, 10).join("\n"); + } + + /** + * Parse pipeline list output from Woodpecker CLI + */ + private parsePipelineList(output: string): PipelineInfo[] { + const lines = output.split("\n").filter((line) => line.trim()); + + // Skip header line and empty lines + const dataLines = lines.slice(1).filter((line) => !line.startsWith("Number")); + + return dataLines + .map((line) => { + // Parse table format: Number | Status | Event | Branch | Commit | ... + const parts = line.split("|").map((p) => p.trim()); + if (parts.length < 5) { + return null; + } + + const number = parseInt(parts[0], 10); + if (isNaN(number)) { + return null; + } + + return { + number, + status: this.parseStatus(parts[1]), + event: parts[2], + branch: parts[3], + commit: parts[4], + }; + }) + .filter((p): p is PipelineInfo => p !== null); + } + + /** + * Parse pipeline info output from Woodpecker CLI + */ + private parsePipelineInfo(output: string, pipelineNumber: number): PipelineInfo { + const lines = output.split("\n"); + + let status = PipelineStatus.PENDING; + let event = ""; + let branch = ""; + let commit = ""; + let started: number | undefined; + let finished: number | undefined; + let error: string | undefined; + + for (const line of lines) { + const trimmed = line.trim(); + + if (trimmed.toLowerCase().includes("status:")) { + const statusText = trimmed.split(":")[1]?.trim() ?? ""; + status = this.parseStatus(statusText); + } else if (trimmed.toLowerCase().includes("event:")) { + event = trimmed.split(":")[1]?.trim() ?? ""; + } else if (trimmed.toLowerCase().includes("branch:")) { + branch = trimmed.split(":")[1]?.trim() ?? ""; + } else if (trimmed.toLowerCase().includes("commit:")) { + commit = trimmed.split(":")[1]?.trim() ?? ""; + } else if (trimmed.toLowerCase().includes("started:")) { + const startedText = trimmed.split(":")[1]?.trim() ?? ""; + started = Date.parse(startedText); + } else if (trimmed.toLowerCase().includes("finished:")) { + const finishedText = trimmed.split(":")[1]?.trim() ?? ""; + finished = Date.parse(finishedText); + } else if (trimmed.toLowerCase().includes("error:")) { + error = trimmed.split(":")[1]?.trim() ?? ""; + } + } + + return { + number: pipelineNumber, + status, + event, + branch, + commit, + started, + finished, + error, + }; + } + + /** + * Parse status string to PipelineStatus enum + */ + private parseStatus(statusText: string): PipelineStatus { + const normalized = statusText.toLowerCase().trim(); + + switch (normalized) { + case "success": + return PipelineStatus.SUCCESS; + case "failure": + case "failed": + return PipelineStatus.FAILURE; + case "running": + return PipelineStatus.RUNNING; + case "pending": + return PipelineStatus.PENDING; + case "killed": + case "cancelled": + return PipelineStatus.KILLED; + case "error": + return PipelineStatus.ERROR; + case "skipped": + return PipelineStatus.SKIPPED; + case "blocked": + return PipelineStatus.BLOCKED; + default: + this.logger.warn(`Unknown pipeline status: ${statusText}`); + return PipelineStatus.PENDING; + } + } + + /** + * Execute Woodpecker CLI command + */ + private async execWoodpecker(args: string[]): Promise<{ stdout: string; stderr: string }> { + const env = { + ...process.env, + WOODPECKER_SERVER: this.woodpeckerServer, + WOODPECKER_TOKEN: this.woodpeckerToken, + }; + + const command = `woodpecker ${args.join(" ")}`; + this.logger.debug(`Executing: ${command}`); + + try { + return await execAsync(command, { env }); + } catch (error) { + if (error instanceof Error && "stderr" in error) { + this.logger.error(`Woodpecker CLI error: ${(error as { stderr: string }).stderr}`); + } + throw error; + } + } + + /** + * Delay helper for polling + */ + private delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } +} diff --git a/apps/orchestrator/src/ci/ci.module.ts b/apps/orchestrator/src/ci/ci.module.ts new file mode 100644 index 0000000..95de643 --- /dev/null +++ b/apps/orchestrator/src/ci/ci.module.ts @@ -0,0 +1,8 @@ +import { Module } from "@nestjs/common"; +import { CIOperationsService } from "./ci-operations.service"; + +@Module({ + providers: [CIOperationsService], + exports: [CIOperationsService], +}) +export class CIModule {} diff --git a/apps/orchestrator/src/ci/index.ts b/apps/orchestrator/src/ci/index.ts new file mode 100644 index 0000000..37fc537 --- /dev/null +++ b/apps/orchestrator/src/ci/index.ts @@ -0,0 +1,3 @@ +export * from "./ci.module"; +export * from "./ci-operations.service"; +export * from "./types"; diff --git a/apps/orchestrator/src/ci/types/ci-operations.types.ts b/apps/orchestrator/src/ci/types/ci-operations.types.ts new file mode 100644 index 0000000..c11de28 --- /dev/null +++ b/apps/orchestrator/src/ci/types/ci-operations.types.ts @@ -0,0 +1,85 @@ +/** + * CI pipeline status + */ +export enum PipelineStatus { + PENDING = "pending", + RUNNING = "running", + SUCCESS = "success", + FAILURE = "failure", + KILLED = "killed", + ERROR = "error", + SKIPPED = "skipped", + BLOCKED = "blocked", +} + +/** + * Pipeline information + */ +export interface PipelineInfo { + number: number; + status: PipelineStatus; + event: string; + branch: string; + commit: string; + started?: number; + finished?: number; + error?: string; +} + +/** + * CI failure diagnosis result + */ +export interface FailureDiagnosis { + category: FailureCategory; + message: string; + suggestion: string; + details?: string; +} + +/** + * Common CI failure categories + */ +export enum FailureCategory { + LINT = "lint", + TYPE_CHECK = "type-check", + TEST = "test", + BUILD = "build", + SECURITY = "security", + UNKNOWN = "unknown", +} + +/** + * Options for waiting on pipeline completion + */ +export interface PipelineWaitOptions { + /** Timeout in seconds (default: 1800 = 30 minutes) */ + timeout?: number; + /** Poll interval in seconds (default: 10) */ + pollInterval?: number; + /** Whether to fetch logs on failure (default: true) */ + fetchLogsOnFailure?: boolean; +} + +/** + * Result of waiting for pipeline + */ +export interface PipelineWaitResult { + success: boolean; + pipeline: PipelineInfo; + logs?: string; + diagnosis?: FailureDiagnosis; +} + +/** + * Custom error for CI operations + */ +export class CIOperationError extends Error { + constructor( + message: string, + public readonly operation: string, + public readonly cause?: Error + ) { + super(message); + this.name = "CIOperationError"; + } +} diff --git a/apps/orchestrator/src/ci/types/index.ts b/apps/orchestrator/src/ci/types/index.ts new file mode 100644 index 0000000..f451ed7 --- /dev/null +++ b/apps/orchestrator/src/ci/types/index.ts @@ -0,0 +1 @@ +export * from "./ci-operations.types";