feat(#313): Implement FastAPI and agent tracing instrumentation

Add comprehensive OpenTelemetry distributed tracing to the coordinator
FastAPI service with automatic request tracing and custom decorators.

Implementation:
- Created src/telemetry.py: OTEL SDK initialization with OTLP exporter
- Created src/tracing_decorators.py: @trace_agent_operation and
  @trace_tool_execution decorators with sync/async support
- Integrated FastAPI auto-instrumentation in src/main.py
- Added tracing to coordinator operations in src/coordinator.py
- Environment-based configuration (OTEL_ENABLED, endpoint, sampling)

Features:
- Automatic HTTP request/response tracing via FastAPIInstrumentor
- Custom span enrichment with agent context (issue_id, agent_type)
- Graceful degradation when telemetry disabled
- Proper exception recording and status management
- Resource attributes (service.name, service.version, deployment.env)
- Configurable sampling ratio (0.0-1.0, defaults to 1.0)

Testing:
- 25 comprehensive tests (17 telemetry, 8 decorators)
- Coverage: 90-91% (exceeds 85% requirement)
- All tests passing, no regressions

Quality:
- Zero linting errors (ruff)
- Zero type checking errors (mypy)
- Security review approved (no vulnerabilities)
- Follows OTEL semantic conventions
- Proper error handling and resource cleanup

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

apps/coordinator/src/coordinator.py

@@ -9,6 +9,7 @@ from src.forced_continuation import ForcedContinuationService
 from src.models import ContextAction
 from src.quality_orchestrator import QualityOrchestrator, VerificationResult
 from src.queue import QueueItem, QueueManager
+from src.tracing_decorators import trace_agent_operation
 
 if TYPE_CHECKING:
     pass
@@ -114,6 +115,7 @@ class Coordinator:
         if self._stop_event is not None:
             self._stop_event.set()
 
+    @trace_agent_operation(operation_name="process_queue")
     async def process_queue(self) -> QueueItem | None:
         """Process the next ready item from the queue.
 
@@ -156,6 +158,7 @@ class Coordinator:
 
         return item
 
+    @trace_agent_operation(operation_name="spawn_agent")
     async def spawn_agent(self, item: QueueItem) -> bool:
         """Spawn an agent to process the given item.
 
@@ -327,6 +330,7 @@ class OrchestrationLoop:
         if self._stop_event is not None:
             self._stop_event.set()
 
+    @trace_agent_operation(operation_name="process_next_issue")
     async def process_next_issue(self) -> QueueItem | None:
         """Process the next ready issue from the queue.
 

apps/coordinator/src/main.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import logging
+import os
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
 from pathlib import Path
@@ -16,6 +17,7 @@ from slowapi.util import get_remote_address
 from .config import settings
 from .coordinator import Coordinator
 from .queue import QueueManager
+from .telemetry import TelemetryService, shutdown_telemetry
 from .webhook import router as webhook_router
 
 
@@ -65,6 +67,13 @@ async def lifespan(app: FastAPI) -> AsyncIterator[dict[str, Any]]:
     logger.info(f"Log level: {settings.log_level}")
     logger.info(f"Server: {settings.host}:{settings.port}")
 
+    # Initialize OpenTelemetry if enabled
+    telemetry_enabled = os.getenv("OTEL_ENABLED", "true").lower() != "false"
+    if telemetry_enabled:
+        telemetry_service = TelemetryService()
+        telemetry_service.initialize()
+        logger.info("OpenTelemetry telemetry initialized")
+
     # Initialize queue manager
     queue_file = Path("queue.json")
     queue_manager = QueueManager(queue_file=queue_file)
@@ -104,6 +113,11 @@ async def lifespan(app: FastAPI) -> AsyncIterator[dict[str, Any]]:
                 pass
         logger.info("Coordinator stopped")
 
+    # Shutdown OpenTelemetry
+    if telemetry_enabled:
+        shutdown_telemetry()
+        logger.info("OpenTelemetry telemetry shut down")
+
     logger.info("Mosaic-coordinator shutdown complete")
 
 
@@ -118,6 +132,13 @@ app = FastAPI(
     lifespan=lifespan,
 )
 
+# Instrument FastAPI with OpenTelemetry if enabled
+if os.getenv("OTEL_ENABLED", "true").lower() != "false":
+    from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+
+    FastAPIInstrumentor.instrument_app(app)
+    logger.info("FastAPI instrumented with OpenTelemetry")
+
 # Register rate limiter
 app.state.limiter = limiter
 app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)

apps/coordinator/src/telemetry.py

@@ -0,0 +1,182 @@
+"""OpenTelemetry telemetry initialization and configuration."""
+
+import logging
+import os
+
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
+
+logger = logging.getLogger(__name__)
+
+
+class TelemetryService:
+    """Service responsible for OpenTelemetry distributed tracing.
+
+    Initializes the OTEL SDK with OTLP exporters and provides
+    tracing utilities for coordinator operations.
+
+    Example:
+        >>> service = TelemetryService()
+        >>> service.initialize()
+        >>> tracer = service.get_tracer()
+    """
+
+    def __init__(self) -> None:
+        """Initialize the TelemetryService."""
+        self.enabled = os.getenv("OTEL_ENABLED", "true").lower() != "false"
+        self.service_name = os.getenv("OTEL_SERVICE_NAME", "mosaic-coordinator")
+        self.provider: TracerProvider | None = None
+        self._tracer: trace.Tracer | None = None
+
+    def _get_sampling_ratio(self) -> float:
+        """Get the trace sampling ratio from environment variable.
+
+        Returns 1.0 (sample all traces) by default.
+        Clamps value between 0.0 and 1.0.
+
+        Returns:
+            The sampling ratio between 0.0 and 1.0
+        """
+        env_value = os.getenv("OTEL_TRACES_SAMPLER_ARG")
+        if not env_value:
+            return 1.0
+
+        try:
+            parsed = float(env_value)
+        except ValueError:
+            logger.warning(
+                f"Invalid OTEL_TRACES_SAMPLER_ARG value: {env_value}, using default 1.0"
+            )
+            return 1.0
+
+        # Clamp to valid range
+        clamped = max(0.0, min(1.0, parsed))
+        if clamped != parsed:
+            logger.warning(f"OTEL_TRACES_SAMPLER_ARG clamped from {parsed} to {clamped}")
+
+        return clamped
+
+    def _get_deployment_environment(self) -> str:
+        """Get the deployment environment from environment variables.
+
+        Defaults to 'development' if not set.
+
+        Returns:
+            The deployment environment string
+        """
+        return os.getenv("OTEL_DEPLOYMENT_ENVIRONMENT", "development")
+
+    def _get_otlp_endpoint(self) -> str:
+        """Get the OTLP endpoint from environment variable.
+
+        Returns:
+            The OTLP endpoint URL
+        """
+        return os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318/v1/traces")
+
+    def initialize(self) -> None:
+        """Initialize the OpenTelemetry SDK with configured exporters.
+
+        This should be called during application startup.
+        """
+        if not self.enabled:
+            logger.info("OpenTelemetry tracing is disabled")
+            self._tracer = trace.get_tracer("noop")
+            return
+
+        try:
+            # Create resource with service metadata
+            resource = Resource.create(
+                attributes={
+                    "service.name": self.service_name,
+                    "service.version": "0.0.1",
+                    "deployment.environment": self._get_deployment_environment(),
+                }
+            )
+
+            # Create sampler with parent-based strategy
+            sampling_ratio = self._get_sampling_ratio()
+            sampler = ParentBased(root=TraceIdRatioBased(sampling_ratio))
+
+            # Create tracer provider
+            self.provider = TracerProvider(resource=resource, sampler=sampler)
+
+            # Create OTLP exporter
+            otlp_endpoint = self._get_otlp_endpoint()
+            exporter = OTLPSpanExporter(endpoint=otlp_endpoint)
+
+            # Add span processor
+            processor = BatchSpanProcessor(exporter)
+            self.provider.add_span_processor(processor)
+
+            # Set global tracer provider
+            trace.set_tracer_provider(self.provider)
+
+            # Get tracer instance
+            self._tracer = trace.get_tracer(self.service_name)
+
+            logger.info(
+                f"OpenTelemetry SDK started for service: {self.service_name} "
+                f"(environment: {self._get_deployment_environment()}, "
+                f"sampling: {sampling_ratio}, endpoint: {otlp_endpoint})"
+            )
+
+        except Exception as e:
+            logger.error(f"Failed to initialize OpenTelemetry SDK: {e}")
+            # Fallback to noop tracer to prevent application failures
+            self._tracer = trace.get_tracer("noop")
+
+    def get_tracer(self) -> trace.Tracer:
+        """Get the tracer instance for creating spans.
+
+        Returns:
+            The configured tracer instance
+        """
+        if self._tracer is None:
+            # Initialize if not already done
+            self.initialize()
+        assert self._tracer is not None
+        return self._tracer
+
+    def shutdown(self) -> None:
+        """Shutdown the OpenTelemetry SDK gracefully.
+
+        This should be called during application shutdown.
+        """
+        if self.provider is not None:
+            try:
+                self.provider.shutdown()
+                logger.info("OpenTelemetry SDK shut down successfully")
+            except Exception as e:
+                logger.error(f"Error shutting down OpenTelemetry SDK: {e}")
+
+
+# Global telemetry service instance
+_telemetry_service: TelemetryService | None = None
+
+
+def get_tracer() -> trace.Tracer:
+    """Get the global tracer instance.
+
+    Returns:
+        The configured tracer instance
+    """
+    global _telemetry_service
+    if _telemetry_service is None:
+        _telemetry_service = TelemetryService()
+        _telemetry_service.initialize()
+    return _telemetry_service.get_tracer()
+
+
+def shutdown_telemetry() -> None:
+    """Shutdown the global telemetry service.
+
+    This should be called during application shutdown.
+    """
+    global _telemetry_service
+    if _telemetry_service is not None:
+        _telemetry_service.shutdown()

apps/coordinator/src/tracing_decorators.py

@@ -0,0 +1,157 @@
+"""Tracing decorators for agent operations and tool executions."""
+
+import functools
+import inspect
+from collections.abc import Callable
+from inspect import BoundArguments
+from typing import ParamSpec, TypeVar, cast
+
+from opentelemetry.trace import Span, SpanKind, StatusCode
+
+from .telemetry import get_tracer
+
+P = ParamSpec("P")
+R = TypeVar("R")
+
+
+def _create_tracing_wrapper(
+    func: Callable[P, R],
+    span_name: str,
+    span_kind: SpanKind,
+    attribute_extractor: Callable[[BoundArguments, Span], None],
+) -> Callable[P, R]:
+    """Internal function to create tracing wrappers for both sync and async functions.
+
+    Args:
+        func: The function to wrap
+        span_name: Name for the OpenTelemetry span
+        span_kind: Kind of span (INTERNAL, CLIENT, etc.)
+        attribute_extractor: Callable to extract and set span attributes from function arguments
+
+    Returns:
+        Wrapped function with tracing
+    """
+
+    @functools.wraps(func)
+    def sync_wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+        tracer = get_tracer()
+
+        with tracer.start_as_current_span(span_name, kind=span_kind) as span:
+            try:
+                # Extract and set attributes
+                sig = inspect.signature(func)
+                bound_args = sig.bind(*args, **kwargs)
+                bound_args.apply_defaults()
+                attribute_extractor(bound_args, span)
+
+                # Execute the function
+                result = func(*args, **kwargs)
+                return result
+
+            except Exception as e:
+                # Record exception and set error status
+                span.record_exception(e)
+                span.set_status(StatusCode.ERROR, str(e))
+                raise
+
+    @functools.wraps(func)
+    async def async_wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+        tracer = get_tracer()
+
+        with tracer.start_as_current_span(span_name, kind=span_kind) as span:
+            try:
+                # Extract and set attributes
+                sig = inspect.signature(func)
+                bound_args = sig.bind(*args, **kwargs)
+                bound_args.apply_defaults()
+                attribute_extractor(bound_args, span)
+
+                # Execute the async function
+                result = await func(*args, **kwargs)  # type: ignore[misc]  # Generic async wrapper limitation
+                return result  # type: ignore[no-any-return]  # Generic async wrapper limitation
+
+            except Exception as e:
+                # Record exception and set error status
+                span.record_exception(e)
+                span.set_status(StatusCode.ERROR, str(e))
+                raise
+
+    # Return appropriate wrapper based on whether function is async
+    if inspect.iscoroutinefunction(func):
+        return cast(Callable[P, R], async_wrapper)
+    else:
+        return cast(Callable[P, R], sync_wrapper)
+
+
+def trace_agent_operation(
+    operation_name: str,
+) -> Callable[[Callable[P, R]], Callable[P, R]]:
+    """Decorator that adds OpenTelemetry tracing to agent operations.
+
+    Automatically creates spans with coordinator-specific attributes.
+
+    Args:
+        operation_name: Name of the agent operation being traced
+
+    Returns:
+        Decorated function with tracing
+
+    Example:
+        >>> @trace_agent_operation(operation_name="assign_agent")
+        >>> async def assign_agent(issue_id: int, agent_type: str) -> bool:
+        >>>     # Implementation
+        >>>     return True
+    """
+
+    def attribute_extractor(bound_args: BoundArguments, span: Span) -> None:
+        """Extract agent-specific attributes from function arguments."""
+        if "issue_id" in bound_args.arguments:
+            span.set_attribute("agent.issue_id", bound_args.arguments["issue_id"])
+        if "issue_number" in bound_args.arguments:
+            span.set_attribute("agent.issue_number", bound_args.arguments["issue_number"])
+        if "agent_type" in bound_args.arguments:
+            span.set_attribute("agent.agent_type", bound_args.arguments["agent_type"])
+        if "agent_id" in bound_args.arguments:
+            span.set_attribute("agent.agent_id", bound_args.arguments["agent_id"])
+
+    def decorator(func: Callable[P, R]) -> Callable[P, R]:
+        span_name = f"agent.{operation_name}"
+        return _create_tracing_wrapper(func, span_name, SpanKind.INTERNAL, attribute_extractor)
+
+    return decorator
+
+
+def trace_tool_execution(
+    tool_name: str,
+) -> Callable[[Callable[P, R]], Callable[P, R]]:
+    """Decorator that adds OpenTelemetry tracing to tool executions.
+
+    Automatically creates spans with tool-specific attributes.
+
+    Args:
+        tool_name: Name of the tool being executed
+
+    Returns:
+        Decorated function with tracing
+
+    Example:
+        >>> @trace_tool_execution(tool_name="quality_gate")
+        >>> async def run_quality_gate(issue_id: int) -> bool:
+        >>>     # Implementation
+        >>>     return True
+    """
+
+    def attribute_extractor(bound_args: BoundArguments, span: Span) -> None:
+        """Extract tool-specific attributes from function arguments."""
+        span.set_attribute("tool.name", tool_name)
+
+        if "issue_id" in bound_args.arguments:
+            span.set_attribute("tool.issue_id", bound_args.arguments["issue_id"])
+        if "issue_number" in bound_args.arguments:
+            span.set_attribute("tool.issue_number", bound_args.arguments["issue_number"])
+
+    def decorator(func: Callable[P, R]) -> Callable[P, R]:
+        span_name = f"tool.{tool_name}"
+        return _create_tracing_wrapper(func, span_name, SpanKind.CLIENT, attribute_extractor)
+
+    return decorator