feat(#313): Implement FastAPI and agent tracing instrumentation

Add comprehensive OpenTelemetry distributed tracing to the coordinator
FastAPI service with automatic request tracing and custom decorators.

Implementation:
- Created src/telemetry.py: OTEL SDK initialization with OTLP exporter
- Created src/tracing_decorators.py: @trace_agent_operation and
  @trace_tool_execution decorators with sync/async support
- Integrated FastAPI auto-instrumentation in src/main.py
- Added tracing to coordinator operations in src/coordinator.py
- Environment-based configuration (OTEL_ENABLED, endpoint, sampling)

Features:
- Automatic HTTP request/response tracing via FastAPIInstrumentor
- Custom span enrichment with agent context (issue_id, agent_type)
- Graceful degradation when telemetry disabled
- Proper exception recording and status management
- Resource attributes (service.name, service.version, deployment.env)
- Configurable sampling ratio (0.0-1.0, defaults to 1.0)

Testing:
- 25 comprehensive tests (17 telemetry, 8 decorators)
- Coverage: 90-91% (exceeds 85% requirement)
- All tests passing, no regressions

Quality:
- Zero linting errors (ruff)
- Zero type checking errors (mypy)
- Security review approved (no vulnerabilities)
- Follows OTEL semantic conventions
- Proper error handling and resource cleanup

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

apps/coordinator/src/main.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import logging
+import os
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
 from pathlib import Path
@@ -16,6 +17,7 @@ from slowapi.util import get_remote_address
 from .config import settings
 from .coordinator import Coordinator
 from .queue import QueueManager
+from .telemetry import TelemetryService, shutdown_telemetry
 from .webhook import router as webhook_router
 
 
@@ -65,6 +67,13 @@ async def lifespan(app: FastAPI) -> AsyncIterator[dict[str, Any]]:
     logger.info(f"Log level: {settings.log_level}")
     logger.info(f"Server: {settings.host}:{settings.port}")
 
+    # Initialize OpenTelemetry if enabled
+    telemetry_enabled = os.getenv("OTEL_ENABLED", "true").lower() != "false"
+    if telemetry_enabled:
+        telemetry_service = TelemetryService()
+        telemetry_service.initialize()
+        logger.info("OpenTelemetry telemetry initialized")
+
     # Initialize queue manager
     queue_file = Path("queue.json")
     queue_manager = QueueManager(queue_file=queue_file)
@@ -104,6 +113,11 @@ async def lifespan(app: FastAPI) -> AsyncIterator[dict[str, Any]]:
                 pass
         logger.info("Coordinator stopped")
 
+    # Shutdown OpenTelemetry
+    if telemetry_enabled:
+        shutdown_telemetry()
+        logger.info("OpenTelemetry telemetry shut down")
+
     logger.info("Mosaic-coordinator shutdown complete")
 
 
@@ -118,6 +132,13 @@ app = FastAPI(
     lifespan=lifespan,
 )
 
+# Instrument FastAPI with OpenTelemetry if enabled
+if os.getenv("OTEL_ENABLED", "true").lower() != "false":
+    from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+
+    FastAPIInstrumentor.instrument_app(app)
+    logger.info("FastAPI instrumented with OpenTelemetry")
+
 # Register rate limiter
 app.state.limiter = limiter
 app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)