"""Tests for success metrics reporting.""" from unittest.mock import MagicMock import pytest from src.coordinator import OrchestrationLoop from src.metrics import SuccessMetrics, generate_metrics_from_orchestrator class TestSuccessMetrics: """Test suite for SuccessMetrics dataclass.""" def test_to_dict(self) -> None: """Test conversion to dictionary.""" metrics = SuccessMetrics( total_issues=10, completed_issues=9, failed_issues=1, autonomy_rate=90.0, quality_pass_rate=90.0, intervention_count=1, cost_optimization_rate=75.0, context_rotations=0, estimation_accuracy=95.0, ) result = metrics.to_dict() assert result["total_issues"] == 10 assert result["completed_issues"] == 9 assert result["failed_issues"] == 1 assert result["autonomy_rate"] == 90.0 assert result["quality_pass_rate"] == 90.0 assert result["intervention_count"] == 1 assert result["cost_optimization_rate"] == 75.0 assert result["context_rotations"] == 0 assert result["estimation_accuracy"] == 95.0 def test_validate_targets_all_met(self) -> None: """Test target validation when all targets are met.""" metrics = SuccessMetrics( total_issues=5, completed_issues=5, failed_issues=0, autonomy_rate=100.0, quality_pass_rate=100.0, intervention_count=0, cost_optimization_rate=80.0, context_rotations=0, estimation_accuracy=95.0, ) validation = metrics.validate_targets() assert validation["autonomy_target_met"] is True assert validation["quality_target_met"] is True assert validation["cost_optimization_target_met"] is True assert validation["context_management_target_met"] is True assert validation["estimation_accuracy_target_met"] is True def test_validate_targets_some_failed(self) -> None: """Test target validation when some targets fail.""" metrics = SuccessMetrics( total_issues=10, completed_issues=7, failed_issues=3, autonomy_rate=70.0, # Below 100% target quality_pass_rate=70.0, # Below 100% target intervention_count=3, cost_optimization_rate=60.0, # Below 70% target context_rotations=2, estimation_accuracy=75.0, # Below 80% target ) validation = metrics.validate_targets() assert validation["autonomy_target_met"] is False assert validation["quality_target_met"] is False assert validation["cost_optimization_target_met"] is False assert validation["context_management_target_met"] is True # Always true currently assert validation["estimation_accuracy_target_met"] is False def test_format_report_all_targets_met(self) -> None: """Test report formatting when all targets are met.""" metrics = SuccessMetrics( total_issues=5, completed_issues=5, failed_issues=0, autonomy_rate=100.0, quality_pass_rate=100.0, intervention_count=0, cost_optimization_rate=80.0, context_rotations=0, estimation_accuracy=95.0, ) report = metrics.format_report() assert "SUCCESS METRICS REPORT" in report assert "Total Issues: 5" in report assert "Completed: 5" in report assert "Failed: 0" in report assert "Autonomy Rate: 100.0%" in report assert "Quality Pass Rate: 100.0%" in report assert "Cost Optimization: 80.0%" in report assert "Context Rotations: 0" in report assert "✓ ALL TARGETS MET" in report def test_format_report_targets_not_met(self) -> None: """Test report formatting when targets are not met.""" metrics = SuccessMetrics( total_issues=10, completed_issues=6, failed_issues=4, autonomy_rate=60.0, quality_pass_rate=60.0, intervention_count=4, cost_optimization_rate=50.0, context_rotations=0, estimation_accuracy=70.0, ) report = metrics.format_report() assert "SUCCESS METRICS REPORT" in report assert "✗ TARGETS NOT MET" in report assert "autonomy_target_met" in report assert "quality_target_met" in report assert "cost_optimization_target_met" in report class TestGenerateMetricsFromOrchestrator: """Test suite for generate_metrics_from_orchestrator function.""" @pytest.fixture def mock_orchestration_loop(self) -> MagicMock: """Create mock orchestration loop with metrics.""" loop = MagicMock(spec=OrchestrationLoop) loop.processed_count = 5 loop.success_count = 5 loop.rejection_count = 0 return loop @pytest.fixture def sample_issue_configs(self) -> list[dict[str, object]]: """Create sample issue configurations.""" return [ { "issue_number": 1001, "assigned_agent": "glm", "difficulty": "easy", "estimated_context": 15000, }, { "issue_number": 1002, "assigned_agent": "glm", "difficulty": "medium", "estimated_context": 35000, }, { "issue_number": 1003, "assigned_agent": "glm", "difficulty": "easy", "estimated_context": 12000, }, { "issue_number": 1004, "assigned_agent": "glm", "difficulty": "medium", "estimated_context": 45000, }, { "issue_number": 1005, "assigned_agent": "opus", "difficulty": "hard", "estimated_context": 80000, }, ] def test_generate_metrics( self, mock_orchestration_loop: MagicMock, sample_issue_configs: list[dict[str, object]], ) -> None: """Test metrics generation from orchestration loop.""" metrics = generate_metrics_from_orchestrator( mock_orchestration_loop, sample_issue_configs ) assert metrics.total_issues == 5 assert metrics.completed_issues == 5 assert metrics.failed_issues == 0 assert metrics.autonomy_rate == 100.0 assert metrics.quality_pass_rate == 100.0 assert metrics.intervention_count == 0 # 4 out of 5 use GLM (free model) = 80% assert metrics.cost_optimization_rate == 80.0 def test_generate_metrics_with_failures( self, sample_issue_configs: list[dict[str, object]] ) -> None: """Test metrics generation with some failures.""" loop = MagicMock(spec=OrchestrationLoop) loop.processed_count = 5 loop.success_count = 3 loop.rejection_count = 2 metrics = generate_metrics_from_orchestrator(loop, sample_issue_configs) assert metrics.total_issues == 5 assert metrics.completed_issues == 3 assert metrics.failed_issues == 2 assert metrics.autonomy_rate == 60.0 assert metrics.quality_pass_rate == 60.0 assert metrics.intervention_count == 2 def test_generate_metrics_empty_issues( self, mock_orchestration_loop: MagicMock ) -> None: """Test metrics generation with no issues.""" metrics = generate_metrics_from_orchestrator(mock_orchestration_loop, []) assert metrics.total_issues == 0 assert metrics.completed_issues == 5 # From loop assert metrics.cost_optimization_rate == 0.0 def test_generate_metrics_invalid_agent(self) -> None: """Test metrics generation with invalid agent name.""" loop = MagicMock(spec=OrchestrationLoop) loop.processed_count = 1 loop.success_count = 1 loop.rejection_count = 0 issue_configs = [ { "issue_number": 1001, "assigned_agent": "invalid_agent", "difficulty": "easy", "estimated_context": 15000, } ] metrics = generate_metrics_from_orchestrator(loop, issue_configs) # Should handle invalid agent gracefully assert metrics.total_issues == 1 assert metrics.cost_optimization_rate == 0.0 # Invalid agent not counted def test_generate_metrics_no_agent_assignment(self) -> None: """Test metrics generation with missing agent assignment.""" loop = MagicMock(spec=OrchestrationLoop) loop.processed_count = 1 loop.success_count = 1 loop.rejection_count = 0 issue_configs = [ { "issue_number": 1001, "difficulty": "easy", "estimated_context": 15000, } ] metrics = generate_metrics_from_orchestrator(loop, issue_configs) # Should handle missing agent gracefully assert metrics.total_issues == 1 assert metrics.cost_optimization_rate == 0.0