from pydantic_settings import BaseSettings
from typing import List, Optional, Dict, Any
from enum import Enum

class EvaluationType(str, Enum):
    CODE_QUALITY = "code_quality"
    SAFETY = "safety" 
    CORRECTNESS = "correctness"
    EFFICIENCY = "efficiency"
    STYLE = "style"

class PromptType(str, Enum):
    CODE_GENERATION = "code_generation"
    CODE_EXPLANATION = "code_explanation"
    BUG_FIXING = "bug_fixing"
    CODE_REVIEW = "code_review"
    SECURITY_AUDIT = "security_audit"

class Settings(BaseSettings):
    # CodeVerdict Configuration
    app_name: str = "CodeVerdict"
    app_version: str = "1.0.0"
    description: str = "Where AI Code Stands Trial"
    
    # MLflow
    mlflow_tracking_uri: str = "sqlite:///mlflow.db"
    mlflow_registry_uri: str = "sqlite:///mlflow.db"
    
    # Argilla
    argilla_api_url: str = "http://localhost:6900"
    argilla_api_key: str = "codeverdict.apikey"
    
    # Neo4j Knowledge Graph
    neo4j_uri: str = "bolt://localhost:7687"
    neo4j_username: str = "neo4j"
    neo4j_password: str = "codeverdict"
    
    # Model APIs
    openai_api_key: Optional[str] = None
    anthropic_api_key: Optional[str] = None
    huggingface_token: Optional[str] = None
    
    # Evaluation
    auto_eval_threshold: float = 0.8
    manual_review_sample_rate: float = 0.5
    min_manual_reviews: int = 10
    max_concurrent_evaluations: int = 50
    
    # Security
    enable_security_scan: bool = True
    enable_bias_detection: bool = True
    
    # Knowledge Graph Settings
    enable_knowledge_graph: bool = True
    kg_auto_ingest: bool = True
    
    class Config:
        env_file = ".env"
        case_sensitive = False

settings = Settings()

from pydantic import BaseModel, Field
from typing import List, Dict, Any, Optional, Union
from datetime import datetime
from enum import Enum
import uuid

class VerdictStatus(str, Enum):
    PENDING = "pending"
    AUTO_APPROVED = "auto_approved"
    MANUAL_REVIEW = "manual_review"
    REJECTED = "rejected"
    APPROVED = "approved"

class CodeVerdict(BaseModel):
    """Final verdict for a code completion"""
    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    completion_id: str
    status: VerdictStatus
    overall_score: float = Field(..., ge=0, le=1)
    confidence: float = Field(..., ge=0, le=1)
    criteria_scores: Dict[str, float] = Field(default_factory=dict)
    human_feedback: Optional[str] = None
    reviewed_by: Optional[str] = None
    created_at: datetime = Field(default_factory=datetime.now)
    updated_at: datetime = Field(default_factory=datetime.now)

class CodeEvaluationCriteria(BaseModel):
    correctness: float = Field(..., ge=0, le=1)
    efficiency: float = Field(..., ge=0, le=1) 
    readability: float = Field(..., ge=0, le=1)
    security: float = Field(..., ge=0, le=1)
    style_adherence: float = Field(..., ge=0, le=1)

class CodePrompt(BaseModel):
    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    prompt: str
    prompt_type: str
    context: Optional[Dict[str, Any]] = None
    expected_output: Optional[str] = None
    test_cases: Optional[List[Dict]] = None
    difficulty: str = "medium"
    tags: List[str] = Field(default_factory=list)

class CodeCompletion(BaseModel):
    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    prompt_id: str
    model_id: str
    completion: str
    metadata: Dict[str, Any] = Field(default_factory=dict)
    timestamp: datetime = Field(default_factory=datetime.now)

class EvaluationResult(BaseModel):
    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    completion_id: str
    evaluator_type: str
    criteria: CodeEvaluationCriteria
    overall_score: float
    confidence: Optional[float] = None
    rater_id: Optional[str] = None
    feedback: Optional[str] = None
    verdict: Optional[VerdictStatus] = None

# codeverdict/evaluation/auto_evaluator.py
import phoenix as px
import pandas as pd
from typing import List, Dict, Any, Tuple
import numpy as np
from codeverdict.data.models import CodeCompletion, CodeEvaluationCriteria

class CodeVerdictAutoEvaluator:
    def __init__(self):
        self.session = px.launch_app()
        self.security_patterns = [
            "eval(", "exec(", "pickle.loads", "os.system", "subprocess.call",
            "shell=True", "input()", "marshal.loads", "__import__", "compile("
        ]
        
    def evaluate_code_correctness(self, completion: CodeCompletion, test_cases: List[Dict]) -> Dict[str, float]:
        """Evaluate code correctness using test cases"""
        passed_tests = 0
        total_tests = len(test_cases)
        
        for test_case in test_cases:
            if self._simulate_test_execution(completion.completion, test_case):
                passed_tests += 1
                
        correctness_score = passed_tests / total_tests if total_tests > 0 else 0.0
        return {"correctness": correctness_score}
    
    def evaluate_code_quality(self, completion: CodeCompletion) -> Dict[str, float]:
        """Evaluate code quality using various metrics"""
        code = completion.completion
        
        readability_score = self._calculate_readability(code)
        efficiency_score = self._estimate_efficiency(code)
        security_score = self._check_security_issues(code)
        style_score = self._check_style_adherence(code)
        
        return {
            "readability": readability_score,
            "efficiency": efficiency_score, 
            "security": security_score,
            "style_adherence": style_score
        }
    
    def generate_verdict(self, completion: CodeCompletion, scores: Dict[str, float]) -> Dict[str, Any]:
        """Generate final verdict based on evaluation scores"""
        overall_score = np.mean(list(scores.values()))
        
        # Determine verdict status based on scores
        if overall_score >= 0.8:
            status = "auto_approved"
            confidence = 0.9
        elif overall_score >= 0.6:
            status = "manual_review"
            confidence = 0.7
        else:
            status = "rejected"
            confidence = 0.8
            
        return {
            "completion_id": completion.id,
            "status": status,
            "overall_score": overall_score,
            "confidence": confidence,
            "criteria_scores": scores,
            "evaluated_at": pd.Timestamp.now().isoformat()
        }
    
    def _calculate_readability(self, code: str) -> float:
        """Calculate code readability score"""
        lines = code.split('\n')
        if len(lines) == 0:
            return 0.0
            
        # Calculate various readability metrics
        avg_line_length = sum(len(line) for line in lines) / len(lines)
        comment_lines = len([line for line in lines if line.strip().startswith('#')])
        comment_ratio = comment_lines / len(lines) if lines else 0
        
        # Combine metrics
        line_length_score = max(0, 1 - (avg_line_length - 40) / 100)
        readability = (line_length_score + min(comment_ratio * 3, 1)) / 2
        
        return min(1.0, max(0.0, readability))
    
    def _check_security_issues(self, code: str) -> float:
        """Check for common security issues"""
        score = 1.0
        issues_found = 0
        
        for pattern in self.security_patterns:
            if pattern in code:
                issues_found += 1
                score -= 0.15  # Penalize for each security issue
                
        # Additional security checks
        if "password" in code.lower() and "getpass" not in code:
            score -= 0.1
                
        return max(0.0, score)
    
    def _estimate_efficiency(self, code: str) -> float:
        """Estimate code efficiency based on patterns"""
        efficiency_indicators = [
            ("for loop in for loop", -0.3),
            ("while True", -0.2),
            ("recursion", -0.1),
            ("list comprehension", 0.1),
            ("generator", 0.2),
            ("built-in functions", 0.1)
        ]
        
        score = 0.7  # Base score
        
        for pattern, impact in efficiency_indicators:
            if pattern in code.lower():
                score += impact
                
        return max(0.0, min(1.0, score))
    
    def _check_style_adherence(self, code: str) -> float:
        """Check basic style adherence"""
        lines = code.split('\n')
        if not lines:
            return 0.0
            
        style_issues = 0
        
        # Check for very long lines
        for line in lines:
            if len(line) > 100:
                style_issues += 1
                
        # Check for proper indentation (basic check)
        if code and not code[0].isspace():
            style_issues += 1
            
        # Calculate score
        max_issues = max(len(lines) // 10, 1)
        style_score = 1.0 - (style_issues / max_issues)
        
        return max(0.0, min(1.0, style_score))
    
    def _simulate_test_execution(self, code: str, test_case: Dict) -> bool:
        """Simulate test execution (placeholder for actual execution)"""
        # In a real implementation, this would actually execute the code
        # For now, we'll simulate based on code patterns
        
        # Simple simulation logic
        test_input = test_case.get('input', '')
        expected_output = test_case.get('output', '')
        
        # Check if code contains expected patterns
        if 'return' in code and expected_output:
            return True  # Simulate passing test
            
        return False  # Simulate failing test

# codeverdict/evaluation/manual_evaluator.py
import argilla as rg
from typing import List, Dict, Any, Optional
from codeverdict.data.models import CodeCompletion, CodeEvaluationCriteria, EvaluationResult
import pandas as pd
import numpy as np

class CodeVerdictManualEvaluator:
    def __init__(self, api_url: str = "http://localhost:6900", api_key: str = "codeverdict.apikey"):
        rg.init(api_url=api_url, api_key=api_key)
        
    def create_evaluation_workspace(self, workspace_name: str, guidelines: str):
        """Create a workspace for manual evaluation in CodeVerdict"""
        try:
            rg.Workspace.create(workspace_name)
            print(f"✅ Created CodeVerdict workspace: {workspace_name}")
        except Exception as e:
            print(f"ℹ️ Workspace may already exist: {e}")
            
    def setup_code_quality_dataset(self, dataset_name: str, workspace: str = "codeverdict"):
        """Setup dataset for code quality evaluation"""
        settings = rg.DatasetSettings(
            guidelines="""
            # CodeVerdict Evaluation Guidelines ⚖️
            
            Evaluate the AI-generated code based on the following criteria:
            
            ## Correctness (1-5)
            - Does the code solve the problem correctly?
            - Are there any logical errors?
            - Does it handle edge cases?
            
            ## Efficiency (1-5)  
            - Is the code optimized for performance?
            - Are there unnecessary computations?
            - Is the algorithm choice appropriate?
            
            ## Readability (1-5)
            - Is the code easy to understand?
            - Are variables and functions well-named?
            - Is the code properly structured?
            
            ## Security (1-5)
            - Does it follow security best practices?
            - Are there potential vulnerabilities?
            - Is input validation performed?
            
            ## Style Adherence (1-5)
            - Does it follow proper coding style?
            - Is the formatting consistent?
            - Are comments used appropriately?
            """,
            fields=[
                rg.TextField(name="prompt", required=True),
                rg.TextField(name="completion", required=True),
                rg.TextField(name="model_id", required=True)
            ],
            questions=[
                rg.RatingQuestion(
                    name="correctness",
                    title="How correct is the code?",
                    description="Does it solve the problem correctly without errors?",
                    required=True,
                    values=[1, 2, 3, 4, 5]
                ),
                rg.RatingQuestion(
                    name="efficiency", 
                    title="How efficient is the code?",
                    description="Is it optimized for performance?",
                    required=True,
                    values=[1, 2, 3, 4, 5]
                ),
                rg.RatingQuestion(
                    name="readability",
                    title="How readable is the code?",
                    description="Is it easy to understand and maintain?",
                    required=True,
                    values=[1, 2, 3, 4, 5]
                ),
                rg.RatingQuestion(
                    name="security",
                    title="How secure is the code?",
                    description="Does it follow security best practices?",
                    required=True,
                    values=[1, 2, 3, 4, 5]
                ),
                rg.RatingQuestion(
                    name="style_adherence",
                    title="How well does it adhere to coding style?",
                    description="Does it follow proper style guidelines?",
                    required=True,
                    values=[1, 2, 3, 4, 5]
                ),
                rg.TextQuestion(
                    name="feedback",
                    title="Additional feedback",
                    description="Any specific issues, suggestions, or comments?",
                    required=False
                )
            ]
        )
        
        dataset = rg.FeedbackDataset(
            settings=settings,
            fields=settings.fields,
            questions=settings.questions
        )
        
        return dataset
    
    def add_completions_for_review(self, dataset: rg.FeedbackDataset, completions: List[CodeCompletion]):
        """Add completions to the CodeVerdict manual evaluation queue"""
        records = []
        
        for completion in completions:
            record = rg.FeedbackRecord(
                fields={
                    "prompt": completion.metadata.get("prompt_text", ""),
                    "completion": completion.completion,
                    "model_id": completion.model_id
                },
                metadata={
                    "completion_id": completion.id,
                    "prompt_id": completion.prompt_id,
                    "prompt_type": completion.metadata.get("prompt_type", "unknown"),
                    "difficulty": completion.metadata.get("difficulty", "medium"),
                    "timestamp": completion.timestamp.isoformat()
                }
            )
            records.append(record)
            
        dataset.add_records(records)
        print(f"📝 Added {len(records)} completions to manual review queue")
        
    def push_to_argilla(self, dataset: rg.FeedbackDataset, dataset_name: str):
        """Push dataset to Argilla server with CodeVerdict branding"""
        dataset.push_to_argilla(name=dataset_name, workspace="codeverdict")
        print(f"🚀 Published CodeVerdict dataset: {dataset_name}")

# codeverdict/evaluation/triage_engine.py
from typing import List, Dict, Any, Tuple
from codeverdict.data.models import CodeCompletion, VerdictStatus
from codeverdict.evaluation.auto_evaluator import CodeVerdictAutoEvaluator
import numpy as np
from enum import Enum

class TriageDecision(Enum):
    AUTO_APPROVE = "auto_approve"
    MANUAL_REVIEW = "manual_review"
    AUDIT_SAMPLE = "audit_sample"

class CodeVerdictTriageEngine:
    def __init__(self, auto_eval_threshold: float = 0.8, manual_sample_rate: float = 0.5):
        self.auto_eval_threshold = auto_eval_threshold
        self.manual_sample_rate = manual_sample_rate
        self.auto_evaluator = CodeVerdictAutoEvaluator()
        
    def triage_completions(self, completions: List[CodeCompletion]) -> Tuple[List[CodeCompletion], List[CodeCompletion]]:
        """Split completions into auto-evaluated and manual review batches"""
        auto_eval_batch = []
        manual_review_batch = []
        
        for completion in completions:
            decision = self._make_triage_decision(completion)
            
            if decision == TriageDecision.AUTO_APPROVE:
                auto_eval_batch.append(completion)
            elif decision == TriageDecision.MANUAL_REVIEW:
                manual_review_batch.append(completion)
            else:  # AUDIT_SAMPLE
                if np.random.random() < 0.5:
                    auto_eval_batch.append(completion)
                else:
                    manual_review_batch.append(completion)
        
        # Ensure we maintain approximately the desired 50/50 split
        total = len(completions)
        if total > 0:
            current_auto_ratio = len(auto_eval_batch) / total
            target_auto_ratio = 1 - self.manual_sample_rate
            
            if abs(current_auto_ratio - target_auto_ratio) > 0.1:
                self._rebalance_batches(auto_eval_batch, manual_review_batch, target_auto_ratio)
        
        print(f"🔀 CodeVerdict Triage: {len(auto_eval_batch)} auto-eval, {len(manual_review_batch)} manual review")
        return auto_eval_batch, manual_review_batch
    
    def _make_triage_decision(self, completion: CodeCompletion) -> TriageDecision:
        """Make triage decision for a single completion"""
        prompt_type = completion.metadata.get('prompt_type', 'code_generation')
        
        # Critical security prompts always go to manual review
        if prompt_type == "security_audit":
            return TriageDecision.MANUAL_REVIEW
            
        # High-risk bug fixes get careful review
        elif prompt_type == "bug_fixing":
            quality_scores = self.auto_evaluator.evaluate_code_quality(completion)
            overall_quality = np.mean(list(quality_scores.values()))
            
            if overall_quality > self.auto_eval_threshold:
                return TriageDecision.AUTO_APPROVE
            elif overall_quality < 0.5:
                return TriageDecision.MANUAL_REVIEW
            else:
                return TriageDecision.AUDIT_SAMPLE
                
        # Standard code generation with sampling
        else:
            if np.random.random() < self.manual_sample_rate:
                return TriageDecision.MANUAL_REVIEW
            else:
                return TriageDecision.AUTO_APPROVE
    
    def _rebalance_batches(self, auto_batch: List[CodeCompletion], manual_batch: List[CodeCompletion], target_auto_ratio: float):
        """Rebalance batches to maintain target ratio"""
        total = len(auto_batch) + len(manual_batch)
        target_auto_count = int(total * target_auto_ratio)
        
        current_auto_count = len(auto_batch)
        
        if current_auto_count > target_auto_count:
            move_count = current_auto_count - target_auto_count
            to_move = auto_batch[-move_count:]
            manual_batch.extend(to_move)
            del auto_batch[-move_count:]
        else:
            move_count = target_auto_count - current_auto_count
            to_move = manual_batch[-move_count:]
            auto_batch.extend(to_move)
            del manual_batch[-move_count:]

# codeverdict/knowledge_graph/neo4j_service.py
from neo4j import GraphDatabase
from typing import Dict, List, Any, Optional
import json
from datetime import datetime
from codeverdict.data.models import CodeCompletion, CodeVerdict, EvaluationResult

class CodeVerdictKnowledgeGraph:
    def __init__(self, uri: str, user: str, password: str):
        self.driver = GraphDatabase.driver(uri, auth=(user, password))
        
    def close(self):
        self.driver.close()
    
    def create_evaluation_run(self, run_id: str, model_id: str, timestamp: str):
        """Create evaluation run node in knowledge graph"""
        query = """
        CREATE (run:EvaluationRun {
            id: $run_id,
            model_id: $model_id,
            timestamp: $timestamp,
            created_at: datetime()
        })
        RETURN run
        """
        with self.driver.session() as session:
            session.run(query, run_id=run_id, model_id=model_id, timestamp=timestamp)
    
    def ingest_verdict_patterns(self, verdicts: List[CodeVerdict], run_id: str):
        """Ingest verdict patterns and create relationships"""
        query = """
        MATCH (run:EvaluationRun {id: $run_id})
        UNWIND $verdicts AS verdict
        CREATE (v:Verdict {
            id: verdict.id,
            status: verdict.status,
            overall_score: verdict.overall_score,
            confidence: verdict.confidence,
            completion_id: verdict.completion_id
        })
        CREATE (run)-[r:CONTAINS_VERDICT]->(v)
        
        // Create failure pattern clusters
        WITH v, verdict
        UNWIND keys(verdict.criteria_scores) AS criterion
        CREATE (c:Criterion {
            name: criterion,
            score: verdict.criteria_scores[criterion]
        })
        CREATE (v)-[sc:HAS_CRITERION]->(c)
        
        // Link similar failure patterns
        WITH v, verdict
        MATCH (existing:Verdict {status: verdict.status})
        WHERE existing.overall_score < 0.7 AND v.overall_score < 0.7
          AND abs(existing.overall_score - v.overall_score) < 0.1
        CREATE (existing)-[sim:SIMILAR_FAILURE]->(v)
        SET sim.strength = 1 - abs(existing.overall_score - v.overall_score)
        
        RETURN COUNT(v) as verdicts_ingested
        """
        
        verdict_dicts = [v.dict() for v in verdicts]
        with self.driver.session() as session:
            result = session.run(query, run_id=run_id, verdicts=verdict_dicts)
            return result.single()["verdicts_ingested"]

    def get_failure_clusters(self, model_id: str, threshold: float = 0.7):
        """Get clusters of similar failures for a model"""
        query = """
        MATCH (run:EvaluationRun {model_id: $model_id})-[:CONTAINS_VERDICT]->(v:Verdict)
        WHERE v.overall_score < $threshold
        MATCH (v)-[:HAS_CRITERION]->(c:Criterion)
        WITH v, COLLECT(c) AS criteria
        RETURN v.status as failure_type,
               COUNT(v) as frequency,
               AVG(v.overall_score) as avg_severity,
               COLLECT(criteria[0].name) as common_criteria
        ORDER BY frequency DESC
        """
        
        with self.driver.session() as session:
            result = session.run(query, model_id=model_id, threshold=threshold)
            return [dict(record) for record in result]

# codeverdict/knowledge_graph/mlflow_neo4j_bridge.py
import mlflow
from typing import Dict, Any
from .neo4j_service import CodeVerdictKnowledgeGraph

class MLflowNeo4jBridge:
    def __init__(self, kg_service: CodeVerdictKnowledgeGraph):
        self.kg = kg_service
        
    def track_mlflow_run_to_kg(self, run_id: str):
        """Bridge MLflow run data to knowledge graph"""
        run = mlflow.get_run(run_id)
        
        # Extract key metrics and parameters
        metrics = run.data.metrics
        params = run.data.params
        tags = run.data.tags
        
        # Create evaluation run in knowledge graph
        self.kg.create_evaluation_run(
            run_id=run_id,
            model_id=params.get('model_id', 'unknown'),
            timestamp=run.info.start_time.isoformat()
        )
        
        # Link metrics and parameters
        self._link_mlflow_artifacts(run_id, metrics, params, tags)
        
    def _link_mlflow_artifacts(self, run_id: str, metrics: Dict, params: Dict, tags: Dict):
        """Link MLflow artifacts to knowledge graph nodes"""
        query = """
        MATCH (run:EvaluationRun {id: $run_id})
        
        // Create metric nodes
        UNWIND keys($metrics) AS metric_name
        CREATE (m:Metric {
            name: metric_name,
            value: $metrics[metric_name]
        })
        CREATE (run)-[rm:HAS_METRIC]->(m)
        
        // Create parameter nodes  
        UNWIND keys($params) AS param_name
        CREATE (p:Parameter {
            name: param_name,
            value: $params[param_name]
        })
        CREATE (run)-[rp:WITH_PARAMETER]->(p)
        
        // Create improvement trajectory
        WITH run, $metrics AS metrics
        MATCH (prev:EvaluationRun {model_id: run.model_id})
        WHERE prev.timestamp < run.timestamp
        WITH run, prev, metrics
        ORDER BY prev.timestamp DESC
        LIMIT 1
        
        CREATE (prev)-[imp:IMPROVES_TO]->(run)
        SET imp.delta_pass1 = metrics.pass1 - prev.metrics.pass1
        SET imp.improvement_areas = $improvement_areas
        
        RETURN run.id
        """
        
        improvement_areas = self._calculate_improvement_areas(metrics, params)
        
        with self.kg.driver.session() as session:
            session.run(query, 
                       run_id=run_id, 
                       metrics=metrics, 
                       params=params,
                       improvement_areas=improvement_areas)

    def _calculate_improvement_areas(self, metrics: Dict, params: Dict) -> List[str]:
        """Calculate key improvement areas from metrics"""
        improvement_areas = []
        
        if metrics.get('correctness', 1) < 0.7:
            improvement_areas.append("correctness")
        if metrics.get('security', 1) < 0.8:
            improvement_areas.append("security")
        if metrics.get('efficiency', 1) < 0.6:
            improvement_areas.append("efficiency")
            
        return improvement_areas

# codeverdict/knowledge_graph/agent_queries.py
from .neo4j_service import CodeVerdictKnowledgeGraph
from typing import List, Dict, Any

class HillClimbingAgentQueries:
    def __init__(self, kg_service: CodeVerdictKnowledgeGraph):
        self.kg = kg_service
        
    def find_high_roi_interventions(self, model_id: str, limit: int = 5):
        """Find highest ROI interventions based on historical patterns"""
        query = """
        // Find failure patterns that are prevalent but easily fixable
        MATCH (run:EvaluationRun {model_id: $model_id})-[:CONTAINS_VERDICT]->(v:Verdict)
        WHERE v.overall_score < 0.7
        MATCH (v)-[:HAS_CRITERION]->(c:Criterion)
        
        WITH c.name AS failure_type, 
             COUNT(v) as failure_count,
             AVG(c.score) as avg_severity,
             COUNT(DISTINCT run) as run_prevalence
        
        // Calculate fixability (inverse of severity and prevalence)
        WITH failure_type, failure_count, avg_severity, run_prevalence,
             (1 - avg_severity) * failure_count as fixability_score
        
        ORDER BY fixability_score DESC
        LIMIT $limit
        
        RETURN failure_type, failure_count, avg_severity, fixability_score
        """
        
        with self.kg.driver.session() as session:
            result = session.run(query, model_id=model_id, limit=limit)
            return [dict(record) for record in result]
    
    def get_improvement_trajectory(self, model_id: str):
        """Get the improvement trajectory for a model"""
        query = """
        MATCH (run:EvaluationRun {model_id: $model_id})
        WITH run
        ORDER BY run.timestamp ASC
        
        RETURN {
            run_id: run.id,
            timestamp: run.timestamp,
            model_id: run.model_id
        } AS trajectory_point
        ORDER BY run.timestamp
        """
        
        with self.kg.driver.session() as session:
            result = session.run(query, model_id=model_id)
            return [record["trajectory_point"] for record in result]
    
    def find_similar_failure_clusters(self, target_failures: List[str]):
        """Find similar failure clusters across models"""
        query = """
        MATCH (v:Verdict)-[:HAS_CRITERION]->(c:Criterion)
        WHERE c.name IN $target_failures AND c.score < 0.6
        
        WITH v, COLLECT(c.name) AS failure_patterns
        MATCH (v)-[:SIMILAR_FAILURE*1..2]-(similar:Verdict)
        
        WITH similar, failure_patterns,
             COUNT(v) AS cluster_size,
             AVG(similar.overall_score) AS cluster_avg_score
        
        WHERE cluster_size > 3  // Significant clusters only
        RETURN similar.status AS cluster_type,
               cluster_size,
               cluster_avg_score,
               COLLECT(DISTINCT similar.completion_id)[0..5] AS example_failures,
               failure_patterns
        ORDER BY cluster_size DESC
        """
        
        with self.kg.driver.session() as session:
            result = session.run(query, target_failures=target_failures)
            return [dict(record) for record in result]

# codeverdict/models/registry.py
import mlflow
import mlflow.pyfunc
from mlflow.tracking import MlflowClient
from typing import List, Dict, Any, Optional
import json
import tempfile
import os
from datetime import datetime

from codeverdict.data.models import CodePrompt

class CodeVerdictRegistry:
    def __init__(self, tracking_uri: str):
        mlflow.set_tracking_uri(tracking_uri)
        self.client = MlflowClient()
        
    def register_prompt_template(self, name: str, template: str, prompt_type: str, version: str = "1.0.0"):
        """Register a prompt template as an MLflow artifact"""
        with tempfile.TemporaryDirectory() as tmp_dir:
            prompt_info = {
                "name": name,
                "template": template,
                "type": prompt_type,
                "version": version,
                "registered_at": datetime.now().isoformat()
            }
            
            prompt_path = os.path.join(tmp_dir, "prompt.json")
            with open(prompt_path, 'w') as f:
                json.dump(prompt_info, f)
            
            with mlflow.start_run(run_name=f"prompt_{name}"):
                mlflow.log_artifact(prompt_path, "prompts")
                mlflow.set_tag("codeverdict.prompt_name", name)
                mlflow.set_tag("codeverdict.prompt_type", prompt_type)
                mlflow.set_tag("codeverdict.version", version)
                
    def register_evaluation_set(self, name: str, prompts: List[CodePrompt], description: str = ""):
        """Register an evaluation set in MLflow"""
        with tempfile.TemporaryDirectory() as tmp_dir:
            eval_set = {
                "name": name,
                "description": description,
                "prompts": [prompt.dict() for prompt in prompts],
                "created_at": datetime.now().isoformat(),
                "size": len(prompts)
            }
            
            eval_path = os.path.join(tmp_dir, "evaluation_set.json")
            with open(eval_path, 'w') as f:
                json.dump(eval_set, f)
            
            with mlflow.start_run(run_name=f"eval_set_{name}"):
                mlflow.log_artifact(eval_path, "evaluation_sets")
                mlflow.set_tag("codeverdict.eval_set_name", name)
                mlflow.set_tag("codeverdict.eval_set_size", len(prompts))
                mlflow.set_tag("codeverdict.type", "evaluation_set")
                
    def register_verdict(self, verdict: Dict[str, Any], model_id: str, prompt_id: str):
        """Register a final verdict in MLflow"""
        with mlflow.start_run(run_name=f"verdict_{verdict['id']}"):
            mlflow.log_params({
                "model_id": model_id,
                "prompt_id": prompt_id,
                "verdict_status": verdict["status"],
                "overall_score": verdict["overall_score"]
            })
            mlflow.log_metrics(verdict.get("criteria_scores", {}))
            mlflow.set_tag("codeverdict.verdict_id", verdict["id"])
            mlflow.set_tag("codeverdict.type", "verdict")

# codeverdict/models/enhanced_registry.py
from .registry import CodeVerdictRegistry
from codeverdict.knowledge_graph.mlflow_neo4j_bridge import MLflowNeo4jBridge
from codeverdict.knowledge_graph.neo4j_service import CodeVerdictKnowledgeGraph
import mlflow
from typing import Dict, Any

class EnhancedCodeVerdictRegistry(CodeVerdictRegistry):
    def __init__(self, tracking_uri: str, neo4j_uri: str, neo4j_auth: tuple):
        super().__init__(tracking_uri)
        self.kg_service = CodeVerdictKnowledgeGraph(neo4j_uri, neo4j_auth[0], neo4j_auth[1])
        self.kg_bridge = MLflowNeo4jBridge(self.kg_service)
    
    def register_verdict_with_insights(self, verdict: Dict[str, Any], model_id: str, prompt_id: str):
        """Enhanced verdict registration with knowledge graph insights"""
        # Original MLflow registration
        super().register_verdict(verdict, model_id, prompt_id)
        
        # Get the current run ID
        current_run = mlflow.active_run()
        if current_run:
            # Bridge to knowledge graph
            self.kg_bridge.track_mlflow_run_to_kg(current_run.info.run_id)
            
            # Generate real-time insights
            insights = self._generate_real_time_insights(verdict, model_id)
            
            # Log insights back to MLflow
            mlflow.log_metrics(insights.get('metrics', {}))
            mlflow.log_params(insights.get('params', {}))
    
    def _generate_real_time_insights(self, verdict: Dict, model_id: str) -> Dict:
        """Generate real-time insights using knowledge graph"""
        from codeverdict.knowledge_graph.agent_queries import HillClimbingAgentQueries
        agent_queries = HillClimbingAgentQueries(self.kg_service)
        
        # Find similar historical patterns
        if verdict['overall_score'] < 0.7:
            failure_types = list(verdict.get('criteria_scores', {}).keys())
            similar_clusters = agent_queries.find_similar_failure_clusters(failure_types)
            
            return {
                'metrics': {
                    'similar_cluster_size': len(similar_clusters) if similar_clusters else 0,
                },
                'params': {
                    'failure_pattern_match': similar_clusters[0]['cluster_type'] if similar_clusters else 'new_pattern'
                }
            }
        return {}

# codeverdict/api/main.py
import uvicorn
from fastapi import FastAPI, HTTPException, BackgroundTasks
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from pydantic import BaseModel
from typing import Dict, Any, List, Optional
import mlflow

from codeverdict.config.settings import settings
from codeverdict.orchestration.workflows import evaluation_pipeline
from codeverdict.data.models import CodePrompt
from codeverdict.models.registry import CodeVerdictRegistry

# Initialize FastAPI app with CodeVerdict branding
app = FastAPI(
    title="CodeVerdict",
    description="⚖️ Where AI Code Stands Trial - Comprehensive AI Code Evaluation Platform",
    version=settings.app_version,
    docs_url="/docs",
    redoc_url="/redoc"
)

# Add CORS middleware
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Initialize components
model_registry = CodeVerdictRegistry(settings.mlflow_tracking_uri)

class EvaluationRequest(BaseModel):
    model_id: str
    eval_set_name: str = "default_code_evals"
    manual_dataset_name: Optional[str] = None

class EvaluationResponse(BaseModel):
    evaluation_id: str
    status: str
    auto_results: int
    manual_tasks: int
    dashboard_url: str

@app.on_event("startup")
async def startup_event():
    """Initialize CodeVerdict services on startup"""
    mlflow.set_tracking_uri(settings.mlflow_tracking_uri)
    
    # Create sample evaluation set for demonstration
    try:
        sample_prompts = [
            CodePrompt(
                prompt="Write a Python function to calculate fibonacci numbers recursively",
                prompt_type="code_generation",
                difficulty="easy",
                tags=["algorithm", "recursion"]
            ),
            CodePrompt(
                prompt="Explain the time complexity of the fibonacci function and suggest optimizations",
                prompt_type="code_explanation", 
                difficulty="medium",
                tags=["complexity", "optimization"]
            ),
            CodePrompt(
                prompt="Find and fix the security vulnerability in this authentication function: [vulnerable code]",
                prompt_type="security_audit",
                difficulty="hard",
                tags=["security", "authentication"]
            )
        ]
        
        model_registry.register_evaluation_set(
            name="codeverdict_sample_evals",
            prompts=sample_prompts,
            description="Sample evaluation set for CodeVerdict demonstration"
        )
        print("✅ CodeVerdict sample evaluation set created")
    except Exception as e:
        print(f"ℹ️ Sample evaluation set may already exist: {e}")

@app.get("/")
async def root():
    """CodeVerdict API root endpoint"""
    return {
        "message": "Welcome to CodeVerdict ⚖️",
        "description": "Where AI Code Stands Trial",
        "version": settings.app_version,
        "endpoints": {
            "evaluate": "/evaluate/{model_id}",
            "results": "/results/{model_id}",
            "health": "/health",
            "docs": "/docs"
        }
    }

@app.get("/health")
async def health_check():
    """Health check endpoint"""
    services = {
        "mlflow": "healthy",
        "argilla": "healthy", 
        "evaluation_engine": "healthy"
    }
    return {"status": "healthy", "services": services}

@app.post("/evaluate", response_model=EvaluationResponse)
async def evaluate_model(request: EvaluationRequest, background_tasks: BackgroundTasks):
    """Trigger CodeVerdict evaluation pipeline for a model"""
    try:
        manual_dataset = request.manual_dataset_name or f"codeverdict_{request.model_id}_{request.eval_set_name}"
        
        # Run evaluation in background
        background_tasks.add_task(
            evaluation_pipeline,
            eval_set_name=request.eval_set_name,
            model_id=request.model_id,
            manual_dataset_name=manual_dataset
        )
        
        return EvaluationResponse(
            evaluation_id=f"eval_{request.model_id}_{request.eval_set_name}",
            status="started",
            auto_results=0,  # Would be updated from actual run
            manual_tasks=0,  # Would be updated from actual run
            dashboard_url=f"http://localhost:6900/datasets/codeverdict/{manual_dataset}/"
        )
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"CodeVerdict evaluation failed: {str(e)}")

@app.get("/results/{model_id}")
async def get_evaluation_results(model_id: str):
    """Get CodeVerdict evaluation results for a model"""
    # This would query your results database
    return {
        "model_id": model_id,
        "status": "evaluation_complete",
        "summary": {
            "total_prompts": 50,
            "auto_evaluated": 25,
            "manual_reviewed": 25,
            "average_score": 0.82,
            "verdict_distribution": {
                "auto_approved": 20,
                "manual_approved": 23,
                "rejected": 7
            }
        },
        "dashboard": {
            "mlflow": "http://localhost:5000",
            "argilla": "http://localhost:6900"
        }
    }

@app.get("/prompts")
async def list_evaluation_sets():
    """List available evaluation sets in CodeVerdict"""
    return {
        "evaluation_sets": [
            {
                "name": "codeverdict_sample_evals",
                "description": "Sample evaluation set for CodeVerdict",
                "size": 3,
                "types": ["code_generation", "code_explanation", "security_audit"]
            }
        ]
    }

if __name__ == "__main__":
    uvicorn.run(
        "codeverdict.api.main:app",
        host="0.0.0.0",
        port=8000,
        reload=True,
        log_config=None
    )

# codeverdict/orchestration/workflows.py
from typing import List, Dict, Any
import asyncio
from codeverdict.data.models import CodeCompletion
from codeverdict.evaluation.triage_engine import CodeVerdictTriageEngine
from codeverdict.evaluation.auto_evaluator import CodeVerdictAutoEvaluator
from codeverdict.evaluation.manual_evaluator import CodeVerdictManualEvaluator
from codeverdict.models.registry import CodeVerdictRegistry
import mlflow

async def evaluation_pipeline(eval_set_name: str, model_id: str, manual_dataset_name: str):
    """Main CodeVerdict evaluation pipeline"""
    print(f"🚀 Starting CodeVerdict evaluation for model: {model_id}")
    
    # Initialize components
    triage_engine = CodeVerdictTriageEngine()
    auto_evaluator = CodeVerdictAutoEvaluator()
    manual_evaluator = CodeVerdictManualEvaluator()
    registry = CodeVerdictRegistry(mlflow.get_tracking_uri())
    
    # Step 1: Generate or load completions
    completions = await _generate_completions(eval_set_name, model_id)
    print(f"📝 Generated {len(completions)} completions for evaluation")
    
    # Step 2: Triage completions
    auto_batch, manual_batch = triage_engine.triage_completions(completions)
    
    # Step 3: Auto-evaluate batch
    auto_results = []
    for completion in auto_batch:
        quality_scores = auto_evaluator.evaluate_code_quality(completion)
        verdict = auto_evaluator.generate_verdict(completion, quality_scores)
        auto_results.append(verdict)
        
        # Register verdict
        registry.register_verdict(verdict, model_id, completion.prompt_id)
    
    # Step 4: Setup manual evaluation
    manual_dataset = manual_evaluator.setup_code_quality_dataset(manual_dataset_name)
    manual_evaluator.add_completions_for_review(manual_dataset, manual_batch)
    manual_evaluator.push_to_argilla(manual_dataset, manual_dataset_name)
    
    # Step 5: Log summary
    with mlflow.start_run(run_name=f"eval_summary_{model_id}"):
        mlflow.log_params({
            "model_id": model_id,
            "eval_set": eval_set_name,
            "total_completions": len(completions),
            "auto_evaluated": len(auto_batch),
            "manual_review": len(manual_batch)
        })
        mlflow.log_metrics({
            "auto_approval_rate": len([r for r in auto_results if r["status"] == "auto_approved"]) / len(auto_results) if auto_results else 0,
            "average_auto_score": sum(r["overall_score"] for r in auto_results) / len(auto_results) if auto_results else 0
        })
    
    print(f"✅ CodeVerdict evaluation complete! Auto: {len(auto_results)}, Manual: {len(manual_batch)}")
    return {
        "auto_results": auto_results,
        "manual_tasks": len(manual_batch),
        "dashboard_url": f"http://localhost:6900/datasets/codeverdict/{manual_dataset_name}/"
    }

async def _generate_completions(eval_set_name: str, model_id: str) -> List[CodeCompletion]:
    """Generate completions for evaluation (placeholder implementation)"""
    # In a real implementation, this would call your model API
    # For now, we'll return sample completions
    
    sample_prompts = [
        {
            "id": "prompt_1",
            "text": "Write a function to reverse a string",
            "type": "code_generation",
            "difficulty": "easy"
        },
        {
            "id": "prompt_2", 
            "text": "Write a function to check if a number is prime",
            "type": "code_generation",
            "difficulty": "medium"
        },
        {
            "id": "prompt_3",
            "text": "Find the security vulnerability in this code: [code snippet]",
            "type": "security_audit", 
            "difficulty": "hard"
        }
    ]
    
    completions = []
    for prompt in sample_prompts:
        completion = CodeCompletion(
            prompt_id=prompt["id"],
            model_id=model_id,
            completion=_generate_sample_completion(prompt["text"]),
            metadata={
                "prompt_text": prompt["text"],
                "prompt_type": prompt["type"],
                "difficulty": prompt["difficulty"]
            }
        )
        completions.append(completion)
    
    return completions

def _generate_sample_completion(prompt: str) -> str:
    """Generate sample completion for demonstration"""
    if "reverse" in prompt.lower():
        return """
def reverse_string(s):
    return s[::-1]
"""
    elif "prime" in prompt.lower():
        return """
def is_prime(n):
    if n < 2:
        return False
    for i in range(2, int(n**0.5) + 1):
        if n % i == 0:
            return False
    return True
"""
    else:
        return """
# Security analysis: The code uses eval() which is dangerous
# Recommendation: Replace eval() with ast.literal_eval()
# or implement proper input validation
"""

# codeverdict/utils/helpers.py
import json
from typing import Dict, Any, List
from datetime import datetime

def calculate_pass_at_k(n: int, c: int, k: int) -> float:
    """
    Calculate Pass@k metric
    
    Args:
        n: Total number of samples
        c: Number of correct samples  
        k: k for Pass@k
        
    Returns:
        Pass@k score
    """
    if n - c < k:
        return 1.0
    return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))

def cluster_failures_by_pattern(failures: List[Dict]) -> Dict[str, List]:
    """Cluster failures by common patterns"""
    clusters = {
        "syntax_errors": [],
        "logical_errors": [], 
        "security_issues": [],
        "efficiency_problems": [],
        "style_violations": []
    }
    
    for failure in failures:
        scores = failure.get("criteria_scores", {})
        
        if scores.get("security", 1) < 0.5:
            clusters["security_issues"].append(failure)
        elif scores.get("correctness", 1) < 0.5:
            clusters["logical_errors"].append(failure)
        elif scores.get("efficiency", 1) < 0.5:
            clusters["efficiency_problems"].append(failure)
        elif scores.get("style_adherence", 1) < 0.5:
            clusters["style_violations"].append(failure)
        else:
            clusters["syntax_errors"].append(failure)
            
    return clusters

def generate_improvement_report(verdicts: List[Dict], model_id: str) -> Dict[str, Any]:
    """Generate improvement report from verdicts"""
    failed_verdicts = [v for v in verdicts if v["overall_score"] < 0.7]
    clusters = cluster_failures_by_pattern(failed_verdicts)
    
    return {
        "model_id": model_id,
        "total_evaluations": len(verdicts),
        "success_rate": len([v for v in verdicts if v["overall_score"] >= 0.7]) / len(verdicts),
        "failure_clusters": {
            cluster: len(items) for cluster, items in clusters.items()
        },
        "recommendations": _generate_recommendations(clusters),
        "generated_at": datetime.now().isoformat()
    }

def _generate_recommendations(clusters: Dict) -> List[str]:
    """Generate improvement recommendations from failure clusters"""
    recommendations = []
    
    if clusters["security_issues"]:
        recommendations.append("Focus on security training with sanitized input examples")
    if clusters["logical_errors"]:
        recommendations.append("Add more edge case testing and validation examples") 
    if clusters["efficiency_problems"]:
        recommendations.append("Include algorithmic complexity and optimization training")
    if clusters["syntax_errors"]:
        recommendations.append("Reinforce language syntax and basic constructs")
        
    return recommendations

# codeverdict/tests/test_evaluation.py
import pytest
from codeverdict.evaluation.auto_evaluator import CodeVerdictAutoEvaluator
from codeverdict.data.models import CodeCompletion

def test_auto_evaluator_initialization():
    evaluator = CodeVerdictAutoEvaluator()
    assert evaluator is not None

def test_code_quality_evaluation():
    evaluator = CodeVerdictAutoEvaluator()
    completion = CodeCompletion(
        prompt_id="test_1",
        model_id="test_model",
        completion="def hello():\n    return 'world'"
    )
    
    scores = evaluator.evaluate_code_quality(completion)
    assert "readability" in scores
    assert "security" in scores
    assert all(0 <= score <= 1 for score in scores.values())

# codeverdict/tests/test_triage.py
import pytest
from codeverdict.evaluation.triage_engine import CodeVerdictTriageEngine
from codeverdict.data.models import CodeCompletion

def test_triage_engine_initialization():
    engine = CodeVerdictTriageEngine()
    assert engine.auto_eval_threshold == 0.8
    assert engine.manual_sample_rate == 0.5

def test_triage_decision_making():
    engine = CodeVerdictTriageEngine()
    completion = CodeCompletion(
        prompt_id="test_1", 
        model_id="test_model",
        completion="test code",
        metadata={"prompt_type": "code_generation"}
    )
    
    decision = engine._make_triage_decision(completion)
    assert decision in engine.TriageDecision

# codeverdict/tests/test_knowledge_graph.py
import pytest
from codeverdict.knowledge_graph.neo4j_service import CodeVerdictKnowledgeGraph

def test_kg_initialization():
    # This would test knowledge graph connection
    # For now, just a placeholder
    pass