name: Run prompt evals on pull request

# The job only starts when at least one *.prompt.yml file is touched.
on:
  pull_request:
    paths:
      - '**/*.prompt.yml'

jobs:
  evaluate-model:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Setup gh-models
        run: gh extension install github/gh-models
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Find changed prompt files
        id: find-prompts
        run: |
          # Get the list of changed files that match *.prompt.yml pattern
          changed_prompts=$(git diff --name-only origin/${{ github.base_ref }}..HEAD | grep '\.prompt\.yml$' | head -1)
          
          if [[ -z "$changed_prompts" ]]; then
            echo "No prompt files found in the changes"
            exit 1
          fi
          
          echo "first_prompt=$changed_prompts" >> "$GITHUB_OUTPUT"
          echo "Found changed prompt file: $changed_prompts"

      - name: Run model evaluation
        id: eval
        run: |
          set -e
          PROMPT_FILE="${{ steps.find-prompts.outputs.first_prompt }}"
          echo "## Model Evaluation Results" >> "$GITHUB_STEP_SUMMARY"
          echo "Evaluating: $PROMPT_FILE" >> "$GITHUB_STEP_SUMMARY"
          echo "" >> "$GITHUB_STEP_SUMMARY"

          if gh models eval "$PROMPT_FILE" > eval_output.txt 2>&1; then
            echo "✅ All evaluations passed!"   >> "$GITHUB_STEP_SUMMARY"
            cat  eval_output.txt               >> "$GITHUB_STEP_SUMMARY"
            echo "eval_status=success"         >> "$GITHUB_OUTPUT"
          else
            echo "❌ Some evaluations failed!" >> "$GITHUB_STEP_SUMMARY"
            cat  eval_output.txt               >> "$GITHUB_STEP_SUMMARY"
            echo "eval_status=failure"         >> "$GITHUB_OUTPUT"
            exit 1
          fi
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Comment on PR (if evaluation failed)
        if: failure() && github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const output = fs.readFileSync('eval_output.txt', 'utf8');
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner:        context.repo.owner,
              repo:         context.repo.repo,
              body: `## 🚨 Model Evaluation Failed

              \`\`\`
              ${output}
              \`\`\`

              Please review the evaluation results and fix any issues before merging.`
            });