Propose PR #807 conflict resolution on the dedicated runner

.github/actions/install-agent-harnesses/action.yml

@@ -9,5 +9,5 @@ runs:
       shell: pwsh
 
     - name: Install GitHub Copilot CLI
-      run: npm install -g @github/copilot@1.0.80
+      run: npm install -g @github/copilot@1.0.79
       shell: pwsh

.github/workflows/copilot-evaluation.yml

@@ -139,23 +139,7 @@ jobs:
       - name: Install evaluation CLIs
         uses: $/.github/actions/install-agent-harnesses
 
-      - name: Run code-review engine for entry ${{ matrix.entry }}
-        if: ${{ inputs.category == 'code-review' }}
-        timeout-minutes: 120
-        shell: pwsh
-        env:
-          COPILOT_GITHUB_TOKEN: ${{ github.token }}
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          Write-Output "::add-mask::$env:COPILOT_GITHUB_TOKEN"
-
-          uv run bcbench evaluate code-review "${{ matrix.entry }}" `
-            --model "${{ inputs.model }}" `
-            --repo-path "${{ steps.setup-env.outputs.repo_path }}" `
-            --output-dir "${{ env.EVALUATION_RESULTS_DIR }}"
-
       - name: Run GitHub Copilot CLI for entry ${{ matrix.entry }}
-        if: ${{ inputs.category != 'code-review' }}
         timeout-minutes: 120
         shell: pwsh
         env:
@@ -189,7 +173,7 @@ jobs:
     with:
       results-dir: ${{ needs.evaluate-with-copilot-cli.outputs.results-dir }}
       model: ${{ inputs.model }}
-      agent: ${{ inputs.category == 'code-review' && 'BC PR Review' || 'GitHub Copilot CLI' }}
+      agent: "GitHub Copilot CLI"
       mock: ${{ inputs.test-run }}
       category: ${{ inputs.category }}
       git-ref: ${{ inputs.git-ref || github.ref_name }}

.github/workflows/pr-review-evaluation.yml

@@ -0,0 +1,171 @@
+name: Evaluation with BC PR Review
+permissions:
+  contents: read
+  actions: write
+
+on:
+  workflow_dispatch:
+    inputs:
+      model:
+        description: "Copilot model used internally by BC PR Review"
+        required: false
+        default: "gpt-5.6-luna"
+        type: choice
+        options:
+          - "claude-sonnet-5"
+          - "claude-opus-5"
+          - "gpt-5.6-sol"
+          - "gpt-5.6-terra"
+          - "gpt-5.6-luna"
+          - "gpt-5.3-codex"
+          - "mai-code-1.1-flash"
+          - "gemini-3.6-flash"
+      test-run:
+        description: "Indicate this is a test run (with few entries)"
+        required: false
+        default: true
+        type: boolean
+      repeat:
+        description: "Number of times to run sequentially (ignored for test runs)"
+        required: false
+        default: "1"
+        type: choice
+        options:
+          - "1"
+          - "2"
+          - "3"
+          - "4"
+          - "5"
+      git-ref:
+        description: "Branch to record for experiment tracking (auto-filled on requeue; leave blank for manual runs)"
+        required: false
+        default: ""
+        type: string
+
+concurrency:
+  group: pr-review-evaluation-${{ inputs.test-run && 'test' || 'full' }}
+  cancel-in-progress: false
+
+env:
+  EVALUATION_RESULTS_DIR: evaluation_results
+
+jobs:
+  pin-commit:
+    uses: $/.github/workflows/pin-evaluation-commit.yml
+    permissions:
+      contents: write
+    with:
+      agent: pr-review
+      test-run: ${{ inputs.test-run }}
+      repeat: ${{ inputs.repeat }}
+
+  get-entries:
+    uses: $/.github/workflows/get-entries.yml
+    with:
+      test-run: ${{ inputs.test-run }}
+      category: code-review
+
+  evaluate-with-pr-review:
+    runs-on: ${{ needs.get-entries.outputs.runner }}
+    needs: get-entries
+    outputs:
+      results-dir: ${{ env.EVALUATION_RESULTS_DIR }}
+    if: needs.get-entries.outputs.entries != '[]'
+    environment:
+      name: ado-read
+      deployment: false
+    permissions:
+      contents: read
+      id-token: write
+      copilot-requests: write
+    name: ${{ matrix.entry }}
+    strategy:
+      fail-fast: false
+      max-parallel: 64
+      matrix:
+        entry: ${{ fromJson(needs.get-entries.outputs.entries) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      - name: Setup Repository
+        id: setup-env
+        timeout-minutes: 40
+        uses: $/.github/actions/setup-bc-container-repo
+        with:
+          instance-id: ${{ matrix.entry }}
+          category: code-review
+          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          skip-container: true
+          skip-repo: false
+
+      - name: Setup Python with UV
+        uses: $/.github/actions/setup-python-uv
+
+      - uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: Install evaluation CLIs
+        uses: $/.github/actions/install-agent-harnesses
+
+      - name: Checkout BC-ALAgents review engine
+        uses: actions/checkout@v5
+        with:
+          repository: microsoft/BC-ALAgents
+          ref: 533dd39dfe29218c09e5e31c39c78bb72fa20aa2
+          path: bc-alagents-engine
+          token: ${{ github.token }}
+
+      - name: Run BC PR Review for entry ${{ matrix.entry }}
+        timeout-minutes: 120
+        shell: pwsh
+        env:
+          COPILOT_GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          Write-Output "::add-mask::$env:COPILOT_GITHUB_TOKEN"
+
+          uv run bcbench evaluate pr-review "${{ matrix.entry }}" `
+            --model "${{ inputs.model }}" `
+            --engine-path "${{ github.workspace }}/bc-alagents-engine" `
+            --repo-path "${{ steps.setup-env.outputs.repo_path }}" `
+            --output-dir "${{ env.EVALUATION_RESULTS_DIR }}"
+
+      - name: Upload evaluation results
+        uses: actions/upload-artifact@v6
+        if: always()
+        with:
+          name: evaluation-results-${{ github.run_id }}-${{ matrix.entry }}
+          path: ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl
+          retention-days: ${{ inputs.test-run && 1 || 30 }}
+
+  summarize-results:
+    needs: evaluate-with-pr-review
+    uses: $/.github/workflows/summarize-results.yml
+    permissions:
+      contents: write
+      id-token: write
+    with:
+      results-dir: ${{ needs.evaluate-with-pr-review.outputs.results-dir }}
+      model: ${{ inputs.model }}
+      agent: "BC PR Review"
+      mock: ${{ inputs.test-run }}
+      category: code-review
+      git-ref: ${{ inputs.git-ref || github.ref_name }}
+    secrets: inherit
+
+  requeue:
+    needs: [summarize-results, pin-commit]
+    if: ${{ !cancelled() && !failure() && !inputs.test-run }}
+    uses: $/.github/workflows/requeue-evaluation.yml
+    permissions:
+      contents: write
+      actions: write
+    with:
+      workflow-file: pr-review-evaluation.yml
+      repeat: ${{ inputs.repeat }}
+      existing-tag: ${{ needs.pin-commit.outputs.tag-name }}
+      workflow-inputs: |
+        {"model": "${{ inputs.model }}", "test-run": "${{ inputs.test-run }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}

CONTRIBUTING.md

@@ -133,6 +133,13 @@ Keep evaluation tools pinned so benchmark runs remain reproducible. For example,
 4. Run focused tests for the integration, then perform a test evaluation for tools exposed to the agent.
 5. Bump the benchmark version according to the Versioning Policy. Tool changes that may affect evaluation results normally require a minor bump.
 
+### Bump the BC PR Review engine
+
+1. Update the pinned `microsoft/BC-ALAgents` commit in `.github/workflows/pr-review-evaluation.yml`
+2. Run a test evaluation through the `pr-review` workflow
+3. Bump the BC-Bench version following the Versioning Policy
+4. Include the exact BC-ALAgents commit SHA in the BC-Bench release notes
+
 ### Create a new release
 
 After you bump the version in [pyproject.toml](https://github.com/microsoft/BC-Bench/blob/main/pyproject.toml#L7) following the Versioning Policy, use the repository's [`create-release` skill](.github/skills/create-release/SKILL.md) to prepare release notes after pushing your changes. The skill screens merged PRs since the previous version tag and returns Markdown covering only changes that may affect evaluation results, without creating the tag or release.

README.md

@@ -31,6 +31,10 @@ The [GitHub Copilot CLI](https://github.com/github/copilot-cli) supports MCP ser
 
 [Claude Code](https://docs.anthropic.com/en/docs/claude-code) is Anthropic's agentic coding tool. It supports MCP servers, custom system prompts, and agent mode. BC-Bench integrates with Claude Code using the same shared configuration as Copilot.
 
+### BC PR Review
+
+BC PR Review is the production-fidelity BC-ALAgents + BCQuality runner for the `code-review` category. It remains separate from the category contract so its results can be compared with GitHub Copilot CLI and Claude Code on the same dataset and scorer.
+
 ## Getting Started
 
 BC-Bench is open source, and you're welcome to fork and adapt it for your own use. We are not accepting external contributions in this repository at this time. You can run evaluations locally and replace the dataset under `dataset/` with tasks from your own codebase.