Add Chrome support to autowebcompat repro agent

agents/autowebcompat-repro/Dockerfile

@@ -28,17 +28,22 @@ ENV PATH="/opt/venv/bin:$PATH"
 
 FROM base AS agent
 
-# The Firefox DevTools MCP server is an npm package launched via `npx`, so the
-# agent image needs Node.js + npm (the python base ships neither). It also
-# needs the shared libraries Firefox requires to run headless; the Firefox
-# binary itself is downloaded at agent startup (a fresh Nightly per run) via
-# mozdownload/mozinstall, not baked in here.
+# The Firefox and Chrome DevTools MCP servers are npm packages launched via
+# `npx`, so the agent image needs Node.js + npm (the python base ships neither).
+# It also needs the shared libraries the browsers require to run headless. The
+# browser binaries themselves are downloaded at agent startup (a fresh build per
+# run): Firefox via mozdownload/mozinstall and Chrome for Testing via the
+# Chrome-for-Testing JSON API (see browser.py). The first `lib*` group below is
+# Firefox's runtime deps; the rest are the extra deps headless Chrome needs.
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
         nodejs npm \
         ca-certificates \
-        libgtk-3-0 libdbus-glib-1-2 libx11-xcb1 libxtst6 libxt6 \
-        libasound2 libpci3 \
+        libgtk-3-0 libdbus-glib-1-2 libx11-xcb1 libxtst6 libxt6 libpci3 \
+        libnss3 libnspr4 libatk1.0-0t64 libatk-bridge2.0-0t64 libatspi2.0-0t64 \
+        libcups2t64 libdbus-1-3 libgbm1 libxcomposite1 libxdamage1 libxfixes3 \
+        libxrandr2 libxkbcommon0 libasound2t64 libpango-1.0-0 libcairo2 \
+        fonts-liberation \
     && rm -rf /var/lib/apt/lists/*
 
 # hackbot.toml lives at the agent root (not inside the package), so copy it into

agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/agent.py

@@ -28,8 +28,9 @@
 from hackbot_runtime.claude import Reporter
 from pydantic import BaseModel
 
-from .browser import FirefoxBrowsers
-from .config import BUGZILLA_READ_TOOLS, DEVTOOLS_TOOLS
+from .browser import ChromeBrowsers, FirefoxBrowsers
+from .chrome_devtools_mcp import build_chrome_devtools_server
+from .config import BUGZILLA_READ_TOOLS, CHROME_DEVTOOLS_TOOLS, DEVTOOLS_TOOLS
 from .devtools_mcp import build_devtools_server
 from .result import (
     RESULT_SERVER_NAME,
@@ -82,6 +83,7 @@ class AutowebcompatReproResult(BaseModel):
     plan_result: TestPlanResult
     reproductions: list[tuple[str, BugReproductionResult | ReproductionResult]]
     chrome_mask_fixed: bool | None
+    chrome_reproduced: bool | None
 
 
 @dataclass
@@ -317,9 +319,13 @@ def __init__(
         input_data: AutoWebcompatInput,
         bugzilla_mcp_server: McpServerConfig,
         screenshot_dir: Path,
+        chrome_path: Path | None = None,
     ):
         super().__init__(task_config, run_tracker)
         self.input_data = input_data
+        # Chrome cross-check is only added when a Chrome binary is supplied
+        # (the initial nightly attempt). Later channel attempts run Firefox-only.
+        self.cross_check_chrome = chrome_path is not None
         self.screenshot_path = make_empty_temp_file(
             screenshot_dir, "reproduction=", ".png"
         )
@@ -334,29 +340,64 @@ def __init__(
             ),
             DEVTOOLS_TOOLS,
         )
+        # The initial reproduction also drives Chrome, so the agent can
+        # cross-check both browsers in one context: if the same steps behave
+        # identically in Firefox and Chrome it can iterate on whether the steps
+        # are wrong versus it genuinely not being a compat issue.
+        if chrome_path is not None:
+            self.add_mcp_server(
+                "chrome-devtools",
+                build_chrome_devtools_server(chrome_path=chrome_path, headless=True),
+                CHROME_DEVTOOLS_TOOLS,
+            )
         if self.input_data.type == "bug_id" != None:
             self.add_mcp_server("bugzilla", bugzilla_mcp_server, BUGZILLA_READ_TOOLS)
 
     def subject(self) -> Any:
         return self.input_data.subject()
 
     def system_prompt(self) -> str:
+        if self.cross_check_chrome:
+            chrome_step = """
+3. Cross-check in Chrome: run the same steps in Chrome using the Chrome DevTools
+   MCP. A genuine web-compat issue reproduces in Firefox but not in in Chrome.
+   - If the behavior is identical in both browsers, your steps may be wrong or
+     this may not be a compat issue. Iterate: refine the steps and re-check both
+     browsers before concluding. Use the difference (or lack of it) between the
+     two browsers to decide whether you have really reproduced the reported
+     issue.
+   - Set `reproduced` for the Firefox outcome and `chrome_reproduced` for the
+     Chrome outcome (`true` if the issue also reproduces in Chrome, `false` if
+     the issue is not reproducible in Chrome)."""
+        else:
+            chrome_step = """
+3. No Chrome cross-check is available for this task: leave `chrome_reproduced`
+   null."""
+
         return (
             super()
             .system_prompt()
             .format(
                 task_details=f"""
 1. Identify the affected URL and the described broken behavior.
-2. Baseline: Navigate to the URL with the Firefox DevTools MCP and
-   try to reproduce the described broken behaviour.
-3. If the issue reproduces AND the breakage is visual in nature (incorrect
-   layout or rendering, not broken interaction), capture a screenshot showing
-   it: call `screenshot_page` with `saveTo` set to `{self.screenshot_path}`.
-   This writes the image to that file instead of returning it — do not capture
-   or paste the image data yourself. Then set `screenshot_path` in your result
-   to exactly `{self.screenshot_path}`. For non-visual issues, take no
-   screenshot and leave `screenshot_path` null.
-4. Submit your findings via `submit_result` (see "Reporting your result").
+2. Baseline: Navigate to the URL with the Firefox DevTools MCP
+   and try to reproduce the described broken behaviour.
+   - Reproduce against the actual reported site. If you cannot reach or
+     reproduce on that site — e.g. it is behind a login wall, blocked,
+     gated by a captcha, or down — do not substitute a reduced testcase,
+     minimal example, or other bug attachment as a stand-in reproduction.
+     Report `reproduced` as false with the appropriate `failure_reason`. A
+     testcase may inform your analysis, but reproducing on it does not count
+     as reproducing the reported site issue.
+{chrome_step}
+4. If the issue reproduces AND the breakage is visual in nature (incorrect
+   layout or rendering, not broken interaction), capture a screenshot in Firefox
+   showing it: call `screenshot_page` with `saveTo` set to
+   `{self.screenshot_path}`. This writes the image to that file instead of
+   returning it — do not capture or paste the image data yourself. Then set
+   `screenshot_path` in your result to exactly `{self.screenshot_path}`. For
+   non-visual issues, take no screenshot and leave `screenshot_path` null.
+5. Submit your findings via `submit_result` (see "Reporting your result").
 """
             )
         )
@@ -502,6 +543,7 @@ def __init__(self, publish_file: PublishFile, plan_result: TestPlanResult):
         ] = {}
         self.initial_repro: InitialReproduction | None = None
         self.chrome_mask_fixed: bool | None = None
+        self.chrome_reproduced: bool | None = None
 
     @property
     def reproduced(self) -> bool:
@@ -551,12 +593,19 @@ def set_result(
         key = (channel, extra)
         if key in self.results:
             raise ValueError(f"Got duplicate results for {channel}, {extra}")
-        if isinstance(result, BugReproductionResult) and result.reproduced:
-            if self.initial_repro is not None:
-                raise ValueError("Got duplicate steps / summary")
-            self.initial_repro = InitialReproduction(
-                channel, result.steps, result.summary, result.screenshot_path
-            )
+        if isinstance(result, BugReproductionResult):
+            if result.reproduced:
+                if self.initial_repro is not None:
+                    raise ValueError("Got duplicate steps / summary")
+                self.initial_repro = InitialReproduction(
+                    channel, result.steps, result.summary, result.screenshot_path
+                )
+            # Only the Chrome-enabled (initial nightly) reproduction reports a
+            # Chrome verdict; other attempts leave it null.
+            if result.chrome_reproduced is not None:
+                if self.chrome_reproduced is not None:
+                    raise ValueError("Got duplicate chrome cross-check results")
+                self.chrome_reproduced = result.chrome_reproduced
         elif isinstance(result, ChromeMaskResult):
             if self.chrome_mask_fixed is not None:
                 raise ValueError("Got duplicate results for chrome mask")
@@ -578,6 +627,7 @@ def into_result(self) -> AutowebcompatReproResult:
                 if isinstance(value, ReproductionResult)
             ],
             chrome_mask_fixed=self.chrome_mask_fixed,
+            chrome_reproduced=self.chrome_reproduced,
         )
 
 
@@ -594,6 +644,7 @@ async def run_autowebcompat_repro(
     :class:`AgentError` if the agent ends in an error.
     """
     firefox_browser = FirefoxBrowsers()
+    chrome_browser = ChromeBrowsers()
 
     test_plan_task = TestPlan(default_config, tracker, input_data, bugzilla_mcp_server)
     test_plan_result = await test_plan_task.run()
@@ -614,6 +665,7 @@ async def next_repro_task(
         channel: FirefoxChannel,
         extra: str | None = None,
         config: TaskConfig = default_config,
+        cross_check_chrome: bool = False,
     ) -> None:
         browser = getattr(firefox_browser, channel.value)
         profile = setup_profile(browser)
@@ -626,6 +678,9 @@ async def next_repro_task(
                 input_data,
                 bugzilla_mcp_server,
                 screenshots_dir,
+                # Only cross-check Chrome when asked (the initial nightly
+                # attempt); later channel attempts stay Firefox-only.
+                chrome_path=chrome_browser.stable if cross_check_chrome else None,
             )
         else:
             task = StepsReproduction(
@@ -644,8 +699,8 @@ async def next_repro_task(
 
     screenshots_dir = Path(tempfile.mkdtemp(prefix="autowebcompat-screenshots-"))
 
-    # Always try in nightly first
-    await next_repro_task(FirefoxChannel.nightly)
+    # Always try in nightly first, cross-checking in Chrome.
+    await next_repro_task(FirefoxChannel.nightly, cross_check_chrome=True)
 
     if not repro_results.reproduced and test_plan_result.affects_platforms == [
         "android"

agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/browser.py

@@ -1,14 +1,24 @@
 import logging
 import platform
+import stat
 import tempfile
+import zipfile
 from pathlib import Path
 from typing import Literal
 
 import mozdownload
 import mozinstall
+import requests
 
 logger = logging.getLogger("autowebcompat-repro")
 
+CHROME_VERSIONS_URL = (
+    "https://googlechromelabs.github.io/chrome-for-testing/"
+    "last-known-good-versions-with-downloads.json"
+)
+CHROME_DOWNLOAD_TIMEOUT = 120
+CHROME_CHUNK_SIZE = 1 << 20
+
 
 def install_firefox(
     channel: Literal["nightly"] | Literal["stable"] | Literal["esr"],
@@ -72,3 +82,82 @@ def esr(self) -> Path:
         if self._esr is None:
             self._esr = install_firefox(channel="esr")
         return self._esr
+
+
+def chrome_platform() -> str:
+    """Chrome for Testing platform string for the current host."""
+    system = platform.system()
+    machine = platform.machine().lower()
+    if system == "Linux":
+        if machine not in {"x86_64", "amd64"}:
+            raise RuntimeError(
+                "Chrome for Testing has no linux build for "
+                f"{platform.machine()}; only x86_64/amd64 is supported. Run the "
+                "agent image as linux/amd64, e.g. DOCKER_DEFAULT_PLATFORM=linux/amd64."
+            )
+        return "linux64"
+    if system == "Darwin":
+        return "mac-arm64" if machine in {"arm64", "aarch64"} else "mac-x64"
+    if system == "Windows":
+        return "win64" if machine in {"x86_64", "amd64"} else "win32"
+    raise RuntimeError(f"Unsupported platform for Chrome for Testing: {system}")
+
+
+def resolve_chrome_download_url(channel: str, cft_platform: str) -> str:
+    """Look up the Chrome for Testing download URL for a channel + platform."""
+    response = requests.get(CHROME_VERSIONS_URL, timeout=CHROME_DOWNLOAD_TIMEOUT)
+    response.raise_for_status()
+    data = response.json()
+
+    entry = data["channels"][channel.capitalize()]
+    logger.info("Chrome for Testing %s: version %s", channel, entry["version"])
+
+    for download in entry["downloads"]["chrome"]:
+        if download["platform"] == cft_platform:
+            return download["url"]
+
+    raise RuntimeError(
+        f"no Chrome for Testing '{cft_platform}' download in {channel} channel"
+    )
+
+
+def install_chrome(channel: Literal["stable"] = "stable") -> Path:
+    """Download Chrome for Testing and return the browser binary path."""
+    cft_platform = chrome_platform()
+    install_dir = Path(tempfile.mkdtemp(prefix=f"chrome-{channel}-", dir=Path.home()))
+
+    url = resolve_chrome_download_url(channel, cft_platform)
+    archive = install_dir / f"chrome-{cft_platform}.zip"
+
+    logger.info("downloading Chrome for Testing from %s", url)
+    with requests.get(url, stream=True, timeout=CHROME_DOWNLOAD_TIMEOUT) as response:
+        response.raise_for_status()
+        with archive.open("wb") as out:
+            for chunk in response.iter_content(chunk_size=CHROME_CHUNK_SIZE):
+                if chunk:
+                    out.write(chunk)
+
+    with zipfile.ZipFile(archive) as zf:
+        zf.extractall(install_dir)
+    archive.unlink()
+
+    binary = install_dir / f"chrome-{cft_platform}" / "chrome"
+    if not binary.exists():
+        raise RuntimeError(f"Chrome binary not found at {binary} after unpacking")
+
+    # zipfile does not preserve the executable bit; restore it.
+    binary.chmod(binary.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
+
+    logger.info("installed Chrome at %s", binary)
+    return binary
+
+
+class ChromeBrowsers:
+    def __init__(self) -> None:
+        self._stable: Path | None = None
+
+    @property
+    def stable(self) -> Path:
+        if self._stable is None:
+            self._stable = install_chrome(channel="stable")
+        return self._stable

agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/chrome_devtools_mcp.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from claude_agent_sdk.types import McpStdioServerConfig
+
+PACKAGE = "chrome-devtools-mcp@latest"
+
+
+def build_chrome_devtools_server(
+    chrome_path: Path | None = None,
+    *,
+    headless: bool = True,
+    no_sandbox: bool = True,
+) -> McpStdioServerConfig:
+    """Build the stdio config for the Chrome DevTools MCP server.
+
+    Args:
+        chrome_path: Chrome binary to drive (the Chrome for Testing build from
+            ``browser.install_chrome``). When ``None`` the server lets its
+            bundled Puppeteer discover a Chrome installation itself.
+        headless: Run Chrome without a visible window (required in
+            container/CI environments).
+        no_sandbox: Pass ``--no-sandbox`` to Chrome. Required when running as an
+            unprivileged user inside a container, where Chrome's setuid sandbox
+            cannot initialize and the browser otherwise fails to launch.
+    """
+    args = ["-y", PACKAGE]
+    if headless:
+        args.append("--headless")
+    if chrome_path is not None:
+        args += ["--executablePath", str(chrome_path)]
+
+    # Opt out of the MCP server's own data collection: its usage statistics and
+    # the CrUX API calls that send performance-trace URLs to Google. This does
+    # not touch Chrome's own behavior, only what the MCP server itself reports.
+    args += ["--usageStatistics=false", "--performanceCrux=false"]
+
+    if no_sandbox:
+        args += ["--chromeArg=--no-sandbox", "--chromeArg=--disable-setuid-sandbox"]
+
+    return McpStdioServerConfig(command="npx", args=args)