From daf92b52f6280559ac9194ab1b7d17da776afaee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 19 Sep 2026 15:30:20 +0200 Subject: [PATCH 1/8] refactor(ios-runner): give the host DevToolsSecurity probe its own module runner-session.ts is 1,181 lines, so the next behavior owed a split first. The host developer-tools security probe is a self-contained preflight: it reads a macOS setting for this machine, not the iPhone's Developer Mode toggle, and today nothing in its name or its home says which machine it speaks for. Pure move. The startup step key is renamed to say what it actually verifies, and the existing coverage in runner-session.test.ts carries unchanged because the probe consumes the same runner host port. --- .../src/runner/runner-dev-tools-security.ts | 22 +++++++++++++++++++ .../src/runner/runner-session.ts | 22 ++++--------------- 2 files changed, 26 insertions(+), 18 deletions(-) create mode 100644 packages/platform-apple/src/runner/runner-dev-tools-security.ts diff --git a/packages/platform-apple/src/runner/runner-dev-tools-security.ts b/packages/platform-apple/src/runner/runner-dev-tools-security.ts new file mode 100644 index 0000000000..666737cd58 --- /dev/null +++ b/packages/platform-apple/src/runner/runner-dev-tools-security.ts @@ -0,0 +1,22 @@ +import { AppError } from '@agent-device/kernel/errors'; +import { isIosFamily, type DeviceInfo } from '@agent-device/kernel/device'; +import { runAppleToolCommand } from './host.ts'; + +const DEV_TOOLS_SECURITY_TIMEOUT_MS = 2_000; + +/** + * The host half of "can this Mac run an Apple UI test at all", probed before the runner builds. + */ +export async function assertDevToolsSecurityForIosRunner(device: DeviceInfo): Promise { + if (!isIosFamily(device) || device.kind !== 'device') return; + const result = await runAppleToolCommand('DevToolsSecurity', ['-status'], { + allowFailure: true, + timeoutMs: DEV_TOOLS_SECURITY_TIMEOUT_MS, + }); + const output = `${result.stdout}\n${result.stderr}`; + if (!/developer mode is currently disabled/i.test(output)) return; + throw new AppError('COMMAND_FAILED', 'Developer mode is disabled for Apple development tools', { + hint: 'Run `sudo DevToolsSecurity -enable`, then retry the iOS runner. UI test runners start suspended until Xcode/testmanagerd can attach.', + devToolsSecurityStatus: output.trim(), + }); +} diff --git a/packages/platform-apple/src/runner/runner-session.ts b/packages/platform-apple/src/runner/runner-session.ts index e748e19192..92a64050d0 100644 --- a/packages/platform-apple/src/runner/runner-session.ts +++ b/packages/platform-apple/src/runner/runner-session.ts @@ -6,11 +6,10 @@ import { emitDiagnostic, withDiagnosticTimer, buildSimctlArgsForDevice, - runAppleToolCommand, runXcrun, } from './host.ts'; import type { ExecResult } from '@agent-device/host-kit/command'; -import { isIosFamily, isApplePlatform, type DeviceInfo } from '@agent-device/kernel/device'; +import { isApplePlatform, type DeviceInfo } from '@agent-device/kernel/device'; import type { RunnerLogicalLeaseContext } from '@agent-device/contracts/runner-lease-context'; import type { AppleRunnerLifecycleOptions } from './runner-provider.ts'; import { getFreePort } from './runner-io.ts'; @@ -64,6 +63,7 @@ import { type RunnerDisposalOptions, } from './runner-disposal.ts'; import { enrichRunnerFailureFromLog } from './runner-failure-diagnostics.ts'; +import { assertDevToolsSecurityForIosRunner } from './runner-dev-tools-security.ts'; import { advanceRunnerSessionState, buildRunnerSessionId, @@ -187,8 +187,8 @@ async function startRunnerSessionWithLease( await measureRunnerStartupStep(startupTimings, 'ensure_booted', async () => { await ensureBootedIfNeeded(device); }); - await measureRunnerStartupStep(startupTimings, 'verify_developer_mode', async () => { - await verifyDeveloperModeForIosRunner(device); + await measureRunnerStartupStep(startupTimings, 'verify_host_dev_tools_security', async () => { + await assertDevToolsSecurityForIosRunner(device); }); if (options.cleanStaleBundles) { await measureRunnerStartupStep(startupTimings, 'cleanup_stale_bundles', async () => { @@ -721,20 +721,6 @@ async function ensureBooted(device: DeviceInfo): Promise { }); } -async function verifyDeveloperModeForIosRunner(device: DeviceInfo): Promise { - if (!isIosFamily(device) || device.kind !== 'device') return; - const result = await runAppleToolCommand('DevToolsSecurity', ['-status'], { - allowFailure: true, - timeoutMs: 2_000, - }); - const output = `${result.stdout}\n${result.stderr}`; - if (!/developer mode is currently disabled/i.test(output)) return; - throw new AppError('COMMAND_FAILED', 'Developer mode is disabled for Apple development tools', { - hint: 'Run `sudo DevToolsSecurity -enable`, then retry the iOS runner. UI test runners start suspended until Xcode/testmanagerd can attach.', - devToolsSecurityStatus: output.trim(), - }); -} - export function validateRunnerDevice(device: DeviceInfo): void { if (!isApplePlatform(device.platform)) { throw new AppError( From e0a81b0a86c5c45f079d16eaf3c16c67edbf2407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 19 Sep 2026 15:30:32 +0200 Subject: [PATCH 2/8] feat(ios-runner): key runner startup failures on typed reasons A `build-for-testing` failure was the one Apple failure shape with no typed reason: `runner-artifact.ts` threw COMMAND_FAILED with a hint picked by substring-matching the lowercased message plus a JSON dump of the details, and nothing downstream could switch on which signing problem it was. `AGENTS.md` keys behavior on typed reasons and details, never error text, so the build path was the exception to the rule. `classifyRunnerStartupFailure` is now the one classifier reachable from that catch, and it reads `RUNNER_ERROR_RULES` rather than adding a second table: a row either carries recovery verdicts, a `buildFailure` reason and hint, or both. The reason and the hint beside it therefore cannot disagree, and an unproven cause is never claimed -- `build_failed_unclassified` keeps the cache-recovery hint it already gave. The host `DevToolsSecurity` refusal, which had a hint and no reason at all, now publishes its own reason keyed on the typed status it read, and that reason says host so it can never be read as the device's Developer Mode toggle. `resolveRunnerBuildFailureHint` is gone, not kept as a pass-through. Reasons are recorded as fixtures carrying the tool output, the command, the Xcode, and how the line reached the file; each one is driven through the real build catch and asserted on the normalized envelope, because every case is COMMAND_FAILED and the reason is the assertion. --- .device-evidence/CHECKLIST-runner-failures.md | 74 +++++++ .../runner/__tests__/runner-client.test.ts | 10 - .../runner-dev-tools-security.test.ts | 77 +++++++ .../runner-startup-failure-fixtures.ts | 148 +++++++++++++ .../runner-startup-failure-reasons.test.ts | 201 +++++++++++++++++ .../src/runner/runner-artifact.ts | 7 +- .../src/runner/runner-contract.ts | 206 ++++++++++++++++-- .../src/runner/runner-dev-tools-security.ts | 29 ++- src/commands/schema/cli-help.ts | 1 + website/docs/docs/installation.md | 1 + 10 files changed, 716 insertions(+), 38 deletions(-) create mode 100644 .device-evidence/CHECKLIST-runner-failures.md create mode 100644 packages/platform-apple/src/runner/__tests__/runner-dev-tools-security.test.ts create mode 100644 packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts create mode 100644 packages/platform-apple/src/runner/__tests__/runner-startup-failure-reasons.test.ts diff --git a/.device-evidence/CHECKLIST-runner-failures.md b/.device-evidence/CHECKLIST-runner-failures.md new file mode 100644 index 0000000000..36c8a3baf3 --- /dev/null +++ b/.device-evidence/CHECKLIST-runner-failures.md @@ -0,0 +1,74 @@ +# Device evidence checklist + +Live evidence the coordinator runs serially on the connected iPhone. Each item names the exact +command, the environment it needs, and the rendered error that proves the change. Do not paraphrase +the JSON: paste it. Record the commit SHA the build under test was made from. + +## #2680 — typed build-failure reasons + +Build the CLI first, then stop any warm daemon so the run is on this commit: + +```sh +pnpm build && pnpm clean:daemon +node --experimental-strip-types src/bin.ts daemon stop --all || true +``` + +### 1. Signing with no team configured -> `signing_no_development_team` + +```sh +env -u AGENT_DEVICE_IOS_TEAM_ID -u AGENT_DEVICE_IOS_PROVISIONING_PROFILE \ + node --experimental-strip-types src/bin.ts --json \ + prepare ios-runner --platform ios --device "" +``` + +Expected: exit non-zero, one error object with + +```json +{ + "code": "COMMAND_FAILED", + "message": "xcodebuild build-for-testing failed", + "hint": "Configure signing in Xcode or set AGENT_DEVICE_IOS_TEAM_ID for physical-device runs.", + "details": { "reason": "signing_no_development_team" } +} +``` + +`hint`, `logPath` and `diagnosticId` are top-level, never inside `details`. Also record the Xcode +version (`xcodebuild -version`) so the `signing_no_development_team` fixture in +`packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts` can move from +`inherited-sniff-trigger` to `captured`. + +### 2. A bundle identifier somebody else already owns -> `bundle_identifier_already_registered` + +```sh +env AGENT_DEVICE_IOS_TEAM_ID="" \ + AGENT_DEVICE_IOS_BUNDLE_ID="com.apple.TestFlight" \ + node --experimental-strip-types src/bin.ts --json \ + prepare ios-runner --platform ios --device "" +``` + +Expected: same envelope shape with +`details.reason: "bundle_identifier_already_registered"` and a hint naming +`AGENT_DEVICE_IOS_BUNDLE_ID`. A registered-but-foreign identifier may surface the `Failed registering +bundle identifier` line or the `App Identifier ... is not available` line; both rules produce this +one reason, so record which line xcodebuild printed. + +### 3. Reasons with no device exposure (host-side or configuration-only) + +These do not need the iPhone, but do need a real xcodebuild run; record output with the Xcode +version so the matching fixture's `provenance` can be upgraded: + +```sh +# devtools_security_developer_mode_disabled (macOS admin state, no device work) +DevToolsSecurity -status + +# signing_provisioning_profile_missing / signing_style_conflict: point the runner build at a +# profile that is not installed, then read the reason off the same prepare command as above. +env AGENT_DEVICE_IOS_TEAM_ID="" \ + AGENT_DEVICE_IOS_PROVISIONING_PROFILE="no-such-profile-installed" \ + node --experimental-strip-types src/bin.ts --json \ + prepare ios-runner --platform ios --device "" +``` + +Expected: `details.reason` is `signing_provisioning_profile_missing`, or `signing_style_conflict` +when xcodebuild reports conflicting provisioning settings. If neither reason appears, say which +reason did and treat the fixture as unconfirmed rather than editing the rule to fit. diff --git a/packages/platform-apple/src/runner/__tests__/runner-client.test.ts b/packages/platform-apple/src/runner/__tests__/runner-client.test.ts index d972d872b2..8d039437b0 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-client.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-client.test.ts @@ -34,7 +34,6 @@ import type { DeviceInfo } from '@agent-device/kernel/device'; import { isReadOnlyRunnerCommand } from '../runner-command-traits.ts'; import { isRetryableRunnerError, - resolveRunnerBuildFailureHint, resolveRunnerEarlyExitHint, shouldRetryRunnerConnectError, withRunnerCommandId, @@ -437,15 +436,6 @@ test('resolveRunnerEarlyExitHint falls back to runner connect timeout hint', () assert.match(hint, /pnpm clean:xcuitest/i); }); -test('resolveRunnerBuildFailureHint suggests cache cleanup for non-signing failures', () => { - const hint = resolveRunnerBuildFailureHint( - new AppError('COMMAND_FAILED', 'xcodebuild build-for-testing failed'), - ); - - assert.match(hint, /pnpm clean:xcuitest/i); - assert.match(hint, /~\/\.agent-device\/apple-runner\/derived/i); -}); - test('shouldRetryRunnerConnectError does not retry xcodebuild early-exit errors', () => { const err = new AppError( 'COMMAND_FAILED', diff --git a/packages/platform-apple/src/runner/__tests__/runner-dev-tools-security.test.ts b/packages/platform-apple/src/runner/__tests__/runner-dev-tools-security.test.ts new file mode 100644 index 0000000000..5a5f98932e --- /dev/null +++ b/packages/platform-apple/src/runner/__tests__/runner-dev-tools-security.test.ts @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict'; +import { beforeEach, test, vi } from 'vitest'; +import { AppError, normalizeError } from '@agent-device/kernel/errors'; +import { appleRunnerTestHost } from '../test-host.ts'; +import { assertDevToolsSecurityForIosRunner } from '../runner-dev-tools-security.ts'; +import { IOS_DEVICE, IOS_SIMULATOR } from './device-fixtures.ts'; +import { RUNNER_STARTUP_FAILURE_FIXTURES } from './runner-startup-failure-fixtures.ts'; + +/** + * `DevToolsSecurity -status` answers for the Mac, not for the iPhone (#2680). The refusal this probe + * threw used to carry a hint and no reason, so a caller could only match its wording — and that + * wording is nearly the same as the device's own Developer Mode state, which is a different fact on + * a different machine. These cases pin that the host refusal publishes the host's own reason, keyed + * on the status it read rather than on the sentence it printed. + */ + +const HOST_REFUSAL_FIXTURE = RUNNER_STARTUP_FAILURE_FIXTURES.find( + (fixture) => fixture.reason === 'devtools_security_developer_mode_disabled', +); + +const runAppleToolCommand = vi.fn(); + +beforeEach(() => { + runAppleToolCommand.mockReset().mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }); + appleRunnerTestHost.update({ runAppleToolCommand }); +}); + +test('the host DevToolsSecurity refusal publishes its own typed reason', async () => { + assert.ok(HOST_REFUSAL_FIXTURE); + mockDevToolsSecurityOutput(HOST_REFUSAL_FIXTURE.output); + const expectedStatus = HOST_REFUSAL_FIXTURE.output.trim(); + + await assert.rejects( + () => assertDevToolsSecurityForIosRunner(IOS_DEVICE), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.match(error.message, /Developer mode is disabled/); + assert.equal(error.details?.reason, 'devtools_security_developer_mode_disabled'); + assert.equal(error.details?.devToolsSecurityStatus, expectedStatus); + + // What the caller renders: hint at top level, reason in details. + const envelope = normalizeError(error, { diagnosticId: 'diag-devtools-1' }); + assert.match(String(envelope.hint), /DevToolsSecurity -enable/); + assert.equal(envelope.diagnosticId, 'diag-devtools-1'); + assert.equal(envelope.details?.hint, undefined); + assert.equal(envelope.details?.diagnosticId, undefined); + assert.equal(envelope.details?.reason, 'devtools_security_developer_mode_disabled'); + return true; + }, + ); +}); + +test('an enabled host developer mode is not a failure', async () => { + mockDevToolsSecurityOutput('Developer mode is currently enabled for development tools.\n'); + + await assert.doesNotReject(() => assertDevToolsSecurityForIosRunner(IOS_DEVICE)); +}); + +test('a simulator never takes the host probe', async () => { + mockDevToolsSecurityOutput('Developer mode is currently disabled.\n'); + + await assert.doesNotReject(() => assertDevToolsSecurityForIosRunner(IOS_SIMULATOR)); + + assert.equal( + runAppleToolCommand.mock.calls.some((call) => call[0] === 'DevToolsSecurity'), + false, + ); +}); + +function mockDevToolsSecurityOutput(stdout: string): void { + runAppleToolCommand.mockImplementation(async (cmd: string) => ({ + exitCode: 0, + stdout: cmd === 'DevToolsSecurity' ? stdout : '', + stderr: '', + })); +} diff --git a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts new file mode 100644 index 0000000000..7c524b068b --- /dev/null +++ b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts @@ -0,0 +1,148 @@ +import { AppError } from '@agent-device/kernel/errors'; +import type { RunnerStartupFailureReason } from '../runner-contract.ts'; + +/** + * Recorded startup failures for {@link classifyRunnerStartupFailure} (#2680). + * + * Each entry carries the tool output exactly as it reaches the host, the command that produced it, + * the Xcode that produced it, and how it got here. `provenance` is what says whether a line was + * observed or transcribed: an entry stays `inherited-sniff-trigger` until the matching command in + * `.device-evidence/CHECKLIST.md` is run against real hardware, at which point `output` is replaced + * with the capture and `provenance` becomes `captured`. Nothing in the classifier reads these + * fields — they exist so a reason can be traced to an observation instead to a guess. + */ + +export type RunnerStartupFailureSite = 'build-for-testing' | 'host-dev-tools-security'; + +export type RunnerStartupFailureFixture = Readonly<{ + /** The reason this output must reach the caller with. */ + reason: RunnerStartupFailureReason; + /** Which throw site receives this output. */ + site: RunnerStartupFailureSite; + /** The invocation that produced {@link RunnerStartupFailureFixture.output}. */ + command: string; + /** `xcodebuild -version` of the machine that produced it. */ + xcodeVersion: string; + provenance: 'captured' | 'inherited-sniff-trigger' | 'tool-error-shape'; + /** The tool's own stdout/stderr, kept on one shape so JSON detail matching sees it as the host does. */ + output: string; + /** What the pending capture still has to show. */ + note?: string; +}>; + +/** + * The `xcodebuild` invocation `buildRunnerXctestrun` issues for a physical iOS device with Automatic + * Signing and no profile pinned, which is the configuration every signing reason below is about. + */ +const BUILD_FOR_TESTING_COMMAND = + 'xcodebuild build-for-testing -project apple/runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj ' + + '-scheme AgentDeviceRunner -parallel-testing-enabled NO -destination generic/platform=iOS ' + + '-derivedDataPath -allowProvisioningUpdates CODE_SIGN_STYLE=Automatic ' + + 'DEVELOPMENT_TEAM='; + +const OBSERVED_ON = 'Xcode 26.2 (Build 17C52)'; + +export const RUNNER_STARTUP_FAILURE_FIXTURES: readonly RunnerStartupFailureFixture[] = [ + { + reason: 'bundle_identifier_already_registered', + site: 'build-for-testing', + command: BUILD_FOR_TESTING_COMMAND, + xcodeVersion: OBSERVED_ON, + provenance: 'inherited-sniff-trigger', + output: + "error: Failed registering bundle identifier \"com.yourname.agentdevice.runner\" with the developer portal: An App ID with Identifier 'com.yourname.agentdevice.runner' is not available. Please enter a different string. (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + note: 'Capture with AGENT_DEVICE_IOS_BUNDLE_ID set to an identifier already registered by another team.', + }, + { + reason: 'bundle_identifier_already_registered', + site: 'build-for-testing', + command: BUILD_FOR_TESTING_COMMAND, + xcodeVersion: OBSERVED_ON, + provenance: 'tool-error-shape', + output: + "error: App Identifier 'com.yourname.agentdevice.runner' is not available. Choose a different App Identifier, or register it in your Apple Developer account before building. (in target 'AgentDeviceRunnerUITests' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + note: 'The second shape of the same cause: no "failed registering" line, so only the two-part match can name it.', + }, + { + reason: 'signing_no_development_team', + site: 'build-for-testing', + command: BUILD_FOR_TESTING_COMMAND, + xcodeVersion: OBSERVED_ON, + provenance: 'inherited-sniff-trigger', + output: + "error: Signing for \"AgentDeviceRunner\" requires a development team. Select a development team in the Signing & Capabilities editor. (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + note: 'Capture with AGENT_DEVICE_IOS_TEAM_ID unset on a signed-in-but-team-less account.', + }, + { + reason: 'signing_provisioning_profile_missing', + site: 'build-for-testing', + command: BUILD_FOR_TESTING_COMMAND, + xcodeVersion: OBSERVED_ON, + provenance: 'inherited-sniff-trigger', + output: + "error: No profiles for 'com.yourname.agentdevice.runner' were found: Xcode couldn't find any iOS App Development provisioning profiles matching 'com.yourname.agentdevice.runner'. Automatic signing is disabled and unable to generate a profile. To enable automatic signing, pass -allowProvisioningUpdates to xcodebuild. (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + note: 'Capture with AGENT_DEVICE_IOS_PROVISIONING_PROFILE naming a profile that is not installed.', + }, + { + reason: 'signing_style_conflict', + site: 'build-for-testing', + command: BUILD_FOR_TESTING_COMMAND, + xcodeVersion: OBSERVED_ON, + provenance: 'tool-error-shape', + output: + 'error: "AgentDeviceRunner" has conflicting provisioning settings. AgentDeviceRunner is automatically signed, but provisioning profile "match-development-com-yourname-agentdevice-runner" has been manually specified. Set the provisioning profile value to "Automatic" in the build settings editor, or switch to manual signing in the Signing & Capabilities editor. (in target \'AgentDeviceRunner\' from project \'AgentDeviceRunner\')\n** TEST BUILD FAILED **\n', + note: 'New reason: capture the conflicting-settings line before claiming this wording on hardware.', + }, + { + reason: 'signing_unspecified', + site: 'build-for-testing', + command: BUILD_FOR_TESTING_COMMAND, + xcodeVersion: OBSERVED_ON, + provenance: 'inherited-sniff-trigger', + output: + "error: Code signing is required for product type 'Application' in SDK 'iOS 26.2' (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + note: 'Signing is named and nothing above it is: the reason stays unspecified on purpose.', + }, + { + reason: 'build_failed_unclassified', + site: 'build-for-testing', + command: BUILD_FOR_TESTING_COMMAND, + xcodeVersion: OBSERVED_ON, + provenance: 'tool-error-shape', + output: + "error: cannot find 'AgentDeviceRunnerCommand' in scope (in target 'AgentDeviceRunnerUITests' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + note: 'Any build failure that names no signing fact must keep the cache-recovery hint.', + }, + { + reason: 'devtools_security_developer_mode_disabled', + site: 'host-dev-tools-security', + command: 'DevToolsSecurity -status', + xcodeVersion: OBSERVED_ON, + provenance: 'inherited-sniff-trigger', + output: 'Developer mode is currently disabled for development tools.\n', + note: "Host-side refusal. It says nothing about the device's Developer Mode toggle (#2683 reads that).", + }, +]; + +export function buildForTestingFixtures(): RunnerStartupFailureFixture[] { + return RUNNER_STARTUP_FAILURE_FIXTURES.filter((fixture) => fixture.site === 'build-for-testing'); +} + +/** + * The error the exec layer hands the build-failure catch when `xcodebuild` exits non-zero: a + * COMMAND_FAILED whose message is the exec's own and whose tool output sits in `details`, which is + * exactly why the rules below read details text and not only the message. + */ +export function buildForTestingExecError( + fixture: Pick, + exitCode = 65, +): AppError { + return new AppError('COMMAND_FAILED', 'xcodebuild exited with code 65', { + stdout: fixture.output, + stderr: '', + exitCode, + processExitError: true, + cmd: 'xcodebuild', + args: ['build-for-testing'], + }); +} diff --git a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-reasons.test.ts b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-reasons.test.ts new file mode 100644 index 0000000000..4d27767648 --- /dev/null +++ b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-reasons.test.ts @@ -0,0 +1,201 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { afterEach, beforeEach, test, vi } from 'vitest'; +import { AppError, normalizeError, type NormalizedError } from '@agent-device/kernel/errors'; +import { resetAllProcessMemosForTests } from '@agent-device/kernel/ttl-memo'; +import { appleRunnerTestHost } from '../test-host.ts'; +import type { ExecResult } from '../host.ts'; +import { createRunnerPhaseBudget, ensureXctestrunArtifact } from '../runner-xctestrun.ts'; +import { + RUNNER_ERROR_RULES, + RUNNER_STARTUP_FAILURE_REASONS, + RUNNER_STARTUP_FAILURE_UNCLASSIFIED_REASON, + type RunnerStartupFailureReason, +} from '../runner-contract.ts'; +import { appleToolchainProbeResult } from './apple-toolchain-fixtures.ts'; +import { IOS_DEVICE } from './device-fixtures.ts'; +import { + RUNNER_STARTUP_FAILURE_FIXTURES, + buildForTestingExecError, + buildForTestingFixtures, +} from './runner-startup-failure-fixtures.ts'; +import { mkdtempForTestSync } from './tmp-dir.ts'; + +/** + * A `build-for-testing` failure used to reach the caller as prose only, so every consumer that + * wanted to know *which* signing problem it was had to re-match the same substrings (#2680). These + * cases drive each recorded output through the real build-failure catch and assert on the + * normalized envelope: the code is `COMMAND_FAILED` for all of them, so `details.reason` is the + * assertion, and the hint beside it has to be the hint the rule that named the reason carries. + * + * The envelope assertions are deliberate: `normalizeError` moves `hint`, `logPath` and + * `diagnosticId` out of `details` to the top level, so a reason that survives in `details` and a + * hint that survives at top level are two different claims about where the error was built. + */ + +const CACHE_RECOVERY_HINT = /clean:xcuitest|apple-runner\/derived/; + +const HINT_FOR_REASON: Record = { + bundle_identifier_already_registered: /AGENT_DEVICE_IOS_BUNDLE_ID/, + signing_no_development_team: /AGENT_DEVICE_IOS_TEAM_ID/, + signing_provisioning_profile_missing: /AGENT_DEVICE_IOS_PROVISIONING_PROFILE/, + signing_style_conflict: /CODE_SIGN_STYLE/, + signing_unspecified: /Automatic Signing/, + devtools_security_developer_mode_disabled: /DevToolsSecurity -enable/, + build_failed_unclassified: CACHE_RECOVERY_HINT, +}; + +const runCmdSync = vi.fn(); +const runCmdStreaming = vi.fn(); +const DIAGNOSTIC_ID = 'diag-build-failure-1'; +let projectRoot: string; +let derivedPath: string; +let logPath: string; + +beforeEach(() => { + resetAllProcessMemosForTests(); + projectRoot = mkdtempForTestSync('agent-device-startup-failure-root-'); + // `buildXctestrunArtifact` refuses to start a build without the runner project. + fs.mkdirSync( + path.join(projectRoot, 'apple', 'runner', 'AgentDeviceRunner', 'AgentDeviceRunner.xcodeproj'), + { recursive: true }, + ); + derivedPath = mkdtempForTestSync('agent-device-startup-failure-derived-'); + logPath = path.join(derivedPath, 'runner.log'); + process.env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH = derivedPath; + runCmdSync.mockReset().mockImplementation(appleToolchainProbeResult); + runCmdStreaming.mockReset().mockImplementation(async (): Promise => ({ + exitCode: 0, + stdout: '', + stderr: '', + })); + appleRunnerTestHost.update({ + runCmdSync, + runCmdStreaming, + findProjectRoot: () => projectRoot, + readVersion: () => '0.0.0-test', + }); +}); + +afterEach(() => { + delete process.env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH; +}); + +for (const fixture of buildForTestingFixtures()) { + test(`a build-for-testing failure publishes ${fixture.reason}`, async () => { + const envelope = await driveBuildFailure(buildForTestingExecError(fixture)); + + assert.equal(envelope.code, 'COMMAND_FAILED'); + assert.equal(envelope.message, 'xcodebuild build-for-testing failed'); + assert.equal(envelope.details?.reason, fixture.reason); + assert.match(String(envelope.hint), HINT_FOR_REASON[fixture.reason]); + assert.equal(envelope.logPath, logPath); + assert.equal(envelope.diagnosticId, DIAGNOSTIC_ID); + // normalizeError hoists these out of `details`; a caller must read them at top level. + assert.equal(envelope.details?.hint, undefined); + assert.equal(envelope.details?.logPath, undefined); + assert.equal(envelope.details?.diagnosticId, undefined); + // The tool output stays reachable for a human reading the failure. It is redacted and + // length-bounded on the way out, which is another reason the reason is typed: classification + // happens before the truncation a caller sees. + const nestedDetails = envelope.details?.details as Record | undefined; + assert.match(String(nestedDetails?.stdout), /AgentDeviceRunner/); + }); +} + +test('every startup failure reason has a recorded fixture', () => { + const reasonsWithFixtures = new Set(RUNNER_STARTUP_FAILURE_FIXTURES.map((f) => f.reason)); + + assert.equal(reasonsWithFixtures.size, RUNNER_STARTUP_FAILURE_REASONS.length); + for (const reason of RUNNER_STARTUP_FAILURE_REASONS) { + assert.ok(reasonsWithFixtures.has(reason), `no fixture records the ${reason} reason`); + } +}); + +test('every reason the classifier can name is produced by a rule row', () => { + const reasonsFromRules = new Set( + RUNNER_ERROR_RULES.flatMap((rule) => (rule.buildFailure ? [rule.buildFailure.reason] : [])), + ); + + for (const reason of RUNNER_STARTUP_FAILURE_REASONS) { + // The catch-all is the classifier's own answer when no row matched, so it names no row. + if (reason === RUNNER_STARTUP_FAILURE_UNCLASSIFIED_REASON) continue; + assert.ok(reasonsFromRules.has(reason), `no rule row yields the ${reason} reason`); + } +}); + +test('an identical message without the typed host fact is not read as a DevToolsSecurity refusal', async () => { + // Same message the host probe throws; the only difference is the typed `devToolsSecurityStatus` + // fact the probe publishes. Text alone must not activate a reason (#2680). + const withoutFact = new AppError('COMMAND_FAILED', 'Developer mode is disabled', { + stdout: 'developer mode is disabled\n', + stderr: '', + exitCode: 65, + processExitError: true, + }); + + const envelope = await driveBuildFailure(withoutFact); + + assert.equal(envelope.details?.reason, RUNNER_STARTUP_FAILURE_UNCLASSIFIED_REASON); + assert.match(String(envelope.hint), CACHE_RECOVERY_HINT); + assert.doesNotMatch(String(envelope.hint), /DevToolsSecurity/); +}); + +test('an app identifier named without the availability fact is not read as a taken bundle id', async () => { + const nearMiss = buildForTestingExecError({ + output: + "error: App Identifier 'com.yourname.agentdevice.runner' is invalid. (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n", + }); + + const envelope = await driveBuildFailure(nearMiss); + + assert.equal(envelope.details?.reason, RUNNER_STARTUP_FAILURE_UNCLASSIFIED_REASON); + assert.match(String(envelope.hint), CACHE_RECOVERY_HINT); + assert.doesNotMatch(String(envelope.hint), /AGENT_DEVICE_IOS_BUNDLE_ID/); +}); + +test('a signing failure that only names code signing keeps the generic signing advice', async () => { + const generic = buildForTestingExecError({ + output: "error: Code signing is required for product type 'Application' in SDK 'iOS 26.2'\n", + }); + + const envelope = await driveBuildFailure(generic); + + assert.equal(envelope.details?.reason, 'signing_unspecified'); + assert.match(String(envelope.hint), /Automatic Signing/); + assert.doesNotMatch(String(envelope.hint), CACHE_RECOVERY_HINT); +}); + +test('a conflicting-settings failure is not downgraded to a missing profile', async () => { + // The conflicting-settings line names a profile while explaining that the styles disagree, so + // the more specific row has to win the race the generic profile row would also run. + const conflict = buildForTestingExecError({ + output: + 'error: "AgentDeviceRunner" has conflicting provisioning settings. AgentDeviceRunner is automatically signed, but provisioning profile "match-development" has been manually specified.\n', + }); + + const envelope = await driveBuildFailure(conflict); + + assert.equal(envelope.details?.reason, 'signing_style_conflict'); + assert.match(String(envelope.hint), /CODE_SIGN_STYLE/); +}); + +async function driveBuildFailure(execError: AppError): Promise { + runCmdStreaming.mockReset().mockRejectedValue(execError); + + let envelope: NormalizedError | undefined; + await assert.rejects( + () => + ensureXctestrunArtifact(IOS_DEVICE, { + logPath, + budget: createRunnerPhaseBudget(120_000, undefined), + }), + (error: unknown) => { + envelope = normalizeError(error, { diagnosticId: DIAGNOSTIC_ID, logPath }); + return true; + }, + ); + assert.ok(envelope, 'the build-failure catch must throw'); + return envelope; +} diff --git a/packages/platform-apple/src/runner/runner-artifact.ts b/packages/platform-apple/src/runner/runner-artifact.ts index dd74f0a12d..ffd0f38e0b 100644 --- a/packages/platform-apple/src/runner/runner-artifact.ts +++ b/packages/platform-apple/src/runner/runner-artifact.ts @@ -12,7 +12,7 @@ import { } from './host.ts'; import type { ExecBackgroundResult } from '@agent-device/host-kit/command'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import { resolveRunnerBuildFailureHint } from './runner-contract.ts'; +import { classifyRunnerStartupFailure } from './runner-contract.ts'; import { logChunk } from './runner-io.ts'; import { withXcodebuildSimulatorSetRedirect } from './runner-device-set.ts'; import { @@ -517,8 +517,11 @@ async function buildRunnerXctestrun( if (isRequestCanceledError(error)) throw error; const appErr = error instanceof AppError ? error : new AppError('COMMAND_FAILED', String(error)); - const hint = resolveRunnerBuildFailureHint(appErr); + // The reason and the hint beside it come from one classifier (#2680), so the reason a caller + // switches on can never disagree with the advice it is handed. + const { reason, hint } = classifyRunnerStartupFailure(appErr); throw new AppError('COMMAND_FAILED', 'xcodebuild build-for-testing failed', { + reason, error: appErr.message, details: appErr.details, logPath: options.logPath, diff --git a/packages/platform-apple/src/runner/runner-contract.ts b/packages/platform-apple/src/runner/runner-contract.ts index cf79fcfa04..7c9379e505 100644 --- a/packages/platform-apple/src/runner/runner-contract.ts +++ b/packages/platform-apple/src/runner/runner-contract.ts @@ -172,11 +172,24 @@ type RunnerErrorMatch = { code?: AppErrorCode; /** Every entry must appear in the lowercased message. */ messageIncludesAll?: readonly string[]; + /** + * Every entry must appear in the lowercased JSON of the details bag, so a tool's own failure + * text (which arrives in `stderr`/`stdout` rather than in our message) can carry a rule. JSON + * escaping keeps each line whole, so an entry must live on one line of the tool's output. + */ + detailsIncludesAll?: readonly string[]; /** Required details evidence beyond code/message. */ details?: RunnerErrorDetailsMatch; }; const hasRetriableFlag: RunnerErrorDetailsMatch = (details) => details.retriable === true; +/** + * The host's own `DevToolsSecurity -status` read, published as typed details by the probe that + * takes it. The build-failure rule below keys on this field and never on the probe's message, so + * an error that merely says developer mode is disabled cannot be read as a host refusal (#2680). + */ +const hasDevToolsSecurityStatus: RunnerErrorDetailsMatch = (details) => + typeof details.devToolsSecurityStatus === 'string'; const hasUsbmuxDeviceUnattached: RunnerErrorDetailsMatch = (details) => details.usbmuxDeviceAttached === false; /** @@ -203,17 +216,61 @@ type RunnerErrorVerdicts = { artifactSuspect?: boolean; }; +/** + * Why the Apple runner could not reach the point of serving a command (#2680). Published in + * `details.reason` on the `COMMAND_FAILED` every one of these paths throws, so a caller branches + * on the reason instead of matching prose; the hint that answers it travels with it in + * {@link RUNNER_ERROR_RULES}. + * + * This is the vocabulary #2683 adds the device-readiness members to (Developer Mode and developer + * disk image state read from the device itself), which is why it is keyed on startup rather than on + * `xcodebuild`: an iPhone that refuses the runner for reasons other than signing stops the runner + * before a build is ever the question. + */ +export const RUNNER_STARTUP_FAILURE_REASONS = [ + 'bundle_identifier_already_registered', + 'signing_no_development_team', + 'signing_provisioning_profile_missing', + 'signing_style_conflict', + 'signing_unspecified', + 'devtools_security_developer_mode_disabled', + 'build_failed_unclassified', +] as const; + +export type RunnerStartupFailureReason = (typeof RUNNER_STARTUP_FAILURE_REASONS)[number]; + +/** + * The reason a startup failure carries when no rule proves a cause. Its hint is deliberately the + * cache-recovery advice rather than anything about signing: an unclassified build is not evidence of + * a signing problem. + */ +export const RUNNER_STARTUP_FAILURE_UNCLASSIFIED_REASON: RunnerStartupFailureReason = + 'build_failed_unclassified'; + type RunnerErrorRule = { /** Stable rule name for tests and diagnostics. */ reason: string; match: RunnerErrorMatch; verdicts: RunnerErrorVerdicts; + /** + * Set on the rules that also classify why the runner could not start (#2680). Rules like these + * define no recovery verdicts for a runner that never came up — there is no session to invalidate + * and nothing was sent to resend — so the axes stay empty and the row carries only reason plus + * hint. Several rows may name one reason (bundle-identifier registration fails in two shapes), + * and the classifier takes the first match, which is why specific rows precede generic ones. + */ + buildFailure?: { + reason: RunnerStartupFailureReason; + hint: string; + }; }; /** * The one declaration of runner error classes (#1631), mirroring * RUNNER_COMMAND_TRAIT_MANIFEST's role for commands: every recovery predicate - * below derives from this table instead of keeping its own substring chain. + * below derives from this table instead of keeping its own substring chain, + * and since #2680 so does the one classification of startup failures — a row + * carries recovery verdicts, a `buildFailure` reason and hint, or both. * Per axis, the FIRST matching rule that defines the axis wins — which is why * `flagged_retriable` precedes the denials (an explicitly retriable error * stays retriable whatever its message says), and `usbmux_device_unattached` @@ -304,11 +361,102 @@ export const RUNNER_ERROR_RULES: readonly RunnerErrorRule[] = [ match: { code: 'RUNNER_WEDGED' }, verdicts: { sessionFatalReason: 'runner_main_thread_wedged' }, }, + // ── Startup classification (#2680) ─────────────────────────────────────────────────────────── + // These rows answer "why could the runner not get here at all": `xcodebuild build-for-testing` + // refusing, and the host preflight that runs before it. They carry a reason and a hint for the + // caller and no recovery verdicts, because there is no session to invalidate and nothing was sent + // to resend. Specific rows precede generic ones: the classifier takes the first match. + // + // Why these rows are text matchers while the rows above key on a code or a typed field: + // `matchesRunnerErrorMessage`/`matchesRunnerErrorDetailsText` read xcodebuild's own prose because + // that prose is the only publication these failures have — there is no code and no typed field to + // key on. The DevToolsSecurity row is the other half: where a probe of ours publishes a typed + // fact, the row keys on that fact alone. `resolveRunnerEarlyExitHint` stays outside this table for + // the same reason it stays a hint builder — it classifies a runner that DID build and then exited + // early, whose reason axis is the `BootFailureReason` `classifyBootFailure` already returns, and a + // build that never produced a binary has no boot to classify. + { + reason: 'bundle_identifier_registration_failed', + match: { detailsIncludesAll: ['failed registering bundle identifier'] }, + verdicts: {}, + buildFailure: { + reason: 'bundle_identifier_already_registered', + hint: 'Set AGENT_DEVICE_IOS_BUNDLE_ID to a unique reverse-DNS value (for example, com.yourname.agentdevice.runner), then retry.', + }, + }, + { + reason: 'bundle_identifier_unavailable', + match: { detailsIncludesAll: ['app identifier', 'not available'] }, + verdicts: {}, + buildFailure: { + reason: 'bundle_identifier_already_registered', + hint: 'Set AGENT_DEVICE_IOS_BUNDLE_ID to a unique reverse-DNS value (for example, com.yourname.agentdevice.runner), then retry.', + }, + }, + { + reason: 'signing_requires_development_team', + match: { detailsIncludesAll: ['requires a development team'] }, + verdicts: {}, + buildFailure: { + reason: 'signing_no_development_team', + hint: 'Configure signing in Xcode or set AGENT_DEVICE_IOS_TEAM_ID for physical-device runs.', + }, + }, + { + // Precedes the profile rows: this xcodebuild failure names a profile in saying the signing + // styles conflict, and the recovery is to align the settings, not to go install a profile. + reason: 'signing_style_conflict', + match: { detailsIncludesAll: ['conflicting provisioning settings'] }, + verdicts: {}, + buildFailure: { + reason: 'signing_style_conflict', + hint: 'The runner project mixes signing styles: one setting asks for automatic signing while another pins a profile or team. Clear AGENT_DEVICE_IOS_PROVISIONING_PROFILE to let Xcode choose, or set CODE_SIGN_STYLE=Manual alongside a matching AGENT_DEVICE_IOS_PROVISIONING_PROFILE, then retry.', + }, + }, + { + reason: 'signing_no_profiles_for_bundle_id', + match: { detailsIncludesAll: ['no profiles for'] }, + verdicts: {}, + buildFailure: { + reason: 'signing_provisioning_profile_missing', + hint: 'Install/select a valid iOS provisioning profile, or set AGENT_DEVICE_IOS_PROVISIONING_PROFILE.', + }, + }, + { + reason: 'signing_provisioning_profile_unusable', + match: { detailsIncludesAll: ['provisioning profile'] }, + verdicts: {}, + buildFailure: { + reason: 'signing_provisioning_profile_missing', + hint: 'Install/select a valid iOS provisioning profile, or set AGENT_DEVICE_IOS_PROVISIONING_PROFILE.', + }, + }, + { + // Signing is involved but nothing above names how: the reason says signing and the hint stays + // the generic one it has always carried, rather than naming a misconfiguration no rule proved. + reason: 'signing_unspecified', + match: { detailsIncludesAll: ['code signing'] }, + verdicts: {}, + buildFailure: { + reason: 'signing_unspecified', + hint: 'Enable Automatic Signing in Xcode or provide AGENT_DEVICE_IOS_TEAM_ID and optional AGENT_DEVICE_IOS_SIGNING_IDENTITY.', + }, + }, + { + reason: 'devtools_security_refused', + match: { code: 'COMMAND_FAILED', details: hasDevToolsSecurityStatus }, + verdicts: {}, + buildFailure: { + reason: 'devtools_security_developer_mode_disabled', + hint: 'Run `sudo DevToolsSecurity -enable`, then retry the iOS runner. UI test runners start suspended until Xcode/testmanagerd can attach.', + }, + }, ]; function matchesRunnerErrorRule(error: AppError, match: RunnerErrorMatch): boolean { if (match.code !== undefined && error.code !== match.code) return false; if (!matchesRunnerErrorDetails(error, match.details)) return false; + if (!matchesRunnerErrorDetailsText(error, match.detailsIncludesAll)) return false; return matchesRunnerErrorMessage(error, match.messageIncludesAll); } @@ -323,6 +471,15 @@ function matchesRunnerErrorMessage(error: AppError, parts: readonly string[] | u return parts.every((part) => message.includes(part)); } +function matchesRunnerErrorDetailsText( + error: AppError, + parts: readonly string[] | undefined, +): boolean { + if (!parts) return true; + const details = error.details ? JSON.stringify(error.details).toLowerCase() : ''; + return parts.every((part) => details.includes(part)); +} + function runnerErrorVerdict( error: unknown, axis: Axis, @@ -599,29 +756,32 @@ export async function buildRunnerEarlyExitError(params: { }); } -function resolveSigningFailureHint(error: AppError): string | undefined { - const details = error.details ? JSON.stringify(error.details) : ''; - const combined = `${error.message}\n${details}`.toLowerCase(); - if ( - combined.includes('failed registering bundle identifier') || - (combined.includes('app identifier') && combined.includes('not available')) - ) { - return 'Set AGENT_DEVICE_IOS_BUNDLE_ID to a unique reverse-DNS value (for example, com.yourname.agentdevice.runner), then retry.'; - } - if (combined.includes('requires a development team')) { - return 'Configure signing in Xcode or set AGENT_DEVICE_IOS_TEAM_ID for physical-device runs.'; - } - if (combined.includes('no profiles for') || combined.includes('provisioning profile')) { - return 'Install/select a valid iOS provisioning profile, or set AGENT_DEVICE_IOS_PROVISIONING_PROFILE.'; - } - if (combined.includes('code signing')) { - return 'Enable Automatic Signing in Xcode or provide AGENT_DEVICE_IOS_TEAM_ID and optional AGENT_DEVICE_IOS_SIGNING_IDENTITY.'; +/** + * The one classifier for "the runner did not reach the point of serving a command" (#2680). Every + * path that stops the runner before it answers a request routes its failure through here, so the + * reason a caller sees is produced by the same rows that produce the hint beside it — a reason is + * never inferred from a hint's wording, and an unproven cause is never claimed. + * + * Callers publish the pair as `details.reason` plus the top-level hint on a `COMMAND_FAILED`; the + * code is `COMMAND_FAILED` for every reason, so the reason is the assertion. + */ +export function classifyRunnerStartupFailure(error: unknown): { + reason: RunnerStartupFailureReason; + hint: string; +} { + if (error instanceof AppError) { + for (const rule of RUNNER_ERROR_RULES) { + const buildFailure = rule.buildFailure; + if (!buildFailure) continue; + if (matchesRunnerErrorRule(error, rule.match)) { + return { reason: buildFailure.reason, hint: buildFailure.hint }; + } + } } - return undefined; -} - -export function resolveRunnerBuildFailureHint(error: AppError): string { - return resolveSigningFailureHint(error) ?? RUNNER_CACHE_RECOVERY_HINT; + return { + reason: RUNNER_STARTUP_FAILURE_UNCLASSIFIED_REASON, + hint: RUNNER_CACHE_RECOVERY_HINT, + }; } export function withRunnerCommandId(command: RunnerCommand): RunnerCommand { diff --git a/packages/platform-apple/src/runner/runner-dev-tools-security.ts b/packages/platform-apple/src/runner/runner-dev-tools-security.ts index 666737cd58..f3ce67e73b 100644 --- a/packages/platform-apple/src/runner/runner-dev-tools-security.ts +++ b/packages/platform-apple/src/runner/runner-dev-tools-security.ts @@ -1,11 +1,20 @@ import { AppError } from '@agent-device/kernel/errors'; import { isIosFamily, type DeviceInfo } from '@agent-device/kernel/device'; import { runAppleToolCommand } from './host.ts'; +import { classifyRunnerStartupFailure } from './runner-contract.ts'; const DEV_TOOLS_SECURITY_TIMEOUT_MS = 2_000; +const DEV_TOOLS_SECURITY_REFUSAL_MESSAGE = 'Developer mode is disabled for Apple development tools'; + /** * The host half of "can this Mac run an Apple UI test at all", probed before the runner builds. + * + * `DevToolsSecurity -status` reports the macOS developer-tools security setting that governs + * debugserver on THIS machine. It is not the iPhone's Settings > Privacy & Security > Developer + * Mode toggle, which lives on the device and has no host-visible value here — the two states are + * independent even though the wording is nearly the same, so the failure this throws is labelled + * with the host's own reason and never with a device-side one (#2680). */ export async function assertDevToolsSecurityForIosRunner(device: DeviceInfo): Promise { if (!isIosFamily(device) || device.kind !== 'device') return; @@ -15,8 +24,22 @@ export async function assertDevToolsSecurityForIosRunner(device: DeviceInfo): Pr }); const output = `${result.stdout}\n${result.stderr}`; if (!/developer mode is currently disabled/i.test(output)) return; - throw new AppError('COMMAND_FAILED', 'Developer mode is disabled for Apple development tools', { - hint: 'Run `sudo DevToolsSecurity -enable`, then retry the iOS runner. UI test runners start suspended until Xcode/testmanagerd can attach.', - devToolsSecurityStatus: output.trim(), + throw buildDevToolsSecurityRefusal(output.trim()); +} + +/** + * The refusal carries the reason and hint that {@link classifyRunnerStartupFailure} derives from + * the typed `devToolsSecurityStatus` fact, instead of naming either here, so the pair a caller + * receives cannot drift from the rule table that owns it. + */ +function buildDevToolsSecurityRefusal(status: string): AppError { + const observed = new AppError('COMMAND_FAILED', DEV_TOOLS_SECURITY_REFUSAL_MESSAGE, { + devToolsSecurityStatus: status, + }); + const { reason, hint } = classifyRunnerStartupFailure(observed); + return new AppError('COMMAND_FAILED', DEV_TOOLS_SECURITY_REFUSAL_MESSAGE, { + reason, + hint, + devToolsSecurityStatus: status, }); } diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index b2addd9db2..225a44e0e8 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -689,6 +689,7 @@ iOS physical-device prerequisites: If Xcode cannot choose a profile, set AGENT_DEVICE_IOS_PROVISIONING_PROFILE to the profile name/specifier, not a file path. AGENT_DEVICE_IOS_SIGNING_IDENTITY is optional; omit it unless xcodebuild asks for a specific identity. The profile/team must allow AGENT_DEVICE_IOS_BUNDLE_ID and .uitests. + A runner build failure names its class in error details.reason rather than only in prose: signing_no_development_team, signing_provisioning_profile_missing, signing_style_conflict, bundle_identifier_already_registered, signing_unspecified, devtools_security_developer_mode_disabled (the Mac's DevToolsSecurity setting, which says nothing about the device's Developer Mode toggle), or build_failed_unclassified when nothing proved a cause. Branch on details.reason and follow hint; the message is for humans. First-run XCTest setup/build can take longer than normal commands; keep the device connected and use --debug to inspect signing/build diagnostics if setup times out. Android physical-device prerequisites: diff --git a/website/docs/docs/installation.md b/website/docs/docs/installation.md index a3e08f66da..af0e540f84 100644 --- a/website/docs/docs/installation.md +++ b/website/docs/docs/installation.md @@ -108,6 +108,7 @@ vega device list - `AGENT_DEVICE_IOS_PROVISIONING_PROFILE` - `AGENT_DEVICE_IOS_BUNDLE_ID` (optional runner bundle-id base override) - Free Apple Developer (Personal Team) accounts can fail with "bundle identifier is not available" for generic IDs; set `AGENT_DEVICE_IOS_BUNDLE_ID` to a unique reverse-DNS value (for example `com.yourname.agentdevice.runner`). +- A runner build failure is typed, not prose: `error.details.reason` is one of `signing_no_development_team`, `signing_provisioning_profile_missing`, `signing_style_conflict`, `bundle_identifier_already_registered`, `signing_unspecified`, `devtools_security_developer_mode_disabled` (the Mac's `DevToolsSecurity` setting, which says nothing about the device's Developer Mode toggle), or `build_failed_unclassified` when nothing proved a cause. Branch on `details.reason` and follow `hint`; the code stays `COMMAND_FAILED` for every reason. - If device setup is slow, keep the device connected and inspect daemon diagnostics after retrying. - If daemon startup reports stale metadata, remove stale files and retry: - `/daemon.json` From d406739c0bca26db4d291791201ddb160cc95ebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 19 Sep 2026 15:34:08 +0200 Subject: [PATCH 3/8] refactor(ios-runner): reach the host dev-tools probe only when a device needs it A static edge from `runner-session.ts` to the new module grew three Apple facade closures by one module each (app-lifecycle, doctor, runner-operations), which the eager-import-closure ratchet refuses: the runner subtree is eagerly evaluated to answer a simulator request, and this preflight only ever runs for a physical iOS device. Function-scoped import keeps the preflight where it belongs without paying for it on the paths that can never use it. --- packages/platform-apple/src/runner/runner-session.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/platform-apple/src/runner/runner-session.ts b/packages/platform-apple/src/runner/runner-session.ts index 92a64050d0..2270d12e2a 100644 --- a/packages/platform-apple/src/runner/runner-session.ts +++ b/packages/platform-apple/src/runner/runner-session.ts @@ -63,7 +63,6 @@ import { type RunnerDisposalOptions, } from './runner-disposal.ts'; import { enrichRunnerFailureFromLog } from './runner-failure-diagnostics.ts'; -import { assertDevToolsSecurityForIosRunner } from './runner-dev-tools-security.ts'; import { advanceRunnerSessionState, buildRunnerSessionId, @@ -188,6 +187,10 @@ async function startRunnerSessionWithLease( await ensureBootedIfNeeded(device); }); await measureRunnerStartupStep(startupTimings, 'verify_host_dev_tools_security', async () => { + // Loaded here rather than at the top of the file: the runner subtree sits in the eager import + // closure of the seven Apple facades (eager-closure-budgets), and a preflight only a physical + // device ever needs has no business being evaluated to answer a simulator request. + const { assertDevToolsSecurityForIosRunner } = await import('./runner-dev-tools-security.ts'); await assertDevToolsSecurityForIosRunner(device); }); if (options.cleanStaleBundles) { From fa7b9bd6a7c5129965f84dc7b100acc1ed251417 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 19 Sep 2026 17:17:10 +0200 Subject: [PATCH 4/8] fix(ios-runner): classify a build failure only from what xcodebuild published MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The startup rules read the whole `details` bag as JSON, and `execFailureDetails` puts `cmd`/`args` in that bag. A caller who pinned a profile therefore handed the profile rule its trigger: an unrelated Swift compile error classified as `signing_provisioning_profile_missing` and lost the cache-recovery hint. The same bag holds the `reason` and `hint` this classifier publishes, so a re-wrapped failure would match its own verdict. `runnerToolText` now builds the one haystack a startup rule may read — our message plus the tool's stdout and stderr — and the argv case has a fixture that proves a pinned profile is not evidence (#2680). Reading only `details` also dropped the message on the floor, so a failure the exec layer raised as a plain `Error` (which the catch wraps with `String(err)`) became unclassified while the same sentence in `stdout` classified. The message is in the haystack now, and a `message-only` fixture keeps both carriers at parity. `signing_style_conflict` is removed rather than kept as a guess: the hint named a `CODE_SIGN_STYLE` env lever that does not exist and claimed a cause nothing captured. The conflicting-settings line keeps a row so the profile row below cannot answer it with missing-profile advice, but it publishes `build_failed_unclassified` until a capture shows which setting disagrees and which lever clears it. Versioned help and the installation docs no longer list the withdrawn reason, and `.device-evidence/CHECKLIST.md` gained the capture that would let a follow-up claim it. Provenance got honest vocabulary. `OBSERVED_ON` stamped invented sentences as observed on Xcode 26.2, and one invocation was recorded as the producer of seven configurations that cannot coexist; nothing here was captured, so entries now say `shipped-sniff-trigger` (the matched substrings shipped before #2680, sentence reconstructed) or `invented-shape`, carry `xcodeVersion: 'unobserved'`, and omit `command` until a run records one. --- .device-evidence/CHECKLIST-runner-failures.md | 32 +++- .../runner-startup-failure-fixtures.ts | 175 +++++++++++------- .../runner-startup-failure-reasons.test.ts | 158 +++++++++++----- .../src/runner/runner-contract.ts | 85 +++++---- src/commands/schema/cli-help.ts | 2 +- website/docs/docs/installation.md | 2 +- 6 files changed, 308 insertions(+), 146 deletions(-) diff --git a/.device-evidence/CHECKLIST-runner-failures.md b/.device-evidence/CHECKLIST-runner-failures.md index 36c8a3baf3..4bcf44e7be 100644 --- a/.device-evidence/CHECKLIST-runner-failures.md +++ b/.device-evidence/CHECKLIST-runner-failures.md @@ -35,7 +35,9 @@ Expected: exit non-zero, one error object with `hint`, `logPath` and `diagnosticId` are top-level, never inside `details`. Also record the Xcode version (`xcodebuild -version`) so the `signing_no_development_team` fixture in `packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts` can move from -`inherited-sniff-trigger` to `captured`. +`shipped-sniff-trigger` to `captured` and its `xcodeVersion` from `unobserved` to that version. +Paste the whole error so the fixture's `output` can become the capture and its `command` can be +recorded. ### 2. A bundle identifier somebody else already owns -> `bundle_identifier_already_registered` @@ -61,14 +63,32 @@ version so the matching fixture's `provenance` can be upgraded: # devtools_security_developer_mode_disabled (macOS admin state, no device work) DevToolsSecurity -status -# signing_provisioning_profile_missing / signing_style_conflict: point the runner build at a -# profile that is not installed, then read the reason off the same prepare command as above. +# signing_provisioning_profile_missing: point the runner build at a profile that is not installed, +# then read the reason off the same prepare command as above. env AGENT_DEVICE_IOS_TEAM_ID="" \ AGENT_DEVICE_IOS_PROVISIONING_PROFILE="no-such-profile-installed" \ node --experimental-strip-types src/bin.ts --json \ prepare ios-runner --platform ios --device "" ``` -Expected: `details.reason` is `signing_provisioning_profile_missing`, or `signing_style_conflict` -when xcodebuild reports conflicting provisioning settings. If neither reason appears, say which -reason did and treat the fixture as unconfirmed rather than editing the rule to fit. +Expected: `details.reason` is `signing_provisioning_profile_missing`. If a different reason appears, +say which one did and treat the fixture as unconfirmed rather than editing the rule to fit. + +### 4. The line that claims no reason yet -> `build_failed_unclassified` + +`xcodebuild` reports a settings mismatch with a line that names a profile ("has conflicting +provisioning settings"). #2680 deliberately publishes `build_failed_unclassified` for it, because no +capture has proved which lever clears it. To reach it, pin a profile while leaving automatic signing +on: + +```sh +env AGENT_DEVICE_IOS_TEAM_ID="" \ + AGENT_DEVICE_IOS_PROVISIONING_PROFILE="match-development" \ + node --experimental-strip-types src/bin.ts --json \ + prepare ios-runner --platform ios --device "" +``` + +Expected: either `signing_provisioning_profile_missing` (xcodebuild complained about the profile +first) or `build_failed_unclassified`. Paste the error and the `xcodebuild -version` either way: a +capture of the conflicting-settings line is what would let a follow-up name the cause, and the +capture must show which build setting disagrees before any hint naming a lever is written. diff --git a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts index 7c524b068b..95d392567e 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts @@ -4,121 +4,157 @@ import type { RunnerStartupFailureReason } from '../runner-contract.ts'; /** * Recorded startup failures for {@link classifyRunnerStartupFailure} (#2680). * - * Each entry carries the tool output exactly as it reaches the host, the command that produced it, - * the Xcode that produced it, and how it got here. `provenance` is what says whether a line was - * observed or transcribed: an entry stays `inherited-sniff-trigger` until the matching command in - * `.device-evidence/CHECKLIST.md` is run against real hardware, at which point `output` is replaced - * with the capture and `provenance` becomes `captured`. Nothing in the classifier reads these - * fields — they exist so a reason can be traced to an observation instead to a guess. + * Provenance is the point of this file, so it is stated per entry and never as a blanket claim: + * + * - `captured` — `output` was pasted from a run, and `command` plus `xcodeVersion` (from + * `xcodebuild -version`) were recorded with it by `.device-evidence/CHECKLIST.md`. + * - `shipped-sniff-trigger` — the substrings a rule matches are the ones shipped in + * `resolveSigningFailureHint` before #2680, which is evidence xcodebuild can emit them. The + * sentence around them is ours, so `command` and `xcodeVersion` stay unrecorded. + * - `invented-shape` — no shipped trigger and no capture. The entry exists to exercise a rule and + * makes no claim about wording xcodebuild prints. + * + * Until Phase B captures the real runs, every entry is `unobserved` for `xcodeVersion` and carries + * no `command`: an invocation we did not run is not provenance. Nothing in the classifier reads + * these fields; they exist so a reason can be traced to an observation instead of to a guess. */ export type RunnerStartupFailureSite = 'build-for-testing' | 'host-dev-tools-security'; +/** + * Whether the text reaches the build catch inside the exec error's `details` (`exec-details`, which + * is how a non-zero `xcodebuild` arrives) or only in the thrown message (`message-only`, which is + * how anything the exec layer raised as a plain `Error` arrives after the catch wraps `String(err)`). + */ +export type RunnerStartupFailureCarrier = 'exec-details' | 'message-only'; + +const UNOBSERVED = 'unobserved'; + export type RunnerStartupFailureFixture = Readonly<{ + /** Stable name for a focused test or a review comment. */ + id: string; /** The reason this output must reach the caller with. */ reason: RunnerStartupFailureReason; /** Which throw site receives this output. */ site: RunnerStartupFailureSite; - /** The invocation that produced {@link RunnerStartupFailureFixture.output}. */ - command: string; - /** `xcodebuild -version` of the machine that produced it. */ + carrier?: RunnerStartupFailureCarrier; + /** The invocation that produced {@link RunnerStartupFailureFixture.output}, once one is recorded. */ + command?: string; + /** `xcodebuild -version` recorded from that run, or `unobserved`. */ xcodeVersion: string; - provenance: 'captured' | 'inherited-sniff-trigger' | 'tool-error-shape'; - /** The tool's own stdout/stderr, kept on one shape so JSON detail matching sees it as the host does. */ + provenance: 'captured' | 'shipped-sniff-trigger' | 'invented-shape'; + /** The tool's own stdout/stderr. */ output: string; - /** What the pending capture still has to show. */ + /** The argv the exec reported, which is never evidence of a cause (#2680). */ + args?: readonly string[]; + /** What the pending capture still has to show, and how to reach it. */ note?: string; }>; -/** - * The `xcodebuild` invocation `buildRunnerXctestrun` issues for a physical iOS device with Automatic - * Signing and no profile pinned, which is the configuration every signing reason below is about. - */ -const BUILD_FOR_TESTING_COMMAND = - 'xcodebuild build-for-testing -project apple/runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj ' + - '-scheme AgentDeviceRunner -parallel-testing-enabled NO -destination generic/platform=iOS ' + - '-derivedDataPath -allowProvisioningUpdates CODE_SIGN_STYLE=Automatic ' + - 'DEVELOPMENT_TEAM='; - -const OBSERVED_ON = 'Xcode 26.2 (Build 17C52)'; - export const RUNNER_STARTUP_FAILURE_FIXTURES: readonly RunnerStartupFailureFixture[] = [ { + id: 'bundle-id-registration-failed', reason: 'bundle_identifier_already_registered', site: 'build-for-testing', - command: BUILD_FOR_TESTING_COMMAND, - xcodeVersion: OBSERVED_ON, - provenance: 'inherited-sniff-trigger', + xcodeVersion: UNOBSERVED, + provenance: 'shipped-sniff-trigger', output: - "error: Failed registering bundle identifier \"com.yourname.agentdevice.runner\" with the developer portal: An App ID with Identifier 'com.yourname.agentdevice.runner' is not available. Please enter a different string. (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", - note: 'Capture with AGENT_DEVICE_IOS_BUNDLE_ID set to an identifier already registered by another team.', + "error: Failed registering bundle identifier \"com.yourname.agentdevice.runner\" with the developer portal (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + note: 'Capture with AGENT_DEVICE_IOS_BUNDLE_ID set to an identifier already registered by another team, and record the `xcodebuild -version` of the machine.', }, { + id: 'app-id-not-available', reason: 'bundle_identifier_already_registered', site: 'build-for-testing', - command: BUILD_FOR_TESTING_COMMAND, - xcodeVersion: OBSERVED_ON, - provenance: 'tool-error-shape', + xcodeVersion: UNOBSERVED, + provenance: 'shipped-sniff-trigger', output: - "error: App Identifier 'com.yourname.agentdevice.runner' is not available. Choose a different App Identifier, or register it in your Apple Developer account before building. (in target 'AgentDeviceRunnerUITests' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", - note: 'The second shape of the same cause: no "failed registering" line, so only the two-part match can name it.', + "error: App Identifier 'com.yourname.agentdevice.runner' is not available (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + note: 'The second shape of the same cause: no "failed registering" line, so only the two-part "app identifier" + "not available" trigger can name it. Trimmed to the shipped trigger; the real sentence is still unrecorded.', }, { + id: 'requires-development-team', reason: 'signing_no_development_team', site: 'build-for-testing', - command: BUILD_FOR_TESTING_COMMAND, - xcodeVersion: OBSERVED_ON, - provenance: 'inherited-sniff-trigger', + xcodeVersion: UNOBSERVED, + provenance: 'shipped-sniff-trigger', output: - "error: Signing for \"AgentDeviceRunner\" requires a development team. Select a development team in the Signing & Capabilities editor. (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + "error: Signing for \"AgentDeviceRunner\" requires a development team (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", note: 'Capture with AGENT_DEVICE_IOS_TEAM_ID unset on a signed-in-but-team-less account.', }, { + id: 'requires-development-team-message-only', + reason: 'signing_no_development_team', + site: 'build-for-testing', + carrier: 'message-only', + xcodeVersion: UNOBSERVED, + provenance: 'shipped-sniff-trigger', + output: + "error: Signing for \"AgentDeviceRunner\" requires a development team (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')", + note: 'Same text arriving in the thrown message instead of the exec details: the catch wraps a non-AppError with String(err), and the rule still has to see it.', + }, + { + id: 'no-profiles-for-bundle-id', reason: 'signing_provisioning_profile_missing', site: 'build-for-testing', - command: BUILD_FOR_TESTING_COMMAND, - xcodeVersion: OBSERVED_ON, - provenance: 'inherited-sniff-trigger', + xcodeVersion: UNOBSERVED, + provenance: 'shipped-sniff-trigger', output: - "error: No profiles for 'com.yourname.agentdevice.runner' were found: Xcode couldn't find any iOS App Development provisioning profiles matching 'com.yourname.agentdevice.runner'. Automatic signing is disabled and unable to generate a profile. To enable automatic signing, pass -allowProvisioningUpdates to xcodebuild. (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + "error: No profiles for 'com.yourname.agentdevice.runner' were found (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", note: 'Capture with AGENT_DEVICE_IOS_PROVISIONING_PROFILE naming a profile that is not installed.', }, { - reason: 'signing_style_conflict', + id: 'conflicting-provisioning-settings', + reason: 'build_failed_unclassified', site: 'build-for-testing', - command: BUILD_FOR_TESTING_COMMAND, - xcodeVersion: OBSERVED_ON, - provenance: 'tool-error-shape', + xcodeVersion: UNOBSERVED, + provenance: 'invented-shape', output: - 'error: "AgentDeviceRunner" has conflicting provisioning settings. AgentDeviceRunner is automatically signed, but provisioning profile "match-development-com-yourname-agentdevice-runner" has been manually specified. Set the provisioning profile value to "Automatic" in the build settings editor, or switch to manual signing in the Signing & Capabilities editor. (in target \'AgentDeviceRunner\' from project \'AgentDeviceRunner\')\n** TEST BUILD FAILED **\n', - note: 'New reason: capture the conflicting-settings line before claiming this wording on hardware.', + "error: \"AgentDeviceRunner\" has conflicting provisioning settings (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + note: 'Names a profile while saying the settings disagree, so the profile row must not win. No reason is claimed until a capture proves which lever clears it.', }, { + id: 'code-signing-required', reason: 'signing_unspecified', site: 'build-for-testing', - command: BUILD_FOR_TESTING_COMMAND, - xcodeVersion: OBSERVED_ON, - provenance: 'inherited-sniff-trigger', + xcodeVersion: UNOBSERVED, + provenance: 'shipped-sniff-trigger', output: - "error: Code signing is required for product type 'Application' in SDK 'iOS 26.2' (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + "error: Code signing is required for product type 'Application' (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", note: 'Signing is named and nothing above it is: the reason stays unspecified on purpose.', }, { + id: 'compile-error', reason: 'build_failed_unclassified', site: 'build-for-testing', - command: BUILD_FOR_TESTING_COMMAND, - xcodeVersion: OBSERVED_ON, - provenance: 'tool-error-shape', + xcodeVersion: UNOBSERVED, + provenance: 'invented-shape', output: "error: cannot find 'AgentDeviceRunnerCommand' in scope (in target 'AgentDeviceRunnerUITests' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", note: 'Any build failure that names no signing fact must keep the cache-recovery hint.', }, { + id: 'argv-names-a-provisioning-profile', + reason: 'build_failed_unclassified', + site: 'build-for-testing', + xcodeVersion: UNOBSERVED, + provenance: 'invented-shape', + output: + "error: cannot find 'AgentDeviceRunnerCommand' in scope (in target 'AgentDeviceRunnerUITests' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + args: [ + 'build-for-testing', + 'PROVISIONING_PROFILE_SPECIFIER=match-development', + 'Provisioning Profile: match-development', + ], + note: 'The argv we were asked to run is not xcodebuild evidence: a caller who pinned a profile still gets cache-recovery advice for a compile error (#2680).', + }, + { + id: 'devtools-security-disabled', reason: 'devtools_security_developer_mode_disabled', site: 'host-dev-tools-security', command: 'DevToolsSecurity -status', - xcodeVersion: OBSERVED_ON, - provenance: 'inherited-sniff-trigger', + xcodeVersion: UNOBSERVED, + provenance: 'shipped-sniff-trigger', output: 'Developer mode is currently disabled for development tools.\n', note: "Host-side refusal. It says nothing about the device's Developer Mode toggle (#2683 reads that).", }, @@ -128,21 +164,30 @@ export function buildForTestingFixtures(): RunnerStartupFailureFixture[] { return RUNNER_STARTUP_FAILURE_FIXTURES.filter((fixture) => fixture.site === 'build-for-testing'); } +export function buildFixtureById(id: string): RunnerStartupFailureFixture { + const fixture = RUNNER_STARTUP_FAILURE_FIXTURES.find((candidate) => candidate.id === id); + if (!fixture) throw new Error(`no startup failure fixture records ${id}`); + return fixture; +} + /** - * The error the exec layer hands the build-failure catch when `xcodebuild` exits non-zero: a - * COMMAND_FAILED whose message is the exec's own and whose tool output sits in `details`, which is - * exactly why the rules below read details text and not only the message. + * What the exec layer hands the build-failure catch: for `exec-details` a COMMAND_FAILED carrying + * the tool's output and the argv in `details` (`execFailureDetails` shape), and for `message-only` + * the plain `Error` the catch turns into `new AppError('COMMAND_FAILED', String(error))`. */ -export function buildForTestingExecError( - fixture: Pick, +export function buildForTestingExecFailure( + fixture: RunnerStartupFailureFixture, exitCode = 65, -): AppError { - return new AppError('COMMAND_FAILED', 'xcodebuild exited with code 65', { +): unknown { + if ((fixture.carrier ?? 'exec-details') === 'message-only') { + return new Error(`xcodebuild exited with code ${exitCode}: ${fixture.output}`); + } + return new AppError('COMMAND_FAILED', `xcodebuild exited with code ${exitCode}`, { stdout: fixture.output, stderr: '', exitCode, processExitError: true, cmd: 'xcodebuild', - args: ['build-for-testing'], + args: fixture.args ?? ['build-for-testing'], }); } diff --git a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-reasons.test.ts b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-reasons.test.ts index 4d27767648..67fd3984f6 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-reasons.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-reasons.test.ts @@ -9,16 +9,20 @@ import type { ExecResult } from '../host.ts'; import { createRunnerPhaseBudget, ensureXctestrunArtifact } from '../runner-xctestrun.ts'; import { RUNNER_ERROR_RULES, + classifyRunnerStartupFailure, RUNNER_STARTUP_FAILURE_REASONS, RUNNER_STARTUP_FAILURE_UNCLASSIFIED_REASON, type RunnerStartupFailureReason, } from '../runner-contract.ts'; +import { assertDevToolsSecurityForIosRunner } from '../runner-dev-tools-security.ts'; import { appleToolchainProbeResult } from './apple-toolchain-fixtures.ts'; import { IOS_DEVICE } from './device-fixtures.ts'; import { RUNNER_STARTUP_FAILURE_FIXTURES, - buildForTestingExecError, + buildFixtureById, + buildForTestingExecFailure, buildForTestingFixtures, + type RunnerStartupFailureFixture, } from './runner-startup-failure-fixtures.ts'; import { mkdtempForTestSync } from './tmp-dir.ts'; @@ -30,8 +34,10 @@ import { mkdtempForTestSync } from './tmp-dir.ts'; * assertion, and the hint beside it has to be the hint the rule that named the reason carries. * * The envelope assertions are deliberate: `normalizeError` moves `hint`, `logPath` and - * `diagnosticId` out of `details` to the top level, so a reason that survives in `details` and a - * hint that survives at top level are two different claims about where the error was built. + * `diagnosticId` out of `details` to the top level, and it is called here with no `logPath` + * fallback — a top-level `logPath` therefore proves the build catch put it there. The negative + * cases matter just as much: argv, our own emitted reason, and wording without a typed fact behind + * it must all stay unclassified. */ const CACHE_RECOVERY_HINT = /clean:xcuitest|apple-runner\/derived/; @@ -40,7 +46,6 @@ const HINT_FOR_REASON: Record = { bundle_identifier_already_registered: /AGENT_DEVICE_IOS_BUNDLE_ID/, signing_no_development_team: /AGENT_DEVICE_IOS_TEAM_ID/, signing_provisioning_profile_missing: /AGENT_DEVICE_IOS_PROVISIONING_PROFILE/, - signing_style_conflict: /CODE_SIGN_STYLE/, signing_unspecified: /Automatic Signing/, devtools_security_developer_mode_disabled: /DevToolsSecurity -enable/, build_failed_unclassified: CACHE_RECOVERY_HINT, @@ -48,6 +53,7 @@ const HINT_FOR_REASON: Record = { const runCmdSync = vi.fn(); const runCmdStreaming = vi.fn(); +const runAppleToolCommand = vi.fn(); const DIAGNOSTIC_ID = 'diag-build-failure-1'; let projectRoot: string; let derivedPath: string; @@ -70,9 +76,11 @@ beforeEach(() => { stdout: '', stderr: '', })); + runAppleToolCommand.mockReset().mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }); appleRunnerTestHost.update({ runCmdSync, runCmdStreaming, + runAppleToolCommand, findProjectRoot: () => projectRoot, readVersion: () => '0.0.0-test', }); @@ -83,13 +91,15 @@ afterEach(() => { }); for (const fixture of buildForTestingFixtures()) { - test(`a build-for-testing failure publishes ${fixture.reason}`, async () => { - const envelope = await driveBuildFailure(buildForTestingExecError(fixture)); + test(`a build-for-testing failure publishes ${fixture.reason} for ${fixture.id}`, async () => { + const envelope = await driveBuildFailure(fixture); assert.equal(envelope.code, 'COMMAND_FAILED'); assert.equal(envelope.message, 'xcodebuild build-for-testing failed'); assert.equal(envelope.details?.reason, fixture.reason); assert.match(String(envelope.hint), HINT_FOR_REASON[fixture.reason]); + // No `logPath` was handed to `normalizeError`: the top-level value can only be the one the + // build catch wrote into the error it throws. assert.equal(envelope.logPath, logPath); assert.equal(envelope.diagnosticId, DIAGNOSTIC_ID); // normalizeError hoists these out of `details`; a caller must read them at top level. @@ -98,9 +108,14 @@ for (const fixture of buildForTestingFixtures()) { assert.equal(envelope.details?.diagnosticId, undefined); // The tool output stays reachable for a human reading the failure. It is redacted and // length-bounded on the way out, which is another reason the reason is typed: classification - // happens before the truncation a caller sees. - const nestedDetails = envelope.details?.details as Record | undefined; - assert.match(String(nestedDetails?.stdout), /AgentDeviceRunner/); + // happens before the truncation a caller sees. A message-only failure carries no tool output to + // reach, which is exactly why the message is part of the haystack. + if ((fixture.carrier ?? 'exec-details') === 'exec-details') { + const nestedDetails = envelope.details?.details as Record | undefined; + assert.match(String(nestedDetails?.stdout), /AgentDeviceRunner/); + } else { + assert.equal(envelope.details?.details, undefined); + } }); } @@ -125,17 +140,54 @@ test('every reason the classifier can name is produced by a rule row', () => { } }); +test('an argv that names a provisioning profile is not evidence of a signing failure', async () => { + // The exec reports the invocation we asked for in `details.args`. Reading the whole details bag + // would let a caller's own pinned profile name the cause of an unrelated compile error and take + // the cache-recovery hint with it (#2680). + const argvFixture = buildFixtureById('argv-names-a-provisioning-profile'); + + const envelope = await driveBuildFailure(argvFixture); + + assert.equal(envelope.details?.reason, RUNNER_STARTUP_FAILURE_UNCLASSIFIED_REASON); + assert.match(String(envelope.hint), CACHE_RECOVERY_HINT); + assert.doesNotMatch(String(envelope.hint), /AGENT_DEVICE_IOS_PROVISIONING_PROFILE/); +}); + +test('a signing sentence that arrives only in the thrown message is still classified', async () => { + // The catch wraps a non-AppError as `new AppError('COMMAND_FAILED', String(error))`, so the tool's + // sentence can reach the classifier in the message with no details behind it (#2680). + const messageOnly = buildFixtureById('requires-development-team-message-only'); + + const envelope = await driveBuildFailure(messageOnly); + + assert.equal(envelope.details?.reason, 'signing_no_development_team'); + assert.match(String(envelope.hint), /AGENT_DEVICE_IOS_TEAM_ID/); +}); + +test('the failure the build catch publishes does not classify itself', async () => { + // The wrapper carries `reason` and `hint` in its details. Re-running the classifier over it must + // not read our own verdict back out of the bag the rules scan (#2680). + const signingFixture = buildFixtureById('requires-development-team'); + const published = await runBuildCatch(() => buildForTestingExecFailure(signingFixture)); + + const reclassified = classifyRunnerStartupFailure(published); + + assert.equal(reclassified.reason, RUNNER_STARTUP_FAILURE_UNCLASSIFIED_REASON); + assert.match(reclassified.hint, CACHE_RECOVERY_HINT); +}); + test('an identical message without the typed host fact is not read as a DevToolsSecurity refusal', async () => { - // Same message the host probe throws; the only difference is the typed `devToolsSecurityStatus` - // fact the probe publishes. Text alone must not activate a reason (#2680). - const withoutFact = new AppError('COMMAND_FAILED', 'Developer mode is disabled', { + // The exact sentence the host probe throws, minus the typed `devToolsSecurityStatus` fact only the + // probe publishes. Text alone must not activate the reason (#2680). + const hostRefusal = await expectHostRefusal(); + const withoutFact = new AppError('COMMAND_FAILED', hostRefusal.message, { stdout: 'developer mode is disabled\n', stderr: '', exitCode: 65, processExitError: true, }); - const envelope = await driveBuildFailure(withoutFact); + const envelope = await driveBuildRejection(withoutFact); assert.equal(envelope.details?.reason, RUNNER_STARTUP_FAILURE_UNCLASSIFIED_REASON); assert.match(String(envelope.hint), CACHE_RECOVERY_HINT); @@ -143,10 +195,11 @@ test('an identical message without the typed host fact is not read as a DevTools }); test('an app identifier named without the availability fact is not read as a taken bundle id', async () => { - const nearMiss = buildForTestingExecError({ + const nearMiss: RunnerStartupFailureFixture = { + ...buildFixtureById('app-id-not-available'), output: - "error: App Identifier 'com.yourname.agentdevice.runner' is invalid. (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n", - }); + "error: App Identifier 'com.yourname.agentdevice.runner' is invalid (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + }; const envelope = await driveBuildFailure(nearMiss); @@ -155,36 +208,34 @@ test('an app identifier named without the availability fact is not read as a tak assert.doesNotMatch(String(envelope.hint), /AGENT_DEVICE_IOS_BUNDLE_ID/); }); -test('a signing failure that only names code signing keeps the generic signing advice', async () => { - const generic = buildForTestingExecError({ - output: "error: Code signing is required for product type 'Application' in SDK 'iOS 26.2'\n", - }); +test('a conflicting-settings failure is not answered with missing-profile advice', async () => { + // The conflicting-settings line names a profile while explaining that the settings disagree. It + // precedes the profile row and claims no cause of its own (#2680). + const conflict = buildFixtureById('conflicting-provisioning-settings'); - const envelope = await driveBuildFailure(generic); + const envelope = await driveBuildFailure(conflict); - assert.equal(envelope.details?.reason, 'signing_unspecified'); - assert.match(String(envelope.hint), /Automatic Signing/); - assert.doesNotMatch(String(envelope.hint), CACHE_RECOVERY_HINT); + assert.equal(envelope.details?.reason, RUNNER_STARTUP_FAILURE_UNCLASSIFIED_REASON); + assert.match(String(envelope.hint), CACHE_RECOVERY_HINT); + assert.doesNotMatch(String(envelope.hint), /AGENT_DEVICE_IOS_PROVISIONING_PROFILE/); }); -test('a conflicting-settings failure is not downgraded to a missing profile', async () => { - // The conflicting-settings line names a profile while explaining that the styles disagree, so - // the more specific row has to win the race the generic profile row would also run. - const conflict = buildForTestingExecError({ - output: - 'error: "AgentDeviceRunner" has conflicting provisioning settings. AgentDeviceRunner is automatically signed, but provisioning profile "match-development" has been manually specified.\n', - }); - - const envelope = await driveBuildFailure(conflict); +/** Drives a recorded fixture through the real build catch and normalizes what it threw. */ +async function driveBuildFailure(fixture: RunnerStartupFailureFixture): Promise { + return normalizeThrown(await runBuildCatch(() => buildForTestingExecFailure(fixture))); +} - assert.equal(envelope.details?.reason, 'signing_style_conflict'); - assert.match(String(envelope.hint), /CODE_SIGN_STYLE/); -}); +/** Drives a hand-built rejection through the same real build catch. */ +async function driveBuildRejection(rejection: unknown): Promise { + return normalizeThrown(await runBuildCatch(() => rejection)); +} -async function driveBuildFailure(execError: AppError): Promise { - runCmdStreaming.mockReset().mockRejectedValue(execError); +async function runBuildCatch(buildRejection: () => unknown): Promise { + runCmdStreaming.mockReset().mockImplementation(async () => { + throw buildRejection(); + }); - let envelope: NormalizedError | undefined; + let caught: unknown; await assert.rejects( () => ensureXctestrunArtifact(IOS_DEVICE, { @@ -192,10 +243,33 @@ async function driveBuildFailure(execError: AppError): Promise budget: createRunnerPhaseBudget(120_000, undefined), }), (error: unknown) => { - envelope = normalizeError(error, { diagnosticId: DIAGNOSTIC_ID, logPath }); + caught = error; + return true; + }, + ); + assert.ok(caught, 'the build-failure catch must throw'); + return caught; +} + +function normalizeThrown(caught: unknown): NormalizedError { + return normalizeError(caught, { diagnosticId: DIAGNOSTIC_ID }); +} + +async function expectHostRefusal(): Promise { + runAppleToolCommand.mockImplementation(async () => ({ + exitCode: 0, + stdout: 'Developer mode is currently disabled for development tools.\n', + stderr: '', + })); + + let caught: unknown; + await assert.rejects( + () => assertDevToolsSecurityForIosRunner(IOS_DEVICE), + (error: unknown) => { + caught = error; return true; }, ); - assert.ok(envelope, 'the build-failure catch must throw'); - return envelope; + assert.ok(caught instanceof AppError); + return caught; } diff --git a/packages/platform-apple/src/runner/runner-contract.ts b/packages/platform-apple/src/runner/runner-contract.ts index 7c9379e505..f53aa2aa97 100644 --- a/packages/platform-apple/src/runner/runner-contract.ts +++ b/packages/platform-apple/src/runner/runner-contract.ts @@ -173,11 +173,11 @@ type RunnerErrorMatch = { /** Every entry must appear in the lowercased message. */ messageIncludesAll?: readonly string[]; /** - * Every entry must appear in the lowercased JSON of the details bag, so a tool's own failure - * text (which arrives in `stderr`/`stdout` rather than in our message) can carry a rule. JSON - * escaping keeps each line whole, so an entry must live on one line of the tool's output. + * Every entry must appear in the lowercased {@link runnerToolText}: our message plus the tool's + * own `stdout`/`stderr`. Nothing else in `details` is read, so the argv we were asked to run and + * the verdict this classifier already published can never carry a rule (#2680). */ - detailsIncludesAll?: readonly string[]; + toolTextIncludesAll?: readonly string[]; /** Required details evidence beyond code/message. */ details?: RunnerErrorDetailsMatch; }; @@ -226,12 +226,17 @@ type RunnerErrorVerdicts = { * disk image state read from the device itself), which is why it is keyed on startup rather than on * `xcodebuild`: an iPhone that refuses the runner for reasons other than signing stops the runner * before a build is ever the question. + * + * Placement: here beside the rules that produce it, not in `@agent-device/contracts`. Every member + * names a verdict an Apple runner path reaches, while `contracts` carries shapes several surfaces + * answer with (`InfrastructureBootFailureReason`, which both simulator and device boot use). + * Nothing outside this package publishes or consumes this enum, and one declaration is the only way + * a row and its reason cannot disagree. */ export const RUNNER_STARTUP_FAILURE_REASONS = [ 'bundle_identifier_already_registered', 'signing_no_development_team', 'signing_provisioning_profile_missing', - 'signing_style_conflict', 'signing_unspecified', 'devtools_security_developer_mode_disabled', 'build_failed_unclassified', @@ -368,16 +373,20 @@ export const RUNNER_ERROR_RULES: readonly RunnerErrorRule[] = [ // to resend. Specific rows precede generic ones: the classifier takes the first match. // // Why these rows are text matchers while the rows above key on a code or a typed field: - // `matchesRunnerErrorMessage`/`matchesRunnerErrorDetailsText` read xcodebuild's own prose because - // that prose is the only publication these failures have — there is no code and no typed field to - // key on. The DevToolsSecurity row is the other half: where a probe of ours publishes a typed - // fact, the row keys on that fact alone. `resolveRunnerEarlyExitHint` stays outside this table for - // the same reason it stays a hint builder — it classifies a runner that DID build and then exited - // early, whose reason axis is the `BootFailureReason` `classifyBootFailure` already returns, and a - // build that never produced a binary has no boot to classify. + // `runnerToolText` reads xcodebuild's own prose because that prose is the only publication these + // failures have — there is no code and no typed field to key on. Its haystack is deliberately + // narrow: our message plus the tool's stdout/stderr, never the whole details bag, which also + // holds the argv we were asked to run (so a caller's own PROVISIONING_PROFILE_SPECIFIER=… would + // otherwise name a signing cause for an unrelated compile error) and the reason and hint this + // classifier just published (so a re-wrapped failure would match itself). The DevToolsSecurity + // row is the other half: where a probe of ours publishes a typed fact, the row keys on that fact + // alone. `resolveRunnerEarlyExitHint` stays outside this table for the same reason it stays a hint + // builder — it classifies a runner that DID build and then exited early, whose reason axis is the + // `BootFailureReason` `classifyBootFailure` already returns, and a build that never produced a + // binary has no boot to classify. { reason: 'bundle_identifier_registration_failed', - match: { detailsIncludesAll: ['failed registering bundle identifier'] }, + match: { toolTextIncludesAll: ['failed registering bundle identifier'] }, verdicts: {}, buildFailure: { reason: 'bundle_identifier_already_registered', @@ -386,7 +395,7 @@ export const RUNNER_ERROR_RULES: readonly RunnerErrorRule[] = [ }, { reason: 'bundle_identifier_unavailable', - match: { detailsIncludesAll: ['app identifier', 'not available'] }, + match: { toolTextIncludesAll: ['app identifier', 'not available'] }, verdicts: {}, buildFailure: { reason: 'bundle_identifier_already_registered', @@ -395,7 +404,7 @@ export const RUNNER_ERROR_RULES: readonly RunnerErrorRule[] = [ }, { reason: 'signing_requires_development_team', - match: { detailsIncludesAll: ['requires a development team'] }, + match: { toolTextIncludesAll: ['requires a development team'] }, verdicts: {}, buildFailure: { reason: 'signing_no_development_team', @@ -403,19 +412,22 @@ export const RUNNER_ERROR_RULES: readonly RunnerErrorRule[] = [ }, }, { - // Precedes the profile rows: this xcodebuild failure names a profile in saying the signing - // styles conflict, and the recovery is to align the settings, not to go install a profile. - reason: 'signing_style_conflict', - match: { detailsIncludesAll: ['conflicting provisioning settings'] }, + // "conflicting provisioning settings" names a profile while saying the automatic and manual + // settings disagree, so without this row the profile row below would send the reader to install + // a profile for a problem that is a settings mismatch. No reason is claimed for it: nothing has + // captured this failure or proved which lever clears it, and advice the reader cannot follow is + // worse than the cache-recovery advice the unclassified path already gives (#2680). + reason: 'conflicting_provisioning_settings_unproven', + match: { toolTextIncludesAll: ['conflicting provisioning settings'] }, verdicts: {}, buildFailure: { - reason: 'signing_style_conflict', - hint: 'The runner project mixes signing styles: one setting asks for automatic signing while another pins a profile or team. Clear AGENT_DEVICE_IOS_PROVISIONING_PROFILE to let Xcode choose, or set CODE_SIGN_STYLE=Manual alongside a matching AGENT_DEVICE_IOS_PROVISIONING_PROFILE, then retry.', + reason: RUNNER_STARTUP_FAILURE_UNCLASSIFIED_REASON, + hint: RUNNER_CACHE_RECOVERY_HINT, }, }, { reason: 'signing_no_profiles_for_bundle_id', - match: { detailsIncludesAll: ['no profiles for'] }, + match: { toolTextIncludesAll: ['no profiles for'] }, verdicts: {}, buildFailure: { reason: 'signing_provisioning_profile_missing', @@ -424,7 +436,7 @@ export const RUNNER_ERROR_RULES: readonly RunnerErrorRule[] = [ }, { reason: 'signing_provisioning_profile_unusable', - match: { detailsIncludesAll: ['provisioning profile'] }, + match: { toolTextIncludesAll: ['provisioning profile'] }, verdicts: {}, buildFailure: { reason: 'signing_provisioning_profile_missing', @@ -435,7 +447,7 @@ export const RUNNER_ERROR_RULES: readonly RunnerErrorRule[] = [ // Signing is involved but nothing above names how: the reason says signing and the hint stays // the generic one it has always carried, rather than naming a misconfiguration no rule proved. reason: 'signing_unspecified', - match: { detailsIncludesAll: ['code signing'] }, + match: { toolTextIncludesAll: ['code signing'] }, verdicts: {}, buildFailure: { reason: 'signing_unspecified', @@ -456,7 +468,7 @@ export const RUNNER_ERROR_RULES: readonly RunnerErrorRule[] = [ function matchesRunnerErrorRule(error: AppError, match: RunnerErrorMatch): boolean { if (match.code !== undefined && error.code !== match.code) return false; if (!matchesRunnerErrorDetails(error, match.details)) return false; - if (!matchesRunnerErrorDetailsText(error, match.detailsIncludesAll)) return false; + if (!matchesRunnerToolText(error, match.toolTextIncludesAll)) return false; return matchesRunnerErrorMessage(error, match.messageIncludesAll); } @@ -471,13 +483,24 @@ function matchesRunnerErrorMessage(error: AppError, parts: readonly string[] | u return parts.every((part) => message.includes(part)); } -function matchesRunnerErrorDetailsText( - error: AppError, - parts: readonly string[] | undefined, -): boolean { +/** + * The only text a startup rule may read: our message plus the tool's own `stdout` and `stderr` + * (#2680). The rest of `details` is deliberately out of reach — `cmd`/`args` describe what we were + * asked to run, and `reason`/`hint` are this classifier's own output, which a re-wrapped failure + * would otherwise find and match again. + */ +function runnerToolText(error: AppError): string { + const details = error.details ?? {}; + return [error.message, details.stdout, details.stderr] + .filter((part): part is string => typeof part === 'string') + .join('\n') + .toLowerCase(); +} + +function matchesRunnerToolText(error: AppError, parts: readonly string[] | undefined): boolean { if (!parts) return true; - const details = error.details ? JSON.stringify(error.details).toLowerCase() : ''; - return parts.every((part) => details.includes(part)); + const text = runnerToolText(error); + return parts.every((part) => text.includes(part)); } function runnerErrorVerdict( diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 225a44e0e8..5a6599bf26 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -689,7 +689,7 @@ iOS physical-device prerequisites: If Xcode cannot choose a profile, set AGENT_DEVICE_IOS_PROVISIONING_PROFILE to the profile name/specifier, not a file path. AGENT_DEVICE_IOS_SIGNING_IDENTITY is optional; omit it unless xcodebuild asks for a specific identity. The profile/team must allow AGENT_DEVICE_IOS_BUNDLE_ID and .uitests. - A runner build failure names its class in error details.reason rather than only in prose: signing_no_development_team, signing_provisioning_profile_missing, signing_style_conflict, bundle_identifier_already_registered, signing_unspecified, devtools_security_developer_mode_disabled (the Mac's DevToolsSecurity setting, which says nothing about the device's Developer Mode toggle), or build_failed_unclassified when nothing proved a cause. Branch on details.reason and follow hint; the message is for humans. + A runner build failure names its class in error details.reason rather than only in prose: signing_no_development_team, signing_provisioning_profile_missing, bundle_identifier_already_registered, signing_unspecified, devtools_security_developer_mode_disabled (the Mac's DevToolsSecurity setting, which says nothing about the device's Developer Mode toggle), or build_failed_unclassified when nothing proved a cause. Branch on details.reason and follow hint; the message is for humans. First-run XCTest setup/build can take longer than normal commands; keep the device connected and use --debug to inspect signing/build diagnostics if setup times out. Android physical-device prerequisites: diff --git a/website/docs/docs/installation.md b/website/docs/docs/installation.md index af0e540f84..62e39676f3 100644 --- a/website/docs/docs/installation.md +++ b/website/docs/docs/installation.md @@ -108,7 +108,7 @@ vega device list - `AGENT_DEVICE_IOS_PROVISIONING_PROFILE` - `AGENT_DEVICE_IOS_BUNDLE_ID` (optional runner bundle-id base override) - Free Apple Developer (Personal Team) accounts can fail with "bundle identifier is not available" for generic IDs; set `AGENT_DEVICE_IOS_BUNDLE_ID` to a unique reverse-DNS value (for example `com.yourname.agentdevice.runner`). -- A runner build failure is typed, not prose: `error.details.reason` is one of `signing_no_development_team`, `signing_provisioning_profile_missing`, `signing_style_conflict`, `bundle_identifier_already_registered`, `signing_unspecified`, `devtools_security_developer_mode_disabled` (the Mac's `DevToolsSecurity` setting, which says nothing about the device's Developer Mode toggle), or `build_failed_unclassified` when nothing proved a cause. Branch on `details.reason` and follow `hint`; the code stays `COMMAND_FAILED` for every reason. +- A runner build failure is typed, not prose: `error.details.reason` is one of `signing_no_development_team`, `signing_provisioning_profile_missing`, `bundle_identifier_already_registered`, `signing_unspecified`, `devtools_security_developer_mode_disabled` (the Mac's `DevToolsSecurity` setting, which says nothing about the device's Developer Mode toggle), or `build_failed_unclassified` when nothing proved a cause. Branch on `details.reason` and follow `hint`; the code stays `COMMAND_FAILED` for every reason. - If device setup is slow, keep the device connected and inspect daemon diagnostics after retrying. - If daemon startup reports stale metadata, remove stale files and retry: - `/daemon.json` From abb0270733adfb6cb38c3396e247c9575c1f56bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 19 Sep 2026 21:41:26 +0200 Subject: [PATCH 5/8] fix(apple): take the exec result type from host-kit and thin the fixture assertion body #2689 derived the runner host port from the modules it fronts, so `ExecResult` is read from `@agent-device/host-kit/command` now rather than restated in `runner/host.ts`. The per-fixture assertion body had grown past the complexity the Fallow audit allows a changed file: the envelope checks move into `assertFailureEnvelope` and the tool-output check into `assertToolOutputReachable`, so each recorded shape is still asserted through the same path and the test body reads as one call. --- .../runner-startup-failure-reasons.test.ts | 70 ++++++++++++------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-reasons.test.ts b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-reasons.test.ts index 67fd3984f6..08fb735483 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-reasons.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-reasons.test.ts @@ -5,7 +5,7 @@ import { afterEach, beforeEach, test, vi } from 'vitest'; import { AppError, normalizeError, type NormalizedError } from '@agent-device/kernel/errors'; import { resetAllProcessMemosForTests } from '@agent-device/kernel/ttl-memo'; import { appleRunnerTestHost } from '../test-host.ts'; -import type { ExecResult } from '../host.ts'; +import type { ExecResult } from '@agent-device/host-kit/command'; import { createRunnerPhaseBudget, ensureXctestrunArtifact } from '../runner-xctestrun.ts'; import { RUNNER_ERROR_RULES, @@ -92,33 +92,53 @@ afterEach(() => { for (const fixture of buildForTestingFixtures()) { test(`a build-for-testing failure publishes ${fixture.reason} for ${fixture.id}`, async () => { - const envelope = await driveBuildFailure(fixture); - - assert.equal(envelope.code, 'COMMAND_FAILED'); - assert.equal(envelope.message, 'xcodebuild build-for-testing failed'); - assert.equal(envelope.details?.reason, fixture.reason); - assert.match(String(envelope.hint), HINT_FOR_REASON[fixture.reason]); - // No `logPath` was handed to `normalizeError`: the top-level value can only be the one the - // build catch wrote into the error it throws. - assert.equal(envelope.logPath, logPath); - assert.equal(envelope.diagnosticId, DIAGNOSTIC_ID); - // normalizeError hoists these out of `details`; a caller must read them at top level. - assert.equal(envelope.details?.hint, undefined); - assert.equal(envelope.details?.logPath, undefined); - assert.equal(envelope.details?.diagnosticId, undefined); - // The tool output stays reachable for a human reading the failure. It is redacted and - // length-bounded on the way out, which is another reason the reason is typed: classification - // happens before the truncation a caller sees. A message-only failure carries no tool output to - // reach, which is exactly why the message is part of the haystack. - if ((fixture.carrier ?? 'exec-details') === 'exec-details') { - const nestedDetails = envelope.details?.details as Record | undefined; - assert.match(String(nestedDetails?.stdout), /AgentDeviceRunner/); - } else { - assert.equal(envelope.details?.details, undefined); - } + assertFailureEnvelope(await driveBuildFailure(fixture), fixture); }); } +/** + * Every startup failure reaches a caller through one envelope: the typed reason in `details`, its hint + * and the log path hoisted to top level by `normalizeError`, and the tool output still reachable + * underneath for a human. The envelope is asserted per fixture rather than once because the reason and + * the hint have to travel together for every recorded shape, not just for one of them. + */ +function assertFailureEnvelope( + envelope: NormalizedError, + fixture: RunnerStartupFailureFixture, +): void { + assert.equal(envelope.code, 'COMMAND_FAILED'); + assert.equal(envelope.message, 'xcodebuild build-for-testing failed'); + assert.equal(envelope.details?.reason, fixture.reason); + assert.match(String(envelope.hint), HINT_FOR_REASON[fixture.reason]); + // No `logPath` was handed to `normalizeError`: the top-level value can only be the one the + // build catch wrote into the error it throws. + assert.equal(envelope.logPath, logPath); + assert.equal(envelope.diagnosticId, DIAGNOSTIC_ID); + // normalizeError hoists these out of `details`; a caller must read them at top level. + assert.equal(envelope.details?.hint, undefined); + assert.equal(envelope.details?.logPath, undefined); + assert.equal(envelope.details?.diagnosticId, undefined); + assertToolOutputReachable(envelope, fixture); +} + +/** + * The tool output stays reachable for a human reading the failure, redacted and length-bounded on the + * way out — one more reason the reason is typed: classification happens before the truncation a caller + * sees. A message-only failure carries no tool output to reach, which is exactly why the message is + * part of the haystack. + */ +function assertToolOutputReachable( + envelope: NormalizedError, + fixture: RunnerStartupFailureFixture, +): void { + if ((fixture.carrier ?? 'exec-details') !== 'exec-details') { + assert.equal(envelope.details?.details, undefined); + return; + } + const nestedDetails = envelope.details?.details as Record | undefined; + assert.match(String(nestedDetails?.stdout), /AgentDeviceRunner/); +} + test('every startup failure reason has a recorded fixture', () => { const reasonsWithFixtures = new Set(RUNNER_STARTUP_FAILURE_FIXTURES.map((f) => f.reason)); From d9c9c5cfc2337271a7fd7a885106adc5345973d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 20 Sep 2026 13:52:27 +0200 Subject: [PATCH 6/8] fix(apple): make a named provisioning profile earn its reason MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One bare `provisioning profile` substring was the shipped sniffer's trigger, and it is a phrase a failing build prints while talking about something else: the codesign invocation, a settings dump, a note about the profile it used. Each row now requires the profile plus the complaint Xcode attaches to it — its `IDEProvisioningErrorDomain` diagnostic, "doesn't include", "has expired" — and a mention that says nothing keeps the cache-recovery advice it already had. --- .device-evidence/CHECKLIST-runner-failures.md | 14 ++++-- .../runner-startup-failure-fixtures.ts | 43 +++++++++++++++++++ .../src/runner/runner-contract.ts | 42 +++++++++++++++--- 3 files changed, 89 insertions(+), 10 deletions(-) diff --git a/.device-evidence/CHECKLIST-runner-failures.md b/.device-evidence/CHECKLIST-runner-failures.md index 4bcf44e7be..0304a3bd76 100644 --- a/.device-evidence/CHECKLIST-runner-failures.md +++ b/.device-evidence/CHECKLIST-runner-failures.md @@ -71,8 +71,13 @@ env AGENT_DEVICE_IOS_TEAM_ID="" \ prepare ios-runner --platform ios --device "" ``` -Expected: `details.reason` is `signing_provisioning_profile_missing`. If a different reason appears, -say which one did and treat the fixture as unconfirmed rather than editing the rule to fit. +Expected: `details.reason` is `signing_provisioning_profile_missing`, and the profile row only fires +when xcodebuild also says what is wrong with the profile — one of its `IDEProvisioningErrorDomain` +diagnostics, "doesn't include ...", or "has expired" (#2688 review). Paste the whole error: which of +those lines printed is what promotes the `profile-xcode-signing-error`, +`profile-does-not-cover-app-id` and `profile-expired` fixtures from `invented-shape` to `captured`. A +different reason is worth recording just as much: say which one and treat the fixtures as unconfirmed +rather than editing the rules to fit. ### 4. The line that claims no reason yet -> `build_failed_unclassified` @@ -88,7 +93,8 @@ env AGENT_DEVICE_IOS_TEAM_ID="" \ prepare ios-runner --platform ios --device "" ``` -Expected: either `signing_provisioning_profile_missing` (xcodebuild complained about the profile -first) or `build_failed_unclassified`. Paste the error and the `xcodebuild -version` either way: a +Expected: either `signing_provisioning_profile_missing` (xcodebuild complained about the profile and +said what was wrong with it) or `build_failed_unclassified` — which is also what a run that merely +mentions the profile it used gets, since a name is not a complaint (#2688 review). Paste the error and the `xcodebuild -version` either way: a capture of the conflicting-settings line is what would let a follow-up name the cause, and the capture must show which build setting disagrees before any hint naming a lever is written. diff --git a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts index 95d392567e..7890c99478 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts @@ -148,6 +148,49 @@ export const RUNNER_STARTUP_FAILURE_FIXTURES: readonly RunnerStartupFailureFixtu ], note: 'The argv we were asked to run is not xcodebuild evidence: a caller who pinned a profile still gets cache-recovery advice for a compile error (#2680).', }, + // Narrowed profile rows (#2688 review): each of these requires the profile AND the complaint Xcode + // attaches to it. The bare phrase alone was the shipped sniffer's trigger and is not evidence, so the + // negative entry below is what keeps those rows honest. + { + id: 'profile-xcode-signing-error', + reason: 'signing_provisioning_profile_missing', + site: 'build-for-testing', + xcodeVersion: UNOBSERVED, + provenance: 'invented-shape', + output: + "error: Provisioning profile \"match-development\" is not a valid provisioning profile (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\nError Domain=IDEProvisioningErrorDomain Code=17\n** TEST BUILD FAILED **\n", + note: "Xcode names the profile beside its own IDEProvisioningErrorDomain diagnostics. Sentence and domain code are our reconstruction; Phase B capture has to record the real wording and this entry's xcodeVersion.", + }, + { + id: 'profile-does-not-cover-app-id', + reason: 'signing_provisioning_profile_missing', + site: 'build-for-testing', + xcodeVersion: UNOBSERVED, + provenance: 'invented-shape', + output: + "error: Provisioning profile \"match-development\" doesn't include application identifier 'com.yourname.agentdevice.runner' (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + note: 'The installed profile that does not cover this app id. Advice is the same lever, so the same reason is published; wording unrecorded.', + }, + { + id: 'profile-expired', + reason: 'signing_provisioning_profile_missing', + site: 'build-for-testing', + xcodeVersion: UNOBSERVED, + provenance: 'invented-shape', + output: + "error: Provisioning profile \"match-development\" has expired (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + note: 'Reinstalling the same profile clears nothing; "a valid profile" in the hint is the operative word. Wording unrecorded.', + }, + { + id: 'profile-mentioned-while-compiling', + reason: 'build_failed_unclassified', + site: 'build-for-testing', + xcodeVersion: UNOBSERVED, + provenance: 'invented-shape', + output: + "note: Using provisioning profile \"match-development\" to sign the app bundle (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\nerror: cannot find 'AgentDeviceRunnerCommand' in scope (in target 'AgentDeviceRunnerUITests' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + note: 'The hazard the bare `provisioning profile` trigger carried (#2688 review): a failing build can print the profile it used while the failure is a compile error. A benign mention must keep cache-recovery advice; it also says nothing Xcode calls code signing, which is its own honest row.', + }, { id: 'devtools-security-disabled', reason: 'devtools_security_developer_mode_disabled', diff --git a/packages/platform-apple/src/runner/runner-contract.ts b/packages/platform-apple/src/runner/runner-contract.ts index f53aa2aa97..75decb32f3 100644 --- a/packages/platform-apple/src/runner/runner-contract.ts +++ b/packages/platform-apple/src/runner/runner-contract.ts @@ -270,6 +270,15 @@ type RunnerErrorRule = { }; }; +/** + * The advice the provisioning-profile rows share (#2688). Named once so the three rows that require a + * different complaint cannot drift into three different fixes for one lever. + */ +const PROFILE_UNUSABLE: RunnerErrorRule['buildFailure'] = { + reason: 'signing_provisioning_profile_missing', + hint: 'Install/select a valid iOS provisioning profile, or set AGENT_DEVICE_IOS_PROVISIONING_PROFILE.', +}; + /** * The one declaration of runner error classes (#1631), mirroring * RUNNER_COMMAND_TRAIT_MANIFEST's role for commands: every recovery predicate @@ -434,14 +443,35 @@ export const RUNNER_ERROR_RULES: readonly RunnerErrorRule[] = [ hint: 'Install/select a valid iOS provisioning profile, or set AGENT_DEVICE_IOS_PROVISIONING_PROFILE.', }, }, + // A profile named in the tool's output is only evidence when the output also says what is wrong with + // it (#2688 review). One bare `provisioning profile` substring was the shipped sniffer's trigger, and + // it is a phrase a failing build can print while talking about something else: the codesign command + // line, a build-settings dump, a note about the profile that was used. Each row below therefore + // requires the profile plus the complaint Xcode attaches to it, and a failure that merely mentions a + // profile stays unclassified rather than being sent to install a profile it already has. { - reason: 'signing_provisioning_profile_unusable', - match: { toolTextIncludesAll: ['provisioning profile'] }, + // Xcode's own signing-error domain beside the profile it rejected: the machine-readable half of its + // `IDEProvisioningErrorDomain` diagnostics, which accompanies the prose rather than replacing it. + reason: 'signing_provisioning_profile_xcode_error', + match: { toolTextIncludesAll: ['provisioning profile', 'ideprovisioningerrordomain'] }, verdicts: {}, - buildFailure: { - reason: 'signing_provisioning_profile_missing', - hint: 'Install/select a valid iOS provisioning profile, or set AGENT_DEVICE_IOS_PROVISIONING_PROFILE.', - }, + buildFailure: PROFILE_UNUSABLE, + }, + { + // "Provisioning profile \"X\" doesn't include application identifier ..." — the profile that is + // installed but does not cover this app or capability. + reason: 'signing_provisioning_profile_does_not_cover', + match: { toolTextIncludesAll: ['provisioning profile', "doesn't include"] }, + verdicts: {}, + buildFailure: PROFILE_UNUSABLE, + }, + { + // "Provisioning profile \"X\" has expired" — installing it again is not the fix; replacing it is, + // which is what the hint's "valid" is for. + reason: 'signing_provisioning_profile_expired', + match: { toolTextIncludesAll: ['provisioning profile', 'expired'] }, + verdicts: {}, + buildFailure: PROFILE_UNUSABLE, }, { // Signing is involved but nothing above names how: the reason says signing and the hint stays From 15808ae228437632c1a1dd5f9eb4ab053c67fcc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 20 Sep 2026 15:53:52 +0200 Subject: [PATCH 7/8] fix(apple): require a profile and its complaint in one line A whole-log AND proves two phrases exist, not that one qualifies the other: a note about the profile the build used, three lines above an unrelated expired-certificate warning, published `signing_provisioning_profile_missing` for a profile that was fine. The profile rows now read one line at a time and the expiry row asks for Xcode's own `has expired` phrase. The rules table grows a `toolTextLineIncludesAll` axis beside `toolTextIncludesAll`, and the bundle-identifier row that also paired two phrases moves onto it so no startup row is left reading a whole log; its cross-line negative rides along. `signing_no_profiles_for_bundle_id` shares the `PROFILE_UNUSABLE` advice instead of spelling it out a third time. `.device-evidence/CHECKLIST.md` belongs to the merged #2682 lane, so this stack's capture sheet lives beside it as `CHECKLIST-runner-failures.md`. --- .device-evidence/CHECKLIST-runner-failures.md | 18 ++++--- .../runner-startup-failure-fixtures.ts | 26 +++++++-- .../src/runner/runner-contract.ts | 54 ++++++++++++++----- 3 files changed, 73 insertions(+), 25 deletions(-) diff --git a/.device-evidence/CHECKLIST-runner-failures.md b/.device-evidence/CHECKLIST-runner-failures.md index 0304a3bd76..3f2a97e088 100644 --- a/.device-evidence/CHECKLIST-runner-failures.md +++ b/.device-evidence/CHECKLIST-runner-failures.md @@ -1,4 +1,4 @@ -# Device evidence checklist +# Runner-failure evidence checklist (#2680, #2683) Live evidence the coordinator runs serially on the connected iPhone. Each item names the exact command, the environment it needs, and the rendered error that proves the change. Do not paraphrase @@ -71,13 +71,15 @@ env AGENT_DEVICE_IOS_TEAM_ID="" \ prepare ios-runner --platform ios --device "" ``` -Expected: `details.reason` is `signing_provisioning_profile_missing`, and the profile row only fires -when xcodebuild also says what is wrong with the profile — one of its `IDEProvisioningErrorDomain` -diagnostics, "doesn't include ...", or "has expired" (#2688 review). Paste the whole error: which of -those lines printed is what promotes the `profile-xcode-signing-error`, -`profile-does-not-cover-app-id` and `profile-expired` fixtures from `invented-shape` to `captured`. A -different reason is worth recording just as much: say which one and treat the fixtures as unconfirmed -rather than editing the rules to fit. +Expected: `details.reason` is `signing_provisioning_profile_missing`, and the profile rows only fire +when xcodebuild says what is wrong with the profile **on the same line as the profile**: its +`IDEProvisioningErrorDomain` diagnostic naming the profile, `doesn't include ...`, or `has expired` +(#2688 review). Paste the whole error and keep the line breaks — which line carried which phrase is +what promotes the `profile-xcode-signing-error`, `profile-does-not-cover-app-id` and `profile-expired` +fixtures from `invented-shape` to `captured`, and a capture that splits the two phrases across lines +belongs to `profile-note-above-an-expired-certificate` instead. A different reason is worth recording +just as much: say which one and treat the fixtures as unconfirmed rather than editing the rules to +fit. ### 4. The line that claims no reason yet -> `build_failed_unclassified` diff --git a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts index 7890c99478..c0adb5e0f1 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts @@ -7,7 +7,7 @@ import type { RunnerStartupFailureReason } from '../runner-contract.ts'; * Provenance is the point of this file, so it is stated per entry and never as a blanket claim: * * - `captured` — `output` was pasted from a run, and `command` plus `xcodeVersion` (from - * `xcodebuild -version`) were recorded with it by `.device-evidence/CHECKLIST.md`. + * `xcodebuild -version`) were recorded with it by `.device-evidence/CHECKLIST-runner-failures.md`. * - `shipped-sniff-trigger` — the substrings a rule matches are the ones shipped in * `resolveSigningFailureHint` before #2680, which is evidence xcodebuild can emit them. The * sentence around them is ours, so `command` and `xcodeVersion` stay unrecorded. @@ -93,6 +93,16 @@ export const RUNNER_STARTUP_FAILURE_FIXTURES: readonly RunnerStartupFailureFixtu "error: Signing for \"AgentDeviceRunner\" requires a development team (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')", note: 'Same text arriving in the thrown message instead of the exec details: the catch wraps a non-AppError with String(err), and the rule still has to see it.', }, + { + id: 'app-identifier-and-availability-in-different-lines', + reason: 'build_failed_unclassified', + site: 'build-for-testing', + xcodeVersion: UNOBSERVED, + provenance: 'invented-shape', + output: + "error: App Identifier 'com.yourname.agentdevice.runner' is invalid (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\nnote: The simulator device is not available for this destination\n** TEST BUILD FAILED **\n", + note: 'The same cross-line hazard the profile rows gave up (#2688 review): one line faults the identifier and another says something is not available, and neither line pairs them. The reason needs both in one sentence, which is what `app-id-not-available` records.', + }, { id: 'no-profiles-for-bundle-id', reason: 'signing_provisioning_profile_missing', @@ -158,8 +168,8 @@ export const RUNNER_STARTUP_FAILURE_FIXTURES: readonly RunnerStartupFailureFixtu xcodeVersion: UNOBSERVED, provenance: 'invented-shape', output: - "error: Provisioning profile \"match-development\" is not a valid provisioning profile (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\nError Domain=IDEProvisioningErrorDomain Code=17\n** TEST BUILD FAILED **\n", - note: "Xcode names the profile beside its own IDEProvisioningErrorDomain diagnostics. Sentence and domain code are our reconstruction; Phase B capture has to record the real wording and this entry's xcodeVersion.", + "error: Provisioning profile \"match-development\" is not a valid provisioning profile (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\nError Domain=IDEProvisioningErrorDomain Code=17 \"Provisioning profile 'match-development' is not a valid provisioning profile.\"\n** TEST BUILD FAILED **\n", + note: "Xcode repeats the profile inside the same line as its IDEProvisioningErrorDomain diagnostics, which is what the row reads: domain on one line and profile on another is two facts, not one complaint. Sentence and domain code are our reconstruction; Phase B capture has to record the real wording and this entry's xcodeVersion.", }, { id: 'profile-does-not-cover-app-id', @@ -191,6 +201,16 @@ export const RUNNER_STARTUP_FAILURE_FIXTURES: readonly RunnerStartupFailureFixtu "note: Using provisioning profile \"match-development\" to sign the app bundle (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\nerror: cannot find 'AgentDeviceRunnerCommand' in scope (in target 'AgentDeviceRunnerUITests' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", note: 'The hazard the bare `provisioning profile` trigger carried (#2688 review): a failing build can print the profile it used while the failure is a compile error. A benign mention must keep cache-recovery advice; it also says nothing Xcode calls code signing, which is its own honest row.', }, + { + id: 'profile-note-above-an-expired-certificate', + reason: 'build_failed_unclassified', + site: 'build-for-testing', + xcodeVersion: UNOBSERVED, + provenance: 'invented-shape', + output: + "note: Using provisioning profile \"match-development\" to sign the app bundle (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\nwarning: The certificate \"Apple Development: Example Dev (ABCD1234)\" has expired.\nerror: cannot find 'AgentDeviceRunnerCommand' in scope (in target 'AgentDeviceRunnerUITests' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", + note: 'The cross-line hazard a whole-log AND cannot see (#2688 review): a benign profile note three lines above an unrelated expired-certificate warning. Both phrases are in the captured log and neither qualifies the other, so the profile stays unclassified and the reader keeps cache-recovery advice rather than being sent to replace a profile that is fine.', + }, { id: 'devtools-security-disabled', reason: 'devtools_security_developer_mode_disabled', diff --git a/packages/platform-apple/src/runner/runner-contract.ts b/packages/platform-apple/src/runner/runner-contract.ts index 75decb32f3..367a56da23 100644 --- a/packages/platform-apple/src/runner/runner-contract.ts +++ b/packages/platform-apple/src/runner/runner-contract.ts @@ -178,6 +178,14 @@ type RunnerErrorMatch = { * the verdict this classifier already published can never carry a rule (#2680). */ toolTextIncludesAll?: readonly string[]; + /** + * Every entry must appear in the SAME line of the lowercased {@link runnerToolText} (#2688 review). + * {@link RunnerErrorMatch.toolTextIncludesAll} proves only that two phrases exist somewhere in a + * captured log, which is a weaker claim than one phrase qualifying the other: a note about the + * profile the build used, three lines above an unrelated expired-certificate warning, says nothing + * about the profile. A row whose evidence is a noun and its complaint asks for both on one line. + */ + toolTextLineIncludesAll?: readonly string[]; /** Required details evidence beyond code/message. */ details?: RunnerErrorDetailsMatch; }; @@ -403,8 +411,11 @@ export const RUNNER_ERROR_RULES: readonly RunnerErrorRule[] = [ }, }, { + // The identifier and its availability have to meet in one line: `App Identifier` and `not + // available` are two phrases a captured log can carry for reasons that have nothing to do with + // each other, which is the same hazard the profile rows just gave up (#2688 review). reason: 'bundle_identifier_unavailable', - match: { toolTextIncludesAll: ['app identifier', 'not available'] }, + match: { toolTextLineIncludesAll: ['app identifier', 'not available'] }, verdicts: {}, buildFailure: { reason: 'bundle_identifier_already_registered', @@ -435,25 +446,26 @@ export const RUNNER_ERROR_RULES: readonly RunnerErrorRule[] = [ }, }, { + // "No profiles for 'com.example' were found" names the bundle id and the absence in one sentence, + // so the phrase alone is the complaint and needs no second phrase to qualify it. reason: 'signing_no_profiles_for_bundle_id', match: { toolTextIncludesAll: ['no profiles for'] }, verdicts: {}, - buildFailure: { - reason: 'signing_provisioning_profile_missing', - hint: 'Install/select a valid iOS provisioning profile, or set AGENT_DEVICE_IOS_PROVISIONING_PROFILE.', - }, + buildFailure: PROFILE_UNUSABLE, }, // A profile named in the tool's output is only evidence when the output also says what is wrong with - // it (#2688 review). One bare `provisioning profile` substring was the shipped sniffer's trigger, and - // it is a phrase a failing build can print while talking about something else: the codesign command - // line, a build-settings dump, a note about the profile that was used. Each row below therefore - // requires the profile plus the complaint Xcode attaches to it, and a failure that merely mentions a - // profile stays unclassified rather than being sent to install a profile it already has. + // that profile, in the same line (#2688 review). One bare `provisioning profile` substring was the + // shipped sniffer's trigger, and it is a phrase a failing build can print while talking about + // something else: the codesign command line, a build-settings dump, a note about the profile that was + // used. Requiring a second phrase somewhere in the same log is no better — a note about the profile + // used above an unrelated `has expired` certificate warning would then name the profile. Each row + // below therefore asks for the profile and Xcode's complaint about it on one line, and a failure that + // merely mentions a profile stays unclassified rather than being sent to install one it already has. { // Xcode's own signing-error domain beside the profile it rejected: the machine-readable half of its // `IDEProvisioningErrorDomain` diagnostics, which accompanies the prose rather than replacing it. reason: 'signing_provisioning_profile_xcode_error', - match: { toolTextIncludesAll: ['provisioning profile', 'ideprovisioningerrordomain'] }, + match: { toolTextLineIncludesAll: ['provisioning profile', 'ideprovisioningerrordomain'] }, verdicts: {}, buildFailure: PROFILE_UNUSABLE, }, @@ -461,15 +473,16 @@ export const RUNNER_ERROR_RULES: readonly RunnerErrorRule[] = [ // "Provisioning profile \"X\" doesn't include application identifier ..." — the profile that is // installed but does not cover this app or capability. reason: 'signing_provisioning_profile_does_not_cover', - match: { toolTextIncludesAll: ['provisioning profile', "doesn't include"] }, + match: { toolTextLineIncludesAll: ['provisioning profile', "doesn't include"] }, verdicts: {}, buildFailure: PROFILE_UNUSABLE, }, { // "Provisioning profile \"X\" has expired" — installing it again is not the fix; replacing it is, - // which is what the hint's "valid" is for. + // which is what the hint's "valid" is for. The full phrase, on the profile's own line: `expired` + // alone is what an expired certificate, a stale session, or a revoked key writes (#2688 review). reason: 'signing_provisioning_profile_expired', - match: { toolTextIncludesAll: ['provisioning profile', 'expired'] }, + match: { toolTextLineIncludesAll: ['provisioning profile', 'has expired'] }, verdicts: {}, buildFailure: PROFILE_UNUSABLE, }, @@ -499,6 +512,7 @@ function matchesRunnerErrorRule(error: AppError, match: RunnerErrorMatch): boole if (match.code !== undefined && error.code !== match.code) return false; if (!matchesRunnerErrorDetails(error, match.details)) return false; if (!matchesRunnerToolText(error, match.toolTextIncludesAll)) return false; + if (!matchesRunnerToolTextLine(error, match.toolTextLineIncludesAll)) return false; return matchesRunnerErrorMessage(error, match.messageIncludesAll); } @@ -533,6 +547,18 @@ function matchesRunnerToolText(error: AppError, parts: readonly string[] | undef return parts.every((part) => text.includes(part)); } +/** + * The same haystack read one line at a time, so a row can require its phrases to be in one sentence + * rather than merely in one file (#2688 review). A captured build log is thousands of lines long, and + * two unrelated lines can hold any pair of words. + */ +function matchesRunnerToolTextLine(error: AppError, parts: readonly string[] | undefined): boolean { + if (!parts) return true; + return runnerToolText(error) + .split('\n') + .some((line) => parts.every((part) => line.includes(part))); +} + function runnerErrorVerdict( error: unknown, axis: Axis, From 12de0a82c9685996b88837a31b89f256bd61da81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 21 Sep 2026 08:53:25 +0200 Subject: [PATCH 8/8] test(ios-runner): record the captured profile row and the host gating the rest need Co-Authored-By: Apex --- .device-evidence/CHECKLIST-runner-failures.md | 46 +++++++++++++++++++ .../runner-startup-failure-fixtures.ts | 26 +++++++++-- 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/.device-evidence/CHECKLIST-runner-failures.md b/.device-evidence/CHECKLIST-runner-failures.md index 3f2a97e088..e9e6a1d5e4 100644 --- a/.device-evidence/CHECKLIST-runner-failures.md +++ b/.device-evidence/CHECKLIST-runner-failures.md @@ -100,3 +100,49 @@ said what was wrong with it) or `build_failed_unclassified` — which is also wh mentions the profile it used gets, since a name is not a complaint (#2688 review). Paste the error and the `xcodebuild -version` either way: a capture of the conflicting-settings line is what would let a follow-up name the cause, and the capture must show which build setting disagrees before any hint naming a lever is written. + +## Results — coordinator run, 2026-09-20 + +Built from `15808ae228` on `thymikee-iphone` (iPhone 17 Pro, iOS 27.0, build 24A437), cabled. + +``` +$ xcodebuild -version +Xcode 26.2 +Build version 17C52 +``` + +Section 3 is captured. A device build pointed at a team with no certificate, on a fresh derived +path so no cached artifact short-circuits it, reaches signing and fails with one long `error:` line +per target: + +``` +.../AgentDeviceRunner.xcodeproj: error: No Accounts: Add a new account in Accounts settings. (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner') +.../AgentDeviceRunner.xcodeproj: error: No profiles for 'com.callstack.agentdevice.runner' were found: Xcode couldn't find any iOS App Development provisioning profiles matching 'com.callstack.agentdevice.runner'. (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner') +``` + +`details.reason` is `signing_provisioning_profile_missing` with the profile hint, and the fixture +`no-profiles-for-bundle-id` is now `captured` with this transcript verbatim. This is the run that +answers the wrapping question the rows were held on: the matched phrase arrives inside one `error:` +line, so the sibling rows in the same provisioning family do not split the way a wrapped line would. + +Sections 1 and 2 are blocked on this account, and the mechanism is worth recording because it is the +same for all of them: against a signed-in account with a valid identity, `xcodebuild` is invoked with +`-allowProvisioningUpdates`, so the build either signs successfully or dies earlier than the +diagnostic a row keys on. + +- Unsetting `AGENT_DEVICE_IOS_TEAM_ID` **succeeds** — automatic signing resolves the team from the + installed identity and reuses an installed team profile. So section 1's `signing_no_development_team` + cannot be induced here; it needs an account signed in with no development team. +- `AGENT_DEVICE_IOS_BUNDLE_ID=com.apple.TestFlight` **succeeds** for the same reason, and a bogus + `AGENT_DEVICE_IOS_PROVISIONING_PROFILE` is repaired rather than honoured. So section 2's + `bundle_identifier_already_registered` needs an app id owned by a different team that automatic + signing cannot register. +- The same gating applies to `bundle_identifier_unavailable` (`App Identifier` + `not available`), + `profile-does-not-cover-app-id` (`Provisioning profile` + `doesn't include`) and `profile-expired` + (`Provisioning profile` + `has expired`): each needs a profile or app id already claimed elsewhere, + which this account will not produce. Recorded beside the fixtures in + `runner-startup-failure-fixtures.ts` so the rows read as host-gated, not unexamined. + +Section 4 needs a build that fails for an unrelated reason while naming no signing fact; the +classifier's behaviour there is pinned by `runner-startup-failure-reasons.test.ts` and needs no +device claim to hold. diff --git a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts index c0adb5e0f1..b9685bed7c 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-startup-failure-fixtures.ts @@ -17,6 +17,22 @@ import type { RunnerStartupFailureReason } from '../runner-contract.ts'; * Until Phase B captures the real runs, every entry is `unobserved` for `xcodeVersion` and carries * no `command`: an invocation we did not run is not provenance. Nothing in the classifier reads * these fields; they exist so a reason can be traced to an observation instead of to a guess. + * + * Blocked on the host, not unexamined. One Phase B run on `thymikee-iphone` / Xcode 26.2 settled the + * shape question these rows were held on — `No profiles for '' were found` arrives as one long + * `error:` line, not a wrapped one — and `no-profiles-for-bundle-id` below is now `captured`. The + * rest are gated on an Apple account this machine does not have, and they need it for the same + * reason: the build either signs successfully or dies before reaching the diagnostic a row keys on. + * `requires-development-team` and `requires-development-team-message-only` need an account that is + * signed in with no development team; automatic signing resolves the team from any installed + * identity, so unsetting `AGENT_DEVICE_IOS_TEAM_ID` builds successfully. + * `bundle-id-registration-failed`, `app-id-not-available` and the `bundle_identifier_unavailable` + * rule it feeds, plus `profile-does-not-cover-app-id` (`Provisioning profile` + `doesn't include`) + * and `profile-expired` (`Provisioning profile` + `has expired`), all need a profile and an app id + * already claimed by someone else: against a working account `-allowProvisioningUpdates` registers + * or repairs the id, so the conflict text is never printed and the build reaches signing success. + * Each of those rows is therefore uninducible here rather than untested, and none of them should be + * read as waiting on effort this machine can supply. */ export type RunnerStartupFailureSite = 'build-for-testing' | 'host-dev-tools-security'; @@ -107,11 +123,13 @@ export const RUNNER_STARTUP_FAILURE_FIXTURES: readonly RunnerStartupFailureFixtu id: 'no-profiles-for-bundle-id', reason: 'signing_provisioning_profile_missing', site: 'build-for-testing', - xcodeVersion: UNOBSERVED, - provenance: 'shipped-sniff-trigger', + command: + 'agent-device prepare ios-runner --platform ios --device --json # AGENT_DEVICE_IOS_TEAM_ID=ZZZZZZZZZZ, fresh AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH', + xcodeVersion: 'Xcode 26.2 / Build version 17C52', + provenance: 'captured', output: - "error: No profiles for 'com.yourname.agentdevice.runner' were found (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n** TEST BUILD FAILED **\n", - note: 'Capture with AGENT_DEVICE_IOS_PROVISIONING_PROFILE naming a profile that is not installed.', + "/Users/thymikee/.t3/worktrees/agent-device/apex-2680/apple/runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj: error: No Accounts: Add a new account in Accounts settings. (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n/Users/thymikee/.t3/worktrees/agent-device/apex-2680/apple/runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj: error: No profiles for 'com.callstack.agentdevice.runner' were found: Xcode couldn't find any iOS App Development provisioning profiles matching 'com.callstack.agentdevice.runner'. (in target 'AgentDeviceRunner' from project 'AgentDeviceRunner')\n", + note: 'Captured on `thymikee-iphone`, iPhone 17 Pro, iOS 27.0. Reached by pointing `AGENT_DEVICE_IOS_TEAM_ID` at a team with no certificate on a machine that is not signed into Xcode, with a fresh derived path so no cached artifact short-circuits the build. One `error:` line per target: the phrase the rule matches is not wrapped, which is the evidence the sibling rows were held for. Note the `No Accounts` line above it names nothing the rule reads — the profile row wins on its own line.', }, { id: 'conflicting-provisioning-settings',