diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 3b5de3cfc1..32320e23d9 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -138,7 +138,7 @@ jobs: path: .tmp/xctest-host if-no-files-found: warn - - name: Build macOS helper + - name: macOS helper build and tests uses: ./.github/actions/run-gate with: { gate: macos-helper } diff --git a/CHANGELOG.md b/CHANGELOG.md index 213f7fd9a9..e7acdb300d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## Unreleased +- Fixed (macos): `press` and `click` on the `frontmost-app`, `desktop`, and `menubar` surfaces post + clicks the way a trackpad does. The helper posted the release in the same tick as the press, so + AppKit dropped it and controls never activated; each press is now held (`--hold-ms`, 60 ms by + default, 40 ms at least). `--count` is that many independent clicks and `--double-tap` posts each + as a double-click pair, the helper's deadline follows the click schedule instead of a fixed 30 s, + and every stop the host applies mid-hold — a cancelled request, a dropped client, the deadline — + reaches the helper as SIGTERM first so it releases the button before it exits; SIGKILL only + follows a helper that has not exited a second later. - Added (diff): `diff screenshot` accepts a JPEG baseline or current image. Both inputs had to be PNG, so a capture exported by another tool had to be converted first and a HarmonyOS capture — which the platform serves as JPEG under whatever name the command was given — could never be compared. Each diff --git a/apple/macos-helper/Package.swift b/apple/macos-helper/Package.swift index 82cd920325..3a472a3ca2 100644 --- a/apple/macos-helper/Package.swift +++ b/apple/macos-helper/Package.swift @@ -11,8 +11,16 @@ let package = Package( ), ], targets: [ + .target( + name: "AgentDeviceMacOSInput" + ), .executableTarget( - name: "AgentDeviceMacOSHelper" + name: "AgentDeviceMacOSHelper", + dependencies: ["AgentDeviceMacOSInput"] + ), + .testTarget( + name: "AgentDeviceMacOSInputTests", + dependencies: ["AgentDeviceMacOSInput"] ), ] ) diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift index 6b77e18683..cc9be1260a 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift @@ -1,3 +1,4 @@ +import AgentDeviceMacOSInput import AppKit import ApplicationServices import CoreGraphics @@ -63,6 +64,10 @@ struct ReadResponse: Encodable { struct PressResponse: Encodable { let x: Double let y: Double + /// Hold actually posted; a short request is raised to the deliverable floor. + let holdMs: Int + let clicks: Int + let doubleClick: Bool let bundleId: String? let surface: String? } @@ -378,10 +383,48 @@ struct AgentDeviceMacOSHelper { throw HelperError.invalidArgs("press requires --x --y ") } + let holdMs = try validatedPressInt( + optionValue(arguments: arguments, name: "--hold-ms"), + name: "--hold-ms", + minimum: 0, + default: 0 + ) + let clicks = try validatedPressInt( + optionValue(arguments: arguments, name: "--clicks"), + name: "--clicks", + minimum: 1, + default: 1 + ) + let intervalMs = try validatedPressInt( + optionValue(arguments: arguments, name: "--interval-ms"), + name: "--interval-ms", + minimum: 0, + default: 120 + ) + + let doubleClick = arguments.contains("--double-click") let bundleId = try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId) let surface = optionValue(arguments: arguments, name: "--surface") - try pressAtPosition(bundleId: bundleId, surface: surface, x: x, y: y) - return SuccessEnvelope(data: PressResponse(x: x, y: y, bundleId: bundleId, surface: surface)) + let request = MouseClickRequest( + x: x, + y: y, + holdMs: holdMs, + clicks: clicks, + doubleClick: doubleClick, + intervalMs: intervalMs + ) + try pressAtPosition(request) + return SuccessEnvelope( + data: PressResponse( + x: x, + y: y, + holdMs: mouseClickHoldMs(requestedMs: holdMs), + clicks: clicks, + doubleClick: doubleClick, + bundleId: bundleId, + surface: surface + ) + ) } static func handleScreenshot(arguments: [String]) throws -> any Encodable { @@ -434,6 +477,21 @@ private func intOption(arguments: [String], name: String) -> Int? { return Int(value) } +private func validatedPressInt( + _ raw: String?, + name: String, + minimum: Int, + default fallback: Int +) throws -> Int { + guard let raw else { + return fallback + } + guard let value = Int(raw), value >= minimum else { + throw HelperError.invalidArgs("press \(name) must be an integer of at least \(minimum)") + } + return value +} + private func readTextAtPosition(bundleId: String?, surface: String?, x: Double, y: Double) throws -> String { let targetApp: NSRunningApplication? if surface == "frontmost-app" || (surface == nil && bundleId != nil) { @@ -481,19 +539,12 @@ private func readTextAtPosition(bundleId: String?, surface: String?, x: Double, throw HelperError.commandFailed("read did not resolve text") } -private func pressAtPosition(bundleId: String?, surface: String?, x: Double, y: Double) throws { - _ = bundleId - _ = surface - let point = CGPoint(x: x, y: y) - guard let move = CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left), - let down = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left), - let up = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left) - else { +private func pressAtPosition(_ request: MouseClickRequest) throws { + do { + try postMouseClick(request) + } catch MouseClickDeliveryError.eventCreationFailed { throw HelperError.commandFailed("press action failed", details: ["reason": "event_creation_failed"]) } - move.post(tap: .cghidEventTap) - down.post(tap: .cghidEventTap) - up.post(tap: .cghidEventTap) } private func captureSurfaceScreenshot(surface: String?, outPath: String, fullscreen: Bool) throws { diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickDelivery.swift b/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickDelivery.swift new file mode 100644 index 0000000000..ee615da7b6 --- /dev/null +++ b/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickDelivery.swift @@ -0,0 +1,116 @@ +import CoreGraphics +import Foundation + +public struct MouseClickRequest: Equatable, Sendable { + public let x: Double + public let y: Double + /// How long the button stays down. Zero asks for the default hold. + public let holdMs: Int + /// Independent presses, each at click state 1. + public let clicks: Int + /// Post every press as a double-click pair with a rising click state. + public let doubleClick: Bool + public let intervalMs: Int + + public init( + x: Double, + y: Double, + holdMs: Int = 0, + clicks: Int = 1, + doubleClick: Bool = false, + intervalMs: Int = 120 + ) { + self.x = x + self.y = y + self.holdMs = holdMs + self.clicks = clicks + self.doubleClick = doubleClick + self.intervalMs = intervalMs + } +} + +public enum MouseClickDeliveryError: Error, Equatable { + case eventCreationFailed +} + +/// The button the helper currently holds down, kept where a signal handler can reach it. +/// A helper killed between a mouse-down and its mouse-up would otherwise leave the system's +/// primary button stuck down for whatever the user touches next. The host stops the helper +/// with SIGTERM before SIGKILL (`runMacOsHelper` in `helper.ts`) so that this handler runs +/// on a deadline and on a cancelled request, not only on a signal sent by hand. +nonisolated(unsafe) private var heldMouseButton: (point: CGPoint, clickState: Int)? + +private func releaseHeldMouseButton() { + guard let held = heldMouseButton else { return } + heldMouseButton = nil + let up = CGEvent( + mouseEventSource: nil, + mouseType: .leftMouseUp, + mouseCursorPosition: held.point, + mouseButton: .left + ) + up?.setIntegerValueField(.mouseEventClickState, value: Int64(held.clickState)) + up?.post(tap: .cghidEventTap) +} + +private func installMouseReleaseOnTermination() { + for terminationSignal in [SIGTERM, SIGINT, SIGHUP] { + signal(terminationSignal) { received in + releaseHeldMouseButton() + _exit(128 + received) + } + } +} + +/// Posts a click the way a trackpad would: one motion to the point, then each press held +/// long enough for the app to accept the release. A termination signal that lands inside a +/// hold releases the button before the process exits. +public func postMouseClick(_ request: MouseClickRequest) throws { + installMouseReleaseOnTermination() + let point = CGPoint(x: request.x, y: request.y) + let hold = mouseClickHoldMs(requestedMs: request.holdMs) + let presses = mouseClickPresses( + clicks: request.clicks, + doubleClick: request.doubleClick, + intervalMs: request.intervalMs + ) + + guard let move = CGEvent( + mouseEventSource: nil, + mouseType: .mouseMoved, + mouseCursorPosition: point, + mouseButton: .left + ) else { + throw MouseClickDeliveryError.eventCreationFailed + } + move.post(tap: .cghidEventTap) + + for press in presses { + if press.delayBeforeMs > 0 { + usleep(UInt32(press.delayBeforeMs) * 1000) + } + guard let down = CGEvent( + mouseEventSource: nil, + mouseType: .leftMouseDown, + mouseCursorPosition: point, + mouseButton: .left + ), let up = CGEvent( + mouseEventSource: nil, + mouseType: .leftMouseUp, + mouseCursorPosition: point, + mouseButton: .left + ) else { + throw MouseClickDeliveryError.eventCreationFailed + } + down.setIntegerValueField(.mouseEventClickState, value: Int64(press.clickState)) + up.setIntegerValueField(.mouseEventClickState, value: Int64(press.clickState)) + heldMouseButton = (point, press.clickState) + down.post(tap: .cghidEventTap) + usleep(UInt32(hold) * 1000) + // The record clears only after the up is posted: a signal that lands between the two + // would otherwise find nothing to release and exit with the button still down. A signal + // that lands after the post releases a button already up, which is harmless. + up.post(tap: .cghidEventTap) + heldMouseButton = nil + } +} diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickSchedule.swift b/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickSchedule.swift new file mode 100644 index 0000000000..0cb31f897d --- /dev/null +++ b/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickSchedule.swift @@ -0,0 +1,61 @@ +import Foundation + +/// AppKit and SwiftUI keep a mouse-down under track long enough to tell a click from a +/// drag, and a mouse-up posted in the same event tick as its mouse-down is never +/// delivered to the app. Measured on an `NSButton`: a 0 ms hold delivered 0 of 15 +/// mouse-ups and activated 0 actions, 20 ms activated 14 of 15, and 40 ms and above +/// activated 15 of 15. Holds shorter than this are raised to it. +public let minimumMouseClickHoldMs = 40 + +/// Hold used when the caller does not name one. Comfortably above +/// `minimumMouseClickHoldMs` so a press does not sit on the measured cliff. +public let defaultMouseClickHoldMs = 60 + +/// The gap between the two presses of one double-click. Well inside the system +/// double-click interval (500 ms by default), and independent of the caller's repeat +/// interval, which separates whole presses rather than the halves of one. +public let mouseClickPairGapMs = 80 + +public func mouseClickHoldMs(requestedMs: Int) -> Int { + if requestedMs <= 0 { + return defaultMouseClickHoldMs + } + return max(requestedMs, minimumMouseClickHoldMs) +} + +/// One press of the button: the click state it is posted with, and how long after the +/// previous release it starts. +public struct MouseClickPress: Equatable, Sendable { + /// `1` for an independent click; `2` for the second half of a double-click. + public let clickState: Int + public let delayBeforeMs: Int + + public init(clickState: Int, delayBeforeMs: Int) { + self.clickState = clickState + self.delayBeforeMs = delayBeforeMs + } +} + +/// The presses one request becomes. `clicks` is always a count of independent presses at +/// click state 1, which is what every other platform means by `--count`; only +/// `doubleClick` raises the state, and it does so per press, so `doubleClick` with three +/// clicks is three double-clicks rather than one triple-click. +public func mouseClickPresses(clicks: Int, doubleClick: Bool, intervalMs: Int) -> [MouseClickPress] { + let gap = max(intervalMs, 0) + var presses: [MouseClickPress] = [] + for index in 0.. Int { + let hold = mouseClickHoldMs(requestedMs: holdMs) + return mouseClickPresses(clicks: clicks, doubleClick: doubleClick, intervalMs: intervalMs) + .reduce(0) { $0 + $1.delayBeforeMs + hold } +} diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSInputTests/MouseClickScheduleTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSInputTests/MouseClickScheduleTests.swift new file mode 100644 index 0000000000..09b30f99ef --- /dev/null +++ b/apple/macos-helper/Tests/AgentDeviceMacOSInputTests/MouseClickScheduleTests.swift @@ -0,0 +1,58 @@ +import XCTest + +@testable import AgentDeviceMacOSInput + +final class MouseClickScheduleTests: XCTestCase { + // The original schedule posted move, down and up back to back. AppKit and SwiftUI + // never delivered the release, so every surface press opened a tracking session that + // no control ever completed: 0 of 15 measured clicks activated their control. + func testHoldShorterThanTheDeliverableFloorIsRaised() { + XCTAssertEqual(mouseClickHoldMs(requestedMs: 0), defaultMouseClickHoldMs) + XCTAssertEqual(mouseClickHoldMs(requestedMs: 1), minimumMouseClickHoldMs) + XCTAssertEqual(mouseClickHoldMs(requestedMs: 39), minimumMouseClickHoldMs) + XCTAssertGreaterThan(defaultMouseClickHoldMs, minimumMouseClickHoldMs) + } + + func testNamedLongPressHoldIsKept() { + XCTAssertEqual(mouseClickHoldMs(requestedMs: 800), 800) + } + + // `--count N` means N independent presses on every platform; a rising click state would + // turn `--count 2` into a double-click and `--count 3` into a triple-click. + func testRepeatClicksAreIndependentPressesAtClickStateOne() { + let presses = mouseClickPresses(clicks: 3, doubleClick: false, intervalMs: 150) + XCTAssertEqual(presses.map(\.clickState), [1, 1, 1]) + XCTAssertEqual(presses.map(\.delayBeforeMs), [0, 150, 150]) + } + + // Only `--double-tap` raises the click state, and it does so per press, so it composes + // with `--count` into that many double-clicks rather than one triple-click. + func testDoubleClickRaisesTheStateInsideEachPress() { + XCTAssertEqual( + mouseClickPresses(clicks: 1, doubleClick: true, intervalMs: 120).map(\.clickState), + [1, 2] + ) + let three = mouseClickPresses(clicks: 3, doubleClick: true, intervalMs: 500) + XCTAssertEqual(three.map(\.clickState), [1, 2, 1, 2, 1, 2]) + XCTAssertEqual( + three.map(\.delayBeforeMs), + [0, mouseClickPairGapMs, 500, mouseClickPairGapMs, 500, mouseClickPairGapMs] + ) + XCTAssertLessThan(mouseClickPairGapMs, 500, "a pair must land inside the system double-click interval") + } + + func testZeroClicksStillPostsOnePress() { + XCTAssertEqual(mouseClickPresses(clicks: 0, doubleClick: false, intervalMs: 0).count, 1) + } + + // The process timeout the caller sets is derived from this: a schedule the timeout does + // not cover is killed mid-hold. + func testScheduleDurationSumsEveryHoldAndGap() { + XCTAssertEqual(mouseClickScheduleMs(holdMs: 0, clicks: 1, doubleClick: false, intervalMs: 120), defaultMouseClickHoldMs) + XCTAssertEqual(mouseClickScheduleMs(holdMs: 10_000, clicks: 4, doubleClick: false, intervalMs: 120), 4 * 10_000 + 3 * 120) + XCTAssertEqual( + mouseClickScheduleMs(holdMs: 60, clicks: 2, doubleClick: true, intervalMs: 100), + 4 * 60 + 2 * mouseClickPairGapMs + 100 + ) + } +} diff --git a/package.json b/package.json index 38886f8796..acb792a2d1 100644 --- a/package.json +++ b/package.json @@ -107,6 +107,8 @@ "package:android-ime-helper:npm": "rm -rf android/ime-helper/dist && AGENT_DEVICE_ANDROID_HELPER=ime sh ./scripts/package-android-helper.sh $(node -p \"require('./package.json').version\") android/ime-helper/dist", "build:macos-helper": "node --experimental-strip-types scripts/swift-toolchain-tmpdir.ts swift build -c release --package-path apple/macos-helper", "build:macos-helper:clean": "node --experimental-strip-types scripts/swift-toolchain-tmpdir.ts swift package --package-path apple/macos-helper clean && pnpm build:macos-helper", + "test:macos-helper": "node --experimental-strip-types scripts/swift-toolchain-tmpdir.ts swift test --package-path apple/macos-helper", + "check:macos-helper": "pnpm build:macos-helper && pnpm test:macos-helper", "prepare:publish-assets": "node scripts/prepare-publish-assets.mjs", "build:package": "pnpm build && pnpm build:xcuitest:ios && pnpm build:xcuitest:macos && pnpm build:xcuitest:tvos && pnpm build:xcuitest:visionos && pnpm build:macos-helper:clean && pnpm prepare:publish-assets", "package:npm": "pnpm build:package && pnpm check:package", diff --git a/packages/host-kit/src/internal/exec-kill-settle.test.ts b/packages/host-kit/src/internal/exec-kill-settle.test.ts index 6b4066b2b0..98de96233d 100644 --- a/packages/host-kit/src/internal/exec-kill-settle.test.ts +++ b/packages/host-kit/src/internal/exec-kill-settle.test.ts @@ -335,6 +335,67 @@ test.runIf(process.platform !== 'win32')( 5_000, ); +// A child that holds something the kill would strand — the macOS helper with the mouse button +// down — asks for a signal it can handle before SIGKILL. The trap script below stands in for +// that helper: it records the release the signal handler performs, and its `sleep` is the +// hold the kill interrupts. + +function releaseMarkerPath(label: string): string { + return path.join(mkdtempForTestSync(`agent-device-exec-${label}-`), 'released'); +} + +function holdUntilSignalledShellScript(markerPath: string): string { + return `trap 'printf released > ${shellQuote(markerPath)}; exit 143' TERM; sleep ${HOLDER_LIFETIME_SECONDS} & wait`; +} + +test.runIf(process.platform !== 'win32')( + 'a cancelled command with a kill policy is signalled so it can release what it holds', + async () => { + const markerPath = releaseMarkerPath('graceful-abort'); + const controller = new AbortController(); + const held = runCmd('/bin/sh', ['-c', holdUntilSignalledShellScript(markerPath)], { + signal: controller.signal, + kill: { signal: 'SIGTERM', graceMs: 2_000 }, + }); + const rejection = settledRejection(held); + await sleep(100); + + controller.abort(); + const outcome = await rejection; + + assert.ok(outcome, 'a cancelled command must not resolve'); + const details = (outcome.error as { details?: Record }).details; + assert.equal(details?.reason, 'request_canceled'); + assert.equal( + fs.readFileSync(markerPath, 'utf8'), + 'released', + 'the child never saw the signal its handler releases on: it was killed outright', + ); + }, +); + +test.runIf(process.platform !== 'win32')( + 'a deadline with a kill policy still ends a child that ignores the first signal', + async () => { + const startedAt = Date.now(); + await assert.rejects( + () => + runCmd('/bin/sh', ['-c', `trap '' TERM; sleep ${HOLDER_LIFETIME_SECONDS} & wait`], { + timeoutMs: DEADLINE_MS, + kill: { signal: 'SIGTERM', graceMs: 200 }, + }), + (error: unknown) => { + assert.equal(isCommandTimeoutError(error), true); + return true; + }, + ); + const elapsedMs = Date.now() - startedAt; + assert.ok(elapsedMs >= DEADLINE_MS + 200, `escalated before the grace passed: ${elapsedMs}ms`); + assert.ok(elapsedMs < 2_000, `the grace became a way to outlive the deadline: ${elapsedMs}ms`); + }, + 10_000, +); + test.runIf(process.platform !== 'win32')( 'runCmd that was never killed still drains output a descendant writes after its parent exited', async () => { diff --git a/packages/host-kit/src/internal/exec.ts b/packages/host-kit/src/internal/exec.ts index 16fc0a092a..7f1f9f8356 100644 --- a/packages/host-kit/src/internal/exec.ts +++ b/packages/host-kit/src/internal/exec.ts @@ -27,6 +27,13 @@ export type ExecOptions = { signal?: AbortSignal; /** Max stdout/stderr bytes for synchronous runs (default Node ~1MB). */ maxBuffer?: number; + /** + * How a deadline or a cancelled request stops the child. Without this it is SIGKILL at + * once. A child that has to undo something before it dies — the macOS helper releasing a + * mouse button it is holding — is sent `signal` first, and SIGKILL only once `graceMs` + * pass without it exiting. + */ + kill?: { readonly signal: NodeJS.Signals; readonly graceMs: number }; }; export type ExecStreamOptions = ExecOptions & { @@ -206,7 +213,7 @@ function runSpawnedCommand( // group kill that would have ended the pipe holder can no longer run through a child // Node already reaped. const settlement = createCommandKillSettlement({ - killProcessTree: () => killProcessTree(child, options.detached), + killProcessTree: () => killProcessTree(child, options), settle, }); const abort = watchCommandAbort(options, settlement.requestKill); @@ -224,7 +231,7 @@ function runSpawnedCommand( if (abort.didAbort || didTimeout) return; if (isEpipeError(error)) return; fail(createStdinError(executable, cmd, args, error)); - killProcessTree(child, options.detached); + killProcessTree(child, options); }); child.stdout.on('data', (chunk) => { @@ -470,7 +477,7 @@ export function runCmdBackground( resolve({ stdout, stderr, exitCode: finalExitCode }); } const settlement = createCommandKillSettlement({ - killProcessTree: () => killProcessTree(child, options.detached), + killProcessTree: () => killProcessTree(child, options), settle, }); const abort = watchCommandAbort(options, settlement.requestKill); @@ -910,9 +917,34 @@ export function signalProcessGroupBestEffort(pid: number, signal: NodeJS.Signals * waiting on. The one group-signal seam reports whether anything was reached rather than * throwing, and a group that is gone or not ours to signal is the case it reports false. */ -function killProcessTree(child: ChildProcess, detached: boolean | undefined): void { +function killProcessTree( + child: ChildProcess, + options: Pick, +): void { + if (!options.kill) { + signalProcessTree(child, options.detached, 'SIGKILL'); + return; + } + // The child is given its chance to clean up, and the escalation is what keeps that + // chance from becoming a way to outlive the deadline. The timer holds nothing open: + // a child that exits on the first signal clears it, and a worker shutting down owes + // a child that ignored the signal nothing further. + signalProcessTree(child, options.detached, options.kill.signal); + const escalation = setTimeout( + () => signalProcessTree(child, options.detached, 'SIGKILL'), + options.kill.graceMs, + ); + escalation.unref(); + child.once('exit', () => clearTimeout(escalation)); +} + +function signalProcessTree( + child: ChildProcess, + detached: boolean | undefined, + signal: NodeJS.Signals, +): void { if (detached && child.pid && process.platform !== 'win32') { - signalProcessGroupBestEffort(child.pid, 'SIGKILL'); + signalProcessGroupBestEffort(child.pid, signal); return; } // A non-detached child leaves its pid free for the kernel to hand to an unrelated @@ -920,7 +952,7 @@ function killProcessTree(child: ChildProcess, detached: boolean | undefined): vo // strike a stranger. Nothing waits for a kill of a child that is already gone: // settlement happens on `exit`. if (child.exitCode !== null || child.signalCode !== null) return; - child.kill('SIGKILL'); + child.kill(signal); } /** diff --git a/packages/platform-apple/src/interactions.ts b/packages/platform-apple/src/interactions.ts index c6e5f07f46..0c61d7486a 100644 --- a/packages/platform-apple/src/interactions.ts +++ b/packages/platform-apple/src/interactions.ts @@ -219,11 +219,20 @@ async function runMacOsSurfacePress( ); } const { runMacOsPressAction } = await import('./os/macos/helper.ts'); - await runMacOsPressAction(point.x, point.y, { + // `count` is independent presses and `doubleTap` raises the click state inside each one, + // the same reading every other platform gives the two flags. + const posted = await runMacOsPressAction(point.x, point.y, { bundleId: context.appBundleId, surface: options.surface, + holdMs: options.holdMs, + clicks: options.count, + doubleClick: options.doubleTap, + intervalMs: options.intervalMs, + signal: context.signal, }); - return {}; + // A hold shorter than what macOS delivers is raised before it is posted, so the + // response carries the hold the helper actually used rather than the request. + return posted.holdMs === undefined ? {} : { holdMs: posted.holdMs }; } async function runAppleAlternateClick( diff --git a/packages/platform-apple/src/os/macos/helper.test.ts b/packages/platform-apple/src/os/macos/helper.test.ts index 1ad8a22eed..eb8d24b0e9 100644 --- a/packages/platform-apple/src/os/macos/helper.test.ts +++ b/packages/platform-apple/src/os/macos/helper.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { createLocalAppleToolProvider, withAppleToolProvider } from '../../core/tool-provider.ts'; -import { runMacOsSnapshotAction } from './helper.ts'; +import { macOsClickScheduleMs, runMacOsPressAction, runMacOsSnapshotAction } from './helper.ts'; test('macOS helper snapshot passes cancellation to the helper process', async () => { const controller = new AbortController(); @@ -34,3 +34,176 @@ test('macOS helper snapshot passes cancellation to the helper process', async () assert.equal(receivedSignal, controller.signal); }); + +function helperReturn(data: Record) { + return { + exitCode: 0, + stdout: JSON.stringify({ ok: true, data }), + stderr: '', + }; +} + +test('macOS helper press carries the hold, repeat count, and interval to the click schedule', async () => { + let receivedArgs: string[] = []; + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async (args) => { + receivedArgs = args; + return helperReturn({ x: 12, y: 34, holdMs: 800, clicks: 2 }); + }, + }, + }); + + const result = await withAppleToolProvider( + provider, + async () => + await runMacOsPressAction(12, 34, { + surface: 'menubar', + bundleId: 'com.example.Menu', + holdMs: 800, + clicks: 2, + intervalMs: 140, + }), + ); + + assert.deepEqual( + [ + receivedArgs.slice(receivedArgs.indexOf('--hold-ms'), receivedArgs.indexOf('--hold-ms') + 2), + receivedArgs.slice(receivedArgs.indexOf('--clicks'), receivedArgs.indexOf('--clicks') + 2), + receivedArgs.slice( + receivedArgs.indexOf('--interval-ms'), + receivedArgs.indexOf('--interval-ms') + 2, + ), + ], + [ + ['--hold-ms', '800'], + ['--clicks', '2'], + ['--interval-ms', '140'], + ], + ); + // The helper reports the hold it delivered, which the caller republishes as the + // press's hold so a clamped request cannot read back as the requested one. + assert.equal(result.holdMs, 800); +}); + +test('macOS helper press carries an explicit zero interval instead of dropping it', async () => { + let receivedArgs: string[] = []; + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async (args) => { + receivedArgs = args; + return helperReturn({ x: 7, y: 8, holdMs: 60, clicks: 2 }); + }, + }, + }); + + await withAppleToolProvider( + provider, + async () => await runMacOsPressAction(7, 8, { surface: 'desktop', clicks: 2, intervalMs: 0 }), + ); + + assert.ok(receivedArgs.includes('--clicks'), receivedArgs.join(' ')); + assert.deepEqual( + receivedArgs.slice( + receivedArgs.indexOf('--interval-ms'), + receivedArgs.indexOf('--interval-ms') + 2, + ), + ['--interval-ms', '0'], + ); +}); + +test('macOS helper press keeps repeats independent and names a double-click explicitly', async () => { + let receivedArgs: string[] = []; + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async (args) => { + receivedArgs = args; + return helperReturn({ x: 7, y: 8, holdMs: 60, clicks: 3, doubleClick: true }); + }, + }, + }); + + await withAppleToolProvider( + provider, + async () => + await runMacOsPressAction(7, 8, { surface: 'frontmost-app', clicks: 3, doubleClick: true }), + ); + + // `--count 3 --double-tap` is three double-clicks: the count stays the press count and the + // rising click state is a separate flag, never derived from the count. + assert.deepEqual( + receivedArgs.slice(receivedArgs.indexOf('--clicks'), receivedArgs.indexOf('--clicks') + 2), + ['--clicks', '3'], + ); + assert.ok(receivedArgs.includes('--double-click'), receivedArgs.join(' ')); +}); + +test('macOS helper press outlives its own click schedule and forwards cancellation', async () => { + let receivedTimeoutMs: number | undefined; + let receivedSignal: AbortSignal | undefined; + let receivedKill: { signal: string; graceMs: number } | undefined; + const controller = new AbortController(); + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async (_args, options) => { + receivedTimeoutMs = options?.timeoutMs; + receivedSignal = options?.signal; + receivedKill = options?.kill; + return helperReturn({ x: 1, y: 2, holdMs: 10_000, clicks: 4 }); + }, + }, + }); + + await withAppleToolProvider( + provider, + async () => + await runMacOsPressAction(1, 2, { + surface: 'desktop', + holdMs: 10_000, + clicks: 4, + intervalMs: 120, + signal: controller.signal, + }), + ); + + // Four ten-second holds are 40.36s of schedule; a fixed 30s timeout would kill the helper + // inside the third hold with the button down. + const scheduleMs = macOsClickScheduleMs({ holdMs: 10_000, clicks: 4, intervalMs: 120 }); + assert.equal(scheduleMs, 40_360); + assert.equal(receivedTimeoutMs, scheduleMs + 30_000); + assert.equal(receivedSignal, controller.signal); + // The host stops the helper with SIGKILL on both routes, and a helper killed between a + // mouse-down and its mouse-up leaves the button stuck. The helper's release handler only + // runs if the stop reaches it as a catchable signal first. + assert.deepEqual(receivedKill, { signal: 'SIGTERM', graceMs: 1_000 }); +}); + +test('macOS click schedule mirrors the helper floors for the timeout it derives', () => { + assert.equal(macOsClickScheduleMs({}), 60); + assert.equal(macOsClickScheduleMs({ holdMs: 5 }), 40); + assert.equal( + macOsClickScheduleMs({ clicks: 2, doubleClick: true, intervalMs: 100 }), + 4 * 60 + 2 * 80 + 100, + ); +}); + +test('macOS helper press stays a single held click when nothing is repeated', async () => { + let receivedArgs: string[] = []; + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async (args) => { + receivedArgs = args; + return helperReturn({ x: 5, y: 6, holdMs: 60, clicks: 1 }); + }, + }, + }); + + await withAppleToolProvider( + provider, + async () => await runMacOsPressAction(5, 6, { surface: 'frontmost-app' }), + ); + + assert.equal(receivedArgs.includes('--clicks'), false); + assert.equal(receivedArgs.includes('--hold-ms'), false); + assert.equal(receivedArgs.includes('--interval-ms'), false); +}); diff --git a/packages/platform-apple/src/os/macos/helper.ts b/packages/platform-apple/src/os/macos/helper.ts index 1b5e660f1a..a0bd002176 100644 --- a/packages/platform-apple/src/os/macos/helper.ts +++ b/packages/platform-apple/src/os/macos/helper.ts @@ -267,14 +267,25 @@ export async function startMacOsAudioProbeProcess(options: { ); } +const MACOS_HELPER_TIMEOUT_MS = 30_000; +/** + * Every stop the host applies to the helper — a deadline, a cancelled request, a client that + * dropped mid-command — reaches it as SIGTERM first. A helper posting a press may be holding + * the mouse button down at that moment, and its SIGTERM handler releases the button before it + * exits; SIGKILL would end it between the down and the up and leave the button stuck for + * whatever the user touches next. A helper that has not exited a second later is killed. + */ +const MACOS_HELPER_KILL_GRACE_MS = 1_000; + async function runMacOsHelper>( args: string[], - options: { signal?: AbortSignal } = {}, + options: { signal?: AbortSignal; timeoutMs?: number } = {}, ): Promise { const helperOptions = { allowFailure: true, - timeoutMs: 30_000, + timeoutMs: options.timeoutMs ?? MACOS_HELPER_TIMEOUT_MS, signal: options.signal, + kill: { signal: 'SIGTERM' as const, graceMs: MACOS_HELPER_KILL_GRACE_MS }, }; const helperProvider = resolveAppleToolProvider().macosHelper; const helperPath = helperProvider @@ -386,19 +397,78 @@ export async function runMacOsReadTextAction( return await runMacOsHelper(args); } +// Mirrors the helper's own floors (`MouseClickSchedule.swift`): the schedule the helper runs +// is derived from the same numbers, so the timeout that must outlast it is derived here too. +const MACOS_CLICK_MINIMUM_HOLD_MS = 40; +const MACOS_CLICK_DEFAULT_HOLD_MS = 60; +const MACOS_CLICK_PAIR_GAP_MS = 80; +const MACOS_CLICK_DEFAULT_INTERVAL_MS = 120; + +/** + * How long the helper stays busy posting one press request: every hold plus every gap, + * exactly as `mouseClickScheduleMs` in the helper sums them. Every path that runs a click + * schedule sets its process timeout from this, because a helper killed mid-hold would leave + * the system's mouse button down. + */ +export function macOsClickScheduleMs(options: { + holdMs?: number; + clicks?: number; + doubleClick?: boolean; + intervalMs?: number; +}): number { + const hold = + options.holdMs && options.holdMs > 0 + ? Math.max(options.holdMs, MACOS_CLICK_MINIMUM_HOLD_MS) + : MACOS_CLICK_DEFAULT_HOLD_MS; + const clicks = Math.max(options.clicks ?? 1, 1); + const interval = Math.max(options.intervalMs ?? MACOS_CLICK_DEFAULT_INTERVAL_MS, 0); + const perPress = options.doubleClick ? 2 * hold + MACOS_CLICK_PAIR_GAP_MS : hold; + return clicks * perPress + (clicks - 1) * interval; +} + export async function runMacOsPressAction( x: number, y: number, - options: { bundleId?: string; surface?: SessionSurface } = {}, + options: { + bundleId?: string; + surface?: SessionSurface; + holdMs?: number; + /** Independent presses, each a single click; `--count` on every platform. */ + clicks?: number; + /** Post each press as a double-click pair; `--double-tap`. */ + doubleClick?: boolean; + intervalMs?: number; + signal?: AbortSignal; + } = {}, ): Promise<{ x: number; y: number; + holdMs?: number; + clicks?: number; + doubleClick?: boolean; bundleId?: string; surface?: SessionSurface; }> { const args = ['press', '--x', String(x), '--y', String(y)]; + if (options.holdMs && options.holdMs > 0) { + args.push('--hold-ms', String(options.holdMs)); + } + const clicks = options.clicks ?? 1; + if (clicks > 1) { + args.push('--clicks', String(clicks)); + // An explicit zero is a request for back-to-back presses, not an unset interval. + if (options.intervalMs !== undefined) { + args.push('--interval-ms', String(Math.max(options.intervalMs, 0))); + } + } + if (options.doubleClick) { + args.push('--double-click'); + } appendMacOsHelperContextArgs(args, options); - return await runMacOsHelper(args); + return await runMacOsHelper(args, { + signal: options.signal, + timeoutMs: macOsClickScheduleMs(options) + MACOS_HELPER_TIMEOUT_MS, + }); } export async function runMacOsScreenshotAction( diff --git a/scripts/check-affected/checks.ts b/scripts/check-affected/checks.ts index 032d3900c6..763df2ca61 100644 --- a/scripts/check-affected/checks.ts +++ b/scripts/check-affected/checks.ts @@ -78,7 +78,7 @@ export const CHECK_CATALOG: readonly CheckSpec[] = [ // helpers packaged into `android/*/dist` (what the replay host verifies), where the // build script writes only the snapshot helper into `.tmp/`. gate('android-helpers', 'Android helper builds (snapshot + IME)', 'build:android', false), - gate('macos-helper', 'macOS helper build', 'build:macos-helper', false), + gate('macos-helper', 'macOS helper build and tests', 'check:macos-helper', false), gate('web-smoke', 'Live web platform smoke', 'test:smoke:web', false), // Needs full history and tags, so it runs in the shared fetch-depth: 0 job // rather than inside the shallow-clone-safe unit lane. diff --git a/test/integration/provider-scenarios/macos-desktop.test.ts b/test/integration/provider-scenarios/macos-desktop.test.ts index f17bad59de..9df52ff9d6 100644 --- a/test/integration/provider-scenarios/macos-desktop.test.ts +++ b/test/integration/provider-scenarios/macos-desktop.test.ts @@ -274,6 +274,24 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help positionals: ['@e2'], expectData: { x: 116, y: 80 }, }, + { + name: 'refresh frontmost refs before the repeat press', + command: 'snapshot', + flags: { snapshotInteractiveOnly: true }, + assert: (snapshot) => { + const general = snapshot.json?.result?.data?.nodes?.find( + (node: { label?: string }) => node.label === 'General', + ); + assert.equal(general?.ref, 'e2', JSON.stringify(snapshot.json)); + }, + }, + { + name: 'double tap snapshot ref', + command: 'press', + positionals: ['@e2'], + flags: { doubleTap: true }, + expectData: { x: 116, y: 80, doubleTap: true }, + }, { name: 'switch to desktop surface', command: 'open', @@ -510,6 +528,19 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help '--surface', 'frontmost-app', ]); + assertFlatToolCall(appleTool.calls, [ + 'macos-helper', + 'press', + '--x', + '116', + '--y', + '80', + '--double-click', + '--bundle-id', + 'com.apple.systempreferences', + '--surface', + 'frontmost-app', + ]); assertFlatToolCall(appleTool.calls, [ 'macos-helper', 'press', diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index ff6c28e4b0..866709a3a0 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -324,6 +324,7 @@ agent-device snapshot -i --platform apple --target desktop - In macOS app sessions, `screenshot` captures the target app window bounds rather than the full desktop. - Prefer selector or `@ref`-driven interactions on macOS. Window position can shift between runs, so raw x/y point commands are less stable than snapshot-derived targets. - Use `click --button secondary` for context menus on macOS, then run `snapshot -i` again. +- On `frontmost-app` and `menubar` surfaces, `press` and `click` post synthetic mouse events through the macOS helper (the `desktop` surface inspects only): `--hold-ms` is how long the button stays down (at least 40 ms, 60 ms by default, because AppKit drops a release posted in the same tick as its press), `--count` is that many independent clicks, and `--double-tap` posts each click as a double-click pair. A long schedule such as `--hold-ms 10000 --count 4` is given the time it needs, and a helper stopped mid-hold — by a cancelled request, a dropped client, or its deadline — releases the button before it exits. `--jitter-px` is not applied on these surfaces. - Mobile-only helpers remain unsupported on macOS: `boot`, `shutdown`, `home`, `orientation`, `app-switcher`, `action-button`, `install`, `reinstall`, `install-from-source`, and `push`. Recommended loops: