From 0dd28a98a2daf217d6ab9e3204d407e3f75d3d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 20 Sep 2026 22:12:31 +0200 Subject: [PATCH 1/4] fix(macos-helper): hold synthetic clicks long enough to be delivered The helper posted mouseMoved, leftMouseDown, and leftMouseUp back to back with no delay. AppKit keeps a mouse-down under track long enough to separate a click from a drag, so the release was dropped: on an NSButton probe a zero-dwell click delivered 0 of 15 mouse-ups and activated 0 actions, 20ms activated 14 of 15, and 40ms and above activated 15 of 15. Every `press` on a macOS surface that has no runner was therefore a visible no-op. Click posting now lives in AgentDeviceMacOSInput as a pure schedule plus a poster, so the dwell and the rising click state are unit-testable without a window server. The default hold is 60ms with a 40ms floor, and --hold-ms, --clicks, and --interval-ms are honoured, which makes --double-tap and --count reach the frontmost-app, desktop, and menubar surfaces as real repeated clicks instead of being dropped. --- apple/macos-helper/Package.swift | 10 +- .../Sources/AgentDeviceMacOSHelper/main.swift | 73 +++++++++++--- .../MouseClickDelivery.swift | 71 ++++++++++++++ .../MouseClickSchedule.swift | 50 ++++++++++ .../MouseClickScheduleTests.swift | 59 ++++++++++++ packages/platform-apple/src/interactions.ts | 10 +- .../src/os/macos/helper.test.ts | 94 ++++++++++++++++++- .../platform-apple/src/os/macos/helper.ts | 20 +++- .../provider-scenarios/macos-desktop.test.ts | 32 +++++++ 9 files changed, 401 insertions(+), 18 deletions(-) create mode 100644 apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickDelivery.swift create mode 100644 apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickSchedule.swift create mode 100644 apple/macos-helper/Tests/AgentDeviceMacOSInputTests/MouseClickScheduleTests.swift diff --git a/apple/macos-helper/Package.swift b/apple/macos-helper/Package.swift index 82cd920325..3a472a3ca2 100644 --- a/apple/macos-helper/Package.swift +++ b/apple/macos-helper/Package.swift @@ -11,8 +11,16 @@ let package = Package( ), ], targets: [ + .target( + name: "AgentDeviceMacOSInput" + ), .executableTarget( - name: "AgentDeviceMacOSHelper" + name: "AgentDeviceMacOSHelper", + dependencies: ["AgentDeviceMacOSInput"] + ), + .testTarget( + name: "AgentDeviceMacOSInputTests", + dependencies: ["AgentDeviceMacOSInput"] ), ] ) diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift index 6b77e18683..3676a2e3d6 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift @@ -1,3 +1,4 @@ +import AgentDeviceMacOSInput import AppKit import ApplicationServices import CoreGraphics @@ -63,6 +64,9 @@ struct ReadResponse: Encodable { struct PressResponse: Encodable { let x: Double let y: Double + /// Hold actually posted; a short request is raised to the deliverable floor. + let holdMs: Int + let clicks: Int let bundleId: String? let surface: String? } @@ -378,10 +382,45 @@ struct AgentDeviceMacOSHelper { throw HelperError.invalidArgs("press requires --x --y ") } + let holdMs = try validatedPressInt( + optionValue(arguments: arguments, name: "--hold-ms"), + name: "--hold-ms", + minimum: 0, + default: 0 + ) + let clicks = try validatedPressInt( + optionValue(arguments: arguments, name: "--clicks"), + name: "--clicks", + minimum: 1, + default: 1 + ) + let intervalMs = try validatedPressInt( + optionValue(arguments: arguments, name: "--interval-ms"), + name: "--interval-ms", + minimum: 0, + default: 120 + ) + let bundleId = try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId) let surface = optionValue(arguments: arguments, name: "--surface") - try pressAtPosition(bundleId: bundleId, surface: surface, x: x, y: y) - return SuccessEnvelope(data: PressResponse(x: x, y: y, bundleId: bundleId, surface: surface)) + let request = MouseClickRequest( + x: x, + y: y, + holdMs: holdMs, + clicks: clicks, + intervalMs: intervalMs + ) + try pressAtPosition(request) + return SuccessEnvelope( + data: PressResponse( + x: x, + y: y, + holdMs: mouseClickHoldMs(requestedMs: holdMs), + clicks: clicks, + bundleId: bundleId, + surface: surface + ) + ) } static func handleScreenshot(arguments: [String]) throws -> any Encodable { @@ -434,6 +473,21 @@ private func intOption(arguments: [String], name: String) -> Int? { return Int(value) } +private func validatedPressInt( + _ raw: String?, + name: String, + minimum: Int, + default fallback: Int +) throws -> Int { + guard let raw else { + return fallback + } + guard let value = Int(raw), value >= minimum else { + throw HelperError.invalidArgs("press \(name) must be an integer of at least \(minimum)") + } + return value +} + private func readTextAtPosition(bundleId: String?, surface: String?, x: Double, y: Double) throws -> String { let targetApp: NSRunningApplication? if surface == "frontmost-app" || (surface == nil && bundleId != nil) { @@ -481,19 +535,12 @@ private func readTextAtPosition(bundleId: String?, surface: String?, x: Double, throw HelperError.commandFailed("read did not resolve text") } -private func pressAtPosition(bundleId: String?, surface: String?, x: Double, y: Double) throws { - _ = bundleId - _ = surface - let point = CGPoint(x: x, y: y) - guard let move = CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left), - let down = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left), - let up = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left) - else { +private func pressAtPosition(_ request: MouseClickRequest) throws { + do { + try postMouseClick(request) + } catch MouseClickDeliveryError.eventCreationFailed { throw HelperError.commandFailed("press action failed", details: ["reason": "event_creation_failed"]) } - move.post(tap: .cghidEventTap) - down.post(tap: .cghidEventTap) - up.post(tap: .cghidEventTap) } private func captureSurfaceScreenshot(surface: String?, outPath: String, fullscreen: Bool) throws { diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickDelivery.swift b/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickDelivery.swift new file mode 100644 index 0000000000..d16f8615c9 --- /dev/null +++ b/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickDelivery.swift @@ -0,0 +1,71 @@ +import CoreGraphics +import Foundation + +public struct MouseClickRequest: Equatable, Sendable { + public let x: Double + public let y: Double + /// How long the button stays down. Zero asks for the default hold. + public let holdMs: Int + public let clicks: Int + public let intervalMs: Int + + public init(x: Double, y: Double, holdMs: Int = 0, clicks: Int = 1, intervalMs: Int = 120) { + self.x = x + self.y = y + self.holdMs = holdMs + self.clicks = clicks + self.intervalMs = intervalMs + } +} + +public enum MouseClickDeliveryError: Error, Equatable { + case eventCreationFailed +} + +/// Posts a click the way a trackpad would: one motion to the point, then a press that is +/// held long enough for the app to accept the release, repeated with a rising click state +/// so a second press reads as a double-click rather than two independent taps. +public func postMouseClick(_ request: MouseClickRequest) throws { + let point = CGPoint(x: request.x, y: request.y) + let steps = mouseClickSteps(holdMs: request.holdMs, clicks: request.clicks, intervalMs: request.intervalMs) + var clickState = 0 + var postedAny = false + + for step in steps { + if postedAny && step.delayBeforeMs > 0 { + usleep(UInt32(step.delayBeforeMs) * 1000) + } + let event: CGEvent? + switch step.kind { + case .move: + event = CGEvent( + mouseEventSource: nil, + mouseType: .mouseMoved, + mouseCursorPosition: point, + mouseButton: .left + ) + case .down: + clickState += 1 + event = CGEvent( + mouseEventSource: nil, + mouseType: .leftMouseDown, + mouseCursorPosition: point, + mouseButton: .left + ) + event?.setIntegerValueField(.mouseEventClickState, value: Int64(clickState)) + case .up: + event = CGEvent( + mouseEventSource: nil, + mouseType: .leftMouseUp, + mouseCursorPosition: point, + mouseButton: .left + ) + event?.setIntegerValueField(.mouseEventClickState, value: Int64(clickState)) + } + guard let event else { + throw MouseClickDeliveryError.eventCreationFailed + } + event.post(tap: .cghidEventTap) + postedAny = true + } +} diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickSchedule.swift b/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickSchedule.swift new file mode 100644 index 0000000000..0e051b9aee --- /dev/null +++ b/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickSchedule.swift @@ -0,0 +1,50 @@ +import Foundation + +public enum MouseClickStepKind: Equatable, Sendable { + case move + case down + case up +} + +public struct MouseClickStep: Equatable, Sendable { + public let kind: MouseClickStepKind + /// Milliseconds to wait after the previous step before posting this one. + public let delayBeforeMs: Int + + public init(kind: MouseClickStepKind, delayBeforeMs: Int) { + self.kind = kind + self.delayBeforeMs = delayBeforeMs + } +} + +/// AppKit and SwiftUI keep a mouse-down under track long enough to tell a click from a +/// drag, and a mouse-up posted in the same event tick as its mouse-down is never +/// delivered to the app. Measured on an `NSButton`: a 0 ms hold delivered 0 of 15 +/// mouse-ups and activated 0 actions, 20 ms activated 14 of 15, and 40 ms and above +/// activated 15 of 15. Holds shorter than this are raised to it. +public let minimumMouseClickHoldMs = 40 + +/// Hold used when the caller does not name one. Comfortably above +/// `minimumMouseClickHoldMs` so a press does not sit on the measured cliff. +public let defaultMouseClickHoldMs = 60 + +public func mouseClickHoldMs(requestedMs: Int) -> Int { + if requestedMs <= 0 { + return defaultMouseClickHoldMs + } + return max(requestedMs, minimumMouseClickHoldMs) +} + +/// The event schedule one synthetic click becomes: park the cursor, then press and +/// release, repeating for multi-clicks. Every `up` is separated from its `down`, which +/// is what makes the release reach the app at all. +public func mouseClickSteps(holdMs: Int, clicks: Int, intervalMs: Int) -> [MouseClickStep] { + let hold = mouseClickHoldMs(requestedMs: holdMs) + let gap = max(intervalMs, 0) + var steps: [MouseClickStep] = [MouseClickStep(kind: .move, delayBeforeMs: 0)] + for index in 0.. { const controller = new AbortController(); @@ -34,3 +34,95 @@ test('macOS helper snapshot passes cancellation to the helper process', async () assert.equal(receivedSignal, controller.signal); }); + +function helperReturn(data: Record) { + return { + exitCode: 0, + stdout: JSON.stringify({ ok: true, data }), + stderr: '', + }; +} + +test('macOS helper press carries the hold, repeat count, and interval to the click schedule', async () => { + let receivedArgs: string[] = []; + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async (args) => { + receivedArgs = args; + return helperReturn({ x: 12, y: 34, holdMs: 800, clicks: 2 }); + }, + }, + }); + + const result = await withAppleToolProvider( + provider, + async () => + await runMacOsPressAction(12, 34, { + surface: 'menubar', + bundleId: 'com.example.Menu', + holdMs: 800, + clicks: 2, + intervalMs: 140, + }), + ); + + assert.deepEqual( + [ + receivedArgs.slice(receivedArgs.indexOf('--hold-ms'), receivedArgs.indexOf('--hold-ms') + 2), + receivedArgs.slice(receivedArgs.indexOf('--clicks'), receivedArgs.indexOf('--clicks') + 2), + receivedArgs.slice( + receivedArgs.indexOf('--interval-ms'), + receivedArgs.indexOf('--interval-ms') + 2, + ), + ], + [ + ['--hold-ms', '800'], + ['--clicks', '2'], + ['--interval-ms', '140'], + ], + ); + // The helper reports the hold it delivered, which the caller republishes as the + // press's hold so a clamped request cannot read back as the requested one. + assert.equal(result.holdMs, 800); +}); + +test('macOS helper press leaves the repeat gap to the click schedule by default', async () => { + let receivedArgs: string[] = []; + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async (args) => { + receivedArgs = args; + return helperReturn({ x: 7, y: 8, holdMs: 60, clicks: 2 }); + }, + }, + }); + + await withAppleToolProvider( + provider, + async () => await runMacOsPressAction(7, 8, { surface: 'desktop', clicks: 2, intervalMs: 0 }), + ); + + assert.ok(receivedArgs.includes('--clicks'), receivedArgs.join(' ')); + assert.equal(receivedArgs.includes('--interval-ms'), false); +}); + +test('macOS helper press stays a single held click when nothing is repeated', async () => { + let receivedArgs: string[] = []; + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async (args) => { + receivedArgs = args; + return helperReturn({ x: 5, y: 6, holdMs: 60, clicks: 1 }); + }, + }, + }); + + await withAppleToolProvider( + provider, + async () => await runMacOsPressAction(5, 6, { surface: 'frontmost-app' }), + ); + + assert.equal(receivedArgs.includes('--clicks'), false); + assert.equal(receivedArgs.includes('--hold-ms'), false); + assert.equal(receivedArgs.includes('--interval-ms'), false); +}); diff --git a/packages/platform-apple/src/os/macos/helper.ts b/packages/platform-apple/src/os/macos/helper.ts index 1b5e660f1a..e6ecd537bf 100644 --- a/packages/platform-apple/src/os/macos/helper.ts +++ b/packages/platform-apple/src/os/macos/helper.ts @@ -389,14 +389,32 @@ export async function runMacOsReadTextAction( export async function runMacOsPressAction( x: number, y: number, - options: { bundleId?: string; surface?: SessionSurface } = {}, + options: { + bundleId?: string; + surface?: SessionSurface; + holdMs?: number; + clicks?: number; + intervalMs?: number; + } = {}, ): Promise<{ x: number; y: number; + holdMs?: number; + clicks?: number; bundleId?: string; surface?: SessionSurface; }> { const args = ['press', '--x', String(x), '--y', String(y)]; + if (options.holdMs && options.holdMs > 0) { + args.push('--hold-ms', String(options.holdMs)); + } + const clicks = options.clicks ?? 1; + if (clicks > 1) { + args.push('--clicks', String(clicks)); + if (options.intervalMs !== undefined && options.intervalMs > 0) { + args.push('--interval-ms', String(options.intervalMs)); + } + } appendMacOsHelperContextArgs(args, options); return await runMacOsHelper(args); } diff --git a/test/integration/provider-scenarios/macos-desktop.test.ts b/test/integration/provider-scenarios/macos-desktop.test.ts index f17bad59de..7813717f5b 100644 --- a/test/integration/provider-scenarios/macos-desktop.test.ts +++ b/test/integration/provider-scenarios/macos-desktop.test.ts @@ -274,6 +274,24 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help positionals: ['@e2'], expectData: { x: 116, y: 80 }, }, + { + name: 'refresh frontmost refs before the repeat press', + command: 'snapshot', + flags: { snapshotInteractiveOnly: true }, + assert: (snapshot) => { + const general = snapshot.json?.result?.data?.nodes?.find( + (node: { label?: string }) => node.label === 'General', + ); + assert.equal(general?.ref, 'e2', JSON.stringify(snapshot.json)); + }, + }, + { + name: 'double tap snapshot ref', + command: 'press', + positionals: ['@e2'], + flags: { doubleTap: true }, + expectData: { x: 116, y: 80, doubleTap: true }, + }, { name: 'switch to desktop surface', command: 'open', @@ -510,6 +528,20 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help '--surface', 'frontmost-app', ]); + assertFlatToolCall(appleTool.calls, [ + 'macos-helper', + 'press', + '--x', + '116', + '--y', + '80', + '--clicks', + '2', + '--bundle-id', + 'com.apple.systempreferences', + '--surface', + 'frontmost-app', + ]); assertFlatToolCall(appleTool.calls, [ 'macos-helper', 'press', From ca831b6c6716c7ef985d84b42ac85292ba164875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 20 Sep 2026 22:12:33 +0200 Subject: [PATCH 2/4] chore(gates): run the macOS helper Swift tests with the macos-helper gate The macos-helper gate only built the helper, so the new AgentDeviceMacOSInput schedule had no gate behind it. Add test:macos-helper and point the gate at check:macos-helper, which builds and tests, keeping the CI step id unchanged. --- package.json | 2 ++ scripts/check-affected/checks.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 38886f8796..acb792a2d1 100644 --- a/package.json +++ b/package.json @@ -107,6 +107,8 @@ "package:android-ime-helper:npm": "rm -rf android/ime-helper/dist && AGENT_DEVICE_ANDROID_HELPER=ime sh ./scripts/package-android-helper.sh $(node -p \"require('./package.json').version\") android/ime-helper/dist", "build:macos-helper": "node --experimental-strip-types scripts/swift-toolchain-tmpdir.ts swift build -c release --package-path apple/macos-helper", "build:macos-helper:clean": "node --experimental-strip-types scripts/swift-toolchain-tmpdir.ts swift package --package-path apple/macos-helper clean && pnpm build:macos-helper", + "test:macos-helper": "node --experimental-strip-types scripts/swift-toolchain-tmpdir.ts swift test --package-path apple/macos-helper", + "check:macos-helper": "pnpm build:macos-helper && pnpm test:macos-helper", "prepare:publish-assets": "node scripts/prepare-publish-assets.mjs", "build:package": "pnpm build && pnpm build:xcuitest:ios && pnpm build:xcuitest:macos && pnpm build:xcuitest:tvos && pnpm build:xcuitest:visionos && pnpm build:macos-helper:clean && pnpm prepare:publish-assets", "package:npm": "pnpm build:package && pnpm check:package", diff --git a/scripts/check-affected/checks.ts b/scripts/check-affected/checks.ts index 032d3900c6..763df2ca61 100644 --- a/scripts/check-affected/checks.ts +++ b/scripts/check-affected/checks.ts @@ -78,7 +78,7 @@ export const CHECK_CATALOG: readonly CheckSpec[] = [ // helpers packaged into `android/*/dist` (what the replay host verifies), where the // build script writes only the snapshot helper into `.tmp/`. gate('android-helpers', 'Android helper builds (snapshot + IME)', 'build:android', false), - gate('macos-helper', 'macOS helper build', 'build:macos-helper', false), + gate('macos-helper', 'macOS helper build and tests', 'check:macos-helper', false), gate('web-smoke', 'Live web platform smoke', 'test:smoke:web', false), // Needs full history and tags, so it runs in the shared fetch-depth: 0 job // rather than inside the shallow-clone-safe unit lane. From 0c6a78f28e6e468bfd79d434d22264af16714311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 21 Sep 2026 09:08:42 +0200 Subject: [PATCH 3/4] fix(macos-helper): keep --count independent, bound the helper timeout to the click schedule, and release the button on termination MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--count N` posted N presses with a rising click state, so `--count 2` read as a double-click and `--count 3` as a triple-click, unlike every other platform where the count is N independent presses. The rising state is now an explicit `doubleClick` field on the request, set only from `--double-tap`, and it rises inside each press: `--count 3 --double-tap` is three double-clicks. The step array is gone; the schedule is one pure function over presses (click state and delay), which the Swift tests pin, and the poster is one loop over it. The helper ran under a fixed 30s timeout while a schedule could last `hold*clicks + interval*(clicks-1)`, so `--hold-ms 10000 --count 4` was killed inside the third hold with the system button down. The bridge now derives the timeout from the same schedule sum plus the usual margin, forwards the request's abort signal, and the helper releases a held button from a termination handler before it exits, so no path can end between a mouse-down and its mouse-up. Also: an explicit `--interval-ms 0` is carried rather than read as unset, the docs say what `--hold-ms`, `--count` and `--double-tap` mean on these surfaces (and that jitter is not applied), and the workflow step carries the gate's name. Verified on an AppKit probe that logs every mouse-down's clickCount and every activation: `--count 3` → three downs at clickCount 1, three activations; `--double-tap` → clickCount 1 then 2; `--double-tap --count 2` → two such pairs; `--hold-ms 10000 --count 4` completed in 40s with four activations, and a plain press afterwards activated normally. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/macos.yml | 2 +- .../Sources/AgentDeviceMacOSHelper/main.swift | 4 + .../MouseClickDelivery.swift | 120 ++++++++++++------ .../MouseClickSchedule.swift | 63 +++++---- .../MouseClickScheduleTests.swift | 71 +++++------ packages/platform-apple/src/interactions.ts | 7 +- .../src/os/macos/helper.test.ts | 81 +++++++++++- .../platform-apple/src/os/macos/helper.ts | 53 +++++++- .../provider-scenarios/macos-desktop.test.ts | 3 +- website/docs/docs/commands.md | 1 + 10 files changed, 290 insertions(+), 115 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 3b5de3cfc1..32320e23d9 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -138,7 +138,7 @@ jobs: path: .tmp/xctest-host if-no-files-found: warn - - name: Build macOS helper + - name: macOS helper build and tests uses: ./.github/actions/run-gate with: { gate: macos-helper } diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift index 3676a2e3d6..cc9be1260a 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift @@ -67,6 +67,7 @@ struct PressResponse: Encodable { /// Hold actually posted; a short request is raised to the deliverable floor. let holdMs: Int let clicks: Int + let doubleClick: Bool let bundleId: String? let surface: String? } @@ -401,6 +402,7 @@ struct AgentDeviceMacOSHelper { default: 120 ) + let doubleClick = arguments.contains("--double-click") let bundleId = try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId) let surface = optionValue(arguments: arguments, name: "--surface") let request = MouseClickRequest( @@ -408,6 +410,7 @@ struct AgentDeviceMacOSHelper { y: y, holdMs: holdMs, clicks: clicks, + doubleClick: doubleClick, intervalMs: intervalMs ) try pressAtPosition(request) @@ -417,6 +420,7 @@ struct AgentDeviceMacOSHelper { y: y, holdMs: mouseClickHoldMs(requestedMs: holdMs), clicks: clicks, + doubleClick: doubleClick, bundleId: bundleId, surface: surface ) diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickDelivery.swift b/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickDelivery.swift index d16f8615c9..2abc405aed 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickDelivery.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickDelivery.swift @@ -6,14 +6,25 @@ public struct MouseClickRequest: Equatable, Sendable { public let y: Double /// How long the button stays down. Zero asks for the default hold. public let holdMs: Int + /// Independent presses, each at click state 1. public let clicks: Int + /// Post every press as a double-click pair with a rising click state. + public let doubleClick: Bool public let intervalMs: Int - public init(x: Double, y: Double, holdMs: Int = 0, clicks: Int = 1, intervalMs: Int = 120) { + public init( + x: Double, + y: Double, + holdMs: Int = 0, + clicks: Int = 1, + doubleClick: Bool = false, + intervalMs: Int = 120 + ) { self.x = x self.y = y self.holdMs = holdMs self.clicks = clicks + self.doubleClick = doubleClick self.intervalMs = intervalMs } } @@ -22,50 +33,79 @@ public enum MouseClickDeliveryError: Error, Equatable { case eventCreationFailed } -/// Posts a click the way a trackpad would: one motion to the point, then a press that is -/// held long enough for the app to accept the release, repeated with a rising click state -/// so a second press reads as a double-click rather than two independent taps. +/// The button the helper currently holds down, kept where a signal handler can reach it. +/// A helper killed between a mouse-down and its mouse-up would otherwise leave the system's +/// primary button stuck down for whatever the user touches next. +nonisolated(unsafe) private var heldMouseButton: (point: CGPoint, clickState: Int)? + +private func releaseHeldMouseButton() { + guard let held = heldMouseButton else { return } + heldMouseButton = nil + let up = CGEvent( + mouseEventSource: nil, + mouseType: .leftMouseUp, + mouseCursorPosition: held.point, + mouseButton: .left + ) + up?.setIntegerValueField(.mouseEventClickState, value: Int64(held.clickState)) + up?.post(tap: .cghidEventTap) +} + +private func installMouseReleaseOnTermination() { + for terminationSignal in [SIGTERM, SIGINT, SIGHUP] { + signal(terminationSignal) { received in + releaseHeldMouseButton() + _exit(128 + received) + } + } +} + +/// Posts a click the way a trackpad would: one motion to the point, then each press held +/// long enough for the app to accept the release. A termination signal that lands inside a +/// hold releases the button before the process exits. public func postMouseClick(_ request: MouseClickRequest) throws { + installMouseReleaseOnTermination() let point = CGPoint(x: request.x, y: request.y) - let steps = mouseClickSteps(holdMs: request.holdMs, clicks: request.clicks, intervalMs: request.intervalMs) - var clickState = 0 - var postedAny = false + let hold = mouseClickHoldMs(requestedMs: request.holdMs) + let presses = mouseClickPresses( + clicks: request.clicks, + doubleClick: request.doubleClick, + intervalMs: request.intervalMs + ) - for step in steps { - if postedAny && step.delayBeforeMs > 0 { - usleep(UInt32(step.delayBeforeMs) * 1000) - } - let event: CGEvent? - switch step.kind { - case .move: - event = CGEvent( - mouseEventSource: nil, - mouseType: .mouseMoved, - mouseCursorPosition: point, - mouseButton: .left - ) - case .down: - clickState += 1 - event = CGEvent( - mouseEventSource: nil, - mouseType: .leftMouseDown, - mouseCursorPosition: point, - mouseButton: .left - ) - event?.setIntegerValueField(.mouseEventClickState, value: Int64(clickState)) - case .up: - event = CGEvent( - mouseEventSource: nil, - mouseType: .leftMouseUp, - mouseCursorPosition: point, - mouseButton: .left - ) - event?.setIntegerValueField(.mouseEventClickState, value: Int64(clickState)) + guard let move = CGEvent( + mouseEventSource: nil, + mouseType: .mouseMoved, + mouseCursorPosition: point, + mouseButton: .left + ) else { + throw MouseClickDeliveryError.eventCreationFailed + } + move.post(tap: .cghidEventTap) + + for press in presses { + if press.delayBeforeMs > 0 { + usleep(UInt32(press.delayBeforeMs) * 1000) } - guard let event else { + guard let down = CGEvent( + mouseEventSource: nil, + mouseType: .leftMouseDown, + mouseCursorPosition: point, + mouseButton: .left + ), let up = CGEvent( + mouseEventSource: nil, + mouseType: .leftMouseUp, + mouseCursorPosition: point, + mouseButton: .left + ) else { throw MouseClickDeliveryError.eventCreationFailed } - event.post(tap: .cghidEventTap) - postedAny = true + down.setIntegerValueField(.mouseEventClickState, value: Int64(press.clickState)) + up.setIntegerValueField(.mouseEventClickState, value: Int64(press.clickState)) + heldMouseButton = (point, press.clickState) + down.post(tap: .cghidEventTap) + usleep(UInt32(hold) * 1000) + heldMouseButton = nil + up.post(tap: .cghidEventTap) } } diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickSchedule.swift b/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickSchedule.swift index 0e051b9aee..0cb31f897d 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickSchedule.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickSchedule.swift @@ -1,22 +1,5 @@ import Foundation -public enum MouseClickStepKind: Equatable, Sendable { - case move - case down - case up -} - -public struct MouseClickStep: Equatable, Sendable { - public let kind: MouseClickStepKind - /// Milliseconds to wait after the previous step before posting this one. - public let delayBeforeMs: Int - - public init(kind: MouseClickStepKind, delayBeforeMs: Int) { - self.kind = kind - self.delayBeforeMs = delayBeforeMs - } -} - /// AppKit and SwiftUI keep a mouse-down under track long enough to tell a click from a /// drag, and a mouse-up posted in the same event tick as its mouse-down is never /// delivered to the app. Measured on an `NSButton`: a 0 ms hold delivered 0 of 15 @@ -28,6 +11,11 @@ public let minimumMouseClickHoldMs = 40 /// `minimumMouseClickHoldMs` so a press does not sit on the measured cliff. public let defaultMouseClickHoldMs = 60 +/// The gap between the two presses of one double-click. Well inside the system +/// double-click interval (500 ms by default), and independent of the caller's repeat +/// interval, which separates whole presses rather than the halves of one. +public let mouseClickPairGapMs = 80 + public func mouseClickHoldMs(requestedMs: Int) -> Int { if requestedMs <= 0 { return defaultMouseClickHoldMs @@ -35,16 +23,39 @@ public func mouseClickHoldMs(requestedMs: Int) -> Int { return max(requestedMs, minimumMouseClickHoldMs) } -/// The event schedule one synthetic click becomes: park the cursor, then press and -/// release, repeating for multi-clicks. Every `up` is separated from its `down`, which -/// is what makes the release reach the app at all. -public func mouseClickSteps(holdMs: Int, clicks: Int, intervalMs: Int) -> [MouseClickStep] { - let hold = mouseClickHoldMs(requestedMs: holdMs) +/// One press of the button: the click state it is posted with, and how long after the +/// previous release it starts. +public struct MouseClickPress: Equatable, Sendable { + /// `1` for an independent click; `2` for the second half of a double-click. + public let clickState: Int + public let delayBeforeMs: Int + + public init(clickState: Int, delayBeforeMs: Int) { + self.clickState = clickState + self.delayBeforeMs = delayBeforeMs + } +} + +/// The presses one request becomes. `clicks` is always a count of independent presses at +/// click state 1, which is what every other platform means by `--count`; only +/// `doubleClick` raises the state, and it does so per press, so `doubleClick` with three +/// clicks is three double-clicks rather than one triple-click. +public func mouseClickPresses(clicks: Int, doubleClick: Bool, intervalMs: Int) -> [MouseClickPress] { let gap = max(intervalMs, 0) - var steps: [MouseClickStep] = [MouseClickStep(kind: .move, delayBeforeMs: 0)] + var presses: [MouseClickPress] = [] for index in 0.. Int { + let hold = mouseClickHoldMs(requestedMs: holdMs) + return mouseClickPresses(clicks: clicks, doubleClick: doubleClick, intervalMs: intervalMs) + .reduce(0) { $0 + $1.delayBeforeMs + hold } } diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSInputTests/MouseClickScheduleTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSInputTests/MouseClickScheduleTests.swift index 9ab69b9612..09b30f99ef 100644 --- a/apple/macos-helper/Tests/AgentDeviceMacOSInputTests/MouseClickScheduleTests.swift +++ b/apple/macos-helper/Tests/AgentDeviceMacOSInputTests/MouseClickScheduleTests.swift @@ -6,54 +6,53 @@ final class MouseClickScheduleTests: XCTestCase { // The original schedule posted move, down and up back to back. AppKit and SwiftUI // never delivered the release, so every surface press opened a tracking session that // no control ever completed: 0 of 15 measured clicks activated their control. - func testEveryReleaseIsHeldApartFromItsPress() { - for holdMs in [0, 5, 40, 60, 800] { - let steps = mouseClickSteps(holdMs: holdMs, clicks: 1, intervalMs: 120) - let press = steps.firstIndex(where: { $0.kind == .down }) - let release = steps.firstIndex(where: { $0.kind == .up }) - guard let pressIndex = press, let releaseIndex = release, - releaseIndex == pressIndex + 1 - else { - XCTFail("hold \(holdMs)ms did not pair a press with the release that follows it: \(steps)") - continue - } - XCTAssertGreaterThanOrEqual( - steps[releaseIndex].delayBeforeMs, - minimumMouseClickHoldMs, - "hold \(holdMs)ms released inside the window where the release is dropped" - ) - } - } - func testHoldShorterThanTheDeliverableFloorIsRaised() { XCTAssertEqual(mouseClickHoldMs(requestedMs: 0), defaultMouseClickHoldMs) XCTAssertEqual(mouseClickHoldMs(requestedMs: 1), minimumMouseClickHoldMs) XCTAssertEqual(mouseClickHoldMs(requestedMs: 39), minimumMouseClickHoldMs) + XCTAssertGreaterThan(defaultMouseClickHoldMs, minimumMouseClickHoldMs) } func testNamedLongPressHoldIsKept() { XCTAssertEqual(mouseClickHoldMs(requestedMs: 800), 800) - let steps = mouseClickSteps(holdMs: 800, clicks: 1, intervalMs: 120) - XCTAssertEqual(steps.last?.delayBeforeMs, 800) } - func testRepeatClicksSeparatePressesAndKeepOneCursorMotion() { - let steps = mouseClickSteps(holdMs: 0, clicks: 3, intervalMs: 150) - XCTAssertEqual(steps.filter { $0.kind == .move }.count, 1) - XCTAssertEqual(steps.filter { $0.kind == .down }.count, 3) - XCTAssertEqual(steps.filter { $0.kind == .up }.count, 3) - XCTAssertEqual(steps.first?.delayBeforeMs, 0) + // `--count N` means N independent presses on every platform; a rising click state would + // turn `--count 2` into a double-click and `--count 3` into a triple-click. + func testRepeatClicksAreIndependentPressesAtClickStateOne() { + let presses = mouseClickPresses(clicks: 3, doubleClick: false, intervalMs: 150) + XCTAssertEqual(presses.map(\.clickState), [1, 1, 1]) + XCTAssertEqual(presses.map(\.delayBeforeMs), [0, 150, 150]) + } + + // Only `--double-tap` raises the click state, and it does so per press, so it composes + // with `--count` into that many double-clicks rather than one triple-click. + func testDoubleClickRaisesTheStateInsideEachPress() { + XCTAssertEqual( + mouseClickPresses(clicks: 1, doubleClick: true, intervalMs: 120).map(\.clickState), + [1, 2] + ) + let three = mouseClickPresses(clicks: 3, doubleClick: true, intervalMs: 500) + XCTAssertEqual(three.map(\.clickState), [1, 2, 1, 2, 1, 2]) + XCTAssertEqual( + three.map(\.delayBeforeMs), + [0, mouseClickPairGapMs, 500, mouseClickPairGapMs, 500, mouseClickPairGapMs] + ) + XCTAssertLessThan(mouseClickPairGapMs, 500, "a pair must land inside the system double-click interval") + } - let downs = steps.enumerated().filter { $0.element.kind == .down }.map { $0.offset } - XCTAssertEqual(downs.first.map { steps[$0].delayBeforeMs }, 0) - for index in downs.dropFirst() { - XCTAssertEqual(steps[index].delayBeforeMs, 150) - } + func testZeroClicksStillPostsOnePress() { + XCTAssertEqual(mouseClickPresses(clicks: 0, doubleClick: false, intervalMs: 0).count, 1) } - func testCursorParksBeforeThePressAndNeverAfterTheRelease() { - let steps = mouseClickSteps(holdMs: 0, clicks: 2, intervalMs: 100) - XCTAssertEqual(steps.first?.kind, .move) - XCTAssertEqual(steps.last?.kind, .up) + // The process timeout the caller sets is derived from this: a schedule the timeout does + // not cover is killed mid-hold. + func testScheduleDurationSumsEveryHoldAndGap() { + XCTAssertEqual(mouseClickScheduleMs(holdMs: 0, clicks: 1, doubleClick: false, intervalMs: 120), defaultMouseClickHoldMs) + XCTAssertEqual(mouseClickScheduleMs(holdMs: 10_000, clicks: 4, doubleClick: false, intervalMs: 120), 4 * 10_000 + 3 * 120) + XCTAssertEqual( + mouseClickScheduleMs(holdMs: 60, clicks: 2, doubleClick: true, intervalMs: 100), + 4 * 60 + 2 * mouseClickPairGapMs + 100 + ) } } diff --git a/packages/platform-apple/src/interactions.ts b/packages/platform-apple/src/interactions.ts index 9b89b36ef3..0c61d7486a 100644 --- a/packages/platform-apple/src/interactions.ts +++ b/packages/platform-apple/src/interactions.ts @@ -219,13 +219,16 @@ async function runMacOsSurfacePress( ); } const { runMacOsPressAction } = await import('./os/macos/helper.ts'); - const clicks = options.doubleTap ? Math.max(2, options.count) : options.count; + // `count` is independent presses and `doubleTap` raises the click state inside each one, + // the same reading every other platform gives the two flags. const posted = await runMacOsPressAction(point.x, point.y, { bundleId: context.appBundleId, surface: options.surface, holdMs: options.holdMs, - clicks, + clicks: options.count, + doubleClick: options.doubleTap, intervalMs: options.intervalMs, + signal: context.signal, }); // A hold shorter than what macOS delivers is raised before it is posted, so the // response carries the hold the helper actually used rather than the request. diff --git a/packages/platform-apple/src/os/macos/helper.test.ts b/packages/platform-apple/src/os/macos/helper.test.ts index f017340c4a..21e2a2a785 100644 --- a/packages/platform-apple/src/os/macos/helper.test.ts +++ b/packages/platform-apple/src/os/macos/helper.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { createLocalAppleToolProvider, withAppleToolProvider } from '../../core/tool-provider.ts'; -import { runMacOsPressAction, runMacOsSnapshotAction } from './helper.ts'; +import { macOsClickScheduleMs, runMacOsPressAction, runMacOsSnapshotAction } from './helper.ts'; test('macOS helper snapshot passes cancellation to the helper process', async () => { const controller = new AbortController(); @@ -86,7 +86,7 @@ test('macOS helper press carries the hold, repeat count, and interval to the cli assert.equal(result.holdMs, 800); }); -test('macOS helper press leaves the repeat gap to the click schedule by default', async () => { +test('macOS helper press carries an explicit zero interval instead of dropping it', async () => { let receivedArgs: string[] = []; const provider = createLocalAppleToolProvider({ macosHelper: { @@ -103,7 +103,82 @@ test('macOS helper press leaves the repeat gap to the click schedule by default' ); assert.ok(receivedArgs.includes('--clicks'), receivedArgs.join(' ')); - assert.equal(receivedArgs.includes('--interval-ms'), false); + assert.deepEqual( + receivedArgs.slice( + receivedArgs.indexOf('--interval-ms'), + receivedArgs.indexOf('--interval-ms') + 2, + ), + ['--interval-ms', '0'], + ); +}); + +test('macOS helper press keeps repeats independent and names a double-click explicitly', async () => { + let receivedArgs: string[] = []; + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async (args) => { + receivedArgs = args; + return helperReturn({ x: 7, y: 8, holdMs: 60, clicks: 3, doubleClick: true }); + }, + }, + }); + + await withAppleToolProvider( + provider, + async () => + await runMacOsPressAction(7, 8, { surface: 'frontmost-app', clicks: 3, doubleClick: true }), + ); + + // `--count 3 --double-tap` is three double-clicks: the count stays the press count and the + // rising click state is a separate flag, never derived from the count. + assert.deepEqual( + receivedArgs.slice(receivedArgs.indexOf('--clicks'), receivedArgs.indexOf('--clicks') + 2), + ['--clicks', '3'], + ); + assert.ok(receivedArgs.includes('--double-click'), receivedArgs.join(' ')); +}); + +test('macOS helper press outlives its own click schedule and forwards cancellation', async () => { + let receivedTimeoutMs: number | undefined; + let receivedSignal: AbortSignal | undefined; + const controller = new AbortController(); + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async (_args, options) => { + receivedTimeoutMs = options?.timeoutMs; + receivedSignal = options?.signal; + return helperReturn({ x: 1, y: 2, holdMs: 10_000, clicks: 4 }); + }, + }, + }); + + await withAppleToolProvider( + provider, + async () => + await runMacOsPressAction(1, 2, { + surface: 'desktop', + holdMs: 10_000, + clicks: 4, + intervalMs: 120, + signal: controller.signal, + }), + ); + + // Four ten-second holds are 40.36s of schedule; a fixed 30s timeout would kill the helper + // inside the third hold with the button down. + const scheduleMs = macOsClickScheduleMs({ holdMs: 10_000, clicks: 4, intervalMs: 120 }); + assert.equal(scheduleMs, 40_360); + assert.equal(receivedTimeoutMs, scheduleMs + 30_000); + assert.equal(receivedSignal, controller.signal); +}); + +test('macOS click schedule mirrors the helper floors for the timeout it derives', () => { + assert.equal(macOsClickScheduleMs({}), 60); + assert.equal(macOsClickScheduleMs({ holdMs: 5 }), 40); + assert.equal( + macOsClickScheduleMs({ clicks: 2, doubleClick: true, intervalMs: 100 }), + 4 * 60 + 2 * 80 + 100, + ); }); test('macOS helper press stays a single held click when nothing is repeated', async () => { diff --git a/packages/platform-apple/src/os/macos/helper.ts b/packages/platform-apple/src/os/macos/helper.ts index e6ecd537bf..1729e88513 100644 --- a/packages/platform-apple/src/os/macos/helper.ts +++ b/packages/platform-apple/src/os/macos/helper.ts @@ -267,13 +267,15 @@ export async function startMacOsAudioProbeProcess(options: { ); } +const MACOS_HELPER_TIMEOUT_MS = 30_000; + async function runMacOsHelper>( args: string[], - options: { signal?: AbortSignal } = {}, + options: { signal?: AbortSignal; timeoutMs?: number } = {}, ): Promise { const helperOptions = { allowFailure: true, - timeoutMs: 30_000, + timeoutMs: options.timeoutMs ?? MACOS_HELPER_TIMEOUT_MS, signal: options.signal, }; const helperProvider = resolveAppleToolProvider().macosHelper; @@ -386,6 +388,35 @@ export async function runMacOsReadTextAction( return await runMacOsHelper(args); } +// Mirrors the helper's own floors (`MouseClickSchedule.swift`): the schedule the helper runs +// is derived from the same numbers, so the timeout that must outlast it is derived here too. +const MACOS_CLICK_MINIMUM_HOLD_MS = 40; +const MACOS_CLICK_DEFAULT_HOLD_MS = 60; +const MACOS_CLICK_PAIR_GAP_MS = 80; +const MACOS_CLICK_DEFAULT_INTERVAL_MS = 120; + +/** + * How long the helper stays busy posting one press request: every hold plus every gap, + * exactly as `mouseClickScheduleMs` in the helper sums them. Every path that runs a click + * schedule sets its process timeout from this, because a helper killed mid-hold would leave + * the system's mouse button down. + */ +export function macOsClickScheduleMs(options: { + holdMs?: number; + clicks?: number; + doubleClick?: boolean; + intervalMs?: number; +}): number { + const hold = + options.holdMs && options.holdMs > 0 + ? Math.max(options.holdMs, MACOS_CLICK_MINIMUM_HOLD_MS) + : MACOS_CLICK_DEFAULT_HOLD_MS; + const clicks = Math.max(options.clicks ?? 1, 1); + const interval = Math.max(options.intervalMs ?? MACOS_CLICK_DEFAULT_INTERVAL_MS, 0); + const perPress = options.doubleClick ? 2 * hold + MACOS_CLICK_PAIR_GAP_MS : hold; + return clicks * perPress + (clicks - 1) * interval; +} + export async function runMacOsPressAction( x: number, y: number, @@ -393,14 +424,19 @@ export async function runMacOsPressAction( bundleId?: string; surface?: SessionSurface; holdMs?: number; + /** Independent presses, each a single click; `--count` on every platform. */ clicks?: number; + /** Post each press as a double-click pair; `--double-tap`. */ + doubleClick?: boolean; intervalMs?: number; + signal?: AbortSignal; } = {}, ): Promise<{ x: number; y: number; holdMs?: number; clicks?: number; + doubleClick?: boolean; bundleId?: string; surface?: SessionSurface; }> { @@ -411,12 +447,19 @@ export async function runMacOsPressAction( const clicks = options.clicks ?? 1; if (clicks > 1) { args.push('--clicks', String(clicks)); - if (options.intervalMs !== undefined && options.intervalMs > 0) { - args.push('--interval-ms', String(options.intervalMs)); + // An explicit zero is a request for back-to-back presses, not an unset interval. + if (options.intervalMs !== undefined) { + args.push('--interval-ms', String(Math.max(options.intervalMs, 0))); } } + if (options.doubleClick) { + args.push('--double-click'); + } appendMacOsHelperContextArgs(args, options); - return await runMacOsHelper(args); + return await runMacOsHelper(args, { + signal: options.signal, + timeoutMs: macOsClickScheduleMs(options) + MACOS_HELPER_TIMEOUT_MS, + }); } export async function runMacOsScreenshotAction( diff --git a/test/integration/provider-scenarios/macos-desktop.test.ts b/test/integration/provider-scenarios/macos-desktop.test.ts index 7813717f5b..9df52ff9d6 100644 --- a/test/integration/provider-scenarios/macos-desktop.test.ts +++ b/test/integration/provider-scenarios/macos-desktop.test.ts @@ -535,8 +535,7 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help '116', '--y', '80', - '--clicks', - '2', + '--double-click', '--bundle-id', 'com.apple.systempreferences', '--surface', diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index ff6c28e4b0..f30353e430 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -324,6 +324,7 @@ agent-device snapshot -i --platform apple --target desktop - In macOS app sessions, `screenshot` captures the target app window bounds rather than the full desktop. - Prefer selector or `@ref`-driven interactions on macOS. Window position can shift between runs, so raw x/y point commands are less stable than snapshot-derived targets. - Use `click --button secondary` for context menus on macOS, then run `snapshot -i` again. +- On `frontmost-app`, `desktop`, and `menubar` surfaces, `press` and `click` post synthetic mouse events through the macOS helper: `--hold-ms` is how long the button stays down (at least 40 ms, 60 ms by default, because AppKit drops a release posted in the same tick as its press), `--count` is that many independent clicks, and `--double-tap` posts each click as a double-click pair. A long schedule such as `--hold-ms 10000 --count 4` is given the time it needs, and a helper interrupted mid-hold releases the button before it exits. `--jitter-px` is not applied on these surfaces. - Mobile-only helpers remain unsupported on macOS: `boot`, `shutdown`, `home`, `orientation`, `app-switcher`, `action-button`, `install`, `reinstall`, `install-from-source`, and `push`. Recommended loops: From 14d1006f722ac29c8d3dfde507504de77577a8d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 21 Sep 2026 11:21:11 +0200 Subject: [PATCH 4/4] fix(macos-helper): stop the helper with SIGTERM before SIGKILL so a held mouse button is released The helper's release handler traps SIGTERM, SIGINT and SIGHUP, but the host stopped the helper only through `killProcessTree`, which sends SIGKILL on both the deadline and a cancelled request. Now that the request signal reaches the surface press, a cancelled `press --hold-ms 10000` or a dropped client killed the helper between the mouse-down and its mouse-up and the button stayed down. `ExecOptions` gains a `kill` policy: the first signal to send and a grace after which SIGKILL follows. `killProcessTree` sends the policy's signal, arms an unref'd escalation timer, and clears it on the child's exit; the group path and the reaped-pid guard are shared by both signals. `runMacOsHelper` asks for SIGTERM with a one-second grace on every helper run, so the release handler runs on every stop the host applies. The helper also posts the mouse-up before it clears the held-button record, so a signal landing between the two can no longer exit with the button down. Two exec tests pin the route with a trap script standing in for the helper: a cancelled command with a kill policy sees the signal and records its release, and a child that ignores the first signal is still ended once the grace passes. The helper test asserts the policy the press hands the provider. The docs note named the `desktop` surface among those that post through the helper; that surface inspects only, and the note now says so. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 8 +++ .../MouseClickDelivery.swift | 9 ++- .../src/internal/exec-kill-settle.test.ts | 61 +++++++++++++++++++ packages/host-kit/src/internal/exec.ts | 44 +++++++++++-- .../src/os/macos/helper.test.ts | 6 ++ .../platform-apple/src/os/macos/helper.ts | 9 +++ website/docs/docs/commands.md | 2 +- 7 files changed, 130 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 213f7fd9a9..e7acdb300d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## Unreleased +- Fixed (macos): `press` and `click` on the `frontmost-app`, `desktop`, and `menubar` surfaces post + clicks the way a trackpad does. The helper posted the release in the same tick as the press, so + AppKit dropped it and controls never activated; each press is now held (`--hold-ms`, 60 ms by + default, 40 ms at least). `--count` is that many independent clicks and `--double-tap` posts each + as a double-click pair, the helper's deadline follows the click schedule instead of a fixed 30 s, + and every stop the host applies mid-hold — a cancelled request, a dropped client, the deadline — + reaches the helper as SIGTERM first so it releases the button before it exits; SIGKILL only + follows a helper that has not exited a second later. - Added (diff): `diff screenshot` accepts a JPEG baseline or current image. Both inputs had to be PNG, so a capture exported by another tool had to be converted first and a HarmonyOS capture — which the platform serves as JPEG under whatever name the command was given — could never be compared. Each diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickDelivery.swift b/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickDelivery.swift index 2abc405aed..ee615da7b6 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickDelivery.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickDelivery.swift @@ -35,7 +35,9 @@ public enum MouseClickDeliveryError: Error, Equatable { /// The button the helper currently holds down, kept where a signal handler can reach it. /// A helper killed between a mouse-down and its mouse-up would otherwise leave the system's -/// primary button stuck down for whatever the user touches next. +/// primary button stuck down for whatever the user touches next. The host stops the helper +/// with SIGTERM before SIGKILL (`runMacOsHelper` in `helper.ts`) so that this handler runs +/// on a deadline and on a cancelled request, not only on a signal sent by hand. nonisolated(unsafe) private var heldMouseButton: (point: CGPoint, clickState: Int)? private func releaseHeldMouseButton() { @@ -105,7 +107,10 @@ public func postMouseClick(_ request: MouseClickRequest) throws { heldMouseButton = (point, press.clickState) down.post(tap: .cghidEventTap) usleep(UInt32(hold) * 1000) - heldMouseButton = nil + // The record clears only after the up is posted: a signal that lands between the two + // would otherwise find nothing to release and exit with the button still down. A signal + // that lands after the post releases a button already up, which is harmless. up.post(tap: .cghidEventTap) + heldMouseButton = nil } } diff --git a/packages/host-kit/src/internal/exec-kill-settle.test.ts b/packages/host-kit/src/internal/exec-kill-settle.test.ts index 6b4066b2b0..98de96233d 100644 --- a/packages/host-kit/src/internal/exec-kill-settle.test.ts +++ b/packages/host-kit/src/internal/exec-kill-settle.test.ts @@ -335,6 +335,67 @@ test.runIf(process.platform !== 'win32')( 5_000, ); +// A child that holds something the kill would strand — the macOS helper with the mouse button +// down — asks for a signal it can handle before SIGKILL. The trap script below stands in for +// that helper: it records the release the signal handler performs, and its `sleep` is the +// hold the kill interrupts. + +function releaseMarkerPath(label: string): string { + return path.join(mkdtempForTestSync(`agent-device-exec-${label}-`), 'released'); +} + +function holdUntilSignalledShellScript(markerPath: string): string { + return `trap 'printf released > ${shellQuote(markerPath)}; exit 143' TERM; sleep ${HOLDER_LIFETIME_SECONDS} & wait`; +} + +test.runIf(process.platform !== 'win32')( + 'a cancelled command with a kill policy is signalled so it can release what it holds', + async () => { + const markerPath = releaseMarkerPath('graceful-abort'); + const controller = new AbortController(); + const held = runCmd('/bin/sh', ['-c', holdUntilSignalledShellScript(markerPath)], { + signal: controller.signal, + kill: { signal: 'SIGTERM', graceMs: 2_000 }, + }); + const rejection = settledRejection(held); + await sleep(100); + + controller.abort(); + const outcome = await rejection; + + assert.ok(outcome, 'a cancelled command must not resolve'); + const details = (outcome.error as { details?: Record }).details; + assert.equal(details?.reason, 'request_canceled'); + assert.equal( + fs.readFileSync(markerPath, 'utf8'), + 'released', + 'the child never saw the signal its handler releases on: it was killed outright', + ); + }, +); + +test.runIf(process.platform !== 'win32')( + 'a deadline with a kill policy still ends a child that ignores the first signal', + async () => { + const startedAt = Date.now(); + await assert.rejects( + () => + runCmd('/bin/sh', ['-c', `trap '' TERM; sleep ${HOLDER_LIFETIME_SECONDS} & wait`], { + timeoutMs: DEADLINE_MS, + kill: { signal: 'SIGTERM', graceMs: 200 }, + }), + (error: unknown) => { + assert.equal(isCommandTimeoutError(error), true); + return true; + }, + ); + const elapsedMs = Date.now() - startedAt; + assert.ok(elapsedMs >= DEADLINE_MS + 200, `escalated before the grace passed: ${elapsedMs}ms`); + assert.ok(elapsedMs < 2_000, `the grace became a way to outlive the deadline: ${elapsedMs}ms`); + }, + 10_000, +); + test.runIf(process.platform !== 'win32')( 'runCmd that was never killed still drains output a descendant writes after its parent exited', async () => { diff --git a/packages/host-kit/src/internal/exec.ts b/packages/host-kit/src/internal/exec.ts index 16fc0a092a..7f1f9f8356 100644 --- a/packages/host-kit/src/internal/exec.ts +++ b/packages/host-kit/src/internal/exec.ts @@ -27,6 +27,13 @@ export type ExecOptions = { signal?: AbortSignal; /** Max stdout/stderr bytes for synchronous runs (default Node ~1MB). */ maxBuffer?: number; + /** + * How a deadline or a cancelled request stops the child. Without this it is SIGKILL at + * once. A child that has to undo something before it dies — the macOS helper releasing a + * mouse button it is holding — is sent `signal` first, and SIGKILL only once `graceMs` + * pass without it exiting. + */ + kill?: { readonly signal: NodeJS.Signals; readonly graceMs: number }; }; export type ExecStreamOptions = ExecOptions & { @@ -206,7 +213,7 @@ function runSpawnedCommand( // group kill that would have ended the pipe holder can no longer run through a child // Node already reaped. const settlement = createCommandKillSettlement({ - killProcessTree: () => killProcessTree(child, options.detached), + killProcessTree: () => killProcessTree(child, options), settle, }); const abort = watchCommandAbort(options, settlement.requestKill); @@ -224,7 +231,7 @@ function runSpawnedCommand( if (abort.didAbort || didTimeout) return; if (isEpipeError(error)) return; fail(createStdinError(executable, cmd, args, error)); - killProcessTree(child, options.detached); + killProcessTree(child, options); }); child.stdout.on('data', (chunk) => { @@ -470,7 +477,7 @@ export function runCmdBackground( resolve({ stdout, stderr, exitCode: finalExitCode }); } const settlement = createCommandKillSettlement({ - killProcessTree: () => killProcessTree(child, options.detached), + killProcessTree: () => killProcessTree(child, options), settle, }); const abort = watchCommandAbort(options, settlement.requestKill); @@ -910,9 +917,34 @@ export function signalProcessGroupBestEffort(pid: number, signal: NodeJS.Signals * waiting on. The one group-signal seam reports whether anything was reached rather than * throwing, and a group that is gone or not ours to signal is the case it reports false. */ -function killProcessTree(child: ChildProcess, detached: boolean | undefined): void { +function killProcessTree( + child: ChildProcess, + options: Pick, +): void { + if (!options.kill) { + signalProcessTree(child, options.detached, 'SIGKILL'); + return; + } + // The child is given its chance to clean up, and the escalation is what keeps that + // chance from becoming a way to outlive the deadline. The timer holds nothing open: + // a child that exits on the first signal clears it, and a worker shutting down owes + // a child that ignored the signal nothing further. + signalProcessTree(child, options.detached, options.kill.signal); + const escalation = setTimeout( + () => signalProcessTree(child, options.detached, 'SIGKILL'), + options.kill.graceMs, + ); + escalation.unref(); + child.once('exit', () => clearTimeout(escalation)); +} + +function signalProcessTree( + child: ChildProcess, + detached: boolean | undefined, + signal: NodeJS.Signals, +): void { if (detached && child.pid && process.platform !== 'win32') { - signalProcessGroupBestEffort(child.pid, 'SIGKILL'); + signalProcessGroupBestEffort(child.pid, signal); return; } // A non-detached child leaves its pid free for the kernel to hand to an unrelated @@ -920,7 +952,7 @@ function killProcessTree(child: ChildProcess, detached: boolean | undefined): vo // strike a stranger. Nothing waits for a kill of a child that is already gone: // settlement happens on `exit`. if (child.exitCode !== null || child.signalCode !== null) return; - child.kill('SIGKILL'); + child.kill(signal); } /** diff --git a/packages/platform-apple/src/os/macos/helper.test.ts b/packages/platform-apple/src/os/macos/helper.test.ts index 21e2a2a785..eb8d24b0e9 100644 --- a/packages/platform-apple/src/os/macos/helper.test.ts +++ b/packages/platform-apple/src/os/macos/helper.test.ts @@ -141,12 +141,14 @@ test('macOS helper press keeps repeats independent and names a double-click expl test('macOS helper press outlives its own click schedule and forwards cancellation', async () => { let receivedTimeoutMs: number | undefined; let receivedSignal: AbortSignal | undefined; + let receivedKill: { signal: string; graceMs: number } | undefined; const controller = new AbortController(); const provider = createLocalAppleToolProvider({ macosHelper: { run: async (_args, options) => { receivedTimeoutMs = options?.timeoutMs; receivedSignal = options?.signal; + receivedKill = options?.kill; return helperReturn({ x: 1, y: 2, holdMs: 10_000, clicks: 4 }); }, }, @@ -170,6 +172,10 @@ test('macOS helper press outlives its own click schedule and forwards cancellati assert.equal(scheduleMs, 40_360); assert.equal(receivedTimeoutMs, scheduleMs + 30_000); assert.equal(receivedSignal, controller.signal); + // The host stops the helper with SIGKILL on both routes, and a helper killed between a + // mouse-down and its mouse-up leaves the button stuck. The helper's release handler only + // runs if the stop reaches it as a catchable signal first. + assert.deepEqual(receivedKill, { signal: 'SIGTERM', graceMs: 1_000 }); }); test('macOS click schedule mirrors the helper floors for the timeout it derives', () => { diff --git a/packages/platform-apple/src/os/macos/helper.ts b/packages/platform-apple/src/os/macos/helper.ts index 1729e88513..a0bd002176 100644 --- a/packages/platform-apple/src/os/macos/helper.ts +++ b/packages/platform-apple/src/os/macos/helper.ts @@ -268,6 +268,14 @@ export async function startMacOsAudioProbeProcess(options: { } const MACOS_HELPER_TIMEOUT_MS = 30_000; +/** + * Every stop the host applies to the helper — a deadline, a cancelled request, a client that + * dropped mid-command — reaches it as SIGTERM first. A helper posting a press may be holding + * the mouse button down at that moment, and its SIGTERM handler releases the button before it + * exits; SIGKILL would end it between the down and the up and leave the button stuck for + * whatever the user touches next. A helper that has not exited a second later is killed. + */ +const MACOS_HELPER_KILL_GRACE_MS = 1_000; async function runMacOsHelper>( args: string[], @@ -277,6 +285,7 @@ async function runMacOsHelper>( allowFailure: true, timeoutMs: options.timeoutMs ?? MACOS_HELPER_TIMEOUT_MS, signal: options.signal, + kill: { signal: 'SIGTERM' as const, graceMs: MACOS_HELPER_KILL_GRACE_MS }, }; const helperProvider = resolveAppleToolProvider().macosHelper; const helperPath = helperProvider diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index f30353e430..866709a3a0 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -324,7 +324,7 @@ agent-device snapshot -i --platform apple --target desktop - In macOS app sessions, `screenshot` captures the target app window bounds rather than the full desktop. - Prefer selector or `@ref`-driven interactions on macOS. Window position can shift between runs, so raw x/y point commands are less stable than snapshot-derived targets. - Use `click --button secondary` for context menus on macOS, then run `snapshot -i` again. -- On `frontmost-app`, `desktop`, and `menubar` surfaces, `press` and `click` post synthetic mouse events through the macOS helper: `--hold-ms` is how long the button stays down (at least 40 ms, 60 ms by default, because AppKit drops a release posted in the same tick as its press), `--count` is that many independent clicks, and `--double-tap` posts each click as a double-click pair. A long schedule such as `--hold-ms 10000 --count 4` is given the time it needs, and a helper interrupted mid-hold releases the button before it exits. `--jitter-px` is not applied on these surfaces. +- On `frontmost-app` and `menubar` surfaces, `press` and `click` post synthetic mouse events through the macOS helper (the `desktop` surface inspects only): `--hold-ms` is how long the button stays down (at least 40 ms, 60 ms by default, because AppKit drops a release posted in the same tick as its press), `--count` is that many independent clicks, and `--double-tap` posts each click as a double-click pair. A long schedule such as `--hold-ms 10000 --count 4` is given the time it needs, and a helper stopped mid-hold — by a cancelled request, a dropped client, or its deadline — releases the button before it exits. `--jitter-px` is not applied on these surfaces. - Mobile-only helpers remain unsupported on macOS: `boot`, `shutdown`, `home`, `orientation`, `app-switcher`, `action-button`, `install`, `reinstall`, `install-from-source`, and `push`. Recommended loops: