fix(macos-helper): hold synthetic clicks long enough to be delivered

.github/workflows/macos.yml

@@ -138,7 +138,7 @@ jobs:
           path: .tmp/xctest-host
           if-no-files-found: warn
 
-      - name: Build macOS helper
+      - name: macOS helper build and tests
         uses: ./.github/actions/run-gate
         with: { gate: macos-helper }
 

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## Unreleased
 
+- Fixed (macos): `press` and `click` on the `frontmost-app`, `desktop`, and `menubar` surfaces post
+  clicks the way a trackpad does. The helper posted the release in the same tick as the press, so
+  AppKit dropped it and controls never activated; each press is now held (`--hold-ms`, 60 ms by
+  default, 40 ms at least). `--count` is that many independent clicks and `--double-tap` posts each
+  as a double-click pair, the helper's deadline follows the click schedule instead of a fixed 30 s,
+  and every stop the host applies mid-hold — a cancelled request, a dropped client, the deadline —
+  reaches the helper as SIGTERM first so it releases the button before it exits; SIGKILL only
+  follows a helper that has not exited a second later.
 - Added (diff): `diff screenshot` accepts a JPEG baseline or current image. Both inputs had to be PNG,
   so a capture exported by another tool had to be converted first and a HarmonyOS capture — which the
   platform serves as JPEG under whatever name the command was given — could never be compared. Each

apple/macos-helper/Package.swift

@@ -11,8 +11,16 @@ let package = Package(
     ),
   ],
   targets: [
+    .target(
+      name: "AgentDeviceMacOSInput"
+    ),
     .executableTarget(
-      name: "AgentDeviceMacOSHelper"
+      name: "AgentDeviceMacOSHelper",
+      dependencies: ["AgentDeviceMacOSInput"]
+    ),
+    .testTarget(
+      name: "AgentDeviceMacOSInputTests",
+      dependencies: ["AgentDeviceMacOSInput"]
     ),
   ]
 )

apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift

@@ -1,3 +1,4 @@
+import AgentDeviceMacOSInput
 import AppKit
 import ApplicationServices
 import CoreGraphics
@@ -63,6 +64,10 @@ struct ReadResponse: Encodable {
 struct PressResponse: Encodable {
   let x: Double
   let y: Double
+  /// Hold actually posted; a short request is raised to the deliverable floor.
+  let holdMs: Int
+  let clicks: Int
+  let doubleClick: Bool
   let bundleId: String?
   let surface: String?
 }
@@ -378,10 +383,48 @@ struct AgentDeviceMacOSHelper {
       throw HelperError.invalidArgs("press requires --x <number> --y <number>")
     }
 
+    let holdMs = try validatedPressInt(
+      optionValue(arguments: arguments, name: "--hold-ms"),
+      name: "--hold-ms",
+      minimum: 0,
+      default: 0
+    )
+    let clicks = try validatedPressInt(
+      optionValue(arguments: arguments, name: "--clicks"),
+      name: "--clicks",
+      minimum: 1,
+      default: 1
+    )
+    let intervalMs = try validatedPressInt(
+      optionValue(arguments: arguments, name: "--interval-ms"),
+      name: "--interval-ms",
+      minimum: 0,
+      default: 120
+    )
+
+    let doubleClick = arguments.contains("--double-click")
     let bundleId = try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId)
     let surface = optionValue(arguments: arguments, name: "--surface")
-    try pressAtPosition(bundleId: bundleId, surface: surface, x: x, y: y)
-    return SuccessEnvelope(data: PressResponse(x: x, y: y, bundleId: bundleId, surface: surface))
+    let request = MouseClickRequest(
+      x: x,
+      y: y,
+      holdMs: holdMs,
+      clicks: clicks,
+      doubleClick: doubleClick,
+      intervalMs: intervalMs
+    )
+    try pressAtPosition(request)
+    return SuccessEnvelope(
+      data: PressResponse(
+        x: x,
+        y: y,
+        holdMs: mouseClickHoldMs(requestedMs: holdMs),
+        clicks: clicks,
+        doubleClick: doubleClick,
+        bundleId: bundleId,
+        surface: surface
+      )
+    )
   }
 
   static func handleScreenshot(arguments: [String]) throws -> any Encodable {
@@ -434,6 +477,21 @@ private func intOption(arguments: [String], name: String) -> Int? {
   return Int(value)
 }
 
+private func validatedPressInt(
+  _ raw: String?,
+  name: String,
+  minimum: Int,
+  default fallback: Int
+) throws -> Int {
+  guard let raw else {
+    return fallback
+  }
+  guard let value = Int(raw), value >= minimum else {
+    throw HelperError.invalidArgs("press \(name) must be an integer of at least \(minimum)")
+  }
+  return value
+}
+
 private func readTextAtPosition(bundleId: String?, surface: String?, x: Double, y: Double) throws -> String {
   let targetApp: NSRunningApplication?
   if surface == "frontmost-app" || (surface == nil && bundleId != nil) {
@@ -481,19 +539,12 @@ private func readTextAtPosition(bundleId: String?, surface: String?, x: Double,
   throw HelperError.commandFailed("read did not resolve text")
 }
 
-private func pressAtPosition(bundleId: String?, surface: String?, x: Double, y: Double) throws {
-  _ = bundleId
-  _ = surface
-  let point = CGPoint(x: x, y: y)
-  guard let move = CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left),
-        let down = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left),
-        let up = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left)
-  else {
+private func pressAtPosition(_ request: MouseClickRequest) throws {
+  do {
+    try postMouseClick(request)
+  } catch MouseClickDeliveryError.eventCreationFailed {
     throw HelperError.commandFailed("press action failed", details: ["reason": "event_creation_failed"])
   }
-  move.post(tap: .cghidEventTap)
-  down.post(tap: .cghidEventTap)
-  up.post(tap: .cghidEventTap)
 }
 
 private func captureSurfaceScreenshot(surface: String?, outPath: String, fullscreen: Bool) throws {

apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickDelivery.swift

@@ -0,0 +1,116 @@
+import CoreGraphics
+import Foundation
+
+public struct MouseClickRequest: Equatable, Sendable {
+  public let x: Double
+  public let y: Double
+  /// How long the button stays down. Zero asks for the default hold.
+  public let holdMs: Int
+  /// Independent presses, each at click state 1.
+  public let clicks: Int
+  /// Post every press as a double-click pair with a rising click state.
+  public let doubleClick: Bool
+  public let intervalMs: Int
+
+  public init(
+    x: Double,
+    y: Double,
+    holdMs: Int = 0,
+    clicks: Int = 1,
+    doubleClick: Bool = false,
+    intervalMs: Int = 120
+  ) {
+    self.x = x
+    self.y = y
+    self.holdMs = holdMs
+    self.clicks = clicks
+    self.doubleClick = doubleClick
+    self.intervalMs = intervalMs
+  }
+}
+
+public enum MouseClickDeliveryError: Error, Equatable {
+  case eventCreationFailed
+}
+
+/// The button the helper currently holds down, kept where a signal handler can reach it.
+/// A helper killed between a mouse-down and its mouse-up would otherwise leave the system's
+/// primary button stuck down for whatever the user touches next. The host stops the helper
+/// with SIGTERM before SIGKILL (`runMacOsHelper` in `helper.ts`) so that this handler runs
+/// on a deadline and on a cancelled request, not only on a signal sent by hand.
+nonisolated(unsafe) private var heldMouseButton: (point: CGPoint, clickState: Int)?
+
+private func releaseHeldMouseButton() {
+  guard let held = heldMouseButton else { return }
+  heldMouseButton = nil
+  let up = CGEvent(
+    mouseEventSource: nil,
+    mouseType: .leftMouseUp,
+    mouseCursorPosition: held.point,
+    mouseButton: .left
+  )
+  up?.setIntegerValueField(.mouseEventClickState, value: Int64(held.clickState))
+  up?.post(tap: .cghidEventTap)
+}
+
+private func installMouseReleaseOnTermination() {
+  for terminationSignal in [SIGTERM, SIGINT, SIGHUP] {
+    signal(terminationSignal) { received in
+      releaseHeldMouseButton()
+      _exit(128 + received)
+    }
+  }
+}
+
+/// Posts a click the way a trackpad would: one motion to the point, then each press held
+/// long enough for the app to accept the release. A termination signal that lands inside a
+/// hold releases the button before the process exits.
+public func postMouseClick(_ request: MouseClickRequest) throws {
+  installMouseReleaseOnTermination()
+  let point = CGPoint(x: request.x, y: request.y)
+  let hold = mouseClickHoldMs(requestedMs: request.holdMs)
+  let presses = mouseClickPresses(
+    clicks: request.clicks,
+    doubleClick: request.doubleClick,
+    intervalMs: request.intervalMs
+  )
+
+  guard let move = CGEvent(
+    mouseEventSource: nil,
+    mouseType: .mouseMoved,
+    mouseCursorPosition: point,
+    mouseButton: .left
+  ) else {
+    throw MouseClickDeliveryError.eventCreationFailed
+  }
+  move.post(tap: .cghidEventTap)
+
+  for press in presses {
+    if press.delayBeforeMs > 0 {
+      usleep(UInt32(press.delayBeforeMs) * 1000)
+    }
+    guard let down = CGEvent(
+      mouseEventSource: nil,
+      mouseType: .leftMouseDown,
+      mouseCursorPosition: point,
+      mouseButton: .left
+    ), let up = CGEvent(
+      mouseEventSource: nil,
+      mouseType: .leftMouseUp,
+      mouseCursorPosition: point,
+      mouseButton: .left
+    ) else {
+      throw MouseClickDeliveryError.eventCreationFailed
+    }
+    down.setIntegerValueField(.mouseEventClickState, value: Int64(press.clickState))
+    up.setIntegerValueField(.mouseEventClickState, value: Int64(press.clickState))
+    heldMouseButton = (point, press.clickState)
+    down.post(tap: .cghidEventTap)
+    usleep(UInt32(hold) * 1000)
+    // The record clears only after the up is posted: a signal that lands between the two
+    // would otherwise find nothing to release and exit with the button still down. A signal
+    // that lands after the post releases a button already up, which is harmless.
+    up.post(tap: .cghidEventTap)
+    heldMouseButton = nil
+  }
+}

apple/macos-helper/Sources/AgentDeviceMacOSInput/MouseClickSchedule.swift

@@ -0,0 +1,61 @@
+import Foundation
+
+/// AppKit and SwiftUI keep a mouse-down under track long enough to tell a click from a
+/// drag, and a mouse-up posted in the same event tick as its mouse-down is never
+/// delivered to the app. Measured on an `NSButton`: a 0 ms hold delivered 0 of 15
+/// mouse-ups and activated 0 actions, 20 ms activated 14 of 15, and 40 ms and above
+/// activated 15 of 15. Holds shorter than this are raised to it.
+public let minimumMouseClickHoldMs = 40
+
+/// Hold used when the caller does not name one. Comfortably above
+/// `minimumMouseClickHoldMs` so a press does not sit on the measured cliff.
+public let defaultMouseClickHoldMs = 60
+
+/// The gap between the two presses of one double-click. Well inside the system
+/// double-click interval (500 ms by default), and independent of the caller's repeat
+/// interval, which separates whole presses rather than the halves of one.
+public let mouseClickPairGapMs = 80
+
+public func mouseClickHoldMs(requestedMs: Int) -> Int {
+  if requestedMs <= 0 {
+    return defaultMouseClickHoldMs
+  }
+  return max(requestedMs, minimumMouseClickHoldMs)
+}
+
+/// One press of the button: the click state it is posted with, and how long after the
+/// previous release it starts.
+public struct MouseClickPress: Equatable, Sendable {
+  /// `1` for an independent click; `2` for the second half of a double-click.
+  public let clickState: Int
+  public let delayBeforeMs: Int
+
+  public init(clickState: Int, delayBeforeMs: Int) {
+    self.clickState = clickState
+    self.delayBeforeMs = delayBeforeMs
+  }
+}
+
+/// The presses one request becomes. `clicks` is always a count of independent presses at
+/// click state 1, which is what every other platform means by `--count`; only
+/// `doubleClick` raises the state, and it does so per press, so `doubleClick` with three
+/// clicks is three double-clicks rather than one triple-click.
+public func mouseClickPresses(clicks: Int, doubleClick: Bool, intervalMs: Int) -> [MouseClickPress] {
+  let gap = max(intervalMs, 0)
+  var presses: [MouseClickPress] = []
+  for index in 0..<max(clicks, 1) {
+    presses.append(MouseClickPress(clickState: 1, delayBeforeMs: index == 0 ? 0 : gap))
+    if doubleClick {
+      presses.append(MouseClickPress(clickState: 2, delayBeforeMs: mouseClickPairGapMs))
+    }
+  }
+  return presses
+}
+
+/// How long the schedule keeps the helper busy: every hold plus every gap. The caller's
+/// process timeout must cover this, or the helper is killed mid-schedule.
+public func mouseClickScheduleMs(holdMs: Int, clicks: Int, doubleClick: Bool, intervalMs: Int) -> Int {
+  let hold = mouseClickHoldMs(requestedMs: holdMs)
+  return mouseClickPresses(clicks: clicks, doubleClick: doubleClick, intervalMs: intervalMs)
+    .reduce(0) { $0 + $1.delayBeforeMs + hold }
+}

apple/macos-helper/Tests/AgentDeviceMacOSInputTests/MouseClickScheduleTests.swift

@@ -0,0 +1,58 @@
+import XCTest
+
+@testable import AgentDeviceMacOSInput
+
+final class MouseClickScheduleTests: XCTestCase {
+  // The original schedule posted move, down and up back to back. AppKit and SwiftUI
+  // never delivered the release, so every surface press opened a tracking session that
+  // no control ever completed: 0 of 15 measured clicks activated their control.
+  func testHoldShorterThanTheDeliverableFloorIsRaised() {
+    XCTAssertEqual(mouseClickHoldMs(requestedMs: 0), defaultMouseClickHoldMs)
+    XCTAssertEqual(mouseClickHoldMs(requestedMs: 1), minimumMouseClickHoldMs)
+    XCTAssertEqual(mouseClickHoldMs(requestedMs: 39), minimumMouseClickHoldMs)
+    XCTAssertGreaterThan(defaultMouseClickHoldMs, minimumMouseClickHoldMs)
+  }
+
+  func testNamedLongPressHoldIsKept() {
+    XCTAssertEqual(mouseClickHoldMs(requestedMs: 800), 800)
+  }
+
+  // `--count N` means N independent presses on every platform; a rising click state would
+  // turn `--count 2` into a double-click and `--count 3` into a triple-click.
+  func testRepeatClicksAreIndependentPressesAtClickStateOne() {
+    let presses = mouseClickPresses(clicks: 3, doubleClick: false, intervalMs: 150)
+    XCTAssertEqual(presses.map(\.clickState), [1, 1, 1])
+    XCTAssertEqual(presses.map(\.delayBeforeMs), [0, 150, 150])
+  }
+
+  // Only `--double-tap` raises the click state, and it does so per press, so it composes
+  // with `--count` into that many double-clicks rather than one triple-click.
+  func testDoubleClickRaisesTheStateInsideEachPress() {
+    XCTAssertEqual(
+      mouseClickPresses(clicks: 1, doubleClick: true, intervalMs: 120).map(\.clickState),
+      [1, 2]
+    )
+    let three = mouseClickPresses(clicks: 3, doubleClick: true, intervalMs: 500)
+    XCTAssertEqual(three.map(\.clickState), [1, 2, 1, 2, 1, 2])
+    XCTAssertEqual(
+      three.map(\.delayBeforeMs),
+      [0, mouseClickPairGapMs, 500, mouseClickPairGapMs, 500, mouseClickPairGapMs]
+    )
+    XCTAssertLessThan(mouseClickPairGapMs, 500, "a pair must land inside the system double-click interval")
+  }
+
+  func testZeroClicksStillPostsOnePress() {
+    XCTAssertEqual(mouseClickPresses(clicks: 0, doubleClick: false, intervalMs: 0).count, 1)
+  }
+
+  // The process timeout the caller sets is derived from this: a schedule the timeout does
+  // not cover is killed mid-hold.
+  func testScheduleDurationSumsEveryHoldAndGap() {
+    XCTAssertEqual(mouseClickScheduleMs(holdMs: 0, clicks: 1, doubleClick: false, intervalMs: 120), defaultMouseClickHoldMs)
+    XCTAssertEqual(mouseClickScheduleMs(holdMs: 10_000, clicks: 4, doubleClick: false, intervalMs: 120), 4 * 10_000 + 3 * 120)
+    XCTAssertEqual(
+      mouseClickScheduleMs(holdMs: 60, clicks: 2, doubleClick: true, intervalMs: 100),
+      4 * 60 + 2 * mouseClickPairGapMs + 100
+    )
+  }
+}