From d362c0d0c2867fd7b94b0eb2e92414cacd3d000e Mon Sep 17 00:00:00 2001 From: shourikan Date: Wed, 15 Apr 2026 17:03:26 +0200 Subject: [PATCH] Fixed the harness method signatures by adding optional keys argument to fit the OpenAI optional key parameter and added displaying of assistance messages --- cua.py | 20 +++++----- local_computer.py | 93 ++++++++++++++++++++++++++++++++--------------- main.py | 8 ++++ 3 files changed, 81 insertions(+), 40 deletions(-) diff --git a/cua.py b/cua.py index cf7f428..ec2a243 100644 --- a/cua.py +++ b/cua.py @@ -55,17 +55,17 @@ async def screenshot(self) -> str: data = bytearray(buffer.getvalue()) return base64.b64encode(data).decode("utf-8") - async def click(self, x: int, y: int, button: str = "left") -> None: + async def click(self, x: int, y: int, button: str = "left", keys: list[str] | None = None) -> None: x, y = self._point_to_screen_coords(x, y) - await self.computer.click(x, y, button=button) + await self.computer.click(x, y, button=button, keys=keys) - async def double_click(self, x: int, y: int) -> None: + async def double_click(self, x: int, y: int, keys: list[str] | None = None) -> None: x, y = self._point_to_screen_coords(x, y) - await self.computer.double_click(x, y) + await self.computer.double_click(x, y, keys=keys) - async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None: + async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int, keys: list[str] | None = None) -> None: x, y = self._point_to_screen_coords(x, y) - await self.computer.scroll(x, y, scroll_x, scroll_y) + await self.computer.scroll(x, y, scroll_x, scroll_y, keys=keys) async def type(self, text: str) -> None: await self.computer.type(text) @@ -73,16 +73,16 @@ async def type(self, text: str) -> None: async def wait(self, ms: int = 1000) -> None: await self.computer.wait(ms) - async def move(self, x: int, y: int) -> None: + async def move(self, x: int, y: int, keys: list[str] | None = None) -> None: x, y = self._point_to_screen_coords(x, y) - await self.computer.move(x, y) + await self.computer.move(x, y, keys=keys) async def keypress(self, keys: list[str]) -> None: await self.computer.keypress(keys) - async def drag(self, path: list[tuple[int, int]]) -> None: + async def drag(self, path: list[tuple[int, int]], keys: list[str] | None = None) -> None: path = [self._point_to_screen_coords(*point) for point in path] - await self.computer.drag(path) + await self.computer.drag(path, keys=keys) def _point_to_screen_coords(self, x, y): width, height = self.dimensions diff --git a/local_computer.py b/local_computer.py index 627c483..ce5fbf9 100644 --- a/local_computer.py +++ b/local_computer.py @@ -1,10 +1,31 @@ import asyncio import base64 +import contextlib import io import pyautogui +_KEYMAP = { + "arrowdown": "down", + "arrowleft": "left", + "arrowright": "right", + "arrowup": "up", + "cmd": "win", + "command": "win", + "meta": "win", + "super": "win", + "control": "ctrl", + "return": "enter", + "esc": "escape", +} + + +def _normalize_key(key: str) -> str: + key = key.lower() + return _KEYMAP.get(key, key) + + class LocalComputer: """Use pyautogui to take screenshots and perform actions on the local computer.""" @@ -18,6 +39,20 @@ def dimensions(self): self.size = screenshot.size return self.size + @contextlib.contextmanager + def _with_modifiers(self, keys: list[str] | None): + """Hold modifier keys down for the duration of a mouse action.""" + pressed: list[str] = [] + try: + for key in keys or []: + normalized = _normalize_key(key) + pyautogui.keyDown(normalized) + pressed.append(normalized) + yield + finally: + for key in reversed(pressed): + pyautogui.keyUp(key) + async def screenshot(self) -> str: screenshot = pyautogui.screenshot() self.size = screenshot.size @@ -27,23 +62,26 @@ async def screenshot(self) -> str: data = bytearray(buffer.getvalue()) return base64.b64encode(data).decode("utf-8") - async def click(self, x: int, y: int, button: str = "left") -> None: + async def click(self, x: int, y: int, button: str = "left", keys: list[str] | None = None) -> None: width, height = self.size if 0 <= x < width and 0 <= y < height: button = "middle" if button == "wheel" else button pyautogui.moveTo(x, y, duration=0.1) - pyautogui.click(x, y, button=button) + with self._with_modifiers(keys): + pyautogui.click(x, y, button=button) - async def double_click(self, x: int, y: int) -> None: + async def double_click(self, x: int, y: int, keys: list[str] | None = None) -> None: width, height = self.size if 0 <= x < width and 0 <= y < height: pyautogui.moveTo(x, y, duration=0.1) - pyautogui.doubleClick(x, y) + with self._with_modifiers(keys): + pyautogui.doubleClick(x, y) - async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None: + async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int, keys: list[str] | None = None) -> None: pyautogui.moveTo(x, y, duration=0.5) - pyautogui.scroll(-scroll_y) - pyautogui.hscroll(scroll_x) + with self._with_modifiers(keys): + pyautogui.scroll(-scroll_y) + pyautogui.hscroll(scroll_x) async def type(self, text: str) -> None: pyautogui.write(text) @@ -51,32 +89,27 @@ async def type(self, text: str) -> None: async def wait(self, ms: int = 1000) -> None: await asyncio.sleep(ms / 1000) - async def move(self, x: int, y: int) -> None: - pyautogui.moveTo(x, y, duration=0.1) + async def move(self, x: int, y: int, keys: list[str] | None = None) -> None: + with self._with_modifiers(keys): + pyautogui.moveTo(x, y, duration=0.1) async def keypress(self, keys: list[str]) -> None: - keys = [key.lower() for key in keys] - keymap = { - "arrowdown": "down", - "arrowleft": "left", - "arrowright": "right", - "arrowup": "up", - } - keys = [keymap.get(key, key) for key in keys] + keys = [_normalize_key(key) for key in keys] for key in keys: pyautogui.keyDown(key) - for key in keys: + for key in reversed(keys): pyautogui.keyUp(key) - async def drag(self, path: list[tuple[int, int]]) -> None: - if len(path) <= 1: - pass - elif len(path) == 2: - pyautogui.moveTo(*path[0], duration=0.5) - pyautogui.dragTo(*path[1], duration=1.0, button="left") - else: - pyautogui.moveTo(*path[0], duration=0.5) - pyautogui.mouseDown(button="left") - for point in path[1:]: - pyautogui.dragTo(*point, duration=1.0, mouseDownUp=False) - pyautogui.mouseUp(button="left") + async def drag(self, path: list[tuple[int, int]], keys: list[str] | None = None) -> None: + with self._with_modifiers(keys): + if len(path) <= 1: + pass + elif len(path) == 2: + pyautogui.moveTo(*path[0], duration=0.5) + pyautogui.dragTo(*path[1], duration=1.0, button="left") + else: + pyautogui.moveTo(*path[0], duration=0.5) + pyautogui.mouseDown(button="left") + for point in path[1:]: + pyautogui.dragTo(*point, duration=1.0, mouseDownUp=False) + pyautogui.mouseUp(button="left") diff --git a/main.py b/main.py index 00f4e70..e3205e7 100644 --- a/main.py +++ b/main.py @@ -77,6 +77,14 @@ async def main(): logger.info(f" {action_type} {action_args}") elif item.type == "function_call": logger.info(f" {item.name}") + elif item.type == "message": + for content in item.content: + text = getattr(content, "text", None) + if text is None and getattr(content, "type", "") == "refusal": + text = getattr(content, "refusal", None) + if text: + logger.info("") + logger.info(f"Assistant: {text}") if __name__ == "__main__": asyncio.run(main())