Fixed the harness method signatures by adding optional keys argument …

cua.py

@@ -55,34 +55,34 @@ async def screenshot(self) -> str:
         data = bytearray(buffer.getvalue())
         return base64.b64encode(data).decode("utf-8")
 
-    async def click(self, x: int, y: int, button: str = "left") -> None:
+    async def click(self, x: int, y: int, button: str = "left", keys: list[str] | None = None) -> None:
         x, y = self._point_to_screen_coords(x, y)
-        await self.computer.click(x, y, button=button)
+        await self.computer.click(x, y, button=button, keys=keys)
 
-    async def double_click(self, x: int, y: int) -> None:
+    async def double_click(self, x: int, y: int, keys: list[str] | None = None) -> None:
         x, y = self._point_to_screen_coords(x, y)
-        await self.computer.double_click(x, y)
+        await self.computer.double_click(x, y, keys=keys)
 
-    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
+    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int, keys: list[str] | None = None) -> None:
         x, y = self._point_to_screen_coords(x, y)
-        await self.computer.scroll(x, y, scroll_x, scroll_y)
+        await self.computer.scroll(x, y, scroll_x, scroll_y, keys=keys)
 
     async def type(self, text: str) -> None:
         await self.computer.type(text)
 
     async def wait(self, ms: int = 1000) -> None:
         await self.computer.wait(ms)
 
-    async def move(self, x: int, y: int) -> None:
+    async def move(self, x: int, y: int, keys: list[str] | None = None) -> None:
         x, y = self._point_to_screen_coords(x, y)
-        await self.computer.move(x, y)
+        await self.computer.move(x, y, keys=keys)
 
     async def keypress(self, keys: list[str]) -> None:
         await self.computer.keypress(keys)
 
-    async def drag(self, path: list[tuple[int, int]]) -> None:
+    async def drag(self, path: list[tuple[int, int]], keys: list[str] | None = None) -> None:
         path = [self._point_to_screen_coords(*point) for point in path]
-        await self.computer.drag(path)
+        await self.computer.drag(path, keys=keys)
 
     def _point_to_screen_coords(self, x, y):
         width, height = self.dimensions

local_computer.py

@@ -1,10 +1,31 @@
 import asyncio
 import base64
+import contextlib
 import io
 
 import pyautogui
 
 
+_KEYMAP = {
+    "arrowdown": "down",
+    "arrowleft": "left",
+    "arrowright": "right",
+    "arrowup": "up",
+    "cmd": "win",
+    "command": "win",
+    "meta": "win",
+    "super": "win",
+    "control": "ctrl",
+    "return": "enter",
+    "esc": "escape",
+}
+
+
+def _normalize_key(key: str) -> str:
+    key = key.lower()
+    return _KEYMAP.get(key, key)
+
+
 class LocalComputer:
     """Use pyautogui to take screenshots and perform actions on the local computer."""
 
@@ -18,6 +39,20 @@ def dimensions(self):
             self.size = screenshot.size
         return self.size
 
+    @contextlib.contextmanager
+    def _with_modifiers(self, keys: list[str] | None):
+        """Hold modifier keys down for the duration of a mouse action."""
+        pressed: list[str] = []
+        try:
+            for key in keys or []:
+                normalized = _normalize_key(key)
+                pyautogui.keyDown(normalized)
+                pressed.append(normalized)
+            yield
+        finally:
+            for key in reversed(pressed):
+                pyautogui.keyUp(key)
+
     async def screenshot(self) -> str:
         screenshot = pyautogui.screenshot()
         self.size = screenshot.size
@@ -27,56 +62,54 @@ async def screenshot(self) -> str:
         data = bytearray(buffer.getvalue())
         return base64.b64encode(data).decode("utf-8")
 
-    async def click(self, x: int, y: int, button: str = "left") -> None:
+    async def click(self, x: int, y: int, button: str = "left", keys: list[str] | None = None) -> None:
         width, height = self.size
         if 0 <= x < width and 0 <= y < height:
             button = "middle" if button == "wheel" else button
             pyautogui.moveTo(x, y, duration=0.1)
-            pyautogui.click(x, y, button=button)
+            with self._with_modifiers(keys):
+                pyautogui.click(x, y, button=button)
 
-    async def double_click(self, x: int, y: int) -> None:
+    async def double_click(self, x: int, y: int, keys: list[str] | None = None) -> None:
         width, height = self.size
         if 0 <= x < width and 0 <= y < height:
             pyautogui.moveTo(x, y, duration=0.1)
-            pyautogui.doubleClick(x, y)
+            with self._with_modifiers(keys):
+                pyautogui.doubleClick(x, y)
 
-    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
+    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int, keys: list[str] | None = None) -> None:
         pyautogui.moveTo(x, y, duration=0.5)
-        pyautogui.scroll(-scroll_y)
-        pyautogui.hscroll(scroll_x)
+        with self._with_modifiers(keys):
+            pyautogui.scroll(-scroll_y)
+            pyautogui.hscroll(scroll_x)
 
     async def type(self, text: str) -> None:
         pyautogui.write(text)
 
     async def wait(self, ms: int = 1000) -> None:
         await asyncio.sleep(ms / 1000)
 
-    async def move(self, x: int, y: int) -> None:
-        pyautogui.moveTo(x, y, duration=0.1)
+    async def move(self, x: int, y: int, keys: list[str] | None = None) -> None:
+        with self._with_modifiers(keys):
+            pyautogui.moveTo(x, y, duration=0.1)
 
     async def keypress(self, keys: list[str]) -> None:
-        keys = [key.lower() for key in keys]
-        keymap = {
-            "arrowdown": "down",
-            "arrowleft": "left",
-            "arrowright": "right",
-            "arrowup": "up",
-        }
-        keys = [keymap.get(key, key) for key in keys]
+        keys = [_normalize_key(key) for key in keys]
         for key in keys:
             pyautogui.keyDown(key)
-        for key in keys:
+        for key in reversed(keys):
             pyautogui.keyUp(key)
 
-    async def drag(self, path: list[tuple[int, int]]) -> None:
-        if len(path) <= 1:
-            pass
-        elif len(path) == 2:
-            pyautogui.moveTo(*path[0], duration=0.5)
-            pyautogui.dragTo(*path[1], duration=1.0, button="left")
-        else:
-            pyautogui.moveTo(*path[0], duration=0.5)
-            pyautogui.mouseDown(button="left")
-            for point in path[1:]:
-                pyautogui.dragTo(*point, duration=1.0, mouseDownUp=False)
-            pyautogui.mouseUp(button="left")
+    async def drag(self, path: list[tuple[int, int]], keys: list[str] | None = None) -> None:
+        with self._with_modifiers(keys):
+            if len(path) <= 1:
+                pass
+            elif len(path) == 2:
+                pyautogui.moveTo(*path[0], duration=0.5)
+                pyautogui.dragTo(*path[1], duration=1.0, button="left")
+            else:
+                pyautogui.moveTo(*path[0], duration=0.5)
+                pyautogui.mouseDown(button="left")
+                for point in path[1:]:
+                    pyautogui.dragTo(*point, duration=1.0, mouseDownUp=False)
+                pyautogui.mouseUp(button="left")

main.py

@@ -77,6 +77,14 @@ async def main():
                     logger.info(f"  {action_type} {action_args}")
             elif item.type == "function_call":
                 logger.info(f"  {item.name}")
+            elif item.type == "message":
+                for content in item.content:
+                    text = getattr(content, "text", None)
+                    if text is None and getattr(content, "type", "") == "refusal":
+                        text = getattr(content, "refusal", None)
+                    if text:
+                        logger.info("")
+                        logger.info(f"Assistant: {text}")
 
 if __name__ == "__main__":
     asyncio.run(main())