fix(cua): handle lowercase modifier keys and unify right-click in Tzafon templates

Author: mttias

pkg/templates/python/tzafon-computer-use/tools/computer.py

@@ -15,7 +15,19 @@
 
 from .base import ToolError
 
-MODIFIER_MAP = {"Control": "Ctrl", "Enter": "Return"}
+MODIFIER_MAP = {
+    "Control": "Ctrl",
+    "control": "Ctrl",
+    "ctrl": "Ctrl",
+    "Enter": "Return",
+    "enter": "Return",
+    "Escape": "Escape",
+    "esc": "Escape",
+    "Shift": "Shift",
+    "shift": "Shift",
+    "Alt": "Alt",
+    "alt": "Alt",
+}
 MODIFIER_NAMES = {"Ctrl", "Shift", "Alt", "Meta", "Super"}
 
 
@@ -65,7 +77,11 @@ async def execute(self, action: Any) -> None:
         """Map a Tzafon model action to Kernel Computer Controls."""
         t = action.type
 
-        if t == "click":
+        if t == "click" and getattr(action, "button", "left") == "right":
+            self.kernel.browsers.computer.click_mouse(
+                self.session_id, x=self._x(action), y=self._y(action), button="right",
+            )
+        elif t == "click":
             self.kernel.browsers.computer.click_mouse(
                 self.session_id, x=self._x(action), y=self._y(action),
             )
@@ -77,10 +93,6 @@ async def execute(self, action: Any) -> None:
             self.kernel.browsers.computer.click_mouse(
                 self.session_id, x=self._x(action), y=self._y(action), num_clicks=3,
             )
-        elif t == "right_click":
-            self.kernel.browsers.computer.click_mouse(
-                self.session_id, x=self._x(action), y=self._y(action), button="right",
-            )
         elif t == "type":
             self.kernel.browsers.computer.type_text(self.session_id, text=action.text)
         elif t in ("key", "keypress"):

pkg/templates/typescript/tzafon-computer-use/tools/computer.ts

@@ -17,7 +17,16 @@ export class ToolError extends Error {
 
 const MODIFIER_MAP: Record<string, string> = {
   Control: 'Ctrl',
+  control: 'Ctrl',
+  ctrl: 'Ctrl',
   Enter: 'Return',
+  enter: 'Return',
+  Escape: 'Escape',
+  esc: 'Escape',
+  Shift: 'Shift',
+  shift: 'Shift',
+  Alt: 'Alt',
+  alt: 'Alt',
 };
 
 const MODIFIER_NAMES = new Set(['Ctrl', 'Shift', 'Alt', 'Meta', 'Super']);
@@ -75,6 +84,13 @@ export class ComputerTool {
   }
 
   async execute(action: any): Promise<void> {
+    if (action.type === 'click' && action.button === 'right') {
+      await this.kernel.browsers.computer.clickMouse(this.sessionId, {
+        x: this.x(action), y: this.y(action), button: 'right',
+      });
+      return;
+    }
+
     switch (action.type) {
       case 'click':
         await this.kernel.browsers.computer.clickMouse(this.sessionId, {
@@ -94,12 +110,6 @@ export class ComputerTool {
         });
         break;
 
-      case 'right_click':
-        await this.kernel.browsers.computer.clickMouse(this.sessionId, {
-          x: this.x(action), y: this.y(action), button: 'right',
-        });
-        break;
-
       case 'type':
         await this.kernel.browsers.computer.typeText(this.sessionId, {
           text: action.text,