From ae3b36ac88be0da677b0f0a8564fcfbdb718d44b Mon Sep 17 00:00:00 2001 From: N3MI-DG <129717180+N3MI-DG@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:42:59 +0000 Subject: [PATCH] Agent cursor follows the agent's own pointer input (ui.move, ui.click, ui.drag, ui.scroll) (#5162) The agent cursor only acted out engine commands: pointer input sent over the control channel reached the app as synthetic events without moving it, so an agent's clicks, right-clicks and drags were invisible to a person watching. While the cursor is shown, each synthetic pointer event now moves it to where the input lands, a press leaves a ripple, and a drag is followed step by step. A request being acted out keeps the pointer; with the cursor hidden nothing changes. Co-authored-by: Claude Opus 5.5 --- crates/ui-egui/src/agent_cursor.rs | 40 +++++++++++++++++++++++++++++- crates/ui-egui/src/lib.rs | 7 ++++++ docs/control-protocol.md | 2 +- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/crates/ui-egui/src/agent_cursor.rs b/crates/ui-egui/src/agent_cursor.rs index 949e847d83..4163308d11 100644 --- a/crates/ui-egui/src/agent_cursor.rs +++ b/crates/ui-egui/src/agent_cursor.rs @@ -4,7 +4,9 @@ //! then glides to each pick in the parameters (faces, edges and points projected to the screen, //! sketch points, the plane of a new sketch), and only then does the command run. The reply //! waits for the animation, so an agent's timing stays honest. With "follow camera" on, a new -//! sketch turns the view to face it and finishing the sketch turns back. +//! sketch turns the view to face it and finishing the sketch turns back. Pointer input the agent +//! sends itself (`ui.move`, `ui.click`, `ui.drag`, `ui.scroll`) moves the pointer too, as each +//! event reaches the app, and a press leaves a ripple. //! //! The show is bounded, whatever the request's size: besides the toolbar button the pointer //! visits at most [`MAX_PICK_STOPS`] picks (a long point list is summarised by its first, middle @@ -168,6 +170,19 @@ impl AgentCursor { self.job.as_ref().filter(|j| j.stops.is_empty()).and_then(|j| j.ahead.as_ref()).map(|a| (a.started, a.label.as_str())) } + /// Follow the pointer input an agent sends itself (`ui.move`, `ui.click`, `ui.drag`, + /// `ui.scroll`), as it reaches the app: the pointer goes where the input lands, and a press + /// leaves a ripple. A request being acted out keeps the pointer meanwhile. + pub(crate) fn follow_input(&mut self, at: Pos2, press: bool, now: f64) { + if self.job.is_some() { + return; + } + self.pos = Some(at); + if press { + self.ripples.push((at, now)); + } + } + /// Stop the request the pointer waits on: answered as stopped, nothing of it applied. pub(crate) fn stop(&mut self) -> Option<(Sender, String)> { self.computing()?; @@ -564,6 +579,29 @@ mod tests { assert!(frames <= 3, "{frames}"); } + #[test] + fn the_pointer_follows_the_agents_own_mouse_input() { + let mut h = crate::scenario::Harness::new(); + // Hidden: control-channel input leaves the pointer alone. + assert_eq!(h.call("ui.agent_cursor", json!({"show": false}))["ok"], true); + h.call("ui.move", json!({"x": 300, "y": 300})); + h.frames(2); + assert_eq!(h.app.agent.pos, None); + + assert_eq!(h.call("ui.agent_cursor", json!({"show": true}))["ok"], true); + h.call("ui.move", json!({"x": 420, "y": 380})); + h.frames(2); + assert_eq!(h.app.agent.pos, Some(pos2(420.0, 380.0)), "a move moves it"); + h.call("ui.click", json!({"x": 500, "y": 400, "button": "right"})); + h.frames(4); + assert_eq!(h.app.agent.pos, Some(pos2(500.0, 400.0)), "a click lands where it clicked"); + assert!(h.app.agent.ripples.iter().any(|(at, _)| *at == pos2(500.0, 400.0)), "and leaves a ripple"); + // A drag is followed (one step a frame) to where it is released. + h.call("ui.drag", json!({"x0": 600, "y0": 300, "x1": 700, "y1": 360, "steps": 4})); + h.frames(8); + assert_eq!(h.app.agent.pos, Some(pos2(700.0, 360.0))); + } + #[test] fn speeds_read_old_and_new_settings() { // Saved settings from before #79: slow and normal are 1×, instant stays. diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index b6c7a4d428..43503e87b7 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -1549,7 +1549,14 @@ impl SolveApp { ) }) .map_or(self.synthetic.len(), |i| i + 1); + let follow = self.ui.agent_cursor.show; for event in self.synthetic.drain(..n) { + // The agent cursor shows the agent's own pointer input too, not only its commands. + match &event { + egui::Event::PointerMoved(at) if follow => self.agent.follow_input(*at, false, self.now), + egui::Event::PointerButton { pos, pressed, .. } if follow => self.agent.follow_input(*pos, *pressed, self.now), + _ => {} + } match &event { egui::Event::ModifiersChanged(modifiers) => self.synthetic_mods = *modifiers, egui::Event::PointerButton { pos, modifiers, pressed, .. } => { diff --git a/docs/control-protocol.md b/docs/control-protocol.md index b711506677..8abeddc385 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -208,7 +208,7 @@ labels; `prefs:measure_axis_distances` identifies the checkbox. | `ui.prefs` | any of `default_units`, `nav` (`fusion`, `solidworks`, `inventor`, `alias`, `blender`, `tinkercad`, `openscad`, `touchpad`), `zoom_reverse`, `pan_reverse_x` (default false: horizontal scrolling pans the other way), `zoom_to_cursor`, `smooth_wheel_zoom` (default true), `trackpad_pan` (default true: two-finger scroll pans, except Touchpad always zooms; pinch zooms), `orbit_around` (`view_centre` (default: what Pan moves and Fit frames), `origin` (the world origin stays still on screen), `cursor` (the model point under the pointer when the drag began); the older `orbit_cursor` boolean still sets `cursor` when true), `look_at_anim`, `look_at_speed`, `middle_double_click` (`home` (default), `fit`, `off`), `msaa`, `show_frame_rate` (default false: frames per second in the view's bottom-right corner, counting the frames the view draws anyway; it reads 0 FPS once the view is idle), `length_decimals`, `angle_decimals`, `theme` (`dark`, `light`, `system` or a theme file's id: a built-in one such as `gruvbox-dark`, or an imported one; the reply's `theme_ids` lists them), `theme_import` (a `solidcraft-theme/1` file's JSON or text: adds the theme and turns it on; the reply's `theme_imported` is its id), `theme_remove` (an imported theme's id), `theme_export: true` (the reply's `theme_file` is the theme on now as a complete file; see [colour themes](book/src/guide/ui-tour.md#colour-themes)), `language` (`en`, `ja`, `de`, `fr`, `ko`, `ms`, `ru`, `uk`, `zh`; see [localization](localization.md)), `ui_scale`, `ui_font` (an installed font family or a font file for the interface; empty for the built-in font), `up_axis` (`z`, `y`), `auto_stand_up` (default true: a STEP or IGES file whose header names a Y-up exporter, SolidWorks, is stood up as it opens), `offer_stand_up`, `color_vision` (`off` (default), `protan`, `deutan`, `tritan`: colour vision modes, #778), `check_updates` (default true; [updates](book/src/guide/updates.md)); `open` (a page: General, Interface, Design, Navigation, Keyboard Shortcuts, Sketch, Graphics, AI, Plugins; Shortcuts, Assistant and Sketch & selection are aliases, and the former Selection page opens Sketch) opens the Preferences dialog there; `button`: `ok`, `cancel`, `apply`, `defaults`; `stand_up: true` stands up a just-opened Y-up file, `stand_up: false` leaves it as it is ("Leave it": the offer closes and `offer_stand_up` stays on); the offer's buttons are the handles `stand_up:stand`, `stand_up:leave` and `stand_up:never` | sets preferences (kept between runs), returns them all plus `dialog` (the open page or null), `dark` and `stand_up_offer` | | `ui.capture` | `action?`: `capture`, `revert`, `cancel` | answers the Capture Position question (asked when a command starts or the design is saved while components were moved without capturing); without `action`, what waits | | `ui.window` | `action`: minimize, maximize, restore, toggle, close | what the title bar's caption buttons do (on Windows and Linux the application bar is the title bar); close asks about unsaved designs first | -| `ui.agent_cursor` | `show` (bool), `speed`: a number from 1 to 5 (times the normal speed) or `"instant"` (no animation: the command runs at once and the pointer jumps to where it acted), `follow_camera` (bool); all optional | the settings. While the cursor is shown, `engine.execute` first glides to the command's toolbar button and to each pick, then runs and replies; later requests wait their turn. `"animate": false` on a call skips it | +| `ui.agent_cursor` | `show` (bool), `speed`: a number from 1 to 5 (times the normal speed) or `"instant"` (no animation: the command runs at once and the pointer jumps to where it acted), `follow_camera` (bool); all optional | the settings. While the cursor is shown, `engine.execute` first glides to the command's toolbar button and to each pick, then runs and replies; later requests wait their turn. It also follows the pointer input sent with `ui.move`, `ui.click`, `ui.drag` and `ui.scroll`, as each event reaches the app, and a press leaves a ripple. `"animate": false` on a call skips it | | `engine.execute` → `view.layout` | `mode`: `floating` or `compact` | Apply and save the workspace layout without opening a dialog, changing the design, resetting the camera or cancelling its active tool. General preferences expose the same choice; `ui.prefs {"layout":"compact"}` also saves it. | | `engine.execute` → `view.history_layout` | `width?`, `dock?`: `bottom`, `left`, `under_browser`, `collapsed?` | Resize/dock the history without opening a dialog or changing the design. Contents respond to measured width; browser-stacked history shares its width and splits height equally. One folded header gives the expanded card the rest; two folded headers stay at the top. | | `ui.agent_chat` | `features?`, `toolbar_button?`, `permissions?` (`ask`, `edits`, `all`), `add_agent?` ({`name`, `kind`: `acp` or `api`, `command` or `base_url` + `model`}; no keys), `default_agent?`, `agent?` (the conversation's agent, by id), `mode?` (the agent's own mode), `model?` (one of its models), `open?`, `side?` (`left`, `right`), `minimized?`, `popped?` (in its own window), `width?`, `picker?`, `refresh?`, `connect?`, `stop?`, `attach?`: a file's path (a picture, a text file or a CAD file, for the next message; CAD files are imported at once), `remove_file?` (index of a file on the next message), `send?`: text (files alone make a message too), `attach_view?`, `permission?`: an option id or `"cancel"` (the newest question of the agent's own), `approve?`: true, false or `"always"` (the oldest request held for Allow/Deny), `cancel?` (Stop: the queue stays), `send_now?` (stop the running step, then send the queued messages), `remove_pending?` (index), `new_chat?` (clear the chat, new session), `undo?` (the last message's changes) | the chat's state: `features`, `agents` (found and added), `currentAgent`, `state` (off, starting, ready, busy, auth, exited), `popped`, `window` (its place and size), `agent`, `modes`, `mode`, `models`, `model`, `authMethods`, `toolsPort`, `pid`, `waiting`, `turn`, `pending` (queued messages, with their `files`), `attachments` (the files on the next message, by name), `loading` (pictures still being read for it, by name), `stopped`, `detection` and the conversation (`items`). With `features` off only the settings apply. Desktop only; see [Agent chat](agent-chat.md) |