From 4355d117efb24f46e2451e27488c19d95a946ecb Mon Sep 17 00:00:00 2001 From: raykholo Date: Sun, 25 Oct 2026 21:37:30 +0000 Subject: [PATCH] Assistant: warn when an Ollama model has no tool support (#4846) (#4916) * claim: #4846 * Warn when Ollama models lack tool support (#4846) Check Ollama model capabilities in the background and cache results per endpoint and model. Show an advisory warning for models without tools, expose it through ui.agent_chat, and document the behavior. Cover capability parsing, warning selection, background probing, caching, and selection changes with synthetic regression tests. * Keep the Ollama capability probe out of the web build The panel never calls it there, so it would only be dead code. * Wrap the model warning instead of cutting it off In a narrow panel the end of the sentence, the part that says what to do, was truncated. * Fix AgentConfig default after rebase onto main Co-Authored-By: Claude Opus 5.5 * Move the Ollama no-tools warning into a pending locale file with ja and zh Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: bherbruck Co-authored-by: Claude Opus 5.5 --- .../pending/ollama-model-no-tools.json | 5 + crates/ui-egui/src/agent_chat/agents.rs | 3 + crates/ui-egui/src/agent_chat/discover.rs | 83 ++++++++++ crates/ui-egui/src/agent_chat/mod.rs | 3 + crates/ui-egui/src/agent_chat/panel.rs | 3 + crates/ui-egui/src/agent_chat/settings.rs | 142 ++++++++++++++++++ docs/agent-chat.md | 7 + docs/control-protocol.md | 2 +- 8 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 assets/locales/pending/ollama-model-no-tools.json diff --git a/assets/locales/pending/ollama-model-no-tools.json b/assets/locales/pending/ollama-model-no-tools.json new file mode 100644 index 0000000000..087498dad0 --- /dev/null +++ b/assets/locales/pending/ollama-model-no-tools.json @@ -0,0 +1,5 @@ +{ + "en": { "ui.agent_chat_model_no_tools": "{model} can't use SolidCraft's tools, so it can only chat. Pick a model with tool support." }, + "ja": { "ui.agent_chat_model_no_tools": "{model} は SolidCraft のツールを使えず、チャットのみ可能です。ツール対応のモデルを選んでください。" }, + "zh": { "ui.agent_chat_model_no_tools": "{model} 无法使用 SolidCraft 的工具，只能聊天。请选择支持工具的模型。" } +} diff --git a/crates/ui-egui/src/agent_chat/agents.rs b/crates/ui-egui/src/agent_chat/agents.rs index 7931760779..8d989a9084 100644 --- a/crates/ui-egui/src/agent_chat/agents.rs +++ b/crates/ui-egui/src/agent_chat/agents.rs @@ -106,6 +106,8 @@ pub struct AgentConfig { /// API: the endpoint (`…/v1`) and model. pub base_url: String, pub model: String, + /// Added with the Ollama preset (even if its display name changes). + pub ollama: bool, /// API: a key is stored for it in the system keychain. pub key: bool, /// API: how long one model request may take, in seconds (clamped to @@ -122,6 +124,7 @@ impl Default for AgentConfig { command: String::new(), base_url: String::new(), model: String::new(), + ollama: false, key: false, request_timeout_s: DEFAULT_REQUEST_TIMEOUT_S, } diff --git a/crates/ui-egui/src/agent_chat/discover.rs b/crates/ui-egui/src/agent_chat/discover.rs index 17663d4df6..0fd03560a9 100644 --- a/crates/ui-egui/src/agent_chat/discover.rs +++ b/crates/ui-egui/src/agent_chat/discover.rs @@ -352,10 +352,93 @@ fn detect_in(dirs: &[PathBuf], home: Option<&Path>) -> Detection { d } +/// Query only the endpoint's origin; OpenAI-compatible paths such as `/v1` are unrelated. +#[cfg(not(target_arch = "wasm32"))] +fn api_origin(base_url: &str) -> Option { + let (scheme, rest) = base_url.trim().split_once("://")?; + if !matches!(scheme, "http" | "https") { + return None; + } + let authority = rest.split(['/', '?', '#']).next()?; + if authority.is_empty() || authority.contains(['@', ' ']) || authority.len() > 2048 { + return None; + } + Some(format!("{scheme}://{authority}")) +} + +/// Blocking, best-effort Ollama probe. Call only on a background thread, without API keys. +#[cfg(not(target_arch = "wasm32"))] +pub(super) fn ollama_tools(http: &super::backend::HttpFn, base_url: &str, model: &str, known_ollama: bool) -> Option { + use super::backend::HttpRequest; + let origin = api_origin(base_url)?; + let request = |method, path: &str, body| { + let (status, bytes) = http(&HttpRequest { + method, + url: format!("{origin}/api/{path}"), + headers: if method == "POST" { vec![("Content-Type".into(), "application/json".into())] } else { Vec::new() }, + body, + timeout_s: 5, + }) + .ok()?; + if !(200..300).contains(&status) || bytes.len() > 4 * 1024 * 1024 { + return None; + } + serde_json::from_slice::(&bytes).ok() + }; + if !known_ollama { + let version = request("GET", "version", None)?; + version.get("version")?.as_str()?; + } + let body = serde_json::to_vec(&serde_json::json!({"model": model})).ok()?; + tools_capability(&request("POST", "show", Some(body))?) +} + +/// Tool support is unknown unless Ollama returns a capabilities list of strings. +#[cfg(not(target_arch = "wasm32"))] +pub(super) fn tools_capability(value: &serde_json::Value) -> Option { + let capabilities = value.get("capabilities")?.as_array()?; + if capabilities.iter().any(|capability| !capability.is_string()) { + return None; + } + Some(capabilities.iter().any(|capability| capability.as_str() == Some("tools"))) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn ollama_capabilities_distinguish_tools_chat_and_unknown() { + use serde_json::json; + assert_eq!(tools_capability(&json!({"capabilities": ["completion", "tools"]})), Some(true)); + assert_eq!(tools_capability(&json!({"capabilities": ["completion"]})), Some(false)); + assert_eq!(tools_capability(&json!({"capabilities": []})), Some(false)); + for value in [json!({}), json!({"capabilities": null}), json!({"capabilities": [false]})] { + assert_eq!(tools_capability(&value), None); + } + assert_eq!(api_origin("http://[::1]:11434/proxy/v1/").as_deref(), Some("http://[::1]:11434")); + assert_eq!(api_origin("file:///tmp/model"), None); + let http: super::super::backend::HttpFn = std::sync::Arc::new(|request| { + assert_eq!(request.timeout_s, 5); + match request.method { + "GET" => { + assert_eq!(request.url, "http://localhost:11434/api/version"); + Ok((200, br#"{"version":"0.0"}"#.to_vec())) + } + "POST" => { + assert_eq!(request.url, "http://localhost:11434/api/show"); + assert_eq!(serde_json::from_slice::(request.body.as_ref().unwrap()).unwrap(), json!({"model":"chat:4b"})); + Ok((200, br#"{"capabilities":["completion"]}"#.to_vec())) + } + _ => Err("unexpected HTTP method".into()), + } + }); + assert_eq!(ollama_tools(&http, "http://localhost:11434/custom/v1", "chat:4b", false), Some(false)); + assert_eq!(ollama_tools(&http, "http://localhost:11434/v1", "chat:4b", true), Some(false)); + let unavailable: super::super::backend::HttpFn = std::sync::Arc::new(|_| Err("unavailable".into())); + assert_eq!(ollama_tools(&unavailable, "http://localhost:11434/v1", "chat:4b", false), None); + } + #[test] fn a_program_is_found_in_the_first_folder_that_has_it() { let root = std::env::temp_dir().join(format!("sc-agent-find-{}", std::process::id())); diff --git a/crates/ui-egui/src/agent_chat/mod.rs b/crates/ui-egui/src/agent_chat/mod.rs index bcd943e3c1..10be0d4a94 100644 --- a/crates/ui-egui/src/agent_chat/mod.rs +++ b/crates/ui-egui/src/agent_chat/mod.rs @@ -1614,6 +1614,8 @@ pub fn poll(app: &mut SolveApp, ctx: &egui::Context) { start_turn(app, text, image, at, files); } } + #[cfg(not(target_arch = "wasm32"))] + settings::poll_capabilities(app, ctx); pump_tools(app); if !app.agent_chat.held.is_empty() || app.agent_chat.state() == State::Busy { ctx.request_repaint_after(std::time::Duration::from_millis(250)); @@ -1964,6 +1966,7 @@ pub fn status(app: &SolveApp) -> Value { "model": c.model, "providers": model_providers(app).iter().map(|(name, count)| json!({"name": name, "count": count})).collect::>(), "provider": shown_provider(app), + "modelWarning": settings::model_warning(app), "modes": c.modes.iter().map(|(id, name)| json!({"id": id, "name": name})).collect::>(), "mode": c.mode, "open": c.open, diff --git a/crates/ui-egui/src/agent_chat/panel.rs b/crates/ui-egui/src/agent_chat/panel.rs index 31a9811a5a..7e4ebd30b0 100644 --- a/crates/ui-egui/src/agent_chat/panel.rs +++ b/crates/ui-egui/src/agent_chat/panel.rs @@ -1163,6 +1163,9 @@ fn switcher(app: &mut SolveApp, ui: &mut egui::Ui) { } } }); + if let Some(warning) = settings::model_warning(app) { + ui.add(egui::Label::new(RichText::new(warning).color(t.warning).size(t.font_small)).wrap()); + } ui.horizontal(|ui| { let b = if app.agent_chat.state().running() { ui_kit::button(ui, tr("ui.agent_chat_stop"), ui_kit::Button::Normal).on_hover_text(tr("ui.agent_chat_stop_tip")) diff --git a/crates/ui-egui/src/agent_chat/settings.rs b/crates/ui-egui/src/agent_chat/settings.rs index 58a8c06fdc..12cc04a089 100644 --- a/crates/ui-egui/src/agent_chat/settings.rs +++ b/crates/ui-egui/src/agent_chat/settings.rs @@ -54,6 +54,92 @@ impl Test { pub struct State { pub add: Option, pub tests: std::collections::HashMap, + #[cfg(not(target_arch = "wasm32"))] + capabilities: std::collections::HashMap<(String, String), Option>, + #[cfg(not(target_arch = "wasm32"))] + checking: Option, +} + +#[cfg(not(target_arch = "wasm32"))] +struct CapabilityCheck { + key: (String, String), + rx: std::sync::mpsc::Receiver>, +} + +/// Poll one bounded background probe at a time. Failures are cached too. +#[cfg(not(target_arch = "wasm32"))] +pub(super) fn poll_capabilities(app: &mut SolveApp, ctx: &egui::Context) { + use std::sync::mpsc::TryRecvError; + let state = &mut app.agent_chat.settings; + if let Some(check) = &state.checking { + let result = match check.rx.try_recv() { + Ok(value) => Some(value), + Err(TryRecvError::Disconnected) => Some(None), + Err(TryRecvError::Empty) => None, + }; + if let Some(value) = result { + if state.capabilities.len() >= 128 { + state.capabilities.clear(); + } + state.capabilities.insert(check.key.clone(), value); + state.checking = None; + } + } + if state.checking.is_some() { + return; + } + let Some((key, known)) = capability_selection(app) else { return }; + if app.agent_chat.settings.capabilities.contains_key(&key) { + return; + } + let Some(http) = app.services.agent_http.clone() else { return }; + if app.agent_chat.settings.capabilities.len() >= 128 { + app.agent_chat.settings.capabilities.clear(); + } + let (tx, rx) = std::sync::mpsc::channel(); + let query = key.clone(); + let wake = ctx.clone(); + if std::thread::Builder::new() + .name("model-capabilities".into()) + .spawn(move || { + let _ = tx.send(discover::ollama_tools(&http, &query.0, &query.1, known)); + wake.request_repaint(); + }) + .is_ok() + { + app.agent_chat.settings.checking = Some(CapabilityCheck { key, rx }); + } else { + app.agent_chat.settings.capabilities.insert(key, None); + } +} + +#[cfg(not(target_arch = "wasm32"))] +fn capability_selection(app: &SolveApp) -> Option<((String, String), bool)> { + let entry = agents::current(app)?; + if entry.kind != Kind::Api { + return None; + } + let model = if app.agent_chat.backend.is_none() || app.agent_chat.model.is_empty() { entry.model } else { app.agent_chat.model.clone() }; + if model.is_empty() || model.len() > 4096 || entry.base_url.len() > 4096 { + return None; + } + let known = app.preferences.agents.iter().any(|a| a.id == entry.id && (a.ollama || a.name.eq_ignore_ascii_case("Ollama"))); + Some(((entry.base_url, model), known)) +} + +/// The same advisory text is used by the panel and the control channel. +pub(super) fn model_warning(app: &SolveApp) -> Option { + #[cfg(not(target_arch = "wasm32"))] + if let Some((key, _)) = capability_selection(app) { + return capability_warning(&key.1, app.agent_chat.settings.capabilities.get(&key).copied().flatten()); + } + let _ = app; + None +} + +#[cfg(not(target_arch = "wasm32"))] +fn capability_warning(model: &str, tools: Option) -> Option { + (tools == Some(false)).then(|| trf("ui.agent_chat_model_no_tools", &[("model", model)])) } /// Open the Add agent dialog. @@ -645,6 +731,7 @@ fn add_agent(app: &mut SolveApp) -> Result<(), String> { } AddKind::Api => { c.kind = s.protocol.kind(); + c.ollama = s.protocol == Protocol::OpenAi && s.preset == 0; c.base_url = agents::normalize_url(&s.base_url)?; c.model = s.model.trim().to_string(); c.request_timeout_s = s.request_timeout_s.clamp(agents::MIN_REQUEST_TIMEOUT_S, agents::MAX_REQUEST_TIMEOUT_S); @@ -676,6 +763,61 @@ mod card_tests { use crate::scenario::{self, Harness}; use serde_json::json; + #[test] + fn cached_model_warning_follows_endpoint_model_and_agent_selection() { + let mut h = Harness::new(); + h.app.agent_chat.detection = Some(Default::default()); + h.app.preferences.agents.push(AgentConfig { + id: "local".into(), + kind: Kind::Api, + base_url: "http://localhost:11434/v1".into(), + model: "chat:4b".into(), + ..Default::default() + }); + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + use std::time::{Duration, Instant}; + let calls = Arc::new(AtomicUsize::new(0)); + let count = calls.clone(); + let ui_thread = std::thread::current().id(); + h.app.services.agent_http = Some(Arc::new(move |request| { + assert_ne!(std::thread::current().id(), ui_thread); + count.fetch_add(1, Ordering::SeqCst); + Ok((200, if request.method == "GET" { br#"{"version":"0.0"}"#.to_vec() } else { br#"{"capabilities":["completion"]}"#.to_vec() })) + })); + poll_capabilities(&mut h.app, &h.ctx); + let deadline = Instant::now() + Duration::from_secs(2); + while h.app.agent_chat.settings.checking.is_some() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(1)); + poll_capabilities(&mut h.app, &h.ctx); + } + assert!(h.app.agent_chat.settings.checking.is_none()); + assert_eq!(calls.load(Ordering::SeqCst), 2); + poll_capabilities(&mut h.app, &h.ctx); + assert_eq!(calls.load(Ordering::SeqCst), 2, "cached selections do not probe again"); + assert_eq!(status(&h.app)["modelWarning"], capability_warning("chat:4b", Some(false)).unwrap()); + h.app.preferences.agents[0].model = "tools:8b".into(); + assert!(status(&h.app)["modelWarning"].is_null()); + h.app.preferences.agents[0].model = "chat:4b".into(); + h.app.preferences.agents[0].base_url = "http://localhost:1234/v1".into(); + assert!(status(&h.app)["modelWarning"].is_null()); + h.app.preferences.agents[0].base_url = "http://localhost:11434/v1".into(); + h.app.preferences.agents[0].kind = Kind::Acp; + assert!(status(&h.app)["modelWarning"].is_null()); + } + + #[test] + fn model_warning_only_for_explicitly_missing_tools() { + assert_eq!( + capability_warning("local:4b", Some(false)).as_deref(), + Some("local:4b can't use SolidCraft's tools, so it can only chat. Pick a model with tool support.") + ); + assert_eq!(capability_warning("local:4b", Some(true)), None); + assert_eq!(capability_warning("local:4b", None), None); + } + #[test] fn add_agent_card_retains_kind_handles_draft_and_validation() { let mut h = Harness::new(); diff --git a/docs/agent-chat.md b/docs/agent-chat.md index 6642568711..57933574ee 100644 --- a/docs/agent-chat.md +++ b/docs/agent-chat.md @@ -224,6 +224,13 @@ Grok Build also supports `XAI_API_KEY` for users who choose API authentication. endpoint's entry under **Agents**, where a change applies from the next request. A request that runs out of time ends the message with the limit it hit and where to raise it. +For Ollama, choosing a model checks its reported capabilities in the background. If its +capabilities list lacks `tools`, a warning below the model picker explains that the model can +only chat and suggests choosing one with tool support. Results are cached in memory per base URL and +model, with a limit of 128 entries. SolidCraft recognises Ollama through `/api/version` at the +endpoint's origin or the Ollama preset. Failed checks and older servers without a capabilities +field show no warning. The check never blocks sending a message. + ### An agent inside WSL (Windows) An agent installed inside WSL keeps its own settings, skills and sign-in there. Add it as a diff --git a/docs/control-protocol.md b/docs/control-protocol.md index 269b515d24..48d81ef370 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -253,7 +253,7 @@ labels; `prefs:measure_axis_distances` identifies the checkbox. | `engine.execute` → `browser.expand_all` | `path?`: occurrence ids; `component?`: id; omit both for the whole tree | Open every Browser row under that scope, including folders that start closed (Origin, Named Views, Features). Display only: not saved in the design, not an undo step. | | `engine.execute` → `browser.collapse_all` | same | Close those rows. The design row stays open when the whole tree is collapsed. | | `engine.execute` → `view.history_layout` | `width?`, `dock?`: `bottom`, `left`, `under_browser`, `collapsed?` | Resize/dock the history without opening a dialog or changing the design. Contents respond to measured width; browser-stacked history shares its width and splits height equally. One folded header gives the expanded card the rest; two folded headers stay at the top. | -| `ui.agent_chat` | `features?`, `toolbar_button?`, `permissions?` (`ask`, `edits`, `all`), `add_agent?` ({`name`, `kind`: `acp`, `api` or `anthropic` (`api` is OpenAI-compatible, `anthropic` Anthropic-compatible), `command` or `base_url` + `model` + `request_timeout_s?` (seconds one model request may take, 300 by default, rounded and clamped to 30 to 3600)}; no keys), `add_dialog?` (`installed`, `custom`, `api`, `anthropic`, or `false` to close), `default_agent?`, `agent?` (the conversation's agent, by id), `mode?` (the agent's own mode), `provider?` (which provider's models the model picker lists, when they come from several), `model?` (one of its models), `open?`, `side?` (`left`, `right`), `minimized?`, `popped?` (in its own window), `width?`, `picker?`, `refresh?`, `connect?`, `stop?`, `attach?`: a file's path (a picture, a text file or a CAD file, for the next message; CAD files are imported at once), `remove_file?` (index of a file on the next message), `send?`: text (files alone make a message too), `attach_view?`, `attach_selection?`, `pick?` (true: open the context picker), `permission?`: an option id or `"cancel"` (the newest question of the agent's own), `approve?`: true, false or `"always"` (the oldest request held for Allow/Deny), `cancel?` (Stop: the queue stays), `send_now?` (stop the running step, then send the queued messages), `remove_pending?` (index), `new_chat?` (clear the chat, new session), `undo?` (the last message's changes) | the chat's state: `features`, `agents` (found and added), `currentAgent`, `state` (off, starting, ready, busy, auth, exited), `popped`, `window` (its place and size), `agent`, `modes`, `mode`, `models`, `model`, `providers` (`name` and `count` of each, empty with one provider), `provider` (the one listed), `authMethods`, `toolsPort`, `pid`, `waiting`, `turn`, `pending` (queued messages, with their `files`), `attachments` (the files on the next message, by name), `loading` (pictures still being read for it, by name), `stopped`, `detection` and the conversation (`items`). With `features` off only the settings apply. Desktop only; see [Agent chat](agent-chat.md) | +| `ui.agent_chat` | `features?`, `toolbar_button?`, `permissions?` (`ask`, `edits`, `all`), `add_agent?` ({`name`, `kind`: `acp`, `api` or `anthropic` (`api` is OpenAI-compatible, `anthropic` Anthropic-compatible), `command` or `base_url` + `model` + `request_timeout_s?` (seconds one model request may take, 300 by default, rounded and clamped to 30 to 3600)}; no keys), `add_dialog?` (`installed`, `custom`, `api`, `anthropic`, or `false` to close), `default_agent?`, `agent?` (the conversation's agent, by id), `mode?` (the agent's own mode), `provider?` (which provider's models the model picker lists, when they come from several), `model?` (one of its models), `open?`, `side?` (`left`, `right`), `minimized?`, `popped?` (in its own window), `width?`, `picker?`, `refresh?`, `connect?`, `stop?`, `attach?`: a file's path (a picture, a text file or a CAD file, for the next message; CAD files are imported at once), `remove_file?` (index of a file on the next message), `send?`: text (files alone make a message too), `attach_view?`, `attach_selection?`, `pick?` (true: open the context picker), `permission?`: an option id or `"cancel"` (the newest question of the agent's own), `approve?`: true, false or `"always"` (the oldest request held for Allow/Deny), `cancel?` (Stop: the queue stays), `send_now?` (stop the running step, then send the queued messages), `remove_pending?` (index), `new_chat?` (clear the chat, new session), `undo?` (the last message's changes) | the chat's state: `features`, `agents` (found and added), `currentAgent`, `state` (off, starting, ready, busy, auth, exited), `popped`, `window` (its place and size), `agent`, `modes`, `mode`, `models`, `model`, `modelWarning` (the model capability warning text, or `null` when unknown or tool support is reported; checked asynchronously and never blocks sending), `providers` (`name` and `count` of each, empty with one provider), `provider` (the one listed), `authMethods`, `toolsPort`, `pid`, `waiting`, `turn`, `pending` (queued messages, with their `files`), `attachments` (the files on the next message, by name), `loading` (pictures still being read for it, by name), `stopped`, `detection` and the conversation (`items`). With `features` off only the settings apply. Desktop only; see [Agent chat](agent-chat.md) | | `ui.resize` | `width`, `height` | | | `app.quit` | `force?` (quit without asking about unsaved designs) | `quitting` (the window closes), `asking` (a "save changes?" prompt is open) |