diff --git a/AGENTS.md b/AGENTS.md index 41624a9..c695ff5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,7 @@ NotesCraft is a native, Obsidian-compatible notes app written in **Rust only** ( - One crate per concern in `crates/`, package `notescraft-`, `[lints] workspace = true`, registered in the root `[workspace.dependencies]` and in `xtask/src/layers.rs`. - **Layering is enforced** by `cargo xtask layers`: a crate may depend only on strictly lower layers; nothing below L5 may use egui, eframe, winit or rfd. Layers: L0 `vault` `markdown`; L1 `index` `canvas` `diagrams` `whiteboard`; L2 `backlinks` `graph` `templates`; L3 `plugins` `themes`; L4 `engine`; L5 `editor` `mcp` `i18n`; L6 `ui-egui`. - Shared crates come from craft-libs (`craft-atomic`, `craft-settings`, `craft-ui`, `craft-quit`, `craft-print`), pinned by git rev in the root `Cargo.toml`. Do not change their API; ask first. -- A new third-party dependency needs a written reason and owner approval. Already approved: `notify`, `pulldown-cmark`, `toml`, `wasmi`, `wat` (dev), `thiserror`, `serde`, `serde_json`, and for sync (2026-10-19): `sha2`, `axum`, `tokio`, `ureq` (rustls), `rusqlite`, and for paste (2026-10-20): `arboard` (image-data), `image` (png feature only). +- A new third-party dependency needs a written reason and owner approval. Already approved: `notify`, `pulldown-cmark`, `toml`, `wasmi`, `wat` (dev), `thiserror`, `serde`, `serde_json`, and for sync (2026-10-19): `sha2`, `axum`, `tokio`, `ureq` (rustls), `rusqlite`, and for paste (2026-10-20): `arboard` (image-data), `image` (png feature only), and for the registry (2026-10-21): `ring` (ed25519 verify, already linked through rustls). ## Never crash (outranks features) diff --git a/Cargo.lock b/Cargo.lock index ff685f3..c38b272 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2847,6 +2847,7 @@ dependencies = [ "notescraft-templates", "notescraft-themes", "notescraft-vault", + "ring", "serde", "serde_json", "thiserror 2.0.21", @@ -2869,6 +2870,13 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "notescraft-htmlmd" +version = "0.1.0" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "notescraft-i18n" version = "0.1.0" @@ -2968,6 +2976,10 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "notescraft-srs" +version = "0.1.0" + [[package]] name = "notescraft-sync" version = "0.1.0" @@ -3038,6 +3050,13 @@ dependencies = [ name = "notescraft-testkit" version = "0.1.0" +[[package]] +name = "notescraft-textdiff" +version = "0.1.0" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "notescraft-themes" version = "0.1.0" @@ -6125,7 +6144,13 @@ checksum = "ec7a2a501ed189703dba8b08142f057e887dfc4b2cc4db2d343ac6376ba3e0b9" name = "xtask" version = "0.1.0" dependencies = [ + "craft-atomic", + "notescraft-engine", + "ring", + "serde", "serde_json", + "sha2", + "toml", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 85b0651..08868c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,9 @@ notescraft-collab = { path = "crates/collab" } notescraft-annotate = { path = "crates/annotate" } notescraft-calendar = { path = "crates/calendar" } notescraft-tasks = { path = "crates/tasks" } +notescraft-textdiff = { path = "crates/textdiff" } +notescraft-htmlmd = { path = "crates/htmlmd" } +notescraft-srs = { path = "crates/srs" } notescraft-templater = { path = "crates/templater" } notescraft-plugins = { path = "crates/plugins" } notescraft-themes = { path = "crates/themes" } @@ -87,6 +90,8 @@ tokio = { version = "1", default-features = false, features = ["rt-multi-thread" rusqlite = { version = "0.37", features = ["bundled"] } # Blocking HTTP client with rustls (sync server tests now, the engine sync client next). ureq = { version = "3", default-features = false, features = ["rustls", "json"] } +# ed25519 verification of registry entries (owner-approved 2026-10-21, already linked through rustls). +ring = { version = "0.17", default-features = false } # Pure-Rust WebAssembly interpreter, same configuration as photocraft-plugins. wasmi = { version = "2.0.0", default-features = false, features = ["std", "validate", "stable", "portable-dispatch"] } wat = "1.228" @@ -113,4 +118,7 @@ debug = "line-tables-only" [profile.release] lto = "thin" -codegen-units = 1 \ No newline at end of file +codegen-units = 1 +# Size: owner-approved 2026-10-21 (specs/registry/measurements.md): -8 MB on macOS; crash backtraces lose symbol names. +opt-level = "s" +strip = "symbols" diff --git a/apps/notescraft/Cargo.toml b/apps/notescraft/Cargo.toml index 65c5301..8146450 100644 --- a/apps/notescraft/Cargo.toml +++ b/apps/notescraft/Cargo.toml @@ -23,7 +23,7 @@ notescraft-calendar = { workspace = true } notescraft-templates = { workspace = true } serde_json = { workspace = true } egui = { workspace = true } -eframe = { workspace = true, features = ["wgpu", "default_fonts", "accesskit", "wayland", "x11"] } +eframe = { workspace = true, features = ["wgpu", "accesskit", "wayland", "x11"] } log = { workspace = true } cpal = { workspace = true } craft-quit = { workspace = true, features = ["egui"] } diff --git a/assets/fonts/NotoEmoji-Regular.ttf b/assets/fonts/NotoEmoji-Regular.ttf new file mode 100644 index 0000000..19b7bad Binary files /dev/null and b/assets/fonts/NotoEmoji-Regular.ttf differ diff --git a/assets/fonts/OFL-NotoEmoji.txt b/assets/fonts/OFL-NotoEmoji.txt new file mode 100644 index 0000000..d952d62 --- /dev/null +++ b/assets/fonts/OFL-NotoEmoji.txt @@ -0,0 +1,92 @@ +This Font Software is licensed under the SIL Open Font License, +Version 1.1. + +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font +creation efforts of academic and linguistic communities, and to +provide a free and open framework in which fonts may be shared and +improved in partnership with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply to +any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software +components as distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, +deleting, or substituting -- in part or in whole -- any of the +components of the Original Version, by changing formats or by porting +the Font Software to a new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, +modify, redistribute, and sell modified and unmodified copies of the +Font Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, in +Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the +corresponding Copyright Holder. This restriction only applies to the +primary font name as presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created using +the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 14fca24..62e9418 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -32,6 +32,7 @@ notescraft-annotate = { workspace = true } notescraft-bases = { workspace = true } notescraft-collab = { workspace = true } ureq = { workspace = true } +ring = { workspace = true } notescraft-regex = { workspace = true } grep-regex = { workspace = true } grep-searcher = { workspace = true } diff --git a/crates/engine/src/addons/commands.rs b/crates/engine/src/addons/commands.rs new file mode 100644 index 0000000..9a8fed7 --- /dev/null +++ b/crates/engine/src/addons/commands.rs @@ -0,0 +1,334 @@ +//! The `registry.*` commands (spec D9): list, info, suggest, check, install, update, remove, +//! rollback, enable and disable. +//! +//! Reads work from the cached index and never touch the network; `registry.check` and the install +//! commands are the only network triggers. Mutating commands are disabled in a read-only session. + +use std::path::PathBuf; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde_json::{Value, json}; + +use super::TRUSTED_KEYS; +use super::install::{self, Ctx, InstallReport}; +use super::model::{Entry, Index, Kind, Version}; +use super::net::{IndexCache, fetch_index, split_code}; +use super::state::{State, VaultFile, key}; +use super::store::Store; +use super::verify::{Ed25519, Verifier}; +use crate::commands::{json, text_param, writable}; +use crate::error::{CommandError, Error}; +use crate::params::{ParamKind, ParamSpec}; +use crate::registry::Registry; +use crate::session::Session; +use crate::spec::{CommandResult, CommandSpec}; + +/// Folder of the per-vault choices. +const VAULT_FILE: &str = ".notescraft/registry.json"; +/// Rows `registry.list` returns when `limit` is not given. +const DEFAULT_LIMIT: usize = 200; +/// Most rows `registry.list` may return. +const MAX_LIMIT: usize = 2000; + +/// The default verifier: the keys embedded in the build. With none configured it refuses with a +/// clear message instead of the generic "unknown key". +#[derive(Debug, Clone, Copy)] +pub struct TrustedVerifier; + +impl Verifier for TrustedVerifier { + fn verify(&self, entry: &Entry) -> crate::error::Result<()> { + if TRUSTED_KEYS.is_empty() { + return Err(super::net::coded("addon", "this build has no trusted registry key, so add-ons cannot be installed yet")); + } + Ed25519::trusted().verify(entry) + } +} + +fn fail(error: &Error) -> CommandError { + match error { + Error::Addon(message) => { + let (code, rest) = split_code(message); + CommandError::new(code, rest) + } + other => crate::commands::command_error(other), + } +} + +fn invalid(message: impl Into) -> CommandError { + CommandError::new("invalid_params", message) +} + +fn kind_param(params: &Value) -> Result { + let name = text_param(params, "kind")?; + Kind::parse(&name).ok_or_else(|| invalid(format!("`kind` is not one of plugin theme pack grammar font snippet: `{name}`"))) +} + +fn now() -> u64 { + SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |elapsed| elapsed.as_secs()) +} + +fn app_version() -> Version { + Version::parse(env!("CARGO_PKG_VERSION")).unwrap_or(Version { major: 0, minor: 0, patch: 0 }) +} + +pub(super) fn store(session: &Session) -> Result { + let base: Option = match session.settings_dir() { + Some(dir) => Some(dir.to_path_buf()), + None => match std::env::var_os(craft_settings::paths::env_var_name(crate::app_settings::APP_ID)) { + Some(dir) if !dir.is_empty() => Some(PathBuf::from(dir)), + _ => craft_settings::paths::data_dir(crate::app_settings::APP_ID), + }, + }; + let base = base.ok_or_else(|| CommandError::new("addon", "there is no data folder for the add-on store"))?; + Store::open(&base).map_err(|e| fail(&e)) +} + +fn cached(store: &Store) -> Result, CommandError> { + let Some(cache) = IndexCache::load(store.root()).map_err(|e| fail(&e))? else { return Ok(None) }; + let index = cache.index().map_err(|e| fail(&e))?; + Ok(Some((cache, index))) +} + +fn need_index(store: &Store) -> Result { + cached(store)?.map(|(_, index)| index).ok_or_else(|| CommandError::new("not_found", "no add-on index is cached yet; run registry.check first")) +} + +/// The state of an entry for the UI: `available`, `installed`, `update`, `revoked`, +/// `incompatible` or `unsupported`. +fn entry_state(index: &Index, entry: &Entry, state: &State) -> &'static str { + let installed = state.installed.get(&key(entry.kind, &entry.id)); + if index.is_revoked(entry) { + return "revoked"; + } + if let Some(row) = installed { + return if row.version < entry.version { "update" } else { "installed" }; + } + if entry.min_app_version > app_version() { + return "incompatible"; + } + if matches!(entry.kind, Kind::Font | Kind::Snippet) { "unsupported" } else { "available" } +} + +fn row(index: &Index, entry: &Entry, state: &State) -> Value { + let installed = state.installed.get(&key(entry.kind, &entry.id)).map(|r| r.version.to_string()); + json!({ + "id": entry.id, "kind": entry.kind.as_str(), "title": entry.title, "summary": entry.summary, + "version": entry.version.to_string(), "installed_version": installed, "size": entry.total_size(), + "state": entry_state(index, entry, state), "permissions": entry.permissions, + }) +} + +fn load_all(session: &Session) -> Result<(Store, Option, State), CommandError> { + let store = store(session)?; + let state = store.load().map_err(|e| fail(&e))?; + let index = cached(&store)?.map(|(_, index)| index); + Ok((store, index, state)) +} + +fn list(session: &mut Session, params: &Value) -> CommandResult { + let (_, index, state) = load_all(session)?; + let kind = params.get("kind").map(|_| kind_param(params)).transpose()?; + let query = params.get("query").and_then(Value::as_str).map(str::to_lowercase); + let wanted = params.get("state").and_then(Value::as_str); + let limit = params.get("limit").and_then(Value::as_u64).map_or(DEFAULT_LIMIT, |n| usize::try_from(n).unwrap_or(MAX_LIMIT).min(MAX_LIMIT)); + let Some(index) = index else { return json(json!({ "entries": [], "cached": false })) }; + let rows: Vec = index + .entries + .iter() + .filter(|e| kind.is_none_or(|k| e.kind == k)) + .filter(|e| query.as_ref().is_none_or(|q| matches_query(e, q))) + .filter(|e| { + let current = entry_state(&index, e, &state); + match wanted { + None => true, + Some("installed") => matches!(current, "installed" | "update"), + Some("updates") => current == "update", + Some(other) => current == other, + } + }) + .take(limit) + .map(|e| row(&index, e, &state)) + .collect(); + json(json!({ "entries": rows, "cached": true })) +} + +fn matches_query(entry: &Entry, query: &str) -> bool { + [entry.id.as_str(), &entry.title, &entry.summary].iter().any(|text| text.to_lowercase().contains(query)) + || entry.aliases.iter().any(|alias| alias.contains(query)) +} + +fn info(session: &mut Session, params: &Value) -> CommandResult { + let (kind, id) = (kind_param(params)?, text_param(params, "id")?); + let (_, index, state) = load_all(session)?; + let index = index.ok_or_else(|| CommandError::new("not_found", "no add-on index is cached yet; run registry.check first"))?; + let entry = index.find(kind, &id).ok_or_else(|| CommandError::new("not_found", format!("the registry has no {} `{id}`", kind.as_str())))?; + let files: Vec = entry.files.iter().map(|f| json!({ "path": f.path, "size": f.size })).collect(); + let mut value = row(&index, entry, &state); + if let Some(map) = value.as_object_mut() { + map.insert("license".into(), json!(entry.license)); + map.insert("source".into(), json!(entry.source)); + map.insert("aliases".into(), json!(entry.aliases)); + map.insert("min_app_version".into(), json!(entry.min_app_version.to_string())); + map.insert("key_id".into(), json!(entry.signature.key_id)); + map.insert("files".into(), json!(files)); + } + json(value) +} + +fn suggest(session: &mut Session, params: &Value) -> CommandResult { + let (_, index, state) = load_all(session)?; + let Some(index) = index else { return json(json!({ "entries": [] })) }; + let rows: Vec = if let Some(label) = params.get("label").and_then(Value::as_str) { + let label = label.trim().to_lowercase(); + index.entries.iter().filter(|e| e.kind == Kind::Grammar && (e.id == label || e.aliases.contains(&label))).map(|e| row(&index, e, &state)).collect() + } else { + let kind = kind_param(params).map_err(|_| invalid("give `label`, or `kind` and `query`"))?; + let query = text_param(params, "query")?.to_lowercase(); + index.entries.iter().filter(|e| e.kind == kind && matches_query(e, &query)).map(|e| row(&index, e, &state)).collect() + }; + json(json!({ "entries": rows })) +} + +fn check(session: &mut Session, _params: &Value) -> CommandResult { + let store = store(session)?; + let old = IndexCache::load(store.root()).map_err(|e| fail(&e))?; + let fresh = fetch_index(&*session.addons_transport, old.as_ref(), now()).map_err(|e| fail(&e))?; + fresh.save(store.root()).map_err(|e| fail(&e))?; + let index = fresh.index().map_err(|e| fail(&e))?; + let state = store.load().map_err(|e| fail(&e))?; + let updates = index.entries.iter().filter(|e| entry_state(&index, e, &state) == "update").count(); + let changed = old.as_ref().is_none_or(|o| o.body != fresh.body); + json(json!({ "entries": index.entries.len(), "skipped": index.skipped, "updates": updates, "changed": changed, "fetched_at": fresh.fetched_at })) +} + +fn report(r: &InstallReport) -> Value { + json!({ + "kind": r.kind.as_str(), "id": r.id, "version": r.version.to_string(), "previous": r.previous.map(|v| v.to_string()), + "source": format!("{:?}", r.source).to_lowercase(), "bytes": r.bytes, + }) +} + +fn with_ctx(session: &Session, run: impl FnOnce(&Ctx<'_>, &Index) -> crate::error::Result) -> Result { + let store = store(session)?; + let index = need_index(&store)?; + let ctx = Ctx { store: &store, transport: &*session.addons_transport, verifier: &*session.addons_verifier, app_version: app_version(), seed: None }; + run(&ctx, &index).map_err(|e| fail(&e)) +} + +fn install_cmd(session: &mut Session, params: &Value) -> CommandResult { + let (kind, id) = (kind_param(params)?, text_param(params, "id")?); + let version = params.get("version").and_then(Value::as_str).map(Version::parse).transpose().map_err(|e| fail(&e))?; + json(report(&with_ctx(session, |ctx, index| install::install(ctx, index, kind, &id, version))?)) +} + +fn update_cmd(session: &mut Session, params: &Value) -> CommandResult { + if params.get("id").is_some() { + let (kind, id) = (kind_param(params)?, text_param(params, "id")?); + return json(report(&with_ctx(session, |ctx, index| install::update(ctx, index, kind, &id))?)); + } + let (store, index, state) = load_all(session)?; + let index = index.ok_or_else(|| CommandError::new("not_found", "no add-on index is cached yet; run registry.check first"))?; + let kind = params.get("kind").map(|_| kind_param(params)).transpose()?; + let due: Vec<&Entry> = index.entries.iter().filter(|e| kind.is_none_or(|k| e.kind == k) && entry_state(&index, e, &state) == "update").collect(); + let ctx = Ctx { store: &store, transport: &*session.addons_transport, verifier: &*session.addons_verifier, app_version: app_version(), seed: None }; + let (mut updated, mut failed) = (Vec::new(), Vec::new()); + for entry in due { + match install::update(&ctx, &index, entry.kind, &entry.id) { + Ok(done) => updated.push(report(&done)), + Err(error) => failed.push(json!({ "kind": entry.kind.as_str(), "id": entry.id, "error": fail(&error) })), + } + } + json(json!({ "updated": updated, "failed": failed })) +} + +fn remove_cmd(session: &mut Session, params: &Value) -> CommandResult { + let (kind, id) = (kind_param(params)?, text_param(params, "id")?); + install::remove(&store(session)?, kind, &id).map_err(|e| fail(&e))?; + json(json!({ "kind": kind.as_str(), "id": id })) +} + +fn rollback_cmd(session: &mut Session, params: &Value) -> CommandResult { + let (kind, id) = (kind_param(params)?, text_param(params, "id")?); + json(report(&install::rollback(&store(session)?, kind, &id).map_err(|e| fail(&e))?)) +} + +fn vault_file(session: &Session) -> Result { + match session.vault().read(VAULT_FILE) { + Ok(bytes) => VaultFile::parse(&bytes).map_err(|e| fail(&e)), + Err(notescraft_vault::Error::Io { source, .. }) if source.kind() == std::io::ErrorKind::NotFound => Ok(VaultFile::default()), + Err(error) => Err(fail(&error.into())), + } +} + +fn set_enabled(session: &mut Session, params: &Value, on: bool) -> CommandResult { + let id = text_param(params, "id")?; + let name = key(Kind::Plugin, &id); + if on && !store(session)?.load().map_err(|e| fail(&e))?.installed.contains_key(&name) { + return Err(CommandError::new("not_found", format!("plugin `{id}` is not installed"))); + } + let mut file = vault_file(session)?; + file.enabled.retain(|known| *known != name); + if on { + file.enabled.push(name); + } + let json_text = file.to_json().map_err(|e| fail(&e))?; + session.vault().write_private(VAULT_FILE, json_text.as_bytes()).map_err(|e| fail(&e.into()))?; + json(json!({ "id": id, "enabled": on })) +} + +fn enable(session: &mut Session, params: &Value) -> CommandResult { + set_enabled(session, params, true) +} + +fn disable(session: &mut Session, params: &Value) -> CommandResult { + set_enabled(session, params, false) +} + +fn kind_id(spec: CommandSpec, what: &str) -> CommandSpec { + spec.with_param(ParamSpec::required("kind", ParamKind::String, "plugin, theme, pack, grammar, font or snippet")).with_param(ParamSpec::required( + "id", + ParamKind::String, + what.to_owned(), + )) +} + +/// Registers the `registry.*` commands. +/// +/// # Errors +/// Whatever [`Registry::register`] returns. +pub(crate) fn register_addons(reg: &mut Registry) -> crate::error::Result<()> { + let specs = vec![ + CommandSpec::new("registry.list", "List add-ons", "The cached add-on index joined with what is installed; no network", |_| true, list) + .with_param(ParamSpec::optional("kind", ParamKind::String, "only this kind")) + .with_param(ParamSpec::optional("query", ParamKind::String, "text in the id, title, summary or aliases")) + .with_param(ParamSpec::optional("state", ParamKind::String, "installed, updates, available, revoked, incompatible or unsupported")) + .with_param(ParamSpec::optional("limit", ParamKind::Integer, "most rows, at most 2000")), + kind_id(CommandSpec::new("registry.info", "Add-on info", "One add-on in full: files, sizes, license, signing key", |_| true, info), "add-on id"), + CommandSpec::new("registry.suggest", "Suggest add-ons", "Add-ons for a fence label, or by kind and text; no network", |_| true, suggest) + .with_param(ParamSpec::optional("label", ParamKind::String, "a fence label such as `twig`")) + .with_param(ParamSpec::optional("kind", ParamKind::String, "kind to search")) + .with_param(ParamSpec::optional("query", ParamKind::String, "text to search")), + CommandSpec::new("registry.check", "Check for add-on updates", "Fetches the registry index (one conditional request) and caches it", writable, check), + kind_id(CommandSpec::new("registry.install", "Install add-on", "Downloads, verifies and installs an add-on", writable, install_cmd), "add-on id") + .with_param(ParamSpec::optional("version", ParamKind::String, "must equal the indexed version")), + CommandSpec::new("registry.update", "Update add-ons", "Installs newer versions; no id means every add-on with an update", writable, update_cmd) + .with_param(ParamSpec::optional("kind", ParamKind::String, "kind of the add-on, or only this kind")) + .with_param(ParamSpec::optional("id", ParamKind::String, "one add-on")), + kind_id(CommandSpec::new("registry.remove", "Remove add-on", "Removes an installed add-on", writable, remove_cmd), "installed add-on id"), + kind_id( + CommandSpec::new("registry.rollback", "Roll back add-on", "Returns to the previous installed version", writable, rollback_cmd), + "installed add-on id", + ), + CommandSpec::new("registry.enable", "Enable plugin", "Switches an installed plugin on in this vault", writable, enable) + .with_param(ParamSpec::required("id", ParamKind::String, "plugin id")), + CommandSpec::new("registry.disable", "Disable plugin", "Switches a plugin off in this vault", writable, disable).with_param(ParamSpec::required( + "id", + ParamKind::String, + "plugin id", + )), + ]; + for spec in specs { + reg.register(spec)?; + } + Ok(()) +} diff --git a/crates/engine/src/addons/fileset.rs b/crates/engine/src/addons/fileset.rs new file mode 100644 index 0000000..e72ce3b --- /dev/null +++ b/crates/engine/src/addons/fileset.rs @@ -0,0 +1,202 @@ +//! Hostile-input rules for an add-on file list: paths, sizes, duplicates and download links. +//! +//! Pure: works on strings and numbers, reads nothing, never follows a link. A path is accepted +//! only if every component passes a character-set check, and [`join_under`] builds the final path +//! from those validated components alone. Nothing is "fixed": a bad input is an `Err`. + +use std::path::{Path, PathBuf}; + +use crate::error::{Error, Result}; + +/// Most files an entry may list. +pub const MAX_FILES: usize = 64; +/// Most path components of one file. +pub const MAX_COMPONENTS: usize = 4; +/// Longest path component, in bytes. +pub const MAX_COMPONENT_BYTES: usize = 64; +/// Longest download link, in bytes. +const MAX_URL_BYTES: usize = 300; +/// One KiB. +const KIB: u64 = 1024; +/// One MiB. +const MIB: u64 = 1024 * KIB; +/// Windows device names that no component may use, with or without an extension. +const RESERVED: [&str; 4] = ["con", "prn", "aux", "nul"]; + +/// What an add-on is, as far as file rules go (task 1's `Kind` maps onto it). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KindRule { + /// A WebAssembly plugin. + Plugin, + /// A colour theme. + Theme, + /// A syntax grammar. + Grammar, + /// A font. + Font, + /// Snippets and templates. + Snippet, + /// A multi-consumer pack. + Pack, +} + +impl From for KindRule { + fn from(kind: super::Kind) -> Self { + match kind { + super::Kind::Plugin => Self::Plugin, + super::Kind::Theme => Self::Theme, + super::Kind::Grammar => Self::Grammar, + super::Kind::Font => Self::Font, + super::Kind::Snippet => Self::Snippet, + super::Kind::Pack => Self::Pack, + } + } +} + +/// A path that passed [`check_path`]: its validated parts. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Checked { + /// The path components, in order. + pub components: Vec, +} + +/// Most bytes the files of one entry of this kind may add up to. +#[must_use] +pub const fn kind_cap(kind: KindRule) -> u64 { + match kind { + KindRule::Plugin => 4 * MIB, + KindRule::Theme => 64 * KIB, + KindRule::Grammar => 512 * KIB, + KindRule::Snippet => MIB, + KindRule::Pack => 8 * MIB, + KindRule::Font => 32 * MIB, + } +} + +fn bad(reason: &str) -> Result { + Err(Error::Addon(reason.to_owned())) +} + +fn is_reserved(component: &str) -> bool { + let stem = component.split('.').next().unwrap_or_default().to_ascii_lowercase(); + let numbered = |prefix: &str| stem.strip_prefix(prefix).is_some_and(|n| matches!(n.as_bytes(), [b'1'..=b'9'])); + RESERVED.contains(&stem.as_str()) || numbered("com") || numbered("lpt") +} + +fn extension_ok(name: &str, extensions: &[&str]) -> bool { + let lower = name.to_ascii_lowercase(); + extensions.iter().any(|ext| lower.ends_with(ext)) +} + +/// Checks one file path of an entry of `kind` (spec D7). +/// +/// # Errors +/// [`Error::Addon`] with the broken rule. +pub fn check_path(kind: KindRule, path: &str) -> Result { + let components: Vec<&str> = path.split('/').collect(); + if components.len() > MAX_COMPONENTS { + return bad("path has too many components"); + } + for component in &components { + if component.is_empty() || component.len() > MAX_COMPONENT_BYTES { + return bad("path component is empty or too long"); + } + if component.starts_with('.') { + return bad("path component starts with a dot"); + } + if !component.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) { + return bad("path has a character outside A-Z a-z 0-9 . _ -"); + } + if is_reserved(component) { + return bad("path uses a reserved device name"); + } + } + let (Some(name), root) = (components.last().copied(), components.first().copied()) else { return bad("empty path") }; + let allowed = match kind { + KindRule::Plugin => components.len() == 1 && matches!(name, "manifest.json" | "plugin.wasm" | "README.md"), + KindRule::Theme => extension_ok(name, &[".toml"]), + KindRule::Grammar => extension_ok(name, &[".sublime-syntax"]), + KindRule::Font => extension_ok(name, &[".ttf", ".otf"]), + KindRule::Snippet => extension_ok(name, &[".md", ".txt", ".toml"]), + KindRule::Pack => components.len() >= 2 && pack_ok(root.unwrap_or_default(), name), + }; + if !allowed { + return bad("file name or extension is not allowed for this kind"); + } + Ok(Checked { components: components.into_iter().map(str::to_owned).collect() }) +} + +fn pack_ok(folder: &str, name: &str) -> bool { + match folder { + "grammars" => extension_ok(name, &[".sublime-syntax"]), + "themes" => extension_ok(name, &[".toml"]), + "templates" => extension_ok(name, &[".md"]), + "icons" => extension_ok(name, &[".json", ".svg"]), + _ => false, + } +} + +/// Joins validated components under `root`. Cannot leave `root`: no part is empty, `.`, `..`, +/// absolute or contains a separator. +#[must_use] +pub fn join_under(root: &Path, checked: &Checked) -> PathBuf { + checked.components.iter().fold(root.to_path_buf(), |path, part| path.join(part)) +} + +/// Checks a whole file list `(path, size)`: every path, the count, case-insensitive duplicates, +/// zero sizes, the overflow-checked total and the kind cap. Returns the total size. +/// +/// # Errors +/// [`Error::Addon`] with the broken rule. +pub fn check_set(kind: KindRule, files: &[(&str, u64)]) -> Result { + if files.is_empty() || files.len() > MAX_FILES { + return bad("file count out of range"); + } + let cap = kind_cap(kind); + let mut seen: Vec = Vec::with_capacity(files.len()); + let mut total = 0_u64; + for (path, size) in files { + check_path(kind, path)?; + if *size == 0 { + return bad("zero-size file"); + } + if *size > cap { + return bad("file is over the size cap"); + } + let lower = path.to_ascii_lowercase(); + if seen.contains(&lower) { + return bad("duplicate path"); + } + seen.push(lower); + total = total.checked_add(*size).map_or_else(|| bad("total size overflows"), Ok)?; + } + if total > cap { + return bad("files are over the size cap"); + } + Ok(total) +} + +/// Checks a download link: [`notescraft_syntax::validate::check_url`] against each allowed host, +/// no `?` or `#`, at most 300 bytes. +/// +/// # Errors +/// [`Error::Addon`] with the reason. +pub fn check_download_url(url: &str, allowed_hosts: &[&str]) -> Result<()> { + if url.len() > MAX_URL_BYTES { + return bad("link is too long"); + } + if url.contains(['?', '#']) { + return bad("link has a query or fragment"); + } + let mut last = String::from("no host is allowed"); + for host in allowed_hosts { + match notescraft_syntax::validate::check_url(url, Some(host)) { + Ok(_) => return Ok(()), + Err(error) => last = error.to_string(), + } + } + Err(Error::Addon(last)) +} + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/addons/fileset/tests.rs b/crates/engine/src/addons/fileset/tests.rs new file mode 100644 index 0000000..22611a8 --- /dev/null +++ b/crates/engine/src/addons/fileset/tests.rs @@ -0,0 +1,109 @@ +use super::*; + +const HOSTS: [&str; 1] = ["git.marcospaulo.dev.br"]; + +#[test] +fn rejected_paths() { + let long = "a".repeat(65); + let cases: Vec<(KindRule, String)> = [ + (KindRule::Theme, "../x"), + (KindRule::Theme, "a/../../b"), + (KindRule::Theme, "/abs"), + (KindRule::Theme, "C:\\x"), + (KindRule::Theme, "a\\b"), + (KindRule::Theme, "a//b"), + (KindRule::Theme, ".hidden"), + (KindRule::Pack, "themes/.b.toml"), + (KindRule::Theme, "a b.toml"), + (KindRule::Theme, "\u{e9}.toml"), + (KindRule::Theme, "x\0.toml"), + (KindRule::Theme, "CON.toml"), + (KindRule::Theme, "nul"), + (KindRule::Theme, "com1/x.toml"), + (KindRule::Theme, "LPT9.toml"), + (KindRule::Theme, ""), + (KindRule::Theme, "a/b/c/d/e.toml"), + (KindRule::Plugin, "evil.wasm"), + (KindRule::Plugin, "sub/plugin.wasm"), + (KindRule::Theme, "x.exe"), + (KindRule::Pack, "bin/x.toml"), + (KindRule::Pack, "x.toml"), + (KindRule::Pack, "grammars/x.toml"), + (KindRule::Grammar, "x.tmLanguage"), + ] + .into_iter() + .map(|(k, p)| (k, p.to_owned())) + .chain([(KindRule::Theme, format!("{long}.toml")), (KindRule::Theme, "dir/".to_owned() + &long)]) + .collect(); + for (kind, path) in cases { + assert!(check_path(kind, &path).is_err(), "{path:?} should be rejected"); + } +} + +#[test] +fn accepted_paths() { + for (kind, path) in [ + (KindRule::Grammar, "twig.sublime-syntax"), + (KindRule::Plugin, "plugin.wasm"), + (KindRule::Plugin, "manifest.json"), + (KindRule::Plugin, "README.md"), + (KindRule::Pack, "grammars/twig.sublime-syntax"), + (KindRule::Pack, "themes/a.toml"), + (KindRule::Pack, "icons/a.svg"), + (KindRule::Font, "Inter-Regular.ttf"), + (KindRule::Snippet, "a.txt"), + ] { + assert!(check_path(kind, path).is_ok(), "{path}"); + } + assert_eq!(check_path(KindRule::Pack, "themes/a.toml").unwrap().components, ["themes", "a.toml"]); +} + +#[test] +fn set_rules() { + let many: Vec = (0..65).map(|i| format!("f{i}.toml")).collect(); + let many: Vec<(&str, u64)> = many.iter().map(|p| (p.as_str(), 1)).collect(); + assert!(check_set(KindRule::Snippet, &many).is_err()); + assert_eq!(check_set(KindRule::Snippet, &many[..64]).unwrap(), 64); + assert!(check_set(KindRule::Theme, &[]).is_err()); + assert!(check_set(KindRule::Theme, &[("a.toml", 1), ("A.toml", 1)]).is_err()); + assert!(check_set(KindRule::Theme, &[("a.toml", 0)]).is_err()); + assert!(check_set(KindRule::Snippet, &[("a.md", u64::MAX), ("b.md", 2)]).is_err()); + let cap = kind_cap(KindRule::Theme); + assert_eq!(check_set(KindRule::Theme, &[("a.toml", cap - 1), ("b.toml", 1)]).unwrap(), cap); + assert!(check_set(KindRule::Pack, &[("themes/a.toml", cap), ("themes/b.toml", 1)]).is_ok()); + assert!(check_set(KindRule::Theme, &[("a.toml", cap), ("b.toml", 1)]).is_err()); + assert!(check_set(KindRule::Font, &[("a.ttf", 33 * 1024 * 1024)]).is_err()); + assert!(check_set(KindRule::Font, &[("a.ttf", 32 * 1024 * 1024)]).is_ok()); + assert!(check_set(KindRule::Theme, &[("../a.toml", 1)]).is_err()); +} + +#[test] +fn download_urls() { + let ok = "https://git.marcospaulo.dev.br/masi/notescraft-registry/raw/branch/main/files/a.toml"; + assert!(check_download_url(ok, &HOSTS).is_ok()); + assert!(check_download_url(ok, &["other.example", "git.marcospaulo.dev.br"]).is_ok()); + for url in [ + "http://git.marcospaulo.dev.br/a", + "https://user:pw@git.marcospaulo.dev.br/a", + "https://git.marcospaulo.dev.br:8443/a", + "https://127.0.0.1/a", + "https://localhost/a", + "https://evil.example/a", + "https://git.marcospaulo.dev.br/a?x=1", + "https://git.marcospaulo.dev.br/a#f", + ] { + assert!(check_download_url(url, &HOSTS).is_err(), "{url}"); + } + let long = format!("https://git.marcospaulo.dev.br/{}", "a".repeat(300 - "https://git.marcospaulo.dev.br/".len() + 1)); + assert_eq!(long.len(), 301); + assert!(check_download_url(&long, &HOSTS).is_err()); + assert!(check_download_url(ok, &[]).is_err()); +} + +#[test] +fn join_stays_under_root() { + let root = Path::new("/data/registry/store"); + let joined = join_under(root, &check_path(KindRule::Pack, "grammars/twig.sublime-syntax").unwrap()); + assert!(joined.starts_with(root)); + assert!(joined.ends_with("grammars/twig.sublime-syntax")); +} diff --git a/crates/engine/src/addons/install.rs b/crates/engine/src/addons/install.rs new file mode 100644 index 0000000..901be91 --- /dev/null +++ b/crates/engine/src/addons/install.rs @@ -0,0 +1,233 @@ +//! Install, update, remove and rollback of add-ons (spec D6). +//! +//! The only commit point is [`Store::commit`]: the version folder is written first and is +//! unreferenced until the state names it, so a crash leaves garbage that [`Store::gc`] removes, +//! never a half-install. The old version is never deleted before the new state is committed. + +use std::fmt::Write as _; +use std::io::ErrorKind; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +use notescraft_syntax::validate::sha256_hex; + +use super::fileset::{KindRule, check_path, check_set, join_under}; +use super::kinds; +use super::model::{Entry, Index, Kind, Version}; +use super::net::{ENTRY_TIMEOUT, Transport, coded, download_file}; +use super::state::{Installed, MAX_INSTALLED, Source, State, key}; +use super::store::Store; +use super::verify::Verifier; +use crate::error::{Error, Result}; + +/// A read-only copy of signed add-on files shipped with the app (spec D11). +pub trait Seed: std::fmt::Debug { + /// The bytes of `path` of this add-on release, if the seed holds it. The caller still checks + /// the sha256 against the entry, so a stale or tampered seed file is simply not used. + fn read(&self, kind: Kind, id: &str, version: Version, path: &str) -> Option>; +} + +/// Everything an operation needs; all parts are injected so tests never touch the network. +pub struct Ctx<'a> { + /// The on-disk store. + pub store: &'a Store, + /// How files are downloaded. + pub transport: &'a dyn Transport, + /// How signatures are checked. + pub verifier: &'a dyn Verifier, + /// This app's version, compared with `min_app_version`. + pub app_version: Version, + /// Files shipped with the app, preferred over the network when their hash matches. + pub seed: Option<&'a dyn Seed>, +} + +/// What an operation changed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InstallReport { + /// Kind of the add-on. + pub kind: Kind, + /// Its id. + pub id: String, + /// The version now active. + pub version: Version, + /// The version rollback returns to. + pub previous: Option, + /// Where the files came from. + pub source: Source, + /// Total bytes of the files. + pub bytes: u64, +} + +/// Downloaded and checked files, not yet on disk. +#[derive(Debug)] +pub struct Staged { + /// `(path, bytes)` in the entry's order. + pub files: Vec<(String, Vec)>, + /// Whether every file came from the seed. + pub all_seed: bool, +} + +fn io(context: &str, error: &std::io::Error) -> Error { + if error.kind() == ErrorKind::StorageFull { coded("disk_full", context) } else { Error::Addon(format!("{context}: {error}")) } +} + +fn now() -> u64 { + SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |elapsed| elapsed.as_secs()) +} + +/// Steps 1 and 2 of D6: every refusal that needs no download. Returns the installed row, if any. +fn preflight(ctx: &Ctx<'_>, index: &Index, entry: &Entry, state: &State) -> Result> { + if entry.min_app_version > ctx.app_version { + return Err(coded("incompatible", format!("`{}` needs NotesCraft {} or newer", entry.id, entry.min_app_version))); + } + if index.is_revoked(entry) { + return Err(coded("revoked", format!("{} of `{}` was pulled", entry.version, entry.id))); + } + if matches!(entry.kind, Kind::Font | Kind::Snippet) { + return Err(Error::Addon(format!("`{}`: this version has no use for {} add-ons", entry.id, entry.kind.as_str()))); + } + ctx.verifier.verify(entry)?; + let listed: Vec<(&str, u64)> = entry.files.iter().map(|f| (f.path.as_str(), f.size)).collect(); + check_set(KindRule::from(entry.kind), &listed)?; + let current = state.installed.get(&key(entry.kind, &entry.id)).cloned(); + match ¤t { + Some(row) if entry.version == row.version => return Err(Error::Addon(format!("`{}` {} is already installed", entry.id, row.version))), + Some(row) if entry.version < row.version => { + return Err(Error::Addon(format!("`{}` {} is older than the installed {}; use rollback", entry.id, entry.version, row.version))); + } + None if state.installed.len() >= MAX_INSTALLED => return Err(coded("limit", format!("more than {MAX_INSTALLED} add-ons installed"))), + _ => {} + } + Ok(current) +} + +/// Steps 3 and 4 of D6: fetch every file (seed first), then run the per-kind checks. +/// +/// # Errors +/// [`Error::Addon`] from the transport, a gate or a content check; nothing is on disk. +pub fn fetch_files(ctx: &Ctx<'_>, entry: &Entry) -> Result { + let cap = entry.kind.cap(); + let started = Instant::now(); + let mut files = Vec::with_capacity(entry.files.len()); + let mut all_seed = true; + for file in &entry.files { + if started.elapsed() > ENTRY_TIMEOUT { + return Err(coded("offline", format!("downloading `{}` took too long", entry.id))); + } + let seeded = ctx.seed.and_then(|seed| seed.read(entry.kind, &entry.id, entry.version, &file.path)).filter(|bytes| sha256_hex(bytes) == file.sha256); + let bytes = if let Some(bytes) = seeded { + bytes + } else { + all_seed = false; + download_file(ctx.transport, file, cap)? + }; + files.push((file.path.clone(), bytes)); + } + kinds::check(entry.kind, entry, &files)?; + Ok(Staged { files, all_seed }) +} + +/// Step 5 of D6: writes the version folder with `write_new` per file. A leftover folder of the +/// same version (garbage of a crash; the state does not reference it) is replaced. +/// +/// # Errors +/// [`Error::Addon`] (`disk_full: ...` when the disk is full). +pub fn write_tree(store: &Store, entry: &Entry, staged: &Staged) -> Result<()> { + let dir = store.version_dir(entry.kind, &entry.id, entry.version)?; + if dir.exists() { + std::fs::remove_dir_all(&dir).map_err(|e| io("cannot replace an unreferenced version folder", &e))?; + } + for (path, bytes) in &staged.files { + let target = join_under(&dir, &check_path(KindRule::from(entry.kind), path)?); + if let Some(parent) = target.parent() { + std::fs::create_dir_all(parent).map_err(|e| io("cannot create a store folder", &e))?; + } + craft_atomic::write_new(&target, bytes).map_err(|e| io("cannot write a store file", &e))?; + } + Ok(()) +} + +fn files_hash(entry: &Entry) -> String { + let mut files: Vec<_> = entry.files.iter().collect(); + files.sort_by(|a, b| a.path.cmp(&b.path)); + let text = files.iter().fold(String::new(), |mut text, f| { + let _ = writeln!(text, "{}\n{}", f.path, f.sha256); + text + }); + sha256_hex(text.as_bytes()) +} + +fn apply(ctx: &Ctx<'_>, index: &Index, kind: Kind, id: &str, version: Option, must_be_installed: bool) -> Result { + let entry = index.find(kind, id).ok_or_else(|| coded("not_found", format!("the registry has no {} `{id}`", kind.as_str())))?; + if version.is_some_and(|wanted| wanted != entry.version) { + return Err(coded("not_found", format!("the registry has no version of `{id}` but {}", entry.version))); + } + let mut state = ctx.store.load()?; + if must_be_installed && !state.installed.contains_key(&key(kind, id)) { + return Err(coded("not_found", format!("`{id}` is not installed"))); + } + let current = preflight(ctx, index, entry, &state)?; + let staged = fetch_files(ctx, entry)?; + write_tree(ctx.store, entry, &staged)?; + let source = if staged.all_seed { Source::Seed } else { Source::Registry }; + let row = Installed { + version: entry.version, + previous: current.map(|row| row.version), + source, + installed_at: now().to_string(), + sha256_of_files: files_hash(entry), + }; + let previous = row.previous; + state.installed.insert(key(kind, id), row); + ctx.store.commit(&state)?; + // ponytail: a failed sweep after the commit only leaves garbage the next start removes. + let _ = ctx.store.gc(&state); + Ok(InstallReport { kind, id: id.to_owned(), version: entry.version, previous, source, bytes: entry.total_size() }) +} + +/// Installs `id` (or, when an older version is installed, updates it). +/// +/// # Errors +/// `not_found`, `incompatible`, `revoked`, `limit`, `offline`, `disk_full` (see +/// [`super::net::split_code`]), or a refused gate: signature, file rules, hash, content. +pub fn install(ctx: &Ctx<'_>, index: &Index, kind: Kind, id: &str, version: Option) -> Result { + apply(ctx, index, kind, id, version, false) +} + +/// Installs the index's newer version of an installed add-on; the old one stays as `previous`. +/// +/// # Errors +/// As [`install`], plus `not_found` when `id` is not installed. +pub fn update(ctx: &Ctx<'_>, index: &Index, kind: Kind, id: &str) -> Result { + apply(ctx, index, kind, id, None, true) +} + +/// Removes an add-on: the state row goes first (the commit), then the folders. +/// +/// # Errors +/// `not_found: ...` when it is not installed, or a store failure. +pub fn remove(store: &Store, kind: Kind, id: &str) -> Result<()> { + let mut state = store.load()?; + if state.installed.remove(&key(kind, id)).is_none() { + return Err(coded("not_found", format!("`{id}` is not installed"))); + } + store.commit(&state)?; + store.gc(&state).map(|_| ()) +} + +/// Swaps `version` and `previous`; the old folder is still in the store. +/// +/// # Errors +/// `not_found: ...` when it is not installed or has no previous version. +pub fn rollback(store: &Store, kind: Kind, id: &str) -> Result { + let mut state = store.load()?; + let row = state.installed.get_mut(&key(kind, id)).ok_or_else(|| coded("not_found", format!("`{id}` is not installed")))?; + let Some(back) = row.previous else { return Err(coded("not_found", format!("`{id}` has no earlier version to return to"))) }; + if !store.version_dir(kind, id, back)?.is_dir() { + return Err(coded("not_found", format!("the files of `{id}` {back} are gone"))); + } + let (version, source) = (back, row.source); + row.previous = Some(row.version); + row.version = back; + store.commit(&state)?; + Ok(InstallReport { kind, id: id.to_owned(), version, previous: state.installed.get(&key(kind, id)).and_then(|r| r.previous), source, bytes: 0 }) +} diff --git a/crates/engine/src/addons/installed.rs b/crates/engine/src/addons/installed.rs new file mode 100644 index 0000000..a37204c --- /dev/null +++ b/crates/engine/src/addons/installed.rs @@ -0,0 +1,32 @@ +//! What the store holds for the consumers: the grammar texts the highlighter loads. + +use super::model::Kind; +use super::state::parse_key; +use crate::session::Session; + +/// Most bytes read from one installed grammar file. +const MAX_GRAMMAR_BYTES: u64 = 512 * 1024; +/// Extension of the one grammar format the registry serves. +const GRAMMAR_EXT: &str = "sublime-syntax"; + +/// `(id, .sublime-syntax text)` for every grammar the add-on store has installed. A missing, +/// oversized or unreadable file is skipped; no network, no writes. +pub(crate) fn installed_grammars(session: &Session) -> Vec<(String, String)> { + let Ok(store) = super::commands::store(session) else { return Vec::new() }; + let Ok(state) = store.load() else { return Vec::new() }; + let mut found = Vec::new(); + for (name, row) in &state.installed { + let Ok((Kind::Grammar, id)) = parse_key(name) else { continue }; + let Ok(dir) = store.version_dir(Kind::Grammar, id, row.version) else { continue }; + let Ok(files) = std::fs::read_dir(&dir) else { continue }; + for file in files.flatten() { + let path = file.path(); + let ok = path.extension().is_some_and(|ext| ext == GRAMMAR_EXT) && file.metadata().is_ok_and(|meta| meta.len() <= MAX_GRAMMAR_BYTES); + if let (true, Ok(text)) = (ok, std::fs::read_to_string(&path)) { + found.push((id.to_owned(), text)); + break; + } + } + } + found +} diff --git a/crates/engine/src/addons/keys.rs b/crates/engine/src/addons/keys.rs new file mode 100644 index 0000000..b59fdc8 --- /dev/null +++ b/crates/engine/src/addons/keys.rs @@ -0,0 +1,10 @@ +//! The ed25519 public keys the app trusts for registry entries (spec D3). +//! +//! A rotation is a release that adds `k2` and later drops `k1`. The private key lives with the +//! owner and never enters this repository. + +/// `(key_id, 32-byte ed25519 public key)` pairs. +/// +/// Empty until the owner supplies the production public key: with no trusted key every entry +/// fails verification, so nothing can be installed from an unsigned or foreign index. +pub const TRUSTED_KEYS: [(&str, [u8; 32]); 0] = []; diff --git a/crates/engine/src/addons/kinds.rs b/crates/engine/src/addons/kinds.rs new file mode 100644 index 0000000..bb8e3d3 --- /dev/null +++ b/crates/engine/src/addons/kinds.rs @@ -0,0 +1,130 @@ +//! Per-kind content checks of staged add-on files (spec D8). +//! +//! Runs on the bytes already received and hash-checked, before anything reaches the store. It +//! parses; it never instantiates WebAssembly (the plugin consumer does the dry load). + +use std::collections::BTreeSet; +use std::path::Path; + +use notescraft_plugins::Manifest; +use notescraft_syntax::validate::check_grammar; +use notescraft_themes::{DiagnosticKind, Theme}; + +use super::model::{Entry, Kind}; +use crate::error::{Error, Result}; + +/// Text every grammar is run over; the index carries no sample of its own. +const GRAMMAR_SAMPLE: &str = "x = 1\nfoo(bar)\n"; +/// First four bytes of a WebAssembly module. +const WASM_MAGIC: &[u8] = b"\0asm"; +/// Accepted font signatures: TrueType, OpenType with CFF, old Apple TrueType. +const FONT_MAGICS: [&[u8]; 3] = [&[0, 1, 0, 0], b"OTTO", b"true"]; +/// Plugin files that may carry any bytes (a README is text, checked by size only). +const PLUGIN_README: &str = "README.md"; + +fn bad(reason: impl Into) -> Result { + Err(Error::Addon(reason.into())) +} + +fn text<'a>(path: &str, bytes: &'a [u8]) -> Result<&'a str> { + let text = std::str::from_utf8(bytes).map_err(|_| Error::Addon(format!("`{path}` is not UTF-8 text")))?; + if text.contains('\0') { + return bad(format!("`{path}` contains a NUL byte")); + } + Ok(text) +} + +fn grammar(path: &str, bytes: &[u8]) -> Result<()> { + check_grammar(text(path, bytes)?, GRAMMAR_SAMPLE).map(|_| ()).map_err(|e| Error::Addon(format!("`{path}` is not a usable grammar: {e}"))) +} + +/// A theme file: loads with no rejected value, and does not reuse a built-in id. With +/// `expect_id` (a theme entry) the theme id must also equal the entry id. +fn theme(path: &str, bytes: &[u8], expect_id: Option<&str>) -> Result<()> { + let (theme, diagnostics) = Theme::from_toml(text(path, bytes)?).map_err(|e| Error::Addon(format!("`{path}` is not a theme: {e}")))?; + if let Some(hard) = diagnostics.iter().find(|d| d.kind == DiagnosticKind::InvalidValue) { + return bad(format!("`{path}` has a rejected value: {hard}")); + } + if notescraft_themes::builtin(&theme.id).is_some() { + return bad(format!("`{path}` reuses the built-in theme id `{}`", theme.id)); + } + match expect_id { + Some(id) if id != theme.id => bad(format!("`{path}` has theme id `{}`, the entry is `{id}`", theme.id)), + _ => Ok(()), + } +} + +fn plugin(entry: &Entry, files: &[(String, Vec)]) -> Result<()> { + let find = |name: &str| files.iter().find(|(path, _)| path == name).map(|(_, bytes)| bytes.as_slice()); + let (Some(manifest), Some(wasm)) = (find("manifest.json"), find("plugin.wasm")) else { + return bad("a plugin needs manifest.json and plugin.wasm"); + }; + let manifest = Manifest::from_json(text("manifest.json", manifest)?).map_err(|e| Error::Addon(format!("manifest.json: {e}")))?; + if manifest.id != entry.id { + return bad(format!("manifest id `{}` is not the entry id `{}`", manifest.id, entry.id)); + } + if manifest.version != entry.version.to_string() { + return bad(format!("manifest version `{}` is not the entry version {}", manifest.version, entry.version)); + } + let declared: BTreeSet<&str> = manifest.permissions.iter().map(|p| p.as_str()).collect(); + let listed: BTreeSet<&str> = entry.permissions.iter().map(String::as_str).collect(); + if declared != listed { + return bad("manifest permissions are not the permissions the index lists"); + } + if !wasm.starts_with(WASM_MAGIC) { + return bad("plugin.wasm is not a WebAssembly module"); + } + if let Some(readme) = find(PLUGIN_README) { + text(PLUGIN_README, readme)?; + } + Ok(()) +} + +fn font(path: &str, bytes: &[u8]) -> Result<()> { + if FONT_MAGICS.iter().any(|magic| bytes.starts_with(magic)) { Ok(()) } else { bad(format!("`{path}` is not a TrueType or OpenType font")) } +} + +fn pack_file(path: &str, bytes: &[u8]) -> Result<()> { + match path.split('/').next() { + Some("grammars") => grammar(path, bytes), + Some("themes") => theme(path, bytes, None), + Some("templates") => text(path, bytes).map(|_| ()), + Some("icons") if Path::new(path).extension().is_some_and(|ext| ext.eq_ignore_ascii_case("json")) => { + serde_json::from_slice::(bytes).map(|_| ()).map_err(|e| Error::Addon(format!("`{path}` is not JSON: {e}"))) + } + Some("icons") => text(path, bytes).map(|_| ()), + _ => bad(format!("`{path}` is not in a pack folder")), + } +} + +/// Checks the staged `files` of `entry` for `kind`: the file set equals the listed one, then each +/// file passes the check of its kind (grammar parses and runs a sample, theme loads with no hard +/// diagnostic, plugin manifest equals the entry and the module has the wasm magic, font magic +/// bytes, snippet UTF-8 without NUL, pack files by folder). +/// +/// # Errors +/// [`Error::Addon`] naming the file and the broken rule. +pub fn check(kind: Kind, entry: &Entry, files: &[(String, Vec)]) -> Result<()> { + let staged: BTreeSet<&str> = files.iter().map(|(path, _)| path.as_str()).collect(); + let listed: BTreeSet<&str> = entry.files.iter().map(|file| file.path.as_str()).collect(); + if staged != listed || staged.len() != files.len() { + return bad("the staged files are not the listed files"); + } + if kind == Kind::Plugin { + return plugin(entry, files); + } + if matches!(kind, Kind::Grammar | Kind::Theme) && files.len() != 1 { + return bad("a grammar or theme entry has exactly one file"); + } + for (path, bytes) in files { + match kind { + Kind::Grammar => grammar(path, bytes)?, + Kind::Theme => theme(path, bytes, Some(&entry.id))?, + Kind::Font => font(path, bytes)?, + Kind::Snippet => text(path, bytes).map(|_| ())?, + Kind::Pack => pack_file(path, bytes)?, + Kind::Plugin => {} + } + } + Ok(()) +} diff --git a/crates/engine/src/addons/mod.rs b/crates/engine/src/addons/mod.rs new file mode 100644 index 0000000..cf9ded6 --- /dev/null +++ b/crates/engine/src/addons/mod.rs @@ -0,0 +1,30 @@ +//! Separable add-ons: the registry index, hostile-input guards, signatures and the local store. +//! +//! Not the command registry (`crate::registry`): this module is the add-on catalogue the +//! `registry.*` commands will drive. + +mod commands; +mod fileset; +mod install; +mod installed; +mod keys; +mod kinds; +mod model; +mod net; +mod state; +mod store; +mod verify; + +pub(crate) use commands::{TrustedVerifier, register_addons}; +pub use fileset::{Checked, KindRule, MAX_COMPONENT_BYTES, MAX_COMPONENTS, MAX_FILES, check_download_url, check_path, check_set, join_under, kind_cap}; +pub use install::{Ctx, InstallReport, Seed, Staged, fetch_files, install, remove, rollback, update, write_tree}; +pub(crate) use installed::installed_grammars; +pub use keys::TRUSTED_KEYS; +pub use kinds::check as check_content; +pub use model::{Entry, File, Index, Kind, MAX_INDEX_BYTES, Revoked, Signature, Version}; +pub use net::{ + ENTRY_TIMEOUT, FILE_TIMEOUT, FakeTransport, INDEX_URL, IndexCache, REGISTRY_HOSTS, Response, Transport, Ureq, coded, download_file, fetch_index, split_code, +}; +pub use state::{FILE_VERSION, Installed, MAX_INSTALLED, MAX_VAULT_IDS, Source, State, VaultFile, key, parse_key}; +pub use store::Store; +pub use verify::{Ed25519, FakeVerifier, Verifier, canonical_text}; diff --git a/crates/engine/src/addons/model.rs b/crates/engine/src/addons/model.rs new file mode 100644 index 0000000..8370292 --- /dev/null +++ b/crates/engine/src/addons/model.rs @@ -0,0 +1,282 @@ +//! The add-on index (schema v2): entries, versions, file lists and their validation. +//! +//! Pure data: no IO, no network, no signature check. [`Index::parse`] refuses a bad top level and +//! skips a broken entry (recording why in [`Index::skipped`]) so one bad row never hides the rest. +//! Nothing is clamped or repaired. + +use serde_json::Value; + +use self::raw::{RawEntry, RawIndex, RawRevoked, validate}; + +use crate::error::{Error, Result}; + +/// Largest index file, in bytes. +pub const MAX_INDEX_BYTES: usize = 1024 * 1024; +/// Most entries an index may hold. +pub const MAX_ENTRIES: usize = 2000; +/// Most files an entry may list. +pub const MAX_ENTRY_FILES: usize = 64; +/// Most aliases an entry may carry. +const MAX_ALIASES: usize = 16; +/// The only index schema version this build reads. +const INDEX_VERSION: u64 = 2; +/// Longest id, in bytes. +const MAX_ID: usize = 48; +/// Longest title, in characters. +const MAX_TITLE: usize = 80; +/// Longest summary, in characters. +const MAX_SUMMARY: usize = 200; +/// Longest license, in characters. +const MAX_LICENSE: usize = 40; +/// Longest alias, in bytes. +const MAX_ALIAS: usize = 32; +/// Longest link or file path, in bytes. +const MAX_TEXT: usize = 2048; +/// Longest signature key id, in bytes. +const MAX_KEY_ID: usize = 32; +/// Longest base64 signature, in bytes. +const MAX_SIG: usize = 128; +/// Longest timestamp, in bytes. +const MAX_STAMP: usize = 40; +/// Largest value of a version component. +const MAX_VERSION_PART: u16 = 9999; +/// Hex length of a SHA-256. +const SHA256_HEX: usize = 64; +/// Permissions a plugin entry may request. +const PERMISSIONS: [&str; 3] = ["vault.read", "vault.write", "editor"]; +/// One KiB. +const KIB: u64 = 1024; +/// One MiB. +const MIB: u64 = 1024 * KIB; + +/// What an add-on is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Kind { + /// A WebAssembly plugin. + Plugin, + /// A colour theme. + Theme, + /// Several files for several consumers, installed as one unit. + Pack, + /// A syntax-highlighting grammar. + Grammar, + /// A font file. + Font, + /// Text snippets and templates. + Snippet, +} + +impl Kind { + /// The name used in the index and in store folders. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Plugin => "plugin", + Self::Theme => "theme", + Self::Pack => "pack", + Self::Grammar => "grammar", + Self::Font => "font", + Self::Snippet => "snippet", + } + } + + /// Parses the name used in the index; anything else is `None`. + #[must_use] + pub fn parse(name: &str) -> Option { + [Self::Plugin, Self::Theme, Self::Pack, Self::Grammar, Self::Font, Self::Snippet].into_iter().find(|kind| kind.as_str() == name) + } + + /// Most bytes all files of one entry of this kind may add up to. + #[must_use] + pub const fn cap(self) -> u64 { + match self { + Self::Plugin => 4 * MIB, + Self::Theme => 64 * KIB, + Self::Grammar => 512 * KIB, + Self::Snippet => MIB, + Self::Pack => 8 * MIB, + Self::Font => 32 * MIB, + } + } +} + +/// `MAJOR.MINOR.PATCH`, each part 0..=9999, no pre-release. Orders numerically. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Version { + /// Major part. + pub major: u16, + /// Minor part. + pub minor: u16, + /// Patch part. + pub patch: u16, +} + +impl Version { + /// Parses `1.2.3`: three decimal parts, no sign, no leading zero, none above 9999. + /// + /// # Errors + /// [`Error::Addon`] naming the problem. + pub fn parse(text: &str) -> Result { + let mut parts = text.split('.'); + let (Some(major), Some(minor), Some(patch), None) = (parts.next(), parts.next(), parts.next(), parts.next()) else { + return Err(Error::Addon(format!("version `{text}` is not MAJOR.MINOR.PATCH"))); + }; + Ok(Self { major: part(text, major)?, minor: part(text, minor)?, patch: part(text, patch)? }) + } +} + +impl std::fmt::Display for Version { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}.{}.{}", self.major, self.minor, self.patch) + } +} + +fn part(text: &str, digits: &str) -> Result { + let bad = || Error::Addon(format!("version `{text}` is not MAJOR.MINOR.PATCH")); + if digits.is_empty() || digits.len() > 4 || !digits.bytes().all(|b| b.is_ascii_digit()) || digits.len() > 1 && digits.starts_with('0') { + return Err(bad()); + } + let value: u16 = digits.parse().map_err(|_| bad())?; + if value > MAX_VERSION_PART { Err(bad()) } else { Ok(value) } +} + +/// One downloadable file of an entry. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct File { + /// Relative path inside the entry (checked again by the fileset guard). + pub path: String, + /// Size in bytes, never zero. + pub size: u64, + /// Lowercase hex SHA-256 of the bytes. + pub sha256: String, + /// Absolute `https` link. + pub url: String, +} + +/// The owner's ed25519 signature over the entry's canonical text. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Signature { + /// Which trusted key signed. + pub key_id: String, + /// Base64 signature. + pub sig: String, +} + +/// One add-on of the index. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Entry { + /// Unique per kind: `[a-z0-9-]{1,48}`. + pub id: String, + /// What it is. + pub kind: Kind, + /// Display name. + pub title: String, + /// One-line description, may be empty. + pub summary: String, + /// Release version. + pub version: Version, + /// Oldest app version that can use it. + pub min_app_version: Version, + /// License name. + pub license: String, + /// Where the source lives (display only). + pub source: Option, + /// Fence labels (grammars) or search words. + pub aliases: Vec, + /// Offered in the first-run list. + pub suggested: bool, + /// Plugin permissions (`vault.read`, `vault.write`, `editor`). + pub permissions: Vec, + /// The files, 1..=64. + pub files: Vec, + /// The signature. + pub signature: Signature, +} + +impl Entry { + /// Sum of the file sizes, saturating. + #[must_use] + pub fn total_size(&self) -> u64 { + self.files.iter().fold(0_u64, |sum, file| sum.saturating_add(file.size)) + } +} + +/// A pulled release. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Revoked { + /// The add-on id. + pub id: String, + /// The pulled version. + pub version: Version, +} + +/// A validated index. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct Index { + /// When the index was generated (opaque text). + pub generated_at: String, + /// The valid entries. + pub entries: Vec, + /// Pulled releases. + pub revoked: Vec, + /// `id: reason` of every entry or revocation that was skipped. + pub skipped: Vec, +} + +impl Index { + /// Parses and validates an index document. + /// + /// # Errors + /// [`Error::Addon`] when the input is over [`MAX_INDEX_BYTES`], is not JSON, is not schema + /// version 2 or holds more than [`MAX_ENTRIES`] entries. A broken entry is skipped, not an error. + pub fn parse(json: &[u8]) -> Result { + if json.len() > MAX_INDEX_BYTES { + return Err(Error::Addon(format!("index is over {MAX_INDEX_BYTES} bytes"))); + } + let raw: RawIndex = serde_json::from_slice(json).map_err(|e| Error::Addon(format!("index is not valid: {e}")))?; + if raw.version != Some(INDEX_VERSION) { + return Err(Error::Addon("index version is not 2".into())); + } + if raw.entries.len() > MAX_ENTRIES { + return Err(Error::Addon(format!("index has more than {MAX_ENTRIES} entries"))); + } + if raw.generated_at.len() > MAX_STAMP { + return Err(Error::Addon("index generated_at is too long".into())); + } + let mut index = Self { generated_at: raw.generated_at, ..Self::default() }; + for value in raw.entries { + let label = value.get("id").and_then(Value::as_str).map_or_else(|| "?".to_owned(), str::to_owned); + match serde_json::from_value::(value).map_err(|e| e.to_string()).and_then(validate) { + Ok(entry) if index.find(entry.kind, &entry.id).is_some() => index.skipped.push(format!("{label}: duplicate id")), + Ok(entry) => index.entries.push(entry), + Err(reason) => index.skipped.push(format!("{label}: {reason}")), + } + } + for value in raw.revoked { + let parsed = serde_json::from_value::(value).map_err(|e| e.to_string()); + match parsed.and_then(|r| Version::parse(&r.version).map(|version| Revoked { id: r.id, version }).map_err(|e| e.to_string())) { + Ok(revoked) => index.revoked.push(revoked), + Err(reason) => index.skipped.push(format!("?: revoked: {reason}")), + } + } + Ok(index) + } + + /// The entry with this kind and id. + #[must_use] + pub fn find(&self, kind: Kind, id: &str) -> Option<&Entry> { + self.entries.iter().find(|e| e.kind == kind && e.id == id) + } + + /// Whether the index pulled this exact release. + #[must_use] + pub fn is_revoked(&self, entry: &Entry) -> bool { + self.revoked.iter().any(|r| r.id == entry.id && r.version == entry.version) + } +} + +mod raw; +pub(crate) use raw::is_valid_id; + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/addons/model/raw.rs b/crates/engine/src/addons/model/raw.rs new file mode 100644 index 0000000..0c6385d --- /dev/null +++ b/crates/engine/src/addons/model/raw.rs @@ -0,0 +1,156 @@ +//! Raw JSON shapes of the index and the rules that turn them into validated [`Entry`] values. + +use serde::Deserialize; +use serde_json::Value; + +use super::{ + Entry, File, Kind, MAX_ALIAS, MAX_ALIASES, MAX_ENTRY_FILES, MAX_ID, MAX_KEY_ID, MAX_LICENSE, MAX_SIG, MAX_SUMMARY, MAX_TEXT, MAX_TITLE, PERMISSIONS, + SHA256_HEX, Signature, Version, +}; + +#[derive(Deserialize)] +pub(super) struct RawIndex { + pub(super) version: Option, + #[serde(default)] + pub(super) generated_at: String, + #[serde(default)] + pub(super) entries: Vec, + #[serde(default)] + pub(super) revoked: Vec, +} + +#[derive(Deserialize)] +pub(super) struct RawFile { + pub(super) path: String, + pub(super) size: u64, + pub(super) sha256: String, + pub(super) url: String, +} + +#[derive(Deserialize)] +pub(super) struct RawSignature { + pub(super) key_id: String, + pub(super) sig: String, +} + +#[derive(Deserialize)] +pub(super) struct RawEntry { + pub(super) id: String, + pub(super) kind: String, + pub(super) title: String, + #[serde(default)] + pub(super) summary: String, + pub(super) version: String, + pub(super) min_app_version: String, + pub(super) license: String, + pub(super) source: Option, + #[serde(default)] + pub(super) aliases: Vec, + #[serde(default)] + pub(super) suggested: bool, + #[serde(default)] + pub(super) permissions: Vec, + pub(super) files: Vec, + pub(super) signature: RawSignature, +} + +#[derive(Deserialize)] +pub(super) struct RawRevoked { + pub(super) id: String, + pub(super) version: String, +} + +/// Whether `id` is a valid add-on id: `[a-z0-9-]{1,48}`. +pub(crate) fn is_valid_id(id: &str) -> bool { + charset(id, MAX_ID, |c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') +} + +fn no_control(text: &str) -> bool { + !text.chars().any(char::is_control) +} + +fn charset(text: &str, max: usize, ok: impl Fn(char) -> bool) -> bool { + !text.is_empty() && text.len() <= max && text.chars().all(ok) +} + +fn https(text: &str) -> bool { + text.len() <= MAX_TEXT && text.starts_with("https://") && !text.chars().any(|c| c.is_control() || c.is_whitespace()) +} + +pub(super) fn validate(raw: RawEntry) -> std::result::Result { + if !is_valid_id(&raw.id) { + return Err("invalid id".into()); + } + let kind = Kind::parse(&raw.kind).ok_or("unknown kind")?; + let title_len = raw.title.chars().count(); + if title_len == 0 || title_len > MAX_TITLE || !no_control(&raw.title) { + return Err("invalid title".into()); + } + if raw.summary.chars().count() > MAX_SUMMARY || !no_control(&raw.summary) { + return Err("invalid summary".into()); + } + let license_len = raw.license.chars().count(); + if license_len == 0 || license_len > MAX_LICENSE || !no_control(&raw.license) { + return Err("invalid license".into()); + } + let version = Version::parse(&raw.version).map_err(|e| e.to_string())?; + let min_app_version = Version::parse(&raw.min_app_version).map_err(|e| e.to_string())?; + if raw.source.as_deref().is_some_and(|s| !https(s)) { + return Err("invalid source".into()); + } + let alias_ok = |a: &String| charset(a, MAX_ALIAS, |c| c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '.' | '_' | '+' | '-')); + if raw.aliases.len() > MAX_ALIASES || !raw.aliases.iter().all(alias_ok) { + return Err("invalid aliases".into()); + } + if !raw.permissions.is_empty() && kind != Kind::Plugin { + return Err("permissions on a non-plugin".into()); + } + if !raw.permissions.iter().all(|p| PERMISSIONS.contains(&p.as_str())) { + return Err("unknown permission".into()); + } + if raw.files.is_empty() || raw.files.len() > MAX_ENTRY_FILES { + return Err("file count out of range".into()); + } + let mut total = 0_u64; + let mut files = Vec::with_capacity(raw.files.len()); + for file in raw.files { + let hex = file.sha256.len() == SHA256_HEX && file.sha256.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)); + if file.path.is_empty() || file.path.len() > MAX_TEXT || !no_control(&file.path) { + return Err("invalid file path".into()); + } + if !hex { + return Err("invalid sha256".into()); + } + if !https(&file.url) { + return Err("file url is not https".into()); + } + if file.size == 0 { + return Err("zero-size file".into()); + } + total = total.saturating_add(file.size); + files.push(File { path: file.path, size: file.size, sha256: file.sha256, url: file.url }); + } + if total > kind.cap() { + return Err("files are over the size cap".into()); + } + let key_ok = charset(&raw.signature.key_id, MAX_KEY_ID, |c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_')); + let sig_ok = charset(&raw.signature.sig, MAX_SIG, |c| c.is_ascii_alphanumeric() || matches!(c, '+' | '/' | '=')); + if !key_ok || !sig_ok { + return Err("invalid signature".into()); + } + Ok(Entry { + id: raw.id, + kind, + title: raw.title, + summary: raw.summary, + version, + min_app_version, + license: raw.license, + source: raw.source, + aliases: raw.aliases, + suggested: raw.suggested, + permissions: raw.permissions, + files, + signature: Signature { key_id: raw.signature.key_id, sig: raw.signature.sig }, + }) +} diff --git a/crates/engine/src/addons/model/tests.rs b/crates/engine/src/addons/model/tests.rs new file mode 100644 index 0000000..3d94fd9 --- /dev/null +++ b/crates/engine/src/addons/model/tests.rs @@ -0,0 +1,135 @@ +use serde_json::{Value, json}; + +use super::*; + +fn entry(id: &str, kind: &str) -> Value { + json!({ + "id": id, "kind": kind, "title": "T", "summary": "s", "version": "1.0.0", "min_app_version": "0.1.0", "license": "MIT", + "aliases": ["a"], "permissions": [], + "files": [{ "path": "x.toml", "size": 10, "sha256": "a".repeat(64), "url": "https://h.example/x.toml" }], + "signature": { "key_id": "k1", "sig": "QUJD" } + }) +} + +fn index(entries: &[Value]) -> Vec { + json!({ "version": 2, "generated_at": "2026-10-21T12:00:00Z", "entries": entries, "revoked": [{ "id": "old", "version": "0.9.0" }] }) + .to_string() + .into_bytes() +} + +fn with(mut e: Value, key: &str, value: Value) -> Value { + e[key] = value; + e +} + +#[test] +fn good_index_round_trip() { + let parsed = Index::parse(&index(&[entry("twig", "grammar"), entry("dice", "plugin")])).unwrap(); + assert_eq!(parsed.entries.len(), 2); + assert_eq!(parsed.skipped.len(), 0); + assert_eq!(parsed.generated_at, "2026-10-21T12:00:00Z"); + let twig = parsed.find(Kind::Grammar, "twig").unwrap(); + assert_eq!(twig.version, Version { major: 1, minor: 0, patch: 0 }); + assert_eq!(twig.total_size(), 10); + assert!(parsed.find(Kind::Theme, "twig").is_none()); + assert_eq!(parsed.revoked.len(), 1); +} + +#[test] +fn top_level_rejections() { + assert!(Index::parse(br#"{"version":1,"entries":[]}"#).is_err()); + assert!(Index::parse(br#"{"version":3,"entries":[]}"#).is_err()); + assert!(Index::parse(br#"{"entries":[]}"#).is_err()); + assert!(Index::parse(b"nope").is_err()); + assert!(Index::parse(&vec![b' '; MAX_INDEX_BYTES + 1]).is_err()); + let many: Vec = (0..=MAX_ENTRIES).map(|i| entry(&format!("e{i}"), "theme")).collect(); + assert!(Index::parse(&index(&many)).is_err()); + let deep = format!("{}{}", "[".repeat(200), "]".repeat(200)); + assert!(Index::parse(deep.as_bytes()).is_err()); + assert!(Index::parse(format!(r#"{{"version":2,"entries":{deep}}}"#).as_bytes()).is_err()); +} + +#[test] +fn broken_entries_are_skipped() { + let long_files: Vec = + (0..65).map(|i| json!({ "path": format!("f{i}"), "size": 1, "sha256": "a".repeat(64), "url": "https://h.example/f" })).collect(); + let file = |patch: Value| { + let mut f = entry("x", "theme")["files"][0].clone(); + for (k, v) in patch.as_object().unwrap() { + f[k] = v.clone(); + } + json!([f]) + }; + let bad = [ + with(entry("Bad Id", "theme"), "id", json!("Bad Id")), + with(entry("sha", "theme"), "files", file(json!({ "sha256": "A".repeat(64) }))), + with(entry("http", "theme"), "files", file(json!({ "url": "http://h.example/x" }))), + with(entry("zero", "theme"), "files", file(json!({ "size": 0 }))), + with(entry("many", "theme"), "files", json!(long_files)), + with(entry("perm", "plugin"), "permissions", json!(["net"])), + with(entry("permtheme", "theme"), "permissions", json!(["editor"])), + entry("exe", "exe"), + with(entry("ver", "theme"), "version", json!("1.2")), + with(entry("nosig", "theme"), "signature", json!({ "key_id": "", "sig": "" })), + json!({ "kind": "theme" }), + ]; + let mut all = vec![entry("ok1", "theme")]; + all.extend(bad.iter().cloned()); + all.push(entry("ok2", "theme")); + let parsed = Index::parse(&index(&all)).unwrap(); + assert_eq!(parsed.entries.iter().map(|e| e.id.as_str()).collect::>(), ["ok1", "ok2"]); + assert_eq!(parsed.skipped.len(), bad.len()); + assert!(parsed.skipped.iter().any(|s| s.starts_with("?:"))); + assert!(parsed.skipped.iter().any(|s| s.starts_with("exe:"))); +} + +#[test] +fn valid_plugin_permissions_and_duplicates() { + let ok = with(entry("dice", "plugin"), "permissions", json!(["vault.read", "editor"])); + let parsed = Index::parse(&index(&[ok.clone(), ok, entry("dice", "theme")])).unwrap(); + assert_eq!(parsed.entries.len(), 2); + assert_eq!(parsed.skipped, ["dice: duplicate id"]); +} + +#[test] +fn per_kind_cap_is_exact() { + let big = |size: u64| { + with(entry("g", "grammar"), "files", json!([{ "path": "g.sublime-syntax", "size": size, "sha256": "b".repeat(64), "url": "https://h.example/g" }])) + }; + let cap = Kind::Grammar.cap(); + assert_eq!(Index::parse(&index(&[big(cap)])).unwrap().entries.len(), 1); + let over = Index::parse(&index(&[big(cap + 1)])).unwrap(); + assert_eq!(over.entries.len(), 0); + assert_eq!(over.skipped.len(), 1); + let huge = Index::parse(&index(&[big(u64::MAX)])).unwrap(); + assert_eq!(huge.entries.len(), 0); +} + +#[test] +fn version_parse_and_order() { + assert_eq!(Version::parse("1.2.3").unwrap(), Version { major: 1, minor: 2, patch: 3 }); + assert_eq!(Version::parse("9999.0.0").unwrap().to_string(), "9999.0.0"); + for bad in ["01.2.3", "1.2", "1.2.3-beta", "10000.0.0", "", "1.2.3.4", "1..3", "+1.2.3", "-1.2.3", "1.2.x"] { + assert!(Version::parse(bad).is_err(), "{bad}"); + } + assert!(Version::parse("1.10.0").unwrap() > Version::parse("1.9.9").unwrap()); +} + +#[test] +fn total_size_saturates() { + let parsed = Index::parse(&index(&[entry("a", "theme")])).unwrap(); + let mut e = parsed.entries[0].clone(); + let mut f = e.files[0].clone(); + f.size = u64::MAX; + e.files = vec![f.clone(), f]; + assert_eq!(e.total_size(), u64::MAX); +} + +#[test] +fn revoked_match() { + let mut e = Index::parse(&index(&[entry("old", "theme")])).unwrap(); + let old = e.entries[0].clone(); + assert!(!e.is_revoked(&old)); + e.revoked.push(Revoked { id: "old".into(), version: old.version }); + assert!(e.is_revoked(&old)); +} diff --git a/crates/engine/src/addons/net.rs b/crates/engine/src/addons/net.rs new file mode 100644 index 0000000..a3de055 --- /dev/null +++ b/crates/engine/src/addons/net.rs @@ -0,0 +1,265 @@ +//! Transport and bounded download of the add-on registry (spec D5, D7). +//! +//! [`Transport`] is injectable so no test touches the network. [`Ureq`] speaks https only, never +//! follows a redirect, has a total timeout and reads at most `max_bytes + 1` so an over-long body +//! is a failure, never a truncation. [`download_file`] adds the entry-level gates: host allow-list, +//! the listed size and the listed sha256, checked on the bytes actually received. + +use std::io::Read; +use std::path::Path; +use std::sync::Mutex; +use std::time::Duration; + +use notescraft_syntax::validate::sha256_hex; +use serde::{Deserialize, Serialize}; + +use super::fileset::check_download_url; +use super::model::{File, Index, MAX_INDEX_BYTES}; +use crate::error::{Error, Result}; + +/// Hosts a download link may point at. +pub const REGISTRY_HOSTS: [&str; 1] = ["git.marcospaulo.dev.br"]; +/// The registry index. +pub const INDEX_URL: &str = "https://git.marcospaulo.dev.br/masi/notescraft-registry/raw/branch/main/index.json"; +/// Total time allowed for one request. +pub const FILE_TIMEOUT: Duration = Duration::from_secs(30); +/// Total time allowed for all files of one entry. +pub const ENTRY_TIMEOUT: Duration = Duration::from_secs(300); +/// Name of the cached index under the registry folder. +const CACHE_FILE: &str = "index-cache.json"; +/// First redirect status. +const REDIRECT_FROM: u16 = 300; +/// Not modified. +const NOT_MODIFIED: u16 = 304; +/// OK. +const OK: u16 = 200; +/// Every machine code an add-on error may carry in its message prefix. +pub const CODES: [&str; 9] = ["offline", "not_found", "incompatible", "revoked", "limit", "disk_full", "needs_dependency", "download_failed", "addon"]; + +/// An [`Error::Addon`] whose message starts with a stable command code, e.g. `offline: ...`. +#[must_use] +pub fn coded(code: &str, message: impl std::fmt::Display) -> Error { + Error::Addon(format!("{code}: {message}")) +} + +/// The command code and message of an add-on error: a tagged message keeps its tag, any other +/// message is a refused gate (`registry_refused`). +#[must_use] +pub fn split_code(message: &str) -> (&str, &str) { + match message.split_once(": ") { + Some((code, rest)) if CODES.contains(&code) => (code, rest), + _ => ("registry_refused", message), + } +} + +/// What a server answered. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Response { + /// HTTP status: 200, or 304 for a conditional request that matched. + pub status: u16, + /// The `ETag` header, when present. + pub etag: Option, + /// The body (empty for 304). + pub bytes: Vec, +} + +/// Fetches one https link, never more than `max_bytes`. +pub trait Transport: std::fmt::Debug + Send { + /// Sends `If-None-Match` when `if_none_match` is set. + /// + /// # Errors + /// `offline: ...` when there is no network, `download_failed: ...` for a redirect, another + /// status or a body over `max_bytes`. + fn get(&self, url: &str, max_bytes: u64, if_none_match: Option<&str>) -> Result; +} + +/// The real transport: `ureq` with https only, no redirects and a global timeout. +#[derive(Debug, Clone)] +pub struct Ureq { + agent: ureq::Agent, +} + +impl Default for Ureq { + fn default() -> Self { + Self::new() + } +} + +impl Ureq { + /// A transport with the registry limits. + #[must_use] + pub fn new() -> Self { + let agent = ureq::Agent::config_builder().http_status_as_error(false).max_redirects(0).timeout_global(Some(FILE_TIMEOUT)).build().into(); + Self { agent } + } +} + +impl Transport for Ureq { + fn get(&self, url: &str, max_bytes: u64, if_none_match: Option<&str>) -> Result { + if !url.starts_with("https://") { + return Err(coded("download_failed", "only https links are allowed")); + } + let mut request = self.agent.get(url).header("User-Agent", concat!("NotesCraft/", env!("CARGO_PKG_VERSION"))); + if let Some(etag) = if_none_match { + request = request.header("If-None-Match", etag); + } + let response = request.call().map_err(|error| match error { + ureq::Error::HostNotFound | ureq::Error::ConnectionFailed | ureq::Error::Timeout(_) | ureq::Error::Io(_) => coded("offline", "no network"), + other => coded("download_failed", other), + })?; + let status = response.status().as_u16(); + let etag = response.headers().get("etag").and_then(|value| value.to_str().ok()).map(str::to_owned); + if status == NOT_MODIFIED { + return Ok(Response { status, etag, bytes: Vec::new() }); + } + if status != OK { + let what = if (REDIRECT_FROM..400).contains(&status) { "redirects are not followed" } else { "the server refused" }; + return Err(coded("download_failed", format!("{what} (status {status})"))); + } + let mut bytes = Vec::new(); + let mut body = response.into_body().into_reader().take(max_bytes.saturating_add(1)); + body.read_to_end(&mut bytes).map_err(|error| coded("offline", error))?; + if u64::try_from(bytes.len()).unwrap_or(u64::MAX) > max_bytes { + return Err(coded("download_failed", format!("the body is over {max_bytes} bytes"))); + } + Ok(Response { status, etag, bytes }) + } +} + +/// A transport for tests: answers from a script, records the links asked for, and enforces +/// `max_bytes` and the status rules like the real one. +#[derive(Debug, Default)] +pub struct FakeTransport { + answers: Mutex)>>, + asked: Mutex>, +} + +impl FakeTransport { + /// An empty script: every link is `offline`. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Answers `url` with a 200 and `bytes`. + #[must_use] + pub fn with(self, url: &str, bytes: &[u8]) -> Self { + self.with_response(url, Ok(Response { status: OK, etag: None, bytes: bytes.to_vec() })) + } + + /// Answers `url` with any response, or fails it with this [`Error::Addon`] message. + #[must_use] + pub fn with_response(self, url: &str, response: std::result::Result) -> Self { + if let Ok(mut answers) = self.answers.lock() { + answers.push((url.to_owned(), response)); + } + self + } + + /// The links asked for so far, in order. + #[must_use] + pub fn asked(&self) -> Vec { + self.asked.lock().map(|asked| asked.clone()).unwrap_or_default() + } +} + +impl Transport for FakeTransport { + fn get(&self, url: &str, max_bytes: u64, _if_none_match: Option<&str>) -> Result { + if let Ok(mut asked) = self.asked.lock() { + asked.push(url.to_owned()); + } + let found = self.answers.lock().ok().and_then(|answers| answers.iter().find(|(known, _)| known == url).map(|(_, answer)| answer.clone())); + let response = found.unwrap_or_else(|| Err("offline: no network".to_owned())).map_err(Error::Addon)?; + if response.status >= REDIRECT_FROM && response.status != NOT_MODIFIED { + return Err(coded("download_failed", format!("status {}", response.status))); + } + if u64::try_from(response.bytes.len()).unwrap_or(u64::MAX) > max_bytes { + return Err(coded("download_failed", format!("the body is over {max_bytes} bytes"))); + } + Ok(response) + } +} + +/// Downloads one listed file and checks it: link on an allowed host, listed size at most `cap`, +/// body exactly the listed size, sha256 equal to the listed one. +/// +/// # Errors +/// [`Error::Addon`]: a gate, `offline: ...` or `download_failed: ...`. +pub fn download_file(transport: &dyn Transport, file: &File, cap: u64) -> Result> { + check_download_url(&file.url, ®ISTRY_HOSTS)?; + if file.size > cap { + return Err(Error::Addon(format!("`{}` is over the size cap", file.path))); + } + let response = transport.get(&file.url, file.size, None)?; + if response.status != OK { + return Err(coded("download_failed", format!("status {} for `{}`", response.status, file.path))); + } + if u64::try_from(response.bytes.len()).ok() != Some(file.size) { + return Err(Error::Addon(format!("`{}` is not the listed size", file.path))); + } + if sha256_hex(&response.bytes) != file.sha256 { + return Err(Error::Addon(format!("`{}` does not match its sha256", file.path))); + } + Ok(response.bytes) +} + +/// The last verified index, kept on disk beside `state.json`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct IndexCache { + /// Unix seconds of the last successful check. + pub fetched_at: u64, + /// The server's `ETag` for `body`. + pub etag: Option, + /// The index document, exactly as received. + pub body: String, +} + +impl IndexCache { + /// Reads `/index-cache.json`; `None` when absent. + /// + /// # Errors + /// [`Error::Addon`] for an unreadable or invalid file. + pub fn load(root: &Path) -> Result> { + match std::fs::read(root.join(CACHE_FILE)) { + Ok(bytes) => serde_json::from_slice(&bytes).map(Some).map_err(|e| Error::Addon(format!("index cache is not valid: {e}"))), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(Error::Addon(format!("cannot read the index cache: {e}"))), + } + } + + /// Writes the cache atomically. + /// + /// # Errors + /// [`Error::Addon`] when the write fails (the old cache stays). + pub fn save(&self, root: &Path) -> Result<()> { + let json = serde_json::to_string(self).map_err(|e| Error::Addon(e.to_string()))?; + craft_atomic::atomic_write(&root.join(CACHE_FILE), json.as_bytes()).map_err(|e| Error::Addon(format!("cannot write the index cache: {e}"))) + } + + /// The parsed index. + /// + /// # Errors + /// [`Error::Addon`] when the cached body no longer parses. + pub fn index(&self) -> Result { + Index::parse(self.body.as_bytes()) + } +} + +/// One conditional GET of the index. Returns the new cache, or `cache` itself (with a fresh +/// `fetched_at`) on a 304. A failed fetch or an invalid index leaves the stored cache untouched. +/// +/// # Errors +/// `offline: ...`, `download_failed: ...` or the reason the index is invalid. +pub fn fetch_index(transport: &dyn Transport, cache: Option<&IndexCache>, now: u64) -> Result { + check_download_url(INDEX_URL, ®ISTRY_HOSTS)?; + let max = u64::try_from(MAX_INDEX_BYTES).unwrap_or(u64::MAX); + let etag = cache.and_then(|c| c.etag.as_deref()); + let response = transport.get(INDEX_URL, max, etag)?; + if response.status == NOT_MODIFIED { + let kept = cache.ok_or_else(|| coded("download_failed", "304 without a cached index"))?; + return Ok(IndexCache { fetched_at: now, ..kept.clone() }); + } + let body = String::from_utf8(response.bytes).map_err(|_| Error::Addon("the index is not UTF-8".into()))?; + Index::parse(body.as_bytes())?; + Ok(IndexCache { fetched_at: now, etag: response.etag, body }) +} diff --git a/crates/engine/src/addons/state.rs b/crates/engine/src/addons/state.rs new file mode 100644 index 0000000..7efe82b --- /dev/null +++ b/crates/engine/src/addons/state.rs @@ -0,0 +1,215 @@ +//! The two small JSON documents of the add-on store: `state.json` (what is installed, global) and +//! `/.notescraft/registry.json` (what this vault enables and wants). Spec D4. +//! +//! Both are versioned (`1`), size and count capped, and strict: an unknown version, a malformed +//! key or a value over a cap is an `Err`; nothing is repaired. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use super::model::{Kind, Version, is_valid_id}; +use crate::error::{Error, Result}; + +/// The version of both documents this build reads and writes. +pub const FILE_VERSION: u32 = 1; +/// Most installed add-ons. +pub const MAX_INSTALLED: usize = 512; +/// Most ids in a vault's `enabled` or `wanted` list. +pub const MAX_VAULT_IDS: usize = 256; +/// Largest document, in bytes. +pub const MAX_DOC_BYTES: usize = 1024 * 1024; +/// Longest timestamp, in bytes. +const MAX_STAMP: usize = 40; +/// Hex length of a SHA-256. +const SHA256_HEX: usize = 64; + +/// Where an installed add-on came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Source { + /// Downloaded from the registry. + Registry, + /// Adopted from the pre-registry grammar settings. + Legacy, + /// Installed from the seed shipped with the app. + Seed, + /// Installed from a file the user picked. + Sideload, +} + +/// One installed add-on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Installed { + /// The active version. + pub version: Version, + /// The version rollback returns to, if one is kept. + pub previous: Option, + /// Where it came from. + pub source: Source, + /// When it was installed (opaque text). + pub installed_at: String, + /// Hex SHA-256 over the files, as recorded at install time. + pub sha256_of_files: String, +} + +/// `state.json`: the installed table, keyed `:`. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct State { + /// Installed add-ons. + pub installed: BTreeMap, +} + +#[derive(Serialize, Deserialize)] +struct RawInstalled { + version: String, + previous: Option, + source: Source, + installed_at: String, + sha256_of_files: String, +} + +#[derive(Serialize, Deserialize)] +struct RawState { + version: u32, + installed: BTreeMap, +} + +fn addon(reason: impl Into) -> Error { + Error::Addon(reason.into()) +} + +/// The table key of an add-on: `:`. +#[must_use] +pub fn key(kind: Kind, id: &str) -> String { + format!("{}:{id}", kind.as_str()) +} + +/// Splits and validates a `:` key. +/// +/// # Errors +/// [`Error::Addon`] when the kind or id is not valid. +pub fn parse_key(text: &str) -> Result<(Kind, &str)> { + let (kind, id) = text.split_once(':').ok_or_else(|| addon(format!("`{text}` is not :")))?; + match Kind::parse(kind) { + Some(kind) if is_valid_id(id) => Ok((kind, id)), + _ => Err(addon(format!("`{text}` is not :"))), + } +} + +fn check_len(bytes: &[u8]) -> Result<()> { + if bytes.len() > MAX_DOC_BYTES { Err(addon(format!("document is over {MAX_DOC_BYTES} bytes"))) } else { Ok(()) } +} + +impl State { + /// Parses `state.json`. + /// + /// # Errors + /// [`Error::Addon`] for bad JSON, an unknown version, more than [`MAX_INSTALLED`] rows, a bad + /// key, version, hash or timestamp. + pub fn parse(bytes: &[u8]) -> Result { + check_len(bytes)?; + let raw: RawState = serde_json::from_slice(bytes).map_err(|e| addon(format!("state is not valid: {e}")))?; + if raw.version != FILE_VERSION { + return Err(addon(format!("state version {} is not {FILE_VERSION}", raw.version))); + } + if raw.installed.len() > MAX_INSTALLED { + return Err(addon(format!("more than {MAX_INSTALLED} installed add-ons"))); + } + let mut installed = BTreeMap::new(); + for (name, row) in raw.installed { + parse_key(&name)?; + let hex = row.sha256_of_files.len() == SHA256_HEX && row.sha256_of_files.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)); + if !hex || row.installed_at.len() > MAX_STAMP || row.installed_at.chars().any(char::is_control) { + return Err(addon(format!("`{name}` has an invalid hash or timestamp"))); + } + let previous = row.previous.as_deref().map(Version::parse).transpose()?; + let version = Version::parse(&row.version)?; + installed.insert(name, Installed { version, previous, source: row.source, installed_at: row.installed_at, sha256_of_files: row.sha256_of_files }); + } + Ok(Self { installed }) + } + + /// Serialises to the `state.json` text. + /// + /// # Errors + /// [`Error::Addon`] when the table is over [`MAX_INSTALLED`] or serialisation fails. + pub fn to_json(&self) -> Result { + if self.installed.len() > MAX_INSTALLED { + return Err(addon(format!("more than {MAX_INSTALLED} installed add-ons"))); + } + let installed = self + .installed + .iter() + .map(|(name, row)| { + let raw = RawInstalled { + version: row.version.to_string(), + previous: row.previous.map(|v| v.to_string()), + source: row.source, + installed_at: row.installed_at.clone(), + sha256_of_files: row.sha256_of_files.clone(), + }; + (name.clone(), raw) + }) + .collect(); + serde_json::to_string_pretty(&RawState { version: FILE_VERSION, installed }).map_err(|e| addon(e.to_string())) + } +} + +/// `/.notescraft/registry.json`: per-vault add-on choices (spec D4). +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct VaultFile { + /// Plugins switched on in this vault, as `plugin:`. + pub enabled: Vec, + /// Add-ons this vault was seen to use, as `:`; never installed automatically. + pub wanted: Vec, +} + +#[derive(Serialize, Deserialize)] +struct RawVault { + version: u32, + #[serde(default)] + enabled: Vec, + #[serde(default)] + wanted: Vec, +} + +impl VaultFile { + /// Parses the per-vault file. + /// + /// # Errors + /// [`Error::Addon`] for bad JSON, an unknown version, a list over [`MAX_VAULT_IDS`] or a bad key. + pub fn parse(bytes: &[u8]) -> Result { + check_len(bytes)?; + let raw: RawVault = serde_json::from_slice(bytes).map_err(|e| addon(format!("registry.json is not valid: {e}")))?; + if raw.version != FILE_VERSION { + return Err(addon(format!("registry.json version {} is not {FILE_VERSION}", raw.version))); + } + let file = Self { enabled: raw.enabled, wanted: raw.wanted }; + file.check()?; + Ok(file) + } + + fn check(&self) -> Result<()> { + if self.enabled.len() > MAX_VAULT_IDS || self.wanted.len() > MAX_VAULT_IDS { + return Err(addon(format!("more than {MAX_VAULT_IDS} ids in a list"))); + } + for name in self.enabled.iter().chain(&self.wanted) { + parse_key(name)?; + } + if let Some(name) = self.enabled.iter().find(|n| !n.starts_with("plugin:")) { + return Err(addon(format!("`{name}` is enabled but is not a plugin"))); + } + Ok(()) + } + + /// Serialises to the file text. + /// + /// # Errors + /// [`Error::Addon`] when a list is over the cap, a key is malformed or serialisation fails. + pub fn to_json(&self) -> Result { + self.check()?; + serde_json::to_string_pretty(&RawVault { version: FILE_VERSION, enabled: self.enabled.clone(), wanted: self.wanted.clone() }) + .map_err(|e| addon(e.to_string())) + } +} diff --git a/crates/engine/src/addons/store.rs b/crates/engine/src/addons/store.rs new file mode 100644 index 0000000..2cf35cf --- /dev/null +++ b/crates/engine/src/addons/store.rs @@ -0,0 +1,147 @@ +//! The on-disk store under `/registry/` (spec D4): `state.json`, immutable version +//! folders and a staging folder. +//! +//! Everything here stays inside the registry folder. `state.json` is written by [`Store::commit`] +//! alone, through `craft-atomic`; the version folders are unreferenced until that commit, so a +//! crash leaves garbage that [`Store::gc`] removes, never a half-install. + +use std::collections::BTreeSet; +use std::fs; +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; + +use super::model::{Kind, Version, is_valid_id}; +use super::state::State; +use crate::error::{Error, Result}; + +/// Folder name under the data dir. +const ROOT: &str = "registry"; +/// The installed table. +const STATE_FILE: &str = "state.json"; +/// Immutable `///` trees. +const STORE_DIR: &str = "store"; +/// Downloads in progress. +const STAGING_DIR: &str = "staging"; + +/// Handle to `/registry/`. +#[derive(Debug, Clone)] +pub struct Store { + root: PathBuf, +} + +fn io(context: &str, error: &std::io::Error) -> Error { + Error::Addon(format!("{context}: {error}")) +} + +impl Store { + /// Opens (creating the folders) the store under `data_dir`. + /// + /// # Errors + /// [`Error::Addon`] when a folder cannot be created. + pub fn open(data_dir: &Path) -> Result { + let root = data_dir.join(ROOT); + for dir in [STORE_DIR, STAGING_DIR] { + fs::create_dir_all(root.join(dir)).map_err(|e| io("cannot create the registry folder", &e))?; + } + Ok(Self { root }) + } + + /// The registry folder itself. + #[must_use] + pub fn root(&self) -> &Path { + &self.root + } + + /// The folder downloads are staged in. + #[must_use] + pub fn staging_dir(&self) -> PathBuf { + self.root.join(STAGING_DIR) + } + + /// `store////`, built from the validated id and the numeric version. + /// + /// # Errors + /// [`Error::Addon`] when `id` is not a valid add-on id. + pub fn version_dir(&self, kind: Kind, id: &str, version: Version) -> Result { + if !is_valid_id(id) { + return Err(Error::Addon(format!("`{id}` is not a valid add-on id"))); + } + Ok(self.root.join(STORE_DIR).join(kind.as_str()).join(id).join(version.to_string())) + } + + /// Reads `state.json`; a missing file is an empty state. + /// + /// # Errors + /// [`Error::Addon`] for an unreadable or invalid file. The file is never touched. + pub fn load(&self) -> Result { + match fs::read(self.root.join(STATE_FILE)) { + Ok(bytes) => State::parse(&bytes), + Err(e) if e.kind() == ErrorKind::NotFound => Ok(State::default()), + Err(e) => Err(io("cannot read state.json", &e)), + } + } + + /// Writes `state.json` atomically: the one commit point of every install and removal. + /// + /// # Errors + /// [`Error::Addon`] when the state is invalid or the write fails (the old file stays). + pub fn commit(&self, state: &State) -> Result<()> { + let json = state.to_json()?; + craft_atomic::atomic_write(&self.root.join(STATE_FILE), json.as_bytes()).map_err(|e| io("cannot write state.json", &e)) + } + + /// Removes every version folder `state` does not reference (its `version` and `previous` are + /// kept), empty parents, and everything in `staging/`. Returns how many folders went. + /// + /// # Errors + /// [`Error::Addon`] when the store cannot be listed or a folder cannot be removed. + pub fn gc(&self, state: &State) -> Result { + let mut keep: BTreeSet = BTreeSet::new(); + for (name, row) in &state.installed { + let (kind, id) = super::state::parse_key(name)?; + keep.insert(self.version_dir(kind, id, row.version)?); + if let Some(previous) = row.previous { + keep.insert(self.version_dir(kind, id, previous)?); + } + } + let mut removed = 0; + for kind in children(&self.root.join(STORE_DIR))? { + for id in children(&kind)? { + for version in children(&id)? { + if !keep.contains(&version) { + remove(&version)?; + removed += 1; + } + } + prune_empty(&id)?; + } + prune_empty(&kind)?; + } + for staged in children(&self.root.join(STAGING_DIR))? { + remove(&staged)?; + } + Ok(removed) + } +} + +fn children(dir: &Path) -> Result> { + match fs::read_dir(dir) { + Ok(entries) => Ok(entries.flatten().map(|e| e.path()).collect()), + Err(e) if e.kind() == ErrorKind::NotFound => Ok(Vec::new()), + Err(e) => Err(io("cannot list the registry folder", &e)), + } +} + +/// Removes a file, a link (not its target) or a folder tree. +fn remove(path: &Path) -> Result<()> { + let is_dir = fs::symlink_metadata(path).is_ok_and(|m| m.is_dir()); + let done = if is_dir { fs::remove_dir_all(path) } else { fs::remove_file(path) }; + done.map_err(|e| io("cannot remove a registry folder", &e)) +} + +fn prune_empty(dir: &Path) -> Result<()> { + if children(dir)?.is_empty() { + fs::remove_dir(dir).map_err(|e| io("cannot remove a registry folder", &e))?; + } + Ok(()) +} diff --git a/crates/engine/src/addons/verify.rs b/crates/engine/src/addons/verify.rs new file mode 100644 index 0000000..22dda4c --- /dev/null +++ b/crates/engine/src/addons/verify.rs @@ -0,0 +1,88 @@ +//! Signature check of an index entry: canonical text and ed25519 verification (spec D3). +//! +//! The signature covers the id, kind, version and every file's path, hash and size, so a changed +//! byte anywhere in a payload invalidates it. An entry without a verifying signature is refused: +//! there is no "trust me" path. + +use std::fmt::Write as _; + +use ring::signature::{ED25519, UnparsedPublicKey}; + +use super::keys::TRUSTED_KEYS; +use super::model::Entry; +use crate::base64; +use crate::error::{Error, Result}; + +/// First line of every canonical text; bumps with the signing format. +const CANONICAL_HEADER: &str = "notescraft-registry-v1"; +/// Byte length of an ed25519 signature. +const SIGNATURE_BYTES: usize = 64; + +/// Decides whether an entry is signed by a trusted key. +pub trait Verifier: std::fmt::Debug { + /// Returns `Ok` only when the entry's signature is valid. + /// + /// # Errors + /// [`Error::Addon`] naming why it was refused. + fn verify(&self, entry: &Entry) -> Result<()>; +} + +/// The text the owner signs: header, id, kind, version, then every file sorted by path. +#[must_use] +pub fn canonical_text(entry: &Entry) -> String { + let mut files: Vec<_> = entry.files.iter().collect(); + files.sort_by(|a, b| a.path.cmp(&b.path)); + let mut text = format!("{CANONICAL_HEADER}\n{}\n{}\n{}\n", entry.id, entry.kind.as_str(), entry.version); + for file in files { + let _ = writeln!(text, "{}\n{}\n{}", file.path, file.sha256, file.size); + } + text +} + +/// Ed25519 verification against a fixed set of `(key_id, public key)` pairs. +#[derive(Debug, Clone)] +pub struct Ed25519 { + keys: Vec<(String, [u8; 32])>, +} + +impl Ed25519 { + /// Trusts exactly these keys. + #[must_use] + pub fn new(keys: Vec<(String, [u8; 32])>) -> Self { + Self { keys } + } + + /// Trusts the keys embedded in the app ([`TRUSTED_KEYS`]). + #[must_use] + pub fn trusted() -> Self { + Self::new(TRUSTED_KEYS.iter().map(|(id, key)| ((*id).to_owned(), *key)).collect()) + } +} + +impl Verifier for Ed25519 { + fn verify(&self, entry: &Entry) -> Result<()> { + let Some((_, key)) = self.keys.iter().find(|(id, _)| *id == entry.signature.key_id) else { + return Err(Error::Addon(format!("unknown signing key `{}`", entry.signature.key_id))); + }; + let sig = base64::decode(&entry.signature.sig)?; + if sig.len() != SIGNATURE_BYTES { + return Err(Error::Addon("signature has the wrong length".into())); + } + UnparsedPublicKey::new(&ED25519, key) + .verify(canonical_text(entry).as_bytes(), &sig) + .map_err(|_| Error::Addon(format!("signature of `{}` does not verify", entry.id))) + } +} + +/// A verifier for other tasks' tests: accepts everything, or refuses everything. +#[derive(Debug, Clone, Copy)] +pub struct FakeVerifier { + /// Whether [`Verifier::verify`] succeeds. + pub accept: bool, +} + +impl Verifier for FakeVerifier { + fn verify(&self, entry: &Entry) -> Result<()> { + if self.accept { Ok(()) } else { Err(Error::Addon(format!("fake verifier refuses `{}`", entry.id))) } + } +} diff --git a/crates/engine/src/ai/assistant_gate.rs b/crates/engine/src/ai/assistant_gate.rs index e5a271a..2ce6089 100644 --- a/crates/engine/src/ai/assistant_gate.rs +++ b/crates/engine/src/ai/assistant_gate.rs @@ -34,6 +34,11 @@ pub const DELETE_COMMANDS: &[&str] = &[ "server.connect", "server.create_vault", "server.delete_vault", + "registry.check", + "registry.install", + "registry.update", + "registry.remove", + "registry.rollback", ]; /// The core plugin a command belongs to, for the per-plugin tool toggle of @@ -169,6 +174,8 @@ pub const WRITE_COMMANDS: &[&str] = &[ "tags.rename", "tags.merge", "export.pandoc", + "registry.enable", + "registry.disable", ]; /// The explicit read allow-list; everything else is treated as a write. @@ -270,6 +277,9 @@ pub const READ_COMMANDS: &[&str] = &[ "maps.status", "maps.tile", "uri.resolve", + "registry.list", + "registry.info", + "registry.suggest", ]; /// Assistant commands the agent itself may never call: they would let it diff --git a/crates/engine/src/builtin/export_commands.rs b/crates/engine/src/builtin/export_commands.rs index aeaa529..c885097 100644 --- a/crates/engine/src/builtin/export_commands.rs +++ b/crates/engine/src/builtin/export_commands.rs @@ -29,12 +29,29 @@ use crate::params::{ParamKind, ParamSpec}; use crate::session::Session; use crate::spec::{CommandResult, CommandSpec}; -/// The body font, embedded whole. -const INTER_REGULAR: &[u8] = include_bytes!("../../../../assets/fonts/Inter-Regular.ttf"); -/// The bold font (headings, strong text), embedded whole. -const INTER_SEMIBOLD: &[u8] = include_bytes!("../../../../assets/fonts/Inter-SemiBold.ttf"); -/// The code font, embedded whole. -const JETBRAINS_MONO: &[u8] = include_bytes!("../../../../assets/fonts/JetBrainsMono-Regular.ttf"); +/// Bytes of the regular `Inter` font, as shipped. +const INTER_REGULAR_BYTES: usize = 411_640; +/// Bytes of the semibold `Inter` font, as shipped. +const INTER_SEMIBOLD_BYTES: usize = 419_744; +/// Bytes of the `JetBrains Mono` font, as shipped. +const JETBRAINS_MONO_BYTES: usize = 269_984; + +/// The body font, embedded whole. The UI installs the same copy. +/// +/// The bytes live in a public static: a private one is copied by the linker into +/// every LTO partition that reads it, so the binary carried the font twice. +pub static INTER_REGULAR_DATA: [u8; INTER_REGULAR_BYTES] = *include_bytes!("../../../../assets/fonts/Inter-Regular.ttf"); +/// The body font, embedded whole. The UI installs the same copy. +pub const INTER_REGULAR: &[u8] = &INTER_REGULAR_DATA; +/// The bold font (headings, strong text), embedded whole. The UI installs the same copy. +pub static INTER_SEMIBOLD_DATA: [u8; INTER_SEMIBOLD_BYTES] = *include_bytes!("../../../../assets/fonts/Inter-SemiBold.ttf"); +/// The bold font (headings, strong text), embedded whole. The UI installs the same copy. +pub const INTER_SEMIBOLD: &[u8] = &INTER_SEMIBOLD_DATA; +/// The code font, embedded whole. The UI installs the same copy: PDF text has no +/// ligature shaping, so the ligature-free file is valid there too. +pub static JETBRAINS_MONO_DATA: [u8; JETBRAINS_MONO_BYTES] = *include_bytes!("../../../../assets/fonts/JetBrainsMono-NoLigatures.ttf"); +/// The code font, embedded whole. The UI installs the same copy. +pub const JETBRAINS_MONO: &[u8] = &JETBRAINS_MONO_DATA; /// The fonts the exporter embeds. fn fonts() -> FontSet { diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index 0a2fb30..631c703 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -68,6 +68,7 @@ pub fn register_core(reg: &mut Registry) -> Result<()> { crate::prop_edit::commands::register_prop_edit(reg)?; crate::uri::register_uri(reg)?; crate::pandoc::register_pandoc(reg)?; + crate::addons::register_addons(reg)?; Ok(()) } diff --git a/crates/engine/src/error.rs b/crates/engine/src/error.rs index 302edd0..0bc4eeb 100644 --- a/crates/engine/src/error.rs +++ b/crates/engine/src/error.rs @@ -236,6 +236,11 @@ pub enum Error { /// holds an unknown version. #[error("smart folder: {0}")] SmartFolder(String), + + /// An add-on index, entry, file list, signature or store file broke a rule of the registry + /// spec (D2, D3, D4, D7). + #[error("add-on: {0}")] + Addon(String), } impl Error { @@ -276,6 +281,7 @@ impl Error { Self::ReviewLimit { .. } => "limit", Self::StaleRange { .. } => "stale_range", Self::SmartFolder(_) => "smart_folder", + Self::Addon(_) => "addon", } } } diff --git a/crates/engine/src/grammars/mod.rs b/crates/engine/src/grammars/mod.rs index a2704b2..b032496 100644 --- a/crates/engine/src/grammars/mod.rs +++ b/crates/engine/src/grammars/mod.rs @@ -223,8 +223,9 @@ fn save(session: &mut Session, id: &str, grammar: &FetchedGrammar) -> CommandRes pub(crate) fn reload_installed(session: &Session) { let Ok((settings, _)) = load_for(session) else { return }; let Some(dir) = grammar_dir(session) else { return }; - let installed: Vec<(String, String)> = + let mut installed: Vec<(String, String)> = settings.grammars.iter().filter_map(|grammar| Some((grammar.id.clone(), std::fs::read_to_string(dir.join(&grammar.file)).ok()?))).collect(); + installed.extend(crate::addons::installed_grammars(session)); load::set_installed(&installed); } diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 81fd21f..83a77b1 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -27,6 +27,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] pub mod activity; +pub mod addons; mod ai; mod base64; mod bases; @@ -145,6 +146,7 @@ pub use crate::sync::{ pub use crate::uri::{UriAction, format_open_uri}; pub use crate::vault::snapshots::Snapshot; pub use crate::viewers::FileKind; +pub use builtin::export_commands::{INTER_REGULAR, INTER_SEMIBOLD, JETBRAINS_MONO}; pub use notescraft_vault::{FileCount, LARGE_FOLDER_FILES, count_files}; #[cfg(test)] diff --git a/crates/engine/src/vault/session.rs b/crates/engine/src/vault/session.rs index ee68fb1..dc5372d 100644 --- a/crates/engine/src/vault/session.rs +++ b/crates/engine/src/vault/session.rs @@ -77,6 +77,10 @@ pub struct Session { pub(crate) batch: crate::batch::BatchState, /// How `grammar.install` downloads; `curl` unless a test swaps it. pub(crate) grammar_transport: Box, + /// How `registry.*` downloads; `ureq` unless a test swaps it. + pub(crate) addons_transport: Box, + /// How `registry.install` checks signatures; the embedded keys unless a test swaps it. + pub(crate) addons_verifier: Box, } impl Session { @@ -284,6 +288,16 @@ impl Session { self.grammar_transport = transport; } + /// Replaces how `registry.*` downloads (tests; the app keeps `ureq`). + pub fn set_addons_transport(&mut self, transport: Box) { + self.addons_transport = transport; + } + + /// Replaces how `registry.*` checks signatures (tests; the app keeps the embedded keys). + pub fn set_addons_verifier(&mut self, verifier: Box) { + self.addons_verifier = verifier; + } + /// Loads the installed syntax grammars into the highlighter (call once after opening). pub fn load_grammars(&self) { crate::grammars::reload_installed(self); @@ -395,7 +409,29 @@ impl Session { /// Builds the session and fills the indexes. pub(crate) fn with_vault(vault: Vault, config: SessionConfig) -> Self { - let mut session = Self { + let mut session = Self::bare(vault, config); + session.begin_scan(); + session + } + + /// A session for a worker thread that only runs `registry.*` commands: the vault is a + /// sibling of the one the app holds (no second lock), `settings_dir` is the app's, and the + /// vault is not scanned. + #[must_use] + pub fn registry_worker(vault: Vault, settings_dir: Option) -> Self { + let mut session = Self::bare(vault, SessionConfig::default()); + session.settings_dir = settings_dir; + session + } + + /// The folder `use_settings_dir` picked, for a worker to share the app's registry store. + #[must_use] + pub fn picked_settings_dir(&self) -> Option { + self.settings_dir.clone() + } + + fn bare(vault: Vault, config: SessionConfig) -> Self { + Self { git: notescraft_git::Git::open(vault.root()), vault, index: Index::new(), @@ -419,13 +455,13 @@ impl Session { scan: crate::indexing::ScanState::default(), batch: crate::batch::BatchState::default(), grammar_transport: Box::new(notescraft_syntax::fetch::Curl), + addons_transport: Box::new(crate::addons::Ureq::new()), + addons_verifier: Box::new(crate::addons::TrustedVerifier), sync: None, rooms: crate::live::rooms::Rooms::new(), sync_attachments: true, calendar_cache: crate::calendar::CalendarCache::default(), - }; - session.begin_scan(); - session + } } } diff --git a/crates/engine/tests/addons_commands.rs b/crates/engine/tests/addons_commands.rs new file mode 100644 index 0000000..b1c8cc8 --- /dev/null +++ b/crates/engine/tests/addons_commands.rs @@ -0,0 +1,187 @@ +//! The `registry.*` commands through the registry: list, check, install, update, remove, +//! rollback, enable, the read-only refusal and the assistant gate classes. + +mod addons_support; +mod common; + +use std::sync::Arc; + +use addons_support::{Shared, TINY, WASM, entry, index_json, manifest, serve}; +use common::{TempVault, temp_config, temp_vault}; +use notescraft_engine::addons::{FakeTransport, FakeVerifier, INDEX_URL, Response}; +use notescraft_engine::{AssistantGate, AssistantSettings, Decision, Error, GateSource, Registry, Session, register_core}; +use serde_json::{Value, json}; + +const GRAMMAR: &str = "twig.sublime-syntax"; + +struct Fixture { + reg: Registry, + session: Session, + net: Arc, + vault_path: std::path::PathBuf, + _guards: (TempVault, TempVault), +} + +fn script(version: &str, bytes: &[u8]) -> FakeTransport { + let doc = index_json(&[entry("grammar", "twig", version, &[(GRAMMAR, bytes)], &[])], &[]); + let net = FakeTransport::new().with_response(INDEX_URL, Ok(Response { status: 200, etag: Some("\"e\"".into()), bytes: doc })); + serve(net, "grammar", "twig", version, &[(GRAMMAR, bytes)]) +} + +fn fixture(name: &str, net: FakeTransport) -> Fixture { + let (vault, config) = (temp_vault(name), temp_config(name)); + let mut session = Session::open(vault.path()).unwrap(); + session.use_settings_dir(config.path()); + let net = Arc::new(net); + session.set_addons_transport(Box::new(Shared(net.clone()))); + session.set_addons_verifier(Box::new(FakeVerifier { accept: true })); + let mut reg = Registry::new(); + register_core(&mut reg).unwrap(); + Fixture { reg, session, net, vault_path: vault.path().to_path_buf(), _guards: (vault, config) } +} + +impl Fixture { + fn run(&mut self, id: &str, params: impl Into) -> Result { + self.reg.run(id, &mut self.session, ¶ms.into()) + } + + fn ok(&mut self, id: &str, params: impl Into) -> Value { + self.run(id, params).unwrap_or_else(|e| panic!("{id}: {e}")) + } + + fn code(&mut self, id: &str, params: impl Into) -> String { + match self.run(id, params) { + Err(Error::Command { source, .. }) => source.code, + other => panic!("{id}: expected a command error, got {other:?}"), + } + } +} + +#[test] +fn list_is_empty_until_a_check_then_joins_the_state() { + let mut f = fixture("list", script("1.0.0", TINY.as_bytes())); + assert_eq!(f.ok("registry.list", json!({}))["cached"], false); + assert!(f.net.asked().is_empty(), "list never touches the network"); + let checked = f.ok("registry.check", json!({})); + assert_eq!((checked["entries"].clone(), checked["updates"].clone()), (json!(1), json!(0))); + let rows = f.ok("registry.list", json!({})); + assert_eq!(rows["entries"][0]["state"], "available"); + f.ok("registry.install", json!({ "kind": "grammar", "id": "twig" })); + assert_eq!(f.ok("registry.list", json!({ "state": "installed" }))["entries"][0]["installed_version"], "1.0.0"); + assert_eq!(f.ok("registry.list", json!({ "state": "available" }))["entries"], json!([])); + assert_eq!(f.ok("registry.list", json!({ "query": "TW" }))["entries"].as_array().map(Vec::len), Some(1)); + assert_eq!(f.ok("registry.info", json!({ "kind": "grammar", "id": "twig" }))["files"][0]["path"], GRAMMAR); + assert_eq!(f.ok("registry.suggest", json!({ "label": "Twig" }))["entries"][0]["id"], "twig"); + assert_eq!(f.ok("registry.suggest", json!({ "label": "nope" }))["entries"], json!([])); +} + +#[test] +fn install_update_rollback_remove_run_through_the_registry() { + let mut f = fixture("flow", script("1.0.0", TINY.as_bytes())); + f.ok("registry.check", json!({})); + let done = f.ok("registry.install", json!({ "kind": "grammar", "id": "twig" })); + assert_eq!(done["version"], "1.0.0"); + let newer = format!("{TINY}# 1.1\n"); + // A new index with 1.1.0 appears after another check. + let net = script("1.1.0", newer.as_bytes()); + f.session.set_addons_transport(Box::new(Shared(Arc::new(net)))); + assert_eq!(f.ok("registry.check", json!({}))["updates"], 1); + assert_eq!(f.ok("registry.list", json!({ "state": "updates" }))["entries"][0]["state"], "update"); + let all = f.ok("registry.update", json!({})); + assert_eq!(all["updated"][0]["previous"], "1.0.0"); + assert_eq!(f.ok("registry.rollback", json!({ "kind": "grammar", "id": "twig" }))["version"], "1.0.0"); + f.ok("registry.remove", json!({ "kind": "grammar", "id": "twig" })); + assert_eq!(f.code("registry.remove", json!({ "kind": "grammar", "id": "twig" })), "not_found"); + assert_eq!(f.code("registry.install", json!({ "kind": "grammar", "id": "nope" })), "not_found"); + assert_eq!(f.code("registry.install", json!({ "kind": "exe", "id": "twig" })), "invalid_params"); +} + +#[test] +fn install_without_a_trusted_key_is_a_clear_addon_error() { + let mut f = fixture("nokey", script("1.0.0", TINY.as_bytes())); + f.ok("registry.check", json!({})); + let config = temp_config("nokey-default"); + let vault = temp_vault("nokey-default"); + let mut plain = Session::open(vault.path()).unwrap(); + plain.use_settings_dir(config.path()); + plain.set_addons_transport(Box::new(Shared(f.net.clone()))); + f.session = plain; + f.ok("registry.check", json!({})); + let error = f.run("registry.install", json!({ "kind": "grammar", "id": "twig" })).unwrap_err().to_string(); + assert!(error.contains("no trusted registry key"), "{error}"); + assert_eq!(f.code("registry.install", json!({ "kind": "grammar", "id": "twig" })), "addon"); + f.session.set_addons_verifier(Box::new(notescraft_engine::addons::Ed25519::new(vec![]))); + assert_eq!(f.code("registry.install", json!({ "kind": "grammar", "id": "twig" })), "registry_refused"); +} + +#[test] +fn offline_check_leaves_the_cache() { + let mut f = fixture("offline", script("1.0.0", TINY.as_bytes())); + f.ok("registry.check", json!({})); + f.session.set_addons_transport(Box::new(Shared(Arc::new(FakeTransport::new())))); + assert_eq!(f.code("registry.check", json!({})), "offline"); + assert_eq!(f.ok("registry.list", json!({}))["entries"].as_array().map(Vec::len), Some(1)); +} + +#[test] +fn install_before_any_check_says_so() { + let mut f = fixture("nocache", script("1.0.0", TINY.as_bytes())); + assert_eq!(f.code("registry.install", json!({ "kind": "grammar", "id": "twig" })), "not_found"); +} + +#[test] +fn enable_and_disable_write_the_vault_file() { + let text = manifest("dice", "1.0.0", &[]); + let files: &[(&str, &[u8])] = &[("manifest.json", text.as_bytes()), ("plugin.wasm", WASM)]; + let doc = index_json(&[entry("plugin", "dice", "1.0.0", files, &[])], &[]); + let net = serve(FakeTransport::new().with(INDEX_URL, &doc), "plugin", "dice", "1.0.0", files); + let mut f = fixture("enable", net); + assert_eq!(f.code("registry.enable", json!({ "id": "dice" })), "not_found", "not installed yet"); + f.ok("registry.check", json!({})); + f.ok("registry.install", json!({ "kind": "plugin", "id": "dice" })); + f.ok("registry.enable", json!({ "id": "dice" })); + let file = std::fs::read_to_string(f.vault_path.join(".notescraft/registry.json")).unwrap(); + assert!(file.contains("plugin:dice"), "{file}"); + f.ok("registry.disable", json!({ "id": "dice" })); + let file = std::fs::read_to_string(f.vault_path.join(".notescraft/registry.json")).unwrap(); + assert!(!file.contains("plugin:dice"), "{file}"); +} + +#[test] +fn a_read_only_session_refuses_the_mutating_ids() { + let vault = temp_vault("ro"); + let mut session = Session::open_read_only(vault.path()).unwrap(); + let mut reg = Registry::new(); + register_core(&mut reg).unwrap(); + for id in ["registry.check", "registry.install", "registry.update", "registry.remove", "registry.rollback", "registry.enable", "registry.disable"] { + let params = json!({ "kind": "grammar", "id": "twig" }); + assert!(matches!(reg.run(id, &mut session, ¶ms), Err(Error::Disabled(_))), "{id}"); + } + for id in ["registry.list", "registry.suggest"] { + assert!(!matches!(reg.run(id, &mut session, &json!({ "label": "x" })), Err(Error::Disabled(_))), "{id}"); + } +} + +#[test] +fn the_gate_classes_hold() { + let f = fixture("gate", FakeTransport::new()); + let mut session = f.session; + let decide = |session: &Session, id: &str| AssistantGate::check(GateSource::Assistant, session, id, &Value::Null); + for id in ["registry.list", "registry.info", "registry.suggest"] { + assert!(matches!(decide(&session, id), Decision::Allow), "{id}"); + } + for id in ["registry.enable", "registry.disable", "registry.check", "registry.install", "registry.update", "registry.remove", "registry.rollback"] { + assert!(matches!(decide(&session, id), Decision::Deny(_)), "{id} needs allow_write"); + } + session.set_assistant_settings(AssistantSettings { allow_write: true, allow_delete: false, review_writes: false, ..AssistantSettings::default() }); + for id in ["registry.enable", "registry.disable"] { + assert!(matches!(decide(&session, id), Decision::Allow), "{id}"); + } + for id in ["registry.check", "registry.install", "registry.update", "registry.remove", "registry.rollback"] { + assert!(matches!(decide(&session, id), Decision::Deny(_)), "{id} needs allow_delete"); + } + session.set_assistant_settings(AssistantSettings { allow_write: true, allow_delete: true, review_writes: false, ..AssistantSettings::default() }); + for id in ["registry.check", "registry.install", "registry.update", "registry.remove", "registry.rollback"] { + assert!(matches!(decide(&session, id), Decision::Allow), "{id}"); + } +} diff --git a/crates/engine/tests/addons_install.rs b/crates/engine/tests/addons_install.rs new file mode 100644 index 0000000..6532ed1 --- /dev/null +++ b/crates/engine/tests/addons_install.rs @@ -0,0 +1,221 @@ +//! Install, update, remove and rollback against a fake transport and a temp store. + +mod addons_support; + +use addons_support::{TINY, WASM, entry, index, manifest, serve}; +use notescraft_engine::addons::{ + Ctx, Ed25519, FakeTransport, FakeVerifier, Index, Kind, Seed, Source, Store, Version, fetch_files, install, key, remove, rollback, update, write_tree, +}; + +const GRAMMAR: &str = "twig.sublime-syntax"; + +struct Env { + dir: std::path::PathBuf, + store: Store, +} + +impl Env { + fn new(name: &str) -> Self { + let dir = std::env::temp_dir().join(format!("notescraft-addons-install-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let store = Store::open(&dir).unwrap(); + Self { dir, store } + } + + fn ctx<'a>(&'a self, transport: &'a FakeTransport, verifier: &'a FakeVerifier) -> Ctx<'a> { + Ctx { store: &self.store, transport, verifier, app_version: v(0, 5, 0), seed: None } + } +} + +impl Drop for Env { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +fn v(major: u16, minor: u16, patch: u16) -> Version { + Version { major, minor, patch } +} + +const OK: FakeVerifier = FakeVerifier { accept: true }; + +fn grammar_index(version: &str, bytes: &[u8]) -> Index { + index(&[entry("grammar", "twig", version, &[(GRAMMAR, bytes)], &[])], &[]) +} + +fn grammar_net(version: &str, bytes: &[u8]) -> FakeTransport { + serve(FakeTransport::new(), "grammar", "twig", version, &[(GRAMMAR, bytes)]) +} + +fn error_of(result: notescraft_engine::Result) -> String { + result.expect_err("must fail").to_string() +} + +#[test] +fn a_clean_install_commits_state_and_files() { + let env = Env::new("clean"); + let report = install(&env.ctx(&grammar_net("1.0.0", TINY.as_bytes()), &OK), &grammar_index("1.0.0", TINY.as_bytes()), Kind::Grammar, "twig", None).unwrap(); + assert_eq!((report.version, report.previous, report.source), (v(1, 0, 0), None, Source::Registry)); + let state = env.store.load().unwrap(); + assert_eq!(state.installed[&key(Kind::Grammar, "twig")].version, v(1, 0, 0)); + let file = env.store.version_dir(Kind::Grammar, "twig", v(1, 0, 0)).unwrap().join(GRAMMAR); + assert_eq!(std::fs::read(file).unwrap(), TINY.as_bytes()); +} + +#[test] +fn a_bad_download_changes_nothing() { + let env = Env::new("bad"); + let idx = grammar_index("1.0.0", TINY.as_bytes()); + let short = grammar_net("1.0.0", &TINY.as_bytes()[..10]); + assert!(install(&env.ctx(&short, &OK), &idx, Kind::Grammar, "twig", None).is_err()); + let wrong = grammar_net("1.0.0", &TINY.to_uppercase().into_bytes()); + assert!(error_of(install(&env.ctx(&wrong, &OK), &idx, Kind::Grammar, "twig", None)).contains("sha256")); + let long = grammar_net("1.0.0", &[TINY.as_bytes(), b"x"].concat()); + assert!(install(&env.ctx(&long, &OK), &idx, Kind::Grammar, "twig", None).is_err()); + assert!(error_of(install(&env.ctx(&FakeTransport::new(), &OK), &idx, Kind::Grammar, "twig", None)).contains("offline")); + assert!(env.store.load().unwrap().installed.is_empty()); + assert_eq!(env.store.gc(&env.store.load().unwrap()).unwrap(), 0, "no folder was written"); +} + +#[test] +fn a_content_check_failure_writes_nothing() { + let env = Env::new("content"); + let junk = b"nope: [".as_slice(); + let idx = grammar_index("1.0.0", junk); + assert!(install(&env.ctx(&grammar_net("1.0.0", junk), &OK), &idx, Kind::Grammar, "twig", None).is_err()); + assert!(!env.store.version_dir(Kind::Grammar, "twig", v(1, 0, 0)).unwrap().exists()); +} + +#[test] +fn the_gates_refuse_before_any_download() { + let env = Env::new("gates"); + let net = grammar_net("1.0.0", TINY.as_bytes()); + let idx = grammar_index("1.0.0", TINY.as_bytes()); + assert!(error_of(install(&env.ctx(&net, &FakeVerifier { accept: false }), &idx, Kind::Grammar, "twig", None)).contains("refuses")); + let real = Ed25519::new(vec![]); + let ctx = Ctx { store: &env.store, transport: &net, verifier: &real, app_version: v(0, 5, 0), seed: None }; + assert!(error_of(install(&ctx, &idx, Kind::Grammar, "twig", None)).contains("unknown signing key")); + let old_app = Ctx { app_version: v(0, 0, 9), ..env.ctx(&net, &OK) }; + assert!(error_of(install(&old_app, &idx, Kind::Grammar, "twig", None)).contains("incompatible")); + let revoked = index(&[entry("grammar", "twig", "1.0.0", &[(GRAMMAR, TINY.as_bytes())], &[])], &[("twig", "1.0.0")]); + assert!(error_of(install(&env.ctx(&net, &OK), &revoked, Kind::Grammar, "twig", None)).contains("revoked")); + assert!(error_of(install(&env.ctx(&net, &OK), &idx, Kind::Grammar, "missing", None)).contains("not_found")); + assert!(error_of(install(&env.ctx(&net, &OK), &idx, Kind::Grammar, "twig", Some(v(2, 0, 0)))).contains("not_found")); + let font = index(&[entry("font", "f", "1.0.0", &[("f.ttf", &[0, 1, 0, 0])], &[])], &[]); + assert!(error_of(install(&env.ctx(&net, &OK), &font, Kind::Font, "f", None)).contains("no use")); + assert!(net.asked().is_empty(), "nothing was fetched: {:?}", net.asked()); + assert!(env.store.load().unwrap().installed.is_empty()); +} + +#[test] +fn a_crash_before_the_state_flip_leaves_the_old_state_and_gc_cleans_up() { + let env = Env::new("crash"); + let (idx, net) = (grammar_index("1.0.0", TINY.as_bytes()), grammar_net("1.0.0", TINY.as_bytes())); + let ctx = env.ctx(&net, &OK); + let entry = idx.find(Kind::Grammar, "twig").unwrap(); + let staged = fetch_files(&ctx, entry).unwrap(); + write_tree(&env.store, entry, &staged).unwrap(); + let dir = env.store.version_dir(Kind::Grammar, "twig", v(1, 0, 0)).unwrap(); + assert!(dir.is_dir()); + assert!(env.store.load().unwrap().installed.is_empty(), "state was never flipped"); + assert_eq!(env.store.gc(&env.store.load().unwrap()).unwrap(), 1); + assert!(!dir.exists()); + write_tree(&env.store, entry, &staged).unwrap(); + write_tree(&env.store, entry, &staged).unwrap(); +} + +#[test] +fn update_keeps_previous_and_rollback_swaps() { + let env = Env::new("update"); + install(&env.ctx(&grammar_net("1.0.0", TINY.as_bytes()), &OK), &grammar_index("1.0.0", TINY.as_bytes()), Kind::Grammar, "twig", None).unwrap(); + let newer = format!("{TINY}# 1.1\n"); + let (idx, net) = (grammar_index("1.1.0", newer.as_bytes()), grammar_net("1.1.0", newer.as_bytes())); + let report = update(&env.ctx(&net, &OK), &idx, Kind::Grammar, "twig").unwrap(); + assert_eq!((report.version, report.previous), (v(1, 1, 0), Some(v(1, 0, 0)))); + assert!(env.store.version_dir(Kind::Grammar, "twig", v(1, 0, 0)).unwrap().is_dir(), "the old version stays for rollback"); + assert!(error_of(update(&env.ctx(&net, &OK), &idx, Kind::Grammar, "twig")).contains("already installed")); + let back = rollback(&env.store, Kind::Grammar, "twig").unwrap(); + assert_eq!((back.version, back.previous), (v(1, 0, 0), Some(v(1, 1, 0)))); + let forward = rollback(&env.store, Kind::Grammar, "twig").unwrap(); + assert_eq!(forward.version, v(1, 1, 0)); + let old = (grammar_index("1.0.0", TINY.as_bytes()), grammar_net("1.0.0", TINY.as_bytes())); + assert!(error_of(install(&env.ctx(&old.1, &OK), &old.0, Kind::Grammar, "twig", None)).contains("rollback"), "a downgrade is never an install"); +} + +#[test] +fn update_and_rollback_need_an_installed_add_on() { + let env = Env::new("needs"); + let (idx, net) = (grammar_index("1.0.0", TINY.as_bytes()), grammar_net("1.0.0", TINY.as_bytes())); + assert!(error_of(update(&env.ctx(&net, &OK), &idx, Kind::Grammar, "twig")).contains("not installed")); + assert!(error_of(rollback(&env.store, Kind::Grammar, "twig")).contains("not installed")); + install(&env.ctx(&net, &OK), &idx, Kind::Grammar, "twig", None).unwrap(); + assert!(error_of(rollback(&env.store, Kind::Grammar, "twig")).contains("earlier")); +} + +#[test] +fn remove_commits_first_then_deletes_the_files() { + let env = Env::new("remove"); + install(&env.ctx(&grammar_net("1.0.0", TINY.as_bytes()), &OK), &grammar_index("1.0.0", TINY.as_bytes()), Kind::Grammar, "twig", None).unwrap(); + remove(&env.store, Kind::Grammar, "twig").unwrap(); + assert!(env.store.load().unwrap().installed.is_empty()); + assert!(!env.store.version_dir(Kind::Grammar, "twig", v(1, 0, 0)).unwrap().exists()); + assert!(error_of(remove(&env.store, Kind::Grammar, "twig")).contains("not_found")); +} + +#[derive(Debug)] +struct Fixed(Vec); + +impl Seed for Fixed { + fn read(&self, _: Kind, _: &str, _: Version, _: &str) -> Option> { + Some(self.0.clone()) + } +} + +#[test] +fn a_seed_installs_offline_and_a_tampered_seed_is_ignored() { + let env = Env::new("seed"); + let idx = grammar_index("1.0.0", TINY.as_bytes()); + let seed = Fixed(TINY.as_bytes().to_vec()); + let offline = FakeTransport::new(); + let ctx = Ctx { seed: Some(&seed), ..env.ctx(&offline, &OK) }; + let report = install(&ctx, &idx, Kind::Grammar, "twig", None).unwrap(); + assert_eq!(report.source, Source::Seed); + assert_eq!(offline.asked().len(), 0); + let env = Env::new("seed-bad"); + let tampered = Fixed(b"tampered".to_vec()); + let ctx = Ctx { seed: Some(&tampered), ..env.ctx(&offline, &OK) }; + assert!(error_of(install(&ctx, &idx, Kind::Grammar, "twig", None)).contains("offline"), "falls back to the network, which is down"); +} + +#[test] +fn a_plugin_installs_with_its_manifest_checked() { + let env = Env::new("plugin"); + let text = manifest("dice", "1.0.0", &["vault.read"]); + let files: &[(&str, &[u8])] = &[("manifest.json", text.as_bytes()), ("plugin.wasm", WASM)]; + let idx = index(&[entry("plugin", "dice", "1.0.0", files, &["vault.read"])], &[]); + let net = serve(FakeTransport::new(), "plugin", "dice", "1.0.0", files); + install(&env.ctx(&net, &OK), &idx, Kind::Plugin, "dice", None).unwrap(); + let dir = env.store.version_dir(Kind::Plugin, "dice", v(1, 0, 0)).unwrap(); + assert!(dir.join("manifest.json").is_file() && dir.join("plugin.wasm").is_file()); +} + +#[test] +fn the_installed_table_cap_is_enforced() { + let env = Env::new("cap"); + let mut state = env.store.load().unwrap(); + for n in 0..512 { + state.installed.insert( + format!("grammar:g{n}"), + notescraft_engine::addons::Installed { + version: v(1, 0, 0), + previous: None, + source: Source::Registry, + installed_at: "t".into(), + sha256_of_files: "e".repeat(64), + }, + ); + } + env.store.commit(&state).unwrap(); + let (idx, net) = (grammar_index("1.0.0", TINY.as_bytes()), grammar_net("1.0.0", TINY.as_bytes())); + assert!(error_of(install(&env.ctx(&net, &OK), &idx, Kind::Grammar, "twig", None)).contains("limit")); +} diff --git a/crates/engine/tests/addons_kinds.rs b/crates/engine/tests/addons_kinds.rs new file mode 100644 index 0000000..4ddddc2 --- /dev/null +++ b/crates/engine/tests/addons_kinds.rs @@ -0,0 +1,81 @@ +//! Per-kind content checks of staged add-on files. + +mod addons_support; + +use addons_support::{THEME, TINY, WASM, entry, index, manifest}; +use notescraft_engine::addons::{Entry, Kind, check_content}; + +type Staged = Vec<(String, Vec)>; + +fn parsed(kind: &str, id: &str, files: &[(&str, &[u8])], permissions: &[&str]) -> Entry { + index(&[entry(kind, id, "1.0.0", files, permissions)], &[]).entries.remove(0) +} + +fn run(kind: Kind, id: &str, files: &[(&str, &[u8])], permissions: &[&str]) -> Result<(), String> { + let staged: Staged = files.iter().map(|(p, b)| ((*p).to_owned(), b.to_vec())).collect(); + check_content(kind, &parsed(kind.as_str(), id, files, permissions), &staged).map_err(|e| e.to_string()) +} + +#[test] +fn grammar() { + assert_eq!(run(Kind::Grammar, "tiny", &[("tiny.sublime-syntax", TINY.as_bytes())], &[]), Ok(())); + assert!(run(Kind::Grammar, "tiny", &[("tiny.sublime-syntax", b"")], &[]).is_err()); + assert!(run(Kind::Grammar, "tiny", &[("tiny.sublime-syntax", &[0xff, 0xfe])], &[]).is_err()); +} + +#[test] +fn theme() { + assert_eq!(run(Kind::Theme, "mine", &[("mine.toml", THEME.as_bytes())], &[]), Ok(())); + let wrong_id = run(Kind::Theme, "other", &[("other.toml", THEME.as_bytes())], &[]); + assert!(wrong_id.unwrap_err().contains("entry")); + let builtin = notescraft_themes::builtin_ids()[0]; + let reuse = format!("id = \"{builtin}\"\nname = \"X\"\nvariant = \"dark\"\n"); + assert!(run(Kind::Theme, builtin, &[("t.toml", reuse.as_bytes())], &[]).unwrap_err().contains("built-in")); + let bad_value = "id = \"mine\"\nname = \"Mine\"\nvariant = \"dark\"\n[colors]\nbackground = \"nope\"\n"; + assert!(run(Kind::Theme, "mine", &[("mine.toml", bad_value.as_bytes())], &[]).unwrap_err().contains("rejected")); +} + +#[test] +fn plugin() { + let good = manifest("dice", "1.0.0", &["vault.read"]); + let files: &[(&str, &[u8])] = &[("manifest.json", good.as_bytes()), ("plugin.wasm", WASM)]; + assert_eq!(run(Kind::Plugin, "dice", files, &["vault.read"]), Ok(())); + let greedy = manifest("dice", "1.0.0", &["vault.read", "vault.write"]); + let files: &[(&str, &[u8])] = &[("manifest.json", greedy.as_bytes()), ("plugin.wasm", WASM)]; + assert!(run(Kind::Plugin, "dice", files, &["vault.read"]).unwrap_err().contains("permissions")); + let other = manifest("other", "1.0.0", &[]); + assert!(run(Kind::Plugin, "dice", &[("manifest.json", other.as_bytes()), ("plugin.wasm", WASM)], &[]).is_err()); + let old = manifest("dice", "0.9.0", &[]); + assert!(run(Kind::Plugin, "dice", &[("manifest.json", old.as_bytes()), ("plugin.wasm", WASM)], &[]).is_err()); + let plain = manifest("dice", "1.0.0", &[]); + assert!(run(Kind::Plugin, "dice", &[("manifest.json", plain.as_bytes()), ("plugin.wasm", b"MZ not wasm")], &[]).is_err()); + assert!(run(Kind::Plugin, "dice", &[("plugin.wasm", WASM)], &[]).is_err()); +} + +#[test] +fn font_and_snippet() { + assert_eq!(run(Kind::Font, "f", &[("f.ttf", &[0, 1, 0, 0, 9])], &[]), Ok(())); + assert_eq!(run(Kind::Font, "f", &[("f.otf", b"OTTO..")], &[]), Ok(())); + assert!(run(Kind::Font, "f", &[("f.ttf", b"GIF89a")], &[]).is_err()); + assert_eq!(run(Kind::Snippet, "s", &[("s.md", b"# hi")], &[]), Ok(())); + assert!(run(Kind::Snippet, "s", &[("s.md", b"a\0b")], &[]).is_err()); + assert!(run(Kind::Snippet, "s", &[("s.md", &[0xff])], &[]).is_err()); +} + +#[test] +fn pack_checks_each_file_by_folder() { + let ok: &[(&str, &[u8])] = &[("grammars/tiny.sublime-syntax", TINY.as_bytes()), ("themes/mine.toml", THEME.as_bytes()), ("icons/a.json", b"{}")]; + assert_eq!(run(Kind::Pack, "p", ok, &[]), Ok(())); + let bad: &[(&str, &[u8])] = &[("grammars/tiny.sublime-syntax", TINY.as_bytes()), ("icons/a.json", b"{nope")]; + assert!(run(Kind::Pack, "p", bad, &[]).is_err(), "one bad file fails the pack"); +} + +#[test] +fn the_staged_files_must_be_the_listed_files() { + let listed: &[(&str, &[u8])] = &[("tiny.sublime-syntax", TINY.as_bytes())]; + let entry = parsed("grammar", "tiny", listed, &[]); + let swapped: Staged = vec![("evil.sublime-syntax".into(), TINY.as_bytes().to_vec())]; + assert!(check_content(Kind::Grammar, &entry, &swapped).is_err()); + let doubled: Staged = vec![("tiny.sublime-syntax".into(), TINY.as_bytes().to_vec()), ("tiny.sublime-syntax".into(), TINY.as_bytes().to_vec())]; + assert!(check_content(Kind::Grammar, &entry, &doubled).is_err()); +} diff --git a/crates/engine/tests/addons_net.rs b/crates/engine/tests/addons_net.rs new file mode 100644 index 0000000..f257e78 --- /dev/null +++ b/crates/engine/tests/addons_net.rs @@ -0,0 +1,112 @@ +//! Bounded download and index fetch of the add-on registry, against a fake transport. + +mod addons_support; + +use addons_support::{entry, index_json, url}; +use notescraft_engine::addons::{FakeTransport, File, INDEX_URL, IndexCache, Response, Transport, Ureq, download_file, fetch_index, split_code}; +use notescraft_syntax::validate::sha256_hex; + +const BODY: &[u8] = b"hello grammar"; +const CAP: u64 = 512 * 1024; + +fn file(link: &str) -> File { + File { path: "a.txt".into(), size: BODY.len() as u64, sha256: sha256_hex(BODY), url: link.into() } +} + +fn good() -> String { + url("grammar", "x", "1.0.0", "a.txt") +} + +fn refused(link: &str) -> String { + let transport = FakeTransport::new().with(link, BODY); + download_file(&transport, &file(link), CAP).expect_err("refused").to_string() +} + +#[test] +fn a_listed_file_downloads() { + let transport = FakeTransport::new().with(&good(), BODY); + assert_eq!(download_file(&transport, &file(&good()), CAP).unwrap(), BODY); +} + +#[test] +fn hostile_links_never_reach_the_transport() { + for link in [ + "https://evil.example.com/a.txt".to_owned(), + good().replacen("https", "http", 1), + good().replacen("https://", "https://user:pw@", 1), + good().replacen("git.marcospaulo.dev.br", "127.0.0.1", 1), + format!("{}?x=1", good()), + ] { + let transport = FakeTransport::new().with(&link, BODY); + assert!(download_file(&transport, &file(&link), CAP).is_err(), "{link}"); + assert!(transport.asked().is_empty(), "{link} was fetched"); + } + assert!(refused("https://evil.example.com/a.txt").contains("registry")); +} + +#[test] +fn a_redirect_status_is_a_failure() { + let transport = FakeTransport::new().with_response(&good(), Ok(Response { status: 302, etag: None, bytes: Vec::new() })); + let error = download_file(&transport, &file(&good()), CAP).expect_err("redirect").to_string(); + assert!(error.contains("download_failed"), "{error}"); +} + +#[test] +fn a_body_one_byte_too_long_or_short_is_refused() { + let mut longer = BODY.to_vec(); + longer.push(b'!'); + let transport = FakeTransport::new().with(&good(), &longer); + assert!(download_file(&transport, &file(&good()), CAP).is_err()); + let transport = FakeTransport::new().with(&good(), &BODY[..BODY.len() - 1]); + assert!(download_file(&transport, &file(&good()), CAP).is_err()); +} + +#[test] +fn a_wrong_hash_or_a_size_over_the_cap_is_refused() { + let transport = FakeTransport::new().with(&good(), b"HELLO grammar"); + assert!(download_file(&transport, &file(&good()), CAP).unwrap_err().to_string().contains("sha256")); + let transport = FakeTransport::new().with(&good(), BODY); + assert!(download_file(&transport, &file(&good()), 3).unwrap_err().to_string().contains("cap")); +} + +#[test] +fn no_network_maps_to_the_offline_code() { + let error = download_file(&FakeTransport::new(), &file(&good()), CAP).unwrap_err().to_string(); + let message = error.strip_prefix("add-on: ").expect("an add-on error"); + assert_eq!(split_code(message).0, "offline"); + assert_eq!(split_code("a gate failed").0, "registry_refused"); +} + +#[test] +fn the_real_transport_refuses_plain_http_without_a_request() { + let error = Ureq::new().get("http://git.marcospaulo.dev.br/x", 10, None).expect_err("http").to_string(); + assert!(error.contains("https"), "{error}"); +} + +#[test] +fn a_304_keeps_the_cache_and_a_failed_fetch_changes_nothing() { + let doc = index_json(&[entry("grammar", "x", "1.0.0", &[("a.txt", BODY)], &[])], &[]); + let first = FakeTransport::new().with_response(INDEX_URL, Ok(Response { status: 200, etag: Some("\"v1\"".into()), bytes: doc.clone() })); + let cache = fetch_index(&first, None, 100).unwrap(); + assert_eq!((cache.fetched_at, cache.etag.as_deref()), (100, Some("\"v1\""))); + let again = FakeTransport::new().with_response(INDEX_URL, Ok(Response { status: 304, etag: None, bytes: Vec::new() })); + let kept = fetch_index(&again, Some(&cache), 200).unwrap(); + assert_eq!((kept.fetched_at, kept.body.as_str(), kept.etag.as_deref()), (200, cache.body.as_str(), Some("\"v1\""))); + let down = fetch_index(&FakeTransport::new(), Some(&cache), 300).expect_err("offline").to_string(); + assert!(down.contains("offline"), "{down}"); + let junk = FakeTransport::new().with(INDEX_URL, b"not json"); + assert!(fetch_index(&junk, Some(&cache), 300).is_err()); +} + +#[test] +fn the_cache_round_trips_on_disk() { + let dir = std::env::temp_dir().join(format!("notescraft-addons-net-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + assert_eq!(IndexCache::load(&dir).unwrap(), None); + let cache = IndexCache { fetched_at: 5, etag: None, body: String::from_utf8(index_json(&[], &[])).unwrap() }; + cache.save(&dir).unwrap(); + assert_eq!(IndexCache::load(&dir).unwrap(), Some(cache)); + std::fs::write(dir.join("index-cache.json"), b"{").unwrap(); + assert!(IndexCache::load(&dir).is_err()); + std::fs::remove_dir_all(&dir).unwrap(); +} diff --git a/crates/engine/tests/addons_state.rs b/crates/engine/tests/addons_state.rs new file mode 100644 index 0000000..467e361 --- /dev/null +++ b/crates/engine/tests/addons_state.rs @@ -0,0 +1,143 @@ +//! State file, per-vault file and store layout of the add-on registry. + +mod common; + +use notescraft_engine::addons::{Installed, Kind, Source, State, Store, VaultFile, Version, key}; + +fn v(major: u16, minor: u16, patch: u16) -> Version { + Version { major, minor, patch } +} + +fn row(version: Version, previous: Option) -> Installed { + Installed { version, previous, source: Source::Registry, installed_at: "2026-10-21T12:00:00Z".into(), sha256_of_files: "e".repeat(64) } +} + +fn sample() -> State { + let mut state = State::default(); + state.installed.insert(key(Kind::Grammar, "twig"), row(v(1, 1, 0), Some(v(1, 0, 0)))); + state.installed.insert(key(Kind::Plugin, "dice"), row(v(0, 1, 0), None)); + state +} + +#[test] +fn state_round_trip() { + let state = sample(); + assert_eq!(State::parse(state.to_json().unwrap().as_bytes()).unwrap(), state); +} + +#[test] +fn state_rejections() { + assert!(State::parse(b"nope").is_err()); + assert!(State::parse(br#"{"version":2,"installed":{}}"#).is_err()); + assert!(State::parse(br#"{"installed":{}}"#).is_err()); + let good = |k: &str| { + format!(r#"{{"version":1,"installed":{{"{k}":{{"version":"1.0.0","source":"seed","installed_at":"t","sha256_of_files":"{}"}}}}}}"#, "e".repeat(64)) + }; + assert!(State::parse(good("grammar:twig").as_bytes()).is_ok()); + for bad in ["twig", "exe:twig", "grammar:../x", "grammar:Twig", "grammar:"] { + assert!(State::parse(good(bad).as_bytes()).is_err(), "{bad}"); + } + assert!(State::parse(good("grammar:a").replace("1.0.0", "1.0").as_bytes()).is_err()); + assert!(State::parse(good("grammar:a").replace("seed", "magic").as_bytes()).is_err()); + let mut big = State::default(); + for i in 0..513 { + big.installed.insert(key(Kind::Theme, &format!("t{i}")), row(v(1, 0, 0), None)); + } + assert!(big.to_json().is_err()); + let rows: Vec = + (0..513).map(|i| format!(r#""theme:t{i}":{{"version":"1.0.0","source":"seed","installed_at":"t","sha256_of_files":"{}"}}"#, "e".repeat(64))).collect(); + assert!(State::parse(format!(r#"{{"version":1,"installed":{{{}}}}}"#, rows.join(",")).as_bytes()).is_err()); +} + +#[test] +fn vault_file_round_trip_and_caps() { + let file = VaultFile { enabled: vec!["plugin:dice".into()], wanted: vec!["grammar:twig".into(), "theme:x".into()] }; + assert_eq!(VaultFile::parse(file.to_json().unwrap().as_bytes()).unwrap(), file); + assert!(VaultFile::parse(br#"{"version":2}"#).is_err()); + assert!(VaultFile::parse(br#"{"version":1,"enabled":["grammar:twig"]}"#).is_err()); + assert!(VaultFile::parse(br#"{"version":1,"wanted":["../x"]}"#).is_err()); + let many: Vec = (0..257).map(|i| format!("grammar:g{i}")).collect(); + assert!(VaultFile { enabled: vec![], wanted: many }.to_json().is_err()); + assert_eq!(VaultFile::parse(br#"{"version":1}"#).unwrap(), VaultFile::default()); +} + +#[test] +fn commit_and_load() { + let dir = common::temp_config("addons-commit"); + let store = Store::open(dir.path()).unwrap(); + assert_eq!(store.load().unwrap(), State::default()); + let state = sample(); + store.commit(&state).unwrap(); + assert_eq!(store.load().unwrap(), state); + assert!(store.root().join("state.json").is_file()); +} + +#[test] +fn corrupt_state_errs_and_file_is_untouched() { + let dir = common::temp_config("addons-corrupt"); + let store = Store::open(dir.path()).unwrap(); + let path = store.root().join("state.json"); + std::fs::write(&path, b"{ not json").unwrap(); + assert!(store.load().is_err()); + assert!(store.gc(&State::default()).is_ok()); + assert_eq!(std::fs::read(&path).unwrap(), b"{ not json"); +} + +#[test] +fn version_dir_uses_validated_parts() { + let dir = common::temp_config("addons-vdir"); + let store = Store::open(dir.path()).unwrap(); + let path = store.version_dir(Kind::Grammar, "twig", v(1, 2, 3)).unwrap(); + assert!(path.starts_with(store.root())); + assert!(path.ends_with("store/grammar/twig/1.2.3")); + assert!(store.version_dir(Kind::Grammar, "../x", v(1, 0, 0)).is_err()); + assert!(store.version_dir(Kind::Grammar, "a/b", v(1, 0, 0)).is_err()); + assert!(store.version_dir(Kind::Grammar, "", v(1, 0, 0)).is_err()); +} + +#[test] +fn gc_keeps_version_and_previous_only() { + let dir = common::temp_config("addons-gc"); + let store = Store::open(dir.path()).unwrap(); + let state = sample(); + let touch = |kind, id, ver| { + let d = store.version_dir(kind, id, ver).unwrap(); + std::fs::create_dir_all(&d).unwrap(); + std::fs::write(d.join("f"), b"x").unwrap(); + d + }; + let active = touch(Kind::Grammar, "twig", v(1, 1, 0)); + let previous = touch(Kind::Grammar, "twig", v(1, 0, 0)); + let old = touch(Kind::Grammar, "twig", v(0, 9, 0)); + let orphan = touch(Kind::Theme, "gone", v(1, 0, 0)); + let plugin = touch(Kind::Plugin, "dice", v(0, 1, 0)); + std::fs::create_dir_all(store.staging_dir().join("abc")).unwrap(); + std::fs::write(store.staging_dir().join("abc/part"), b"x").unwrap(); + std::fs::write(store.staging_dir().join("loose"), b"x").unwrap(); + let outside = dir.path().join("outside"); + std::fs::create_dir_all(&outside).unwrap(); + std::fs::write(outside.join("keep"), b"x").unwrap(); + + assert_eq!(store.gc(&state).unwrap(), 2); + assert!(active.is_dir() && previous.is_dir() && plugin.is_dir()); + assert!(!old.exists() && !orphan.exists()); + assert!(!orphan.parent().unwrap().exists(), "empty parents are pruned"); + assert_eq!(std::fs::read_dir(store.staging_dir()).unwrap().count(), 0); + assert!(outside.join("keep").is_file()); +} + +#[cfg(unix)] +#[test] +fn gc_does_not_follow_links() { + let dir = common::temp_config("addons-link"); + let store = Store::open(dir.path()).unwrap(); + let outside = dir.path().join("outside"); + std::fs::create_dir_all(&outside).unwrap(); + std::fs::write(outside.join("keep"), b"x").unwrap(); + let parent = store.version_dir(Kind::Theme, "t", v(1, 0, 0)).unwrap().parent().unwrap().to_path_buf(); + std::fs::create_dir_all(&parent).unwrap(); + std::os::unix::fs::symlink(&outside, parent.join("2.0.0")).unwrap(); + store.gc(&State::default()).unwrap(); + assert!(outside.join("keep").is_file()); + assert!(!parent.join("2.0.0").exists()); +} diff --git a/crates/engine/tests/addons_support/mod.rs b/crates/engine/tests/addons_support/mod.rs new file mode 100644 index 0000000..5c8de72 --- /dev/null +++ b/crates/engine/tests/addons_support/mod.rs @@ -0,0 +1,68 @@ +//! Shared fixtures of the add-on tests: signed-looking entries, index documents and a fake +//! transport that serves their files. Signatures are not real here (tests use `FakeVerifier`). + +#![allow(dead_code)] // each test target uses a different part + +use std::sync::Arc; + +use notescraft_engine::addons::{FakeTransport, Index, Response, Transport}; +use notescraft_syntax::validate::sha256_hex; +use serde_json::{Value, json}; + +/// Where the first-party registry serves files from. +pub const BASE: &str = "https://git.marcospaulo.dev.br/masi/notescraft-registry/raw/branch/main/files"; +/// A grammar the syntax crate accepts. +pub const TINY: &str = "%YAML 1.2\n---\nname: Tiny\nfile_extensions: [tiny]\nscope: source.tiny\ncontexts:\n main:\n - match: '\\bfoo\\b'\n scope: keyword.control.tiny\n"; +/// A theme the themes crate accepts. +pub const THEME: &str = "id = \"mine\"\nname = \"Mine\"\nvariant = \"dark\"\n"; +/// The smallest WebAssembly module header. +pub const WASM: &[u8] = b"\0asm\x01\0\0\0"; + +/// A plugin manifest for `id` at `version` with `permissions`. +pub fn manifest(id: &str, version: &str, permissions: &[&str]) -> String { + json!({ "id": id, "name": "Dice", "version": version, "min_app_version": "0.1.0", "permissions": permissions, "commands": [{ "id": "roll", "name": "Roll" }] }) + .to_string() +} + +/// The download link of one file of an entry. +pub fn url(kind: &str, id: &str, version: &str, path: &str) -> String { + format!("{BASE}/{kind}/{id}/{version}/{path}") +} + +/// An entry as the index writes it, with the real hashes and sizes of `files`. +pub fn entry(kind: &str, id: &str, version: &str, files: &[(&str, &[u8])], permissions: &[&str]) -> Value { + let listed: Vec = files + .iter() + .map(|(path, bytes)| json!({ "path": path, "size": bytes.len(), "sha256": sha256_hex(bytes), "url": url(kind, id, version, path) })) + .collect(); + json!({ + "id": id, "kind": kind, "title": id, "summary": "s", "version": version, "min_app_version": "0.1.0", "license": "MIT", + "aliases": [id], "permissions": permissions, "files": listed, "signature": { "key_id": "k1", "sig": "QUJD" }, + }) +} + +/// An index document. +pub fn index_json(entries: &[Value], revoked: &[(&str, &str)]) -> Vec { + let revoked: Vec = revoked.iter().map(|(id, version)| json!({ "id": id, "version": version })).collect(); + json!({ "version": 2, "generated_at": "2026-10-21T00:00:00Z", "entries": entries, "revoked": revoked }).to_string().into_bytes() +} + +/// The parsed index of `entries`. +pub fn index(entries: &[Value], revoked: &[(&str, &str)]) -> Index { + Index::parse(&index_json(entries, revoked)).expect("the fixture index parses") +} + +/// A transport serving the files of `entry` (as written by [`entry`]) from `files`. +pub fn serve(transport: FakeTransport, kind: &str, id: &str, version: &str, files: &[(&str, &[u8])]) -> FakeTransport { + files.iter().fold(transport, |t, (path, bytes)| t.with(&url(kind, id, version, path), bytes)) +} + +/// A transport shared between the test and the session that owns the box. +#[derive(Debug, Clone)] +pub struct Shared(pub Arc); + +impl Transport for Shared { + fn get(&self, url: &str, max_bytes: u64, if_none_match: Option<&str>) -> notescraft_engine::Result { + self.0.get(url, max_bytes, if_none_match) + } +} diff --git a/crates/engine/tests/addons_verify.rs b/crates/engine/tests/addons_verify.rs new file mode 100644 index 0000000..bb14789 --- /dev/null +++ b/crates/engine/tests/addons_verify.rs @@ -0,0 +1,115 @@ +//! Signature canonical text and ed25519 verification of registry entries. + +use notescraft_engine::addons::{Ed25519, Entry, FakeVerifier, Index, Verifier, canonical_text}; +use ring::signature::{Ed25519KeyPair, KeyPair}; +use serde_json::{Value, json}; + +/// Test-only seed; not a secret and not the production key. +const TEST_SEED: [u8; 32] = [7; 32]; +const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +fn b64(bytes: &[u8]) -> String { + let mut out = String::new(); + for chunk in bytes.chunks(3) { + let n = chunk.iter().enumerate().fold(0_u32, |n, (i, b)| n | u32::from(*b) << (16 - 8 * i)); + for i in 0..4 { + if i <= chunk.len() { + out.push(char::from(B64[usize::try_from((n >> (18 - 6 * i)) & 63).unwrap()])); + } else { + out.push('='); + } + } + } + out +} + +fn pair() -> Ed25519KeyPair { + Ed25519KeyPair::from_seed_unchecked(&TEST_SEED).unwrap() +} + +fn verifier() -> Ed25519 { + Ed25519::new(vec![("k1".into(), pair().public_key().as_ref().try_into().unwrap())]) +} + +fn files(order: &[&str]) -> Value { + Value::Array(order.iter().map(|p| json!({ "path": p, "size": 5, "sha256": "c".repeat(64), "url": format!("https://h.example/{p}") })).collect()) +} + +fn entry_json(files: &Value, sig: &str, key_id: &str) -> Value { + json!({ + "id": "p", "kind": "pack", "title": "P", "version": "1.2.3", "min_app_version": "0.1.0", "license": "MIT", + "files": files, "signature": { "key_id": key_id, "sig": sig } + }) +} + +fn parse(value: &Value) -> Entry { + let index = Index::parse(json!({ "version": 2, "entries": [value] }).to_string().as_bytes()).unwrap(); + index.entries[0].clone() +} + +fn signed(order: &[&str]) -> Entry { + let unsigned = parse(&entry_json(&files(order), "QUJD", "k1")); + let sig = b64(pair().sign(canonical_text(&unsigned).as_bytes()).as_ref()); + parse(&entry_json(&files(order), &sig, "k1")) +} + +const A: &str = "grammars/a.sublime-syntax"; +const B: &str = "themes/b.toml"; + +#[test] +fn known_answer_verifies() { + let entry = signed(&[A, B]); + assert_eq!(canonical_text(&entry), format!("notescraft-registry-v1\np\npack\n1.2.3\n{A}\n{}\n5\n{B}\n{}\n5\n", "c".repeat(64), "c".repeat(64))); + verifier().verify(&entry).unwrap(); +} + +#[test] +fn file_order_does_not_change_text_or_signature() { + let entry = signed(&[A, B]); + let mut shuffled = entry.clone(); + shuffled.files.reverse(); + assert_eq!(canonical_text(&entry), canonical_text(&shuffled)); + verifier().verify(&shuffled).unwrap(); +} + +#[test] +fn tampering_fails() { + let entry = signed(&[A, B]); + let mut hash = entry.clone(); + hash.files[0].sha256 = format!("d{}", "c".repeat(63)); + assert!(verifier().verify(&hash).is_err()); + let mut size = entry.clone(); + size.files[0].size += 1; + assert!(verifier().verify(&size).is_err()); + let mut version = entry.clone(); + version.version.patch += 1; + assert!(verifier().verify(&version).is_err()); + let mut id = entry.clone(); + id.id = "q".into(); + assert!(verifier().verify(&id).is_err()); + let mut kind = entry; + kind.kind = notescraft_engine::addons::Kind::Theme; + assert!(verifier().verify(&kind).is_err()); +} + +#[test] +fn unknown_key_bad_length_and_empty_trust_fail() { + let entry = signed(&[A]); + let mut other = entry.clone(); + other.signature.key_id = "k2".into(); + assert!(verifier().verify(&other).is_err()); + let mut short = entry.clone(); + short.signature.sig = b64(&[1, 2, 3]); + assert!(verifier().verify(&short).is_err()); + let mut junk = entry.clone(); + junk.signature.sig = "!!!!".into(); + assert!(verifier().verify(&junk).is_err()); + assert!(Ed25519::trusted().verify(&entry).is_err()); +} + +#[test] +fn fake_verifier() { + let entry = signed(&[A]); + assert!(FakeVerifier { accept: true }.verify(&entry).is_ok()); + assert!(FakeVerifier { accept: false }.verify(&entry).is_err()); +} diff --git a/crates/engine/tests/core_commands.rs b/crates/engine/tests/core_commands.rs index 072192c..556b170 100644 --- a/crates/engine/tests/core_commands.rs +++ b/crates/engine/tests/core_commands.rs @@ -39,6 +39,16 @@ const CORE_IDS: &[&str] = &[ "grammar.install", "grammar.install_text", "grammar.remove", + "registry.check", + "registry.disable", + "registry.enable", + "registry.info", + "registry.install", + "registry.list", + "registry.remove", + "registry.rollback", + "registry.suggest", + "registry.update", "json.format_block", "json.minify_block", "vault.info", diff --git a/crates/htmlmd/Cargo.toml b/crates/htmlmd/Cargo.toml new file mode 100644 index 0000000..fe07c7c --- /dev/null +++ b/crates/htmlmd/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "notescraft-htmlmd" +description = "forgiving HTML tokenizer for the clean paste pipeline" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +publish = false + +[lib] +path = "src/lib.rs" + +[dependencies] +thiserror = { workspace = true } + +[lints] +workspace = true diff --git a/crates/htmlmd/src/entities.rs b/crates/htmlmd/src/entities.rs new file mode 100644 index 0000000..7679637 --- /dev/null +++ b/crates/htmlmd/src/entities.rs @@ -0,0 +1,94 @@ +//! Decoding of HTML character references. + +/// The named entities the tokenizer understands. +const NAMED: &[(&str, char)] = &[ + ("amp", '&'), + ("lt", '<'), + ("gt", '>'), + ("quot", '"'), + ("apos", '\''), + ("nbsp", '\u{00a0}'), + ("ndash", '\u{2013}'), + ("mdash", '\u{2014}'), + ("hellip", '\u{2026}'), + ("lsquo", '\u{2018}'), + ("rsquo", '\u{2019}'), + ("ldquo", '\u{201c}'), + ("rdquo", '\u{201d}'), + ("copy", '\u{00a9}'), + ("reg", '\u{00ae}'), + ("trade", '\u{2122}'), + ("bull", '\u{2022}'), + ("middot", '\u{00b7}'), + ("laquo", '\u{00ab}'), + ("raquo", '\u{00bb}'), + ("euro", '\u{20ac}'), + ("pound", '\u{00a3}'), + ("yen", '\u{00a5}'), + ("cent", '\u{00a2}'), + ("deg", '\u{00b0}'), + ("times", '\u{00d7}'), + ("divide", '\u{00f7}'), +]; + +/// The longest entity reference accepted, in characters. +const MAX_ENTITY_LEN: usize = 32; + +/// Replace the character references of `text` by the characters they stand for. +/// +/// Named, decimal and hexadecimal references are understood. An unknown name +/// or an invalid code point is left verbatim. +#[must_use] +pub fn decode_entities(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(pos) = rest.find('&') { + out.push_str(rest.get(..pos).unwrap_or("")); + let tail = rest.get(pos..).unwrap_or(""); + if let Some((ch, len)) = decode_one(tail) { + out.push(ch); + rest = tail.get(len..).unwrap_or(""); + } else { + out.push('&'); + rest = tail.get(1..).unwrap_or(""); + } + } + out.push_str(rest); + out +} + +/// Decode the reference at the start of `tail`, which begins with `&`. +/// +/// Returns the character and the number of bytes the reference uses. +fn decode_one(tail: &str) -> Option<(char, usize)> { + let body = tail.get(1..)?; + let Some(digits) = body.strip_prefix('#') else { + return decode_named(body); + }; + let (radix, digits) = match digits.strip_prefix('x').or_else(|| digits.strip_prefix('X')) { + Some(hex) => (16, hex), + None => (10, digits), + }; + let end = digits.find(';')?; + if end == 0 || end > MAX_ENTITY_LEN { + return None; + } + let number = digits.get(..end)?; + let code = u32::from_str_radix(number, radix).ok()?; + if code == 0 { + return None; + } + let ch = char::from_u32(code)?; + Some((ch, 1 + 1 + usize::from(radix == 16) + end + 1)) +} + +/// Decode a named reference; `body` is the text after the `&`. +fn decode_named(body: &str) -> Option<(char, usize)> { + let end = body.find(';')?; + if end == 0 || end > MAX_ENTITY_LEN { + return None; + } + let name = body.get(..end)?; + let ch = NAMED.iter().find(|(n, _)| *n == name).map(|(_, ch)| *ch)?; + Some((ch, 1 + end + 1)) +} diff --git a/crates/htmlmd/src/lib.rs b/crates/htmlmd/src/lib.rs new file mode 100644 index 0000000..a82a6c2 --- /dev/null +++ b/crates/htmlmd/src/lib.rs @@ -0,0 +1,33 @@ +//! A forgiving HTML tokenizer for the clean paste pipeline. +//! +//! The crate is pure: no IO, no network, no third-party dependency. It turns +//! the HTML of a paste into a stream of [`Token`]s that a converter can walk +//! without building a DOM, so malformed markup can never make it loop or +//! panic. +//! +//! ``` +//! use notescraft_htmlmd::{Token, tokenize}; +//! +//! let tokens = tokenize("

Hi & bye

", 64).expect("tokens"); +//! assert_eq!(tokens[0], Token::Open { name: "p".to_string(), attrs: vec![("class".to_string(), "a".to_string())], self_closing: false }); +//! assert_eq!(tokens[1], Token::Text("Hi & bye".to_string())); +//! ``` + +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +pub mod entities; +pub mod tokens; + +pub use entities::decode_entities; +pub use tokens::{Token, tokenize}; + +/// The errors of this crate. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// The input is longer than the limit, or it produced too many tokens. + #[error("input too large")] + TooLarge, +} + +/// The result of this crate. +pub type Result = std::result::Result; diff --git a/crates/htmlmd/src/tokens.rs b/crates/htmlmd/src/tokens.rs new file mode 100644 index 0000000..fac1acf --- /dev/null +++ b/crates/htmlmd/src/tokens.rs @@ -0,0 +1,393 @@ +//! A forgiving tokenizer for pasted HTML. +//! +//! The tokenizer never rejects malformed input: markup that does not parse is +//! returned as text. It only has to be good enough to find tags, attributes +//! and entities for the clean paste pipeline. +//! +//! ``` +//! use notescraft_htmlmd::{Token, tokenize}; +//! +//! let tokens = tokenize("

Hi & bye

", 64).expect("tokens"); +//! assert_eq!(tokens.len(), 3); +//! assert_eq!(tokens[1], Token::Text("Hi & bye".to_string())); +//! ``` + +use crate::entities::decode_entities; +use crate::{Error, Result}; + +/// Longest input accepted by [`tokenize`], in bytes. +pub const MAX_INPUT_BYTES: usize = 2 * 1024 * 1024; + +/// Elements that never have a closing tag. +const VOID_ELEMENTS: &[&str] = &["br", "hr", "img", "input", "meta", "link"]; + +/// Elements whose content is raw text, not markup. +const RAW_TEXT_ELEMENTS: &[&str] = &["script", "style"]; + +/// One token of the HTML stream. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Token { + /// Text between tags, with entities decoded. + Text(String), + /// An opening tag. + Open { + /// The lowercased tag name. + name: String, + /// The attributes in source order; names lowercased, values decoded. + attrs: Vec<(String, String)>, + /// True for void elements and for tags closed with `/>`. + self_closing: bool, + }, + /// A closing tag, holding the lowercased tag name. + Close(String), + /// The body of an HTML comment. + Comment(String), +} + +/// What a `<` opens. +enum Kind { + /// Not a tag: the `<` is text. + Text, + /// A comment. + Comment, + /// A declaration (``, ``): dropped. + Dropped, + /// An opening tag. + Open, + /// A closing tag. + Close, +} + +/// The outcome of reading a tag. +enum TagResult { + /// The tag and the position just after it. + Parsed(Token, usize), + /// The tag never ended: the rest of the input is text. + Unterminated, +} + +/// The scanner state. +struct Scanner<'a> { + /// The whole input. + src: &'a str, + /// The current byte position, always on a character boundary. + pos: usize, + /// The tokens produced so far. + out: Vec, + /// The maximum number of tokens allowed. + max_tokens: usize, +} + +/// Split `html` into tokens. +/// +/// Tag and attribute names are lowercased, attribute values are decoded and +/// the first of two attributes with the same name wins. The content of +/// `script` and `style` is returned raw. Comments are returned as +/// [`Token::Comment`]; declarations are dropped. Markup that is not closed +/// is returned as text. +/// +/// # Errors +/// Returns [`Error::TooLarge`] when the input is longer than +/// [`MAX_INPUT_BYTES`] or when more than `max_tokens` tokens are produced. +pub fn tokenize(html: &str, max_tokens: usize) -> Result> { + if html.len() > MAX_INPUT_BYTES { + return Err(Error::TooLarge); + } + let scanner = Scanner { src: html, pos: 0, out: Vec::new(), max_tokens }; + scanner.run() +} + +impl Scanner<'_> { + fn run(mut self) -> Result> { + let mut text = String::new(); + while let Some(tail) = self.src.get(self.pos..) { + let Some(rel) = tail.find('<') else { + text.push_str(tail); + self.pos = self.src.len(); + break; + }; + let lt = self.pos + rel; + text.push_str(self.src.get(self.pos..lt).unwrap_or("")); + self.pos = lt; + match self.classify() { + Kind::Text => { + text.push('<'); + self.pos += 1; + } + Kind::Comment => { + let Some((content, next)) = self.read_comment() else { + text.push_str(self.src.get(self.pos..).unwrap_or("")); + self.pos = self.src.len(); + self.flush(&mut text)?; + break; + }; + self.flush(&mut text)?; + self.pos = next; + self.push(Token::Comment(content))?; + } + Kind::Dropped => { + let Some(next) = self.read_declaration() else { + text.push_str(self.src.get(self.pos..).unwrap_or("")); + self.pos = self.src.len(); + self.flush(&mut text)?; + break; + }; + self.flush(&mut text)?; + self.pos = next; + } + Kind::Open => { + self.flush(&mut text)?; + let TagResult::Parsed(token, next) = self.read_open_tag() else { + text.push_str(self.src.get(self.pos..).unwrap_or("")); + self.pos = self.src.len(); + self.flush(&mut text)?; + break; + }; + self.pos = next; + self.push(token.clone())?; + self.read_raw_text(&token)?; + } + Kind::Close => { + self.flush(&mut text)?; + let TagResult::Parsed(token, next) = self.read_close_tag(self.pos) else { + text.push_str(self.src.get(self.pos..).unwrap_or("")); + self.pos = self.src.len(); + self.flush(&mut text)?; + break; + }; + self.pos = next; + self.push(token)?; + } + } + } + self.flush(&mut text)?; + Ok(self.out) + } + + /// Push a token, refusing to go over the cap. + fn push(&mut self, token: Token) -> Result<()> { + if self.out.len() >= self.max_tokens { + return Err(Error::TooLarge); + } + self.out.push(token); + Ok(()) + } + + /// Move the pending text into a token. + fn flush(&mut self, text: &mut String) -> Result<()> { + if text.is_empty() { + return Ok(()); + } + let decoded = decode_entities(text); + text.clear(); + self.push(Token::Text(decoded)) + } + + /// Decide what the `<` at the current position opens. + fn classify(&self) -> Kind { + let Some(after) = self.src.get(self.pos + 1..) else { return Kind::Text }; + let Some(first) = after.chars().next() else { return Kind::Text }; + match first { + '!' if after.starts_with("!--") => Kind::Comment, + '!' | '?' => Kind::Dropped, + '/' => { + if after.get(1..).and_then(|s| s.chars().next()).is_some_and(|c| c.is_ascii_alphabetic()) { + Kind::Close + } else { + Kind::Text + } + } + c if c.is_ascii_alphabetic() => Kind::Open, + _ => Kind::Text, + } + } + + /// Read a comment; `None` when it is never closed. + fn read_comment(&self) -> Option<(String, usize)> { + let after = self.src.get(self.pos + 4..)?; + let end = after.find("-->")?; + let content = after.get(..end)?.to_string(); + Some((content, self.pos + 4 + end + 3)) + } + + /// Skip a declaration; `None` when it is never closed. + fn read_declaration(&self) -> Option { + let after = self.src.get(self.pos + 1..)?; + if let Some(rest) = after.strip_prefix("![CDATA[") { + let end = rest.find("]]>")?; + return Some(self.pos + 1 + 8 + end + 3); + } + let end = after.find('>')?; + Some(self.pos + 1 + end + 1) + } + + /// Read an opening tag at `lt`. + fn read_open_tag(&self) -> TagResult { + let src = self.src; + let (name, mut p) = read_name(src, self.pos + 1); + if name.is_empty() { + return TagResult::Unterminated; + } + let mut attrs: Vec<(String, String)> = Vec::new(); + let mut self_closing = false; + loop { + let Some(tail) = src.get(p..) else { return TagResult::Unterminated }; + let Some(first) = tail.chars().next() else { return TagResult::Unterminated }; + if first.is_whitespace() { + p += first.len_utf8(); + continue; + } + if first == '>' { + p += 1; + break; + } + if first == '/' { + if tail.get(1..).is_some_and(|s| s.starts_with('>')) { + self_closing = true; + p += 2; + break; + } + p += 1; + continue; + } + let (attr, after) = read_attr_name(src, p); + if attr.is_empty() { + p += first.len_utf8(); + continue; + } + p = after; + let (value, next) = read_attr_value(src, p); + let Some(value) = value else { return TagResult::Unterminated }; + p = next; + if !attrs.iter().any(|(n, _)| n == &attr) { + attrs.push((attr, decode_entities(&value))); + } + } + let self_closing = self_closing || VOID_ELEMENTS.contains(&name.as_str()); + TagResult::Parsed(Token::Open { name, attrs, self_closing }, p) + } + + /// Read a closing tag at `lt`. + fn read_close_tag(&self, lt: usize) -> TagResult { + let src = self.src; + let (name, mut p) = read_name(src, lt + 2); + if name.is_empty() { + return TagResult::Unterminated; + } + loop { + let Some(tail) = src.get(p..) else { return TagResult::Unterminated }; + let Some(first) = tail.chars().next() else { return TagResult::Unterminated }; + p += first.len_utf8(); + if first == '>' { + break; + } + } + TagResult::Parsed(Token::Close(name), p) + } + + /// Read the raw content of `script` and `style` elements. + fn read_raw_text(&mut self, token: &Token) -> Result<()> { + let Token::Open { name, self_closing, .. } = token else { return Ok(()) }; + if *self_closing || !RAW_TEXT_ELEMENTS.contains(&name.as_str()) { + return Ok(()); + } + let start = self.pos; + let Some(lt) = find_raw_end(self.src, start, name) else { + let content = self.src.get(start..).unwrap_or("").to_string(); + self.pos = self.src.len(); + if !content.is_empty() { + self.push(Token::Text(content))?; + } + return Ok(()); + }; + let content = self.src.get(start..lt).unwrap_or("").to_string(); + self.pos = lt; + if !content.is_empty() { + self.push(Token::Text(content))?; + } + if let TagResult::Parsed(close, next) = self.read_close_tag(lt) { + self.pos = next; + self.push(close)?; + } + Ok(()) + } +} + +/// Read a tag name at `pos`, lowercased. +fn read_name(src: &str, pos: usize) -> (String, usize) { + let mut end = pos; + for (offset, ch) in src.get(pos..).unwrap_or("").char_indices() { + if !ch.is_ascii_alphanumeric() && ch != '-' { + break; + } + end = pos + offset + ch.len_utf8(); + } + (src.get(pos..end).unwrap_or("").to_ascii_lowercase(), end) +} + +/// Read an attribute name at `pos`, lowercased. +fn read_attr_name(src: &str, pos: usize) -> (String, usize) { + let mut end = pos; + for (offset, ch) in src.get(pos..).unwrap_or("").char_indices() { + if ch.is_whitespace() || ch == '=' || ch == '>' || ch == '/' { + break; + } + end = pos + offset + ch.len_utf8(); + } + (src.get(pos..end).unwrap_or("").to_ascii_lowercase(), end) +} + +/// Read the value of the attribute whose name ends at `pos`. +/// +/// Returns the raw value and the position after it, or `None` when the +/// value is never closed. +fn read_attr_value(src: &str, pos: usize) -> (Option, usize) { + let mut p = pos; + while let Some(first) = src.get(p..).and_then(|s| s.chars().next()) { + if !first.is_whitespace() { + break; + } + p += first.len_utf8(); + } + if !src.get(p..).is_some_and(|s| s.starts_with('=')) { + return (Some(String::new()), p); + } + p += 1; + while let Some(first) = src.get(p..).and_then(|s| s.chars().next()) { + if !first.is_whitespace() { + break; + } + p += first.len_utf8(); + } + let Some(tail) = src.get(p..) else { return (None, p) }; + let Some(first) = tail.chars().next() else { return (None, p) }; + if first == '"' || first == '\'' { + let rest = src.get(p + 1..).unwrap_or(""); + let Some(end) = rest.find(first) else { return (None, p) }; + return (Some(rest.get(..end).unwrap_or("").to_string()), p + 1 + end + 1); + } + let end = tail.find(|c: char| c.is_whitespace() || c == '>').unwrap_or(tail.len()); + (Some(tail.get(..end).unwrap_or("").to_string()), p + end) +} + +/// Find the `<` of the closing tag of `name` at or after `from`. +fn find_raw_end(src: &str, from: usize, name: &str) -> Option { + let mut pos = from; + while let Some(tail) = src.get(pos..) { + let rel = tail.find('<')?; + let lt = pos + rel; + let after = src.get(lt + 1..)?; + let Some(tail2) = after.strip_prefix('/') else { + pos = lt + 1; + continue; + }; + if let Some(head) = tail2.get(..name.len()) { + let rest = tail2.get(name.len()..).unwrap_or(""); + if head.eq_ignore_ascii_case(name) && (rest.starts_with('>') || rest.starts_with(char::is_whitespace)) { + return Some(lt); + } + } + pos = lt + 1; + } + None +} diff --git a/crates/htmlmd/tests/tokens.rs b/crates/htmlmd/tests/tokens.rs new file mode 100644 index 0000000..6a219b0 --- /dev/null +++ b/crates/htmlmd/tests/tokens.rs @@ -0,0 +1,171 @@ +//! Tests of the forgiving HTML tokenizer. + +use notescraft_htmlmd::{Error, Token, decode_entities, tokenize}; + +/// Build an open token. +fn open(name: &str, attrs: &[(&str, &str)], self_closing: bool) -> Token { + Token::Open { name: name.to_string(), attrs: attrs.iter().map(|(n, v)| (n.to_string(), v.to_string())).collect(), self_closing } +} + +#[test] +fn tags_attributes_and_text() { + let tokens = tokenize("

Hi & bye

", 64).expect("tokens"); + assert_eq!(tokens, vec![open("p", &[("class", "a b")], false), Token::Text("Hi & bye".to_string()), Token::Close("p".to_string()),]); +} + +#[test] +fn attribute_values_may_be_unquoted_or_single_quoted() { + let tokens = tokenize("t", 64).expect("tokens"); + assert_eq!(tokens[0], open("a", &[("href", "/foo"), ("title", "x y")], false)); +} + +#[test] +fn attribute_values_are_decoded() { + let tokens = tokenize("t", 64).expect("tokens"); + assert_eq!(tokens[0], open("a", &[("href", "?a=1&b=2")], false)); +} + +#[test] +fn a_duplicate_attribute_keeps_the_first_value() { + let tokens = tokenize("

x

", 64).expect("tokens"); + assert_eq!(tokens[0], open("p", &[("class", "a")], false)); +} + +#[test] +fn names_are_lowercased() { + let tokens = tokenize("

x

", 64).expect("tokens"); + assert_eq!(tokens[0], open("p", &[("class", "A")], false)); + assert_eq!(tokens[2], Token::Close("p".to_string())); +} + +#[test] +fn void_elements_are_self_closing() { + let tokens = tokenize("
", 64).expect("tokens"); + assert_eq!(tokens[0], open("br", &[], true)); + assert_eq!(tokens[1], open("img", &[("src", "x")], true)); + assert_eq!(tokens[2], open("hr", &[], true)); +} + +#[test] +fn script_content_is_raw() { + let tokens = tokenize("", 64).expect("tokens"); + assert_eq!(tokens, vec![open("script", &[], false), Token::Text("if (a < b) { x(); }".to_string()), Token::Close("script".to_string()),]); +} + +#[test] +fn style_content_is_raw() { + let tokens = tokenize("", 64).expect("tokens"); + assert_eq!(tokens[1], Token::Text("a > b { color: red }".to_string())); +} + +#[test] +fn comments_are_returned() { + let tokens = tokenize("", 64).expect("tokens"); + assert_eq!(tokens, vec![Token::Comment(" hi ".to_string())]); +} + +#[test] +fn conditional_comments_are_returned() { + let html = ""; + let tokens = tokenize(html, 64).expect("tokens"); + assert_eq!(tokens, vec![Token::Comment("[if gte mso 9]>

x

text"; + let tokens = tokenize(html, 64).expect("tokens"); + assert_eq!(tokens, vec![Token::Text("text".to_string())]); +} + +#[test] +fn unterminated_markup_is_text() { + let tokens = tokenize("

\" ' \u{a0} โ€ฆ โ€” ยฉ".to_string())]); +} + +#[test] +fn numeric_entities_are_decoded() { + let tokens = tokenize("' ' 😀", 64).expect("tokens"); + assert_eq!(tokens, vec![Token::Text("' ' ๐Ÿ˜€".to_string())]); +} + +#[test] +fn invalid_and_unknown_entities_stay_verbatim() { + let tokens = tokenize("� � � &nope;", 64).expect("tokens"); + assert_eq!(tokens, vec![Token::Text("� � � &nope;".to_string())]); +} + +#[test] +fn emoji_survive() { + let tokens = tokenize("hello ๐Ÿ˜€", 64).expect("tokens"); + assert_eq!(tokens, vec![Token::Text("hello ๐Ÿ˜€".to_string())]); +} + +#[test] +fn decode_entities_is_public() { + assert_eq!(decode_entities("a & b"), "a & b"); +} + +#[test] +fn too_large_input_is_rejected() { + let html = "a".repeat(3 * 1024 * 1024); + let err = tokenize(&html, 64).expect_err("too large"); + assert!(matches!(err, Error::TooLarge)); +} + +#[test] +fn the_token_cap_is_enforced() { + let err = tokenize("

a

", 2).expect_err("too many tokens"); + assert!(matches!(err, Error::TooLarge)); + let tokens = tokenize("

a

", 3).expect("three tokens"); + assert_eq!(tokens.len(), 3); +} + +/// A tiny linear congruential generator, so the soups are reproducible. +struct Lcg(u64); + +impl Lcg { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); + self.0 >> 33 + } +} + +#[test] +fn random_soups_never_panic() { + const ALPHABET: &[char] = &['<', '>', '/', '=', '"', '\'', ' ', 'a', '&', ';', '!', '-']; + let mut rng = Lcg(0x5eed); + for _ in 0..5000 { + let len = usize::try_from(rng.next() % 64).unwrap_or(0); + let mut html = String::new(); + for _ in 0..len { + let index = usize::try_from(rng.next()).unwrap_or(0) % ALPHABET.len(); + if let Some(ch) = ALPHABET.get(index) { + html.push(*ch); + } + } + let _ = tokenize(&html, 4096); + } +} diff --git a/crates/i18n/locales/en.toml b/crates/i18n/locales/en.toml index 7df55ae..46bb419 100644 --- a/crates/i18n/locales/en.toml +++ b/crates/i18n/locales/en.toml @@ -1915,3 +1915,56 @@ remove-banner = "Remove banner" no-colour = "No colour" more = "Refine the search to see more icons" no-images = "No image in the vault" + +[ui.registry] +title = "Registry" +description = "Add tools, themes and languages to your workspace." +search = "Search add-ons" +check = "Check for updates" +never = "Never checked" +last-checked = "Last checked {time}" +auto-check = "Check for updates automatically" +all = "All" +plugins = "Plugins" +themes = "Themes" +packs = "Packs" +grammars = "Grammars" +fonts = "Fonts" +snippets = "Snippets" +install = "Install" +installed = "Installed" +update = "Update to {version}" +updates = "{count} updates" +incompatible = "Needs NotesCraft {version}" +revoked = "Revoked" +unsupported = "Not available in this version" +details = "Details" +close = "Close details" +remove = "Remove" +rollback = "Rollback" +enable = "Enable in this vault" +disable = "Disable in this vault" +offline = "Offline โ€” showing cached add-ons. Try checking again when connected." +empty = "No add-ons found. Try another search or check for updates." +loading = "Workingโ€ฆ" +permissions = "Permissions" +read = "Read notes and search" +write = "Create and change notes" +edit = "Edit the open note" +permission-unknown = "Additional permission: {permission}" +license = "License: {license}" +source = "Source: {source}" +signature = "Signature key: {key}" +files = "Files" +bytes = "{size} bytes" +version = "Version {version} ยท {size}" +invalid-reply = "The registry returned an invalid catalogue." +file = "{path} ยท {size} bytes" +worker-stopped = "Registry worker stopped" +backend-unavailable = "Registry installation is not available until the command worker is connected." +just-now = "just now" +install-grammar = "Install {name} grammar ({size})" +suggestions-title = "Add-ons you may want" +suggestions-dismiss = "Not now" +suggestions-empty = "No suggestions for this vault yet." +wanted-title = "This vault uses these add-ons" diff --git a/crates/srs/Cargo.toml b/crates/srs/Cargo.toml new file mode 100644 index 0000000..df3e94a --- /dev/null +++ b/crates/srs/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "notescraft-srs" +description = "SM-2 spaced repetition scheduler for NotesCraft flashcards" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +publish = false + +[lib] +path = "src/lib.rs" + +[dependencies] + +[lints] +workspace = true diff --git a/crates/srs/src/lib.rs b/crates/srs/src/lib.rs new file mode 100644 index 0000000..c67ae9d --- /dev/null +++ b/crates/srs/src/lib.rs @@ -0,0 +1,25 @@ +//! Spaced repetition scheduling for `NotesCraft` flashcards. +//! +//! The crate is pure: no clock, no file, no environment. The caller passes +//! the current day (the number of days since 1970-01-01) and gets the next +//! scheduling state back. +//! +//! [`new_state`] builds the state of a card that was never reviewed, +//! [`review`] applies one grade and [`is_due`] tells whether a card is due. +//! +//! ``` +//! use notescraft_srs::{Grade, is_due, new_state, review}; +//! +//! let state = new_state(100); +//! assert!(!is_due(&state, 99)); +//! +//! let state = review(state, Grade::Good, 100); +//! assert_eq!(state.interval_days, 1); +//! assert!(is_due(&state, 101)); +//! ``` + +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +mod sm2; + +pub use sm2::{EASE_MAX, EASE_MIN, EASE_START, Grade, MAX_INTERVAL_DAYS, State, is_due, new_state, review}; diff --git a/crates/srs/src/sm2.rs b/crates/srs/src/sm2.rs new file mode 100644 index 0000000..6301130 --- /dev/null +++ b/crates/srs/src/sm2.rs @@ -0,0 +1,155 @@ +//! The SM-2 scheduler: the public `SuperMemo` 2 algorithm. +//! +//! Every number of the algorithm is a named constant in thousandths, so the +//! arithmetic stays exact and no floating point enters the state. + +/// The ease factor of a card that was never reviewed, in thousandths (2.5). +pub const EASE_START: u16 = 2500; + +/// The lowest ease factor a card may reach, in thousandths (1.3). +pub const EASE_MIN: u16 = 1300; + +/// The highest ease factor a card may reach, in thousandths (4.0). +pub const EASE_MAX: u16 = 4000; + +/// The longest interval between two reviews, in days (one hundred years). +pub const MAX_INTERVAL_DAYS: u32 = 36_500; + +/// The interval multiplier of a Hard answer, in thousandths (1.2). +pub const HARD_FACTOR_X1000: u64 = 1200; + +/// The interval bonus of an Easy answer, in thousandths (1.3). +pub const EASY_BONUS_X1000: u64 = 1300; + +/// The ease penalty of a lapsed card, in thousandths. +pub const AGAIN_EASE_PENALTY: u16 = 200; + +/// The ease penalty of a Hard answer, in thousandths. +pub const HARD_EASE_PENALTY: u16 = 150; + +/// The ease bonus of an Easy answer, in thousandths. +pub const EASY_EASE_BONUS: u16 = 150; + +/// The interval of the first successful review of a card, in days. +pub const GOOD_FIRST_INTERVAL: u32 = 1; + +/// The interval of the second successful review of a card, in days. +pub const GOOD_SECOND_INTERVAL: u32 = 6; + +/// The interval of the first Easy review of a card, in days. +pub const EASY_FIRST_INTERVAL: u32 = 4; + +/// The interval of the second Easy review of a card, in days. +pub const EASY_SECOND_INTERVAL: u32 = 10; + +/// One scheduling step of a card, in thousandths of a day. +const MILLI: u64 = 1000; + +/// The scheduling state of one card. +/// +/// The state is the whole history the scheduler needs: the ease factor, the +/// current interval, how many times the card was answered, how many times it +/// lapsed and the day it comes back. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct State { + /// The ease factor, in thousandths (2.5 is 2500). + pub ease_x1000: u16, + /// The number of days between the last review and the next one. + pub interval_days: u32, + /// The number of successful reviews since the last lapse. + pub reps: u32, + /// The number of times the card was answered with `Again`. + pub lapses: u32, + /// The day the card comes back, in days since 1970-01-01. + pub due_day: u32, +} + +/// The grade the user gives to a card. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Grade { + /// The card was not remembered: it lapses and comes back today. + Again, + /// The card was hard to recall: a small interval, a small ease penalty. + Hard, + /// The card was recalled: the usual interval. + Good, + /// The card was easy: a long interval and an ease bonus. + Easy, +} + +/// Builds the state of a card that was never reviewed. +/// +/// The card starts with the starting ease, no interval, no review and it is +/// due on `today`. +#[must_use] +pub fn new_state(today: u32) -> State { + State { ease_x1000: EASE_START, interval_days: 0, reps: 0, lapses: 0, due_day: today } +} + +/// Applies one grade to a card and returns its next state. +/// +/// `today` is the number of days since 1970-01-01. The new due day is +/// `today` plus the new interval; it saturates instead of overflowing. +#[must_use] +pub fn review(state: State, grade: Grade, today: u32) -> State { + let (reps, lapses, interval_days, ease_x1000) = match grade { + Grade::Again => (0, state.lapses.saturating_add(1), 0, ease_down(state.ease_x1000, AGAIN_EASE_PENALTY)), + Grade::Hard => (state.reps.saturating_add(1), state.lapses, grow_hard(state), ease_down(state.ease_x1000, HARD_EASE_PENALTY)), + Grade::Good => (state.reps.saturating_add(1), state.lapses, grow_good(state), state.ease_x1000), + Grade::Easy => (state.reps.saturating_add(1), state.lapses, grow_easy(state), ease_up(state.ease_x1000, EASY_EASE_BONUS)), + }; + State { ease_x1000, interval_days, reps, lapses, due_day: today.saturating_add(interval_days) } +} + +/// Tells whether a card is due on `today`. +#[must_use] +pub fn is_due(state: &State, today: u32) -> bool { + state.due_day <= today +} + +/// The interval of a Hard answer: the current interval times 1.2, at least a day. +fn grow_hard(state: State) -> u32 { + let grown = scale(u64::from(state.interval_days).saturating_mul(HARD_FACTOR_X1000), MILLI); + clamp_interval(grown.max(1)) +} + +/// The interval of a Good answer: one day, then six, then the interval times the ease. +fn grow_good(state: State) -> u32 { + match state.reps { + 0 => GOOD_FIRST_INTERVAL, + 1 => GOOD_SECOND_INTERVAL, + _ => clamp_interval(scale(u64::from(state.interval_days).saturating_mul(u64::from(state.ease_x1000)), MILLI)), + } +} + +/// The interval of an Easy answer: four days, then ten, then the interval times the ease and the bonus. +fn grow_easy(state: State) -> u32 { + match state.reps { + 0 => EASY_FIRST_INTERVAL, + 1 => EASY_SECOND_INTERVAL, + _ => { + let base = u64::from(state.interval_days).saturating_mul(u64::from(state.ease_x1000)).saturating_mul(EASY_BONUS_X1000); + clamp_interval(scale(base, MILLI.saturating_mul(MILLI))) + } + } +} + +/// Lowers an ease factor by a penalty and keeps it inside its bounds. +fn ease_down(ease: u16, penalty: u16) -> u16 { + ease.saturating_sub(penalty).max(EASE_MIN) +} + +/// Raises an ease factor by a bonus and keeps it inside its bounds. +fn ease_up(ease: u16, bonus: u16) -> u16 { + ease.saturating_add(bonus).min(EASE_MAX) +} + +/// Divides two numbers and rounds the result to the nearest integer, halves up. +fn scale(num: u64, den: u64) -> u64 { + num.saturating_add(den / 2) / den +} + +/// Turns a day count into the interval range of the algorithm. +fn clamp_interval(days: u64) -> u32 { + u32::try_from(days).unwrap_or(MAX_INTERVAL_DAYS).min(MAX_INTERVAL_DAYS) +} diff --git a/crates/srs/tests/sm2.rs b/crates/srs/tests/sm2.rs new file mode 100644 index 0000000..a0ff130 --- /dev/null +++ b/crates/srs/tests/sm2.rs @@ -0,0 +1,139 @@ +//! Table tests of the SM-2 scheduler. + +use notescraft_srs::{EASE_MAX, EASE_MIN, EASE_START, Grade, MAX_INTERVAL_DAYS, State, is_due, new_state, review}; + +const TODAY: u32 = 20_000; + +#[test] +fn new_state_is_due_today_with_no_interval() { + let state = new_state(TODAY); + assert_eq!(state.ease_x1000, EASE_START); + assert_eq!(state.interval_days, 0); + assert_eq!(state.reps, 0); + assert_eq!(state.lapses, 0); + assert_eq!(state.due_day, TODAY); +} + +#[test] +fn good_gives_one_day_then_six_then_ease_times_interval() { + let first = review(new_state(TODAY), Grade::Good, TODAY); + assert_eq!(first.interval_days, 1); + assert_eq!(first.reps, 1); + assert_eq!(first.ease_x1000, EASE_START); + + let second = review(first, Grade::Good, TODAY); + assert_eq!(second.interval_days, 6); + assert_eq!(second.reps, 2); + + let third = review(second, Grade::Good, TODAY); + assert_eq!(third.interval_days, 15); + assert_eq!(third.reps, 3); +} + +#[test] +fn easy_gives_four_then_ten_then_ease_and_bonus() { + let first = review(new_state(TODAY), Grade::Easy, TODAY); + assert_eq!(first.interval_days, 4); + assert_eq!(first.ease_x1000, 2650); + + let second = review(first, Grade::Easy, TODAY); + assert_eq!(second.interval_days, 10); + assert_eq!(second.ease_x1000, 2800); + + let third = review(second, Grade::Easy, TODAY); + assert_eq!(third.interval_days, 36); + assert_eq!(third.ease_x1000, 2950); +} + +#[test] +fn hard_grows_the_interval_and_lowers_the_ease() { + let first = review(new_state(TODAY), Grade::Hard, TODAY); + assert_eq!(first.interval_days, 1); + assert_eq!(first.ease_x1000, 2350); + + let second = review(first, Grade::Hard, TODAY); + assert_eq!(second.interval_days, 1); + assert_eq!(second.ease_x1000, 2200); +} + +#[test] +fn again_resets_the_card_and_lowers_the_ease() { + let mut state = new_state(TODAY); + for _ in 0..3 { + state = review(state, Grade::Good, TODAY); + } + assert_eq!(state.interval_days, 15); + + let state = review(state, Grade::Again, TODAY); + assert_eq!(state.reps, 0); + assert_eq!(state.interval_days, 0); + assert_eq!(state.lapses, 1); + assert_eq!(state.ease_x1000, 2300); + assert_eq!(state.due_day, TODAY); +} + +#[test] +fn ease_never_falls_below_the_floor() { + let mut state = new_state(TODAY); + for _ in 0..10 { + state = review(state, Grade::Again, TODAY); + } + assert_eq!(state.ease_x1000, EASE_MIN); + assert_eq!(state.lapses, 10); +} + +#[test] +fn ease_never_rises_above_the_ceiling() { + let mut state = new_state(TODAY); + for _ in 0..40 { + state = review(state, Grade::Easy, TODAY); + assert!(state.ease_x1000 <= EASE_MAX); + } + assert_eq!(state.ease_x1000, EASE_MAX); +} + +#[test] +fn the_interval_is_capped() { + let state = State { ease_x1000: EASE_MAX, interval_days: MAX_INTERVAL_DAYS, reps: 5, lapses: 0, due_day: TODAY }; + let state = review(state, Grade::Good, TODAY); + assert_eq!(state.interval_days, MAX_INTERVAL_DAYS); + + let state = review(state, Grade::Easy, TODAY); + assert_eq!(state.interval_days, MAX_INTERVAL_DAYS); +} + +#[test] +fn a_day_at_the_end_of_the_range_does_not_overflow() { + let state = review(new_state(u32::MAX), Grade::Good, u32::MAX); + assert_eq!(state.due_day, u32::MAX); + assert!(is_due(&state, u32::MAX)); +} + +#[test] +fn is_due_holds_at_the_due_day() { + let state = review(new_state(TODAY), Grade::Good, TODAY); + assert_eq!(state.due_day, TODAY + 1); + assert!(!is_due(&state, TODAY)); + assert!(is_due(&state, TODAY + 1)); + assert!(is_due(&state, TODAY + 2)); +} + +#[test] +fn every_grade_over_a_grid_keeps_the_card_sane() { + let grid = [ + new_state(TODAY), + State { ease_x1000: EASE_MIN, interval_days: 0, reps: 0, lapses: 3, due_day: TODAY }, + State { ease_x1000: EASE_MAX, interval_days: MAX_INTERVAL_DAYS, reps: 9, lapses: 1, due_day: TODAY }, + ]; + let grades = [Grade::Again, Grade::Hard, Grade::Good, Grade::Easy]; + for state in grid { + for grade in grades { + let next = review(state, grade, TODAY); + assert!(next.due_day >= TODAY, "{grade:?} moved the card into the past"); + assert!(next.ease_x1000 >= EASE_MIN); + assert!(next.ease_x1000 <= EASE_MAX); + assert!(next.interval_days <= MAX_INTERVAL_DAYS); + assert_eq!(next.lapses, state.lapses + u32::from(grade == Grade::Again)); + } + } +} diff --git a/crates/textdiff/Cargo.toml b/crates/textdiff/Cargo.toml new file mode 100644 index 0000000..48f3cd9 --- /dev/null +++ b/crates/textdiff/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "notescraft-textdiff" +description = "word-level diff between two texts, with bounded work" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +publish = false + +[lib] +path = "src/lib.rs" + +[dependencies] +thiserror = { workspace = true } + +[lints] +workspace = true diff --git a/crates/textdiff/src/lib.rs b/crates/textdiff/src/lib.rs new file mode 100644 index 0000000..cf34317 --- /dev/null +++ b/crates/textdiff/src/lib.rs @@ -0,0 +1,202 @@ +//! Word-level diff between two texts. +//! +//! The crate splits both texts into tokens (runs of word characters, runs of +//! whitespace and single other characters), finds the shortest edit script +//! between the token sequences and reports the result as a list of [`Span`]s: +//! text present in both texts ([`Op::Same`]), only in the new text +//! ([`Op::Add`]) or only in the old one ([`Op::Del`]). +//! +//! The work is bounded: [`Limits`] caps the bytes, the tokens and the cells +//! the diff may use. When a limit is hit the call returns [`Error::TooLarge`]; +//! nothing is ever truncated silently. +//! +//! ``` +//! use notescraft_textdiff::{Limits, Op, word_diff}; +//! +//! let spans = word_diff("the quick brown fox", "the quick red fox", &Limits::default()).expect("a diff"); +//! let adds = spans.iter().filter(|span| span.op == Op::Add).count(); +//! let dels = spans.iter().filter(|span| span.op == Op::Del).count(); +//! assert_eq!((adds, dels), (1, 1)); +//! ``` + +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +mod myers; +mod tokens; + +use myers::Edit; +use tokens::Token; + +/// What happened to a run of text. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Op { + /// The text is in the old text and in the new text. + Same, + /// The text is only in the new text. + Add, + /// The text is only in the old text. + Del, +} + +/// A run of text with one operation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Span { + /// What happened to the run. + pub op: Op, + /// The text of the run. + pub text: String, +} + +/// The bounds of one diff run. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Limits { + /// The most bytes the two texts may have together. + pub max_bytes: usize, + /// The most tokens the two texts may have together. + pub max_tokens: usize, + /// The most cells the diff may compute. + pub max_cells: usize, +} + +/// Default byte budget: 2 MiB for the two texts together. +pub const DEFAULT_MAX_BYTES: usize = 2 * 1024 * 1024; + +/// Default token budget: 40 000 tokens for the two texts together. +pub const DEFAULT_MAX_TOKENS: usize = 40_000; + +/// Default cell budget: 16 000 000 cells. +pub const DEFAULT_MAX_CELLS: usize = 16_000_000; + +impl Default for Limits { + fn default() -> Self { + Self { max_bytes: DEFAULT_MAX_BYTES, max_tokens: DEFAULT_MAX_TOKENS, max_cells: DEFAULT_MAX_CELLS } + } +} + +/// The errors of the diff. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// The texts are bigger than the limits allow. + #[error("the texts are too large to diff within the given limits")] + TooLarge, +} + +/// The result of a diff call. +pub type Result = std::result::Result; + +/// The diff of two texts, as a list of spans. +/// +/// The spans keep the order of the texts: the text of every [`Op::Same`] and +/// [`Op::Del`] span, in order, is the old text and the text of every +/// [`Op::Same`] and [`Op::Add`] span, in order, is the new text. +/// +/// # Errors +/// +/// Returns [`Error::TooLarge`] when a limit of `limits` is exceeded. +pub fn word_diff(old: &str, new: &str, limits: &Limits) -> Result> { + let bytes = old.len().saturating_add(new.len()); + if bytes > limits.max_bytes { + return Err(Error::TooLarge); + } + + let old_tokens = tokens::tokenize(old); + let new_tokens = tokens::tokenize(new); + let count = old_tokens.len().saturating_add(new_tokens.len()); + if count > limits.max_tokens { + return Err(Error::TooLarge); + } + + let (prefix, suffix) = common_edges(&old_tokens, &new_tokens); + let old_mid = middle(&old_tokens, prefix, suffix); + let new_mid = middle(&new_tokens, prefix, suffix); + + let edits = myers::diff(&old_mid, &new_mid, limits.max_cells)?; + + let mut spans: Vec = Vec::new(); + for token in old_tokens.iter().take(prefix) { + push_span(&mut spans, Op::Same, &token.text); + } + let mut old_iter = old_mid.iter(); + let mut new_iter = new_mid.iter(); + for edit in edits { + match edit { + Edit::Equal(len) => { + for (old_token, _) in old_iter.by_ref().zip(new_iter.by_ref()).take(len) { + push_span(&mut spans, Op::Same, &old_token.text); + } + } + Edit::Delete(len) => { + for token in old_iter.by_ref().take(len) { + push_span(&mut spans, Op::Del, &token.text); + } + } + Edit::Insert(len) => { + for token in new_iter.by_ref().take(len) { + push_span(&mut spans, Op::Add, &token.text); + } + } + } + } + for token in old_tokens.iter().skip(old_tokens.len() - suffix) { + push_span(&mut spans, Op::Same, &token.text); + } + Ok(spans) +} + +/// The number of words added and removed by a diff. +/// +/// Words are runs of word characters; whitespace and punctuation are not +/// counted. +#[must_use] +pub fn stats(spans: &[Span]) -> (usize, usize) { + let mut added = 0usize; + let mut removed = 0usize; + for span in spans { + let words = tokens::count_words(&span.text); + match span.op { + Op::Add => added += words, + Op::Del => removed += words, + Op::Same => {} + } + } + (added, removed) +} + +/// The number of tokens both texts share at the start and at the end. +fn common_edges(old: &[Token], new: &[Token]) -> (usize, usize) { + let shortest = old.len().min(new.len()); + let mut prefix = 0usize; + while prefix < shortest && old.get(prefix) == new.get(prefix) { + prefix += 1; + } + let mut suffix = 0usize; + while suffix < shortest - prefix { + let old_at = old.len() - 1 - suffix; + let new_at = new.len() - 1 - suffix; + if old.get(old_at) != new.get(new_at) { + break; + } + suffix += 1; + } + (prefix, suffix) +} + +/// The tokens between the common prefix and the common suffix. +fn middle(tokens: &[Token], prefix: usize, suffix: usize) -> Vec { + tokens.iter().skip(prefix).take(tokens.len() - prefix - suffix).cloned().collect() +} + +/// Appends a run of text to the spans, merging it with the last one when the +/// operation is the same. +fn push_span(spans: &mut Vec, op: Op, text: &str) { + if text.is_empty() { + return; + } + if let Some(last) = spans.last_mut() + && last.op == op + { + last.text.push_str(text); + return; + } + spans.push(Span { op, text: text.to_string() }); +} diff --git a/crates/textdiff/src/myers.rs b/crates/textdiff/src/myers.rs new file mode 100644 index 0000000..57e7126 --- /dev/null +++ b/crates/textdiff/src/myers.rs @@ -0,0 +1,191 @@ +//! Myers' shortest edit script over two token sequences. +//! +//! The script is built in two passes: the forward pass walks the diagonals of +//! the edit graph and keeps the furthest point reached on each diagonal; the +//! back pass walks the kept points backwards to recover the script. The work +//! of both passes is counted in cells and the diff stops with +//! [`Error::TooLarge`] when the budget is spent. + +use crate::tokens::Token; +use crate::{Error, Result}; + +/// One run of the edit script. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Edit { + /// The next tokens are equal in both texts. + Equal(usize), + /// The next tokens of the old text are gone. + Delete(usize), + /// The next tokens of the new text are new. + Insert(usize), +} + +/// One step of the script while it is being recovered. +#[derive(Debug, Clone, Copy)] +enum Step { + Equal, + Delete, + Insert, +} + +/// The points the forward pass kept for the back pass. +struct Forward { + /// One row per level, holding the diagonals of the level. + rows: Vec>, + /// The last level the forward pass reached. + last_d: usize, +} + +/// The diff of two token sequences, as the runs of its edit script. +pub(crate) fn diff(old: &[Token], new: &[Token], max_cells: usize) -> Result> { + let forward = forward_pass(old, new, max_cells)?; + let steps = back_pass(old, new, &forward)?; + Ok(compress(steps)) +} + +/// Walks the diagonals of the edit graph and keeps the points of every level. +fn forward_pass(old: &[Token], new: &[Token], max_cells: usize) -> Result { + let old_len = old.len(); + let new_len = new.len(); + let old_last = i32::try_from(old_len).map_err(|_| Error::TooLarge)?; + let new_last = i32::try_from(new_len).map_err(|_| Error::TooLarge)?; + let offset = old_last + new_last; + let mut points = vec![0i32; 2 * (old_len + new_len) + 1]; + let mut rows: Vec> = Vec::new(); + let mut cells = 0usize; + let mut last_d = None; + + for level in 0..=old_len + new_len { + let level_i = i32::try_from(level).map_err(|_| Error::TooLarge)?; + if level >= 1 { + let len = 2 * level - 1; + let start = usize::try_from(offset - (level_i - 1)).map_err(|_| Error::TooLarge)?; + let row: Vec = points.iter().skip(start).take(len).copied().collect(); + cells += len; + if cells > max_cells { + return Err(Error::TooLarge); + } + rows.push(row); + } + let mut diag = -level_i; + while diag <= level_i { + let down = diag == -level_i || (diag != level_i && diagonal(&points, offset, diag - 1) < diagonal(&points, offset, diag + 1)); + let mut pos_x = if down { diagonal(&points, offset, diag + 1) } else { diagonal(&points, offset, diag - 1) + 1 }; + let mut pos_y = pos_x - diag; + let mut steps = 0usize; + while same(old, new, pos_x, pos_y) { + pos_x += 1; + pos_y += 1; + steps += 1; + } + cells += 1 + steps; + if cells > max_cells { + return Err(Error::TooLarge); + } + let index = usize::try_from(diag + offset).map_err(|_| Error::TooLarge)?; + if let Some(slot) = points.get_mut(index) { + *slot = pos_x; + } + if pos_x >= old_last && pos_y >= new_last { + last_d = Some(level); + break; + } + diag += 2; + } + if last_d.is_some() { + break; + } + } + + let last_d = last_d.ok_or(Error::TooLarge)?; + Ok(Forward { rows, last_d }) +} + +/// Walks the kept points backwards to recover the steps of the script. +fn back_pass(old: &[Token], new: &[Token], forward: &Forward) -> Result> { + let old_last = i32::try_from(old.len()).map_err(|_| Error::TooLarge)?; + let new_last = i32::try_from(new.len()).map_err(|_| Error::TooLarge)?; + let mut steps: Vec = Vec::new(); + let mut pos_x = old_last; + let mut pos_y = new_last; + + for (level, row) in (1..=forward.last_d).rev().zip(forward.rows.iter().rev()) { + let level_i = i32::try_from(level).map_err(|_| Error::TooLarge)?; + let base = -(level_i - 1); + let diag = pos_x - pos_y; + let down = diag == -level_i || (diag != level_i && row_diagonal(row, base, diag - 1) < row_diagonal(row, base, diag + 1)); + let prev_diag = if down { diag + 1 } else { diag - 1 }; + let prev_x = row_diagonal(row, base, prev_diag); + let prev_y = prev_x - prev_diag; + while pos_x > prev_x && pos_y > prev_y { + steps.push(Step::Equal); + pos_x -= 1; + pos_y -= 1; + } + if pos_x == prev_x { + steps.push(Step::Insert); + pos_y -= 1; + } else { + steps.push(Step::Delete); + pos_x -= 1; + } + } + + while pos_x > 0 && pos_y > -1 { + steps.push(Step::Equal); + pos_x -= 1; + pos_y -= 1; + } + steps.reverse(); + Ok(steps) +} + +/// Joins the steps of the script into runs. +fn compress(steps: Vec) -> Vec { + let mut edits: Vec = Vec::new(); + for step in steps { + match step { + Step::Equal => merge(&mut edits, Edit::Equal(1)), + Step::Delete => merge(&mut edits, Edit::Delete(1)), + Step::Insert => merge(&mut edits, Edit::Insert(1)), + } + } + edits +} + +/// Adds a run to the script, joining it with the last one when they match. +fn merge(edits: &mut Vec, edit: Edit) { + match (edits.last_mut(), edit) { + (Some(Edit::Equal(count)), Edit::Equal(more)) | (Some(Edit::Delete(count)), Edit::Delete(more)) | (Some(Edit::Insert(count)), Edit::Insert(more)) => { + *count += more; + } + _ => edits.push(edit), + } +} + +/// Whether the tokens at the given positions are equal. +fn same(old: &[Token], new: &[Token], pos_x: i32, pos_y: i32) -> bool { + let (Ok(old_at), Ok(new_at)) = (usize::try_from(pos_x), usize::try_from(pos_y)) else { + return false; + }; + match (old.get(old_at), new.get(new_at)) { + (Some(left), Some(right)) => left == right, + _ => false, + } +} + +/// The furthest point reached on a diagonal of the live graph. +fn diagonal(points: &[i32], offset: i32, diag: i32) -> i32 { + match usize::try_from(diag + offset) { + Ok(at) => points.get(at).copied().unwrap_or_default(), + Err(_) => 0, + } +} + +/// The furthest point reached on a diagonal of a kept row. +fn row_diagonal(row: &[i32], base: i32, diag: i32) -> i32 { + match usize::try_from(diag - base) { + Ok(at) => row.get(at).copied().unwrap_or_default(), + Err(_) => 0, + } +} diff --git a/crates/textdiff/src/tokens.rs b/crates/textdiff/src/tokens.rs new file mode 100644 index 0000000..3eba434 --- /dev/null +++ b/crates/textdiff/src/tokens.rs @@ -0,0 +1,101 @@ +//! Splitting a text into the tokens the diff works on. + +/// What a token is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TokenKind { + /// A run of word characters: letters, digits or `_`. + Word, + /// A run of whitespace. + Space, + /// A single other character. + Other, +} + +/// One token of a text. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Token { + /// The kind of the token. + pub kind: TokenKind, + /// The text of the token. + pub text: String, +} + +/// Splits a text into word runs, whitespace runs and single other characters. +pub fn tokenize(text: &str) -> Vec { + let mut tokens: Vec = Vec::new(); + let mut chars = text.chars().peekable(); + while let Some(first) = chars.next() { + if is_word_char(first) { + let mut run = String::new(); + run.push(first); + while let Some(&next) = chars.peek() + && is_word_char(next) + { + run.push(next); + chars.next(); + } + tokens.push(Token { kind: TokenKind::Word, text: run }); + } else if first.is_whitespace() { + let mut run = String::new(); + run.push(first); + while let Some(&next) = chars.peek() + && next.is_whitespace() + { + run.push(next); + chars.next(); + } + tokens.push(Token { kind: TokenKind::Space, text: run }); + } else { + tokens.push(Token { kind: TokenKind::Other, text: first.to_string() }); + } + } + tokens +} + +/// The number of word tokens in a text. +pub fn count_words(text: &str) -> usize { + tokenize(text).iter().filter(|token| token.kind == TokenKind::Word).count() +} + +/// Whether a character belongs to a word. +fn is_word_char(ch: char) -> bool { + ch.is_alphanumeric() || ch == '_' +} + +#[cfg(test)] +mod tests { + use super::{TokenKind, count_words, tokenize}; + + #[test] + fn splits_words_spaces_and_single_characters() { + let tokens = tokenize("hi there!"); + assert_eq!(tokens.len(), 4); + assert_eq!(tokens.first().map(|token| token.text.as_str()), Some("hi")); + assert_eq!(tokens.first().map(|token| token.kind), Some(TokenKind::Word)); + assert_eq!(tokens.get(1).map(|token| token.kind), Some(TokenKind::Space)); + assert_eq!(tokens.get(2).map(|token| token.text.as_str()), Some("there")); + assert_eq!(tokens.get(3).map(|token| token.text.as_str()), Some("!")); + assert_eq!(tokens.get(3).map(|token| token.kind), Some(TokenKind::Other)); + } + + #[test] + fn keeps_underscores_and_digits_in_words() { + let tokens = tokenize("a_1 2b"); + assert_eq!(tokens.first().map(|token| token.text.as_str()), Some("a_1")); + assert_eq!(tokens.get(2).map(|token| token.text.as_str()), Some("2b")); + } + + #[test] + fn treats_a_combining_mark_as_its_own_character() { + let tokens = tokenize("e\u{0301}"); + assert_eq!(tokens.len(), 2); + assert_eq!(tokens.get(1).map(|token| token.kind), Some(TokenKind::Other)); + } + + #[test] + fn counts_only_words() { + assert_eq!(count_words("one two, three!"), 3); + assert_eq!(count_words(" "), 0); + assert_eq!(count_words(""), 0); + } +} diff --git a/crates/textdiff/tests/word_diff.rs b/crates/textdiff/tests/word_diff.rs new file mode 100644 index 0000000..67127a5 --- /dev/null +++ b/crates/textdiff/tests/word_diff.rs @@ -0,0 +1,195 @@ +//! Word-level diff of two texts. + +use notescraft_textdiff::{Error, Limits, Op, Span, stats, word_diff}; + +/// A small linear congruential generator, so the cases are the same on every +/// run. +struct Lcg(u64); + +impl Lcg { + fn new(seed: u64) -> Self { + Self(seed) + } + + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); + self.0 >> 33 + } + + fn below(&mut self, limit: usize) -> usize { + let limit = u64::try_from(limit).unwrap_or(1); + usize::try_from(self.next() % limit).unwrap_or(0) + } +} + +const WORDS: [&str; 8] = ["alpha", "beta", "gamma", "delta", "eps", "zeta", "eta", "theta"]; + +fn random_text(rng: &mut Lcg) -> String { + let count = rng.below(12); + let mut out = String::new(); + for index in 0..count { + if index > 0 { + out.push(' '); + } + out.push_str(WORDS.get(rng.below(WORDS.len())).copied().unwrap_or("word")); + if rng.below(6) == 0 { + out.push(','); + } + } + out +} + +/// The old text of a diff: the same spans and the deletions, in order. +fn old_text(spans: &[Span]) -> String { + let mut out = String::new(); + for span in spans { + match span.op { + Op::Same | Op::Del => out.push_str(&span.text), + Op::Add => {} + } + } + out +} + +/// The new text of a diff: the same spans and the additions, in order. +fn new_text(spans: &[Span]) -> String { + let mut out = String::new(); + for span in spans { + match span.op { + Op::Same | Op::Add => out.push_str(&span.text), + Op::Del => {} + } + } + out +} + +fn count(spans: &[Span], op: Op) -> usize { + spans.iter().filter(|span| span.op == op).count() +} + +#[test] +fn reproduces_both_texts_for_a_thousand_pairs() { + let mut rng = Lcg::new(0x5EED_1234); + for _ in 0..1000 { + let old = random_text(&mut rng); + let new = if rng.below(2) == 0 { random_text(&mut rng) } else { old.clone() }; + let spans = word_diff(&old, &new, &Limits::default()).expect("a diff"); + assert_eq!(old_text(&spans), old, "the old text must be reproduced"); + assert_eq!(new_text(&spans), new, "the new text must be reproduced"); + } +} + +#[test] +fn one_changed_word_gives_one_deletion_and_one_addition() { + let old = "the quick brown fox jumps over the lazy dog and keeps running until the end of the paragraph"; + let new = old.replace("brown", "red"); + let spans = word_diff(old, &new, &Limits::default()).expect("a diff"); + assert_eq!(count(&spans, Op::Del), 1); + assert_eq!(count(&spans, Op::Add), 1); + assert_eq!(old_text(&spans), old); + assert_eq!(new_text(&spans), new); + assert_eq!(stats(&spans), (1, 1)); +} + +#[test] +fn inserts_at_the_start_and_at_the_end() { + let old = "middle"; + let new = "start middle end"; + let spans = word_diff(old, new, &Limits::default()).expect("a diff"); + assert_eq!(old_text(&spans), old); + assert_eq!(new_text(&spans), new); + assert_eq!(count(&spans, Op::Add), 2); + assert_eq!(count(&spans, Op::Same), 1); + assert_eq!(stats(&spans), (2, 0)); +} + +#[test] +fn an_empty_old_text_is_all_additions() { + let spans = word_diff("", "hello world", &Limits::default()).expect("a diff"); + assert_eq!(count(&spans, Op::Add), 1); + assert_eq!(count(&spans, Op::Del), 0); + assert_eq!(new_text(&spans), "hello world"); + assert_eq!(old_text(&spans), ""); +} + +#[test] +fn an_empty_new_text_is_all_deletions() { + let spans = word_diff("hello world", "", &Limits::default()).expect("a diff"); + assert_eq!(count(&spans, Op::Del), 1); + assert_eq!(count(&spans, Op::Add), 0); + assert_eq!(old_text(&spans), "hello world"); + assert_eq!(new_text(&spans), ""); +} + +#[test] +fn two_empty_texts_give_no_spans() { + let spans = word_diff("", "", &Limits::default()).expect("a diff"); + assert_eq!(spans, Vec::::new()); + assert_eq!(stats(&spans), (0, 0)); +} + +#[test] +fn carriage_returns_stay_in_the_whitespace() { + let old = "first\r\nsecond\r\n"; + let new = "first\r\nthird\r\n"; + let spans = word_diff(old, new, &Limits::default()).expect("a diff"); + assert_eq!(old_text(&spans), old); + assert_eq!(new_text(&spans), new); + assert_eq!(count(&spans, Op::Del), 1); + assert_eq!(count(&spans, Op::Add), 1); +} + +#[test] +fn emoji_and_combining_marks_are_never_lost() { + let old = "cafe\u{0301} \u{1F600} e\u{0301}"; + let new = "caf\u{e9} \u{1F600} \u{e8}"; + let spans = word_diff(old, new, &Limits::default()).expect("a diff"); + assert_eq!(old_text(&spans), old); + assert_eq!(new_text(&spans), new); +} + +#[test] +fn too_many_bytes_is_refused() { + let limits = Limits { max_bytes: 8, ..Limits::default() }; + let result = word_diff("a longer old text", "a longer new text", &limits); + assert!(matches!(result, Err(Error::TooLarge))); +} + +#[test] +fn too_many_tokens_is_refused() { + let limits = Limits { max_tokens: 4, ..Limits::default() }; + let result = word_diff("one two three four five", "one two three four five", &limits); + assert!(matches!(result, Err(Error::TooLarge))); +} + +#[test] +fn too_many_cells_is_refused() { + let limits = Limits { max_cells: 0, ..Limits::default() }; + let result = word_diff("one two", "three four", &limits); + assert!(matches!(result, Err(Error::TooLarge))); +} + +#[test] +fn the_default_limits_accept_a_normal_note() { + let note = "a note with a few hundred words, repeated for a while to look like a real note in a vault"; + let spans = word_diff(note, note, &Limits::default()).expect("a diff"); + assert_eq!(count(&spans, Op::Same), 1); + assert_eq!(count(&spans, Op::Add), 0); + assert_eq!(count(&spans, Op::Del), 0); +} + +#[test] +#[ignore = "slow: two texts of 2 MiB"] +fn two_big_texts_are_fast_or_too_large() { + let big = "word ".repeat(400_000); + let start = std::time::Instant::now(); + let result = word_diff(big.as_str(), big.as_str(), &Limits::default()); + let elapsed = start.elapsed(); + match result { + Ok(spans) => { + assert_eq!(count(&spans, Op::Same), 1); + assert!(elapsed.as_millis() < 500, "the diff took {elapsed:?}"); + } + Err(Error::TooLarge) => {} + } +} diff --git a/crates/ui-egui/examples/snapshot.rs b/crates/ui-egui/examples/snapshot.rs index d00e1a4..de7b47a 100644 --- a/crates/ui-egui/examples/snapshot.rs +++ b/crates/ui-egui/examples/snapshot.rs @@ -117,6 +117,9 @@ fn main() { if args.iter().any(|a| a == "--settings") { app.open_note(notescraft_ui_egui::tabs::SETTINGS_PATH); } + if args.iter().any(|arg| arg == "--registry") { + app.open_registry(); + } // `--right-tab related` opens that right-card tab; `--related-json ` shows a canned `knowledge.related` result. if let Some(tab) = arg(&args, "--right-tab") { use notescraft_ui_egui::right_view::RightTab; diff --git a/crates/ui-egui/src/design/fonts/mod.rs b/crates/ui-egui/src/design/fonts/mod.rs index efd0964..0bfe604 100644 --- a/crates/ui-egui/src/design/fonts/mod.rs +++ b/crates/ui-egui/src/design/fonts/mod.rs @@ -1,6 +1,9 @@ //! Fonts: `Inter` for the UI, `JetBrains Mono` for code and `STIX Two Math` for math, //! shipped in `assets/fonts` like the other Craft apps (SIL OFL 1.1, see `ATTRIBUTION.md`). //! +//! The `Inter` and `JetBrains Mono` programs come from the engine, which embeds the same +//! bytes for the PDF export: the binary carries each font once. +//! //! Every other script (Indic, Thai, CJK, ...) is loaded lazily from the system fonts: [`ensure_scripts`] scans //! text and appends the font of each script it finds, once. egui does no OpenType shaping, so conjuncts and //! reordering marks (Bengali, Devanagari, Tamil, Thai, Khmer, ...) draw as unjoined letters. @@ -14,34 +17,29 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use egui::{FontData, FontDefinitions, FontFamily}; +use notescraft_engine::{INTER_REGULAR, INTER_SEMIBOLD, JETBRAINS_MONO}; -const INTER_REGULAR: &[u8] = include_bytes!("../../../../../assets/fonts/Inter-Regular.ttf"); const INTER_MEDIUM: &[u8] = include_bytes!("../../../../../assets/fonts/Inter-Medium.ttf"); -const INTER_SEMIBOLD: &[u8] = include_bytes!("../../../../../assets/fonts/Inter-SemiBold.ttf"); -const JETBRAINS_MONO: &[u8] = include_bytes!("../../../../../assets/fonts/JetBrainsMono-NoLigatures.ttf"); const STIX_MATH: &[u8] = include_bytes!("../../../../../assets/fonts/STIXTwoMath-Regular.ttf"); +const NOTO_EMOJI: &[u8] = include_bytes!("../../../../../assets/fonts/NotoEmoji-Regular.ttf"); +const NOTO_EMOJI_NAME: &str = "noto-emoji"; -/// Fonts the system ships with Hebrew and Arabic letters, read when present: the bundled fonts have none, and a -/// line of right-to-left text would otherwise show empty boxes. -const SYSTEM_FALLBACKS: [&str; 7] = [ - "/System/Library/Fonts/SFHebrew.ttf", - "/System/Library/Fonts/SFArabic.ttf", - "C:\\Windows\\Fonts\\segoeui.ttf", - "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", - "/usr/share/fonts/dejavu/DejaVuSans.ttf", - "/usr/share/fonts/truetype/noto/NotoSansHebrew-Regular.ttf", - "/usr/share/fonts/noto/NotoSansHebrew-Regular.ttf", -]; -/// Largest system font read, in bytes. -const MAX_FALLBACK_BYTES: u64 = 24 * 1024 * 1024; +/// The regular `Inter` bytes the UI installs, for the shared-copy test. +#[cfg(test)] +fn inter_regular() -> &'static [u8] { + INTER_REGULAR +} -/// The bytes of the system fallback fonts that exist here. -fn system_fallbacks() -> Vec> { - SYSTEM_FALLBACKS - .iter() - .filter(|path| std::fs::metadata(path).is_ok_and(|meta| meta.len() <= MAX_FALLBACK_BYTES)) - .filter_map(|path| std::fs::read(path).ok()) - .collect() +/// The semibold `Inter` bytes the UI installs, for the shared-copy test. +#[cfg(test)] +fn inter_semibold() -> &'static [u8] { + INTER_SEMIBOLD +} + +/// The `JetBrains Mono` bytes the UI installs, for the shared-copy test. +#[cfg(test)] +fn jetbrains_mono() -> &'static [u8] { + JETBRAINS_MONO } /// Named family for medium-weight text. @@ -70,32 +68,27 @@ pub fn semibold() -> FontFamily { } /// Installs `Inter` as the proportional font, `JetBrains Mono` as the monospace font and the two -/// named weight families. egui's bundled fonts stay behind them as glyph fallbacks. +/// named weight families. Noto Emoji and STIX follow them as glyph fallbacks; egui's own default fonts are not +/// compiled in (`default_fonts` is off), so the binary carries each face once. pub fn install(ctx: &egui::Context) { - let mut fonts = FontDefinitions::default(); - for (name, bytes) in [("inter", INTER_REGULAR), (MEDIUM, INTER_MEDIUM), (SEMIBOLD, INTER_SEMIBOLD), ("jetbrains-mono", JETBRAINS_MONO), (MATH, STIX_MATH)] { + let mut fonts = FontDefinitions::empty(); + for (name, bytes) in [ + ("inter", INTER_REGULAR), + (MEDIUM, INTER_MEDIUM), + (SEMIBOLD, INTER_SEMIBOLD), + ("jetbrains-mono", JETBRAINS_MONO), + (MATH, STIX_MATH), + (NOTO_EMOJI_NAME, NOTO_EMOJI), + ] { fonts.font_data.insert(name.to_owned(), Arc::new(FontData::from_static(bytes))); } - let proportional = fonts.families.entry(FontFamily::Proportional).or_default(); - proportional.insert(0, "inter".to_owned()); - let mut fallback_names = Vec::new(); - for (index, bytes) in system_fallbacks().into_iter().enumerate() { - let name = format!("system-fallback-{index}"); - fonts.font_data.insert(name.clone(), Arc::new(FontData::from_owned(bytes))); - if let Some(chain) = fonts.families.get_mut(&FontFamily::Proportional) { - chain.push(name.clone()); - } - fallback_names.push(name); - } + // Hebrew, Arabic and the other scripts load on demand through `note_text`; nothing is read from disk here. // STIX Two Math is the last resort of every text family, so symbols, arrows and math operators never show boxes. - if let Some(chain) = fonts.families.get_mut(&FontFamily::Proportional) { - chain.push(MATH.to_owned()); - } + let proportional = fonts.families.entry(FontFamily::Proportional).or_default(); + proportional.extend(["inter", NOTO_EMOJI_NAME, MATH].map(str::to_owned)); let monospace = fonts.families.entry(FontFamily::Monospace).or_default(); - monospace.insert(0, "jetbrains-mono".to_owned()); - // Hebrew and Arabic inside code (strings, comments) must not show boxes either. - monospace.extend(fallback_names); - monospace.push(MATH.to_owned()); + // Inter backs the monospace chain for the few symbols JetBrains Mono lacks (the removed egui fonts did the same). + monospace.extend(["jetbrains-mono", "inter", NOTO_EMOJI_NAME, MATH].map(str::to_owned)); for (family, face) in [(MEDIUM, MEDIUM), (SEMIBOLD, SEMIBOLD)] { let mut chain = vec![face.to_owned(), "inter".to_owned()]; if let Some(base) = fonts.families.get(&FontFamily::Proportional) { @@ -141,7 +134,7 @@ fn covers(script: &scripts::Script, code: u32) -> bool { fn detect(text: &str) -> u64 { let mut found = 0_u64; let mut bytes = 0_usize; - let mut last = 0_usize; + let mut last: Option = None; for c in text.chars().take(MAX_SCAN_CHARS) { bytes += c.len_utf8(); if bytes > MAX_SCAN_BYTES { @@ -149,12 +142,12 @@ fn detect(text: &str) -> u64 { } let code = u32::from(c); // Runs of one script are the common case: test the previous hit first. - if code < scripts::FIRST_SCRIPT_CHAR || scripts::SCRIPTS.get(last).is_some_and(|script| covers(script, code)) { + if code < scripts::FIRST_SCRIPT_CHAR || last.and_then(|at| scripts::SCRIPTS.get(at)).is_some_and(|script| covers(script, code)) { continue; } if let Some(index) = scripts::SCRIPTS.iter().position(|script| covers(script, code)) { found |= 1 << index; - last = index; + last = Some(index); } } found diff --git a/crates/ui-egui/src/design/fonts/scripts.rs b/crates/ui-egui/src/design/fonts/scripts.rs index 9005cf6..f160eee 100644 --- a/crates/ui-egui/src/design/fonts/scripts.rs +++ b/crates/ui-egui/src/design/fonts/scripts.rs @@ -45,6 +45,8 @@ macro_rules! face { /// Every lazily loaded script; the position is the bit in the loaded set (at most 64 entries). #[rustfmt::skip] pub(super) const SCRIPTS: &[Script] = &[ + Script { name: "hebrew", ranges: &[(0x0590, 0x05FF), (0xFB1D, 0xFB4F)], files: &[face!("/System/Library/Fonts/", "SFHebrew.ttf"), face!("C:\\Windows\\Fonts\\", "segoeui.ttf")], noto: &[("NotoSansHebrew-Regular.ttf", 0)] }, + Script { name: "arabic", ranges: &[(0x0600, 0x06FF), (0x0750, 0x077F), (0x08A0, 0x08FF), (0xFB50, 0xFDFF), (0xFE70, 0xFEFF)], files: &[face!("/System/Library/Fonts/", "SFArabic.ttf"), face!("C:\\Windows\\Fonts\\", "segoeui.ttf")], noto: &[("NotoSansArabic-Regular.ttf", 0)] }, Script { name: "armenian", ranges: &[(0x0530, 0x058F), (0xFB13, 0xFB17)], files: &[face!("/System/Library/Fonts/", "SFArmenian.ttf"), face!("/System/Library/Fonts/Supplemental/", "Mshtakan.ttc"), face!("C:\\Windows\\Fonts\\", "segoeui.ttf")], noto: &[("NotoSansArmenian-Regular.ttf", 0)] }, Script { name: "georgian", ranges: &[(0x10A0, 0x10FF), (0x1C90, 0x1CBF), (0x2D00, 0x2D2F)], files: &[face!("/System/Library/Fonts/", "SFGeorgian.ttf"), face!("C:\\Windows\\Fonts\\", "segoeui.ttf")], noto: &[("NotoSansGeorgian-Regular.ttf", 0)] }, Script { name: "devanagari", ranges: &[(0x0900, 0x097F), (0xA8E0, 0xA8FF)], files: &[face!("/System/Library/Fonts/Supplemental/", "Devanagari Sangam MN.ttc"), face!("/System/Library/Fonts/Supplemental/", "DevanagariMT.ttc"), face!("C:\\Windows\\Fonts\\", "Nirmala.ttf")], noto: &[("NotoSansDevanagari-Regular.ttf", 0)] }, diff --git a/crates/ui-egui/src/design/fonts/tests.rs b/crates/ui-egui/src/design/fonts/tests.rs index 6829f71..308bd57 100644 --- a/crates/ui-egui/src/design/fonts/tests.rs +++ b/crates/ui-egui/src/design/fonts/tests.rs @@ -19,12 +19,13 @@ fn detects_each_script() { assert_eq!(detect("ใฒใ‚‰ใŒใช"), bit("kana")); assert_eq!(detect("ํ•œ๊ตญ์–ด"), bit("hangul")); assert_eq!(detect("\u{20000}"), bit("han")); + assert_eq!(detect("ืขื‘ืจื™ืช"), bit("hebrew")); + assert_eq!(detect("ุงู„ุนุฑุจูŠุฉ"), bit("arabic")); } #[test] fn bundled_and_startup_scripts_need_nothing() { assert_eq!(detect("Hello ะŸั€ะธะฒะตั‚ ฮ“ฮตฮนฮฌ"), 0); - assert_eq!(detect("ืขื‘ืจื™ืช ุงู„ุนุฑุจูŠุฉ"), 0); assert_eq!(detect("๐Ÿ˜€ โœ“"), 0); assert_eq!(detect(""), 0); } @@ -143,6 +144,22 @@ fn symbols_never_show_boxes_in_any_text_family() { } } +#[test] +fn the_ui_and_the_exporter_share_one_inter_copy() { + assert!( + std::ptr::eq(notescraft_engine::INTER_REGULAR.as_ptr(), inter_regular().as_ptr()), + "the UI must install the engine's Inter copy, not embed its own" + ); + assert!( + std::ptr::eq(notescraft_engine::INTER_SEMIBOLD.as_ptr(), inter_semibold().as_ptr()), + "the UI must install the engine's Inter copy, not embed its own" + ); + assert!( + std::ptr::eq(notescraft_engine::JETBRAINS_MONO.as_ptr(), jetbrains_mono().as_ptr()), + "the UI must install the engine's JetBrains Mono copy, not embed its own" + ); +} + #[test] fn math_stays_last_when_a_script_font_is_added() { let ctx = egui::Context::default(); @@ -158,3 +175,34 @@ fn math_stays_last_when_a_script_font_is_added() { assert!(chain.iter().any(|n| n == "script-bengali")); } } + +#[test] +fn install_reads_no_system_font_and_hebrew_arabic_load_on_demand() { + let ctx = installed(); + let before = font_count(&ctx); + let reads = std::cell::Cell::new(0_u32); + let read = |_: &str| { + reads.set(reads.get() + 1); + Some(vec![0_u8; 4]) + }; + assert!(!add_scripts(&ctx, detect("Hello"), |_| Some(10), read)); + assert_eq!(reads.get(), 0, "plain text reads nothing"); + assert!(add_scripts(&ctx, detect("ืขื‘ืจื™ืช ุงู„ุนุฑุจูŠุฉ"), |_| Some(10), read)); + assert_eq!(reads.get(), 2); + assert_eq!(font_count(&ctx), before + 2); +} + +#[test] +// Astral colour emoji (U+1F600) have no glyph here or in egui's default fonts; the symbol range still resolves. +fn emoji_and_symbols_resolve_without_the_egui_default_fonts() { + let ctx = installed(); + ctx.run_ui(egui::RawInput::default(), |_| {}).textures_delta.clear(); + for family in [FontFamily::Proportional, FontFamily::Monospace] { + let missing: Vec = ['\u{2764}', '\u{2605}', '\u{2192}', '\u{2713}'] + .into_iter() + .filter(|ch| !resolves(&ctx, family.clone(), *ch)) + .map(|ch| format!("U+{:04X}", u32::from(ch))) + .collect(); + assert!(missing.is_empty(), "{family:?} lacks {missing:?}"); + } +} diff --git a/crates/ui-egui/src/shell/app.rs b/crates/ui-egui/src/shell/app.rs index 02368d4..6161e7e 100644 --- a/crates/ui-egui/src/shell/app.rs +++ b/crates/ui-egui/src/shell/app.rs @@ -56,6 +56,7 @@ const REMINDER_PERIOD: f64 = 30.0; /// UI-string constants (they move to the i18n crate later). pub use large_folder::LargeFolderGate; pub use vault::explorer::focus_filter_id as explorer_focus_filter_id; +pub use vault::registry::RegistryError; pub use vault_switch::VaultRequest; pub use window::menubar::{EditAction, Menu, MenuEntry, MenuItem, MenuRole, MenuTarget, Predefined, menu_model}; @@ -169,6 +170,8 @@ pub struct NotesApp { pub(crate) batch_review: Option, /// The "Add a grammar" dialog, while it is open. pub(crate) grammar_dialog: Option, + /// Registry command queue and its owned worker dispatcher. + pub(crate) registry_jobs: vault::registry::RegistryJobs, recent_vaults: Vec, watch_updates: Option>, /// When the reminders are next checked (egui time); none until the start-up check ran. @@ -304,6 +307,7 @@ impl NotesApp { link_hover: None, batch_review: None, grammar_dialog: None, + registry_jobs: vault::registry::RegistryJobs::default(), recent_vaults: Vec::new(), watch_updates, reminder_next: None, diff --git a/crates/ui-egui/src/shell/app/vault/first_run/mod.rs b/crates/ui-egui/src/shell/app/vault/first_run/mod.rs new file mode 100644 index 0000000..a99687f --- /dev/null +++ b/crates/ui-egui/src/shell/app/vault/first_run/mod.rs @@ -0,0 +1,133 @@ +//! First-run add-on suggestions, rendered from cached data supplied by the shell. + +#[path = "wanted_banner.rs"] +mod wanted_banner; + +use crate::shell::app::NotesApp; +use crate::state::UiState; +use serde_json::json; +use std::collections::BTreeMap; + +/// Cached alias lookups, discarded whenever the catalogue changes. +#[derive(Clone, Default)] +struct Offers { + catalogue: String, + labels: BTreeMap>, +} + +impl NotesApp { + /// Pumps cache-only grammar suggestions and explicit install clicks after painting. + pub(in crate::shell::app) fn pump_registry_offers(&mut self, ctx: &egui::Context) { + let vault = self.session.vault().root().to_path_buf(); + let opened_id = egui::Id::new(("registry-offers-opened", &vault)); + if !ctx.data(|data| data.get_temp::(opened_id)).unwrap_or(false) { + if let Some(reply) = self.engine_command("registry.list", &json!({"limit": 512})) { + self.preferences.view.registry.apply_reply(&reply); + } + ctx.data_mut(|data| data.insert_temp(opened_id, true)); + } + let cache_id = egui::Id::new(("registry-offers", &vault)); + let mut offers = ctx.data_mut(|data| data.remove_temp::(cache_id)).unwrap_or_default(); + let catalogue = serde_json::to_string(&self.preferences.view.registry.rows).unwrap_or_default(); + if offers.catalogue != catalogue { + for label in offers.labels.keys() { + crate::note_view::clear_registry_offer(ctx, label); + } + offers.labels.clear(); + offers.catalogue = catalogue; + } + for label in crate::note_view::take_registry_labels(ctx) { + if !offers.labels.contains_key(&label) { + let entry = self + .engine_command("registry.suggest", &json!({"label": label})) + .and_then(|reply| reply.get("entries")?.as_array()?.first().cloned()) + .and_then(|entry| serde_json::from_value::(entry).ok()) + .filter(|entry| entry.state == "available"); + if entry.is_some() { + ctx.request_repaint(); + } + offers.labels.insert(label.clone(), entry); + } + if let Some(Some(entry)) = offers.labels.get(&label) { + crate::note_view::set_registry_offer(ctx, &label, entry.id.clone(), entry.title.clone(), size_text(entry.size)); + } + } + if let Some(request) = crate::note_view::take_registry_install_request(ctx) { + self.state.pending_registry_install = Some(request); + } + if let Some((kind, id)) = self.state.pending_registry_install.take() { + self.preferences.view.registry_actions.push(Action::Install(kind, id)); + } + let mut entries: Vec = + self.preferences.view.registry.rows.iter().filter(|entry| entry.suggested && entry.state == "available").cloned().collect(); + for entry in offers.labels.values().flatten() { + if !entries.iter().any(|known| known.kind == entry.kind && known.id == entry.id) { + entries.push(entry.clone()); + } + } + ctx.data_mut(|data| data.insert_temp(cache_id, offers)); + self.state.registry_update_badge = + if self.preferences.view.registry.last_checked.is_some() { u32::try_from(self.preferences.view.registry.updates()).unwrap_or(u32::MAX) } else { 0 }; + let dismissed = self.state.registry_suggestions_dismissed; + if !entries.is_empty() && !dismissed { + egui::Area::new(egui::Id::new("registry-first-run")).anchor(egui::Align2::RIGHT_BOTTOM, egui::vec2(-24.0, -24.0)).show(ctx, |ui| { + self.preferences.view.registry_actions.extend(show(ui, &self.tokens, &mut self.state, &entries)); + }); + } + if self.state.registry_suggestions_dismissed != dismissed { + self.persist_ui(); + } + wanted_banner::show(self, ctx); + } +} + +/// Formats a download size with the existing translated byte label. +fn size_text(bytes: u64) -> String { + crate::text::registry::bytes().replace("{size}", &bytes.to_string()) +} + +use crate::views::registry_view::{Action, Entry}; + +/// Paints the one-time suggestion card and returns explicit install requests. +pub(crate) fn show(ui: &mut egui::Ui, tokens: &crate::theme::Tokens, state: &mut UiState, entries: &[Entry]) -> Vec { + if state.registry_suggestions_dismissed { + return Vec::new(); + } + let mut actions = Vec::new(); + let colors = &tokens.theme.colors; + egui::Frame::new().fill(crate::theme::color(colors.surface)).corner_radius(tokens.radius_medium()).inner_margin(egui::Margin::same(16)).show(ui, |ui| { + ui.heading(crate::text::registry::suggestions_title()); + for entry in entries.iter().filter(|entry| entry.state == "available") { + ui.horizontal(|ui| { + ui.label(&entry.title); + if crate::views::registry_view::button(ui, tokens, ("first-run-install", &entry.kind, &entry.id), crate::text::registry::install(), true, true) + { + actions.push(Action::Install(entry.kind.clone(), entry.id.clone())); + } + }); + } + if entries.is_empty() { + ui.label(crate::text::registry::suggestions_empty()); + } + if crate::views::registry_view::button(ui, tokens, "first-run-dismiss", crate::text::registry::suggestions_dismiss(), false, true) { + state.registry_suggestions_dismissed = true; + } + }); + actions +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dismissed_card_stays_hidden_and_suggestions_do_not_fetch() { + let Some(theme) = notescraft_themes::builtin("catppuccin-mocha") else { return }; + let mut state = UiState::new(); + state.registry_suggestions_dismissed = true; + let mut actions = Vec::new(); + let tokens = crate::theme::Tokens::new(theme); + crate::testsupport::run_frames_ui(800.0, 600.0, |ui| actions = show(ui, &tokens, &mut state, &[])); + assert_eq!(actions, Vec::::new()); + } +} diff --git a/crates/ui-egui/src/shell/app/vault/first_run/wanted_banner.rs b/crates/ui-egui/src/shell/app/vault/first_run/wanted_banner.rs new file mode 100644 index 0000000..782747c --- /dev/null +++ b/crates/ui-egui/src/shell/app/vault/first_run/wanted_banner.rs @@ -0,0 +1,70 @@ +//! Cached vault add-on requirements; this banner never fetches or installs automatically. + +use crate::shell::app::NotesApp; +use crate::views::registry_view::{Action, Entry}; + +/// Shows available add-ons already recorded in the vault's wanted list. +pub(super) fn show(app: &mut NotesApp, ctx: &egui::Context) { + let rows = &app.preferences.view.registry.rows; + let generation = serde_json::to_string(rows).unwrap_or_default(); + let id = egui::Id::new(("registry-wanted-banner", app.session.vault().root(), generation)); + let entries = ctx.data(|data| data.get_temp::>(id)).unwrap_or_else(|| { + let wanted = app + .session + .vault() + .read(".notescraft/registry.json") + .ok() + .and_then(|bytes| notescraft_engine::addons::VaultFile::parse(&bytes).ok()) + .map(|file| file.wanted) + .unwrap_or_default(); + let entries = wanted_entries(rows, &wanted); + ctx.data_mut(|data| data.insert_temp(id, entries.clone())); + entries + }); + if entries.is_empty() { + return; + } + egui::Area::new(egui::Id::new("registry-wanted")).anchor(egui::Align2::LEFT_BOTTOM, egui::vec2(24.0, -24.0)).show(ctx, |ui| { + egui::Frame::new() + .fill(crate::theme::color(app.tokens.theme.colors.surface)) + .corner_radius(app.tokens.radius_medium()) + .inner_margin(egui::Margin::same(16)) + .show(ui, |ui| { + ui.heading(crate::text::registry::wanted_title()); + for entry in &entries { + ui.horizontal(|ui| { + ui.label(format!("{} ยท {}", entry.title, super::size_text(entry.size))); + if crate::views::registry_view::button( + ui, + &app.tokens, + ("wanted-install", &entry.kind, &entry.id), + crate::text::registry::install(), + true, + !app.preferences.view.registry.busy, + ) { + app.preferences.view.registry_actions.push(Action::Install(entry.kind.clone(), entry.id.clone())); + } + }); + } + }); + }); +} + +fn wanted_entries(rows: &[Entry], wanted: &[String]) -> Vec { + rows.iter().filter(|entry| entry.state == "available" && wanted.iter().any(|key| key == &format!("{}:{}", entry.kind, entry.id))).cloned().collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wanted_banner_excludes_installed_and_unknown_entries() { + let available = Entry { kind: "grammar".into(), id: "twig".into(), state: "available".into(), ..Entry::default() }; + let installed = Entry { kind: "grammar".into(), id: "blade".into(), state: "installed".into(), ..Entry::default() }; + let wanted = vec!["grammar:twig".into(), "grammar:blade".into(), "grammar:unknown".into()]; + let entries = wanted_entries(&[available, installed], &wanted); + assert_eq!(entries.len(), 1); + assert_eq!(entries.first().map(|entry| entry.id.as_str()), Some("twig")); + } +} diff --git a/crates/ui-egui/src/shell/app/vault/mod.rs b/crates/ui-egui/src/shell/app/vault/mod.rs index 559355a..f9b5788 100644 --- a/crates/ui-egui/src/shell/app/vault/mod.rs +++ b/crates/ui-egui/src/shell/app/vault/mod.rs @@ -3,6 +3,7 @@ pub(super) mod batch_review; pub(super) mod explorer; pub(super) mod file_commands; +pub(super) mod first_run; pub(super) mod grammar_dialog; pub(super) mod hibernation; pub(super) mod history; @@ -11,6 +12,7 @@ pub(super) mod large_folder; pub(super) mod lifecycle; pub(super) mod link_preview; pub(super) mod preferences; +pub(super) mod registry; pub(super) mod switcher; pub(super) mod vault_switch; pub(super) mod workspaces; diff --git a/crates/ui-egui/src/shell/app/vault/registry/e2e.rs b/crates/ui-egui/src/shell/app/vault/registry/e2e.rs new file mode 100644 index 0000000..9e92c5d --- /dev/null +++ b/crates/ui-egui/src/shell/app/vault/registry/e2e.rs @@ -0,0 +1,123 @@ +//! End to end: stub index and files -> Registry view -> Check -> Install -> the fence loses its banner. + +use super::*; +use egui_kittest::kittest::Queryable; +use notescraft_engine::addons::{FakeTransport, FakeVerifier, INDEX_URL, Response}; +use notescraft_syntax::validate::sha256_hex; + +const ID: &str = "twigstub"; +const FILE: &str = "twigstub.sublime-syntax"; +const BASE: &str = "https://git.marcospaulo.dev.br/masi/notescraft-registry/raw/branch/main/files"; +const GRAMMAR: &str = "%YAML 1.2\n---\nname: TwigStub\nfile_extensions: [twigstub]\nscope: source.twigstub\ncontexts:\n main:\n - match: '\\bfoo\\b'\n scope: keyword.control.twigstub\n"; + +fn stub() -> FakeTransport { + let url = format!("{BASE}/grammar/{ID}/1.0.0/{FILE}"); + let entry = json!({ + "id": ID, "kind": "grammar", "title": "TwigStub", "summary": "s", "version": "1.0.0", "min_app_version": "0.1.0", "license": "MIT", + "aliases": [ID], "permissions": [], "signature": { "key_id": "k1", "sig": "QUJD" }, + "files": [{ "path": FILE, "size": GRAMMAR.len(), "sha256": sha256_hex(GRAMMAR.as_bytes()), "url": url }], + }); + let index = json!({ "version": 2, "generated_at": "2026-10-21T00:00:00Z", "entries": [entry], "revoked": [] }).to_string().into_bytes(); + FakeTransport::new().with_response(INDEX_URL, Ok(Response { status: 200, etag: None, bytes: index })).with(&url, GRAMMAR.as_bytes()) +} + +/// A worker backend over the stub; `trusted` keeps the build's own (empty) key list. +fn backend(app: &crate::app::NotesApp, trusted: bool) -> impl FnMut(&str, &Value) -> Result + Send + 'static { + let (vault, dir) = (app.session.vault().sibling(), app.session.picked_settings_dir()); + move |id, params| { + let mut commands = notescraft_engine::Registry::new(); + notescraft_engine::register_core(&mut commands).map_err(|e| RegistryError { code: e.code().into(), message: e.to_string() })?; + let mut session = notescraft_engine::Session::registry_worker(vault.sibling(), dir.clone()); + session.set_addons_transport(Box::new(stub())); + if !trusted { + session.set_addons_verifier(Box::new(FakeVerifier { accept: true })); + } + commands.run(id, &mut session, params).map_err(|error| match error { + notescraft_engine::Error::Command { source, .. } => RegistryError { code: source.code, message: source.message }, + other => RegistryError { code: other.code().into(), message: other.to_string() }, + }) + } +} + +fn app(name: &str) -> crate::app::NotesApp { + let root = crate::testsupport::temp_vault(name); + crate::testsupport::write_note(&root, "t", &format!("```{ID}\nfoo\n```\n")); + let mut session = notescraft_engine::Session::open(&root).unwrap(); + session.use_settings_dir(root.join(".notescraft/test-config")); + let theme = notescraft_themes::builtin("catppuccin-mocha").unwrap(); + let mut app = crate::app::NotesApp::new(session, theme); + app.state.animations = false; + app +} + +/// Steps until the registry worker is idle again (bounded), then a few frames more. +fn settle(harness: &mut egui_kittest::Harness<'_, crate::app::NotesApp>) { + for _ in 0..1000 { + harness.step(); + std::thread::sleep(std::time::Duration::from_millis(5)); + if !harness.state().preferences.view.registry.busy { + break; + } + } + for _ in 0..10 { + harness.step(); + } +} + +fn harness(app: crate::app::NotesApp) -> egui_kittest::Harness<'static, crate::app::NotesApp> { + egui_kittest::Harness::builder().with_size(egui::vec2(1440.0, 1000.0)).build_ui_state(|ui, app: &mut crate::app::NotesApp| app.ui(ui.ctx()), app) +} + +#[test] +fn check_then_install_a_grammar_removes_the_fence_banner() { + let mut app = app("registry-e2e"); + let stub_backend = backend(&app, false); + app.set_registry_backend(stub_backend); + app.open_note("t.md"); + let mut harness = harness(app); + settle(&mut harness); + let add = crate::text::note_view::add_grammar(); + assert!(harness.query_by_label(add).is_some(), "an unknown fence shows the banner"); + harness.state_mut().open_registry(); + settle(&mut harness); + harness.get_by_label(crate::text::registry::check()).click(); + settle(&mut harness); + harness.get_by_label(crate::text::registry::install()).click(); + settle(&mut harness); + harness.state_mut().open_note("t.md"); + // The highlighter is process-global and other tests' apps reload it as they start, so reload + // from the store and look again instead of trusting one frame. + let gone = (0..50).any(|_| { + harness.state().session.load_grammars(); + harness.step(); + notescraft_syntax::load::is_known(ID) && harness.query_by_label(add).is_none() && harness.query_by_label_contains("TwigStub").is_none() + }); + assert!(gone, "the installed grammar reached the highlighter and the banner is gone"); +} + +#[test] +fn no_trusted_key_shows_the_addon_error() { + let mut app = app("registry-e2e-key"); + let stub_backend = backend(&app, true); + app.set_registry_backend(stub_backend); + app.open_registry(); + let mut harness = harness(app); + settle(&mut harness); + harness.get_by_label(crate::text::registry::check()).click(); + settle(&mut harness); + harness.get_by_label(crate::text::registry::install()).click(); + settle(&mut harness); + assert!(harness.query_by_label_contains("no trusted registry key").is_some()); + assert!(!notescraft_syntax::load::is_known(ID)); +} + +#[test] +fn default_backend_lists_the_cache_without_network_or_error() { + let mut app = app("registry-e2e-default"); + app.open_registry(); + let mut harness = harness(app); + settle(&mut harness); + let view = &harness.state().preferences.view.registry; + assert!(view.error.is_none() && !view.offline && !view.busy); + assert!(view.rows.is_empty()); +} diff --git a/crates/ui-egui/src/shell/app/vault/registry/mod.rs b/crates/ui-egui/src/shell/app/vault/registry/mod.rs new file mode 100644 index 0000000..bbcab7a --- /dev/null +++ b/crates/ui-egui/src/shell/app/vault/registry/mod.rs @@ -0,0 +1,253 @@ +//! Serial registry jobs: the shell owns the backend and painting only queues actions. + +use std::collections::VecDeque; +use std::sync::mpsc::{Receiver, TryRecvError, channel}; + +use serde_json::{Value, json}; + +use crate::views::registry_view::Action; + +/// Delay between worker result polls. +const POLL: std::time::Duration = std::time::Duration::from_millis(50); + +/// A registry refusal, preserving its command error code for the offline banner. +#[derive(Debug, Clone)] +pub struct RegistryError { + /// Engine command error code. + pub code: String, + /// Engine diagnostic. + pub message: String, +} + +/// Owned command dispatcher moved to a worker; it must never borrow the live UI session. +pub type Backend = Box Result + Send>; + +type Reply = (Backend, &'static str, Result); + +/// Serial worker queue for the Registry Settings category. +#[derive(Default)] +pub struct RegistryJobs { + backend: Option, + receiver: Option>, + queue: VecDeque<(&'static str, Value)>, + consumers_changed: bool, + /// A backend supplied by the host (tests, embedders); the shell never replaces it. + explicit: bool, + /// Vault the default backend was built for. + root: Option, + /// Whether the opt-in automatic check already ran for this backend. + auto_checked: bool, +} + +impl RegistryJobs { + /// Installs the owned command backend before any work starts. + pub fn set_backend(&mut self, backend: Backend) { + self.backend = Some(backend); + } + + /// Whether a backend has been supplied or is currently running. + #[must_use] + pub fn has_backend(&self) -> bool { + self.backend.is_some() || self.receiver.is_some() + } + + /// Reads the cached index only. Opening never queues `registry.check`. + pub fn open(&mut self) { + self.queue.push_back(("registry.list", json!({}))); + } + + /// Queues exactly one command for a user action; preferences stay in the shell. + pub fn request(&mut self, action: Action) { + if let Some(call) = command(action) { + self.queue.push_back(call); + } + } + + /// Polls without waiting and starts the next command on an owned worker thread. + pub fn pump(&mut self, view: &mut crate::views::registry_view::RegistryView, ctx: &egui::Context) { + if let Some(receiver) = &self.receiver { + match receiver.try_recv() { + Ok((backend, id, reply)) => { + ctx.request_repaint(); + self.receiver = None; + self.backend = Some(backend); + match reply { + Ok(value) => { + view.error = None; + if matches!(id, "registry.install" | "registry.update" | "registry.remove" | "registry.rollback") { + self.consumers_changed = true; + } + if id == "registry.info" { + view.apply_info(&value); + } else if id == "registry.list" { + view.apply_reply(&value); + } else { + if id == "registry.check" { + view.last_checked = Some(crate::text::registry::just_now().to_owned()); + view.offline = false; + } + self.open(); + } + } + Err(error) => view.set_error(&error.code, &error.message), + } + } + Err(TryRecvError::Disconnected) => { + ctx.request_repaint(); + self.receiver = None; + self.queue.clear(); + view.set_error("needs_dependency", crate::text::registry::worker_stopped()); + } + Err(TryRecvError::Empty) => {} + } + } + if self.receiver.is_none() + && !self.queue.is_empty() + && let Some(mut backend) = self.backend.take() + && let Some((id, params)) = self.queue.pop_front() + { + let (sender, receiver) = channel(); + match std::thread::Builder::new().name("notescraft-registry".into()).spawn(move || { + let reply = backend(id, ¶ms); + let _ = sender.send((backend, id, reply)); + }) { + Ok(_) => self.receiver = Some(receiver), + Err(error) => { + self.queue.clear(); + view.set_error("needs_dependency", &error.to_string()); + } + } + } + view.busy = self.receiver.is_some() || !self.queue.is_empty(); + if view.busy { + ctx.request_repaint_after(POLL); + } + } +} + +impl crate::app::NotesApp { + /// Connects an owned engine command dispatcher to the registry worker. + /// The dispatcher runs serially off the UI thread and must preserve engine write gates. + /// The host must reconnect the dispatcher when switching vaults. + pub fn set_registry_backend(&mut self, backend: impl FnMut(&str, &Value) -> Result + Send + 'static) { + self.registry_jobs.set_backend(Box::new(backend)); + self.registry_jobs.explicit = true; + self.preferences.view.registry_open = false; + } + + /// Opens Settings directly on the cache-only Registry category. + pub fn open_registry(&mut self) { + self.open_note(crate::tabs::SETTINGS_PATH); + self.preferences.view.selected = crate::settings_view::SettingsCategory::Registry; + } + + /// Registers the localized Registry navigation command for the palette. + pub(in crate::shell::app) fn register_registry_command(&mut self) { + let spec = notescraft_engine::CommandSpec::new( + "ui.open_registry", + crate::text::registry::title(), + crate::text::registry::description(), + |_: &crate::state::UiState| true, + crate::shell::app::save_stub, + ); + if let Err(error) = self.ui_registry.register(spec) { + self.toast = Some(error.to_string()); + } + } + + /// Handles the Registry navigation command before engine dispatch. + pub(in crate::shell::app) fn run_registry_command(&mut self, id: &str) -> bool { + if id != "ui.open_registry" { + return false; + } + self.open_registry(); + true + } + + /// Builds the default worker backend for this vault unless the host supplied one. + /// The worker owns a vault sibling and a fresh engine session; the UI session never crosses threads. + fn ensure_registry_backend(&mut self) { + let root = self.session.vault().root().to_path_buf(); + let jobs = &mut self.registry_jobs; + if jobs.explicit || jobs.receiver.is_some() || (jobs.has_backend() && jobs.root.as_ref() == Some(&root)) { + return; + } + jobs.set_backend(engine_backend(self.session.vault().sibling(), self.session.picked_settings_dir())); + jobs.root = Some(root); + jobs.auto_checked = false; + self.preferences.view.registry_open = false; + } + + /// Hands off registry actions after painting and polls without waiting. + pub(in crate::shell::app) fn pump_registry(&mut self, ctx: &egui::Context) { + self.ensure_registry_backend(); + self.preferences.view.registry.auto_check = self.state.registry_auto_check; + let active = + self.tabs.active_path() == Some(crate::tabs::SETTINGS_PATH) && self.preferences.view.selected == crate::settings_view::SettingsCategory::Registry; + if !active { + self.preferences.view.registry_open = false; + } else if !self.preferences.view.registry_open { + self.preferences.view.registry_open = true; + self.registry_jobs.open(); + } + if self.state.registry_auto_check && !self.registry_jobs.auto_checked && self.registry_jobs.has_backend() { + self.registry_jobs.auto_checked = true; + self.registry_jobs.request(Action::Check); + } + for action in std::mem::take(&mut self.preferences.view.registry_actions) { + if let Action::AutoCheck(enabled) = action { + self.state.registry_auto_check = enabled; + self.preferences.view.registry.auto_check = enabled; + self.persist_ui(); + } else { + self.registry_jobs.request(action); + } + } + let previous_error = self.preferences.view.registry.error.clone(); + self.registry_jobs.pump(&mut self.preferences.view.registry, ctx); + if !active && self.preferences.view.registry.error != previous_error { + self.toast.clone_from(&self.preferences.view.registry.error); + } + if std::mem::take(&mut self.registry_jobs.consumers_changed) { + self.session.load_grammars(); + ctx.request_repaint(); + } + self.state.registry_update_badge = u32::try_from(self.preferences.view.registry.updates()).unwrap_or(u32::MAX); + } +} + +/// The default backend: each command runs on a fresh worker session over a vault sibling, so it +/// shares the app's registry store and never takes a second vault lock. +fn engine_backend(vault: notescraft_vault::Vault, settings_dir: Option) -> Backend { + Box::new(move |id, params| { + let refuse = |code: &str, message: String| RegistryError { code: code.to_owned(), message }; + let mut commands = notescraft_engine::Registry::new(); + notescraft_engine::register_core(&mut commands).map_err(|error| refuse(error.code(), error.to_string()))?; + let mut session = notescraft_engine::Session::registry_worker(vault.sibling(), settings_dir.clone()); + commands.run(id, &mut session, params).map_err(|error| match error { + notescraft_engine::Error::Command { source, .. } => RegistryError { code: source.code, message: source.message }, + other => refuse(other.code(), other.to_string()), + }) + }) +} + +fn command(action: Action) -> Option<(&'static str, Value)> { + let (command, kind, id) = match action { + Action::Check => return Some(("registry.check", json!({}))), + Action::Enable(id) => return Some(("registry.enable", json!({ "id": id }))), + Action::Disable(id) => return Some(("registry.disable", json!({ "id": id }))), + Action::AutoCheck(_) => return None, + Action::Install(kind, id) => ("registry.install", kind, id), + Action::Update(kind, id) => ("registry.update", kind, id), + Action::Remove(kind, id) => ("registry.remove", kind, id), + Action::Rollback(kind, id) => ("registry.rollback", kind, id), + Action::Info(kind, id) => ("registry.info", kind, id), + }; + Some((command, json!({ "kind": kind, "id": id }))) +} + +#[cfg(test)] +mod tests; + +#[cfg(test)] +mod e2e; diff --git a/crates/ui-egui/src/shell/app/vault/registry/tests.rs b/crates/ui-egui/src/shell/app/vault/registry/tests.rs new file mode 100644 index 0000000..19091f0 --- /dev/null +++ b/crates/ui-egui/src/shell/app/vault/registry/tests.rs @@ -0,0 +1,165 @@ +//! Registry worker and Settings interaction tests. + +use super::*; +use std::sync::{Arc, Mutex}; + +fn finish(jobs: &mut RegistryJobs, view: &mut crate::views::registry_view::RegistryView) { + let ctx = egui::Context::default(); + for _ in 0..100 { + jobs.pump(view, &ctx); + if !view.busy { + return; + } + std::thread::sleep(std::time::Duration::from_millis(2)); + } + assert!(!view.busy, "registry worker did not finish"); +} + +#[test] +fn opening_reads_cache_without_network_and_install_runs_once() { + let calls = Arc::new(Mutex::new(Vec::new())); + let recorded = calls.clone(); + let mut jobs = RegistryJobs::default(); + jobs.set_backend(Box::new(move |id, params| { + recorded.lock().unwrap().push((id.to_owned(), params.clone())); + Ok(json!({"entries": []})) + })); + let mut view = crate::views::registry_view::RegistryView::default(); + jobs.open(); + finish(&mut jobs, &mut view); + assert_eq!(*calls.lock().unwrap(), vec![("registry.list".into(), json!({}))]); + jobs.request(Action::Install("grammar".into(), "twig".into())); + finish(&mut jobs, &mut view); + let calls = calls.lock().unwrap(); + assert_eq!(calls.iter().filter(|(id, _)| id == "registry.install").count(), 1); + assert!(!calls.iter().any(|(id, _)| id == "registry.check")); + assert!(calls.iter().any(|(id, params)| id == "registry.install" && params == &json!({"kind":"grammar","id":"twig"}))); +} + +#[test] +fn an_install_click_runs_one_worker_command() { + use egui_kittest::kittest::Queryable; + let calls = Arc::new(Mutex::new(Vec::new())); + let recorded = calls.clone(); + let mut jobs = RegistryJobs::default(); + jobs.set_backend(Box::new(move |id, _| { + recorded.lock().unwrap().push(id.to_owned()); + Ok(json!({"entries": [{"id":"twig", "kind":"grammar", "title":"Twig", "state":"available"}]})) + })); + let tokens = crate::theme::Tokens::new(notescraft_themes::builtin("catppuccin-mocha").unwrap()); + let view = crate::views::registry_view::RegistryView::default(); + jobs.open(); + let mut harness = egui_kittest::Harness::builder().with_size(egui::vec2(1000.0, 800.0)).build_ui_state( + move |ui, (jobs, view): &mut (RegistryJobs, crate::views::registry_view::RegistryView)| { + crate::fonts::install(ui.ctx()); + crate::style::apply(ui.ctx(), &tokens); + crate::motion::set_enabled(ui.ctx(), false); + for action in view.show(ui, &tokens) { + jobs.request(action); + } + jobs.pump(view, ui.ctx()); + }, + (jobs, view), + ); + for _ in 0..20 { + harness.step(); + std::thread::sleep(std::time::Duration::from_millis(2)); + } + assert_eq!(*calls.lock().unwrap(), vec!["registry.list"]); + harness.get_by_label(crate::text::registry::install()).click(); + for _ in 0..20 { + harness.step(); + std::thread::sleep(std::time::Duration::from_millis(2)); + } + assert_eq!(calls.lock().unwrap().iter().filter(|id| id.as_str() == "registry.install").count(), 1); +} + +#[test] +fn opening_settings_registry_reads_only_cache_then_click_installs_once() { + use egui_kittest::kittest::Queryable; + let root = crate::testsupport::temp_vault("registry-category"); + let mut session = notescraft_engine::Session::open(&root).unwrap(); + session.use_settings_dir(root.join(".notescraft/test-config")); + let theme = notescraft_themes::builtin("catppuccin-mocha").unwrap(); + let mut app = crate::app::NotesApp::new(session, theme); + app.state.animations = false; + let calls = Arc::new(Mutex::new(Vec::new())); + let recorded = calls.clone(); + app.set_registry_backend(move |id, _| { + recorded.lock().unwrap().push(id.to_owned()); + Ok(json!({"entries": [{"id":"twig", "kind":"grammar", "title":"Twig", "state":"available"}]})) + }); + app.run_command("ui.open_registry", &json!({})); + assert_eq!(app.preferences.view.selected, crate::settings_view::SettingsCategory::Registry); + app.preferences.view.selected = crate::settings_view::SettingsCategory::Appearance; + let mut harness = + egui_kittest::Harness::builder().with_size(egui::vec2(1440.0, 1000.0)).build_ui_state(|ui, app: &mut crate::app::NotesApp| app.ui(ui.ctx()), app); + for _ in 0..20 { + harness.step(); + std::thread::sleep(std::time::Duration::from_millis(2)); + } + assert!(calls.lock().unwrap().is_empty()); + harness.get_by_label(crate::text::registry::title()).click(); + for _ in 0..20 { + harness.step(); + std::thread::sleep(std::time::Duration::from_millis(2)); + } + assert_eq!(*calls.lock().unwrap(), vec!["registry.list"]); + harness.get_by_label(crate::text::registry::install()).click(); + for _ in 0..20 { + harness.step(); + std::thread::sleep(std::time::Duration::from_millis(2)); + } + assert_eq!(calls.lock().unwrap().iter().filter(|id| id.as_str() == "registry.install").count(), 1); + assert!(!calls.lock().unwrap().iter().any(|id| id == "registry.check")); +} + +#[test] +fn offline_check_keeps_cached_rows() { + let mut jobs = RegistryJobs::default(); + jobs.set_backend(Box::new(|_, _| Err(RegistryError { code: "offline".into(), message: "offline fixture".into() }))); + let mut view = crate::views::registry_view::RegistryView::default(); + view.apply_reply(&json!({"entries":[{"id":"twig", "kind":"grammar"}]})); + jobs.request(Action::Check); + finish(&mut jobs, &mut view); + assert!(view.offline); + assert!(view.last_checked.is_none()); + assert_eq!(view.rows.len(), 1); + assert_eq!(view.rows.first().map(|row| row.id.as_str()), Some("twig")); +} + +#[test] +fn command_mapping_preserves_kind_and_id() { + for (action, id) in [ + (Action::Update("theme".into(), "frappe".into()), "registry.update"), + (Action::Remove("theme".into(), "frappe".into()), "registry.remove"), + (Action::Rollback("theme".into(), "frappe".into()), "registry.rollback"), + (Action::Info("theme".into(), "frappe".into()), "registry.info"), + ] { + assert_eq!(command(action), Some((id, json!({"kind":"theme","id":"frappe"})))); + } + assert!(command(Action::AutoCheck(true)).is_none()); +} + +#[test] +fn successful_mutation_marks_consumers_for_refresh_but_failure_preserves_diagnostic() { + let mut jobs = RegistryJobs::default(); + jobs.set_backend(Box::new(|id, _| { + if id == "registry.install" { + return Ok(json!({"kind":"grammar", "id":"twig"})); + } + Ok(json!({"entries": []})) + })); + let mut view = crate::views::registry_view::RegistryView::default(); + jobs.request(Action::Install("grammar".into(), "twig".into())); + finish(&mut jobs, &mut view); + assert!(std::mem::take(&mut jobs.consumers_changed)); + jobs.set_backend(Box::new(|_, _| { + Err(RegistryError { code: "addon".into(), message: "this build has no trusted registry key, so add-ons cannot be installed yet".into() }) + })); + jobs.request(Action::Install("grammar".into(), "twig".into())); + finish(&mut jobs, &mut view); + assert!(!jobs.consumers_changed); + assert!(view.error.as_deref().is_some_and(|error| error.contains("no trusted registry key"))); + assert!(!view.busy); +} diff --git a/crates/ui-egui/src/shell/app/window/commands.rs b/crates/ui-egui/src/shell/app/window/commands.rs index f4fbca2..90b96d8 100644 --- a/crates/ui-egui/src/shell/app/window/commands.rs +++ b/crates/ui-egui/src/shell/app/window/commands.rs @@ -76,6 +76,7 @@ impl NotesApp { self.toast = Some(error.to_string()); } self.register_hotkey_commands(); + self.register_registry_command(); } pub(crate) fn start_pdf_export(&mut self, params: Value) { @@ -165,7 +166,7 @@ impl NotesApp { /// Dispatches a registered command and applies its result to the live shell. /// Errors become the app toast; settings use the engine's validated atomic store. pub fn run_command(&mut self, id: &str, params: &Value) { - if self.run_vault_command(id) || self.run_hotkey_command(id, params) { + if self.run_vault_command(id) || self.run_hotkey_command(id, params) || self.run_registry_command(id) { return; } match id { diff --git a/crates/ui-egui/src/shell/app/window/frame.rs b/crates/ui-egui/src/shell/app/window/frame.rs index e291f50..5647607 100644 --- a/crates/ui-egui/src/shell/app/window/frame.rs +++ b/crates/ui-egui/src/shell/app/window/frame.rs @@ -273,6 +273,8 @@ impl NotesApp { let embed_edit = response.embed_edit.take(); self.handle_center(response, now, ctx); self.pump_bases(ctx); + self.pump_registry_offers(ctx); + self.pump_registry(ctx); // The rooms talk to the editor: the local edits go out, the remote ones come in. self.push_live_edits(ctx); self.poll_live(ctx); diff --git a/crates/ui-egui/src/shell/state.rs b/crates/ui-egui/src/shell/state.rs index 97bb5cd..da49fdd 100644 --- a/crates/ui-egui/src/shell/state.rs +++ b/crates/ui-egui/src/shell/state.rs @@ -92,6 +92,16 @@ pub struct UiState { pub volume: f32, /// Whether animations run (off: every transition jumps). pub animations: bool, + /// Whether the first-run add-on suggestions card was dismissed. + pub registry_suggestions_dismissed: bool, + /// Opt-in: check the registry index once per start (off by default). + pub registry_auto_check: bool, + /// Number of available add-on updates, populated only by a successful check. + #[serde(skip)] + pub registry_update_badge: u32, + /// Grammar add-on selected from a code card, awaiting an explicit install action. + #[serde(skip)] + pub pending_registry_install: Option<(String, String)>, /// Per-note property card expansion, bounded to 200 notes. #[serde(deserialize_with = "restore_properties_expanded")] pub properties_expanded: std::collections::BTreeMap, @@ -197,6 +207,10 @@ impl UiState { sound: true, volume: DEFAULT_VOLUME, animations: true, + registry_suggestions_dismissed: false, + registry_auto_check: false, + registry_update_badge: 0, + pending_registry_install: None, properties_expanded: std::collections::BTreeMap::new(), pinned_folders: Vec::new(), pinned_tabs: Vec::new(), diff --git a/crates/ui-egui/src/text/views/mod.rs b/crates/ui-egui/src/text/views/mod.rs index df29b95..f079abe 100644 --- a/crates/ui-egui/src/text/views/mod.rs +++ b/crates/ui-egui/src/text/views/mod.rs @@ -19,6 +19,7 @@ pub mod lightbox; pub mod maps; pub mod media_view; pub mod note_view; +pub mod registry; pub mod slash_popup; -pub mod whiteboard_view_arrange; -pub mod whiteboard_view_layers_panel; +mod whiteboard; +pub use whiteboard::{whiteboard_view_arrange, whiteboard_view_layers_panel}; diff --git a/crates/ui-egui/src/text/views/registry.rs b/crates/ui-egui/src/text/views/registry.rs new file mode 100644 index 0000000..3fd88ef --- /dev/null +++ b/crates/ui-egui/src/text/views/registry.rs @@ -0,0 +1,55 @@ +//! Copy for the offline add-on registry browser. + +table! { + just_now = "ui.registry.just-now"; + worker_stopped = "ui.registry.worker-stopped"; + backend_unavailable = "ui.registry.backend-unavailable"; + title = "ui.registry.title"; + description = "ui.registry.description"; + search = "ui.registry.search"; + check = "ui.registry.check"; + never = "ui.registry.never"; + last_checked = "ui.registry.last-checked"; + auto_check = "ui.registry.auto-check"; + all = "ui.registry.all"; + plugins = "ui.registry.plugins"; + themes = "ui.registry.themes"; + packs = "ui.registry.packs"; + grammars = "ui.registry.grammars"; + fonts = "ui.registry.fonts"; + snippets = "ui.registry.snippets"; + install = "ui.registry.install"; + installed = "ui.registry.installed"; + update = "ui.registry.update"; + updates = "ui.registry.updates"; + incompatible = "ui.registry.incompatible"; + revoked = "ui.registry.revoked"; + unsupported = "ui.registry.unsupported"; + details = "ui.registry.details"; + close = "ui.registry.close"; + remove = "ui.registry.remove"; + rollback = "ui.registry.rollback"; + enable = "ui.registry.enable"; + disable = "ui.registry.disable"; + offline = "ui.registry.offline"; + empty = "ui.registry.empty"; + loading = "ui.registry.loading"; + permissions = "ui.registry.permissions"; + read = "ui.registry.read"; + write = "ui.registry.write"; + edit = "ui.registry.edit"; + permission_unknown = "ui.registry.permission-unknown"; + license = "ui.registry.license"; + source = "ui.registry.source"; + signature = "ui.registry.signature"; + file = "ui.registry.file"; + files = "ui.registry.files"; + bytes = "ui.registry.bytes"; + version = "ui.registry.version"; + invalid_reply = "ui.registry.invalid-reply"; + install_grammar = "ui.registry.install-grammar"; + wanted_title = "ui.registry.wanted-title"; + suggestions_title = "ui.registry.suggestions-title"; + suggestions_dismiss = "ui.registry.suggestions-dismiss"; + suggestions_empty = "ui.registry.suggestions-empty"; +} diff --git a/crates/ui-egui/src/text/views/whiteboard/mod.rs b/crates/ui-egui/src/text/views/whiteboard/mod.rs new file mode 100644 index 0000000..e423323 --- /dev/null +++ b/crates/ui-egui/src/text/views/whiteboard/mod.rs @@ -0,0 +1,4 @@ +//! Copy for whiteboard arranging and layer controls. + +pub mod whiteboard_view_arrange; +pub mod whiteboard_view_layers_panel; diff --git a/crates/ui-egui/src/text/views/whiteboard_view_arrange.rs b/crates/ui-egui/src/text/views/whiteboard/whiteboard_view_arrange.rs similarity index 100% rename from crates/ui-egui/src/text/views/whiteboard_view_arrange.rs rename to crates/ui-egui/src/text/views/whiteboard/whiteboard_view_arrange.rs diff --git a/crates/ui-egui/src/text/views/whiteboard_view_layers_panel.rs b/crates/ui-egui/src/text/views/whiteboard/whiteboard_view_layers_panel.rs similarity index 100% rename from crates/ui-egui/src/text/views/whiteboard_view_layers_panel.rs rename to crates/ui-egui/src/text/views/whiteboard/whiteboard_view_layers_panel.rs diff --git a/crates/ui-egui/src/views/mod.rs b/crates/ui-egui/src/views/mod.rs index 855e0ae..d35b037 100644 --- a/crates/ui-egui/src/views/mod.rs +++ b/crates/ui-egui/src/views/mod.rs @@ -19,6 +19,7 @@ pub mod math_view; pub mod media_view; pub mod note_view; pub mod properties_view; +pub mod registry_view; pub mod settings_view; pub mod share_dialog; pub mod trash_view; diff --git a/crates/ui-egui/src/views/note_view.rs b/crates/ui-egui/src/views/note_view.rs index a46785b..d50f4e8 100644 --- a/crates/ui-egui/src/views/note_view.rs +++ b/crates/ui-egui/src/views/note_view.rs @@ -35,6 +35,7 @@ pub use blocks::rich::image_targets; use blocks::rich::text_line::render_text_line; pub use blocks::{EmbedNotes, dismissed_offers, request_grammar, set_dismissed_offers, set_embeds, take_grammar_request}; use blocks::{callout_card, code_block, display_math, properties_card, table}; +pub(crate) use blocks::{clear_registry_offer, set_registry_offer, take_registry_install_request, take_registry_labels}; pub use edit::NoteEdit; pub use header::{banner, title_icon}; pub use inlines::tag_target; diff --git a/crates/ui-egui/src/views/note_view/blocks.rs b/crates/ui-egui/src/views/note_view/blocks.rs index 695c3c9..2658334 100644 --- a/crates/ui-egui/src/views/note_view/blocks.rs +++ b/crates/ui-egui/src/views/note_view/blocks.rs @@ -28,6 +28,7 @@ pub(super) use base_embed::{opens_base, show_inline as show_base_block}; pub(super) use code_actions::take_edit as code_card_edit; pub use code_actions::{dismissed_offers, request_grammar, set_dismissed_offers, take_grammar_request}; pub(super) use code_card::take_click as code_card_clicked; +pub(crate) use code_card::{clear_registry_offer, set_registry_offer, take_registry_install_request, take_registry_labels}; pub use embed::set_embeds; pub(super) use embed::{Call, details, show as show_embed}; pub use embed_data::EmbedNotes; diff --git a/crates/ui-egui/src/views/note_view/blocks/code_card.rs b/crates/ui-egui/src/views/note_view/blocks/code_card.rs index c240c9f..141115e 100644 --- a/crates/ui-egui/src/views/note_view/blocks/code_card.rs +++ b/crates/ui-egui/src/views/note_view/blocks/code_card.rs @@ -5,6 +5,11 @@ //! repaint wake-up brings the colours in. A block over [`VIRTUAL_FROM`] lines draws only the rows //! near the viewport (fixed pitch, no wrapping) and asks the cache for just that window. +mod registry; + +pub(crate) use registry::{clear_registry_offer, set_registry_offer, take_registry_install_request, take_registry_labels}; +use registry::{registry_install_id, registry_offer, request_registry_label}; + use std::ops::Range; use egui::{Align, Color32, FontId, Id, Layout, Margin, RichText, Ui, vec2}; @@ -199,6 +204,7 @@ fn header(ui: &mut Ui, tokens: &Tokens, card: &Card<'_>) -> bool { let shown = if card.label.is_empty() { strings::code_plain() } else { card.label }.to_uppercase(); let json_labelled = card.label.eq_ignore_ascii_case("json"); let suggestion = suggestion(ui.ctx(), card.label); + let registry_offer = registry_offer(ui.ctx(), card.label); let unknown = !card.label.is_empty() && !json_labelled && !notescraft_syntax::load::is_known(card.label); egui::Frame::new().inner_margin(Margin { left: 16, right: 8, top: HEADER_PAD_TOP, bottom: 0 }).show(ui, |ui| { ui.set_min_width(ui.available_width()); @@ -220,8 +226,16 @@ fn header(ui: &mut Ui, tokens: &Tokens, card: &Card<'_>) -> bool { { post(ui.ctx(), edit); } - if unknown && action(ui, tokens, strings::add_grammar(), color(colors.text_muted), None) { - request_grammar(ui.ctx(), card.label); + if unknown { + request_registry_label(ui.ctx(), card.label); + if let Some((id, title, size)) = registry_offer { + let text = crate::text::registry::install_grammar().replace("{name}", &title).replace("{size}", &size); + if action(ui, tokens, &text, color(colors.accent), None) { + ui.ctx().data_mut(|data| data.insert_temp(registry_install_id(), ("grammar".to_owned(), id))); + } + } else if action(ui, tokens, strings::add_grammar(), color(colors.text_muted), None) { + request_grammar(ui.ctx(), card.label); + } } ui.with_layout(Layout::right_to_left(Align::Center), |ui| { let copied_at = ui.data(|data| data.get_temp::(card.id.with("copied"))); @@ -373,24 +387,5 @@ fn row_spans(text: &str, tokens: Option<&[TokenSpan]>, palette: &[Color32; 8], p } #[cfg(test)] -mod tests { - use notescraft_syntax::TokenKind; - - use super::*; - - #[test] - fn row_spans_cover_the_text_in_order_and_survive_bad_ranges() { - let palette = [Color32::RED; 8]; - let plain = Color32::WHITE; - let tokens = [TokenSpan { start: 0, end: 2, kind: TokenKind::Keyword }, TokenSpan { start: 3, end: 7, kind: TokenKind::Function }]; - let spans = row_spans("fn main()", Some(&tokens), &palette, plain); - let joined: String = spans.iter().map(|s| if let Span::CodeTok(t, _) = s { t.as_str() } else { "" }).collect(); - assert_eq!(joined, "fn main()"); - assert!(matches!(spans.first(), Some(Span::CodeTok(t, c)) if t == "fn" && *c == Color32::RED)); - let hostile = [TokenSpan { start: 5, end: 99, kind: TokenKind::String }, TokenSpan { start: 1, end: 3, kind: TokenKind::String }]; - let joined: String = - row_spans("hรฉllo wรถrld", Some(&hostile), &palette, plain).iter().map(|s| if let Span::CodeTok(t, _) = s { t.as_str() } else { "" }).collect(); - assert_eq!(joined, "hรฉllo wรถrld", "out-of-range and non-boundary ranges are skipped, text is intact"); - assert!(matches!(row_spans("", None, &palette, plain).as_slice(), [Span::CodeTok(t, _)] if t.is_empty())); - } -} +#[path = "code_card/tests.rs"] +mod tests; diff --git a/crates/ui-egui/src/views/note_view/blocks/code_card/registry.rs b/crates/ui-egui/src/views/note_view/blocks/code_card/registry.rs new file mode 100644 index 0000000..1f9bea2 --- /dev/null +++ b/crates/ui-egui/src/views/note_view/blocks/code_card/registry.rs @@ -0,0 +1,103 @@ +//! Cached grammar offers and explicit requests shared with the shell. + +use egui::Id; + +fn registry_offer_id(label: &str) -> Id { + Id::new(("notescraft-registry-grammar-offer", label.to_lowercase())) +} + +pub(super) fn registry_install_id() -> Id { + Id::new("notescraft-registry-grammar-install") +} + +/// Stores a cached registry suggestion for a grammar label; this performs no engine call. +pub(crate) fn set_registry_offer(ctx: &egui::Context, label: &str, id: String, title: String, size: String) { + ctx.data_mut(|data| data.insert_temp(registry_offer_id(label), (id, title, size))); +} + +/// Clears an offer whose cached catalogue entry changed or disappeared. +pub(crate) fn clear_registry_offer(ctx: &egui::Context, label: &str) { + ctx.data_mut(|data| { + data.remove_temp::<(String, String, String)>(registry_offer_id(label)); + }); +} + +/// Takes the cached registry suggestion matching `label` for painting. +pub(super) fn registry_offer(ctx: &egui::Context, label: &str) -> Option<(String, String, String)> { + ctx.data(|data| data.get_temp(registry_offer_id(label))) +} + +/// Takes the grammar install request made by an explicit code-card click. +#[must_use] +pub(crate) fn take_registry_install_request(ctx: &egui::Context) -> Option<(String, String)> { + ctx.data_mut(|data| data.remove_temp(registry_install_id())) +} + +/// Queues a cache-only alias lookup for a rendered fence. +pub(super) fn request_registry_label(ctx: &egui::Context, label: &str) { + ctx.data_mut(|data| { + let id = Id::new("registry-labels"); + let mut labels = data.get_temp::>(id).unwrap_or_default(); + let label = label.to_lowercase(); + if !labels.contains(&label) { + labels.push(label); + } + data.insert_temp(id, labels); + }); +} + +/// Takes visible fence labels without any network operation. +pub(crate) fn take_registry_labels(ctx: &egui::Context) -> Vec { + ctx.data_mut(|data| data.remove_temp(Id::new("registry-labels"))).unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cached_offer_does_not_request_install() { + let ctx = egui::Context::default(); + set_registry_offer(&ctx, "Twig", "twig".into(), "Twig".into(), "3.9 KB".into()); + assert_eq!(registry_offer(&ctx, "twig"), Some(("twig".into(), "Twig".into(), "3.9 KB".into()))); + assert_eq!(take_registry_install_request(&ctx), None); + request_registry_label(&ctx, "Twig"); + request_registry_label(&ctx, "twig"); + assert_eq!(take_registry_labels(&ctx), vec!["twig"]); + assert_eq!(take_registry_labels(&ctx), Vec::::new()); + } + + #[test] + fn code_card_click_queues_one_registry_install() { + use egui_kittest::kittest::Queryable; + let Some(theme) = notescraft_themes::builtin("catppuccin-mocha") else { return }; + let tokens = crate::theme::Tokens::new(theme); + let source = "```twig-registry-test\nhello\n```"; + let lines: Vec<(usize, usize)> = source + .lines() + .scan(0, |offset, line| { + let start = *offset; + *offset += line.len() + 1; + Some((start, start + line.len())) + }) + .collect(); + let mut warmed = false; + let mut harness = egui_kittest::Harness::builder().with_size(egui::vec2(900.0, 400.0)).build_ui(move |ui| { + if !warmed { + crate::fonts::install(ui.ctx()); + crate::style::apply(ui.ctx(), &tokens); + warmed = true; + return; + } + set_registry_offer(ui.ctx(), "twig-registry-test", "twig".into(), "Twig".into(), "3900 bytes".into()); + super::super::show(ui, &tokens, source, &lines, 0, lines.len()); + }); + let label = crate::text::registry::install_grammar().replace("{name}", "Twig").replace("{size}", "3900 bytes"); + harness.get_by_label(&label).click(); + harness.run(); + assert_eq!(take_registry_install_request(&harness.ctx), Some(("grammar".into(), "twig".into()))); + assert_eq!(take_registry_install_request(&harness.ctx), None); + clear_registry_offer(&harness.ctx, "twig-registry-test"); + assert!(registry_offer(&harness.ctx, "twig-registry-test").is_none()); + } +} diff --git a/crates/ui-egui/src/views/note_view/blocks/code_card/tests.rs b/crates/ui-egui/src/views/note_view/blocks/code_card/tests.rs new file mode 100644 index 0000000..b40bbe1 --- /dev/null +++ b/crates/ui-egui/src/views/note_view/blocks/code_card/tests.rs @@ -0,0 +1,21 @@ +//! Code-card token rendering checks. + +use notescraft_syntax::TokenKind; + +use super::*; + +#[test] +fn row_spans_cover_the_text_in_order_and_survive_bad_ranges() { + let palette = [Color32::RED; 8]; + let plain = Color32::WHITE; + let tokens = [TokenSpan { start: 0, end: 2, kind: TokenKind::Keyword }, TokenSpan { start: 3, end: 7, kind: TokenKind::Function }]; + let spans = row_spans("fn main()", Some(&tokens), &palette, plain); + let joined: String = spans.iter().map(|s| if let Span::CodeTok(t, _) = s { t.as_str() } else { "" }).collect(); + assert_eq!(joined, "fn main()"); + assert!(matches!(spans.first(), Some(Span::CodeTok(t, c)) if t == "fn" && *c == Color32::RED)); + let hostile = [TokenSpan { start: 5, end: 99, kind: TokenKind::String }, TokenSpan { start: 1, end: 3, kind: TokenKind::String }]; + let joined: String = + row_spans("hรฉllo wรถrld", Some(&hostile), &palette, plain).iter().map(|s| if let Span::CodeTok(t, _) = s { t.as_str() } else { "" }).collect(); + assert_eq!(joined, "hรฉllo wรถrld", "out-of-range and non-boundary ranges are skipped, text is intact"); + assert!(matches!(row_spans("", None, &palette, plain).as_slice(), [Span::CodeTok(t, _)] if t.is_empty())); +} diff --git a/crates/ui-egui/src/views/registry_view/detail.rs b/crates/ui-egui/src/views/registry_view/detail.rs new file mode 100644 index 0000000..fdef3a3 --- /dev/null +++ b/crates/ui-egui/src/views/registry_view/detail.rs @@ -0,0 +1,100 @@ +//! Cached entry drawer and human-readable plugin permissions. + +use super::{Action, Entry, RegistryView, list}; +use crate::{ + text::registry as strings, + theme::{Tokens, color}, +}; + +/// Detail card width, points, bounded by the viewport. +const WIDTH: f32 = 440.0; +/// Padding around drawer content, points. +const PAD: i8 = 24; +/// Viewport margin around drawer, points. +const MARGIN: f32 = 16.0; + +/// Shows the selected cached entry; the drawer never fetches. +pub(super) fn show(view: &mut RegistryView, ui: &mut egui::Ui, tokens: &Tokens, actions: &mut Vec) { + let Some((kind, id)) = &view.selected else { return }; + let Some(entry) = view.rows.iter().find(|row| &row.kind == kind && &row.id == id).cloned() else { return }; + let ctx = ui.ctx(); + let mut close = ctx.input(|input| input.key_pressed(egui::Key::Escape)); + crate::dialog::paint_backdrop(ctx, "registry-detail", 1.0); + egui::Area::new(egui::Id::new("registry-detail")).order(crate::dialog::CARD_ORDER).anchor(egui::Align2::RIGHT_TOP, egui::vec2(-MARGIN, MARGIN)).show( + ctx, + |ui| { + egui::Frame::new().fill(color(tokens.theme.colors.surface)).corner_radius(crate::theme::PANEL_RADIUS).inner_margin(PAD).show(ui, |ui| { + ui.set_max_height(ctx.viewport_rect().height() - MARGIN * 2.0 - f32::from(PAD) * 2.0); + ui.set_width(WIDTH.min((ctx.viewport_rect().width() - MARGIN * 2.0 - f32::from(PAD) * 2.0).max(0.0))); + egui::ScrollArea::vertical() + .id_salt("notescraft-registry-detail-scroll") + .max_height(ctx.viewport_rect().height() - MARGIN * 2.0 - f32::from(PAD) * 2.0) + .show(ui, |ui| { + ui.heading(&entry.title); + ui.label(&entry.summary); + ui.add_space(MARGIN); + metadata(ui, &entry); + if entry.kind == "plugin" { + ui.add_space(MARGIN); + ui.label(egui::RichText::new(strings::permissions()).strong()); + for permission in &entry.permissions { + ui.label(permission_sentence(permission)); + } + } + ui.add_space(MARGIN); + ui.horizontal_wrapped(|ui| { + if entry.installed_version.is_some() { + if list::button(ui, tokens, "registry-remove", strings::remove(), false, !view.busy) { + actions.push(Action::Remove(entry.kind.clone(), entry.id.clone())); + } + if entry.previous.is_some() && list::button(ui, tokens, "registry-rollback", strings::rollback(), false, !view.busy) { + actions.push(Action::Rollback(entry.kind.clone(), entry.id.clone())); + } + } + close |= list::button(ui, tokens, "registry-close", strings::close(), false, true); + }); + if entry.kind == "plugin" && entry.installed_version.is_some() { + ui.horizontal_wrapped(|ui| { + let label = if entry.enabled { strings::disable() } else { strings::enable() }; + ui.label(label); + list::switch(ui, tokens, "registry-enable", label, entry.enabled, !view.busy, |next| { + actions.push(if next { Action::Enable(entry.id.clone()) } else { Action::Disable(entry.id.clone()) }); + }); + }); + } + }); + }); + }, + ); + if close { + view.selected = None; + } +} + +fn metadata(ui: &mut egui::Ui, entry: &Entry) { + ui.label(strings::license().replace("{license}", &entry.license)); + let source = strings::source().replace("{source}", &entry.source); + if entry.source.starts_with("https://") { + ui.hyperlink_to(source, &entry.source); + } else { + ui.label(source); + } + ui.label(strings::signature().replace("{key}", &entry.key_id)); + if !entry.files.is_empty() { + ui.label(egui::RichText::new(strings::files()).strong()); + for file in &entry.files { + let path = file.get("path").and_then(serde_json::Value::as_str).unwrap_or_default(); + let size = file.get("size").and_then(serde_json::Value::as_u64).unwrap_or_default(); + ui.label(strings::file().replace("{path}", path).replace("{size}", &size.to_string())); + } + } +} + +fn permission_sentence(permission: &str) -> String { + match permission { + "vault.read" | "read" | "notes.read" | "search" => strings::read().to_owned(), + "vault.write" | "write" | "notes.write" => strings::write().to_owned(), + "editor.edit" | "editor" | "editor.write" => strings::edit().to_owned(), + _ => strings::permission_unknown().replace("{permission}", permission), + } +} diff --git a/crates/ui-egui/src/views/registry_view/list.rs b/crates/ui-egui/src/views/registry_view/list.rs new file mode 100644 index 0000000..68258d3 --- /dev/null +++ b/crates/ui-egui/src/views/registry_view/list.rs @@ -0,0 +1,194 @@ +//! Registry toolbar, local filters and tonal catalogue cards. + +use super::{Action, Entry, RegistryView}; +use crate::{ + text::registry as strings, + theme::{Tokens, color}, +}; +use egui::{RichText, Ui}; + +/// Gap between cards, points. +const GAP: f32 = 12.0; +/// Card inner padding, points. +const PAD: i8 = 16; +/// Default control height, points; compact controls grow to 44. +const CONTROL_HEIGHT: f32 = 32.0; +/// Space around control text, points. +const BUTTON_PAD: f32 = 24.0; +/// Body text size, points. +const BODY_SIZE: f32 = 13.0; +/// Search field width on a wide viewport, points. +const SEARCH_WIDTH: f32 = 320.0; +/// Browser heading size, points. +const TITLE_SIZE: f32 = 24.0; +/// Catalogue title size, points. +const ENTRY_TITLE_SIZE: f32 = 16.0; +/// Switch hit width, points. +const SWITCH_WIDTH: f32 = 48.0; +/// Switch track width and height, points. +const SWITCH_TRACK: egui::Vec2 = egui::vec2(40.0, 24.0); +/// Switch track radius and thumb inset, points. +const SWITCH_RADIUS: f32 = 12.0; +/// Switch thumb radius, points. +const THUMB_RADIUS: f32 = 8.0; + +/// A stable, accessible tonal button, with the same tokens as dialog buttons. +pub(crate) fn button(ui: &mut Ui, tokens: &Tokens, salt: impl std::hash::Hash + std::fmt::Debug, label: &str, accent: bool, enabled: bool) -> bool { + let colors = &tokens.theme.colors; + let tint = if accent { colors.on_accent } else { colors.text }; + let galley = ui.painter().layout_no_wrap(label.to_owned(), egui::FontId::proportional(BODY_SIZE), color(tint)); + let height = crate::style::compact::target(ui.ctx(), CONTROL_HEIGHT); + let size = egui::vec2((galley.size().x + BUTTON_PAD).max(height), height); + let (rect, _) = ui.allocate_exact_size(size, egui::Sense::hover()); + let response = ui.interact(rect, ui.id().with(salt), if enabled { egui::Sense::click() } else { egui::Sense::hover() }); + let fill = if accent { + colors.accent + } else if response.hovered() { + colors.selection + } else { + colors.surface_2 + }; + ui.painter().rect_filled(rect, crate::style::radius_u8(tokens.radius_medium()), color(fill).gamma_multiply(if enabled { 1.0 } else { 0.4 })); + ui.painter().galley(rect.center() - galley.size() / 2.0, galley, color(tint)); + response.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, enabled, label)); + response.clicked() && enabled +} + +/// Toolbar with cache-only search and explicit network check. +pub(super) fn header(view: &mut RegistryView, ui: &mut Ui, tokens: &Tokens, actions: &mut Vec) { + ui.label(RichText::new(strings::title()).size(TITLE_SIZE).strong().color(color(tokens.theme.colors.text))); + ui.label(RichText::new(strings::description()).color(color(tokens.theme.colors.text_muted))); + ui.add_space(GAP); + ui.horizontal_wrapped(|ui| { + let width = SEARCH_WIDTH.min(ui.available_width()); + ui.push_id("registry-search", |ui| { + crate::settings_view::controls::input(ui, tokens, &mut view.query, width, strings::search()); + }); + if button(ui, tokens, "registry-check", strings::check(), true, !view.busy) { + actions.push(Action::Check); + } + }); + ui.add_space(GAP); + ui.horizontal_wrapped(|ui| { + for (kind, label) in kinds() { + if button(ui, tokens, ("registry-kind", kind), label, view.kind == kind, true) { + kind.clone_into(&mut view.kind); + } + } + }); + ui.add_space(GAP); + ui.horizontal_wrapped(|ui| { + let checked = view.last_checked.as_ref().map_or_else(|| strings::never().to_owned(), |time| strings::last_checked().replace("{time}", time)); + ui.label(RichText::new(checked).color(color(tokens.theme.colors.text_muted))); + if view.updates() > 0 { + ui.label(RichText::new(strings::updates().replace("{count}", &view.updates().to_string())).color(color(tokens.theme.colors.accent))); + } + }); + ui.horizontal_wrapped(|ui| { + let label = strings::auto_check(); + ui.label(label); + switch(ui, tokens, "registry-auto-check", label, view.auto_check, !view.busy, |next| { + view.auto_check = next; + actions.push(Action::AutoCheck(next)); + }); + }); + ui.add_space(GAP); +} + +/// An accessible switch with compact touch sizing and stable identity. +pub(super) fn switch(ui: &mut Ui, tokens: &Tokens, salt: &str, label: &str, value: bool, enabled: bool, mut changed: impl FnMut(bool)) { + let height = crate::style::compact::target(ui.ctx(), CONTROL_HEIGHT); + let (rect, _) = ui.allocate_exact_size(egui::vec2(SWITCH_WIDTH, height), egui::Sense::hover()); + let response = ui.interact(rect, ui.id().with(salt), if enabled { egui::Sense::click() } else { egui::Sense::hover() }); + let track = egui::Rect::from_center_size(rect.center(), SWITCH_TRACK); + let colors = &tokens.theme.colors; + ui.painter().rect_filled(track, SWITCH_RADIUS, color(if value { colors.accent } else { colors.surface_2 })); + let center = egui::pos2(if value { track.right() - SWITCH_RADIUS } else { track.left() + SWITCH_RADIUS }, track.center().y); + ui.painter().circle_filled(center, THUMB_RADIUS, color(colors.on_accent)); + response.widget_info(|| egui::WidgetInfo::selected(egui::WidgetType::Checkbox, enabled, value, label)); + if enabled && response.clicked() { + changed(!value); + } +} + +fn kinds() -> [(&'static str, &'static str); 7] { + [ + ("", strings::all()), + ("plugin", strings::plugins()), + ("theme", strings::themes()), + ("pack", strings::packs()), + ("grammar", strings::grammars()), + ("font", strings::fonts()), + ("snippet", strings::snippets()), + ] +} + +/// User-facing kind label from the i18n catalogue. +pub(super) fn kind_label(kind: &str) -> &'static str { + kinds().into_iter().find(|(id, _)| *id == kind).map_or(strings::all(), |(_, label)| label) +} + +/// Visible catalogue, filtered locally with no actions for search or chips. +pub(super) fn show(view: &mut RegistryView, ui: &mut Ui, tokens: &Tokens, actions: &mut Vec) { + let query = view.query.trim().to_lowercase(); + let rows: Vec<_> = view + .rows + .iter() + .filter(|row| { + (view.kind.is_empty() || row.kind == view.kind) + && [row.title.as_str(), row.id.as_str(), row.summary.as_str()].iter().any(|text| text.to_lowercase().contains(&query)) + }) + .collect(); + egui::ScrollArea::vertical().id_salt("notescraft-registry-scroll").auto_shrink([false, false]).show(ui, |ui| { + if rows.is_empty() { + ui.label(strings::empty()); + } + for row in rows { + ui.push_id(("registry-entry", &row.kind, &row.id), |ui| { + egui::Frame::new().fill(color(tokens.theme.colors.surface)).corner_radius(crate::theme::PANEL_RADIUS).inner_margin(PAD).show(ui, |ui| { + ui.set_width(ui.available_width()); + ui.label(RichText::new(&row.title).strong().size(ENTRY_TITLE_SIZE).color(color(tokens.theme.colors.text))); + ui.label(RichText::new(kind_label(&row.kind)).color(color(tokens.theme.colors.accent))); + let size = strings::bytes().replace("{size}", &row.size.to_string()); + ui.label( + RichText::new(strings::version().replace("{version}", &row.version).replace("{size}", &size)) + .color(color(tokens.theme.colors.text_muted)), + ); + if !row.summary.is_empty() { + ui.label(&row.summary); + } + ui.add_space(GAP); + ui.horizontal_wrapped(|ui| { + state_button(ui, tokens, row, !view.busy, actions); + if button(ui, tokens, "details", strings::details(), false, !view.busy) { + view.selected = Some((row.kind.clone(), row.id.clone())); + actions.push(Action::Info(row.kind.clone(), row.id.clone())); + } + }); + }); + }); + ui.add_space(GAP); + } + }); +} + +fn state_button(ui: &mut Ui, tokens: &Tokens, row: &Entry, enabled: bool, actions: &mut Vec) { + let action = match row.state.as_str() { + "available" => Some((strings::install().to_owned(), Action::Install(row.kind.clone(), row.id.clone()))), + "update" => Some((strings::update().replace("{version}", &row.version), Action::Update(row.kind.clone(), row.id.clone()))), + _ => None, + }; + if let Some((label, action)) = action { + if button(ui, tokens, "entry-action", &label, true, enabled) { + actions.push(action); + } + return; + } + let label = match row.state.as_str() { + "installed" => strings::installed().to_owned(), + "revoked" => strings::revoked().to_owned(), + "incompatible" => strings::incompatible().replace("{version}", &row.min_app_version), + _ => strings::unsupported().to_owned(), + }; + ui.label(RichText::new(label).color(color(tokens.theme.colors.text_muted))); +} diff --git a/crates/ui-egui/src/views/registry_view/mod.rs b/crates/ui-egui/src/views/registry_view/mod.rs new file mode 100644 index 0000000..776f4fe --- /dev/null +++ b/crates/ui-egui/src/views/registry_view/mod.rs @@ -0,0 +1,163 @@ +//! Offline registry browser. Painting emits actions; the shell owns every engine call. + +mod detail; +mod list; +pub(crate) use list::button; + +use crate::text::registry as strings; +use crate::theme::Tokens; +use serde::Deserialize; +use serde_json::Value; + +/// A cached add-on row, joined with its installed state by `registry.list`. +#[derive(Clone, Debug, Default, Deserialize, serde::Serialize)] +#[serde(default)] +pub struct Entry { + /// Stable registry identifier. + pub id: String, + /// Registry kind. + pub kind: String, + /// Display title. + pub title: String, + /// Available version. + pub version: String, + /// Currently installed version. + pub installed_version: Option, + /// Download size in bytes. + pub size: u64, + /// Engine availability state. + pub state: String, + /// Whether the publisher recommends this entry on first run. + pub suggested: bool, + /// Publisher description. + pub summary: String, + /// Plugin capabilities, displayed as sentences. + pub permissions: Vec, + /// SPDX license identifier. + pub license: String, + /// Publisher source URL. + pub source: String, + /// Signature verification key identifier. + pub key_id: String, + /// Published file descriptors. + pub files: Vec, + /// Minimum supported application version. + pub min_app_version: String, + /// Whether this plugin is enabled in the current vault. + pub enabled: bool, + /// Retained version available for rollback. + pub previous: Option, +} + +/// An explicit request for the shell; the view never performs IO. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Action { + /// Refresh the signed index. + Check, + /// Install kind and identifier. + Install(String, String), + /// Update kind and identifier. + Update(String, String), + /// Remove kind and identifier. + Remove(String, String), + /// Restore the previous version. + Rollback(String, String), + /// Approve and enable a plugin in this vault. + Enable(String), + /// Disable a plugin in this vault. + Disable(String), + /// Persist automatic update checks. + AutoCheck(bool), + /// Load cached details for kind and identifier. + Info(String, String), +} + +/// Browser state independent of engine and network implementations. +#[derive(Clone, Debug, Default)] +pub struct RegistryView { + /// Rows from the last successful cache read. + pub rows: Vec, + /// Local search text; typing never requests network access. + pub query: String, + /// Kind filter; empty means all kinds. + pub kind: String, + /// Last network operation failed offline; cached rows remain visible. + pub offline: bool, + /// Latest command failure. + pub error: Option, + /// Whether a command is running. + pub busy: bool, + /// Human-readable time supplied by the shell. + pub last_checked: Option, + /// Automatic checks are off by default. + pub auto_check: bool, + /// Open detail drawer, keyed by kind and identifier. + pub selected: Option<(String, String)>, +} + +impl RegistryView { + /// Applies a cache-only list reply, preserving existing rows on malformed replies. + pub fn apply_reply(&mut self, reply: &Value) { + let rows = reply.as_array().map(|_| reply).or_else(|| reply.get("entries")).or_else(|| reply.get("items")); + if let Some(rows) = rows { + match serde_json::from_value(rows.clone()) { + Ok(rows) => { + self.rows = rows; + self.error = None; + } + Err(_) => self.error = Some(strings::invalid_reply().to_owned()), + } + } + } + + /// Merges a cache-only detail reply into its existing catalogue row. + pub fn apply_info(&mut self, reply: &Value) { + let data = reply.get("entry").unwrap_or(reply); + let Some(id) = data.get("id").and_then(Value::as_str) else { return }; + let Some(row) = self.rows.iter_mut().find(|row| row.id == id && data.get("kind").and_then(Value::as_str).is_none_or(|kind| row.kind == kind)) else { + return; + }; + let Ok(mut merged) = serde_json::to_value(&*row) else { return }; + if let (Some(target), Some(source)) = (merged.as_object_mut(), data.as_object()) { + target.extend(source.clone()); + if let Some(key) = data.get("signature").and_then(|signature| signature.get("key_id")) { + target.insert("key_id".to_owned(), key.clone()); + } + } + if let Ok(updated) = serde_json::from_value(merged) { + *row = updated; + } + } + + /// Applies an engine error without discarding the cached catalogue. + pub fn set_error(&mut self, code: &str, message: &str) { + if code == "offline" { + self.offline = true; + } + self.error = Some(message.to_owned()); + } + + /// Update count shown only after a successful explicit check. + #[must_use] + pub fn updates(&self) -> usize { + if self.last_checked.is_some() { self.rows.iter().filter(|row| row.state == "update").count() } else { 0 } + } + + /// Paints the browser and returns explicit user actions from this frame. + pub fn show(&mut self, ui: &mut egui::Ui, tokens: &Tokens) -> Vec { + let mut actions = Vec::new(); + list::header(self, ui, tokens, &mut actions); + if self.offline { + ui.label(strings::offline()); + } + if let Some(error) = &self.error { + ui.colored_label(crate::theme::color(tokens.theme.colors.danger), error); + } + if self.busy { + ui.label(strings::loading()); + } + list::show(self, ui, tokens, &mut actions); + detail::show(self, ui, tokens, &mut actions); + actions + } +} diff --git a/crates/ui-egui/src/views/settings_view.rs b/crates/ui-egui/src/views/settings_view.rs index 69d0fdd..4e1b4b4 100644 --- a/crates/ui-egui/src/views/settings_view.rs +++ b/crates/ui-egui/src/views/settings_view.rs @@ -56,6 +56,8 @@ pub enum SettingsCategory { CorePlugins, /// Installed community plugins. CommunityPlugins, + /// Cached add-on browser. + Registry, /// External assistant permissions and command. AiAssistant, /// Advanced preferences. @@ -74,6 +76,7 @@ impl SettingsCategory { Self::Calendar, Self::CorePlugins, Self::CommunityPlugins, + Self::Registry, Self::AiAssistant, Self::Advanced, ]; @@ -88,6 +91,7 @@ impl SettingsCategory { Self::DailyNotes, Self::CorePlugins, Self::CommunityPlugins, + Self::Registry, Self::AiAssistant, Self::Advanced, ]; @@ -105,6 +109,7 @@ impl SettingsCategory { Self::Calendar => strings::calendar(), Self::CorePlugins => strings::core_plugins(), Self::CommunityPlugins => strings::community_plugins(), + Self::Registry => crate::text::registry::title(), Self::AiAssistant => strings::ai_assistant(), Self::Advanced => strings::advanced(), } @@ -123,6 +128,7 @@ impl SettingsCategory { Self::Calendar => "calendar", Self::CorePlugins => "layout-dashboard", Self::CommunityPlugins => "git-branch", + Self::Registry => "download", Self::AiAssistant => "sparkles", Self::Advanced => "bug", } @@ -157,6 +163,12 @@ impl ThemeInfo { pub struct SettingsView { /// Currently selected navigation category. pub selected: SettingsCategory, + /// Cached registry browser state. + pub registry: crate::views::registry_view::RegistryView, + /// Registry actions waiting for the shell after painting. + pub(crate) registry_actions: Vec, + /// Whether the shell has read the cache during this visit. + pub(crate) registry_open: bool, /// Pages whose sub-items are open in the list. pub(crate) expanded: BTreeSet, /// Search text applied across all setting rows. @@ -190,6 +202,9 @@ impl Default for SettingsView { fn default() -> Self { Self { selected: SettingsCategory::Appearance, + registry: crate::views::registry_view::RegistryView::default(), + registry_actions: Vec::new(), + registry_open: false, expanded: BTreeSet::new(), search_text: String::new(), pending_ui_scale: None, diff --git a/crates/ui-egui/src/views/settings_view/categories.rs b/crates/ui-egui/src/views/settings_view/categories.rs index 855e87d..543383e 100644 --- a/crates/ui-egui/src/views/settings_view/categories.rs +++ b/crates/ui-egui/src/views/settings_view/categories.rs @@ -52,6 +52,10 @@ pub(super) fn show_category( SettingsCategory::DailyNotes => daily_notes(ui, tokens, settings, state, category, changes), SettingsCategory::Calendar => calendar::show(ui, tokens, settings, state, category, changes), SettingsCategory::Sync => sync::show(ui, tokens, state, category), + SettingsCategory::Registry if matches(state, category, crate::text::registry::title(), crate::text::registry::description()) => { + state.registry_actions.extend(state.registry.show(ui, tokens)); + } + SettingsCategory::Registry => {} SettingsCategory::CorePlugins => { core_cards::show(ui, tokens, settings, state, category, changes); maps::show(ui, tokens, settings, state, category, changes); diff --git a/crates/ui-egui/src/views/settings_view/nav.rs b/crates/ui-egui/src/views/settings_view/nav.rs index 05f8176..a184500 100644 --- a/crates/ui-egui/src/views/settings_view/nav.rs +++ b/crates/ui-egui/src/views/settings_view/nav.rs @@ -9,6 +9,8 @@ use crate::theme::{Tokens, color}; const CHILD_INDENT: f32 = 22.0; /// Id of the section heading the page should scroll to once it is drawn. const ANCHOR_ID: &str = "settings-anchor"; +/// Registry update badge radius, points. +const BADGE_RADIUS: f32 = 4.0; /// One visible row of the list. struct Row { @@ -77,7 +79,15 @@ pub(super) fn navigation(ui: &mut Ui, tokens: &Tokens, state: &mut SettingsView) ); for row in &list { let active = is_active(row); - let (rect, hit) = ui.allocate_exact_size(Vec2::new(width, NAV_ROW_HEIGHT), Sense::click()); + let sense = if row.page == SettingsCategory::Registry { Sense::hover() } else { Sense::click() }; + let (rect, hit) = ui.allocate_exact_size(Vec2::new(width, NAV_ROW_HEIGHT), sense); + let hit = if row.page == SettingsCategory::Registry { + let hit = ui.interact(rect, ui.id().with("settings-registry-category"), Sense::click()); + hit.widget_info(|| egui::WidgetInfo::selected(egui::WidgetType::SelectableLabel, ui.is_enabled(), active, row.label)); + hit + } else { + hit + }; let hover = crate::motion::hover(ui.ctx(), ("settings-category", row.label), &hit); let wobble = crate::motion::wobble(ui.ctx(), ("settings-category-wobble", row.label), hit.clicked(), 0.45); if !active { @@ -100,6 +110,9 @@ pub(super) fn navigation(ui: &mut Ui, tokens: &Tokens, state: &mut SettingsView) let arrow = egui::Rect::from_center_size(egui::Pos2::new(rect.right() - 18.0, rect.center().y), Vec2::splat(14.0)); crate::icons::paint(ui, name, arrow, tint); } + if row.page == SettingsCategory::Registry && state.registry.last_checked.is_some() && state.registry.updates() > 0 { + ui.painter().circle_filled(egui::pos2(rect.right() - 18.0, rect.center().y), BADGE_RADIUS, color(c.accent)); + } if hit.clicked() { click(ui, state, row); } @@ -116,11 +129,22 @@ pub(super) fn strip(ui: &mut Ui, tokens: &Tokens, state: &mut SettingsView) { let active = state.selected == category; let galley = ui.painter().layout_no_wrap(category.label().to_owned(), egui::FontId::proportional(13.0), color(c.text)); let size = Vec2::new(galley.size().x + 32.0, crate::style::compact::MIN_TARGET_COMPACT); - let (rect, hit) = ui.allocate_exact_size(size, Sense::click()); + let sense = if category == SettingsCategory::Registry { Sense::hover() } else { Sense::click() }; + let (rect, hit) = ui.allocate_exact_size(size, sense); + let hit = if category == SettingsCategory::Registry { + let hit = ui.interact(rect, ui.id().with("settings-registry-category"), Sense::click()); + hit.widget_info(|| egui::WidgetInfo::selected(egui::WidgetType::SelectableLabel, ui.is_enabled(), active, category.label())); + hit + } else { + hit + }; let fill = if active { color(c.selection) } else { color(c.surface) }; ui.painter().rect_filled(rect, CornerRadius::same(12), fill); let tint = if active { color(c.text) } else { color(c.text_muted) }; ui.painter().galley(rect.center() - galley.size() / 2.0, galley, tint); + if category == SettingsCategory::Registry && state.registry.last_checked.is_some() && state.registry.updates() > 0 { + ui.painter().circle_filled(egui::pos2(rect.right() - 8.0, rect.top() + 8.0), BADGE_RADIUS, color(c.accent)); + } if hit.clicked() { state.selected = category; } diff --git a/crates/ui-egui/tests/registry_view.rs b/crates/ui-egui/tests/registry_view.rs new file mode 100644 index 0000000..454c4fc --- /dev/null +++ b/crates/ui-egui/tests/registry_view.rs @@ -0,0 +1,170 @@ +//! Registry browser driven through real egui frames with cached engine-shaped replies. + +use egui::{Event, Vec2}; +use egui_kittest::{Harness, kittest::Queryable}; +use notescraft_ui_egui::{ + theme::Tokens, + views::registry_view::{Action, RegistryView}, +}; +use serde_json::json; + +struct Page { + view: RegistryView, + actions: Vec, + tokens: Tokens, + warmed: bool, +} + +fn rows() -> serde_json::Value { + json!([ + {"id":"twig","kind":"grammar","title":"Twig","summary":"Syntax highlighting for Twig templates.","version":"1.2.0","size":3900,"state":"available"}, + {"id":"dice","kind":"plugin","title":"Dice","summary":"Roll dice in your notes.","version":"1.2.0","installed_version":"1.0.0","previous":"0.9.0","size":4800,"state":"update","permissions":["vault.read","vault.write","editor"],"license":"MIT","source":"https://example.test/dice","key_id":"first-party","files":[{"path":"plugin.wasm","size":4800}]}, + {"id":"theme","kind":"theme","title":"Frappe","version":"1.0.0","state":"installed","installed_version":"1.0.0"}, + {"id":"old","kind":"plugin","title":"Revoked plugin","state":"revoked"}, + {"id":"future","kind":"plugin","title":"Future plugin","state":"incompatible","min_app_version":"0.3"}, + {"id":"font","kind":"font","title":"Math font","state":"unsupported"} + ]) +} + +fn page(theme: &str, width: f32) -> Harness<'static, Page> { + let mut view = RegistryView::default(); + view.apply_reply(&rows()); + let page = Page { view, actions: Vec::new(), tokens: Tokens::new(notescraft_themes::builtin(theme).expect("theme")), warmed: false }; + Harness::builder().with_size(Vec2::new(width, 1200.0)).build_ui_state( + |ui, page: &mut Page| { + if !page.warmed { + notescraft_ui_egui::fonts::install(ui.ctx()); + notescraft_ui_egui::motion::set_enabled(ui.ctx(), false); + notescraft_ui_egui::style::apply(ui.ctx(), &page.tokens); + page.warmed = true; + return; + } + egui::Frame::new().fill(notescraft_ui_egui::theme::color(page.tokens.theme.colors.canvas)).inner_margin(24).show(ui, |ui| { + page.actions.extend(page.view.show(ui, &page.tokens)); + }); + }, + page, + ) +} + +fn settle(h: &mut Harness<'_, Page>) { + h.run_steps(4); +} + +#[test] +fn every_row_state_and_install_click() { + let mut h = page("catppuccin-mocha", 1000.0); + settle(&mut h); + for label in ["Install", "Update to 1.2.0", "Installed", "Revoked", "Needs NotesCraft 0.3", "Not available in this version"] { + assert!(h.query_by_label(label).is_some(), "missing {label}"); + } + h.get_by_label("Install").click(); + settle(&mut h); + assert_eq!(h.state().actions, vec![Action::Install("grammar".to_owned(), "twig".to_owned())]); +} + +#[test] +fn typing_search_and_kind_filter_are_local() { + let mut h = page("catppuccin-latte", 1000.0); + settle(&mut h); + h.get_by_role(egui::accesskit::Role::TextInput).click(); + h.step(); + h.input_mut().events.push(Event::Text("Twig".to_owned())); + settle(&mut h); + assert!(h.query_by_label("Twig").is_some()); + assert!(h.query_by_label("Dice").is_none()); + assert_eq!(h.state().actions, Vec::::new()); + h.get_by_role_and_label(egui::accesskit::Role::Button, "Plugins").click(); + settle(&mut h); + assert!(h.query_by_label("Twig").is_none()); + assert!(h.query_by_label("No add-ons found. Try another search or check for updates.").is_some()); + assert_eq!(h.state().actions, Vec::::new()); +} + +#[test] +fn offline_error_and_empty_states_preserve_cache() { + let mut h = page("catppuccin-mocha", 1000.0); + h.state_mut().view.set_error("offline", "Connection refused"); + settle(&mut h); + assert!(h.query_by_label("Offline โ€” showing cached add-ons. Try checking again when connected.").is_some()); + assert!(h.query_by_label("Connection refused").is_some()); + assert!(h.query_by_label("Twig").is_some()); + h.state_mut().view.rows.clear(); + settle(&mut h); + assert!(h.query_by_label("No add-ons found. Try another search or check for updates.").is_some()); +} + +#[test] +fn unknown_signing_key_error_stays_visible_without_discarding_cache() { + let mut h = page("catppuccin-mocha", 1000.0); + let message = notescraft_engine::Error::Addon("unknown signing key `first-party`".to_owned()).to_string(); + h.state_mut().view.set_error("addon", &message); + settle(&mut h); + assert!(h.query_by_label(&message).is_some()); + assert!(h.query_by_label("Twig").is_some()); + assert!(!h.state().view.offline); + assert_eq!(h.state().actions, Vec::::new()); +} + +#[test] +fn permissions_are_sentences_and_detail_actions_are_explicit() { + let mut h = page("catppuccin-mocha", 1000.0); + h.state_mut().view.selected = Some(("plugin".to_owned(), "dice".to_owned())); + settle(&mut h); + for label in ["Read notes and search", "Create and change notes", "Edit the open note", "Remove", "Rollback"] { + assert!(h.query_by_label(label).is_some(), "missing {label}"); + } + assert_eq!(h.state().actions, Vec::::new()); + assert!(h.query_by_role_and_label(egui::accesskit::Role::CheckBox, "Enable in this vault").is_some()); + h.get_by_label("Rollback").click(); + settle(&mut h); + assert_eq!(h.state().actions, vec![Action::Rollback("plugin".to_owned(), "dice".to_owned())]); +} + +#[test] +fn update_badge_requires_a_successful_check() { + let mut h = page("catppuccin-mocha", 1000.0); + settle(&mut h); + assert_eq!(h.state().view.updates(), 0); + assert!(h.query_by_label("1 updates").is_none()); + h.get_by_label("Check for updates").click(); + settle(&mut h); + assert_eq!(h.state().actions, vec![Action::Check]); + assert_eq!(h.state().view.updates(), 0); + h.state_mut().view.last_checked = Some("just now".to_owned()); + settle(&mut h); + assert!(h.query_by_label("1 updates").is_some()); +} + +#[test] +fn malformed_reply_does_not_discard_cache_and_info_keeps_install_state() { + let mut view = RegistryView::default(); + view.apply_reply(&rows()); + view.apply_reply(&json!({"entries":[{"permissions":12}]})); + assert_eq!(view.rows.len(), 6); + assert!(view.error.is_some()); + view.apply_info(&json!({"kind":"plugin","id":"dice","summary":"Updated metadata"})); + let row = view.rows.iter().find(|row| row.id == "dice").expect("dice"); + assert_eq!(row.summary, "Updated metadata"); + assert_eq!(row.installed_version.as_deref(), Some("1.0.0")); +} + +#[test] +#[ignore = "offscreen PNG visual review"] +fn render_registry_pngs() { + let output = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/agent-sol/png"); + std::fs::create_dir_all(&output).expect("png folder"); + for (theme, width, name) in [ + ("catppuccin-mocha", 1000.0, "registry-mocha.png"), + ("catppuccin-latte", 1000.0, "registry-latte.png"), + ("catppuccin-mocha", 390.0, "registry-compact.png"), + ] { + let mut h = page(theme, width); + settle(&mut h); + h.render().expect("render").save(output.join(name)).expect("png"); + } + let mut h = page("catppuccin-mocha", 1000.0); + h.state_mut().view.selected = Some(("plugin".to_owned(), "dice".to_owned())); + settle(&mut h); + h.render().expect("render detail").save(output.join("registry-detail.png")).expect("png"); +} diff --git a/packaging/linux/package.sh b/packaging/linux/package.sh index fd664de..a473a27 100755 --- a/packaging/linux/package.sh +++ b/packaging/linux/package.sh @@ -61,6 +61,8 @@ rm -rf "$WORK" # ---- stage an FHS tree (shared by every format) ------------------------------------------------- install -Dm755 "$BIN/notescraft" "$STAGE/usr/bin/notescraft" install -Dm755 "$BIN/notescraft-cli" "$STAGE/usr/bin/notescraft-cli" +# Signed snapshot of the add-ons moved out of the binary (registry spec section 11); absent until it is built. +if [ -d "$ROOT/registry-seed" ]; then cp -R "$ROOT/registry-seed" "$STAGE/usr/bin/registry-seed"; fi "${CROSS_COMPILE:-}strip" "$STAGE/usr/bin/notescraft" "$STAGE/usr/bin/notescraft-cli" 2>/dev/null || true install -Dm644 "$HERE/$APP_ID.desktop" "$STAGE/usr/share/applications/$APP_ID.desktop" install -Dm644 "$HERE/$APP_ID.mime.xml" "$STAGE/usr/share/mime/packages/$APP_ID.xml" diff --git a/packaging/macos/package.sh b/packaging/macos/package.sh index e78d48d..b687ad7 100755 --- a/packaging/macos/package.sh +++ b/packaging/macos/package.sh @@ -105,6 +105,8 @@ cp "$WORK/bin/notescraft" "$APP/Contents/MacOS/NotesCraft" # The assistant hands agents `notescraft-cli mcp` as the vault MCP server and looks for it beside the app binary. cp "$WORK/bin/notescraft-cli" "$APP/Contents/MacOS/notescraft-cli" cp "$ROOT/assets/app-icon/notescraft.icns" "$APP/Contents/Resources/NotesCraft.icns" +# Signed snapshot of the add-ons moved out of the binary (registry spec section 11); absent until it is built. +if [ -d "$ROOT/registry-seed" ]; then cp -R "$ROOT/registry-seed" "$APP/Contents/Resources/registry-seed"; fi sed -e "s/@VERSION@/$VERSION/g" -e "s/@SHORT_VERSION@/$SHORT_VERSION/g" \ -e "s/@BUILD_SHA@/${NOTESCRAFT_BUILD_SHA:-unknown}/g" \ "$HERE/Info.plist.in" >"$APP/Contents/Info.plist" diff --git a/packaging/windows/package.ps1 b/packaging/windows/package.ps1 index b98d7a8..060312b 100644 --- a/packaging/windows/package.ps1 +++ b/packaging/windows/package.ps1 @@ -100,6 +100,9 @@ $Stage = Join-Path $TargetDir "windows-package\$Arch" Remove-Item -Recurse -Force $Stage -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force -Path $Stage | Out-Null Copy-Item (Join-Path $Bin 'notescraft.exe'), (Join-Path $Bin 'notescraft-cli.exe') $Stage +# Signed snapshot of the add-ons moved out of the binary (registry spec section 11); absent until it is built. +$SeedSrc = Join-Path $Root 'registry-seed' +if (Test-Path $SeedSrc) { Copy-Item $SeedSrc (Join-Path $Stage 'registry-seed') -Recurse } & (Join-Path $PSScriptRoot 'sign.ps1') (Join-Path $Stage 'notescraft.exe') (Join-Path $Stage 'notescraft-cli.exe') @@ -119,6 +122,7 @@ $Portable = Join-Path $TargetDir "windows-package\notescraft-$Version-windows-$A Remove-Item -Recurse -Force $Portable -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force -Path $Portable | Out-Null Copy-Item (Join-Path $Stage '*.exe') $Portable +if (Test-Path (Join-Path $Stage 'registry-seed')) { Copy-Item (Join-Path $Stage 'registry-seed') $Portable -Recurse } foreach ($f in 'README.md', 'LICENSE', 'LICENSE-MIT', 'LICENSE-APACHE') { $p = Join-Path $Root $f if (Test-Path $p) { Copy-Item $p $Portable } diff --git a/specs/registry/tasks.md b/specs/registry/tasks.md index 0d58961..cf67019 100644 --- a/specs/registry/tasks.md +++ b/specs/registry/tasks.md @@ -8,27 +8,27 @@ One task = one agent (`specs/agents/opencode-runbook.md`). `[ ]` open. Spec: `sp ## Phase 0 - size fixes without a registry (do these first, they are 25 % of the binary) -- [ ] 0.1 LEAD: release profile. OWNS: root `Cargo.toml` (`[profile.release]` only), `packaging/macos/*.sh` and `packaging/windows/*` only if they strip separately. GOAL: add `strip = "symbols"`; confirm Linux `package.sh` strip is now redundant but harmless. VERIFY: `CARGO_TARGET_DIR=target/agent-size cargo build --release -p notescraft`, `ls -l` is at most 34.9 MB (measured 34,851,464), the app still starts (`measure.py` in `measurements.md`), `cargo xtask ci`. DO NOT: set `panic = "abort"` (the never-crash policy and tests rely on unwinding). -- [ ] 0.2 Share fonts (brief `briefs/29-font-dedupe.md`). -- [ ] 0.3 Drop egui `default_fonts`. OWNS: `apps/notescraft/Cargo.toml` (eframe feature list only, LEAD edit), `crates/ui-egui/src/design/fonts/mod.rs`, `assets/fonts/NotoEmoji-Regular.ttf` (copy from `epaint_default_fonts`, with its OFL text), `crates/ui-egui/tests/fonts_emoji.rs`. FILES TO OPEN: `crates/ui-egui/src/design/fonts/mod.rs` (`install`), `~/.cargo/registry/src/*/epaint_default_fonts-*/fonts/`. GOAL: `eframe` without `default_fonts`, `FontDefinitions::empty()` plus Inter, JetBrains Mono, STIX, NotoEmoji (last in every chain), the system fallbacks as today. TESTS: kittest renders `๐Ÿ˜€ โ˜… โ†’ โœ“` in proportional and monospace with no missing-glyph box (compare against the egui replacement glyph); the snapshot of the Settings and Explorer screens differs only by sub-pixel noise. VERIFY: standing + `cargo run -p notescraft-ui-egui --example snapshot -- --vault --out a.png --size 1440x900` before and after, read both PNGs; binary shrinks by at least 0.9 MiB. DO NOT: change any font size or weight. +- [x] 0.1 LEAD: release profile. OWNS: root `Cargo.toml` (`[profile.release]` only), `packaging/macos/*.sh` and `packaging/windows/*` only if they strip separately. GOAL: add `strip = "symbols"`; confirm Linux `package.sh` strip is now redundant but harmless. VERIFY: `CARGO_TARGET_DIR=target/agent-size cargo build --release -p notescraft`, `ls -l` is at most 34.9 MB (measured 34,851,464), the app still starts (`measure.py` in `measurements.md`), `cargo xtask ci`. DO NOT: set `panic = "abort"` (the never-crash policy and tests rely on unwinding). +- [x] 0.2 Share fonts (brief `briefs/29-font-dedupe.md`). +- [x] 0.3 Drop egui `default_fonts`. OWNS: `apps/notescraft/Cargo.toml` (eframe feature list only, LEAD edit), `crates/ui-egui/src/design/fonts/mod.rs`, `assets/fonts/NotoEmoji-Regular.ttf` (copy from `epaint_default_fonts`, with its OFL text), `crates/ui-egui/tests/fonts_emoji.rs`. FILES TO OPEN: `crates/ui-egui/src/design/fonts/mod.rs` (`install`), `~/.cargo/registry/src/*/epaint_default_fonts-*/fonts/`. GOAL: `eframe` without `default_fonts`, `FontDefinitions::empty()` plus Inter, JetBrains Mono, STIX, NotoEmoji (last in every chain), the system fallbacks as today. TESTS: kittest renders `๐Ÿ˜€ โ˜… โ†’ โœ“` in proportional and monospace with no missing-glyph box (compare against the egui replacement glyph); the snapshot of the Settings and Explorer screens differs only by sub-pixel noise. VERIFY: standing + `cargo run -p notescraft-ui-egui --example snapshot -- --vault --out a.png --size 1440x900` before and after, read both PNGs; binary shrinks by at least 0.9 MiB. DO NOT: change any font size or weight. - [ ] 0.4 LEAD: `opt-level = "s"` decision. GOAL: measure on `target/agent-size` with the profile env vars: index build of the 6500-note vault (CPU seconds, `measure.py --large-ok`), `cargo test --release -p notescraft-ui-egui --test perf_code_blocks -- --ignored --nocapture`, a graph timelapse frame time; adopt only if none is worse than 10 %. Write the numbers in `measurements.md` section 1.3. DO NOT: adopt without numbers. -- [ ] 0.5 Lazy system fallback fonts. OWNS: `crates/ui-egui/src/design/fonts/mod.rs`, `.../scripts.rs` (add Hebrew and Arabic to the on-demand table if missing), `crates/ui-egui/src/design/fonts/tests.rs`. FILES TO OPEN: same, `note_text` / `apply_pending`. GOAL: remove `SYSTEM_FALLBACKS` eager read at `install`; Hebrew and Arabic load through `note_text` like other scripts, so launch does not read `SFHebrew.ttf` and `SFArabic.ttf`. TESTS: a note with Hebrew shows no box after one frame loop (existing test style in `tests.rs`); a note without them never reads the files (counter in the injected `file_size`). VERIFY: standing. DO NOT: change the Latin chain. +- [x] 0.5 Lazy system fallback fonts. OWNS: `crates/ui-egui/src/design/fonts/mod.rs`, `.../scripts.rs` (add Hebrew and Arabic to the on-demand table if missing), `crates/ui-egui/src/design/fonts/tests.rs`. FILES TO OPEN: same, `note_text` / `apply_pending`. GOAL: remove `SYSTEM_FALLBACKS` eager read at `install`; Hebrew and Arabic load through `note_text` like other scripts, so launch does not read `SFHebrew.ttf` and `SFArabic.ttf`. TESTS: a note with Hebrew shows no box after one frame loop (existing test style in `tests.rs`); a note without them never reads the files (counter in the injected `file_size`). VERIFY: standing. DO NOT: change the Latin chain. ## Phase 1 - pure registry core (engine, no IO unless stated) -- [ ] 1. Index model, validation, version compare (brief `briefs/30-registry-index-model.md`). -- [ ] 2. Fileset guard: path and size rules (brief `briefs/31-registry-fileset-guard.md`). -- [ ] 3. BLOCKED (owner Q1). Signature canonical text and verify. +- [x] 1. Index model, validation, version compare (brief `briefs/30-registry-index-model.md`). +- [x] 2. Fileset guard: path and size rules (brief `briefs/31-registry-fileset-guard.md`). +- [x] 3. Signature canonical text and verify. OWNS: `crates/engine/src/addons/verify.rs`, `crates/engine/src/addons/keys.rs`, `crates/engine/tests/addons_verify.rs`. FILES TO OPEN: `spec.md` D3, `crates/engine/src/addons/model.rs` (task 1), `crates/syntax/src/validate.rs` (`sha256_hex`). GOAL: `pub trait Verifier { fn verify(&self, entry: &Entry) -> Result<()>; }`, `pub fn canonical_text(entry: &Entry) -> String`, `pub struct Ed25519 { keys }` implementing it with the dependency the owner approved, `TRUSTED_KEYS` with the owner's public key (placeholder key in tests only), a `FakeVerifier` for other tasks. TESTS: known-answer signature generated by the xtask tool (task 18) verifies; one flipped hash byte fails; unknown `key_id` fails; wrong kind or version in the text fails; files in a different order produce the same text. VERIFY: standing. DO NOT: put a private key anywhere; accept an entry without `signature`. -- [ ] 4. State and store layout. +- [x] 4. State and store layout. OWNS: `crates/engine/src/addons/state.rs`, `crates/engine/src/addons/store.rs`, `crates/engine/tests/addons_state.rs`. FILES TO OPEN: `spec.md` D4, `crates/engine/src/activity/outbox.rs` (versioned JSON pattern), `crates/engine/src/grammars/mod.rs` (`grammar_dir`, settings dir), `craft-libs/crates/atomic/src/lib.rs` (`atomic_write`, `write_new`). GOAL: `State { installed: BTreeMap }` with `parse/to_json` (version 1, cap 512, unknown version `Err`), `Store::open(data_dir)`, `Store::version_dir(kind, id, version) -> PathBuf` built from validated components, `Store::gc(&State)` removing version dirs not in state and `staging/`, `commit(&State)` via `atomic_write`, `VaultFile { enabled, wanted }` with `parse/to_json` (cap 256 each). TESTS: round trip; corrupt JSON returns `Err` and leaves the file byte-identical; gc keeps `version` and `previous` and deletes the rest; commit is the only writer of `state.json`. VERIFY: standing. DO NOT: touch the vault; delete anything outside `/registry/`. -- [ ] 5. Transport and bounded download. +- [x] 5. Transport and bounded download. OWNS: `crates/engine/src/addons/net.rs`, `crates/engine/tests/addons_net.rs`. FILES TO OPEN: `crates/syntax/src/fetch.rs` (`Transport`, `Curl`), `crates/syntax/src/validate.rs` (`check_url`), `crates/engine/src/sync/` (how `ureq` is used), `spec.md` D5, D7. GOAL: `pub trait Transport { fn get(&self, url: &str, max_bytes: u64, if_none_match: Option<&str>) -> Result; }`, `Response { status, etag, bytes }` (304 allowed), `Ureq` implementation (https only, no redirects, total timeout, reads at most `max_bytes + 1` and fails above), `FakeTransport` for tests, `download_file(&dyn Transport, &File, cap) -> Result>` that checks host allow-list, size equals entry size, sha256 equals entry. TESTS: host not allow-listed; `http://`; credentials; IP literal; redirect status; body one byte too long; one byte short; wrong sha256; 304 keeps cache; error maps to `offline`. VERIFY: standing; no real network in tests. DO NOT: spawn `curl`; follow redirects; add a header with an identifier beyond `NotesCraft/`. -- [ ] 6. Per-kind content checks. +- [x] 6. Per-kind content checks. OWNS: `crates/engine/src/addons/kinds.rs`, `crates/engine/tests/addons_kinds.rs`. FILES TO OPEN: `spec.md` D8, `crates/syntax/src/validate.rs` (`check_grammar`), `crates/themes/src/load.rs` (`Theme::from_toml` diagnostics), `crates/plugins/src/manifest.rs`, `crates/plugins/src/lib.rs`. GOAL: `check(kind, &Entry, &[(path, bytes)]) -> Result<()>` dispatching to grammar, theme, plugin (manifest equals entry: id, version, permissions, command ids; wasm magic `\0asm`), font (magic bytes), snippet (UTF-8, no NUL), pack (each file by its subfolder). TESTS: one good and one bad fixture per kind; plugin manifest with extra permission fails; theme reusing a built-in id fails; `.tmLanguage` renamed to `.sublime-syntax` fails. VERIFY: standing. DO NOT: instantiate wasm here (task 12 does the dry load). -- [ ] 7. BLOCKED on 3. Install, update, remove, rollback pipeline. +- [x] 7. BLOCKED on 3. Install, update, remove, rollback pipeline. OWNS: `crates/engine/src/addons/install.rs`, `crates/engine/tests/addons_install.rs`. FILES TO OPEN: `spec.md` D6, tasks 1 to 6 outputs, `crates/engine/src/grammars/mod.rs` (`save`). GOAL: `install(&Ctx, &Index, kind, id, version) -> Result`, `update`, `remove`, `rollback` exactly as D6 steps 1 to 7; `Ctx { store, transport: &dyn Transport, verifier: &dyn Verifier, app_version, seed: Option<&SeedDir> }`; seed file used when its sha256 equals the entry. TESTS: the list in `spec.md` D13 IO bullet (hash mismatch, short and long body, crash before the state flip leaves the old state and gc removes the new dir, update keeps `previous`, rollback, remove, seed offline, revoked, app too old, bad signature, cap reached). VERIFY: standing. DO NOT: write anywhere but the store and `state.json`; delete the old version before the new state is committed. -- [ ] 8. Commands, gate, command list. +- [x] 8. Commands, gate, command list. OWNS: `crates/engine/src/addons/commands.rs`, `crates/engine/src/addons/mod.rs`, one `mod addons;` in `crates/engine/src/lib.rs`, one `register_addons(reg)` call in `crates/engine/src/commands.rs::register_core`, `crates/engine/src/ai/assistant_gate.rs`, `crates/engine/tests/core_commands.rs` (`CORE_IDS`), `crates/engine/tests/addons_commands.rs`. FILES TO OPEN: `spec.md` D9, `crates/engine/src/bases/commands.rs` (group shape), `crates/engine/src/ai/assistant_gate.rs` (`READ_COMMANDS`, `WRITE_COMMANDS`, `DELETE_COMMANDS`), `docs/CODE-MAP.md`. GOAL: the nine commands of D9, error codes, session fields `addons_transport` and `addons_verifier` injectable like `grammar_transport`. TESTS: list joins state and cache; `registry.install` through the registry with a fake transport; read-only session refuses mutating ids; gate: `list|info|suggest` READ, `enable|disable` WRITE, the five others need `allow_delete`; offline `check` leaves the cache. VERIFY: standing + `--test core_commands --test addons_commands`. DO NOT: touch UI; change the `grammar.*` codes. - [ ] 9. CLI sugar. OWNS: `apps/notescraft-cli/src/args.rs`, `apps/notescraft-cli/src/commands/addons.rs`, `apps/notescraft-cli/tests/registry.rs`. FILES TO OPEN: `apps/notescraft-cli/src/commands.rs`, `apps/notescraft-cli/src/commands/notes.rs`, `apps/notescraft-cli/src/output.rs`. GOAL: `notescraft registry list|info|check|install|update|remove|rollback` mapped to the commands, `--json` like the rest, exit code 3 only for write commands while the app holds the lock (`list` and `info` use `open_read_only`). TESTS: parse table; `list --json` shape; `install` against a fake transport env var is not allowed: test the argument-to-params mapping only. VERIFY: standing, `-p notescraft-cli`. DO NOT: add network code to the CLI. @@ -41,19 +41,19 @@ One task = one agent (`specs/agents/opencode-runbook.md`). `[ ]` open. Spec: `sp ## Phase 3 - UI -- [ ] 14. Registry browser view. OWNS: `crates/ui-egui/src/views/registry_view/mod.rs`, `.../list.rs`, `.../detail.rs`, `crates/ui-egui/src/text/views/registry.rs`, `[ui.registry]` at the END of `crates/i18n/locales/en.toml`, `crates/ui-egui/tests/registry_view.rs`. FILES TO OPEN: `spec.md` D10, `crates/ui-egui/src/views/settings_view/categories.rs`, `crates/ui-egui/src/views/share_dialog/mod.rs` (modal pattern), `.claude/skills/notescraft-egui-screen/`. GOAL: the view returns `Action`s only (`Check`, `Install(kind,id)`, `Update`, `Remove`, `Rollback`, `Enable(id)`), draws rows by state, the drawer with permission sentences, the offline banner; no engine calls inside the view. TESTS: kittest: each state renders its button; Install emits one action; search filters; offline banner; permissions listed as sentences. VERIFY: standing + `cargo xtask i18n`. DO NOT: use raw egui buttons or inputs; hardcode strings; fetch from the view. -- [ ] 15. Shell wiring: category, commands, snapshot flag. OWNS: `crates/ui-egui/src/views/settings_view/categories.rs` (one Registry entry), `crates/ui-egui/src/shell/app/vault/registry.rs`, `crates/ui-egui/examples/snapshot.rs` (`--registry`). GOAL: actions call the engine commands on a worker thread (poll like `grammar_dialog.rs`), results update the view data; `registry.list` on open reads the cache only; Check/Install are the network triggers. TESTS: kittest: opening the category makes zero network calls (counter on the fake transport); Install runs one command. VERIFY: standing + snapshot PNG read. DO NOT: block the UI thread. -- [ ] 16. Code card, first-run card, update badge. OWNS: `crates/ui-egui/src/views/note_view/blocks/code_card.rs` (the Add-grammar link only), `crates/ui-egui/src/shell/app/vault/first_run.rs`, `crates/ui-egui/src/text/views/registry.rs` (append). GOAL: D10 bullets 5 and 6; card dismissal remembered in `UiState`. TESTS: code card shows "Install Twig grammar" when `registry.suggest` hits, the old dialog link otherwise; first-run card never fetches; badge only after a check. VERIFY: standing + `cargo xtask i18n`. DO NOT: install without a click, unless `registry.auto_install_for_vault` is on and the file comes from the seed. +- [x] 14. Registry browser view. OWNS: `crates/ui-egui/src/views/registry_view/mod.rs`, `.../list.rs`, `.../detail.rs`, `crates/ui-egui/src/text/views/registry.rs`, `[ui.registry]` at the END of `crates/i18n/locales/en.toml`, `crates/ui-egui/tests/registry_view.rs`. FILES TO OPEN: `spec.md` D10, `crates/ui-egui/src/views/settings_view/categories.rs`, `crates/ui-egui/src/views/share_dialog/mod.rs` (modal pattern), `.claude/skills/notescraft-egui-screen/`. GOAL: the view returns `Action`s only (`Check`, `Install(kind,id)`, `Update`, `Remove`, `Rollback`, `Enable(id)`), draws rows by state, the drawer with permission sentences, the offline banner; no engine calls inside the view. TESTS: kittest: each state renders its button; Install emits one action; search filters; offline banner; permissions listed as sentences. VERIFY: standing + `cargo xtask i18n`. DO NOT: use raw egui buttons or inputs; hardcode strings; fetch from the view. +- [x] 15. Shell wiring: category, commands, snapshot flag. OWNS: `crates/ui-egui/src/views/settings_view/categories.rs` (one Registry entry), `crates/ui-egui/src/shell/app/vault/registry.rs`, `crates/ui-egui/examples/snapshot.rs` (`--registry`). GOAL: actions call the engine commands on a worker thread (poll like `grammar_dialog.rs`), results update the view data; `registry.list` on open reads the cache only; Check/Install are the network triggers. TESTS: kittest: opening the category makes zero network calls (counter on the fake transport); Install runs one command. VERIFY: standing + snapshot PNG read. DO NOT: block the UI thread. +- [x] 16. Code card, first-run card, update badge. OWNS: `crates/ui-egui/src/views/note_view/blocks/code_card.rs` (the Add-grammar link only), `crates/ui-egui/src/shell/app/vault/first_run.rs`, `crates/ui-egui/src/text/views/registry.rs` (append). GOAL: D10 bullets 5 and 6; card dismissal remembered in `UiState`. TESTS: code card shows "Install Twig grammar" when `registry.suggest` hits, the old dialog link otherwise; first-run card never fetches; badge only after a check. VERIFY: standing + `cargo xtask i18n`. DO NOT: install without a click, unless `registry.auto_install_for_vault` is on and the file comes from the seed. - [ ] 17. Vault `wanted` scan and banner. OWNS: `crates/engine/src/addons/wanted.rs`, `crates/engine/tests/addons_wanted.rs`, `crates/ui-egui/src/shell/app/vault/wanted_banner.rs`. FILES TO OPEN: `crates/engine/src/vault/session.rs`, the content-search code that uses `ignore`/`grep-*`, `spec.md` D11 step 4. GOAL: `scan_wanted(session, budget_ms) -> Vec` over fence labels and `$`/`$$` in the vault with a 3 s budget, run once per vault and registry generation, written to `.notescraft/registry.json` `wanted`; banner lists not-installed entries with size. TESTS: a vault with a ```` ```twig ```` fence yields `grammar:twig`; budget stops a 6500-note scan early without error; installed items are not listed; no network. VERIFY: standing. DO NOT: read notes over 2 MiB; write outside `.notescraft/registry.json`. ## Phase 4 - first-party content and release -- [ ] 18. Registry repo layout and signing tool. OWNS: `xtask/src/registry_sign.rs`, one subcommand line in `xtask/src/main.rs`, `xtask/tests/registry_sign.rs`; outside this repo (separate commit by the lead): `notescraft-registry/README.md` (schema v2, `id` not `name`), `index.json`, `files/`. FILES TO OPEN: `spec.md` D1 to D3, `/Volumes/Storage/Projects/notescraft-registry/README.md`, `xtask/src/main.rs`. GOAL: `cargo xtask registry-sign --dir --key ` rebuilds `index.json` from `files////` plus a small `entry.toml` per item (title, license, aliases), fills size and sha256, signs; `--check` verifies an existing index. TESTS: sign then verify with the engine verifier; tampered file fails `--check`. VERIFY: standing, `-p xtask`. DO NOT: read the key from the repo; print it. +- [x] 18. Registry repo layout and signing tool. OWNS: `xtask/src/registry_sign.rs`, one subcommand line in `xtask/src/main.rs`, `xtask/tests/registry_sign.rs`; outside this repo (separate commit by the lead): `notescraft-registry/README.md` (schema v2, `id` not `name`), `index.json`, `files/`. FILES TO OPEN: `spec.md` D1 to D3, `/Volumes/Storage/Projects/notescraft-registry/README.md`, `xtask/src/main.rs`. GOAL: `cargo xtask registry-sign --dir --key ` rebuilds `index.json` from `files////` plus a small `entry.toml` per item (title, license, aliases), fills size and sha256, signs; `--check` verifies an existing index. TESTS: sign then verify with the engine verifier; tampered file fails `--check`. VERIFY: standing, `-p xtask`. DO NOT: read the key from the repo; print it. - [ ] 19. Move the 25 grammars out. OWNS: `crates/syntax/src/embedded.rs` (shrinks to an empty list or removed), `crates/syntax/grammars/` (moved to the registry repo and to the seed), `crates/syntax/src/load.rs` (drop the embedded loop), `crates/engine/tests/addons_migration_grammars.rs`. GOAL: only after 7, 10, 16, 17, 18, 23 are done; registry entries `twig`, `blade`, ... one per file; tests per D11 step 5; the TOML and TypeScript grammars stay if the owner wants highlighting out of the box (they are the two most common: decide with the owner, default keep). TESTS: for every moved id: suggest finds it, seed verifies, offline install from the seed makes `is_known` true; removal restores plain text. VERIFY: standing + measure `load::current()` (expect about 75 ms instead of 110 ms) and binary delta (about -69 KB). DO NOT: remove a grammar that has no seed copy. - [ ] 20. STIX Two Math as a font entry. OWNS: `crates/ui-egui/src/design/fonts/mod.rs` (STIX loaded from the store when present), `crates/ui-egui/src/views/math_view.rs` (Install button when absent), `assets/fonts/STIXTwoMath-Regular.ttf` (moves to the seed), tests. GOAL: after 7, 16, 18, 23; math without the font uses the existing Inter/JetBrains chain and shows "Install math font (376 KB)". TESTS: with and without the font; rendering never panics; the migration test of D11. VERIFY: binary delta about -0.36 MiB; snapshot of a note with display math both ways. - [ ] 21. Lottie icons as a pack. OWNS: `crates/ui-egui/src/design/lottie_icon/files.rs` (store lookup, static fallback), `assets/icons-anim/` (moves to the seed), tests. GOAL: after 13, 18, 23; `icons-animated` pack; without it every animated icon draws its static Lucide icon (the skill `notescraft-icon-motion` rules still hold). TESTS: each icon id has a static fallback; motion tests run with the pack installed from a temp store. VERIFY: binary delta about -0.14 MiB. DO NOT: change any animation JSON. - [ ] 22. First plugins and themes in the registry. OWNS (registry repo, plus this repo's tests only): `files/plugin/dice/1.0.0/`, `files/theme/{catppuccin-frappe,catppuccin-macchiato,solarized-light,solarized-dark}/1.0.0/` (themes copied from `crates/themes/src/builtin.rs` and kept built-in until a migration test passes), `plugins/dice/README.md` (point to the registry), `crates/engine/tests/addons_first_party.rs`. GOAL: dice builds with `plugins/dice/build.sh`, its `manifest.json` equals the entry; citations follows when roadmap 10 starts. TESTS: dice installs offline from the seed, rolls, `2d6+3` result in range. DO NOT: remove built-in themes in this task. -- [ ] 23. Seed packaging. OWNS: `packaging/macos/*`, `packaging/windows/*`, `packaging/linux/package.sh` (copy `registry-seed/` next to the binary), `crates/engine/src/addons/seed.rs`, `crates/engine/tests/addons_seed.rs`. FILES TO OPEN: `packaging/README.md`, `packaging/macos/`. GOAL: `SeedDir::locate()` finds `Contents/Resources/registry-seed` (macOS), `registry-seed/` beside the exe (Windows, Linux), verified signed snapshot of moved items only. TESTS: locate with a fake exe path; missing seed is fine (network install still works); tampered seed file rejected. DO NOT: put anything unsigned in the seed. +- [x] 23. Seed packaging. OWNS: `packaging/macos/*`, `packaging/windows/*`, `packaging/linux/package.sh` (copy `registry-seed/` next to the binary), `crates/engine/src/addons/seed.rs`, `crates/engine/tests/addons_seed.rs`. FILES TO OPEN: `packaging/README.md`, `packaging/macos/`. GOAL: `SeedDir::locate()` finds `Contents/Resources/registry-seed` (macOS), `registry-seed/` beside the exe (Windows, Linux), verified signed snapshot of moved items only. TESTS: locate with a fake exe path; missing seed is fine (network install still works); tampered seed file rejected. DO NOT: put anything unsigned in the seed. - [ ] 24. Docs, numbers, status. OWNS: `ROADMAP.md`, `docs/ARCHITECTURE.md`, `docs/CODE-MAP.md` (addons rows), `SPEC.md` sections 6.1 and 7 (a "Registry" subsection, community store no longer deferred), `specs/registry/measurements.md` (new numbers), `specs/plugins-parity/roadmap.md` (add the rows). VERIFY: `cargo xtask ci`. DO NOT: add features. ## Order and parallelism diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 30701f0..a6e64bf 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -12,7 +12,13 @@ name = "xtask" path = "src/main.rs" [dependencies] +craft-atomic = { workspace = true } +notescraft-engine = { workspace = true } +ring = { workspace = true } +serde = { workspace = true } serde_json = { workspace = true } +sha2 = { workspace = true } +toml = { workspace = true } [lints] workspace = true \ No newline at end of file diff --git a/xtask/src/layers.rs b/xtask/src/layers.rs index 7420883..4c5a688 100644 --- a/xtask/src/layers.rs +++ b/xtask/src/layers.rs @@ -42,6 +42,9 @@ pub const TABLE: &[(&str, Class)] = &[ ("math", Class::Layer(0)), ("lottie", Class::Layer(0)), ("acp", Class::Layer(0)), + ("htmlmd", Class::Layer(0)), + ("srs", Class::Layer(0)), + ("textdiff", Class::Layer(0)), ("pdf", Class::Layer(1)), // L1 ("index", Class::Layer(1)), diff --git a/xtask/src/main.rs b/xtask/src/main.rs index cb08cf5..904bd66 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -9,6 +9,7 @@ mod layers; mod lex; #[cfg(test)] mod lex_tests; +mod registry_sign; mod rules; mod tree; mod version; @@ -26,6 +27,7 @@ commands: layers enforce the crate dependency layering (SPEC ยง3.1) tree fail when a source directory holds more than 12 loose .rs files ci verify, layers, tree, i18n --strict + registry-sign sign or --check the add-on registry index; `keygen --out ` makes a key (see registry_sign.rs) ico ico OUT.ico IN.png... packs PNGs into a Windows .ico (packaging/icons.sh) version print the workspace version; `version set X.Y.Z[-pre]` changes it i18n scan ui-egui/src and apps/*/src for hardcoded user strings (SPEC ยง12); @@ -40,6 +42,7 @@ fn main() -> ExitCode { Some("tree") => tree::run(&root()), Some("ci") => cmd_ci(), Some("i18n") => cmd_i18n(&args), + Some("registry-sign") => registry_sign::run(&args.iter().skip(1).map(String::as_str).collect::>()), Some("ico") => ico::run(&args.iter().skip(1).map(String::as_str).collect::>()), Some("version") => version::run(&root(), &args.iter().skip(1).map(String::as_str).collect::>()), Some("-h" | "--help" | "help") | None => { diff --git a/xtask/src/registry_sign.rs b/xtask/src/registry_sign.rs new file mode 100644 index 0000000..3fd6cce --- /dev/null +++ b/xtask/src/registry_sign.rs @@ -0,0 +1,365 @@ +//! `cargo xtask registry-sign`: builds, signs and checks the add-on registry index (spec D1 to D3). +//! +//! - `registry-sign --dir (--key | --key-env ) [--key-id ]` rebuilds +//! `index.json` from `files///entry.toml` and `files////` +//! (the highest version folder is the one indexed), signs every entry and writes the index. +//! Pulled releases (`revoked`) of an existing schema-2 index are kept. +//! - `registry-sign --check --dir --public-key [--key-id ]` verifies the +//! index signatures with the engine verifier and every payload's size and sha256. +//! - `registry-sign keygen --out ` writes a new PKCS#8 ed25519 key (base64) to a path +//! outside this repository and prints only the public key (base64). +//! +//! The private key is read from a file or an environment variable and is never printed. A key +//! path inside this repository or the registry checkout is refused. + +use std::path::{Component, Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use notescraft_engine::addons::{Ed25519, Entry, File, Index, Kind, Revoked, Signature, Verifier, Version, canonical_text}; +use notescraft_engine::decode_base64; +use ring::rand::SystemRandom; +use ring::signature::{Ed25519KeyPair, KeyPair}; +use serde::Deserialize; +use serde_json::{Value, json}; + +use self::encode::{encode_base64, sha256_hex}; +use crate::root; + +mod encode; + +/// The index file at the checkout root. +const INDEX_FILE: &str = "index.json"; +/// The payload folder at the checkout root. +const FILES_DIR: &str = "files"; +/// The per-item metadata file, next to the version folders. +const ENTRY_FILE: &str = "entry.toml"; +/// Key id written when `--key-id` is not given. +const DEFAULT_KEY_ID: &str = "k1"; +/// Where the registry is served from; the raw tree of the Gitea repository. +const RAW_BASE: &str = "https://git.marcospaulo.dev.br/masi/notescraft-registry/raw/branch/main"; +/// The only index schema this tool writes and reads. +const INDEX_VERSION: u64 = 2; +/// Byte length of an ed25519 public key. +const PUBLIC_KEY_BYTES: usize = 32; + +/// Command-line options after the subcommand. +#[derive(Default)] +struct Options { + dir: Option, + key: Option, + key_env: Option, + key_id: Option, + public_key: Option, + out: Option, + check: bool, +} + +/// `entry.toml`: the fields the folder layout cannot carry. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ItemMeta { + title: String, + #[serde(default)] + summary: String, + license: String, + source: Option, + min_app_version: String, + #[serde(default)] + aliases: Vec, + #[serde(default)] + suggested: bool, + #[serde(default)] + permissions: Vec, +} + +/// Runs `registry-sign` with the arguments after the subcommand name. +/// +/// # Errors +/// A message for the user: bad options, a key inside the repository, an unreadable or malformed +/// registry, or a failed verification. +pub fn run(args: &[&str]) -> Result<(), String> { + if let Some(&"keygen") = args.first() { + return keygen(&parse(args.get(1..).unwrap_or_default())?); + } + let options = parse(args)?; + if options.check { check(&options) } else { sign(&options) } +} + +fn parse(args: &[&str]) -> Result { + let mut options = Options::default(); + let mut iter = args.iter(); + while let Some(&flag) = iter.next() { + match flag { + "--check" => options.check = true, + "--dir" => options.dir = Some(value(&mut iter, flag)?.into()), + "--key" => options.key = Some(value(&mut iter, flag)?.into()), + "--key-env" => options.key_env = Some(value(&mut iter, flag)?.to_owned()), + "--key-id" => options.key_id = Some(value(&mut iter, flag)?.to_owned()), + "--public-key" => options.public_key = Some(value(&mut iter, flag)?.to_owned()), + "--out" => options.out = Some(value(&mut iter, flag)?.into()), + other => return Err(format!("unknown option `{other}`")), + } + } + Ok(options) +} + +fn value<'a>(iter: &mut std::slice::Iter<'_, &'a str>, flag: &str) -> Result<&'a str, String> { + iter.next().copied().ok_or_else(|| format!("{flag} needs a value")) +} + +fn sign(options: &Options) -> Result<(), String> { + let dir = required(options.dir.as_ref(), "--dir")?; + let key_id = options.key_id.as_deref().unwrap_or(DEFAULT_KEY_ID); + let pair = load_key(options, dir)?; + let public_key: [u8; PUBLIC_KEY_BYTES] = pair.public_key().as_ref().try_into().map_err(|_| "public key has the wrong length".to_owned())?; + let mut entries = scan(dir)?; + for entry in &mut entries { + let sig = pair.sign(canonical_text(entry).as_bytes()); + entry.signature = Signature { key_id: key_id.to_owned(), sig: encode_base64(sig.as_ref()) }; + } + let bytes = index_bytes(&entries, &existing_revoked(&dir.join(INDEX_FILE))?)?; + craft_atomic::atomic_write(&dir.join(INDEX_FILE), &bytes).map_err(|e| format!("cannot write index.json: {e}"))?; + println!("signed {} entries with key `{key_id}` (public key {})", entries.len(), encode_base64(&public_key)); + println!("wrote {}", dir.join(INDEX_FILE).display()); + Ok(()) +} + +fn check(options: &Options) -> Result<(), String> { + let dir = required(options.dir.as_ref(), "--dir")?; + let key_text = required(options.public_key.as_ref(), "--public-key")?; + let key: [u8; PUBLIC_KEY_BYTES] = + decode_base64(key_text).map_err(|e| e.to_string())?.try_into().map_err(|_| "public key must be 32 bytes of base64".to_owned())?; + let key_id = options.key_id.as_deref().unwrap_or(DEFAULT_KEY_ID); + let bytes = std::fs::read(dir.join(INDEX_FILE)).map_err(|e| format!("cannot read index.json: {e}"))?; + let index = Index::parse(&bytes).map_err(|e| e.to_string())?; + if !index.skipped.is_empty() { + return Err(format!("index has refused entries: {}", index.skipped.join("; "))); + } + let verifier = Ed25519::new(vec![(key_id.to_owned(), key)]); + for entry in &index.entries { + verifier.verify(entry).map_err(|e| e.to_string())?; + for file in &entry.files { + check_payload(dir, entry, file)?; + } + } + println!("OK: {} entries verified against key `{key_id}`", index.entries.len()); + Ok(()) +} + +/// Compares one payload file on disk with the size and hash the index records. +fn check_payload(dir: &Path, entry: &Entry, file: &File) -> Result<(), String> { + let path = payload_path(dir, entry.kind, &entry.id, entry.version, &file.path)?; + let bytes = std::fs::read(&path).map_err(|e| format!("{}: cannot read: {e}", path.display()))?; + if u64::try_from(bytes.len()).ok() != Some(file.size) { + return Err(format!("{}: size differs from the index", path.display())); + } + if sha256_hex(&bytes) != file.sha256 { + return Err(format!("{}: sha256 differs from the index", path.display())); + } + Ok(()) +} + +/// Refuses a payload name that is not a single normal path component. +fn payload_path(dir: &Path, kind: Kind, id: &str, version: Version, name: &str) -> Result { + let mut parts = Path::new(name).components(); + if !matches!((parts.next(), parts.next()), (Some(Component::Normal(_)), None)) { + return Err(format!("payload name `{name}` is not a plain file name")); + } + Ok(dir.join(FILES_DIR).join(kind.as_str()).join(id).join(version.to_string()).join(name)) +} + +fn keygen(options: &Options) -> Result<(), String> { + let out = required(options.out.as_ref(), "--out")?; + let out = resolve(out)?; + refuse_inside(&out, &[root()])?; + let pkcs8 = Ed25519KeyPair::generate_pkcs8(&SystemRandom::new()).map_err(|_| "cannot generate a key".to_owned())?; + let pair = Ed25519KeyPair::from_pkcs8(pkcs8.as_ref()).map_err(|_| "generated key is invalid".to_owned())?; + let text = format!("{}\n", encode_base64(pkcs8.as_ref())); + craft_atomic::write_new(&out, text.as_bytes()).map_err(|e| format!("cannot write {}: {e}", out.display()))?; + eprintln!("wrote private key to {} (owner only, outside the repository)", out.display()); + println!("{}", encode_base64(pair.public_key().as_ref())); + Ok(()) +} + +/// The private key, from `--key ` or `--key-env `; never echoed. +fn load_key(options: &Options, dir: &Path) -> Result { + let text = match (&options.key, &options.key_env) { + (Some(path), None) => { + refuse_inside(&resolve(path)?, &[root(), dir.to_path_buf()])?; + std::fs::read_to_string(path).map_err(|e| format!("cannot read the key file: {e}"))? + } + (None, Some(name)) => std::env::var(name).map_err(|_| format!("environment variable `{name}` is not set"))?, + _ => return Err("give exactly one of --key or --key-env ".to_owned()), + }; + let der = decode_base64(text.trim()).map_err(|_| "the key is not base64 PKCS#8".to_owned())?; + Ed25519KeyPair::from_pkcs8(&der).map_err(|_| "the key is not an ed25519 PKCS#8 key".to_owned()) +} + +/// Absolute form of `path`; a file that does not exist yet is resolved through its folder. +fn resolve(path: &Path) -> Result { + if path.exists() { + return std::fs::canonicalize(path).map_err(|e| format!("{}: {e}", path.display())); + } + let parent = match path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent, + _ => Path::new("."), + }; + let name = path.file_name().ok_or_else(|| format!("{}: not a file path", path.display()))?; + Ok(std::fs::canonicalize(parent).map_err(|e| format!("{}: {e}", parent.display()))?.join(name)) +} + +/// Refuses a path under any of `repos`, so a key cannot end up in a repository. +fn refuse_inside(path: &Path, repos: &[PathBuf]) -> Result<(), String> { + for repo in repos { + let repo = std::fs::canonicalize(repo).map_err(|e| format!("{}: {e}", repo.display()))?; + if path.starts_with(&repo) { + return Err(format!("refusing a key path inside a repository ({})", repo.display())); + } + } + Ok(()) +} + +fn required<'a, T>(option: Option<&'a T>, flag: &str) -> Result<&'a T, String> { + option.ok_or_else(|| format!("{flag} is required")) +} + +/// Unsigned entries from the checkout's folder layout, sorted by kind then id. +fn scan(dir: &Path) -> Result, String> { + let files = dir.join(FILES_DIR); + if !files.is_dir() { + return Err(format!("{} has no {FILES_DIR}/ folder", dir.display())); + } + let mut entries = Vec::new(); + for kind_dir in subdirs(&files)? { + let kind_name = name_of(&kind_dir)?; + let kind = Kind::parse(&kind_name).ok_or_else(|| format!("unknown kind folder `{kind_name}`"))?; + for id_dir in subdirs(&kind_dir)? { + entries.push(scan_item(kind, &name_of(&id_dir)?, &id_dir)?); + } + } + entries.sort_by(|a, b| a.kind.as_str().cmp(b.kind.as_str()).then_with(|| a.id.cmp(&b.id))); + Ok(entries) +} + +/// One item: `entry.toml` plus the highest version folder's files. +fn scan_item(kind: Kind, id: &str, id_dir: &Path) -> Result { + let meta_path = id_dir.join(ENTRY_FILE); + let meta: ItemMeta = toml::from_str(&std::fs::read_to_string(&meta_path).map_err(|e| format!("{}: {e}", meta_path.display()))?) + .map_err(|e| format!("{}: {e}", meta_path.display()))?; + let version = subdirs(id_dir)? + .iter() + .map(|sub| name_of(sub).and_then(|name| Version::parse(&name).map_err(|e| e.to_string()))) + .collect::, _>>()? + .into_iter() + .max() + .ok_or_else(|| format!("{id}: no version folder"))?; + let version_dir = id_dir.join(version.to_string()); + let mut files = Vec::new(); + for path in files_in(&version_dir)? { + let name = name_of(&path)?; + let size = std::fs::metadata(&path).map_err(|e| format!("{}: {e}", path.display()))?.len(); + let sha256 = sha256_hex(&std::fs::read(&path).map_err(|e| format!("{}: {e}", path.display()))?); + let url = format!("{RAW_BASE}/{FILES_DIR}/{}/{id}/{version}/{name}", kind.as_str()); + files.push(File { path: name, size, sha256, url }); + } + if files.is_empty() { + return Err(format!("{id}: version {version} has no files")); + } + Ok(Entry { + id: id.to_owned(), + kind, + title: meta.title, + summary: meta.summary, + version, + min_app_version: Version::parse(&meta.min_app_version).map_err(|e| format!("{id}: {e}"))?, + license: meta.license, + source: meta.source, + aliases: meta.aliases, + suggested: meta.suggested, + permissions: meta.permissions, + files, + signature: Signature { key_id: String::new(), sig: String::new() }, + }) +} + +/// Sorted sub-folders of `dir`; plain files are ignored. +fn subdirs(dir: &Path) -> Result, String> { + let mut out = Vec::new(); + for item in std::fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))? { + let path = item.map_err(|e| format!("{}: {e}", dir.display()))?.path(); + if path.is_dir() { + out.push(path); + } + } + out.sort(); + Ok(out) +} + +/// Sorted regular files directly inside `dir`; sub-folders are refused, payloads are flat. +fn files_in(dir: &Path) -> Result, String> { + let mut out = Vec::new(); + for item in std::fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))? { + let path = item.map_err(|e| format!("{}: {e}", dir.display()))?.path(); + if path.is_dir() { + return Err(format!("{}: sub-folders are not allowed in a version folder", path.display())); + } + if path.is_file() { + out.push(path); + } + } + out.sort(); + Ok(out) +} + +fn name_of(path: &Path) -> Result { + path.file_name().and_then(|name| name.to_str()).map(str::to_owned).ok_or_else(|| format!("{}: name is not UTF-8", path.display())) +} + +/// The `revoked` list of an existing schema-2 index; an older or missing index has none. +fn existing_revoked(index: &Path) -> Result, String> { + if !index.is_file() { + return Ok(Vec::new()); + } + let bytes = std::fs::read(index).map_err(|e| format!("cannot read index.json: {e}"))?; + let schema: Value = serde_json::from_slice(&bytes).map_err(|e| format!("index.json is not JSON: {e}"))?; + if schema.get("version").and_then(Value::as_u64) != Some(INDEX_VERSION) { + return Ok(Vec::new()); + } + Index::parse(&bytes).map(|parsed| parsed.revoked).map_err(|e| e.to_string()) +} + +/// The serialized index. Refuses output that the engine's own validator would skip an entry of. +fn index_bytes(entries: &[Entry], revoked: &[Revoked]) -> Result, String> { + let generated_at = SystemTime::now().duration_since(UNIX_EPOCH).map_err(|e| format!("clock: {e}"))?.as_secs(); + let document = json!({ + "version": INDEX_VERSION, + "generated_at": generated_at.to_string(), + "entries": entries.iter().map(entry_json).collect::>(), + "revoked": revoked.iter().map(|r| json!({ "id": r.id, "version": r.version.to_string() })).collect::>(), + }); + let mut bytes = serde_json::to_vec_pretty(&document).map_err(|e| e.to_string())?; + bytes.push(b'\n'); + let parsed = Index::parse(&bytes).map_err(|e| e.to_string())?; + if !parsed.skipped.is_empty() { + return Err(format!("the engine refuses entries: {}", parsed.skipped.join("; "))); + } + Ok(bytes) +} + +fn entry_json(entry: &Entry) -> Value { + json!({ + "id": entry.id, + "kind": entry.kind.as_str(), + "title": entry.title, + "summary": entry.summary, + "version": entry.version.to_string(), + "min_app_version": entry.min_app_version.to_string(), + "license": entry.license, + "source": entry.source, + "aliases": entry.aliases, + "suggested": entry.suggested, + "permissions": entry.permissions, + "files": entry.files.iter().map(|f| json!({ "path": f.path, "size": f.size, "sha256": f.sha256, "url": f.url })).collect::>(), + "signature": { "key_id": entry.signature.key_id, "sig": entry.signature.sig }, + }) +} diff --git a/xtask/src/registry_sign/encode.rs b/xtask/src/registry_sign/encode.rs new file mode 100644 index 0000000..a2e784a --- /dev/null +++ b/xtask/src/registry_sign/encode.rs @@ -0,0 +1,46 @@ +//! Hashing and base64 for the registry tool: a SHA-256 hex digest and standard base64 with padding. + +use std::fmt::Write as _; + +use sha2::{Digest, Sha256}; + +/// Standard base64 alphabet (RFC 4648). +const BASE64_ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +/// Bits per base64 symbol. +const SYMBOL_BITS: u32 = 6; +/// Mask of one base64 symbol. +const SYMBOL_MASK: u32 = 0x3f; +/// Bits of a full base64 group, three bytes. +const GROUP_BITS: u32 = 24; +/// Base64 symbols per three-byte group. +const GROUP_SYMBOLS: usize = 4; +/// Bytes per base64 group. +const GROUP_BYTES: usize = 3; + +/// Lowercase hex SHA-256 of `bytes`. +pub fn sha256_hex(bytes: &[u8]) -> String { + let mut out = String::with_capacity(Sha256::output_size() * 2); + for byte in Sha256::digest(bytes) { + let _ = write!(out, "{byte:02x}"); + } + out +} + +/// Standard base64 with padding. +pub fn encode_base64(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len().div_ceil(GROUP_BYTES) * GROUP_SYMBOLS); + for chunk in bytes.chunks(GROUP_BYTES) { + let byte = |i: usize| chunk.get(i).copied().map_or(0, u32::from); + let group = (byte(0) << 16) | (byte(1) << 8) | byte(2); + for symbol in 0..GROUP_SYMBOLS { + if symbol > chunk.len() { + out.push('='); + continue; + } + let shift = GROUP_BITS - SYMBOL_BITS * u32::try_from(symbol).unwrap_or(0) - SYMBOL_BITS; + let index = usize::try_from((group >> shift) & SYMBOL_MASK).unwrap_or(0); + out.push(BASE64_ALPHABET.get(index).copied().map_or('=', char::from)); + } + } + out +} diff --git a/xtask/tests/registry_sign.rs b/xtask/tests/registry_sign.rs new file mode 100644 index 0000000..f37892f --- /dev/null +++ b/xtask/tests/registry_sign.rs @@ -0,0 +1,110 @@ +//! Integration tests for `cargo xtask registry-sign`: sign, verify with the engine, tamper, refuse. +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +use notescraft_engine::addons::{Ed25519, Index, Kind, Verifier}; +use notescraft_engine::decode_base64; + +/// Key id the tool signs with by default. +const KEY_ID: &str = "k1"; +/// This crate's manifest, which lives inside the repository. +const REPO_CARGO_TOML: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml"); + +/// A fresh scratch directory outside the repository. +fn scratch(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("registry-sign-test-{}-{name}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir +} + +fn xtask(args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_xtask")).arg("registry-sign").args(args).output().unwrap() +} + +fn path_arg(path: &Path) -> &str { + path.to_str().unwrap() +} + +/// A registry checkout with one grammar that has two versions; `1.0.0` is the one indexed. +fn fixture(root: &Path) { + let item = root.join("files/grammar/twig"); + fs::create_dir_all(item.join("1.0.0")).unwrap(); + fs::create_dir_all(item.join("0.9.0")).unwrap(); + fs::write(item.join("entry.toml"), "title = \"Twig\"\nlicense = \"MIT\"\nmin_app_version = \"0.1.0\"\naliases = [\"twig\"]\n").unwrap(); + fs::write(item.join("0.9.0/twig.sublime-syntax"), "old\n").unwrap(); + fs::write(item.join("1.0.0/twig.sublime-syntax"), "name: Twig\n").unwrap(); +} + +/// Generates a key outside the repository and returns `(key file, public key base64)`. +fn keygen(dir: &Path) -> (PathBuf, String) { + let key = dir.join("owner.key"); + let out = xtask(&["keygen", "--out", path_arg(&key)]); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let public = String::from_utf8(out.stdout).unwrap().trim().to_owned(); + (key, public) +} + +fn public_bytes(public: &str) -> [u8; 32] { + decode_base64(public).unwrap().try_into().unwrap() +} + +#[test] +fn keygen_prints_only_the_public_key_and_refuses_to_overwrite() { + let dir = scratch("keygen"); + let (key, public) = keygen(&dir); + assert_eq!(public_bytes(&public).len(), 32); + let again = xtask(&["keygen", "--out", path_arg(&key)]); + assert!(!again.status.success(), "a second keygen must not replace the key"); + fs::remove_dir_all(&dir).unwrap(); +} + +#[test] +fn signed_index_verifies_with_the_engine_and_check_catches_tampering() { + let dir = scratch("sign"); + let reg = dir.join("registry"); + fixture(®); + let (key, public) = keygen(&dir); + + let signed = xtask(&["--dir", path_arg(®), "--key", path_arg(&key)]); + assert!(signed.status.success(), "{}", String::from_utf8_lossy(&signed.stderr)); + + let index = Index::parse(&fs::read(reg.join("index.json")).unwrap()).unwrap(); + assert!(index.skipped.is_empty(), "{:?}", index.skipped); + let entry = index.find(Kind::Grammar, "twig").expect("twig is indexed"); + assert_eq!(entry.version.to_string(), "1.0.0", "the highest version folder is indexed"); + Ed25519::new(vec![(KEY_ID.to_owned(), public_bytes(&public))]).verify(entry).expect("the signature verifies"); + + let check = |extra: &[&str]| { + let mut args = vec!["--check", "--dir", path_arg(®), "--public-key", public.as_str()]; + args.extend_from_slice(extra); + xtask(&args) + }; + assert!(check(&[]).status.success()); + + fs::write(reg.join("files/grammar/twig/1.0.0/twig.sublime-syntax"), "name: Tw1g\n").unwrap(); + let tampered = check(&[]); + assert!(!tampered.status.success()); + assert!(String::from_utf8_lossy(&tampered.stderr).contains("differs from the index")); + fs::remove_dir_all(&dir).unwrap(); +} + +#[test] +fn key_inside_the_repository_is_refused_and_nothing_is_written() { + let dir = scratch("refuse"); + let reg = dir.join("registry"); + fixture(®); + let out = xtask(&["--dir", path_arg(®), "--key", REPO_CARGO_TOML]); + assert!(!out.status.success()); + assert!(String::from_utf8_lossy(&out.stderr).contains("inside a repository")); + assert!(!reg.join("index.json").exists()); + + let in_repo = concat!(env!("CARGO_MANIFEST_DIR"), "/should-not-exist.key"); + let made = xtask(&["keygen", "--out", in_repo]); + assert!(!made.status.success()); + assert!(!Path::new(in_repo).exists()); + fs::remove_dir_all(&dir).unwrap(); +}