masi/notescraft
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

feat: Obsidian basics, editor previews, graph rework, query blocks, stacked tabs, pop-out windows
ci / ci (macos-latest) (push) Has been cancelled
Details
ci / ci (ubuntu-latest) (push) Has been cancelled
Details
ci / ci (windows-latest) (push) Has been cancelled
Details

Browse Source 
Editor previews (colour swatches, arrows, embeds), the 18 obsidian-basics tasks (find and replace
with regex, back/forward, auto-pair, footnotes, folding, search operators, split view, PDF/audio/
video embeds, spelling, drag and drop, pinned tabs, settings) and round 2: ```query blocks, stacked
tabs, pop-out windows, tasks ticked inside embeds, text around an embed, embeds that follow edits,
block menu shortcut. New crates: notescraft-regex (linear-time engine) and notescraft-syntax.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Marcos Paulo
2026-10-19 11:21:19 -03:00
parent 15d7f5e873
commit 5e8020aa38
294 changed files with 30429 additions and 676 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+326 -5
View File
@@ -134,6 +134,15 @@ dependencies = [
"zerocopy",
]
[[package]]
name = "aho-corasick"
version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
dependencies = [
"memchr",
]
[[package]]
name = "allocator-api2"
version = "0.2.21"
@@ -429,15 +438,39 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bincode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
dependencies = [
"serde",
]
[[package]]
name = "bit-set"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
dependencies = [
"bit-vec 0.8.0",
]
[[package]]
name = "bit-set"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2f926cc3060f09db9ebc5b52823d85268d24bb917e472c0c4bea35780a7d"
dependencies = [
"bit-vec",
"bit-vec 0.9.1",
]
[[package]]
name = "bit-vec"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
[[package]]
name = "bit-vec"
version = "0.9.1"
@@ -456,6 +489,15 @@ version = "2.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
[[package]]
name = "block-buffer"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"
dependencies = [
"hybrid-array",
]
[[package]]
name = "block2"
version = "0.5.1"
@@ -487,6 +529,17 @@ dependencies = [
"piper",
]
[[package]]
name = "bstr"
version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"
dependencies = [
"memchr",
"regex-automata",
"serde_core",
]
[[package]]
name = "bumpalo"
version = "3.20.3"
@@ -702,6 +755,12 @@ dependencies = [
"libc",
]
[[package]]
name = "core_detect"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48"
[[package]]
name = "coreaudio-rs"
version = "0.13.0"
@@ -742,6 +801,15 @@ dependencies = [
"windows 0.54.0",
]
[[package]]
name = "cpufeatures"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566"
dependencies = [
"libc",
]
[[package]]
name = "craft-atomic"
version = "0.1.0"
@@ -799,6 +867,25 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.23"
@@ -811,6 +898,15 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
[[package]]
name = "crypto-common"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453"
dependencies = [
"hybrid-array",
]
[[package]]
name = "cursor-icon"
version = "1.2.0"
@@ -829,6 +925,16 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376"
[[package]]
name = "digest"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
dependencies = [
"block-buffer",
"crypto-common",
]
[[package]]
name = "dirs"
version = "7.0.0"
@@ -1067,6 +1173,29 @@ dependencies = [
"bytemuck",
]
[[package]]
name = "encoding_rs"
version = "0.8.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679"
dependencies = [
"cfg-if",
"core_detect",
"multiversion_no_op",
"rustversion",
"scopeguard",
"simdutf8",
]
[[package]]
name = "encoding_rs_io"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fba3fe847045ecff794b9c138293a80db914678c453ad63fbf0c6a9eb6e00b22"
dependencies = [
"encoding_rs",
]
[[package]]
name = "endi"
version = "1.1.1"
@@ -1196,6 +1325,17 @@ dependencies = [
"pin-project-lite",
]
[[package]]
name = "fancy-regex"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "998b056554fbe42e03ae0e152895cd1a7e1002aec800fdc6635d20270260c46f"
dependencies = [
"bit-set 0.8.0",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "fastrand"
version = "2.5.0"
@@ -1246,6 +1386,12 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4"
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
version = "0.1.5"
@@ -1418,6 +1564,19 @@ dependencies = [
"vello_common",
]
[[package]]
name = "globset"
version = "0.4.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988"
dependencies = [
"aho-corasick",
"bstr",
"log",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "glow"
version = "0.17.0"
@@ -1510,6 +1669,43 @@ dependencies = [
"windows 0.62.2",
]
[[package]]
name = "grep-matcher"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9417543f4870fc8f1c8e1af870afae2431007626d9e703fce6471c468d33847"
dependencies = [
"memchr",
]
[[package]]
name = "grep-regex"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce0c256c3ad82bcc07b812c15a45ec1d398122e8e15124f96695234db7112ef"
dependencies = [
"bstr",
"grep-matcher",
"log",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "grep-searcher"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72348823a0eafc4bc2e9051064f28b5b42cc100b571b3a35d67918d711efcbc6"
dependencies = [
"bstr",
"encoding_rs",
"encoding_rs_io",
"grep-matcher",
"log",
"memchr",
"memmap2",
]
[[package]]
name = "guillotiere"
version = "0.7.0"
@@ -1584,6 +1780,31 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "hybrid-array"
version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17"
dependencies = [
"typenum",
]
[[package]]
name = "ignore"
version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b69833ed729dc5aa7d19541d96d6cf8e9137194207a04916d658e43168402f"
dependencies = [
"crossbeam-deque",
"globset",
"log",
"memchr",
"regex-automata",
"same-file",
"walkdir",
"winapi-util",
]
[[package]]
name = "image"
version = "0.25.10"
@@ -1870,6 +2091,12 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4a5ff6bcca6c4867b1c4fd4ef63e4db7436ef363e0ad7531d1558856bae64f4"
[[package]]
name = "linked-hash-map"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]]
name = "linux-raw-sys"
version = "0.4.15"
@@ -1996,6 +2223,12 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "multiversion_no_op"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d"
[[package]]
name = "naga"
version = "30.0.1"
@@ -2003,7 +2236,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a616d2fb8c89516ac2723a581f69d6c18576046bed761bd6b305e5618e6ae130"
dependencies = [
"arrayvec",
"bit-set",
"bit-set 0.10.0",
"bitflags 2.13.2",
"cfg-if",
"cfg_aliases",
@@ -2113,7 +2346,9 @@ name = "notescraft-blocks"
version = "0.1.0"
dependencies = [
"notescraft-markdown",
"notescraft-regex",
"serde",
"serde_json",
"thiserror 2.0.21",
]
@@ -2164,6 +2399,7 @@ dependencies = [
"egui",
"notescraft-blocks",
"notescraft-markdown",
"notescraft-syntax",
"serde",
"thiserror 2.0.21",
]
@@ -2172,8 +2408,12 @@ dependencies = [
name = "notescraft-engine"
version = "0.1.0"
dependencies = [
"craft-atomic",
"craft-settings",
"flate2",
"grep-regex",
"grep-searcher",
"ignore",
"notescraft-blocks",
"notescraft-calendar",
"notescraft-git",
@@ -2184,6 +2424,8 @@ dependencies = [
"notescraft-math",
"notescraft-pdf",
"notescraft-plugins",
"notescraft-regex",
"notescraft-syntax",
"notescraft-templates",
"notescraft-themes",
"notescraft-vault",
@@ -2300,6 +2542,22 @@ dependencies = [
"wat",
]
[[package]]
name = "notescraft-regex"
version = "0.1.0"
dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "notescraft-syntax"
version = "0.1.0"
dependencies = [
"sha2",
"syntect",
"thiserror 2.0.21",
]
[[package]]
name = "notescraft-templates"
version = "0.1.0"
@@ -2345,6 +2603,7 @@ dependencies = [
"notescraft-lottie",
"notescraft-markdown",
"notescraft-math",
"notescraft-syntax",
"notescraft-templates",
"notescraft-themes",
"notescraft-vault",
@@ -3216,6 +3475,23 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "regex-automata"
version = "0.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
[[package]]
name = "renderdoc-sys"
version = "1.1.0"
@@ -3427,6 +3703,17 @@ dependencies = [
"serde_core",
]
[[package]]
name = "sha2"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]
name = "shlex"
version = "2.0.1"
@@ -3655,6 +3942,25 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "syntect"
version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "656b45c05d95a5704399aeef6bd0ddec7b2b3531b7c9e900abbf7c4d2190c925"
dependencies = [
"bincode",
"fancy-regex",
"flate2",
"fnv",
"once_cell",
"regex-syntax",
"serde",
"serde_derive",
"thiserror 2.0.21",
"walkdir",
"yaml-rust",
]
[[package]]
name = "tempfile"
version = "3.27.0"
@@ -3861,6 +4167,12 @@ dependencies = [
"rustc-hash 2.1.3",
]
[[package]]
name = "typenum"
version = "1.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
[[package]]
name = "uds_windows"
version = "1.2.1"
@@ -4340,8 +4652,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c018fce9b6270aa203c2fdd56f3cce996713534bd757e4ea58c8560b121f14"
dependencies = [
"arrayvec",
"bit-set",
"bit-vec",
"bit-set 0.10.0",
"bit-vec 0.9.1",
"bitflags 2.13.2",
"bytemuck",
"cfg_aliases",
@@ -4413,7 +4725,7 @@ dependencies = [
"android_system_properties",
"arrayvec",
"ash",
"bit-set",
"bit-set 0.10.0",
"bitflags 2.13.2",
"block2 0.6.2",
"bytemuck",
@@ -5007,6 +5319,15 @@ dependencies = [
"serde_json",
]
[[package]]
name = "yaml-rust"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
dependencies = [
"linked-hash-map",
]
[[package]]
name = "zbus"
version = "5.19.0"
Cargo.toml
+10
View File
@@ -26,6 +26,8 @@ notescraft-lottie = { path = "crates/lottie" }
notescraft-acp = { path = "crates/acp" }
notescraft-pdf = { path = "crates/pdf" }
notescraft-markdown = { path = "crates/markdown" }
notescraft-syntax = { path = "crates/syntax" }
notescraft-regex = { path = "crates/regex" }
notescraft-index = { path = "crates/index" }
notescraft-backlinks = { path = "crates/backlinks" }
notescraft-blocks = { path = "crates/blocks" }
@@ -62,6 +64,14 @@ cpal = "0.16"
notify = "8"
pulldown-cmark = { version = "0.13", default-features = false }
toml = "1"
# Content search over the files on disk before the index is ready (the ripgrep library crates; engine only; owner-approved 2026-10-18).
ignore = "0.4"
grep-regex = "0.1"
grep-searcher = "0.1"
# Syntax highlighting for fenced code; pure-Rust regex backend, no onig (owner-approved).
syntect = { version = "5", default-features = false, features = ["parsing", "default-syntaxes", "yaml-load", "regex-fancy"] }
# SHA-256 of downloaded grammars (pure Rust, RustCrypto; owner-approved 2026-10-18).
sha2 = { version = "0.11", default-features = false }
# Pure-Rust WebAssembly interpreter, same configuration as photocraft-plugins.
wasmi = { version = "2.0.0", default-features = false, features = ["std", "validate", "stable", "portable-dispatch"] }
wat = "1.228"
ROADMAP.md
+11 -2
View File
@@ -2,7 +2,7 @@
**Stage: pre-alpha** · spec approved 2026-10-13 · phase 0 done · phases 1 to 10 implemented; phase 11 (packaging, i18n migration, release) open
> **Last updated:** 2026-10-17 (vault knowledge, settings sub-menus, macOS menu bar; calendar plugin: events, recurrence, quick add, filters, keyboard, reminders, ICS and Google feeds; assistant panel with ACP bridge and write gate, control channel, properties panel, slash menu, image paste/drop, PDF export, LaTeX math, Vim mode, resizable panels, whiteboard upgrade) · **Target:** Obsidian-compatible vault, native egui, see [SPEC.md](SPEC.md)
> **Last updated:** 2026-10-19 (vault knowledge, settings sub-menus, macOS menu bar; calendar plugin: events, recurrence, quick add, filters, keyboard, reminders, ICS and Google feeds; assistant panel with ACP bridge and write gate, control channel, properties panel, slash menu, image paste/drop, PDF export, LaTeX math, Vim mode, resizable panels, whiteboard upgrade, fast open, editor blocks and lint, code blocks, grammars and formatting, editor previews) · **Target:** Obsidian-compatible vault, native egui, see [SPEC.md](SPEC.md)
One-page summary of where NotesCraft stands. The contract is [SPEC.md](SPEC.md); the definition of done per feature is its section 12. One PR per phase, each reviewed before merge.
@@ -24,13 +24,15 @@ One-page summary of where NotesCraft stands. The contract is [SPEC.md](SPEC.md);
| 12 | Calendar plugin | `calendar`, engine `calendar.*`, calendar view, `apps/notescraft` services | done: events as notes, repeat subset, quick add, five views, filters and saved filters, keyboard model, reminders with inbox, ICS feeds, Google (view-only); not built: Apple, Outlook, CalDAV, two-way sync, background agent, non-English quick add (specs/calendar-plugin-redesign) |
| 13 | Vault knowledge | `knowledge`, engine `knowledge.*`, Related tab, assistant sources, Settings > Knowledge | done: passage index, vocabulary expansion, link and tag signals, five commands, MCP `vault_context`, auto context, sources, Related tab, compact layout; not built: embeddings (only if recall@6 falls under 80%), non-English stemming, attachments (specs/vault-knowledge) |
| 14 | Editor blocks and lint | `lint`, `blocks`, engine `lint.*` and `block.*`, slash menu, grip, Lint tab, Settings > Editor and lint | done: 12 lint rules (spelling slot only), one-click and safe fixes, learned style, dismissals, `fix_on_save`, slash menu with groups and preview in Source and Visual, block grip, drag, menu, Vim `]]` `[[` `gb`, Visual tables, note embeds, `%%` comments; not built: spell-check, mobile bottom toolbar, Fix popover in the editor, keyboard use of the block menu (specs/editor-blocks-and-lint) |
| 15 | Fast open and large vaults | `vault` listing and `count_files`, engine `open_lazy`, `index_step`, `ContentSearch`, `index.*`, Explorer virtualization and navigation, large-folder warning | done: window in under a second on 10 000 notes, background indexing with a status card, search from disk while indexing, pause and resume, rename and move refused until the index is complete, virtualized Explorer (50 000 rows), note counts, filter, collapse all, reveal and follow, pinned folders, sticky folder path, warning above 5 000 files; no index cache (not needed); not built: honouring `.gitignore` in non-vault folders, jump-to-letter strip, key bindings for the Explorer commands (specs/fast-open-and-explorer) |
| 16 | Editor previews | `editor::preview`, engine `editor.previews.*`, colour swatches, external-link arrows, tree card, Mermaid Source toggle | done: `detect_colors` (hex, `rgb`/`rgba`, `hsl`/`hsla` and the basic colour names) and `detect_tree`, both pure and bounded, swatches in body text, inline code and ten fenced languages, the right-click colour menu (Copy color, Copy as rgb, the two switches), external-link arrows, the TREE card, the Mermaid Diagram/Source toggle and the error/unsupported/too-big chips; not built: swatches and arrows in Source mode and in rows with inline math, swatches for a hex Obsidian reads as a tag (specs/editor-previews) |
| 11 | PDF export, packaging, i18n coverage, hardening | `pdf`, `packaging/`, release | PDF export done (crate, `export.pdf`, CLI, dialog; LaTeX prints as source, `craft-print` not wired); packaging done for macOS (built and smoke-tested), Windows (MSI scripts, compile-checked only) and Linux (deb/rpm/AppImage scripts, untested here); i18n migration done (252 keys in `en.toml`, `cargo xtask i18n --strict` is part of `ci`) |
Also built beyond the phase list: Vim mode (`vim` crate and app-wide navigation), resizable and collapsable cards with rails, Windows/Linux custom title bar.
## Implemented so far
23 crates plus the CLI: `acp`, `backlinks`, `calendar`, `canvas`, `diagrams`, `editor`, `engine`, `git`, `graph`, `i18n`, `index`, `markdown`, `math`, `mcp`, `pdf`, `plugins`, `templates`, `testkit` (empty), `themes`, `ui-egui`, `vault`, `vim`, `whiteboard`; about 110 000 source lines. `cargo xtask ci` is green on `develop`. Per-crate API and numbers: [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md).
28 crates plus the CLI: `acp`, `backlinks`, `blocks`, `calendar`, `canvas`, `diagrams`, `editor`, `engine`, `git`, `graph`, `i18n`, `index`, `knowledge`, `lint`, `lottie`, `markdown`, `math`, `mcp`, `pdf`, `plugins`, `syntax`, `templates`, `testkit` (empty), `themes`, `ui-egui`, `vault`, `vim`, `whiteboard`; about 166 000 source lines. `cargo xtask ci` is green on `develop`. Per-crate API and numbers: [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md).
Also done on 2026-10-17: the assistant starts opencode with a lean private config (`crates/acp/src/lean.rs`: no plugins, MCP, skills or agents, unused built-in tools off; about 5 000 tokens for a first prompt instead of 60 000 or more), the in-app assistant's MCP list holds only the aliases, `search` and plugin tools, the assistant's Preview of a pending change is a full-window side-by-side diff (still without Accept/Reject, and it opens only the first pending change), mouse-wheel scrolling works in every card, many more LaTeX commands, and a configurable note padding (Settings, Appearance).
@@ -49,9 +51,16 @@ Done (crossed off):
- [x] ~~macOS: traffic-light buttons hidden, native menu bar (File, Edit, View, Window, Help), top space reclaimed (8 pt drag strip)~~
- [x] ~~Settings page at phone width: category pills on top, rows stack, 44 pt targets~~
- [x] ~~Designs made in PaperCraft: slash menu, WYSIWYG blocks, lint and fixes~~
- [x] ~~Fast open and large vaults: lazy session open, `index_step` in the frame loop, content search from disk, indexing card and states, virtualized Explorer with filter, pins and sticky path, large-folder warning; 12 tasks in specs/fast-open-and-explorer~~
- [x] ~~Editor blocks and lint: `notescraft-lint`, `notescraft-blocks`, `lint.*` and `block.*` commands (engine, CLI, MCP), Lint tab, slash menu with preview, block grip and menu, Settings > Editor and lint, Visual tables and note embeds~~
- [x] ~~Code blocks, grammars and formatting: `notescraft-syntax` (highlighting on a worker thread with a bounded cache, grammar fetch/validate/install, language suggestions), JSON check, format and minify in `notescraft-blocks`, the eight `[syntax]` theme colours, the rendered code card (Copy, Wrap, Valid chip, JSON offer, typo suggestion, "Add grammar"), rounded inline-code chips, the batch format and lint engine (`batch.*`, a dry run that writes nothing) and the file-by-file review view with Accept/Skip per change and Apply, the original TOML and TypeScript grammars embedded in `notescraft-syntax` and the third batch check "Add missing language labels"; 15 tasks in specs/code-blocks-and-formatting~~
- [x] ~~Editor previews: colour swatches in text, inline code and fenced stylesheets, the right-click colour menu (Copy color, Copy as rgb, two switches), external-link arrows, the TREE card, the Mermaid Diagram/Source toggle and the fallback chips; 11 tasks in specs/editor-previews~~
Pending, by area:
- **Code blocks, grammars and formatting (done, 15 of 15 tasks, specs/code-blocks-and-formatting):** `toml` and `ts` fences now highlight through two original grammars embedded in `notescraft-syntax` (`crates/syntax/grammars/*.sublime-syntax`), and the batch review has a third check, "Add missing language labels": `blocks::detect_language` names the language of an unlabeled fence conservatively (shebang, PHP, HTML, XML, JSON, YAML, diff, shell prompts, Rust, Python, SQL, TOML, on the first 4 KiB and 40 non-blank lines) and each `ChangeKind::Label` is a zero-length insert right after the fence run that always needs review (`batch.start` takes `labels`, on by default). Not built — the language picker of the design (a mistyped label is fixed by the suggestion chip, and nothing offers to add a label while typing in source mode), the table-alignment checks of the design, and the "more info" card of the grammar dialog. A note with 50 code blocks draws its first frame in 15 ms median (release), 87 ms in a cold process.
- **Editor previews (done, 11 of 11 tasks, specs/editor-previews):** not built or limited — no swatches or arrows in Source mode or in rows carrying inline math, and a hex that Obsidian reads as a tag (`#cba6f7` in prose) stays a tag, so only inline code and non-tag text get swatches; `Unsupported` is reachable only if the diagram parser returns an unrenderable kind (today it answers `NoHeader` for one it does not know, so those show the Error chip). Scanning 100 rows of 10 colours (1,000 hits) takes 0.05 ms median, 0.15 ms worst, in release.
- **Obsidian basics (specs/obsidian-basics, audit in audit.md): all 18 tasks built.** Find and replace (with regex), back/forward, auto-pair, footnotes, explorer sort and drag-and-drop move, heading and list folding, search operators and `/regex/` search, vault-wide replace, quick switcher headings and blocks, random and unique notes, Open in default app, Copy Obsidian URL, pinned and reorderable tabs, link hover preview, outgoing links, split view (two panes with their own tabs; a click in a pane makes it the editing pane), spelling lint rule, settings (default view, font size, readable width, line numbers, new-note folder, link format and path, deleted files), PDF embeds drawn as an image of the page (`#page=N`; macOS PDFKit, elsewhere `pdftoppm` when installed, else a file card), audio embeds with Play/Stop through the system player, video embeds with a still, and in-note playback (ffmpeg decodes frames, ffplay the sound) when ffmpeg is installed, else Open in the system player, and a graph view with force sliders and colour groups that scales to 6.5k notes. Strict line breaks and show-hidden-files are settings too. Limits kept on purpose: video playback needs `ffmpeg` on the machine (no decoder is linked), no bundled spelling dictionary (the vault is the vocabulary). `crates/ui-egui/tests/driven_features.rs` drives tab drag, pinning, the link hover card, list folding, split view, hidden files, the regex find bar and Explorer drag-and-drop through real input events.
- **Obsidian basics round 2 (specs/obsidian-basics/tasks.md 19-24): query embeds (19) and stacked tabs (21) built; editor follow-ups (23) built; pop-out windows (22) built but not seen in a native window; unbuilt: RTL (egui has no bidi or Arabic shaping), hand check of the packaged app and a Windows playback run.
- **Editor (built, see phase 14):** spell-check needs an owner decision (new dependency or in-house checker); open: mobile bottom toolbar, inline fix popover, block menu by keyboard, embed with other text on one line, checkboxes inside embeds, embed refresh after editing another tab.
- **Editor settings:** the page has only Vim mode; more settings need editor features first (line numbers, spellcheck, font size, ...), then sub-menus for them.
- **Community plugins:** a browse page and sub-menu.
apps/notescraft/src/main.rs
+51 -13
View File
@@ -23,7 +23,10 @@ use std::process::ExitCode;
use eframe::egui;
use notescraft_engine::Session;
use notescraft_ui_egui::app::{NotesApp, VaultRequest};
use notescraft_ui_egui::app::{LargeFolderGate, NotesApp, VaultRequest};
/// Passed to the relaunched window after the user confirmed the large-folder warning.
const LARGE_OK_FLAG: &str = "--large-ok";
/// Why the app did not start.
#[derive(Debug)]
@@ -70,11 +73,20 @@ impl eframe::App for NotesCraftWindow {
craft_quit::on_close_request(ctx);
// A vault is bound to its Session, so switching starts a new instance on the chosen folder.
let Some(request) = self.app.take_vault_request() else { return };
let target = match request {
VaultRequest::Open(path) => Some(path),
VaultRequest::Pick => rfd::FileDialog::new().set_title("Open or create a vault folder").pick_folder(),
// A folder the user confirmed in the large-folder warning is relaunched with the check skipped.
let (target, confirmed) = match request {
VaultRequest::Open(path) => (Some(path), false),
VaultRequest::OpenLarge(path) => (Some(path), true),
VaultRequest::Pick => (rfd::FileDialog::new().set_title("Open or create a vault folder").pick_folder(), false),
};
if target.is_some_and(|target| std::env::current_exe().and_then(|exe| std::process::Command::new(exe).arg(target).spawn()).is_ok()) {
let relaunch = |target: std::path::PathBuf| {
let mut command = std::process::Command::new(std::env::current_exe()?);
if confirmed {
command.arg(LARGE_OK_FLAG);
}
command.arg(target).spawn()
};
if target.is_some_and(|target| relaunch(target).is_ok()) {
ctx.send_viewport_cmd(egui::ViewportCommand::Close);
}
}
@@ -160,8 +172,20 @@ fn remember_vault(path: &std::path::Path) {
}
}
/// The warning to show before `path` is opened: it holds more than the allowed number of files and
/// the user has not confirmed yet. Counting only reads, so nothing is written to the folder.
fn large_folder_gate(path: &std::path::Path, confirmed: bool) -> Option<LargeFolderGate> {
if confirmed {
return None;
}
let counted = notescraft_engine::count_files(path, notescraft_engine::LARGE_FOLDER_FILES).ok()?;
counted.over_limit.then(|| LargeFolderGate { path: path.to_path_buf(), files: counted.files, notes: counted.notes, over_limit: true })
}
fn main() -> ExitCode {
let args: Vec<std::ffi::OsString> = std::env::args_os().skip(1).collect();
let mut args: Vec<std::ffi::OsString> = std::env::args_os().skip(1).collect();
let large_ok = args.iter().any(|arg| arg == LARGE_OK_FLAG);
args.retain(|arg| arg != LARGE_OK_FLAG);
let path = match parse_launch(&args) {
Ok(Launch::Print(text)) => {
println!("{text}");
@@ -177,7 +201,13 @@ fn main() -> ExitCode {
return ExitCode::from(2);
}
};
let session = match Session::open(&path) {
// A folder with thousands of files is confirmed before anything is written to it: the window
// opens on an empty throw-away vault and shows the warning, and only a confirmed relaunch
// opens the folder itself.
let gate = large_folder_gate(&path, large_ok);
let gate_dir = std::env::temp_dir().join(format!("notescraft-gate-{}", std::process::id()));
let opened = if gate.is_some() { Session::open_read_only(&gate_dir) } else { Session::open_lazy(&path) };
let session = match opened {
Ok(session) => session,
Err(error) => {
eprintln!("{}", StartError::Vault(error));
@@ -186,14 +216,20 @@ fn main() -> ExitCode {
};
// The control channel (SPEC 10, D9): loopback only, descriptor in the vault
// state folder. A failure never blocks the app — the CLI just falls back.
let control = match notescraft_engine::ControlServer::start(session.vault()) {
Ok(handle) => Some(handle),
Err(error) => {
eprintln!("notescraft: the control channel did not start ({error})");
None
let control = if gate.is_some() {
None
} else {
match notescraft_engine::ControlServer::start(session.vault()) {
Ok(handle) => Some(handle),
Err(error) => {
eprintln!("notescraft: the control channel did not start ({error})");
None
}
}
};
remember_vault(&path);
if gate.is_none() {
remember_vault(&path);
}
let Some(theme) = notescraft_themes::builtin("catppuccin-mocha") else {
eprintln!("built-in theme catppuccin-mocha missing (broken build)");
return ExitCode::from(1);
@@ -219,6 +255,7 @@ fn main() -> ExitCode {
options,
Box::new(move |creation| {
let mut app = NotesApp::new(session, theme);
app.set_large_folder_gate(gate);
app.set_services(std::sync::Arc::new(desktop::Desktop::new()));
app.set_recent_vaults(&recent_vaults());
app.control = control;
@@ -241,6 +278,7 @@ fn main() -> ExitCode {
}) as Box<dyn eframe::App>)
}),
);
let _ = std::fs::remove_dir_all(&gate_dir);
match result {
Ok(()) => ExitCode::SUCCESS,
Err(error) => {
crates/backlinks/src/lib.rs
+1 -1
View File
@@ -36,7 +36,7 @@ mod source;
pub(crate) mod text;
pub use error::{Error, Result};
pub use mentions::link::link_mention;
pub use mentions::link::{link_mention, link_mention_as};
pub use mentions::linked::linked_mentions;
pub use mentions::unlinked::{counts, unlinked_mentions};
pub use model::{Counts, LinkedMention, MAX_MENTIONS, MAX_NOTE_BYTES, MAX_SNIPPET_CHARS, UnlinkedMention, UnlinkedReport, group_by_note};
crates/backlinks/src/mentions/link.rs
+14 -2
View File
@@ -26,16 +26,28 @@ pub fn link_mention(content: &str, mention: &UnlinkedMention, target_title: &str
if target_title.is_empty() {
return Err(Error::EmptyTitle);
}
let link = if mention.matched == target_title { format!("[[{target_title}]]") } else { format!("[[{target_title}|{}]]", mention.matched) };
link_mention_as(content, mention, &link)
}
/// [`link_mention`] with the link text already written (the caller picks wikilink or Markdown form).
///
/// # Errors
///
/// Same as [`link_mention`], except [`Error::EmptyTitle`] for an empty `link`.
pub fn link_mention_as(content: &str, mention: &UnlinkedMention, link: &str) -> Result<String> {
if link.is_empty() {
return Err(Error::EmptyTitle);
}
check_range(content, mention)?;
if mention.matched.is_empty() || content.get(mention.range.start..mention.range.end) != Some(mention.matched.as_str()) {
return Err(Error::Stale);
}
let link = if mention.matched == target_title { format!("[[{target_title}]]") } else { format!("[[{target_title}|{}]]", mention.matched) };
let (head, rest) = content.split_at_checked(mention.range.start).ok_or(invalid(content, mention))?;
let (_, tail) = rest.split_at_checked(mention.range.end - mention.range.start).ok_or(invalid(content, mention))?;
let mut out = String::with_capacity(content.len().saturating_sub(mention.matched.len()).saturating_add(link.len()));
out.push_str(head);
out.push_str(&link);
out.push_str(link);
out.push_str(tail);
Ok(out)
}
crates/blocks/Cargo.toml
+2
View File
@@ -15,5 +15,7 @@ workspace = true
[dependencies]
notescraft-markdown = { workspace = true }
notescraft-regex = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
crates/blocks/src/detect.rs
+235
View File
@@ -0,0 +1,235 @@
//! Conservative language detection for unlabeled fenced code blocks.
//!
//! [`detect_language`] returns a fence label only when the evidence is strong;
//! prose, one-liners and ambiguous snippets get `None`, so a batch review never
//! offers a guess it cannot defend. Only the first [`MAX_BYTES`] bytes and the
//! first [`MAX_LINES`] non-blank lines are examined, so a huge body costs no
//! more than a small one.
/// Largest prefix of a body the detector examines, in bytes.
const MAX_BYTES: usize = 4 * 1024;
/// Most non-blank lines the detector examines.
const MAX_LINES: usize = 40;
/// Most non-blank lines the YAML shape rule examines.
const YAML_LINES: usize = 20;
/// Fewest `$` prompt lines the shell rule accepts.
const MIN_PROMPT_LINES: usize = 2;
/// Fewest shaped lines the YAML rule accepts.
const MIN_YAML_LINES: usize = 3;
/// Fewest `key:` lines the YAML rule accepts.
const MIN_YAML_KEYS: usize = 2;
/// The fence label a code-block body is written in, `None` when the evidence is weak.
///
/// The rules run in order and the first match wins: shebang, PHP, HTML, XML,
/// JSON, YAML, diff, shell prompts, Rust, Python, SQL, TOML. Anything else is
/// `None`.
#[must_use]
pub fn detect_language(body: &str) -> Option<&'static str> {
let window = head(body);
if window.trim().is_empty() {
return None;
}
let lines = split_lines(window);
let seen: Vec<&str> = lines.iter().copied().filter(|line| !line.trim().is_empty()).take(MAX_LINES).collect();
if let Some(found) = shebang(seen.first().copied()) {
return Some(found);
}
let lead = window.trim_start();
if lead.starts_with("<?php") {
return Some("php");
}
if starts_html(lead) {
return Some("html");
}
if lead.starts_with("<?xml") {
return Some("xml");
}
if crate::looks_like_json(window) {
return Some("json");
}
if is_yaml(&seen, window) {
return Some("yaml");
}
if is_diff(&seen) {
return Some("diff");
}
if is_prompt(&seen) {
return Some("bash");
}
if is_rust(&seen) {
return Some("rust");
}
if is_python(&seen) {
return Some("python");
}
if is_sql(&seen, window) {
return Some("sql");
}
if is_toml(&seen, window) {
return Some("toml");
}
None
}
/// The first at most [`MAX_BYTES`] bytes of `body`, cut on a char boundary.
fn head(body: &str) -> &str {
let mut end = body.len().min(MAX_BYTES);
while end > 0 && !body.is_char_boundary(end) {
end -= 1;
}
body.get(..end).unwrap_or("")
}
/// `window` split into lines, each without its CR.
fn split_lines(window: &str) -> Vec<&str> {
window.split('\n').map(|line| line.strip_suffix('\r').unwrap_or(line)).collect()
}
/// The interpreter of a `#!` first line: `python`, `bash` (also `sh`/`zsh`) or `javascript`.
fn shebang(line: Option<&str>) -> Option<&'static str> {
let rest = line?.trim_start().strip_prefix("#!")?;
let mut words = rest.split_whitespace();
let first = words.next()?;
let named = if first.rsplit('/').next() == Some("env") { words.next()? } else { first.rsplit('/').next()? };
match named {
"python" => Some("python"),
"bash" | "sh" | "zsh" => Some("bash"),
"node" => Some("javascript"),
_ => None,
}
}
/// `true` when `lead` opens with `<!DOCTYPE html` or `<html`, whatever the case.
fn starts_html(lead: &str) -> bool {
let lower: String = lead.chars().take(16).flat_map(char::to_lowercase).collect();
lower.starts_with("<!doctype html") || lower.starts_with("<html")
}
/// `true` when the first lines are a `key: value` / `- item` YAML shape: every non-blank,
/// non-comment line of the first [`YAML_LINES`] matches, at least [`MIN_YAML_KEYS`] hold a
/// colon key, at least [`MIN_YAML_LINES`] are shaped, and no `{` or `;` appears.
fn is_yaml(seen: &[&str], window: &str) -> bool {
if window.contains('{') || window.contains(';') {
return false;
}
let mut shaped = 0;
let mut keys = 0;
for line in seen.iter().take(YAML_LINES) {
let trimmed = line.trim();
if trimmed.starts_with('#') {
continue; // a comment line is exempt from the shape test
}
if yaml_key(trimmed) {
keys += 1;
shaped += 1;
} else if trimmed.starts_with("- ") || trimmed == "-" {
shaped += 1;
} else {
return false; // one line outside the shape: not YAML
}
}
keys >= MIN_YAML_KEYS && shaped >= MIN_YAML_LINES
}
/// `true` when `line` is `key:` or `key: value` with an ASCII-word key, a colon and a space (or nothing) after it.
fn yaml_key(line: &str) -> bool {
let key = line.bytes().take_while(|b| b.is_ascii_alphanumeric() || *b == b'_' || *b == b'-').count();
if line.as_bytes().get(key) != Some(&b':') {
return false;
}
let after = line.get(key + 1..).unwrap_or("");
after.is_empty() || after.starts_with(' ')
}
/// `true` for a `diff --git ` opener or a `--- a/` line with a `+++ b/` line after it.
fn is_diff(seen: &[&str]) -> bool {
if seen.first().is_some_and(|line| line.starts_with("diff --git ")) {
return true;
}
let mut minus = false;
seen.iter().take(MAX_LINES).any(|line| {
if line.starts_with("--- a/") {
minus = true;
} else if minus && line.starts_with("+++ b/") {
return true;
}
false
})
}
/// `true` when every examined line starts with `$ `, so the body is a transcript of shell commands.
fn is_prompt(seen: &[&str]) -> bool {
seen.len() >= MIN_PROMPT_LINES && seen.iter().all(|line| line.starts_with("$ "))
}
/// `true` on Rust evidence: a `fn `/`pub fn `/`impl `/`use std::` line and a `let `/`->`/`::` line.
fn is_rust(seen: &[&str]) -> bool {
let opens = seen.iter().any(|line| {
let trimmed = line.trim_start();
trimmed.starts_with("fn ") || trimmed.starts_with("pub fn ") || trimmed.starts_with("impl ") || trimmed.starts_with("use std::")
});
let marks = seen.iter().any(|line| line.contains("let ") || line.contains("->") || line.contains("::"));
opens && marks
}
/// `true` on Python evidence: a `def `/`import `/`from x import y` line and an indented, `class ` or `print(` line.
fn is_python(seen: &[&str]) -> bool {
let opens = seen.iter().any(|line| {
let trimmed = line.trim_start();
trimmed.starts_with("def ") || trimmed.starts_with("import ") || (trimmed.starts_with("from ") && trimmed.contains(" import "))
});
let marks = seen.iter().any(|line| line.starts_with(" ") || line.trim_start().starts_with("class ") || line.contains("print("));
opens && marks
}
/// `true` when the first keyword is a DML/DDL opener (case-insensitive) and the body holds a `;` or a FROM/VALUES/WHERE word.
fn is_sql(seen: &[&str], window: &str) -> bool {
let Some(first) = seen.first().copied().map(str::trim) else {
return false;
};
let upper = first.trim().to_ascii_uppercase();
let opens = upper.starts_with("SELECT ")
|| upper.starts_with("INSERT INTO")
|| upper.starts_with("CREATE TABLE")
|| upper.starts_with("DELETE FROM")
|| (upper.starts_with("UPDATE ") && upper.contains(" SET "));
if !opens {
return false;
}
let all_upper = window.to_ascii_uppercase();
all_upper.contains(';') || has_word(&all_upper, "FROM") || has_word(&all_upper, "VALUES") || has_word(&all_upper, "WHERE")
}
/// `true` when `word` appears in `upper` at the start or after a space or newline.
fn has_word(upper: &str, word: &str) -> bool {
upper.starts_with(word) || upper.contains(&format!(" {word}")) || upper.contains(&format!("\n{word}"))
}
/// `true` on TOML evidence: a `[section]` line or two `key = value` lines, and no `{` or `;`.
fn is_toml(seen: &[&str], window: &str) -> bool {
if window.contains('{') || window.contains(';') {
return false;
}
let section = seen.iter().any(|line| {
let trimmed = line.trim();
trimmed.len() > 2
&& trimmed.starts_with('[')
&& trimmed.ends_with(']')
&& !trimmed.contains('"')
&& !trimmed.contains(',')
&& !trimmed.chars().any(char::is_control)
});
let assigns = seen.iter().filter(|line| toml_assign(line.trim())).count();
section || assigns >= 2
}
/// `true` when `line` is a `key = value` assignment with an ASCII-word key and a non-empty value.
fn toml_assign(line: &str) -> bool {
let Some(at) = line.find('=') else {
return false;
};
let key = line.get(..at).unwrap_or("").trim();
let value = line.get(at + 1..).unwrap_or("").trim();
!key.is_empty() && !value.is_empty() && key.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-')
}
crates/blocks/src/find.rs
+323
View File
@@ -0,0 +1,323 @@
//! Plain find and replace: the byte range of every match, and a text with chosen
//! ranges replaced.
//!
//! Matches never overlap and never run past [`MAX_MATCHES`]. Case folding goes through
//! [`char::to_lowercase`] one char at a time, so a folded match keeps the byte range of
//! the text it was found in (`İ` and `ß` included).
use std::ops::Range;
/// Most matches [`find_all`] returns; a text with more matches reports the first ones.
const MAX_MATCHES: usize = 10_000;
/// How [`find_all`] matches: case sensitivity and whole-word boundaries.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct FindOptions {
/// Upper and lower case must match as written.
pub case_sensitive: bool,
/// Only match when neither neighbour is a letter, a digit or `_`.
pub whole_word: bool,
}
impl Default for FindOptions {
fn default() -> Self {
Self { case_sensitive: true, whole_word: false }
}
}
/// The matches of a regular expression, with what is needed to expand `$1` in a replacement.
#[derive(Debug, Clone)]
pub struct RegexFind {
/// Byte range of every match, as [`find_all`] reports them.
pub ranges: Vec<Range<usize>>,
regex: notescraft_regex::Regex,
matches: Vec<notescraft_regex::Match>,
}
/// Every non-empty match of the regular expression `pattern` in `text` (at most [`MAX_MATCHES`]).
///
/// # Errors
/// The reason the pattern is invalid, or that the text is too large to search.
pub fn find_regex(text: &str, pattern: &str, case_sensitive: bool) -> Result<RegexFind, String> {
let regex = notescraft_regex::Regex::new(pattern, !case_sensitive).map_err(|error| error.to_string())?;
let matches = regex.find_all(text, MAX_MATCHES).map_err(|error| error.to_string())?;
Ok(RegexFind { ranges: matches.iter().map(|found| found.range.clone()).collect(), regex, matches })
}
impl RegexFind {
/// The replacement of match `index` for `template` (`$1` group, `$0` whole match, `$$` dollar).
#[must_use]
pub fn replacement(&self, index: usize, text: &str, template: &str) -> String {
self.matches.get(index).map_or_else(String::new, |found| self.regex.expand(found, text, template))
}
/// `text` with every match replaced.
#[must_use]
pub fn replace_all(&self, text: &str, template: &str) -> String {
let mut out = String::with_capacity(text.len());
let mut at = 0;
for (index, range) in self.ranges.iter().enumerate() {
out.push_str(text.get(at..range.start).unwrap_or_default());
out.push_str(&self.replacement(index, text, template));
at = range.end;
}
out.push_str(text.get(at..).unwrap_or_default());
out
}
}
/// Every match of `needle` in `text`: byte ranges on char boundaries, non-overlapping, left to
/// right. An empty `needle` matches nothing, and at most [`MAX_MATCHES`] ranges come back.
#[must_use]
pub fn find_all(text: &str, needle: &str, options: FindOptions) -> Vec<Range<usize>> {
let wanted: Vec<char> = if options.case_sensitive { needle.chars().collect() } else { needle.chars().flat_map(char::to_lowercase).collect() };
if wanted.is_empty() {
return Vec::new();
}
let mut found: Vec<Range<usize>> = Vec::new();
let mut at = 0;
while at <= text.len() && found.len() < MAX_MATCHES {
match match_at(text, at, &wanted, options.case_sensitive) {
Some(end) if whole_word_ok(text, at, end, options.whole_word) => {
found.push(at..end);
at = end; // matches never overlap: the next one starts past this one
}
_ => at = next_char(text, at),
}
}
found
}
/// `text` with each range of `ranges` replaced by `with`, spliced from the end so the byte
/// offsets of the ranges left stay valid. Ranges are taken in order of their start; one that is
/// out of bounds, off a char boundary or overlapping an earlier replacement is skipped.
#[must_use]
pub fn replace_ranges(text: &str, ranges: &[Range<usize>], with: &str) -> String {
let mut order: Vec<&Range<usize>> = ranges.iter().collect();
order.sort_by_key(|range| (range.start, range.end));
let mut out = String::with_capacity(text.len());
let mut at = 0usize;
for range in order {
if range.start < at || !valid(text, range) {
continue; // invalid or overlapping an earlier replacement: the text stays
}
out.push_str(text.get(at..range.start).unwrap_or_default());
out.push_str(with);
at = range.end;
}
out.push_str(text.get(at..).unwrap_or_default());
out
}
/// Whether a range can be spliced out of `text`: inside it, ordered and on char boundaries.
fn valid(text: &str, range: &Range<usize>) -> bool {
range.start <= range.end && range.end <= text.len() && text.is_char_boundary(range.start) && text.is_char_boundary(range.end)
}
/// The end of the match of `wanted` at `at`, `None` when `text` does not hold it there. One text
/// char can stand for several wanted chars when its lowercase folds longer (`ß` to `ss`).
fn match_at(text: &str, at: usize, wanted: &[char], case_sensitive: bool) -> Option<usize> {
if case_sensitive {
let mut end = at;
for want in wanted {
let c = text.get(end..).and_then(|rest| rest.chars().next())?;
if c != *want {
return None;
}
end += c.len_utf8();
}
return Some(end);
}
let mut end = at;
let mut matched = 0usize;
while matched < wanted.len() {
let c = text.get(end..).and_then(|rest| rest.chars().next())?;
let mut done = false;
for got in c.to_lowercase() {
if got != wanted.get(matched).copied()? {
return None;
}
matched += 1;
if matched == wanted.len() {
done = true; // the needle ran out inside this char, so the whole char is in the match
break;
}
}
end += c.len_utf8();
if done {
return Some(end);
}
}
Some(end)
}
/// Whether both neighbours of a match are word-free, so `whole_word` keeps it.
fn whole_word_ok(text: &str, start: usize, end: usize, whole_word: bool) -> bool {
if !whole_word {
return true;
}
let before = text.get(..start).and_then(|head| head.chars().next_back());
let after = text.get(end..).and_then(|rest| rest.chars().next());
[before, after].into_iter().flatten().all(|c| !c.is_alphanumeric() && c != '_')
}
/// The next place a match may start: the char after `at`, past the end of the text.
fn next_char(text: &str, at: usize) -> usize {
match text.get(at..).and_then(|rest| rest.chars().next()) {
Some(c) => at + c.len_utf8(),
None => text.len() + 1,
}
}
#[cfg(test)]
mod tests {
use super::*;
const PLAIN: FindOptions = FindOptions { case_sensitive: true, whole_word: false };
const FOLDED: FindOptions = FindOptions { case_sensitive: false, whole_word: false };
const WORD: FindOptions = FindOptions { case_sensitive: true, whole_word: true };
/// What `find_all` matched, as text.
fn hits<'a>(text: &'a str, needle: &str, options: FindOptions) -> Vec<&'a str> {
find_all(text, needle, options).iter().filter_map(|range| text.get(range.clone())).collect()
}
/// `(text, needle, options, expected matches)`.
type Row = (&'static str, &'static str, FindOptions, Vec<&'static str>);
#[test]
fn find_all_rows() {
let rows: Vec<Row> = vec![
("hello world", "world", PLAIN, vec!["world"]),
("aaa", "aa", PLAIN, vec!["aa"]),
("aaaa", "aa", PLAIN, vec!["aa", "aa"]),
("abcabcabc", "abc", PLAIN, vec!["abc", "abc", "abc"]),
("abc", "z", PLAIN, vec![]),
("abc", "", PLAIN, vec![]),
("", "a", PLAIN, vec![]),
("", "", PLAIN, vec![]),
("a b", " ", PLAIN, vec![" "]),
("line\nline", "line", PLAIN, vec!["line", "line"]),
("x\ty", "\t", PLAIN, vec!["\t"]),
("aaa", "aaaa", PLAIN, vec![]),
("foo bar foo", "foo", PLAIN, vec!["foo", "foo"]),
("FOO foo", "foo", PLAIN, vec!["foo"]),
("FOO foo", "foo", FOLDED, vec!["FOO", "foo"]),
("Foo Bar", "fOO", FOLDED, vec!["Foo"]),
("Straße", "asse", FOLDED, vec![]),
("straße", "STRASSE", FOLDED, vec![]), // `ß` folds to itself, not to `ss`
("ẞ", "ß", FOLDED, vec!["ẞ"]),
("Istanbul", "istanbul", FOLDED, vec!["Istanbul"]),
("İstanbul", "istanbul", FOLDED, vec![]), // `İ` folds to `i` plus a dot, so it is not `i`
("ÄÖÜ", "äöü", FOLDED, vec!["ÄÖÜ"]),
("é", "e", FOLDED, vec![]),
("cat dog cat", "cat", WORD, vec!["cat", "cat"]),
("cat.", "cat", WORD, vec!["cat"]),
("(cat)", "cat", WORD, vec!["cat"]),
("concat", "cat", WORD, vec![]),
("cat_x", "cat", WORD, vec![]),
("cat9", "cat", WORD, vec![]),
("9cat", "cat", WORD, vec![]),
("_cat_", "cat", WORD, vec![]),
("cat, dog, cat!", "cat", WORD, vec!["cat", "cat"]),
("concat", "cat", PLAIN, vec!["cat"]),
("cat", "cat", WORD, vec!["cat"]),
("CAT cat", "cat", FOLDED, vec!["CAT", "cat"]),
("a cat CAT", "cat", WORD, vec!["cat"]),
("a cat CAT", "cat", FindOptions { case_sensitive: false, whole_word: true }, vec!["cat", "CAT"]),
("ß", "s", FOLDED, vec![]),
("ß", "ss", FOLDED, vec![]),
("ß", "ß", FOLDED, vec!["ß"]),
("İ", "i", FOLDED, vec!["İ"]),
("ss", "ß", FOLDED, vec![]),
("naïve", "Ïve", FOLDED, vec!["ïve"]),
("naïve", "ïve", FOLDED, vec!["ïve"]),
(" spaced ", "spaced", WORD, vec!["spaced"]),
("--flag --flag", "-", WORD, vec!["-", "-"]),
("a\tb", "a", WORD, vec!["a"]),
("[a]", "a", WORD, vec!["a"]),
("🚀 rocket 🚀", "🚀", PLAIN, vec!["🚀", "🚀"]),
("🚀 rocket 🚀", "rocket", WORD, vec!["rocket"]),
];
for (text, needle, options, expected) in rows {
assert_eq!(hits(text, needle, options), expected, "find {needle:?} in {text:?}");
}
}
#[test]
fn ranges_are_byte_ranges_on_char_boundaries() {
let text = "İİ istanbul";
let range = find_all(text, "istanbul", FOLDED).first().cloned().expect("one match");
assert_eq!(range, 5..13);
assert_eq!(text.get(range.clone()), Some("istanbul"));
let sharp = find_all("İ", "i", FOLDED).first().cloned().expect("one match");
assert_eq!(sharp, 0..2, "`İ` folds to two chars but keeps its own byte range");
}
#[test]
fn matches_never_overlap_and_stop_at_the_cap() {
let text = "a".repeat(20_000);
let found = find_all(&text, "a", PLAIN);
assert_eq!(found.len(), MAX_MATCHES);
assert!(found.windows(2).all(|pair| pair[0].end <= pair[1].start), "no overlaps");
let stepped = "aaaaa".repeat(4);
let ranges = find_all(&stepped, "aa", PLAIN);
assert_eq!(ranges.len(), 10);
assert_eq!(ranges.last().cloned(), Some(18..20));
}
#[test]
fn a_mib_of_text_stays_bounded() {
let text = "needle in a haystack, ".repeat(50_000);
assert!(text.len() > 1024 * 1024, "{} bytes", text.len());
let began = std::time::Instant::now();
let found = find_all(&text, "needle", PLAIN);
assert_eq!(found.len(), MAX_MATCHES, "a megabyte of matches is capped");
assert!(began.elapsed().as_secs_f64() < 2.0, "took {}s", began.elapsed().as_secs_f64());
}
#[test]
fn replace_ranges_splices_from_the_end() {
let text = "one two three";
let ranges = find_all(text, "two", PLAIN);
assert_eq!(replace_ranges(text, &ranges, "2"), "one 2 three");
assert_eq!(replace_ranges(text, &[], "x"), text);
assert_eq!(replace_ranges(text, &find_all(text, "o", PLAIN), "0"), "0ne tw0 three");
}
/// `replace_ranges` with a single `start..end` range.
fn one(text: &str, start: usize, end: usize, with: &str) -> String {
let range = start..end;
replace_ranges(text, std::slice::from_ref(&range), with)
}
#[test]
fn replace_ranges_skips_invalid_and_overlapping_ranges() {
let text = "café au lait";
let all_a = find_all(text, "a", PLAIN);
assert_eq!(replace_ranges(text, &all_a, "A"), "cAfé Au lAit");
assert_eq!(replace_ranges(text, &[3..5, 5..6], "-"), "caf--au lait");
assert_eq!(one(text, 0, text.len() + 5, "x"), text, "out of bounds");
assert_eq!(one(text, 3, 1, "x"), text, "reversed");
assert_eq!(one(text, 4, 5, "x"), text, "off a char boundary");
let sorted = "one two three";
assert_eq!(replace_ranges(sorted, &[8..13, 0..3], "!"), "! two !", "ranges are sorted first");
assert_eq!(replace_ranges(text, &[], "-"), text, "no range changes nothing");
}
}
#[cfg(test)]
mod regex_tests {
use super::*;
#[test]
fn regex_matches_expand_groups_and_bad_patterns_report() {
let text = "a-1 b-22";
let found = find_regex(text, r"(\w)-(\d+)", true).unwrap();
assert_eq!(found.ranges, [0..3, 4..8]);
assert_eq!(found.replace_all(text, "$2$1"), "1a 22b");
assert_eq!(found.replacement(1, text, "<$0>"), "<b-22>");
assert!(find_regex(text, "(", true).is_err());
assert_eq!(find_regex("ABC", "b", false).unwrap().ranges.first(), Some(&(1..2)));
}
}
crates/blocks/src/json.rs
+260
View File
@@ -0,0 +1,260 @@
//! JSON in code blocks: detect, validate, format and minify.
//!
//! Formatting re-indents the validated text instead of re-serialising it, so key order, number
//! spelling (`1e5`, `1.0`) and string escapes stay byte-for-byte what the author wrote.
use std::ops::Range;
use serde::de::IgnoredAny;
use crate::{BlockKind, Edited, Error, Result, block, splice};
/// Largest JSON text that is checked or rewritten, in bytes.
pub const MAX_JSON_BYTES: usize = 1024 * 1024;
/// Spaces per indent level.
const INDENT: usize = 2;
/// Checks that `text` is one JSON value (the parser's nesting limit applies).
///
/// # Errors
/// [`Error::Json`] with the position, or for text over [`MAX_JSON_BYTES`].
pub fn check_json(text: &str) -> Result<()> {
if text.len() > MAX_JSON_BYTES {
return Err(Error::Json { message: format!("larger than {MAX_JSON_BYTES} bytes"), line: 1, column: 1 });
}
serde_json::from_str::<IgnoredAny>(text).map(|_| ()).map_err(|error| Error::Json { message: error.to_string(), line: error.line(), column: error.column() })
}
/// The byte range of the JSON inside a code-block body: the whole trimmed body, or the inside of
/// a single backtick-quoted template string. `None` when neither is a JSON object or array.
#[must_use]
pub fn json_span(body: &str) -> Option<Range<usize>> {
let lead = body.len() - body.trim_start().len();
let core = body.trim();
let (start, inner) = match core.strip_prefix('`').and_then(|rest| rest.strip_suffix('`')) {
Some(inner) if !inner.contains('`') => (lead + 1, inner),
_ => (lead, core),
};
let first = inner.chars().next()?;
let last = inner.chars().next_back()?;
let shaped = matches!((first, last), ('{', '}') | ('[', ']'));
(shaped && check_json(inner).is_ok()).then(|| start..start + inner.len())
}
/// `true` when the body holds JSON that [`format_json_block`] would act on.
#[must_use]
pub fn looks_like_json(body: &str) -> bool {
json_span(body).is_some()
}
/// Pretty-prints valid JSON with two-space indents; empty containers stay `{}` and `[]`.
///
/// # Errors
/// [`Error::Json`] when `text` is not valid JSON.
pub fn format_json(text: &str) -> Result<String> {
check_json(text)?;
let mut out = String::with_capacity(text.len() + text.len() / 4);
let mut depth = 0usize;
let mut in_string = false;
let mut escaped = false;
let mut chars = text.chars().peekable();
while let Some(c) = chars.next() {
if in_string {
out.push(c);
match (escaped, c) {
(true, _) => escaped = false,
(false, '\\') => escaped = true,
(false, '"') => in_string = false,
_ => {}
}
continue;
}
match c {
'"' => {
in_string = true;
out.push(c);
}
'{' | '[' => {
out.push(c);
while chars.next_if(|next| next.is_whitespace()).is_some() {}
if chars.next_if(|next| matches!((c, next), ('{', '}') | ('[', ']'))).is_some() {
out.push(if c == '{' { '}' } else { ']' });
} else {
depth += 1;
newline(&mut out, depth);
}
}
'}' | ']' => {
depth = depth.saturating_sub(1);
newline(&mut out, depth);
out.push(c);
}
',' => {
out.push(c);
newline(&mut out, depth);
}
':' => out.push_str(": "),
c if c.is_whitespace() => {}
c => out.push(c),
}
}
Ok(out)
}
fn newline(out: &mut String, depth: usize) {
out.push('\n');
out.extend(std::iter::repeat_n(' ', depth.saturating_mul(INDENT)));
}
/// Removes every insignificant space from valid JSON.
///
/// # Errors
/// [`Error::Json`] when `text` is not valid JSON.
pub fn minify_json(text: &str) -> Result<String> {
check_json(text)?;
let mut out = String::with_capacity(text.len());
let mut in_string = false;
let mut escaped = false;
for c in text.chars() {
if in_string {
match (escaped, c) {
(true, _) => escaped = false,
(false, '\\') => escaped = true,
(false, '"') => in_string = false,
_ => {}
}
out.push(c);
} else if c == '"' {
in_string = true;
out.push(c);
} else if !c.is_whitespace() {
out.push(c);
}
}
Ok(out)
}
/// `replacement` with its line endings made to match `note`: CRLF when the note uses CRLF, else as given.
#[must_use]
pub fn match_line_endings(note: &str, replacement: &str) -> String {
if note.contains("\r\n") { replacement.replace("\r\n", "\n").replace('\n', "\r\n") } else { replacement.to_owned() }
}
/// Formats the JSON in code block `index` as one edit.
///
/// # Errors
/// [`Error::BlockIndex`], [`Error::Unsupported`] when the block is not a fence, or [`Error::Json`]
/// when the body holds no valid JSON.
pub fn format_json_block(text: &str, index: usize) -> Result<Edited> {
rewrite(text, index, format_json)
}
/// Minifies the JSON in code block `index` as one edit; errors as [`format_json_block`].
///
/// # Errors
/// See [`format_json_block`].
pub fn minify_json_block(text: &str, index: usize) -> Result<Edited> {
rewrite(text, index, minify_json)
}
fn rewrite(text: &str, index: usize, run: fn(&str) -> Result<String>) -> Result<Edited> {
let found = block(text, index)?;
if found.kind != BlockKind::CodeFence {
return Err(Error::Unsupported("only code blocks hold JSON"));
}
let body = text.get(found.text_range.clone()).ok_or(Error::CharBoundary { start: found.text_range.start, end: found.text_range.end })?;
let Some(span) = json_span(body) else {
check_json(body.trim())?;
return Err(Error::Unsupported("the code block holds no JSON object or array"));
};
let inner = body.get(span.clone()).ok_or(Error::CharBoundary { start: span.start, end: span.end })?;
let new = match_line_endings(text, &run(inner)?);
let start = found.text_range.start + span.start;
splice(text, start..found.text_range.start + span.end, new, start)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn check_accepts_valid_and_reports_position_for_invalid() {
assert!(check_json("{\"a\": [1, 2, {\"b\": null}]}").is_ok());
assert!(matches!(check_json("{\"a\": }"), Err(Error::Json { line: 1, .. })));
assert!(matches!(check_json("{\"a\":\n 1,}"), Err(Error::Json { line: 2, .. })));
assert!(check_json("").is_err());
}
#[test]
fn format_keeps_key_order_numbers_and_escapes_byte_for_byte() {
let src = r#"{"z":1e5,"a":[1.0,-0,"q\"uote \\ \u00e9"],"m":{"n":true,"e":{},"l":[ ]}}"#;
let pretty = format_json(src).expect("valid");
let expected = "{\n \"z\": 1e5,\n \"a\": [\n 1.0,\n -0,\n \"q\\\"uote \\\\ \\u00e9\"\n ],\n \"m\": {\n \"n\": true,\n \"e\": {},\n \"l\": []\n }\n}";
assert_eq!(pretty, expected);
assert_eq!(minify_json(&pretty).expect("valid"), src.replace("[ ]", "[]"));
assert_eq!(format_json(&pretty).expect("valid"), pretty, "formatting is idempotent");
}
#[test]
fn strings_with_structure_characters_are_untouched() {
let src = r#"{"k": "a, b: {c} [d] e"}"#;
assert_eq!(minify_json(src).expect("valid"), r#"{"k":"a, b: {c} [d] e"}"#);
assert!(format_json(src).expect("valid").contains("\"a, b: {c} [d] e\""));
}
#[test]
fn invalid_depth_and_size_are_errors_not_panics() {
assert!(format_json("{\"a\":").is_err());
assert!(format_json(&"[".repeat(5000)).is_err(), "nesting limit");
let big = format!("[\"{}\"]", "x".repeat(MAX_JSON_BYTES));
assert!(format_json(&big).is_err() && minify_json(&big).is_err());
for hostile in ["\u{0}", "{\"a\":\"\u{fffd}\u{0}\"", "\u{1f600}", "[1,,]", "{,}"] {
assert!(format_json(hostile).is_err(), "{hostile:?}");
}
}
#[test]
fn spans_cover_plain_bodies_and_single_template_strings() {
assert_eq!(json_span("\n {\"a\":1}\n"), Some(3..10));
assert_eq!(json_span("`{\"a\":1}`\n"), Some(1..8));
assert!(looks_like_json("[1, 2]\n"));
assert!(!looks_like_json("let x = {};\n"));
assert!(!looks_like_json("`{\"a\":${x}}`"));
assert!(!looks_like_json("`{\"a\":1}` + `{}`"));
assert!(!looks_like_json("42"), "only objects and arrays are offered");
assert!(!looks_like_json("{not json}"));
}
#[test]
fn block_edits_replace_only_the_json_and_keep_the_fence() {
let note = "# T\n\n```json\n{\"a\":1,\"b\":[2]}\n```\n\nafter\n";
let edited = format_json_block(note, 1).expect("formats");
assert_eq!(edited.text, "# T\n\n```json\n{\n \"a\": 1,\n \"b\": [\n 2\n ]\n}\n```\n\nafter\n");
let back = minify_json_block(&edited.text, 1).expect("minifies");
assert_eq!(back.text, note);
assert_eq!(&note[edited.replace.0.clone()], "{\"a\":1,\"b\":[2]}", "the replacement range is the old JSON");
}
#[test]
fn crlf_notes_get_crlf_inside_the_formatted_json_too() {
let note = "```json\r\n{\"a\":1}\r\n```\r\n";
assert_eq!(format_json_block(note, 0).expect("formats").text, "```json\r\n{\r\n \"a\": 1\r\n}\r\n```\r\n");
assert_eq!(match_line_endings("lf\n", "a\nb"), "a\nb");
assert_eq!(match_line_endings("crlf\r\n", "a\nb\r\nc"), "a\r\nb\r\nc");
}
#[test]
fn template_strings_keep_their_backticks() {
let note = "```js\n`{\"a\":1}`\n```\n";
assert_eq!(format_json_block(note, 0).expect("formats").text, "```js\n`{\n \"a\": 1\n}`\n```\n");
}
#[test]
fn bad_input_leaves_the_caller_with_an_error() {
assert!(matches!(format_json_block("```json\n{\"a\":\n```\n", 0), Err(Error::Json { .. })));
assert!(matches!(format_json_block("hello\n", 0), Err(Error::Unsupported(_))));
assert!(matches!(format_json_block("```\nplain\n```\n", 0), Err(Error::Json { .. })));
assert!(matches!(format_json_block("```\n42\n```\n", 0), Err(Error::Unsupported(_))));
assert!(matches!(format_json_block("x\n", 9), Err(Error::BlockIndex(9))));
}
}
crates/blocks/src/lib.rs
+16
View File
@@ -7,14 +7,20 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
mod convert;
mod detect;
mod find;
mod insert;
mod json;
mod ops;
mod parse;
use std::ops::Range;
pub use convert::{Target, turn_into};
pub use detect::detect_language;
pub use find::{FindOptions, RegexFind, find_all, find_regex, replace_ranges};
pub use insert::{Component, insert_component};
pub use json::{MAX_JSON_BYTES, check_json, format_json, format_json_block, json_span, looks_like_json, match_line_endings, minify_json, minify_json_block};
pub use ops::{Direction, delete_block, duplicate_block, move_block};
pub use parse::parse_blocks;
@@ -48,6 +54,16 @@ pub enum Error {
/// The operation does not apply to this block.
#[error("{0}")]
Unsupported(&'static str),
/// The text is not valid JSON.
#[error("invalid JSON at line {line}, column {column}: {message}")]
Json {
/// What the parser said.
message: String,
/// 1-based line of the problem.
line: usize,
/// 1-based column of the problem.
column: usize,
},
}
/// Result alias used across the crate.
crates/blocks/tests/detect_language.rs
+146
View File
@@ -0,0 +1,146 @@
//! Table-driven tests of [`notescraft_blocks::detect_language`]: every rule positive, near-miss
//! negatives, CRLF input, empty and hostile bodies, and the 4 KiB / 40 line bounds.
use notescraft_blocks::detect_language;
/// (body, expected label) rows: a rule positive or a near miss that must stay `None`.
#[test]
fn the_table() {
let rows: Vec<(&str, Option<&str>)> = vec![
// shebang
("#!/usr/bin/env python\nprint('hi')\n", Some("python")),
("#!/usr/bin/python\nimport os\n", Some("python")),
("#!/usr/bin/env bash\necho hi\n", Some("bash")),
("#!/bin/bash\necho hi\n", Some("bash")),
("#!/bin/sh\necho hi\n", Some("bash")),
("#!/usr/bin/env zsh\necho hi\n", Some("bash")),
("#!/usr/bin/env node\nconsole.log(1)\n", Some("javascript")),
("#!/usr/bin/env ruby\nputs 'no'\n", None),
("# not a shebang\n", None),
// php, html, xml by opener
("<?php\necho 1;\n", Some("php")),
("<?PHP\necho 1;\n", None),
("<!DOCTYPE html>\n<html><body></body></html>\n", Some("html")),
("<!doctype html>\n<p>hi</p>\n", Some("html")),
("<HTML>\n</HTML>\n", Some("html")),
("<div>no label for fragments</div>\n", None),
("<?xml version=\"1.0\"?>\n<a/>\n", Some("xml")),
// json
("{\"a\": 1}\n", Some("json")),
("[1, 2, 3]\n", Some("json")),
("`{\"a\":1}`\n", Some("json")),
("{\"a\": 1}\r\n", Some("json")),
("{a: 1}\nb: 2\n", None),
("42\n", None),
// yaml
("name: NotesCraft\nversion: 1.0\nitems:\n - one\n - two\n", Some("yaml")),
("title: a\n# a comment\n- one\nkey: value\n", Some("yaml")),
("a: {x}\nb: 1\nc: 2\n", None),
("a: b\nc: d\n", None),
("x: 1; y\nz: 2\nw: 3\n", None),
// diff
("diff --git a/x b/x\nindex 1..2 100644\n", Some("diff")),
("--- a/x\n+++ b/x\n@@ -1 +1 @@\n-old\n+new\n", Some("diff")),
("+++ b/x\n--- a/x\n", None),
("- removed\n+ added\n", None),
// shell transcripts
("$ cargo build\n$ cargo test\n", Some("bash")),
("$ ls\nfile1 file2\n$ pwd\n/home\n", None),
("$ make\n", None),
// rust
("fn main() {\n let x = 1;\n}\n", Some("rust")),
("impl Foo {\n fn bar(&self) -> u32 {\n 0\n }\n}\n", Some("rust")),
("use std::collections::HashMap;\n", Some("rust")),
("pub fn id(x: u32) -> u32 {\n x\n}\n", Some("rust")),
("let x = 1;\n", None),
("fn main() {}\n", None),
("use foo::bar;\n", None),
// python
("def add(a, b):\n return a + b\n", Some("python")),
("import os\nprint(os.getcwd())\n", Some("python")),
("from os import path\nprint(path)\n", Some("python")),
("class A:\n def x(self):\n pass\n", Some("python")),
("def f(): pass\n", None),
("import os\n", None),
("from x y\n", None),
// sql
("SELECT * FROM users;\n", Some("sql")),
("select name\nfrom t\nwhere id = 1\n", Some("sql")),
("INSERT INTO t VALUES (1);\n", Some("sql")),
("CREATE TABLE t (id INT);\n", Some("sql")),
("DELETE FROM t WHERE id = 1;\n", Some("sql")),
("UPDATE t SET a = 1 WHERE b = 2;\n", Some("sql")),
("UPDATE t SET a = 1\n", None),
("SELECT 1\n", None),
("insert into\n", None),
// toml
("[package]\nname = \"x\"\n", Some("toml")),
("a = 1\nb = 2\n", Some("toml")),
("a = 1\n", None),
("a = 1\nb = 2;\n", None),
("[\"a\",]\n", None),
("[section]\n", Some("toml")),
// prose, lists, one-liners, hostile
("The plan is simple: we ship it.\nAnother line here: no code.\nMostly prose without any shape at all.\n", None),
("Note: this\nWarning: that\n", None),
("- one\n- two\n- three\n", None),
("x = 1", None),
("a: b", None),
("just a sentence\n", None),
("1 + 1\n2 + 2\n", None),
("hello: world\nfoo: bar\n", None),
("\u{0}\u{0}\u{fffd}\n", None),
("caf\u{e9}: no\n", None),
// crlf versions of the keyword rules
("fn main() {\r\n let x = 1;\r\n}\r\n", Some("rust")),
("def add(a, b):\r\n return a + b\r\n", Some("python")),
("$ cargo build\r\n$ cargo test\r\n", Some("bash")),
("[package]\r\nname = \"x\"\r\n", Some("toml")),
];
for (body, expected) in rows {
assert_eq!(detect_language(body), expected, "body {body:?}");
}
}
#[test]
fn empty_and_blank_bodies_are_none() {
assert_eq!(detect_language(""), None);
assert_eq!(detect_language(" \n\n \n"), None);
assert_eq!(detect_language("\r\n\r\n"), None);
}
#[test]
fn only_the_first_4_kib_and_40_non_blank_lines_are_examined() {
// 1500 `$ x` lines (6000 bytes) push the bad line past the 4 KiB window.
let beyond_bytes = format!("{}not a prompt\n", "$ x\n".repeat(1500));
assert_eq!(detect_language(&beyond_bytes), Some("bash"));
// 40 prompt lines: the 41st is never examined.
let beyond_lines = format!("{}nope\n", "$ x\n".repeat(40));
assert_eq!(detect_language(&beyond_lines), Some("bash"));
// 39 prompt lines plus the bad one: the rule fails.
let within = format!("{}nope\n", "$ x\n".repeat(39));
assert_eq!(detect_language(&within), None);
}
#[test]
fn a_mib_body_stays_bounded_in_time() {
let began = std::time::Instant::now();
let big = "x".repeat(1024 * 1024);
assert_eq!(detect_language(&big), None);
let mut pythonish = String::from("def add(a, b):\n return a + b\n");
pythonish.push_str(&"x".repeat(1024 * 1024));
assert_eq!(detect_language(&pythonish), Some("python"));
let mut multibyte = String::from("fn main() {\n let x = \"\u{e9}\";\n}\n");
multibyte.push_str(&"\u{e9}".repeat(700 * 1024));
assert_eq!(detect_language(&multibyte), Some("rust"));
assert!(began.elapsed().as_secs_f64() < 1.0, "took {}s", began.elapsed().as_secs_f64());
}
#[test]
fn hostile_bytes_never_panic_and_stay_none() {
for hostile in ["\u{0}", "\u{0}\n\u{0}", "\u{7f}\u{8}", "\u{fffd}".repeat(5000).as_str(), "#!/usr/bin/env\u{0}python\n", "<<??php", "[]", "{}", "[\u{0}]"] {
let _ = detect_language(hostile);
}
assert_eq!(detect_language("[\u{0}]"), None);
assert_eq!(detect_language("#!/usr/bin/env python\u{0}"), None);
}
crates/blocks/tests/json_hostile.rs
+241
View File
@@ -0,0 +1,241 @@
//! Hostile input for the JSON half of the blocks crate: random alphabet soup, deep nesting,
//! half a million escaped quotes and absurd numbers.
//!
//! Invariant everywhere: nothing panics, the size and nesting caps are errors, and for any input
//! `format_json` accepts, formatting is idempotent and minifying loses nothing.
use notescraft_blocks::{Edited, check_json, format_json, format_json_block, json_span, looks_like_json, minify_json, minify_json_block};
use serde_json::Value;
/// Strings in the random sweep.
const RANDOM_CASES: usize = 3000;
/// Longest random string, in chars.
const RANDOM_MAX: usize = 120;
/// Bytes of the escaped-quote monster.
const QUOTE_PAIR: &str = "\\\"";
/// xorshift64: deterministic, tiny, no dependency.
struct Rng(u64);
impl Rng {
fn next(&mut self) -> u64 {
let mut x = self.0;
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
self.0 = x;
x
}
/// Bytes from the JSON alphabet plus NUL and a few multi-byte shapes.
fn text(&mut self, len: usize) -> String {
const ALPHABET: &[char] = &[
'{', '}', '[', ']', '"', ',', ':', ' ', '\n', '\t', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'e', 'E', '.', '+', '-', 't', 'f', 'n', 'u',
'l', '\0', '\u{7f}', 'é', '😀', '\u{2028}', '\u{feff}',
];
(0..len)
.map(|_| {
let at = u64::try_from(ALPHABET.len()).unwrap_or(u64::MAX);
let at = usize::try_from(self.next() % at).unwrap_or(0);
*ALPHABET.get(at).unwrap_or(&'?')
})
.collect()
}
}
/// One note with the hostile text as the body of a `json` code block.
fn note(body: &str) -> String {
format!("# n\n\n```json\n{body}\n```\n\nafter\n")
}
/// The edit must be exactly one splice of the old note, and the caret must be inside the new text.
fn assert_edit_is_consistent(old: &str, edited: &Edited) {
let (range, new) = &edited.replace;
let head = old.get(..range.start).unwrap_or_default();
let tail = old.get(range.end..).unwrap_or_default();
let rebuilt = format!("{head}{new}{tail}");
assert_eq!(rebuilt, edited.text, "the edit is one splice of the old text");
assert!(edited.caret <= edited.text.len(), "the caret is past the text");
assert!(edited.text.is_char_boundary(edited.caret), "the caret is not on a char boundary");
assert!(edited.text.contains("after\n"), "the rest of the note survives");
}
/// Whatever the input, formatting must be idempotent and minifying must lose nothing.
fn assert_format_properties(text: &str) {
let Ok(formatted) = format_json(text) else { return };
let minified = minify_json(text).unwrap_or_default();
assert_eq!(minify_json(&formatted).unwrap_or_default(), minified, "minify(format(x)) == minify(x) for {text:.40}");
assert_eq!(format_json(&formatted).unwrap_or_default(), formatted, "format is idempotent for {text:.40}");
assert_eq!(serde_json::from_str::<Value>(text).ok(), serde_json::from_str::<Value>(&formatted).ok(), "same value for {text:.40}");
assert!(check_json(&formatted).is_ok());
}
#[test]
fn random_alphabet_soup_never_panics() {
let mut rng = Rng(0x1064_5dca_9e37_79b9);
for _ in 0..RANDOM_CASES {
let cap = u64::try_from(RANDOM_MAX).unwrap_or(u64::MAX).saturating_add(1);
let len = usize::try_from(rng.next() % cap).unwrap_or(0);
let text = rng.text(len);
let _ = check_json(&text);
let _ = looks_like_json(&text);
if let Some(span) = json_span(&text) {
assert!(span.start <= span.end && span.end <= text.len(), "span out of the body: {text:.40}");
assert!(text.is_char_boundary(span.start) && text.is_char_boundary(span.end), "span off a char boundary");
let inner = text.get(span.clone()).unwrap_or_default();
assert!(check_json(inner).is_ok(), "the span must hold valid JSON");
assert!(matches!((inner.chars().next(), inner.chars().next_back()), (Some('{'), Some('}')) | (Some('['), Some(']'))));
assert!(looks_like_json(&text));
}
assert_format_properties(&text);
}
}
/// `{"a":{"a":...1...}}` `depth` levels deep.
fn nest(depth: usize) -> String {
format!("{}1{}", "{\"a\":".repeat(depth), "}".repeat(depth))
}
#[test]
fn deep_nesting_is_handled_without_a_stack_overflow() {
let deep_ok = nest(100);
assert!(check_json(&deep_ok).is_ok(), "100 levels is valid JSON");
assert_format_properties(&deep_ok);
// 129 levels: valid JSON, and nothing overflows. See the ignored BUG test below: the rustdoc
// claims serde_json's limit of 128 applies, but `IgnoredAny` skips values iteratively.
let at_129 = nest(129);
assert!(check_json(&at_129).is_ok(), "129 levels parses");
assert!(check_json(&format!("{}0{}", "[".repeat(129), "]".repeat(129))).is_ok());
assert_format_properties(&at_129);
// Deep but small: under the byte cap, so the parser sees it and survives it.
let deep = format!("{}0{}", "[".repeat(200_000), "]".repeat(200_000));
assert!(deep.len() <= notescraft_blocks::MAX_JSON_BYTES, "{} bytes", deep.len());
assert!(check_json(&deep).is_ok(), "no stack overflow on deep nesting");
assert!(minify_json(&deep).is_ok());
// Unbalanced nesting is an error, at any depth.
for (open, close) in [("{", ""), ("", "}"), ("[[", "]]"), ("{\"a\":", ""), ("{\"a\": 1,", "")] {
let junk = format!("{}{}", open.repeat(200), close);
assert!(check_json(&junk).is_err(), "{junk:.20}");
assert!(format_json(&junk).is_err(), "{junk:.20}");
assert!(minify_json(&junk).is_err(), "{junk:.20}");
}
}
#[test]
#[ignore = "BUG: check_json's rustdoc says the parser's nesting limit applies, but 129 levels is accepted (IgnoredAny skips iteratively)"]
fn nesting_over_128_is_refused() {
let too_deep = nest(129);
assert!(check_json(&too_deep).is_err(), "129 levels should be over serde_json's limit of 128");
assert!(format_json(&too_deep).is_err());
assert!(minify_json(&too_deep).is_err());
}
#[test]
#[ignore = "BUG: format_json expands quadratically with depth: 2000 levels (6 KB in) already make a 4 MB string, so 1 MiB of nesting would try to build ~275 GB"]
fn formatting_deep_json_stays_bounded() {
let text = nest(2_000);
let formatted = format_json(&text).unwrap_or_default();
assert!(formatted.len() <= notescraft_blocks::MAX_JSON_BYTES, "{} bytes out of {} in", formatted.len(), text.len());
}
#[test]
fn a_string_with_half_a_million_escaped_quotes_survives() {
let text = format!("[{}]{}]", "\"", "").to_owned() + &QUOTE_PAIR.repeat(500_000) + "\"]";
assert!(text.len() > 1_000_000 && text.len() < notescraft_blocks::MAX_JSON_BYTES, "{} bytes", text.len());
assert!(check_json(&text).is_ok(), "a monster string is still valid JSON");
assert_format_properties(&text);
let formatted = format_json(&text).unwrap_or_default();
assert_eq!(formatted.matches(QUOTE_PAIR).count(), 500_000, "no escape is lost");
assert_eq!(minify_json(&formatted).unwrap_or_default(), minify_json(&text).unwrap_or_default());
}
#[test]
fn number_spelling_survives_formatting_byte_for_byte() {
for number in ["1e999999999", "-0", "0.1e-5", "1E5", "-0.0e0", "123456789012345678901234567890", "0", "-1.7976931348623157e308", "1e-999999"] {
let text = format!("{{\"a\":{number},\"b\":[{number}]}}");
let formatted = format_json(&text).unwrap_or_default();
assert_eq!(formatted.matches(number).count(), 2, "{number} must survive byte for byte: {formatted}");
assert_eq!(minify_json(&formatted).unwrap_or_default(), minify_json(&text).unwrap_or_default());
assert_eq!(format_json(&formatted).unwrap_or_default(), formatted.clone());
assert_eq!(serde_json::from_str::<Value>(&text).ok(), serde_json::from_str::<Value>(&formatted).ok(), "{number}");
}
assert_format_properties(r#"{"deep":[[[[{"x":1e999999999}]]]],"neg":[-0,-0.1e-5]}"#);
}
#[test]
fn the_size_cap_is_an_error_and_huge_bodies_do_not_panic() {
let over = format!("[\"{}\"]", "x".repeat(notescraft_blocks::MAX_JSON_BYTES));
assert!(check_json(&over).is_err(), "over the size cap");
assert!(format_json(&over).is_err() && minify_json(&over).is_err());
assert!(!looks_like_json(&over), "an oversized block holds no offerable JSON");
assert_eq!(json_span(&over), None);
let under = format!("[\"{}\"]", "x".repeat(100_000));
assert!(check_json(&under).is_ok());
assert_format_properties(&under);
}
#[test]
fn block_edits_on_hostile_bodies_never_panic_and_stay_consistent() {
let mut rng = Rng(0x5851_f42d_4c95_7f2d);
for case in 0..400 {
let len = usize::try_from(rng.next() % 80).unwrap_or(0);
let text = note(&rng.text(len));
let formatted = format_json_block(&text, 0);
let minified = minify_json_block(&text, 0);
if let Ok(edited) = &formatted {
assert_edit_is_consistent(&text, edited);
}
if let Ok(edited) = &minified {
assert_edit_is_consistent(&text, edited);
}
if let (Ok(f), Ok(m)) = (&formatted, &minified) {
assert_eq!(f.text, m.text, "case {case}: formatting and minifying differ");
}
}
// Absurd block indices are errors, never panics; only the fence (index 1) holds JSON.
for index in [0usize, 2, 3, 9, 10_000, usize::MAX] {
let text = note("{\"a\":1}");
assert!(format_json_block(&text, index).is_err(), "index {index}");
assert!(minify_json_block(&text, index).is_err(), "index {index}");
}
// Hostile shapes inside a fence: backticks, new lines, nul, fences of their own.
let bodies: Vec<String> = vec![
"`{\"a\":1}`".to_owned(),
"`{\"a\":${x}}`".to_owned(),
"`{}` + `{}`".to_owned(),
"{\"a\":\"```\"}".to_owned(),
"\u{0}{\"a\":1}".to_owned(),
"{\"a\":\"\\u0000\\uD83D\\uDE00\"}".to_owned(),
"```".to_owned(),
"{\"a\":1}\n{\"b\":2}".to_owned(),
"\u{feff}{\"a\":1}\u{2028}".to_owned(),
"\t{\"a\" : 1}\t".to_owned(),
"[".repeat(500),
format!("[\"{}\"]", "x".repeat(50_000)),
];
for body in &bodies {
let text = note(body);
let before = json_span(body).and_then(|span| serde_json::from_str::<Value>(body.get(span).unwrap_or_default()).ok());
for index in [0usize, 1, usize::MAX] {
if let Ok(edited) = format_json_block(&text, index) {
assert_edit_is_consistent(&text, &edited);
let after_body = json_of(&edited.text);
if let (Some(span), Some(want)) = (json_span(after_body), before.clone()) {
let got = after_body.get(span).and_then(|inner| serde_json::from_str::<Value>(inner).ok());
assert_eq!(got, Some(want), "formatting keeps the value of {body:.20}");
}
}
if let Ok(edited) = minify_json_block(&text, index) {
assert_edit_is_consistent(&text, &edited);
}
}
}
}
/// The body of the first fenced block in `text`.
fn json_of(text: &str) -> &str {
text.split("```").nth(1).unwrap_or_default()
}
crates/editor/Cargo.toml
+1
View File
@@ -17,6 +17,7 @@ workspace = true
notescraft-blocks = { workspace = true }
egui = { workspace = true }
notescraft-markdown = { workspace = true }
notescraft-syntax = { workspace = true }
thiserror = { workspace = true }
serde = { workspace = true }
crates/editor/src/colors.rs
+12
View File
@@ -31,6 +31,8 @@ pub struct EditorColors {
pub accent: Color32,
/// Text selection rectangle (passed through for consistency).
pub selection: Color32,
/// Token colours inside fenced code, in `notescraft_syntax::TokenKind::ALL` order.
pub syntax: [Color32; 8],
}
impl EditorColors {
@@ -51,6 +53,16 @@ impl EditorColors {
highlight: Color32::from_rgba_unmultiplied(0x50, 0x4D, 0x2E, 0xFF),
accent: Color32::from_rgb(0x89, 0xB4, 0xFA),
selection: Color32::from_rgb(0x36, 0x3A, 0x4F),
syntax: [
Color32::from_rgb(0xCB, 0xA6, 0xF7),
Color32::from_rgb(0xA6, 0xE3, 0xA1),
Color32::from_rgb(0xFA, 0xB3, 0x87),
Color32::from_rgb(0x7F, 0x84, 0x9C),
Color32::from_rgb(0xF9, 0xE2, 0xAF),
Color32::from_rgb(0x89, 0xB4, 0xFA),
Color32::from_rgb(0x94, 0xE2, 0xD5),
Color32::from_rgb(0x89, 0xDC, 0xEB),
],
}
}
crates/editor/src/editor.rs
+8
View File
@@ -238,6 +238,14 @@ fn handle_event(state: &mut EditorState, sel: &mut (usize, usize), event: &mut E
}
false // Tab is ours, always
}
Event::Key { key: Key::Backspace, pressed: true, modifiers, .. } if !modifiers.command && !modifiers.alt => {
// Backspace inside an empty pair (`()`, `[]`, `""`) removes both halves.
if let Some(edit) = crate::pairs::on_backspace(&state.text, sel.0..sel.1) {
*edited |= apply_edit(state, sel, &edit);
return false;
}
true
}
Event::Key { key, pressed: true, modifiers, .. } if matches!(key, Key::B | Key::I | Key::K) => {
if !(modifiers.command) {
return true;
crates/editor/src/fence_syntax.rs
+132
View File
@@ -0,0 +1,132 @@
//! Syntax colours inside fenced code in source mode.
//!
//! The fence label picks the grammar; the shared highlight cache parses off the UI thread, so a
//! block that is not ready yet is painted as plain code and gains colour on a later frame.
//! Source mode has no notion of the visible window, so big fences stay plain
//! ([`MAX_FENCE_BYTES`], [`MAX_FENCE_LINES`]); rendered mode colours them window by window.
use std::ops::Range;
use notescraft_syntax::global;
use crate::highlight::Sty;
/// Largest fence body that is coloured in source mode, in bytes.
pub const MAX_FENCE_BYTES: usize = 256 * 1024;
/// Most lines of a fence body that are coloured in source mode.
pub const MAX_FENCE_LINES: usize = 2000;
/// Style pieces covering the whole `fence` range, in order: token pieces in their colour and
/// everything else as plain code. A fence with no usable grammar yields one plain piece.
pub fn fence_assigns(text: &str, fence: &Range<usize>) -> Vec<(Range<usize>, Sty)> {
let plain = || vec![(fence.clone(), Sty::Code)];
let Some(tokens) = tokens(text, fence) else { return plain() };
let mut out = Vec::with_capacity(tokens.len().saturating_mul(2).saturating_add(1));
let mut pos = fence.start;
for (range, kind) in tokens {
if range.start < pos || range.end > fence.end || range.start >= range.end {
continue;
}
if range.start > pos {
out.push((pos..range.start, Sty::Code));
}
pos = range.end;
out.push((range, Sty::Syntax(kind)));
}
if pos < fence.end {
out.push((pos..fence.end, Sty::Code));
}
out
}
/// Absolute token ranges of a fence, sorted and non-overlapping; `None` while not ready or when plain.
fn tokens(text: &str, fence: &Range<usize>) -> Option<Vec<(Range<usize>, notescraft_syntax::TokenKind)>> {
let block = text.get(fence.clone())?;
let (opener, rest) = block.split_once('\n')?;
let marker = opener.trim_start().chars().next().filter(|c| matches!(c, '`' | '~'))?;
let label = opener.trim_start().trim_start_matches(marker).split_whitespace().next()?;
let body_start = fence.start.checked_add(opener.len())?.checked_add(1)?;
let body = body_without_closer(rest, marker);
if body.len() > MAX_FENCE_BYTES || body.bytes().filter(|b| *b == b'\n').count() > MAX_FENCE_LINES {
return None;
}
let lines = body.split_inclusive('\n').count();
let found = global().get(label, body, 0..lines)?;
let mut out = Vec::new();
let mut offset = body_start;
for (index, line) in body.split_inclusive('\n').enumerate() {
let spans = found.first_line.checked_add(0).and_then(|first| index.checked_sub(first)).and_then(|at| found.lines.get(at));
for span in spans.into_iter().flatten() {
out.push((offset.saturating_add(span.start)..offset.saturating_add(span.end), span.kind));
}
offset = offset.saturating_add(line.len());
}
(!out.is_empty()).then_some(out)
}
/// `rest` (everything after the opening line) without a trailing closing-fence line.
fn body_without_closer(rest: &str, marker: char) -> &str {
let last_start = rest.trim_end_matches(['\n', '\r']).rfind('\n').map_or(0, |at| at + 1);
let last = rest.get(last_start..).unwrap_or_default().trim();
let closes = last.len() >= 3 && last.chars().all(|c| c == marker);
if closes { rest.get(..last_start).unwrap_or(rest) } else { rest }
}
#[cfg(test)]
mod tests {
use std::time::{Duration, Instant};
use super::*;
fn wait_for_colour(text: &str, fence: &Range<usize>) -> Vec<(Range<usize>, Sty)> {
let until = Instant::now() + Duration::from_secs(10);
loop {
let pieces = fence_assigns(text, fence);
if pieces.iter().any(|(_, sty)| matches!(sty, Sty::Syntax(_))) || Instant::now() > until {
return pieces;
}
std::thread::sleep(Duration::from_millis(5));
}
}
#[test]
fn a_labelled_fence_gets_token_pieces_that_tile_the_whole_fence() {
let text = "intro\n\n```rust\nfn main() { let n = 42; }\n```\n\nafter\n";
let fences = crate::inline::code_fences(text);
let fence = fences.first().expect("one fence").clone();
let pieces = wait_for_colour(text, &fence);
assert!(pieces.iter().any(|(_, sty)| matches!(sty, Sty::Syntax(_))), "{pieces:?}");
let mut pos = fence.start;
for (range, _) in &pieces {
assert_eq!(range.start, pos, "pieces tile the fence without gaps or overlap");
pos = range.end;
}
assert_eq!(pos, fence.end);
let first_token = pieces.iter().find(|(_, sty)| matches!(sty, Sty::Syntax(_))).expect("token");
assert!(text[first_token.0.clone()].starts_with("fn"), "{:?}", &text[first_token.0.clone()]);
}
#[test]
fn unlabelled_unknown_unclosed_and_huge_fences_stay_plain_or_cover_the_text() {
for text in ["```\nfn main() {}\n```\n", "```nosuchlang\nfn main() {}\n```\n", "```", "```rust"] {
let fences = crate::inline::code_fences(text);
for fence in fences {
let pieces = fence_assigns(text, &fence);
assert_eq!(pieces.first().map(|(r, _)| r.start), Some(fence.start), "{text:?}");
assert_eq!(pieces.last().map(|(r, _)| r.end), Some(fence.end), "{text:?}");
}
}
let big = format!("```rust\n{}\n```\n", "let a = 1;\n".repeat(MAX_FENCE_LINES + 5));
let fence = crate::inline::code_fences(&big).first().expect("fence").clone();
assert_eq!(fence_assigns(&big, &fence), vec![(fence, Sty::Code)]);
}
#[test]
fn an_unclosed_fence_colours_to_the_end_of_the_text() {
let text = "```rust\nlet s = \"x\";\n";
let fence = crate::inline::code_fences(text).first().expect("fence").clone();
let pieces = wait_for_colour(text, &fence);
assert!(pieces.iter().any(|(_, sty)| matches!(sty, Sty::Syntax(_))));
assert_eq!(pieces.last().map(|(r, _)| r.end), Some(text.len()));
}
}
crates/editor/src/highlight.rs
+10 -5
View File
@@ -26,7 +26,7 @@ struct Assign {
/// Style buckets used by the styler.
#[derive(Debug, Clone, Copy, PartialEq)]
enum Sty {
pub(crate) enum Sty {
/// Body text.
Plain,
/// Frontmatter, comments, block ids.
@@ -47,6 +47,8 @@ enum Sty {
TagColor,
/// Callout header and task marker.
Accent,
/// A token inside a fenced code block: monospace in the theme's token colour on the code background.
Syntax(notescraft_syntax::TokenKind),
}
fn heading_size(base: f32, level: u8) -> f32 {
@@ -59,7 +61,7 @@ fn heading_size(base: f32, level: u8) -> f32 {
}
fn text_format(sty: Sty, colors: &EditorColors, base: f32) -> TextFormat {
use Sty::{Accent, Code, Dim, Heading, Highlighted, Link, LinkUnresolved, Muted, Plain, TagColor};
use Sty::{Accent, Code, Dim, Heading, Highlighted, Link, LinkUnresolved, Muted, Plain, Syntax, TagColor};
let prop = FontId::proportional(base);
let mono = FontId::monospace(base);
match sty {
@@ -73,6 +75,7 @@ fn text_format(sty: Sty, colors: &EditorColors, base: f32) -> TextFormat {
LinkUnresolved => fmt(prop, colors.link_unresolved, Color32::TRANSPARENT),
TagColor => fmt(prop, colors.tag, Color32::TRANSPARENT),
Accent => fmt(prop, colors.accent, Color32::TRANSPARENT),
Syntax(kind) => fmt(mono, colors.syntax.get(kind.index()).copied().unwrap_or(colors.code), colors.code_background),
}
}
@@ -138,7 +141,9 @@ fn frontmatter(text: &str) -> Vec<Assign> {
fn own_scans(text: &str) -> Vec<Assign> {
let mut out: Vec<Assign> = Vec::new();
let fences = crate::inline::code_fences(text);
out.extend(fences.iter().cloned().map(|r| Assign { range: r, prio: PRI_FENCE, sty: Sty::Code }));
for fence in &fences {
out.extend(crate::fence_syntax::fence_assigns(text, fence).into_iter().map(|(range, sty)| Assign { range, prio: PRI_FENCE, sty }));
}
let mut skip: Vec<std::ops::Range<usize>> = Vec::new();
let fm = notescraft_markdown::yaml::block_range(text);
skip.extend(fm);
@@ -184,9 +189,9 @@ fn item_assign(item: &notescraft_markdown::Item, resolver: &dyn Fn(&str) -> bool
ItemKind::Comment => (PRI_ITEM_COMMENT, Sty::Muted),
ItemKind::Math { .. } => (PRI_ITEM_MATH, Sty::Code),
ItemKind::Highlight => (PRI_ITEM_OTHER, Sty::Highlighted),
ItemKind::Callout { .. } | ItemKind::TaskItem { .. } => (PRI_ITEM_OTHER, Sty::Accent),
ItemKind::Callout { .. } | ItemKind::TaskItem { .. } | ItemKind::Footnote { .. } | ItemKind::InlineFootnote => (PRI_ITEM_OTHER, Sty::Accent),
ItemKind::Tag { .. } => (PRI_ITEM_OTHER, Sty::TagColor),
ItemKind::BlockId { .. } => (PRI_ITEM_OTHER, Sty::Muted),
ItemKind::BlockId { .. } | ItemKind::FootnoteDef { .. } => (PRI_ITEM_OTHER, Sty::Muted),
ItemKind::WikiLink { target, .. } | ItemKind::Embed { target, .. } => (PRI_ITEM_OTHER, if resolver(target) { Sty::Link } else { Sty::LinkUnresolved }),
ItemKind::Link { dest, internal, .. } => (PRI_ITEM_OTHER, if *internal || resolver(dest) { Sty::Link } else { Sty::LinkUnresolved }),
};
crates/editor/src/inline.rs
+1 -1
View File
@@ -257,7 +257,7 @@ abc",
assert_eq!(spans.len(), 2);
assert_eq!(text_range(text, &spans[0]), "`x y`");
assert_eq!(text_range(text, &spans[1]), "```c```");
assert_eq!(inline_code("`unterminated", &[]), []);
assert_eq!(inline_code("`unterminated", &[]), Vec::<Region>::new());
}
#[test]
crates/editor/src/keys.rs
+36 -22
View File
@@ -7,6 +7,8 @@
use std::fmt::Write as _;
use crate::pairs;
/// Tab produces four spaces.
pub const TAB: &str = " ";
@@ -123,17 +125,30 @@ pub fn wrap_selection(text: &str, sel: std::ops::Range<usize>, open: &str, close
Some(Edit { range: start..end, insert, sel: inner })
}
/// Auto-close pairs and skip-over. `typed` is the character being inserted;
/// the selection must be empty (else `None` and the caller lets `TextEdit` act).
/// Auto-close pairs, wrap the selection and skip over. `typed` is the
/// character being inserted.
///
/// With a selection, an opener (`(`, `[`, `{`, `"`, `` ` ``) wraps it and keeps
/// it selected; `*` (bold is `Cmd-B`/`wrap_selection`) and `'` never wrap.
/// With the caret empty an opener inserts its pair and leaves the caret
/// between, except where a suppression rule applies: `(`, `[` and `{` do not
/// pair when the next character is a word character, `"` and `` ` `` do not
/// pair when the previous character is one (apostrophes and quotes in words
/// type normally), and `'` never pairs. Typing a closer that matches the
/// character at the caret steps over it instead of inserting.
#[must_use]
pub fn auto_close(text: &str, sel: std::ops::Range<usize>, typed: char) -> Option<Edit> {
let (start, end) = norm_sel(text, &sel);
if start != end || !text.is_char_boundary(start) {
return None;
if typed == '\'' {
return None; // apostrophes (`don't`, `it's`) never pair
}
if start != end {
let close = pairs::closer(typed)?;
return wrap_selection(text, start..end, &typed.to_string(), &close.to_string());
}
let at_cursor = text.get(start..).and_then(|t| t.chars().next());
// Skip over an existing closer.
if start > 0 && at_cursor == Some(typed) && matches!(typed, ')' | ']' | '"' | '*' | '`') {
if start > 0 && at_cursor == Some(typed) && matches!(typed, ')' | ']' | '}' | '"' | '*' | '`') {
let step = typed.len_utf8();
return Some(Edit { range: start..start, insert: String::new(), sel: (start + step)..(start + step) });
}
@@ -146,31 +161,30 @@ pub fn auto_close(text: &str, sel: std::ops::Range<usize>, typed: char) -> Optio
}
return None;
}
let (open, close) = pair(typed)?;
if at_cursor.is_some_and(char::is_alphanumeric) && typed == '"' {
return None; // apostrophe-like quoting inside a word
let close = pairs::closer(typed)?;
let prev = text.get(..start).and_then(|t| t.chars().next_back());
if matches!(typed, '(' | '[' | '{') && at_cursor.is_some_and(is_word) {
return None; // mid-word openers do not pair
}
let insert = format!("{open}{close}");
let half = open.len_utf8();
if matches!(typed, '"' | '`') && prev.is_some_and(is_word) {
return None; // quote right after a word does not pair
}
let insert = format!("{typed}{close}");
let half = typed.len_utf8();
Some(Edit { range: start..start, insert, sel: (start + half)..(start + half) })
}
/// The auto-close pair for `typed`, if any. `[` closes with `]`, which also
/// turns `#[[]]` (`[` typed right before the cursor `]` of a previous pair)
/// into the `[[ ]]` wikilink shape naturally.
const fn pair(typed: char) -> Option<(char, char)> {
match typed {
'(' => Some(('(', ')')),
'[' => Some(('[', ']')),
'"' => Some(('"', '"')),
'`' => Some(('`', '`')),
_ => None,
}
/// `true` for word characters (letters, digits, `_`) used by the pairing
/// suppression rules.
fn is_word(c: char) -> bool {
c == '_' || c.is_alphanumeric()
}
// -- shared helpers --------------------------------------------------------
fn norm_sel(text: &str, sel: &std::ops::Range<usize>) -> (usize, usize) {
/// Normalize a selection: floor each end to a char boundary, clamp to the
/// text length and order them. Shared with [`crate::pairs::on_backspace`].
pub(crate) fn norm_sel(text: &str, sel: &std::ops::Range<usize>) -> (usize, usize) {
let floor = |p: usize| {
if text.is_char_boundary(p) { usize::min(p, text.len()) } else { prev_boundary(text, p) }
};
crates/editor/src/lib.rs
+3
View File
@@ -19,8 +19,11 @@ pub mod editor;
pub mod highlight;
pub mod keys;
pub mod marks;
pub mod pairs;
pub mod preview;
pub mod stats;
mod fence_syntax;
mod inline;
pub use notescraft_blocks::{Error, Result};
crates/editor/src/pairs.rs
+50
View File
@@ -0,0 +1,50 @@
//! Auto-pair characters: the opener/closer table shared with
//! [`crate::keys::auto_close`] and the Backspace behaviour for an empty pair.
//!
//! All byte offsets are on char boundaries; positions not on a boundary are
//! floored to one before use (see [`crate::keys`]). No function here panics
//! on hostile input: bad positions are normalized, and anything that is not
//! exactly the shape described returns `None`.
use crate::keys::{Edit, norm_sel};
/// The closer auto-paired with `open` as an opener, if any.
///
/// `*`, `_`, `~`, `=` (emphasis markers) and `'` (apostrophes) are
/// deliberately absent: emphasis wraps through [`crate::keys::wrap_selection`]
/// (only the `**` bold completion lives in [`crate::keys::auto_close`]) and
/// `'` never pairs so `don't` and `it's` type normally.
#[must_use]
pub const fn closer(open: char) -> Option<char> {
match open {
'(' => Some(')'),
'[' => Some(']'),
'{' => Some('}'),
'"' => Some('"'),
'`' => Some('`'),
_ => None,
}
}
/// Backspace between an empty pair (`()`, `[]`, `{}`, two quotes or two
/// backticks) deletes both characters and puts the caret where the opener
/// was.
///
/// Any other position returns `None` and the caller lets `TextEdit` do its
/// normal delete: a non-empty selection, no pair around the caret, or `''`
/// and `**` (not auto-pairs, so their halves are deleted one by one).
#[must_use]
pub fn on_backspace(text: &str, sel: std::ops::Range<usize>) -> Option<Edit> {
let (start, end) = norm_sel(text, &sel);
if start != end || !text.is_char_boundary(start) {
return None;
}
let prev = text.get(..start).and_then(|t| t.chars().next_back())?;
let next = text.get(start..).and_then(|t| t.chars().next())?;
if closer(prev) != Some(next) {
return None; // not an empty pair: prev is no opener or next is not its closer
}
let from = start.checked_sub(prev.len_utf8())?;
let to = start.checked_add(next.len_utf8())?;
Some(Edit { range: from..to, insert: String::new(), sel: from..from })
}
crates/editor/src/preview/color.rs
+382
View File
@@ -0,0 +1,382 @@
//! Hex and functional colour scanning for one line of note text.
use super::{Detected, is_word_byte};
/// Scans one `line` (which starts at byte `base` of the full text) for hex and
/// functional colour literals, appending them to `out` in scan order.
pub(super) fn scan_line(line: &str, base: usize, out: &mut Vec<Detected>) {
let bytes = line.as_bytes();
let mut at = 0usize;
while let Some(byte) = bytes.get(at) {
at = match *byte {
b'#' => hex(line, base, at, out),
b'r' | b'R' | b'h' | b'H' => functional(line, base, at, out),
_ => at + 1,
};
}
}
/// Hex literal at `hash`: one of `#rgb`, `#rgba`, `#rrggbb`, `#rrggbbaa`.
///
/// Digits are ASCII hex. A word character (`[A-Za-z0-9_-]`) right before the `#` or right
/// after the digits disqualifies the candidate (`note#abc`, `#abcdefg`, `#fff_`): such text is
/// an anchor, an issue number or a longer word, not a colour. Returns the index scanning
/// continues from.
fn hex(line: &str, base: usize, hash: usize, out: &mut Vec<Detected>) -> usize {
let bytes = line.as_bytes();
let prev = hash.checked_sub(1).and_then(|at| bytes.get(at).copied());
let prev_word = prev.is_some_and(is_word_byte);
let mut len = 0usize;
let mut value: u64 = 0;
let mut end = hash + 1;
while len < 9 {
match bytes.get(end).copied().and_then(hex_digit) {
Some(digit) => {
value = value * 16 + u64::from(digit);
len += 1;
end += 1;
}
None => break,
}
}
if !matches!(len, 3 | 4 | 6 | 8) || prev_word || bytes.get(end).copied().is_some_and(is_word_byte) {
return hash + 1;
}
let (rgb, alpha) = match len {
3 => ([nibble(value, 8), nibble(value, 4), nibble(value, 0)], 255),
4 => ([nibble(value, 12), nibble(value, 8), nibble(value, 4)], nibble(value, 0)),
6 => ([octet(value, 16), octet(value, 8), octet(value, 0)], 255),
8 => ([octet(value, 24), octet(value, 16), octet(value, 8)], octet(value, 0)),
_ => return hash + 1,
};
out.push(Detected { range: base + hash..base + end, rgb, alpha });
end
}
/// Numeric value of one ASCII hex digit.
fn hex_digit(byte: u8) -> Option<u8> {
match byte {
b'0'..=b'9' => Some(byte - b'0'),
b'a'..=b'f' => Some(byte - b'a' + 10),
b'A'..=b'F' => Some(byte - b'A' + 10),
_ => None,
}
}
/// Short-form digit at `shift * 4` bits, doubled (`a` expands to `0xaa`).
fn nibble(value: u64, shift: u32) -> u8 {
u8::try_from(((value >> shift) & 0xF) * 17).unwrap_or(0)
}
/// Byte at `shift` bits of a long-form literal.
fn octet(value: u64, shift: u32) -> u8 {
u8::try_from((value >> shift) & 0xFF).unwrap_or(0)
}
/// Colour-space kind of a functional literal.
#[derive(Clone, Copy)]
enum Kind {
Rgb,
Hsl,
}
/// Unit suffix allowed after a numeric argument.
#[derive(Clone, Copy, PartialEq)]
enum Unit {
None,
Percent,
Degrees,
}
/// One parsed functional argument: a number plus its unit.
struct Arg {
value: f64,
unit: Unit,
}
/// Functional literal at `start`: `rgb(`, `rgba(`, `hsl(` or `hsla(`
/// (case-insensitive). Malformed or out-of-range candidates produce nothing.
fn functional(line: &str, base: usize, start: usize, out: &mut Vec<Detected>) -> usize {
let bytes = line.as_bytes();
if start.checked_sub(1).and_then(|at| bytes.get(at).copied()).is_some_and(is_word_byte) {
return start + 1;
}
let Some((kind, open)) = kind_at(line, start) else {
return start + 1;
};
let Some((args, alpha, close)) = parse_args(line, open) else {
return start + 1;
};
let Some((rgb, alpha)) = build(kind, &args, alpha.as_ref()) else {
return start + 1;
};
out.push(Detected { range: base + start..base + close, rgb, alpha });
close
}
/// Matches a function name and its `(` at `start`; returns the kind and the
/// index just after the `(`. Longest names are tried first so `rgba(` is not
/// read as `rgb` plus a stray `a`.
fn kind_at(line: &str, start: usize) -> Option<(Kind, usize)> {
const NAMES: [(&str, Kind); 4] = [("rgba", Kind::Rgb), ("hsla", Kind::Hsl), ("rgb", Kind::Rgb), ("hsl", Kind::Hsl)];
for (name, kind) in NAMES {
let open = start + name.len();
let named = line.get(start..open).is_some_and(|head| head.eq_ignore_ascii_case(name)) && line.as_bytes().get(open) == Some(&b'(');
if named {
return Some((kind, open + 1));
}
}
None
}
/// Parses the argument list starting just after `(`. Returns the positional
/// arguments, the `/`-separated alpha (if any) and the index just after the
/// closing `)`. Comma and space both separate; a stray, double or trailing
/// comma is malformed.
fn parse_args(line: &str, start: usize) -> Option<(Vec<Arg>, Option<Arg>, usize)> {
let bytes = line.as_bytes();
let mut at = start;
let mut args: Vec<Arg> = Vec::new();
let mut alpha: Option<Arg> = None;
loop {
at = skip_spaces(bytes, at);
match bytes.get(at).copied() {
Some(b')') => return Some((args, alpha, at + 1)),
Some(b',') => {
at = skip_spaces(bytes, at + 1);
if matches!(bytes.get(at), Some(b',' | b')' | b'/')) {
return None;
}
}
Some(b'/') => {
if args.len() != 3 || alpha.is_some() {
return None;
}
let after = skip_spaces(bytes, at + 1);
let (value, next) = parse_arg(line, after)?;
alpha = Some(value);
at = skip_spaces(bytes, next);
if bytes.get(at) != Some(&b')') {
return None;
}
return Some((args, alpha, at + 1));
}
_ => {
if args.len() == 4 {
return None;
}
let (value, next) = parse_arg(line, at)?;
args.push(value);
at = next;
}
}
}
}
/// Skips spaces and tabs from `at`, returning the next index.
fn skip_spaces(bytes: &[u8], mut at: usize) -> usize {
while matches!(bytes.get(at), Some(b' ' | b'\t')) {
at += 1;
}
at
}
/// Parses one numeric argument with an optional `%` or `deg` unit.
fn parse_arg(line: &str, start: usize) -> Option<(Arg, usize)> {
let bytes = line.as_bytes();
let mut end = start;
while matches!(bytes.get(end), Some(b'0'..=b'9' | b'.' | b'+' | b'-' | b'e' | b'E')) {
end += 1;
}
if end == start {
return None;
}
let value: f64 = line.get(start..end)?.parse().ok()?;
if !value.is_finite() {
return None;
}
let unit = match bytes.get(end).copied() {
Some(b'%') => {
end += 1;
Unit::Percent
}
Some(byte) if byte.is_ascii_alphabetic() => {
let tail = line.get(end..end + 3)?;
if tail.eq_ignore_ascii_case("deg") {
end += 3;
Unit::Degrees
} else {
return None;
}
}
_ => Unit::None,
};
Some((Arg { value, unit }, end))
}
/// Validates the arguments and converts them to 8-bit channels plus alpha;
/// `None` when anything is malformed or out of range.
fn build(kind: Kind, args: &[Arg], alpha: Option<&Arg>) -> Option<([u8; 3], u8)> {
let alpha = match alpha.or_else(|| args.get(3)) {
Some(arg) => alpha_byte(arg)?,
None => 255,
};
let rgb = match kind {
Kind::Rgb => [channel_byte(args.first()?)?, channel_byte(args.get(1)?)?, channel_byte(args.get(2)?)?],
Kind::Hsl => {
let hue = hue_value(args.first()?)?;
let sat = percent_value(args.get(1)?)?;
let light = percent_value(args.get(2)?)?;
hue_to_rgb(hue, sat, light)
}
};
Some((rgb, alpha))
}
/// One `rgb` channel: bare 0-255 or a percentage of 255.
fn channel_byte(arg: &Arg) -> Option<u8> {
match arg.unit {
Unit::None => ranged(arg.value, 255.0).map(to_byte),
Unit::Percent => ranged(arg.value, 100.0).map(|v| to_byte(v / 100.0 * 255.0)),
Unit::Degrees => None,
}
}
/// Saturation or lightness: a percentage 0-100, scaled to 0-1.
fn percent_value(arg: &Arg) -> Option<f64> {
if arg.unit == Unit::Percent { ranged(arg.value, 100.0).map(|v| v / 100.0) } else { None }
}
/// Hue: any finite number, with or without the `deg` unit.
fn hue_value(arg: &Arg) -> Option<f64> {
matches!(arg.unit, Unit::None | Unit::Degrees).then_some(arg.value)
}
/// Alpha: bare 0-1 or a percentage, scaled to 0-255.
fn alpha_byte(arg: &Arg) -> Option<u8> {
match arg.unit {
Unit::None => ranged(arg.value, 1.0).map(|v| to_byte(v * 255.0)),
Unit::Percent => ranged(arg.value, 100.0).map(|v| to_byte(v / 100.0 * 255.0)),
Unit::Degrees => None,
}
}
/// `Some(value)` when it lies in `0.0..=max`.
fn ranged(value: f64, max: f64) -> Option<f64> {
(0.0..=max).contains(&value).then_some(value)
}
/// Rounds a 0-255 channel, clamping float error.
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] // clamped to 0..=255, and the cast saturates
fn to_byte(value: f64) -> u8 {
value.round().clamp(0.0, 255.0) as u8
}
/// CSS hue-to-channel step: `n` is 0 for red, 8 for green and 4 for blue.
fn hue_channel(hue: f64, sat: f64, light: f64, n: f64) -> f64 {
let k = (n + hue / 30.0) % 12.0;
let chroma = sat * light.min(1.0 - light);
light - chroma * (-1.0_f64).max((k - 3.0).min(9.0 - k).min(1.0))
}
/// Converts HSL (hue in degrees, saturation and lightness 0-1) to 8-bit RGB
/// with CSS rounding. The hue wraps, so `400deg` is `40deg`.
fn hue_to_rgb(hue: f64, sat: f64, light: f64) -> [u8; 3] {
let hue = hue.rem_euclid(360.0);
[to_byte(hue_channel(hue, sat, light, 0.0) * 255.0), to_byte(hue_channel(hue, sat, light, 8.0) * 255.0), to_byte(hue_channel(hue, sat, light, 4.0) * 255.0)]
}
#[cfg(test)]
mod tests {
use super::super::{DEFAULT_LIMITS, Detected, detect_colors};
fn one(text: &str) -> Option<Detected> {
detect_colors(text, false, false, DEFAULT_LIMITS).into_iter().next()
}
fn rgba(text: &str) -> Option<([u8; 3], u8)> {
one(text).map(|hit| (hit.rgb, hit.alpha))
}
fn span(text: &str, literal: &str) -> std::ops::Range<usize> {
let start = text.find(literal).unwrap_or(usize::MAX);
start..start.saturating_add(literal.len())
}
#[test]
fn hex_forms() {
let rows: &[(&str, [u8; 3], u8)] = &[
("#fff", [255, 255, 255], 255),
("#abc", [170, 187, 204], 255),
("#f00", [255, 0, 0], 255),
("#1234", [17, 34, 51], 68),
("#abcd", [170, 187, 204], 221),
("#000000", [0, 0, 0], 255),
("#ff0000", [255, 0, 0], 255),
("#ABCDEF", [171, 205, 239], 255),
("#deadbeef", [222, 173, 190], 239),
("#ff000080", [255, 0, 0], 128),
("#00000000", [0, 0, 0], 0),
];
for (text, rgb, alpha) in rows {
assert_eq!(rgba(text), Some((*rgb, *alpha)), "{text}");
}
}
#[test]
fn hex_adjacency_needs_clean_edges() {
let colors: &[&str] = &["#fff.", "(#fff)", "\"#fff\"", "color: #fff;", "é#fff", "#fffé", "##fff", " #fff ", "#fff\r"];
for text in colors {
assert_eq!(rgba(text), Some(([255, 255, 255], 255)), "{text}");
}
}
#[test]
fn hex_rejected() {
let rejected: &[&str] = &[
"",
"#",
"#f",
"#ff",
"#ffffg",
"#12345",
"#1234567",
"#123456789",
"#ggg",
"#fffg",
"x#fff",
"a#fff",
"f#fff",
"#abcdefg",
"note#123",
"x#abcdef",
"a#abcdef",
"word#abc123x",
"_#fff",
"-#fff",
"#fff_",
"#fff-x",
"no colors here",
];
for text in rejected {
assert_eq!(rgba(text), None, "{text}");
}
}
#[test]
fn hex_ranges() {
for (text, literal) in [("color: #ff0000;", "#ff0000"), ("é #abc x", "#abc")] {
assert_eq!(one(text).map(|h| h.range), Some(span(text, literal)), "{text}");
}
assert_eq!(rgba("color: #ffaa00;"), Some(([255, 170, 0], 255)));
}
#[test]
fn hostile_bytes_stay_on_char_boundaries() {
for text in ["\u{0}#fff\u{1}", "😀#abc", "e\u{301}#fff", "#fff\u{2028}"] {
let hit = one(text);
let checked = hit.is_some_and(|h| text.get(h.range).is_some());
assert!(checked, "{text:?}");
}
}
}
crates/editor/src/preview/mod.rs
+260
View File
@@ -0,0 +1,260 @@
//! Pure preview scanners for the editor: colour swatches and tree fences.
//!
//! [`detect_colors`] finds hex and functional (`rgb`/`rgba`/`hsl`/`hsla`)
//! colour literals in note text plus, when asked, the basic named CSS colours.
//! [`detect_tree`] recognises fenced folder-structure blocks. Both are linear,
//! bounded and free of UI types, so `ui-egui` only draws.
mod color;
mod named;
mod tree;
pub use named::NAMED_COLORS;
pub use tree::{TreeRow, detect_tree};
use std::ops::Range;
/// One colour found in note text.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Detected {
/// Byte range of the literal, always on `char` boundaries of the text.
pub range: Range<usize>,
/// Red, green and blue channels, each 0-255.
pub rgb: [u8; 3],
/// Opacity 0-255, where 255 is fully opaque.
pub alpha: u8,
}
/// Bounds that keep detection linear and quick on hostile input.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Limits {
/// Detection stops after this many colours in one note.
pub max_colors: usize,
/// Lines longer than this many bytes are not scanned at all.
pub max_line_bytes: usize,
}
/// Default bounds: 2,000 colours per note, lines up to 4 KiB.
pub const DEFAULT_LIMITS: Limits = Limits { max_colors: 2000, max_line_bytes: 4096 };
/// Finds colour literals in `text`, sorted by position and never overlapping.
///
/// Scanned forms: hex (`#rgb`, `#rgba`, `#rrggbb`, `#rrggbbaa`, digits are
/// ASCII hex) and functional `rgb`/`rgba`/`hsl`/`hsla` (case-insensitive, 3 or
/// 4 arguments, comma or space separated, optional `/ alpha`). A hex literal
/// must not touch a word character (`[A-Za-z0-9_-]`): a non-hex word character
/// before the `#` disqualifies it, a hex-digit neighbour before a short form is
/// tolerated, and a long form ignores what follows its last digit. Functional
/// arguments out of range (such as `256` or `101%`) or malformed give nothing.
///
/// The basic named CSS colours match whole words, case-insensitively, only
/// when `named` is on and `in_code` says the text is inline code.
///
/// Lines longer than [`Limits::max_line_bytes`] are skipped and scanning stops
/// after [`Limits::max_colors`] hits. Ranges are byte ranges on `char`
/// boundaries of `text`.
#[must_use]
pub fn detect_colors(text: &str, named: bool, in_code: bool, limits: Limits) -> Vec<Detected> {
let mut found = Vec::new();
let mut base = 0usize;
for line in text.split('\n') {
if found.len() >= limits.max_colors {
break;
}
if line.len() <= limits.max_line_bytes {
color::scan_line(line, base, &mut found);
if named && in_code {
named::scan_line(line, base, &mut found);
}
}
base = base.saturating_add(line.len()).saturating_add(1);
}
found.sort_by_key(|hit| (hit.range.start, hit.range.end));
if found.len() > limits.max_colors {
found.truncate(limits.max_colors);
}
let mut hits = Vec::with_capacity(found.len());
let mut end = 0usize;
for hit in found {
if hit.range.start >= end {
end = hit.range.end;
hits.push(hit);
}
}
hits
}
/// ASCII characters that continue a word: alphanumerics, `_` and `-`.
fn is_word_byte(byte: u8) -> bool {
byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-'
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn lines_scan_sorted_and_in_order() {
let text = "red #fff\nhsl(0, 100%, 50%) #000";
let hits = detect_colors(text, false, false, DEFAULT_LIMITS);
assert_eq!(hits.len(), 3);
for (hit, literal) in hits.iter().zip(["#fff", "hsl(0, 100%, 50%)", "#000"]) {
let start = text.find(literal).unwrap_or(usize::MAX);
assert_eq!(hit.range, start..start.saturating_add(literal.len()));
}
}
#[test]
fn crlf_lines_scan() {
let text = "#fff\r\n#0f0\r\n";
let hits = detect_colors(text, false, false, DEFAULT_LIMITS);
assert_eq!(hits.len(), 2);
for (hit, literal) in hits.iter().zip(["#fff", "#0f0"]) {
let start = text.find(literal).unwrap_or(usize::MAX);
assert_eq!(hit.range, start..start.saturating_add(literal.len()));
}
}
#[test]
fn multibyte_text_yields_char_boundary_ranges() {
let text = "é #fff 😀 hsl(0, 100%, 50%)";
let hits = detect_colors(text, false, false, DEFAULT_LIMITS);
assert_eq!(hits.len(), 2);
for (hit, literal) in hits.iter().zip(["#fff", "hsl(0, 100%, 50%)"]) {
let start = text.find(literal).unwrap_or(usize::MAX);
assert_eq!(hit.range, start..start.saturating_add(literal.len()));
assert!(text.get(hit.range.clone()).is_some());
}
}
#[test]
fn named_hits_sort_before_hex_in_the_same_line() {
let text = "red #fff";
let hits = detect_colors(text, true, true, DEFAULT_LIMITS);
assert_eq!(hits.len(), 2);
let red = text.find("red").unwrap_or(usize::MAX);
assert_eq!(hits.first().map(|h| h.range.clone()), Some(red..red.saturating_add("red".len())));
}
#[test]
fn long_lines_skipped() {
let tiny = Limits { max_colors: 10, max_line_bytes: 3 };
assert_eq!(detect_colors("#fff", false, false, tiny), Vec::new());
let long_line = format!("{}#fff", "x".repeat(5000));
assert_eq!(detect_colors(&long_line, false, false, DEFAULT_LIMITS), Vec::new());
}
#[test]
fn scanning_stops_at_max_colors() {
let text = "#fff\n".repeat(3000);
let hits = detect_colors(&text, false, false, DEFAULT_LIMITS);
assert_eq!(hits.len(), DEFAULT_LIMITS.max_colors);
}
#[test]
fn hostile_sizes_finish_quickly() {
let started = std::time::Instant::now();
let one_line = format!("{}#fff", "a".repeat(1024 * 1024));
assert_eq!(detect_colors(&one_line, false, false, DEFAULT_LIMITS), Vec::new());
let hashes = "#\n".repeat(100_000);
assert_eq!(detect_colors(&hashes, false, false, DEFAULT_LIMITS), Vec::new());
let one_hash_line = "#".repeat(100_000);
assert_eq!(detect_colors(&one_hash_line, false, false, DEFAULT_LIMITS), Vec::new());
let elapsed = started.elapsed();
assert!(elapsed.as_secs() < 1, "took {elapsed:?}");
}
fn rgba(text: &str) -> Option<([u8; 3], u8)> {
detect_colors(text, false, false, DEFAULT_LIMITS).into_iter().next().map(|hit| (hit.rgb, hit.alpha))
}
#[test]
fn rgb_functional() {
let rows: &[(&str, [u8; 3], u8)] = &[
("rgb(255, 0, 0)", [255, 0, 0], 255),
("RGB(1,2,3)", [1, 2, 3], 255),
("rgba(0, 0, 255, 0.5)", [0, 0, 255], 128),
("rgba(1,2,3)", [1, 2, 3], 255),
("rgb(0 0 0)", [0, 0, 0], 255),
("rgb( 1 , 2 , 3 )", [1, 2, 3], 255),
("rgb(100%, 0%, 0%)", [255, 0, 0], 255),
("rgb(50%,50%,50%)", [128, 128, 128], 255),
("rgb(127.5, 0, 0)", [128, 0, 0], 255),
("rgb(1,2,3,0.25)", [1, 2, 3], 64),
("rgb(1 2 3 / 50%)", [1, 2, 3], 128),
("rgb(1,2,3/0.5)", [1, 2, 3], 128),
("rgb(0,0,0)x", [0, 0, 0], 255),
];
for (text, rgb, alpha) in rows {
assert_eq!(rgba(text), Some((*rgb, *alpha)), "{text}");
}
}
#[test]
fn rgb_rejected() {
let rejected: &[&str] = &[
"rgb(256,0,0)",
"rgb(-1,0,0)",
"rgb(101%,0%,0%)",
"rgb(1,2)",
"rgb(1,2,3,4,5)",
"rgb(1,2,3,)",
"rgb(1,,2,3)",
"rgb()",
"rgb(1,2,3",
"xrgb(1,2,3)",
"argb(1,2,3)",
"rgb(1deg,2,3)",
"rgb (1,2,3)",
"rgb(1,2,3/0.6/0.7)",
"rgb(1.5.5,2,3)",
"rgb(a,2,3)",
];
for text in rejected {
assert_eq!(rgba(text), None, "{text}");
}
}
#[test]
fn hsl_functional() {
let rows: &[(&str, [u8; 3], u8)] = &[
("hsl(0, 100%, 50%)", [255, 0, 0], 255),
("HSL(120deg, 100%, 50%)", [0, 255, 0], 255),
("hsl(240, 100%, 50%)", [0, 0, 255], 255),
("hsl(-30, 100%, 50%)", [255, 0, 128], 255),
("hsl(400, 100%, 50%)", [255, 170, 0], 255),
("hsl(120, 50%, 50%)", [64, 191, 64], 255),
("hsla(0, 100%, 50%, 0.5)", [255, 0, 0], 128),
("hsl(0 100% 50% / 25%)", [255, 0, 0], 64),
("hsl(0,100%,50%)deg", [255, 0, 0], 255),
];
for (text, rgb, alpha) in rows {
assert_eq!(rgba(text), Some((*rgb, *alpha)), "{text}");
}
}
#[test]
fn hsl_rejected() {
let rejected: &[&str] = &[
"hsl(1e999, 100%, 50%)",
"hsl(120, 100, 50%)",
"hsl(0.5turn, 100%, 50%)",
"hsl(0, 100%, 50%, 120)",
"hsl(120, 50%, 50)",
"xhsl(0,100%,50%)",
"hsl(0,101%,50%)",
];
for text in rejected {
assert_eq!(rgba(text), None, "{text}");
}
}
#[test]
fn functional_range_covers_parenthesis() {
let text = "x rgb(0, 0, 0) y";
let start = text.find("rgb(0, 0, 0)").unwrap_or(usize::MAX);
let want = start..start.saturating_add("rgb(0, 0, 0)".len());
let got = detect_colors(text, false, false, DEFAULT_LIMITS).into_iter().next().map(|hit| hit.range);
assert_eq!(got, Some(want));
}
}
crates/editor/src/preview/named.rs
+110
View File
@@ -0,0 +1,110 @@
//! Named CSS colour detection: the basic 16 plus `orange`, whole words only.
use super::{Detected, is_word_byte};
/// The basic CSS colour names (plus `orange`) with their exact CSS values.
pub const NAMED_COLORS: &[(&str, [u8; 3])] = &[
("aqua", [0x00, 0xff, 0xff]),
("black", [0x00, 0x00, 0x00]),
("blue", [0x00, 0x00, 0xff]),
("fuchsia", [0xff, 0x00, 0xff]),
("gray", [0x80, 0x80, 0x80]),
("green", [0x00, 0x80, 0x00]),
("grey", [0x80, 0x80, 0x80]),
("lime", [0x00, 0xff, 0x00]),
("maroon", [0x80, 0x00, 0x00]),
("navy", [0x00, 0x00, 0x80]),
("olive", [0x80, 0x80, 0x00]),
("orange", [0xff, 0xa5, 0x00]),
("purple", [0x80, 0x00, 0x80]),
("red", [0xff, 0x00, 0x00]),
("silver", [0xc0, 0xc0, 0xc0]),
("teal", [0x00, 0x80, 0x80]),
("white", [0xff, 0xff, 0xff]),
("yellow", [0xff, 0xff, 0x00]),
];
/// Scans one `line` (which starts at byte `base` of the full text) for named
/// colours, appending whole-word matches to `out`. The caller gates this on the
/// named-colours setting and the inline-code context; `reddish`, `red-ish` and
/// `bored` never match because `-`, `_` and alphanumerics continue a word.
pub(super) fn scan_line(line: &str, base: usize, out: &mut Vec<Detected>) {
let bytes = line.as_bytes();
let mut at = 0usize;
while let Some(byte) = bytes.get(at) {
if !is_word_byte(*byte) {
at += 1;
continue;
}
let start = at;
while bytes.get(at).copied().is_some_and(is_word_byte) {
at += 1;
}
let word = line.get(start..at).unwrap_or("");
let hit = NAMED_COLORS.iter().find(|(name, _)| word.eq_ignore_ascii_case(name));
if let Some((_, rgb)) = hit {
out.push(Detected { range: base + start..base + at, rgb: *rgb, alpha: 255 });
}
}
}
#[cfg(test)]
mod tests {
use super::super::{DEFAULT_LIMITS, detect_colors};
use super::NAMED_COLORS;
fn hits(text: &str, named: bool, in_code: bool) -> Vec<(std::ops::Range<usize>, [u8; 3])> {
detect_colors(text, named, in_code, DEFAULT_LIMITS).into_iter().map(|hit| (hit.range, hit.rgb)).collect()
}
#[test]
fn table_names_match_with_exact_css_values() {
for (name, rgb) in NAMED_COLORS {
assert_eq!(hits(name, true, true), vec![(0..name.len(), *rgb)], "{name}");
}
}
#[test]
fn names_match_case_insensitive_whole_words() {
let rows: &[(&str, [u8; 3])] = &[
("RED", [255, 0, 0]),
("Blue", [0, 0, 255]),
("fUCHsia", [255, 0, 255]),
("the color red.", [255, 0, 0]),
("(gray)", [128, 128, 128]),
("red and blue", [255, 0, 0]),
];
for (text, rgb) in rows {
let found = hits(text, true, true);
assert_eq!(found.first().map(|(_, channels)| *channels), Some(*rgb), "{text}");
}
}
#[test]
fn word_neighbours_do_not_match() {
let rejected: &[&str] = &["reddish", "red-ish", "red_ish", "bored", "red1", "1red", "redred", "read", "shred", "grayish"];
for text in rejected {
assert_eq!(hits(text, true, true), Vec::new(), "{text}");
}
}
#[test]
fn named_needs_both_switches() {
assert_eq!(hits("red", true, false), Vec::new());
assert_eq!(hits("red", false, true), Vec::new());
assert_eq!(hits("red", false, false), Vec::new());
assert_eq!(hits("red", true, true).len(), 1);
}
#[test]
fn named_offsets_in_multibyte_text() {
let text = "é red 😀 gray";
let found = hits(text, true, true);
assert_eq!(found.len(), 2);
let red = text.find("red").unwrap_or(usize::MAX);
assert_eq!(found.first().map(|(range, _)| range.clone()), Some(red..red.saturating_add("red".len())));
for (range, _) in &found {
assert!(text.is_char_boundary(range.start) && text.is_char_boundary(range.end));
}
}
}
crates/editor/src/preview/tree.rs
+233
View File
@@ -0,0 +1,233 @@
//! Fenced folder-structure (tree) detection.
/// One row of a detected folder tree.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TreeRow {
/// Guide glyphs and the branch connector before the name, no trailing
/// spaces (`├──`, `│ └──`, `|--`, `'--`).
pub prefix: String,
/// Row name, trailing spaces removed.
pub name: String,
/// True for folders: the name ends with `/` or the next row nests deeper.
pub folder: bool,
}
/// Largest fence body scanned, in bytes.
const MAX_BODY_BYTES: usize = 64 * 1024;
/// Largest fence body scanned, in lines.
const MAX_BODY_LINES: usize = 2000;
/// Table corners and joints that never appear in a folder tree.
const TABLE_CHARS: [char; 6] = ['┌', '┐', '┘', '┬', '┴', '┼'];
/// Glyphs that must not appear inside a row name (box drawing and pipe).
const NAME_REJECT: [char; 11] = ['┌', '┐', '┘', '┬', '┴', '┼', '│', '├', '└', '─', '|'];
/// Recognises a folder-structure fence: `label` must be empty, `text` or
/// `tree` (case-insensitive).
///
/// The body needs at least two rows whose prefix is a glyph run of `│ ├ └ ─ |`
/// (or `'` and `` ` ``) followed by dashes — `├── name`, `└── name`,
/// `|-- name`, `'-- name`, `` `-- name `` — plus an optional first root line
/// without glyphs. Box-drawing tables, ASCII art without names and bodies over
/// 64 KiB or 2,000 lines are rejected with `None`. The copy text stays with
/// the caller; the rows never include it.
#[must_use]
pub fn detect_tree(label: &str, body: &str) -> Option<Vec<TreeRow>> {
let label = label.trim();
if !label.is_empty() && !label.eq_ignore_ascii_case("text") && !label.eq_ignore_ascii_case("tree") {
return None;
}
if body.len() > MAX_BODY_BYTES {
return None;
}
let content = body.strip_suffix('\n').unwrap_or(body);
let lines: Vec<&str> = content.split('\n').collect();
if lines.len() > MAX_BODY_LINES {
return None;
}
let mut rows: Vec<(String, String)> = Vec::new();
for line in lines {
let line = line.trim_end();
if line.is_empty() {
continue;
}
if line.contains(TABLE_CHARS) || line.ends_with('│') || line.ends_with('|') {
return None;
}
let first = line.chars().next();
if is_prefix_char(first) {
let (prefix, name) = parse_row(line)?;
rows.push((prefix, name));
} else if rows.is_empty() {
if !name_clean(line) {
return None;
}
rows.push((String::new(), line.to_string()));
} else {
return None;
}
}
if rows.iter().filter(|(prefix, _)| qualifies(prefix)).count() < 2 {
return None;
}
let depths: Vec<usize> = rows.iter().map(|(prefix, _)| prefix.chars().count()).collect();
let mut out = Vec::with_capacity(rows.len());
for (index, (prefix, name)) in rows.iter().enumerate() {
let mine = depths.get(index).copied().unwrap_or(0);
let folder = name.ends_with('/') || depths.get(index + 1).is_some_and(|next| *next > mine);
out.push(TreeRow { prefix: prefix.clone(), name: name.clone(), folder });
}
Some(out)
}
/// Characters that may start a tree row prefix: guides and connectors.
fn is_prefix_char(first: Option<char>) -> bool {
matches!(first, Some('│' | '├' | '└' | '─' | '|' | '\'' | '`' | ' '))
}
/// A prefix carries a tree glyph when it holds a box-drawing character or an
/// ASCII connector (`|--`, `'--`, `` `-- ``); plain dashes do not count.
fn qualifies(prefix: &str) -> bool {
prefix.contains(['│', '├', '└', '─', '|']) || prefix.contains("'--") || prefix.contains("`--")
}
/// Splits a row into its guide `prefix` (dashes included) and `name`.
fn parse_row(line: &str) -> Option<(String, String)> {
let mut at = 0usize;
while let Some(c) = char_at(line, at) {
match c {
'│' | ' ' | '|' | '\'' | '`' | '├' | '└' => at += c.len_utf8(),
_ => break,
}
}
let dash_start = at;
while let Some(c) = char_at(line, at) {
match c {
'─' | '-' => at += c.len_utf8(),
_ => break,
}
}
if at == dash_start {
return None;
}
let prefix = line.get(0..at).unwrap_or("").to_string();
let mut name_start = at;
while char_at(line, name_start) == Some(' ') {
name_start += 1;
}
if name_start == at {
return None;
}
let name = line.get(name_start..).unwrap_or("").trim_end();
if name.is_empty() || !name_clean(name) {
return None;
}
Some((prefix, name.to_string()))
}
/// True when the name holds no drawing or pipe characters.
fn name_clean(name: &str) -> bool {
!name.contains(NAME_REJECT)
}
/// Character at byte `at`, if any.
fn char_at(text: &str, at: usize) -> Option<char> {
text.get(at..).and_then(|rest| rest.chars().next())
}
#[cfg(test)]
mod tests {
use super::{MAX_BODY_BYTES, TreeRow, detect_tree};
fn row(prefix: &str, name: &str, folder: bool) -> TreeRow {
TreeRow { prefix: prefix.into(), name: name.into(), folder }
}
#[test]
fn box_tree_with_root() {
let body = "root\n├── a\n│ └── b\n└── c\n";
let want = vec![row("", "root", true), row("├──", "a", true), row("│ └──", "b", false), row("└──", "c", false)];
assert_eq!(detect_tree("", body), Some(want));
}
#[test]
fn ascii_tree() {
let body = "root\n|-- a\n| `-- b\n'-- c\n";
let want = vec![row("", "root", true), row("|--", "a", true), row("| `--", "b", false), row("'--", "c", false)];
assert_eq!(detect_tree("text", body), Some(want));
}
#[test]
fn backtick_and_single_dash_connectors() {
let body = "`-- a\n|- b\n├- c\n";
let want = vec![row("`--", "a", false), row("|-", "b", false), row("├-", "c", false)];
assert_eq!(detect_tree("", body), Some(want));
}
#[test]
fn tree_without_root_and_with_blank_lines() {
let body = "├── a\n\n└── b\n";
let want = vec![row("├──", "a", false), row("└──", "b", false)];
assert_eq!(detect_tree("tree", body), Some(want));
}
#[test]
fn label_rules() {
let body = "├── a\n└── b\n";
for label in ["", "text", "tree", "TEXT", " Tree "] {
assert!(detect_tree(label, body).is_some(), "label {label}");
}
for label in ["rust", "mermaid", "texty", "ascii-tree"] {
assert_eq!(detect_tree(label, body), None, "label {label}");
}
}
#[test]
fn folders_by_slash_and_depth() {
let body = "root\n├── src/\n│ ├── a\n└── b\n";
let want = vec![row("", "root", true), row("├──", "src/", true), row("│ ├──", "a", false), row("└──", "b", false)];
assert_eq!(detect_tree("", body), Some(want));
}
#[test]
fn rejects_tables_art_and_prose() {
let rejected: &[&str] = &[
"┌───┬───┐\n│ a │ b │\n└───┴───┘\n",
"│ a │ b │\n│ c │ d │\n",
"├───┤\n└───┘\n",
"├── a│\n└── b│\n",
"├── a |\n└── b |\n",
"├── a│b\n└── c\n",
"- a\n- b\n",
"├──\n└── x\n",
"├──x\n└── y\n",
"root\n├── a\nplain\n└── b\n",
"only one line",
"├── a\n│\n└── b\n",
];
for body in rejected {
assert_eq!(detect_tree("", body), None, "{body}");
}
}
#[test]
fn crlf_body() {
let body = "root\r\n├── a\r\n└── b\r\n";
let want = vec![row("", "root", true), row("├──", "a", false), row("└──", "b", false)];
assert_eq!(detect_tree("", body), Some(want));
}
#[test]
fn oversized_bodies_rejected() {
let too_many_lines = "├── x\n".repeat(2001);
assert_eq!(detect_tree("", &too_many_lines), None);
let just_enough = "├── x\n".repeat(2000);
let rows = detect_tree("", &just_enough).map(|rows| rows.len());
assert_eq!(rows, Some(2000));
let mut huge = String::new();
while huge.len() <= MAX_BODY_BYTES {
huge.push_str("├── x\n");
}
assert_eq!(detect_tree("", &huge), None);
}
}
crates/editor/tests/fence_hostile.rs
+152
View File
@@ -0,0 +1,152 @@
//! Hostile fence inputs for `layout_job`: random fence soup, thousands of fences, huge bodies and
//! labels, unclosed fences and mismatched fence lengths never panic, never lose a byte and stay fast.
use std::time::{Duration, Instant};
use notescraft_editor::colors::EditorColors;
use notescraft_editor::highlight::layout_job;
/// Pieces a fence-heavy note is built from; CRLF, wide chars and bare ticks included.
const PIECES: [&str; 13] = ["```", "~~~", "rust", "json", "\n", "`", "é", "{", "}", "\"", " ", "\r\n", " "];
/// How many random documents the sweep builds.
const CASES: u32 = 3000;
/// No call may take longer than this.
const BUDGET: Duration = Duration::from_secs(3);
fn resolver(_target: &str) -> bool {
true
}
/// Tiny deterministic xorshift generator (no rand dependency).
struct Rng(u32);
impl Rng {
fn next(&mut self) -> u32 {
let mut x = self.0;
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
self.0 = x;
x
}
}
/// Layout one document and assert the invariants: same text, contiguous sections, full coverage,
/// no section past the end and no empty-range section between two others.
fn check(name: &str, text: &str, started: Instant) {
let colors = EditorColors::fallback_dark();
let job = layout_job(text, &colors, 14.0, 400.0, &resolver);
assert_eq!(job.text, text, "{name}: the job must carry the source");
let mut next = 0usize;
for section in &job.sections {
let (start, end) = (section.byte_range.start.0, section.byte_range.end.0);
assert!(start <= end, "{name}: section range {start}..{end} is reversed");
assert!(end <= text.len(), "{name}: section ends at {end}, past the text");
assert_eq!(start, next, "{name}: gap or overlap at {next}");
assert!(text.is_char_boundary(start) && text.is_char_boundary(end), "{name}: range {start}..{end} splits a char");
next = end;
}
assert_eq!(next, text.len(), "{name}: coverage incomplete");
let elapsed = started.elapsed();
assert!(elapsed < BUDGET, "{name}: layout took {elapsed:?}, over the {BUDGET:?} budget");
}
#[test]
fn random_fence_soup_keeps_the_text_and_the_time() {
let started = Instant::now();
let mut rng = Rng(0xF3C7_5EED);
for case in 0..CASES {
let pieces = rng.next() as usize % 40 + 1;
let mut text = String::new();
for _ in 0..pieces {
let pick = (rng.next() % u32::try_from(PIECES.len()).unwrap_or(1)) as usize;
if let Some(piece) = PIECES.get(pick) {
text.push_str(piece);
}
}
check(&format!("case {case}"), &text, started);
}
}
#[test]
fn five_thousand_fences_in_a_row() {
let started = Instant::now();
let mut text = String::new();
for n in 0..5000 {
text.push_str("```rust\nfn f");
text.push_str(&n.to_string());
text.push_str("() {}\n```\n");
}
check("5000 fences", &text, started);
}
#[test]
fn a_three_hundred_kilobyte_fence_body() {
let started = Instant::now();
let mut text = String::from("```json\n{\"a\":[");
text.push_str(&"1,".repeat(160_000));
text.push_str("1]}\n```\n");
assert!(text.len() > 300 * 1024, "the fixture must be over 300 KB");
check("300 KB body", &text, started);
}
#[test]
fn a_ten_kilobyte_fence_label() {
let started = Instant::now();
let text = format!("```{}\nbody\n```\n", "r".repeat(10 * 1024));
check("10 KB label", &text, started);
}
#[test]
fn an_unclosed_fence_at_eof() {
let started = Instant::now();
for text in ["```rust\nfn f() {}", "```", "````rust\n```\n````", "~~~json\n{", "\n```\n", "```\r\n"] {
check("unclosed", text, started);
}
}
#[test]
fn mismatched_and_nested_fence_lengths() {
let started = Instant::now();
for text in [
"````rust\n```\ninner\n```\n````\n",
"```rust\n````\n```\n",
"`````\n`````\n",
"``````json\n```\n``````\n",
"~~~rust\n```json\n{x}\n```\n~~~\n",
"``` \n```\n",
"```json\r\n{\"a\":1}\r\n```\r\n",
] {
check("mismatched", text, started);
}
}
#[test]
fn crlf_documents_with_mixed_fences() {
let started = Instant::now();
let mut rng = Rng(0x0BAD_C0DE);
for case in 0..200 {
let pieces = rng.next() as usize % 30 + 1;
let mut text = String::new();
for _ in 0..pieces {
let pick = (rng.next() % u32::try_from(PIECES.len()).unwrap_or(1)) as usize;
if let Some(piece) = PIECES.get(pick) {
text.push_str(piece);
}
}
check(&format!("crlf case {case}"), &text, started);
}
check("lone cr", "# T\r```rust\rfn f() {}\r```\r", started);
}
#[test]
fn documents_of_every_size_around_the_fence_cap() {
let started = Instant::now();
for len in [0_usize, 1, 2, 3, 7, 15, 16, 17, 63, 64, 65, 1023, 1024, 4095] {
let mut text = "```rust\n".repeat(len);
text.push('x');
check(&format!("{len} fences"), &text, started);
}
}
crates/editor/tests/highlight.rs
+41
View File
@@ -116,3 +116,44 @@ fn never_panics_on_degenerate_input() {
assert_eq!(job.text, text);
}
}
#[test]
fn fence_tokens_take_the_theme_syntax_colours_and_cover_the_text() {
use std::time::{Duration, Instant};
// Token colours unlike every other colour, so a match means "painted as a token".
let colors = EditorColors { syntax: [0u8, 1, 2, 3, 4, 5, 6, 7].map(|n| egui::Color32::from_rgb(1, 2, 200 + n)), ..EditorColors::fallback_dark() };
let text = "# T\n\n```rust\nfn main() { let s = \"x\"; } // c\n```\n\nafter `code` here\n";
let until = Instant::now() + Duration::from_secs(10);
let job = loop {
let job = layout_job(text, &colors, 14.0, 400.0, &resolver);
if job.sections.iter().any(|s| colors.syntax.contains(&s.format.color)) || Instant::now() > until {
break job;
}
std::thread::sleep(Duration::from_millis(5));
};
assert_eq!(job.text, text, "the job covers the whole text");
let mut end = 0;
for section in &job.sections {
assert_eq!(section.byte_range.start.0, end, "sections are contiguous");
end = section.byte_range.end.0;
}
assert_eq!(end, text.len());
let coloured: Vec<&str> =
job.sections.iter().filter(|s| colors.syntax.contains(&s.format.color)).map(|s| &text[s.byte_range.start.0..s.byte_range.end.0]).collect();
assert!(coloured.contains(&"fn"), "{coloured:?}");
assert!(coloured.iter().any(|t| t.contains('"')), "{coloured:?}");
assert!(
job.sections.iter().all(|s| s.format.background != egui::Color32::TRANSPARENT || !text[s.byte_range.start.0..s.byte_range.end.0].contains("fn")),
"tokens keep the code background"
);
}
#[test]
fn hostile_fences_do_not_panic() {
let colors = EditorColors::fallback_dark();
let long = format!("```rust\n{}\n```\n", "x".repeat(50_000));
for text in ["```rust\n", "```rust\n`\n```json\n{", "```\u{0}rust\n\u{0}\n```", long.as_str(), "~~~py\nprint('é')\n~~~\n```js\n"] {
let job = layout_job(text, &colors, 14.0, 400.0, &resolver);
assert_eq!(job.text, text);
}
}
crates/editor/tests/pairs.rs
+211
View File
@@ -0,0 +1,211 @@
//! Tables for auto-pairing: pair and suppress rules, wrap the selection,
//! step over closers, and Backspace inside an empty pair. Positions come
//! from `find`/`len`, never hand-counted; `|` marks the caret and `‹…›` a
//! selection in every row, so byte offsets are never written by hand.
use notescraft_editor::keys::{self, Edit};
use notescraft_editor::pairs;
fn applied(text: &str, edit: &Edit) -> String {
let mut out = String::from(text);
out.replace_range(edit.range.clone(), &edit.insert);
out
}
/// Split a marked string into (text, selection): `|` = empty caret, `‹…›` =
/// selection. Offsets come from `find`/`len`, never hand-counted.
fn parse(marked: &str) -> (String, std::ops::Range<usize>) {
if let Some((before, mid, after)) = marked.split_once('‹').and_then(|(b, rest)| rest.split_once('›').map(|(mid, after)| (b, mid, after))) {
let start = before.len();
return (format!("{before}{mid}{after}"), start..start + mid.len());
}
let at = marked.find('|').unwrap_or(marked.len());
(marked.replace('|', ""), at..at)
}
/// One `auto_close` expectation: the edit's applied text and new selection
/// must match the marked expectation, or both must be `None`.
fn check_close(marked: &str, typed: char, expected: Option<&str>) {
let (text, sel) = parse(marked);
let edit = keys::auto_close(&text, sel, typed);
assert_eq!(edit.is_some(), expected.is_some(), "input {marked:?} + {typed}");
if let (Some(edit), Some(exp)) = (edit, expected) {
let (want, want_sel) = parse(exp);
assert_eq!(applied(&text, &edit), want, "input {marked:?} + {typed}");
assert_eq!(edit.sel, want_sel, "input {marked:?} + {typed}");
}
}
/// One `on_backspace` expectation in the same marked format.
fn check_backspace(marked: &str, expected: Option<&str>) {
let (text, sel) = parse(marked);
let edit = pairs::on_backspace(&text, sel);
assert_eq!(edit.is_some(), expected.is_some(), "input {marked:?}");
if let (Some(edit), Some(exp)) = (edit, expected) {
let (want, want_sel) = parse(exp);
assert_eq!(applied(&text, &edit), want, "input {marked:?}");
assert_eq!(edit.sel, want_sel, "input {marked:?}");
}
}
#[test]
fn pair_openers_and_suppression_rules() {
let rows: &[(&str, char, Option<&str>)] = &[
("|", '(', Some("(|)")),
("|", '[', Some("[|]")),
("|", '{', Some("{|}")),
("|", '"', Some("\"|\"")),
("|", '`', Some("`|`")),
("|", '\'', None),
("ab|c", '(', None),
("ab|c", '[', None),
("ab|c", '{', None),
("don|t", '"', None),
("it|s", '\'', None),
("a |b", '"', Some("a \"|\"b")),
("a| b", '(', Some("a(|) b")),
("a| b", '[', Some("a[|] b")),
("a| b", '{', Some("a{|} b")),
("a(|)", ')', Some("a()|")),
("a[|]", ']', Some("a[]|")),
("a{|}", '}', Some("a{}|")),
("a\"|\"", '"', Some("a\"\"|")),
("a`|`", '`', Some("a``|")),
("a*|*", '*', Some("a**|")),
("a|", '*', None),
("a*|", '*', Some("a**|**")),
("(|)", '*', None),
("(|)", '\'', None),
("[|", '[', Some("[[|]")),
("x[|]", '[', Some("x[[|]]")),
("é|", '(', Some("é(|)")),
("éx|", '(', Some("éx(|)")),
("a😀|", '{', Some("a😀{|}")),
("é|x", '"', None),
("don|", '`', None),
("a |", '"', Some("a \"|\"")),
("word|", '"', None),
("foo_|", '"', None),
("word_|", '(', Some("word_(|)")),
("|word", '(', None),
("don\"|\"", '"', Some("don\"\"|")),
];
for (marked, typed, expected) in rows {
check_close(marked, *typed, *expected);
}
}
#[test]
fn selection_wraps_and_stays_selected() {
let rows: &[(&str, char, Option<&str>)] = &[
("say ‹hi›", '[', Some("say [‹hi›]")),
("say ‹hi›", '(', Some("say (‹hi›)")),
("say ‹hi›", '{', Some("say {‹hi›}")),
("say ‹hi›", '"', Some("say \"‹hi›\"")),
("say ‹hi›", '`', Some("say `‹hi›`")),
("say ‹hi›", '*', None),
("say ‹hi›", '_', None),
("say ‹hi›", '~', None),
("say ‹hi›", '=', None),
("say ‹hi›", '\'', None),
("‹é😀›", '(', Some("(‹é😀›)")),
("a‹bc›d", '[', Some("a[‹bc›]d")),
];
for (marked, typed, expected) in rows {
check_close(marked, *typed, *expected);
}
}
#[test]
fn wrap_selection_range_is_the_inner_text() {
let text = "say hi";
let sel = text.find("hi").unwrap()..text.len();
let edit = keys::auto_close(text, sel, '[').unwrap();
let new_text = applied(text, &edit);
assert_eq!(new_text, "say [hi]");
let open = new_text.find('[').unwrap();
let close = new_text.find(']').unwrap();
assert_eq!(edit.sel, open + 1..close);
}
#[test]
fn wrap_multibyte_selection_keeps_char_boundaries() {
let text = "é😀x";
let sel = text.find('😀').unwrap()..text.find('x').unwrap();
let edit = keys::auto_close(text, sel, '(').unwrap();
let new_text = applied(text, &edit);
assert_eq!(new_text, "é(😀)x");
let open = new_text.find('(').unwrap();
let close = new_text.find(')').unwrap();
assert_eq!(edit.sel, open + 1..close);
}
#[test]
fn backspace_deletes_empty_pairs() {
let rows: &[(&str, Option<&str>)] = &[
("(|)", Some("|")),
("[|]", Some("|")),
("{|}", Some("|")),
("\"|\"", Some("|")),
("`|`", Some("|")),
("a(|)b", Some("a|b")),
("a[|]b", Some("a|b")),
("a{|}b", Some("a|b")),
("(\"|\")", Some("(|)")),
("x[[|]]", Some("x[|]")),
("é(|)", Some("é|")),
("😀(|)", Some("😀|")),
("'|'", None),
("**|**", None),
("(|a", None),
("a(|", None),
("ab|", None),
];
for (marked, expected) in rows {
check_backspace(marked, *expected);
}
}
#[test]
fn backspace_ignores_selections_and_hostile_positions() {
let text = "()";
// A selection is deleted by `TextEdit`, not by the pair rule.
assert_eq!(pairs::on_backspace(text, 0..text.len()), None);
// Inverted out-of-bounds selection normalizes to a real selection.
let inverted = usize::from(9u8)..usize::from(0u8);
assert_eq!(pairs::on_backspace(text, inverted), None);
// Caret past the end clamps: no pair around it.
assert_eq!(pairs::on_backspace(text, 9..9), None);
// A caret splitting `é` floors to a boundary and finds no pair.
let hostile = "é()";
assert_eq!(pairs::on_backspace(hostile, 1..1), None);
assert_eq!(pairs::on_backspace(hostile, 2..2), None);
// One character before the pair: single delete, not a pair delete.
assert_eq!(pairs::on_backspace("a()", 1..1), None);
}
#[test]
fn multibyte_caret_lands_on_char_boundary() {
let text = "a😀b";
let at = text.find('b').unwrap();
let edit = keys::auto_close(text, at..at, '"').unwrap();
let new_text = applied(text, &edit);
assert_eq!(new_text, "a😀\"\"b");
let q = new_text.find('"').unwrap();
assert_eq!(edit.sel, q + 1..q + 1);
}
#[test]
fn closer_table() {
assert_eq!(pairs::closer('('), Some(')'));
assert_eq!(pairs::closer('['), Some(']'));
assert_eq!(pairs::closer('{'), Some('}'));
assert_eq!(pairs::closer('"'), Some('"'));
assert_eq!(pairs::closer('`'), Some('`'));
assert_eq!(pairs::closer('\''), None);
assert_eq!(pairs::closer('*'), None);
assert_eq!(pairs::closer('_'), None);
assert_eq!(pairs::closer('~'), None);
assert_eq!(pairs::closer('='), None);
assert_eq!(pairs::closer(')'), None);
}
crates/engine/Cargo.toml
+6
View File
@@ -11,6 +11,7 @@ publish = false
path = "src/lib.rs"
[dependencies]
craft-atomic = { workspace = true }
craft-settings = { workspace = true }
notescraft-vault = { workspace = true }
notescraft-git = { workspace = true }
@@ -25,6 +26,11 @@ notescraft-templates = { workspace = true }
notescraft-calendar = { workspace = true }
notescraft-themes = { workspace = true }
notescraft-plugins = { workspace = true }
notescraft-syntax = { workspace = true }
notescraft-regex = { workspace = true }
grep-regex = { workspace = true }
grep-searcher = { workspace = true }
ignore = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
crates/engine/src/ai/assistant_gate.rs
+15
View File
@@ -94,6 +94,13 @@ pub const WRITE_COMMANDS: &[&str] = &[
"block.duplicate",
"block.delete",
"block.turn_into",
"json.format_block",
"json.minify_block",
"batch.start",
"batch.step",
"batch.cancel",
"batch.decide",
"batch.apply",
];
/// The explicit read allow-list; everything else is treated as a write.
@@ -107,6 +114,8 @@ pub const READ_COMMANDS: &[&str] = &[
"links.unresolved",
"tags.list",
"search",
"switcher.search",
"note.random",
"plugins.list",
"template.list",
"theme.list",
@@ -128,7 +137,13 @@ pub const READ_COMMANDS: &[&str] = &[
"knowledge.related",
"knowledge.status",
"knowledge.rebuild",
"index.pause",
"index.resume",
"batch.status",
"batch.proposals",
"batch.proposal",
"block.list",
"grammar.info",
"lint.check",
"lint.rules",
"lint.state",
crates/engine/src/batch/commands.rs
+388
View File
@@ -0,0 +1,388 @@
//! The `batch.*` commands: start a scope, read it into proposals in steps, list
//! and decide the changes, then apply the accepted ones.
//!
//! Starting and applying are writes (the gate queues them); stepping reads
//! notes only, so it works in a read-only session too. Applying writes each
//! note through [`Session::lint_commit`](crate::session::Session::lint_commit),
//! one edit per note, and skips a note whose text changed since the check.
use std::time::{Duration, Instant};
use serde_json::{Value, json};
use crate::batch::replace::{checks_param, replace_param};
use crate::batch::state::BatchState;
use crate::batch::{Checks, Decision, Proposal, Status, apply_changes, hash_text, propose_note};
use crate::commands::{command_error, flag_param, json, text_param, writable};
use crate::error::CommandError;
use crate::params::{ParamKind, ParamSpec};
use crate::session::Session;
use crate::spec::{CommandResult, CommandSpec};
/// Largest note a batch reads, in bytes; larger notes are reported, not checked.
const MAX_NOTE_BYTES: usize = 2 * 1024 * 1024;
/// Most proposals `batch.proposals` lists in one answer.
const MAX_LIST: usize = 5000;
/// Default milliseconds of reading per `batch.step`.
const DEFAULT_BUDGET_MS: i64 = 8;
/// Smallest `budget_ms` a step accepts.
const MIN_BUDGET_MS: i64 = 1;
/// Largest `budget_ms` a step accepts.
const MAX_BUDGET_MS: i64 = 1000;
/// The `batch.*` commands of the registry.
pub(crate) fn specs() -> Vec<CommandSpec<Session>> {
vec![
CommandSpec::new(
"batch.start",
"Start batch",
"Queues a note, folder or the whole vault for format, lint and replace review; nothing is written",
writable,
start,
)
.with_param(ParamSpec::required("scope", ParamKind::String, "`note`, `folder` or `vault`"))
.with_param(ParamSpec::optional("path", ParamKind::String, "the note, or the folder (required for `note` and `folder`)"))
.with_param(ParamSpec::optional("json", ParamKind::Bool, "format JSON in code blocks, on by default"))
.with_param(ParamSpec::optional("lint", ParamKind::Bool, "turn lint fixes into changes, on by default"))
.with_param(ParamSpec::optional("labels", ParamKind::Bool, "label unlabeled code blocks whose language is detected, on by default"))
.with_param(ParamSpec::optional(
"replace_find",
ParamKind::String,
"plain text to look for; with it and no other check named, the other three default to off",
))
.with_param(ParamSpec::optional("replace_with", ParamKind::String, "what every match becomes, the match itself by default"))
.with_param(ParamSpec::optional("replace_case", ParamKind::Bool, "the find text must match the case, on by default"))
.with_param(ParamSpec::optional("replace_word", ParamKind::Bool, "only match whole words, off by default")),
CommandSpec::new("batch.step", "Batch step", "Reads queued notes into proposals, within a time budget", |_| true, step)
.with_param(ParamSpec::optional("budget_ms", ParamKind::Integer, "1 to 1000 milliseconds of reading, 8 by default")),
CommandSpec::new("batch.cancel", "Cancel batch", "Empties the queue and keeps the proposals read so far", |_| true, cancel),
CommandSpec::new("batch.status", "Batch status", "How far the batch has got", |_| true, status),
CommandSpec::new("batch.proposals", "Batch proposals", "The proposals read so far, optionally filtered by status", |_| true, proposals)
.with_param(ParamSpec::optional("status", ParamKind::String, "`proposed`, `needs_review`, `not_edited` or `clean`")),
CommandSpec::new("batch.proposal", "Batch proposal", "One note's proposal with every change", |_| true, proposal).with_param(ParamSpec::required(
"path",
ParamKind::String,
"note path",
)),
CommandSpec::new("batch.decide", "Decide change", "Accepts or skips one change, or every change of a note", |_| true, decide)
.with_param(ParamSpec::optional("path", ParamKind::String, "note path; absent with `all` decides every proposed note"))
.with_param(ParamSpec::optional("change_id", ParamKind::Integer, "one change of the note, from `batch.proposal`"))
.with_param(ParamSpec::required("decision", ParamKind::String, "`accept`, `skip` or `pending`"))
.with_param(ParamSpec::optional("all", ParamKind::Bool, "with no `path`: accept every change of every proposed note")),
CommandSpec::new("batch.apply", "Apply batch", "Writes the accepted changes, one edit per note", writable, apply),
]
}
fn invalid(message: impl Into<String>) -> CommandError {
CommandError::new("invalid_params", message)
}
/// The wire name of a proposal status.
fn status_name(status: Status) -> &'static str {
match status {
Status::Proposed => "proposed",
Status::NeedsReview => "needs_review",
Status::NotEdited => "not_edited",
Status::Clean => "clean",
}
}
/// The status a `batch.proposals` filter names.
fn parse_status(name: &str) -> Option<Status> {
match name {
"proposed" => Some(Status::Proposed),
"needs_review" => Some(Status::NeedsReview),
"not_edited" => Some(Status::NotEdited),
"clean" => Some(Status::Clean),
_ => None,
}
}
/// Whether a vault path sits in a reserved folder the batch never queues.
fn reserved(path: &str) -> bool {
path.starts_with(".obsidian/") || path.starts_with(".notescraft/")
}
/// The progress object every batch command answers with.
fn progress(session: &Session) -> Value {
let batch = &session.batch;
let count = |status| batch.proposals.iter().filter(|proposal| proposal.status == status).count();
json!({
"total": batch.total,
"done": batch.proposals.len(),
"finished": batch.queue.is_empty(),
"running": batch.running,
"cancelled": batch.cancelled,
"proposed": count(Status::Proposed),
"needs_review": count(Status::NeedsReview),
"not_edited": count(Status::NotEdited),
"clean": count(Status::Clean),
"accepted": batch.proposals.iter().flat_map(|proposal| proposal.changes.iter()).filter(|change| change.decision == Decision::Accept).count(),
})
}
/// `batch.start`: builds the queue from the indexed notes, replacing any
/// previous batch. Nothing is written.
fn start(session: &mut Session, params: &Value) -> CommandResult {
session.require_full_index().map_err(|error| command_error(&error))?;
let scope = text_param(params, "scope")?;
let replace = replace_param(params)?;
let checks = checks_param(params, replace);
let path = params.get("path").and_then(Value::as_str).map(str::to_owned);
let notes = session.index.notes();
let mut queue: Vec<String> = Vec::new();
match scope.as_str() {
"vault" => queue.extend(notes.iter().filter(|note| !reserved(note)).map(|note| (*note).to_owned())),
"folder" => {
let folder = path.ok_or_else(|| invalid("`path` is required for scope `folder`"))?;
let folder = session.file_path(&folder).map_err(|error| command_error(&error))?;
let prefix = format!("{folder}/");
queue.extend(notes.iter().filter(|note| note.starts_with(&prefix) && !reserved(note)).map(|note| (*note).to_owned()));
}
"note" => {
let note = path.ok_or_else(|| invalid("`path` is required for scope `note`"))?;
let note = session.note_path(&note).map_err(|error| command_error(&error))?;
if reserved(&note) || !notes.iter().any(|indexed| *indexed == note) {
return Err(CommandError::new("not_found", format!("`{note}` is not a note of the vault")));
}
queue.push(note);
}
other => return Err(invalid(format!("`scope` is `note`, `folder` or `vault`, not `{other}`"))),
}
queue.reverse(); // `pop` reads the next note in index order
let total = queue.len();
session.batch = BatchState {
queue,
total,
titles: if checks.lint { session.lint_titles() } else { Vec::new() },
checks,
proposals: Vec::new(),
running: true,
cancelled: false,
};
json(json!({ "total": total }))
}
/// The step budget, 1 to 1000 milliseconds, 8 by default.
fn budget_param(params: &Value) -> Result<Duration, CommandError> {
let millis = params.get("budget_ms").and_then(Value::as_i64).unwrap_or(DEFAULT_BUDGET_MS);
if !(MIN_BUDGET_MS..=MAX_BUDGET_MS).contains(&millis) {
return Err(invalid(format!("`budget_ms` must be {MIN_BUDGET_MS} to {MAX_BUDGET_MS} milliseconds")));
}
Ok(Duration::from_millis(u64::try_from(millis).unwrap_or(0)))
}
/// `batch.step`: reads queued notes into proposals until the budget is spent,
/// at least one per call.
fn step(session: &mut Session, params: &Value) -> CommandResult {
let budget = budget_param(params)?;
let checks = session.batch.checks.clone();
let started = Instant::now();
while let Some(path) = session.batch.queue.pop() {
let proposal = read_note(session, &path, &checks);
session.batch.proposals.push(proposal);
if started.elapsed() >= budget {
break;
}
}
json(progress(session))
}
/// Reads one queued note into a proposal, or into a `NotEdited` report when the
/// note cannot be read or checked.
fn read_note(session: &mut Session, path: &str, checks: &Checks) -> Proposal {
let refused =
|reason: &str| Proposal { path: path.to_owned(), content_hash: 0, status: Status::NotEdited, reason: Some(reason.to_owned()), changes: Vec::new() };
let Ok(text) = session.lint_text(path) else {
return refused("cannot read the note");
};
if text.len() > MAX_NOTE_BYTES {
return refused("the note is over 2 MiB");
}
let issues = if checks.lint {
if !session.config.lint.enabled {
return refused("lint is off");
}
let mut titles = std::mem::take(&mut session.batch.titles);
let checked = session.lint_check_with(path, &mut titles);
session.batch.titles = titles;
match checked {
Ok((_, issues)) => issues,
Err(_) => return refused("lint failed"),
}
} else {
Vec::new()
};
propose_note(path, &text, &issues, checks.clone())
}
/// `batch.cancel`: empties the queue, keeps the proposals read.
fn cancel(session: &mut Session, _params: &Value) -> CommandResult {
session.batch.queue.clear();
session.batch.cancelled = true;
session.batch.running = false;
json(progress(session))
}
/// `batch.status`: the progress object, changed by nothing.
fn status(session: &mut Session, _params: &Value) -> CommandResult {
json(progress(session))
}
/// `batch.proposals`: the proposals read so far, optionally by status.
fn proposals(session: &mut Session, params: &Value) -> CommandResult {
let status = match params.get("status").and_then(Value::as_str) {
None => None,
Some(name) => {
Some(parse_status(name).ok_or_else(|| invalid(format!("`status` is `proposed`, `needs_review`, `not_edited` or `clean`, not `{name}`")))?)
}
};
let matched: Vec<&Proposal> = session.batch.proposals.iter().filter(|proposal| status.is_none_or(|wanted| proposal.status == wanted)).collect();
let truncated = matched.len() > MAX_LIST;
let entries: Vec<Value> = matched
.iter()
.take(MAX_LIST)
.map(|proposal| json!({ "path": proposal.path, "status": status_name(proposal.status), "reason": proposal.reason, "changes": proposal.changes.len() }))
.collect();
json(json!({ "proposals": entries, "truncated": truncated }))
}
/// `batch.proposal`: one note's proposal with every change.
fn proposal(session: &mut Session, params: &Value) -> CommandResult {
let path = text_param(params, "path")?;
let found = session
.batch
.proposals
.iter()
.find(|proposal| proposal.path == path)
.ok_or_else(|| CommandError::new("not_found", format!("`{path}` has no proposal yet")))?;
let proposal = found.clone();
let mut value = serde_json::to_value(&proposal).map_err(|error| CommandError::new("not_serialisable", error.to_string()))?;
// Lines are only meaningful against the text that was checked.
if let (Ok(text), Some(changes)) = (session.lint_text(&path), value.get_mut("changes").and_then(Value::as_array_mut))
&& hash_text(&text) == proposal.content_hash
{
for (change, entry) in proposal.changes.iter().zip(changes) {
let line = text.get(..change.range.start).map(|head| head.bytes().filter(|byte| *byte == b'\n').count().saturating_add(1));
if let (Some(line), Some(map)) = (line, entry.as_object_mut()) {
map.insert("line".to_owned(), json!(line));
}
}
}
json(value)
}
/// The `change_id` parameter, an optional non-negative integer.
fn change_id_param(params: &Value) -> Result<Option<usize>, CommandError> {
match params.get("change_id").filter(|raw| !raw.is_null()) {
None => Ok(None),
Some(raw) => raw.as_i64().and_then(|id| usize::try_from(id).ok()).map(Some).ok_or_else(|| invalid("`change_id` must be an integer from 0 up")),
}
}
/// `batch.decide`: one change of a note, every change of a note, or (with
/// `all` and no `path`) every change of every proposed note.
fn decide(session: &mut Session, params: &Value) -> CommandResult {
let decision = match text_param(params, "decision")?.as_str() {
"accept" => Decision::Accept,
"skip" => Decision::Skip,
"pending" => Decision::Pending,
other => return Err(invalid(format!("`decision` is `accept`, `skip` or `pending`, not `{other}`"))),
};
let change_id = change_id_param(params)?;
match params.get("path").and_then(Value::as_str) {
Some(path) => decide_note(session, path, change_id, decision),
None => decide_all(session, params, change_id, decision),
}
}
/// Decides one change (or all of them when `change_id` is absent) of one note.
fn decide_note(session: &mut Session, path: &str, change_id: Option<usize>, decision: Decision) -> CommandResult {
let Some(proposal) = session.batch.proposals.iter_mut().find(|proposal| proposal.path == path) else {
return Err(CommandError::new("not_found", format!("`{path}` has no proposal yet")));
};
let decided = if let Some(id) = change_id {
let Some(change) = proposal.changes.iter_mut().find(|change| change.id == id) else {
return Err(CommandError::new("not_found", format!("`{path}` has no change `{id}`")));
};
change.decision = decision;
1_usize
} else {
let decided = proposal.changes.len();
for change in &mut proposal.changes {
change.decision = decision;
}
decided
};
json(json!({ "path": path, "decided": decided }))
}
/// Accepts every change of every proposed note (only `accept`; needs `all`).
fn decide_all(session: &mut Session, params: &Value, change_id: Option<usize>, decision: Decision) -> CommandResult {
if change_id.is_some() {
return Err(invalid("`change_id` needs a `path`"));
}
if !flag_param(params, "all", false) {
return Err(invalid("`path` is required, unless `all` is true"));
}
if decision != Decision::Accept {
return Err(invalid("`all` only accepts: a whole vault is reviewed change by change otherwise"));
}
let mut decided = 0_usize;
for proposal in &mut session.batch.proposals {
if proposal.status == Status::Proposed {
for change in &mut proposal.changes {
change.decision = decision;
decided = decided.saturating_add(1);
}
}
}
json(json!({ "decided": decided }))
}
/// `batch.apply`: writes the accepted changes of every proposal, one edit per
/// note, skipping notes that changed since the check.
fn apply(session: &mut Session, _params: &Value) -> CommandResult {
let mut written = 0_usize;
let mut skipped: Vec<Value> = Vec::new();
let mut failed: Vec<Value> = Vec::new();
let count = session.batch.proposals.len();
for at in 0..count {
let Some((path, content_hash)) = session.batch.proposals.get(at).map(|proposal| (proposal.path.clone(), proposal.content_hash)) else {
continue;
};
let accepted = session.batch.proposals.get(at).is_some_and(|proposal| proposal.changes.iter().any(|change| change.decision == Decision::Accept));
if !accepted {
continue;
}
let text = match session.lint_text(&path) {
Ok(text) => text,
Err(error) => {
failed.push(json!({ "path": path, "error": error.to_string() }));
continue;
}
};
if hash_text(&text) != content_hash {
skipped.push(json!({ "path": path, "reason": "changed since the check" }));
continue;
}
let Some(new) = session.batch.proposals.get(at).map(|proposal| apply_changes(&text, &proposal.changes).0) else {
continue;
};
if new == text {
continue; // nothing applied: the accepted changes were stale or overlapping
}
match session.lint_commit(&path, &new) {
Ok(_) => {
written = written.saturating_add(1);
if let Some(proposal) = session.batch.proposals.get_mut(at) {
proposal.status = Status::Clean;
proposal.changes.clear();
}
}
Err(error) => failed.push(json!({ "path": path, "error": error.to_string() })),
}
}
session.batch.running = false;
json(json!({ "written": written, "skipped": skipped, "failed": failed }))
}
crates/engine/src/batch/labels.rs
+38
View File
@@ -0,0 +1,38 @@
//! The `labels` batch check: naming the language of unlabeled fenced code blocks.
use notescraft_blocks::{BlockKind, detect_language, parse_blocks};
use crate::batch::propose::change;
use crate::batch::types::{Change, ChangeKind};
/// The first word after the fence run of a fence's opening line, `None` when the fence is unlabeled.
pub(crate) fn fence_label(open: &str) -> Option<&str> {
let c = open.chars().next()?;
let run = open.chars().take_while(|x| *x == c).count();
if run < 3 || !matches!(c, '`' | '~') {
return None;
}
open.get(run..).and_then(|rest| rest.split_whitespace().next())
}
/// One `Label` change per unlabeled fence block whose language [`detect_language`] names, skipping
/// the blocks a JSON change already covers (an unlabeled JSON block keeps its combined change). The
/// change is a zero-length insertion right after the opening fence run and is never safe: the
/// language is a guess the user must approve.
pub(crate) fn label_changes(text: &str, json_blocks: &[usize], out: &mut Vec<Change>) {
for block in parse_blocks(text) {
if block.kind != BlockKind::CodeFence || json_blocks.contains(&block.range.start) {
continue;
}
let Some(open) = text.get(block.range.start..block.text_range.start) else { continue };
if fence_label(open).is_some() {
continue; // already labeled
}
let Some(body) = text.get(block.text_range.clone()) else { continue };
let Some(language) = detect_language(body) else { continue };
let trimmed = open.trim_start();
let run = trimmed.chars().take_while(|c| matches!(c, '`' | '~')).count();
let Some(at) = (block.range.start + (open.len() - trimmed.len())).checked_add(run) else { continue };
out.push(change(ChangeKind::Label, at..at, "", language.to_owned(), false));
}
}
crates/engine/src/batch/mod.rs
+24
View File
@@ -0,0 +1,24 @@
//! Batch review: turning one note's text into a reviewable list of changes.
//!
//! Pure functions only: [`propose_note`] reads a note (plus the lint issues
//! found in it) and returns a [`Proposal`] of non-overlapping changes (JSON
//! formatting, lint fixes, language labels and plain find-and-replace); the
//! user flips each [`Decision`]; [`apply_changes`] splices the accepted ones
//! back into the text. Nothing here touches files.
//!
//! The `batch.*` commands (queueing a scope, stepping through it, deciding and
//! applying) live in [`crate::batch::commands`] on top of this module and of
//! [`crate::batch::state`], the per-session [`BatchState`].
mod commands;
mod labels;
mod propose;
mod replace;
mod state;
mod types;
pub(crate) use commands::specs;
pub(crate) use state::BatchState;
pub use propose::{apply_changes, hash_text, propose_note};
pub use types::{Change, ChangeKind, Checks, Decision, Proposal, ReplaceSpec, Status};
crates/engine/src/batch/propose.rs
+388
View File
@@ -0,0 +1,388 @@
//! Turning one note's text into reviewable changes and applying the accepted ones.
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::ops::Range;
use notescraft_blocks::{Block, BlockKind, check_json, format_json, json_span, parse_blocks};
use notescraft_lint::Issue;
use crate::batch::labels::{fence_label, label_changes};
use crate::batch::replace::replace_changes;
use crate::batch::types::{Change, ChangeKind, Checks, Decision, Proposal, Status};
/// Stable hash of a note's text; a proposal keeps the hash of the text it was read from, so a
/// later mismatch means the note was edited in between.
#[must_use]
pub fn hash_text(text: &str) -> u64 {
let mut hasher = DefaultHasher::new();
text.hash(&mut hasher);
hasher.finish()
}
/// Turns one note's `text` (plus the lint issues found in it) into a [`Proposal`]: JSON blocks that
/// would be reformatted and lint fixes, one [`Change`] each, sorted by range, free of overlaps and
/// `Pending`. Fixes with several edits and stale or invalid edits are skipped, not errors: the
/// batch stays a list of things to review.
///
/// `checks` picks what runs: [`Checks::json`] adds JSON formatting changes, [`Checks::lint`]
/// turns one-edit fixes into changes, [`Checks::labels`] inserts a language label into unlabeled
/// code blocks whose language [`detect_language`] names and [`Checks::replace`] adds one change
/// per match of a plain find string (never safe, so the note needs review).
#[must_use]
pub fn propose_note(path: &str, text: &str, issues: &[Issue], checks: Checks) -> Proposal {
let Checks { json, lint, labels, replace } = checks;
let mut changes = Vec::new();
let mut json_blocks: Vec<usize> = Vec::new();
if json {
json_changes(text, &mut json_blocks, &mut changes);
}
if lint {
lint_changes(text, issues, &mut changes);
}
if labels {
label_changes(text, &json_blocks, &mut changes);
}
if let Some(spec) = replace.as_ref() {
replace_changes(text, spec, &mut changes);
}
changes.sort_by(|a, b| a.range.start.cmp(&b.range.start).then(a.range.end.cmp(&b.range.end)));
let mut kept = Vec::new();
let mut end = 0;
for change in changes {
if change.range.start < end {
continue; // overlaps an earlier kept change: the first one wins
}
end = change.range.end;
kept.push(change);
}
let status = if kept.is_empty() {
Status::Clean
} else if kept.iter().any(|c| !c.safe) {
Status::NeedsReview
} else {
Status::Proposed
};
let changes = kept
.into_iter()
.enumerate()
.map(|(id, mut c)| {
c.id = id;
c
})
.collect();
Proposal { path: path.to_owned(), content_hash: hash_text(text), status, reason: None, changes }
}
/// Applies the changes whose decision is `Accept`, from the highest start to the lowest (spliced
/// ascending, which is the same thing: no change shifts another's range). Returns the new text,
/// the ids applied and the ids skipped. `Accept` changes are skipped when their range is out of
/// bounds, off a `char` boundary, no longer holds `before`, or overlaps another accepted change
/// already taken; `Pending` and `Skip` changes are neither applied nor reported as skipped.
#[must_use]
pub fn apply_changes(text: &str, changes: &[Change]) -> (String, Vec<usize>, Vec<usize>) {
let mut valid = Vec::new();
let mut skipped = Vec::new();
for change in changes.iter().filter(|c| c.decision == Decision::Accept) {
let ok = change.range.start <= change.range.end
&& change.range.end <= text.len()
&& text.is_char_boundary(change.range.start)
&& text.is_char_boundary(change.range.end)
&& text.get(change.range.start..change.range.end) == Some(change.before.as_str());
if ok {
valid.push(change);
} else {
skipped.push(change.id);
}
}
valid.sort_by(|a, b| a.range.start.cmp(&b.range.start).then(a.range.end.cmp(&b.range.end)));
let mut applied = Vec::new();
let mut out = String::with_capacity(text.len());
let mut at = 0;
let mut end = 0;
for change in valid {
if change.range.start < end {
skipped.push(change.id); // overlaps a change already applied
continue;
}
if let Some(seg) = text.get(at..change.range.start) {
out.push_str(seg);
}
out.push_str(&change.after);
at = change.range.end;
end = change.range.end;
applied.push(change.id);
}
if let Some(rest) = text.get(at..) {
out.push_str(rest);
}
applied.sort_unstable();
skipped.sort_unstable();
(out, applied, skipped)
}
/// One fresh, `Pending` change of `kind` at `range`.
pub(crate) fn change(kind: ChangeKind, range: Range<usize>, before: &str, after: String, safe: bool) -> Change {
Change { id: 0, kind, range, before: before.to_owned(), after, rule: None, safe, decision: Decision::Pending }
}
/// The JSON formatting change for one fence block, `None` when the block needs none.
fn json_change(text: &str, block: &Block) -> Option<Change> {
if block.kind != BlockKind::CodeFence {
return None;
}
let open = text.get(block.range.start..block.text_range.start)?;
let body = text.get(block.text_range.clone())?;
let at = block.text_range.start;
match fence_label(open) {
Some(label) if label.eq_ignore_ascii_case("json") => labeled_json(body, at),
// A labeled block of another language (js, rust, ...) is left alone even when it holds JSON.
Some(_) => None,
None => unlabeled_json(body, at),
}
}
/// A `json`-labeled fence: the core is the whole trimmed body; safe to reformat.
fn labeled_json(body: &str, at: usize) -> Option<Change> {
let lead = body.len() - body.trim_start().len();
let core = body.trim();
if check_json(core).is_err() {
return None; // invalid JSON: no change, nothing to review
}
let formatted = format_json(core).ok()?;
if formatted == core {
return None;
}
Some(change(ChangeKind::Json, at + lead..at + lead + core.len(), core, formatted, true))
}
/// An unlabeled fence: only a body whose [`json_span`] is a JSON value is offered, and it needs
/// review, because the block's language is a guess.
fn unlabeled_json(body: &str, at: usize) -> Option<Change> {
let span = json_span(body)?;
let inner = body.get(span.clone())?;
let formatted = format_json(inner).ok()?;
if formatted == inner {
return None;
}
Some(change(ChangeKind::Json, at + span.start..at + span.end, inner, formatted, false))
}
/// Every JSON formatting change of the note's fence blocks; `json_blocks` collects the byte
/// start of each block a change was proposed for, so the label check can leave those alone.
fn json_changes(text: &str, json_blocks: &mut Vec<usize>, out: &mut Vec<Change>) {
for block in parse_blocks(text) {
if let Some(one) = json_change(text, &block) {
json_blocks.push(block.range.start);
out.push(one);
}
}
}
/// One change per lint issue whose fix has exactly one edit. Fixes with several edits are skipped
/// on purpose: the batch reviews one contiguous replacement at a time, and a multi-edit fix cannot
/// be accepted one edit at a time. Edits that no longer match the text are skipped as well.
fn lint_changes(text: &str, issues: &[Issue], out: &mut Vec<Change>) {
for issue in issues {
let Some(fix) = &issue.fix else { continue };
let [edit] = fix.edits.as_slice() else { continue };
let range = edit.range.start..edit.range.end;
let Some(before) = text.get(range.clone()) else { continue };
let mut one = change(ChangeKind::Lint, range, before, edit.text.clone(), fix.safe);
one.rule = Some(issue.rule.clone());
out.push(one);
}
}
#[cfg(test)]
mod tests {
use super::*;
use notescraft_lint::{Edit, Fix, Range as LintRange, Severity};
/// An issue whose fix is one edit replacing `start..end` with `replacement`.
fn lint_issue(rule: &str, start: usize, end: usize, replacement: &str, safe: bool) -> Issue {
Issue {
rule: rule.to_owned(),
severity: Severity::Info,
range: LintRange { start, end },
line: 1,
message: String::new(),
fix: Some(Fix { label: String::new(), edits: vec![Edit { range: LintRange { start, end }, text: replacement.to_owned() }], safe }),
extra_fixes: Vec::new(),
hash: 0,
}
}
#[test]
fn labeled_json_formats_and_is_safe() {
let note = "```json\n{\"a\":1,\"b\":2}\n```\n";
let proposal = propose_note("n.md", note, &[], Checks { json: true, lint: false, labels: false, replace: None });
assert_eq!(proposal.status, Status::Proposed);
assert_eq!(proposal.reason, None);
assert_eq!(proposal.content_hash, hash_text(note));
let c = proposal.changes.first().expect("one change");
let start = note.find("{\"a\":1,\"b\":2}").expect("core in note");
assert_eq!(c.id, 0);
assert_eq!(c.kind, ChangeKind::Json);
assert_eq!(c.range, start..start + "{\"a\":1,\"b\":2}".len());
assert_eq!(c.before, "{\"a\":1,\"b\":2}");
assert_eq!(c.after, "{\n \"a\": 1,\n \"b\": 2\n}");
assert!(c.safe);
assert_eq!(c.rule, None);
assert_eq!(c.decision, Decision::Pending);
}
#[test]
fn invalid_json_yields_no_change_and_other_labels_are_left_alone() {
for note in ["```json\n{\"a\": }\n```\n", "```js\n{\"a\":1}\n```\n", "```json\n{\n \"a\": 1\n}\n```\n"] {
let proposal = propose_note("n.md", note, &[], Checks { json: true, lint: false, labels: false, replace: None });
assert_eq!(proposal.changes, Vec::new(), "{note:?}");
assert_eq!(proposal.status, Status::Clean, "{note:?}");
}
}
#[test]
fn unlabeled_json_needs_review() {
let note = "~~~\n{\"a\":1}\n~~~\n";
let proposal = propose_note("n.md", note, &[], Checks { json: true, lint: false, labels: false, replace: None });
assert_eq!(proposal.status, Status::NeedsReview);
let c = proposal.changes.first().expect("one change");
assert_eq!(c.kind, ChangeKind::Json);
assert!(!c.safe);
let start = note.find("{\"a\":1}").expect("core in note");
assert_eq!(c.range, start..start + "{\"a\":1}".len());
}
#[test]
fn lint_issue_with_one_edit_becomes_a_lint_change() {
let note = "a b\n";
let start = note.find(" ").expect("two spaces");
let issue = lint_issue("double-space", start, start + 2, " ", true);
let proposal = propose_note("n.md", note, std::slice::from_ref(&issue), Checks { json: false, lint: true, labels: false, replace: None });
assert_eq!(proposal.status, Status::Proposed);
let c = proposal.changes.first().expect("one change");
assert_eq!(c.kind, ChangeKind::Lint);
assert_eq!(c.rule, Some("double-space".to_owned()));
assert_eq!((c.before.as_str(), c.after.as_str()), (" ", " "));
assert!(c.safe);
}
#[test]
fn multi_edit_and_stale_fixes_are_skipped() {
let note = "a b\n";
let start = note.find(" ").expect("two spaces");
let mut multi = lint_issue("multi", start, start + 2, " ", true);
if let Some(fix) = &mut multi.fix {
fix.edits.push(Edit { range: LintRange { start: start + 2, end: start + 3 }, text: String::new() });
}
let stale = lint_issue("stale", note.len() + 5, note.len() + 7, " ", true);
let proposal = propose_note("n.md", note, &[multi, stale], Checks { json: false, lint: true, labels: false, replace: None });
assert_eq!(proposal.changes, Vec::new());
assert_eq!(proposal.status, Status::Clean);
}
#[test]
fn overlapping_change_is_dropped_and_ids_stay_sequential() {
let note = "```json\n{\"a\":1,\"b\":2}\n```\n\n```json\n{\"c\":3}\n```\n";
let inside = note.find("\"b\":2").expect("inside the first core");
let issue = lint_issue("inside", inside, inside + 3, "x", false);
let proposal = propose_note("n.md", note, std::slice::from_ref(&issue), Checks { json: true, lint: true, labels: false, replace: None });
assert_eq!(proposal.changes.len(), 2, "the lint change inside the JSON core is dropped");
assert_eq!(proposal.changes.iter().map(|c| c.id).collect::<Vec<_>>(), vec![0, 1]);
assert_eq!(proposal.changes.iter().map(|c| c.kind).collect::<Vec<_>>(), vec![ChangeKind::Json, ChangeKind::Json]);
}
#[test]
fn clean_note_is_clean() {
let proposal = propose_note("n.md", "# Title\n\nplain note\n", &[], Checks { json: true, lint: true, labels: false, replace: None });
assert_eq!(proposal.status, Status::Clean);
assert_eq!(proposal.changes, Vec::new());
}
#[test]
fn apply_changes_accepts_and_skips() {
let note = "```json\n{\"a\":1}\n```\n";
let mut proposal = propose_note("n.md", note, &[], Checks { json: true, lint: false, labels: false, replace: None });
for c in &mut proposal.changes {
c.decision = Decision::Accept;
}
let (new, applied, skipped) = apply_changes(note, &proposal.changes);
assert_eq!(applied, vec![0]);
assert_eq!(skipped, Vec::<usize>::new());
assert_eq!(new, "```json\n{\n \"a\": 1\n}\n```\n");
for c in &mut proposal.changes {
c.decision = Decision::Skip;
}
let (new, applied, skipped) = apply_changes(note, &proposal.changes);
assert_eq!(new, note);
assert_eq!((applied, skipped), (Vec::new(), Vec::new()), "skipped decisions are not reported");
for c in &mut proposal.changes {
c.decision = Decision::Pending;
}
let (new, applied, skipped) = apply_changes(note, &proposal.changes);
assert_eq!((new, applied, skipped), (note.to_owned(), Vec::new(), Vec::new()), "pending decisions are not applied");
}
#[test]
fn apply_changes_skips_stale_and_overlapping() {
let note = "one two\n";
let at = note.find(" ").expect("two spaces");
let stale = change(ChangeKind::Lint, at..at + 3, " ", " ".to_owned(), true); // `before` does not match
let accepted = |mut c: Change, decision: Decision| {
c.decision = decision;
c
};
let (new, applied, skipped) = apply_changes(note, &[accepted(stale.clone(), Decision::Accept)]);
assert_eq!((new, applied, skipped), (note.to_owned(), Vec::new(), vec![stale.id]));
let first = change(ChangeKind::Lint, at..at + 2, " ", " ".to_owned(), true);
let second = change(ChangeKind::Lint, at + 1..at + 3, " t", "x".to_owned(), true);
let (new, applied, skipped) = apply_changes(note, &[accepted(first.clone(), Decision::Accept), accepted(second.clone(), Decision::Accept)]);
assert_eq!(applied, vec![first.id]);
assert_eq!(skipped, vec![second.id]);
assert_eq!(new, "one two\n");
}
#[test]
fn apply_changes_handles_multibyte() {
let note = "caf\u{e9} x\n";
let at = note.find("\u{e9}").expect("e-acute");
let mut ok = change(ChangeKind::Lint, at..at + 2, "\u{e9}", "e".to_owned(), true);
ok.decision = Decision::Accept;
let (new, applied, skipped) = apply_changes(note, std::slice::from_ref(&ok));
assert_eq!((new, applied, skipped), ("cafe x\n".to_owned(), vec![ok.id], Vec::new()));
let mut mid = change(ChangeKind::Lint, at + 1..at + 2, "junk", "e".to_owned(), true); // splits the two-byte char
mid.decision = Decision::Accept;
let (new, applied, skipped) = apply_changes(note, std::slice::from_ref(&mid));
assert_eq!((new, applied, skipped), (note.to_owned(), Vec::new(), vec![mid.id]));
}
#[test]
fn a_mib_note_with_5000_changes_stays_fast() {
let mut note = String::new();
let mut issues = Vec::new();
for _ in 0..5000 {
let start = note.len();
note.push_str(&"x".repeat(210));
note.push_str(" word\n");
let at = note.rfind(" word\n").expect("the fresh pair of spaces");
issues.push(lint_issue("double-space", at, at + 2, " ", true));
debug_assert_eq!(start + 210, at);
}
debug_assert!(note.len() >= 1024 * 1024);
let began = std::time::Instant::now();
let mut proposal = propose_note("big.md", &note, &issues, Checks { json: true, lint: true, labels: false, replace: None });
assert_eq!(proposal.changes.len(), 5000);
for c in &mut proposal.changes {
c.decision = Decision::Accept;
}
let (new, applied, skipped) = apply_changes(&note, &proposal.changes);
assert_eq!(applied.len(), 5000);
assert_eq!(skipped, Vec::<usize>::new());
assert!(new.len() < note.len());
assert!(began.elapsed().as_secs_f64() < 2.0, "took {}s", began.elapsed().as_secs_f64());
}
}
crates/engine/src/batch/replace.rs
+111
View File
@@ -0,0 +1,111 @@
//! The `replace` batch check: one change per match of a plain find string.
use notescraft_blocks::{FindOptions, find_all};
use serde_json::Value;
use crate::batch::propose::change;
use crate::batch::types::{Change, ChangeKind, Checks, ReplaceSpec};
use crate::commands::flag_param;
use crate::error::CommandError;
/// Largest find string a replace check takes, in bytes.
pub(crate) const MAX_FIND_BYTES: usize = 500;
/// Most changes one note gets from a replace check; the rest wait for a narrower find.
const MAX_CHANGES: usize = 2000;
/// The replace check the `batch.start` params ask for, `None` when `replace_find` is absent or
/// empty. The registry has already refused a param of the wrong type.
///
/// # Errors
/// [`CommandError`] `invalid_params` when `replace_find` is over [`MAX_FIND_BYTES`].
pub(crate) fn replace_param(params: &Value) -> Result<Option<ReplaceSpec>, CommandError> {
let find = params.get("replace_find").and_then(Value::as_str).unwrap_or_default();
if find.is_empty() {
return Ok(None);
}
if find.len() > MAX_FIND_BYTES {
return Err(CommandError::new("invalid_params", format!("`replace_find` is at most {MAX_FIND_BYTES} bytes")));
}
Ok(Some(ReplaceSpec {
find: find.to_owned(),
with: params.get("replace_with").and_then(Value::as_str).unwrap_or_default().to_owned(),
case_sensitive: flag_param(params, "replace_case", true),
whole_word: flag_param(params, "replace_word", false),
}))
}
/// The checks a batch starts with. The format and lint checks are on by default, unless the run
/// asks for a replace and names no other check: a replace run then changes matches and nothing else.
pub(crate) fn checks_param(params: &Value, replace: Option<ReplaceSpec>) -> Checks {
let named = ["json", "lint", "labels"].iter().any(|name| params.get(*name).is_some_and(|value| !value.is_null()));
let default = replace.is_none() || named;
Checks { json: flag_param(params, "json", default), lint: flag_param(params, "lint", default), labels: flag_param(params, "labels", default), replace }
}
/// One `Replace` change per match of `spec.find`, everywhere in the note, the frontmatter block
/// included. A change is never safe: replacing text always needs a human look. Nothing is
/// proposed when the find string is empty or over [`MAX_FIND_BYTES`].
pub(crate) fn replace_changes(text: &str, spec: &ReplaceSpec, out: &mut Vec<Change>) {
if spec.find.is_empty() || spec.find.len() > MAX_FIND_BYTES {
return;
}
let options = FindOptions { case_sensitive: spec.case_sensitive, whole_word: spec.whole_word };
for range in find_all(text, &spec.find, options).into_iter().take(MAX_CHANGES) {
let Some(before) = text.get(range.clone()) else { continue };
out.push(change(ChangeKind::Replace, range, before, spec.with.clone(), false));
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::batch::propose::propose_note;
use crate::batch::types::{Checks, Status};
/// A replace check with the given switches, nothing else.
fn checks(find: &str, with: &str, case_sensitive: bool, whole_word: bool) -> Checks {
Checks {
json: false,
lint: false,
labels: false,
replace: Some(ReplaceSpec { find: find.to_owned(), with: with.to_owned(), case_sensitive, whole_word }),
}
}
#[test]
fn every_match_becomes_one_change_that_needs_review() {
let note = "---\ntitle: tea\n---\n\ntea time\n";
let proposal = propose_note("n.md", note, &[], checks("tea", "coffee", true, false));
assert_eq!(proposal.status, Status::NeedsReview);
assert_eq!(proposal.changes.len(), 2, "the frontmatter match is offered too");
for change in &proposal.changes {
assert_eq!(change.kind, ChangeKind::Replace);
assert_eq!(change.before, "tea");
assert_eq!(change.after, "coffee");
assert_eq!(change.rule, None);
assert!(!change.safe);
}
let front = note.find("tea").expect("the first match");
assert_eq!(proposal.changes.first().map(|c| c.range.clone()), Some(front..front + 3));
}
#[test]
fn an_empty_or_huge_find_proposes_nothing() {
let empty = propose_note("n.md", "tea\n", &[], checks("", "x", true, false));
assert_eq!(empty.changes, Vec::new());
assert_eq!(empty.status, Status::Clean);
let huge = "a".repeat(MAX_FIND_BYTES + 1);
let long = propose_note("n.md", &format!("{huge}\n"), &[], checks(&huge, "x", true, false));
assert_eq!(long.changes, Vec::new());
let exact = "a".repeat(MAX_FIND_BYTES);
let edge = propose_note("n.md", &exact, &[], checks(&exact, "x", true, false));
assert_eq!(edge.changes.len(), 1, "500 bytes is still accepted");
}
#[test]
fn one_note_never_gets_more_than_the_cap() {
let note = "x ".repeat(3000);
let proposal = propose_note("n.md", &note, &[], checks("x", "y", true, false));
assert_eq!(proposal.changes.len(), MAX_CHANGES);
}
}
crates/engine/src/batch/state.rs
+40
View File
@@ -0,0 +1,40 @@
//! The state one format/lint batch keeps between its commands.
use crate::batch::{Checks, Proposal};
/// The batch state of a session: the notes still to read, the proposals
/// already read and the checks the batch runs with.
///
/// [`BatchState::queue`] is reversed: [`Vec::pop`] takes the last element, so
/// the batch reads the notes in the order the index lists them.
#[derive(Debug)]
pub(crate) struct BatchState {
/// Paths still to read, reversed so `pop` returns the next one.
pub(crate) queue: Vec<String>,
/// Notes queued when the batch started.
pub(crate) total: usize,
/// Which checks the batch runs on each note.
pub(crate) checks: Checks,
/// One proposal per read note, in reading order.
pub(crate) proposals: Vec<Proposal>,
/// Whether a batch is open (started, not cancelled or applied).
pub(crate) running: bool,
/// Whether the user cancelled the batch; the proposals read stay.
pub(crate) cancelled: bool,
/// `(path, title)` of every note, built once per batch for lint's link rules.
pub(crate) titles: Vec<(String, String)>,
}
impl Default for BatchState {
fn default() -> Self {
Self {
queue: Vec::new(),
total: 0,
checks: Checks { json: false, lint: false, labels: false, replace: None },
proposals: Vec::new(),
running: false,
cancelled: false,
titles: Vec::new(),
}
}
}
crates/engine/src/batch/types.rs
+106
View File
@@ -0,0 +1,106 @@
//! Types of a review batch: one note, its proposed changes and the user's decisions.
use std::ops::Range;
use serde::Serialize;
/// Which check produced a change.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub enum ChangeKind {
/// Pretty-printing of JSON in a code block.
Json,
/// A lint rule's fix.
Lint,
/// A language label inserted after the opening fence of an unlabeled code block.
Label,
/// One match of a plain find string, replaced by what the user asked for.
Replace,
}
/// The user's decision on a change.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub enum Decision {
/// Not decided yet.
Pending,
/// Apply it.
Accept,
/// Leave the text as it is.
Skip,
}
/// How a note's proposal ended.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub enum Status {
/// Changes proposed, all safe.
Proposed,
/// At least one change needs a human look (not safe).
NeedsReview,
/// The note changed after it was read, so nothing was applied.
NotEdited,
/// Nothing to change.
Clean,
}
/// One reviewable change: replace `before` at `range` with `after`.
///
/// `range` is a byte range of the text that was read.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Change {
/// Position in the batch, `0..` in final order.
pub id: usize,
/// Where the change comes from.
pub kind: ChangeKind,
/// Byte range of the old text.
pub range: Range<usize>,
/// The old text, as read.
pub before: String,
/// The replacement.
pub after: String,
/// Lint rule id, when the change comes from lint.
pub rule: Option<String>,
/// True when the change cannot alter meaning.
pub safe: bool,
/// The user's decision.
pub decision: Decision,
}
/// All changes proposed for one note.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Proposal {
/// Vault-relative path of the note.
pub path: String,
/// [`crate::batch::hash_text`] of the text the changes were read from.
pub content_hash: u64,
/// How the proposal ended.
pub status: Status,
/// Why the status is what it is.
pub reason: Option<String>,
/// The changes, sorted by range and without overlaps.
pub changes: Vec<Change>,
}
/// What a replace check looks for and what it writes instead.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct ReplaceSpec {
/// The plain text to look for; never empty, at most 500 bytes.
pub find: String,
/// What every match becomes (may be empty).
pub with: String,
/// Upper and lower case must match as written.
pub case_sensitive: bool,
/// Only match when neither neighbour is a letter, a digit or `_`.
pub whole_word: bool,
}
/// Which checks run in a batch.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Checks {
/// Format JSON in code blocks.
pub json: bool,
/// Turn lint fixes into changes.
pub lint: bool,
/// Label unlabeled code blocks whose language is detected.
pub labels: bool,
/// Turn every match of a plain find string into a change, when asked for.
pub replace: Option<ReplaceSpec>,
}
crates/engine/src/block_edit/mod.rs
+23 -2
View File
@@ -1,9 +1,10 @@
//! The `block.*` commands: list the blocks of a note and move, duplicate, delete or retype one.
//! The `block.*` commands: list the blocks of a note and move, duplicate, delete or retype one; plus
//! `json.format_block` and `json.minify_block` for the JSON inside a code block.
//!
//! The pure edits live in `notescraft-blocks`. Each write is one replacement, so an open buffer takes it
//! as one undo step; the assistant gate queues the writes like any other edit.
use notescraft_blocks::{Direction, Edited, Target, delete_block, duplicate_block, move_block, parse_blocks, turn_into};
use notescraft_blocks::{Direction, Edited, Target, delete_block, duplicate_block, format_json_block, minify_json_block, move_block, parse_blocks, turn_into};
use serde_json::{Value, json};
use crate::commands::{command_error, json, text_param, writable};
@@ -34,6 +35,18 @@ pub(crate) fn specs() -> Vec<CommandSpec<Session>> {
ParamKind::String,
"`paragraph`, `heading1` to `heading6`, `quote`, `task`, `bullet`, `numbered` or `callout:<kind>`",
)),
CommandSpec::new("json.format_block", "Format JSON block", "Pretty-prints the JSON in a code block as one undoable edit", writable, format_json)
.with_param(path())
.with_param(index()),
CommandSpec::new(
"json.minify_block",
"Minify JSON block",
"Removes the spaces of the JSON in a code block as one undoable edit",
writable,
minify_json,
)
.with_param(path())
.with_param(index()),
]
}
@@ -105,6 +118,14 @@ fn delete(session: &mut Session, params: &Value) -> CommandResult {
edit(session, params, delete_block)
}
fn format_json(session: &mut Session, params: &Value) -> CommandResult {
edit(session, params, format_json_block)
}
fn minify_json(session: &mut Session, params: &Value) -> CommandResult {
edit(session, params, minify_json_block)
}
fn turn(session: &mut Session, params: &Value) -> CommandResult {
let target = target_param(params)?;
edit(session, params, |text, index| turn_into(text, index, &target))
crates/engine/src/commands.rs
+5
View File
@@ -28,11 +28,16 @@ pub fn register_core(reg: &mut Registry<Session>) -> Result<()> {
.chain(crate::attachment_commands::specs())
.chain(crate::folder_commands::specs())
.chain(crate::search::search_specs())
.chain(crate::switcher::switcher_specs())
.chain(crate::random_unique::specs())
.chain(crate::indexing::specs())
.chain(crate::daily::daily_specs())
.chain(crate::calendar::specs())
.chain(crate::knowledge::specs())
.chain(crate::lint::specs())
.chain(crate::block_edit::specs())
.chain(crate::grammars::specs())
.chain(crate::batch::specs())
.chain(crate::themes::theme_specs())
.chain(crate::settings::settings_specs())
.chain(crate::plugin_commands::plugin_specs())
crates/engine/src/error.rs
+9
View File
@@ -146,6 +146,14 @@ pub enum Error {
/// One message per note that could not be rewritten.
failures: Vec<String>,
},
/// The opening scan has not finished, so an operation that needs the whole index is refused.
#[error("still indexing the vault ({done} of {total} notes read); try again when it finishes")]
Indexing {
/// Notes handled so far.
done: usize,
/// Notes found when the vault was opened.
total: usize,
},
/// The session configuration is malformed: an empty format, a path with
/// `..`, an absolute path or a reserved folder.
#[error("invalid configuration: {0}")]
@@ -220,6 +228,7 @@ impl Error {
Self::Vault(_) => "vault_error",
Self::Index(_) => "index_error",
Self::RenameIncomplete { .. } => "rename_incomplete",
Self::Indexing { .. } => "indexing",
Self::Config(_) => "config",
Self::Query(_) => "invalid_query",
Self::AttachmentTooLarge { .. } => "attachment_too_large",
crates/engine/src/grammars/mod.rs
+249
View File
@@ -0,0 +1,249 @@
//! The `grammar.*` commands: look up, install and remove syntax grammars.
//!
//! Installing runs every gate in `notescraft-syntax` before anything is written: https-only link, size,
//! format and parse always; registry host, checksum and sample only for registry entries (a pasted link has
//! none to check against); a failure saves nothing and changes no state.
//! The file goes to the `grammars` folder next to the settings through `craft-atomic`.
use std::path::{Path, PathBuf};
use std::time::{SystemTime, UNIX_EPOCH};
use notescraft_syntax::fetch::{Request, Transport, download_grammar};
use notescraft_syntax::validate::check_grammar;
use notescraft_syntax::validate::check_url;
use notescraft_syntax::{Error as SyntaxError, load, suggest_language};
use serde_json::{Value, json};
use crate::app_settings::{GrammarInfo, MAX_GRAMMARS, is_grammar_id};
use crate::commands::{json, text_param, writable};
use crate::error::CommandError;
use crate::params::{ParamKind, ParamSpec};
use crate::session::Session;
use crate::settings::{load_for, save_for};
use crate::spec::{CommandResult, CommandSpec};
/// Host of the grammar registry; registry links must be on it.
pub const REGISTRY_HOST: &str = "git.marcospaulo.dev.br";
/// Folder the registry serves raw files from.
const REGISTRY_RAW: &str = "https://git.marcospaulo.dev.br/masi/notescraft-registry/raw/branch/main/";
/// Largest registry index, in bytes.
const MAX_INDEX_BYTES: usize = 256 * 1024;
/// Longest sample text taken from the index, in characters.
const MAX_SAMPLE_BYTES: usize = 16 * 1024;
/// The only grammar format the registry may list.
const FORMAT: &str = "sublime-syntax";
pub(crate) fn specs() -> Vec<CommandSpec<Session>> {
vec![
CommandSpec::new("grammar.info", "Grammar info", "Installed grammars, and for a fence label whether it is known and the closest name", |_| true, info)
.with_param(ParamSpec::optional("label", ParamKind::String, "a fence label such as `pyhton`")),
CommandSpec::new(
"grammar.install",
"Install grammar",
"Downloads a syntax grammar from the registry (by id) or from a link, checks it, then saves it",
writable,
install,
)
.with_param(ParamSpec::required("id", ParamKind::String, "grammar id: lowercase letters, digits and `-`"))
.with_param(ParamSpec::optional(
"url",
ParamKind::String,
"https link to a .sublime-syntax file; without it the registry entry with this id is used",
)),
CommandSpec::new("grammar.install_text", "Install grammar from text", "Saves grammar source the caller fetched and checked", writable, install_text)
.with_param(ParamSpec::required("id", ParamKind::String, "grammar id"))
.with_param(ParamSpec::required("text", ParamKind::String, "the .sublime-syntax source"))
.with_param(ParamSpec::optional("license", ParamKind::String, "license name")),
CommandSpec::new("grammar.remove", "Remove grammar", "Deletes an installed grammar", writable, remove).with_param(ParamSpec::required(
"id",
ParamKind::String,
"installed grammar id",
)),
]
}
fn refused(error: &SyntaxError) -> CommandError {
let code = match error {
SyntaxError::Offline => "offline",
SyntaxError::Transport(_) => "download_failed",
_ => "grammar_refused",
};
CommandError::new(code, error.to_string())
}
fn invalid(message: impl Into<String>) -> CommandError {
CommandError::new("invalid_params", message)
}
/// The folder grammar files live in: next to the settings the session uses.
fn grammar_dir(session: &Session) -> Option<PathBuf> {
let base = match session.settings_dir() {
Some(dir) => dir.to_path_buf(),
None => match std::env::var_os(craft_settings::paths::env_var_name(crate::app_settings::APP_ID)) {
Some(dir) if !dir.is_empty() => PathBuf::from(dir),
_ => craft_settings::paths::data_dir(crate::app_settings::APP_ID)?,
},
};
Some(base.join("grammars"))
}
fn id_param(params: &Value) -> Result<String, CommandError> {
let id = text_param(params, "id")?;
if is_grammar_id(&id) { Ok(id) } else { Err(invalid("`id` is lowercase letters, digits and `-`, at most 32 characters")) }
}
fn info(session: &mut Session, params: &Value) -> CommandResult {
let (settings, _) = load_for(session)?;
let installed: Vec<Value> = settings
.grammars
.iter()
.map(
|grammar| json!({ "id": grammar.id, "name": grammar.name, "size": grammar.size, "license": grammar.license, "installed_at": grammar.installed_at }),
)
.collect();
let label = params.get("label").and_then(Value::as_str).map(|label| {
let id = label.trim().to_lowercase();
json!({
"label": label,
"bundled": load::is_known(label) && !settings.grammars.iter().any(|grammar| grammar.id == id),
"installed": settings.grammars.iter().any(|grammar| grammar.id == id),
"suggestion": suggest_language(label),
})
});
json(json!({ "installed": installed, "label": label }))
}
/// The registry entry for `id`: the file path to download, its checksum, license and sample.
struct Entry {
file: String,
sha256: String,
license: String,
sample: String,
}
fn registry_entry(transport: &dyn Transport, id: &str) -> Result<Entry, CommandError> {
let url = format!("{REGISTRY_RAW}index.json");
check_url(&url, Some(REGISTRY_HOST)).map_err(|error| refused(&error))?;
let bytes = transport.get(&url, MAX_INDEX_BYTES).map_err(|error| refused(&error))?;
let index: Value =
serde_json::from_slice(&bytes).map_err(|error| CommandError::new("grammar_refused", format!("the registry index is not valid JSON: {error}")))?;
let found = index
.get("grammars")
.and_then(Value::as_array)
.and_then(|list| list.iter().find(|entry| entry.get("id").and_then(Value::as_str) == Some(id)))
.ok_or_else(|| CommandError::new("not_found", format!("the registry has no grammar `{id}`")))?;
let text = |key: &str| found.get(key).and_then(Value::as_str).map(str::to_owned);
if text("format").as_deref() != Some(FORMAT) {
return Err(CommandError::new("grammar_refused", format!("the registry lists `{id}` in a format other than {FORMAT}")));
}
let file = text("file").filter(|file| !file.is_empty() && !file.starts_with('/') && !file.contains("..") && !file.contains(':') && !file.contains('\\'));
let (Some(file), Some(sha256)) = (file, text("sha256")) else {
return Err(CommandError::new("grammar_refused", "the registry entry lacks a safe `file` or a `sha256`"));
};
let sample = text("sample").unwrap_or_default().chars().take(MAX_SAMPLE_BYTES).collect();
Ok(Entry { file, sha256, license: text("license").unwrap_or_default(), sample })
}
/// A grammar that passed every gate, ready to be saved.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FetchedGrammar {
/// The `.sublime-syntax` source.
pub text: String,
/// The grammar's own name.
pub name: String,
/// License the registry lists; empty for a pasted link.
pub license: String,
}
/// Downloads and checks a grammar without touching the session or the disk, so the UI can run it on
/// a worker thread. With `url` it is a pasted link (no checksum); without, the registry entry `id`.
///
/// # Errors
/// The first failing gate, as a [`CommandError`] (`grammar_refused`, `offline`, `download_failed`, `not_found`).
pub fn fetch_grammar(transport: &dyn Transport, id: &str, url: Option<&str>) -> Result<FetchedGrammar, CommandError> {
if !is_grammar_id(id) {
return Err(invalid("`id` is lowercase letters, digits and `-`, at most 32 characters"));
}
let (got, license) = if let Some(url) = url {
(download_grammar(transport, &Request { url, only_host: None, sha256: None, sample: "" }), String::new())
} else {
let entry = registry_entry(transport, id)?;
let url = format!("{REGISTRY_RAW}{}", entry.file);
(download_grammar(transport, &Request { url: &url, only_host: Some(REGISTRY_HOST), sha256: Some(&entry.sha256), sample: &entry.sample }), entry.license)
};
let grammar = got.map_err(|error| refused(&error))?;
Ok(FetchedGrammar { text: grammar.text, name: grammar.name, license })
}
fn install(session: &mut Session, params: &Value) -> CommandResult {
let id = id_param(params)?;
let fetched = fetch_grammar(&*session.grammar_transport, &id, params.get("url").and_then(Value::as_str))?;
save(session, &id, &fetched)
}
/// `grammar.install_text`: saves grammar source the caller already fetched (and the UI checked),
/// after the same format, parse and size gates.
fn install_text(session: &mut Session, params: &Value) -> CommandResult {
let id = id_param(params)?;
let text = text_param(params, "text")?;
let name = check_grammar(&text, "").map_err(|error| refused(&error))?;
let license = params.get("license").and_then(Value::as_str).unwrap_or_default().to_owned();
save(session, &id, &FetchedGrammar { text, name, license })
}
/// Writes the file, records it in the settings and reloads the highlighter's grammars.
fn save(session: &mut Session, id: &str, grammar: &FetchedGrammar) -> CommandResult {
let (mut settings, _) = load_for(session)?;
if settings.grammars.len() >= MAX_GRAMMARS && !settings.grammars.iter().any(|grammar| grammar.id == id) {
return Err(CommandError::new("limit", format!("at most {MAX_GRAMMARS} grammars can be installed")));
}
let dir = grammar_dir(session).ok_or_else(|| CommandError::new("no_config_dir", "there is no folder to keep grammars in"))?;
let file = format!("{id}.sublime-syntax");
write_file(&dir, &file, grammar.text.as_bytes())?;
let info = GrammarInfo {
id: id.to_owned(),
file: file.clone(),
name: grammar.name.clone(),
size: u64::try_from(grammar.text.len()).unwrap_or(u64::MAX),
license: grammar.license.clone(),
installed_at: SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |elapsed| elapsed.as_secs()),
};
settings.grammars.retain(|grammar| grammar.id != id);
settings.grammars.push(info.clone());
settings.grammars.sort_by(|a, b| a.id.cmp(&b.id));
if let Err(error) = save_for(session, &settings) {
let _ = std::fs::remove_file(dir.join(&file));
return Err(error);
}
reload_installed(session);
json(json!({ "id": id, "name": info.name, "size": info.size, "license": info.license }))
}
/// Loads every installed grammar file into the highlighter. A missing or unreadable file is skipped.
pub(crate) fn reload_installed(session: &Session) {
let Ok((settings, _)) = load_for(session) else { return };
let Some(dir) = grammar_dir(session) else { return };
let installed: Vec<(String, String)> =
settings.grammars.iter().filter_map(|grammar| Some((grammar.id.clone(), std::fs::read_to_string(dir.join(&grammar.file)).ok()?))).collect();
load::set_installed(&installed);
}
fn write_file(dir: &Path, file: &str, bytes: &[u8]) -> Result<(), CommandError> {
std::fs::create_dir_all(dir).map_err(|error| CommandError::new("io", format!("cannot create the grammars folder: {error}")))?;
craft_atomic::atomic_write(&dir.join(file), bytes).map_err(|error| CommandError::new("io", format!("cannot save the grammar: {error}")))
}
fn remove(session: &mut Session, params: &Value) -> CommandResult {
let id = id_param(params)?;
let (mut settings, _) = load_for(session)?;
let Some(info) = settings.grammars.iter().find(|grammar| grammar.id == id).cloned() else {
return Err(CommandError::new("not_found", format!("grammar `{id}` is not installed")));
};
settings.grammars.retain(|grammar| grammar.id != id);
save_for(session, &settings)?;
if let Some(dir) = grammar_dir(session) {
let _ = std::fs::remove_file(dir.join(&info.file));
}
reload_installed(session);
json(json!({ "id": id, "removed": true }))
}
crates/engine/src/lib.rs
+12 -3
View File
@@ -28,6 +28,7 @@
mod ai;
mod base64;
mod batch;
mod block_edit;
mod builtin;
mod calendar;
@@ -35,6 +36,7 @@ mod commands;
mod control;
mod error;
mod files;
mod grammars;
mod id;
mod info;
mod knowledge;
@@ -49,6 +51,7 @@ mod slash_kit;
mod spec;
mod vault;
pub use crate::vault::switcher;
pub(crate) use ai::connections;
pub use ai::{assistant, assistant_chat, assistant_commands, assistant_gate, assistant_queue};
pub(crate) use builtin::{
@@ -57,10 +60,11 @@ pub(crate) use builtin::{
pub(crate) use plugin_host::{plugin_bridge, plugin_run, plugins};
pub(crate) use prefs::{app_settings, calendar_config, config, reminders_config, settings, settings_store, themes};
pub(crate) use slash_kit::{slash, slash_menu};
pub(crate) use vault::{attachments, bookmarks, buffers, daily, external, notes, properties, search, session};
pub(crate) use vault::{attachments, bookmarks, buffers, content_search, daily, external, indexing, notes, properties, random_unique, search, session};
pub use crate::app_settings::{
APP_ID, AppSettings, AssistantSettings, EditorSettings, GitSettings, MAX_RECENT_VAULTS, MAX_SHORTCUT_LEN, MAX_UI_BYTES, PluginSettings, SCHEMA_VERSION,
APP_ID, AppSettings, AssistantSettings, EditorSettings, GitSettings, GrammarInfo, MAX_FONT_SIZE, MAX_RECENT_VAULTS, MAX_SHORTCUT_LEN, MAX_UI_BYTES,
MIN_FONT_SIZE, PluginSettings, PreviewSettings, SCHEMA_VERSION, ViewDefault,
};
pub use crate::assistant::{is_gated_source, run_as_assistant};
pub use crate::assistant_chat::{
@@ -72,6 +76,7 @@ pub use crate::assistant_queue::{
};
pub use crate::attachments::SavedAttachment;
pub use crate::base64::{MAX_DECODED_BYTES, decode as decode_base64};
pub use crate::batch::{Change, ChangeKind, Checks, Decision as BatchDecision, Proposal, ReplaceSpec, Status, apply_changes, hash_text, propose_note};
pub use crate::bookmarks::{Bookmark, MAX_BOOKMARK_BYTES, MAX_BOOKMARKS};
pub use crate::buffers::{Buffer, MAX_BUFFER_BYTES, MAX_UNDO_STEPS};
pub use crate::calendar::{
@@ -79,15 +84,18 @@ pub use crate::calendar::{
};
pub use crate::calendar_config::CalendarConfig;
pub use crate::commands::register_core;
pub use crate::config::{FilesConfig, SessionConfig};
pub use crate::config::{DeletedFiles, FilesConfig, LinkFormat, LinkPath, SessionConfig};
pub use crate::connections::{Connection, ConnectionKind, MAX_CONNECTIONS};
pub use crate::content_search::{ContentHit, ContentPoll, ContentSearch};
pub use crate::control::client::ControlClient;
pub use crate::control::pump::ControlHandle;
pub use crate::control::server::{ControlOptions, ControlServer};
pub use crate::error::{CommandError, Error, Result};
pub use crate::export_commands::render_pdf;
pub use crate::external::Applied;
pub use crate::grammars::{FetchedGrammar, REGISTRY_HOST, fetch_grammar};
pub use crate::id::{MAX_ID_LEN, PLUGIN_NAMESPACE, plugin_prefix};
pub use crate::indexing::IndexProgress;
pub use crate::info::CommandInfo;
pub use crate::notes::RenameReport;
pub use crate::panic_hunt::adversarial_params;
@@ -99,6 +107,7 @@ pub use crate::session::Session;
pub use crate::settings_store::{load, load_at, load_checked, load_checked_at, save, save_at};
pub use crate::slash::{SlashEntry, filter_entries, trigger_ok};
pub use crate::spec::{CommandResult, CommandSpec};
pub use notescraft_vault::{FileCount, LARGE_FOLDER_FILES, count_files};
#[cfg(test)]
mod tests {
crates/engine/src/lint/fixes.rs
+20 -3
View File
@@ -20,7 +20,8 @@ impl Session {
/// Rule switches and severities from `session.lint`.
pub(crate) fn lint_rules(&self) -> RuleSet {
let config = &self.config.lint;
let enabled = config.disabled.iter().map(|id| (id.clone(), false)).collect();
let mut enabled: std::collections::BTreeMap<String, bool> = config.disabled.iter().map(|id| (id.clone(), false)).collect();
enabled.insert("spelling".to_owned(), config.spelling);
let severity = config
.severities
.iter()
@@ -44,24 +45,40 @@ impl Session {
learned
}
/// `(path, title)` of every indexed note, for link resolution.
pub(crate) fn lint_titles(&self) -> Vec<(String, String)> {
self.index.notes().into_iter().filter_map(|note| Some((note.to_owned(), self.index.note(note)?.title.clone()))).collect()
}
/// Checks `path`, returning the text that was checked and its issues.
///
/// # Errors
/// Whatever reading the note returns.
pub(crate) fn lint_check(&mut self, path: &str) -> Result<(String, Vec<Issue>)> {
let mut titles = self.lint_titles();
self.lint_check_with(path, &mut titles)
}
/// [`Session::lint_check`] with the note titles supplied (and handed back), so a caller that checks
/// many notes builds the titles once instead of once per note.
///
/// # Errors
/// Whatever reading the note returns.
pub(crate) fn lint_check_with(&mut self, path: &str, titles: &mut Vec<(String, String)>) -> Result<(String, Vec<Issue>)> {
let text = self.lint_text(path)?;
if !self.config.lint.enabled {
return Ok((text, Vec::new()));
}
let titles = self.index.notes().into_iter().filter_map(|note| Some((note.to_owned(), self.index.note(note)?.title.clone()))).collect();
let context = Context {
titles,
titles: std::mem::take(titles),
current_path: path.to_owned(),
style: self.lint_style(),
rules: self.lint_rules(),
dismissed: self.lint.dismissed.get(path).cloned().unwrap_or_default(),
known_words: if self.config.lint.spelling { self.lint_known_words() } else { std::sync::Arc::default() },
};
let issues = check(&text, &context);
*titles = context.titles;
Ok((text, issues))
}
crates/engine/src/lint/mod.rs
+9
View File
@@ -31,4 +31,13 @@ pub(crate) struct Lint {
pub(crate) total: Option<Vote>,
/// Dismissed issue keys per note path.
pub(crate) dismissed: BTreeMap<String, Vec<String>>,
/// The words of each note, so one note can change without a rescan.
pub(crate) note_words: HashMap<String, std::collections::HashSet<String>>,
/// In how many notes each word occurs.
pub(crate) word_notes: HashMap<String, u32>,
/// The words that occur in at least [`MIN_NOTES_FOR_KNOWN`] notes; `None` until needed after a change.
pub(crate) known: Option<std::sync::Arc<std::collections::HashSet<String>>>,
}
/// A word is part of the vault's vocabulary once this many notes use it.
pub(crate) const MIN_NOTES_FOR_KNOWN: u32 = 2;
crates/engine/src/lint/sync.rs
+33
View File
@@ -19,11 +19,44 @@ impl Session {
vote.add_note(text);
self.lint.votes.insert(path.to_owned(), vote);
self.lint.total = None;
let words = notescraft_lint::vocabulary(text);
self.lint_forget_words(path);
for word in &words {
*self.lint.word_notes.entry(word.clone()).or_default() += 1;
}
self.lint.note_words.insert(path.to_owned(), words);
self.lint.known = None;
}
/// Takes a note's words out of the counts.
fn lint_forget_words(&mut self, path: &str) {
for word in self.lint.note_words.remove(path).into_iter().flatten() {
if let Some(count) = self.lint.word_notes.get_mut(&word) {
*count = count.saturating_sub(1);
if *count == 0 {
self.lint.word_notes.remove(&word);
}
}
}
}
/// The words the `spelling` rule accepts: those that occur in two or more notes.
pub(crate) fn lint_known_words(&mut self) -> std::sync::Arc<std::collections::HashSet<String>> {
if let Some(known) = &self.lint.known {
return known.clone();
}
let known: std::collections::HashSet<String> =
self.lint.word_notes.iter().filter(|(_, count)| **count >= super::MIN_NOTES_FOR_KNOWN).map(|(word, _)| word.clone()).collect();
let known = std::sync::Arc::new(known);
self.lint.known = Some(known.clone());
known
}
/// Forgets a note.
pub(crate) fn lint_remove(&mut self, path: &str) {
self.lint.votes.remove(path);
self.lint_forget_words(path);
self.lint.known = None;
self.lint.dismissed.remove(path);
self.lint.total = None;
}
crates/engine/src/prefs/app_settings.rs
+77 -10
View File
@@ -20,6 +20,8 @@ use notescraft_plugins::Permission;
use crate::config::SessionConfig;
use crate::id::id_problem;
pub use crate::prefs::editor_settings::{EditorSettings, MAX_FONT_SIZE, MIN_FONT_SIZE, ViewDefault};
/// The app id `craft-settings` stores the file under: `notescraft/config.toml`.
pub const APP_ID: &str = "notescraft";
@@ -64,6 +66,27 @@ pub struct AppSettings {
/// Command id to shortcut; unknown command ids are kept (validation never
/// requires a registered id).
pub hotkeys: BTreeMap<String, String>,
/// Syntax grammars the user installed, sorted by id; the files live in the `grammars` folder.
/// A list, not a table: the settings store merges tables key by key, so a removed key would come back.
pub grammars: Vec<GrammarInfo>,
}
/// One installed syntax grammar.
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct GrammarInfo {
/// The grammar id: lowercase letters, digits and `-`.
pub id: String,
/// File name inside the grammars folder, always `<id>.sublime-syntax`.
pub file: String,
/// The grammar's own name, as the fence picker shows it.
pub name: String,
/// Size of the file in bytes.
pub size: u64,
/// License the registry lists; empty for a pasted link.
pub license: String,
/// Seconds since the Unix epoch when it was installed.
pub installed_at: u64,
}
impl Default for AppSettings {
@@ -79,6 +102,7 @@ impl Default for AppSettings {
git: GitSettings::default(),
editor: EditorSettings::default(),
hotkeys: BTreeMap::new(),
grammars: Vec::new(),
}
}
}
@@ -109,9 +133,11 @@ impl AppSettings {
#[must_use]
pub fn validate(&mut self) -> Vec<String> {
let mut notes: Vec<String> = self.session.repair().into_iter().map(|note| format!("session.{note}")).collect();
notes.extend(self.editor.validate().into_iter().map(|note| format!("editor.{note}")));
notes.extend(repair_ui(&mut self.ui));
notes.extend(repair_recents(&mut self.recent_vaults));
notes.extend(repair_hotkeys(&mut self.hotkeys));
notes.extend(repair_grammars(&mut self.grammars));
let minutes = self.git.auto_commit_minutes.clamp(0, MAX_AUTO_COMMIT_MINUTES);
if minutes != self.git.auto_commit_minutes {
self.git.auto_commit_minutes = minutes;
@@ -121,19 +147,34 @@ impl AppSettings {
}
}
/// Longest grammar id, in characters.
pub const MAX_GRAMMAR_ID: usize = 32;
/// Most grammars that can be installed.
pub const MAX_GRAMMARS: usize = 64;
/// Whether `id` is a grammar id: lowercase letters, digits and `-`, `1..=MAX_GRAMMAR_ID` long.
#[must_use]
pub fn is_grammar_id(id: &str) -> bool {
(1..=MAX_GRAMMAR_ID).contains(&id.chars().count()) && id.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-')
}
/// Drops grammar entries whose id is not a grammar id, is repeated, or whose file is not
/// `<id>.sublime-syntax`, and keeps the list sorted by id.
fn repair_grammars(grammars: &mut Vec<GrammarInfo>) -> Vec<String> {
let before = grammars.len();
grammars.retain(|info| is_grammar_id(&info.id) && info.file == format!("{}.sublime-syntax", info.id));
grammars.sort_by(|a, b| a.id.cmp(&b.id));
grammars.dedup_by(|a, b| a.id == b.id);
grammars.truncate(MAX_GRAMMARS);
match before.saturating_sub(grammars.len()) {
0 => Vec::new(),
dropped => vec![format!("grammars: dropped {dropped} invalid entries")],
}
}
/// Longest interval accepted for automatic local snapshots.
const MAX_AUTO_COMMIT_MINUTES: i64 = 240;
/// Editor preferences (SPEC 6 item 29).
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct EditorSettings {
/// Whether Vim modal editing is enabled; off by default.
pub vim: bool,
/// Block editing in the Visual view.
pub blocks: BlocksSettings,
}
/// Block editing switches (grip, floating toolbar, slash preview).
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
@@ -152,6 +193,22 @@ impl Default for BlocksSettings {
}
}
/// Preview switches (color swatches and named colors in the Visual view).
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct PreviewSettings {
/// Color swatches next to the colors the text holds.
pub colors: bool,
/// Swatches for the sixteen basic CSS color names (off by default).
pub named_colors: bool,
}
impl Default for PreviewSettings {
fn default() -> Self {
Self { colors: true, named_colors: false }
}
}
/// Local git snapshot preferences; the UI owns the inactivity timer.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
@@ -365,6 +422,16 @@ mod tests {
assert!(notes[0].contains("1 shortcuts with characters outside"), "{}", notes[0]);
}
#[test]
fn editor_repairs_are_prefixed_and_applied() {
let mut settings = AppSettings::default();
settings.editor.font_size = 3;
let notes = settings.validate();
assert_eq!(notes.len(), 1, "{notes:?}");
assert_eq!(notes.first().map(String::as_str), Some("editor.font_size: 3 is outside 10..=32 and is clamped to 10"));
assert_eq!(settings.editor.font_size, MIN_FONT_SIZE);
}
#[test]
fn session_repairs_are_prefixed_and_applied() {
let mut settings = AppSettings::default();
crates/engine/src/prefs/config.rs
+15 -16
View File
@@ -15,6 +15,8 @@ use super::knowledge_config::KnowledgeConfig;
use super::lint_config::LintConfig;
use crate::error::{Error, Result};
pub use crate::prefs::files_config::{DeletedFiles, FilesConfig, LinkFormat, LinkPath};
/// Default folder of the daily notes.
const DEFAULT_DAILY_FOLDER: &str = "Daily";
/// Default file-name format of a daily note.
@@ -28,28 +30,13 @@ const DEFAULT_CALENDAR_FOLDER: &str = "Calendar";
/// Default theme id.
const DEFAULT_THEME: &str = "catppuccin-mocha";
/// Default cap on one attachment, in MiB (SPEC 9).
const DEFAULT_MAX_ATTACHMENT_MIB: u32 = 64;
pub(super) const DEFAULT_MAX_ATTACHMENT_MIB: u32 = 64;
/// Smallest and largest accepted attachment cap, in MiB.
const MIN_ATTACHMENT_MIB: u32 = 1;
/// Largest accepted attachment cap, in MiB: a config value cannot ask for
/// petabyte attachments.
const MAX_ATTACHMENT_MIB: u32 = 4096;
/// File and attachment settings (SPEC 9).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct FilesConfig {
/// Largest attachment (pasted image, dropped file) accepted, in MiB; a
/// larger one is refused before any byte is written.
pub max_attachment_mib: u32,
}
impl Default for FilesConfig {
fn default() -> Self {
Self { max_attachment_mib: DEFAULT_MAX_ATTACHMENT_MIB }
}
}
/// The engine settings the user may change (SPEC 6 item 14).
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
@@ -131,6 +118,7 @@ impl SessionConfig {
self.reminders.validate()?;
self.knowledge.validate()?;
self.lint.validate()?;
self.files.validate()?;
if !(MIN_ATTACHMENT_MIB..=MAX_ATTACHMENT_MIB).contains(&self.files.max_attachment_mib) {
return Err(Error::Config(format!(
"files.max_attachment_mib: {} is outside {}..={MAX_ATTACHMENT_MIB}",
@@ -190,6 +178,7 @@ impl SessionConfig {
notes.extend(self.reminders.repair());
notes.extend(self.knowledge.repair());
notes.extend(self.lint.repair());
notes.extend(self.files.repair());
notes
}
}
@@ -327,4 +316,14 @@ mod tests {
SessionConfig { daily_folder: String::from("Notes/Daily"), daily_template: Some(String::from("Templates/Seed.md")), ..SessionConfig::default() };
assert_eq!(config.repair(), Vec::<String>::new());
}
#[test]
fn the_files_section_validates_and_repairs_with_the_document() {
let mut config: SessionConfig = serde_json::from_str(r#"{"files": {"new_note_folder": "../up"}}"#).unwrap_or_default();
assert!(config.validate().is_err(), "a folder that escapes the vault is refused");
let notes = config.repair();
assert_eq!(config.files.new_note_folder, "", "it is cleared");
assert!(notes.first().is_some_and(|note| note.starts_with("files.new_note_folder")), "{notes:?}");
assert_eq!(config.validate(), Ok(()), "the repaired document is valid");
}
}
crates/engine/src/prefs/editor_settings.rs
+141
View File
@@ -0,0 +1,141 @@
//! Editor preferences: which view a new tab opens in and how the text is laid
//! out (SPEC 6 item 29).
//!
//! The keys are flat (`editor.default_view`, `editor.font_size`,
//! `editor.readable_line_length`, `editor.line_numbers`,
//! `editor.strict_line_breaks`) and live outside every vault, so they apply to
//! whichever vault is open. [`EditorSettings::validate`] repairs the one value
//! that can be bad; [`crate::app_settings::AppSettings::validate`] puts
//! `editor.` in front of its notes.
use serde::{Deserialize, Serialize};
/// Default editor font size, in points.
const DEFAULT_FONT_SIZE: u8 = 16;
/// Smallest accepted editor font size, in points: below it no text is legible.
pub const MIN_FONT_SIZE: u8 = 10;
/// Largest accepted editor font size, in points.
pub const MAX_FONT_SIZE: u8 = 32;
/// The view a new tab opens in.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ViewDefault {
/// The rendered view, with formatting.
#[default]
Visual,
/// The Markdown source view.
Source,
}
/// What a new tab shows and how the text is laid out.
///
/// Four independent switches, each its own `editor.*` key: a layout group
/// would move them out of the flat key space the settings commands use.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[expect(clippy::struct_excessive_bools, reason = "the layout switches are separate settings keys")]
#[serde(default)]
pub struct EditorSettings {
/// The view a new tab opens in.
pub default_view: ViewDefault,
/// Font size in points; [`EditorSettings::validate`] clamps it into
/// [`MIN_FONT_SIZE`]..=[`MAX_FONT_SIZE`].
pub font_size: u8,
/// Whether lines wrap at a readable width instead of at the window edge.
pub readable_line_length: bool,
/// Whether the gutter numbers the lines.
pub line_numbers: bool,
/// `CommonMark` line breaks: when on, the lines of a paragraph join into one wrapped row in the rendered
/// view; when off (the default, as in Obsidian) every source line is its own row.
pub strict_line_breaks: bool,
/// Whether Vim modal editing is enabled; off by default.
pub vim: bool,
/// Block editing in the Visual view.
pub blocks: crate::app_settings::BlocksSettings,
/// Color and link previews in the Visual view.
pub previews: crate::app_settings::PreviewSettings,
}
impl Default for EditorSettings {
fn default() -> Self {
Self {
default_view: ViewDefault::default(),
font_size: DEFAULT_FONT_SIZE,
readable_line_length: true,
line_numbers: false,
strict_line_breaks: false,
vim: false,
blocks: crate::app_settings::BlocksSettings::default(),
previews: crate::app_settings::PreviewSettings::default(),
}
}
}
impl EditorSettings {
/// Repairs bad values in place and returns one note per repair, never an
/// error: a settings file the user can still read must not keep the app
/// from starting.
///
/// Only the font size can be bad. Like `git.auto_commit_minutes` it is
/// clamped back into [`MIN_FONT_SIZE`]..=[`MAX_FONT_SIZE`] with a note
/// naming the value it replaced.
#[must_use]
pub fn validate(&mut self) -> Vec<String> {
let wanted = self.font_size;
let clamped = wanted.clamp(MIN_FONT_SIZE, MAX_FONT_SIZE);
if clamped == wanted {
return Vec::new();
}
self.font_size = clamped;
vec![format!("font_size: {wanted} is outside {MIN_FONT_SIZE}..={MAX_FONT_SIZE} and is clamped to {clamped}")]
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::{Value, json};
#[test]
fn defaults_and_the_font_clamp() {
let mut editor = EditorSettings::default();
let value = serde_json::to_value(&editor).expect("serialize the editor defaults");
for (pointer, want) in [
("/default_view", json!("visual")),
("/font_size", json!(16)),
("/readable_line_length", json!(true)),
("/line_numbers", json!(false)),
("/strict_line_breaks", json!(false)),
] {
assert_eq!(value.pointer(pointer), Some(&want), "{value}");
}
assert_eq!(editor.validate(), Vec::<String>::new());
for (wanted, clamped) in [(3_u8, MIN_FONT_SIZE), (200, MAX_FONT_SIZE)] {
editor.font_size = wanted;
let notes = editor.validate();
assert_eq!(editor.font_size, clamped);
assert_eq!(notes.len(), 1, "{notes:?}");
assert_eq!(notes.first().map(String::as_str), Some(format!("font_size: {wanted} is outside 10..=32 and is clamped to {clamped}").as_str()));
}
}
#[test]
fn old_documents_load_and_wrong_types_are_refused() {
for document in [json!({}), json!({ "editor": {} }), json!({ "editor": { "vim": true } })] {
let shown = document.to_string();
let settings: crate::app_settings::AppSettings = serde_json::from_value(document).expect("missing keys use the defaults");
assert_eq!(settings.editor.font_size, DEFAULT_FONT_SIZE, "{shown}");
}
for value in [json!(1), json!([]), json!({}), Value::Null, json!("Visual"), json!("rendering")] {
let document = json!({ "editor": { "default_view": value } });
assert!(serde_json::from_value::<crate::app_settings::AppSettings>(document).is_err(), "{value} must be refused");
}
for value in [json!("16"), json!(-1), json!(null)] {
let document = json!({ "editor": { "font_size": value } });
assert!(serde_json::from_value::<crate::app_settings::AppSettings>(document).is_err(), "{value} must be refused");
}
}
}
crates/engine/src/prefs/files_config.rs
+222
View File
@@ -0,0 +1,222 @@
//! File settings: what a new note lands in, how links are written, what a
//! deleted file does and whether hidden files are listed (SPEC 9).
//!
//! The keys live in the nested `session.files` group
//! (`session.files.new_note_folder`) and travel with the session, so the UI
//! writes them next to the attachment cap. [`FilesConfig::validate`] refuses a
//! new-note folder that would escape or reserve vault space; [`FilesConfig::repair`]
//! clears it with a note instead.
use serde::{Deserialize, Serialize};
use crate::error::Result;
/// How a link is written into a note (SPEC 4.3).
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum LinkFormat {
/// `[[Note]]`, with an alias after `|`.
#[default]
Wikilink,
/// `[Note](Note.md)`, Markdown link syntax.
Markdown,
}
/// The path a link carries.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum LinkPath {
/// `[[Note]]`: no path at all.
#[default]
Shortest,
/// The shortest path that reaches the note from the linking note.
Relative,
/// The path from the vault root.
Absolute,
}
/// What happens to a deleted file.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum DeletedFiles {
/// The file goes to the system trash, where it can be restored.
#[default]
Trash,
/// The file is erased for good.
Permanent,
}
/// File and attachment settings (SPEC 9).
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct FilesConfig {
/// Largest attachment (pasted image, dropped file) accepted, in MiB; a
/// larger one is refused before any byte is written.
pub max_attachment_mib: u32,
/// Folder a new note is created in; empty means the vault root or the
/// folder the command was invoked from.
pub new_note_folder: String,
/// How a link is written into a note.
pub link_format: LinkFormat,
/// The path a link carries.
pub link_path: LinkPath,
/// What happens to a deleted file.
pub deleted_files: DeletedFiles,
/// Whether dot-files and dot-folders are listed.
pub show_hidden_files: bool,
}
impl Default for FilesConfig {
fn default() -> Self {
Self {
max_attachment_mib: crate::prefs::config::DEFAULT_MAX_ATTACHMENT_MIB,
new_note_folder: String::new(),
link_format: LinkFormat::default(),
link_path: LinkPath::default(),
deleted_files: DeletedFiles::default(),
show_hidden_files: false,
}
}
}
impl FilesConfig {
/// The text of a link to `target` (a vault path) written from the note `from` (`""` for none),
/// by `link_format` and `link_path`; `alias` is the shown text, the note name when `None`.
#[must_use]
pub fn link_text(&self, from: &str, target: &str, alias: Option<&str>) -> String {
let name = target.rsplit('/').next().unwrap_or(target);
let stem = name.strip_suffix(".md").unwrap_or(name);
let shown = alias.unwrap_or(stem);
let path = match self.link_path {
LinkPath::Shortest => name.to_owned(),
LinkPath::Absolute => target.to_owned(),
LinkPath::Relative => relative_path(from, target),
};
match self.link_format {
LinkFormat::Wikilink => {
let inner = if self.link_path == LinkPath::Shortest { stem } else { path.strip_suffix(".md").unwrap_or(&path) };
if alias.is_some_and(|alias| alias != inner) { format!("[[{inner}|{shown}]]") } else { format!("[[{inner}]]") }
}
LinkFormat::Markdown => format!("[{shown}]({})", path.replace(' ', "%20")),
}
}
/// Rejects a new-note folder the engine must not write into.
///
/// # Errors
///
/// [`crate::error::Error::Config`] naming `files.new_note_folder` and the
/// reason. Empty is accepted: it means the vault root.
pub fn validate(&self) -> Result<()> {
if !self.new_note_folder.is_empty() {
crate::prefs::config::check_folder_path(&self.new_note_folder)
.map_err(|reason| crate::error::Error::Config(format!("files.new_note_folder: {reason}")))?;
}
Ok(())
}
/// Replaces a bad new-note folder with the default (empty) and returns one
/// note per repair.
#[must_use]
pub fn repair(&mut self) -> Vec<String> {
if self.new_note_folder.is_empty() || crate::prefs::config::check_folder_path(&self.new_note_folder).is_ok() {
return Vec::new();
}
let bad = std::mem::take(&mut self.new_note_folder);
let reason = crate::prefs::config::check_folder_path(&bad).err().unwrap_or_default();
vec![format!("files.new_note_folder: the bad `{bad}` is cleared, so new notes land in the vault root ({reason})")]
}
}
/// `target` as a path from the folder of `from`: `a/x.md` from `a/b.md` is `x.md`, from `c/d.md` it is `../a/x.md`.
fn relative_path(from: &str, target: &str) -> String {
let base: Vec<&str> = from.rsplit_once('/').map_or(Vec::new(), |(dir, _)| dir.split('/').collect());
let goal: Vec<&str> = target.split('/').collect();
let shared = base.iter().zip(&goal).take_while(|(a, b)| a == b).count().min(goal.len().saturating_sub(1));
let ups = base.len().saturating_sub(shared);
let mut parts: Vec<&str> = vec![".."; ups];
parts.extend(goal.get(shared..).unwrap_or_default());
parts.join("/")
}
#[cfg(test)]
mod tests {
use super::*;
use crate::error::Error;
#[test]
fn defaults_are_the_documented_ones() {
let json = serde_json::to_string(&FilesConfig::default()).unwrap_or_default();
assert_eq!(
json,
r#"{"max_attachment_mib":64,"new_note_folder":"","link_format":"wikilink","link_path":"shortest","deleted_files":"trash","show_hidden_files":false}"#
);
}
#[test]
fn an_empty_new_note_folder_is_the_vault_root_and_a_folder_is_accepted() {
let mut files = FilesConfig::default();
assert!(files.validate().is_ok(), "the vault root is a valid answer");
assert_eq!(files.repair(), Vec::<String>::new());
files.new_note_folder = String::from("Notes/Inbox");
assert!(files.validate().is_ok(), "{:?} must be accepted", files.validate());
assert_eq!(files.repair(), Vec::<String>::new());
}
#[test]
fn a_bad_new_note_folder_is_refused_and_cleared_with_a_note() {
for bad in ["../up", "/etc", "c:/tmp", ".obsidian", "Notes/.notescraft"] {
let mut files = FilesConfig { new_note_folder: bad.to_owned(), ..FilesConfig::default() };
match files.validate() {
Err(Error::Config(message)) => assert!(message.starts_with("files.new_note_folder"), "got {message}"),
other => panic!("expected a config error for {bad}, got {other:?}"),
}
let notes = files.repair();
assert_eq!(files.new_note_folder, "", "{bad} is cleared");
assert_eq!(notes.len(), 1, "{notes:?}");
assert!(notes.first().is_some_and(|note| note.starts_with(&format!("files.new_note_folder: the bad `{bad}`"))), "{notes:?}");
}
}
#[test]
fn unknown_enum_names_are_refused_and_known_ones_round_trip() {
for (key, names) in [
("link_format", &["wikilink", "markdown"][..]),
("link_path", &["shortest", "relative", "absolute"][..]),
("deleted_files", &["trash", "permanent"][..]),
] {
for name in names {
let json = format!(r#"{{"{key}":"{name}"}}"#);
let files: FilesConfig = serde_json::from_str(&json).unwrap_or_default();
assert_eq!(files.validate(), Ok(()), "{json} must be accepted");
let back = serde_json::to_string(&files).unwrap_or_default();
assert!(back.contains(&format!(r#""{key}":"{name}""#)), "{back}");
}
for bad in ["WikiLink", "trash ", "link", ""] {
let json = format!(r#"{{"{key}":"{bad}"}}"#);
assert!(serde_json::from_str::<FilesConfig>(&json).is_err(), "{json} must be refused");
}
}
}
}
#[cfg(test)]
mod link_text_tests {
use super::*;
fn config(format: LinkFormat, path: LinkPath) -> FilesConfig {
FilesConfig { link_format: format, link_path: path, ..FilesConfig::default() }
}
#[test]
fn links_follow_the_format_and_path_settings() {
let wiki = FilesConfig::default();
assert_eq!(wiki.link_text("x.md", "a/My Note.md", None), "[[My Note]]");
assert_eq!(wiki.link_text("x.md", "a/My Note.md", Some("shown")), "[[My Note|shown]]");
assert_eq!(config(LinkFormat::Wikilink, LinkPath::Absolute).link_text("x.md", "a/n.md", None), "[[a/n]]");
assert_eq!(config(LinkFormat::Wikilink, LinkPath::Relative).link_text("c/d.md", "a/n.md", None), "[[../a/n]]");
assert_eq!(config(LinkFormat::Markdown, LinkPath::Shortest).link_text("x.md", "a/My Note.md", None), "[My Note](My%20Note.md)");
assert_eq!(config(LinkFormat::Markdown, LinkPath::Relative).link_text("a/b.md", "a/n.md", Some("hi")), "[hi](n.md)");
assert_eq!(relative_path("", "n.md"), "n.md");
}
}
crates/engine/src/prefs/lint_config.rs
+3
View File
@@ -27,6 +27,8 @@ pub struct LintConfig {
pub learn_date_format: bool,
/// Every safe fix is applied when a note is saved.
pub fix_on_save: bool,
/// The spelling rule runs: words the vault does not use in two notes are flagged.
pub spelling: bool,
/// Ids of rules that are switched off.
pub disabled: Vec<String>,
/// Severity per rule id (`error`, `warning`, `info`); rules not listed keep their default.
@@ -41,6 +43,7 @@ impl Default for LintConfig {
learn_list_marker: true,
learn_date_format: true,
fix_on_save: false,
spelling: false,
disabled: Vec::new(),
severities: BTreeMap::new(),
}
crates/engine/src/prefs/mod.rs
+2
View File
@@ -3,6 +3,8 @@
pub(crate) mod app_settings;
pub(crate) mod calendar_config;
pub(crate) mod config;
pub(crate) mod editor_settings;
pub(crate) mod files_config;
pub(crate) mod knowledge_config;
pub(crate) mod lint_config;
pub(crate) mod reminders_config;
crates/engine/src/prefs/settings.rs
+6 -4
View File
@@ -8,7 +8,9 @@
//! `assistant.review_writes`, `hotkeys.buffer.save`): sections
//! `session`, `editor`, `recent_vaults`, `plugins` and the `assistant` fields only
//! accept their existing keys, while `ui`, `hotkeys` and
//! `assistant.plugin_tools` are open maps the engine does not interpret.
//! `assistant.plugin_tools` are open maps the engine does not interpret. The
//! `editor` groups `blocks` and `previews` take one more level
//! (`editor.previews.colors`).
//! Values are checked by deserialising the whole document back
//! ([`AppSettings`]) and by [`AppSettings::validate`], which repairs and
//! returns notes; an unknown key or an ill-typed value is an error, nothing
@@ -93,7 +95,7 @@ fn settings_reset(session: &mut Session, params: &Value) -> CommandResult {
/// Loads the settings for `session`: the explicit folder when the session has
/// one (tests, the CLI `--config-dir` flag), otherwise the configured place.
fn load_for(session: &Session) -> Result<(AppSettings, Vec<String>), CommandError> {
pub(crate) fn load_for(session: &Session) -> Result<(AppSettings, Vec<String>), CommandError> {
match session.settings_dir().map(Path::to_path_buf) {
Some(dir) => crate::settings_store::load_checked_at(&dir),
None => crate::settings_store::load_checked(),
@@ -102,7 +104,7 @@ fn load_for(session: &Session) -> Result<(AppSettings, Vec<String>), CommandErro
}
/// Saves the settings for `session`, at the same place [`load_for`] reads.
fn save_for(session: &Session, settings: &AppSettings) -> Result<(), CommandError> {
pub(crate) fn save_for(session: &Session, settings: &AppSettings) -> Result<(), CommandError> {
match session.settings_dir().map(Path::to_path_buf) {
Some(dir) => crate::settings_store::save_at(&dir, settings),
None => crate::settings_store::save(settings),
@@ -191,7 +193,7 @@ fn write_section(section: &mut Value, name: &str, rest: &[String], value: Value)
Ok(())
}
[field] => write_field(section, name, field, value),
[group, field] if name == "editor" && group == "blocks" => {
[group, field] if name == "editor" && (group == "blocks" || group == "previews") => {
let blocks = field_mut(section, name, group)?;
write_field(blocks, group, field, value)
}
crates/engine/src/vault/buffers.rs
+1
View File
@@ -233,6 +233,7 @@ impl crate::session::Session {
/// cannot be read.
pub fn open_buffer(&mut self, path: &str) -> Result<&Buffer> {
let path = self.note_path(path)?;
self.index_now(&path);
if !self.buffers.contains_key(&path) {
let text = self.read_text(&path)?;
self.buffers.insert(path.clone(), Buffer::new(path.clone(), text));
crates/engine/src/vault/find/content_search.rs
+173
View File
@@ -0,0 +1,173 @@
//! Content search straight from the files on disk, for the time before the index is ready.
//!
//! The ranked search needs every note indexed; a freshly opened large vault does not have
//! that yet. [`ContentSearch`] walks the notes on its own thread with the ripgrep library
//! crates and streams the lines that contain every term of the query (case-insensitive,
//! plain text). It is bounded (file size, binary files, snippet length, number of files) and
//! cancellable: a new query drops the old search, and dropping stops the thread.
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{Receiver, Sender, channel};
use grep_regex::RegexMatcherBuilder;
use grep_searcher::sinks::Lossy;
use grep_searcher::{BinaryDetection, SearcherBuilder};
use ignore::WalkBuilder;
use crate::error::{Error, Result};
/// Notes larger than this are skipped, in bytes.
const MAX_FILE_BYTES: u64 = 8 * 1024 * 1024;
/// Longest snippet returned, in characters.
const MAX_SNIPPET_CHARS: usize = 240;
/// Lines reported per note.
const MAX_LINES_PER_FILE: usize = 3;
/// Deepest folder walked, as in the vault listing.
const MAX_DEPTH: usize = 64;
/// Memory the line buffer of one file may use, in bytes.
const HEAP_LIMIT: usize = 16 * 1024 * 1024;
/// One matching line.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ContentHit {
/// Vault-relative note path.
pub path: String,
/// 1-based line number.
pub line: usize,
/// The line, cut to [`MAX_SNIPPET_CHARS`] characters.
pub text: String,
}
/// What [`ContentSearch::poll`] found since the last call.
#[derive(Debug, Default)]
pub struct ContentPoll {
/// New hits, in the order the walk met them.
pub hits: Vec<ContentHit>,
/// The walk ended (finished, hit the limit, or was cancelled); no more hits will come.
pub done: bool,
}
enum Message {
Hits(Vec<ContentHit>),
Done,
}
/// A running search over the notes of a folder.
#[derive(Debug)]
pub struct ContentSearch {
receiver: Receiver<Message>,
cancel: Arc<AtomicBool>,
finished: bool,
}
impl ContentSearch {
/// Starts searching the notes under `root` for lines holding every word of `query`,
/// reporting at most `limit` notes.
///
/// # Errors
///
/// [`Error::Query`] when the query has no words.
pub fn start(root: &Path, query: &str, limit: usize) -> Result<Self> {
let mut terms: Vec<String> = query.split_whitespace().map(str::to_lowercase).collect();
terms.sort_by_key(|term| std::cmp::Reverse(term.len()));
let Some(longest) = terms.first().cloned() else {
return Err(Error::Query("the query has no words".to_owned()));
};
let matcher = RegexMatcherBuilder::new().case_insensitive(true).fixed_strings(true).build(&longest).map_err(|error| Error::Query(error.to_string()))?;
let (sender, receiver) = channel();
let cancel = Arc::new(AtomicBool::new(false));
let job = Job { root: root.to_path_buf(), terms, limit, cancel: Arc::clone(&cancel), sender };
std::thread::Builder::new().name("content-search".to_owned()).spawn(move || job.run(&matcher)).map_err(|error| Error::Vault(error.to_string()))?;
Ok(Self { receiver, cancel, finished: false })
}
/// Takes the hits found since the last call.
pub fn poll(&mut self) -> ContentPoll {
let mut poll = ContentPoll { hits: Vec::new(), done: self.finished };
if self.finished {
return poll;
}
loop {
match self.receiver.try_recv() {
Ok(Message::Hits(hits)) => poll.hits.extend(hits),
Ok(Message::Done) | Err(std::sync::mpsc::TryRecvError::Disconnected) => {
self.finished = true;
poll.done = true;
return poll;
}
Err(std::sync::mpsc::TryRecvError::Empty) => return poll,
}
}
}
/// Asks the thread to stop; hits already sent may still be polled.
pub fn cancel(&self) {
self.cancel.store(true, Ordering::Relaxed);
}
}
impl Drop for ContentSearch {
fn drop(&mut self) {
self.cancel();
}
}
/// The work of the search thread.
struct Job {
root: PathBuf,
terms: Vec<String>,
limit: usize,
cancel: Arc<AtomicBool>,
sender: Sender<Message>,
}
impl Job {
fn run(self, matcher: &grep_regex::RegexMatcher) {
let mut searcher = SearcherBuilder::new().binary_detection(BinaryDetection::quit(0)).line_number(true).heap_limit(Some(HEAP_LIMIT)).build();
let walker = WalkBuilder::new(&self.root).standard_filters(false).hidden(true).follow_links(false).max_depth(Some(MAX_DEPTH)).build();
let mut reported = 0_usize;
for entry in walker.flatten() {
if reported >= self.limit || self.cancel.load(Ordering::Relaxed) {
break;
}
let path = entry.path();
if !entry.file_type().is_some_and(|kind| kind.is_file()) || !is_note(path) || !entry.metadata().is_ok_and(|meta| meta.len() <= MAX_FILE_BYTES) {
continue;
}
let mut hits = Vec::new();
let relative = relative_path(&self.root, path);
let outcome = searcher.search_path(
matcher,
path,
Lossy(|number, line| {
let lowered = line.to_lowercase();
if self.terms.iter().all(|term| lowered.contains(term.as_str())) {
let text: String = line.trim_end().chars().take(MAX_SNIPPET_CHARS).collect();
hits.push(ContentHit { path: relative.clone(), line: usize::try_from(number).unwrap_or(usize::MAX), text });
}
Ok(hits.len() < MAX_LINES_PER_FILE && !self.cancel.load(Ordering::Relaxed))
}),
);
if outcome.is_ok() && !hits.is_empty() {
reported = reported.saturating_add(1);
if self.sender.send(Message::Hits(hits)).is_err() {
return;
}
}
}
let _ = self.sender.send(Message::Done);
}
}
/// Whether `path` names a note (`.md`, case-insensitive).
fn is_note(path: &Path) -> bool {
path.extension().is_some_and(|extension| extension.eq_ignore_ascii_case("md"))
}
/// `path` below `root` with `/` separators.
fn relative_path(root: &Path, path: &Path) -> String {
let below = path.strip_prefix(root).unwrap_or(path);
below.components().map(|part| part.as_os_str().to_string_lossy()).collect::<Vec<_>>().join("/")
}
crates/engine/src/vault/find/mod.rs
+6
View File
@@ -0,0 +1,6 @@
//! Finding notes: indexed and disk search, the quick switcher and the random-note commands.
pub(crate) mod content_search;
pub(crate) mod random_unique;
pub(crate) mod search;
pub mod switcher;
crates/engine/src/vault/find/random_unique.rs
+147
View File
@@ -0,0 +1,147 @@
//! `note.random` and `note.unique` (task 15 of the Obsidian basics): a random
//! indexed note, and Obsidian's unique-note creator.
//!
//! `note.random` picks one indexed note, optionally under a folder; the
//! randomness is a small xorshift seeded from the clock's nanoseconds, so no
//! new dependency is needed. `note.unique` creates an empty note named
//! `YYYYMMDDHHmm.md` in the local time the daily note helper computes,
//! appending `-1`, `-2`, … when the name is taken, and never overwrites.
use std::time::{SystemTime, UNIX_EPOCH};
use serde_json::{Value, json};
use notescraft_templates::DateTime;
use crate::calendar::local_utc_offset_minutes;
use crate::commands::{command_error, json, writable};
use crate::error::Error;
use crate::params::{ParamKind, ParamSpec};
use crate::session::Session;
use crate::spec::{CommandResult, CommandSpec};
/// The vault-relative folder `note.unique` creates its notes in: `None` means
/// the vault root. No setting owns this yet (task 14 covers default
/// locations); a value here would be the folder path, like `daily_folder`.
pub const UNIQUE_FOLDER_SETTING: Option<&str> = None;
/// How many `-N` suffixes are tried before giving up.
const MAX_UNIQUE_TRIES: u32 = 1000;
/// Clock nanoseconds as a non-zero xorshift seed.
fn clock_seed() -> u64 {
let nanos = SystemTime::now().duration_since(UNIX_EPOCH).ok().and_then(|since| u64::try_from(since.as_nanos()).ok()).unwrap_or(0);
nanos.max(1)
}
/// A small xorshift64 step, so `note.random` needs no dependency.
fn xorshift(mut state: u64) -> u64 {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
state
}
/// The random note and unique-note operations of a session.
impl Session {
/// A random indexed note path, optionally under `folder`.
///
/// # Errors
///
/// [`Error::NotFound`] when no indexed note is under `folder`, or the
/// vault holds none at all.
pub fn random_note(&self, folder: Option<&str>) -> crate::error::Result<String> {
let folder = folder.map(str::trim).filter(|folder| !folder.is_empty());
let notes: Vec<&str> = self
.index()
.notes()
.into_iter()
.filter(|path| folder.is_none_or(|folder| path.strip_prefix(folder).is_some_and(|rest| rest.is_empty() || rest.starts_with('/'))))
.collect();
let len = u64::try_from(notes.len()).unwrap_or(u64::MAX);
let pick = usize::try_from(xorshift(clock_seed()) % len.max(1)).unwrap_or(0);
notes.get(pick).map(|path| (*path).to_owned()).ok_or_else(|| {
Error::NotFound(match folder {
Some(folder) => format!("no note is under `{folder}`"),
None => "the vault holds no note".to_owned(),
})
})
}
/// Creates an empty note named after the current local time, like
/// Obsidian's unique note creator: `YYYYMMDDHHmm.md` in
/// [`UNIQUE_FOLDER_SETTING`] (the vault root when it is `None`), appending
/// `-1`, `-2`, … when the name is taken. Nothing is ever overwritten.
///
/// # Errors
///
/// [`Error::Config`] for a bad local offset, [`Error::ReadOnly`],
/// [`Error::Reserved`], or [`Error::Exists`] when every suffix is taken.
pub fn unique_note(&mut self) -> crate::error::Result<String> {
let now = DateTime::now_with_offset(local_utc_offset_minutes())?;
let date = now.date();
let stem =
format!("{:04}{:02}{:02}{:02}{:02}", date.year(), u32::from(date.month()), u32::from(date.day()), u32::from(now.hour()), u32::from(now.minute()));
let folder = UNIQUE_FOLDER_SETTING.unwrap_or("").trim_end_matches('/');
for suffix in 0..MAX_UNIQUE_TRIES {
let name = match suffix {
0 => format!("{stem}.md"),
n => format!("{stem}-{n}.md"),
};
let path = if folder.is_empty() { name } else { format!("{folder}/{name}") };
if self.vault.resolve(&path)?.exists() {
continue;
}
self.create_note(&path, "")?;
return Ok(path);
}
Err(Error::Exists(format!("{stem} and every suffix up to -{MAX_UNIQUE_TRIES} are taken")))
}
}
/// Registers the `note.random` and `note.unique` commands.
pub(crate) fn specs() -> Vec<CommandSpec<Session>> {
vec![
CommandSpec::new("note.random", "Open random note", "Picks one indexed note at random", |_| true, command_random)
.with_menu(vec!["File".to_owned(), "Open random note".to_owned()])
.with_param(ParamSpec::optional("folder", ParamKind::String, "only notes under this folder")),
CommandSpec::new(
"note.unique",
"New unique note",
"Creates an empty note named after the current local time, never overwriting",
writable,
command_unique,
)
.with_menu(vec!["File".to_owned(), "New unique note".to_owned()]),
]
}
/// `note.random`: the path of one random indexed note.
fn command_random(session: &mut Session, params: &Value) -> CommandResult {
let folder = params.get("folder").and_then(Value::as_str);
let path = session.random_note(folder).map_err(|error| command_error(&error))?;
json(json!({ "path": path }))
}
/// `note.unique`: the path of the created note.
fn command_unique(session: &mut Session, _params: &Value) -> CommandResult {
let path = session.unique_note().map_err(|error| command_error(&error))?;
json(json!({ "path": path }))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn xorshift_moves_and_never_returns_zero_from_a_seed() {
let state = xorshift(clock_seed());
assert!(state > 0, "a non-zero seed never maps to the fixed point");
assert_ne!(xorshift(state), state, "one step changes the state");
}
#[test]
fn the_unique_folder_setting_defaults_to_the_vault_root() {
assert_eq!(UNIQUE_FOLDER_SETTING, None);
}
}
crates/engine/src/vault/find/search.rs
+200
View File
@@ -0,0 +1,200 @@
//! Full-text search: the [`Session::search`] query and the `search` command
//! (SPEC 6 item 6).
//!
//! The [`SearchIndex`] mirror lives on the session and is fed everywhere the
//! link index is, so a hit never reflects a state that was already dropped
//! from the disk. Ranking, operators and snippets are the `notescraft-index`
//! implementation; this module only wires them up.
use serde_json::{Value, json};
use notescraft_index::search::{Hit, SearchOptions, SearchQuery};
use crate::error::CommandError;
use crate::params::{ParamKind, ParamSpec};
use crate::session::Session;
use crate::spec::{CommandResult, CommandSpec};
/// Hits returned when the params do not name a limit (`search` command).
const DEFAULT_SEARCH_LIMIT: usize = 20;
/// Largest `limit` the `search` command accepts.
const MAX_SEARCH_LIMIT: usize = 200;
/// Most notes a regular-expression search reads.
const REGEX_MAX_NOTES: usize = 50_000;
/// Largest note a regular-expression search reads, in bytes.
const REGEX_MAX_NOTE_BYTES: usize = 1024 * 1024;
/// Matches counted per line.
const REGEX_LINE_MATCHES: usize = 50;
/// Snippet lines kept per note.
const REGEX_SNIPPETS: usize = 3;
/// Characters kept of a snippet line.
const REGEX_SNIPPET_CHARS: usize = 200;
/// The pattern of a `/pattern/` query, `None` for an ordinary query.
fn regex_query(query: &str) -> Option<&str> {
let inner = query.trim().strip_prefix('/')?.strip_suffix('/')?;
(!inner.is_empty()).then_some(inner)
}
/// Maps a session [`crate::error::Error`] onto the `{ code, message }` a
/// transport expects; the `.map_err(|error| s(&error))?` idiom.
fn s(error: &crate::error::Error) -> CommandError {
CommandError::new(error.code(), error.to_string())
}
/// Ranked search over the whole vault (SPEC 6 item 6).
impl Session {
/// Runs `query` against the session's search mirror.
///
/// `limit` caps the hits; the best ones come first. An empty or
/// operator-only query is refused, not treated as "match everything".
///
/// # Errors
///
/// [`SearchQuery::parse`] rejects malformed queries. Searching itself never
/// fails.
pub fn search(&self, query: &str, limit: usize) -> Result<Vec<Hit>, crate::error::Error> {
self.search_matching(query, limit, false)
}
/// As [`Session::search`], but `match_case` turns the case switch on
/// for the whole query, exactly as a `match-case:` piece in it would.
///
/// # Errors
///
/// The same as [`Session::search`].
pub fn search_matching(&self, query: &str, limit: usize, match_case: bool) -> Result<Vec<Hit>, crate::error::Error> {
if let Some(pattern) = regex_query(query) {
return self.regex_search(pattern, limit, match_case);
}
let options = if match_case { SearchOptions::matching_case() } else { SearchOptions::new() };
let parsed = SearchQuery::parse_with(query, &options).map_err(|error| crate::error::Error::Query(error.to_string()))?;
Ok(self.search_index.search(&parsed, limit))
}
/// Notes with a line matching the regular expression `pattern` (the `/…/` query form), most matches first.
fn regex_search(&self, pattern: &str, limit: usize, match_case: bool) -> Result<Vec<Hit>, crate::error::Error> {
let regex = notescraft_regex::Regex::new(pattern, !match_case).map_err(|error| crate::error::Error::Query(error.to_string()))?;
let mut hits: Vec<Hit> = Vec::new();
for path in self.index().node_ids().into_iter().filter(|path| crate::session::is_markdown(path)).take(REGEX_MAX_NOTES) {
let Ok(text) = self.read_note(&path) else { continue };
if text.len() > REGEX_MAX_NOTE_BYTES {
continue;
}
let (mut count, mut snippets) = (0_u32, Vec::new());
for (number, line) in (1_u32..).zip(text.lines()) {
let Ok(found) = regex.ranges(line, REGEX_LINE_MATCHES) else { continue };
if found.is_empty() {
continue;
}
count = count.saturating_add(u32::try_from(found.len()).unwrap_or(u32::MAX));
if snippets.len() < REGEX_SNIPPETS {
let shown: String = line.chars().take(REGEX_SNIPPET_CHARS).collect();
let ranges = found.into_iter().filter(|range| range.end <= shown.len()).collect();
snippets.push(notescraft_index::search::Snippet { line: number, text: shown, ranges });
}
}
if count > 0 {
hits.push(Hit { path, score: f32::from(u16::try_from(count).unwrap_or(u16::MAX)), title_match: false, snippets });
}
}
hits.sort_by(|a, b| b.score.total_cmp(&a.score).then_with(|| a.path.cmp(&b.path)));
hits.truncate(limit);
Ok(hits)
}
/// Whether [`Session::search`] still sees only part of the vault: the opening scan has not
/// finished, so the UI shows the results as partial and may add a disk search.
#[must_use]
pub fn search_partial(&self) -> bool {
!self.index_progress().finished
}
}
/// Registers the `search` command.
pub(crate) fn search_specs() -> Vec<CommandSpec<Session>> {
vec![
CommandSpec::new("search", "Search", "Ranked full-text search with snippets", |_| true, command_search)
.with_menu(vec!["Edit".to_owned(), "Search".to_owned()])
.with_param(ParamSpec::required(
"query",
ParamKind::String,
"the query: bare terms, \"phrase\", -term, OR, path:, file:, tag:, line:, section:, block:, task:, task-todo:, task-done:",
))
.with_param(ParamSpec::optional("limit", ParamKind::Integer, "maximum hits, 20 by default, at most 200"))
.with_param(ParamSpec::optional(
"match_case",
ParamKind::Bool,
"terms and phrases must be written exactly as in the query, as `match-case:` does inside it",
)),
]
}
/// The `limit` param, refused when it is negative or past `max`.
fn limit_param(params: &Value, default: usize, max: usize) -> std::result::Result<usize, CommandError> {
let Some(raw) = params.get("limit") else { return Ok(default) };
let value = raw.as_i64().ok_or_else(|| CommandError::new("invalid_params", "`limit` must be an integer"))?;
let limit = usize::try_from(value).map_err(|_| CommandError::new("invalid_params", format!("`limit` is {value}, it must be from 0 up")))?;
(limit <= max).then_some(limit).ok_or_else(|| CommandError::new("invalid_params", format!("`limit` is {limit}, the maximum is {max}")))
}
/// `search`: ranked hits with snippets.
fn command_search(session: &mut Session, params: &Value) -> CommandResult {
let query = crate::commands::text_param(params, "query")?;
let limit = limit_param(params, DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT)?;
let match_case = params.get("match_case").and_then(Value::as_bool).unwrap_or(false);
let hits = session.search_matching(&query, limit, match_case).map_err(|error| s(&error))?;
let listed: Vec<Value> = hits
.iter()
.map(|hit| json!({ "path": hit.path, "score": f64::from(hit.score), "title_match": hit.title_match, "snippets": snippet_json(hit) }))
.collect();
crate::commands::json(json!({ "count": listed.len(), "hits": listed }))
}
/// The snippets of one hit as `[{"line": 1, "text": "…"}]`.
fn snippet_json(hit: &Hit) -> Vec<Value> {
hit.snippets.iter().map(|snippet| json!({ "line": snippet.line, "text": snippet.text })).collect()
}
#[cfg(test)]
mod tests {
#[test]
fn a_slash_query_is_a_regular_expression() {
assert_eq!(super::regex_query(" /a+b/ "), Some("a+b"));
assert_eq!(super::regex_query("//"), None);
assert_eq!(super::regex_query("path/to"), None);
}
use super::*;
#[test]
fn limit_defaults_to_the_cap_and_refuses_the_rest() {
let params = Value::Null;
assert_eq!(limit_param(&params, DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT).unwrap_or_default(), DEFAULT_SEARCH_LIMIT);
let zero = json!({ "limit": 0 });
assert_eq!(limit_param(&zero, DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT).unwrap_or_default(), 0);
let big = json!({ "limit": MAX_SEARCH_LIMIT + 1 });
assert!(limit_param(&big, DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT).is_err());
let negative = json!({ "limit": -1 });
assert!(limit_param(&negative, DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT).is_err());
let not_integer = json!({ "limit": "twenty" });
assert!(limit_param(&not_integer, DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT).is_err());
}
#[test]
fn an_empty_query_does_not_match_everything() {
assert!(SearchQuery::parse("").is_err(), "the parser refuses an empty query");
assert!(SearchQuery::parse("-only-negations").is_err(), "and operator-only queries");
}
#[test]
fn the_match_case_param_is_optional_and_off_by_default() {
let params = json!({ "query": "ship" });
assert_eq!(params.get("match_case"), None, "the param is optional");
assert!(!params.get("match_case").and_then(Value::as_bool).unwrap_or(false));
let spec = search_specs().into_iter().find(|spec| spec.id == "search");
assert!(spec.as_ref().is_some_and(|spec| spec.params.iter().any(|param| param.name == "match_case")), "the spec offers the switch");
assert!(spec.as_ref().is_some_and(|spec| spec.params.iter().any(|param| param.kind == ParamKind::Bool)), "as a boolean");
}
}
crates/engine/src/vault/find/switcher.rs
+313
View File
@@ -0,0 +1,313 @@
//! The quick switcher ranking behind `switcher.search` (tasks 9 and 15 of the
//! Obsidian basics).
//!
//! Note titles and aliases match as a case-insensitive fuzzy subsequence with
//! word-start and streak boosts (the idea the command palette's `fuzzy_score`
//! uses, kept here as a pure function so the engine never depends on the UI),
//! plus an exact-match and a prefix tier above them. A query holding `#`
//! narrows to headings of the notes whose title matched the part before it
//! (`#head` alone searches every note), a query holding `^` does the same for
//! `^block-id`s. An empty query lists the most recently modified notes.
use std::collections::HashMap;
use serde::Serialize;
use serde_json::{Value, json};
use notescraft_index::NoteInfo;
use crate::error::{CommandError, Error};
use crate::params::{ParamKind, ParamSpec};
use crate::session::Session;
use crate::spec::{CommandResult, CommandSpec};
/// Longest query the switcher accepts, in characters.
pub const MAX_QUERY_CHARS: usize = 200;
/// Hits returned when the params name no `limit`.
pub const DEFAULT_SWITCHER_LIMIT: usize = 30;
/// Largest `limit` the switcher accepts.
pub const MAX_SWITCHER_LIMIT: usize = 100;
/// Score of a case-insensitive exact match, above every fuzzy score.
const EXACT_SCORE: u32 = 3_000_000;
/// Score of a prefix match, above every plain subsequence score.
const PREFIX_SCORE: u32 = 2_000_000;
/// Fuzzy match score of `query` inside `text`: `None` when `query` is not a
/// subsequence of `text`.
///
/// Case-insensitive. An exact match scores [`EXACT_SCORE`], a prefix match
/// [`PREFIX_SCORE`] plus the subsequence score, and the subsequence score
/// itself rewards runs of consecutive characters and matches at word starts.
///
/// ```
/// use notescraft_engine::switcher::fuzzy_score;
///
/// assert_eq!(fuzzy_score("", "anything"), Some(0));
/// assert!(fuzzy_score("nt", "New note").is_some());
/// assert!(fuzzy_score("nt", "Save").is_none());
/// assert!(fuzzy_score("note", "Note") > fuzzy_score("no", "Note"));
/// assert!(fuzzy_score("no", "Note") > fuzzy_score("te", "Note"));
/// ```
#[must_use]
pub fn fuzzy_score(query: &str, text: &str) -> Option<u32> {
if query.is_empty() {
return Some(0);
}
let query_chars: Vec<char> = query.chars().flat_map(char::to_lowercase).collect();
let text_chars: Vec<char> = text.chars().flat_map(char::to_lowercase).collect();
let mut score: u32 = 0;
let mut streak: u32 = 0;
let mut matched: usize = 0;
let mut last_hit: usize = 0;
for (ti, tc) in text_chars.iter().enumerate() {
let Some(&want) = query_chars.get(matched) else { break };
if *tc != want {
continue;
}
let word_start = ti == 0 || text_chars.get(ti.wrapping_sub(1)).is_some_and(|p| matches!(*p, ' ' | '/' | '-'));
streak = if matched > 0 && ti == last_hit.saturating_add(1) { streak.saturating_add(1) } else { 0 };
score = score.saturating_add(1).saturating_add(streak).saturating_add(u32::from(word_start) * 2);
last_hit = ti;
matched = matched.saturating_add(1);
}
if matched < query_chars.len() {
return None;
}
if query_chars == text_chars {
Some(EXACT_SCORE)
} else if text_chars.starts_with(&query_chars) {
Some(PREFIX_SCORE.saturating_add(score))
} else {
Some(score)
}
}
/// The three parts of a switcher query: the note filter before the first `#`
/// or `^`, the heading text after `#` and the block id after `^`.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
struct QueryParts<'a> {
/// What the note title (or an alias) must match.
note: &'a str,
/// The heading text after `#`, when the query holds one.
heading: Option<&'a str>,
/// The block id after `^`, when the query holds one.
block: Option<&'a str>,
}
/// Splits `query` at its first `#` and `^`: `note#head^blk` keeps the heading
/// text up to the `^` and the block id up to the `#`, so either order works.
/// Byte offsets come from `find`, never arithmetic on hostile input.
fn split_query(query: &str) -> QueryParts<'_> {
let hash = query.find('#');
let caret = query.find('^');
let first = hash.zip(caret).map_or_else(|| hash.or(caret), |(h, c)| Some(h.min(c)));
let note = first.map_or(query, |cut| query.get(..cut).unwrap_or_default());
let tail = |start: Option<usize>, other: Option<usize>| -> Option<&str> {
let start = start?;
let end = other.filter(|o| *o > start).unwrap_or(query.len());
let from = start.saturating_add(1);
if from <= end { query.get(from..end) } else { Some("") }
};
QueryParts { note, heading: tail(hash, caret), block: tail(caret, hash) }
}
/// A row of the switcher: a note, a heading or a block hit.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct SwitchHit {
/// `note`, `heading` or `block`.
pub kind: &'static str,
/// Vault-relative path of the note.
pub path: String,
/// Title (the file stem) of the note.
pub title: String,
/// Heading text, heading hits only.
#[serde(skip_serializing_if = "Option::is_none")]
pub heading: Option<String>,
/// Block id, block hits only.
#[serde(skip_serializing_if = "Option::is_none")]
pub block: Option<String>,
/// The alias the query matched, when it beat the title.
#[serde(skip_serializing_if = "Option::is_none")]
pub alias: Option<String>,
/// Rank, bigger is better.
pub score: u32,
}
/// The best note-level score of `query` against `info`: the title match, or
/// the best alias match when only an alias (or a better-scoring alias) fits.
/// An empty `query` matches every note with score 0.
fn best_note_match(info: &NoteInfo, query: &str) -> Option<(u32, Option<String>)> {
if query.is_empty() {
return Some((0, None));
}
let title = fuzzy_score(query, &info.title).map(|score| (score, None));
let alias = info.aliases.iter().filter_map(|alias| fuzzy_score(query, alias).map(|score| (score, Some(alias.clone())))).max_by_key(|(score, _)| *score);
match (title, alias) {
(Some(t), Some(a)) if a.0 > t.0 => Some(a),
(Some(t), _) => Some(t),
(None, Some(a)) => Some(a),
(None, None) => None,
}
}
/// Switcher queries on a session.
impl Session {
/// Runs the quick-switcher query over the whole vault, best hits first.
///
/// `limit` caps the rows. An empty or blank query lists the most recently
/// modified notes, alphabetical when the file system has no times.
///
/// # Errors
///
/// [`Error::Query`] when the query is longer than [`MAX_QUERY_CHARS`].
pub fn switcher(&self, query: &str, limit: usize) -> crate::error::Result<Vec<SwitchHit>> {
let query = query.trim();
if query.chars().count() > MAX_QUERY_CHARS {
return Err(Error::Query(format!("the query is over {MAX_QUERY_CHARS} characters")));
}
if query.is_empty() {
return Ok(self.recent_notes(limit));
}
let parts = split_query(query);
let mut hits = Vec::new();
for path in self.index().notes() {
let Some(info) = self.index().note(path) else { continue };
let Some((score, alias)) = best_note_match(info, parts.note) else { continue };
if parts.heading.is_none() && parts.block.is_none() {
hits.push(SwitchHit { kind: "note", path: info.path.clone(), title: info.title.clone(), heading: None, block: None, alias, score });
continue;
}
if let Some(heading) = parts.heading {
for h in &info.headings {
if let Some(hit) = fuzzy_score(heading, &h.text) {
hits.push(SwitchHit {
kind: "heading",
path: info.path.clone(),
title: info.title.clone(),
heading: Some(h.text.clone()),
block: None,
alias: None,
score: hit.saturating_add(score),
});
}
}
}
if let Some(block) = parts.block {
for id in &info.block_ids {
if let Some(hit) = fuzzy_score(block, id) {
hits.push(SwitchHit {
kind: "block",
path: info.path.clone(),
title: info.title.clone(),
heading: None,
block: Some(id.clone()),
alias: None,
score: hit.saturating_add(score),
});
}
}
}
}
hits.sort_by(|a, b| b.score.cmp(&a.score).then(a.path.cmp(&b.path)));
hits.truncate(limit);
Ok(hits)
}
/// The most recently modified notes, alphabetical among equals.
fn recent_notes(&self, limit: usize) -> Vec<SwitchHit> {
let mut modified: HashMap<String, u64> = HashMap::new();
for entry in self.vault.list().unwrap_or_default() {
if !entry.is_dir {
modified.insert(entry.path, entry.modified.unwrap_or(0));
}
}
let mut rows: Vec<(u64, String)> = self.index().notes().into_iter().map(|path| (modified.get(path).copied().unwrap_or(0), path.to_owned())).collect();
rows.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1)));
rows.truncate(limit);
rows.into_iter()
.map(|(_, path)| SwitchHit {
kind: "note",
title: self.index().note(&path).map_or_else(String::new, |info| info.title.clone()),
path,
heading: None,
block: None,
alias: None,
score: 0,
})
.collect()
}
}
/// Registers the `switcher.search` command.
pub(crate) fn switcher_specs() -> Vec<CommandSpec<Session>> {
vec![
CommandSpec::new(
"switcher.search",
"Quick switcher",
"Ranked notes and aliases, `note#heading` and `note^block`; empty lists the recent notes",
|_| true,
command_switcher,
)
.with_param(ParamSpec::required("query", ParamKind::String, "the switcher query"))
.with_param(ParamSpec::optional("limit", ParamKind::Integer, "maximum hits, 30 by default, at most 100")),
]
}
/// The `limit` param, refused outside `1..=100`.
fn limit_param(params: &Value) -> std::result::Result<usize, CommandError> {
let Some(raw) = params.get("limit") else { return Ok(DEFAULT_SWITCHER_LIMIT) };
let value = raw.as_i64().ok_or_else(|| CommandError::new("invalid_params", "`limit` must be an integer"))?;
let limit = usize::try_from(value).map_err(|_| CommandError::new("invalid_params", format!("`limit` is {value}, it must be from 1 up")))?;
(1..=MAX_SWITCHER_LIMIT)
.contains(&limit)
.then_some(limit)
.ok_or_else(|| CommandError::new("invalid_params", format!("`limit` is {limit}, it must be from 1 to {MAX_SWITCHER_LIMIT}")))
}
/// `switcher.search`: the ranked rows of the quick switcher.
fn command_switcher(session: &mut Session, params: &Value) -> CommandResult {
let query = crate::commands::text_param(params, "query")?;
let limit = limit_param(params)?;
let hits = session.switcher(&query, limit).map_err(|error| crate::commands::command_error(&error))?;
crate::commands::json(json!({ "count": hits.len(), "hits": hits }))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn exact_beats_prefix_beats_subsequence() {
let exact = fuzzy_score("note", "Note");
let prefix = fuzzy_score("no", "Note");
let sub = fuzzy_score("te", "Note");
assert!(exact.is_some_and(|s| s >= EXACT_SCORE));
assert!(prefix.is_some_and(|s| s >= PREFIX_SCORE));
assert!(sub.is_some_and(|s| s < PREFIX_SCORE));
assert!(exact > prefix && prefix > sub, "exact > prefix > subsequence");
assert!(fuzzy_score("nt", "Save").is_none());
assert_eq!(fuzzy_score("", "anything"), Some(0));
assert!(fuzzy_score("ünïcode", "Ünïcode").is_some_and(|s| s >= EXACT_SCORE), "case folding covers non-ASCII");
}
#[test]
fn a_query_splits_at_the_first_hash_and_caret() {
let parts = split_query("note#head^blk");
assert_eq!(parts, QueryParts { note: "note", heading: Some("head"), block: Some("blk") });
assert_eq!(split_query("#head"), QueryParts { note: "", heading: Some("head"), block: None });
assert_eq!(split_query("note^"), QueryParts { note: "note", heading: None, block: Some("") });
assert_eq!(split_query("note"), QueryParts { note: "note", heading: None, block: None });
assert_eq!(split_query("^blk#head"), QueryParts { note: "", heading: Some("head"), block: Some("blk") });
assert_eq!(split_query("#^"), QueryParts { note: "", heading: Some(""), block: Some("") });
}
#[test]
fn the_limit_param_stays_between_one_and_the_cap() {
assert_eq!(limit_param(&json!({})).unwrap_or_default(), DEFAULT_SWITCHER_LIMIT);
assert_eq!(limit_param(&json!({ "limit": 1 })).unwrap_or_default(), 1);
assert_eq!(limit_param(&json!({ "limit": MAX_SWITCHER_LIMIT })).unwrap_or_default(), MAX_SWITCHER_LIMIT);
for bad in [json!({ "limit": 0 }), json!({ "limit": MAX_SWITCHER_LIMIT + 1 }), json!({ "limit": -3 }), json!({ "limit": "30" })] {
assert!(limit_param(&bad).is_err(), "{bad}");
}
}
}
crates/engine/src/vault/folders.rs
+3 -2
View File
@@ -58,7 +58,7 @@ impl Session {
pub fn delete_folder(&mut self, path: &str) -> Result<()> {
self.existing_folder(path)?;
let files = self.folder_files(path)?;
self.vault.delete_to_trash(path)?;
self.remove_file_or_folder(path)?;
for file in files {
self.buffers.remove(&file);
if is_markdown(&file) {
@@ -77,8 +77,9 @@ impl Session {
/// Moves a folder, rewriting inbound links and bookmark paths, preserving dirty buffers.
/// # Errors
/// Returns path, collision, read-only, rewrite or vault/index errors.
/// Returns path, collision, read-only, rewrite or vault/index errors, and [`Error::Indexing`] while the opening scan runs.
pub fn rename_folder(&mut self, from: &str, to: &str, update_links: bool) -> Result<RenameReport> {
self.require_full_index()?;
self.existing_folder(from)?;
self.folder_path(to)?;
if from == to {
crates/engine/src/vault/indexing.rs
+160
View File
@@ -0,0 +1,160 @@
//! Opening a vault in steps: the file list at once, the notes in time-boxed batches.
//!
//! [`Session::open_lazy`] lists the vault and registers attachments, then leaves every note
//! in a queue. The caller (the UI frame loop) calls [`Session::index_step`] with a time budget
//! until [`IndexProgress::finished`]. The eager [`Session::open`] runs the same steps to the
//! end, so both paths build the same index. A note that cannot be read or indexed becomes a
//! diagnostic, never an error.
use std::collections::VecDeque;
use std::path::Path;
use std::time::{Duration, Instant};
use serde_json::Value;
use crate::error::{Error, Result};
use crate::session::{Session, is_markdown};
use crate::spec::{CommandResult, CommandSpec};
/// How far the opening scan has got.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct IndexProgress {
/// Notes found when the vault was opened.
pub total: usize,
/// Notes handled so far, readable or not.
pub done: usize,
/// No step does any work while this is set.
pub paused: bool,
/// The queue is empty and the index is complete.
pub finished: bool,
}
/// The notes still to read.
#[derive(Debug, Default)]
pub(crate) struct ScanState {
queue: VecDeque<String>,
progress: IndexProgress,
}
impl Session {
/// Opens the vault at `root` for writing without reading any note: the file list and the
/// attachments are known at once, the notes follow through [`Session::index_step`].
///
/// # Errors
///
/// The same as [`Session::open`].
pub fn open_lazy(root: &Path) -> Result<Self> {
let config = crate::config::SessionConfig::default();
config.validate()?;
Ok(Self::with_vault(notescraft_vault::Vault::open(root)?, config))
}
/// How far the opening scan has got.
#[must_use]
pub fn index_progress(&self) -> IndexProgress {
self.scan.progress
}
/// Indexes queued notes until `budget` has passed, at least one when any is queued, and
/// finishes the scan when the queue is empty. Does nothing once finished.
pub fn index_step(&mut self, budget: Duration) -> IndexProgress {
if self.scan.progress.finished || self.scan.progress.paused {
return self.scan.progress;
}
let start = Instant::now();
while let Some(path) = self.scan.queue.pop_front() {
self.index_queued(&path);
if start.elapsed() >= budget {
break;
}
}
if self.scan.queue.is_empty() {
self.knowledge_end_scan();
self.scan.progress.finished = true;
}
self.scan.progress
}
/// Reads one queued note. A note deleted since the listing is skipped quietly; any other
/// failure becomes a diagnostic.
fn index_queued(&mut self, path: &str) {
match self.index_note(path) {
Ok(()) | Err(Error::NotFound(_)) => {}
Err(error) => self.diagnostics.push(format!("{path}: {error}")),
}
self.scan.progress.done = self.scan.progress.done.saturating_add(1);
}
/// Indexes `path` now when it is still queued: the note the user just opened must not wait
/// for its turn.
pub(crate) fn index_now(&mut self, path: &str) {
let Some(position) = self.scan.queue.iter().position(|queued| queued == path) else { return };
self.scan.queue.remove(position);
self.index_queued(path);
}
/// Refuses an operation that needs every note indexed (renames and moves rewrite links
/// from the backlinks, which are incomplete until the scan ends).
///
/// # Errors
///
/// [`Error::Indexing`] until the scan has finished.
pub(crate) fn require_full_index(&self) -> Result<()> {
let progress = self.scan.progress;
if progress.finished { Ok(()) } else { Err(Error::Indexing { done: progress.done, total: progress.total }) }
}
/// Prepares the opening scan: the per-feature state first, then the file list.
pub(crate) fn begin_scan(&mut self) {
self.knowledge_begin_scan();
self.lint_begin_scan();
let entries = match self.vault.list() {
Ok(entries) => entries,
Err(error) => {
self.diagnostics.push(format!("cannot list the vault: {error}"));
return;
}
};
let mut notes = Vec::new();
for entry in entries.iter().filter(|entry| !entry.is_dir) {
if is_markdown(&entry.path) {
notes.push((entry.modified, entry.path.clone()));
} else if let Err(error) = self.index.add_file(&entry.path) {
self.diagnostics.push(format!("{}: {error}", entry.path));
}
}
// Newest first; the listing is sorted by path, so equal times keep a stable order.
notes.sort_by_key(|(modified, _)| std::cmp::Reverse(*modified));
self.scan.queue = notes.into_iter().map(|(_, path)| path).collect();
self.scan.progress.total = self.scan.queue.len();
}
/// Runs the scan to the end, for the callers that want a complete index at once.
pub(crate) fn finish_scan(&mut self) {
self.scan.progress.paused = false;
while !self.index_step(Duration::MAX).finished {}
}
}
/// The `index.pause` and `index.resume` commands.
pub(crate) fn specs() -> Vec<CommandSpec<Session>> {
vec![
CommandSpec::new("index.pause", "Pause indexing", "Stops reading notes until resumed", |_| true, pause),
CommandSpec::new("index.resume", "Resume indexing", "Continues reading notes", |_| true, resume),
]
}
fn pause(session: &mut Session, _params: &Value) -> CommandResult {
session.scan.progress.paused = !session.scan.progress.finished;
crate::commands::json(progress_json(session))
}
fn resume(session: &mut Session, _params: &Value) -> CommandResult {
session.scan.progress.paused = false;
crate::commands::json(progress_json(session))
}
fn progress_json(session: &Session) -> Value {
let progress = session.scan.progress;
serde_json::json!({ "total": progress.total, "done": progress.done, "paused": progress.paused, "finished": progress.finished })
}
crates/engine/src/vault/mod.rs
+5 -1
View File
@@ -5,8 +5,12 @@ pub(crate) mod bookmarks;
pub(crate) mod buffers;
pub(crate) mod daily;
pub(crate) mod external;
pub(crate) mod find;
pub(crate) mod folders;
pub(crate) mod indexing;
pub(crate) mod notes;
pub(crate) mod pdf_preview;
pub(crate) mod properties;
pub(crate) mod search;
pub(crate) mod session;
pub use find::switcher;
pub(crate) use find::{content_search, random_unique, search};
crates/engine/src/vault/notes.rs
+12 -3
View File
@@ -68,6 +68,14 @@ impl Session {
Ok(())
}
/// Removes a file or folder the way `files.deleted_files` says: to the vault trash, or for good.
pub(crate) fn remove_file_or_folder(&self, path: &str) -> std::result::Result<(), notescraft_vault::Error> {
match self.config().files.deleted_files {
crate::prefs::files_config::DeletedFiles::Trash => self.vault.delete_to_trash(path).map(|_| ()),
crate::prefs::files_config::DeletedFiles::Permanent => self.vault.delete_permanently(path),
}
}
/// Moves `path` to the trash and drops it from the index.
///
/// A buffer open on the note is closed, since its file is gone.
@@ -78,8 +86,8 @@ impl Session {
/// [`Error::Vault`].
pub fn delete_note(&mut self, path: &str) -> Result<()> {
let path = self.any_path(path)?;
match self.vault.delete_to_trash(&path) {
Ok(_) => {}
match self.remove_file_or_folder(&path) {
Ok(()) => {}
Err(notescraft_vault::Error::Io { source, .. }) if source.kind() == std::io::ErrorKind::NotFound => {
return Err(Error::NotFound(format!("`{path}` does not exist")));
}
@@ -107,8 +115,9 @@ impl Session {
///
/// [`Error::InvalidPath`], [`Error::NotFound`] when `from` is not a note of
/// this vault, [`Error::Exists`] when `to` is taken, [`Error::ReadOnly`],
/// [`Error::Reserved`] or [`Error::RenameIncomplete`].
/// [`Error::Reserved`], [`Error::RenameIncomplete`] or [`Error::Indexing`] while the opening scan runs.
pub fn rename_note(&mut self, from: &str, to: &str, update_links: bool) -> Result<RenameReport> {
self.require_full_index()?;
let from = self.any_path(from)?;
let to = self.any_path(to)?;
if from == to {
crates/engine/src/vault/pdf_preview.rs
+165
View File
@@ -0,0 +1,165 @@
//! An image of one page of a PDF in the vault, made by the system's own tool (no PDF library is linked):
//! `sips` on macOS (first page), `pdftoppm` elsewhere (any page, when installed). The PNG is cached in the
//! vault's private folder, keyed by the file's size and modified time, so a page is rendered once.
use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
use crate::error::{Error, Result};
use crate::session::Session;
/// JavaScript for Automation that renders page `argv[1]` of the PDF `argv[0]` to the PNG `argv[2]`, longest side
/// `argv[3]` points; no file is written when the page does not exist.
const PDFKIT_PAGE: &str = r"ObjC.import('PDFKit'); ObjC.import('AppKit');
function run(argv) {
var doc = $.PDFDocument.alloc.initWithURL($.NSURL.fileURLWithPath(argv[0]));
var number = parseInt(argv[1]);
if (!doc || number < 1 || doc.pageCount < number) { return 'range'; }
var page = doc.pageAtIndex(number - 1);
var box = page.boundsForBox($.kPDFDisplayBoxMediaBox);
var scale = parseFloat(argv[3]) / Math.max(box.size.width, box.size.height);
var image = page.thumbnailOfSizeForBox($.NSMakeSize(box.size.width * scale, box.size.height * scale), $.kPDFDisplayBoxMediaBox);
var rep = $.NSBitmapImageRep.imageRepWithData(image.TIFFRepresentation);
rep.representationUsingTypeProperties($.NSBitmapImageFileTypePNG, $({})).writeToFileAtomically(argv[2], true);
return 'ok';
}";
/// Largest PDF handed to the tool, in bytes.
const MAX_PDF_BYTES: u64 = 200 * 1024 * 1024;
/// Longest the tool may run, in seconds.
const TOOL_TIMEOUT: Duration = Duration::from_secs(15);
/// Longest side of the image, in pixels.
const MAX_SIDE: &str = "1100";
/// Highest page number accepted.
const MAX_PAGE: u32 = 10_000;
/// Largest image kept, in bytes.
const MAX_PNG_BYTES: usize = 16 * 1024 * 1024;
impl Session {
/// PNG bytes of a still from the video at `path`, cached like [`Session::pdf_preview`].
///
/// # Errors
/// [`Error::InvalidPath`] outside the vault, [`Error::Vault`] when the file is too large or no tool renders it.
pub fn video_poster(&self, path: &str) -> Result<Vec<u8>> {
let source = self.vault.resolve(path).map_err(|error| Error::Vault(error.to_string()))?;
let meta = source.metadata().map_err(|error| Error::Vault(error.to_string()))?;
let stamp = meta.modified().ok().and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()).map_or(0, |d| d.as_secs());
let key = format!("{:016x}", fnv(&format!("{path}|{}|{stamp}|poster", meta.len())));
let cached = format!(".notescraft/cache/poster-{key}.png");
if let Ok(bytes) = self.vault.read(&cached) {
return Ok(bytes);
}
let bytes = poster(&source)?;
let _ = self.vault.write_private(&cached, &bytes);
Ok(bytes)
}
/// PNG bytes of page `page` (1-based) of the PDF at `path`.
///
/// # Errors
/// [`Error::InvalidPath`] for a path outside the vault, [`Error::Vault`] when the file is too large, the
/// page is out of range, or no tool for rendering is installed or it fails.
pub fn pdf_preview(&self, path: &str, page: u32) -> Result<Vec<u8>> {
if !(1..=MAX_PAGE).contains(&page) {
return Err(Error::Vault(format!("page {page} is out of range")));
}
let source = self.vault.resolve(path).map_err(|error| Error::Vault(error.to_string()))?;
let meta = source.metadata().map_err(|error| Error::Vault(error.to_string()))?;
if meta.len() > MAX_PDF_BYTES {
return Err(Error::Vault("the PDF is too large to preview".to_owned()));
}
let stamp = meta.modified().ok().and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()).map_or(0, |d| d.as_secs());
let key = format!("{:016x}", fnv(&format!("{path}|{}|{stamp}|{page}", meta.len())));
let cached = format!(".notescraft/cache/pdf-{key}.png");
if let Ok(bytes) = self.vault.read(&cached) {
return Ok(bytes);
}
let bytes = render(&source, page)?;
let _ = self.vault.write_private(&cached, &bytes);
Ok(bytes)
}
}
/// FNV-1a of `text`.
fn fnv(text: &str) -> u64 {
text.bytes().fold(0xcbf2_9ce4_8422_2325, |hash, byte| (hash ^ u64::from(byte)).wrapping_mul(0x0100_0000_01b3))
}
/// Runs the platform tool for a PDF page and returns the PNG it wrote.
fn render(pdf: &std::path::Path, page: u32) -> Result<Vec<u8>> {
let out = temp_stem(pdf);
let png = out.with_extension("png");
let mut command;
if cfg!(target_os = "macos") {
// PDFKit through JavaScript for Automation, so any page can be asked for; `sips` (first page only)
// is the fallback when that fails.
command = Command::new("osascript");
command.args(["-l", "JavaScript", "-e", PDFKIT_PAGE]).arg(pdf).arg(page.to_string()).arg(&png).arg(MAX_SIDE);
if let Ok(bytes) = run_tool(command, &png) {
return Ok(bytes);
}
if page != 1 {
return Err(Error::Vault(format!("page {page} could not be rendered")));
}
command = Command::new("sips");
command.args(["-s", "format", "png", "-Z", MAX_SIDE]).arg(pdf).arg("--out").arg(&png);
} else {
command = Command::new("pdftoppm");
command.args(["-png", "-singlefile", "-scale-to", MAX_SIDE, "-f", &page.to_string(), "-l", &page.to_string()]).arg(pdf).arg(&out);
}
run_tool(command, &png)
}
/// A temporary path (no extension) for one tool run.
fn temp_stem(source: &std::path::Path) -> std::path::PathBuf {
std::env::temp_dir().join(format!("notescraft-media-{}-{}", std::process::id(), fnv(&source.to_string_lossy())))
}
/// Runs `command` (which must write `png`), waits at most [`TOOL_TIMEOUT`] and returns the image.
fn run_tool(mut command: Command, png: &std::path::Path) -> Result<Vec<u8>> {
command.stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null());
let mut child = command.spawn().map_err(|error| Error::Vault(format!("no renderer available: {error}")))?;
let started = Instant::now();
let status = loop {
match child.try_wait() {
Ok(Some(status)) => break status,
Ok(None) if started.elapsed() < TOOL_TIMEOUT => std::thread::sleep(Duration::from_millis(20)),
_ => {
let _ = child.kill();
let _ = child.wait();
let _ = std::fs::remove_file(png);
return Err(Error::Vault("the renderer did not finish".to_owned()));
}
}
};
let bytes = std::fs::read(png);
let _ = std::fs::remove_file(png);
match bytes {
Ok(bytes) if status.success() && !bytes.is_empty() && bytes.len() <= MAX_PNG_BYTES => Ok(bytes),
_ => Err(Error::Vault("the renderer produced no image".to_owned())),
}
}
/// A still from a video, made by `qlmanage` (macOS) or `ffmpeg` (elsewhere, when installed).
fn poster(video: &std::path::Path) -> Result<Vec<u8>> {
let stem = temp_stem(video);
if cfg!(target_os = "macos") {
let dir = stem.with_extension("d");
std::fs::create_dir_all(&dir).map_err(|error| Error::Vault(error.to_string()))?;
let mut command = Command::new("qlmanage");
command.args(["-t", "-s", MAX_SIDE, "-o"]).arg(&dir).arg(video);
let name = video.file_name().map(|name| format!("{}.png", name.to_string_lossy())).unwrap_or_default();
let result = run_tool(command, &dir.join(name));
let _ = std::fs::remove_dir_all(&dir);
result
} else {
let png = stem.with_extension("png");
let mut command = Command::new("ffmpeg");
command
.args(["-y", "-loglevel", "error", "-ss", "1", "-i"])
.arg(video)
.args(["-frames:v", "1", "-vf", &format!("scale={MAX_SIDE}:-2:force_original_aspect_ratio=decrease")])
.arg(&png);
run_tool(command, &png)
}
}
crates/engine/src/vault/search.rs
-104
View File
@@ -1,104 +0,0 @@
//! Full-text search: the [`Session::search`] query and the `search` command
//! (SPEC 6 item 6).
//!
//! The [`SearchIndex`] mirror lives on the session and is fed everywhere the
//! link index is, so a hit never reflects a state that was already dropped
//! from the disk. Ranking, operators and snippets are the `notescraft-index`
//! implementation; this module only wires them up.
use serde_json::{Value, json};
use notescraft_index::search::{Hit, SearchQuery};
use crate::error::CommandError;
use crate::params::{ParamKind, ParamSpec};
use crate::session::Session;
use crate::spec::{CommandResult, CommandSpec};
/// Hits returned when the params do not name a limit (`search` command).
const DEFAULT_SEARCH_LIMIT: usize = 20;
/// Largest `limit` the `search` command accepts.
const MAX_SEARCH_LIMIT: usize = 200;
/// Maps a session [`crate::error::Error`] onto the `{ code, message }` a
/// transport expects; the `.map_err(|error| s(&error))?` idiom.
fn s(error: &crate::error::Error) -> CommandError {
CommandError::new(error.code(), error.to_string())
}
/// Ranked search over the whole vault (SPEC 6 item 6).
impl Session {
/// Runs `query` against the session's search mirror.
///
/// `limit` caps the hits; the best ones come first. An empty or
/// operator-only query is refused, not treated as "match everything".
///
/// # Errors
///
/// [`SearchQuery::parse`] rejects malformed queries. Searching itself never
/// fails.
pub fn search(&self, query: &str, limit: usize) -> Result<Vec<Hit>, crate::error::Error> {
let parsed = SearchQuery::parse(query).map_err(|error| crate::error::Error::Query(error.to_string()))?;
Ok(self.search_index.search(&parsed, limit))
}
}
/// Registers the `search` command.
pub(crate) fn search_specs() -> Vec<CommandSpec<Session>> {
vec![
CommandSpec::new("search", "Search", "Ranked full-text search with snippets", |_| true, command_search)
.with_menu(vec!["Edit".to_owned(), "Search".to_owned()])
.with_param(ParamSpec::required("query", ParamKind::String, "the query: bare terms, \"phrase\", -term, OR, path:, file:, tag:, line:"))
.with_param(ParamSpec::optional("limit", ParamKind::Integer, "maximum hits, 20 by default, at most 200")),
]
}
/// The `limit` param, refused when it is negative or past `max`.
fn limit_param(params: &Value, default: usize, max: usize) -> std::result::Result<usize, CommandError> {
let Some(raw) = params.get("limit") else { return Ok(default) };
let value = raw.as_i64().ok_or_else(|| CommandError::new("invalid_params", "`limit` must be an integer"))?;
let limit = usize::try_from(value).map_err(|_| CommandError::new("invalid_params", format!("`limit` is {value}, it must be from 0 up")))?;
(limit <= max).then_some(limit).ok_or_else(|| CommandError::new("invalid_params", format!("`limit` is {limit}, the maximum is {max}")))
}
/// `search`: ranked hits with snippets.
fn command_search(session: &mut Session, params: &Value) -> CommandResult {
let query = crate::commands::text_param(params, "query")?;
let limit = limit_param(params, DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT)?;
let hits = session.search(&query, limit).map_err(|error| s(&error))?;
let listed: Vec<Value> = hits
.iter()
.map(|hit| json!({ "path": hit.path, "score": f64::from(hit.score), "title_match": hit.title_match, "snippets": snippet_json(hit) }))
.collect();
crate::commands::json(json!({ "count": listed.len(), "hits": listed }))
}
/// The snippets of one hit as `[{"line": 1, "text": "…"}]`.
fn snippet_json(hit: &Hit) -> Vec<Value> {
hit.snippets.iter().map(|snippet| json!({ "line": snippet.line, "text": snippet.text })).collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn limit_defaults_to_the_cap_and_refuses_the_rest() {
let params = Value::Null;
assert_eq!(limit_param(&params, DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT).unwrap_or_default(), DEFAULT_SEARCH_LIMIT);
let zero = json!({ "limit": 0 });
assert_eq!(limit_param(&zero, DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT).unwrap_or_default(), 0);
let big = json!({ "limit": MAX_SEARCH_LIMIT + 1 });
assert!(limit_param(&big, DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT).is_err());
let negative = json!({ "limit": -1 });
assert!(limit_param(&negative, DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT).is_err());
let not_integer = json!({ "limit": "twenty" });
assert!(limit_param(&not_integer, DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT).is_err());
}
#[test]
fn an_empty_query_does_not_match_everything() {
assert!(SearchQuery::parse("").is_err(), "the parser refuses an empty query");
assert!(SearchQuery::parse("-only-negations").is_err(), "and operator-only queries");
}
}
crates/engine/src/vault/session.rs
+27 -25
View File
@@ -60,6 +60,12 @@ pub struct Session {
pub(crate) pending: Vec<PendingChange>,
/// The highest pending-change id handed out so far.
pub(crate) next_pending_id: u64,
/// The notes still to read after a lazy open.
pub(crate) scan: crate::indexing::ScanState,
/// The format/lint batch: its queue and the proposals read so far.
pub(crate) batch: crate::batch::BatchState,
/// How `grammar.install` downloads; `curl` unless a test swaps it.
pub(crate) grammar_transport: Box<dyn notescraft_syntax::fetch::Transport>,
}
impl Session {
@@ -83,7 +89,9 @@ impl Session {
/// lock is held by a live process.
pub fn open_with(root: &Path, config: SessionConfig) -> Result<Self> {
config.validate()?;
Ok(Self::with_vault(Vault::open(root)?, config))
let mut session = Self::with_vault(Vault::open(root)?, config);
session.finish_scan();
Ok(session)
}
/// Opens the vault at `root` without the lock: the second instance and the
@@ -96,7 +104,9 @@ impl Session {
pub fn open_read_only(root: &Path) -> Result<Self> {
let config = SessionConfig::default();
config.validate()?;
Ok(Self::with_vault(Vault::open_read_only(root)?, config))
let mut session = Self::with_vault(Vault::open_read_only(root)?, config);
session.finish_scan();
Ok(session)
}
/// The vault, for the UI to ask for the root and the watcher.
@@ -208,6 +218,16 @@ impl Session {
self.focused.as_deref()
}
/// Replaces how `grammar.install` downloads (tests; the app keeps `curl`).
pub fn set_grammar_transport(&mut self, transport: Box<dyn notescraft_syntax::fetch::Transport>) {
self.grammar_transport = transport;
}
/// Loads the installed syntax grammars into the highlighter (call once after opening).
pub fn load_grammars(&self) {
crate::grammars::reload_installed(self);
}
/// The folder this session's settings commands read and write, when one
/// was picked explicitly.
pub(crate) fn settings_dir(&self) -> Option<&Path> {
@@ -299,7 +319,7 @@ impl Session {
}
/// Builds the session and fills the indexes.
fn with_vault(vault: Vault, config: SessionConfig) -> Self {
pub(crate) fn with_vault(vault: Vault, config: SessionConfig) -> Self {
let mut session = Self {
git: notescraft_git::Git::open(vault.root()),
vault,
@@ -321,31 +341,13 @@ impl Session {
write_granted: false,
pending: Vec::new(),
next_pending_id: 0,
scan: crate::indexing::ScanState::default(),
batch: crate::batch::BatchState::default(),
grammar_transport: Box::new(notescraft_syntax::fetch::Curl),
};
session.knowledge_begin_scan();
session.lint_begin_scan();
session.scan();
session.knowledge_end_scan();
session.begin_scan();
session
}
/// Walks the vault once: notes into the index, everything else as an
/// attachment. Failures become diagnostics, never an error.
fn scan(&mut self) {
let entries = match self.vault.list() {
Ok(entries) => entries,
Err(error) => {
self.diagnostics.push(format!("cannot list the vault: {error}"));
return;
}
};
for entry in entries.iter().filter(|entry| !entry.is_dir) {
let outcome = if is_markdown(&entry.path) { self.index_note(&entry.path) } else { self.index.add_file(&entry.path).map_err(Error::from) };
if let Err(error) = outcome {
self.diagnostics.push(format!("{}: {error}", entry.path));
}
}
}
}
/// Whether a vault-relative path names a note (case-insensitive `.md`).
crates/engine/tests/batch_commands.rs
+380
View File
@@ -0,0 +1,380 @@
//! The `batch.*` commands: queue a scope, step through it, decide and apply.
mod common;
use std::path::Path;
use std::time::Duration;
use common::{on_disk, seed, temp_vault};
use notescraft_engine::{Error, Registry, Session, SessionConfig, register_core};
use serde_json::{Value, json};
/// A note whose `json` block is not formatted yet.
const JSON_NOTE: &str = "# J\n\n```json\n{\"b\":1,\"a\":2}\n```\n";
/// The same note after the block is formatted.
const JSON_FORMATTED: &str = "# J\n\n```json\n{\n \"b\": 1,\n \"a\": 2\n}\n```\n";
/// An unlabeled fence holding JSON: the change needs review.
const UNLABELED_NOTE: &str = "# U\n\n```\n{\"b\":1}\n```\n";
/// A single trailing space: `trailing-space` fires with one safe edit.
const LINT_NOTE: &str = "# L\n\ntrailing space \n";
const LINT_FIXED: &str = "# L\n\ntrailing space\n";
const CLEAN_NOTE: &str = "# C\n\na clean note\n";
const MAIN_NOTES: &[(&str, &str)] = &[("json.md", JSON_NOTE), ("unlabeled.md", UNLABELED_NOTE), ("lint.md", LINT_NOTE), ("clean.md", CLEAN_NOTE)];
/// A registry and a session on a seeded vault.
fn setup(name: &str, files: &[(&str, &str)]) -> (Registry<Session>, Session, common::TempVault) {
let vault = temp_vault(name);
seed(vault.path(), files);
let mut registry = Registry::new();
register_core(&mut registry).expect("register");
let session = Session::open(vault.path()).expect("open");
(registry, session, vault)
}
fn run(registry: &Registry<Session>, session: &mut Session, id: &str, params: &Value) -> Value {
registry.run(id, session, params).unwrap_or_else(|error| panic!("{id}: {error:?}"))
}
/// Steps until the batch finishes and returns the last progress.
fn drain(registry: &Registry<Session>, session: &mut Session) -> Value {
loop {
let progress = run(registry, session, "batch.step", &json!({ "budget_ms": 1 }));
if progress["finished"] == json!(true) {
return progress;
}
}
}
/// The bytes of every note, to prove the batch wrote nothing.
fn snapshot(vault: &Path, files: &[(&str, &str)]) -> Vec<(String, Vec<u8>)> {
files.iter().map(|(path, _)| ((*path).to_owned(), std::fs::read(vault.join(path)).unwrap_or_default())).collect()
}
/// The error code of a refused command call.
fn code(result: Result<Value, Error>) -> String {
match result {
Err(Error::Command { source, .. }) => source.code.clone(),
Err(other) => panic!("expected a command error, got {other:?}"),
Ok(value) => panic!("expected a refusal, got {value}"),
}
}
/// The row of `path` in a `batch.proposals` answer.
fn row(rows: &[Value], path: &str) -> Value {
rows.iter().find(|row| row["path"] == json!(path)).cloned().unwrap_or_else(|| panic!("no row for {path}"))
}
#[test]
fn start_and_step_review_without_writing() {
let (registry, mut session, vault) = setup("batch-review", MAIN_NOTES);
let before = snapshot(vault.path(), MAIN_NOTES);
let started = run(&registry, &mut session, "batch.start", &json!({ "scope": "vault" }));
assert_eq!(started, json!({ "total": 4 }));
let progress = drain(&registry, &mut session);
assert_eq!(progress["total"], json!(4));
assert_eq!(progress["done"], json!(4));
assert_eq!(progress["finished"], json!(true));
assert_eq!(progress["proposed"], json!(2), "json.md and lint.md");
assert_eq!(progress["needs_review"], json!(1), "unlabeled.md");
assert_eq!(progress["clean"], json!(1), "clean.md");
assert_eq!(progress["not_edited"], json!(0));
assert_eq!(snapshot(vault.path(), MAIN_NOTES), before, "reading the batch writes nothing");
assert_eq!(run(&registry, &mut session, "batch.status", &Value::Null), progress, "status changes nothing");
let list = run(&registry, &mut session, "batch.proposals", &Value::Null);
let rows = list["proposals"].as_array().expect("rows").clone();
assert_eq!(rows.len(), 4);
assert_eq!(row(&rows, "json.md")["status"], json!("proposed"));
assert_eq!(row(&rows, "json.md")["changes"], json!(1));
assert_eq!(row(&rows, "unlabeled.md")["status"], json!("needs_review"));
assert_eq!(row(&rows, "clean.md")["status"], json!("clean"));
assert_eq!(row(&rows, "clean.md")["changes"], json!(0));
assert_eq!(list["truncated"], json!(false));
let only_clean = run(&registry, &mut session, "batch.proposals", &json!({ "status": "clean" }));
assert_eq!(only_clean["proposals"].as_array().expect("rows").len(), 1);
let full = run(&registry, &mut session, "batch.proposal", &json!({ "path": "json.md" }));
assert_eq!(full["path"], json!("json.md"));
assert_eq!(full["status"], json!("Proposed"));
let change = &full["changes"][0];
let at = JSON_NOTE.find("{\"b\":1,\"a\":2}").expect("the JSON core");
assert_eq!(change["id"], json!(0));
assert_eq!(change["kind"], json!("Json"));
assert_eq!(change["range"], json!({ "start": at, "end": at + "{\"b\":1,\"a\":2}".len() }));
assert_eq!(change["before"], json!("{\"b\":1,\"a\":2}"));
assert_eq!(change["after"], json!("{\n \"b\": 1,\n \"a\": 2\n}"));
assert_eq!(change["safe"], json!(true));
assert_eq!(change["decision"], json!("Pending"));
assert_eq!(change["rule"], Value::Null);
let lint_full = run(&registry, &mut session, "batch.proposal", &json!({ "path": "lint.md" }));
let lint_change = &lint_full["changes"][0];
assert_eq!(lint_change["kind"], json!("Lint"));
assert_eq!(lint_change["rule"], json!("trailing-space"));
assert_eq!(lint_change["safe"], json!(true));
}
#[test]
fn decide_then_apply_writes_only_the_accepted_changes() {
let (registry, mut session, vault) = setup("batch-apply", MAIN_NOTES);
run(&registry, &mut session, "batch.start", &json!({ "scope": "vault" }));
drain(&registry, &mut session);
let accepted = run(&registry, &mut session, "batch.decide", &json!({ "path": "json.md", "change_id": 0, "decision": "accept" }));
assert_eq!(accepted["decided"], json!(1));
let skipped = run(&registry, &mut session, "batch.decide", &json!({ "path": "lint.md", "decision": "skip" }));
assert_eq!(skipped["decided"], json!(1));
let applied = run(&registry, &mut session, "batch.apply", &Value::Null);
assert_eq!(applied["written"], json!(1));
assert_eq!(applied["skipped"], json!([]));
assert_eq!(applied["failed"], json!([]));
assert_eq!(on_disk(vault.path(), "json.md"), JSON_FORMATTED);
assert_eq!(on_disk(vault.path(), "lint.md"), LINT_NOTE, "the skipped fix is not applied");
assert_eq!(on_disk(vault.path(), "unlabeled.md"), UNLABELED_NOTE, "pending is not applied");
assert_eq!(on_disk(vault.path(), "clean.md"), CLEAN_NOTE);
let clean = run(&registry, &mut session, "batch.proposals", &json!({ "status": "clean" }));
let rows = clean["proposals"].as_array().expect("rows").clone();
assert_eq!(rows.len(), 2, "json.md became clean; clean.md already was");
assert_eq!(row(&rows, "json.md")["changes"], json!(0), "the written note has no changes left");
let again = run(&registry, &mut session, "batch.apply", &Value::Null);
assert_eq!(again["written"], json!(0));
}
#[test]
fn all_true_accepts_every_change_of_the_proposed_notes() {
let (registry, mut session, vault) = setup("batch-all", MAIN_NOTES);
run(&registry, &mut session, "batch.start", &json!({ "scope": "vault" }));
drain(&registry, &mut session);
let decided = run(&registry, &mut session, "batch.decide", &json!({ "all": true, "decision": "accept" }));
assert_eq!(decided["decided"], json!(2), "the two proposed notes, not the needs-review one");
let applied = run(&registry, &mut session, "batch.apply", &Value::Null);
assert_eq!(applied["written"], json!(2));
assert_eq!(on_disk(vault.path(), "json.md"), JSON_FORMATTED);
assert_eq!(on_disk(vault.path(), "lint.md"), LINT_FIXED);
assert_eq!(on_disk(vault.path(), "unlabeled.md"), UNLABELED_NOTE, "needs review waits for a per-change decision");
}
#[test]
fn apply_skips_a_note_edited_after_the_check() {
let files: &[(&str, &str)] = &[("a.md", JSON_NOTE), ("b.md", JSON_NOTE), ("lint.md", LINT_NOTE)];
let (registry, mut session, vault) = setup("batch-stale", files);
run(&registry, &mut session, "batch.start", &json!({ "scope": "vault" }));
drain(&registry, &mut session);
for path in ["a.md", "b.md"] {
run(&registry, &mut session, "batch.decide", &json!({ "path": path, "decision": "accept" }));
}
let edited = "# A\n\nedited by hand\n";
std::fs::write(vault.path().join("a.md"), edited).expect("edit a.md after the check");
let applied = run(&registry, &mut session, "batch.apply", &Value::Null);
assert_eq!(applied["written"], json!(1));
assert_eq!(applied["skipped"], json!([{ "path": "a.md", "reason": "changed since the check" }]));
assert_eq!(applied["failed"], json!([]));
assert_eq!(on_disk(vault.path(), "a.md"), edited, "the fresh edit stays");
assert_eq!(on_disk(vault.path(), "b.md"), JSON_FORMATTED);
}
#[test]
fn cancel_empties_the_queue_and_keeps_the_proposals() {
let (registry, mut session, _vault) = setup("batch-cancel", MAIN_NOTES);
run(&registry, &mut session, "batch.start", &json!({ "scope": "vault" }));
let first = run(&registry, &mut session, "batch.step", &json!({ "budget_ms": 1 }));
let done = first["done"].as_u64().expect("done");
assert!(done >= 1, "a step always reads at least one note");
let cancelled = run(&registry, &mut session, "batch.cancel", &Value::Null);
assert_eq!(cancelled["done"], json!(done));
assert_eq!(cancelled["finished"], json!(true));
assert_eq!(cancelled["cancelled"], json!(true));
let after = run(&registry, &mut session, "batch.step", &json!({ "budget_ms": 1 }));
assert_eq!(after["done"], json!(done), "nothing is read after the cancel");
let list = run(&registry, &mut session, "batch.proposals", &Value::Null);
assert_eq!(list["proposals"].as_array().expect("rows").len(), usize::try_from(done).unwrap_or(0));
}
#[test]
fn scopes_pick_their_notes_and_reserved_folders_are_never_queued() {
let files: &[(&str, &str)] =
&[("root.md", JSON_NOTE), ("sub/a.md", JSON_NOTE), ("sub/b.md", CLEAN_NOTE), (".obsidian/x.md", JSON_NOTE), (".notescraft/y.md", JSON_NOTE)];
let (registry, mut session, _vault) = setup("batch-scopes", files);
let vault_scope = run(&registry, &mut session, "batch.start", &json!({ "scope": "vault" }));
assert_eq!(vault_scope["total"], json!(3), "the reserved folders are never queued");
let folder = run(&registry, &mut session, "batch.start", &json!({ "scope": "folder", "path": "sub" }));
assert_eq!(folder["total"], json!(2));
let empty = run(&registry, &mut session, "batch.start", &json!({ "scope": "folder", "path": "nope" }));
assert_eq!(empty["total"], json!(0));
let note = run(&registry, &mut session, "batch.start", &json!({ "scope": "note", "path": "sub/a.md" }));
assert_eq!(note["total"], json!(1));
let drained = run(&registry, &mut session, "batch.step", &json!({ "budget_ms": 1000 }));
assert_eq!(drained["done"], json!(1));
let rows = run(&registry, &mut session, "batch.proposals", &Value::Null);
assert_eq!(rows["proposals"][0]["path"], json!("sub/a.md"));
}
#[test]
fn start_is_refused_until_the_opening_scan_ends() {
let vault = temp_vault("batch-indexing");
seed(vault.path(), MAIN_NOTES);
let mut registry = Registry::new();
register_core(&mut registry).expect("register");
let mut session = Session::open_lazy(vault.path()).expect("lazy open");
let refused = registry.run("batch.start", &mut session, &json!({ "scope": "vault" }));
assert_eq!(code(refused), "indexing");
while !session.index_progress().finished {
session.index_step(Duration::from_secs(30));
}
let started = run(&registry, &mut session, "batch.start", &json!({ "scope": "vault" }));
assert_eq!(started["total"], json!(4));
}
#[test]
fn hostile_params_are_refused_and_change_nothing() {
let (registry, mut session, vault) = setup("batch-hostile", MAIN_NOTES);
run(&registry, &mut session, "batch.start", &json!({ "scope": "vault" }));
assert_eq!(code(registry.run("batch.start", &mut session, &json!({ "scope": "x" }))), "invalid_params");
assert_eq!(code(registry.run("batch.start", &mut session, &json!({ "scope": "note" }))), "invalid_params");
assert_eq!(code(registry.run("batch.start", &mut session, &json!({ "scope": "note", "path": "missing.md" }))), "not_found");
assert_eq!(
code(registry.run("batch.start", &mut session, &json!({ "scope": "note", "path": ".obsidian/x.md" }))),
"not_found",
"a reserved path is never a batch note"
);
assert_eq!(code(registry.run("batch.start", &mut session, &json!({ "scope": "folder" }))), "invalid_params");
assert_eq!(code(registry.run("batch.step", &mut session, &json!({ "budget_ms": 0 }))), "invalid_params");
assert_eq!(code(registry.run("batch.step", &mut session, &json!({ "budget_ms": 1_000_000_000 }))), "invalid_params");
assert_eq!(code(registry.run("batch.step", &mut session, &json!({ "budget_ms": -5 }))), "invalid_params");
assert_eq!(code(registry.run("batch.proposals", &mut session, &json!({ "status": "bogus" }))), "invalid_params");
assert_eq!(code(registry.run("batch.proposal", &mut session, &json!({ "path": "missing.md" }))), "not_found");
assert_eq!(code(registry.run("batch.decide", &mut session, &json!({ "path": "missing.md", "decision": "accept" }))), "not_found");
assert_eq!(code(registry.run("batch.decide", &mut session, &json!({ "path": "json.md", "change_id": 1_000_000_000, "decision": "accept" }))), "not_found");
assert_eq!(code(registry.run("batch.decide", &mut session, &json!({ "path": "json.md", "change_id": -1, "decision": "accept" }))), "invalid_params");
assert_eq!(code(registry.run("batch.decide", &mut session, &json!({ "decision": "weird" }))), "invalid_params");
assert_eq!(code(registry.run("batch.decide", &mut session, &json!({ "decision": "accept" }))), "invalid_params");
assert_eq!(code(registry.run("batch.decide", &mut session, &json!({ "all": true, "decision": "skip" }))), "invalid_params");
assert_eq!(code(registry.run("batch.decide", &mut session, &json!({ "all": true, "decision": "accept", "change_id": 3 }))), "invalid_params");
let progress = run(&registry, &mut session, "batch.status", &Value::Null);
assert_eq!(progress["done"], json!(0), "no hostile call touched the batch");
for (path, content) in MAIN_NOTES {
assert_eq!(on_disk(vault.path(), path), *content);
}
}
#[test]
fn unreadable_and_oversized_notes_are_reported_not_edited() {
let huge = "x".repeat(3 * 1024 * 1024);
let files: Vec<(&str, &str)> = vec![("gone.md", CLEAN_NOTE), ("huge.md", huge.as_str()), ("ok.md", JSON_NOTE)];
let (registry, mut session, vault) = setup("batch-refused", &files);
run(&registry, &mut session, "batch.start", &json!({ "scope": "vault" }));
std::fs::remove_file(vault.path().join("gone.md")).expect("delete a note after the start");
let progress = drain(&registry, &mut session);
assert_eq!(progress["not_edited"], json!(2));
assert_eq!(progress["proposed"], json!(1), "ok.md still reviews fine");
let list = run(&registry, &mut session, "batch.proposals", &json!({ "status": "not_edited" }));
let rows = list["proposals"].as_array().expect("rows").clone();
assert_eq!(rows.len(), 2);
let reason = |path: &str| row(&rows, path)["reason"].as_str().map(str::to_owned).unwrap_or_default();
assert!(reason("gone.md").contains("read"), "{}", reason("gone.md"));
assert!(reason("huge.md").contains("2 MiB"), "{}", reason("huge.md"));
}
#[test]
fn a_session_with_lint_off_reports_lint_notes_as_not_edited() {
let vault = temp_vault("batch-lint-off");
seed(vault.path(), MAIN_NOTES);
let mut config = SessionConfig::default();
config.lint.enabled = false;
let mut registry = Registry::new();
register_core(&mut registry).expect("register");
let mut session = Session::open_with(vault.path(), config).expect("open");
// a batch that asks for lint while lint is off reports every note as not edited
run(&registry, &mut session, "batch.start", &json!({ "scope": "vault" }));
let progress = drain(&registry, &mut session);
assert_eq!(progress["not_edited"], json!(4));
assert_eq!(progress["proposed"], json!(0));
// with the lint check off the batch still reviews JSON
run(&registry, &mut session, "batch.start", &json!({ "scope": "vault", "lint": false }));
let progress = drain(&registry, &mut session);
assert_eq!(progress["proposed"], json!(1), "json.md only: no lint fixes were requested");
assert_eq!(progress["needs_review"], json!(1));
assert_eq!(progress["not_edited"], json!(0));
}
/// An unlabeled Rust fence: the label change is a zero-length insertion right after the fence run.
const RUST_NOTE: &str = "# R\n\n```\nfn main() {\n let x = 1;\n}\n```\n";
/// The same note with the detected label inserted.
const RUST_LABELED: &str = "# R\n\n```rust\nfn main() {\n let x = 1;\n}\n```\n";
/// An unlabeled Python fence.
const PYTHON_NOTE: &str = "```\ndef add(a, b):\n return a + b\n```\n";
/// A fence that already carries its label.
const LABELED_NOTE: &str = "```rust\nfn main() {}\n```\n";
#[test]
fn unlabeled_code_blocks_get_label_changes_and_apply_inserts_the_label() {
let files: &[(&str, &str)] = &[("rust.md", RUST_NOTE), ("python.md", PYTHON_NOTE), ("labeled.md", LABELED_NOTE)];
let (registry, mut session, vault) = setup("batch-labels", files);
let started = run(&registry, &mut session, "batch.start", &json!({ "scope": "vault" }));
assert_eq!(started, json!({ "total": 3 }));
let progress = drain(&registry, &mut session);
assert_eq!(progress["needs_review"], json!(2), "rust.md and python.md guess their language");
assert_eq!(progress["clean"], json!(1), "labeled.md already has its label");
let full = run(&registry, &mut session, "batch.proposal", &json!({ "path": "rust.md" }));
assert_eq!(full["status"], json!("NeedsReview"));
let change = &full["changes"][0];
assert_eq!(change["kind"], json!("Label"));
assert_eq!(change["before"], json!(""));
assert_eq!(change["after"], json!("rust"));
assert_eq!(change["safe"], json!(false));
let fence = RUST_NOTE.find("```\n").expect("the opening fence line") + "```".len();
assert_eq!(change["range"], json!({ "start": fence, "end": fence }));
assert_eq!(change["line"], json!(3), "the insertion sits on the fence line");
run(&registry, &mut session, "batch.decide", &json!({ "path": "python.md", "decision": "accept" }));
let applied = run(&registry, &mut session, "batch.apply", &Value::Null);
assert_eq!(applied["written"], json!(1));
assert_eq!(on_disk(vault.path(), "python.md"), "```python\ndef add(a, b):\n return a + b\n```\n");
assert_eq!(on_disk(vault.path(), "rust.md"), RUST_NOTE, "pending is not applied");
assert_eq!(on_disk(vault.path(), "labeled.md"), LABELED_NOTE, "a labeled fence is never relabeled");
}
#[test]
fn labels_false_proposes_nothing_and_an_open_buffer_gets_one_undo_step() {
let files: &[(&str, &str)] = &[("rust.md", RUST_NOTE), ("labeled.md", LABELED_NOTE)];
let (registry, mut session, _vault) = setup("batch-labels-off", files);
run(&registry, &mut session, "batch.start", &json!({ "scope": "vault", "labels": false }));
let progress = drain(&registry, &mut session);
assert_eq!(progress["clean"], json!(2), "with the label check off nothing is guessed");
assert_eq!(progress["needs_review"], json!(0));
assert_eq!(progress["proposed"], json!(0));
// the same note, labels on, applied to an open buffer: ONE undo step takes it back
run(&registry, &mut session, "buffer.open", &json!({ "path": "rust.md" }));
run(&registry, &mut session, "batch.start", &json!({ "scope": "vault", "labels": true }));
drain(&registry, &mut session);
run(&registry, &mut session, "batch.decide", &json!({ "path": "rust.md", "decision": "accept" }));
let applied = run(&registry, &mut session, "batch.apply", &Value::Null);
assert_eq!(applied["written"], json!(1));
let opened = run(&registry, &mut session, "buffer.open", &json!({ "path": "rust.md" }));
assert_eq!(opened["text"], json!(RUST_LABELED));
let undone = run(&registry, &mut session, "buffer.undo", &json!({ "path": "rust.md" }));
assert_eq!(undone["undone"], json!(true), "one undo step covers the whole apply");
let back = run(&registry, &mut session, "buffer.open", &json!({ "path": "rust.md" }));
assert_eq!(back["text"], json!(RUST_NOTE));
}
crates/engine/tests/batch_labels.rs
+118
View File
@@ -0,0 +1,118 @@
//! The `labels` batch check: proposing and applying language labels for unlabeled code blocks.
use notescraft_engine::{BatchDecision, Change, ChangeKind, Checks, Status, apply_changes, propose_note};
use notescraft_lint::{Edit, Fix, Issue, Range as LintRange, Severity};
/// The `Checks` of a labels-only batch: no JSON formatting, no lint fixes.
const LABELS: Checks = Checks { json: false, lint: false, labels: true, replace: None };
/// An issue whose fix is one edit replacing `start..end` with `replacement`.
fn lint_issue(rule: &str, start: usize, end: usize, replacement: &str, safe: bool) -> Issue {
Issue {
rule: rule.to_owned(),
severity: Severity::Info,
range: LintRange { start, end },
line: 1,
message: String::new(),
fix: Some(Fix { label: String::new(), edits: vec![Edit { range: LintRange { start, end }, text: replacement.to_owned() }], safe }),
extra_fixes: Vec::new(),
hash: 0,
}
}
#[test]
fn unlabeled_fences_get_label_changes_that_need_review() {
let note = "# T\n\n```\nfn main() {\n let x = 1;\n}\n```\n";
let proposal = propose_note("n.md", note, &[], LABELS);
assert_eq!(proposal.status, Status::NeedsReview);
let c = proposal.changes.first().expect("one change");
assert_eq!(c.kind, ChangeKind::Label);
let fence = note.find("```\n").expect("the opening fence") + "```".len();
assert_eq!(c.range, fence..fence, "zero-length range right after the fence run");
assert_eq!(c.before, "");
assert_eq!(c.after, "rust");
assert!(!c.safe, "a guess always needs review");
assert_eq!(c.rule, None);
assert_eq!(c.decision, BatchDecision::Pending);
}
#[test]
fn labeled_fences_and_labels_off_get_no_label_changes() {
for note in ["```rust\nfn main() {}\n```\n", "~~~python\nprint(1)\n~~~\n"] {
let proposal = propose_note("n.md", note, &[], LABELS);
assert_eq!(proposal.changes, Vec::new(), "{note:?}");
assert_eq!(proposal.status, Status::Clean, "{note:?}");
}
let unlabeled = "```\nfn main() {\n let x = 1;\n}\n```\n";
let off = Checks { labels: false, ..LABELS };
let proposal = propose_note("n.md", unlabeled, &[], off);
assert_eq!(proposal.changes, Vec::new());
assert_eq!(proposal.status, Status::Clean);
}
#[test]
fn an_unlabeled_json_block_keeps_its_json_change_only() {
let note = "```\n{\"a\":1,\"b\":2}\n```\n";
let both = || Checks { json: true, ..LABELS };
let proposal = propose_note("n.md", note, &[], both());
assert_eq!(proposal.changes.len(), 1, "no label change on top of the JSON change");
assert_eq!(proposal.changes.first().map(|c| c.kind), Some(ChangeKind::Json));
// already-formatted unlabeled JSON still gets its label: there is no JSON change to keep
let formatted = "```\n{\n \"a\": 1\n}\n```\n";
let proposal = propose_note("n.md", formatted, &[], both());
let c = proposal.changes.first().expect("one change");
assert_eq!(c.kind, ChangeKind::Label);
assert_eq!(c.after, "json");
assert!(!c.safe);
}
#[test]
fn applying_a_label_change_inserts_right_after_the_fence_run() {
let note = "# T\n\n```\ndef add(a, b):\n return a + b\n```\n";
let mut proposal = propose_note("n.md", note, &[], LABELS);
for c in &mut proposal.changes {
c.decision = BatchDecision::Accept;
}
let (new, applied, skipped) = apply_changes(note, &proposal.changes);
assert_eq!(applied, vec![0]);
assert_eq!(skipped, Vec::<usize>::new());
assert_eq!(new, "# T\n\n```python\ndef add(a, b):\n return a + b\n```\n");
}
#[test]
fn a_label_point_inside_another_change_range_is_dropped() {
let note = "```\nfn main() {\n let x = 1;\n}\n```\n";
let fence_line = note.find("```\n").expect("the opening fence line") + "```\n".len();
let issue = lint_issue("whole-line", 0, fence_line, "```rust\n", true);
let with_lint = Checks { lint: true, ..LABELS };
let proposal = propose_note("n.md", note, std::slice::from_ref(&issue), with_lint);
assert_eq!(proposal.changes.len(), 1, "the label inside the lint change is dropped");
assert_eq!(proposal.changes.first().map(|c| c.kind), Some(ChangeKind::Lint));
}
#[test]
fn prose_and_ambiguous_blocks_get_no_label_change() {
for note in ["```\njust a sentence\n```\n", "```\na: b\n```\n", "```\nx = 1\n```\n"] {
let proposal = propose_note("n.md", note, &[], LABELS);
assert_eq!(proposal.changes, Vec::new(), "{note:?}");
assert_eq!(proposal.status, Status::Clean, "{note:?}");
}
}
#[test]
fn many_blocks_in_one_note_each_get_their_own_label_change() {
let note = "```\nfn main() {\n let x = 1;\n}\n```\n\ntext\n\n```\n$ cargo build\n$ cargo test\n```\n\n```\ndef add(a, b):\n return a + b\n```\n";
let proposal = propose_note("n.md", note, &[], LABELS);
let after: Vec<&str> = proposal.changes.iter().map(|c: &Change| c.after.as_str()).collect();
assert_eq!(after, vec!["rust", "bash", "python"]);
assert_eq!(proposal.status, Status::NeedsReview);
for (change, language) in proposal.changes.iter().zip(["rust", "bash", "python"]) {
let fence = match language {
"rust" => note.find("```\nfn main"),
"bash" => note.find("```\n$ cargo build"),
_ => note.find("```\ndef add"),
}
.expect("the opening fence");
assert_eq!(change.range, fence + "```".len()..fence + "```".len(), "{language} inserts after its own fence run");
}
}
crates/engine/tests/batch_replace.rs
+311
View File
@@ -0,0 +1,311 @@
//! The `replace` batch check: a vault-wide find and replace reviewed change by change.
mod common;
use std::path::Path;
use common::{on_disk, seed, temp_vault};
use notescraft_engine::{Error, Registry, Session, register_core};
use serde_json::{Value, json};
/// A note with two matches of `tea`.
const TEA_NOTE: &str = "---\ntitle: tea\n---\n\ntea time, tea again\n";
/// `TEA_NOTE` with every `tea` replaced by `coffee`.
const TEA_REPLACED: &str = "---\ntitle: coffee\n---\n\ncoffee time, coffee again\n";
/// A note with no match at all.
const CLEAN_NOTE: &str = "# C\n\nwater\n";
/// A note holding `tea` inside words, a code fence and the frontmatter.
const TRICKY_NOTE: &str = "teapot tea\n\n```\ntea\n```\n";
/// A note a replace run touches, and the format and lint checks would too.
const MIXED_NOTE: &str = "```json\n{\"a\":1,\"b\":2}\n```\n\ntea \n";
const NOTES: &[(&str, &str)] = &[("tea.md", TEA_NOTE), ("clean.md", CLEAN_NOTE), ("tricky.md", TRICKY_NOTE)];
/// A registry and a session on a seeded vault.
fn setup(name: &str, files: &[(&str, &str)]) -> (Registry<Session>, Session, common::TempVault) {
let vault = temp_vault(name);
seed(vault.path(), files);
let mut registry = Registry::new();
register_core(&mut registry).expect("register");
let session = Session::open(vault.path()).expect("open");
(registry, session, vault)
}
fn run(registry: &Registry<Session>, session: &mut Session, id: &str, params: &Value) -> Value {
registry.run(id, session, params).unwrap_or_else(|error| panic!("{id}: {error:?}"))
}
/// The bytes of every note, to prove the batch wrote nothing.
fn snapshot(vault: &Path, files: &[(&str, &str)]) -> Vec<(String, Vec<u8>)> {
files.iter().map(|(path, _)| ((*path).to_owned(), std::fs::read(vault.join(path)).unwrap_or_default())).collect()
}
/// Steps until the batch finishes and returns the last progress.
fn drain(registry: &Registry<Session>, session: &mut Session) -> Value {
loop {
let progress = run(registry, session, "batch.step", &json!({ "budget_ms": 1 }));
if progress["finished"] == json!(true) {
return progress;
}
}
}
/// The error code of a refused command call.
fn code(result: Result<Value, Error>) -> String {
match result {
Err(Error::Command { source, .. }) => source.code.clone(),
Err(Error::InvalidParams { .. }) => "invalid_params".to_owned(),
Err(other) => panic!("expected a command error, got {other:?}"),
Ok(value) => panic!("expected a refusal, got {value}"),
}
}
/// The row of `path` in a `batch.proposals` answer.
fn row(rows: &[Value], path: &str) -> Value {
rows.iter().find(|row| row["path"] == json!(path)).cloned().unwrap_or_else(|| panic!("no row for {path}"))
}
/// Starts a replace batch over the whole vault and drains it.
fn replace_batch(registry: &Registry<Session>, session: &mut Session, params: &Value) -> Value {
let mut start = json!({ "scope": "vault", "replace_find": "tea" });
merge(&mut start, params);
run(registry, session, "batch.start", &start);
drain(registry, session)
}
/// Copies the pairs of `extra` into `base`.
fn merge(base: &mut Value, extra: &Value) {
if let (Some(map), Some(extra)) = (base.as_object_mut(), extra.as_object()) {
for (key, value) in extra {
map.insert(key.clone(), value.clone());
}
}
}
#[test]
fn a_replace_run_reviews_without_writing() {
let (registry, mut session, vault) = setup("batch-replace-dry", NOTES);
let before = snapshot(vault.path(), NOTES);
let progress = replace_batch(&registry, &mut session, &json!({ "replace_with": "coffee" }));
assert_eq!(progress["total"], json!(3));
assert_eq!(progress["needs_review"], json!(2), "every replace change needs a look");
assert_eq!(progress["clean"], json!(1), "clean.md holds no match");
assert_eq!(progress["proposed"], json!(0), "a replace change is never safe");
assert_eq!(snapshot(vault.path(), NOTES), before, "reading the batch writes nothing");
let list = run(&registry, &mut session, "batch.proposals", &Value::Null);
let rows = list["proposals"].as_array().expect("rows").clone();
assert_eq!(row(&rows, "tea.md")["changes"], json!(3), "the frontmatter match counts too");
assert_eq!(row(&rows, "tricky.md")["changes"], json!(3), "code fence and words count too");
assert_eq!(row(&rows, "clean.md")["changes"], json!(0));
let full = run(&registry, &mut session, "batch.proposal", &json!({ "path": "tea.md" }));
let change = &full["changes"][0];
assert_eq!(change["kind"], json!("Replace"));
assert_eq!(change["before"], json!("tea"));
assert_eq!(change["after"], json!("coffee"));
assert_eq!(change["safe"], json!(false));
assert_eq!(change["decision"], json!("Pending"));
assert_eq!(change["rule"], Value::Null);
assert_eq!(change["line"], json!(2), "the frontmatter match sits on line 2");
let at = TEA_NOTE.find("tea").expect("the first match");
assert_eq!(change["range"], json!({ "start": at, "end": at + 3 }));
}
#[test]
fn case_and_whole_word_switches_change_what_matches() {
let (registry, mut session, _vault) = setup("batch-replace-switches", NOTES);
replace_batch(&registry, &mut session, &json!({ "replace_case": false }));
let rows = run(&registry, &mut session, "batch.proposals", &Value::Null);
let rows = rows["proposals"].as_array().expect("rows").clone();
assert_eq!(row(&rows, "tea.md")["changes"], json!(3), "lowercase text matches the same either way");
replace_batch(&registry, &mut session, &json!({ "replace_case": false, "replace_find": "TEA" }));
let rows = run(&registry, &mut session, "batch.proposals", &Value::Null);
let rows = rows["proposals"].as_array().expect("rows").clone();
assert_eq!(row(&rows, "tea.md")["changes"], json!(3), "folded case finds the lowercase matches");
assert_eq!(row(&rows, "tricky.md")["changes"], json!(3));
replace_batch(&registry, &mut session, &json!({ "replace_word": true }));
let rows = run(&registry, &mut session, "batch.proposals", &Value::Null);
let rows = rows["proposals"].as_array().expect("rows").clone();
assert_eq!(row(&rows, "tricky.md")["changes"], json!(2), "`teapot` is not a whole word");
assert_eq!(row(&rows, "tea.md")["changes"], json!(3));
replace_batch(&registry, &mut session, &json!({ "replace_case": false, "replace_word": true }));
let rows = run(&registry, &mut session, "batch.proposals", &Value::Null);
let rows = rows["proposals"].as_array().expect("rows").clone();
assert_eq!(row(&rows, "tricky.md")["changes"], json!(2), "both switches together");
}
#[test]
fn decide_one_change_or_all_then_apply_writes_only_what_was_accepted() {
let (registry, mut session, vault) = setup("batch-replace-apply", NOTES);
// an open buffer shows each written file as ONE commit
run(&registry, &mut session, "buffer.open", &json!({ "path": "tea.md" }));
run(&registry, &mut session, "buffer.open", &json!({ "path": "tricky.md" }));
replace_batch(&registry, &mut session, &json!({ "replace_with": "coffee" }));
let one = run(&registry, &mut session, "batch.decide", &json!({ "path": "tea.md", "change_id": 0, "decision": "accept" }));
assert_eq!(one["decided"], json!(1));
for change_id in 1..3 {
let rest = run(&registry, &mut session, "batch.decide", &json!({ "path": "tea.md", "change_id": change_id, "decision": "skip" }));
assert_eq!(rest["decided"], json!(1), "the other matches are skipped one by one");
}
let every = run(&registry, &mut session, "batch.decide", &json!({ "path": "tricky.md", "decision": "accept" }));
assert_eq!(every["decided"], json!(3));
let applied = run(&registry, &mut session, "batch.apply", &Value::Null);
assert_eq!(applied["written"], json!(2), "only the notes with accepted changes are written");
assert_eq!(applied["skipped"], json!([]));
assert_eq!(applied["failed"], json!([]));
assert_eq!(on_disk(vault.path(), "clean.md"), CLEAN_NOTE, "a note with no match is never written");
// an open buffer takes the whole file in ONE commit, so one undo takes it all back
for (path, changed, before) in
[("tea.md", "---\ntitle: coffee\n---\n\ntea time, tea again\n", TEA_NOTE), ("tricky.md", "coffeepot coffee\n\n```\ncoffee\n```\n", TRICKY_NOTE)]
{
let now = run(&registry, &mut session, "buffer.open", &json!({ "path": path }));
assert_eq!(now["text"], json!(changed), "{path}: only the accepted matches");
let undone = run(&registry, &mut session, "buffer.undo", &json!({ "path": path }));
assert_eq!(undone["undone"], json!(true), "{path}: one undo step");
let back = run(&registry, &mut session, "buffer.open", &json!({ "path": path }));
assert_eq!(back["text"], json!(before), "{path}: the whole apply is undone at once");
assert_eq!(run(&registry, &mut session, "buffer.undo", &json!({ "path": path }))["undone"], json!(false), "{path}: nothing left");
}
assert_eq!(run(&registry, &mut session, "batch.apply", &Value::Null)["written"], json!(0));
}
#[test]
fn a_note_edited_after_the_check_is_skipped() {
let (registry, mut session, vault) = setup("batch-replace-stale", NOTES);
replace_batch(&registry, &mut session, &json!({ "replace_with": "coffee" }));
for path in ["tea.md", "tricky.md"] {
run(&registry, &mut session, "batch.decide", &json!({ "path": path, "decision": "accept" }));
}
let edited = "# tea\n\nedited by hand\n";
std::fs::write(vault.path().join("tea.md"), edited).expect("edit tea.md after the check");
let applied = run(&registry, &mut session, "batch.apply", &Value::Null);
assert_eq!(applied["written"], json!(1));
assert_eq!(applied["skipped"], json!([{ "path": "tea.md", "reason": "changed since the check" }]));
assert_eq!(applied["failed"], json!([]));
assert_eq!(on_disk(vault.path(), "tea.md"), edited, "the fresh edit stays");
assert_eq!(on_disk(vault.path(), "tricky.md"), "coffeepot coffee\n\n```\ncoffee\n```\n");
}
#[test]
fn reserved_folders_are_never_queued_nor_written() {
let files: &[(&str, &str)] = &[("tea.md", TEA_NOTE), (".obsidian/tea.md", TEA_NOTE), (".notescraft/tea.md", TEA_NOTE)];
let (registry, mut session, vault) = setup("batch-replace-reserved", files);
let started = replace_batch(&registry, &mut session, &json!({ "replace_with": "coffee" }));
assert_eq!(started["total"], json!(1));
run(&registry, &mut session, "batch.decide", &json!({ "path": "tea.md", "decision": "accept" }));
let applied = run(&registry, &mut session, "batch.apply", &Value::Null);
assert_eq!(applied["written"], json!(1));
assert_eq!(on_disk(vault.path(), "tea.md"), TEA_REPLACED);
assert_eq!(on_disk(vault.path(), ".obsidian/tea.md"), TEA_NOTE, "reserved notes are never written");
assert_eq!(on_disk(vault.path(), ".notescraft/tea.md"), TEA_NOTE);
}
#[test]
fn one_note_stops_at_two_thousand_changes() {
let huge = "tea ".repeat(3000);
let files: Vec<(&str, &str)> = vec![("huge.md", huge.as_str()), ("small.md", CLEAN_NOTE)];
let (registry, mut session, vault) = setup("batch-replace-cap", &files);
let progress = replace_batch(&registry, &mut session, &json!({ "replace_with": "coffee" }));
assert_eq!(progress["needs_review"], json!(1));
let list = run(&registry, &mut session, "batch.proposals", &Value::Null);
let rows = list["proposals"].as_array().expect("rows").clone();
assert_eq!(row(&rows, "huge.md")["changes"], json!(2000), "a note never gets more than 2000 changes");
assert_eq!(row(&rows, "huge.md")["status"], json!("needs_review"));
run(&registry, &mut session, "batch.decide", &json!({ "path": "huge.md", "decision": "accept" }));
let applied = run(&registry, &mut session, "batch.apply", &Value::Null);
assert_eq!(applied["written"], json!(1));
let written = on_disk(vault.path(), "huge.md");
assert_eq!(written.matches("coffee").count(), 2000, "only the accepted matches are written");
assert_eq!(written.matches("tea").count(), 1000, "the rest of the note is left alone");
}
#[test]
fn unicode_notes_and_find_strings_are_matched_on_char_boundaries() {
let note = "café ISTANBUL ẞ\n\nstraße\n";
let files: &[(&str, &str)] = &[("uni.md", note)];
let (registry, mut session, vault) = setup("batch-replace-unicode", files);
// `ß` folds to itself: a lowercase `ß` find does not touch `ẞ` unless case is off
replace_batch(&registry, &mut session, &json!({ "replace_find": "ß" }));
let list = run(&registry, &mut session, "batch.proposals", &Value::Null);
let rows = list["proposals"].as_array().expect("rows").clone();
assert_eq!(row(&rows, "uni.md")["changes"], json!(1), "only the lowercase `ß` matches");
replace_batch(&registry, &mut session, &json!({ "replace_find": "ß", "replace_case": false }));
let list = run(&registry, &mut session, "batch.proposals", &Value::Null);
let rows = list["proposals"].as_array().expect("rows").clone();
assert_eq!(row(&rows, "uni.md")["changes"], json!(2), "`ẞ` folds to `ß` too");
replace_batch(&registry, &mut session, &json!({ "replace_find": "istanbul", "replace_case": false }));
let list = run(&registry, &mut session, "batch.proposals", &Value::Null);
let rows = list["proposals"].as_array().expect("rows").clone();
assert_eq!(row(&rows, "uni.md")["changes"], json!(1), "folded case finds `ISTANBUL`");
replace_batch(&registry, &mut session, &json!({ "replace_find": "é", "replace_with": "e", "replace_case": false }));
run(&registry, &mut session, "batch.decide", &json!({ "path": "uni.md", "decision": "accept" }));
let applied = run(&registry, &mut session, "batch.apply", &Value::Null);
assert_eq!(applied["written"], json!(1));
assert_eq!(on_disk(vault.path(), "uni.md"), "cafe ISTANBUL ẞ\n\nstraße\n", "only the one match is written");
}
#[test]
fn the_other_checks_default_to_off_for_a_pure_replace_run() {
let files: &[(&str, &str)] = &[("tea.md", MIXED_NOTE)];
let (registry, mut session, vault) = setup("batch-replace-only", files);
let progress = replace_batch(&registry, &mut session, &json!({ "replace_with": "coffee" }));
assert_eq!(progress["needs_review"], json!(1));
let full = run(&registry, &mut session, "batch.proposal", &json!({ "path": "tea.md" }));
assert_eq!(full["changes"].as_array().expect("changes").len(), 1, "the JSON and lint changes are off");
assert_eq!(full["changes"][0]["kind"], json!("Replace"));
run(&registry, &mut session, "batch.decide", &json!({ "path": "tea.md", "decision": "accept" }));
run(&registry, &mut session, "batch.apply", &Value::Null);
assert_eq!(on_disk(vault.path(), "tea.md"), "```json\n{\"a\":1,\"b\":2}\n```\n\ncoffee \n", "only the match is written");
// naming another check turns the defaults back on
std::fs::write(vault.path().join("tea.md"), MIXED_NOTE).expect("put the note back");
replace_batch(&registry, &mut session, &json!({ "replace_with": "coffee", "json": true }));
let full = run(&registry, &mut session, "batch.proposal", &json!({ "path": "tea.md" }));
let kinds: Vec<&str> = full["changes"].as_array().expect("changes").iter().filter_map(|change| change["kind"].as_str()).collect();
assert_eq!(kinds, vec!["Json", "Replace", "Lint"], "an explicit `json` keeps the other defaults");
// the checks alone, without a find, still work
let started = run(&registry, &mut session, "batch.start", &json!({ "scope": "vault" }));
assert_eq!(started, json!({ "total": 1 }));
let progress = drain(&registry, &mut session);
assert_eq!(progress["proposed"], json!(1), "no replace check, no replace change");
let full = run(&registry, &mut session, "batch.proposal", &json!({ "path": "tea.md" }));
assert_eq!(full["changes"][0]["kind"], json!("Json"));
}
#[test]
fn hostile_replace_params_are_refused_and_change_nothing() {
let (registry, mut session, vault) = setup("batch-replace-hostile", NOTES);
let huge = "a".repeat(501);
let refused = registry.run("batch.start", &mut session, &json!({ "scope": "vault", "replace_find": huge }));
assert_eq!(code(refused), "invalid_params", "a find string over 500 bytes is refused");
let refused = registry.run("batch.start", &mut session, &json!({ "scope": "vault", "replace_find": "tea", "replace_with": 7 }));
assert_eq!(code(refused), "invalid_params", "a non-string replacement is refused");
let refused = registry.run("batch.start", &mut session, &json!({ "scope": "vault", "replace_find": "tea", "replace_case": "yes" }));
assert_eq!(code(refused), "invalid_params", "a non-bool switch is refused");
// the empty find is not an error, it just turns the check off
let empty = run(&registry, &mut session, "batch.start", &json!({ "scope": "vault", "replace_find": "" }));
assert_eq!(empty["total"], json!(3));
let progress = drain(&registry, &mut session);
assert_eq!(progress["clean"], json!(3), "an empty find changes nothing");
for (path, content) in NOTES {
assert_eq!(on_disk(vault.path(), path), *content);
}
}
crates/engine/tests/block_commands.rs
+30
View File
@@ -61,3 +61,33 @@ fn hostile_input_is_refused_not_applied() {
assert!(registry.run("block.turn_into", &mut session, &json!({ "path": "n.md", "index": 1, "target": "heading9" })).is_err());
assert_eq!(on_disk(vault.path(), "n.md"), NOTE);
}
const JSON_NOTE: &str = "# T\n\n```json\n{\"b\":1,\"a\":[1e5]}\n```\n";
#[test]
fn json_format_and_minify_rewrite_only_the_block_body() {
let vault = temp_vault("json-block");
seed(vault.path(), &[("j.md", JSON_NOTE)]);
let mut registry = Registry::new();
register_core(&mut registry).expect("register");
let mut session = Session::open(vault.path()).expect("open");
let result = run(&registry, &mut session, "json.format_block", &json!({ "path": "j.md", "index": 1 }));
assert_eq!(result["changed"], json!(true));
assert_eq!(on_disk(vault.path(), "j.md"), "# T\n\n```json\n{\n \"b\": 1,\n \"a\": [\n 1e5\n ]\n}\n```\n");
run(&registry, &mut session, "json.minify_block", &json!({ "path": "j.md", "index": 1 }));
assert_eq!(on_disk(vault.path(), "j.md"), JSON_NOTE);
}
#[test]
fn invalid_json_is_an_error_and_the_note_is_untouched() {
let vault = temp_vault("json-invalid");
let broken = "```json\n{\"a\":\n```\n";
seed(vault.path(), &[("j.md", broken), ("p.md", "plain\n")]);
let mut registry = Registry::new();
register_core(&mut registry).expect("register");
let mut session = Session::open(vault.path()).expect("open");
assert!(registry.run("json.format_block", &mut session, &json!({ "path": "j.md", "index": 0 })).is_err());
assert_eq!(on_disk(vault.path(), "j.md"), broken);
assert!(registry.run("json.format_block", &mut session, &json!({ "path": "p.md", "index": 0 })).is_err());
assert_eq!(on_disk(vault.path(), "p.md"), "plain\n");
}
crates/engine/tests/content_search.rs
+108
View File
@@ -0,0 +1,108 @@
//! Content search over the files on disk: works with no index, stops on request, bounded.
mod common;
use std::time::{Duration, Instant};
use common::{seed, temp_vault};
use notescraft_engine::{ContentHit, ContentSearch};
fn collect(mut search: ContentSearch) -> Vec<ContentHit> {
let deadline = Instant::now() + Duration::from_secs(30);
let mut hits = Vec::new();
loop {
let poll = search.poll();
hits.extend(poll.hits);
if poll.done {
return hits;
}
assert!(Instant::now() < deadline, "the search never finished");
std::thread::sleep(Duration::from_millis(5));
}
}
fn paths(hits: &[ContentHit]) -> Vec<&str> {
let mut paths: Vec<&str> = hits.iter().map(|hit| hit.path.as_str()).collect();
paths.dedup();
paths.sort_unstable();
paths
}
#[test]
fn finds_notes_with_every_term_on_one_line_ignoring_case() {
let vault = temp_vault("cs_basic");
seed(
vault.path(),
&[
("a.md", "first line\nThe Zebra crossed the road\n"),
("sub/b.md", "zebra only\nroad only\n"),
("c.md", "nothing here"),
("pic.png", "zebra road"),
(".hidden/d.md", "zebra road"),
],
);
let hits = collect(ContentSearch::start(vault.path(), "zebra ROAD", 10).expect("start"));
assert_eq!(paths(&hits), vec!["a.md"]);
assert_eq!((hits[0].line, hits[0].text.as_str()), (2, "The Zebra crossed the road"));
}
#[test]
fn an_empty_query_is_refused() {
let vault = temp_vault("cs_empty");
assert!(ContentSearch::start(vault.path(), " ", 10).is_err());
}
#[test]
fn the_file_limit_stops_the_search() {
let vault = temp_vault("cs_limit");
let files: Vec<(String, String)> = (0..30).map(|n| (format!("n{n:02}.md"), "needle".to_owned())).collect();
let borrowed: Vec<(&str, &str)> = files.iter().map(|(p, t)| (p.as_str(), t.as_str())).collect();
seed(vault.path(), &borrowed);
let hits = collect(ContentSearch::start(vault.path(), "needle", 5).expect("start"));
assert_eq!(paths(&hits).len(), 5);
}
#[test]
fn hostile_files_are_skipped_without_panic() {
let vault = temp_vault("cs_hostile");
seed(vault.path(), &[("ok.md", "needle in a good note\n")]);
std::fs::write(vault.path().join("binary.md"), b"needle\0\0\x01\x02 binary").expect("write");
std::fs::write(vault.path().join("latin1.md"), b"caf\xE9 needle \xFF\xFE\n").expect("write");
let huge = vec![b'x'; 9 * 1024 * 1024];
std::fs::write(vault.path().join("huge.md"), [b"needle ".as_slice(), &huge].concat()).expect("write");
let long_line = format!("needle {}", "y".repeat(100_000));
std::fs::write(vault.path().join("long.md"), long_line).expect("write");
let hits = collect(ContentSearch::start(vault.path(), "needle", 50).expect("start"));
let found = paths(&hits);
assert!(found.contains(&"ok.md") && found.contains(&"latin1.md"), "{found:?}");
assert!(!found.contains(&"binary.md") && !found.contains(&"huge.md"), "{found:?}");
let long = hits.iter().find(|hit| hit.path == "long.md").expect("the long line is found");
assert!(long.text.chars().count() <= 241, "the snippet is cut: {}", long.text.len());
}
#[test]
fn cancel_stops_the_search_and_drops_late_hits() {
let vault = temp_vault("cs_cancel");
let files: Vec<(String, String)> = (0..400).map(|n| (format!("d{}/n{n}.md", n % 20), "needle\n".to_owned())).collect();
let borrowed: Vec<(&str, &str)> = files.iter().map(|(p, t)| (p.as_str(), t.as_str())).collect();
seed(vault.path(), &borrowed);
let mut search = ContentSearch::start(vault.path(), "needle", 1000).expect("start");
search.cancel();
let deadline = Instant::now() + Duration::from_secs(30);
loop {
let poll = search.poll();
assert!(poll.hits.is_empty() || !poll.done || poll.hits.len() < 400, "cancelled search kept going");
if poll.done {
break;
}
assert!(Instant::now() < deadline);
std::thread::sleep(Duration::from_millis(5));
}
assert!(search.poll().hits.is_empty(), "nothing arrives after done");
}
#[test]
fn dropping_a_search_cancels_it() {
let vault = temp_vault("cs_drop");
seed(vault.path(), &[("a.md", "needle")]);
let search = ContentSearch::start(vault.path(), "needle", 10).expect("start");
drop(search);
}
crates/engine/tests/core_commands.rs
+19
View File
@@ -9,11 +9,25 @@ use serde_json::{Value, json};
/// Every id the core registers.
const CORE_IDS: &[&str] = &[
"batch.apply",
"batch.cancel",
"batch.decide",
"batch.proposal",
"batch.proposals",
"batch.start",
"batch.status",
"batch.step",
"block.delete",
"block.duplicate",
"block.list",
"block.move",
"block.turn_into",
"grammar.info",
"grammar.install",
"grammar.install_text",
"grammar.remove",
"json.format_block",
"json.minify_block",
"vault.info",
"note.list",
"note.read",
@@ -40,6 +54,9 @@ const CORE_IDS: &[&str] = &[
"buffer.redo",
// Slice 3: search, daily notes and templates, themes.
"search",
"switcher.search",
"note.random",
"note.unique",
"daily.open",
"calendar.month",
"calendar.agenda",
@@ -71,6 +88,8 @@ const CORE_IDS: &[&str] = &[
"calendar.google_state",
"knowledge.context",
"knowledge.rebuild",
"index.pause",
"index.resume",
"knowledge.related",
"knowledge.search",
"knowledge.status",
crates/engine/tests/deleted_files.rs
+37
View File
@@ -0,0 +1,37 @@
//! `session.files.deleted_files`: trash by default, erase for good when asked.
mod common;
use common::{temp_config, temp_vault};
use notescraft_engine::{Registry, Session, register_core};
use serde_json::json;
fn delete_with(setting: Option<&str>, name: &str) -> (std::path::PathBuf, bool) {
let vault = temp_vault(name);
let config = temp_config(name);
std::fs::write(vault.path().join("gone.md"), "# gone\n").expect("write");
let mut registry = Registry::new();
register_core(&mut registry).expect("register");
let mut session = Session::open(vault.path()).expect("open");
session.use_settings_dir(config.path());
if let Some(value) = setting {
registry.run("settings.set", &mut session, &json!({ "key": "session.files.deleted_files", "value": value })).expect("set");
}
registry.run("note.delete", &mut session, &json!({ "path": "gone.md" })).expect("delete");
let trash = vault.path().join(".notescraft/trash");
(vault.path().join("gone.md"), trash.exists())
}
#[test]
fn the_default_keeps_a_trash_copy() {
let (file, trash) = delete_with(None, "deleted-default");
assert!(!file.exists());
assert!(trash, "moved to the vault trash");
}
#[test]
fn permanent_erases_the_file_without_a_copy() {
let (file, trash) = delete_with(Some("permanent"), "deleted-permanent");
assert!(!file.exists());
assert!(!trash, "no trash folder was made");
}
crates/engine/tests/editor_settings.rs
+147
View File
@@ -113,3 +113,150 @@ fn toggle_vim_refuses_to_overwrite_a_newer_schema() {
assert!(matches!(result, Err(Error::Command { ref source, .. }) if source.code == "settings"), "{result:?}");
assert_eq!(std::fs::read_to_string(path).expect("read unchanged settings"), original);
}
/// The editor defaults the document holds, as JSON.
fn editor_defaults() -> serde_json::Value {
serde_json::json!({
"default_view": "visual",
"font_size": 16,
"readable_line_length": true,
"line_numbers": false,
"strict_line_breaks": false,
})
}
/// The defaults of the layout keys, with the switches the file may already have.
fn editor_defaults_document() -> serde_json::Value {
serde_json::json!({
"blocks": { "grip": true, "toolbar": true, "slash_preview": true },
"default_view": "visual",
"font_size": 16,
"line_numbers": false,
"previews": { "colors": true, "named_colors": false },
"readable_line_length": true,
"strict_line_breaks": false,
"vim": false,
})
}
/// The editor section as it sits in `config.toml`.
fn saved_editor(config: &TempVault) -> Value {
let (settings, notes) = load_at(config.path());
assert_eq!(notes, &[] as &[String], "{notes:?}");
serde_json::to_value(&settings).expect("serialize settings").pointer("/editor").cloned().unwrap_or_default()
}
/// Every layout key defaults the way the design says, and a `config.toml`
/// written before they existed loads unchanged.
#[test]
fn editor_layout_defaults_and_older_documents() {
let mut settings = AppSettings::default();
assert_eq!(settings.validate(), Vec::<String>::new());
let editor = serde_json::to_value(&settings).expect("serialize settings").pointer("/editor").cloned().unwrap_or_default();
assert_eq!(editor, editor_defaults_document(), "the whole editor section is at its defaults");
let config = temp_config("editor-layout-old-file");
std::fs::write(config.path().join("config.toml"), b"schema_version = 1\n\n[editor]\nvim = false\n\n[editor.blocks]\ngrip = false\n")
.expect("write the old file");
let (loaded, notes) = load_at(config.path());
assert_eq!(notes, &[] as &[String], "{notes:?}");
assert!(!loaded.editor.blocks.grip, "the keys the file has are kept");
let editor = serde_json::to_value(&loaded).expect("serialize settings").pointer("/editor").cloned().unwrap_or_default();
assert_eq!(editor.pointer("/default_view"), Some(&serde_json::json!("visual")), "{editor}");
assert_eq!(editor.pointer("/font_size"), Some(&serde_json::json!(16)), "{editor}");
assert_eq!(editor.pointer("/readable_line_length"), Some(&serde_json::json!(true)), "{editor}");
assert_eq!(editor.pointer("/line_numbers"), Some(&serde_json::json!(false)), "{editor}");
assert_eq!(editor.pointer("/strict_line_breaks"), Some(&serde_json::json!(false)), "{editor}");
}
/// Every layout key round-trips through `settings.set`, comes back out of
/// `config.toml`, and `settings.reset` puts it back.
#[test]
fn editor_layout_keys_round_trip_and_reset() {
let (registry, mut session, _vault, config) = fixture("editor-layout-round-trip");
let set = |session: &mut Session, key: &str, value: Value| {
let params = serde_json::json!({ "key": key, "value": value });
registry.run("settings.set", session, &params)
};
let changed = [
("editor.default_view", serde_json::json!("source")),
("editor.font_size", serde_json::json!(22)),
("editor.readable_line_length", serde_json::json!(false)),
("editor.line_numbers", serde_json::json!(true)),
("editor.strict_line_breaks", serde_json::json!(true)),
];
for (key, value) in &changed {
assert!(set(&mut session, key, value.clone()).is_ok(), "{key} must be settable");
}
let got = registry.run("settings.get", &mut session, &Value::Null).expect("get settings");
let editor = got.pointer("/settings/editor").cloned().unwrap_or_default();
for (key, value) in &changed {
let field = key.trim_start_matches("editor.");
assert_eq!(editor.pointer(&format!("/{field}")), Some(value), "{key} in settings.get: {editor}");
assert_eq!(saved_editor(&config).pointer(&format!("/{field}")), Some(value), "{key} in config.toml");
}
for (key, _) in &changed {
let params = serde_json::json!({ "key": key });
assert!(registry.run("settings.reset", &mut session, &params).is_ok(), "{key} must be resettable");
let field = key.trim_start_matches("editor.");
let default = editor_defaults().pointer(&format!("/{field}")).cloned().unwrap_or_default();
assert_eq!(saved_editor(&config).pointer(&format!("/{field}")), Some(&default), "{key} back to its default");
}
}
/// A font size outside `10..=32` is clamped with a note, both from a file and
/// through `settings.set` (the same convention as `git.auto_commit_minutes`).
#[test]
fn editor_font_size_is_clamped_with_a_note() {
let config = temp_config("editor-font-clamp-file");
std::fs::write(config.path().join("config.toml"), "schema_version = 1\n\n[editor]\nfont_size = 3\n").expect("write a tiny font");
let (settings, notes) = load_at(config.path());
assert_eq!(settings.editor.font_size, 10, "3 is clamped to the smallest legible size");
let joined = notes.join("\n");
assert!(joined.contains("editor.font_size: 3 is outside 10..=32"), "{joined}");
let (registry, mut session, _vault, config) = fixture("editor-font-clamp-set");
let result =
registry.run("settings.set", &mut session, &serde_json::json!({ "key": "editor.font_size", "value": 200 })).expect("the set succeeds, repaired");
assert!(result.to_string().contains("editor.font_size: 200"), "{result}");
assert_eq!(saved_editor(&config).pointer("/font_size"), Some(&serde_json::json!(32)));
// A value that does not even fit a `u8` is refused, not clamped.
let too_big = registry.run("settings.set", &mut session, &serde_json::json!({ "key": "editor.font_size", "value": 300 }));
assert!(matches!(too_big, Err(Error::Command { ref source, .. }) if source.code == "invalid_value"), "{too_big:?}");
assert_eq!(saved_editor(&config).pointer("/font_size"), Some(&serde_json::json!(32)), "the clamped value survives the refusal");
}
/// Wrong types, unknown keys and unknown view names are refused, and a refused
/// set leaves `config.toml` byte-identical.
#[test]
fn editor_layout_refuses_bad_values_without_touching_the_file() {
let (registry, mut session, _vault, config) = fixture("editor-layout-refusals");
let set = |session: &mut Session, key: &str, value: Value| {
let params = serde_json::json!({ "key": key, "value": value });
registry.run("settings.set", session, &params)
};
assert!(set(&mut session, "editor.font_size", serde_json::json!(20)).is_ok(), "one good value first");
let before = std::fs::read(config.path().join("config.toml")).expect("read the config file");
for (key, value, code) in [
("editor.default_view", serde_json::json!("rendering"), "invalid_value"),
("editor.default_view", serde_json::json!("Visual"), "invalid_value"),
("editor.default_view", serde_json::json!(1), "invalid_value"),
("editor.font_size", serde_json::json!("20"), "invalid_value"),
("editor.font_size", serde_json::json!(-1), "invalid_value"),
("editor.readable_line_length", serde_json::json!("yes"), "invalid_value"),
("editor.line_numbers", serde_json::json!(0), "invalid_value"),
("editor.strict_line_breaks", serde_json::json!(null), "invalid_value"),
("editor.unknown", serde_json::json!(true), "unknown_key"),
("editor.line_numbers.too", serde_json::json!(true), "unknown_key"),
] {
let result = set(&mut session, key, value.clone());
assert!(matches!(result, Err(Error::Command { ref source, .. }) if source.code == code), "`{key}` = {value} must fail with {code}, got {result:?}");
}
let after = std::fs::read(config.path().join("config.toml")).expect("read the config file again");
assert_eq!(before, after, "a refused value leaves config.toml byte-identical");
assert_eq!(saved_editor(&config).pointer("/font_size"), Some(&serde_json::json!(20)), "the last good value survives");
}
crates/engine/tests/grammars.rs
+170
View File
@@ -0,0 +1,170 @@
//! `grammar.install`, `grammar.remove` and `grammar.info`, with a fake transport (no network).
mod common;
use std::collections::BTreeMap;
use std::sync::{Arc, Mutex};
use common::{temp_config, temp_vault};
use notescraft_engine::{AppSettings, Registry, Session, register_core};
use notescraft_syntax::Error as SyntaxError;
use notescraft_syntax::fetch::Transport;
use notescraft_syntax::validate::sha256_hex;
use serde_json::{Value, json};
/// The highlighter's grammar list is process-wide, so these tests take turns.
static TURN: Mutex<()> = Mutex::new(());
fn turn() -> std::sync::MutexGuard<'static, ()> {
TURN.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
const TINY: &str = "%YAML 1.2\n---\nname: Tiny\nfile_extensions: [tiny]\nscope: source.tiny\ncontexts:\n main:\n - match: '\\bfoo\\b'\n scope: keyword.control.tiny\n";
const RAW: &str = "https://git.marcospaulo.dev.br/masi/notescraft-registry/raw/branch/main/";
/// Serves canned bodies by URL and records every request.
#[derive(Debug, Clone, Default)]
struct Fake {
files: Arc<Mutex<BTreeMap<String, Vec<u8>>>>,
asked: Arc<Mutex<Vec<String>>>,
}
impl Transport for Fake {
fn get(&self, url: &str, _max: usize) -> notescraft_syntax::Result<Vec<u8>> {
self.asked.lock().expect("lock").push(url.to_owned());
self.files.lock().expect("lock").get(url).cloned().ok_or(SyntaxError::Offline)
}
}
fn index(sha: &str) -> String {
json!({ "version": 1, "grammars": [
{ "id": "tiny", "name": "Tiny", "format": "sublime-syntax", "file": "grammars/tiny.sublime-syntax", "sha256": sha, "license": "MIT", "sample": "foo\n" },
{ "id": "plist", "format": "tmLanguage", "file": "grammars/plist.tmLanguage", "sha256": "00" },
{ "id": "evil", "format": "sublime-syntax", "file": "../../etc/passwd", "sha256": "00" }
] })
.to_string()
}
struct Setup {
registry: Registry<Session>,
session: Session,
fake: Fake,
config: common::TempVault,
_vault: common::TempVault,
}
fn setup(name: &str, body: &str) -> Setup {
let vault = temp_vault(name);
let config = temp_config(name);
let mut registry = Registry::new();
register_core(&mut registry).expect("register");
let mut session = Session::open(vault.path()).expect("open");
session.use_settings_dir(config.path());
let fake = Fake::default();
{
let mut files = fake.files.lock().expect("lock");
files.insert(format!("{RAW}index.json"), index(&sha256_hex(body.as_bytes())).into_bytes());
files.insert(format!("{RAW}grammars/tiny.sublime-syntax"), body.as_bytes().to_vec());
}
session.set_grammar_transport(Box::new(fake.clone()));
Setup { registry, session, fake, config, _vault: vault }
}
fn run(s: &mut Setup, id: &str, params: &Value) -> Result<Value, notescraft_engine::Error> {
s.registry.run(id, &mut s.session, params)
}
fn installed(s: &Setup) -> AppSettings {
notescraft_engine::load_at(s.config.path()).0
}
#[test]
fn a_registry_install_writes_the_file_and_records_it() {
let _turn = turn();
let mut s = setup("gram-ok", TINY);
let result = run(&mut s, "grammar.install", &json!({ "id": "tiny" })).expect("installs");
assert_eq!(result["name"], json!("Tiny"));
assert_eq!(result["license"], json!("MIT"));
let file = s.config.path().join("grammars/tiny.sublime-syntax");
assert_eq!(std::fs::read_to_string(&file).expect("file"), TINY);
assert_eq!(installed(&s).grammars.iter().find(|g| g.id == "tiny").map(|g| g.name.as_str()), Some("Tiny"));
assert!(notescraft_syntax::load::is_known("tiny"), "the installed grammar is live in the highlighter");
let info = run(&mut s, "grammar.info", &json!({ "label": "pyhton" })).expect("info");
assert_eq!(info["installed"].as_array().map(Vec::len), Some(1));
assert_eq!(info["label"]["suggestion"], json!("python"));
assert_eq!(info["label"]["bundled"], json!(false));
}
#[test]
fn a_pasted_link_installs_without_a_checksum() {
let _turn = turn();
let mut s = setup("gram-url", TINY);
s.fake.files.lock().expect("lock").insert("https://example.org/my.sublime-syntax".to_owned(), TINY.as_bytes().to_vec());
run(&mut s, "grammar.install", &json!({ "id": "mine", "url": "https://example.org/my.sublime-syntax" })).expect("installs");
assert!(s.config.path().join("grammars/mine.sublime-syntax").exists());
}
#[test]
fn every_failed_gate_saves_nothing() {
let _turn = turn();
let mut s = setup("gram-bad", TINY);
let before = installed(&s);
for params in [
json!({ "id": "plist" }),
json!({ "id": "evil" }),
json!({ "id": "absent" }),
json!({ "id": "Bad Id" }),
json!({ "id": "x", "url": "http://example.org/a.sublime-syntax" }),
json!({ "id": "x", "url": "https://127.0.0.1/a.sublime-syntax" }),
json!({ "id": "x", "url": "https://example.org/offline.sublime-syntax" }),
] {
assert!(run(&mut s, "grammar.install", &params).is_err(), "{params}");
}
s.fake.files.lock().expect("lock").insert(format!("{RAW}grammars/tiny.sublime-syntax"), b"tampered".to_vec());
assert!(run(&mut s, "grammar.install", &json!({ "id": "tiny" })).is_err(), "checksum mismatch");
assert_eq!(installed(&s), before);
assert!(!s.config.path().join("grammars").exists() || std::fs::read_dir(s.config.path().join("grammars")).expect("dir").next().is_none());
}
#[test]
fn remove_deletes_the_file_and_the_entry() {
let _turn = turn();
let mut s = setup("gram-remove", TINY);
run(&mut s, "grammar.install", &json!({ "id": "tiny" })).expect("installs");
run(&mut s, "grammar.remove", &json!({ "id": "tiny" })).expect("removes");
assert!(!s.config.path().join("grammars/tiny.sublime-syntax").exists());
assert_eq!(installed(&s).grammars, Vec::new());
assert!(!notescraft_syntax::load::is_known("tiny"), "removal unloads it");
assert!(run(&mut s, "grammar.remove", &json!({ "id": "tiny" })).is_err(), "second removal is an error");
}
#[test]
fn bad_entries_are_dropped_and_the_rest_sorted() {
let _turn = turn();
let entry = |id: &str, file: &str| notescraft_engine::GrammarInfo { id: id.into(), file: file.into(), ..Default::default() };
let mut settings = AppSettings {
grammars: vec![
entry("zed", "zed.sublime-syntax"),
entry("ok", "ok.sublime-syntax"),
entry("../x", "../x.sublime-syntax"),
entry("mismatch", "other.sublime-syntax"),
entry("ok", "ok.sublime-syntax"),
],
..AppSettings::default()
};
let notes = settings.validate();
assert_eq!(settings.grammars.iter().map(|g| g.id.as_str()).collect::<Vec<_>>(), ["ok", "zed"]);
assert!(notes.iter().any(|n| n.contains("dropped 3")), "{notes:?}");
}
#[test]
fn install_text_checks_the_grammar_then_saves_it() {
let _turn = turn();
let mut s = setup("gram-text", TINY);
run(&mut s, "grammar.install_text", &json!({ "id": "tiny", "text": TINY, "license": "MIT" })).expect("installs");
assert!(s.config.path().join("grammars/tiny.sublime-syntax").exists());
assert!(notescraft_syntax::load::is_known("tiny"));
assert!(run(&mut s, "grammar.install_text", &json!({ "id": "bad", "text": "nope: [" })).is_err());
assert!(run(&mut s, "grammar.install_text", &json!({ "id": "bad", "text": "<?xml version=\"1.0\"?><plist/>" })).is_err());
assert!(!s.config.path().join("grammars/bad.sublime-syntax").exists());
}
crates/engine/tests/grammars_hostile.rs
+259
View File
@@ -0,0 +1,259 @@
//! Hostile params for `grammar.install`, `grammar.install_text`, `grammar.remove`, `grammar.info`
//! and `json.format_block` / `json.minify_block`: every bad shape is a clean `Err`, nothing is
//! written under the grammars folder, the settings keep their grammar list and no note on disk moves.
mod common;
use std::collections::BTreeMap;
use std::path::Path;
use std::sync::{Arc, Mutex};
use common::{seed, temp_config, temp_vault};
use notescraft_engine::{AppSettings, Error, Registry, Session, adversarial_params, register_core};
use notescraft_syntax::Error as SyntaxError;
use notescraft_syntax::fetch::Transport;
use serde_json::json;
/// The highlighter's grammar list is process-wide, so these tests take turns.
static TURN: Mutex<()> = Mutex::new(());
fn turn() -> std::sync::MutexGuard<'static, ()> {
TURN.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
const TINY: &str = "%YAML 1.2\n---\nname: Tiny\nfile_extensions: [tiny]\nscope: source.tiny\ncontexts:\n main:\n - match: '\\bfoo\\b'\n scope: keyword.control.tiny\n";
/// The note the `json.*` commands are pointed at: a heading and one JSON code block.
const JSON_NOTE: &str = "# T\n\n```json\n{\"b\":1,\"a\":[1e5]}\n```\n";
/// The command ids under attack.
const IDS: [&str; 6] = ["grammar.install", "grammar.install_text", "grammar.remove", "grammar.info", "json.format_block", "json.minify_block"];
/// Serves canned bodies by URL and records every request; empty by default, so every download fails.
#[derive(Debug, Clone, Default)]
struct Fake {
files: Arc<Mutex<BTreeMap<String, Vec<u8>>>>,
asked: Arc<Mutex<Vec<String>>>,
}
impl Transport for Fake {
fn get(&self, url: &str, _max: usize) -> notescraft_syntax::Result<Vec<u8>> {
self.asked.lock().expect("lock").push(url.to_owned());
self.files.lock().expect("lock").get(url).cloned().ok_or(SyntaxError::Offline)
}
}
struct Setup {
registry: Registry<Session>,
session: Session,
fake: Fake,
config: common::TempVault,
vault: common::TempVault,
}
fn setup(name: &str) -> Setup {
let vault = temp_vault(name);
seed(vault.path(), &[("n.md", JSON_NOTE), ("code.md", TINY)]);
let config = temp_config(name);
let mut registry = Registry::new();
register_core(&mut registry).expect("register");
let mut session = Session::open(vault.path()).expect("open");
session.use_settings_dir(config.path());
let fake = Fake::default();
session.set_grammar_transport(Box::new(fake.clone()));
Setup { registry, session, fake, config, vault }
}
/// The file names sitting in `<config>/grammars/`, sorted; `Vec::new()` when the folder is absent.
fn saved_files(config: &Path) -> Vec<String> {
let Ok(entries) = std::fs::read_dir(config.join("grammars")) else { return Vec::new() };
let mut names: Vec<String> = entries.flatten().map(|entry| entry.file_name().to_string_lossy().into_owned()).collect();
names.sort();
names
}
fn settings(config: &Path) -> AppSettings {
notescraft_engine::load_at(config).0
}
/// Every hostile id must be refused by every id-taking grammar command, writing nothing.
#[test]
fn hostile_grammar_ids_are_refused() {
let _turn = turn();
let mut s = setup("gram-id");
let before = settings(s.config.path()).grammars.clone();
let ids = [
json!(""),
json!("../../x"),
json!("A"),
json!("a".repeat(33)),
json!("x\0y"),
json!("gráma"),
json!("é".repeat(512)),
json!("a".repeat(10 * 1024)),
json!("Bad Id"),
json!("x\ny"),
json!(3),
json!(null),
json!(["tiny"]),
json!({ "id": "tiny" }),
];
for id in &ids {
assert!(s.registry.run("grammar.install", &mut s.session, &json!({ "id": id })).is_err(), "install must refuse {id}");
assert!(s.registry.run("grammar.install_text", &mut s.session, &json!({ "id": id, "text": TINY })).is_err(), "install_text must refuse {id}");
assert!(s.registry.run("grammar.remove", &mut s.session, &json!({ "id": id })).is_err(), "remove must refuse {id}");
}
assert!(s.registry.run("grammar.install", &mut s.session, &json!({})).is_err(), "a missing id is an error");
assert_eq!(saved_files(s.config.path()), Vec::<String>::new());
assert_eq!(settings(s.config.path()).grammars, before);
assert_eq!(common::on_disk(s.vault.path(), "n.md"), JSON_NOTE, "no note may move");
assert_eq!(common::on_disk(s.vault.path(), "code.md"), TINY, "no note may move");
}
/// A hostile link is refused by the url gate: not one of them reaches the transport.
#[test]
fn hostile_urls_are_refused_before_any_download() {
let _turn = turn();
let mut s = setup("gram-url");
let urls = [
json!("http://example.org/a.sublime-syntax"),
json!("ftp://example.org/a.sublime-syntax"),
json!("file:///etc/passwd"),
json!("https://user:pw@h.dev/x"),
json!("https://[::1]/x"),
json!("https://127.0.0.1/x"),
json!("https://2130706433/x"),
json!("https://localhost/x"),
json!("https://x.dev:8443/x"),
json!(format!("https://x.dev/{}", "a".repeat(3000))),
json!("https://x.dev/a b.sublime-syntax"),
json!("https://x.dev/a\nb.sublime-syntax"),
json!("https://"),
json!(""),
json!(7),
json!(null),
];
for url in &urls {
assert!(s.registry.run("grammar.install", &mut s.session, &json!({ "id": "mine", "url": url })).is_err(), "install must refuse {url}");
}
assert_eq!(*s.fake.asked.lock().expect("lock"), Vec::<String>::new(), "a refused link must not be fetched");
assert_eq!(saved_files(s.config.path()), Vec::<String>::new());
assert_eq!(settings(s.config.path()).grammars, Vec::new());
}
/// Hostile grammar sources never reach the grammars folder.
#[test]
fn hostile_grammar_texts_never_reach_the_disk() {
let _turn = turn();
let mut s = setup("gram-text");
let texts = [
json!(""),
json!(" ".repeat(600 * 1024)),
json!("<?xml version=\"1.0\"?><plist><dict/></plist>"),
json!("{}"),
json!("[]"),
json!("name: nope\nlist:\n - 1\n - 2\n"),
json!("\u{0}\u{1}\u{2}\u{3}\u{feff}\u{202e}"),
json!("\u{fffd}\u{10ffff}\u{7f}"),
json!(["a"]),
json!(null),
json!(12),
];
for (case, text) in texts.iter().enumerate() {
let id = format!("hostile-{case}");
assert!(s.registry.run("grammar.install_text", &mut s.session, &json!({ "id": id, "text": text })).is_err(), "text must be refused: {text}");
}
assert!(s.registry.run("grammar.install_text", &mut s.session, &json!({ "id": "no-text" })).is_err(), "a missing text is an error");
assert_eq!(saved_files(s.config.path()), Vec::<String>::new());
assert_eq!(settings(s.config.path()).grammars, Vec::new());
assert_eq!(common::on_disk(s.vault.path(), "n.md"), JSON_NOTE);
}
/// Removing something that was never installed is an error, and changes nothing.
#[test]
fn remove_refuses_everything_that_is_not_installed() {
let _turn = turn();
let mut s = setup("gram-remove");
for id in ["tiny", "notescraft", "code", ""] {
assert!(s.registry.run("grammar.remove", &mut s.session, &json!({ "id": id })).is_err(), "remove must refuse `{id}`");
}
assert_eq!(saved_files(s.config.path()), Vec::<String>::new());
assert_eq!(settings(s.config.path()).grammars, Vec::new());
}
/// `grammar.info` is read-only: any label shape answers, none of them install anything.
#[test]
fn grammar_info_answers_any_label_and_installs_nothing() {
let _turn = turn();
let mut s = setup("gram-info");
for label in [json!(""), json!("pyhton"), json!("x\0y"), json!(" "), json!("\u{202e}rtl"), json!("a".repeat(4096))] {
let result = s.registry.run("grammar.info", &mut s.session, &json!({ "label": label })).expect("info answers");
assert_eq!(result["installed"].as_array().map(Vec::len), Some(0), "label {label}");
}
for label in [json!(3), json!(null), json!(["rust"]), json!({ "a": 1 })] {
assert!(s.registry.run("grammar.info", &mut s.session, &json!({ "label": label })).is_err(), "a non-string label is an error");
}
assert!(s.registry.run("grammar.info", &mut s.session, &json!({ "id": "../../x", "text": TINY })).is_err(), "params it does not declare are refused");
assert_eq!(saved_files(s.config.path()), Vec::<String>::new());
assert_eq!(settings(s.config.path()).grammars, Vec::new());
}
/// `json.*` refuses a path that leaves the vault and every index that is not a block.
#[test]
fn json_block_commands_refuse_hostile_paths_and_indexes() {
let _turn = turn();
let mut s = setup("json-block");
for id in ["json.format_block", "json.minify_block"] {
for path in ["../x.md", "/etc/passwd", "n\0d.md", "", "nope.md", "../n.md"] {
assert!(s.registry.run(id, &mut s.session, &json!({ "path": path, "index": 1 })).is_err(), "{id} must refuse `{path}`");
}
for index in [json!(-1), json!(99), json!(u64::MAX), json!(1.5), json!("1"), json!(null), json!([1])] {
assert!(s.registry.run(id, &mut s.session, &json!({ "path": "n.md", "index": index })).is_err(), "{id} must refuse {index}");
}
for params in [json!({}), json!({ "path": "n.md" }), json!({ "index": 1 }), json!({ "path": 3, "index": 1 })] {
assert!(s.registry.run(id, &mut s.session, &params).is_err(), "{id} must refuse {params}");
}
}
assert_eq!(common::on_disk(s.vault.path(), "n.md"), JSON_NOTE, "a refused edit leaves the note alone");
assert_eq!(common::on_disk(s.vault.path(), "code.md"), TINY);
}
/// The control case: a good link really does install, so the refusals above mean something.
#[test]
fn a_good_link_installs_from_the_transport() {
let _turn = turn();
let mut s = setup("gram-ok");
s.fake.files.lock().expect("lock").insert("https://example.org/my.sublime-syntax".to_owned(), TINY.as_bytes().to_vec());
s.registry.run("grammar.install", &mut s.session, &json!({ "id": "mine", "url": "https://example.org/my.sublime-syntax" })).expect("installs");
assert_eq!(saved_files(s.config.path()), vec!["mine.sublime-syntax".to_owned()]);
assert_eq!(settings(s.config.path()).grammars.iter().map(|g| g.id.as_str()).collect::<Vec<_>>(), ["mine"]);
s.registry.run("grammar.remove", &mut s.session, &json!({ "id": "mine" })).expect("removes");
assert_eq!(saved_files(s.config.path()), Vec::<String>::new());
assert_eq!(settings(s.config.path()).grammars, Vec::new());
}
/// No adversarial params object panics a grammar or json command, nor writes to the vault.
#[test]
fn adversarial_params_never_panic_a_grammar_or_json_command() {
let _turn = turn();
let mut s = setup("gram-fuzz");
let previous = std::panic::take_hook();
std::panic::set_hook(Box::new(|_| {}));
let mut panicked = Vec::new();
let mut accepted = 0_usize;
for id in IDS {
for params in adversarial_params() {
match s.registry.run(id, &mut s.session, &params) {
Ok(_) => accepted += 1,
Err(Error::Panicked(other)) => panicked.push(other),
Err(_) => {}
}
}
}
std::panic::set_hook(previous);
assert!(panicked.is_empty(), "commands panicked: {panicked:?}");
assert!(accepted > 0, "at least `grammar.info` answers some fixtures");
assert_eq!(saved_files(s.config.path()), Vec::<String>::new());
assert_eq!(settings(s.config.path()).grammars, Vec::new());
assert_eq!(common::on_disk(s.vault.path(), "n.md"), JSON_NOTE);
}
crates/engine/tests/indexing.rs
+170
View File
@@ -0,0 +1,170 @@
//! Lazy open: names first, notes indexed in time-boxed steps, the same index in the end.
mod common;
use std::time::Duration;
use common::{linked_vault, seed, temp_vault};
use notescraft_engine::{IndexProgress, Registry, Session, register_core};
use serde_json::Value;
const GENEROUS: Duration = Duration::from_secs(30);
fn run_to_end(session: &mut Session) -> IndexProgress {
let mut progress = session.index_progress();
while !progress.finished {
progress = session.index_step(GENEROUS);
}
progress
}
fn run(session: &mut Session, id: &str) {
let mut registry = Registry::new();
register_core(&mut registry).expect("register");
registry.run(id, session, &Value::Null).unwrap_or_else(|error| panic!("{id}: {error:?}"));
}
#[test]
fn a_lazy_session_has_the_file_list_but_no_parsed_notes() {
let vault = linked_vault("lazy_empty", 5);
seed(vault.path(), &[("pic.png", "x")]);
let session = Session::open_lazy(vault.path()).expect("lazy");
let progress = session.index_progress();
assert_eq!((progress.total, progress.done, progress.finished), (5, 0, false));
assert!(session.index().is_empty(), "no note is parsed before the first step");
assert!(session.index().files().contains(&"pic.png"), "attachments are registered at open");
}
#[test]
fn a_zero_budget_still_indexes_one_note_per_step() {
let vault = linked_vault("lazy_budget", 4);
let mut session = Session::open_lazy(vault.path()).expect("lazy");
for expected in 1..=3 {
let progress = session.index_step(Duration::ZERO);
assert_eq!((progress.done, progress.finished), (expected, false));
assert_eq!(session.index().len(), expected);
}
let progress = session.index_step(Duration::ZERO);
assert_eq!((progress.done, progress.total), (4, 4));
assert!(progress.finished);
}
#[test]
fn steps_after_the_end_change_nothing() {
let vault = linked_vault("lazy_idempotent", 3);
let mut session = Session::open_lazy(vault.path()).expect("lazy");
let end = run_to_end(&mut session);
assert_eq!(session.index_step(GENEROUS), end);
assert_eq!(session.index_step(Duration::ZERO), end);
}
#[test]
fn lazy_open_plus_all_steps_builds_the_same_index_as_eager_open() {
let vault = linked_vault("lazy_same", 12);
seed(
vault.path(),
&[("sub/Extra.md", "---\naliases: [Spark]\n---\n# H\n[[Missing]] [[spark]] #deep/tag zebra word\n"), ("pic.png", "x"), ("notes/bad-utf8.md", "ok")],
);
std::fs::write(vault.path().join("notes/bad-utf8.md"), [0xFF, 0xFE, 0x00]).expect("write bad bytes");
let eager = Session::open(vault.path()).expect("eager");
drop(eager);
let mut lazy = Session::open_lazy(vault.path()).expect("lazy");
let progress = run_to_end(&mut lazy);
let eager = Session::open_read_only(vault.path()).expect("eager again");
assert_eq!(progress.done, progress.total);
assert_eq!(lazy.index().notes(), eager.index().notes());
assert_eq!(lazy.index().files(), eager.index().files());
assert_eq!(lazy.index().tags().len(), eager.index().tags().len());
assert_eq!(lazy.index().unresolved().len(), eager.index().unresolved().len());
assert_eq!(lazy.index().backlinks("note-03.md").len(), eager.index().backlinks("note-03.md").len());
assert_eq!(lazy.diagnostics().len(), eager.diagnostics().len(), "the unreadable note is a diagnostic in both");
let hits = |session: &Session| session.search("zebra", 10).expect("search").into_iter().map(|hit| hit.path).collect::<Vec<_>>();
assert_eq!(hits(&lazy), hits(&eager));
assert_eq!(hits(&lazy), vec!["sub/Extra.md".to_owned()]);
}
#[test]
fn an_empty_vault_finishes_on_the_first_step() {
let vault = temp_vault("lazy_empty_vault");
let mut session = Session::open_lazy(vault.path()).expect("lazy");
let progress = session.index_step(GENEROUS);
assert_eq!((progress.total, progress.done, progress.finished), (0, 0, true));
}
#[test]
fn pause_stops_the_steps_and_resume_continues_without_rereading() {
let vault = linked_vault("lazy_pause", 6);
let mut session = Session::open_lazy(vault.path()).expect("lazy");
session.index_step(Duration::ZERO);
run(&mut session, "index.pause");
let paused = session.index_step(GENEROUS);
assert!(paused.paused && !paused.finished);
assert_eq!(paused.done, 1);
assert_eq!(session.index().len(), 1);
run(&mut session, "index.resume");
let end = run_to_end(&mut session);
assert_eq!((end.done, end.total, end.paused), (6, 6, false));
assert_eq!(session.index().len(), 6);
}
#[test]
fn the_newest_notes_are_read_first() {
let vault = temp_vault("lazy_order");
seed(vault.path(), &[("a-old.md", "old"), ("b-new.md", "new"), ("c-mid.md", "mid")]);
let now = std::time::SystemTime::now();
for (name, age) in [("a-old.md", 300), ("c-mid.md", 200), ("b-new.md", 100)] {
let file = std::fs::File::options().write(true).open(vault.path().join(name)).expect("open");
file.set_modified(now - Duration::from_secs(age)).expect("set mtime");
}
let mut session = Session::open_lazy(vault.path()).expect("lazy");
session.index_step(Duration::ZERO);
assert_eq!(session.index().notes(), vec!["b-new.md"]);
session.index_step(Duration::ZERO);
assert_eq!(session.index().notes(), vec!["b-new.md", "c-mid.md"]);
}
#[test]
fn opening_a_buffer_indexes_that_note_at_once() {
let vault = linked_vault("lazy_open_buffer", 5);
let mut session = Session::open_lazy(vault.path()).expect("lazy");
session.open_buffer("note-03.md").expect("buffer");
assert!(session.index().note("note-03.md").is_some());
let progress = session.index_progress();
assert_eq!((progress.done, progress.finished), (1, false));
let end = run_to_end(&mut session);
assert_eq!((end.done, end.total), (5, 5), "the note is not counted twice");
assert_eq!(session.index().len(), 5);
}
#[test]
fn a_note_deleted_before_its_turn_is_skipped_quietly() {
let vault = linked_vault("lazy_deleted", 3);
let mut session = Session::open_lazy(vault.path()).expect("lazy");
std::fs::remove_file(vault.path().join("note-01.md")).expect("remove");
let end = run_to_end(&mut session);
assert_eq!(end.done, 3);
assert!(session.diagnostics().is_empty(), "{:?}", session.diagnostics());
assert!(session.index().note("note-01.md").is_none());
}
#[test]
fn renames_and_moves_are_refused_until_the_index_is_complete() {
let vault = linked_vault("lazy_gate", 4);
seed(vault.path(), &[("dir/inner.md", "[[note-00]]")]);
let mut session = Session::open_lazy(vault.path()).expect("lazy");
let error = session.rename_note("note-00.md", "renamed.md", true).expect_err("refused");
assert_eq!(error.code(), "indexing");
assert!(error.to_string().contains("still indexing"), "{error}");
assert_eq!(session.rename_folder("dir", "moved", true).expect_err("refused").code(), "indexing");
assert!(vault.path().join("note-00.md").exists(), "nothing was renamed");
run_to_end(&mut session);
session.rename_note("note-00.md", "renamed.md", true).expect("rename after the scan");
session.rename_folder("dir", "moved", true).expect("move after the scan");
}
#[test]
fn search_reports_partial_results_until_the_scan_ends() {
let vault = linked_vault("lazy_partial", 3);
let mut session = Session::open_lazy(vault.path()).expect("lazy");
assert!(session.search_partial());
run_to_end(&mut session);
assert!(!session.search_partial());
}
crates/engine/tests/json_undo.rs
+147
View File
@@ -0,0 +1,147 @@
//! Acceptance: a JSON format or minify rewrite is ONE undo step.
//!
//! `json.format_block` and `json.minify_block` go through `Session::lint_commit`, which edits
//! the open buffer as a single replacement: one `buffer.undo` takes the whole format back,
//! `buffer.redo` re-applies it, and unsaved edits elsewhere in the note survive. Without an
//! open buffer the same edit is one atomic write, and formatting an already formatted block
//! reports `changed: false` and leaves the file untouched. For comparison, `block.turn_into`
//! and `block.move` are checked to be one undo step as well.
mod common;
use common::{TempVault, on_disk, seed, temp_vault};
use notescraft_engine::{Registry, Session, register_core};
use serde_json::{Value, json};
/// The fixture note: one fenced JSON block with compact JSON.
const NOTE: &str = "# T\n\n```json\n{\"b\":1,\"a\":[1e5]}\n```\n";
/// The same note with the JSON pretty-printed; the ONLY change a format makes.
const FORMATTED: &str = "# T\n\n```json\n{\n \"b\": 1,\n \"a\": [\n 1e5\n ]\n}\n```\n";
/// A vault with `n.md` holding `note`, a registry with the core commands, and an open session.
fn setup(name: &str, note: &str) -> (Registry<Session>, Session, TempVault) {
let vault = temp_vault(name);
seed(vault.path(), &[("n.md", note)]);
let mut registry = Registry::new();
register_core(&mut registry).expect("register");
let session = Session::open(vault.path()).expect("open");
(registry, session, vault)
}
/// Runs `id` on the session, panicking with the error instead of returning a bare `Err`.
fn run(registry: &Registry<Session>, session: &mut Session, id: &str, params: &Value) -> Value {
registry.run(id, session, params).unwrap_or_else(|error| panic!("{id}: {error:?}"))
}
/// The current text of the `n.md` buffer.
fn text(session: &Session) -> String {
session.buffer("n.md").expect("the buffer is open").text().to_owned()
}
/// One `json.format_block` is one undo step: undo restores the exact original, redo re-applies
/// it, and a later `json.minify_block` is one undo step too.
#[test]
fn json_format_and_minify_are_one_undo_step_each() {
let (registry, mut session, vault) = setup("json-undo", NOTE);
run(&registry, &mut session, "buffer.open", &json!({ "path": "n.md" }));
let result = run(&registry, &mut session, "json.format_block", &json!({ "path": "n.md", "index": 1 }));
assert_eq!(result["changed"], json!(true));
assert_eq!(text(&session), FORMATTED);
assert_eq!(on_disk(vault.path(), "n.md"), NOTE, "a buffer edit is not a save");
let result = run(&registry, &mut session, "buffer.undo", &json!({ "path": "n.md" }));
assert_eq!(result["undone"], json!(true));
assert_eq!(text(&session), NOTE, "ONE undo restores the exact original text");
assert_eq!(result["dirty"], json!(false), "the buffer is back to the saved bytes");
let result = run(&registry, &mut session, "buffer.undo", &json!({ "path": "n.md" }));
assert_eq!(result["undone"], json!(false), "the format was ONE step: nothing more to undo");
let result = run(&registry, &mut session, "buffer.redo", &json!({ "path": "n.md" }));
assert_eq!(result["redone"], json!(true));
assert_eq!(text(&session), FORMATTED, "ONE redo re-applies the format");
let result = run(&registry, &mut session, "json.minify_block", &json!({ "path": "n.md", "index": 1 }));
assert_eq!(result["changed"], json!(true));
assert_eq!(text(&session), NOTE, "the minified body is the original compact body");
run(&registry, &mut session, "buffer.undo", &json!({ "path": "n.md" }));
assert_eq!(text(&session), FORMATTED, "the minify is ONE undo step, not more");
}
/// For comparison: `block.turn_into` and `block.move` are one undo step each in an open buffer.
#[test]
fn turn_into_and_move_are_one_undo_step_each() {
const PROSE: &str = "# Title\n\nFirst para\n\nSecond para\n";
let (registry, mut session, _vault) = setup("block-undo-turn", PROSE);
run(&registry, &mut session, "buffer.open", &json!({ "path": "n.md" }));
run(&registry, &mut session, "block.turn_into", &json!({ "path": "n.md", "index": 1, "target": "heading2" }));
assert_eq!(text(&session), "# Title\n\n## First para\n\nSecond para\n");
run(&registry, &mut session, "buffer.undo", &json!({ "path": "n.md" }));
assert_eq!(text(&session), PROSE, "ONE undo restores the original");
let result = run(&registry, &mut session, "buffer.undo", &json!({ "path": "n.md" }));
assert_eq!(result["undone"], json!(false), "the turn into was ONE step");
let (registry, mut session, _vault) = setup("block-undo-move", PROSE);
run(&registry, &mut session, "buffer.open", &json!({ "path": "n.md" }));
run(&registry, &mut session, "block.move", &json!({ "path": "n.md", "index": 2, "direction": "up" }));
assert_eq!(text(&session), "# Title\n\nSecond para\n\nFirst para\n");
run(&registry, &mut session, "buffer.undo", &json!({ "path": "n.md" }));
assert_eq!(text(&session), PROSE, "ONE undo restores the original");
}
/// A buffer with unsaved user edits elsewhere keeps them: the format touches only the JSON
/// block, and one undo removes only the format.
#[test]
fn format_keeps_unsaved_edits_elsewhere_in_the_buffer() {
const ORIGINAL: &str = "# T\n\nIntro\n\n```json\n{\"b\":1,\"a\":[1e5]}\n```\n";
const EDITED: &str = "# T\n\nOutro\n\n```json\n{\"b\":1,\"a\":[1e5]}\n```\n";
let (registry, mut session, vault) = setup("json-undo-dirty", ORIGINAL);
run(&registry, &mut session, "buffer.open", &json!({ "path": "n.md" }));
let start = text(&session).find("Intro").unwrap_or_default();
let end = start.saturating_add("Intro".len());
session.buffer_mut("n.md").expect("the buffer is open").edit(start..end, "Outro").expect("user edit");
assert_eq!(text(&session), EDITED);
let result = run(&registry, &mut session, "json.format_block", &json!({ "path": "n.md", "index": 2 }));
assert_eq!(result["changed"], json!(true));
assert_eq!(text(&session), "# T\n\nOutro\n\n```json\n{\n \"b\": 1,\n \"a\": [\n 1e5\n ]\n}\n```\n");
assert_eq!(on_disk(vault.path(), "n.md"), ORIGINAL, "nothing is saved behind the user's back");
run(&registry, &mut session, "buffer.undo", &json!({ "path": "n.md" }));
assert_eq!(text(&session), EDITED, "undo removes only the format, the user edit stays");
run(&registry, &mut session, "buffer.redo", &json!({ "path": "n.md" }));
assert_eq!(text(&session), "# T\n\nOutro\n\n```json\n{\n \"b\": 1,\n \"a\": [\n 1e5\n ]\n}\n```\n");
}
/// Without an open buffer the format is one write: an already formatted block reports
/// `changed: false` and leaves the file bytes and mtime untouched.
#[test]
fn format_without_an_open_buffer_writes_the_file_once() {
let (registry, mut session, vault) = setup("json-no-buffer", NOTE);
let result = run(&registry, &mut session, "json.format_block", &json!({ "path": "n.md", "index": 1 }));
assert_eq!(result["changed"], json!(true));
assert_eq!(on_disk(vault.path(), "n.md"), FORMATTED, "the format reached the disk");
let mtime = std::fs::metadata(vault.path().join("n.md")).and_then(|meta| meta.modified()).expect("mtime");
let result = run(&registry, &mut session, "json.format_block", &json!({ "path": "n.md", "index": 1 }));
assert_eq!(result["changed"], json!(false), "the block is already formatted");
assert_eq!(on_disk(vault.path(), "n.md"), FORMATTED, "the bytes are identical");
let again = std::fs::metadata(vault.path().join("n.md")).and_then(|meta| meta.modified()).expect("mtime");
assert_eq!(again, mtime, "an unchanged format does not rewrite the file");
}
/// A CRLF note keeps CRLF everywhere, also inside the formatted JSON (no mixed line endings).
#[test]
fn format_in_a_crlf_note_keeps_crlf_outside_the_json() {
const CRLF: &str = "# T\r\n\r\n```json\r\n{\"b\":1,\"a\":[1e5]}\r\n```\r\n";
let (registry, mut session, _vault) = setup("json-crlf", CRLF);
run(&registry, &mut session, "buffer.open", &json!({ "path": "n.md" }));
let result = run(&registry, &mut session, "json.format_block", &json!({ "path": "n.md", "index": 1 }));
assert_eq!(result["changed"], json!(true));
let after = text(&session);
assert_eq!(after, "# T\r\n\r\n```json\r\n{\r\n \"b\": 1,\r\n \"a\": [\r\n 1e5\r\n ]\r\n}\r\n```\r\n");
assert!(after.starts_with("# T\r\n\r\n```json\r\n"), "the lines outside the JSON keep CRLF");
run(&registry, &mut session, "buffer.undo", &json!({ "path": "n.md" }));
assert_eq!(text(&session), CRLF, "ONE undo restores the CRLF original");
}
crates/engine/tests/lint_spelling.rs
+29
View File
@@ -0,0 +1,29 @@
//! The `spelling` rule: a word the vault uses in fewer than two notes is flagged once the rule is on.
mod common;
use common::{temp_config, temp_vault};
use notescraft_engine::{Registry, Session, register_core};
use serde_json::json;
#[test]
fn unknown_words_are_flagged_with_a_suggestion_only_when_spelling_is_on() {
let vault = temp_vault("spelling");
let config = temp_config("spelling");
for (name, text) in [("a.md", "Please receive the letter today.\n"), ("b.md", "We receive letters often.\n"), ("c.md", "Did you recieve the letter?\n")] {
std::fs::write(vault.path().join(name), text).expect("write");
}
let mut registry = Registry::new();
register_core(&mut registry).expect("register");
let mut session = Session::open(vault.path()).expect("open");
session.use_settings_dir(config.path());
let spelling = |registry: &Registry<Session>, session: &mut Session| -> Vec<serde_json::Value> {
let result = registry.run("lint.check", session, &json!({ "path": "c.md" })).expect("check");
result["issues"].as_array().cloned().unwrap_or_default().into_iter().filter(|issue| issue["rule"] == "spelling").collect()
};
assert!(spelling(&registry, &mut session).is_empty(), "off by default");
registry.run("settings.set", &mut session, &json!({ "key": "session.lint.spelling", "value": true })).expect("enable");
let issues = spelling(&registry, &mut session);
assert_eq!(issues.len(), 1, "{issues:?}");
assert_eq!(issues[0]["fix"]["label"], "Change to receive");
}
crates/engine/tests/pdf_preview.rs
+43
View File
@@ -0,0 +1,43 @@
//! `Session::pdf_preview`: a page image from the system's renderer, cached in the vault.
mod common;
use common::temp_vault;
use notescraft_engine::Session;
/// A one-page PDF with a line of text.
const PDF: &str = "%PDF-1.1\n1 0 obj<</Type/Catalog/Pages 2 0 R>>endobj\n2 0 obj<</Type/Pages/Kids[3 0 R]/Count 1>>endobj\n3 0 obj<</Type/Page/Parent 2 0 R/MediaBox[0 0 200 100]/Contents 4 0 R/Resources<</Font<</F1 5 0 R>>>>>>endobj\n4 0 obj<</Length 44>>stream\nBT /F1 24 Tf 20 40 Td (Hello) Tj ET\nendstream\nendobj\n5 0 obj<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>endobj\ntrailer<</Root 1 0 R>>\n%%EOF\n";
#[test]
fn a_pdf_page_becomes_a_cached_png_or_a_clear_error() {
let vault = temp_vault("pdf-preview");
std::fs::write(vault.path().join("doc.pdf"), PDF).expect("write");
let session = Session::open(vault.path()).expect("open");
assert!(session.pdf_preview("doc.pdf", 0).is_err(), "page numbers start at 1");
assert!(session.pdf_preview("../outside.pdf", 1).is_err());
match session.pdf_preview("doc.pdf", 1) {
Ok(png) => {
assert!(png.starts_with(b"\x89PNG"), "a PNG came back");
let cached = std::fs::read_dir(vault.path().join(".notescraft/cache")).expect("cache").count();
assert!(cached >= 1, "the image is cached");
assert_eq!(session.pdf_preview("doc.pdf", 1).expect("again"), png, "the second call reads the cache");
}
Err(error) => eprintln!("no PDF renderer on this machine: {error}"),
}
}
/// A two-page PDF: page 2 says "Second".
const TWO_PAGES: &str = "%PDF-1.1\n1 0 obj<</Type/Catalog/Pages 2 0 R>>endobj\n2 0 obj<</Type/Pages/Kids[3 0 R 6 0 R]/Count 2>>endobj\n3 0 obj<</Type/Page/Parent 2 0 R/MediaBox[0 0 200 100]/Contents 4 0 R/Resources<</Font<</F1 5 0 R>>>>>>endobj\n4 0 obj<</Length 44>>stream\nBT /F1 24 Tf 20 40 Td (First) Tj ET\nendstream\nendobj\n5 0 obj<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>endobj\n6 0 obj<</Type/Page/Parent 2 0 R/MediaBox[0 0 300 200]/Contents 7 0 R/Resources<</Font<</F1 5 0 R>>>>>>endobj\n7 0 obj<</Length 45>>stream\nBT /F1 24 Tf 20 40 Td (Second) Tj ET\nendstream\nendobj\ntrailer<</Root 1 0 R>>\n%%EOF\n";
#[test]
fn the_page_asked_for_is_the_page_rendered() {
let vault = temp_vault("pdf-pages");
std::fs::write(vault.path().join("two.pdf"), TWO_PAGES).expect("write");
let session = Session::open(vault.path()).expect("open");
let (Ok(first), Ok(second)) = (session.pdf_preview("two.pdf", 1), session.pdf_preview("two.pdf", 2)) else {
eprintln!("no PDF renderer able to pick a page on this machine");
return;
};
assert_ne!(first, second, "page 2 is a different image (it is also a different size)");
assert!(session.pdf_preview("two.pdf", 3).is_err(), "a page past the end is an error");
}
crates/engine/tests/perf_open.rs
+65
View File
@@ -0,0 +1,65 @@
//! Open-time measurements for large vaults (run on demand, release build):
//! `cargo test --release -p notescraft-engine --test perf_open -- --ignored --nocapture`.
//!
//! Prints where the time goes when a vault is read: the file list, reading the files, and each
//! index that is fed with a note. The numbers decide whether an index cache is worth having.
mod common;
use std::time::{Duration, Instant};
use common::temp_vault;
use notescraft_engine::Session;
const NOTES: usize = 5000;
const FOLDERS: usize = 50;
fn body(n: usize) -> String {
let next = (n + 1) % NOTES;
format!(
"---\ntags: [t{}]\n---\n# Note {n}\n\nParagraph about topic {} and project {} with [[n{next}]] and a #tag{}.\n\n- [ ] task {n}\n- item two\n\n## Section\n\nMore words: alpha beta gamma delta epsilon zeta eta theta iota kappa {n}.\n",
n % 50,
n % 97,
n % 31,
n % 13
)
}
fn timed<T>(label: &str, work: impl FnOnce() -> T) -> (T, Duration) {
let start = Instant::now();
let out = work();
let took = start.elapsed();
println!("{label:<34} {took:>10.2?}");
(out, took)
}
#[test]
#[ignore = "measurement, run on demand in a release build"]
fn where_the_time_goes_when_a_vault_is_read() {
let vault = temp_vault("perf_open");
for n in 0..NOTES {
let folder = vault.path().join(format!("f{}", n % FOLDERS));
std::fs::create_dir_all(&folder).expect("folder");
std::fs::write(folder.join(format!("n{n}.md")), body(n)).expect("note");
}
println!("--- {NOTES} notes in {FOLDERS} folders");
let (listing, _) = timed("list the vault (names, sizes)", || notescraft_vault::Vault::open_read_only(vault.path()).and_then(|v| v.list()).expect("list"));
let paths: Vec<String> = listing.into_iter().filter(|e| !e.is_dir).map(|e| e.path).collect();
let (texts, _) = timed("read every note", || paths.iter().map(|p| std::fs::read_to_string(vault.path().join(p)).expect("read")).collect::<Vec<_>>());
let mut links = notescraft_index::Index::new();
timed("link index: update_note", || paths.iter().zip(&texts).for_each(|(p, t)| links.update_note(p, t).expect("index")));
let mut search = notescraft_index::search::SearchIndex::new();
timed("search index: update", || paths.iter().zip(&texts).for_each(|(p, t)| search.update(p, t).expect("search")));
let mut passages = notescraft_knowledge::Index::default();
timed("knowledge passages: update", || paths.iter().zip(&texts).for_each(|(p, t)| drop(passages.update(p, t))));
let mut vocab = notescraft_knowledge::Vocab::default();
timed("knowledge vocabulary: update", || paths.iter().zip(&texts).for_each(|(p, t)| drop(vocab.update(p, t))));
let (mut session, lazy) = timed("open_lazy (names only)", || Session::open_lazy(vault.path()).expect("lazy"));
let ((), scan) = timed("index every note in steps", || while !session.index_step(Duration::from_millis(8)).finished {});
drop(session);
let (_, eager) = timed("eager open (whole thing)", || Session::open(vault.path()).expect("eager"));
println!("--- first window after {lazy:.2?}; everything indexed after {scan:.2?} more; eager open {eager:.2?}");
assert!(lazy < Duration::from_secs(1), "the first window should not wait for the notes: {lazy:?}");
}
crates/engine/tests/regex_search.rs
+20
View File
@@ -0,0 +1,20 @@
//! The `/pattern/` search form.
mod common;
use common::temp_vault;
use notescraft_engine::Session;
#[test]
fn a_slash_query_matches_lines_by_regular_expression() {
let vault = temp_vault("regex-search");
std::fs::write(vault.path().join("a.md"), "intro\nid: AB-123 and AB-9\n").expect("write");
std::fs::write(vault.path().join("b.md"), "nothing here\n").expect("write");
std::fs::write(vault.path().join("c.md"), "ab-45\n").expect("write");
let session = Session::open(vault.path()).expect("open");
let hits = session.search("/AB-\\d+/", 10).expect("search");
assert_eq!(hits.iter().map(|hit| hit.path.as_str()).collect::<Vec<_>>(), ["a.md", "c.md"], "most matches first, case-insensitive");
assert_eq!(hits.first().and_then(|hit| hit.snippets.first()).map(|snippet| (snippet.line, snippet.ranges.len())), Some((2, 2)));
assert!(session.search_matching("/AB-\\d+/", 10, true).expect("case").iter().all(|hit| hit.path == "a.md"));
assert!(session.search("/(/", 10).is_err(), "a bad pattern is an error, not a crash");
}
crates/engine/tests/search_operators.rs
+62
View File
@@ -0,0 +1,62 @@
//! The new search operators through the `search` command: `section:`,
//! `block:`, `task:`, `task-todo:`, `task-done:` and the `match_case`
//! param, over a real vault and a real session.
mod common;
use common::{seed, temp_vault};
use notescraft_engine::{Registry, Session, register_core};
use serde_json::{Value, json};
/// A registry plus a session on a vault with one note per shape.
fn fixture(name: &str) -> (Registry<Session>, Session, common::TempVault) {
let vault = temp_vault(name);
seed(
vault.path(),
&[
("sections.md", "# Alpha\n\nalpha text here\n\n# Beta\n\nbeta text\n"),
("tasks.md", "# Tasks\n\n- [ ] ship the release\n- [x] ship the docs\n"),
("blocks.md", "alpha alone\n\nship the release\n"),
],
);
let mut registry = Registry::new();
register_core(&mut registry).expect("register");
let session = Session::open(vault.path()).expect("open");
(registry, session, vault)
}
/// The paths a query returns, through the `search` command.
fn hits(registry: &Registry<Session>, session: &mut Session, params: &Value) -> Vec<String> {
let result = registry.run("search", session, params).expect("search");
let list = result.get("hits").and_then(Value::as_array).map_or_else(Vec::new, Clone::clone);
list.iter().map(|hit| hit.get("path").and_then(Value::as_str).unwrap_or_default().to_string()).collect()
}
/// The operators reach the engine unchanged.
#[test]
fn the_structural_operators_run_through_the_command() {
let (registry, mut session, _vault) = fixture("operators");
assert_eq!(hits(&registry, &mut session, &json!({ "query": "section:(alpha text)" })), vec!["sections.md"]);
assert_eq!(hits(&registry, &mut session, &json!({ "query": "section:(alpha beta)" })), Vec::<String>::new());
assert_eq!(hits(&registry, &mut session, &json!({ "query": "block:(ship the release)" })), vec!["tasks.md", "blocks.md"]);
assert_eq!(hits(&registry, &mut session, &json!({ "query": "task:release" })), vec!["tasks.md"]);
assert_eq!(hits(&registry, &mut session, &json!({ "query": "task-todo:" })), vec!["tasks.md"]);
assert_eq!(hits(&registry, &mut session, &json!({ "query": "task-done:docs" })), vec!["tasks.md"]);
assert_eq!(hits(&registry, &mut session, &json!({ "query": "task:release path:blocks/" })), Vec::<String>::new());
}
/// `match_case` is the param form of the `match-case:` operator.
#[test]
fn the_match_case_param_narrows_the_result() {
let (registry, mut session, _vault) = fixture("match-case");
assert_eq!(hits(&registry, &mut session, &json!({ "query": "ALPHA" })), vec!["sections.md", "blocks.md"]);
assert_eq!(hits(&registry, &mut session, &json!({ "query": "ALPHA", "match_case": true })), Vec::<String>::new());
assert_eq!(hits(&registry, &mut session, &json!({ "query": "Alpha", "match_case": true })), vec!["sections.md"]);
assert_eq!(hits(&registry, &mut session, &json!({ "query": "match-case: Alpha" })), vec!["sections.md"]);
assert_eq!(hits(&registry, &mut session, &json!({ "query": "alpha", "match_case": false })), vec!["sections.md", "blocks.md"]);
// A malformed query is still refused, switch or not.
assert!(registry.run("search", &mut session, &json!({ "query": "match-case:", "match_case": true })).is_err());
}
crates/engine/tests/settings.rs
+199 -1
View File
@@ -183,7 +183,7 @@ fn the_attachment_cap_is_a_session_files_setting() {
assert_eq!(session.config().files.max_attachment_mib, 128, "applied to the live session");
// 0 is refused at the config boundary...
let config = SessionConfig { files: FilesConfig { max_attachment_mib: 0 }, ..SessionConfig::default() };
let config = SessionConfig { files: FilesConfig { max_attachment_mib: 0, ..FilesConfig::default() }, ..SessionConfig::default() };
assert!(config.validate().is_err(), "0 is refused");
// ...and repaired to the default with a note through the settings document.
@@ -254,3 +254,201 @@ fn lint_and_block_settings_round_trip_through_settings_set() {
assert!(set(&mut session, "editor.blocks.grip", serde_json::json!(false)).is_ok());
assert!(set(&mut session, "editor.blocks.nope", serde_json::json!(true)).is_err(), "unknown key refused");
}
/// A session with its own config folder and a registry holding the core commands.
fn preview_session(name: &str) -> (common::TempVault, notescraft_engine::Registry<notescraft_engine::Session>, notescraft_engine::Session) {
let vault = temp_config(name);
let config = temp_config(&format!("{name}-cfg"));
let mut session = notescraft_engine::Session::open(vault.path()).expect("open the vault");
session.use_settings_dir(config.path());
let mut registry = notescraft_engine::Registry::new();
notescraft_engine::register_core(&mut registry).expect("register the core commands");
(config, registry, session)
}
/// The two preview switches as the document holds them.
fn saved_previews(config: &common::TempVault) -> (bool, bool) {
let (settings, notes) = load_from(config);
assert_eq!(notes, &[] as &[String], "{notes:?}");
(settings.editor.previews.colors, settings.editor.previews.named_colors)
}
/// `editor.previews.*` set, save, come back out of `config.toml` and refuse an
/// ill-typed value without touching the file.
#[test]
fn preview_settings_round_trip_through_settings_set() {
let defaults = notescraft_engine::PreviewSettings::default();
assert!(defaults.colors, "color swatches are on by default");
assert!(!defaults.named_colors, "named colors are off by default");
let (config, registry, mut session) = preview_session("previews-set");
let set = |session: &mut notescraft_engine::Session, key: &str, value: serde_json::Value| {
let params = serde_json::json!({ "key": key, "value": value });
registry.run("settings.set", session, &params)
};
assert!(set(&mut session, "editor.previews.colors", serde_json::json!(false)).is_ok());
assert!(set(&mut session, "editor.previews.named_colors", serde_json::json!(true)).is_ok());
assert_eq!(saved_previews(&config), (false, true), "both switches survive the file");
let got = registry.run("settings.get", &mut session, &serde_json::json!({})).expect("get");
let previews = got.get("settings").and_then(|settings| settings.get("editor")).and_then(|editor| editor.get("previews")).cloned();
assert_eq!(previews, Some(serde_json::json!({ "colors": false, "named_colors": true })), "{got}");
let before = std::fs::read(config.path().join("config.toml")).expect("read the config file");
for (key, value) in [("editor.previews.colors", serde_json::json!("yes")), ("editor.previews.named_colors", serde_json::json!(3))] {
let refused = set(&mut session, key, value);
assert!(refused.is_err(), "`{key}` with the wrong type is refused");
}
let after = std::fs::read(config.path().join("config.toml")).expect("read the config file again");
assert_eq!(before, after, "a refused value leaves config.toml byte-identical");
}
/// A `config.toml` written before the switches existed loads with the defaults.
#[test]
fn preview_settings_default_for_older_config_files() {
let fix = Fix::new("previews-old");
std::fs::write(config_file(&fix), b"schema_version = 1\n\n[editor]\nvim = false\n\n[editor.blocks]\ngrip = false\n").expect("write the old file");
let (settings, notes) = load_from(&fix.config);
assert_eq!(notes, &[] as &[String], "{notes:?}");
assert_eq!(settings.editor.previews, notescraft_engine::PreviewSettings::default(), "a file without the group gets the defaults");
assert_eq!(saved_previews(&fix.config), (true, false));
assert_eq!(AppSettings::default().editor.previews, notescraft_engine::PreviewSettings::default());
}
/// `settings.reset editor.previews` puts both switches back.
#[test]
fn settings_reset_restores_the_preview_defaults() {
let (config, registry, mut session) = preview_session("previews-reset");
for (key, value) in [("editor.previews.colors", serde_json::json!(false)), ("editor.previews.named_colors", serde_json::json!(true))] {
let params = serde_json::json!({ "key": key, "value": value });
assert!(registry.run("settings.set", &mut session, &params).is_ok(), "{key}");
}
assert_eq!(saved_previews(&config), (false, true));
let params = serde_json::json!({ "key": "editor.previews.named_colors" });
assert!(registry.run("settings.reset", &mut session, &params).is_ok());
assert_eq!(saved_previews(&config), (false, false), "one key resets to its own default");
for (key, value) in [("editor.previews.colors", serde_json::json!(false)), ("editor.previews.named_colors", serde_json::json!(true))] {
let params = serde_json::json!({ "key": key, "value": value });
assert!(registry.run("settings.set", &mut session, &params).is_ok(), "{key}");
}
let params = serde_json::json!({ "key": "editor.previews" });
assert!(registry.run("settings.reset", &mut session, &params).is_ok());
assert_eq!(saved_previews(&config), (true, false), "the whole group resets too");
}
/// The `session.files` group as it sits in `config.toml`.
fn saved_files(config: &common::TempVault) -> serde_json::Value {
let (settings, notes) = load_from(config);
assert_eq!(notes, &[] as &[String], "{notes:?}");
serde_json::to_value(&settings).expect("serialize settings").pointer("/session/files").cloned().unwrap_or_default()
}
/// The new-note folder, the link format, the link path, the deleted-file
/// behaviour and the hidden-file switch live in `session.files`, default the
/// way the design says, and set, save and reset like every other key.
#[test]
fn new_note_and_link_settings_round_trip_through_settings_set() {
let defaults = serde_json::to_value(notescraft_engine::FilesConfig::default()).expect("the default files section");
assert_eq!(defaults.pointer("/new_note_folder"), Some(&serde_json::json!("")));
assert_eq!(defaults.pointer("/link_format"), Some(&serde_json::json!("wikilink")));
assert_eq!(defaults.pointer("/link_path"), Some(&serde_json::json!("shortest")));
assert_eq!(defaults.pointer("/deleted_files"), Some(&serde_json::json!("trash")));
assert_eq!(defaults.pointer("/show_hidden_files"), Some(&serde_json::json!(false)));
let (config, registry, mut session) = preview_session("files-set");
let set = |session: &mut notescraft_engine::Session, key: &str, value: serde_json::Value| {
let params = serde_json::json!({ "key": key, "value": value });
registry.run("settings.set", session, &params)
};
let changed = [
("session.files.new_note_folder", serde_json::json!("Notes/Inbox")),
("session.files.link_format", serde_json::json!("markdown")),
("session.files.link_path", serde_json::json!("relative")),
("session.files.deleted_files", serde_json::json!("permanent")),
("session.files.show_hidden_files", serde_json::json!(true)),
];
for (key, value) in &changed {
assert!(set(&mut session, key, value.clone()).is_ok(), "{key} must be settable");
let field = key.trim_start_matches("session.files.");
assert_eq!(saved_files(&config).pointer(&format!("/{field}")), Some(value), "{key} in config.toml");
}
for (key, _) in &changed {
let params = serde_json::json!({ "key": key });
assert!(registry.run("settings.reset", &mut session, &params).is_ok(), "{key} must be resettable");
let field = key.trim_start_matches("session.files.");
assert_eq!(saved_files(&config).pointer(&format!("/{field}")), defaults.pointer(&format!("/{field}")), "{key} back to its default");
}
for (key, value) in &changed {
let params = serde_json::json!({ "key": key, "value": value.clone() });
assert!(registry.run("settings.set", &mut session, &params).is_ok(), "{key} must be settable again");
}
let params = serde_json::json!({ "key": "session.files" });
assert!(registry.run("settings.reset", &mut session, &params).is_ok());
assert_eq!(saved_files(&config), defaults, "the whole group resets too");
}
/// A `config.toml` written before these keys existed loads with the defaults.
#[test]
fn file_settings_default_for_older_config_files() {
let fix = Fix::new("files-old");
std::fs::write(config_file(&fix), b"schema_version = 1\n\n[session]\ndaily_folder = \"Notes\"\n\n[session.files]\nmax_attachment_mib = 8\n")
.expect("write the old file");
let (settings, notes) = load_from(&fix.config);
assert_eq!(notes, &[] as &[String], "{notes:?}");
assert_eq!(settings.session.files.max_attachment_mib, 8, "the keys the file has are kept");
assert_eq!(settings.session.files, notescraft_engine::FilesConfig { max_attachment_mib: 8, ..notescraft_engine::FilesConfig::default() });
}
/// An unusable new-note folder is repaired to the vault root with a note, a
/// wrong type and an unknown enum name are refused, and a refused set leaves
/// `config.toml` byte-identical.
#[test]
fn file_settings_repair_and_refuse_bad_values() {
let fix = Fix::new("files-bad-folder");
std::fs::write(config_file(&fix), b"schema_version = 1\n\n[session]\n\n[session.files]\nnew_note_folder = \"../up\"\n").expect("write the bad folder");
let (settings, notes) = load_from(&fix.config);
assert_eq!(settings.session.files.new_note_folder, "", "a folder that escapes the vault is cleared");
let joined = notes.join("\n");
assert!(joined.contains("session.files.new_note_folder"), "{joined}");
let (config, registry, mut session) = preview_session("files-refusals");
let set = |session: &mut notescraft_engine::Session, key: &str, value: serde_json::Value| {
let params = serde_json::json!({ "key": key, "value": value });
registry.run("settings.set", session, &params)
};
assert!(set(&mut session, "session.files.link_path", serde_json::json!("absolute")).is_ok(), "one good value first");
let before = std::fs::read(config.path().join("config.toml")).expect("read the config file");
for (key, value, code) in [
("session.files.new_note_folder", serde_json::json!(["Notes"]), "invalid_value"),
("session.files.link_format", serde_json::json!("WikiLink"), "invalid_value"),
("session.files.link_path", serde_json::json!("shortest "), "invalid_value"),
("session.files.deleted_files", serde_json::json!(true), "invalid_value"),
("session.files.show_hidden_files", serde_json::json!("no"), "invalid_value"),
("session.files.nope", serde_json::json!(true), "unknown_key"),
] {
let result = set(&mut session, key, value.clone());
assert!(matches!(result, Err(Error::Command { ref source, .. }) if source.code == code), "`{key}` = {value} must fail with {code}, got {result:?}");
}
let after = std::fs::read(config.path().join("config.toml")).expect("read the config file again");
assert_eq!(before, after, "a refused value leaves config.toml byte-identical");
assert_eq!(saved_files(&config).pointer("/link_path"), Some(&serde_json::json!("absolute")), "the last good value survives");
}
/// A new-note folder that escapes the vault is repaired to the vault root
/// through `settings.set`, with the note that says so.
#[test]
fn a_bad_new_note_folder_is_cleared_through_settings_set() {
let (config, registry, mut session) = preview_session("files-repair-set");
let params = serde_json::json!({ "key": "session.files.new_note_folder", "value": ".obsidian" });
let result = registry.run("settings.set", &mut session, &params).expect("the set succeeds, repaired");
let notes = result.to_string();
assert!(notes.contains("session.files.new_note_folder"), "{notes}");
assert_eq!(saved_files(&config).pointer("/new_note_folder"), Some(&serde_json::json!("")));
}
crates/engine/tests/switcher_commands.rs
+216
View File
@@ -0,0 +1,216 @@
//! The `switcher.search`, `note.random` and `note.unique` commands: ranking,
//! `#`/`^` queries, unicode, speed, randomness and collision suffixes.
mod common;
use std::time::{Duration, Instant};
use common::{on_disk, seed, temp_vault};
use notescraft_engine::{Registry, Session, register_core};
use serde_json::{Value, json};
fn setup(name: &str, files: &[(&str, &str)]) -> (Registry<Session>, Session, common::TempVault) {
let vault = temp_vault(name);
seed(vault.path(), files);
let mut registry = Registry::new();
register_core(&mut registry).expect("register");
let session = Session::open(vault.path()).expect("open");
(registry, session, vault)
}
fn run(registry: &Registry<Session>, session: &mut Session, id: &str, params: &Value) -> Value {
registry.run(id, session, params).unwrap_or_else(|error| panic!("{id}: {error:?}"))
}
/// The `hits` rows of a `switcher.search` result.
fn rows(result: &Value) -> Vec<Value> {
result["hits"].as_array().cloned().unwrap_or_default()
}
/// The hit paths, in the order the switcher ranked them.
fn paths(result: &Value) -> Vec<String> {
rows(result).iter().map(|hit| hit["path"].as_str().unwrap_or_default().to_owned()).collect()
}
fn search(registry: &Registry<Session>, session: &mut Session, query: &str) -> Value {
run(registry, session, "switcher.search", &json!({ "query": query }))
}
#[test]
fn exact_beats_prefix_beats_subsequence() {
let (registry, mut session, _vault) = setup("switcher-rank", &[("Not.md", ""), ("Note.md", ""), ("Xenolith.md", "")]);
let result = search(&registry, &mut session, "not");
assert_eq!(paths(&result), vec!["Not.md", "Note.md", "Xenolith.md"], "exact, then prefix, then subsequence");
assert!(rows(&result)[0]["score"].as_u64() > rows(&result)[1]["score"].as_u64(), "the exact hit outscores the prefix hit");
assert!(rows(&result)[1]["score"].as_u64() > rows(&result)[2]["score"].as_u64(), "the prefix hit outscores the subsequence hit");
}
#[test]
fn aliases_match_and_report_the_alias() {
let (registry, mut session, _vault) =
setup("switcher-alias", &[("Real Name.md", "---\naliases: [Spark]\n---\n"), ("Sparkle.md", ""), ("Unrelated.md", "")]);
let result = search(&registry, &mut session, "spark");
assert_eq!(paths(&result), vec!["Real Name.md", "Sparkle.md"], "the exact alias outranks the prefix title");
let first = &rows(&result)[0];
assert_eq!(first["kind"], "note");
assert_eq!(first["alias"], "Spark");
assert_eq!(first["title"], "Real Name");
assert_eq!(first["heading"], Value::Null, "a note hit carries no heading");
}
#[test]
fn a_hash_query_searches_headings_of_title_matching_notes() {
let (registry, mut session, _vault) =
setup("switcher-hash", &[("Alpha.md", "# Intro\n## Alpha Setup\n"), ("Beta.md", "# Alpha Care\n"), ("Gamma.md", "# Deep Alpha\n")]);
let scoped = search(&registry, &mut session, "alp#alpha");
assert_eq!(paths(&scoped), vec!["Alpha.md"], "only notes whose title matches the part before `#`");
assert_eq!(rows(&scoped)[0]["kind"], "heading");
assert_eq!(rows(&scoped)[0]["heading"], "Alpha Setup");
let everywhere = search(&registry, &mut session, "#alpha");
assert_eq!(paths(&everywhere), vec!["Alpha.md", "Beta.md", "Gamma.md"], "`#head` alone searches every note");
assert!(rows(&everywhere).iter().all(|hit| hit["kind"] == "heading"), "a `#` query returns heading hits only");
}
#[test]
fn a_caret_query_searches_block_ids() {
let (registry, mut session, _vault) = setup("switcher-caret", &[("Alpha.md", "para ^blk-alpha\n"), ("Beta.md", "para ^alpha-1\n")]);
let scoped = search(&registry, &mut session, "alp^alpha");
assert_eq!(paths(&scoped), vec!["Alpha.md"], "only notes whose title matches the part before `^`");
assert_eq!(rows(&scoped)[0]["kind"], "block");
assert_eq!(rows(&scoped)[0]["block"], "blk-alpha");
let everywhere = search(&registry, &mut session, "^alpha");
assert_eq!(paths(&everywhere), vec!["Beta.md", "Alpha.md"], "the prefix block id outranks the subsequence one");
assert!(rows(&everywhere).iter().all(|hit| hit["kind"] == "block"));
}
#[test]
fn unicode_matches_case_insensitively() {
let (registry, mut session, _vault) = setup("switcher-unicode", &[("Ünïcode.md", "# Grüber Kopf\n"), ("Plain.md", "")]);
let prefix = search(&registry, &mut session, "ünï");
assert_eq!(paths(&prefix), vec!["Ünïcode.md"]);
let heading = search(&registry, &mut session, "#grüber");
assert_eq!(paths(&heading), vec!["Ünïcode.md"]);
assert_eq!(rows(&heading)[0]["heading"], "Grüber Kopf");
}
#[test]
fn an_empty_query_lists_the_most_recently_modified_notes() {
let (registry, mut session, _vault) = setup("switcher-recent", &[("a.md", "a"), ("b.md", "b"), ("c.md", "c")]);
std::thread::sleep(Duration::from_millis(10));
run(&registry, &mut session, "note.write", &json!({ "path": "b.md", "content": "touched" }));
let result = search(&registry, &mut session, "");
assert_eq!(paths(&result).first(), Some(&"b.md".to_owned()), "the most recently modified note comes first");
assert_eq!(paths(&result).len(), 3);
let limited = run(&registry, &mut session, "switcher.search", &json!({ "query": "", "limit": 2 }));
assert_eq!(paths(&limited).len(), 2, "`limit` caps the recent list too");
}
#[test]
fn five_thousand_notes_answer_in_milliseconds() {
let vault = temp_vault("switcher-perf");
let files: Vec<(String, String)> = (0..5000).map(|i| (format!("note-{i:04}.md"), format!("# Heading {i}\n\nBody ^blk-{i:04}\n"))).collect();
let borrowed: Vec<(&str, &str)> = files.iter().map(|(path, body)| (path.as_str(), body.as_str())).collect();
seed(vault.path(), &borrowed);
let mut registry = Registry::new();
register_core(&mut registry).expect("register");
let mut session = Session::open(vault.path()).expect("open");
for query in ["note", "#head", "^blk", "zzz-no-such-thing"] {
let start = Instant::now();
let result = search(&registry, &mut session, query);
let elapsed = start.elapsed();
assert!(rows(&result).len() <= 30, "the default limit holds");
assert!(elapsed.as_millis() < 500, "`{query}` took {elapsed:?}");
}
}
#[test]
fn random_stays_inside_a_folder_and_varies() {
let mut files: Vec<(String, String)> = (0..10).map(|i| (format!("root-{i:02}.md"), String::new())).collect();
for i in 0..10 {
files.push((format!("sub/sub-{i:02}.md"), String::new()));
}
let borrowed: Vec<(&str, &str)> = files.iter().map(|(path, body)| (path.as_str(), body.as_str())).collect();
let (registry, mut session, _vault) = setup("note-random-folder", &borrowed);
let mut seen: Vec<String> = Vec::new();
for _ in 0..50 {
let path = run(&registry, &mut session, "note.random", &json!({ "folder": "sub" }))["path"].as_str().unwrap_or_default().to_owned();
assert!(path.starts_with("sub/"), "{path} escaped the folder");
seen.push(path);
}
seen.sort();
seen.dedup();
assert!(seen.len() > 1, "50 calls varied over the folder's notes");
let whole = run(&registry, &mut session, "note.random", &json!({}));
assert!(whole["path"].as_str().unwrap_or_default().to_ascii_lowercase().ends_with(".md"));
let outside = registry.run("note.random", &mut session, &json!({ "folder": "nope" }));
assert!(outside.is_err(), "a folder without notes is an error");
}
#[test]
fn random_on_an_empty_vault_is_not_found() {
let (registry, mut session, _vault) = setup("note-random-empty", &[]);
let Err(notescraft_engine::Error::Command { source, .. }) = registry.run("note.random", &mut session, &json!({})) else {
panic!("no note to pick");
};
assert_eq!(source.code, "not_found");
}
#[test]
fn unique_notes_never_overwrite_and_take_the_next_suffix() {
let (registry, mut session, vault) = setup("note-unique", &[("keep.md", "keep me\n")]);
let first = run(&registry, &mut session, "note.unique", &json!({}))["path"].as_str().unwrap_or_default().to_owned();
let stem = first.get(..first.find(".md").unwrap_or(first.len())).unwrap_or_default();
assert_eq!(stem.len(), 12, "`{first}` is a YYYYMMDDHHmm name");
assert!(stem.chars().all(|c| c.is_ascii_digit()), "`{first}` is all digits");
assert_eq!(on_disk(vault.path(), &first), "", "the note is created empty");
assert_eq!(on_disk(vault.path(), "keep.md"), "keep me\n", "an existing note is untouched");
let second = run(&registry, &mut session, "note.unique", &json!({}))["path"].as_str().unwrap_or_default().to_owned();
assert_eq!(second, format!("{stem}-1.md"), "a collision takes the -1 suffix");
let third = run(&registry, &mut session, "note.unique", &json!({}))["path"].as_str().unwrap_or_default().to_owned();
assert_eq!(third, format!("{stem}-2.md"), "the next collision takes -2");
assert_eq!(on_disk(vault.path(), &first), "", "the first note keeps its empty content");
}
#[test]
fn unique_needs_a_writable_session() {
let vault = temp_vault("note-unique-readonly");
let mut registry = Registry::new();
register_core(&mut registry).expect("register");
let mut session = Session::open_read_only(vault.path()).expect("open read-only");
assert!(registry.run("note.unique", &mut session, &json!({})).is_err(), "a read-only session refuses the write");
}
#[test]
fn hostile_params_are_refused_not_panicked() {
let (registry, mut session, _vault) = setup("switcher-hostile", &[("n.md", "# Head\n")]);
let long = "x".repeat(201);
for params in [
json!({ "query": long }),
json!({ "query": "ok", "limit": 0 }),
json!({ "query": "ok", "limit": 101 }),
json!({ "query": "ok", "limit": -1 }),
json!({ "query": "ok", "limit": "30" }),
json!({}),
json!({ "query": 7 }),
json!({ "query": "ok", "extra": true }),
] {
assert!(registry.run("switcher.search", &mut session, &params).is_err(), "{params}");
}
for query in ["#", "^", "###^^^#", "\u{0}nul\u{1}", "note#", "note^", "^", "#", "\u{10FFFF}"] {
let result = search(&registry, &mut session, query);
assert!(result["count"].as_u64().unwrap_or_default() <= 30, "`{query}` answered without panicking");
}
for params in [json!({ "folder": "../outside" }), json!({ "folder": 7 }), json!({ "folder": "nope/deep" })] {
let outcome = registry.run("note.random", &mut session, &params);
assert!(outcome.is_err(), "{params}");
}
let registry_still_usable = search(&registry, &mut session, "n");
assert_eq!(paths(&registry_still_usable), vec!["n.md"], "the registry still answers");
}
crates/i18n/locales/en.toml
+206
View File
@@ -173,8 +173,11 @@ process-exited = "agent process exited"
linked = "Linked mentions"
unlinked = "Unlinked mentions"
link-button = "Link"
outgoing = "Outgoing links"
not-created = "This note does not exist yet"
more = "and {{n}} more"
skipped = "{{n}} large notes skipped"
partial = "Still reading the vault: this list may be incomplete."
[ui.canvas-view-render]
image = "Image preview"
@@ -195,6 +198,11 @@ diagnostics = "Diagnostics"
show-source = "Show source"
hide-source = "Hide source"
source-error = "Could not render this diagram"
diagram = "Diagram"
source = "Source"
chip-unsupported = "Unsupported diagram: {{kind}}"
chip-error = "Error: {{message}}"
chip-big = "Too big to draw: {{message}}"
[ui.editing-prompt]
title = "Unsaved changes"
@@ -207,6 +215,7 @@ discard = "Discard"
cancel = "Cancel"
[ui.graph-view]
indexing = "The graph appears when the vault is read. Indexing"
orphans = "Orphans"
unresolved = "Unresolved"
tags = "Tags"
@@ -215,6 +224,13 @@ global = "Global"
less = "−"
more = "+"
hops = "hops"
forces = "Forces"
centre-force = "Center force"
repel-force = "Repel force"
link-force = "Link force"
link-distance = "Link distance"
groups = "Groups"
group-hint = "path or tag:name, Enter"
[ui.menubar]
app = "NotesCraft"
@@ -254,12 +270,40 @@ search-commands = "Search Commands…"
[ui.note-view]
add-to-chat = "Add to chat"
copy = "Copy"
code-copied = "Copied"
code-tree = "Tree"
query-label = "Query"
query-empty = "No notes match"
query-invalid = "Invalid query"
color-copy = "Copy color"
color-copy-rgb = "Copy as rgb()"
color-previews = "Color previews"
color-named = "Named colors (red, gray)"
code-wrap = "Wrap"
code-no-wrap = "No wrap"
code-plain = "Text"
json-valid = "Valid"
json-format = "Format"
json-minify = "Minify"
json-error = "Line {{line}}: {{message}}"
did-you-mean = "Did you mean {{language}}?"
add-grammar = "Add grammar"
offer-title = "This block looks like JSON"
offer-body = "A single string without a language label parses as JSON. Format it? Nothing changes until you accept."
offer-not-now = "Not now"
truncated = "Large note: showing the first 2 MiB"
embed = "Embed: {{target}}"
embed-missing = "Cannot find {{target}}"
embed-cycle = "{{target}} is already open above (circular embed)"
embed-depth = "Not shown: {{target}} is nested too deeply"
embed-open = "Open note"
media-pdf = "PDF"
media-audio = "Audio"
media-video = "Video"
media-open = "Open in default app"
media-play = "Play"
media-stop = "Stop"
media-no-player = "No audio player found on this system; opened files stay in your default app"
embed-section-missing = "Section not found: {{target}}"
image-failed = "Cannot display image {{target}}"
@@ -359,11 +403,16 @@ operators-0 = "path:"
operators-1 = "tag:"
operators-2 = "\"phrase\""
operators-3 = "-term"
operators-4 = "section:"
operators-5 = "block:"
operators-6 = "task:"
operators-7 = "match-case:"
[ui.search-panel-results]
results = "RESULTS"
empty = "Search your vault"
no-results = "No results for"
partial = "Still reading the vault: matches found on disk are shown too."
help-0 = "Find notes by title or content."
help-1 = "Use filters to narrow your search."
@@ -455,6 +504,35 @@ ui-sounds = "UI sounds"
ui-sounds-description = "Play interface sounds"
ui-scale = "UI scale"
ui-scale-description = "Preview interface size before applying"
new-note-folder = "Default location for new notes"
new-note-folder-description = "Folder new notes are created in; empty uses the folder of the open note"
show-hidden = "Show hidden files"
show-hidden-description = "List folders and notes whose name starts with a dot in the Explorer; the search and graph still skip them"
link-format = "New link format"
link-format-description = "How links written by NotesCraft look: [[Wikilink]] or [Markdown](link.md)"
link-wikilink = "Wikilink"
link-markdown = "Markdown link"
link-path = "Path in new links"
link-path-description = "Shortest is the file name, relative starts at the current note, absolute starts at the vault root"
path-shortest = "Shortest"
path-relative = "Relative"
path-absolute = "Absolute"
deleted-files = "Deleted files"
deleted-files-description = "Move deleted notes to the vault trash, or erase them for good"
deleted-trash = "Vault trash"
deleted-permanent = "Erase permanently"
line-numbers = "Line numbers"
line-numbers-description = "Number the lines of the Markdown source"
strict-breaks = "Strict line breaks"
strict-breaks-description = "Join the lines of a paragraph into one, as CommonMark does; off keeps every line break"
default-view = "Default view for new tabs"
default-view-description = "Whether a note opens rendered or as Markdown source"
view-visual = "Visual"
view-source = "Source"
font-size = "Editor font size"
font-size-description = "Text size of the Markdown source and the rendered note, in points"
readable-line = "Readable line length"
readable-line-description = "Keeps note text in a comfortable column instead of the full window width"
note-padding = "Note padding"
note-padding-description = "Space between a note and the sides of its card, in points"
sound-volume = "Sound volume"
@@ -503,7 +581,133 @@ preview = "PREVIEW"
note = "Welcome note"
open = "Open"
[ui.tabs]
more = "{{n}} more"
close-others = "Close other tabs"
close-all = "Close all tabs"
close-tab = "Close tab"
[ui.large-folder]
title = "Open this large folder?"
body = "Opening it reads every file and adds a .notescraft folder here. If this is not a vault, cancel."
more-than = "More than "
files = "files"
notes = "notes"
open-anyway = "Open anyway"
cancel = "Cancel"
[ui.batch-view]
title = "Batch format and lint"
scope = "Scope"
current-note = "Current note"
folder = "Folder {{name}}"
whole-vault = "Whole vault"
checks = "Checks"
check-json = "Format JSON blocks"
check-lint = "Apply lint fixes"
check-labels = "Add missing language labels"
replace-heading = "Replace text"
replace-find = "Find"
replace-with = "Replace with"
replace-case = "Match case"
replace-word = "Whole word"
kind-replace = "Replace"
title-replace = "Replace text"
run = "Run dry run"
running = "Reading notes…"
cancel = "Cancel"
note = "Dry run only. Nothing is written until you accept changes file by file."
finished = "Dry run finished"
would-change = "Would change"
needs-review = "Needs review"
not-edited = "Not edited"
clean = "Already clean"
tab-all = "All {{n}}"
tab-changes = "Changes {{n}}"
tab-review = "Review {{n}}"
tab-skipped = "Skipped {{n}}"
badge-pending = "Pending"
badge-ready = "Ready"
badge-review = "Review"
badge-skipped = "Skipped"
badge-clean = "Clean"
changes = "{{n}} changes"
one-change = "1 change"
reviewed = "Reviewed {{done}} of {{total}}"
accepted = "{{n}} accepted"
file-of = "File {{i}} of {{n}}"
accept = "Accept"
skip = "Skip"
previous = "Previous"
next = "Next"
accept-all = "Accept all in file"
apply = "Apply {{n}} accepted"
kind-json = "JSON"
kind-lint = "Lint"
kind-label = "Label"
title-label = "Add language label"
title-json = "Format JSON block"
title-unlabeled = "Format JSON block (no language label)"
close = "Close"
applied = "Wrote {{written}} files, skipped {{skipped}}, failed {{failed}}"
empty = "Nothing to review"
truncated = "Showing the first 5,000 notes"
pick = "Pick a file to review its changes"
[ui.find-bar]
find = "Find"
replace = "Replace with"
none = "No matches"
count = "{{n}} of {{total}}"
previous = "Previous match"
next = "Next match"
close = "Close"
case = "Match case"
regex = "Regular expression"
word = "Whole word"
replace-one = "Replace"
replace-all = "Replace all"
toggle-replace = "Show replace"
replaced = "Replaced {{n}}"
[ui.grammar-dialog]
title = "Add a grammar"
url-label = "Link to a .sublime-syntax file (leave empty to look in the NotesCraft grammar repository)"
intro = "Only https links are accepted. The file is checked before anything is saved."
link-ok = "https link"
checking = "Checking the grammar…"
test-install = "Test and install"
add = "Add grammar"
cancel = "Cancel"
ready-size = "{{size}} KB"
ready-valid = "Valid grammar: {{name}}"
ready-license = "License: {{license}}"
installed = "Installed grammar {{name}}"
start-failed = "Cannot start the download"
stopped = "The download stopped"
[ui.index-status]
indexing = "Indexing"
paused = "Indexing paused"
hint = "Search and file names work now. Backlinks fill in as notes are read."
pause = "Pause indexing"
resume = "Resume indexing"
[ui.explorer]
truncated = "more items are not shown (the tree is full)"
pin-limit = "You can pin up to 20 folders."
filter = "Filter files"
collapse-all = "Collapse all folders"
follow = "Follow the active note"
pinned = "PINNED"
clear-filter = "Clear filter"
[ui.sidebar]
tab-explorer = "Explorer"
tab-search = "Search"
tab-tags = "Tags"
tab-buffers = "Buffers"
tab-source = "Source"
search = "Search notes"
shortcut = "⌘K"
tags = "TAGS"
@@ -914,6 +1118,8 @@ toolbar-description = "Appears when text is selected"
lint = "Lint"
check-typing = "Check notes while typing"
check-typing-description = "Underline problems and offer a fix"
spelling = "Spelling"
spelling-description = "Underline words the vault does not use in at least two notes, and suggest the closest one"
fix-on-save = "Fix all safe on save"
fix-on-save-description = "Apply only fixes marked safe (spacing, list markers). Links are never changed without asking"
rules = "Rules"
crates/index/src/search/mod.rs
+37 -4
View File
@@ -17,7 +17,10 @@
//! # Query language
//!
//! See [`SearchQuery::parse`]: bare terms (AND), `"phrases"`, `-negations`,
//! `OR`, and the `path:`, `file:`, `tag:` and `line:` operators.
//! `OR`, the `path:`, `file:`, `tag:` and `line:` operators, the structural
//! `section:`, `block:` and `task:` operators (`task-todo:` and `task-done:`
//! filter by checkbox state), and the `match-case:` / `ignore-case:` flags
//! ([`SearchOptions`]).
//!
//! ```
//! use notescraft_index::search::{SearchIndex, SearchQuery};
@@ -38,13 +41,25 @@
//! notes.remove("work/Plan.md").unwrap();
//! assert!(notes.search(&SearchQuery::parse("ship").unwrap(), 10).is_empty());
//! ```
//!
//! # Verification pass
//!
//! The inverted index only narrows the candidates; the structural operators
//! and the case-sensitive terms are decided on the text of a candidate note
//! ([`verify`]). An alternative that needs that pass is checked on at most
//! [`MAX_CANDIDATES`] notes, in path order, so a vault of huge notes cannot
//! make one query unbounded; narrowing with `path:` or `tag:` keeps such a
//! query exact.
pub(crate) mod doc;
pub(crate) mod options;
pub(crate) mod query;
pub(crate) mod rank;
pub(crate) mod scan;
pub(crate) mod tokenize;
pub(crate) mod verify;
pub use options::SearchOptions;
pub use query::SearchQuery;
pub use rank::{Hit, Snippet};
@@ -56,6 +71,11 @@ use crate::reading::path::{self, PathKind};
use crate::search::doc::Doc;
use crate::search::query::Clause;
/// Most candidate notes one alternative is verified against the note text
/// on, and only for the alternatives that need that pass: a documented cap,
/// chosen so a vault of large notes cannot make one query unbounded.
pub const MAX_CANDIDATES: usize = 5_000;
/// In-memory full-text index of a vault: per-note fields, BM25 ranking, and
/// one global map of term to matched notes.
///
@@ -160,15 +180,17 @@ impl SearchIndex {
#[must_use]
pub fn search(&self, query: &SearchQuery, limit: usize) -> Vec<rank::Hit> {
let terms = query.positive_terms();
let case_sensitive = query.case_sensitive();
let mut found: BTreeSet<String> = BTreeSet::new();
for alternative in query.alternatives() {
let candidates: Vec<String> = match seed_list(alternative, &self.postings) {
Some(list) => list.clone(),
None => self.docs.keys().cloned().collect(),
};
let candidates = capped(candidates, query.needs_text(alternative));
for path in candidates {
let Some(doc_data) = self.docs.get(&path) else { continue };
if alternative.iter().all(|clause| rank::matches(doc_data, clause)) {
if alternative.iter().all(|clause| rank::matches(doc_data, clause, case_sensitive)) {
found.insert(path);
}
}
@@ -176,7 +198,7 @@ impl SearchIndex {
for clause in query.negatives() {
found.retain(|path| {
let Some(doc_data) = self.docs.get(path) else { return false };
!rank::matches(doc_data, clause)
!rank::matches(doc_data, clause, case_sensitive)
});
}
let mut hits: Vec<rank::Hit> = found
@@ -245,6 +267,17 @@ impl SearchIndex {
/// The empty candidate list shown for terms the index has never seen.
static NO_CANDIDATES: LazyLock<Vec<String>> = LazyLock::new(Vec::new);
/// The candidate list of an alternative that is verified on the note text,
/// cut to [`MAX_CANDIDATES`] in path order so the cut does not depend on
/// the hash order of the note map. Every other alternative is exact.
fn capped(mut candidates: Vec<String>, verified: bool) -> Vec<String> {
if verified && candidates.len() > MAX_CANDIDATES {
candidates.sort_unstable();
candidates.truncate(MAX_CANDIDATES);
}
candidates
}
/// The smallest candidate list among the alternative's term clauses, so the
/// intersection starts small. [`NO_CANDIDATES`] when a term is absent from
/// the index; `None` when the alternative has no term clause, in which case
@@ -252,7 +285,7 @@ static NO_CANDIDATES: LazyLock<Vec<String>> = LazyLock::new(Vec::new);
fn seed_list<'a>(alternative: &[Clause], postings: &'a HashMap<String, Vec<String>>) -> Option<&'a Vec<String>> {
let mut best: Option<&'a Vec<String>> = None;
for clause in alternative {
let Clause::Term { term } = clause else { continue };
let Clause::Term { term, .. } = clause else { continue };
let Some(list) = postings.get(term) else {
return Some(&NO_CANDIDATES);
};
crates/index/src/search/options.rs
+44
View File
@@ -0,0 +1,44 @@
//! Options of a search: the case switch, today the only one.
//!
//! The options are resolved when the query is parsed
//! ([`SearchQuery::parse_with`](crate::search::SearchQuery::parse_with)), so
//! the query itself carries the mode and the index needs no extra state.
/// How a query is matched. The default ([`SearchOptions::default`]) is
/// case-insensitive, as every text search in the wild is.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct SearchOptions {
/// When `true`, bare terms and `"phrases"` must occur with the exact
/// spelling of the query: `Ship` no longer matches `ship`.
///
/// The structural operators (`line:`, `section:`, `block:`, `task:`)
/// and the metadata operators (`path:`, `file:`, `tag:`) stay
/// case-insensitive, like the note data they read.
pub case_sensitive: bool,
}
impl SearchOptions {
/// Case-insensitive, the default.
#[must_use]
pub const fn new() -> Self {
Self { case_sensitive: false }
}
/// Case-sensitive terms and phrases.
#[must_use]
pub const fn matching_case() -> Self {
Self { case_sensitive: true }
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_default_is_case_insensitive() {
assert_eq!(SearchOptions::default(), SearchOptions::new());
assert!(!SearchOptions::default().case_sensitive);
assert!(SearchOptions::matching_case().case_sensitive);
}
}
crates/index/src/search/query.rs
+145 -101
View File
@@ -1,34 +1,63 @@
//! The search query language and its parser: `term`, `"phrase"`, `-negated`,
//! `OR`, `path:folder/`, `file:name`, `tag:#a/b`, `line:(a b)`.
//! `OR`, `path:folder/`, `file:name`, `tag:#a/b`, `line:(a b)`,
//! `section:(a b)`, `block:(a b)`, `task:term`, `task-todo:`,
//! `task-done:` and the case switches `match-case:` / `ignore-case:`.
//!
//! `/regex/` is rejected with [`Error::Unsupported`](crate::Error::Unsupported):
//! no regex crate is an approved dependency yet. Queries are parsed eagerly
//! and every malformed input is an error, never a panic.
//! and every malformed input is an error, never a panic. An unknown
//! `something:` prefix is not an operator: it stays a plain term.
pub(crate) mod parse;
use std::collections::BTreeSet;
use crate::search::options::SearchOptions;
use crate::search::query::parse::{case_flag, clause_of};
use crate::search::scan::scan;
use crate::error::{Error, Result};
use crate::search::tokenize;
/// Longest query accepted, in chars.
pub(crate) const MAX_QUERY_CHARS: usize = 1000;
/// Most terms per query, negations and phrase and `line:` members included.
pub(crate) const MAX_QUERY_TERMS: usize = 32;
/// Which task lines `task:`, `task-todo:` and `task-done:` accept.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum TaskState {
/// `task:`: any `- [ ]` or `- [x]` line.
Any,
/// `task-todo:`: unchecked lines only.
Todo,
/// `task-done:`: checked lines only.
Done,
}
impl TaskState {
/// Whether a task line in `line` satisfies the operator.
pub(crate) fn accepts(self, line: TaskState) -> bool {
self == TaskState::Any || self == line
}
}
/// One whitespace-separated clause of a query.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum Clause {
/// A bare term: it must occur somewhere in the note.
Term {
/// Lowercased token.
/// Lowercased token, the inverted index key.
term: String,
/// The token as the query spells it, used by the case-sensitive
/// verification pass.
raw: String,
},
/// A quoted phrase: the terms must occur in order in one unit.
Phrase {
/// Lowercased tokens, in phrase order, at least one.
terms: Vec<String>,
/// The tokens as the query spells them, in the same order.
raw: Vec<String>,
},
/// `path:folder/`: the note path must live under `folder`.
Path {
@@ -50,6 +79,26 @@ pub(crate) enum Clause {
/// Lowercased terms, at least one.
terms: Vec<String>,
},
/// `section:(a b)`: all terms must be inside one heading section.
Section {
/// Lowercased terms, at least one.
terms: Vec<String>,
},
/// `block:(a b)`: all terms must be inside one block, that is one
/// paragraph, list item or table row, blocks being separated by blank
/// lines.
Block {
/// Lowercased terms, at least one.
terms: Vec<String>,
},
/// `task:term`, `task-todo:` and `task-done:`: task lines, filtered by
/// state and carrying every term.
Task {
/// Lowercased terms, none for the bare `task-todo:` filter.
terms: Vec<String>,
/// Which checkbox states the operator accepts.
state: TaskState,
},
}
/// A parsed query: OR-separated alternatives of positive clauses, plus
@@ -62,6 +111,8 @@ pub struct SearchQuery {
alternatives: Vec<Vec<Clause>>,
/// Clauses whose match removes a note from the result.
negatives: Vec<Clause>,
/// Whether terms and phrases must match the query spelling exactly.
case_sensitive: bool,
}
impl SearchQuery {
@@ -74,6 +125,18 @@ impl SearchQuery {
/// [`Error`] variants for a misplaced `OR`, operators without an
/// argument, `/regex/`, and queries over the length or term caps.
pub fn parse(query: &str) -> Result<Self> {
Self::parse_with(query, &SearchOptions::default())
}
/// Parse `query` with `options`: the case mode starts from the options
/// and a `match-case:` or `ignore-case:` piece in the query flips it
/// again. Both switches are flags, not clauses: they carry no argument
/// to search for and a lone `match-case:` is an empty query.
///
/// # Errors
///
/// The same as [`SearchQuery::parse`].
pub fn parse_with(query: &str, options: &SearchOptions) -> Result<Self> {
let chars = query.chars().count();
if chars > MAX_QUERY_CHARS {
return Err(Error::QueryTooLong { len: chars, max: MAX_QUERY_CHARS });
@@ -84,7 +147,12 @@ impl SearchQuery {
let mut negatives: Vec<Clause> = vec![];
let mut counted = 0_usize;
let mut pending_or = false;
let mut case_sensitive = options.case_sensitive;
for (negated, text) in raw {
if let Some(flag) = case_flag(&text) {
case_sensitive = flag;
continue;
}
if !negated && text == "OR" {
if current.is_empty() {
return Err(Error::InvalidQuery { reason: String::from("misplaced OR") });
@@ -116,7 +184,7 @@ impl SearchQuery {
if alternatives.is_empty() {
return Err(Error::EmptyQuery);
}
Ok(Self { alternatives, negatives })
Ok(Self { alternatives, negatives, case_sensitive })
}
/// OR alternatives, each an AND of positive clauses.
@@ -133,23 +201,51 @@ impl SearchQuery {
pub(crate) fn positive_terms(&self) -> Vec<String> {
self.alternatives.iter().flat_map(|alt| alt.iter()).flat_map(|clause| clause.terms().into_iter()).collect::<BTreeSet<_>>().into_iter().collect()
}
}
impl Clause {
/// The terms the clause searches for: one for `Term`, all the members of
/// a phrase or `line:` group, none for the path/file/tag operators.
pub(crate) fn terms(&self) -> Vec<String> {
match self {
Self::Term { term } => vec![term.clone()],
Self::Phrase { terms } | Self::Line { terms } => terms.clone(),
Self::Path { .. } | Self::File { .. } | Self::Tag { .. } => vec![],
}
/// Whether terms and phrases must match the spelling of the query.
pub(crate) fn case_sensitive(&self) -> bool {
self.case_sensitive
}
/// Whether the alternative is decided by a pass over the note text, in
/// which case its candidate list is capped (see
/// [`MAX_CANDIDATES`](crate::search::MAX_CANDIDATES)).
pub(crate) fn needs_text(&self, alternative: &[Clause]) -> bool {
alternative.iter().any(|clause| clause.is_text_pass(self.case_sensitive))
}
}
/// A `path:`, `file:` or `tag:` filter.
impl Clause {
/// The terms the clause searches for, as inverted index keys: one for
/// `Term`, all the members of a phrase, `line:`, `section:`, `block:`
/// or `task:` group, none for the path/file/tag operators.
pub(crate) fn terms(&self) -> Vec<String> {
match self {
Self::Term { term, .. } => vec![term.clone()],
Self::Phrase { terms, .. } | Self::Line { terms } | Self::Section { terms } | Self::Block { terms } | Self::Task { terms, .. } => terms.clone(),
Self::Path { .. } | Self::File { .. } | Self::Tag { .. } => vec![],
}
}
/// The spellings the case-sensitive pass compares with: bare terms and
/// phrases only, the structural operators stay case-insensitive.
pub(crate) fn spelled(&self) -> Option<&[String]> {
match self {
Self::Term { raw, .. } => Some(std::slice::from_ref(raw)),
Self::Phrase { raw, .. } => Some(raw),
_ => None,
}
}
/// Whether deciding this clause needs a pass over the note text.
fn is_text_pass(&self, case_sensitive: bool) -> bool {
matches!(self, Self::Section { .. } | Self::Block { .. } | Self::Task { .. }) || (case_sensitive && self.spelled().is_some())
}
}
/// A `path:`, `file:`, `tag:` or task filter: decidable without a term.
fn is_filter(clause: &Clause) -> bool {
matches!(clause, Clause::Path { .. } | Clause::File { .. } | Clause::Tag { .. })
matches!(clause, Clause::Path { .. } | Clause::File { .. } | Clause::Tag { .. } | Clause::Section { .. } | Clause::Block { .. } | Clause::Task { .. })
}
fn searchable(clause: &Clause) -> bool {
@@ -160,112 +256,59 @@ fn searchable(clause: &Clause) -> bool {
fn term_count(clause: &Clause) -> usize {
match clause {
Clause::Term { .. } | Clause::Path { .. } | Clause::File { .. } | Clause::Tag { .. } => 1,
Clause::Phrase { terms } | Clause::Line { terms } => terms.len(),
Clause::Phrase { terms, .. } | Clause::Line { terms } | Clause::Section { terms } | Clause::Block { terms } => terms.len(),
Clause::Task { terms, .. } => terms.len().max(1),
}
}
/// Turn one raw piece into a clause, lowercasing operator arguments and
/// tokenizing terms and phrases.
fn clause_of(raw: &str) -> Result<Clause> {
if raw.len() >= 2 && raw.starts_with('/') && raw.ends_with('/') {
return Err(Error::Unsupported(String::from("regex search")));
}
let prefix =
["path:", "file:", "tag:", "line:"].into_iter().find(|p| raw.len() >= p.len() && raw.get(..p.len()).is_some_and(|head| head.eq_ignore_ascii_case(p)));
if let Some(prefix) = prefix {
let (name, arg) = raw.split_once(':').unwrap_or((prefix, ""));
let name = name.to_lowercase();
let inner = arg.strip_prefix('(').and_then(|a| a.strip_suffix(')')).unwrap_or(arg).trim().to_lowercase();
return match name.as_str() {
"path" => folder_clause(&inner),
"file" => (!inner.is_empty()).then_some(Clause::File { name: inner }),
"tag" => tag_clause(&inner),
_ => line_clause(&inner),
}
.ok_or_else(|| Error::InvalidQuery { reason: format!("{name}: needs an argument") });
}
let terms = tokenize::tokens(raw).into_iter().map(|token| token.text).collect::<Vec<_>>();
if terms.is_empty() {
return Err(Error::InvalidQuery { reason: String::from("empty term") });
}
if terms.len() == 1 {
let term = terms.first().map_or(String::new(), Clone::clone);
Ok(Clause::Term { term })
} else {
Ok(Clause::Phrase { terms })
}
}
/// `line:(a b)` and `line:a`: one or more terms that must share a line.
fn line_clause(inner: &str) -> Option<Clause> {
let terms = tokenize::tokens(inner).into_iter().map(|token| token.text).collect::<Vec<_>>();
if terms.is_empty() {
return None;
}
Some(Clause::Line { terms })
}
/// `path:folder/`: the folder prefix, without the trailing slash.
fn folder_clause(inner: &str) -> Option<Clause> {
let folder = inner.trim_end_matches('/');
(!folder.is_empty()).then(|| Clause::Path { folder: folder.to_string() })
}
/// `tag:#a/b`: the tag without its leading `#`, one `#` only.
fn tag_clause(inner: &str) -> Option<Clause> {
let tag = inner.trim_start_matches('#');
if tag.is_empty() || tag.contains('#') {
return None;
}
Some(Clause::Tag { tag: tag.to_string() })
}
#[cfg(test)]
mod tests {
use super::*;
use crate::error::Error;
/// A query that must parse, or the test is wrong.
fn parse(query: &str) -> SearchQuery {
let parsed = SearchQuery::parse(query);
assert!(parsed.is_ok(), "{query:?} must parse");
parsed.unwrap_or_default()
}
/// A query that must not parse.
fn err(query: &str) {
assert!(SearchQuery::parse(query).is_err(), "{query:?} must not parse");
}
/// Lowercased words, as the index keys of a clause look.
fn words(list: &[&str]) -> Vec<String> {
list.iter().map(|word| (*word).to_string()).collect()
}
/// A bare term carrying the spelling it was written with.
fn term(written: &str) -> Clause {
Clause::Term { term: written.to_lowercase(), raw: String::from(written) }
}
#[test]
fn bare_terms_are_one_and_alternative() {
let query = parse("Ship THE Road");
assert_eq!(
query.alternatives(),
&[vec![Clause::Term { term: String::from("ship") }, Clause::Term { term: String::from("the") }, Clause::Term { term: String::from("road") }]]
);
assert_eq!(query.alternatives(), &[vec![term("Ship"), term("THE"), term("Road")]]);
}
#[test]
fn or_splits_alternatives() {
let query = parse("a OR b c");
assert_eq!(query.alternatives()[0], vec![Clause::Term { term: String::from("a") }]);
assert_eq!(query.alternatives()[1], vec![Clause::Term { term: String::from("b") }, Clause::Term { term: String::from("c") }]);
assert_eq!(query.alternatives()[0], vec![term("a")]);
assert_eq!(query.alternatives()[1], vec![term("b"), term("c")]);
}
#[test]
fn phrases_and_negations_and_groups_parse() {
let query = parse("-\"a b\" -path:work/ -file:note tag:#tag/sub line:(x y) (p OR q)");
assert_eq!(query.negatives()[0], Clause::Phrase { terms: vec![String::from("a"), String::from("b")] });
assert_eq!(query.negatives()[0], Clause::Phrase { terms: words(&["a", "b"]), raw: words(&["a", "b"]) });
assert_eq!(query.negatives()[1], Clause::Path { folder: String::from("work") });
// The group continues the current alternative; its OR splits it.
assert_eq!(
query.alternatives()[0],
vec![
Clause::Tag { tag: String::from("tag/sub") },
Clause::Line { terms: vec![String::from("x"), String::from("y")] },
Clause::Term { term: String::from("p") }
]
);
assert_eq!(query.alternatives()[1], vec![Clause::Term { term: String::from("q") }]);
assert_eq!(query.alternatives()[0], vec![Clause::Tag { tag: String::from("tag/sub") }, Clause::Line { terms: words(&["x", "y"]) }, term("p")]);
assert_eq!(query.alternatives()[1], vec![term("q")]);
}
#[test]
@@ -287,6 +330,15 @@ mod tests {
err("(a");
err("tag:");
err("tag:a#b");
err("section:");
err("section:()");
err("block:");
}
#[test]
fn a_task_filter_without_terms_is_a_whole_note_filter() {
let query = parse("task:()");
assert_eq!(query.alternatives()[0], vec![Clause::Task { terms: Vec::new(), state: TaskState::Any }]);
}
#[test]
@@ -301,19 +353,11 @@ mod tests {
assert!(SearchQuery::parse("t ".repeat(32).trim()).is_ok(), "32 terms pass");
}
#[test]
fn operators_carry_their_argument_shape() {
let query = parse("term tag:#Work/Sub file:Note path:Deep/Nested/");
assert_eq!(query.alternatives()[0][1], Clause::Tag { tag: String::from("work/sub") });
assert_eq!(query.alternatives()[0][2], Clause::File { name: String::from("note") });
assert_eq!(query.alternatives()[0][3], Clause::Path { folder: String::from("deep/nested") });
let query = parse("line:(Alpha Beta)");
assert_eq!(query.alternatives()[0][0], Clause::Line { terms: vec![String::from("alpha"), String::from("beta")] });
}
#[test]
fn positive_terms_are_deduplicated_and_sorted() {
let query = parse("b a \"a c\" line:(c d)");
assert_eq!(query.positive_terms(), vec![String::from("a"), String::from("b"), String::from("c"), String::from("d")]);
assert_eq!(query.positive_terms(), words(&["a", "b", "c", "d"]));
let query = parse("section:(a b) block:(b c) task:b");
assert_eq!(query.positive_terms(), words(&["a", "b", "c"]), "the structural operators score their terms too");
}
}
crates/index/src/search/query/parse.rs
+186
View File
@@ -0,0 +1,186 @@
//! Turning one scanned query piece into a [`Clause`]: the operator table,
//! the term and phrase tokenizer and the case flags.
//!
//! Everything here is a pure function of the piece, so a malformed operator
//! is an [`Error`] and never a panic.
use crate::error::{Error, Result};
use crate::search::query::{Clause, TaskState};
use crate::search::tokenize;
/// The operator names the parser knows, without their `:`.
pub(crate) const OPERATORS: [&str; 9] = ["path", "file", "tag", "line", "section", "block", "task", "task-todo", "task-done"];
/// Turn one raw piece into a clause, lowercasing operator arguments and
/// tokenizing terms and phrases.
pub(crate) fn clause_of(raw: &str) -> Result<Clause> {
if raw.len() >= 2 && raw.starts_with('/') && raw.ends_with('/') {
return Err(Error::Unsupported(String::from("regex search")));
}
if let Some((name, arg)) = operator_of(raw) {
let inner = arg.strip_prefix('(').and_then(|a| a.strip_suffix(')')).unwrap_or(arg).trim().to_lowercase();
return operator_clause(&name, &inner).ok_or_else(|| Error::InvalidQuery { reason: format!("{name}: needs an argument") });
}
let tokens = tokenize::tokens(raw);
let terms = tokens.iter().map(|token| token.text.clone()).collect::<Vec<_>>();
let spelled = tokens.iter().map(|token| raw.get(token.range.clone()).unwrap_or_default().to_string()).collect::<Vec<_>>();
if tokens.is_empty() {
return Err(Error::InvalidQuery { reason: String::from("empty term") });
}
if let [term] = terms.as_slice() {
return Ok(Clause::Term { term: term.clone(), raw: spelled.first().cloned().unwrap_or_default() });
}
Ok(Clause::Phrase { terms, raw: spelled })
}
/// The operator name and argument of `raw`, or `None` when `raw` is a plain
/// term: an unknown `something:` prefix is searched for as a word.
fn operator_of(raw: &str) -> Option<(String, &str)> {
let (name, arg) = raw.split_once(':')?;
let name = name.to_lowercase();
OPERATORS.contains(&name.as_str()).then_some((name, arg))
}
/// The clause of one known operator; `None` when it has no usable argument.
fn operator_clause(name: &str, inner: &str) -> Option<Clause> {
match name {
"path" => folder_clause(inner),
"file" => (!inner.is_empty()).then(|| Clause::File { name: inner.to_string() }),
"tag" => tag_clause(inner),
"line" => words_clause(inner, |terms| Clause::Line { terms }),
"section" => words_clause(inner, |terms| Clause::Section { terms }),
"block" => words_clause(inner, |terms| Clause::Block { terms }),
"task" => Some(task_clause(inner, TaskState::Any)),
"task-todo" => Some(task_clause(inner, TaskState::Todo)),
"task-done" => Some(task_clause(inner, TaskState::Done)),
_ => None,
}
}
/// `line:(a b)`, `section:(a b)` and `block:(a b)`: one or more terms that
/// must share a unit of the note.
fn words_clause(inner: &str, make: impl Fn(Vec<String>) -> Clause) -> Option<Clause> {
let terms = tokenize::tokens(inner).into_iter().map(|token| token.text).collect::<Vec<_>>();
(!terms.is_empty()).then(|| make(terms))
}
/// `task:word`, `task-todo:` and `task-done:`: task lines in `state`, with
/// no term the operator is a plain state filter.
fn task_clause(inner: &str, state: TaskState) -> Clause {
let terms = tokenize::tokens(inner).into_iter().map(|token| token.text).collect::<Vec<_>>();
Clause::Task { terms, state }
}
/// `path:folder/`: the folder prefix, without the trailing slash.
fn folder_clause(inner: &str) -> Option<Clause> {
let folder = inner.trim_end_matches('/');
(!folder.is_empty()).then(|| Clause::Path { folder: folder.to_string() })
}
/// `tag:#a/b`: the tag without its leading `#`, one `#` only.
fn tag_clause(inner: &str) -> Option<Clause> {
let tag = inner.trim_start_matches('#');
if tag.is_empty() || tag.contains('#') {
return None;
}
Some(Clause::Tag { tag: tag.to_string() })
}
/// The case mode a `match-case:` / `ignore-case:` piece asks for; `None`
/// for anything else, including the plain terms `match-case` and
/// `ignore-case`. The argument is ignored: the switch is a flag.
pub(crate) fn case_flag(raw: &str) -> Option<bool> {
let (name, _) = raw.split_once(':')?;
match name.to_lowercase().as_str() {
"match-case" => Some(true),
"ignore-case" => Some(false),
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::search::options::SearchOptions;
use crate::search::query::SearchQuery;
/// Lowercased words, as the index keys of a clause look.
fn words(list: &[&str]) -> Vec<String> {
list.iter().map(|word| (*word).to_string()).collect()
}
/// A bare term carrying the spelling it was written with.
fn term(written: &str) -> Clause {
Clause::Term { term: written.to_lowercase(), raw: String::from(written) }
}
/// The clauses of the single alternative of a query that must parse.
fn one(query: &str) -> Vec<Clause> {
let parsed = SearchQuery::parse(query);
assert!(parsed.is_ok(), "{query:?} must parse");
parsed.unwrap_or_default().alternatives().first().cloned().unwrap_or_default()
}
#[test]
fn a_term_keeps_its_spelling_for_the_case_switch() {
let clauses = one("Ship \"Big Rock\"");
assert_eq!(clauses[0], term("Ship"));
assert_eq!(clauses[0].spelled(), Some(words(&["Ship"]).as_slice()));
assert_eq!(
clauses[1],
Clause::Phrase { terms: words(&["big", "rock"]), raw: words(&["Big", "Rock"]) },
"the index keys are lowercase, the spellings are not"
);
assert_eq!(clauses[1].spelled(), Some(words(&["Big", "Rock"]).as_slice()));
}
#[test]
fn the_case_switch_comes_from_the_options_or_the_query() {
assert!(SearchQuery::parse_with("ship", &SearchOptions::matching_case()).is_ok_and(|q| q.case_sensitive()));
assert!(SearchQuery::parse_with("match-case: ship", &SearchOptions::default()).is_ok_and(|q| q.case_sensitive()));
assert!(SearchQuery::parse_with("match-case: ship ignore-case:", &SearchOptions::default()).is_ok_and(|q| !q.case_sensitive()), "the last flag wins");
assert!(SearchQuery::parse_with("ignore-case: ship", &SearchOptions::matching_case()).is_ok_and(|q| !q.case_sensitive()));
assert!(SearchQuery::parse_with("MATCH-CASE: ship", &SearchOptions::default()).is_ok_and(|q| q.case_sensitive()));
assert!(!SearchQuery::parse("match-case").is_ok_and(|q| q.case_sensitive()), "without a colon it is a plain term");
assert!(SearchQuery::parse("match-case:").is_err(), "a lone flag searches for nothing");
assert!(!SearchQuery::parse("a").unwrap_or_default().case_sensitive(), "the default is case-insensitive");
}
#[test]
fn the_structural_operators_parse_their_groups() {
let clauses = one("section:(Alpha Beta) block:(gamma) task:milk task-todo: task-done:house");
assert_eq!(clauses[0], Clause::Section { terms: words(&["alpha", "beta"]) });
assert_eq!(clauses[1], Clause::Block { terms: words(&["gamma"]) });
assert_eq!(clauses[2], Clause::Task { terms: words(&["milk"]), state: TaskState::Any });
assert_eq!(clauses[3], Clause::Task { terms: Vec::new(), state: TaskState::Todo });
assert_eq!(clauses[4], Clause::Task { terms: words(&["house"]), state: TaskState::Done });
assert!(clauses[0].is_text_pass(false), "a section is decided on the note text");
assert!(!term("a").is_text_pass(false), "a plain term is decided by the index alone");
assert!(term("a").is_text_pass(true), "unless it must match its spelling");
assert_eq!(one("task:()"), vec![Clause::Task { terms: Vec::new(), state: TaskState::Any }], "a bare task filter is valid");
}
#[test]
fn an_unknown_operator_prefix_stays_a_plain_term() {
assert_eq!(
one("todo:milk author:me"),
vec![
Clause::Phrase { terms: words(&["todo", "milk"]), raw: words(&["todo", "milk"]) },
Clause::Phrase { terms: words(&["author", "me"]), raw: words(&["author", "me"]) }
],
"the colon is punctuation, both stay terms"
);
let query = SearchQuery::parse("todo:milk");
assert!(query.is_ok_and(|q| !q.needs_text(q.alternatives().first().map_or(&[][..], Vec::as_slice))), "no text pass needed");
}
#[test]
fn operators_carry_their_argument_shape() {
let clauses = one("term tag:#Work/Sub file:Note path:Deep/Nested/");
assert_eq!(clauses[1], Clause::Tag { tag: String::from("work/sub") });
assert_eq!(clauses[2], Clause::File { name: String::from("note") });
assert_eq!(clauses[3], Clause::Path { folder: String::from("deep/nested") });
assert_eq!(one("line:(Alpha Beta)")[0], Clause::Line { terms: words(&["alpha", "beta"]) });
assert_eq!(one("line:Alpha")[0], Clause::Line { terms: words(&["alpha"]) }, "the group is optional");
}
}
crates/index/src/search/rank.rs
+66 -20
View File
@@ -11,6 +11,7 @@ use std::ops::Range;
use crate::search::doc::{self, Doc};
use crate::search::query::Clause;
use crate::search::tokenize;
use crate::search::verify;
/// BM25 k1.
pub(crate) const BM25_K1: f32 = 1.2;
@@ -46,16 +47,20 @@ pub struct Snippet {
pub ranges: Vec<Range<usize>>,
}
/// Whether a note satisfies one clause.
#[must_use]
pub(crate) fn matches(doc: &Doc, clause: &Clause) -> bool {
/// Whether a note satisfies one clause. The structural operators
/// (`section:`, `block:`, `task:`) and the case-sensitive terms and
/// phrases are verified on the note text in [`verify`].
pub(crate) fn matches(doc: &Doc, clause: &Clause, case_sensitive: bool) -> bool {
match clause {
Clause::Term { term } => doc.terms.contains_key(term),
Clause::Phrase { terms } => phrase_matches(doc, terms),
Clause::Term { term, raw } => doc.terms.contains_key(term) && (!case_sensitive || verify::cased_term(doc, raw)),
Clause::Phrase { terms, raw } => phrase_matches(doc, terms) && (!case_sensitive || verify::cased_phrase(doc, raw)),
Clause::Path { folder } => path_matches(&doc.path, folder),
Clause::File { name } => doc.title.to_lowercase().contains(name),
Clause::Tag { tag } => doc.tags.iter().any(|t| tag_matches(t, tag)),
Clause::Line { terms } => shared_line(doc, terms),
Clause::Section { terms } => verify::section(doc, terms),
Clause::Block { terms } => verify::block(doc, terms),
Clause::Task { terms, state } => verify::task(doc, terms, *state),
}
}
@@ -172,30 +177,71 @@ fn one_snippet(note: &Doc, line: u32, wanted: &HashSet<&str>) -> Option<Snippet>
mod tests {
use super::*;
use crate::search::doc::build;
use crate::search::query::TaskState;
fn terms(list: &[&str]) -> Vec<String> {
list.iter().map(|term| (*term).to_string()).collect()
}
#[test]
fn path_file_tag_clauses_match_notes() {
let doc = build("work/A/B.md", "body #team/ops\n");
assert!(matches(&doc, &Clause::Path { folder: String::from("work") }));
assert!(matches(&doc, &Clause::Path { folder: String::from("work/a") }));
assert!(!matches(&doc, &Clause::Path { folder: String::from("other") }));
assert!(matches(&doc, &Clause::File { name: String::from("b") }));
assert!(matches(&doc, &Clause::Tag { tag: String::from("team") }));
assert!(matches(&doc, &Clause::Tag { tag: String::from("team/ops") }));
assert!(!matches(&doc, &Clause::Tag { tag: String::from("ops") }));
assert!(!matches(&doc, &Clause::Tag { tag: String::from("team/other") }));
assert!(matches(&doc, &Clause::Path { folder: String::from("work") }, false));
assert!(matches(&doc, &Clause::Path { folder: String::from("work/a") }, false));
assert!(!matches(&doc, &Clause::Path { folder: String::from("other") }, false));
assert!(matches(&doc, &Clause::File { name: String::from("b") }, false));
assert!(matches(&doc, &Clause::Tag { tag: String::from("team") }, false));
assert!(matches(&doc, &Clause::Tag { tag: String::from("team/ops") }, false));
assert!(!matches(&doc, &Clause::Tag { tag: String::from("ops") }, false));
assert!(!matches(&doc, &Clause::Tag { tag: String::from("team/other") }, false));
}
#[test]
fn phrase_and_line_clauses_need_the_right_layout() {
let doc = build("n.md", "alpha beta gamma\nalpha alone\ngamma\n");
assert!(matches(&doc, &Clause::Phrase { terms: vec![String::from("alpha"), String::from("beta")] }));
assert!(!matches(&doc, &Clause::Phrase { terms: vec![String::from("beta"), String::from("alpha")] }));
assert!(!matches(&doc, &Clause::Phrase { terms: vec![String::from("beta"), String::from("alone")] }), "different lines");
assert!(matches(&doc, &Clause::Line { terms: vec![String::from("alpha")] }));
assert!(matches(&doc, &Clause::Line { terms: vec![String::from("alpha"), String::from("beta")] }));
assert!(matches(&doc, &Clause::Line { terms: vec![String::from("alpha"), String::from("gamma")] }), "line 1 carries both");
assert!(!matches(&doc, &Clause::Line { terms: vec![String::from("alone"), String::from("gamma")] }));
assert!(matches(&doc, &Clause::Phrase { terms: terms(&["alpha", "beta"]), raw: terms(&["Alpha", "Beta"]) }, false));
assert!(!matches(&doc, &Clause::Phrase { terms: terms(&["beta", "alpha"]), raw: terms(&["beta", "alpha"]) }, false));
assert!(!matches(&doc, &Clause::Phrase { terms: terms(&["beta", "alone"]), raw: terms(&["beta", "alone"]) }, false), "different lines");
assert!(matches(&doc, &Clause::Line { terms: terms(&["alpha"]) }, false));
assert!(matches(&doc, &Clause::Line { terms: terms(&["alpha", "beta"]) }, false));
assert!(matches(&doc, &Clause::Line { terms: terms(&["alpha", "gamma"]) }, false), "line 1 carries both");
assert!(!matches(&doc, &Clause::Line { terms: terms(&["alone", "gamma"]) }, false));
}
#[test]
fn case_sensitive_terms_and_phrases_check_the_spelling() {
let doc = build("Ship.md", "Ship the Ship\n");
let term = Clause::Term { term: String::from("ship"), raw: String::from("Ship") };
assert!(matches(&doc, &term, false), "case-insensitive, the index key decides");
assert!(matches(&doc, &term, true));
let lower = Clause::Term { term: String::from("ship"), raw: String::from("ship") };
assert!(matches(&doc, &lower, false));
assert!(!matches(&doc, &lower, true), "`ship` does not match `Ship` under match-case");
let phrase = Clause::Phrase { terms: terms(&["ship", "the"]), raw: terms(&["Ship", "the"]) };
assert!(matches(&doc, &phrase, true));
let wrong = Clause::Phrase { terms: terms(&["ship", "the"]), raw: terms(&["ship", "The"]) };
assert!(!matches(&doc, &wrong, true));
}
#[test]
fn structural_clauses_are_verified_on_the_note_text() {
let doc = build("n.md", "# Alpha\nalpha text\n\n# Beta\n- [ ] beta task\n");
assert!(matches(&doc, &Clause::Section { terms: terms(&["alpha"]) }, false));
assert!(!matches(&doc, &Clause::Section { terms: terms(&["alpha", "beta"]) }, false), "two sections");
assert!(matches(&doc, &Clause::Block { terms: terms(&["alpha"]) }, false));
assert!(!matches(&doc, &Clause::Block { terms: terms(&["alpha", "beta"]) }, false));
assert!(matches(&doc, &Clause::Task { terms: terms(&["beta"]), state: TaskState::Todo }, false));
assert!(!matches(&doc, &Clause::Task { terms: terms(&["beta"]), state: TaskState::Done }, false));
}
#[test]
fn task_states_are_accepted_by_their_operator() {
assert!(TaskState::Any.accepts(TaskState::Todo));
assert!(TaskState::Any.accepts(TaskState::Done));
assert!(TaskState::Todo.accepts(TaskState::Todo));
assert!(!TaskState::Todo.accepts(TaskState::Done));
assert!(!TaskState::Done.accepts(TaskState::Todo));
assert!(TaskState::Done.accepts(TaskState::Done));
}
#[test]
crates/index/src/search/verify.rs
+356
View File
@@ -0,0 +1,356 @@
//! The verification pass: the structural search operators (`section:`,
//! `block:`, `task:`) and the case-sensitive terms and phrases are decided
//! on the text of a candidate note, never on the whole index.
//!
//! The inverted index stays as it is: it narrows the candidates, this pass
//! confirms them. A note is capped at [`doc::MAX_NOTE_BYTES`] when it is
//! indexed, so one pass over a note is bounded by construction, and every
//! pass is linear in the note: no term is ever searched for twice.
//!
//! Lines are read with [`str::trim`], so a `\r` of a CRLF note is no
//! marker, and the `#` and `- [ ]` of a fenced code block are text, not
//! structure.
use std::collections::HashSet;
use crate::search::doc::Doc;
use crate::search::query::TaskState;
use crate::search::tokenize;
/// Deepest heading level (`######`).
const MAX_HEADING_LEVEL: usize = 6;
/// The terms of one clause and the ones already found in the unit being
/// read: at most the query cap of them, whatever the note holds.
struct Wanted<'a> {
/// The lowercased terms the clause looks for.
terms: Vec<&'a str>,
/// The subset found in the current unit.
found: HashSet<&'a str>,
}
impl<'a> Wanted<'a> {
/// Watch for `terms`, which may be empty (then every unit matches).
fn new(terms: &'a [String]) -> Self {
Self { terms: terms.iter().map(String::as_str).collect(), found: HashSet::new() }
}
/// Whether every term has been found in the current unit.
fn complete(&self) -> bool {
self.found.len() == self.terms.len()
}
/// Forget the unit, keeping the terms.
fn reset(&mut self) {
self.found.clear();
}
/// Add the terms `text` carries.
fn scan(&mut self, text: &str) {
if self.complete() {
return;
}
for token in tokenize::tokens(text) {
if let Some(term) = self.terms.iter().find(|wanted| **wanted == token.text) {
self.found.insert(term);
}
if self.complete() {
break;
}
}
}
}
/// Whether every term of `terms` is inside one heading section: a heading
/// and the lines up to the next heading of the same or a shallower level,
/// so a nested `##` stays inside its `#`. The text before the first
/// heading is one preamble section.
#[must_use]
pub(crate) fn section(doc: &Doc, terms: &[String]) -> bool {
if terms.is_empty() {
return false;
}
let mut wanted = Wanted::new(terms);
let mut level = usize::MAX;
let mut fence = None;
for line in &doc.lines {
let text = line.trim();
let (open, fenced) = fence_step(text, fence);
fence = open;
if !fenced && let Some((heading, new_level)) = heading(text) {
if wanted.complete() {
return true;
}
if new_level <= level {
wanted.reset();
}
level = new_level;
wanted.scan(heading);
continue;
}
// A fenced line is text of the section it sits in, never structure.
wanted.scan(text);
}
wanted.complete()
}
/// Whether every term of `terms` is inside one block: one paragraph, list
/// item or table row, blocks being separated by blank lines. A fenced code
/// block is never split by its blank lines.
#[must_use]
pub(crate) fn block(doc: &Doc, terms: &[String]) -> bool {
if terms.is_empty() {
return false;
}
let mut wanted = Wanted::new(terms);
let mut fence = None;
for line in &doc.lines {
let text = line.trim();
let (open, fenced) = fence_step(text, fence);
fence = open;
if fenced {
wanted.scan(text);
continue;
}
if text.is_empty() {
if wanted.complete() {
return true;
}
wanted.reset();
continue;
}
wanted.scan(text);
}
wanted.complete()
}
/// Whether the note has a task line of `state` carrying every term; with
/// no term (`task-todo:`) any such line matches.
#[must_use]
pub(crate) fn task(doc: &Doc, terms: &[String], state: TaskState) -> bool {
let mut wanted = Wanted::new(terms);
let mut fence = None;
for line in &doc.lines {
let text = line.trim();
let (open, fenced) = fence_step(text, fence);
fence = open;
if fenced {
continue;
}
let Some((line_state, content)) = task_line(text) else { continue };
if !state.accepts(line_state) {
continue;
}
wanted.reset();
wanted.scan(content);
if wanted.complete() {
return true;
}
}
false
}
/// Whether `spelled` occurs as a token written exactly that way, the note
/// title included.
#[must_use]
pub(crate) fn cased_term(doc: &Doc, spelled: &str) -> bool {
spellings(&doc.title).any(|word| word == spelled) || doc.lines.iter().any(|line| spellings(line).any(|word| word == spelled))
}
/// Whether `spelled` occurs as consecutive tokens of one line (or of the
/// title), written exactly that way.
#[must_use]
pub(crate) fn cased_phrase(doc: &Doc, spelled: &[String]) -> bool {
if spelled.is_empty() {
return false;
}
doc.lines.iter().any(|line| cased_phrase_in(line, spelled)) || cased_phrase_in(&doc.title, spelled)
}
/// The tokens of `text` as they are written in it: the tokenizer gives the
/// ranges, the source text the spelling.
fn spellings(text: &str) -> impl Iterator<Item = &str> {
tokenize::tokens(text).into_iter().filter_map(move |token| text.get(token.range))
}
/// Whether `spelled` is a run of consecutive tokens of `text`.
fn cased_phrase_in(text: &str, spelled: &[String]) -> bool {
let words: Vec<&str> = spellings(text).collect();
words.windows(spelled.len()).any(|window| window.iter().zip(spelled).all(|(word, want)| *word == want.as_str()))
}
/// The fence state after one line and whether the line is inside a fence,
/// the opening and closing lines included.
fn fence_step(text: &str, open: Option<char>) -> (Option<char>, bool) {
let marker = fence_marker(text);
let next = match (open, marker) {
(Some(fence), Some(marker)) if fence == marker => None,
(None, Some(marker)) => Some(marker),
(fence, _) => fence,
};
(next, marker.is_some() || next.is_some())
}
/// The fence a line opens or closes, or `None` for ordinary text.
fn fence_marker(text: &str) -> Option<char> {
if text.starts_with("```") {
Some('`')
} else if text.starts_with("~~~") {
Some('~')
} else {
None
}
}
/// The heading text and level of `line`, or `None` when it is not one: a
/// `#tag` at the start of a line is a tag, not a heading.
fn heading(line: &str) -> Option<(&str, usize)> {
let hashes = line.chars().take_while(|c| *c == '#').count();
if hashes == 0 || hashes > MAX_HEADING_LEVEL {
return None;
}
let rest = line.get(hashes..)?;
if !(rest.is_empty() || rest.starts_with(' ')) {
return None;
}
Some((rest.trim().trim_end_matches('#').trim(), hashes))
}
/// The checkbox state and the text of a task line, or `None` when the line
/// is not one. Blockquote markers, list bullets and ordered markers are
/// stripped first, so `> - [ ] nested` is a task too.
fn task_line(line: &str) -> Option<(TaskState, &str)> {
let rest = strip_quote(line);
let rest = strip_bullet(rest)?;
if rest.as_bytes().first() != Some(&b'[') {
return None;
}
let state = match rest.as_bytes().get(1) {
Some(b' ') => TaskState::Todo,
Some(b'x' | b'X') => TaskState::Done,
_ => return None,
};
if rest.as_bytes().get(2) != Some(&b']') {
return None;
}
let content = rest.get(3..)?;
if !(content.is_empty() || content.starts_with([' ', '\t'])) {
return None;
}
Some((state, content.trim()))
}
/// Strip the `>` markers of a blockquote, with the spaces after them.
fn strip_quote(line: &str) -> &str {
let mut rest = line.trim_start();
while let Some(after) = rest.strip_prefix('>') {
rest = after.trim_start();
}
rest
}
/// Strip one list bullet or ordered marker and the space after it.
fn strip_bullet(line: &str) -> Option<&str> {
for marker in ['-', '*', '+'] {
if let Some(rest) = line.strip_prefix(marker) {
return rest.strip_prefix(char::is_whitespace).map(str::trim_start);
}
}
let digits = line.chars().take_while(char::is_ascii_digit).count();
if digits > 0 {
let after = line.get(digits..)?.strip_prefix(['.', ')'])?;
return after.strip_prefix(char::is_whitespace).map(str::trim_start);
}
None
}
#[cfg(test)]
mod tests {
use super::*;
use crate::search::doc::build;
fn terms(list: &[&str]) -> Vec<String> {
list.iter().map(|t| (*t).to_string()).collect()
}
#[test]
fn sections_stop_at_the_next_heading_of_the_same_level() {
let doc = build("n.md", "# Alpha\nalpha one\n## Beta\nbeta with gamma\n# Delta\ndelta only\n");
assert!(section(&doc, &terms(&["alpha", "gamma"])), "a nested ## stays inside its #");
assert!(!section(&doc, &terms(&["alpha", "delta"])), "# Delta opens a new section");
assert!(section(&doc, &terms(&["delta"])));
assert!(!section(&doc, &terms(&["alpha", "epsilon"])));
assert!(!section(&doc, &terms(&[])), "no term, no section match");
}
#[test]
fn a_heading_inside_a_fence_is_text() {
let doc = build("n.md", "```md\n# Fake\nalpha gamma\n```\n# Real\nalpha only\n");
assert!(section(&doc, &terms(&["fake", "alpha"])), "the fenced text belongs to the section it sits in");
assert!(!section(&doc, &terms(&["fake", "gamma", "real"])), "# Real opens a new section");
assert!(!section(&doc, &terms(&["gamma", "alpha only"])));
}
#[test]
fn blocks_are_split_on_blank_lines_only() {
let doc = build("n.md", "alpha in one block\nbeta in another\n\nalpha beta together\n");
assert!(block(&doc, &terms(&["alpha", "beta"])), "the third line is its own block");
assert!(!block(&doc, &terms(&["alpha", "gamma"])));
let fenced = build("n.md", "```\nalpha\n\nbeta\n```\n");
assert!(block(&fenced, &terms(&["alpha", "beta"])), "a fence is not split by its blank line");
}
#[test]
fn crlf_lines_are_trimmed_the_same_way() {
let doc = build("n.md", "# Alpha\r\nalpha one\r\n\r\nbeta one\r\n");
assert!(section(&doc, &terms(&["alpha"])), "the `\\r` is no marker");
assert!(section(&doc, &terms(&["alpha", "beta"])), "a section runs to the next heading, not to the next blank line");
assert!(!block(&doc, &terms(&["alpha", "beta"])), "but a block stops at the blank line");
}
#[test]
fn tasks_are_found_in_quotes_and_nested_lists() {
let doc = build("n.md", "- [ ] alpha todo\n- [x] beta done\n> - [ ] gamma quoted\n - [ ] nested alpha\n1. [x] ordered delta\n* [ ]asterisk\n");
assert!(task(&doc, &terms(&["alpha"]), TaskState::Any));
assert!(task(&doc, &terms(&["alpha"]), TaskState::Todo));
assert!(!task(&doc, &terms(&["alpha"]), TaskState::Done));
assert!(task(&doc, &terms(&["beta"]), TaskState::Done));
assert!(task(&doc, &terms(&["gamma"]), TaskState::Todo), "inside a blockquote");
assert!(task(&doc, &terms(&["nested"]), TaskState::Todo), "in a nested list");
assert!(task(&doc, &terms(&["ordered"]), TaskState::Done));
assert!(!task(&doc, &terms(&["asterisk"]), TaskState::Any), "a checkbox needs a space");
assert!(task(&doc, &terms(&[]), TaskState::Todo), "an empty term list accepts any line of the state");
assert!(task(&doc, &terms(&[]), TaskState::Done));
}
#[test]
fn a_task_inside_a_fence_is_text() {
let doc = build("n.md", "```\n- [ ] fake\n```\n- [ ] real\n");
assert!(!task(&doc, &terms(&["fake"]), TaskState::Any));
assert!(task(&doc, &terms(&["real"]), TaskState::Todo));
}
#[test]
fn case_sensitive_terms_and_phrases_need_the_spelling() {
let doc = build("Ship.md", "# Ship Plan\nplan the Ships #Tag\n");
assert!(cased_term(&doc, "Ship"), "the title and the heading");
assert!(cased_term(&doc, "Ships"), "as it is written on its line");
assert!(cased_term(&doc, "Tag"), "a tag is written on its line");
assert!(!cased_term(&doc, "ship"), "the note never spells it that way");
assert!(!cased_term(&doc, "ship plan"));
assert!(cased_phrase(&doc, &terms(&["Ship", "Plan"])));
assert!(cased_phrase(&doc, &terms(&["plan", "the"])));
assert!(!cased_phrase(&doc, &terms(&["Plan", "Ship"])), "in order");
assert!(!cased_phrase(&doc, &terms(&["Ship", "plan"])), "with the spelling it is not written with");
assert!(!cased_phrase(&doc, &[]), "an empty phrase never matches");
}
#[test]
fn a_hash_starting_a_line_is_a_tag_not_a_heading() {
assert_eq!(heading("# Real heading"), Some(("Real heading", 1)));
assert_eq!(heading("## Closed ##"), Some(("Closed", 2)));
assert_eq!(heading("#tag"), None);
assert_eq!(heading("####### too deep"), None);
assert_eq!(heading("plain"), None);
}
}
crates/index/tests/operators.rs
+154
View File
@@ -0,0 +1,154 @@
//! The search operators end to end over the public API: `section:`,
//! `block:`, `task:`, `task-todo:`, `task-done:` and the case switch, alone
//! and composed with `path:`, `file:`, `tag:`, `line:`, `-term` and `OR`.
//!
//! Every row is `(query, case_sensitive, notes it must hit)` over one small
//! vault, so a regression names the row that broke.
use notescraft_index::search::{MAX_CANDIDATES, SearchIndex, SearchOptions, SearchQuery};
/// One note per shape: sections, blocks, tasks, fences, CRLF and unicode.
fn vault() -> SearchIndex {
let mut notes = SearchIndex::new();
notes.update("n/sections.md", "# Alpha\n\nalpha text here\n\n## Beta\n\nbeta text with gamma\n\n# Delta\n\ndelta text\n").unwrap();
notes.update("n/blocks.md", "alpha in one block\n\nbeta in another\n\nalpha beta together in a block\n").unwrap();
notes.update("n/tasks.md", "- [ ] alpha todo #work\n\n- [x] beta done\n> - [ ] gamma quoted\n - [ ] nested alpha\n\n1. [x] ordered delta\n").unwrap();
notes.update("n/fenced.md", "```md\n# Fake heading\n\n- [ ] fake task\n```\n\n# Real heading\n\n- [x] real task\n").unwrap();
notes.update("play/n/other.md", "alpha beta\n").unwrap();
notes.update("u/unicode.md", "# 見出し\n\n漢字 ひらがな #日本語\n\n- [ ] 漢字 task\n").unwrap();
// A CRLF note: the `\r` is no marker for headings, blocks or tasks.
notes.update("work/n/crlf.md", "# Alpha\r\nalpha one\r\n\r\nbeta one\r\n\r\n- [ ] alpha task\r\n").unwrap();
notes
}
fn search(query: &str, sensitive: bool) -> Vec<String> {
let options = if sensitive { SearchOptions::matching_case() } else { SearchOptions::new() };
let parsed = SearchQuery::parse_with(query, &options).unwrap_or_else(|error| panic!("{query:?} must parse: {error}"));
let mut paths: Vec<String> = vault().search(&parsed, MAX_CANDIDATES).into_iter().map(|hit| hit.path).collect();
paths.sort();
paths
}
/// `(query, case_sensitive, notes it must hit)`.
const ROWS: &[(&str, bool, &[&str])] = &[
// section:
("section:(alpha text)", false, &["n/sections.md"]),
("section:(alpha gamma)", false, &["n/sections.md", "n/tasks.md"]),
("section:(alpha delta)", false, &["n/tasks.md"]),
("section:(gamma delta)", false, &["n/tasks.md"]),
("section:(delta text)", false, &["n/sections.md"]),
("section:(alpha one)", false, &["n/blocks.md", "work/n/crlf.md"]),
("section:(alpha beta)", false, &["n/blocks.md", "n/sections.md", "n/tasks.md", "play/n/other.md", "work/n/crlf.md"]),
("section:alpha", false, &["n/blocks.md", "n/sections.md", "n/tasks.md", "play/n/other.md", "work/n/crlf.md"]),
("section:nosuch", false, &[]),
("section:(見出し 漢字)", false, &["u/unicode.md"]),
("section:(漢字 ひらがな)", false, &["u/unicode.md"]),
("section:(fake heading)", false, &["n/fenced.md"]),
("section:(fake real)", false, &[]),
("section:(real task)", false, &["n/fenced.md"]),
("section:(todo nested)", false, &["n/tasks.md"]),
("section:alpha path:n/sections.md", false, &["n/sections.md"]),
("section:alpha -path:n/", false, &["play/n/other.md", "work/n/crlf.md"]),
("section:alpha tag:#work", false, &["n/tasks.md"]),
("section:alpha -alpha", false, &[]),
("section:alpha file:Other", false, &["play/n/other.md"]),
("section:(alpha gamma) OR section:(real task)", false, &["n/fenced.md", "n/sections.md", "n/tasks.md"]),
("(section:alpha OR section:fake)", false, &["n/blocks.md", "n/fenced.md", "n/sections.md", "n/tasks.md", "play/n/other.md", "work/n/crlf.md"]),
("section:alpha line:(nested alpha)", false, &["n/tasks.md"]),
// block:
("block:(alpha beta)", false, &["n/blocks.md", "n/tasks.md", "play/n/other.md"]),
("block:(alpha in one block)", false, &["n/blocks.md"]),
("block:together", false, &["n/blocks.md"]),
("block:(together in)", false, &["n/blocks.md"]),
("block:(alpha one)", false, &["n/blocks.md", "work/n/crlf.md"]),
("block:(alpha one beta)", false, &[]),
("block:fake", false, &["n/fenced.md"]),
("block:real", false, &["n/fenced.md"]),
("block:(fake real)", false, &[]),
("block:fake real", false, &["n/fenced.md"]),
("block:(fake task)", false, &["n/fenced.md"]),
("block:漢字", false, &["u/unicode.md"]),
("block:alpha path:work/", false, &["work/n/crlf.md"]),
("block:alpha -block", false, &["n/sections.md", "n/tasks.md", "play/n/other.md", "work/n/crlf.md"]),
("block:alpha -one", false, &["n/sections.md", "n/tasks.md", "play/n/other.md"]),
("block:(alpha beta) tag:#work", false, &["n/tasks.md"]),
("block:(delta text)", false, &["n/sections.md"]),
("block:gamma", false, &["n/sections.md", "n/tasks.md"]),
("block:(quoted nested)", false, &["n/tasks.md"]),
("block:(beta done quoted)", false, &["n/tasks.md"]),
// task:
("task:alpha", false, &["n/tasks.md", "work/n/crlf.md"]),
("task:(alpha nested)", false, &["n/tasks.md"]),
("task:(alpha beta)", false, &[]),
("task:gamma", false, &["n/tasks.md"]),
("task:quoted", false, &["n/tasks.md"]),
("task:ordered", false, &["n/tasks.md"]),
("task:fake", false, &[]),
("task:real", false, &["n/fenced.md"]),
("task:漢字", false, &["u/unicode.md"]),
("task:todo", false, &["n/tasks.md"]),
("task:alpha tag:#work", false, &["n/tasks.md"]),
("task:alpha -path:play/", false, &["n/tasks.md", "work/n/crlf.md"]),
("task:alpha path:work/", false, &["work/n/crlf.md"]),
("task:alpha file:Tasks", false, &["n/tasks.md"]),
("task:alpha OR task:gamma", false, &["n/tasks.md", "work/n/crlf.md"]),
("task:alpha -task-done:", false, &["work/n/crlf.md"]),
("task-todo:", false, &["n/tasks.md", "u/unicode.md", "work/n/crlf.md"]),
("task-done:", false, &["n/fenced.md", "n/tasks.md"]),
("task-todo: -path:work/", false, &["n/tasks.md", "u/unicode.md"]),
("task-done:delta", false, &["n/tasks.md"]),
("task-todo:delta", false, &[]),
("task-done:(ordered delta)", false, &["n/tasks.md"]),
// The structural operators stay case-insensitive, the terms do not.
("match-case: task-done: real", false, &["n/fenced.md"]),
("match-case: task-done: REAL", false, &[]),
// match-case: and ignore-case:
("match-case: alpha", false, &["n/blocks.md", "n/sections.md", "n/tasks.md", "play/n/other.md", "work/n/crlf.md"]),
("match-case: Alpha", false, &["n/sections.md", "work/n/crlf.md"]),
("match-case: ALPHA", false, &[]),
("ignore-case: ALPHA", false, &["n/blocks.md", "n/sections.md", "n/tasks.md", "play/n/other.md", "work/n/crlf.md"]),
("match-case: ALPHA ignore-case:", false, &["n/blocks.md", "n/sections.md", "n/tasks.md", "play/n/other.md", "work/n/crlf.md"]),
("ignore-case: ALPHA match-case:", false, &[]),
("match-case: \"alpha text\"", false, &["n/sections.md"]),
("match-case: \"Alpha text\"", false, &[]),
("match-case: \"Alpha text\" OR ignore-case: \"alpha text\"", false, &["n/sections.md"]),
("match-case: alpha -path:n/", false, &["play/n/other.md", "work/n/crlf.md"]),
("match-case: alpha path:n/sections.md", false, &["n/sections.md"]),
("match-case: alpha tag:#work", false, &["n/tasks.md"]),
("match-case: alpha -ALPHA", false, &["n/blocks.md", "n/sections.md", "n/tasks.md", "play/n/other.md", "work/n/crlf.md"]), // the negation is cased too: `-ALPHA` subtracts nothing
("match-case: crlf", false, &["work/n/crlf.md"]),
("match-case: Tasks", false, &[]),
("match-case: 漢字", false, &["u/unicode.md"]),
("match-case: 漢字の", false, &[]),
("match-case: section:(ALPHA)", false, &["n/blocks.md", "n/sections.md", "n/tasks.md", "play/n/other.md", "work/n/crlf.md"]),
// The options parameter says the same as the in-query flag.
("alpha", true, &["n/blocks.md", "n/sections.md", "n/tasks.md", "play/n/other.md", "work/n/crlf.md"]),
("Alpha", true, &["n/sections.md", "work/n/crlf.md"]),
("ALPHA", true, &[]),
("alpha", false, &["n/blocks.md", "n/sections.md", "n/tasks.md", "play/n/other.md", "work/n/crlf.md"]),
("\"alpha text\"", true, &["n/sections.md"]),
("section:(ALPHA text)", true, &["n/sections.md"]),
];
#[test]
fn every_operator_row_behaves() {
assert!(ROWS.len() >= 50, "the table keeps {} rows", ROWS.len());
let mut broken: Vec<String> = Vec::new();
for (query, sensitive, want) in ROWS {
let mut expected: Vec<String> = want.iter().map(|path| (*path).to_string()).collect();
expected.sort();
let got = search(query, *sensitive);
if got != expected {
broken.push(format!("{query:?} (case_sensitive={sensitive}) got {got:?}, want {expected:?}"));
}
}
assert_eq!(broken, Vec::<String>::new());
}
#[test]
fn the_case_switch_needs_a_term_to_search_for() {
for query in ["match-case:", "ignore-case:", "match-case: ignore-case:", "-match-case:"] {
assert!(SearchQuery::parse(query).is_err(), "{query:?} searches for nothing");
}
assert!(SearchQuery::parse_with("match-case:", &SearchOptions::matching_case()).is_err());
}
crates/index/tests/operators_hostile.rs
+113
View File
@@ -0,0 +1,113 @@
//! Hostile input for the new search operators: huge notes, huge operator
//! arguments, deeply nested groups, thousands of terms and a vault bigger
//! than the candidate cap. Nothing here may panic or take unbounded time.
use std::fmt::Write as _;
use notescraft_index::Error;
use notescraft_index::search::{MAX_CANDIDATES, SearchIndex, SearchOptions, SearchQuery};
/// Most terms a query may carry, mirroring the parser cap.
const TERM_CAP: usize = 32;
fn paths(notes: &SearchIndex, query: &str) -> Vec<String> {
let parsed = SearchQuery::parse(query);
assert!(parsed.is_ok(), "{query:?} must parse");
notes.search(&parsed.unwrap_or_default(), usize::MAX).into_iter().map(|hit| hit.path).collect()
}
/// `count` distinct space-separated terms.
fn terms(count: usize) -> String {
let mut out = String::new();
for n in 0..count {
let _ = write!(&mut out, "t{n} ");
}
out
}
#[test]
fn thousands_of_terms_are_refused_by_the_caps_not_by_a_crash() {
// Over the term cap first, then over the char cap.
let many = "t ".repeat(10_000);
assert!(matches!(SearchQuery::parse(many.trim()), Err(Error::QueryTooLong { max: 1000, .. })));
let over_cap = terms(TERM_CAP + 5);
assert!(matches!(SearchQuery::parse(over_cap.trim()), Err(Error::TooManyTerms { max: TERM_CAP, .. })));
let inside = terms(TERM_CAP);
assert!(SearchQuery::parse(inside.trim()).is_ok(), "exactly the cap passes");
// A group that carries the whole cap is fine, one more term is not.
assert!(SearchQuery::parse(&format!("section:({inside})")).is_ok());
assert!(SearchQuery::parse(&format!("section:({inside} one)")).is_err());
}
#[test]
fn deeply_nested_groups_and_huge_operators_are_errors() {
for depth in [9_usize, 64, 512] {
let query = format!("{}alpha{}", "(".repeat(depth), ")".repeat(depth));
assert!(SearchQuery::parse(&query).is_err(), "{depth} nested groups must not parse");
}
for query in [
format!("section:({})", "a ".repeat(2_000)),
format!("block:{}", "b".repeat(4_000)),
format!("task:{}", "c".repeat(4_000)),
format!("{}(", "(".repeat(1_000)),
"section:((((((((((".to_string(),
] {
assert!(SearchQuery::parse(&query).is_err(), "{query:?} must not parse");
}
// A huge operator with no token at all is refused, not silently empty.
assert!(SearchQuery::parse(&format!("section:{}", "#".repeat(2_000))).is_err());
assert!(SearchQuery::parse(&format!("task:{}", "漢".repeat(2_000))).is_err());
}
/// Bytes of note text the verification pass reads; a vault note is capped
/// at 2 MiB by the index, so the pass is bounded by construction.
const NOTE_BYTES: usize = 256 * 1024;
#[test]
fn a_note_of_a_quarter_megabyte_is_verified_once_per_clause() {
let mut body = String::with_capacity(NOTE_BYTES);
let mut section = 0_usize;
while body.len() < NOTE_BYTES {
section += 1;
let _ = write!(&mut body, "# Section {section}\n\n- [ ] task {section} filler\n\n```\n# not a heading\n- [ ] not a task\n```\n\n");
}
let mut notes = SearchIndex::new();
notes.update("huge.md", &body).unwrap();
assert_eq!(paths(&notes, &format!("section:(task {section})")), vec![String::from("huge.md")]);
assert_eq!(paths(&notes, "task-done:"), Vec::<String>::new(), "every task of the note is open");
assert_eq!(paths(&notes, "section:(not a task)"), vec![String::from("huge.md")], "the fenced text is section text");
assert_eq!(paths(&notes, "block:(not a task filler)"), Vec::<String>::new(), "the fence is not split");
assert_eq!(paths(&notes, "task:not"), Vec::<String>::new(), "a fenced checkbox is not a task");
assert_eq!(notes.len(), 1);
}
#[test]
fn a_vault_larger_than_the_candidate_cap_stays_bounded_and_stable() {
let mut notes = SearchIndex::new();
for i in 0..MAX_CANDIDATES + 10 {
notes.update(&format!("n/Note {i:05}.md"), "# Head\n\nhay filler\n").unwrap();
}
notes.update("z/last.md", "# Head\n\nneedle in a section\n").unwrap();
assert_eq!(notes.len(), MAX_CANDIDATES + 11);
// The structural operator needs the text pass, so its candidates are
// capped and the last note in path order is the one left out.
let capped = paths(&notes, "section:needle");
assert!(capped.len() <= MAX_CANDIDATES, "the cap holds: {}", capped.len());
assert_eq!(capped, paths(&notes, "section:needle"), "and the cut is stable");
// A plain term is decided by the index alone and stays exact.
assert_eq!(paths(&notes, "hay").len(), MAX_CANDIDATES + 10);
assert_eq!(paths(&notes, "head").len(), MAX_CANDIDATES + 11);
}
#[test]
fn the_case_switch_on_a_big_vault_is_exact() {
let mut notes = SearchIndex::new();
notes.update("a/Ship.md", "ship the Ship\n").unwrap();
notes.update("b/Other.md", "ship only lowercase\n").unwrap();
let insensitive = SearchQuery::parse_with("ship", &SearchOptions::matching_case()).unwrap_or_default();
assert_eq!(notes.search(&insensitive, 10).into_iter().map(|hit| hit.path).collect::<Vec<_>>(), vec![String::from("a/Ship.md"), String::from("b/Other.md")]);
let cased = SearchQuery::parse_with("Ship", &SearchOptions::matching_case()).unwrap_or_default();
assert_eq!(notes.search(&cased, 10).into_iter().map(|hit| hit.path).collect::<Vec<_>>(), vec![String::from("a/Ship.md")]);
}
crates/lint/src/lib.rs
+1 -1
View File
@@ -28,7 +28,7 @@ use std::collections::HashSet;
pub use error::{Error, Result};
pub use model::{Context, Edit, Fix, HeadingStyle, Issue, Range, RuleSet, Severity, Style};
pub use rules::{MAX_ISSUES, RuleInfo, rules};
pub use rules::{MAX_ISSUES, RuleInfo, rules, vocabulary};
pub use style::Vote;
/// Check `text` and return the issues sorted by position (then rule id).
crates/lint/src/model.rs
+3
View File
@@ -132,6 +132,9 @@ pub struct Context {
pub rules: RuleSet,
/// Dismissed [`Issue::key`]s.
pub dismissed: Vec<String>,
/// Lowercase words the `spelling` rule accepts; empty disables it.
#[serde(skip)]
pub known_words: std::sync::Arc<std::collections::HashSet<String>>,
}
pub(crate) fn hash_of(matched: &str, line: &str) -> u64 {
crates/lint/src/rules.rs
+5 -1
View File
@@ -4,6 +4,9 @@ mod blank;
mod frontmatter;
mod lines;
mod links;
mod spelling;
pub use spelling::vocabulary;
use serde::{Deserialize, Serialize};
@@ -35,7 +38,7 @@ pub const FM_DUPLICATE: &str = "frontmatter-duplicate-key";
pub const DUPLICATE_HEADING: &str = "duplicate-heading";
/// `unresolved-link` rule id.
pub const UNRESOLVED_LINK: &str = "unresolved-link";
/// `spelling` rule id (slot only, never reports).
/// `spelling` rule id (off by default).
pub const SPELLING: &str = "spelling";
/// Static description of a rule.
@@ -113,4 +116,5 @@ pub(crate) fn run(sink: &mut Sink<'_>) {
blank::run(sink);
frontmatter::run(sink);
links::run(sink);
spelling::run(sink);
}
crates/lint/src/rules/spelling.rs
+208
View File
@@ -0,0 +1,208 @@
//! `spelling`: words the vault itself does not know. There is no bundled dictionary: a word is known
//! when it is in [`crate::Context::known_words`] (the engine fills it with the words that occur in at
//! least two notes), and a suggestion is the closest known word. Off by default.
use crate::model::{Edit, Fix, Range};
use crate::rules::{SPELLING, Sink};
use crate::scan::{Kind, capped, code_spans, in_spans};
/// Shortest word that is checked, in characters.
const MIN_WORD: usize = 4;
/// Longest word that is checked, in characters.
const MAX_WORD: usize = 30;
/// Most suggestions kept per word.
const MAX_SUGGESTIONS: usize = 3;
/// Largest edit distance of a suggestion.
const MAX_DISTANCE: usize = 2;
/// Known words compared per misspelling.
const MAX_CANDIDATES: usize = 200_000;
/// Misspellings that get suggestions per check.
const MAX_SUGGESTED: usize = 40;
pub(super) fn run(s: &mut Sink<'_>) {
let ctx = s.ctx();
if !s.on(SPELLING) || ctx.known_words.is_empty() {
return;
}
let doc = s.doc;
let mut suggested = 0;
for (i, line) in doc.lines.iter().enumerate() {
if matches!(line.kind, Kind::Blank | Kind::Code | Kind::Frontmatter | Kind::FrontmatterFence | Kind::FenceOpen | Kind::FenceClose) {
continue;
}
let body = capped(doc.body(i));
let skip = skipped_spans(body);
for (at, word) in words(body) {
if in_spans(&skip, at) || !is_checked(word) || ctx.known_words.contains(&word.to_lowercase()) {
continue;
}
let range = Range { start: line.start + at, end: line.start + at + word.len() };
let near = if suggested < MAX_SUGGESTED { nearest(word, ctx.known_words.iter().map(String::as_str)) } else { Vec::new() };
suggested += usize::from(!near.is_empty());
let mut fixes = near.into_iter().map(|better| Fix {
label: format!("Change to {better}"),
edits: vec![Edit { range, text: matching_case(word, &better) }],
safe: false,
});
let first = fixes.next();
s.add(SPELLING, range, format!("\"{word}\" is not in this vault's vocabulary"), first, fixes.collect());
}
}
}
/// The lowercase words of `text` the spelling rule would check, from prose only (no code, links, URLs, tags).
#[must_use]
pub fn vocabulary(text: &str) -> std::collections::HashSet<String> {
let doc = crate::scan::Doc::new(text);
let mut out = std::collections::HashSet::new();
for (i, line) in doc.lines.iter().enumerate() {
if matches!(line.kind, Kind::Blank | Kind::Code | Kind::Frontmatter | Kind::FrontmatterFence | Kind::FenceOpen | Kind::FenceClose) {
continue;
}
let body = capped(doc.body(i));
let skip = skipped_spans(body);
out.extend(words(body).into_iter().filter(|(at, word)| !in_spans(&skip, *at) && word.chars().count() >= MIN_WORD).map(|(_, word)| word.to_lowercase()));
}
out
}
/// Byte spans of a line that hold no prose: inline code, `[[links]]`, URLs, tags and `[text](target)` targets.
fn skipped_spans(body: &str) -> Vec<(usize, usize)> {
let mut spans = code_spans(body);
for (open, close) in [("[[", "]]"), ("](", ")")] {
let mut from = 0;
while let Some(start) = body.get(from..).and_then(|rest| rest.find(open)).map(|p| p + from) {
let end = body.get(start..).and_then(|rest| rest.find(close)).map_or(body.len(), |p| start + p + close.len());
spans.push((start, end));
from = end.max(start + 1);
}
}
let mut at = 0;
for token in body.split_inclusive(char::is_whitespace) {
if token.contains("://") || token.starts_with("www.") || token.starts_with('#') || token.starts_with('@') {
spans.push((at, at + token.len()));
}
at += token.len();
}
spans
}
/// Maximal runs of letters, with an apostrophe or hyphen kept inside: `(byte offset, word)`.
fn words(body: &str) -> Vec<(usize, &str)> {
let mut out = Vec::new();
let mut start: Option<usize> = None;
let mut chars = body.char_indices().peekable();
while let Some((at, c)) = chars.next() {
let inside = c.is_alphabetic() || (start.is_some() && matches!(c, '\'' | '’' | '-') && chars.peek().is_some_and(|(_, next)| next.is_alphabetic()));
match (inside, start) {
(true, None) => start = Some(at),
(false, Some(from)) => {
out.extend(body.get(from..at).map(|word| (from, word)));
start = None;
}
_ => {}
}
}
if let Some(from) = start {
out.extend(body.get(from..).map(|word| (from, word)));
}
out
}
/// Whether `word` is a candidate: long enough, not an acronym, not a compound with a hyphen or apostrophe.
fn is_checked(word: &str) -> bool {
let length = word.chars().count();
(MIN_WORD..=MAX_WORD).contains(&length) && word.chars().all(char::is_alphabetic) && word.chars().skip(1).any(char::is_lowercase)
}
/// The known words closest to `word`, nearest first.
fn nearest<'a>(word: &str, known: impl Iterator<Item = &'a str>) -> Vec<String> {
let want: Vec<char> = word.to_lowercase().chars().collect();
let mut found: Vec<(usize, &str)> = Vec::new();
for candidate in known.take(MAX_CANDIDATES) {
let have: Vec<char> = candidate.chars().collect();
if have.len().abs_diff(want.len()) > MAX_DISTANCE {
continue;
}
let distance = edit_distance(&want, &have, MAX_DISTANCE);
if distance <= MAX_DISTANCE {
found.push((distance, candidate));
}
}
found.sort_unstable();
found.into_iter().take(MAX_SUGGESTIONS).map(|(_, word)| word.to_owned()).collect()
}
/// Levenshtein distance, giving up (returning `limit + 1`) once every cell passes `limit`.
fn edit_distance(a: &[char], b: &[char], limit: usize) -> usize {
let mut row: Vec<usize> = (0..=b.len()).collect();
for (i, ca) in a.iter().enumerate() {
let mut diag = row.first().copied().unwrap_or(0);
if let Some(first) = row.first_mut() {
*first = i + 1;
}
let mut best = row.first().copied().unwrap_or(0);
for (j, cb) in b.iter().enumerate() {
let up = row.get(j + 1).copied().unwrap_or(0);
let left = row.get(j).copied().unwrap_or(0);
let cell = (diag + usize::from(ca != cb)).min(up + 1).min(left + 1);
if let Some(slot) = row.get_mut(j + 1) {
*slot = cell;
}
diag = up;
best = best.min(cell);
}
if best > limit {
return limit + 1;
}
}
row.last().copied().unwrap_or(limit + 1)
}
/// `better` written with the capital of `word`.
fn matching_case(word: &str, better: &str) -> String {
let mut chars = better.chars();
match (word.chars().next().is_some_and(char::is_uppercase), chars.next()) {
(true, Some(first)) => first.to_uppercase().chain(chars).collect(),
_ => better.to_owned(),
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use crate::{Context, check};
fn context(words: &[&str]) -> Context {
let mut ctx = Context { known_words: Arc::new(words.iter().map(|w| (*w).to_owned()).collect()), ..Context::default() };
ctx.rules.enabled.insert("spelling".to_owned(), true);
ctx
}
#[test]
fn unknown_words_get_the_closest_known_word_and_prose_only() {
let ctx = context(&["receive", "the", "letter"]);
let text = "I recieve the leter.\n`recieve` [[recieve]] https://x.io/recieve #recieve\n```\nrecieve\n```\n";
let issues: Vec<_> = check(text, &ctx).into_iter().filter(|issue| issue.rule == "spelling").collect();
let shown: Vec<&str> = issues.iter().filter_map(|issue| text.get(issue.range.start..issue.range.end)).collect();
assert_eq!(shown, ["recieve", "leter"], "code, links, URLs, tags and fences are not checked");
assert_eq!(issues[0].fix.as_ref().map(|fix| fix.label.as_str()), Some("Change to receive"));
assert_eq!(issues[0].fix.as_ref().map(|fix| fix.safe), Some(false));
}
#[test]
fn the_rule_is_silent_when_off_or_without_a_vocabulary() {
let mut off = context(&["word"]);
off.rules.enabled.insert("spelling".to_owned(), false);
assert_eq!(check("badwrd here\n", &off), Vec::new());
assert!(check("badwrd here\n", &Context::default()).iter().all(|issue| issue.rule != "spelling"));
}
#[test]
fn short_words_acronyms_and_capitals_are_left_alone() {
let ctx = context(&["something"]);
let issues = check("NASA is on API day. Zed Zorro\n", &ctx);
assert_eq!(issues.iter().filter(|issue| issue.rule == "spelling").count(), 1, "only `Zorro`: {issues:?}");
}
}
crates/markdown/src/lib.rs
+24 -2
View File
@@ -9,7 +9,7 @@
//! `pulldown-cmark` handles the `CommonMark`/GFM structure (headings, links,
//! code, tables, task lists, footnotes, math) and hands over the spans where no
//! extension may be recognised; the Obsidian scanner adds the rest (wikilinks,
//! embeds, callouts, tags, block ids, highlights, comments). A second math
//! embeds, callouts, tags, block ids, highlights, comments, footnotes). A second math
//! pass (`math_scan`) adds the `$..$` / `$$..$$` formulas the `CommonMark`
//! scanner gives up on and the `\(..\)` / `\[..\]` delimiters it does not
//! know, outside code and taken ranges (SPEC 5, item 30).
@@ -51,7 +51,7 @@ pub use yaml::{Frontmatter, Value, block_range, edit_key};
use std::collections::HashSet;
use std::ops::Range;
use scanning::{common, math_scan, scan};
use scanning::{common, footnotes, math_scan, scan};
/// Maximum number of container levels (`> `, list markers) a line prefix may
/// have before the scanner stops unwrapping it. Deeper input still parses:
@@ -149,6 +149,21 @@ pub enum ItemKind {
/// `+` open, `-` collapsed, absent otherwise.
fold: Fold,
},
/// `[^label]`: a footnote reference. Reported even when `label` has no
/// definition anywhere in the note; the view decides what to do with it.
Footnote {
/// The label, without `[^` and `]`.
label: String,
},
/// `[^label]:` at the start of a definition line. The range covers the
/// marker only; the definition body stays in the source.
FootnoteDef {
/// The label, without `[^`, `]` and the `:`.
label: String,
},
/// `^[inline footnote]`, delimiters included. The text may nest balanced
/// `[` and `]` but never crosses a blank line (footnote scanner bounds).
InlineFootnote,
}
/// A parsed note: the original source plus the items found in it.
@@ -354,6 +369,13 @@ pub fn parse(src: &str) -> Document {
skip.extend(comments);
skip.sort_by_key(|span| (span.start, span.end));
// Footnotes are found before the body scan and their ranges join the
// skip set, so nothing inside a footnote is reported twice.
let notes = footnotes::scan(src, &skip, &items);
skip.extend(notes.iter().map(|item| item.range.clone()));
items.extend(notes);
skip.sort_by_key(|span| (span.start, span.end));
scan::scan(src, &skip, &mut items, MAX_NESTING_DEPTH);
items.extend(math_scan::missed(src, &skip, &items));
items.sort_by_key(|item| (item.range.start, item.range.end));
crates/markdown/src/scanning/footnotes.rs
+299
View File
@@ -0,0 +1,299 @@
//! Footnotes: `[^label]` references, `[^label]:` definition markers and
//! `^[..]` inline footnotes (Obsidian basics, SPEC 5).
//!
//! The pass runs after the `CommonMark` and comment passes and before the
//! Obsidian body scan, and the ranges it finds join the skip set, so no
//! later scanner reports a construct inside a footnote. Openers never sit in
//! code, math, HTML, comments, frontmatter, wikilinks or markdown links; an
//! inline footnote body may cross a code span but never the start of another
//! item, so a footnote can only sit inside a heading or a highlight, the way
//! an inline tag sits inside a heading. A reference whose label has no
//! definition anywhere is still reported: the view decides.
//!
//! Bounded work: label, body, `[[..]]` and `](..)` probes charge a byte
//! budget; a document made of unclosed probes stops the pass early instead
//! of running quadratic time. Items never change the source, so parsing
//! stays lossless.
use std::ops::Range;
use crate::scanning::math_scan::Covered;
use crate::scanning::scan::{char_len, content_start, is, link_span, parens_end};
use crate::{Item, ItemKind, MAX_NESTING_DEPTH};
/// A footnote label holds at most this many characters.
const MAX_LABEL_CHARS: usize = 100;
/// An inline footnote body holds at most this many bytes.
const MAX_INLINE_BYTES: usize = 2_000;
/// Probe budget charged per source byte.
const BUDGET_PER_BYTE: usize = 32;
/// Budget floor, so small notes are never starved.
const BUDGET_FLOOR: usize = 4_096;
/// The footnote items of `src`, in source order.
///
/// `skip` are the spans where nothing may be recognised (code, HTML, math,
/// comments, frontmatter) and `items` the constructs already taken; a
/// footnote never starts inside either.
#[must_use]
pub(crate) fn scan(src: &str, skip: &[Range<usize>], items: &[Item]) -> Vec<Item> {
// A markdown link owns its label and its destination, so nothing inside
// one is a footnote.
let covered = Covered::new({
let mut ranges = skip.to_vec();
ranges.extend(items.iter().filter(|item| matches!(item.kind, ItemKind::Link { .. })).map(|item| item.range.clone()));
ranges
});
// Sorted starts of the items a footnote body may never cross.
let mut starts: Vec<usize> = items.iter().map(|item| item.range.start).collect();
starts.sort_unstable();
let budget = src.len().saturating_mul(BUDGET_PER_BYTE).saturating_add(BUDGET_FLOOR);
let mut pass = Pass { src, bytes: src.as_bytes(), covered, starts, budget, out: Vec::new() };
pass.run();
pass.out
}
/// One forward pass over the source.
struct Pass<'a> {
/// The note being scanned.
src: &'a str,
/// `src` as raw bytes.
bytes: &'a [u8],
/// Spans where no opener may sit: `skip` plus markdown links.
covered: Covered,
/// Sorted starts of the items a footnote body may not cross.
starts: Vec<usize>,
/// Bytes of probe budget left.
budget: usize,
/// The items found so far.
out: Vec<Item>,
}
impl Pass<'_> {
/// Walk the source once, left to right.
fn run(&mut self) {
let len = self.bytes.len();
let mut at = 0;
while at < len {
if let Some(after) = self.covered.jump(at) {
at = after;
continue;
}
let Some(&b) = self.bytes.get(at) else { break };
let next = self.bytes.get(at + 1).copied();
at = match b {
// `\^` is an escaped caret, so `\^[` never opens; `\\` is
// an escaped backslash, so the `^[` after it stays live. A
// backslash before `[` is left to the body scanner, which
// does not treat it as an escape either.
b'\\' if is(next, b'\\') || is(next, b'^') => self.step(at, 2),
b'\\' => self.step(at, 1),
b'!' if is(next, b'[') && is(self.bytes.get(at + 2).copied(), b'[') => self.wiki_jump(at, 1),
b'[' if is(next, b'[') => self.wiki_jump(at, 0),
b']' if is(next, b'(') => self.parens_jump(at),
b'[' if is(next, b'^') => self.reference(at),
b'^' if is(next, b'[') => self.inline_footnote(at),
_ => self.step(at, char_len(self.src, at)),
};
}
}
/// Jump over a wikilink (`bang == 0`) or an embed (`bang == 1`) so
/// nothing inside it is reported.
fn wiki_jump(&mut self, at: usize, bang: usize) -> usize {
let end = line_end(self.src, at);
match link_span(self.src, at + bang, end) {
Some((_, close)) if self.charge(close - at) => close.min(end),
_ => self.step(at, 1),
}
}
/// Jump over a link destination: `]( ... )`.
fn parens_jump(&mut self, at: usize) -> usize {
let after = parens_end(self.src, at + 1).clamp(at + 1, line_end(self.src, at));
if !self.charge(after - at) {
return self.src.len();
}
after
}
/// A `[^label]` reference or a `[^label]:` definition marker at `at`.
fn reference(&mut self, at: usize) -> usize {
match self.reference_probe(at) {
Some((item, after)) => {
self.out.push(item);
after
}
None => self.step(at, 1),
}
}
/// The item and the offset past it, or nothing.
fn reference_probe(&mut self, at: usize) -> Option<(Item, usize)> {
if self.covered.jump(at + 1).is_some() {
return None;
}
// The label is any run without whitespace or `]`, capped.
let mut pos = at + 2;
let mut chars = 0usize;
loop {
// A label never runs through code, math or any other taken span.
if self.covered.jump(pos).is_some() {
return None;
}
let rest = self.src.get(pos..)?;
let head = rest.chars().next()?;
if head == ']' || head.is_whitespace() {
break;
}
chars += 1;
if chars > MAX_LABEL_CHARS {
return None;
}
pos += head.len_utf8();
if !self.charge(head.len_utf8()) {
return None;
}
}
if self.bytes.get(pos) != Some(&b']') || chars == 0 {
return None;
}
let label = self.src.get(at + 2..pos)?.to_string();
let close = pos + 1;
let def = self.bytes.get(close) == Some(&b':') && self.at_content_start(at);
let (kind, end) = if def { (ItemKind::FootnoteDef { label }, close + 1) } else { (ItemKind::Footnote { label }, close) };
Some((Item { kind, range: at..end }, end))
}
/// Whether `open` is the content start of its line, after container
/// prefixes, so a `[^..]:` there is a definition marker.
fn at_content_start(&mut self, open: usize) -> bool {
let Some(head) = self.src.get(..open) else { return false };
let line_start = head.rfind('\n').map_or(0, |at| at + 1);
if !self.charge(open - line_start) {
return false;
}
content_start(self.src, line_start, line_end(self.src, open), MAX_NESTING_DEPTH) == open
}
/// An inline footnote `^[..]` opening at the `^` at `at`.
fn inline_footnote(&mut self, at: usize) -> usize {
match self.body_probe(at) {
Some(end) => {
self.out.push(Item { kind: ItemKind::InlineFootnote, range: at..end });
end
}
None => self.step(at, 1),
}
}
/// The offset past the closing `]` of the body, or nothing. The body
/// balances `[` and `]`, stops at a blank line, is capped and never
/// crosses the start of another item.
fn body_probe(&mut self, at: usize) -> Option<usize> {
if self.covered.jump(at + 1).is_some() {
return None;
}
let next_start = self.starts.partition_point(|&start| start <= at + 1);
let bytes = self.bytes;
let mut pos = at + 2;
let mut depth = 1usize;
let mut size = 0usize;
loop {
if let Some(after) = self.covered.jump(pos) {
// A body may cross code and other skip spans, but never the
// start of an item hiding inside them (math, comments).
if self.starts.get(next_start).is_some_and(|&start| start < after) {
return None;
}
let crossed = after - pos;
if !self.charge(crossed) {
return None;
}
pos = after;
size += crossed;
if size > MAX_INLINE_BYTES {
return None;
}
continue;
}
if self.starts.get(next_start).is_some_and(|&start| start <= pos) {
return None; // the body would swallow another item
}
let &b = bytes.get(pos)?;
if b == b'\n' || b == b'\r' {
let break_len = 1 + usize::from(b == b'\r' && bytes.get(pos + 1) == Some(&b'\n'));
if blank_after(bytes, pos + break_len) {
return None; // an inline footnote never crosses a blank line
}
if !self.charge(break_len) {
return None;
}
pos += break_len;
size += break_len;
if size > MAX_INLINE_BYTES {
return None;
}
continue;
}
if b == b'[' {
depth += 1;
} else if b == b']' {
depth = depth.checked_sub(1)?;
if depth == 0 {
return Some(pos + 1);
}
}
let advance = char_len(self.src, pos);
if !self.charge(advance) {
return None;
}
pos += advance;
size += advance;
if size > MAX_INLINE_BYTES {
return None;
}
}
}
/// Advance `advance` bytes, charging the budget; the source length when
/// the budget is gone, which ends the pass.
fn step(&mut self, at: usize, advance: usize) -> usize {
if !self.charge(advance) {
return self.src.len();
}
at + advance
}
/// Spend `n` bytes of budget; false when the pass must stop.
fn charge(&mut self, n: usize) -> bool {
if self.budget <= n {
self.budget = 0;
return false;
}
self.budget -= n;
true
}
}
/// The content end of the line holding `from`, before the line ending.
fn line_end(src: &str, from: usize) -> usize {
let Some(rest) = src.get(from..) else { return from };
match rest.find('\n') {
Some(nl) => {
let absolute = from + nl;
if nl > 0 && src.as_bytes().get(absolute - 1) == Some(&b'\r') { absolute - 1 } else { absolute }
}
None => src.len(),
}
}
/// Whether the line after the break ending at `from` is blank: only spaces
/// and tabs, then another break.
fn blank_after(bytes: &[u8], from: usize) -> bool {
let mut at = from;
while matches!(bytes.get(at), Some(b' ' | b'\t')) {
at += 1;
}
matches!(bytes.get(at), Some(b'\n' | b'\r'))
}

Some files were not shown because too many files have changed in this diff Show More