feat: smart folder model, media time fragments, link card URL guard (MiniMax); specs for 10 more gaps (Codex) and the registry with bloat measurements
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Marcos Paulo
@@ -230,6 +230,12 @@ pub enum Error {
|
||||
/// Why the range is stale.
|
||||
reason: String,
|
||||
},
|
||||
/// A smart folder definition breaks a rule of its spec: a bad id, an
|
||||
/// empty or too long name, a filter that cannot be read, a path that is
|
||||
/// not a `.base` file of the vault, a limit out of range, or a file that
|
||||
/// holds an unknown version.
|
||||
#[error("smart folder: {0}")]
|
||||
SmartFolder(String),
|
||||
}
|
||||
|
||||
impl Error {
|
||||
@@ -269,6 +275,7 @@ impl Error {
|
||||
Self::Chat(_) => "chat",
|
||||
Self::ReviewLimit { .. } => "limit",
|
||||
Self::StaleRange { .. } => "stale_range",
|
||||
Self::SmartFolder(_) => "smart_folder",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,6 +44,7 @@ mod grammars;
|
||||
mod id;
|
||||
mod info;
|
||||
mod knowledge;
|
||||
mod linkcard;
|
||||
mod lint;
|
||||
mod live;
|
||||
pub mod pandoc;
|
||||
@@ -58,6 +59,7 @@ pub mod review;
|
||||
mod share;
|
||||
mod size;
|
||||
mod slash_kit;
|
||||
pub(crate) mod smart_folders;
|
||||
mod spec;
|
||||
pub mod sync;
|
||||
mod tag_edit;
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
//! The link card feature: the URL guard that stands in front of the fetcher
|
||||
//! (SPEC link-cards, D1).
|
||||
//!
|
||||
//! The guard is pure: it never touches the network, never resolves a name and
|
||||
//! never reads or writes a file. It decides whether a URL may be fetched at
|
||||
//! all, and what the cache key of the URL is.
|
||||
|
||||
#![allow(dead_code, reason = "the link card feature lands in steps; the guard is not wired into a command yet")]
|
||||
|
||||
pub(crate) mod url;
|
||||
@@ -0,0 +1,305 @@
|
||||
//! The URL guard of the link card feature: normalisation, the private-address
|
||||
//! rules and the cache key (SPEC link-cards, D1).
|
||||
//!
|
||||
//! Pure code: no network, no name resolution, no file. A URL that fails the
|
||||
//! guard is never fetched, and a name that points at a private address is
|
||||
//! only caught when it is written as an address.
|
||||
|
||||
use std::net::IpAddr;
|
||||
|
||||
use notescraft_sync::hash_hex;
|
||||
|
||||
mod host;
|
||||
|
||||
use host::{check_host, is_private_v4};
|
||||
|
||||
/// Longest URL the guard accepts, in bytes.
|
||||
const MAX_URL_BYTES: usize = 2048;
|
||||
|
||||
/// A URL that passed the guard.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct Checked {
|
||||
/// The normalised URL: lowercase scheme and host, the default port
|
||||
/// dropped, the fragment dropped, the query kept.
|
||||
pub normalised: String,
|
||||
/// The host of the URL, lowercase and without the brackets of an IPv6
|
||||
/// literal.
|
||||
pub host: String,
|
||||
}
|
||||
|
||||
/// Why the guard refused a URL.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum Refusal {
|
||||
/// The scheme is not `http` or `https`.
|
||||
Scheme,
|
||||
/// The authority carries `user:password@`.
|
||||
Userinfo,
|
||||
/// The URL is longer than 2048 bytes.
|
||||
TooLong,
|
||||
/// The host is empty, local, or not a usable name.
|
||||
Host,
|
||||
/// The host is an address inside a loopback, private or reserved range.
|
||||
Private,
|
||||
/// The URL does not parse.
|
||||
Malformed,
|
||||
}
|
||||
|
||||
/// Checks a URL against the guard.
|
||||
///
|
||||
/// The URL is parsed, never repaired: a URL that cannot be understood is
|
||||
/// refused, not fixed.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// The [`Refusal`] that stopped the URL: the scheme, the userinfo, the
|
||||
/// length, the host, a private address or the shape of the URL.
|
||||
pub(crate) fn check(raw: &str) -> Result<Checked, Refusal> {
|
||||
let parts = parse(raw)?;
|
||||
let normalised = parts.normalised();
|
||||
let host = parts.host;
|
||||
Ok(Checked { normalised, host })
|
||||
}
|
||||
|
||||
/// Whether an address is inside a range the guard refuses: loopback, private,
|
||||
/// link-local, unique-local, unspecified, multicast, reserved or the
|
||||
/// carrier-grade NAT.
|
||||
#[must_use]
|
||||
pub(crate) fn is_private_ip(ip: IpAddr) -> bool {
|
||||
match ip {
|
||||
IpAddr::V4(v4) => is_private_v4(v4),
|
||||
IpAddr::V6(v6) => {
|
||||
// An IPv4-mapped or IPv4-compatible address is judged as the IPv4
|
||||
// address it stands for.
|
||||
if let Some(v4) = v6.to_ipv4() {
|
||||
return is_private_v4(v4);
|
||||
}
|
||||
if v6.is_unspecified() || v6.is_loopback() || v6.is_multicast() {
|
||||
return true;
|
||||
}
|
||||
let head = v6.segments().first().copied().unwrap_or(0);
|
||||
// fc00::/7, the unique local range.
|
||||
if head & 0xfe00 == 0xfc00 {
|
||||
return true;
|
||||
}
|
||||
// fe80::/10, the link-local range.
|
||||
if head & 0xffc0 == 0xfe80 {
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The cache key of a normalised URL: the SHA-256 of the URL, as 64 lowercase
|
||||
/// hex digits.
|
||||
#[must_use]
|
||||
pub(crate) fn cache_key(normalised: &str) -> String {
|
||||
hash_hex(normalised.as_bytes())
|
||||
}
|
||||
|
||||
/// Resolves the `Location` of a redirect against the URL it came from.
|
||||
///
|
||||
/// The result is not checked: the caller runs it through [`check`] again, so a
|
||||
/// redirect that points at a private address is refused there. A location
|
||||
/// that cannot be resolved is [`None`].
|
||||
#[must_use]
|
||||
pub(crate) fn resolve_redirect(base: &str, location: &str) -> Option<String> {
|
||||
if location.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if location.chars().any(|ch| ch.is_whitespace() || ch.is_control()) {
|
||||
return None;
|
||||
}
|
||||
if let Some(scheme) = scheme_of(location) {
|
||||
return (scheme == "http" || scheme == "https").then(|| location.to_owned());
|
||||
}
|
||||
let base = parse(base).ok()?;
|
||||
if let Some(rest) = location.strip_prefix("//") {
|
||||
return Some(format!("{}://{rest}", base.scheme));
|
||||
}
|
||||
if location.starts_with('/') {
|
||||
return Some(format!("{}://{}{location}", base.scheme, base.authority()));
|
||||
}
|
||||
Some(format!("{}://{}{}{location}", base.scheme, base.authority(), folder_of(&base.path)))
|
||||
}
|
||||
|
||||
/// The parts of a URL the guard works with.
|
||||
struct Parsed {
|
||||
/// The lowercase scheme.
|
||||
scheme: String,
|
||||
/// The lowercase host, without the brackets of an IPv6 literal.
|
||||
host: String,
|
||||
/// The port, when the URL names one.
|
||||
port: Option<u16>,
|
||||
/// The path: empty, or starting with a slash.
|
||||
path: String,
|
||||
/// The query, without the question mark.
|
||||
query: Option<String>,
|
||||
}
|
||||
|
||||
impl Parsed {
|
||||
/// The URL in its normalised shape.
|
||||
fn normalised(&self) -> String {
|
||||
let mut out = String::with_capacity(self.scheme.len() + self.host.len() + self.path.len() + 16);
|
||||
out.push_str(&self.scheme);
|
||||
out.push_str("://");
|
||||
out.push_str(&host_text(&self.host));
|
||||
if let Some(port) = self.port
|
||||
&& !is_default_port(&self.scheme, port)
|
||||
{
|
||||
out.push(':');
|
||||
out.push_str(&port.to_string());
|
||||
}
|
||||
out.push_str(&self.path);
|
||||
if let Some(query) = &self.query {
|
||||
out.push('?');
|
||||
out.push_str(query);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The authority of the URL: the host and the port.
|
||||
fn authority(&self) -> String {
|
||||
let mut out = host_text(&self.host);
|
||||
if let Some(port) = self.port {
|
||||
out.push(':');
|
||||
out.push_str(&port.to_string());
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a URL into its parts.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// The [`Refusal`] that stopped the URL.
|
||||
fn parse(raw: &str) -> Result<Parsed, Refusal> {
|
||||
if raw.len() > MAX_URL_BYTES {
|
||||
return Err(Refusal::TooLong);
|
||||
}
|
||||
if raw.chars().any(|ch| ch.is_whitespace() || ch.is_control()) {
|
||||
return Err(Refusal::Malformed);
|
||||
}
|
||||
let (scheme, rest) = raw.split_once(':').ok_or(Refusal::Malformed)?;
|
||||
let scheme = scheme.to_ascii_lowercase();
|
||||
if scheme != "http" && scheme != "https" {
|
||||
return Err(Refusal::Scheme);
|
||||
}
|
||||
let rest = rest.strip_prefix("//").ok_or(Refusal::Malformed)?;
|
||||
let (authority, remainder) = split_authority(rest);
|
||||
if authority.contains('@') {
|
||||
return Err(Refusal::Userinfo);
|
||||
}
|
||||
let (raw_host, raw_port) = split_host_port(authority)?;
|
||||
let host = check_host(raw_host)?;
|
||||
let port = match raw_port {
|
||||
Some(text) => Some(parse_port(text)?),
|
||||
None => None,
|
||||
};
|
||||
let (path, query) = split_path_query(remainder);
|
||||
Ok(Parsed { scheme, host, port, path, query })
|
||||
}
|
||||
|
||||
/// The authority of a URL and the rest of it.
|
||||
fn split_authority(rest: &str) -> (&str, &str) {
|
||||
match rest.find(['/', '?', '#']) {
|
||||
Some(end) => rest.split_at(end),
|
||||
None => (rest, ""),
|
||||
}
|
||||
}
|
||||
|
||||
/// The host and the port of an authority.
|
||||
fn split_host_port(authority: &str) -> Result<(&str, Option<&str>), Refusal> {
|
||||
if let Some(rest) = authority.strip_prefix('[') {
|
||||
let (inside, after) = rest.split_once(']').ok_or(Refusal::Malformed)?;
|
||||
let port = match after {
|
||||
"" => None,
|
||||
other => Some(other.strip_prefix(':').ok_or(Refusal::Malformed)?),
|
||||
};
|
||||
return Ok((inside, port));
|
||||
}
|
||||
if authority.contains(']') {
|
||||
return Err(Refusal::Malformed);
|
||||
}
|
||||
if authority.matches(':').count() > 1 {
|
||||
// An IPv6 literal without brackets names no port.
|
||||
return Ok((authority, None));
|
||||
}
|
||||
Ok(match authority.rsplit_once(':') {
|
||||
Some((host, port)) => (host, Some(port)),
|
||||
None => (authority, None),
|
||||
})
|
||||
}
|
||||
|
||||
/// The port of a URL.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`Refusal::Malformed`] when the port is not a number of 1 to 65535.
|
||||
fn parse_port(text: &str) -> Result<u16, Refusal> {
|
||||
if text.is_empty() || !text.bytes().all(|byte| byte.is_ascii_digit()) {
|
||||
return Err(Refusal::Malformed);
|
||||
}
|
||||
let port = text.parse::<u16>().map_err(|_| Refusal::Malformed)?;
|
||||
if port == 0 {
|
||||
return Err(Refusal::Malformed);
|
||||
}
|
||||
Ok(port)
|
||||
}
|
||||
|
||||
/// The path and the query of the part of a URL after the authority.
|
||||
fn split_path_query(remainder: &str) -> (String, Option<String>) {
|
||||
let without_fragment = match remainder.split_once('#') {
|
||||
Some((head, _)) => head,
|
||||
None => remainder,
|
||||
};
|
||||
match without_fragment.split_once('?') {
|
||||
Some((path, "")) => (path.to_owned(), None),
|
||||
Some((path, query)) => (path.to_owned(), Some(query.to_owned())),
|
||||
None => (without_fragment.to_owned(), None),
|
||||
}
|
||||
}
|
||||
|
||||
/// The host of a URL, with the brackets of an IPv6 literal.
|
||||
fn host_text(host: &str) -> String {
|
||||
let mut out = String::with_capacity(host.len() + 2);
|
||||
if host.contains(':') {
|
||||
out.push('[');
|
||||
out.push_str(host);
|
||||
out.push(']');
|
||||
} else {
|
||||
out.push_str(host);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Whether a port is the default of its scheme.
|
||||
fn is_default_port(scheme: &str, port: u16) -> bool {
|
||||
(scheme == "http" && port == 80) || (scheme == "https" && port == 443)
|
||||
}
|
||||
|
||||
/// The folder a relative path is resolved against.
|
||||
fn folder_of(path: &str) -> String {
|
||||
match path.rsplit_once('/') {
|
||||
Some((head, _)) => format!("{head}/"),
|
||||
None => "/".to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The scheme of an absolute URL, if it has one.
|
||||
fn scheme_of(location: &str) -> Option<String> {
|
||||
let (head, _) = location.split_once(':')?;
|
||||
let mut chars = head.chars();
|
||||
let first = chars.next()?;
|
||||
if !first.is_ascii_alphabetic() {
|
||||
return None;
|
||||
}
|
||||
if !head.chars().all(|ch| ch.is_ascii_alphanumeric() || ch == '+' || ch == '-' || ch == '.') {
|
||||
return None;
|
||||
}
|
||||
Some(head.to_ascii_lowercase())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,161 @@
|
||||
//! The host rules of the URL guard: the names that point at the machine
|
||||
//! itself, and the addresses inside a refused range.
|
||||
|
||||
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
|
||||
use std::str::FromStr;
|
||||
|
||||
use super::{Refusal, is_private_ip};
|
||||
|
||||
/// Checks the host of a URL and returns it in lowercase.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`Refusal::Host`] when the host is empty or local, [`Refusal::Private`]
|
||||
/// when it is an address inside a refused range and [`Refusal::Malformed`]
|
||||
/// when it is neither an address nor a usable name.
|
||||
pub(super) fn check_host(raw_host: &str) -> Result<String, Refusal> {
|
||||
let host = raw_host.to_ascii_lowercase();
|
||||
if host.is_empty() {
|
||||
return Err(Refusal::Host);
|
||||
}
|
||||
if host.contains(':') {
|
||||
let address = Ipv6Addr::from_str(&host).map_err(|_| Refusal::Malformed)?;
|
||||
if is_private_ip(IpAddr::V6(address)) {
|
||||
return Err(Refusal::Private);
|
||||
}
|
||||
return Ok(host);
|
||||
}
|
||||
let candidates = ipv4_candidates(&host);
|
||||
if !candidates.is_empty() {
|
||||
if candidates.iter().any(|address| is_private_ip(IpAddr::V4(*address))) {
|
||||
return Err(Refusal::Private);
|
||||
}
|
||||
return Ok(host);
|
||||
}
|
||||
if !is_host_name(&host) {
|
||||
return Err(Refusal::Malformed);
|
||||
}
|
||||
if is_local_name(&host) {
|
||||
return Err(Refusal::Host);
|
||||
}
|
||||
Ok(host)
|
||||
}
|
||||
|
||||
/// Whether a name points at the machine itself.
|
||||
fn is_local_name(host: &str) -> bool {
|
||||
let last = host.rsplit('.').next().unwrap_or(host);
|
||||
matches!(last, "localhost" | "local" | "internal")
|
||||
}
|
||||
|
||||
/// Whether a name is a usable host name.
|
||||
fn is_host_name(host: &str) -> bool {
|
||||
if host.len() > 253 {
|
||||
return false;
|
||||
}
|
||||
let host = host.strip_suffix('.').unwrap_or(host);
|
||||
if host.is_empty() {
|
||||
return false;
|
||||
}
|
||||
host.split('.').all(|label| !label.is_empty() && label.len() <= 63 && label.chars().all(|ch| ch.is_alphanumeric() || ch == '-' || ch == '_'))
|
||||
}
|
||||
|
||||
/// Whether an IPv4 address is inside a refused range.
|
||||
pub(super) fn is_private_v4(ip: Ipv4Addr) -> bool {
|
||||
let [first, second, ..] = ip.octets();
|
||||
if first == 0 || first == 10 || first == 127 {
|
||||
// "This network", the private range 10/8 and the loopback range.
|
||||
return true;
|
||||
}
|
||||
if first == 100 && (64..=127).contains(&second) {
|
||||
// The carrier-grade NAT range, 100.64/10.
|
||||
return true;
|
||||
}
|
||||
if first == 169 && second == 254 {
|
||||
// The link-local range, 169.254/16.
|
||||
return true;
|
||||
}
|
||||
if first == 172 && (16..=31).contains(&second) {
|
||||
// The private range, 172.16/12.
|
||||
return true;
|
||||
}
|
||||
if first == 192 && second == 168 {
|
||||
// The private range, 192.168/16.
|
||||
return true;
|
||||
}
|
||||
if (224..=239).contains(&first) {
|
||||
// Multicast.
|
||||
return true;
|
||||
}
|
||||
if (240..=255).contains(&first) {
|
||||
// Reserved, and the broadcast address.
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// How a part of a numeric host is read.
|
||||
#[derive(Clone, Copy)]
|
||||
enum NumberStyle {
|
||||
/// The classic `inet_aton` reading: `0x` is hexadecimal, a leading zero is
|
||||
/// octal, anything else is decimal.
|
||||
Classic,
|
||||
/// Every part is decimal.
|
||||
Decimal,
|
||||
}
|
||||
|
||||
/// The addresses a numeric host can stand for.
|
||||
///
|
||||
/// A host made only of digits and dots is an address in one of the old
|
||||
/// notations: `2130706433`, `0x7f.1`, `0177.0.0.1`. The classic reading is
|
||||
/// the one `inet_aton` used; the plain decimal reading is added when it
|
||||
/// differs, so a host that means a private address under either reading is
|
||||
/// refused.
|
||||
fn ipv4_candidates(host: &str) -> Vec<Ipv4Addr> {
|
||||
let parts: Vec<&str> = host.split('.').collect();
|
||||
if parts.is_empty() || parts.len() > 4 {
|
||||
return Vec::new();
|
||||
}
|
||||
let Some(classic) = ipv4_from_parts(&parts, NumberStyle::Classic) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut out = vec![classic];
|
||||
if let Some(decimal) = ipv4_from_parts(&parts, NumberStyle::Decimal)
|
||||
&& decimal != classic
|
||||
{
|
||||
out.push(decimal);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The address a numeric host stands for, in one reading.
|
||||
fn ipv4_from_parts(parts: &[&str], style: NumberStyle) -> Option<Ipv4Addr> {
|
||||
let count = parts.len();
|
||||
let mut total: u64 = 0;
|
||||
for (index, part) in parts.iter().enumerate() {
|
||||
let value = match style {
|
||||
NumberStyle::Classic => classic_number(part)?,
|
||||
NumberStyle::Decimal => part.parse::<u32>().ok()?,
|
||||
};
|
||||
let shift = 8 * (count - 1 - index);
|
||||
let limit = if index + 1 == count { 1_u64 << (32 - 8 * (count - 1)) } else { 256 };
|
||||
if u64::from(value) >= limit {
|
||||
return None;
|
||||
}
|
||||
total |= u64::from(value) << shift;
|
||||
}
|
||||
u32::try_from(total).ok().map(Ipv4Addr::from)
|
||||
}
|
||||
|
||||
/// One part of a numeric host, in the classic reading.
|
||||
fn classic_number(part: &str) -> Option<u32> {
|
||||
if let Some(digits) = part.strip_prefix("0x").or_else(|| part.strip_prefix("0X")) {
|
||||
return u32::from_str_radix(digits, 16).ok();
|
||||
}
|
||||
if part.len() > 1
|
||||
&& let Some(digits) = part.strip_prefix('0')
|
||||
&& digits.bytes().all(|byte| (b'0'..=b'7').contains(&byte))
|
||||
{
|
||||
return u32::from_str_radix(digits, 8).ok();
|
||||
}
|
||||
part.parse::<u32>().ok()
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
//! The tests of the URL guard.
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_public_url_is_normalised() {
|
||||
let checked = check("https://Example.com/a?x=1#frag").unwrap();
|
||||
assert_eq!(checked.normalised, "https://example.com/a?x=1");
|
||||
assert_eq!(checked.host, "example.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_default_port_and_the_fragment_are_dropped() {
|
||||
let checked = check("http://example.com:80/").unwrap();
|
||||
assert_eq!(checked.normalised, "http://example.com/");
|
||||
let checked = check("https://example.com:443/a").unwrap();
|
||||
assert_eq!(checked.normalised, "https://example.com/a");
|
||||
let checked = check("https://example.com:8443/a").unwrap();
|
||||
assert_eq!(checked.normalised, "https://example.com:8443/a");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_ipv6_url_keeps_its_brackets() {
|
||||
let checked = check("https://[2001:db8::1]/").unwrap();
|
||||
assert_eq!(checked.normalised, "https://[2001:db8::1]/");
|
||||
assert_eq!(checked.host, "2001:db8::1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_query_is_dropped() {
|
||||
let checked = check("https://example.com/a?").unwrap();
|
||||
assert_eq!(checked.normalised, "https://example.com/a");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_http_and_https_are_accepted() {
|
||||
assert_eq!(check("file:///etc/passwd"), Err(Refusal::Scheme));
|
||||
assert_eq!(check("ftp://x"), Err(Refusal::Scheme));
|
||||
assert_eq!(check("javascript:alert(1)"), Err(Refusal::Scheme));
|
||||
assert_eq!(check("HTTPS://example.com/"), Ok(Checked { normalised: "https://example.com/".to_owned(), host: "example.com".to_owned() }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_url_with_userinfo_is_refused() {
|
||||
assert_eq!(check("http://user:pw@example.com/"), Err(Refusal::Userinfo));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_loopback_and_the_private_ranges_are_refused() {
|
||||
for url in [
|
||||
"http://127.0.0.1/",
|
||||
"http://[::1]/",
|
||||
"http://10.0.0.1/",
|
||||
"http://172.16.0.1/",
|
||||
"http://192.168.1.1/",
|
||||
"http://169.254.169.254/",
|
||||
"http://100.64.0.1/",
|
||||
"http://0.0.0.0/",
|
||||
"http://[::ffff:127.0.0.1]/",
|
||||
"http://[fc00::1]/",
|
||||
"http://[fe80::1]/",
|
||||
"http://[ff02::1]/",
|
||||
"http://255.255.255.255/",
|
||||
] {
|
||||
assert_eq!(check(url), Err(Refusal::Private), "{url}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_public_address_is_accepted() {
|
||||
assert!(check("http://8.8.8.8/").is_ok());
|
||||
let public = "8.8.8.8".parse::<IpAddr>().expect("a literal address");
|
||||
assert!(!is_private_ip(public));
|
||||
let private = "10.0.0.1".parse::<IpAddr>().expect("a literal address");
|
||||
assert!(is_private_ip(private));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_old_numeric_notations_are_refused() {
|
||||
assert_eq!(check("http://2130706433/"), Err(Refusal::Private));
|
||||
assert_eq!(check("http://0x7f.1/"), Err(Refusal::Private));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_local_name_is_refused() {
|
||||
assert_eq!(check("http://localhost:8080"), Err(Refusal::Host));
|
||||
assert_eq!(check("http://foo.localhost/"), Err(Refusal::Host));
|
||||
assert_eq!(check("http://printer.local/"), Err(Refusal::Host));
|
||||
assert_eq!(check("http://host.internal/"), Err(Refusal::Host));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_too_long_url_is_refused() {
|
||||
let long = format!("http://example.com/{}", "a".repeat(2030));
|
||||
assert_eq!(long.len(), 2049);
|
||||
assert_eq!(check(&long), Err(Refusal::TooLong));
|
||||
let fits = format!("http://example.com/{}", "a".repeat(2029));
|
||||
assert_eq!(fits.len(), 2048);
|
||||
assert!(check(&fits).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_malformed_url_is_refused() {
|
||||
assert_eq!(check("http://"), Err(Refusal::Host));
|
||||
assert_eq!(check("http:///a"), Err(Refusal::Host));
|
||||
assert_eq!(check("example.com"), Err(Refusal::Malformed));
|
||||
assert_eq!(check("http://example.com/a b"), Err(Refusal::Malformed));
|
||||
assert_eq!(check("http://example.com/a\u{00a0}b"), Err(Refusal::Malformed));
|
||||
assert_eq!(check("http://example.com/a\u{3000}b"), Err(Refusal::Malformed));
|
||||
assert_eq!(check("http://example.com:0/"), Err(Refusal::Malformed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redirect_is_resolved_against_its_base() {
|
||||
assert_eq!(resolve_redirect("https://example.com/a/b", "/x"), Some("https://example.com/x".to_owned()));
|
||||
assert_eq!(resolve_redirect("https://example.com/a/b", "//host/x"), Some("https://host/x".to_owned()));
|
||||
assert_eq!(resolve_redirect("https://example.com/a/b", "https://other.test/x"), Some("https://other.test/x".to_owned()));
|
||||
assert_eq!(resolve_redirect("https://example.com/a/b", "c"), Some("https://example.com/a/c".to_owned()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_broken_redirect_is_refused() {
|
||||
assert_eq!(resolve_redirect("https://example.com/a", ""), None);
|
||||
assert_eq!(resolve_redirect("https://example.com/a", "file:///etc/passwd"), None);
|
||||
assert_eq!(resolve_redirect("https://example.com/a", "not a url"), None);
|
||||
assert_eq!(resolve_redirect("https://example.com/a", "javascript:alert(1)"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redirect_to_a_private_address_is_refused_by_the_check() {
|
||||
let location = resolve_redirect("https://example.com/", "http://127.0.0.1/").unwrap_or_default();
|
||||
assert_eq!(check(&location), Err(Refusal::Private));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_cache_key_is_the_hash_of_the_normalised_url() {
|
||||
assert_eq!(cache_key(""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
|
||||
let key = cache_key("https://example.com/a?x=1");
|
||||
assert_eq!(key.len(), 64);
|
||||
assert!(key.bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)));
|
||||
assert_eq!(key, cache_key("https://example.com/a?x=1"));
|
||||
assert_ne!(key, cache_key("https://example.com/a?x=2"));
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
//! Smart folders: the saved queries the Explorer shows as folders.
|
||||
//!
|
||||
//! A smart folder is a named Bases query that the Explorer shows as a folder
|
||||
//! (SPEC smart folders D1). This module holds the model: the definition of a
|
||||
//! folder, the rules a definition must follow and the JSON shape of the file
|
||||
//! that keeps them. Reading and writing that file is the store's job.
|
||||
|
||||
#![allow(dead_code, unused_imports, reason = "the smart folder commands use the model in a later task")]
|
||||
|
||||
pub(crate) mod model;
|
||||
|
||||
pub(crate) use model::{
|
||||
FILE_VERSION, MAX_FILE_BYTES, MAX_FOLDERS, MAX_ICON_CHARS, MAX_LIMIT, MAX_NAME_CHARS, MAX_PATH_CHARS, MAX_VIEW_CHARS, MAX_YAML_BYTES, MIN_LIMIT,
|
||||
SmartFolder, Source, Store, new_id, validate,
|
||||
};
|
||||
@@ -0,0 +1,543 @@
|
||||
//! The model of the smart folders: the definitions and the rules they follow.
|
||||
//!
|
||||
//! A smart folder is a named, saved Bases query that the Explorer shows as a
|
||||
//! folder (SPEC smart folders D1). This module holds the definition itself,
|
||||
//! the validation that keeps a definition usable and the JSON shape of the
|
||||
//! file that stores them; reading and writing that file is the store's job.
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
/// The version of the file this client writes.
|
||||
pub const FILE_VERSION: u32 = 1;
|
||||
/// Most smart folders one vault holds.
|
||||
pub const MAX_FOLDERS: usize = 200;
|
||||
/// Longest name of a smart folder, in characters.
|
||||
pub const MAX_NAME_CHARS: usize = 80;
|
||||
/// Largest inline base of a smart folder, in bytes.
|
||||
pub const MAX_YAML_BYTES: usize = 16 * 1024;
|
||||
/// Largest file the parser reads, in bytes.
|
||||
pub const MAX_FILE_BYTES: usize = 4 * 1024 * 1024;
|
||||
/// Smallest limit a smart folder may ask for.
|
||||
pub const MIN_LIMIT: usize = 1;
|
||||
/// Largest limit a smart folder may ask for.
|
||||
pub const MAX_LIMIT: usize = 2000;
|
||||
/// Longest name of an icon, in characters.
|
||||
pub const MAX_ICON_CHARS: usize = 64;
|
||||
/// Longest name of a base view, in characters.
|
||||
pub const MAX_VIEW_CHARS: usize = 80;
|
||||
/// Longest path of a base file, in characters.
|
||||
pub const MAX_PATH_CHARS: usize = 512;
|
||||
|
||||
/// The prefix every smart folder id carries.
|
||||
const ID_PREFIX: &str = "sf_";
|
||||
/// The letters and digits the random part of an id is made of.
|
||||
const ID_ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyz012345";
|
||||
/// Characters of the random part of a new id.
|
||||
const ID_CHARS: usize = 6;
|
||||
/// Smallest random part of an id, in characters.
|
||||
const MIN_ID_CHARS: usize = 4;
|
||||
/// Largest random part of an id, in characters.
|
||||
const MAX_ID_CHARS: usize = 12;
|
||||
/// The offset the hash of an id starts from.
|
||||
const HASH_OFFSET: u64 = 0xcbf2_9ce4_8422_2325;
|
||||
/// The prime the hash of an id multiplies with.
|
||||
const HASH_PRIME: u64 = 0x0000_0100_0000_01b3;
|
||||
/// The bits one character of an id carries.
|
||||
const CHUNK_BITS: u32 = 5;
|
||||
/// The mask that keeps the bits of one character.
|
||||
const CHUNK_MASK: u64 = 0x1f;
|
||||
/// The file written when the folders cannot be serialised.
|
||||
const EMPTY_FILE: &str = r#"{"version":1,"folders":[]}"#;
|
||||
|
||||
/// The counter that keeps the ids of one process apart.
|
||||
static COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
/// A smart folder: a named, saved query.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SmartFolder {
|
||||
/// The stable id of the folder: `sf_` plus 4 to 12 letters or digits.
|
||||
pub id: String,
|
||||
/// The name the Explorer shows.
|
||||
pub name: String,
|
||||
/// The icon shown next to the name, if any.
|
||||
pub icon: Option<String>,
|
||||
/// Where the query comes from.
|
||||
pub source: Source,
|
||||
/// How many rows the folder shows at most.
|
||||
pub limit: Option<usize>,
|
||||
}
|
||||
|
||||
/// Where a smart folder takes its query from.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Source {
|
||||
/// An inline base, the same yaml the `base.query` command takes.
|
||||
Yaml(String),
|
||||
/// A `.base` file of the vault and the view to read from it.
|
||||
Base {
|
||||
/// The vault-relative path of the `.base` file.
|
||||
path: String,
|
||||
/// The view to read; the first one when it is empty.
|
||||
view: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
/// The smart folders of a vault.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct Store {
|
||||
/// The folders, in the order the file holds them.
|
||||
pub folders: Vec<SmartFolder>,
|
||||
}
|
||||
|
||||
/// The file that holds the smart folders of a vault.
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct File {
|
||||
/// The version of the file.
|
||||
#[serde(default)]
|
||||
version: u32,
|
||||
/// The folders the file holds.
|
||||
#[serde(default)]
|
||||
folders: Vec<FolderFile>,
|
||||
}
|
||||
|
||||
/// One folder as the file holds it: the query sits beside the name, not
|
||||
/// inside a nested object.
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct FolderFile {
|
||||
/// The stable id of the folder.
|
||||
#[serde(default)]
|
||||
id: String,
|
||||
/// The name the Explorer shows.
|
||||
#[serde(default)]
|
||||
name: String,
|
||||
/// The icon shown next to the name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
icon: Option<String>,
|
||||
/// The inline base, when the folder holds one.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
yaml: Option<String>,
|
||||
/// The `.base` file, when the folder points at one.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
path: Option<String>,
|
||||
/// The view of the `.base` file.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
view: Option<String>,
|
||||
/// How many rows the folder shows at most.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// Reads the folders of a file.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`Error::SmartFolder`] when the file is not JSON, holds an
|
||||
/// unknown version or a definition that breaks a rule of the spec.
|
||||
pub fn parse(json: &str) -> Result<Self> {
|
||||
if json.len() > MAX_FILE_BYTES {
|
||||
return Err(Error::SmartFolder(format!("the smart folder file is {} bytes long, the cap is {MAX_FILE_BYTES}", json.len())));
|
||||
}
|
||||
let file: File = serde_json::from_str(json).map_err(|error| Error::SmartFolder(format!("the smart folder file is not valid JSON: {error}")))?;
|
||||
if file.version != FILE_VERSION {
|
||||
return Err(Error::SmartFolder(format!("the smart folder file has the unknown version {}", file.version)));
|
||||
}
|
||||
if file.folders.len() > MAX_FOLDERS {
|
||||
return Err(Error::SmartFolder(format!("the smart folder file holds {} folders, the cap is {MAX_FOLDERS}", file.folders.len())));
|
||||
}
|
||||
let mut folders = Vec::with_capacity(file.folders.len());
|
||||
for raw in file.folders {
|
||||
let folder = raw.into_folder()?;
|
||||
validate(&folder)?;
|
||||
folders.push(folder);
|
||||
}
|
||||
Ok(Self { folders })
|
||||
}
|
||||
|
||||
/// Writes the folders as the file the app keeps them in.
|
||||
#[must_use]
|
||||
pub fn to_json(&self) -> String {
|
||||
let file = File { version: FILE_VERSION, folders: self.folders.iter().map(FolderFile::from_folder).collect() };
|
||||
serde_json::to_string(&file).unwrap_or_else(|_| EMPTY_FILE.to_owned())
|
||||
}
|
||||
|
||||
/// Adds a folder, or replaces the one that carries the same id.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`Error::SmartFolder`] when the definition breaks a rule or
|
||||
/// the vault already holds [`MAX_FOLDERS`] folders.
|
||||
pub fn upsert(&mut self, folder: SmartFolder) -> Result<()> {
|
||||
validate(&folder)?;
|
||||
if let Some(slot) = self.folders.iter_mut().find(|other| other.id == folder.id) {
|
||||
*slot = folder;
|
||||
return Ok(());
|
||||
}
|
||||
if self.folders.len() >= MAX_FOLDERS {
|
||||
return Err(Error::SmartFolder(format!("the vault already holds {MAX_FOLDERS} smart folders")));
|
||||
}
|
||||
self.folders.push(folder);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Removes the folder that carries that id and says whether there was one.
|
||||
pub fn remove(&mut self, id: &str) -> bool {
|
||||
let before = self.folders.len();
|
||||
self.folders.retain(|folder| folder.id != id);
|
||||
self.folders.len() != before
|
||||
}
|
||||
}
|
||||
|
||||
impl FolderFile {
|
||||
/// Reads the folder the file holds.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`Error::SmartFolder`] when the folder holds a yaml and a
|
||||
/// path, or holds neither of them.
|
||||
fn into_folder(self) -> Result<SmartFolder> {
|
||||
let Self { id, name, icon, yaml, path, view, limit } = self;
|
||||
let source = match (yaml, path) {
|
||||
(Some(yaml), None) => Source::Yaml(yaml),
|
||||
(None, Some(path)) => Source::Base { path, view },
|
||||
(Some(_), Some(_)) => {
|
||||
return Err(Error::SmartFolder("a smart folder holds either a yaml filter or a base path, never both".to_owned()));
|
||||
}
|
||||
(None, None) => {
|
||||
return Err(Error::SmartFolder("a smart folder holds either a yaml filter or a base path".to_owned()));
|
||||
}
|
||||
};
|
||||
Ok(SmartFolder { id, name, icon, source, limit })
|
||||
}
|
||||
|
||||
/// Writes the folder as the file holds it.
|
||||
fn from_folder(folder: &SmartFolder) -> Self {
|
||||
let (yaml, path, view) = match &folder.source {
|
||||
Source::Yaml(yaml) => (Some(yaml.clone()), None, None),
|
||||
Source::Base { path, view } => (None, Some(path.clone()), view.clone()),
|
||||
};
|
||||
Self { id: folder.id.clone(), name: folder.name.clone(), icon: folder.icon.clone(), yaml, path, view, limit: folder.limit }
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks that a definition can be used.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`Error::SmartFolder`] when the id, the name, the query or the
|
||||
/// limit breaks a rule of the spec.
|
||||
pub fn validate(folder: &SmartFolder) -> Result<()> {
|
||||
if !valid_id(&folder.id) {
|
||||
return Err(Error::SmartFolder(format!("`{}` is not a smart folder id", folder.id)));
|
||||
}
|
||||
validate_name(&folder.name)?;
|
||||
if let Some(icon) = &folder.icon {
|
||||
validate_text(icon, MAX_ICON_CHARS, "icon")?;
|
||||
}
|
||||
match &folder.source {
|
||||
Source::Yaml(yaml) => validate_yaml(yaml)?,
|
||||
Source::Base { path, view } => {
|
||||
validate_path(path)?;
|
||||
if let Some(view) = view {
|
||||
validate_text(view, MAX_VIEW_CHARS, "view")?;
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(limit) = folder.limit
|
||||
&& !(MIN_LIMIT..=MAX_LIMIT).contains(&limit)
|
||||
{
|
||||
return Err(Error::SmartFolder(format!("the limit {limit} is outside {MIN_LIMIT}..={MAX_LIMIT}")));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Builds an id for a new smart folder.
|
||||
///
|
||||
/// The id is `sf_` plus six letters or digits. A candidate that repeats the
|
||||
/// id of a folder in `existing` grows by one character, up to the twelve the
|
||||
/// spec allows.
|
||||
#[must_use]
|
||||
pub fn new_id(existing: &[SmartFolder]) -> String {
|
||||
let counter = COUNTER.fetch_add(1, Ordering::Relaxed);
|
||||
let hash = hash(existing.len(), millis(), counter);
|
||||
let mut id = String::new();
|
||||
for length in ID_CHARS..=MAX_ID_CHARS {
|
||||
id = format!("{ID_PREFIX}{}", encode(hash, length));
|
||||
if !existing.iter().any(|folder| folder.id == id) {
|
||||
return id;
|
||||
}
|
||||
}
|
||||
id
|
||||
}
|
||||
|
||||
/// Says whether the id looks like a smart folder id.
|
||||
fn valid_id(id: &str) -> bool {
|
||||
let Some(rest) = id.strip_prefix(ID_PREFIX) else { return false };
|
||||
let length = rest.chars().count();
|
||||
(MIN_ID_CHARS..=MAX_ID_CHARS).contains(&length) && rest.chars().all(|letter| letter.is_ascii_lowercase() || letter.is_ascii_digit())
|
||||
}
|
||||
|
||||
/// Checks the name of a smart folder.
|
||||
fn validate_name(name: &str) -> Result<()> {
|
||||
if name.trim() != name {
|
||||
return Err(Error::SmartFolder("the name of a smart folder must not start or end with a space".to_owned()));
|
||||
}
|
||||
if name.is_empty() {
|
||||
return Err(Error::SmartFolder("a smart folder needs a name".to_owned()));
|
||||
}
|
||||
let length = name.chars().count();
|
||||
if length > MAX_NAME_CHARS {
|
||||
return Err(Error::SmartFolder(format!("the name is {length} characters long, the cap is {MAX_NAME_CHARS}")));
|
||||
}
|
||||
if name.chars().any(char::is_control) {
|
||||
return Err(Error::SmartFolder("the name of a smart folder holds a control character".to_owned()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Checks a short text of a definition, such as an icon or a view name.
|
||||
fn validate_text(text: &str, cap: usize, what: &str) -> Result<()> {
|
||||
let length = text.chars().count();
|
||||
if length > cap {
|
||||
return Err(Error::SmartFolder(format!("the {what} is {length} characters long, the cap is {cap}")));
|
||||
}
|
||||
if text.chars().any(char::is_control) {
|
||||
return Err(Error::SmartFolder(format!("the {what} holds a control character")));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Checks the inline base of a smart folder.
|
||||
fn validate_yaml(yaml: &str) -> Result<()> {
|
||||
if yaml.len() > MAX_YAML_BYTES {
|
||||
return Err(Error::SmartFolder(format!("the filter is {} bytes long, the cap is {MAX_YAML_BYTES}", yaml.len())));
|
||||
}
|
||||
notescraft_bases::Base::parse(yaml).map_err(|error| Error::SmartFolder(format!("the filter cannot be read: {error}")))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Checks the path of the base file a smart folder points at.
|
||||
fn validate_path(path: &str) -> Result<()> {
|
||||
let length = path.chars().count();
|
||||
if length == 0 {
|
||||
return Err(Error::SmartFolder("a smart folder needs the path of its base file".to_owned()));
|
||||
}
|
||||
if length > MAX_PATH_CHARS {
|
||||
return Err(Error::SmartFolder(format!("the base path is {length} characters long, the cap is {MAX_PATH_CHARS}")));
|
||||
}
|
||||
if path.chars().any(char::is_control) {
|
||||
return Err(Error::SmartFolder("the base path holds a control character".to_owned()));
|
||||
}
|
||||
if path.contains('\\') {
|
||||
return Err(Error::SmartFolder("the base path holds a backslash".to_owned()));
|
||||
}
|
||||
if path.starts_with('/') {
|
||||
return Err(Error::SmartFolder("the base path is absolute".to_owned()));
|
||||
}
|
||||
if path.split('/').any(|segment| segment == "..") {
|
||||
return Err(Error::SmartFolder("the base path leaves the vault".to_owned()));
|
||||
}
|
||||
if path.starts_with(".obsidian/") || path.starts_with(".notescraft/") {
|
||||
return Err(Error::SmartFolder(format!("`{path}` is a reserved path")));
|
||||
}
|
||||
if !path.to_ascii_lowercase().ends_with(".base") {
|
||||
return Err(Error::SmartFolder(format!("`{path}` is not a `.base` file")));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The milliseconds since the epoch, used to spread the ids apart.
|
||||
fn millis() -> u64 {
|
||||
SystemTime::now().duration_since(UNIX_EPOCH).ok().and_then(|delta| u64::try_from(delta.as_millis()).ok()).unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Hashes the parts of an id into one number.
|
||||
fn hash(length: usize, stamp: u64, counter: u64) -> u64 {
|
||||
let mut value = HASH_OFFSET;
|
||||
for part in [u64::try_from(length).unwrap_or(0), stamp, counter] {
|
||||
for byte in part.to_le_bytes() {
|
||||
value ^= u64::from(byte);
|
||||
value = value.wrapping_mul(HASH_PRIME);
|
||||
}
|
||||
}
|
||||
value
|
||||
}
|
||||
|
||||
/// Writes the hash as letters and digits.
|
||||
fn encode(hash: u64, length: usize) -> String {
|
||||
let mut out = String::with_capacity(length);
|
||||
for index in 0..length {
|
||||
let shift = usize::try_from(CHUNK_BITS).unwrap_or(5).saturating_mul(length.saturating_sub(1).saturating_sub(index));
|
||||
let chunk = (hash >> shift) & CHUNK_MASK;
|
||||
let letter = ID_ALPHABET.get(usize::try_from(chunk).unwrap_or(0)).copied().unwrap_or(b'a');
|
||||
out.push(char::from(letter));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
/// A filter every test can use.
|
||||
const YAML: &str = "filters:\n and:\n - file.hasTag(\"urgent\")\n";
|
||||
|
||||
/// A folder with an inline base.
|
||||
fn folder(id: &str) -> SmartFolder {
|
||||
SmartFolder { id: id.to_owned(), name: "Urgent".to_owned(), icon: None, source: Source::Yaml(YAML.to_owned()), limit: None }
|
||||
}
|
||||
|
||||
/// A file that holds that many folders.
|
||||
fn file_with(count: usize) -> String {
|
||||
let folders: Vec<serde_json::Value> =
|
||||
(0..count).map(|index| json!({"id": format!("sf_a{index:04}"), "name": format!("F{index}"), "yaml": YAML})).collect();
|
||||
json!({"version": 1, "folders": folders}).to_string()
|
||||
}
|
||||
|
||||
/// A yaml of exactly that many bytes that the bases parser reads.
|
||||
fn yaml_of_size(size: usize) -> String {
|
||||
let head = "filters:\n and:\n - file.hasTag(\"urgent\")\npadding: \"";
|
||||
let tail = "\"\n";
|
||||
let fill = size.saturating_sub(head.len() + tail.len());
|
||||
format!("{head}{}{tail}", "x".repeat(fill))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_folder_survives_a_round_trip() {
|
||||
let mut store = Store::default();
|
||||
store.upsert(folder("sf_ab12")).unwrap();
|
||||
let json = store.to_json();
|
||||
assert_eq!(
|
||||
json, r#"{"version":1,"folders":[{"id":"sf_ab12","name":"Urgent","yaml":"filters:\n and:\n - file.hasTag(\"urgent\")\n"}]}"#,
|
||||
"the file keeps the shape of the spec"
|
||||
);
|
||||
assert!(json.contains("\"yaml\""), "the file holds the filter beside the name");
|
||||
assert!(!json.contains("\"path\""), "a folder with a yaml holds no path");
|
||||
let back = Store::parse(&json).unwrap();
|
||||
assert_eq!(back, store);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_folder_holds_either_a_yaml_or_a_path() {
|
||||
let both = json!({"version": 1, "folders": [{"id": "sf_ab12", "name": "x", "yaml": YAML, "path": "a.base"}]}).to_string();
|
||||
assert!(Store::parse(&both).is_err(), "a folder holds one query, not two");
|
||||
let neither = json!({"version": 1, "folders": [{"id": "sf_ab12", "name": "x"}]}).to_string();
|
||||
assert!(Store::parse(&neither).is_err(), "a folder holds a query");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_vault_holds_at_most_two_hundred_folders() {
|
||||
assert!(Store::parse(&file_with(MAX_FOLDERS)).is_ok());
|
||||
assert!(Store::parse(&file_with(MAX_FOLDERS + 1)).is_err());
|
||||
let mut store = Store::default();
|
||||
for index in 0..MAX_FOLDERS {
|
||||
store.upsert(folder(&format!("sf_a{index:04}"))).unwrap();
|
||||
}
|
||||
assert!(store.upsert(folder("sf_b0000")).is_err(), "the two hundred and first folder is refused");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_name_is_checked() {
|
||||
let mut folder = folder("sf_ab12");
|
||||
folder.name = "a".repeat(MAX_NAME_CHARS + 1);
|
||||
assert!(validate(&folder).is_err(), "a long name is refused");
|
||||
folder.name = String::new();
|
||||
assert!(validate(&folder).is_err(), "an empty name is refused");
|
||||
folder.name = "a\u{7}b".to_owned();
|
||||
assert!(validate(&folder).is_err(), "a control character is refused");
|
||||
folder.name = "a".repeat(MAX_NAME_CHARS);
|
||||
assert!(validate(&folder).is_ok(), "a name of the cap is kept");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_filter_is_at_most_sixteen_kib() {
|
||||
let mut folder = folder("sf_ab12");
|
||||
folder.source = Source::Yaml(yaml_of_size(MAX_YAML_BYTES + 1));
|
||||
assert!(validate(&folder).is_err(), "a filter over the cap is refused");
|
||||
folder.source = Source::Yaml(yaml_of_size(MAX_YAML_BYTES));
|
||||
assert!(validate(&folder).is_ok(), "a filter of the cap is kept");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_filter_that_cannot_be_read_is_refused() {
|
||||
let mut folder = folder("sf_ab12");
|
||||
folder.source = Source::Yaml("filters: [".to_owned());
|
||||
assert!(validate(&folder).is_err(), "a broken filter is refused");
|
||||
folder.source = Source::Yaml(String::new());
|
||||
assert!(validate(&folder).is_ok(), "an empty filter reads as a base with no filter");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_path_must_be_a_base_file_of_the_vault() {
|
||||
let mut folder = folder("sf_ab12");
|
||||
folder.source = Source::Base { path: "x.md".to_owned(), view: None };
|
||||
assert!(validate(&folder).is_err(), "a path that is not a base file is refused");
|
||||
folder.source = Source::Base { path: ".notescraft/a.base".to_owned(), view: None };
|
||||
assert!(validate(&folder).is_err(), "a reserved path is refused");
|
||||
folder.source = Source::Base { path: ".obsidian/a.base".to_owned(), view: None };
|
||||
assert!(validate(&folder).is_err(), "a reserved path is refused");
|
||||
folder.source = Source::Base { path: "../a.base".to_owned(), view: None };
|
||||
assert!(validate(&folder).is_err(), "a path that leaves the vault is refused");
|
||||
folder.source = Source::Base { path: "views/a.base".to_owned(), view: Some("Table".to_owned()) };
|
||||
assert!(validate(&folder).is_ok(), "a base file of the vault is kept");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_id_must_look_like_a_smart_folder_id() {
|
||||
assert!(validate(&folder("sf_ab12")).is_ok());
|
||||
assert!(validate(&folder("sf_abc")).is_err(), "the id is too short");
|
||||
assert!(validate(&folder("sf_abcdefghijklm")).is_err(), "the id is too long");
|
||||
assert!(validate(&folder("sf_AB12")).is_err(), "the id holds a capital");
|
||||
assert!(validate(&folder("ab12")).is_err(), "the id misses the prefix");
|
||||
assert!(validate(&folder("sf_ab-12")).is_err(), "the id holds a dash");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_limit_is_between_one_and_two_thousand() {
|
||||
let mut folder = folder("sf_ab12");
|
||||
folder.limit = Some(0);
|
||||
assert!(validate(&folder).is_err(), "a limit of zero is refused");
|
||||
folder.limit = Some(MAX_LIMIT + 1);
|
||||
assert!(validate(&folder).is_err(), "a limit over the cap is refused");
|
||||
folder.limit = Some(MAX_LIMIT);
|
||||
assert!(validate(&folder).is_ok(), "a limit of the cap is kept");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unknown_version_is_refused() {
|
||||
let json = json!({"version": 2, "folders": []}).to_string();
|
||||
assert!(Store::parse(&json).is_err(), "a version this client does not know is refused");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_broken_file_is_refused() {
|
||||
assert!(Store::parse("{oops").is_err(), "a file that is not JSON is refused");
|
||||
assert!(Store::parse("").is_err(), "an empty file is refused");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_new_id_does_not_collide() {
|
||||
let mut store = Store::default();
|
||||
let first = new_id(&store.folders);
|
||||
assert!(valid_id(&first), "the id looks like a smart folder id");
|
||||
store.upsert(folder(&first)).unwrap();
|
||||
let second = new_id(&store.folders);
|
||||
assert_ne!(first, second, "a new id does not repeat an old one");
|
||||
let mut ids: Vec<String> = Vec::new();
|
||||
for _ in 0..MAX_FOLDERS {
|
||||
let id = new_id(&ids.iter().map(|id| folder(id)).collect::<Vec<_>>());
|
||||
assert!(!ids.contains(&id), "every id of a vault is its own");
|
||||
ids.push(id);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_removed_folder_is_gone() {
|
||||
let mut store = Store::default();
|
||||
store.upsert(folder("sf_ab12")).unwrap();
|
||||
assert!(store.remove("sf_ab12"), "the folder was there");
|
||||
assert!(!store.remove("sf_ab12"), "the folder is gone");
|
||||
assert_eq!(store.folders, [] as [SmartFolder; 0]);
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,7 @@
|
||||
|
||||
mod frontmatter;
|
||||
mod scanning;
|
||||
mod timefrag;
|
||||
|
||||
pub mod links;
|
||||
pub use frontmatter::yaml;
|
||||
@@ -47,6 +48,7 @@ pub use frontmatter::frontmatter_edit::{
|
||||
pub use links::{LinkTarget, anchor, anchors_match, decode_percent, is_internal_dest, parse_link_target};
|
||||
pub use scanning::callouts::{CalloutKind, Fold};
|
||||
pub use scanning::fences::{Fence, FenceBlock, fence_open, fenced_blocks};
|
||||
pub use timefrag::{MEDIA_EXTS, TimeFragment, format_time, is_media_ext, parse_time_fragment};
|
||||
pub use yaml::{Frontmatter, Value, block_range, edit_key};
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
//! Media timestamp fragments inside link subpaths (`[[note.mp3#t=04:12]]`).
|
||||
//!
|
||||
//! The grammar follows the `Media Fragments` URI: `t=` then an optional
|
||||
//! `npt:` prefix, a time and an optional `,` with the end of a range. A time
|
||||
//! is `S[.f]`, `M:SS[.f]` or `H:MM:SS[.f]`. Anything else has no time
|
||||
//! fragment: the link still opens the file, it just does not seek.
|
||||
|
||||
/// Maximum size of the whole subpath, in bytes.
|
||||
const MAX_SUBPATH_BYTES: usize = 64;
|
||||
|
||||
/// Longest total duration of a fragment, in milliseconds (100 hours).
|
||||
const MAX_TOTAL_MS: u64 = 100 * 60 * 60 * 1000;
|
||||
|
||||
/// Largest number of minutes in the `M:SS` form.
|
||||
const MAX_MINUTES: u64 = 6000;
|
||||
|
||||
/// Largest number of hours in the `H:MM:SS` form.
|
||||
const MAX_HOURS: u64 = 100;
|
||||
|
||||
/// Largest number of digits in the fraction part.
|
||||
const MAX_FRACTION_DIGITS: usize = 3;
|
||||
|
||||
/// Seconds in a minute.
|
||||
const SECONDS_PER_MINUTE: u64 = 60;
|
||||
|
||||
/// Seconds in an hour.
|
||||
const SECONDS_PER_HOUR: u64 = 60 * 60;
|
||||
|
||||
/// Milliseconds in a second.
|
||||
const MS_PER_SECOND: u64 = 1000;
|
||||
|
||||
/// Milliseconds in an hour.
|
||||
const MS_PER_HOUR: u64 = SECONDS_PER_HOUR * MS_PER_SECOND;
|
||||
|
||||
/// File extensions that can carry a media timestamp, lowercase.
|
||||
///
|
||||
/// Keep in sync with `AUDIO_EXTS` and `VIDEO_EXTS` in
|
||||
/// `crates/ui-egui/src/widgets/tabs/kind.rs`.
|
||||
pub const MEDIA_EXTS: &[&str] = &["mp3", "wav", "ogg", "m4a", "flac", "opus", "aac", "3gp", "mp4", "webm", "mov", "mkv", "m4v"];
|
||||
|
||||
/// A parsed media timestamp.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct TimeFragment {
|
||||
/// Start of the fragment, in milliseconds.
|
||||
pub start_ms: u64,
|
||||
/// End of the fragment, in milliseconds. `None` for a single point.
|
||||
pub end_ms: Option<u64>,
|
||||
}
|
||||
|
||||
/// Parse the subpath of a link as a media timestamp.
|
||||
///
|
||||
/// `subpath` is the text after the `#` (`t=04:12`). The result is `None` for
|
||||
/// anything that is not a valid timestamp: an empty value, a negative or
|
||||
/// exponential number, a field out of range, a range that does not move
|
||||
/// forward or a subpath over [`MAX_SUBPATH_BYTES`].
|
||||
///
|
||||
/// ```
|
||||
/// # use notescraft_markdown::parse_time_fragment;
|
||||
/// let Some(frag) = parse_time_fragment("t=04:12") else { return };
|
||||
/// assert_eq!(frag.start_ms, 252_000);
|
||||
/// assert_eq!(frag.end_ms, None);
|
||||
/// assert!(parse_time_fragment("t=99:99").is_none());
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn parse_time_fragment(subpath: &str) -> Option<TimeFragment> {
|
||||
if subpath.len() > MAX_SUBPATH_BYTES {
|
||||
return None;
|
||||
}
|
||||
let rest = subpath.strip_prefix("t=")?;
|
||||
let rest = rest.strip_prefix("npt:").unwrap_or(rest);
|
||||
let (start_text, end_text) = match rest.split_once(',') {
|
||||
Some((start_text, end_text)) => (start_text, Some(end_text)),
|
||||
None => (rest, None),
|
||||
};
|
||||
let start_ms = parse_time(start_text)?;
|
||||
let end_ms = match end_text {
|
||||
Some(text) => {
|
||||
let end_ms = parse_time(text)?;
|
||||
if end_ms <= start_ms {
|
||||
return None;
|
||||
}
|
||||
Some(end_ms)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
Some(TimeFragment { start_ms, end_ms })
|
||||
}
|
||||
|
||||
/// Write a duration as `M:SS`, or as `H:MM:SS` from one hour.
|
||||
///
|
||||
/// Seconds are truncated, never rounded.
|
||||
///
|
||||
/// ```
|
||||
/// # use notescraft_markdown::format_time;
|
||||
/// assert_eq!(format_time(252_000), "4:12");
|
||||
/// assert_eq!(format_time(3_723_000), "1:02:03");
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn format_time(ms: u64) -> String {
|
||||
let total_seconds = ms / MS_PER_SECOND;
|
||||
let hours = total_seconds / SECONDS_PER_HOUR;
|
||||
let minutes = (total_seconds % SECONDS_PER_HOUR) / SECONDS_PER_MINUTE;
|
||||
let seconds = total_seconds % SECONDS_PER_MINUTE;
|
||||
if hours > 0 { format!("{hours}:{minutes:02}:{seconds:02}") } else { format!("{minutes}:{seconds:02}") }
|
||||
}
|
||||
|
||||
/// Whether `ext` is a media file extension, ignoring case.
|
||||
///
|
||||
/// ```
|
||||
/// # use notescraft_markdown::is_media_ext;
|
||||
/// assert!(is_media_ext("MP3"));
|
||||
/// assert!(!is_media_ext("md"));
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn is_media_ext(ext: &str) -> bool {
|
||||
MEDIA_EXTS.iter().any(|known| known.eq_ignore_ascii_case(ext))
|
||||
}
|
||||
|
||||
/// Parse one time of the grammar, in milliseconds.
|
||||
fn parse_time(text: &str) -> Option<u64> {
|
||||
let (int_part, fraction_ms) = match text.split_once('.') {
|
||||
Some((int_part, fraction)) => (int_part, parse_fraction(fraction)?),
|
||||
None => (text, 0),
|
||||
};
|
||||
let mut parts = int_part.split(':');
|
||||
let first = parts.next()?;
|
||||
let second = parts.next();
|
||||
let third = parts.next();
|
||||
if parts.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
let total = match (second, third) {
|
||||
(None, None) => parse_number(first)?.checked_mul(MS_PER_SECOND)?,
|
||||
(Some(seconds), None) => {
|
||||
let minutes = parse_number(first)?;
|
||||
let seconds = parse_number(seconds)?;
|
||||
if minutes > MAX_MINUTES || seconds >= SECONDS_PER_MINUTE {
|
||||
return None;
|
||||
}
|
||||
minutes.checked_mul(SECONDS_PER_MINUTE)?.checked_mul(MS_PER_SECOND)?.checked_add(seconds.checked_mul(MS_PER_SECOND)?)?
|
||||
}
|
||||
(Some(minutes), Some(seconds)) => {
|
||||
let hours = parse_number(first)?;
|
||||
let minutes = parse_number(minutes)?;
|
||||
let seconds = parse_number(seconds)?;
|
||||
if hours > MAX_HOURS || minutes >= SECONDS_PER_MINUTE || seconds >= SECONDS_PER_MINUTE {
|
||||
return None;
|
||||
}
|
||||
hours
|
||||
.checked_mul(MS_PER_HOUR)?
|
||||
.checked_add(minutes.checked_mul(SECONDS_PER_MINUTE)?.checked_mul(MS_PER_SECOND)?)?
|
||||
.checked_add(seconds.checked_mul(MS_PER_SECOND)?)?
|
||||
}
|
||||
(None, Some(_)) => return None,
|
||||
};
|
||||
let total = total.checked_add(fraction_ms)?;
|
||||
if total > MAX_TOTAL_MS {
|
||||
return None;
|
||||
}
|
||||
Some(total)
|
||||
}
|
||||
|
||||
/// Parse the fraction part after the dot, in milliseconds.
|
||||
fn parse_fraction(text: &str) -> Option<u64> {
|
||||
if text.is_empty() || text.len() > MAX_FRACTION_DIGITS {
|
||||
return None;
|
||||
}
|
||||
let scale = match text.len() {
|
||||
1 => 100,
|
||||
2 => 10,
|
||||
3 => 1,
|
||||
_ => return None,
|
||||
};
|
||||
parse_number(text)?.checked_mul(scale)
|
||||
}
|
||||
|
||||
/// Parse a run of ASCII digits, refusing anything else.
|
||||
fn parse_number(text: &str) -> Option<u64> {
|
||||
if text.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut value: u64 = 0;
|
||||
for ch in text.chars() {
|
||||
if !ch.is_ascii_digit() {
|
||||
return None;
|
||||
}
|
||||
let digit = u64::from(ch.to_digit(10)?);
|
||||
value = value.checked_mul(10)?.checked_add(digit)?;
|
||||
}
|
||||
Some(value)
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
//! Integration tests for the media timestamp parser (`[[note.mp3#t=04:12]]`).
|
||||
|
||||
use notescraft_markdown::{MEDIA_EXTS, TimeFragment, format_time, is_media_ext, parse_time_fragment};
|
||||
|
||||
/// Start offset of a subpath, if it is a time fragment.
|
||||
fn start(subpath: &str) -> Option<u64> {
|
||||
parse_time_fragment(subpath).map(|frag| frag.start_ms)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_the_documented_forms() {
|
||||
assert_eq!(start("t=04:12"), Some(252_000));
|
||||
assert_eq!(start("t=4:12"), Some(252_000));
|
||||
assert_eq!(start("t=252"), Some(252_000));
|
||||
assert_eq!(start("t=1:02:03"), Some(3_723_000));
|
||||
assert_eq!(start("t=0:04:12.5"), Some(252_500));
|
||||
assert_eq!(start("t=npt:10.5"), Some(10_500));
|
||||
assert_eq!(start("t=10.5"), Some(10_500));
|
||||
assert_eq!(start("t=0"), Some(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_a_range() {
|
||||
assert_eq!(parse_time_fragment("t=10,20"), Some(TimeFragment { start_ms: 10_000, end_ms: Some(20_000) }));
|
||||
assert_eq!(parse_time_fragment("t=npt:10.5,20"), Some(TimeFragment { start_ms: 10_500, end_ms: Some(20_000) }));
|
||||
assert_eq!(parse_time_fragment("t=10"), Some(TimeFragment { start_ms: 10_000, end_ms: None }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_a_range_that_does_not_move_forward() {
|
||||
assert_eq!(parse_time_fragment("t=20,10"), None);
|
||||
assert_eq!(parse_time_fragment("t=10,10"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_invalid_subpaths() {
|
||||
for bad in [
|
||||
"t=",
|
||||
"t=-1",
|
||||
"t=1e3",
|
||||
"t=1.2345",
|
||||
"t=99:99",
|
||||
"t=1:75",
|
||||
"t=360001",
|
||||
"t= 10",
|
||||
"t=10 ",
|
||||
"t=10&x=1",
|
||||
"t=10,20,30",
|
||||
"t=10:",
|
||||
"t=:10",
|
||||
"t=1.2.3",
|
||||
"t=10.",
|
||||
"t=.5",
|
||||
"page=3",
|
||||
"T=10",
|
||||
"",
|
||||
] {
|
||||
assert_eq!(parse_time_fragment(bad), None, "{bad} is not a time fragment");
|
||||
}
|
||||
assert_eq!(parse_time_fragment("t=\u{661}\u{662}"), None, "non-ASCII digits are not digits");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_subpaths_over_the_byte_limit() {
|
||||
let long = format!("t={}", "0".repeat(63));
|
||||
assert_eq!(long.len(), 65);
|
||||
assert_eq!(parse_time_fragment(&long), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_the_total_limit() {
|
||||
assert_eq!(start("t=100:00:00"), Some(360_000_000));
|
||||
assert_eq!(start("t=100:00:00.001"), None);
|
||||
assert_eq!(start("t=6000:00"), Some(360_000_000));
|
||||
assert_eq!(start("t=6000:00.001"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn formats_times() {
|
||||
assert_eq!(format_time(0), "0:00");
|
||||
assert_eq!(format_time(252_000), "4:12");
|
||||
assert_eq!(format_time(3_723_000), "1:02:03");
|
||||
assert_eq!(format_time(3_600_000), "1:00:00");
|
||||
assert_eq!(format_time(59_999), "0:59", "seconds are truncated, not rounded");
|
||||
assert_eq!(format_time(3_723_999), "1:02:03");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn formats_and_parses_round_trip() {
|
||||
for step in 0..1000_u64 {
|
||||
let ms = step * 1000;
|
||||
let text = format!("t={}", format_time(ms));
|
||||
assert_eq!(start(&text), Some(ms), "{text} does not round trip");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn media_extensions_are_case_insensitive() {
|
||||
assert!(is_media_ext("MP3"));
|
||||
assert!(is_media_ext("mp4"));
|
||||
assert!(is_media_ext("MKV"));
|
||||
assert!(!is_media_ext("md"));
|
||||
assert!(!is_media_ext(""));
|
||||
assert!(MEDIA_EXTS.contains(&"mp3"));
|
||||
assert!(MEDIA_EXTS.contains(&"mp4"));
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
# Attachment OCR (agy gap 8, roadmap 37)
|
||||
|
||||
## Research (2026-10-21; agy claims checked)
|
||||
[Joplin OCR](https://joplinapp.org/help/apps/ocr/) confirms offline attachment search and desktop processing; this validates workflow, not NotesCraft's proposed `.ocr.db` storage or handwriting accuracy. [Apple Vision VNRecognizeTextRequest](https://developer.apple.com/documentation/vision/vnrecognizetextrequest) is native text-recognition API; its dynamic docs did not yield details through this research tool, so do not infer language/accuracy guarantees. [Windows OcrEngine](https://learn.microsoft.com/en-us/uwp/api/windows.media.ocr.ocrengine?view=winrt-26100) documents supported-language discovery, image-size limit and word boxes. [ocrs](https://github.com/robertknight/ocrs) is real Rust OCR project with model requirements; agy's “pure Rust on Linux” is not justification for shipping new inference/model dependencies automatically. Bear/OneNote parity claims not needed and dropped.
|
||||
|
||||
## Goal and decision
|
||||
Offline image/scanned-PDF text extraction, attachment search results with page/box navigation, and explicit Copy detected text. Prefer OS Vision on macOS and Windows.Media.Ocr on Windows over bundled engine in v1: avoids model distribution and duplicating OS resources. Linux returns structured unsupported capability, no cloud or opportunistic executable download. This is selected architecture subject to owner approval of safe binding dependencies and supported packaging. No native adapter implementation until approval; contract/model/index tests can proceed with deterministic fake backend. No new crate: pure result types in existing index crate, orchestration/platform adapters in engine. Direct Objective-C/COM FFI often requires unsafe, forbidden here: do not weaken lint, add C/Swift helper or put unsafe in sibling crate. Approve vetted safe wrapper or stop adapter task and present concrete alternative. Rust-only constraint remains.
|
||||
|
||||
## D1 Result and bounded worker contract
|
||||
`OcrDocument {version:1,path,source_sha256,backend,backend_version,languages,pages}`; page `{number,width,height,lines:[{text,rect,words:[{text,rect}]}]}`. Rectangles normalized top-left after image orientation/PDF rotation, finite positive area within unit square; reading order returned by backend, source stays byte-identical. Never fabricate confidence when API has none; optional backend confidence only. Caps: 64 MiB source image, 40 million decoded pixels, existing PDF 200 MiB cap, <= 500 pages/job, <= 2 MiB text/page and 16 MiB/job, <= 100,000 words/page. Reject oversized input before recognition, no silent truncation/downscale. User may explicitly select PDF page range <=500; job reports that range so partial extraction cannot masquerade as complete.
|
||||
One worker per vault, bounded queue 32, cancellation on vault switch/removal, 30 seconds/page and 5 minutes/job. Each completion checks current source hash/job generation before publishing; stale result discarded explicitly. Job states queued/running/complete/cancelled/failed/unsupported, progress per page. No UI thread OCR. Platform image cap may be smaller: report supported bound, reject with actionable error. No network for recognition, no auto language/model download.
|
||||
|
||||
## D2 Cache and indexing
|
||||
`.notescraft/cache/ocr/<source-sha256>/<identity-sha256>.json`, schema 1, content hash + backend revision + requested language set in cache identity; atomic write_private. `identity-sha256` hashes canonical backend ID/version plus sorted requested language set, so differing recognition configurations cannot collide. Cache derived and local-only, not sync default. Source hash is approved existing sha2 via viewer hashing pattern. Validate cache exactly like hostile backend data; corrupt cache never indexed, diagnostic and explicit regeneration action. No sqlite dependency/new DB. Cache budget 256 MiB, bounded in-memory result snapshot; eviction deletes only recognized OCR cache entries via explicit cache management API, never source files.
|
||||
Existing `crates/index/src/search/doc.rs` builds Markdown documents; do not pretend attachment text has Markdown line positions. Add separate typed attachment insertion/removal API and existing `Hit` (`crates/index/src/search/rank.rs`) attachment location (path, page, normalized rectangles, snippet) preserving note-hit JSON compatibility. Reuse tokenizer/ranker, never index OCR into Markdown/frontmatter or overwrite native text. Source removal/hash change invalidates OCR result and postings. Search image opens image view; PDF hit opens existing PDF tab at page and overlays result box. No live re-recognition per query. Native PDF text extraction, when available from PDF annotation backend, is preferred for textual pages; scanned pages OCR raster from existing pdf_preview path, resolution/capability explicit. If native extraction unavailable, OCR pages with provenance `ocr`, never describe as original text.
|
||||
|
||||
## D3 Engine/UI
|
||||
`ocr.capabilities {}` -> supported/backend/languages/limits; `ocr.start {path,pages?,languages?}` -> job ID; `ocr.status {job}` -> progress/error; `ocr.text {path,page?}` -> cached text/geometry; `ocr.cancel {job}`. Start/cancel write-gated because cache/job state changes; capability/status/text read-gated, locked encryption domains refuse all extraction and cached reads. Existing read-only vault can read valid cache, cannot start persistent job. Commands registered in engine, assistant gate and CORE_IDS. Persist top-level `AppSettings.ocr` (not `SessionConfig`) with `enabled` and `background` false by default; user enables background image indexing or runs explicit attachment action. Installed-language absence has recoverable error; empty result succeeds with no-text state. Strings through i18n, keyboard-accessible Copy detected text, search badge makes provenance clear. Attachment import schedules only after successful save/index, never on paint; no handwritten-text accuracy promise.
|
||||
|
||||
## Acceptance criteria
|
||||
1. Fake backend makes image text searchable once; hit points to attachment/page/rect, note results unchanged; source bytes unchanged.
|
||||
2. Hash replacement/removal/cancel/locked domain removes stale postings and prevents late worker publication; read-only and invalid paths reject before writes.
|
||||
3. Hostile dimensions, NaN/out-of-bounds box, invalid UTF-8 transport, corrupt cache, unknown schema and all caps fail safely; no silent truncation.
|
||||
4. Native fixture suite runs offline on approved macOS/Windows targets with installed languages; unsupported OS/language/packaging reports capability, never cloud fallback. Actual supported builds must pass before marked complete.
|
||||
5. PDF page selection, rotated image, no-text page, cancellation and cache reopen tested; OCR never modifies PDF or Markdown.
|
||||
6. UI screenshots/kittest cover image/PDF hit navigation, Copy text and progress/error states; background work never restarts every frame.
|
||||
|
||||
## Open owner questions
|
||||
- Q1: Approve OS-first macOS/Windows v1 with Linux explicitly unsupported? Bundled `ocrs` alternative requires separate dependency/model/license/size approval.
|
||||
- Q2: Approve concrete safe Rust bindings for Vision/WinRT after adapter spike, minimum OS and Windows packaging verification? If no safe interface works under forbidden unsafe/Rust-only rules, adapter blocked; do not relax either rule silently.
|
||||
- Q3: Accept opt-in background indexing and local-only 256 MiB derived cache? Should OCR text eventually sync (privacy and backend-version policy needed)?
|
||||
- Q4: Which languages and fixture accuracy targets are required for launch? Proposed printed English + Portuguese; no universal/handwriting guarantee.
|
||||
|
||||
## Feature ideas (researched, ranked)
|
||||
1. Accessible PDF export with hidden selectable text, separate feature inspired by [Joplin OCR accessible documents](https://joplinapp.org/help/apps/ocr/).
|
||||
2. Linux offline engine after explicit model/dependency approval, informed by [ocrs](https://github.com/robertknight/ocrs).
|
||||
3. Installed-language selector and word overlay based on [Windows OcrEngine capabilities](https://learn.microsoft.com/en-us/uwp/api/windows.media.ocr.ocrengine?view=winrt-26100).
|
||||
@@ -0,0 +1,60 @@
|
||||
# Tasks: attachment OCR (roadmap 37)
|
||||
|
||||
Spec: `spec.md`. Rules: `AGENTS.md`, `docs/CODE-MAP.md`.
|
||||
One task = one agent; execute in order, dependencies are preceding tasks. Each task has its own test boundary; backend tests use fakes until approval. Existing paths verified with `rg --files` and source reads on 2026-10-21; paths marked new do not exist yet. No new crate; no workspace/layer registration needed. If owner instead approves a new crate, first revise tasks to own its manifest, root `Cargo.toml` and `xtask/src/layers.rs`, package/layer/dependency registration and crate smoke test before use.
|
||||
**First tool call rule:** first tool call reads at most three listed inputs; write FIRST file in OWNS in SECOND tool call, with compiling stub and one failing test, then make it pass. Do not spend second call exploring. For document-only tasks write initial acceptance checklist in second call and review/complete it instead of a code stub.
|
||||
**Standing VERIFY (future implementation only):** `export PATH="$HOME/.cargo/bin:$PATH"`; `cargo test -p <owned-package>`; `cargo clippy -p <owned-package> --all-targets -- -D warnings`; `rustfmt --edition 2024 <owned Rust files>`. UI tasks add `cargo xtask i18n` and render/read snapshot PNG. Last implementation task adds `cargo xtask layers`, `cargo xtask tree`, `cargo xtask ci`. These commands are instructions, not commands run during this spec-writing request.
|
||||
**Standing DO NOT:** touch outside OWNS; change git state; run `cargo fmt --all`; add dependency without owner response; unsafe/panics/unchecked slicing; rewrite source attachments; hardcode UI strings; bypass engine commands. No production claim until platform acceptance passes.
|
||||
|
||||
## Phase 1 — validated contract
|
||||
- [ ] 1. Pure OCR result model and capability contract.
|
||||
OWNS: `crates/index/src/ocr.rs` (new, FIRST), `crates/index/src/lib.rs` (declare/re-export), `crates/index/src/error.rs` (validation error only).
|
||||
FILES TO OPEN: `specs/plugins-parity/attachment-ocr/spec.md` D1; `crates/index/src/lib.rs`; `crates/index/src/error.rs`.
|
||||
GOAL: `OcrDocument`, `OcrPage`, `OcrLine`, `OcrWord`, `OcrRect`, `OcrCapabilities`, `validate()` with named caps; pure model, no OCR engine/no IO. Optional confidence only, no fabricated values; backend-neutral typed unsupported status. JSON persistence stays engine if serde unavailable here; no manifest edit.
|
||||
TESTS: image/page geometry orientation, empty successful result, NaN/outside box, page 0, 501 pages, duplicate page, text/word cap, unsupported language/capability description; fixture reading order retained.
|
||||
VERIFY: standing index; `cargo test -p notescraft-index ocr` runs actual new module tests.
|
||||
DO NOT: native APIs, dependency, OCR implementation, IO or synthetic Markdown postings.
|
||||
## Phase 2 — backend gate and worker
|
||||
- [ ] 2. Approve native adapter boundary (document-only decision).
|
||||
OWNS: `specs/plugins-parity/attachment-ocr/backend-decision.md` (new, FIRST).
|
||||
FILES TO OPEN: spec questions; root `Cargo.toml`; `crates/engine/Cargo.toml`.
|
||||
GOAL: inventory already-installed safe bindings; primary-source verified API/OS/packaging/language availability; exact proposed dependency versions/licenses and safe-call proof. Compare OS-first vs bundled ocrs/model footprint. Obtain recorded owner response before native task; unavailable safe path blocks task 4, not contract tests. No new crate proposed.
|
||||
TESTS: documented compile feasibility cases macOS Vision request creation and Windows desktop WinRT OCR initialization without project unsafe; identify any unsupported package configuration explicitly.
|
||||
VERIFY: manual source/manifest review, no cargo required for decision document; owner approval remains pending until actual response.
|
||||
DO NOT: edit manifests, unsafe adapter, C/Swift helper or auto download.
|
||||
- [ ] 3. Bounded worker, cache and registry commands with fake backend.
|
||||
OWNS: `crates/engine/src/ocr/mod.rs` (new, FIRST), `crates/engine/src/ocr/worker.rs` (new), `crates/engine/src/ocr/cache.rs` (new), `crates/engine/src/ocr/commands.rs` (new), `crates/engine/src/lib.rs`, `crates/engine/src/commands.rs`, `crates/engine/src/ai/assistant_gate.rs`, `crates/engine/tests/core_commands.rs`, `crates/engine/tests/ocr.rs` (new).
|
||||
FILES TO OPEN: spec D1–D3; `crates/engine/src/viewers/info.rs`; `crates/engine/src/vault/pdf_preview.rs`; later remaining owned registry files and `crates/engine/tests/common/mod.rs`.
|
||||
GOAL: bounded one-worker queue/cancel/generation, explicit progress, validated hash-keyed atomic cache, five command IDs with read/write policy; fake backend in tests and production unsupported capability until native adapter exists. Reuse existing sha2 hash implementation pattern, never mtime-only cache key. No pretend recognition.
|
||||
TESTS: queue full, cancel races, deadline, hash replacement same size/mtime, corrupt schema/cache, read-only, bounds, source unchanged, no late publication after vault switch; cache hit schedules zero recognition.
|
||||
VERIFY: standing engine; command integration tests and fake worker tests deterministic without installed OCR.
|
||||
DO NOT: native dependency before Q2, silent empty success for unavailable backend, unbounded channels.
|
||||
- [ ] 4. Native macOS/Windows adapters (blocked on task 2 owner response).
|
||||
OWNS: `crates/engine/src/ocr/platform.rs` (new, FIRST), `crates/engine/src/ocr/platform/macos.rs` (new), `crates/engine/src/ocr/platform/windows.rs` (new), `crates/engine/src/ocr/mod.rs`, `crates/engine/Cargo.toml` (approved target-only bindings), root `Cargo.toml` (approved dependency versions only), `Cargo.lock` (approved resolution only), `crates/engine/tests/ocr_native.rs` (new).
|
||||
FILES TO OPEN: approved backend-decision.md; spec D1/D3; task 3 worker interface.
|
||||
GOAL: safe Rust API adapter, runtime installed language/size discovery, normalized coordinates, image orientation, explicit Windows package capability; offline recognition. Linux unsupported. No workspace crate added; layer table remains unchanged. If bindings require project unsafe or non-Rust helper, stop and ask owner, never bypass lint.
|
||||
TESTS: native offline printed English/Portuguese fixtures, rotated image/no-text image, installed-language absent, API image limit, runtime unavailable/package failure; report true text and boxes with tolerance, no claim of handwriting parity.
|
||||
VERIFY: standing engine on each approved target and native fixture evidence, not fake suite alone.
|
||||
DO NOT: add unapproved dependency/model; weaken Rust-only/unsafe; hidden cloud fallback.
|
||||
## Phase 3 — search and UI
|
||||
- [ ] 5. Attachment postings and invalidation.
|
||||
OWNS: `crates/index/src/search/attachment.rs` (new, FIRST), `crates/index/src/search/mod.rs`, `crates/index/src/search/doc.rs`, `crates/index/src/search/options.rs`, `crates/index/src/search/rank.rs`, `crates/index/src/lib.rs`, `crates/engine/src/ocr/mod.rs`, `crates/engine/src/vault/find/content_search.rs`, `crates/index/tests/ocr_search.rs` (new), `crates/engine/tests/ocr_search.rs` (new).
|
||||
FILES TO OPEN: spec D2; `crates/index/src/search/mod.rs`; `crates/index/src/search/doc.rs`; later `crates/index/src/search/options.rs` and engine search response before edit.
|
||||
GOAL: typed attachment insert/remove and hit location/provenance; reuse tokenizer/ranker; integrate validated completion snapshots; preserve existing note hit wire shape. Invalidate on hash/remove and exclude encrypted/locked domains before index insertion or reads; if encryption not implemented, no claim it is supported.
|
||||
TESTS: note and attachment matches together; no fake Markdown line offset; page/rect stable; cache reload; delete/hash change removes terms; late cancelled completion cannot reinsert; existing search tests unchanged.
|
||||
VERIFY: standing index/engine packages, both named OCR search integration tests.
|
||||
DO NOT: write extracted text into note files, return OCR text from locked domains, duplicate ranker.
|
||||
- [ ] 6. Attachment actions, settings and search navigation.
|
||||
OWNS: `crates/ui-egui/src/views/media_view/ocr.rs` (new, FIRST), `crates/ui-egui/src/views/media_view/mod.rs`, `crates/ui-egui/src/views/media_view/pdf.rs`, `crates/ui-egui/src/views/image_view/mod.rs`, `crates/ui-egui/src/panels/search_panel.rs`, `crates/ui-egui/src/shell/app/window/views.rs`, `crates/ui-egui/src/text/views/media_view/mod.rs`, `crates/i18n/locales/en.toml`, `crates/ui-egui/tests/ocr.rs` (new), `crates/engine/src/prefs/ocr_config.rs` (new), `crates/engine/src/prefs/mod.rs`, `crates/engine/src/prefs/app_settings.rs`, `crates/engine/src/prefs/settings.rs`, `crates/ui-egui/src/views/settings_view/categories/core_cards.rs`, `crates/engine/src/ocr/mod.rs`, `crates/engine/src/vault/attachments.rs` (successful-import scheduling hook only).
|
||||
FILES TO OPEN: spec D3; `crates/ui-egui/src/panels/search_panel.rs`; `crates/ui-egui/src/views/media_view/pdf.rs`; later read `crates/engine/src/prefs/maps_config.rs`, owned settings/app_settings/module files and `crates/ui-egui/src/views/settings_view/categories/core_cards.rs` before editing settings wiring.
|
||||
GOAL: explicit recognize/cancel/copy actions, opt-in background setting, progress and unsupported/language errors; hit opens image/PDF page and overlays boxes. Persist `ocr.enabled`/background preference as top-level `AppSettings.ocr` via owned app_settings/settings/module files using existing maps config pattern. No default background work before consent.
|
||||
TESTS: kittest action sends once; disabled unsupported; Copy order; PDF/image hit correct box; progress/cancel; opt-in false schedules no import jobs; source unchanged.
|
||||
VERIFY: standing UI/engine + i18n; read snapshots Mocha/Latte/scale 2.
|
||||
DO NOT: OS recognition during paint, hardcoded strings, broad settings refactor.
|
||||
- [ ] 7. Acceptance and status.
|
||||
OWNS: `specs/plugins-parity/attachment-ocr/acceptance.md` (new, FIRST), `specs/plugins-parity/roadmap.md` (rank37 status), `ROADMAP.md`, `docs/ARCHITECTURE.md`, `docs/CODE-MAP.md` (OCR API/status only).
|
||||
FILES TO OPEN: spec acceptance/questions; native and fake test evidence; relevant existing docs sections.
|
||||
GOAL: map every criterion to evidence, explicitly report unsupported Linux/packaging/languages and outstanding approvals.
|
||||
TESTS: all six criteria traced; distinguish native acceptance from fake contract success.
|
||||
VERIFY: final standing gates. Do not execute this future task during documentation-only request, which permits writes only under `specs/plugins-parity/`.
|
||||
DO NOT: mark feature complete before native platform fixtures/UI checks; broaden launch accuracy guarantees.
|
||||
@@ -0,0 +1,30 @@
|
||||
# TASK pdf-annotation-model: PDF annotation geometry, sidecar and fragment model
|
||||
Runs through OpenCode (`specs/agents/opencode-runbook.md` is attached automatically). Task 1 of `specs/plugins-parity/pdf-annotations/tasks.md`.
|
||||
|
||||
## OWNS
|
||||
- `crates/pdf/src/annotation.rs` (new, FIRST), `crates/pdf/src/annotation/fragment.rs` (new)
|
||||
- `crates/pdf/src/lib.rs` (declare/re-export), `crates/pdf/src/error.rs` (specific validation variant only)
|
||||
|
||||
## FILES TO OPEN
|
||||
- `specs/plugins-parity/pdf-annotations/spec.md` D1–D2
|
||||
- `crates/pdf/src/lib.rs`
|
||||
- `crates/pdf/src/error.rs`
|
||||
|
||||
## GOAL / API
|
||||
Write `crates/pdf/src/annotation.rs` in SECOND tool call: compiling type/signature stub and one failing validation test, then make pass. First call reads at most three listed files.
|
||||
|
||||
Pure no-IO `Rect::validate`, `Annotations::parse/to_json`, `PdfTarget::parse/to_fragment`; normalized top-left rotated-crop-box rectangles, page 1..10,000. Sidecar version 1 and exact spec D1 fields; duplicate IDs/version/size rejected. First fragment supports `page`, optional stable `annotation` or normalized `rect` per D2; preserve page-only links. Existing serde/serde_json verified in PDF manifest; no new dependency. Declare module so tests run. No arbitrary-PDF reader implied by existing exporter crate.
|
||||
|
||||
## TESTS
|
||||
Round trip, NaN/infinity, zero area/outside unit square, page 0/10001, duplicate ID, all D1 caps, unknown version, malformed/duplicate fragment fields, page-only link, Unicode target, rotated geometry fixture.
|
||||
|
||||
## VERIFY
|
||||
```sh
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
cargo test -p notescraft-pdf annotation
|
||||
cargo clippy -p notescraft-pdf --all-targets -- -D warnings
|
||||
rustfmt --edition 2024 crates/pdf/src/annotation.rs crates/pdf/src/annotation/fragment.rs crates/pdf/src/lib.rs crates/pdf/src/error.rs
|
||||
```
|
||||
|
||||
## DO NOT
|
||||
IO, engine registration, UI, Cargo/dependencies/new crate, PDF export refactor, unsafe/panics/unchecked indexing; git state changes.
|
||||
@@ -0,0 +1,30 @@
|
||||
# TASK web-capture-envelope: web-clipper first-task model
|
||||
Runs through OpenCode (`specs/agents/opencode-runbook.md` attached automatically). Task 1 of `specs/plugins-parity/web-clipper/tasks.md`.
|
||||
|
||||
## OWNS
|
||||
- `crates/engine/src/capture/web_model.rs` (new, first write), unit tests inside it.
|
||||
- `crates/engine/src/capture/mod.rs` (one module declaration only).
|
||||
|
||||
## FILES TO OPEN
|
||||
- `specs/plugins-parity/web-clipper/spec.md` D1.
|
||||
- `crates/engine/src/capture/rules.rs` and `crates/engine/src/capture/mod.rs`.
|
||||
|
||||
## GOAL / API
|
||||
Write `crates/engine/src/capture/web_model.rs` in your SECOND tool call: compiling stub plus ONE failing test first. Batch at most three relevant reads in first call, then make test pass. Module declarations listed in OWNS make unit tests discoverable.
|
||||
|
||||
Expose crate-private `WebCapture::parse(&str) -> Result<Self, CommandError>` with metadata, Target, Asset and named caps exactly from spec D1. Envelope v1 accepts Markdown, not arbitrary HTML. Source http(s) without credentials, safe target, title/control validation; base64 decode bounded and PNG/JPEG signatures checked. Eight assets maximum, <=256 KiB each, <=512 KiB aggregate, text <=256 KiB and wire <=1 MiB. Reject duplicate/missing asset ids. Recognize placeholders only in Markdown image destination spans; retain prose/code literal asset text. Reuse existing base64 decoder and Markdown parser; no dependencies. All invalid values return structured errors, never clamp/truncate.
|
||||
|
||||
## TESTS
|
||||
Minimal roundtrip; unknown version/field; malformed JSON/types; boundary/+1 for each cap; UTF-8 bytes; credentials/control characters; reserved/traversal target; malformed base64/signature/type; duplicate ids/missing placeholder; asset text inside fenced/inline code unchanged; whitespace-only invalid; valid PNG/JPEG accepted.
|
||||
|
||||
## VERIFY
|
||||
```sh
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
cargo test -p notescraft-engine capture::web_model
|
||||
cargo clippy -p notescraft-engine --all-targets -- -D warnings
|
||||
rustfmt --edition 2024 crates/engine/src/capture/web_model.rs
|
||||
```
|
||||
Format other owned Rust files individually too; no cargo fmt --all. Verification commands apply when implementing brief; current request is docs only.
|
||||
|
||||
## DO NOT
|
||||
Register commands, do IO, open sockets, fetch URLs, convert HTML, touch Cargo or add dependencies. No UI. Owner questions Q1-Q5 remain in spec; pure model needs no dependency decision. Do not edit outside OWNS, change Git state, commit, use unsafe, panic, unwrap/expect or input-derived indexing. No shared craft-libs API changes.
|
||||
@@ -0,0 +1,24 @@
|
||||
# TASK encryption-threat-model: per-note and secret-folder design only
|
||||
Runs through OpenCode (`specs/agents/opencode-runbook.md` is attached automatically). Task 1 of `specs/plugins-parity/note-encryption/tasks.md`.
|
||||
|
||||
## OWNS
|
||||
- `specs/plugins-parity/note-encryption/threat-model.md` (new, FIRST; only writable file)
|
||||
|
||||
## FILES TO OPEN
|
||||
- `specs/plugins-parity/note-encryption/spec.md` D1
|
||||
- `crates/engine/src/vault/session.rs`
|
||||
- `crates/vault/src/lib.rs`
|
||||
|
||||
## GOAL / API
|
||||
Write `specs/plugins-parity/note-encryption/threat-model.md` in SECOND tool call with initial acceptance checklist; first call reads at most three listed inputs. Document-only task: no code stub or running crypto.
|
||||
|
||||
Produce asset/adversary/trust-boundary matrix for per-note `.md.enc` and proposed opaque secret-folder domain. Distinguish offline locked-vault theft, sync storage, unlocked/compromised host, rollback/deletion, path/size/timestamp leakage, shared attachments and historical plaintext. Separate local at-rest lock from sync E2EE. Trace protected/excluded/blocked states to source boundary. Q1/Q4 pending owner decisions remain explicit; no implementation authorized.
|
||||
|
||||
## TESTS
|
||||
Tabletop cases: stolen locked vault, replay old ciphertext, swap encrypted objects, hostile same-user process while unlocked, outside plaintext attachment, known filename and old backup. Every case maps to protection/exclusion/blocker with reason; no false secure-erasure/zero-knowledge claim.
|
||||
|
||||
## VERIFY
|
||||
Use rg --files to confirm inspected source paths; read resulting document against spec D1 and acceptance 1/4. Confirm only owned document changed. No cargo or crypto execution for design task.
|
||||
|
||||
## DO NOT
|
||||
Code/Cargo/git changes, dependencies, crypto implementation, secure-erasure promise, choose algorithm on owner's behalf, mark owner approval granted.
|
||||
@@ -0,0 +1,28 @@
|
||||
# TASK object-type-model: Schema definitions, inheritance and validation
|
||||
Runs through OpenCode (`specs/agents/opencode-runbook.md` attached automatically). Task 1 of `specs/plugins-parity/object-types/tasks.md`.
|
||||
|
||||
## OWNS
|
||||
`crates/bases/src/schema/model.rs` (new, first write), `crates/bases/src/schema/mod.rs` (new), `crates/bases/src/schema/resolve.rs` (new), `crates/bases/src/lib.rs` (schema export), `crates/bases/src/error.rs` (schema errors), unit tests in new modules
|
||||
|
||||
## FILES TO OPEN
|
||||
`spec.md` D1, `crates/bases/src/value.rs`, `crates/bases/src/model.rs`, `crates/bases/src/lib.rs`, `crates/bases/src/error.rs`, `crates/markdown/src/frontmatter/frontmatter_edit/value.rs`, `crates/engine/src/vault/properties.rs`
|
||||
|
||||
## GOAL / API
|
||||
Write FIRST OWNS file in SECOND tool call (compiling stub plus one failing behavior test first).
|
||||
Pure `TypeRegistry::new(definitions) -> Result<Self>`, `resolve(tags) -> Result<ResolvedSchema>`, and `validate(values) -> Vec<Diagnostic>`. JSON codec belongs in engine task 2 using existing serde_json; Bases has no serde dependency. Use same scalar/list semantics as existing property model. Represent D1 schema, bounded inheritance and deterministic multi-tag merge; no new IO.
|
||||
Follow `specs/plugins-parity/object-types/spec.md` limits, error semantics and acceptance criteria exactly. New modules need rustdoc and parent declaration/re-exports; no IO.
|
||||
|
||||
## TESTS
|
||||
Model construction; all model size limits; duplicate ID/tag; missing parent; depth16/17; cycles; inherited required OR; default override; incompatible overrides and multi-tag collisions; bad typed defaults and values; unknown properties preserved.
|
||||
|
||||
## VERIFY
|
||||
```sh
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
cargo test -p notescraft-bases
|
||||
cargo clippy -p notescraft-bases --all-targets -- -D warnings
|
||||
# rustfmt --edition 2024 followed by the owned Rust paths only
|
||||
```
|
||||
Confirm new tests appear in test output. No cargo execution during specification authoring.
|
||||
|
||||
## DO NOT
|
||||
Register commands or touch property mutation behavior. No dependency additions, Git mutations, Cargo edits, UI work or files outside OWNS. Reject hostile input with errors; never clamp/truncate or panic. Root Cargo.toml / xtask/src/layers.rs registration is unnecessary because this task extends existing crate.
|
||||
@@ -0,0 +1,28 @@
|
||||
# TASK synced-block-target: Resolve stable block IDs and replace source content
|
||||
Runs through OpenCode (`specs/agents/opencode-runbook.md` attached automatically). Task 1 of `specs/plugins-parity/synced-blocks/tasks.md`.
|
||||
|
||||
## OWNS
|
||||
`crates/blocks/src/transclusion.rs` (new, first write), `crates/blocks/src/lib.rs` (module, exports and error variants), unit tests in new module
|
||||
|
||||
## FILES TO OPEN
|
||||
`spec.md` D1, `crates/blocks/src/ids.rs`, `crates/blocks/src/parse.rs`, `crates/markdown/src/scanning/scan.rs`, `crates/blocks/src/lib.rs`, `crates/ui-egui/src/views/note_view/blocks/embed_data.rs`
|
||||
|
||||
## GOAL / API
|
||||
Write FIRST OWNS file in SECOND tool call (compiling stub plus one failing behavior test first).
|
||||
`resolve_by_id(source,id) -> Result<BlockTarget>` with block/content/marker ranges; `replace_by_id(source,id,expected_block,replacement) -> Result<Edited>`. Parse real markdown IDs and blocks, preserve ID/structure/CRLF, reject ambiguity and changed target, enforce 1 MiB replacement cap.
|
||||
Follow `specs/plugins-parity/synced-blocks/spec.md` limits, error semantics and acceptance criteria exactly. New modules need rustdoc and parent declaration/re-exports; no IO.
|
||||
|
||||
## TESTS
|
||||
Paragraph Unicode/CRLF; list/quote and standalone table ID; ID-like code not target; duplicate/missing ID; unrelated offset shift; stale expected block; marker injection; unsupported structural replacement; over-cap refusal; unrelated bytes unchanged.
|
||||
|
||||
## VERIFY
|
||||
```sh
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
cargo test -p notescraft-blocks
|
||||
cargo clippy -p notescraft-blocks --all-targets -- -D warnings
|
||||
# rustfmt --edition 2024 followed by the owned Rust paths only
|
||||
```
|
||||
Confirm new tests appear in test output. No cargo execution during specification authoring.
|
||||
|
||||
## DO NOT
|
||||
Do IO, register commands or implement own wikilink path resolver. No dependency additions, Git mutations, Cargo edits, UI work or files outside OWNS. Reject hostile input with errors; never clamp/truncate or panic. Root Cargo.toml / xtask/src/layers.rs registration is unnecessary because this task extends existing crate.
|
||||
@@ -0,0 +1,30 @@
|
||||
# TASK ocr-contract: bounded backend-neutral attachment OCR result model
|
||||
Runs through OpenCode (`specs/agents/opencode-runbook.md` is attached automatically). Task 1 of `specs/plugins-parity/attachment-ocr/tasks.md`.
|
||||
|
||||
## OWNS
|
||||
- `crates/index/src/ocr.rs` (new, FIRST)
|
||||
- `crates/index/src/lib.rs` (declare/re-export), `crates/index/src/error.rs` (validation error only)
|
||||
|
||||
## FILES TO OPEN
|
||||
- `specs/plugins-parity/attachment-ocr/spec.md` D1
|
||||
- `crates/index/src/lib.rs`
|
||||
- `crates/index/src/error.rs`
|
||||
|
||||
## GOAL / API
|
||||
Write `crates/index/src/ocr.rs` in SECOND tool call: compiling model stub plus one failing rectangle-validation test, then make pass. First call reads at most three listed inputs.
|
||||
|
||||
Define `OcrDocument`, `OcrPage`, `OcrLine`, `OcrWord`, `OcrRect`, `OcrCapabilities` with named spec caps and `validate()`; normalized top-left geometry, preserved reading order, explicit unsupported state and optional backend confidence. Pure no-IO model; no serde dependency in index manifest, so JSON persistence belongs engine. No fake Markdown line offsets. OS-first decision does not authorize native bindings: tasks 2/4 require owner response; fake contract testing can proceed.
|
||||
|
||||
## TESTS
|
||||
Valid/empty result, orientation/page geometry, NaN/infinity/outside/zero rect, page 0, duplicate page, 501 pages, 2 MiB/page and 16 MiB/job text overflow, word cap, optional confidence and unsupported language/capability shape. Declare module so unit tests execute.
|
||||
|
||||
## VERIFY
|
||||
```sh
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
cargo test -p notescraft-index ocr
|
||||
cargo clippy -p notescraft-index --all-targets -- -D warnings
|
||||
rustfmt --edition 2024 crates/index/src/ocr.rs crates/index/src/lib.rs crates/index/src/error.rs
|
||||
```
|
||||
|
||||
## DO NOT
|
||||
IO, native API/FFI, unsafe, dependency/Cargo/new crate, bundled models, OCR implementation, index postings, UI, git state changes.
|
||||
@@ -0,0 +1,32 @@
|
||||
# TASK quick-capture-model: global-quick-capture first-task model
|
||||
Runs through OpenCode (`specs/agents/opencode-runbook.md` attached automatically). Task 1 of `specs/plugins-parity/global-quick-capture/tasks.md`.
|
||||
|
||||
## OWNS
|
||||
- `crates/ui-egui/src/capture/model.rs` (new, first write), unit tests inside it.
|
||||
- `crates/ui-egui/src/capture/mod.rs` (new, module/re-export only).
|
||||
- `crates/ui-egui/src/lib.rs` (declare capture module only).
|
||||
|
||||
## FILES TO OPEN
|
||||
- `specs/plugins-parity/global-quick-capture/spec.md` D1.
|
||||
- `crates/engine/src/capture/mod.rs` (note.capture parameter names/caps).
|
||||
- `crates/ui-egui/src/shell/app/popout.rs` (existing child window pattern; do not edit).
|
||||
|
||||
## GOAL / API
|
||||
Write `crates/ui-egui/src/capture/model.rs` in your SECOND tool call: compiling stub plus ONE failing test first. Batch at most three relevant reads in first call, then make test pass. Module declarations listed in OWNS make unit tests discoverable.
|
||||
|
||||
Implement pure Draft { text, title, target }, Target { Daily, Inbox } and CaptureState { Hidden, Editing, Saving, Failed } with explicit generation counter and matching completion. `submit()` validates then returns one note.capture params object/generation; while Saving it emits nothing. `hide()` preserves draft; `discard()` clears explicitly. Failure retains text and error; matching success clears; stale result cannot clear newer draft. Validate whitespace-only text, 1 MiB UTF-8 and title 200-character caps, reject over-limit rather than truncating. Report unsent draft for later quit confirmation. No native/window dependency needed.
|
||||
|
||||
## TESTS
|
||||
Valid Daily/Inbox params; empty/whitespace; exact 1 MiB/+1 and multibyte bytes; title 200/+1; double-submit one request; failed completion then retry; stale completion preserves newer generation; hide/reopen retains; explicit Discard clears; quit requires confirmation iff unsent draft.
|
||||
|
||||
## VERIFY
|
||||
```sh
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
cargo test -p notescraft-ui-egui capture::model
|
||||
cargo clippy -p notescraft-ui-egui --all-targets -- -D warnings
|
||||
rustfmt --edition 2024 crates/ui-egui/src/capture/model.rs
|
||||
```
|
||||
Format other owned Rust files individually too; no cargo fmt --all. Verification commands apply when implementing brief; current request is docs only.
|
||||
|
||||
## DO NOT
|
||||
Create viewport, register OS shortcuts or commands, do IO, read clipboard, persist draft, touch Cargo, add dependency. Native global-hotkey and Wayland binding approvals are later gates, not prerequisites for pure model. Do not edit outside OWNS, change Git state, commit, use unsafe, panic, unwrap/expect or input-derived indexing. No shared craft-libs API changes.
|
||||
@@ -0,0 +1,29 @@
|
||||
# TASK citation-model: normalized bibliography model and CSL JSON (Citations)
|
||||
Runs through OpenCode (`specs/agents/opencode-runbook.md` is attached automatically). Task 1 of `specs/plugins-parity/citations/tasks.md` (rank 39).
|
||||
|
||||
## OWNS
|
||||
- `crates/engine/src/citations/model.rs` (new, FIRST; unit tests inside).
|
||||
- `crates/engine/src/citations/mod.rs` (new, `//!`, module declaration and internal exports), one `mod citations;` in `crates/engine/src/lib.rs`.
|
||||
- `crates/engine/src/error.rs` only a narrow bibliography variant if no fitting error exists.
|
||||
|
||||
## FILES TO OPEN
|
||||
- `specs/plugins-parity/citations/spec.md` D1 and tasks.md task 1.
|
||||
- `crates/engine/src/lib.rs`, `crates/engine/src/error.rs`, `crates/engine/src/activity/outbox.rs`.
|
||||
|
||||
## GOAL / API
|
||||
Write `crates/engine/src/citations/model.rs` in SECOND tool call (compiling stub plus one failing test, then pass).
|
||||
Pure `Entry`, `Creator`, `Library`; `parse_csl_json(&str) -> Result<Library>` and `validate_key(&str) -> Result<()>`. No IO or CSL formatting. CSL JSON `id` becomes stable key; preserve family/given or literal authors and issued date precision. Missing optional metadata remains absent, never invented. Bounds and key subset exactly D1: 8 MiB, 20,000 entries, 256-byte keys, 64 KiB fields, 32 levels, 100 creators. Detect duplicate keys before returning any library; invalid input returns Err. No new crate or dependency.
|
||||
|
||||
## TESTS
|
||||
Equivalent entries, Unicode/literal creators, partial dates, optional fields, invalid JSON/duplicate keys, key subset and boundary cases; every D1 cap including nesting. Module declared so tests execute.
|
||||
|
||||
## VERIFY
|
||||
```
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
cargo test -p notescraft-engine citations
|
||||
cargo clippy -p notescraft-engine --all-targets -- -D warnings
|
||||
rustfmt --edition 2024 crates/engine/src/citations/model.rs crates/engine/src/citations/mod.rs
|
||||
```
|
||||
|
||||
## DO NOT
|
||||
Edit Cargo files, add dependency, read/write bibliography files, register commands, implement BibTeX/CSL formatting, access Zotero/network, touch UI. Q2 parser decision belongs to later adapter scope.
|
||||
@@ -0,0 +1,28 @@
|
||||
# TASK table-formula-model: Bounded formula grammar and cell dependency evaluation
|
||||
Runs through OpenCode (`specs/agents/opencode-runbook.md` attached automatically). Task 1 of `specs/plugins-parity/table-formulas/tasks.md`.
|
||||
|
||||
## OWNS
|
||||
`crates/blocks/src/table_formulas/mod.rs` (new, first write), `crates/blocks/src/table_formulas/parse.rs` (new), `crates/blocks/src/table_formulas/eval.rs` (new), `crates/blocks/src/lib.rs` (module/export and errors), unit tests in new modules
|
||||
|
||||
## FILES TO OPEN
|
||||
`spec.md` D1/D2, `crates/blocks/src/lib.rs`, `crates/bases/src/eval/ops.rs` (finite number handling reference only), `crates/ui-egui/src/views/note_view/blocks/table.rs`
|
||||
|
||||
## GOAL / API
|
||||
Write FIRST OWNS file in SECOND tool call (compiling stub plus one failing behavior test first).
|
||||
`evaluate(cells: &[Vec<String>]) -> Result<Evaluation>` with stable cell diagnostics; parse formulas into bounded AST, compute dependency DAG iteratively, no scripts or IO. Implement D1 coordinate/numeric/aggregate semantics and every D2 resource cap.
|
||||
Follow `specs/plugins-parity/table-formulas/spec.md` limits, error semantics and acceptance criteria exactly. New modules need rustdoc and parent declaration/re-exports; no IO.
|
||||
|
||||
## TESTS
|
||||
Arithmetic precedence/unary; A1 body coords; ranges; each aggregate including empty/text; cycles and propagation; nonfinite/divzero/out-of-bounds; depth/formula/cell/edge/operation caps; deterministic results; raw cell strings unchanged.
|
||||
|
||||
## VERIFY
|
||||
```sh
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
cargo test -p notescraft-blocks
|
||||
cargo clippy -p notescraft-blocks --all-targets -- -D warnings
|
||||
# rustfmt --edition 2024 followed by the owned Rust paths only
|
||||
```
|
||||
Confirm new tests appear in test output. No cargo execution during specification authoring.
|
||||
|
||||
## DO NOT
|
||||
Add dependency or reuse Bases expression grammar as if A1 were note property syntax. No dependency additions, Git mutations, Cargo edits, UI work or files outside OWNS. Reject hostile input with errors; never clamp/truncate or panic. Root Cargo.toml / xtask/src/layers.rs registration is unnecessary because this task extends existing crate.
|
||||
@@ -0,0 +1,35 @@
|
||||
# TASK slide-model: lossless Markdown slide ranges (Markdown slides)
|
||||
Runs through OpenCode (`specs/agents/opencode-runbook.md` is attached automatically). Task 1 of `specs/plugins-parity/markdown-slides/tasks.md` (rank 41, supersedes 19).
|
||||
|
||||
## OWNS
|
||||
- `crates/markdown/src/slides.rs` (new, FIRST; unit tests inside).
|
||||
- `crates/markdown/src/lib.rs` (module and public exports).
|
||||
- `crates/markdown/src/error.rs` only a narrow deck-size error if needed.
|
||||
|
||||
## FILES TO OPEN
|
||||
- `specs/plugins-parity/markdown-slides/spec.md` D1 and tasks.md task 1.
|
||||
- `crates/markdown/src/lib.rs`, `crates/markdown/src/frontmatter/mod.rs`, `crates/markdown/src/scanning/fences.rs`, `crates/markdown/src/scanning/common.rs`.
|
||||
- `crates/markdown/Cargo.toml` read only: existing pulldown-cmark parser; no dependency needed.
|
||||
|
||||
## GOAL / API
|
||||
Write `crates/markdown/src/slides.rs` in SECOND tool call (compiling stub plus one failing test first).
|
||||
```rust
|
||||
pub struct Slide { pub range: std::ops::Range<usize> }
|
||||
pub struct Deck { pub slides: Vec<Slide> }
|
||||
pub fn parse_deck(source: &str) -> Result<Deck>;
|
||||
```
|
||||
Use existing offset parser, frontmatter boundary and protected spans. Only root-level CommonMark thematic-break event for exact unindented `---`, with body boundary/blank lines around it, becomes separator. Preserve explicit empty slides and original UTF-8 byte ranges; remove frontmatter/separators from slide ranges without rewriting source. Fenced code, quote/list content, setext underline, comments, math and HTML protected regions do not split. Limit source 2 MiB and deck 500 slides; reject excess, never truncate. Empty/frontmatter-only note yields one empty slide. No IO/new crate/dependency.
|
||||
|
||||
## TESTS
|
||||
D1 fixtures: frontmatter, setext, code/comments/math/HTML, quote/list, alternate rules, unclosed fence/comment, CRLF, Unicode, empty/body-only/frontmatter-only, leading/trailing/repeated valid boundaries, missing final newline, exact/over byte and slide caps. Assert every `source.get(range)` succeeds and concatenation/ranges retain expected bytes; original source unchanged.
|
||||
|
||||
## VERIFY
|
||||
```
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
cargo test -p notescraft-markdown slides
|
||||
cargo clippy -p notescraft-markdown --all-targets -- -D warnings
|
||||
rustfmt --edition 2024 crates/markdown/src/slides.rs crates/markdown/src/lib.rs crates/markdown/src/error.rs
|
||||
```
|
||||
|
||||
## DO NOT
|
||||
Edit Cargo files, add dependency, read/write files, touch engine/UI, split raw source on `---`, silently drop empty slides, implement rank19 independently. User-facing behavior belongs to subsequent engine commands.
|
||||
@@ -0,0 +1,34 @@
|
||||
# TASK font-dedupe: embed each font once (Registry spec, task 0.2)
|
||||
Runs through OpenCode (`specs/agents/opencode-runbook.md` is attached automatically). Task 0.2 of `specs/registry/tasks.md`. Measured in `specs/registry/measurements.md` 1.2: `Inter-Regular.ttf` and `Inter-SemiBold.ttf` are in the release binary twice (once in `engine` PDF export, once in `ui-egui`), 0.79 MiB wasted; PDF export also embeds `JetBrainsMono-Regular.ttf` (264 KiB) that nothing else uses.
|
||||
|
||||
## OWNS
|
||||
- `crates/engine/src/builtin/export_commands.rs` (make the three font constants public and switch the mono one).
|
||||
- `crates/engine/src/lib.rs` (ONE line: `pub use builtin::export_commands::{INTER_REGULAR, INTER_SEMIBOLD, JETBRAINS_MONO};`; other people have uncommitted edits in this file, touch nothing else).
|
||||
- `crates/ui-egui/src/design/fonts/mod.rs` (the two Inter constants and the mono constant now come from the engine; keep `INTER_MEDIUM` and `STIX_MATH` as they are).
|
||||
- `crates/ui-egui/src/design/fonts/tests.rs` only to add the test below.
|
||||
|
||||
## FILES TO OPEN
|
||||
- `crates/engine/src/builtin/export_commands.rs` lines 30-45 (the constants and `fonts()`).
|
||||
- `crates/ui-egui/src/design/fonts/mod.rs` lines 1-30 and `install`.
|
||||
- `crates/engine/src/builtin/mod.rs` (`pub(crate) mod export_commands`, so the re-export in `lib.rs` is the only public path).
|
||||
|
||||
## GOAL / API
|
||||
In `export_commands.rs`: `pub const INTER_REGULAR: &[u8]`, `pub const INTER_SEMIBOLD: &[u8]` (same files as today) and `pub const JETBRAINS_MONO: &[u8] = include_bytes!("../../../../assets/fonts/JetBrainsMono-NoLigatures.ttf");` (the UI font; PDF text has no ligature shaping, widths come from the file, so the PDF stays valid). Rustdoc on each (keep the existing sentences). `lib.rs` re-exports them. In `ui-egui` fonts `mod.rs`: delete the two Inter `include_bytes!` and the mono one, `use notescraft_engine::{INTER_REGULAR, INTER_SEMIBOLD, JETBRAINS_MONO};`. Do NOT delete any file under `assets/fonts/` (the `pdf` crate's tests still read `JetBrainsMono-Regular.ttf`).
|
||||
|
||||
Expected result: the release binary shrinks by about 1.05 MiB (0.79 + 0.26).
|
||||
|
||||
## TESTS
|
||||
Add in `ui-egui/src/design/fonts/tests.rs`: `the_ui_and_the_exporter_share_one_inter_copy` asserting `std::ptr::eq(notescraft_engine::INTER_REGULAR.as_ptr(), <the slice the UI installs>.as_ptr())` (expose the slice through a `#[cfg(test)] pub(super) fn inter_regular() -> &'static [u8]` or compare lengths and first 64 bytes if pointer access is awkward). Existing PDF export tests must still pass: if one compares exact PDF bytes or glyph widths of the mono font and fails, STOP and report the test name and the diff; do not edit the test.
|
||||
|
||||
## VERIFY
|
||||
```
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
cargo test -p notescraft-engine export
|
||||
cargo test -p notescraft-ui-egui fonts
|
||||
cargo clippy -p notescraft-engine -p notescraft-ui-egui --all-targets -- -D warnings
|
||||
rustfmt --edition 2024 crates/engine/src/builtin/export_commands.rs crates/ui-egui/src/design/fonts/mod.rs crates/ui-egui/src/design/fonts/tests.rs
|
||||
```
|
||||
Optional if time: `CARGO_TARGET_DIR=target/agent-fonts cargo build --release -p notescraft` and report `ls -l` against 40,235,208 bytes.
|
||||
|
||||
## DO NOT
|
||||
Edit Cargo files; change `INTER_MEDIUM`, `STIX_MATH`, any font size, or the `eframe` features (task 0.3 does that); delete asset files; add a `fonts` module or crate; touch anything else in `engine/src/lib.rs`.
|
||||
@@ -0,0 +1,46 @@
|
||||
# TASK registry-index-model: add-on index schema v2, validation and version compare (Registry spec, task 1)
|
||||
Runs through OpenCode (`specs/agents/opencode-runbook.md` is attached automatically). Task 1 of `specs/registry/tasks.md`. Pure: no IO, no network, no signature check (task 3).
|
||||
|
||||
## OWNS
|
||||
- `crates/engine/src/addons/model.rs` (new, FIRST; unit tests inside `#[cfg(test)]`).
|
||||
- `crates/engine/src/addons/mod.rs` (new, `//!` doc, `mod model; pub(crate) use model::{...};`), one `mod addons;` in `crates/engine/src/lib.rs` (other people have uncommitted edits in that file: add only that line, next to the other `mod` lines).
|
||||
- `crates/engine/src/error.rs`: add ONE variant `Addon(String)` only if no fitting one exists.
|
||||
The module is NOT named `registry`: `crates/engine/src/registry` is the command registry.
|
||||
|
||||
## FILES TO OPEN
|
||||
- `specs/registry/spec.md` D2 (schema, caps) and D8 (kinds).
|
||||
- `crates/plugins/src/manifest.rs` (`Permission::as_str`, the permission strings and the id rules to copy).
|
||||
- `crates/engine/src/activity/outbox.rs` (versioned JSON file pattern), `crates/engine/src/lib.rs`, `crates/engine/src/error.rs`.
|
||||
|
||||
## GOAL / API
|
||||
Write `crates/engine/src/addons/model.rs` in your SECOND tool call (stub + one failing test first).
|
||||
```rust
|
||||
pub enum Kind { Plugin, Theme, Pack, Grammar, Font, Snippet }
|
||||
pub struct Version { pub major: u16, pub minor: u16, pub patch: u16 } // each 0..=9999, derive Ord
|
||||
pub struct File { pub path: String, pub size: u64, pub sha256: String, pub url: String }
|
||||
pub struct Signature { pub key_id: String, pub sig: String }
|
||||
pub struct Entry { pub id: String, pub kind: Kind, pub title: String, pub summary: String, pub version: Version,
|
||||
pub min_app_version: Version, pub license: String, pub source: Option<String>, pub aliases: Vec<String>,
|
||||
pub suggested: bool, pub permissions: Vec<String>, pub files: Vec<File>, pub signature: Signature }
|
||||
pub struct Revoked { pub id: String, pub version: Version }
|
||||
pub struct Index { pub generated_at: String, pub entries: Vec<Entry>, pub revoked: Vec<Revoked>, pub skipped: Vec<String> }
|
||||
impl Index { pub fn parse(json: &[u8]) -> Result<Index>; pub fn find(&self, kind: Kind, id: &str) -> Option<&Entry>; pub fn is_revoked(&self, e: &Entry) -> bool; }
|
||||
impl Version { pub fn parse(s: &str) -> Result<Version>; }
|
||||
impl Entry { pub fn total_size(&self) -> u64; } // saturating sum
|
||||
pub const MAX_INDEX_BYTES: usize = 1024 * 1024;
|
||||
```
|
||||
Rules (spec D2): top-level `"version": 2` else `Err`; input over `MAX_INDEX_BYTES` `Err`; at most 2000 entries (more is `Err`); `id` `[a-z0-9-]{1,48}`, unique per kind (a duplicate skips the later one); `title` 1..=80 chars no control char; `summary` 0..=200; `license` 1..=40; `aliases` at most 16 each `[a-z0-9._+-]{1,32}`; `permissions` only `vault.read`, `vault.write`, `editor` and only for `plugin` (anything else invalidates the entry); `files` 1..=64; `sha256` 64 lowercase hex; `url` starts with `https://`; `files[].size` > 0; per-kind total caps: plugin 4 MiB, theme 64 KiB, grammar 512 KiB, snippet 1 MiB, pack 8 MiB, font 32 MiB. **A broken entry is skipped and its id (or `?` if none) plus the reason is pushed to `skipped`; the rest loads.** Invalid top level or non-JSON returns `Err`. Nothing is clamped or repaired. No `[i]`, no `[a..b]`, no panics; use `serde_json::from_slice` into private raw structs, then validate into the public ones.
|
||||
|
||||
## TESTS
|
||||
Good index round trip of two entries; version 1 and 3 `Err`; 1 MiB + 1 `Err`; 2001 entries `Err`; one bad entry among three (bad id, bad sha, `http://` url, unknown permission on a plugin, permission on a theme, zero-size file, 65 files, kind `exe`) keeps the other two and fills `skipped`; duplicate id same kind skipped, same id different kind kept; `Version::parse` of `1.2.3`, `01.2.3` (reject), `1.2`, `1.2.3-beta`, `10000.0.0`, empty; ordering `1.10.0 > 1.9.9`; `total_size` with sizes near `u64::MAX` saturates; per-kind cap exceeded by one byte; revoked match; corrupt JSON and a depth-bomb `[[[[...` (200 levels) return `Err`.
|
||||
|
||||
## VERIFY
|
||||
```
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
cargo test -p notescraft-engine addons
|
||||
cargo clippy -p notescraft-engine --all-targets -- -D warnings
|
||||
rustfmt --edition 2024 crates/engine/src/addons/mod.rs crates/engine/src/addons/model.rs
|
||||
```
|
||||
|
||||
## DO NOT
|
||||
Edit Cargo files; add a dependency (serde and serde_json are in the engine already); read or write any file; check signatures or hashes against bytes (tasks 3 and 5); register commands; touch the UI; create more than the two new files.
|
||||
@@ -0,0 +1,42 @@
|
||||
# TASK registry-fileset-guard: hostile path, size and URL rules for an add-on file list (Registry spec, task 2)
|
||||
Runs through OpenCode (`specs/agents/opencode-runbook.md` is attached automatically). Task 2 of `specs/registry/tasks.md`. Pure: no IO. Works on plain strings and numbers so it does not depend on task 1 (a later task wires it to `Entry`).
|
||||
|
||||
## OWNS
|
||||
- `crates/engine/src/addons/fileset.rs` (new, FIRST; unit tests inside `#[cfg(test)]`).
|
||||
- `crates/engine/src/addons/mod.rs`: if it does not exist yet (task 1 creates it), create it with `//!` doc and `mod fileset;`; if it exists, add ONLY `mod fileset;` and `pub(crate) use fileset::{...};`.
|
||||
- `crates/engine/src/lib.rs`: add `mod addons;` ONLY if task 1 has not (check first; other people have uncommitted edits there).
|
||||
Not named `registry`: that is the command registry.
|
||||
|
||||
## FILES TO OPEN
|
||||
- `specs/registry/spec.md` D7 (the rules) and D8 (allowed extensions per kind).
|
||||
- `crates/syntax/src/validate.rs` (`check_url`, to call, and its tests for style).
|
||||
- `crates/engine/src/vault/` grep for the existing relative-path guard used for vault writes (copy the style, not the rules).
|
||||
|
||||
## GOAL / API
|
||||
```rust
|
||||
pub enum KindRule { Plugin, Theme, Grammar, Font, Snippet, Pack } // local copy; task 1's Kind maps onto it
|
||||
pub struct Checked { pub components: Vec<String> } // validated path parts
|
||||
pub fn check_path(kind: KindRule, path: &str) -> Result<Checked>;
|
||||
pub fn join_under(root: &std::path::Path, checked: &Checked) -> std::path::PathBuf; // only built from validated parts
|
||||
pub fn check_set(kind: KindRule, files: &[(&str, u64)]) -> Result<u64>; // paths, duplicates (case-insensitive), count, sizes; returns the total
|
||||
pub fn check_download_url(url: &str, allowed_hosts: &[&str]) -> Result<()>;
|
||||
pub const MAX_FILES: usize = 64;
|
||||
pub const MAX_COMPONENTS: usize = 4;
|
||||
pub const MAX_COMPONENT_BYTES: usize = 64;
|
||||
pub fn kind_cap(kind: KindRule) -> u64; // plugin 4 MiB, theme 64 KiB, grammar 512 KiB, snippet 1 MiB, pack 8 MiB, font 32 MiB
|
||||
```
|
||||
Rules for `check_path`: relative, `/` separated, 1..=4 components, each `[A-Za-z0-9._-]{1,64}`, none empty, `.`, `..` or starting with `.`; no `\`, `:`, NUL, space or non-ASCII (all fall out of the character set); no Windows reserved device names (`CON PRN AUX NUL COM1..COM9 LPT1..LPT9`, case-insensitive, with or without an extension) as a component; the file name's extension must be on the kind's allow-list: Plugin `manifest.json`, `plugin.wasm`, `README.md` only (exact names, at the root); Theme `.toml`; Grammar `.sublime-syntax`; Font `.ttf` `.otf`; Snippet `.md` `.txt` `.toml`; Pack: first component is `grammars`, `themes`, `templates` or `icons` and the extension is `.sublime-syntax`, `.toml`, `.md`, `.json`/`.svg` respectively. `check_set`: at most `MAX_FILES`, no case-insensitive duplicate, no zero size, total with `checked_add` (overflow is `Err`), total at most `kind_cap`, one font file at most 32 MiB. `check_download_url`: delegates to `notescraft_syntax::validate::check_url(url, Some(host))` for each allowed host until one accepts, rejects a `?` or `#`, and rejects over 300 bytes. `join_under` never receives raw server text. Every failure is `Err` with a short reason; nothing is "fixed".
|
||||
|
||||
## TESTS
|
||||
Table-driven. Reject: `../x`, `a/../../b`, `/abs`, `C:\x`, `a\b`, `a//b`, `.hidden`, `a/.b`, `a b`, `é.toml`, `x\0.toml`, `CON.toml`, `nul`, `com1/x.toml`, 65-char component, 5 components, empty string, a plugin file `evil.wasm`, a theme `x.exe`, a pack file `bin/x.toml`, a grammar `x.tmLanguage`. Accept: `twig.sublime-syntax`, `plugin.wasm`, `manifest.json`, pack `grammars/twig.sublime-syntax`, `themes/a.toml`. `check_set`: 65 files, two files differing only in case, a zero size, sizes `u64::MAX` + 1, total one byte over the kind cap, font of 33 MiB. `check_download_url`: `http://`, credentials, port, IP literal, `localhost`, unlisted host, query string, 301 bytes, listed host accepted. `join_under` of a checked path stays inside `root` (assert `starts_with`).
|
||||
|
||||
## VERIFY
|
||||
```
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
cargo test -p notescraft-engine addons
|
||||
cargo clippy -p notescraft-engine --all-targets -- -D warnings
|
||||
rustfmt --edition 2024 crates/engine/src/addons/fileset.rs crates/engine/src/addons/mod.rs
|
||||
```
|
||||
|
||||
## DO NOT
|
||||
Edit Cargo files; add a dependency; read or write any file; use `Path::canonicalize` or follow symlinks (nothing is read); duplicate `check_url` instead of calling it; build a path with `format!` from server text; touch the UI, the grammar code or `Entry` (task 1).
|
||||
@@ -0,0 +1,61 @@
|
||||
# Zotero/BibTeX citations (agy gap 12, roadmap 39)
|
||||
|
||||
Part of `specs/plugins-parity/`. Existing engine, Markdown/editor and native UI; no new crate or third-party dependency in the baseline.
|
||||
|
||||
## Research (2026-10-21; agy claims checked)
|
||||
- Pandoc documents `[@key]`, grouped citations, bibliographies and CSL processing. Syntax and bibliography formatting are separate concerns: recognizing a key does not implement CSL. https://pandoc.org/MANUAL.html#citations and https://www.pandoc.org/demo/example33/8.20-citation-syntax.html
|
||||
- Zotero documents BibTeX and CSL JSON exchange. Better BibTeX is a Zotero add-on; its “Keep updated” export writes a refreshed file. It is not NotesCraft-to-Zotero bidirectional sync. https://www.zotero.org/support/dev/data_formats and https://retorque.re/zotero-better-bibtex/exporting/auto/
|
||||
- Zotero has a supported API, but reading its live SQLite database is not this contract. https://www.zotero.org/support/dev/web_api/v3/
|
||||
- agy's Logseq-native and “Obsidian Citation Suite” comparison was not verified and is dropped. “1-click metadata sync” is narrowed to explicit refresh of a local export; cloud sync and arbitrary CSL formatting are not implied by a citation picker.
|
||||
|
||||
## Problem / goal
|
||||
Academic notes need stable citation keys, searchable bibliographic metadata, insertion without copying titles, and an export path that can produce real citations. Preserve normal Markdown `[@key]` source and users' bibliography files.
|
||||
|
||||
## Non-goals
|
||||
A Zotero database editor, cloud credentials, attachment imports, downloading PDFs/DOIs, native CSL implementation, full BibLaTeX/TeX execution, arbitrary embedded HTML, changing citation keys automatically.
|
||||
|
||||
## Design decisions
|
||||
### D1 Sources and normalized model
|
||||
Explicitly select one vault-relative `.bib` or CSL JSON source per note via frontmatter `bibliography: references.bib` (path relative to the note). Optional `csl: styles/apa.csl` is also vault-relative; no URLs, absolute paths, traversal or reserved paths. V1 supports one source; a list returns `unsupported_bibliography`, never silently chooses its first element. Configuration stays in existing note frontmatter; no private database required. Zotero users export manually or configure Better BibTeX auto-export to that file, then choose Refresh bibliography.
|
||||
|
||||
Normalized `Entry { key, kind, title, authors, issued, doi, url }`; creator names support literal institutional names and family/given components, dates retain supplied precision. Retain unknown source fields for round trips only if a future writer exists: v1 never rewrites source files. CSL JSON `id` is the citation key; duplicate keys reject the whole refresh. The pure model lives in `crates/engine/src/citations/{mod,model}.rs`; loading and command plumbing remain separate. No new crate, hence no Cargo/layer registration.
|
||||
|
||||
Input caps: 8 MiB source, 20,000 entries, 256 bytes/key, 64 KiB/field, nesting 32, 100 authors/entry. Keys start ASCII letter/digit/underscore and continue ASCII letters/digits/underscore/hyphen/colon/period; this intentionally documented subset is narrower than full Pandoc syntax. Reject invalid key, invalid UTF-8, duplicate, malformed record or over-limit input with a structured error. No truncation or guessed repairs.
|
||||
|
||||
BibTeX subset: `@article`, `@book`, `@inproceedings`, `@misc` records with balanced braced/quoted fields, numeric years, nested title braces, `and`-separated authors, `%` comments outside values. Unknown record kinds and fields may retain strings but must not be presented as supported semantic mappings. `@string`, `@preamble`, concatenation, crossref inheritance and unsupported TeX commands return `unsupported_bibtex` with location. A fixture containing an unsupported construct fails the refresh; offer CSL JSON/Pandoc conversion, never silently produce corrupted authors. Full BibTeX coverage is owner Q2, not an excuse to grow an ad hoc TeX interpreter.
|
||||
|
||||
### D2 Commands and buffers
|
||||
| id | params | result | writes |
|
||||
|---|---|---|---|
|
||||
| `citation.refresh` | `{path}` note path | `{source, count, generation}` | memory cache only |
|
||||
| `citation.search` | `{path, query, limit?}` max 100, default 20 | `{items:[{key,title,authors,issued}], total, generation}` | no |
|
||||
| `citation.resolve` | `{path, keys}` max 100 | `{items, missing}` | no |
|
||||
| `citation.insert` | `{path, at, key, expected_source}` | `{path, inserted}` | buffer edit |
|
||||
|
||||
Refresh validates all records before swapping the previous snapshot; failure preserves snapshot, marks it stale and shows the diagnostic. Cache keyed by canonical guarded source path plus source bytes/version, bounded to 8 sources/64 MiB; explicit refresh invalidates it. Search stable order: exact key, key prefix, title/author substring, key lexical tie-break. No engine IO while painting or per citation per frame. Insert uses current open `Buffer::edit`, refuses stale `expected_source`, invalid UTF-8 offset/read-only/unknown key, inserts exactly `[@key]`, and participates in undo. It must not save unrelated edits. Reads work in read-only sessions. Classify refresh/search/resolve as READ, insert as WRITE in assistant gate, and update `CORE_IDS`.
|
||||
|
||||
### D3 UI and compatibility
|
||||
Picker searches key/title/author/year and inserts via `citation.insert`; shortcut is assigned only after hotkey conflict check. Read preview recognizes standalone `[@key]` and grouped `[@one; @two]` outside code, frontmatter, math, links and comments; unsupported locator syntax remains literal source in v1 (Pandoc still handles it on export). Resolved citations show key plus metadata tooltip, missing keys show a localized unresolved indicator and keep their literal text. Never relabel unformatted keys as APA/CSL output. Reuse note-view inline scanner and text tables; no HTML renderer. Refresh button exposes stale diagnostics, including malformed auto-export files; source is never rewritten.
|
||||
|
||||
### D4 CSL export through existing Pandoc adapter
|
||||
`crates/engine/src/pandoc/{commands,run}.rs` already runs an optional installed executable, with argv, timeout, bounded stderr and vault output write. Extend its typed request with explicit `citeproc`, guarded bibliography and optional local CSL paths; use citation-capable Pandoc Markdown reader for this mode (existing `gfm+wikilinks_title_after_pipe` is not a citation guarantee). Restrict this to explicit `export.pandoc` opt-in `citations:true`; ordinary export remains unchanged. Send validated source snapshot and style bytes from controlled temporary inputs rather than allowing note metadata to introduce remote sources or arbitrary paths. Disable raw HTML in the native preview. Missing Pandoc/invalid style/unsupported source produces actionable failure and no final output. Do not install/bundle Pandoc or add a CSL crate. Library formatting is explicitly unavailable without this opt-in backend.
|
||||
|
||||
## Acceptance criteria
|
||||
1. CSL JSON and supported `.bib` fixtures resolve same keys/title/authors/year, including Unicode and institutional names; duplicates and unsupported BibTeX fail without replacing last good snapshot.
|
||||
2. Picker insertion changes only intended buffer range, preserves surrounding UTF-8, undo restores bytes, read-only/stale source refuses insertion.
|
||||
3. Code, comments and literal emails remain unchanged; known and unknown citations preserve source on load/save and grouped keys resolve predictably.
|
||||
4. Refresh reads user export, never writes it; malformed mid-export file yields stale diagnostic and successful later refresh recovers.
|
||||
5. Stub Pandoc verifies argv contains explicit citation mode, guarded files and no shell invocation; timeout/missing tool/path traversal leaves no final export. Optional real-tool fixture produces bibliography from two cited works.
|
||||
6. Warm search over 20,000 entries stays under 50 ms in an ignored release perf fixture; no reads or calls per inline citation during paint.
|
||||
7. Localized picker, keyboard operation, missing-key tooltip and Mocha/Latte screenshots pass; ordinary notes/export regressions remain green.
|
||||
|
||||
## Open owner questions
|
||||
- Q1: approve optional installed Pandoc as CSL backend for citation exports? Proposal: reuse existing optional executable, no new bundled code. Native CSL preview remains deferred.
|
||||
- Q2: is documented BibTeX subset plus full CSL JSON acceptable for v1, or approve a maintained Rust BibTeX/BibLaTeX parser (name/version/license and reason required before adding dependency)? No dependency currently authorized.
|
||||
- Q3: should Zotero local-API integration follow export-file refresh later? Proposal: export files first; no credentials or network in v1.
|
||||
|
||||
## Feature ideas (researched, ranked)
|
||||
1. Better BibTeX auto-export guidance and change detection after reliable refresh ships: https://retorque.re/zotero-better-bibtex/exporting/auto/
|
||||
2. Complete locator/prefix/suffix groups and author-in-text citations using Pandoc grammar: https://www.pandoc.org/demo/example33/8.20-citation-syntax.html
|
||||
3. Native CSL preview after approving a maintained processor and conformance fixtures: https://www.zotero.org/support/styles
|
||||
4. Read-only Zotero API source with explicit connection and stable key mapping: https://www.zotero.org/support/dev/web_api/v3/
|
||||
@@ -0,0 +1,47 @@
|
||||
# Tasks: Zotero/BibTeX citations (roadmap 39)
|
||||
|
||||
Spec: `spec.md`. One task = one agent. No new crate; no Cargo/layer registration. New dependencies require owner response to Q2.
|
||||
**First tool call rule:** first call batches required reads; write first OWNS file in SECOND tool call, compiling stub plus one failing test, then make it pass. First OWNS is always a source file, not a directory. Module wiring listed explicitly below.
|
||||
**Standing VERIFY:** `export PATH="$HOME/.cargo/bin:$PATH"`; `cargo test -p <owned package>`; `cargo clippy -p <owned package> --all-targets -- -D warnings`; `rustfmt --edition 2024 <owned Rust files>`; UI adds `cargo xtask i18n`; final phase `cargo xtask layers` and `cargo xtask tree`. These are future implementation checks; specification author does not run Cargo.
|
||||
**Standing DO NOT:** edit outside OWNS; git mutations; dependencies/Cargo changes; shared craft API changes; panics, unchecked input slices, blanket allows; per-frame IO; `cargo fmt --all`; rewrite bibliography exports.
|
||||
|
||||
## Phase 1 — pure input model
|
||||
- [ ] 1. CSL JSON normalization and limits.
|
||||
OWNS: `crates/engine/src/citations/model.rs` (new, FIRST), `crates/engine/src/citations/mod.rs` (new), one `mod citations;` declaration in `crates/engine/src/lib.rs`, narrowly scoped error variant in `crates/engine/src/error.rs` if existing error cannot express invalid bibliography.
|
||||
FILES TO OPEN: `spec.md` D1, `crates/engine/src/lib.rs`, `crates/engine/src/error.rs`, `crates/engine/src/activity/outbox.rs` (versioned structured data pattern).
|
||||
GOAL: pure `Entry`, `Creator`, `Library`, `parse_csl_json(&str) -> Result<Library>`, `validate_key(&str) -> Result<()>`; bounded deterministic normalization, preserve literal names/date precision; no source writes.
|
||||
TESTS: two items, Unicode, institutional author, partial date, absent optional fields; invalid JSON, duplicates, invalid/oversize key, 20,001 items, nesting/field/author caps; invalid inputs return Err.
|
||||
VERIFY: standing `-p notescraft-engine` with `citations` filter, confirm tests run through module declaration.
|
||||
DO NOT: IO, commands, BibTeX parser, CSL formatting or dependencies.
|
||||
- [ ] 2. Explicit BibTeX subset adapter.
|
||||
OWNS: `crates/engine/src/citations/bibtex.rs` (new, FIRST), `crates/engine/src/citations/mod.rs` (module declaration), fixtures/tests inside `bibtex.rs`.
|
||||
FILES TO OPEN: D1 and task 1 model, `crates/engine/src/error.rs`.
|
||||
GOAL: `parse_bibtex(&str) -> Result<Library>` maps supported records, balanced nested values and creators, reports unsupported constructs with location. Keep bounded parser under 400 lines/file; no TeX execution.
|
||||
TESTS: article/book/misc/inproceedings, nested braces, escaped quotes, `%` comments, `and` authors, institutional braces, Unicode; duplicate, malformed/unclosed entry, macros/concatenation/crossref/unsupported TeX, depth/byte caps; equivalent CSL fixture normalization.
|
||||
VERIFY: standing `-p notescraft-engine`, filter `citations`.
|
||||
DO NOT: silently discard unsupported semantics, download parser or claim complete BibLaTeX support. If Q2 requires a new parser, stop dependency work until approved.
|
||||
|
||||
## Phase 2 — engine surface
|
||||
- [ ] 3. Refresh/search/resolve/insert with guarded sources.
|
||||
OWNS: `crates/engine/src/citations/commands.rs` (new, FIRST), `crates/engine/src/citations/source.rs` (new), `crates/engine/src/citations/mod.rs`, `crates/engine/src/vault/session.rs` (bounded cache field and constructor initialization), `crates/engine/src/commands.rs` (one register call), `crates/engine/src/ai/assistant_gate.rs`, `crates/engine/tests/core_commands.rs`, `crates/engine/tests/citations.rs` (new).
|
||||
FILES TO OPEN: D2, `crates/engine/src/capture/mod.rs`, `crates/engine/src/vault/buffers.rs`, `crates/engine/tests/common/mod.rs`, `crates/markdown/src/frontmatter/mod.rs`.
|
||||
GOAL: D2 command table, bounded cache and atomic snapshot replacement, note-relative guarded bibliography path, stable search, compare-before-edit insertion into open buffer.
|
||||
TESTS: TempVault sources in two folders, relative path resolution, read-only reads/insert refusal, stale buffer/invalid UTF-8 offset, undo, duplicate source refresh keeps previous snapshot, recovery after failed refresh, traversal/reserved/absolute/URL source rejection; command ids/gate classification; ignored warm-search fixture.
|
||||
VERIFY: standing `-p notescraft-engine`, `--test citations --test core_commands`.
|
||||
DO NOT: read Zotero SQLite, write source, perform network calls, save unrelated dirty edits.
|
||||
|
||||
## Phase 3 — export and native UI
|
||||
- [ ] 4. Explicit citation-enabled Pandoc export (Q1 backend decision first).
|
||||
OWNS: `crates/engine/src/pandoc/run.rs` (FIRST), `crates/engine/src/pandoc/commands.rs`, `crates/engine/src/pandoc/mod.rs`, `crates/engine/tests/pandoc.rs`.
|
||||
FILES TO OPEN: D4, current Pandoc request/reader/timeout and engine source guard from task 3.
|
||||
GOAL: opt-in `citations:true`, citation-capable reader, `--citeproc`, guarded controlled bibliography/style snapshots; default export behavior stays intact. No shell, remote references or arbitrary metadata paths.
|
||||
TESTS: stub executable captures argv/input; citation mode vs default; missing executable, bad style, source path traversal, timeout, malformed source, failed run never creates final output; optional ignored real-Pandoc bibliography assertion.
|
||||
VERIFY: standing `-p notescraft-engine`, `--test pandoc --test citations`.
|
||||
DO NOT: bundle/install Pandoc, add CSL dependency, call this native CSL rendering, enable unvalidated remote resource fetches.
|
||||
- [ ] 5. Citation picker, inline metadata and release checks.
|
||||
OWNS: `crates/ui-egui/src/views/citation_picker/mod.rs` (new, FIRST), `crates/ui-egui/src/views/mod.rs`, `crates/ui-egui/src/views/note_view/inlines/scan.rs`, `crates/ui-egui/src/views/note_view/inlines.rs`, `crates/ui-egui/src/views/note_view/inlines/scan_tests.rs`, `crates/ui-egui/src/views/note_view/painting.rs` (citation span/tooltip arm only), `crates/ui-egui/src/shell/app/window/commands.rs`, `crates/ui-egui/src/shell/app/window/views.rs`, `crates/ui-egui/src/text/views/citations.rs` (new), `crates/ui-egui/src/text/views/mod.rs`, `[ui.citations]` at END of `crates/i18n/locales/en.toml`, `crates/ui-egui/tests/citations.rs` (new), `ROADMAP.md`, `docs/ARCHITECTURE.md`, `docs/CODE-MAP.md`, rank 39 status in `specs/plugins-parity/roadmap.md`.
|
||||
FILES TO OPEN: D3, `crates/ui-egui/src/views/note_view.rs`, existing inline scanner/tests, `crates/ui-egui/src/widgets/palette.rs`, `crates/ui-egui/src/shell/hotkeys/defaults.rs`, `crates/ui-egui/src/views/share_dialog/mod.rs` worker pattern.
|
||||
GOAL: keyboard-search picker and insert through engine, one source snapshot per note/version, known/missing/grouped citation display and refresh diagnostics. Wire actual handlers in owned shell files; no unowned “lead wiring” remains. Update documentation only after actual checks.
|
||||
TESTS: kittest search and select inserts once, cancel no write, undo, missing key and malformed refresh, no code/comment/email recognition, unsupported locators literal, no per-span engine calls.
|
||||
VERIFY: standing engine/UI; i18n, Mocha/Latte snapshot PNGs read, `cargo xtask ci`. Record optional real-Pandoc check skipped if unavailable.
|
||||
DO NOT: invent shortcut conflicts, style key text as a real CSL bibliography, silently alter note source.
|
||||
@@ -0,0 +1,59 @@
|
||||
# Global quick-capture window (agy gap 11, roadmap 38)
|
||||
|
||||
Part of `specs/plugins-parity/`. Reuse `note.capture`, native eframe viewport and desktop host. No new crate. Native shortcut dependency is an explicit owner gate.
|
||||
|
||||
## Research (2026-10-21; agy claims checked)
|
||||
- Apple documents Fn-Q for Quick Note: [Notes keyboard shortcuts](https://support.apple.com/en-kg/guide/notes/apd46c25187e/mac). Confirms global-capture pattern; does not prove Reflect/Raycast parity, which is dropped.
|
||||
- Candidate safe facade [global-hotkey documentation](https://docs.rs/global-hotkey/latest/global_hotkey/) supports Windows/macOS/Linux X11, requires platform event-loop integration, and does **not** support Wayland. Do not copy its unwrap examples or claim every desktop works.
|
||||
- Wayland alternative is user-authorized [XDG GlobalShortcuts portal](https://flatpak.github.io/xdg-desktop-portal/docs/doc-org.freedesktop.portal.GlobalShortcuts.html), a later adapter with its own binding/approval gate. No bundled privileged helper.
|
||||
|
||||
## Problem / goal
|
||||
While another application has focus, explicitly registered shortcut opens compact NotesCraft capture viewport. Submit to chosen live vault's Daily note or Inbox through `note.capture`; preserve unsent text on dismissal or save failure. App must already run; no background daemon or launch-at-login in v1.
|
||||
|
||||
## Verified baseline / files
|
||||
`apps/notescraft/src/main.rs`: each opened vault is separate process and `NotesCraftWindow` wraps `NotesApp`; `launcher_window.rs` spawns vault processes. `crates/engine/src/capture/{mod,rules}.rs` already handles daily/Inbox/folder, 1 MiB text, source/title and attachment. `crates/ui-egui/src/shell/hotkeys/{defaults,dispatch,commands}.rs` implements **focused-app** hotkeys only; not global registration. `crates/ui-egui/src/shell/app.rs` holds app state and `shell/app/window/{commands,frame}.rs` dispatches actions. `apps/notescraft/Cargo.toml` has no direct global-hotkey dependency. `crates/engine/src/prefs/{app_settings,settings,mod}.rs` owns app preferences; `prefs/catalog/list.rs` provides remembered vaults.
|
||||
|
||||
## Non-goals
|
||||
Auto-start after quit; screenshot/clipboard monitoring; frontmost-app text extraction; cross-vault writes without live session; silent capture when locked/read-only; mobile global shortcut. No changes to shared craft-libs API.
|
||||
|
||||
## Design decisions
|
||||
### D1 Pure draft state
|
||||
New `crates/ui-egui/src/capture/model.rs`, declared by new `capture/mod.rs` and existing `src/lib.rs`. `Draft { text:String, title:Option<String>, target:Target }`, Target Daily or Inbox only; retain in memory, never auto-persist plaintext. `CaptureState` models Hidden, Editing, Saving, Failed with a monotonically increasing submission generation. Validate <=1 MiB UTF-8, title <=200 characters, whitespace-only invalid; no truncation. `submit()` produces one request only from Editing/Failed, rejects repeats during Saving. Complete matching generation clears draft on success; stale result cannot clear newer draft. Esc/close hides preserving draft. User's explicit Discard clears. Quit with unsent draft asks Discard/Cancel through existing close flow; crash loses draft, documented.
|
||||
|
||||
### D2 Host/window lifecycle
|
||||
Single secondary viewport per owning process, about 520x280 points, theme tokens, title/target, multiline input, Save/Discard; optional frameless styling must retain accessible Close and drag affordance. Open requests coalesce, repeat shortcut focuses existing window; never toggles it closed. Set focus/visibility via eframe viewport commands, no busy-loop or IO in paint. Request repaint on native hotkey event so minimized main window still processes it. Main editor does not receive capture keystrokes. Save Ctrl/Cmd+Enter, Escape hide; Tab navigation/focus restored on reopening. Submit failure keeps draft/window, success hides. Closing main process follows normal unsaved-data checks, unregisters hotkey, closes child.
|
||||
|
||||
### D3 Shared command surface
|
||||
Existing `note.capture` is sole persistence command. New engine `capture.window.request {}` -> `{requested:true}` queues one ephemeral UI event on Session; drained once by live shell. `enabled` requires writable live Session; no text/draft/token returned. Headless sessions return `unsupported` instead of claiming a window opened. `capture.window.status {}` -> `{supported,available}` reports host capability only. Host registers capability with Session at startup; registry never depends on egui. New `capture/window.rs` + registration in capture/mod.rs, add IDs to core_commands and assistant_gate (status READ, request WRITE; request may focus UI and must respect assistant gate). Native event and palette invoke same request command. CLI/MCP/control can request live window; no new public IPC server.
|
||||
|
||||
### D4 Multi-process ownership
|
||||
Global preference `quick_capture.owner_vault` is explicit canonical existing vault path selected by user, **not** active/recent vault chosen implicitly. Only process whose live Session matches may register chord; other vault windows show owner name and enabled/disabled status. Launcher alone cannot capture. Two processes for same vault: first successful OS registration wins, second shows conflict and never steals. Owner closes -> feature unavailable until owner vault reopened; no implicit rerouting or process spawning. Changing owner unregisters current registration before saving/applying replacement; never operate two registrations. Chord change: attempt replacement safely, on failure preserve prior effective registration and display configured/effective values rather than silent mismatch.
|
||||
|
||||
### D5 OS registration / dependency gate
|
||||
Default `quick_capture.enabled=false`; configurable chord, proposed Mod+Alt+Shift+N (validate against app table; OS conflicts checked at registration), selected owner path. Persist via existing settings; local machine preferences, never vault sync. Use candidate `global-hotkey` **only after owner approval**, version/pinning/license audited then; root workspace and target-specific app manifest changes belong to task 3. Register on macOS main event-loop thread; Windows manager/event-loop same thread. Poll channel without blocking and unregister on disable/rebind/exit. Repo forbids unsafe: no handwritten FFI, no exception to lints. No eframe/winit platform callbacks assumed available without compile probe against pinned versions. If adapter cannot integrate supported event loop, stop/report, do not install low-level keyboard hooks.
|
||||
|
||||
macOS/Windows/X11 candidates; OS denial/chord conflict -> visible actionable error and ordinary palette request still works. Wayland reports unsupported in v1, with instructions to use focused-app capture; do not label focused-app binding global. Native Wayland portal postponed until owner approves binding and permission UX. `global-hotkey` approval is not approval for portal/objc/windows dependency additions.
|
||||
|
||||
### D6 UI and privacy
|
||||
Strings `[ui.quick_capture]` appended to `crates/i18n/locales/en.toml`, table module through `text/features/mod.rs`. Reuse craft-ui inputs/buttons/modal and theme tokens; no logging draft text. Settings enable/chord/owner/status and unsaved-draft warning. Memory draft belongs only to selected owner process. If selected vault becomes read-only/locked, refuse submit; preserve draft and explain. No clipboard reads, screenshot capture, network or automatic focus tracking.
|
||||
|
||||
## Acceptance criteria
|
||||
1. Pure model tests: whitespace/oversize rejected; repeated Save emits one request; failed save preserves text; success clears only matching generation; Escape retains, Discard clears.
|
||||
2. `capture.window.request` via control and native event shows same viewport once; headless returns unsupported. No UI imports below L5.
|
||||
3. Actual OS smoke test from another app while owner main window minimized: shortcut shows/focuses capture, input lands there, Cmd/Ctrl+Enter saves to selected owner Daily/Inbox exactly once. Main editor unchanged except intended daily-buffer update.
|
||||
4. Dirty daily note receives capture through existing Session buffer, undo remains correct. Read-only or write error leaves draft editable and retryable.
|
||||
5. Two vault processes do not compete; non-owner cannot register. Second identical owner reports conflict; disable/quit releases chord. No foreground-app shortcut mistaken for global.
|
||||
6. Chord change failure preserves previous effective shortcut; OS unsupported/denied status explicit. Wayland does not claim support.
|
||||
7. Close/reopen child preserves draft; process quit asks before discard. No capture text in settings, disk recovery, logs or control status.
|
||||
8. Kittest covers keyboard/focus/Save/Discard, localized strings; visual snapshots read at Mocha/Latte and compact width. Main-window close retains existing safety flow.
|
||||
|
||||
## Open owner questions
|
||||
- Q1: approve new `global-hotkey` dependency for desktop host (reason: safe Rust global registration under unsafe-forbidden policy), macOS/Windows/X11 first? Version/lock changes require approval before task 3.
|
||||
- Q2: accept Wayland unsupported v1 or approve separate portal dependency/permission design? No implicit native bindings added.
|
||||
- Q3: confirm one explicit owner vault, app-running-only lifecycle and no automatic transfer/launch-on-login.
|
||||
- Q4: confirm memory-only draft with quit confirmation, rather than disk plaintext recovery; proposed chord Mod+Alt+Shift+N remains user-configurable.
|
||||
|
||||
## Feature ideas (researched, ranked)
|
||||
1. Wayland [GlobalShortcuts portal](https://flatpak.github.io/xdg-desktop-portal/docs/doc-org.freedesktop.portal.GlobalShortcuts.html), permission-driven binding after owner approval.
|
||||
2. Quick Note resume pattern from [Apple Notes shortcuts](https://support.apple.com/en-kg/guide/notes/apd46c25187e/mac): one retained draft; later multiple draft tabs only on demand.
|
||||
3. Platform-native shortcut preferences using [global-hotkey](https://docs.rs/global-hotkey/latest/global_hotkey/) supported backends, with real conflict diagnostics.
|
||||
@@ -0,0 +1,52 @@
|
||||
# Tasks: global quick capture (roadmap 38)
|
||||
|
||||
One task = one bounded implementation handoff. Spec: `spec.md`; rules: `AGENTS.md`, `docs/CODE-MAP.md`. Paths marked new do not exist yet. Tasks run in order; each leaves its own testable result. Later implementation owns code paths listed below; this documentation run writes only `specs/plugins-parity/`.
|
||||
|
||||
**First tool call rule:** batch at most three relevant reads in first tool call; write first OWNS file in SECOND tool call (compiling stub plus one failing test), then implement and pass that test. Do not spend second call exploring. For docs-only task, write first OWNS document in second call and validate its acceptance checklist.
|
||||
**Standing VERIFY (future implementation only):** `export PATH="$HOME/.cargo/bin:$PATH"`; `cargo test -p <package>`; `cargo clippy -p <package> --all-targets -- -D warnings`; `rustfmt --edition 2024 <owned Rust files>`; UI tasks add `cargo xtask i18n` and read snapshot PNGs; final integration adds `cargo xtask layers`, `cargo xtask tree`, `cargo xtask ci`. Stop on failure; never bypass checks.
|
||||
**Standing DO NOT:** edit outside OWNS, change Git state, commit, use `cargo fmt --all`, add unapproved dependency, modify craft-libs API, use unsafe/panics/input indexing, write disk outside atomic Vault/Session/settings APIs, hardcode UI strings. Parent module declarations/test harnesses must be owned explicitly.
|
||||
|
||||
## Phase 1 — pure draft lifecycle
|
||||
- [ ] 1. Draft validation and submission state (brief `briefs/25-quick-capture-model.md`).
|
||||
OWNS: `crates/ui-egui/src/capture/model.rs` (new, first write), `crates/ui-egui/src/capture/mod.rs` (new), `crates/ui-egui/src/lib.rs` (module declaration only).
|
||||
FILES TO OPEN: `specs/plugins-parity/global-quick-capture/spec.md` D1, `crates/engine/src/capture/mod.rs`, `crates/ui-egui/src/shell/app/popout.rs`.
|
||||
GOAL: pure Draft/Target/CaptureState transitions; <=1 MiB text, <=200 title characters; one in-flight submission, generation-matched completion, hide preserves text, explicit discard clears. Use existing serde_json only to form note.capture params; no IO.
|
||||
TESTS: empty/whitespace/over-limit/multibyte; repeated submit; failure then retry; stale result cannot clear newer draft; hide/reopen/discard; valid Daily/Inbox param shape; quitting with draft reports confirmation required.
|
||||
VERIFY: standing `-p notescraft-ui-egui`, `cargo test -p notescraft-ui-egui capture::model`.
|
||||
DO NOT: native hotkey registration, viewport creation, disk draft recovery, dependency/Cargo changes, engine command registration.
|
||||
|
||||
## Phase 2 — live command surface
|
||||
- [ ] 2. Ephemeral capture-window request in live Session.
|
||||
OWNS: `crates/engine/src/capture/window.rs` (new, first write), `crates/engine/src/capture/mod.rs`, `crates/engine/src/vault/session.rs`, `crates/engine/src/commands.rs`, `crates/engine/src/ai/assistant_gate.rs`, `crates/engine/tests/core_commands.rs`, `crates/engine/tests/capture_window.rs` (new).
|
||||
FILES TO OPEN: spec D3, `crates/engine/src/capture/mod.rs`, `crates/engine/src/vault/session.rs`, `crates/engine/src/control/pump.rs`.
|
||||
GOAL: capture.window.request/status; host registers capability, request coalesces to one pending event drained by shell; headless unsupported; writable gate; never return draft. Public rustdoc for minimal capability/drain methods so app can use without egui in engine.
|
||||
TESTS: unsupported headless; host-enabled request drains once; repeated request coalesces; read-only refused; request/status permission classification and core ids; existing note.capture regression.
|
||||
VERIFY: standing engine, `--test capture_window --test capture --test core_commands`.
|
||||
DO NOT: add UI dependencies below L5, create IPC server, save notes outside note.capture, infer global support from focused-app hotkeys.
|
||||
|
||||
## Phase 3 — native registration (owner gate)
|
||||
- [ ] 3. Platform hotkey owner and settings; stop until Q1/Q2 answered.
|
||||
OWNS: `apps/notescraft/src/capture_hotkey.rs` (new, first write), `apps/notescraft/src/main.rs`, `apps/notescraft/Cargo.toml`, `Cargo.toml` (approved global-hotkey workspace dependency ONLY), `Cargo.lock`, `crates/engine/src/prefs/quick_capture_config.rs` (new), `crates/engine/src/prefs/mod.rs`, `crates/engine/src/prefs/app_settings.rs`, `crates/engine/src/prefs/settings.rs`.
|
||||
FILES TO OPEN: spec D4-D5 and owner decisions, `apps/notescraft/src/main.rs`, `apps/notescraft/src/menu.rs` (main-thread event polling), `crates/engine/src/prefs/app_settings.rs`, `crates/engine/src/prefs/settings.rs`.
|
||||
GOAL: default-off local config and explicit owner; safe approved facade only, macOS main loop and Windows same-thread loop probe against pinned eframe. Only owner process registers. Conflict/unsupported states explicit; disable/rebind/exit unregisters, prior effective chord preserved if replacement fails. Repaint wakeup while minimized; native event requests command from task 2. This is not a new crate: no layers entry needed; if extraction becomes necessary, stop for plan update and own root Cargo.toml plus xtask/src/layers.rs registration before building it.
|
||||
TESTS: fake registration backend conflict/replace/rollback/drop, owner vs non-owner; settings validation/defaults/roundtrip. Actual manual smoke macOS/Windows/X11 from another foreground app, minimized main window, disable/rebind/exit. Wayland reports unsupported. Record each tested OS; no extrapolation.
|
||||
VERIFY: standing `-p notescraft` and engine; approved-platform compile probes and manual matrix; layers/tree.
|
||||
DO NOT: add dependency before explicit approval, handwritten unsafe FFI, global low-level key hooks, second listener, login daemon, automatically transfer owner.
|
||||
|
||||
## Phase 4 — compact window
|
||||
- [ ] 4. Viewport, focus, submit and settings UX.
|
||||
OWNS: `crates/ui-egui/src/capture/window.rs` (new, first write), `crates/ui-egui/src/capture/mod.rs`, `crates/ui-egui/src/shell/app.rs`, `crates/ui-egui/src/shell/app/window/commands.rs`, `crates/ui-egui/src/shell/app/window/frame.rs`, `crates/ui-egui/src/shell/hotkeys/commands.rs`, `crates/ui-egui/src/views/settings_view/categories/core_cards.rs`, `crates/ui-egui/src/text/features/quick_capture.rs` (new), `crates/ui-egui/src/text/features/mod.rs`, `crates/i18n/locales/en.toml` (append ui.quick_capture), `crates/ui-egui/tests/quick_capture.rs` (new), `apps/notescraft/src/main.rs` (quit confirmation integration only).
|
||||
FILES TO OPEN: spec D1-D6, `crates/ui-egui/src/shell/app/popout.rs`, `crates/ui-egui/src/shell/app/window/frame.rs`, `crates/ui-egui/src/shell/app/window/commands.rs`, `crates/ui-egui/src/views/settings_view/categories/core_cards.rs`.
|
||||
GOAL: one themed native child viewport; consume coalesced engine request; focus/reopen preserves draft; text/target/title; keyboard submit/Esc/Discard; async result generation safety; note.capture sole writer. Settings includes explicit owner/chord/effective status. Pending draft joins existing quit confirmation. Use current craft-ui controls, no shared API change.
|
||||
TESTS: kittest focus and keys, repeated request/save, failed write, read-only, dirty Daily buffer/undo, close/reopen, discard, quit cancel; no draft content in settings/control status/logs; all text i18n.
|
||||
VERIFY: standing ui-egui + engine + desktop; i18n; snapshot PNGs read in Mocha/Latte, desktop and compact; actual foreground/minimized capture smoke from task 3.
|
||||
DO NOT: block painting, read clipboard, silently clear draft, hardcode labels, persist plaintext draft.
|
||||
|
||||
## Phase 5 — acceptance and documentation
|
||||
- [ ] 5. Record measured platform coverage and final status.
|
||||
OWNS: `specs/plugins-parity/global-quick-capture/validation.md` (new, first write), `specs/plugins-parity/global-quick-capture/spec.md`, `specs/plugins-parity/global-quick-capture/tasks.md`, `specs/plugins-parity/roadmap.md` (38 only), `ROADMAP.md`, `docs/ARCHITECTURE.md`, `docs/CODE-MAP.md` (feature entries only).
|
||||
FILES TO OPEN: spec AC1-8 and open decisions, `crates/ui-egui/tests/quick_capture.rs`, `specs/plugins-parity/roadmap.md`.
|
||||
GOAL: acceptance-to-test mapping, owner approval record, actual platform/event-loop results, unsupported Wayland status, app-running-only/owner/draft-loss limits. Update command/path maps only after implementation passes.
|
||||
TESTS: manually verify two vault processes, native foreground test, conflict/rebind/exit and quit-with-draft; document exact platform/backend versions. Each AC has evidence or remains visibly blocked.
|
||||
VERIFY: standing final gates incl cargo xtask ci; links and task status agree. These commands are future implementation checks, not part of current docs run.
|
||||
DO NOT: claim unsupported backend works, remove approval gates without owner decision, add features or commit.
|
||||
@@ -0,0 +1,57 @@
|
||||
# Markdown slides mode (agy gap 16, roadmap 41)
|
||||
|
||||
Part of `specs/plugins-parity/`. **Supersedes roadmap 19 “Slides view”**: one implementation, one acceptance set; do not create a second slides plugin. SPEC 6.1 still calls Slides deferred; implementation completion must reconcile that status. Existing Markdown, engine and native UI; no new crate/dependency.
|
||||
|
||||
## Research (2026-10-21; agy claims checked)
|
||||
- Obsidian's core Slides documents note-tab/command entry, `---` slide separators, arrow/Space navigation and Escape exit. Verified directly; prefer current lowercase help URL over agy's legacy path. https://help.obsidian.md/plugins/slides
|
||||
- Pandoc documents presentations and browser-backed slide output. That is inspiration, not NotesCraft's runtime: NotesCraft remains Rust/egui, without reveal.js, JavaScript or webview. https://pandoc.org/MANUAL.html#slide-shows
|
||||
- agy's Marp and SiYuan comparisons are not necessary to this design and are dropped rather than treated as verified feature parity. Effort “S” is unverified: parser ambiguity, buffer consistency and focus/fullscreen restoration require tests.
|
||||
|
||||
## Problem / goal
|
||||
Present an existing Markdown note as navigable slides without copying it into another format. Text, local images, lists, tables, math and links reuse native note rendering. Enter and exit without modifying source, losing editor selection or changing normal reading mode.
|
||||
|
||||
## Non-goals
|
||||
HTML/browser export, animation, speaker notes, remote control, presenter second display, arbitrary CSS/Marp directives, slide editing during presentation, automatic PDF slide pagination, new renderer.
|
||||
|
||||
## Design decisions
|
||||
### D1 Deck model and separators
|
||||
Pure model in `crates/markdown/src/slides.rs` uses `pulldown-cmark` already available in that crate, offset events plus existing frontmatter boundaries; do not split the raw string on `---`.
|
||||
`Deck { slides: Vec<Slide> }`, `Slide { range: Range<usize> }`, `parse_deck(&str) -> Result<Deck>`. Ranges are source UTF-8 byte ranges, excludes frontmatter and separator lines, preserve contents/newline bytes; source always remains unchanged. A separator is a root-level CommonMark thematic-break event whose complete line, after optional CR removal, is exactly `---`, with a blank line or body boundary on each side. Not `***`, `___`, `----`, indented/list/quote breaks, setext underlines, code-fence contents, HTML/Obsidian comments, math blocks or frontmatter delimiters. Existing protected spans plus parser container context determine eligibility. Unclosed fenced/comment regions never create slides inside their content.
|
||||
|
||||
Empty/body-only/frontmatter-only note yields one possibly empty slide; explicit leading/trailing/repeated valid separators preserve empty slides (no silent dropping). Cap input 2 MiB and 500 slides, reject excess (`too_large`) instead of truncating. No unchecked slicing; ranges refer to original source and all `get` accesses checked. Keep module publicly re-exported from `crates/markdown/src/lib.rs`.
|
||||
|
||||
### D2 Engine session and command surface
|
||||
| id | params | result | effect |
|
||||
|---|---|---|---|
|
||||
| `slides.start` | `{path, slide?}` 1-based default 1 | `{path, slide, total, generation}` | capture latest open-buffer source into session presentation |
|
||||
| `slides.status` | none | `{active:false}` or active path/slide/total/generation | no mutation |
|
||||
| `slides.goto` | `{slide}` 1-based | active status | exact validated slide index |
|
||||
| `slides.stop` | none | `{active:false}` | release snapshot; idempotent |
|
||||
|
||||
`Presentation` state in engine `slides/{mod,state,commands}.rs`, one optional field in `Session`. Commands do not write disk or dirty buffers and work read-only. Start uses buffer source including unsaved edits when open, guarded note read otherwise. Invalid deck/path/index returns structured error and preserves prior active state. Goto outside 1..total returns error, never clamps; UI disables boundary arrows. Only UI maps Next/Previous to a known-valid index. Store one bounded immutable `Arc<str>` snapshot plus ranges and a generation; view obtains snapshot via internal session accessor (not huge JSON in status). If source changes externally during a show, current snapshot stays fixed; Restart presentation explicitly builds a fresh snapshot. Switching/closing vault clears presentation. Assistant gate grants status READ; start/goto/stop use the existing WRITE-class gate because they alter session state (this does not mean the commands write files). Existing write grants/review policy applies; no new permission system. Do not let assistant status reads start fullscreen; if review cannot represent a presentation-only change, return the existing unsupported-preview refusal instead of bypassing review. CLI/MCP/control callers can inspect and navigate the same state; desktop shell observes generation and active status.
|
||||
|
||||
### D3 Native UI
|
||||
New `views/slides_view/{mod,controls}.rs` renders current slide through existing `note_view::show_body_with_fences` hooks with `editable:false`. Relative paths resolve against original note, never a copied temporary note. Slide subsections are parsed separately for painting but all link/asset lookup uses source note path. Disable task toggles, toolbar mutations and editable transclusions; links can open a note only through explicit exit/open action. Missing media uses existing placeholder; diagram/media resources use existing bounded caches. No IO/engine calls while painting a block.
|
||||
|
||||
Presentation occupies active window content and requests fullscreen using eframe viewport commands; preserve prior fullscreen flag, focused tab/pane, cursor/selection, scroll and panel layout. Exit restores exactly previous fullscreen state (already-fullscreen windows remain fullscreen). Shortcut input only consumed while presentation active and no modal/input owns it: Right/PageDown/Space next, Left/PageUp previous, Home/End first/last, Escape exit. One keypress makes one state transition. On-window controls offer previous/next, current/total, exit and restart; localized labels and accessible focus. Panel chrome hidden for show, not persisted to workspace layout. No global hotkey registration.
|
||||
|
||||
Default 16:9 content area fits viewport with themed letterboxing. Reuse theme tokens and note type scale; scroll tall slide vertically with clear overflow affordance rather than clipping unreadable content or silently reducing text below readable size. Scroll resets on navigation, preserved while repainting same slide. Frame remains bounded by same source/render caps. Native widgets only; no browser engine. Entry via note tab menu and command palette; no mandatory new chord.
|
||||
|
||||
## Acceptance criteria
|
||||
1. Frontmatter, setext headings, fenced `---`, quoted/list rules, comments/math and CRLF fixtures produce documented slide counts/ranges and lossless original source.
|
||||
2. Empty, repeated-boundary and no-final-newline sources work; non-UTF-8 offsets cannot occur; input/slide overflow errors preserve current deck.
|
||||
3. Start reads unsaved buffer; navigation and exit leave buffer bytes, dirty flag and undo history unchanged; read-only vault supports full presentation.
|
||||
4. Engine/control and UI use same slide index; invalid start/goto does not replace active state; stop idempotent; no filesystem writes.
|
||||
5. Arrow/Space/Home/End/Escape, pointer controls, modal focus and repeated key events behave predictably; editor caret/focus/scroll, panels and former fullscreen state restore after exit.
|
||||
6. Local asset relative resolution, code/table/math, tall-slide overflow and missing image render correctly; source edits during presentation affect only Restart, never shift current deck unexpectedly.
|
||||
7. Mocha/Latte screenshots at 1440x900 and narrow/2x viewport read; no raw user strings or per-block IO; parsing/rendering bounded; regression gates green.
|
||||
|
||||
## Open owner questions
|
||||
- Q1: accept exact `---` with blank-line boundaries as v1 compatibility rule, including explicit empty slides? Proposal yes; other thematic-break markers stay ordinary note content.
|
||||
- Q2: approve fullscreen in existing active window rather than second presenter window? Proposal yes; capture/restore previous fullscreen and editor state.
|
||||
- Q3: tall-slide handling: proposal scroll with visible overflow affordance; automatic fit-to-text and paginated PDF export remain later features.
|
||||
|
||||
## Feature ideas (researched, ranked)
|
||||
1. Source-position-aware “Start from current slide”, using same deck ranges: https://help.obsidian.md/plugins/slides
|
||||
2. Export slides through optional installed Pandoc after a separate native-to-export compatibility contract: https://pandoc.org/MANUAL.html#slide-shows
|
||||
3. Speaker notes and incremental reveal require an explicit syntax and native rendering policy; compare documented Pandoc presentation conventions before extending grammar: https://pandoc.org/MANUAL.html#slide-shows
|
||||
@@ -0,0 +1,40 @@
|
||||
# Tasks: Markdown slides mode (roadmap 41; replaces 19)
|
||||
|
||||
Spec: `spec.md`. One task = one agent. No new crate, Cargo registration, third-party dependency or browser runtime.
|
||||
**First tool call rule:** batch needed reads in first call; write first OWNS source file in SECOND tool call, compiling stub plus one failing test, then make it pass.
|
||||
**Standing VERIFY:** `export PATH="$HOME/.cargo/bin:$PATH"`; `cargo test -p <owned package>`; `cargo clippy -p <owned package> --all-targets -- -D warnings`; `rustfmt --edition 2024 <owned Rust files>`; UI adds `cargo xtask i18n`; final phase adds `cargo xtask layers` and `cargo xtask tree`. Future implementation checks only, no Cargo during spec authoring.
|
||||
**Standing DO NOT:** edit outside OWNS, mutate git, add dependencies/Cargo changes, unchecked source slicing/panics/blanket allows, `cargo fmt --all`, source rewriting or second independent rank-19 implementation.
|
||||
|
||||
## Phase 1 — pure deck model
|
||||
- [ ] 1. Lossless Markdown slide ranges.
|
||||
OWNS: `crates/markdown/src/slides.rs` (new, FIRST; unit tests inside), `crates/markdown/src/lib.rs` (module and exports), `crates/markdown/src/error.rs` (narrow deck-size error if needed).
|
||||
FILES TO OPEN: `spec.md` D1, `crates/markdown/src/lib.rs`, `crates/markdown/src/frontmatter/mod.rs`, `crates/markdown/src/scanning/fences.rs`, `crates/markdown/src/scanning/common.rs`, `crates/markdown/Cargo.toml` (read only, existing pulldown-cmark).
|
||||
GOAL: `Deck`, `Slide`, `parse_deck(&str) -> Result<Deck>` with D1 eligible root thematic-break events, exact source ranges, protected-region handling and byte/slide caps.
|
||||
TESTS: frontmatter, setext, root `---`, quoted/list/indented breaks, `***`/`----`, fenced/inline code, comments/math/HTML, unclosed fence/comment, CRLF, Unicode, empty note/frontmatter-only, leading/trailing/repeated valid boundaries, no-final-newline, 500/501 slides, 2 MiB/+1 bytes. Assert original source untouched and every range `get` succeeds.
|
||||
VERIFY: standing `-p notescraft-markdown`, filter `slides`, confirm tests run via module declaration.
|
||||
DO NOT: engine/UI/IO, split raw text on `---`, add parser dependency.
|
||||
|
||||
## Phase 2 — shared engine presentation
|
||||
- [ ] 2. Bounded session snapshot and commands.
|
||||
OWNS: `crates/engine/src/slides/state.rs` (new, FIRST), `crates/engine/src/slides/mod.rs` (new), `crates/engine/src/slides/commands.rs` (new), `crates/engine/src/lib.rs`, `crates/engine/src/vault/session.rs` (presentation field, initialization and accessor), `crates/engine/src/commands.rs` (one register call), `crates/engine/src/ai/assistant_gate.rs`, `crates/engine/tests/core_commands.rs`, `crates/engine/tests/slides.rs` (new).
|
||||
FILES TO OPEN: D2, `crates/engine/src/vault/buffers.rs`, `crates/engine/src/capture/mod.rs`, `crates/engine/tests/common/mod.rs`, `crates/engine/src/control/protocol.rs`.
|
||||
GOAL: D2 exact start/status/goto/stop table, guarded path and current buffer snapshot, immutable bounded deck, generation and UI accessor; start/goto/stop use existing WRITE-class gate while status READ; preserve existing grant/review behavior and refuse unsupported previews. Read-only presentation allowed.
|
||||
TESTS: registry/control dispatch same status, dirty unsaved source takes priority, invalid path/index/deck leaves active state unchanged, navigation boundaries errors, stop twice, snapshot stays fixed until restart, read-only success, no disk/buffer/undo mutations, session close releases state.
|
||||
VERIFY: standing `-p notescraft-engine`, `--test slides --test core_commands` plus existing control tests.
|
||||
DO NOT: store egui types below L5, fullscreen/window IO, serialize full source on status, write any file during navigation, bypass assistant review or invent a separate permission system.
|
||||
|
||||
## Phase 3 — native presentation surface
|
||||
- [ ] 3. Slide viewport and deterministic controls.
|
||||
OWNS: `crates/ui-egui/src/views/slides_view/mod.rs` (new, FIRST), `crates/ui-egui/src/views/slides_view/controls.rs` (new), `crates/ui-egui/src/views/mod.rs`, `crates/ui-egui/src/text/views/slides.rs` (new), `crates/ui-egui/src/text/views/mod.rs`, `[ui.slides]` at END of `crates/i18n/locales/en.toml`, `crates/ui-egui/tests/slides_view.rs` (new).
|
||||
FILES TO OPEN: D3, `crates/ui-egui/src/views/note_view.rs` (`show_body_with_fences`, `BodyHooks`), `crates/ui-egui/src/views/note_view/blocks/embed.rs`, `crates/ui-egui/src/views/media_view/bar.rs`, `crates/ui-egui/src/design/style.rs`.
|
||||
GOAL: read-only native slide paint using supplied snapshot, origin-relative assets, themed 16:9 viewport and tall-slide scrolling; controls return bounded navigation/stop/restart intents for shell. No engine calls in paint.
|
||||
TESTS: kittest arrows disabled at bounds, current/total, tall slide scroll/reset, read-only task/transclusion clicks, origin-relative image/missing asset and code/table/math; Mocha/Latte standalone view snapshots.
|
||||
VERIFY: standing `-p notescraft-ui-egui`, i18n and read snapshot PNGs.
|
||||
DO NOT: task edits/source mutations, remote fetch during paint, raw hardcoded UI strings, introduce another Markdown renderer or webview.
|
||||
- [ ] 4. Shell entry/exit, focus/fullscreen and completion gates.
|
||||
OWNS: `crates/ui-egui/src/shell/app/window/views.rs` (FIRST), `crates/ui-egui/src/shell/app/window/frame.rs`, `crates/ui-egui/src/shell/app/window/commands.rs`, `crates/ui-egui/src/shell/app/window/menus.rs`, `crates/ui-egui/src/shell/hotkeys/dispatch.rs`, `crates/ui-egui/src/shell/hotkeys/commands.rs`, `crates/ui-egui/tests/slides_shell.rs` (new), `SPEC.md` (6.1 status only after completion), `ROADMAP.md`, `docs/ARCHITECTURE.md`, `docs/CODE-MAP.md`, ranks 19/41 in `specs/plugins-parity/roadmap.md`.
|
||||
FILES TO OPEN: D2/D3, current window frame/view state, `crates/ui-egui/src/shell/hotkeys/defaults.rs`, `crates/ui-egui/src/shell/hotkeys/focus.rs`, task 2 session accessor and task 3 view API.
|
||||
GOAL: palette/tab-menu entry, observe shared session presentation generation, consume active-only navigation once, restore tab/pane/editor selection/scroll/panels/fullscreen on exit; explicit restart refreshes source. Control-channel start reaches same surface. Mark rank19 superseded by41, never duplicate completion counts.
|
||||
TESTS: kittest start/navigate/exit, Space consumed once, modal/input owns keys, boundary no goto error toast, prior fullscreen true/false restoration, dirty editor unchanged, external source mutation/restart, close/switch vault clears state; offscreen view plus manual OS fullscreen smoke check.
|
||||
VERIFY: standing engine/UI plus `cargo xtask ci`; read 1440x900, narrow and 2x Mocha/Latte PNGs; record manual platform checks still unavailable instead of claiming them passed.
|
||||
DO NOT: register global shortcut, persist hidden chrome to workspace, broaden native menus or update completed status before gates.
|
||||
@@ -0,0 +1,40 @@
|
||||
# Per-note and secret-folder encryption: design only (agy gap 5, roadmap 34)
|
||||
|
||||
## Research (2026-10-21; agy claims checked)
|
||||
[Apple Platform Security](https://support.apple.com/guide/security/secure-features-in-the-notes-app-sec1782bcab1/web) documents secured notes; it does not establish encrypted folders. [Joplin E2EE](https://joplinapp.org/help/apps/sync/e2ee/) describes encrypted synchronization, not a per-note local-disk lock. Drop agy's implied equivalence and “small effort” rating. AES-256-GCM and Argon2id are candidate choices, not existing NotesCraft capability or an approved dependency list. [RFC 9106](https://www.rfc-editor.org/rfc/rfc9106) defines Argon2 and parameter profiles; it does not remove need for platform cost measurement/security review.
|
||||
|
||||
## Goal and non-goal
|
||||
Produce reviewable threat model, versioned container proposal, key lifecycle, integration inventory and approval checklist. NO crypto implementation, Cargo edits, lock UI, production format, or claim of secure encryption in this phase. No new crate now. A later approved implementation plan must separately register any new crate in root `Cargo.toml` and `xtask/src/layers.rs` with explicit layer rationale.
|
||||
|
||||
## D1 Threat model
|
||||
Protect locked content against offline theft of vault/ciphertext and unauthorized reading by sync storage. Attacker may read/copy/change/replay encrypted files and observe path, size and timestamps. AEAD detects tampering, not rollback/deletion: rollback needs trusted external state and is out of v1. No protection against compromised OS/app, keylogger, hostile same-user process, unlocked app, clipboard/screenshot/print, swap/core dumps or prior plaintext backups. Do not call local lock “zero knowledge” for all app operations. Whole-disk encryption still recommended as complementary scope, not prerequisite.
|
||||
Per-note visible filename leaks title by default. Secret folder proposal uses opaque container name with encrypted manifest hiding child filenames/tree; container size/timing and existence still leak. Plain external wikilinks, backlink snippets, recent lists and previously indexed metadata can leak. Reject automatic encryption if unmanaged attachments or external plaintext copies would make privacy claim false; explicit migration review required.
|
||||
|
||||
## D2 Candidate format (not frozen)
|
||||
Per-note `.md.enc`, secret folder `.ncfolder.enc` (single encrypted manifest plus separately encrypted opaque objects to avoid full-folder rewrite). Container envelope: magic `NCENC`, version `1`, bounded header length, suite/KDF IDs, bounded KDF parameters, salt, key-slot metadata, random object ID, generation, nonce, ciphertext length, ciphertext, AEAD tag. Fixed canonical binary encoding and byte order to be specified in design task before approval; no ambiguous JSON canonicalization. Authenticated associated data covers full immutable header, object identity and purpose; key slots separately authenticate wrapped data key. Payload holds original UTF-8 note and metadata/manifest; attachment bytes encrypted under separate object IDs. Path is not AEAD identity so rename need not decrypt/re-encrypt; object substitution rejected by expected ID and manifest binding.
|
||||
Candidate: AES-256-GCM with fresh cryptographic nonce per write and Argon2id passphrase KEK wrapping random data keys. Never derive nonce from time/counter alone, never reuse across rewrites. Owner must approve RNG, AEAD, KDF, secret-memory and keychain dependencies as a reviewed set. Enforce pre-KDF parameter ceilings and payload bounds before allocation; unknown suite/version and authentication failure fail closed. Proposed ceiling 256 MiB ciphertext per object and 16 KiB envelope; KDF accepted range requires measured profile, not arbitrary caller values. No hand-written crypto or fake test encryption.
|
||||
|
||||
## D3 Key lifecycle and state
|
||||
States Locked -> Unlocking -> Unlocked -> Locking -> Locked; authentication error returns Locked. Keys only enter memory after explicit local unlock; never env/settings/logs/control-channel/MCP params. Passphrase change rewraps data keys after old authentication; interrupted replacement preserves previous valid key slot. Forgotten passphrase is unrecoverable unless owner chooses recovery-key design; never password-reset promise. Session-only unlock default; auto-lock on vault switch, screen lock/suspend and configurable idle timeout. Dirty buffers must atomically save encrypted content or visibly refuse lock; failed save cannot discard work or falsely report locked. Clear previews, history, search postings, undo/redo, extracted text, clipboard references, AI context and jobs before locking. Memory zeroization best effort, cannot promise erasure of framework/OS copies. Keychain/biometric persistence is opt-in separate approval; no unlock key synced beside ciphertext.
|
||||
|
||||
## D4 Integration and migration boundaries
|
||||
Audit actual `crates/engine/src/vault/session.rs`, `crates/engine/src/vault/buffers.rs`, `crates/engine/src/vault/attachments.rs`, `crates/engine/src/vault/find/content_search.rs`, `crates/engine/src/ai/assistant_gate.rs`, `crates/engine/src/sync/worker.rs`, `crates/vault/src/lib.rs` and UI previews. Capture, OCR, PDF crops, publish/share/export/recovery/trash must never emit plaintext while locked. All programmatic reads denied while locked even if path known; unlocking UI-only, no passphrase command. Unlocked disclosure to AI/plugins/export requires explicit action and documented policy. Secret folder is security domain, not filesystem hidden flag; inherited encryption enforced on move/create, shared attachments require copy into encrypted domain or refusal. Existing plaintext index/cache/recovery/sync versions need inventoried purge/migration, with warning that SSD snapshots/remote history cannot be securely erased by app.
|
||||
Migration journal uses craft-atomic through vault boundary, writes/authenticates ciphertext before removing plaintext; removal is explicit destructive action requiring owner authorization. Crash matrix covers every stage, mixed encrypted/plain states and rollback; never plaintext temp files. Design must decide conflict handling and multi-device key rotation before sync is enabled for encrypted domains. No existing engine command bypass permitted.
|
||||
|
||||
## Acceptance criteria (documents only)
|
||||
1. Threat/asset/adversary matrix distinguishes local at-rest protection, sync E2EE, metadata leakage and excluded attacks.
|
||||
2. Envelope byte layout, bounds, authenticated fields, nonce uniqueness strategy, key slots, version rejection and test-vector source documented; no implementation or unapproved dependency.
|
||||
3. State machine covers wrong password, cancellation, dirty save failure, auto-lock, crash and lost password; memory limitations stated honestly.
|
||||
4. Integration matrix maps every plaintext producer/consumer to existing file, required gate, test and owner; unresolved gaps block implementation.
|
||||
5. Migration/recovery and hostile-header test plans cover interrupted writes, nonce reuse prevention, tamper, replay limitation and backups; owner decisions recorded explicitly.
|
||||
|
||||
## Open owner questions
|
||||
- Q1: Approve exact threat scope, visible per-note names and opaque secret-folder domain proposal? Is hidden tree a requirement?
|
||||
- Q2: Which audited Rust AEAD/KDF/RNG/zeroization/keychain dependencies and security reviewer may be approved? AES-GCM/Argon2id are proposals only; no dependency before response.
|
||||
- Q3: Session-only unlock default, idle policy, OS keychain/biometric opt-in, recovery key or deliberately unrecoverable passphrase?
|
||||
- Q4: Encrypted attachments, multi-device sync, share/publish restrictions and handling historical plaintext copies: required launch scope or explicitly excluded? No implementation until reviewed.
|
||||
|
||||
## Feature ideas (researched, ranked)
|
||||
1. Optional local biometric/keychain unlock, informed by [Apple secured notes](https://support.apple.com/guide/security/secure-features-in-the-notes-app-sec1782bcab1/web), requiring separate safe-platform design.
|
||||
2. Multi-device wrapped-key onboarding, informed by [Joplin E2EE setup](https://joplinapp.org/help/apps/sync/e2ee/), not a copy of its local-storage guarantees.
|
||||
3. Measured KDF profiles and migration policy anchored in [RFC 9106](https://www.rfc-editor.org/rfc/rfc9106).
|
||||
@@ -0,0 +1,35 @@
|
||||
# Tasks: note and secret-folder encryption design (roadmap 34)
|
||||
|
||||
Spec: `spec.md`. DESIGN ONLY: every OWNS path stays below this feature directory; no cargo/code/dependency edits. Tasks are sequential reviewable documents, individually checked against explicit cases. Existing source paths are inspection inputs, never writable in this phase.
|
||||
**First tool call rule:** read at most three listed inputs in first call; write FIRST OWNS document in SECOND call with initial checklist, then complete it. Code-stub/failing-test rule intentionally replaced with document acceptance cases because no crypto code is authorized.
|
||||
**Standing VERIFY:** resolve each referenced existing path with `rg --files`; compare document to spec acceptance matrix and cited primary sources; check no secret values/examples and no runnable crypto. Do not run cargo during this design phase.
|
||||
**Standing DO NOT:** change code/Cargo/git state; add dependency; implement encryption/decryption/KDF/RNG; weaken unsafe policy; freeze algorithm/format without owner approval; claim secure erasure.
|
||||
|
||||
- [ ] 1. Threat model and trust boundaries.
|
||||
OWNS: `specs/plugins-parity/note-encryption/threat-model.md` (new, FIRST).
|
||||
FILES TO OPEN: `specs/plugins-parity/note-encryption/spec.md` D1; `crates/engine/src/vault/session.rs`; `crates/vault/src/lib.rs`.
|
||||
GOAL: asset/adversary/trust-boundary table for per-note and folder domains; separate locked/unlocked behavior, metadata and prior backups; explain local-at-rest vs sync E2EE; owner Q1/Q4 decision checklist with explicit pending status.
|
||||
TESTS: tabletop cases stolen locked vault, replayed old ciphertext, swapped object, hostile same-user process while unlocked, plaintext shared attachment, known filename and old snapshot; each maps protected/excluded/blocked plus reason.
|
||||
VERIFY: standing; D1 and acceptance 1/4 traced with no undocumented assumption.
|
||||
DO NOT: crypto snippets, promise zero knowledge or rollback protection, approve own threat model.
|
||||
- [ ] 2. Container format and bounded parser design.
|
||||
OWNS: `specs/plugins-parity/note-encryption/format.md` (new, FIRST).
|
||||
FILES TO OPEN: spec D2; task 1 threat model; `crates/engine/src/vault/attachments.rs`.
|
||||
GOAL: byte-by-byte candidate envelope, endian/length encoding, AEAD associated-data byte layout, encrypted manifest/object identity, key-slot integrity, versioning and pre-KDF bounds. Record AES-GCM/Argon2id as proposed; list dependency/RNG review questions. No format claims frozen before independent review.
|
||||
TESTS: design vectors for truncation at every field, oversized lengths/KDF cost, nonce reuse across retries, object swap, tampered header/tag, missing manifest object, unknown suite and version; specify expected rejection stage.
|
||||
VERIFY: standing + field/length/AAD consistency review; RFC 9106 parameter rationale measured-plan required.
|
||||
DO NOT: choose unapproved crate or write toy crypto; pass arbitrary hostile KDF cost to runtime.
|
||||
- [ ] 3. Key state machine and migration failure matrix.
|
||||
OWNS: `specs/plugins-parity/note-encryption/key-lifecycle.md` (new, FIRST), `specs/plugins-parity/note-encryption/migration.md` (new).
|
||||
FILES TO OPEN: spec D3–D4; `crates/engine/src/vault/buffers.rs`; `crates/engine/src/sync/worker.rs`.
|
||||
GOAL: unlock/lock state transitions, dirty save error/cancel, zeroization limits, rotation/key-slot recovery, idle/suspend races, session-only defaults and keychain/recovery alternatives; crash-safe atomic migration sequence and explicit plaintext removal authorization point.
|
||||
TESTS: tabletop wrong password, closed dirty tab, crash after every migration stage, low disk on lock, passphrase change interrupted, sync old key slot, forgotten password, OS lock during extraction; preserve work without claiming locked after failure.
|
||||
VERIFY: standing; each state transition names data retained/cleared and error behavior; key material never enters settings/logs/command params.
|
||||
DO NOT: guarantee wiping SSD/swap/backups; silently delete plaintext; implement keychain.
|
||||
- [ ] 4. Leakage integration inventory and owner review package.
|
||||
OWNS: `specs/plugins-parity/note-encryption/integration-review.md` (new, FIRST), `specs/plugins-parity/note-encryption/approval-checklist.md` (new).
|
||||
FILES TO OPEN: spec acceptance/questions; `crates/engine/src/vault/find/content_search.rs`; `crates/engine/src/ai/assistant_gate.rs`; later inspect `crates/engine/src/vault/pdf_preview.rs`, `crates/engine/src/vault/attachments.rs`, `crates/engine/src/sync/worker.rs`.
|
||||
GOAL: input/output/cache/history/trash/preview/OCR/export/AI/plugin/share/capture matrix, verified source path per boundary, proposed locked behavior and future regression test. Prepare owner questions with exact proposed dependencies only after safe-Rust/security/license investigation. Future implementation must have new approved spec/tasks; if new crypto crate chosen, first task explicitly owns/registers `crates/crypto/Cargo.toml`, `crates/crypto/src/lib.rs`, root `Cargo.toml` workspace dependency and `xtask/src/layers.rs` layer entry.
|
||||
TESTS: known encrypted path via all external command surfaces while locked denied; worker late publication after lock; stale index/cache snippets; shared attachment reference leakage; encrypted sync conflicts. Each unresolved surface blocks implementation approval.
|
||||
VERIFY: standing; trace all five spec acceptance criteria and Q1–Q4 to checklist; no approvals inferred.
|
||||
DO NOT: implement future task, modify root docs/status as shipped, mark design approval granted.
|
||||
@@ -0,0 +1,39 @@
|
||||
# Object types and supertags (agy gap 6, roadmap 35)
|
||||
|
||||
Part of plugins parity. Reuse Bases note properties and existing frontmatter edits; no new crate or dependency.
|
||||
|
||||
## Research (2026-10-21; agy claims checked)
|
||||
Tana documents supertags as typed nodes, field templates, defaults and tag extension: https://outliner.tana.inc/learn/features/supertags . This verifies inspiration, not strict database enforcement. Drop unverified Capacities/Anytype parity and the implication every named product shares inheritance semantics. Obsidian Bases reads note frontmatter and uses `.base` YAML for view configuration, not a schema registry: https://obsidian.md/help/bases/syntax . NotesCraft's type registry is its own extension.
|
||||
|
||||
## Problem and goal
|
||||
A book or person note needs repeatable typed properties without a second object database. A supertag is an explicitly configured frontmatter tag mapped to an object type; plain inline/body tags remain ordinary tags. Notes remain ordinary Obsidian-readable markdown.
|
||||
|
||||
## Design decisions
|
||||
### D1 Model and storage
|
||||
New pure `crates/bases/src/schema/{mod,model,resolve}.rs`; registry JSON codec and IO in engine (Bases has no serde dependency). Store `.notescraft/object-types.json` through `vault.write_private`, version 1, with definitions `{id,label,tag,extends?,fields:[{key,type,required,default?}]}`. IDs/tags unique; one parent per type; cycles or missing parents are errors. Reuse `notescraft_bases::Value` and existing markdown property kinds at the boundary, supporting only text, number, checkbox, date, datetime and text-list mappings already writable by properties commands. No custom type system in `.base` YAML.
|
||||
|
||||
Limits: 256 types, 64 fields per resolved type, depth 16, registry 1 MiB, identifiers 1..64 ASCII letters/digits/underscore/dash. Field keys obey existing markdown key validation. Required combines with OR; overriding an inherited field's kind is forbidden; child may override same-kind default. Multiple applicable tags merge schemas; conflicting kinds/defaults return a visible diagnostic, never choose an arbitrary winner. Required means present and non-null, with empty text/list treated as missing. Unknown note properties are retained. Invalid registry stays byte-identical and disables schema actions with diagnostic; no empty-registry overwrite.
|
||||
|
||||
### D2 Commands and edits
|
||||
`objecttype.list {}` -> `{version,types}`; `objecttype.save {definition}` and `objecttype.delete {id}` -> `{changed}`; `objecttype.validate {path}` -> `{types,diagnostics:[{key,code,message}]}`; `objecttype.apply {path,id,expected_text}` -> `{changed,path}`. Read commands remain available in read-only vaults; mutations are disabled and guarded. Delete refuses a referenced parent. Applying adds the configured frontmatter tag and missing defaults only; never overwrites values or expands body text. Build one replacement with existing frontmatter functions, commit once through session buffer/atomic path, and compare exact expected source before changing anything. Validation reads dirty buffers before disk. Existing `note.property_set` and `base.row.set` remain permissive for ordinary Obsidian editing; schema diagnostics explain mismatches rather than silently coercing values. Registry edits never bulk rewrite notes.
|
||||
|
||||
### D3 UI and wiring
|
||||
Properties panel shows type chips, inherited fields and diagnostics; apply type is explicit, with preview of defaults. Base queries consume ordinary note values, so existing filters work unchanged. Registry editor uses existing widgets and translated strings. Register IDs in engine core and assistant READ/WRITE gate, update `CORE_IDS`. No engine calls during painting; refresh on source/registry changes.
|
||||
|
||||
## Acceptance criteria
|
||||
1. Applying Book adds its frontmatter tag and absent defaults as one undo step; unrelated source bytes and existing values remain unchanged.
|
||||
2. A child inherits parent required fields; cycle, unknown parent, incompatible override and incompatible multi-tag schema each return deterministic errors.
|
||||
3. Existing Bases filter on a newly added property returns the note using the shared property/index path.
|
||||
4. Invalid date/number and missing required value produce diagnostics in Source/Visual property UI without blocking unrelated note edits.
|
||||
5. Read-only, stale expected source, corrupt registry and invalid definitions cause zero disk/buffer mutation.
|
||||
6. Unknown properties and plain tags survive apply/remove-schema operations; deleting a definition never deletes note content.
|
||||
|
||||
## Open owner questions
|
||||
- Q1: approve frontmatter tags as type membership, or prefer explicit `types` property? Proposed tags; never infer body tags.
|
||||
- Q2: approve advisory validation rather than blocking all property edits? Proposed advisory preserves external-editor compatibility.
|
||||
- Q3: should registry sync as vault user data? Proposal yes; coordinate existing private-path sync policy first.
|
||||
|
||||
## Feature ideas (researched, ranked)
|
||||
1. Pinned fields and per-type Bases views, inspired by Tana pinned fields: https://outliner.tana.inc/learn/features/supertags .
|
||||
2. Explicit template-body insertion as a separate previewed action, inspired by Tana templates at the same URL; not automatic on tagging.
|
||||
3. Computed display fields through existing Bases formulas, avoiding writes: https://obsidian.md/help/bases/syntax .
|
||||
@@ -0,0 +1,68 @@
|
||||
# Tasks: object-types (roadmap 35)
|
||||
|
||||
Spec: `spec.md`. Ordered tasks depend on preceding tasks only; each finishes with independently runnable behavior tests.
|
||||
|
||||
**First tool call rule:** after at most three focused reads, write FIRST OWNS file in SECOND tool call: compiling stub plus one failing behavior test, then make it pass. New paths below are explicitly marked new; other paths verified in source on 2026-10-21.
|
||||
**Standing VERIFY:** `export PATH="$HOME/.cargo/bin:$PATH"`; `cargo test -p <pkg>`; `cargo clippy -p <pkg> --all-targets -- -D warnings`; `rustfmt --edition 2024 <owned Rust files>`; UI tasks also `cargo xtask i18n` and inspect snapshot PNG at 1440x900 Mocha/Latte. Final task: `cargo xtask ci`. These are FUTURE implementation checks; documentation author runs no cargo.
|
||||
**Standing DO NOT:** edit outside OWNS, mutate Git, add dependencies, blanket lint allows, panic/unchecked input indexing, IO during paint, overwrite malformed user data, bypass atomic vault writes, `cargo fmt --all`. No new crate planned: no root Cargo.toml or xtask/src/layers.rs edits required. If implementation introduces a crate, stop for scope approval then explicitly own root Cargo.toml and xtask/src/layers.rs registration before compiling.
|
||||
|
||||
## Phase 1
|
||||
- [ ] 1. Schema definitions, inheritance and validation (brief `briefs/22-object-type-model.md`).
|
||||
|
||||
OWNS: `crates/bases/src/schema/model.rs` (new, first write), `crates/bases/src/schema/mod.rs` (new), `crates/bases/src/schema/resolve.rs` (new), `crates/bases/src/lib.rs` (schema export), `crates/bases/src/error.rs` (schema errors), unit tests in new modules
|
||||
|
||||
FILES TO OPEN: `spec.md` D1, `crates/bases/src/value.rs`, `crates/bases/src/model.rs`, `crates/bases/src/lib.rs`, `crates/bases/src/error.rs`, `crates/markdown/src/frontmatter/frontmatter_edit/value.rs`, `crates/engine/src/vault/properties.rs`
|
||||
|
||||
GOAL: Pure `TypeRegistry::new(definitions) -> Result<Self>`, `resolve(tags) -> Result<ResolvedSchema>`, and `validate(values) -> Vec<Diagnostic>`. JSON codec belongs in engine task 2 using existing serde_json; Bases has no serde dependency. Use same scalar/list semantics as existing property model. Represent D1 schema, bounded inheritance and deterministic multi-tag merge; no new IO.
|
||||
|
||||
TESTS: Model construction; all model size limits; duplicate ID/tag; missing parent; depth16/17; cycles; inherited required OR; default override; incompatible overrides and multi-tag collisions; bad typed defaults and values; unknown properties preserved.
|
||||
|
||||
VERIFY: standing, `-p notescraft-bases`.
|
||||
|
||||
DO NOT: Register commands or touch property mutation behavior.
|
||||
|
||||
## Phase 2
|
||||
- [ ] 2. Registry persistence and objecttype commands.
|
||||
|
||||
OWNS: `crates/engine/src/object_types/mod.rs` (new), `crates/engine/src/object_types/store.rs` (new), `crates/engine/src/object_types/commands.rs` (new), `crates/engine/src/lib.rs` (module), `crates/engine/src/commands.rs` (registration), `crates/engine/src/ai/assistant_gate.rs`, `crates/engine/tests/core_commands.rs` (CORE_IDS), `crates/engine/tests/object_types.rs` (new)
|
||||
|
||||
FILES TO OPEN: `spec.md` D2, `crates/engine/src/vault/properties.rs`, `crates/engine/src/bases/write.rs`, `crates/engine/src/builtin/properties_commands.rs`, `crates/engine/src/activity/outbox.rs`
|
||||
|
||||
GOAL: Implement all D2 commands; versioned JSON round-trip and unknown-version errors in engine; parse/store registry via private atomic API; apply tag/default frontmatter as single buffer-first commit, expected-text check, assistant gate, read-only guard and ordinary Bases refresh. Parent deletion refused.
|
||||
|
||||
TESTS: Temporary vault tests for save/list/delete parent refusal; apply twice idempotent; dirty source plus undo; stale source zero writes; corrupt registry unchanged; readonly mutation rejected; base.query sees new fields.
|
||||
|
||||
VERIFY: standing, `-p notescraft-engine`.
|
||||
|
||||
DO NOT: Bulk-migrate notes or change permissive base.row.set behavior.
|
||||
|
||||
## Phase 3
|
||||
- [ ] 3. Properties integration and schema editor.
|
||||
|
||||
OWNS: `crates/ui-egui/src/views/properties_view/object_types.rs` (new), `crates/ui-egui/src/views/properties_view/mod.rs` (declare/wire), `crates/ui-egui/src/views/properties_view/rows.rs`, `crates/ui-egui/src/views/properties_view/tests.rs`, `crates/ui-egui/src/text/views/object_types.rs` (new), `crates/ui-egui/src/text/views/mod.rs` (declare), `crates/i18n/locales/en.toml` (append namespace)
|
||||
|
||||
FILES TO OPEN: `spec.md` D3, `crates/ui-egui/src/views/properties_view/controls.rs`, `crates/ui-egui/src/views/properties_view/validation.rs`, `crates/ui-egui/src/views/properties_view/raw.rs`
|
||||
|
||||
GOAL: Type chips, inherited field/error display, explicit apply preview, minimal registry editor within properties view using established widgets; queued commands after paint.
|
||||
|
||||
TESTS: UI tests apply once; cancel zero changes; invalid inherited field visible; editor save/read-only disabled; screenshot both themes; command counters unchanged frame zero.
|
||||
|
||||
VERIFY: standing, `-p notescraft-ui-egui`.
|
||||
|
||||
DO NOT: Hardcode copy or perform engine writes during painting.
|
||||
|
||||
## Phase 4
|
||||
- [ ] 4. Compatibility and status.
|
||||
|
||||
OWNS: `crates/engine/tests/object_types.rs`, `ROADMAP.md`, `docs/ARCHITECTURE.md`, `docs/CODE-MAP.md`, `specs/plugins-parity/roadmap.md` (rank35)
|
||||
|
||||
FILES TO OPEN: `spec.md` acceptance criteria and prior task tests
|
||||
|
||||
GOAL: Complete end-to-end frontmatter/Bases/advisory validation corpus; mark delivered only once owner questions resolved and acceptance evidence recorded.
|
||||
|
||||
TESTS: External-editor malformed data preserved; schema deletion retains notes; query refresh and read-only fixture.
|
||||
|
||||
VERIFY: standing, `-p notescraft-engine`.
|
||||
|
||||
DO NOT: Implement deferred features.
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# PDF annotations and visual excerpts (agy gap 1, roadmap 32)
|
||||
|
||||
## Research (2026-10-21; agy claims checked)
|
||||
[Heptabase](https://heptabase.com/) documents PDF annotation; its [deep links](https://support.heptabase.com/en/articles/11176386-use-deep-links) include PDF pages and highlights. [Obsidian embeds](https://help.obsidian.md/embeds) document page links. These verify demand, not the agy `rect=` syntax: that syntax below is a NotesCraft extension. Logseq/SiYuan feature equivalence was not established and is dropped. The suggested new parser crate is not justified: `crates/pdf/src/lib.rs` is an exporter, not an arbitrary-PDF reader. `crates/engine/src/vault/pdf_preview.rs` already rasterizes via PDFKit/JXA on macOS and `pdftoppm` elsewhere; UI `crates/ui-egui/src/views/media_view/pdf.rs` has paging/zoom but no text geometry or page count. Its initial comment understates current macOS support; inspect implementation.
|
||||
|
||||
## Problem and goal
|
||||
Annotate an existing PDF without rewriting it; select a text passage or page rectangle, attach a comment, copy a linked quote or visual excerpt into a note, and navigate back to exact page/region. Reuse existing PDF crate for pure geometry/sidecar types, engine for IO, existing PDF tab for interaction. No new crate.
|
||||
|
||||
## Design decisions
|
||||
### D1 Portable sidecar and anchors
|
||||
Version 1 sidecar at `.notescraft/pdf-annotations/<sha256-of-vault-relative-path>.json`, via `Vault::write_private` only; original PDF byte-identical. Shape `{version:1,path,source_sha256,annotations:[{id,page,kind,rects,quote,comment,colour}]}`. `page` is 1-based; rectangles `[x,y,width,height]` normalized to displayed crop box, top-left origin after page rotation. `kind` is `highlight` (one or more line rectangles) or `excerpt` (one rectangle). Colour is a theme-independent named token, never arbitrary UI colour. IDs are opaque stable local IDs, unique within document; edits retain IDs. No fresh randomness dependency required for sequential IDs under serialized engine writes.
|
||||
Caps: 4 MiB sidecar, 10,000 annotations, 256 rects per annotation, quote/comment each 16 KiB; finite coordinates, positive area wholly in unit square; page 1..10,000. Unknown version, duplicate ID, invalid kind/geometry return Err before mutation. Corrupt sidecar remains byte-identical. Store source SHA-256 and mark anchors stale on content replacement; never silently remap. Renames migrate path-keyed sidecar atomically with existing rename semantics; failure preserves original association and reports error.
|
||||
### D2 Link and excerpt contract
|
||||
`[[paper.pdf#page=4&annotation=pa_1]]` is stable annotation navigation. `![[paper.pdf#page=4&rect=0.1,0.2,0.3,0.1]]` is derived raster crop, not a mutation to PDF. Plain `#page=4` remains supported. Parse only these recognized keys, reject duplicate/malformed/out-of-range values; do not reinterpret as heading. Stable-ID navigation requires sidecar; standalone rect navigation remains usable without it. Copy quote emits Markdown blockquote plus stable-ID link; excerpt command returns link and bounded PNG via attachment helper only when explicitly saving excerpt. Cache crop by content hash/page/geometry/render version, no source mutation. Missing/stale anchor shows diagnostic and opens page without claiming exact match.
|
||||
### D3 Engine and backend capabilities
|
||||
Commands `pdf.annotations.list {path}`, `pdf.annotations.upsert {path,annotation,expected_revision}`, `pdf.annotations.delete {path,id,expected_revision}`, `pdf.excerpt {path,page,rect,save?}`, `pdf.page_info {path,page}`. Return revision/hash and explicit stale/capability status. Mutation rejects read-only; optimistic revision avoids lost update. `pdf.excerpt` defaults `save=false`; classify entire command WRITE in assistant gate because `save=true` writes an attachment (including when default is used); read-only permits only `save=false`. `list`/`page_info` READ; upsert/delete WRITE. Register in existing viewer group, AI read/write gate, CORE_IDS tests. `pdf.page_info` returns rotated crop geometry and text runs with normalized rectangles only when backend supports them. Existing PNG renderer cannot supply text runs: add bounded native-tool extraction adapter only after Q1 approval; unsupported backend disables text selection with explanation, retains rectangle tools. Never claim raster OCR is original PDF text. Worker performs rendering/extraction; timeout 15 seconds, 200 MiB source cap, 16 MiB PNG cap match existing preview; text <= 2 MiB/page, <= 100,000 runs, cancellation on tab close/change. Original text highlight capability must pass platform fixture gate before feature marked complete.
|
||||
### D4 UI and integration
|
||||
PDF toolbar: Select text / Rectangle / Annotate; keyboard access, selected-region highlight across zoom and scroll, comment editor, delete confirmation, copy quote/excerpt. Engine mutations queued after frame; no IO during paint. Embed renderer draws clipped page texture and click opens page/annotation through shared link dispatch; no arbitrary path access. i18n tables/English keys use existing media-view pattern. No PDF editing/export annotation import, freehand drawing, or OCR in this feature.
|
||||
|
||||
## Acceptance criteria
|
||||
1. Multi-line highlight, comment and rectangle survive reopen; original PDF SHA-256 unchanged; source replacement visibly stale.
|
||||
2. Text copy on supported backend matches fixture text; raster-only backend reports unavailable, never invents text.
|
||||
3. Stable ID opens page and overlays exact region after zoom/scroll; crop round trip works with 90/180/270-degree pages and nonzero crop-box origin.
|
||||
4. Read-only/invalid geometry/stale revision/corrupt JSON reject without any writes; missing renderer produces recoverable error.
|
||||
5. Engine tests cover rename, Unicode path, traversal/symlink rejection, duplicate IDs, malformed fragments, caps and timeout/cancel; existing page links unchanged.
|
||||
6. egui tests and read screenshots cover annotation, crop embed, keyboard navigation, Mocha/Latte and scale 2; no per-frame worker restart.
|
||||
|
||||
## Open owner questions
|
||||
- Q1: Approve native PDFKit text extraction and installed Poppler tools (`pdftotext -bbox` plus geometry inspection) as platform capability contract? Or fund/approve a bundled PDF parser dependency and license review? No dependency added without answer; arbitrary-PDF text highlighting blocked until adapter proven.
|
||||
- Q2: Sync annotation sidecars as user data? Proposal yes, derived raster caches no; revision conflicts remain visible, no last-write overwrite.
|
||||
- Q3: Accept NotesCraft normalized `rect`/`annotation` extension and stale-anchor behavior? No claim of Obsidian rect interoperability.
|
||||
|
||||
## Feature ideas (researched, ranked)
|
||||
1. PDF highlight cards on canvas, informed by [Heptabase PDF annotation](https://heptabase.com/).
|
||||
2. Cross-device highlight deep links, informed by [Heptabase deep links](https://support.heptabase.com/en/articles/11176386-use-deep-links).
|
||||
3. Page-only fallback for other apps, preserving [Obsidian PDF embed syntax](https://help.obsidian.md/embeds).
|
||||
@@ -0,0 +1,46 @@
|
||||
# Tasks: PDF annotations (roadmap 32)
|
||||
|
||||
Spec: `spec.md`. Rules: `AGENTS.md`, `docs/CODE-MAP.md`.
|
||||
One task = one agent; execute in order, dependencies are preceding tasks. Each task has its own test boundary; backend tests use fakes until approval. Existing paths verified with `rg --files` and source reads on 2026-10-21; paths marked new do not exist yet. No new crate; no workspace/layer registration needed. If owner instead approves a new crate, first revise tasks to own its manifest, root `Cargo.toml` and `xtask/src/layers.rs`, package/layer/dependency registration and crate smoke test before use.
|
||||
**First tool call rule:** first tool call reads at most three listed inputs; write FIRST file in OWNS in SECOND tool call, with compiling stub and one failing test, then make it pass. Do not spend second call exploring. For document-only tasks write initial acceptance checklist in second call and review/complete it instead of a code stub.
|
||||
**Standing VERIFY (future implementation only):** `export PATH="$HOME/.cargo/bin:$PATH"`; `cargo test -p <owned-package>`; `cargo clippy -p <owned-package> --all-targets -- -D warnings`; `rustfmt --edition 2024 <owned Rust files>`. UI tasks add `cargo xtask i18n` and render/read snapshot PNG. Last implementation task adds `cargo xtask layers`, `cargo xtask tree`, `cargo xtask ci`. These commands are instructions, not commands run during this spec-writing request.
|
||||
**Standing DO NOT:** touch outside OWNS; change git state; run `cargo fmt --all`; add dependency without owner response; unsafe/panics/unchecked slicing; rewrite source attachments; hardcode UI strings; bypass engine commands. No production claim until platform acceptance passes.
|
||||
|
||||
## Phase 1 — pure model
|
||||
- [ ] 1. Annotation geometry, sidecar and fragment model.
|
||||
OWNS: `crates/pdf/src/annotation.rs` (new, FIRST), `crates/pdf/src/annotation/fragment.rs` (new), `crates/pdf/src/lib.rs` (declare/re-export), `crates/pdf/src/error.rs` (specific validation variant only).
|
||||
FILES TO OPEN: `specs/plugins-parity/pdf-annotations/spec.md` D1–D2; `crates/pdf/src/lib.rs`; `crates/pdf/src/error.rs`.
|
||||
GOAL: `Rect::validate`, `Annotations::parse/to_json`, `PdfTarget::parse/to_fragment`, named caps and typed errors, pure no-IO APIs. Canonical normalized rect and stable annotation ID links, no PDF parser. Use existing serde/serde_json dependencies verified in `crates/pdf/Cargo.toml`; no manifest edit.
|
||||
TESTS: geometry round trip, rotated geometry mapping fixture, NaN/infinity, zero/negative/outside bounds, page 0/10001, duplicate ID, caps, unknown version, duplicate fragment keys, existing `page=3`, Unicode target and unknown fragment refusal.
|
||||
VERIFY: standing `-p notescraft-pdf`; tests named `annotation` execute, exporter tests unchanged.
|
||||
DO NOT: IO, engine registration, parser dependency, modify PDF export pipeline.
|
||||
## Phase 2 — engine and backend
|
||||
- [ ] 2. Sidecar commands, revisions and crop worker.
|
||||
OWNS: `crates/engine/src/viewers/pdf_annotations.rs` (new, FIRST), `crates/engine/src/viewers/pdf_crop.rs` (new), `crates/engine/src/viewers/mod.rs`, `crates/engine/src/ai/assistant_gate.rs`, `crates/engine/tests/core_commands.rs`, `crates/engine/tests/pdf_annotations.rs` (new), `crates/engine/src/vault/notes.rs` (file rename association hook only), `crates/engine/src/vault/folders.rs` (folder rename association hook only), `crates/engine/src/vault/external.rs` (external move association only).
|
||||
FILES TO OPEN: spec D1–D3; `crates/engine/src/viewers/annotations.rs`; `crates/engine/src/viewers/info.rs`; later `crates/engine/src/vault/attachments.rs`, `crates/engine/tests/common/mod.rs`.
|
||||
GOAL: register list/upsert/delete/excerpt commands; excerpt defaults save=false but entire assistant command WRITE because save=true persists; read-only refuses save=true only; source-hash stale detection, optimistic revision checks, atomic private persistence, bounded crop work using existing PNG tooling; save crop via attachment helper. Propagate persistence errors; do not copy image-sidecar `.bad` rewrite behavior because PDF spec preserves corrupt input.
|
||||
TESTS: TempVault round trip; byte-identical PDF; corrupt sidecar unchanged; concurrent revision rejected; traversal/symlink; rename/replacement; read-only; crop pixel boundaries and cap; fake worker cancellation.
|
||||
VERIFY: standing engine; `cargo test -p notescraft-engine --test pdf_annotations --test core_commands`.
|
||||
DO NOT: native backend dependency or silent source remap; call engine during paint.
|
||||
- [ ] 3. PDF geometry/text capability adapter (blocked on Q1).
|
||||
OWNS: `crates/engine/src/viewers/pdf_backend.rs` (new, FIRST), `crates/engine/src/viewers/pdf_annotations.rs`, `crates/engine/src/viewers/mod.rs`, `crates/engine/src/vault/pdf_preview.rs` (shared bounded worker boundary only), `crates/engine/tests/pdf_backend.rs` (new), `specs/plugins-parity/pdf-annotations/backend-decision.md` (new).
|
||||
FILES TO OPEN: spec D3/Q1; `crates/engine/src/vault/pdf_preview.rs`; `crates/engine/src/viewers/mod.rs`.
|
||||
GOAL: record approved native-tool scope first; register page_info; rotated crop geometry/text runs through bounded worker. Unsupported backend explicitly lacks text selection. If approved approach needs dependency, stop and amend owned files/approval before edits; no mock masquerading as implementation.
|
||||
TESTS: synthetic adapter output limits/errors/timeouts; native fixtures selectable text, scanned page, rotated/cropped page, missing executable and page range. Confirm actual macOS and supported Poppler behavior.
|
||||
VERIFY: standing engine, backend fixture suite on each supported platform; record unsupported targets.
|
||||
DO NOT: invent text from raster; unsafe, unapproved dependency, hidden fallback.
|
||||
## Phase 3 — PDF tab and linked embeds
|
||||
- [ ] 4. Annotation tools, embed crops and shared navigation.
|
||||
OWNS: `crates/ui-egui/src/views/media_view/pdf_annotations.rs` (new, FIRST), `crates/ui-egui/src/views/media_view/pdf.rs`, `crates/ui-egui/src/views/media_view/mod.rs`, `crates/ui-egui/src/views/note_view/blocks/embed.rs`, `crates/ui-egui/src/views/note_view/blocks/embed_data.rs`, `crates/ui-egui/src/shell/app/window/views.rs`, `crates/ui-egui/src/text/views/media_view/mod.rs`, `crates/i18n/locales/en.toml` (append keys), `crates/ui-egui/tests/pdf_annotations.rs` (new).
|
||||
FILES TO OPEN: spec D2/D4; `crates/ui-egui/src/views/media_view/pdf.rs`; `crates/ui-egui/src/views/note_view/blocks/embed.rs`; later open remaining owned dispatch files before edit.
|
||||
GOAL: keyboard-accessible region/text selection, comments, copy quote/excerpt, delete confirmation, stale/unsupported messages; route PDF fragment actions to tab state; use viewport-to-normalized mapping independent of zoom. Queue engine work after frame.
|
||||
TESTS: kittest select/comment/reopen; crop click page jump; rotated coordinates; unsupported text; failed save retains input; frame counter prevents repeated jobs; old page links unchanged.
|
||||
VERIFY: standing ui-egui + PNG snapshots Mocha/Latte/scale 2 read visually.
|
||||
DO NOT: raw UI patterns contrary to existing shared controls, IO in paint, claim Obsidian rect compatibility.
|
||||
- [ ] 5. Acceptance evidence and status.
|
||||
OWNS: `specs/plugins-parity/pdf-annotations/acceptance.md` (new, FIRST), `specs/plugins-parity/roadmap.md` (rank 32 status only), `ROADMAP.md`, `docs/ARCHITECTURE.md`, `docs/CODE-MAP.md` (PDF status/API only).
|
||||
FILES TO OPEN: spec acceptance; completed test reports; existing docs relevant PDF sections.
|
||||
GOAL: record criterion-to-test/platform evidence, unresolved owner decisions and accurate capability table; mark complete only when mandatory text backend and UI tests pass.
|
||||
TESTS: trace every acceptance item to passing assertion/visual evidence, no unsupported-platform claim.
|
||||
VERIFY: final standing gates. Documentation-only spec-writing request must NOT execute this future task or edit outside `specs/plugins-parity/`.
|
||||
DO NOT: broaden feature, mark approval-gated backend done.
|
||||
@@ -24,7 +24,7 @@ of the daily workflow it changes. "Brief" = ready to run in `briefs/`.
|
||||
| 16 | Icons and banners from properties (`icon`, `color`, `banner`) in Explorer, tabs, note top | Iconize 2.3M, Banners, Pretty Properties | M | high | 1 (shares property cell code) | - |
|
||||
| 17 | Importer / Format converter (Notion zip, Evernote ENEX, Bear, HTML, Roam JSON) | Importer 1.8M, core Format converter | L | high (adoption) | none | spec first |
|
||||
| 18 | Git remotes and auto-commit timer (shell out to `git`) | Git 3.3M | M | high | none | - |
|
||||
| 19 | Slides view (Markdown to slides, present mode) | Advanced Slides 0.84M, core Slides | L | medium | none | spec first |
|
||||
| 19 | Slides view (superseded by scoped spec/tasks at rank 41) | Advanced Slides 0.84M, core Slides | L | medium | none | spec first |
|
||||
| 20 | Charts block and Calendar heatmap | Charts 0.33M, Heatmap Calendar | M | medium | 1 (summaries) | - |
|
||||
| 21 | Mind map diagram kind + "outline as mind map" | Mind Map 0.9M | M | medium | none | - |
|
||||
| 22 | Latex Suite snippets for `$..$` | Latex Suite 0.68M | M | medium | none | - |
|
||||
@@ -38,6 +38,18 @@ of the daily workflow it changes. "Brief" = ready to run in `briefs/`.
|
||||
| 30 | Link bookmark cards from OpenGraph, cached under `.notescraft/cache/web/`, opt-in network (`link-cards/`) | Notion, Capacities, Craft; Obsidian card plugins | M | medium (privacy-gated) | 29 task 1 (tokenizer) | 17 |
|
||||
| 31 | Spaced repetition: cards in notes, SM-2, review tab (`spaced-repetition/`, new crate `srs`; replaces row 23) | Spaced Repetition 0.5M (listing 498k), Logseq/RemNote | L | medium | none | 18 |
|
||||
|
||||
|
||||
| 32 | PDF deep annotations, stable links and visual excerpts (`pdf-annotations/`) | PDF annotation workflows; native text capability gated | L | high | existing pdf export + viewer backend; owner backend decision | [19](briefs/19-pdf-annotation-model.md) |
|
||||
| 33 | Web clipper: scoped local HTTP capture bridge, local assets and bookmarklet/extension contract (`web-clipper/`) | Official Obsidian Clipper workflow | M | high | existing engine capture/control; external adapter contract | [20](briefs/20-web-capture-envelope.md) |
|
||||
| 34 | Per-note and secret-folder encryption **design only** (`note-encryption/`) | Local at-rest lock; distinct from sync E2EE | L | high (privacy) | owner threat/format/key/dependency/security review; no implementation authorized | [21](briefs/21-encryption-threat-model.md) |
|
||||
| 35 | Supertags/object types over Bases properties (`object-types/`) | Tana-inspired typed notes; plain frontmatter retained | M | high | 1; owner membership/validation policy | [22](briefs/22-object-type-model.md) |
|
||||
| 36 | Synced blocks: editable `![[note#^id]]` source transclusion (`synced-blocks/`) | Notion-style source-owned edits | M | high | existing blocks + embed resolver + Session undo | [23](briefs/23-synced-block-target.md) |
|
||||
| 37 | Offline attachment OCR and typed search hits (`attachment-ocr/`) | Joplin-style attachment search | L | high | owner safe Vision/Windows bindings + packaging; 32 text backend optional | [24](briefs/24-ocr-contract.md) |
|
||||
| 38 | Global quick-capture hotkey and compact window (`global-quick-capture/`) | Apple Quick Note workflow | M | high | existing note.capture; owner native hotkey dependency/OS scope | [25](briefs/25-quick-capture-model.md) |
|
||||
| 39 | Zotero/BibTeX citation picker, local export refresh and optional CSL export (`citations/`) | Academic citation workflow; no bidirectional Zotero sync | M | medium-high | local bibliography; 8 for optional approved Pandoc CSL backend | [26](briefs/26-citation-model.md) |
|
||||
| 40 | Opt-in formulas inside Markdown pipe tables (`table-formulas/`) | Small spreadsheet workflow; own syntax, not Advanced Tables compatibility | M | medium | existing blocks/Markdown table ranges | [27](briefs/27-table-formula-model.md) |
|
||||
| 41 | Markdown slides and native presentation mode (`markdown-slides/`; replaces row 19) | Obsidian core Slides workflow | M | medium | existing Markdown parser and note renderer | [28](briefs/28-slide-model.md) |
|
||||
|
||||
## Reasoning
|
||||
|
||||
- **Bases first, and its pieces are reused**: the expression language is the engine of Templater (12),
|
||||
@@ -51,12 +63,26 @@ of the daily workflow it changes. "Brief" = ready to run in `briefs/`.
|
||||
owner wants third parties; first-party features stay in-tree behind core-plugin cards, which ships faster
|
||||
and safer.
|
||||
- **Skipped on purpose** (see `community.md`): Style Settings, BRAT, Buttons, Shell commands, Commander macros
|
||||
(JS or shell execution), Remotely Save (cloud adapters), OCR (new dependency).
|
||||
(JS or shell execution), Remotely Save (cloud adapters). OCR now has approval-gated design at rank 37; no dependency is approved by this roadmap.
|
||||
- **Mismatch to fix in the docs**: SPEC 6.1 still lists Bases, Dataview-style queries and Slides as deferred;
|
||||
update it when items 1, 11 and 19 start.
|
||||
update it when items 1, 11 and 41 (superseding 19) start.
|
||||
|
||||
## Network policy reminder
|
||||
|
||||
Only three items in this roadmap touch the network, all opt-in: Maps tiles (item 2, spec D13), link cards
|
||||
(item 30, `link-cards/spec.md` D1, setting off by default, never automatic) and, later, Git remotes (item 18, the user's own remote). Nothing else may fetch (no remote cover images in Cards, no
|
||||
title fetching for pasted links) without its own spec section and a settings switch.
|
||||
Network access remains explicit and opt-in: Maps tiles (2, Bases D13), link cards
|
||||
(30, `link-cards/spec.md` D1, setting off by default) and Git remotes (18, user's own remote).
|
||||
Web clipper (33) adds an off-by-default **loopback-only** HTTP bridge on the control lifecycle,
|
||||
with capture-only credentials; browser adapter fetches assets only after explicit browser permission.
|
||||
The app does not scrape source URLs or fetch remote images. Clipboard bookmarklet import works without
|
||||
HTTP. Citations (39) refreshes local exports only; Zotero cloud API is a future owner decision.
|
||||
OCR (37) and slides (41) add no remote service. Other new fetching needs its own specification and switch.
|
||||
|
||||
## Gap-research follow-up (ranks 32–41)
|
||||
|
||||
Each linked spec records verified primary sources, corrections/dropped Gemini claims, acceptance
|
||||
criteria, owner questions and researched feature ideas. No research estimate implies approval or
|
||||
shipped status. All new rows remain planned. Encryption is a document-only review phase; OCR, PDF
|
||||
text extraction and native global shortcuts have explicit platform/dependency gates. These plans
|
||||
reuse existing crates; any later new crate requires a revised task owning root `Cargo.toml` and
|
||||
`xtask/src/layers.rs` registration before use. Briefs 19–28 are first-task handoffs, not authorization
|
||||
to execute implementation during this documentation-only request.
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# Synced blocks: editable transclusion (agy gap 7, roadmap 36)
|
||||
|
||||
Reuse `crates/blocks`, engine block edit/buffers and current embed resolver. No new crate or dependency.
|
||||
|
||||
## Research (2026-10-21; agy claims checked)
|
||||
Notion confirms editing one synced instance changes other instances and edit access belongs to source: https://www.notion.com/help/synced-blocks . Adopt source ownership and propagation, not its cloud/copy storage format. Drop unverified product-list parity and agy's specific issue-number attribution. Existing NotesCraft embed rendering already resolves note/heading/block content, loads dirty buffers first and limits nesting; editable targets are the missing behavior, not transclusion itself.
|
||||
|
||||
## Problem and goal
|
||||
`![[note#^id]]` displays a source block but requires navigation to edit. An explicit Edit source block action edits that source from any embed while retaining one source markdown block and ordinary Obsidian syntax.
|
||||
|
||||
## Design decisions
|
||||
### D1 Pure block target
|
||||
Add `crates/blocks/src/transclusion.rs`, export from `lib.rs`. Resolve existing IDs using markdown block-id items and `parse_blocks`, not substring matching. Return `{id,block_range,content_range,marker_range}`; `replace_by_id(source,id,expected_block,replacement)` returns `Edited`. Support paragraphs, quote/callout/list blocks and standalone IDs following table/list/quote where supported by markdown contract. Preserve ID marker and line endings. Reject duplicate IDs, missing IDs, invalid UTF-8 offsets, oversized replacement and a replacement that injects/rebinds target IDs. Replacement is markdown content; preserve required structural markers or return unsupported. Keep unsupported targets readable with Open source action.
|
||||
|
||||
### D2 Engine contract
|
||||
`block.transclusion.read {from,target}` resolves wiki target through index relative to host; target must contain a `#^id` fragment; result `{path,id,block,expected_block,editable,reason?}`. `block.transclusion.replace {path,id,expected_block,replacement}` -> `{path,id,changed}`. Exact expected block is optimistic concurrency token (not a noncryptographic hash); resolve ID again against current buffer-first text at commit. Changes elsewhere in source may proceed, changed/deleted target returns `conflict`, duplicates return `ambiguous_block`. Serialize through session command dispatch; never read source then overwrite entire stale file. Use existing `lint_commit`/buffer edit flow so open target remains dirty, one undo step; closed target writes atomically through existing session path. Preserve source id and host embed bytes. Writes are assistant-gated. Validate vault paths and read-only policy before reading/editing.
|
||||
|
||||
### D3 UI and propagation
|
||||
Existing `views/note_view/blocks/embed_data.rs` preloads source; `embed.rs` paints it. New edit state keyed by `(host path, resolved source path, block id, embed occurrence)`; no edits against rendered host offsets. Edit button shows source path; explicit Apply/Cancel, draft retained on conflict. Only one active draft per source target; another instance focuses it. After commit invalidate all embed caches for that path, including closed-file watcher changes, source tabs and split panes. No filesystem calls while painting. Self/cyclic embeds retain current depth 3 limit, additionally track `(path,id)` to show cycle placeholder. Cache never turns truncated source into editable target; target >1 MiB is read-only with explanation. Draft cap 1 MiB; 32 source notes and depth 3 remain bounded.
|
||||
|
||||
## Acceptance criteria
|
||||
1. Two host notes embed one paragraph: editing either updates source and both renderings; host markdown byte-identical.
|
||||
2. Open source dirty buffer is authoritative; undo in source restores previous block, propagated everywhere.
|
||||
3. Stale target returns conflict without writes; unrelated source changes remain intact when ID resolves at a new offset.
|
||||
4. Missing/duplicate ID, readonly vault, malformed target, cycle and deleted source display diagnostics and never panic/write.
|
||||
5. CRLF, Unicode, nested list/quote, and standalone table ID round-trip with marker preserved; unsupported targets remain readable.
|
||||
6. Changing a closed source externally refreshes cached embeds; no repeated engine calls per row/frame.
|
||||
|
||||
## Open owner questions
|
||||
- Q1: approve explicit Apply/Cancel v1 versus keystroke-level autosave? Proposal explicit, matching conflict/undo semantics.
|
||||
- Q2: approve source-buffer undo ownership? Proposal source owns undo; host offers Open source, no second undo log.
|
||||
|
||||
## Feature ideas (researched, ranked)
|
||||
1. Show other embedding notes and navigate to source, inspired by Notion's editing-location list: https://www.notion.com/help/synced-blocks .
|
||||
2. Explicit Detach copy (host edit only), inspired by Notion unsync at the same URL; defer until safe source-range serialization exists.
|
||||
3. Multi-block targets require separate stable identity contract; do not copy Notion cloud identifiers into markdown.
|
||||
@@ -0,0 +1,68 @@
|
||||
# Tasks: synced-blocks (roadmap 36)
|
||||
|
||||
Spec: `spec.md`. Ordered tasks depend on preceding tasks only; each finishes with independently runnable behavior tests.
|
||||
|
||||
**First tool call rule:** after at most three focused reads, write FIRST OWNS file in SECOND tool call: compiling stub plus one failing behavior test, then make it pass. New paths below are explicitly marked new; other paths verified in source on 2026-10-21.
|
||||
**Standing VERIFY:** `export PATH="$HOME/.cargo/bin:$PATH"`; `cargo test -p <pkg>`; `cargo clippy -p <pkg> --all-targets -- -D warnings`; `rustfmt --edition 2024 <owned Rust files>`; UI tasks also `cargo xtask i18n` and inspect snapshot PNG at 1440x900 Mocha/Latte. Final task: `cargo xtask ci`. These are FUTURE implementation checks; documentation author runs no cargo.
|
||||
**Standing DO NOT:** edit outside OWNS, mutate Git, add dependencies, blanket lint allows, panic/unchecked input indexing, IO during paint, overwrite malformed user data, bypass atomic vault writes, `cargo fmt --all`. No new crate planned: no root Cargo.toml or xtask/src/layers.rs edits required. If implementation introduces a crate, stop for scope approval then explicitly own root Cargo.toml and xtask/src/layers.rs registration before compiling.
|
||||
|
||||
## Phase 1
|
||||
- [ ] 1. Resolve stable block IDs and replace source content (brief `briefs/23-synced-block-target.md`).
|
||||
|
||||
OWNS: `crates/blocks/src/transclusion.rs` (new, first write), `crates/blocks/src/lib.rs` (module, exports and error variants), unit tests in new module
|
||||
|
||||
FILES TO OPEN: `spec.md` D1, `crates/blocks/src/ids.rs`, `crates/blocks/src/parse.rs`, `crates/markdown/src/scanning/scan.rs`, `crates/blocks/src/lib.rs`, `crates/ui-egui/src/views/note_view/blocks/embed_data.rs`
|
||||
|
||||
GOAL: `resolve_by_id(source,id) -> Result<BlockTarget>` with block/content/marker ranges; `replace_by_id(source,id,expected_block,replacement) -> Result<Edited>`. Parse real markdown IDs and blocks, preserve ID/structure/CRLF, reject ambiguity and changed target, enforce 1 MiB replacement cap.
|
||||
|
||||
TESTS: Paragraph Unicode/CRLF; list/quote and standalone table ID; ID-like code not target; duplicate/missing ID; unrelated offset shift; stale expected block; marker injection; unsupported structural replacement; over-cap refusal; unrelated bytes unchanged.
|
||||
|
||||
VERIFY: standing, `-p notescraft-blocks`.
|
||||
|
||||
DO NOT: Do IO, register commands or implement own wikilink path resolver.
|
||||
|
||||
## Phase 2
|
||||
- [ ] 2. Buffer-first transclusion commands.
|
||||
|
||||
OWNS: `crates/engine/src/block_edit/transclusion.rs` (new), `crates/engine/src/block_edit/mod.rs` (module/spec registration), `crates/engine/src/ai/assistant_gate.rs`, `crates/engine/tests/core_commands.rs` (CORE_IDS), `crates/engine/tests/transclusion.rs` (new)
|
||||
|
||||
FILES TO OPEN: `spec.md` D2, `crates/engine/src/block_edit/mod.rs`, `crates/engine/src/vault/buffers.rs`, `crates/engine/src/lint/fixes.rs`, `crates/ui-egui/src/views/note_view/blocks/embed_data.rs`
|
||||
|
||||
GOAL: Implement read/replace contracts with shared index resolve, strict path validation, expected-block check at serial commit and source-owned undo through lint_commit. Classify read READ, replacement WRITE.
|
||||
|
||||
TESTS: Two hosts one source; dirty source wins over disk; undo; stale/missing/ambiguous/read-only zero mutation; unrelated edits retained; closed source atomic persistence; host bytes unchanged.
|
||||
|
||||
VERIFY: standing, `-p notescraft-engine`.
|
||||
|
||||
DO NOT: Copy rendered host ranges into source or bypass session commit.
|
||||
|
||||
## Phase 3
|
||||
- [ ] 3. Editable embeds and cache invalidation.
|
||||
|
||||
OWNS: `crates/ui-egui/src/views/note_view/blocks/synced.rs` (new), `crates/ui-egui/src/views/note_view/blocks.rs` (module), `crates/ui-egui/src/views/note_view/blocks/embed.rs`, `crates/ui-egui/src/views/note_view/blocks/embed_data.rs`, `crates/ui-egui/src/shell/app/editor/blocks.rs`, `crates/ui-egui/src/shell/app/editor/blocks_tests.rs`, `crates/ui-egui/src/text/views/blocks.rs`, `crates/i18n/locales/en.toml` (append synced-block keys)
|
||||
|
||||
FILES TO OPEN: `spec.md` D3, `crates/ui-egui/src/views/note_view/blocks/embed.rs`, `crates/ui-egui/src/views/note_view/blocks/embed_data.rs`, `crates/ui-egui/src/shell/app/editor/blocks.rs`
|
||||
|
||||
GOAL: Wire edit draft and Apply/Cancel to source command; queue after paint; deduplicate drafts per source; refresh embeds including closed-file watch changes and source undo; bounded cycle/oversize placeholders and conflict draft retention.
|
||||
|
||||
TESTS: UI two embeds reflect edit; one draft per target; cancel; conflict keeps draft; source undo refresh; external closed-source write invalidation; self-cycle; same source in split panes; no IO during paint.
|
||||
|
||||
VERIFY: standing, `-p notescraft-ui-egui`.
|
||||
|
||||
DO NOT: Change existing note/heading embeds to editable targets without block identity.
|
||||
|
||||
## Phase 4
|
||||
- [ ] 4. Acceptance and status.
|
||||
|
||||
OWNS: `crates/engine/tests/transclusion.rs`, `ROADMAP.md`, `docs/ARCHITECTURE.md`, `docs/CODE-MAP.md`, `specs/plugins-parity/roadmap.md` (rank36)
|
||||
|
||||
FILES TO OPEN: `spec.md` acceptance criteria and prior tests
|
||||
|
||||
GOAL: Complete source ownership, cycle and conflict regression corpus; document exact supported targets and undo semantics.
|
||||
|
||||
TESTS: Host/source byte preservation with all supported block fixtures and readonly source.
|
||||
|
||||
VERIFY: standing, `-p notescraft-engine`.
|
||||
|
||||
DO NOT: Add detach/multi-block sync.
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# Markdown pipe-table formulas (agy gap 15, roadmap 40)
|
||||
|
||||
Pure calculation in existing `crates/blocks`; engine command and current markdown table UI. No new crate/dependency. Bases computed columns stay separate.
|
||||
|
||||
## Research (2026-10-21; agy claims checked)
|
||||
agy's Advanced Tables URL is WRONG (404). Correct project: https://github.com/tgrosinger/advanced-tables-obsidian . Its own help says formulas are on the line following a table, evaluated explicitly: https://raw.githubusercontent.com/tgrosinger/advanced-tables-obsidian/main/docs/help.md . Thus agy's `=SUM(B1:B5)` cells are a requested design, not verified Advanced Tables syntax. Drop Notion spreadsheet-equivalence claim; a database formula property is a different model. No claim of full Excel/Org/Advanced Tables compatibility.
|
||||
|
||||
## Problem and goal
|
||||
Small budgets in markdown pipe tables need simple arithmetic without moving into a database. Formula cells remain literal markdown source; Visual mode shows calculated values and exposes formula source on edit. Source/Obsidian retain formulas legibly.
|
||||
|
||||
## Design decisions
|
||||
### D1 Opt-in syntax and coordinates
|
||||
Enable only a table immediately followed by `<!-- notescraft:table-formulas v1 -->` on next line, so existing `=...` prose is unchanged. Numeric data-row coordinates: A1 is first body row, first column; header and delimiter rows are excluded. Formula cell starts `=` after whitespace trim; literal text starting `=` uses inline code. Numbers are finite signed decimal with optional exponent, decimal dot only. Empty referenced cell is 0; nonnumeric text is a typed error, never silently parsed from formatted markup. Grammar: arithmetic `+ - * /`, unary sign, parentheses, cell refs and rectangular ranges only as aggregate args; `SUM`, `AVG`, `MIN`, `MAX`, `COUNT`. COUNT counts numeric cells, ignores empty cells, errors on text. AVG/MIN/MAX empty range errors. No dates, strings, scripts, network, external/cross-table refs, implicit locale conversions or volatile functions.
|
||||
|
||||
### D2 Evaluation and safety
|
||||
New `crates/blocks/src/table_formulas/{mod,parse,eval}.rs` accepts cell-source grid, emits per-cell `Number` or `{code,span}` error. Dependency graph evaluates once per source change, detects cycles and propagates dependency errors; no recursive traversal without bounds. Caps: 10,000 cells, 1 KiB per formula, 128 AST depth, 100,000 dependency edges/expanded-range visits and 1,000,000 evaluation operations per table. Exceeding cap is explicit error, never truncation. Out-of-grid refs, division by zero, nonfinite arithmetic, unknown function and cycles have stable codes. Display finite numbers with Rust shortest-roundtrip formatting; no promise of exact decimal accounting. Row/column insert/delete preserves formula text verbatim: references are positional and may change result; UI warns before structural edit of enabled table, no hidden Excel-style rewrite.
|
||||
|
||||
### D3 Commands/UI and source integrity
|
||||
`table.formulas.evaluate {markdown,table_index}` -> `{cells:[{row,column,value?,error?}],diagnostics}` read-only; zero-based table_index, one-based result coordinates. Engine uses shared pure table extraction based on markdown source ranges; UI parser at `views/note_view/blocks/table.rs` delegates cell splitting to pure module where necessary, preserving existing rendering. No second table interpretation for engine/UI. Read-only Visual evaluation cached by exact table source plus opt-in marker; formula editor always shows original literal. Editing goes through existing `NoteEdit`, never writes computed values. Header/body rendering exposes localized error tooltip with source span. Marker absent -> exact existing behavior. CSV/export remains raw formula source in v1, visibly documented; no automatic spreadsheet execution export. Engine registry and assistant gate classify evaluate READ.
|
||||
|
||||
## Acceptance criteria
|
||||
1. Opted-in table `A1=2`, `B1=3`, `C1 = A1*B1`, aggregate over a numeric range render correct values while source stays byte-identical.
|
||||
2. Marker absent and inline-code `=literal` cells keep existing rendering and never evaluate.
|
||||
3. Cycle, unknown function, text operand, out-of-bounds, division by zero, overflow and every cap return bounded diagnostics without panic.
|
||||
4. Unicode text, escaped pipes, code-span pipes, CRLF and short rows produce identical coordinates in command and Visual view.
|
||||
5. Changing one literal updates dependent results; unchanged frame performs no reparse/evaluation (counter test).
|
||||
6. Editing/undo retains formula syntax, table marker and unrelated bytes; structural edit requires confirmation and preserves refs verbatim.
|
||||
|
||||
## Open owner questions
|
||||
- Q1: approve opt-in comment and A1 body-only coordinates? Proposal yes; explicitly NotesCraft syntax, not Advanced Tables compatibility.
|
||||
- Q2: approve finite f64 v1 rather than accounting-grade decimal? New decimal dependency needs owner approval before addition.
|
||||
- Q3: approve positional references unchanged on structural edits, or require reference rewrite before release? Proposal warning plus unchanged formulas.
|
||||
|
||||
## Feature ideas (researched, ranked)
|
||||
1. Separate explicit materialize-results command with preview, inspired by Advanced Tables explicit evaluation: https://raw.githubusercontent.com/tgrosinger/advanced-tables-obsidian/main/docs/help.md .
|
||||
2. Import Advanced Tables trailing formulas only after a compatibility spec against upstream syntax: https://github.com/tgrosinger/md-advanced-tables/blob/main/docs/formulas.md . URL is linked from verified help; syntax has not been validated for this v1.
|
||||
3. Offer Bases computed properties when table grows into a note database: https://obsidian.md/help/bases/syntax .
|
||||
@@ -0,0 +1,68 @@
|
||||
# Tasks: table-formulas (roadmap 40)
|
||||
|
||||
Spec: `spec.md`. Ordered tasks depend on preceding tasks only; each finishes with independently runnable behavior tests.
|
||||
|
||||
**First tool call rule:** after at most three focused reads, write FIRST OWNS file in SECOND tool call: compiling stub plus one failing behavior test, then make it pass. New paths below are explicitly marked new; other paths verified in source on 2026-10-21.
|
||||
**Standing VERIFY:** `export PATH="$HOME/.cargo/bin:$PATH"`; `cargo test -p <pkg>`; `cargo clippy -p <pkg> --all-targets -- -D warnings`; `rustfmt --edition 2024 <owned Rust files>`; UI tasks also `cargo xtask i18n` and inspect snapshot PNG at 1440x900 Mocha/Latte. Final task: `cargo xtask ci`. These are FUTURE implementation checks; documentation author runs no cargo.
|
||||
**Standing DO NOT:** edit outside OWNS, mutate Git, add dependencies, blanket lint allows, panic/unchecked input indexing, IO during paint, overwrite malformed user data, bypass atomic vault writes, `cargo fmt --all`. No new crate planned: no root Cargo.toml or xtask/src/layers.rs edits required. If implementation introduces a crate, stop for scope approval then explicitly own root Cargo.toml and xtask/src/layers.rs registration before compiling.
|
||||
|
||||
## Phase 1
|
||||
- [ ] 1. Bounded formula grammar and cell dependency evaluation (brief `briefs/27-table-formula-model.md`).
|
||||
|
||||
OWNS: `crates/blocks/src/table_formulas/mod.rs` (new, first write), `crates/blocks/src/table_formulas/parse.rs` (new), `crates/blocks/src/table_formulas/eval.rs` (new), `crates/blocks/src/lib.rs` (module/export and errors), unit tests in new modules
|
||||
|
||||
FILES TO OPEN: `spec.md` D1/D2, `crates/blocks/src/lib.rs`, `crates/bases/src/eval/ops.rs` (finite number handling reference only), `crates/ui-egui/src/views/note_view/blocks/table.rs`
|
||||
|
||||
GOAL: `evaluate(cells: &[Vec<String>]) -> Result<Evaluation>` with stable cell diagnostics; parse formulas into bounded AST, compute dependency DAG iteratively, no scripts or IO. Implement D1 coordinate/numeric/aggregate semantics and every D2 resource cap.
|
||||
|
||||
TESTS: Arithmetic precedence/unary; A1 body coords; ranges; each aggregate including empty/text; cycles and propagation; nonfinite/divzero/out-of-bounds; depth/formula/cell/edge/operation caps; deterministic results; raw cell strings unchanged.
|
||||
|
||||
VERIFY: standing, `-p notescraft-blocks`.
|
||||
|
||||
DO NOT: Add dependency or reuse Bases expression grammar as if A1 were note property syntax.
|
||||
|
||||
## Phase 2
|
||||
- [ ] 2. Shared table extraction and read-only command.
|
||||
|
||||
OWNS: `crates/blocks/src/table_formulas/table.rs` (new), `crates/blocks/src/table_formulas/mod.rs` (declaration/export), `crates/engine/src/block_edit/formulas.rs` (new), `crates/engine/src/block_edit/mod.rs` (module/registration), `crates/engine/src/ai/assistant_gate.rs`, `crates/engine/tests/core_commands.rs` (CORE_IDS), `crates/engine/tests/table_formulas.rs` (new)
|
||||
|
||||
FILES TO OPEN: `spec.md` D3, `crates/ui-egui/src/views/note_view/blocks/table.rs`, `crates/ui-egui/src/views/note_view/blocks/table/edit.rs`, `crates/blocks/src/parse.rs`
|
||||
|
||||
GOAL: Extract shared cell tokenizer/ranges and opt-in marker detection; implement table.formulas.evaluate JSON contract; reject unmarked table as not_enabled, malformed/table_index bad params with zero writes.
|
||||
|
||||
TESTS: Escaped/code-span pipes; Unicode/CRLF/short row normalization; marked/unmarked; malformed marker; multiple tables/index bounds; readonly works; engine coordinates equal pure fixtures; disk untouched.
|
||||
|
||||
VERIFY: standing, `-p notescraft-engine`.
|
||||
|
||||
DO NOT: Materialize computed values or alter source.
|
||||
|
||||
## Phase 3
|
||||
- [ ] 3. Visual formula rendering and editing.
|
||||
|
||||
OWNS: `crates/ui-egui/src/views/note_view/blocks/table/formulas.rs` (new), `crates/ui-egui/src/views/note_view/blocks/table.rs` (module and shared extraction), `crates/ui-egui/src/views/note_view/blocks/table/edit.rs`, `crates/ui-egui/src/text/views/blocks.rs`, `crates/i18n/locales/en.toml` (append formula keys), tests in owned table modules
|
||||
|
||||
FILES TO OPEN: `spec.md` D3, `crates/ui-egui/src/views/note_view/blocks/table.rs`, `crates/ui-egui/src/views/note_view/blocks/table/edit.rs`
|
||||
|
||||
GOAL: Render computed display/error tooltip only when opted in; exact-source cache; raw formula editing with existing NoteEdit/undo; confirm structural edits and preserve references; render literal code uncomputed.
|
||||
|
||||
TESTS: UI values versus formula editor; edit/undo raw bytes; unmarked snapshot unchanged; structural confirm/cancel; source/engine coordinate parity; unchanged frame evaluation counter zero; both-theme screenshot.
|
||||
|
||||
VERIFY: standing, `-p notescraft-ui-egui`.
|
||||
|
||||
DO NOT: Rewrite refs, write result values automatically or export executable CSV silently.
|
||||
|
||||
## Phase 4
|
||||
- [ ] 4. Compatibility corpus and status.
|
||||
|
||||
OWNS: `crates/engine/tests/table_formulas.rs`, `ROADMAP.md`, `docs/ARCHITECTURE.md`, `docs/CODE-MAP.md`, `specs/plugins-parity/roadmap.md` (rank40)
|
||||
|
||||
FILES TO OPEN: `spec.md` acceptance criteria
|
||||
|
||||
GOAL: Close acceptance with hostile-input and source preservation fixtures; document NotesCraft-only opt-in syntax, f64 limits and raw export.
|
||||
|
||||
TESTS: All caps and error codes; marker remains after edit; unmarked tables unaffected; source byte checks.
|
||||
|
||||
VERIFY: standing, `-p notescraft-engine`.
|
||||
|
||||
DO NOT: Claim Excel/Advanced Tables formula compatibility.
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
# Web clipper: browser capture into Markdown and local assets (agy gap 2, roadmap 33)
|
||||
|
||||
Part of `specs/plugins-parity/`. Reuse engine `capture/` and live-session control dispatch. No new crate. Browser adapter is a contract, not JavaScript embedded in NotesCraft.
|
||||
|
||||
## Research (2026-10-21; agy claims checked)
|
||||
- Official Obsidian Clipper exists and saves web content/highlights privately into notes: [product](https://obsidian.md/clipper). This supports capture UX, not a claim that every image is automatically localized. Drop unverified Notion/Joplin/Bear parity and effort estimates.
|
||||
- Extension service workers can perform cross-origin requests with host permissions; content scripts retain page-origin restrictions: [Chrome network contract](https://developer.chrome.com/docs/extensions/develop/concepts/network-requests). Therefore endpoint calls and credentials belong to extension background code.
|
||||
- Bookmarklets can be blocked by CSP: [MDN javascript URLs](https://developer.mozilla.org/en-US/docs/Web/URI/Reference/Schemes/javascript). Do not promise universal one-click localhost access; clipboard/manual import is required fallback.
|
||||
|
||||
## Problem / goal
|
||||
Capture selected text or article Markdown, source URL, title and explicitly supplied image bytes into an open writable vault. Browser handles DOM extraction; NotesCraft validates input and saves through engine commands. No telemetry, server-side scraping, remote fetching, cookies, or cloud account.
|
||||
|
||||
## Verified baseline / files
|
||||
`crates/engine/src/capture/mod.rs` registers `note.capture`, appends to a daily buffer or creates an Inbox/folder note; accepts one base64 attachment. `capture/rules.rs` formats frontmatter/slugs. `control/{server,protocol,pump,token}.rs` implements loopback NDJSON, **not HTTP**, capped at 1 MiB per line. `apps/notescraft/src/main.rs` starts server per vault process. `crates/engine/tests/{capture,control,control_client}.rs` are integration patterns. `prefs/{app_settings,settings,mod}.rs` own persistent settings. `specs/plugins-parity/clean-paste/spec.md` proposes `htmlmd`; it is a dependency only for optional HTML import, not needed for Markdown v1.
|
||||
|
||||
## Non-goals
|
||||
Bundled extension/store publication; executing page HTML; automatic crawling; bypassing login/paywalls; claiming full Readability extraction; automatic article deduplication; raw control-command access from web pages. Bookmarklet captures selection/title/link only; richer article extraction belongs to extension.
|
||||
|
||||
## Design decisions
|
||||
### D1 Versioned envelope (new `capture/web_model.rs`)
|
||||
```json
|
||||
{"version":1,"request_id":"clip_0123456789abcdef","title":"Example","source":"https://example.org/article","text":"# Article\n\n","target":"inbox","assets":[{"id":"figure","name":"figure.png","media_type":"image/png","base64":"..."}]}
|
||||
```
|
||||
Exactly these fields; reject unknown version/fields and malformed types. `request_id` is `clip_` plus 16..64 lowercase hex characters. Source must be absolute http(s), no credentials/control characters, <=2000 characters; store source as inert metadata, never fetch. Title 1..200 characters without controls. Text UTF-8 <=256 KiB, nonempty after whitespace unless assets present. Target `inbox` default, `daily`, or safe relative vault folder <=1000 characters; reserved paths forbidden through existing Vault rules. At most 8 PNG/JPEG assets, each <=256 KiB decoded, aggregate <=512 KiB, envelope <=1 MiB encoded; reject excess, never truncate. Validate signature/media type/name, duplicate ids and missing `asset:id` references before any write. No SVG/HTML assets. Browser may explicitly leave uncaptured http(s) image links; show remote-link warning in preview, never fetch silently.
|
||||
|
||||
Use Markdown image destination spans from existing Markdown parser, not global string replacement: rewrite `asset:id` only in image destinations, never inside prose/code. Escaped names and duplicate filenames must remain valid. Save assets through Session attachment helpers and note through existing capture path. Factor shared capture execution only as needed; do not duplicate daily/Inbox formatting.
|
||||
|
||||
### D2 Engine surface
|
||||
| Command | Params | Result / effects |
|
||||
|---|---|---|
|
||||
| `capture.web.preview` | envelope | `{title,source,target,assets:[{id,name,bytes}],remote_images,warnings}`; validate, no IO/network |
|
||||
| `capture.web.save` | `{envelope,retry_only?}` (default false) | `{request_id,path,appended,assets:[{id,path}],replayed}`; writable gate |
|
||||
Add both to `commands.rs` registration, `tests/core_commands.rs`, `ai/assistant_gate.rs` (preview READ, save WRITE). CLI/MCP call same registry surface. Existing `note.capture` behavior/tests remain unchanged.
|
||||
|
||||
Per-process idempotency: retain at most 128 completed request ids plus envelope digest and result, clear on close. Exact retry while id is retained returns same result without write; changed envelope with same id -> `request_conflict`. `retry_only:true` never creates a capture: unknown/evicted id returns `outcome_unknown` and writes nothing. After eviction/restart user can explicitly choose a fresh-id submission with duplicate warning; no automatic blind resubmit. Browser retains unsent payload and must show uncertain outcome after restart; no cross-restart exactly-once promise. Use existing sha2 workspace dependency only if approved manifest usage already covers it; otherwise store bounded canonical envelopes for equality (128 x 1 MiB upper bound, prefer digest; Q3).
|
||||
|
||||
Preflight all paths/assets/read-only/destination checks before writes. Asset writes precede note insertion. On later failure return `partial_capture` with exact newly-created paths; never remove pre-existing/deduplicated files or overwrite a note. User can retry or inspect orphaned assets; no false multi-file atomicity claim. Every individual disk write goes through existing atomic Vault/Session APIs. Include save failures in result UI; no silent partial success.
|
||||
|
||||
### D3 Local HTTP capture endpoint on control channel
|
||||
Keep NDJSON listener and descriptor backward compatible. Start a separate loopback-only HTTP listener owned by same `ControlHandle`; it shares the control job/pump path and shuts down with it. Bind `127.0.0.1:0`, never LAN. Descriptor gains optional `capture_http:{port,version:1}` (no capture credential); old clients ignore it. Use already approved workspace `axum`/`tokio`, added to engine manifest explicitly in task 3; no new third-party dependency or crate. HTTP parser is framework-owned, not a new hand-written parser.
|
||||
|
||||
Disabled by default: app setting `capture.web_enabled=false`. Explicit local pairing screen displays port and a separate capture-only bearer token from at least 16 OS-cryptographic random bytes (reuse Unix `/dev/urandom` read pattern, not `new_token()` noncryptographic fallback); never expose broad control token. Capture token rotates on restart/disable/revoke, remains memory-only, never logged, put in URLs, clipboard payloads or page DOM. Owner manually enters it in extension settings; no unauthenticated remote pairing endpoint. Enable and token display are local UI actions only. Existing `control/token.rs::new_token` falls back to RandomState/time/pid on Windows or RNG error; that fallback is NOT allowed for browser-facing capture. Add a fallible capture-only token helper; RNG failure keeps bridge disabled. Unix OS randomness works without new dependency; Windows HTTP bridge stays unsupported until owner approves secure-RNG binding/dependency Q5. Clipboard import remains available there.
|
||||
|
||||
Routes: `POST /v1/capture/preview`, `POST /v1/capture` map ONLY to above commands; HTTP body stays the D1 envelope, bridge wraps save params. Header `X-NotesCraft-Retry: 1` sets `retry_only:true`; include that header in exact-origin preflight allowlist. Browser uses it for every retry after uncertain response; `OPTIONS` implements exact allowlisted extension origins. No generic `run`, commands listing or note reads. Require `application/json` and `Authorization: Bearer ...`, exact numeric-loopback Host/port, reject foreign Host (DNS rebinding), web/null origins and missing/invalid token before queuing. Origin-less authenticated CLI allowed; exact enabled extension ids configured locally. No wildcard CORS/credentials; approved preflight does not authorize POST. Return 200 preview, 201 saved, 200 replay, 400 invalid, 401 auth, 403 disabled/origin/read-only, 409 conflict, 413 over limit, 429 busy, 500 structured failure. Bodies/errors never echo bearer token or source body.
|
||||
|
||||
Body cap 1 MiB; header cap 16 KiB and read timeout 5 s; at most 4 connections, 1 capture in flight, no unbounded queue. Validate/decode on HTTP worker; session mutation still runs through pump. Set 10 s response deadline: before pump execution, expired job must be skipped; once mutation starts do not claim cancellation/rollback. Timeout reply carries `outcome_unknown` and request id so retry-only lookup resolves it while retained, otherwise returns uncertainty without another write. NDJSON 1 MiB cap unchanged.
|
||||
|
||||
### D4 Bookmarklet / extension contract
|
||||
Extension extracts Markdown locally after explicit click, preview includes destination and missing/localized images. Background worker calls endpoint; host permissions limited to loopback plus explicit image origins, never arbitrary page-supplied fetch proxy. Tokens in extension-local storage, never sync storage/content script messages. Pair each open vault explicitly; no scan of arbitrary ports or fallback to another vault. Browser fetches assets only after permission/confirmation; passes bytes, not URLs for app to fetch. Browser contract tests cover CSP/host permission failure and stale pairing. External adapter implementation/publication needs separate owner scope.
|
||||
|
||||
Bookmarklet v1 returns a version-1 envelope with selection as plain Markdown-escaped text, current URL/title, Inbox target, empty assets and fresh id; copies it where clipboard permission permits, otherwise shows selectable text in browser UI. User pastes JSON into NotesCraft's Import web capture dialog, previews, saves via commands. No bearer token in bookmarklet; no promise of direct HTTP calls from arbitrary websites. Browser-restricted pages show unsupported. This fallback does not require extension or an app webview.
|
||||
|
||||
### D5 UI / settings
|
||||
New import/pairing dialog in `crates/ui-egui/src/views/web_capture/`, declared in existing `views/mod.rs`; shell wiring in `shell/app.rs`, `shell/app/window/{commands,frame}.rs`. Reuse craft-ui controls and tokens. Local setting is persisted through existing `prefs/settings.rs`; listener enable/disable reconciles outside paint. Strings `[ui.web_capture]` at end of `crates/i18n/locales/en.toml`, table module through `text/features/mod.rs`. Preview, explicit save, retry/partial errors and copy/import instructions; no HTML execution or token in screenshots/logs.
|
||||
|
||||
## Acceptance criteria
|
||||
1. Registry preview writes zero files; save from CLI and HTTP creates same Markdown and local image links. Daily capture respects dirty buffer/undo; Inbox naming collisions preserve both captures.
|
||||
2. Invalid URL/path/version/asset/missing reference rejects before writes. Boundary tests exercise exact limits and +1; HTML/script/SVG never execute.
|
||||
3. NDJSON clients remain compatible; disabled bridge binds no HTTP socket. Endpoint cannot run arbitrary commands, read notes, or use broad control token.
|
||||
4. Forged Host, disallowed origin, missing auth and oversized bodies never enqueue mutation. Enable/disable/restart rotates token and stops listener; read-only refuses save.
|
||||
5. Lost reply plus retry-only request while retained saves once; changed payload/id conflicts. Unknown/evicted id and retry-only after restart write nothing and return outcome_unknown; fresh submission requires explicit duplicate-risk confirmation. Expiry before pump writes nothing; post-dispatch timeout is explicitly uncertain.
|
||||
6. Inject failure after first asset: structured partial result lists only newly-created assets, existing files intact. Retry does not overwrite prior files.
|
||||
7. Bookmarklet clipboard import works without HTTP; CSP/clipboard refusal displays manual fallback. No extension support is claimed without testing actual adapter.
|
||||
8. UI snapshots in Mocha/Latte at 1440x900 and compact width show preview and errors without secret values; all strings localized. Flood tests show bounded memory/queue.
|
||||
|
||||
## Open owner questions
|
||||
- Q1: approve selection-only bookmarklet plus clipboard/manual import and external extension contract as v1? Direct bookmarklet localhost auth would expose credentials to page context.
|
||||
- Q2: confirm 512 KiB aggregate local-asset limit for v1; raising beyond existing control budget requires separate streaming/worker plan.
|
||||
- Q3: approve adding already-approved `sha2` workspace usage to engine for bounded idempotency digests (reason: retain 128 results without 128 MiB payload copies)? No unapproved dependency may be added.
|
||||
- Q4: confirm explicit partial-capture errors with recoverable orphaned assets rather than a durable multi-file capture journal.
|
||||
- Q5: approve a safe cross-platform OS-RNG dependency such as `getrandom` (exact version/license reviewed before addition) for Windows bridge tokens, or accept Unix-only HTTP bridge v1? Reason: existing control-token fallback is explicitly noncryptographic. Never relax project unsafe policy.
|
||||
|
||||
## Feature ideas (researched, ranked)
|
||||
1. User templates/site-specific extraction, inspired by [Obsidian Clipper](https://obsidian.md/clipper); keep conversion in browser, never execute remote code in app.
|
||||
2. Extension highlights and reader selection workflow, same [official Clipper](https://obsidian.md/clipper), after basic contract adapter is tested.
|
||||
3. Native-messaging browser transport to avoid localhost permission friction: [Chrome native messaging](https://developer.chrome.com/docs/extensions/develop/concepts/native-messaging); requires separate packaging/owner approval.
|
||||
4. Larger streamed images and durable request journal after measuring local endpoint limits; [Chrome network permissions](https://developer.chrome.com/docs/extensions/develop/concepts/network-requests) informs explicit origin grants.
|
||||
@@ -0,0 +1,52 @@
|
||||
# Tasks: web clipper (roadmap 33)
|
||||
|
||||
One task = one bounded implementation handoff. Spec: `spec.md`; rules: `AGENTS.md`, `docs/CODE-MAP.md`. Paths marked new do not exist yet. Tasks run in order; each leaves its own testable result. Later implementation owns code paths listed below; this documentation run writes only `specs/plugins-parity/`.
|
||||
|
||||
**First tool call rule:** batch at most three relevant reads in first tool call; write first OWNS file in SECOND tool call (compiling stub plus one failing test), then implement and pass that test. Do not spend second call exploring. For docs-only task, write first OWNS document in second call and validate its acceptance checklist.
|
||||
**Standing VERIFY (future implementation only):** `export PATH="$HOME/.cargo/bin:$PATH"`; `cargo test -p <package>`; `cargo clippy -p <package> --all-targets -- -D warnings`; `rustfmt --edition 2024 <owned Rust files>`; UI tasks add `cargo xtask i18n` and read snapshot PNGs; final integration adds `cargo xtask layers`, `cargo xtask tree`, `cargo xtask ci`. Stop on failure; never bypass checks.
|
||||
**Standing DO NOT:** edit outside OWNS, change Git state, commit, use `cargo fmt --all`, add unapproved dependency, modify craft-libs API, use unsafe/panics/input indexing, write disk outside atomic Vault/Session/settings APIs, hardcode UI strings. Parent module declarations/test harnesses must be owned explicitly.
|
||||
|
||||
## Phase 1 — pure contract
|
||||
- [ ] 1. Versioned envelope and validation (brief `briefs/20-web-capture-envelope.md`).
|
||||
OWNS: `crates/engine/src/capture/web_model.rs` (new, first write), `crates/engine/src/capture/mod.rs` (module declaration only).
|
||||
FILES TO OPEN: `specs/plugins-parity/web-clipper/spec.md` D1, `crates/engine/src/capture/rules.rs`, `crates/engine/src/capture/mod.rs`.
|
||||
GOAL: `WebCapture::parse(&str) -> Result<Self, CommandError>`, validated metadata/assets and normalized capture target; named caps from D1; decoded signatures/duplicate ids/placeholder validation; pure function, no IO. Keep result type crate-private and tests local. Use existing base64 decoder, no added dependency.
|
||||
TESTS: roundtrip minimal envelope; unknown version/fields; exact byte caps/+1; UTF-8 size; bad source/credentials; traversal/reserved paths; duplicate/missing asset ids; bad base64/MIME/signature; no replacement inside code; empty capture; valid PNG/JPEG fixtures.
|
||||
VERIFY: standing `-p notescraft-engine`, `cargo test -p notescraft-engine capture::web_model` passes all named cases.
|
||||
DO NOT: register commands, save assets, open listener, convert arbitrary HTML or add htmlmd dependency.
|
||||
|
||||
## Phase 2 — engine save semantics
|
||||
- [ ] 2. Preview/save, local assets and retry identity. Depends on 1 and owner Q3 decision.
|
||||
OWNS: `crates/engine/src/capture/web.rs` (new, first write), `crates/engine/src/capture/mod.rs`, `crates/engine/src/capture/rules.rs` (shared capture execution only), `crates/engine/src/vault/session.rs` (bounded replay state only), `crates/engine/src/commands.rs`, `crates/engine/src/ai/assistant_gate.rs`, `crates/engine/tests/core_commands.rs`, `crates/engine/tests/web_capture.rs` (new); `crates/engine/Cargo.toml` ONLY if owner Q3 approves existing workspace sha2 usage.
|
||||
FILES TO OPEN: spec D1-D2, `crates/engine/src/capture/mod.rs`, `crates/engine/src/vault/attachments.rs`, `crates/engine/tests/capture.rs`, `crates/engine/tests/common/mod.rs`.
|
||||
GOAL: two D2 commands, shared existing capture executor, attachment destination span rewrite; preflight before writes; replay ledger 128 ids/digests/results; explicit partial_capture path report. Do not promise transactional multi-file save.
|
||||
TESTS: TempVault preview zero writes; text+PNG emits local embed; Daily dirty buffer and undo; Inbox collisions; same-id replay once; changed-id payload conflict; ledger eviction at 129 entries; retry_only unknown/evicted id zero writes; bad asset no writes; injected later write failure lists only new assets; read-only; existing note.capture tests remain passing.
|
||||
VERIFY: standing engine plus `--test web_capture --test capture --test core_commands`.
|
||||
DO NOT: network fetch, overwrite existing files, delete orphaned assets automatically, change NDJSON limit, add a new crate.
|
||||
|
||||
## Phase 3 — loopback transport
|
||||
- [ ] 3. HTTP bridge owned by control lifecycle. Depends on 2; existing axum/tokio approved, no new dependency proposal.
|
||||
OWNS: `crates/engine/src/control/capture_http.rs` (new, first write), `crates/engine/src/control/mod.rs`, `crates/engine/src/control/server.rs`, `crates/engine/src/control/pump.rs`, `crates/engine/src/control/protocol.rs`, `crates/engine/src/control/token.rs`, `crates/engine/Cargo.toml` (existing workspace axum/tokio), `Cargo.lock` (resolved approved workspace dependencies only), `crates/engine/tests/web_capture_http.rs` (new).
|
||||
FILES TO OPEN: spec D3, `crates/engine/src/control/server.rs`, `crates/engine/src/control/pump.rs`, `crates/engine/src/control/protocol.rs`, `crates/engine/tests/control.rs`, `apps/notescraft-sync-server/Cargo.toml` (existing approved HTTP framework).
|
||||
GOAL: separate HTTP loopback listener, same control pump/registry dispatch, optional descriptor field; explicit local enable/token rotation API using fallible OS CSPRNG (no RandomState fallback); capture-only auth and exact origin/Host gates; fixed body/header/connection/inflight/deadline limits. Existing NDJSON payload cap/protocol untouched. Extend Job with expiry so skipped-before-execution jobs do not mutate after timeout. Reuse framework HTTP parsing.
|
||||
TESTS: raw HTTP/axum tests: no bind disabled; POST preview/save/replay with explicit envelope-to-command wrapping and retry-only header; arbitrary route/command refused; token separation, wrong Host, null/foreign origin, preflight, missing bearer, oversized body/header; queued expiry zero writes; drop/disable shuts listener; RNG failure leaves listener disabled; Unix token source and unsupported Windows status until Q5 approval; slow client/flood bounded; old NDJSON control tests pass.
|
||||
VERIFY: standing engine, `--test web_capture_http --test control --test control_client`; `cargo xtask layers`, `cargo xtask tree`.
|
||||
DO NOT: expose control token, log payload, permit wildcard CORS or LAN, allow page origins, implement handwritten HTTP parser, change workspace registrations (no new crate). Windows bridge is blocked on Q5; before any approved RNG dependency addition, amend OWNS to include root Cargo.toml and Cargo.lock plus engine manifest. Unix-only transport can proceed without it.
|
||||
|
||||
## Phase 4 — import and pairing UI
|
||||
- [ ] 4. Explicit settings, pairing and clipboard-envelope preview/save.
|
||||
OWNS: `crates/ui-egui/src/views/web_capture/mod.rs` (new, first write), `crates/ui-egui/src/views/mod.rs`, `crates/ui-egui/src/shell/app.rs`, `crates/ui-egui/src/shell/app/window/commands.rs`, `crates/ui-egui/src/shell/app/window/frame.rs`, `crates/ui-egui/src/text/features/web_capture.rs` (new), `crates/ui-egui/src/text/features/mod.rs`, `crates/i18n/locales/en.toml` (append ui.web_capture), `crates/engine/src/prefs/app_settings.rs`, `crates/engine/src/prefs/settings.rs`, `crates/ui-egui/tests/web_capture.rs` (new), `apps/notescraft/src/main.rs` (control startup/local bridge lifecycle hook only).
|
||||
FILES TO OPEN: spec D3-D5, `crates/ui-egui/src/views/share_dialog/mod.rs`, `crates/engine/src/prefs/app_settings.rs`, `crates/engine/src/prefs/settings.rs`, `crates/ui-egui/src/shell/app/window/frame.rs`.
|
||||
GOAL: off-by-default web enable setting and extension-origin list, local token display/revoke, listener reconciliation outside paint; Import web capture dialog takes copied JSON, previews, confirms save, reports remote images/partial capture/uncertain retry. Save stays registry-backed. Secrets absent from diagnostics/snapshot fixtures.
|
||||
TESTS: kittest disabled/enabled/revoke; invalid JSON and selection-only envelope; Save once; retry-only unknown result requires explicit fresh-submission confirmation; missing assets warning; read-only; queued failure keeps draft. Settings restart preserves enabled/origins but rotates token. TOML parses.
|
||||
VERIFY: standing engine + ui-egui; `cargo xtask i18n`; offscreen screenshots Mocha/Latte, normal/compact, read PNGs.
|
||||
DO NOT: write browser extension, silently change destination, execute HTML, read clipboard automatically, call engine per row/frame.
|
||||
|
||||
## Phase 5 — external client contract and status
|
||||
- [ ] 5. Browser adapter contract with executable transport fixtures; close acceptance gaps.
|
||||
OWNS: `specs/plugins-parity/web-clipper/browser-contract.md` (new, first write), `crates/engine/tests/web_capture_http.rs`, `specs/plugins-parity/web-clipper/spec.md`, `specs/plugins-parity/web-clipper/tasks.md`, `specs/plugins-parity/roadmap.md` (33 only), `ROADMAP.md`, `docs/ARCHITECTURE.md`, `docs/CODE-MAP.md` (feature entries only).
|
||||
FILES TO OPEN: spec D4 and AC1-8, `crates/engine/tests/web_capture_http.rs`, `specs/plugins-parity/roadmap.md`.
|
||||
GOAL: publish exact JSON/route/error/pairing contract, selection-only bookmarklet algorithm and clipboard fallback, extension background/host-permission and asset rules. Include request/response examples and manual browser matrix for restricted page/CSP/permission/stale token/offline app. Mark external adapter unimplemented until separate work exists; update status with actual scope.
|
||||
TESTS: example envelopes accepted by registry/HTTP fixtures; invalid examples rejected; manual browser checks only for actual adapters, record untested contract claims as such. Trace every AC to prior tests or explicit gated adapter check.
|
||||
VERIFY: fixture tests plus standing final gates; docs links resolve. No cargo is run while writing these plans.
|
||||
DO NOT: publish/install extension, add JS to native app, mark full adapter shipped without real browser test, infer owner approval from time elapsed.
|
||||
@@ -0,0 +1,188 @@
|
||||
# Measurements: where NotesCraft weight comes from (2026-10-21)
|
||||
|
||||
Raw numbers and the commands that produced them. Spec: `spec.md`. Nothing in the repo was changed: the build ran in a scratch copy and every number below can be reproduced with the commands in "Reproduce".
|
||||
|
||||
Machine: Apple Silicon, 10 cores, macOS 25.3. **The machine was busy** (load average 8 to 10 from other agents' builds), so compile wall times and CPU seconds are upper bounds. Binary sizes, section sizes, asset copy counts and RSS plateaus do not depend on load.
|
||||
|
||||
## 1. Release binary
|
||||
|
||||
Scratch copy `/private/tmp/claude-501/scratch/nc` (rsync of the tree without `target*`, `.git`, `.notescraft`, `.mm-agents`, `dist`). **Pitfall**: the pattern `target*` also drops `crates/ui-egui/assets/icons/lucide/target.svg`, which `icon_catalogue.rs` embeds, so the first build failed until the file was copied back. Profile is the repo's: `lto = "thin"`, `codegen-units = 1`, no `strip`, no `opt-level` (so 3), no `panic = "abort"`.
|
||||
|
||||
| Item | Bytes | MiB |
|
||||
|---|---|---|
|
||||
| `notescraft` release binary (this tree) | 40,235,208 | 38.4 |
|
||||
| installed `/Applications/NotesCraft.app/Contents/MacOS/NotesCraft` (older build) | 37.9 M by `ls` | 37.9 |
|
||||
| `notescraft-cli` release (`dist/release`) | 12.9 M by `ls` | 12.9 |
|
||||
| DMG (compressed, `dist/release`) | 14.1 M by `ls` | 14.1 |
|
||||
|
||||
Mach-O sections of the binary (`size -m`):
|
||||
|
||||
| Section | Bytes | MiB | Share of file |
|
||||
|---|---|---|---|
|
||||
| `__TEXT,__text` (code) | 22,962,444 | 21.9 | 57 % |
|
||||
| `__TEXT,__const` (read-only data: fonts, tables, packdump) | 6,850,888 | 6.5 | 17 % |
|
||||
| `__LINKEDIT` (symbol table, string table, fixups) | 5,947,392 | 5.7 | 15 % |
|
||||
| `__eh_frame` + `__unwind_info` | 1,904,672 + 299,088 | 2.1 | 5 % |
|
||||
| `__gcc_except_tab` | 837,156 | 0.8 | 2 % |
|
||||
| `__DATA_CONST` const | 804,816 | 0.8 | 2 % |
|
||||
| `__cstring` | 273,834 | 0.3 | 0.7 % |
|
||||
|
||||
### 1.1 Code by crate (`cargo bloat --release --crates`, .text 21.9 MiB)
|
||||
|
||||
Attribution is approximate (cargo-bloat guesses by symbol path; generic code lands in `std` and the caller).
|
||||
|
||||
| Group | .text | Share of .text | Notes |
|
||||
|---|---|---|---|
|
||||
| all `notescraft_*` crates | 7.9 MiB | 36 % | `ui_egui` 4.2, `engine` 1.8, then each under 0.2 |
|
||||
| `std` (generic instantiations) | 4.1 MiB | 19 % | |
|
||||
| GPU: `naga` 0.90, `wgpu_core` 0.67, `wgpu_hal` 0.15, `wgpu_types` 0.05, `wgpu` 0.04 | 1.8 MiB | 8 % | required by eframe/wgpu |
|
||||
| egui stack: `egui` 0.84, `epaint` 0.16, `winit` 0.18, `egui_wgpu` 0.05 | 1.3 MiB | 6 % | required |
|
||||
| image codecs: `image` 0.42, `image_webp` 0.41, `zune_jpeg` 0.30, `tiff` 0.19, `png` 0.05 | 1.4 MiB | 6 % | notes embed images; `tiff` is pulled by `arboard` (macOS pasteboard) |
|
||||
| text shaping: `harfrust` 0.37, `skrifa` 0.27, `read_fonts` 0.17 | 0.8 MiB | 4 % | required |
|
||||
| `serde_core` 0.43, `serde_json` 0.30, `toml_edit` 0.22, `toml` 0.07 | 1.1 MiB | 5 % | |
|
||||
| TLS and HTTP: `rustls` 0.34, `ring` 0.14, `ureq` 0.11, `ureq_proto` 0.05 | 0.65 MiB | 3 % | sync client; grammar fetch still shells out to `curl` |
|
||||
| regex: `regex_automata` 0.33, `regex_syntax` 0.16, `aho_corasick` 0.10, `fancy_regex` 0.06 | 0.65 MiB | 3 % | |
|
||||
| SVG: `usvg` 0.20, `tiny_skia` 0.14, `resvg` 0.06 | 0.45 MiB | 2 % | |
|
||||
| `syntect` | 0.10 MiB | 0.4 % | the packdump data is in `__const`, not here |
|
||||
| **`wasmi`** | **0** | **0 %** | **not in the binary at all** (see 1.4) |
|
||||
|
||||
Workspace crates, `.text` in KiB, largest first: `ui_egui` 4300, `engine` 1840, `bases` 183, `calendar` 170, `index` 139, `diagrams` 131, `pdf` 110, `knowledge` 101, `acp` 88, `vim` 86, `markdown` 86, `lint` 77, `editor` 72, `lottie` 71, `whiteboard` 57, `math` 44, `blocks` 44, `git` 36, `syntax` 32, `canvas` 27, `annotate` 25, `sync` 21, `backlinks` 21, `themes` 19, `templates` 17, `collab` 16, `regex` 16, `graph` 11, `i18n` 7, `plugins` 1.5.
|
||||
|
||||
The "optional-looking" crates (`bases`, `calendar`, `diagrams`, `pdf`, `knowledge`, `acp`, `vim`, `lottie`, `whiteboard`, `math`, `git`, `canvas`, `annotate`, `collab`, `graph`) add up to **1.15 MiB** of `.text`, 5 % of code. Their UI halves live inside `ui_egui` and their commands inside `engine`; the source-line share of the view folders that belong to them (whiteboard 5906, calendar 2883, graph 2072, media 3942, canvas 768, diagram 218 of 103,212 lines in `ui-egui/src`) is about 15 % of `ui_egui`, or 0.6 MiB. **Estimate, not measured**: compiling every optional feature out together would save 2.5 to 3.5 MiB (7 to 9 % of the file).
|
||||
|
||||
### 1.2 Embedded data (`__const` 6.5 MiB)
|
||||
|
||||
Counted by searching the binary for whole-file byte runs (`python3 -I`, `bytes.count`):
|
||||
|
||||
| Asset | Size | Copies in binary | Bytes in binary |
|
||||
|---|---|---|---|
|
||||
| `Inter-Regular.ttf` | 411,640 | **2** (`engine` PDF export + `ui-egui`) | 823,280 |
|
||||
| `Inter-SemiBold.ttf` | 419,744 | **2** (same) | 839,488 |
|
||||
| `Inter-Medium.ttf` | 417,300 | 1 | 417,300 |
|
||||
| `JetBrainsMono-Regular.ttf` (only PDF export) | 270,224 | 1 | 270,224 |
|
||||
| `JetBrainsMono-NoLigatures.ttf` (UI) | 269,984 | 1 | 269,984 |
|
||||
| `STIXTwoMath-Regular.ttf` (math only) | 375,596 | 1 | 375,596 |
|
||||
| subtotal NotesCraft fonts | | | **2,995,872 (2.86 MiB)** |
|
||||
| egui `default_fonts` (feature of `eframe` in `apps/notescraft/Cargo.toml`): `Hack-Regular` 309,408, `Ubuntu-Light` 361,676, `NotoEmoji-Regular` 418,804, `emoji-icon-font` 324,132 | | 1 each | **1,414,020 (1.35 MiB)** |
|
||||
| **all fonts** | | | **4,409,892 (4.2 MiB, 11 % of the file, 65 % of `__const`)** |
|
||||
| syntect default packdump (`default_newlines.packdump` 368,467 + metadata 4,976) | | 1 | about 0.37 MiB |
|
||||
| Lucide catalogue (278 SVG, `ui-egui/assets/icons/lucide`) | 115,773 | 1 | 0.11 MiB |
|
||||
| Lottie icons (65 JSON, `assets/icons-anim`) | 147,922 | 1 | 0.14 MiB |
|
||||
| UI icons (89 SVG, `assets/icons`) | 31,228 | 1 | 0.03 MiB |
|
||||
| 25 embedded grammars (`crates/syntax/grammars`) | 68,844 | 1 | 0.07 MiB |
|
||||
| `en.toml` (only locale) | 61,122 | 1 | 0.06 MiB |
|
||||
|
||||
The PDF crate's own `include_bytes!` of the fonts sit inside `#[cfg(test)]`, so they are not in the binary; the duplicate comes from `crates/engine/src/builtin/export_commands.rs`.
|
||||
|
||||
Findings: fonts are the biggest single data cost and **larger than all fifteen optional crates together** (4.2 MiB against 1.15 MiB). Two copies of Inter Regular and SemiBold waste 0.79 MiB; the egui default set is mostly unused (the app's `fonts::install` puts Inter first in the proportional chain and JetBrains Mono first in the monospace chain; Hack and Ubuntu-Light are only fallbacks; NotoEmoji matters for emoji). Everything else packable (Lucide, Lottie, grammars, locale, packdump) is about 0.8 MiB in total.
|
||||
|
||||
### 1.3 What stripping and optimisation do (measured, no code change)
|
||||
|
||||
| Variant | Bytes | vs baseline |
|
||||
|---|---|---|
|
||||
| baseline (thin LTO, opt-level 3, symbols kept) | 40,235,208 | - |
|
||||
| `strip -x` (drop local symbols) | 35,767,528 | -4.47 MB (-11.1 %) |
|
||||
| `strip` (drop all symbols) | 34,851,464 | **-5.38 MB (-13.4 %)** |
|
||||
| `opt-level = "s"` + `strip = "symbols"` (`CARGO_PROFILE_RELEASE_*`, own target dir) | 32,088,240 | **-8.15 MB (-20.3 %)** |
|
||||
|
||||
`packaging/linux/package.sh` already runs `strip`; the macOS and Windows packaging do not, so the macOS app ships 5.4 MB of symbol table. `opt-level = "s"` removes another 2.8 MB; its speed cost was measured only for startup (section 2) and must be measured on the index build and a scroll benchmark before adopting.
|
||||
|
||||
### 1.4 The wasm plugin host costs 0 bytes today, and does nothing
|
||||
|
||||
`wasmi` and `ns_run_command` are absent from the binary (`bytes.count` = 0; `nm | grep -c wasmi` = 0). Reason: `Session::load_plugins` and `Session::run_plugin_command` have no caller outside `crates/engine/tests` (grep over `apps/`, `crates/ui-egui/src`, `crates/mcp/src`, `crates/engine/src`), so the linker drops the whole host. Consequences: (a) `plugins/dice` cannot run in the shipped app; (b) wasm plugin support costs compile time only (37 CPU-s for `wasmi`, 3 % of the build); (c) wiring plugin loading, which the registry needs for plugins, will **add** an estimated 0.8 to 1.2 MiB of code (the wasmi interpreter; estimate, not measured).
|
||||
|
||||
## 2. Runtime
|
||||
|
||||
Method: `measure.py` (below) starts the binary with `NOTESCRAFT_CONFIG_DIR` pointing at a fresh scratch dir (the owner's settings are untouched) and a vault argument, samples `ps -o rss=,time=,%cpu=` every 0.1 s for 3 s then every 1 s, then kills it. `vmmap -summary` and `footprint` give the memory breakdown. Synthetic vaults in `/private/tmp/claude-501/scratch/vaults`: `small` 50 notes, `big` 6500 notes in 60 folders, 25 MB, 5 wikilinks, 3 tags and frontmatter each, a Rust fence in every tenth note.
|
||||
|
||||
| Run | RSS at 0.3 to 0.6 s | RSS plateau | CPU time to idle | Notes |
|
||||
|---|---|---|---|---|
|
||||
| installed app, `small` | 120 MB at 0.31 s, 131.6 at 0.62 s | 132.0 MB | 0.29 s (0.47 s after 20 s) | idle CPU 0.0 to 0.1 % |
|
||||
| new release, `small` (2 runs) | 131 MB | 131.0 and 131.2 MB | 0.30 s | |
|
||||
| `opt-level=s` + strip, `small` | 131 MB | 131.2 MB | similar | no visible startup difference |
|
||||
| new release, `big`, **no flag** | 125.6 MB | 125.7 MB | 0.9 s over 60 s | shows the large-folder warning (`LargeFolderGate`), vault not loaded |
|
||||
| new release, `big`, `--large-ok`, **first open** (no index cache) | 156.8 MB at 0.66 s | 221.7 to 230 MB after 17 s | 3.4 s at 5 s, 4.4 s at 70 s | **peak 494.3 MB at about 5 s** (index build), then falls |
|
||||
| new release, `big`, `--large-ok`, **later open** (cache in the vault) | not sampled | 378.6 MB | 1.9 s | |
|
||||
| `opt-level=s`, `big`, `--large-ok`, later open | not sampled | 380.8 MB | 2.1 s | within noise of the line above |
|
||||
|
||||
`vmmap -summary` of the release build on `small` after 8 s: **physical footprint 72.4 MB** (peak 189.6 MB during startup); `MALLOC_SMALL` 23.1 MB resident, `IOSurface` 11.5 MB (GPU surfaces), `__DATA_CONST` 17.1 MB resident, `__DATA` 7.7 MB, `__TEXT` 314 MB resident of 1.2 GB mapped (system frameworks from the shared cache, not the app's 33 MB). RSS (131 MB) is larger than the footprint because it counts shared system library pages.
|
||||
|
||||
Readings:
|
||||
- Cold start is already fast: about 0.3 s of CPU and a window at roughly 0.6 s (sampling resolution 0.3 s early on; time to first frame was not instrumented). A 21.9 MiB `.text` is mapped, not read, and only touched pages count.
|
||||
- The app's own binary is a small part of RSS (clean, evictable file pages). Removing 5 MB of symbols changes nothing at runtime (`__LINKEDIT` is not touched); removing 1 MiB of optional code changes RSS by well under 1 MB.
|
||||
- **What is heavy is vault data**: a 6500-note vault peaks at 494 MB while indexing and settles at 220 to 380 MB. That is index memory, unrelated to plugins and the registry. If "heavy" is the owner's complaint, an index-memory spec is worth more than this one (see spec, open question 9).
|
||||
- Eager work at startup that a lazy load could remove: `fonts::install` reads `SFHebrew.ttf` (64,604 B) and `SFArabic.ttf` (724,608 B) from disk on every launch into the heap; the script-on-demand loader already exists for the other scripts. Saves about 0.8 MB RSS and one disk read. Small.
|
||||
|
||||
### 2.1 Grammar set build (first fence in a note)
|
||||
|
||||
Example run in the scratch copy (release, `notescraft-syntax`), several runs each:
|
||||
|
||||
| Step | Time |
|
||||
|---|---|
|
||||
| `SyntaxSet::load_defaults_newlines()` | 0.3 to 0.6 ms (lazy) |
|
||||
| `into_builder()` (deserialise all 75 default syntaxes) | 17.4 ms |
|
||||
| `build()` with defaults only | 60.3 ms |
|
||||
| parse and add the 25 embedded grammars | 30.3 to 31.8 ms |
|
||||
| `build()` with defaults + embedded | 62.6 to 66.2 ms |
|
||||
| `load::current()` total | **108 to 113 ms** |
|
||||
|
||||
It runs on first use inside the highlight cache worker, so it does not block the UI thread. The 25 NotesCraft grammars cost about 35 ms of the 110 ms (parse 31 ms plus 3 to 6 ms link); the other 75 ms belong to syntect's default pack. `load::set_installed` repeats the whole build for every install or removal. Moving the 25 grammars out removes 69 KB of binary and about 35 ms of the first highlight; it does not touch the 75 ms.
|
||||
|
||||
## 3. Compile-time weight
|
||||
|
||||
`cargo build --release -p notescraft --timings` (375 units; the first build aborted at `ui-egui` on the missing `target.svg`, the second finished; CPU seconds are the sum of per-unit durations from the two timing reports):
|
||||
|
||||
| Metric | Value |
|
||||
|---|---|
|
||||
| total unit CPU | 1,151 s |
|
||||
| workspace + craft-libs crates | 392 s (34 %) |
|
||||
| external crates | 759 s (66 %) |
|
||||
| `ui-egui` + `notescraft` bin after the dependencies were built (wall, loaded machine) | 2 m 41 s |
|
||||
| full `opt-level=s` + strip build, wall (loaded machine) | 2 m 01 s |
|
||||
| unique packages in `cargo tree -p notescraft -e normal` | 386 (47 workspace or craft) |
|
||||
| crates present in more than one version (`cargo tree -d`) | 18 |
|
||||
|
||||
Top units: `notescraft-ui-egui` 101 s, `notescraft-engine` 98 s, `notescraft` bin 59 s, `naga` 42.5, `wgpu-core` 41.6, **`wasmi` 37.1 (0 bytes shipped)**, `read-fonts` 35.0, `regex-automata` 33.4, `vello_cpu` 31.4, `objc2-metal` 30.0, `objc2-foundation` 29.2, `skrifa` 28.9, `harfrust` 28.2, `ring` build script 28.0, `egui` 21.9, `egui_extras` 21.4, `zune-jpeg` 21.0, `image` 19.9. The only compile cost a feature could remove without touching the UI is `wasmi` (3 %).
|
||||
|
||||
## 4. Ranking by measured cost (what a user could save)
|
||||
|
||||
| Rank | Item | Measured or estimated saving | Needs the registry? |
|
||||
|---|---|---|---|
|
||||
| 1 | strip symbols on macOS and Windows | **5.38 MB (13.4 %)** measured | no, release profile |
|
||||
| 2 | `opt-level = "s"` (after a perf check) | 2.8 MB more, measured | no |
|
||||
| 3 | fonts: remove the duplicate Inter pair | 0.79 MiB, measured copy count | no |
|
||||
| 4 | fonts: stop shipping Hack, Ubuntu-Light, emoji-icon-font through `default_fonts` (keep NotoEmoji) | about 0.95 MiB; needs a visual check for tofu | no |
|
||||
| 5 | PDF export reuses `JetBrainsMono-NoLigatures` | 0.26 MiB | no |
|
||||
| 6 | STIX Two Math as a font pack (math only) | 0.36 MiB; math then needs a pack or a seed | yes |
|
||||
| 7 | Lottie icons, Lucide catalogue, 25 grammars | 0.14 + 0.11 + 0.07 MiB; about 35 ms off the first highlight | yes, each tiny |
|
||||
| 8 | compile out every optional feature | 2.5 to 3.5 MiB estimated; large refactor | no (cargo features); the registry can only host the assets |
|
||||
| 9 | lazy system fallback fonts | 0.8 MB RSS | no |
|
||||
|
||||
Items 1 to 5 total about 10 MB (25 %) and need no registry. The registry-only items 6 and 7 are about 0.7 MiB (2 %).
|
||||
|
||||
## Reproduce
|
||||
|
||||
```sh
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
S=/private/tmp/claude-501/scratch/nc
|
||||
rsync -a --exclude 'target*' --exclude .git --exclude .notescraft --exclude .mm-agents --exclude dist ./ $S/
|
||||
cp crates/ui-egui/assets/icons/lucide/target.svg $S/crates/ui-egui/assets/icons/lucide/ # the 'target*' exclude dropped it
|
||||
cd $S && export CARGO_TARGET_DIR=/Volumes/Storage/Projects/notescraft/target/agent-bloat
|
||||
cargo build --release -p notescraft --timings
|
||||
cargo install cargo-bloat --locked --root /private/tmp/claude-501/scratch/tools # not touching ~/.cargo
|
||||
/private/tmp/claude-501/scratch/tools/bin/cargo-bloat bloat --release -p notescraft --crates -n 0
|
||||
size -m $CARGO_TARGET_DIR/release/notescraft
|
||||
cp $CARGO_TARGET_DIR/release/notescraft nc-strip && strip nc-strip && ls -l nc-strip
|
||||
CARGO_PROFILE_RELEASE_OPT_LEVEL=s CARGO_PROFILE_RELEASE_STRIP=symbols \
|
||||
CARGO_TARGET_DIR=/Volumes/Storage/Projects/notescraft/target/agent-bloat-s cargo build --release -p notescraft
|
||||
cargo tree -p notescraft --offline -e normal --prefix none | sort -u | wc -l # 386
|
||||
cargo tree -p notescraft --offline -e normal -d | grep -E '^[a-z_-]+ v' | sort -u | wc -l # 18
|
||||
# font copies: python3 -I, B = open(binary,'rb').read(); B.count(open(font,'rb').read())
|
||||
# runtime: python3 -I measure.py <binary> <vault> <seconds> <label> [--large-ok]
|
||||
# Popen([bin, vault]), env NOTESCRAFT_CONFIG_DIR=<fresh dir>, ps -o rss=,time=,%cpu= -p PID
|
||||
# memory: vmmap -summary PID ; footprint PID
|
||||
# grammar timing: an example in the scratch copy calling notescraft_syntax::load::current() and syntect directly
|
||||
# wasm host absent: python3 -I -c "B=open(bin,'rb').read(); print(B.count(b'wasmi'), B.count(b'ns_run_command'))"
|
||||
```
|
||||
|
||||
Scratch artefacts (not in the repo): `/private/tmp/claude-501/scratch/{nc,tools,vaults,measure.py,samples_*.txt,crates_bloat.txt}`; build outputs in `target/agent-bloat` and `target/agent-bloat-s` (safe to delete, about 2 GB).
|
||||
@@ -0,0 +1,283 @@
|
||||
# NotesCraft registry: separable add-ons, and an honest debloat plan
|
||||
|
||||
Status: draft for owner review (2026-10-21). Numbers: `measurements.md`. Tasks: `tasks.md`. Owner request: "separate some of the plugins and certain things from the current application, add them to the notescraft registry, to debloat the app so it doesn't feel heavy".
|
||||
|
||||
## 0. Verdict first
|
||||
|
||||
1. **The registry is the right mechanism for choice and growth, not for the weight the app has today.** Measured on this tree: binary 38.4 MiB, RSS 131 MB idle (physical footprint 72 MB), 0.3 s CPU to idle. Everything that could move to a registry today (extra grammars, math font, Lottie icons, Lucide long tail, extra themes) is **about 0.7 MiB, 2 % of the binary**, and moves no startup or RSS number you can feel.
|
||||
2. **The real, cheap debloat needs no registry** and is about 10 MB (25 %): strip symbols on macOS and Windows (-5.4 MB measured), fonts (-0.8 MiB duplicate Inter, about -1 MiB unused egui default fonts, -0.26 MiB duplicate mono), `opt-level = "s"` (-2.8 MB more, perf unmeasured). Fonts are 4.2 MiB, 11 % of the file, more than all the optional crates together.
|
||||
3. **Why it "feels heavy" is vault data, not features**: a 6500-note vault peaks at 494 MB while indexing and settles at 220 to 380 MB. No add-on mechanism changes that; it needs its own spec (open question 9).
|
||||
4. **The wasm plugin host is not wired into the app at all** (`Session::load_plugins` has no non-test caller; `wasmi` is 0 bytes in the shipped binary). The registry cannot deliver plugins until that is wired, and wiring adds an estimated 0.8 to 1.2 MiB. Order of work matters: do the no-registry savings first, or the registry makes the binary bigger before it makes it smaller.
|
||||
5. The registry still earns its place: it stops the binary growing (every new grammar, theme, template, small command lands outside it), gives users choice, and finishes a half-built feature (`grammar.install`, an empty `notescraft-registry` repo, `plugins/dice`).
|
||||
|
||||
So the plan has three layers, in this order: **P0** size fixes without a registry; **P1** a small, signed, offline-first registry (one engine module, no new crate); **P2** move the few things that fit, to prove the pipeline and bank the 35 ms and 0.7 MiB.
|
||||
|
||||
## 1. What exists today (reuse, do not rebuild)
|
||||
|
||||
- Registry repo `https://git.marcospaulo.dev.br/masi/notescraft-registry` (local clone `/Volumes/Storage/Projects/notescraft-registry`), `index.json` with four empty lists, README schema. **Schema mismatch to fix**: the README names the key `name`, the engine (`crates/engine/src/grammars/mod.rs::registry_entry`) reads `id`. No entry was ever published, so no compatibility burden.
|
||||
- Grammar flow: `grammar.info|install|install_text|remove`, gates in `crates/syntax/src/{fetch,validate}.rs` (https only, no credentials or IP literals, host allow-list `REGISTRY_HOST`, no redirects, 512 KiB cap, sha256, parse and sample), transport `Curl` (a `curl` subprocess) behind a `Transport` trait, state in `AppSettings::grammars` (max 64) and files in `<settings dir>/grammars/`.
|
||||
- Themes: user themes are `<settings dir>/themes/*.toml` (`prefs/themes.rs`, max `MAX_USER_THEMES`, ids may not reuse a built-in).
|
||||
- Plugins: `crates/plugins` (wasmi host, ABI v1, manifest with `permissions` allow-list), folder `<vault>/.notescraft/plugins/<id>/{manifest.json,plugin.wasm}`, approvals kept in settings, sample `plugins/dice` (4.8 KB). No view, panel, settings or event hooks (ABI v1); UI-contribution ABI is roadmap item 25.
|
||||
- `sha2` is approved; `ureq` (rustls, which brings `ring`) is already linked into the engine; `craft-atomic` offers `atomic_write(path, bytes)` and `write_new(path, bytes)`.
|
||||
- Core plugin cards in Settings (Calendar, Templates, Diagrams, Whiteboard) only toggle features that are compiled in. They are organisational, not size.
|
||||
- Name clash to avoid: `crates/engine/src/registry` is the **command** registry. The new engine module is `addons/`; command ids are `registry.*`.
|
||||
|
||||
## 2. Classification of everything that could move
|
||||
|
||||
Legend: **core** always in the binary; **lazy** stays in the binary, initialised on first use; **feature** cargo feature (compile-out); **pack** downloadable assets; **wasm** plugin on ABI v1; **UI-ABI** blocked on roadmap item 25.
|
||||
"Effect" is measured unless marked est.
|
||||
|
||||
| Item | Class | Effect on size / startup / RSS | Honest note |
|
||||
|---|---|---|---|
|
||||
| egui, wgpu, winit, text shaping, image codecs, serde, regex, TLS | core | 9 MiB of `.text` | cannot move |
|
||||
| `ui-egui` (4.2 MiB), `engine` (1.8 MiB), vault, markdown, index, editor, i18n, themes, search, backlinks, links, tags, properties, bases core, templates, tasks, lint, blocks | core | | the product |
|
||||
| Inter Regular/SemiBold/Medium, JetBrains Mono NoLigatures | core | | |
|
||||
| Duplicate Inter Regular + SemiBold in `engine` PDF export | **fix in place** (share one `const`) | -0.79 MiB | no registry |
|
||||
| PDF export JetBrains Mono Regular | **fix in place** (use NoLigatures) | -0.26 MiB | no registry |
|
||||
| egui `default_fonts` (Hack, Ubuntu-Light, emoji-icon-font; keep NotoEmoji) | **fix in place** (drop the eframe feature, embed NotoEmoji only) | about -0.95 MiB (est) | needs a snapshot check for tofu |
|
||||
| Symbols (`__LINKEDIT`) | **release profile** `strip = "symbols"` | **-5.38 MB** | losing symbol names in crash backtraces is the cost |
|
||||
| `opt-level = "s"` | **release profile** | -2.8 MB more | measure index build and scroll first |
|
||||
| syntect default pack (75 grammars, 0.37 MiB, built on first fence: 75 ms) | lazy (already) | | trimming it needs a custom dump; skip |
|
||||
| 25 NotesCraft grammars (Twig, Blade, Svelte, Vue, Nix, Zig...) | **pack** `grammar` | -69 KB, **-35 ms** first highlight, repeated at every `set_installed` | best first mover: tiny, pure text, proves the pipeline |
|
||||
| STIX Two Math font (math only) | **pack** `font` | -0.36 MiB | math falls back to Inter/JetBrains glyphs and boxes until installed; needs the seed (section 11) |
|
||||
| Lottie icon animations (65 JSON, 0.14 MiB) | **pack** `pack` | -0.14 MiB | static icons already exist; animation is polish |
|
||||
| Lucide long tail (278 catalogue vs the ~89 used) | pack | at most -0.07 MiB | not worth the complexity; skip |
|
||||
| `en.toml` (61 KB) | core | | future locales ship as `pack` (language packs): zero cost now, the one place the registry clearly pays |
|
||||
| Extra built-in themes (Frappe, Macchiato, Solarized x2) | **pack** `theme` | under -10 KB | worth it only as the first theme entries, for the browser to show |
|
||||
| Community-style themes, template sets, snippets | **pack** / `theme` / `snippet` | 0 (they never were in the binary) | the real value of a registry |
|
||||
| Extra Hebrew/Arabic system fallback read at launch (SFHebrew, SFArabic) | lazy | -0.8 MB RSS, 1 disk read | use the existing script-on-demand loader |
|
||||
| CJK and other script fonts | none | | already **not** bundled: `fonts::install` loads system fonts per script on demand. A Noto download pack is only for Linux without them; defer |
|
||||
| Wasm plugin host (`wasmi`) | **feature** `plugins`, default on | 0 bytes today (not linked), -37 CPU-s of compile; +0.8 to 1.2 MiB (est) once wired | wire it first (task 12), then the feature exists for CLI/server builds |
|
||||
| Dice roller, Citations (roadmap 10), word tools, small text commands | **wasm** (ABI v1: vault.read, vault.write, editor) | 4.8 KB each, 0 in the binary | the only real "plugins" that fit today |
|
||||
| Importers (Notion zip, ENEX, Roam) | needs a host call for zip/file import, or stays in-tree | | ABI v1 has no file import and no binary input: **not** wasm yet |
|
||||
| Calendar (170 KB + 2.9k UI lines + 2k engine lines), Whiteboard (57 KB + 5.9k UI), Diagrams, Canvas, Graph, Bases Map/Cards, Kanban, Slides, Charts, Mind map, Maps | **UI-ABI** (item 25) to become wasm; **feature** to compile out | all optional crates: 1.15 MiB `.text`; with their UI and engine halves 2.5 to 3.5 MiB (est), 7 to 9 % | extracting them to wasm needs views, panels, settings, events. Cargo features alone save little for the desktop app and cost a large refactor; only do it for the CLI/server builds |
|
||||
| Vim, assistant (ACP), knowledge (RAG), pdf export, git, collab | **feature** candidates | 0.1 to 0.4 MiB each, est | organisational for the desktop app; useful for slim CLI/MCP builds |
|
||||
| Map tile providers | **snippet** presets | 0 | network policy already opt-in (bases D13); a preset list is data |
|
||||
| Pandoc | external helper, detected on `PATH` | 0 | **do not** download executables in v1 (open question 5) |
|
||||
| Video | external player via `shell/app/video/launch.rs` | 0 | already a helper, not bundled |
|
||||
| OCR backend (attachment-ocr spec) | helper binary pack, later | | needs its own trust model; out of scope here |
|
||||
|
||||
What actually reduces what:
|
||||
- **Binary size**: P0 (about 10 MB), then fonts-as-pack and Lottie (0.5 MiB). Cargo-feature compile-outs are the only big code lever and are a refactor, not a registry job.
|
||||
- **Startup**: the grammar rebuild (-35 ms of 110 ms on first fence) and lazy fallback fonts. Nothing else measured moves.
|
||||
- **RSS**: nothing in this spec measurably does; the levers are index memory and egui/GPU surfaces.
|
||||
- **Only organisational**: core plugin cards, moving crates between folders, the Lucide tail.
|
||||
|
||||
## 3. Goals and non-goals
|
||||
|
||||
Goals: a signed, static, offline-first catalogue of add-ons the user installs on purpose; one engine surface (commands, CLI, MCP, UI); zero loss for anyone upgrading; hostile content cannot write outside its own folder or run anything without consent.
|
||||
|
||||
Non-goals (v1): third-party publishing flow or accounts; a download counter; payments; automatic code execution; executable downloads (pandoc, ffmpeg); archive formats; plugin sandbox changes; UI-contribution ABI (item 25); a custom registry URL (open question 4).
|
||||
|
||||
## D1. Hosting and index
|
||||
|
||||
- **v1 host: the existing Gitea repo** `masi/notescraft-registry`, raw layout `raw/branch/main/`. No server work, history is the audit log, the host allow-list `REGISTRY_HOSTS = ["git.marcospaulo.dev.br"]` already exists (becomes a list). Mirror later by adding a host to the allow-list and `url` in entries.
|
||||
- **Optional later: the sync server** serves the same tree from a directory with one read-only route `GET /registry/{*path}` (about 30 lines in `apps/notescraft-sync-server/src/api/routes.rs`, public like `/p/{slug}`, no auth, no vault data, file allow-list by extension, caches by ETag). Not needed for v1; listed because the owner asked whether it could host one: yes, but a git repo is simpler and already trusted.
|
||||
- Files: `index.json` at the root; payload files under `files/<kind>/<id>/<version>/<name>`.
|
||||
- The app caches the last verified index at `<data dir>/registry/index-cache.json` with `fetched_at` and `etag`; a conditional GET (`If-None-Match`) keeps checks near-free.
|
||||
- Index cap 1 MiB, at most 2000 entries; a bigger or non-JSON index is refused and the cache stays.
|
||||
|
||||
## D2. Index schema v2
|
||||
|
||||
```json
|
||||
{
|
||||
"version": 2,
|
||||
"generated_at": "2026-10-21T12:00:00Z",
|
||||
"entries": [
|
||||
{
|
||||
"id": "twig",
|
||||
"kind": "grammar",
|
||||
"title": "Twig",
|
||||
"summary": "Twig templates",
|
||||
"version": "1.0.0",
|
||||
"min_app_version": "0.1.0",
|
||||
"license": "MIT",
|
||||
"source": "https://github.com/...",
|
||||
"aliases": ["twig", "html.twig"],
|
||||
"suggested": false,
|
||||
"permissions": [],
|
||||
"files": [
|
||||
{ "path": "twig.sublime-syntax", "size": 3912, "sha256": "<64 hex>", "url": "https://git.marcospaulo.dev.br/masi/notescraft-registry/raw/branch/main/files/grammar/twig/1.0.0/twig.sublime-syntax" }
|
||||
],
|
||||
"signature": { "key_id": "k1", "sig": "<base64 ed25519>" }
|
||||
}
|
||||
],
|
||||
"revoked": [ { "id": "old-thing", "version": "0.9.0" } ]
|
||||
}
|
||||
```
|
||||
|
||||
Fields (all required unless "optional"): `id` `[a-z0-9-]{1,48}` unique per `kind`; `kind` one of `plugin theme pack grammar font snippet`; `title` 1..80; `summary` 0..200 (optional); `version` `MAJOR.MINOR.PATCH`, each 0..9999, no pre-release; `min_app_version` same shape; `license` SPDX-like 1..40; `source` https URL (optional, display only); `aliases` (grammar: fence labels; others: search words), at most 16, each `[a-z0-9._+-]{1,32}`; `suggested` bool (optional, first-run list); `permissions` (plugin only, must be a subset of the known `Permission` strings; unknown rejects the entry); `files` 1..64, each with `path`, `size`, `sha256`, `url`; `signature`. The README's `file` becomes `files[].url` relative form is **not** allowed: urls are absolute https on an allow-listed host.
|
||||
|
||||
Derived, never in the file: total size = sum of `files[].size`.
|
||||
|
||||
Caps per kind (sum of file sizes): plugin 4 MiB, theme 64 KiB, grammar 512 KiB (the existing cap), snippet 1 MiB, pack 8 MiB, font 32 MiB. One file at most half of the cap of 32 MiB for fonts, otherwise the whole cap. All checked against both the index size and the bytes actually received.
|
||||
|
||||
Validation lives in `addons/index.rs`; one broken entry is skipped with a diagnostic, the rest of the index loads (same rule as plugin folders). A wrong `version` at the top rejects the index.
|
||||
|
||||
## D3. Trust and signatures
|
||||
|
||||
- Every entry carries an **ed25519 signature** by the owner-held key over a canonical text:
|
||||
`notescraft-registry-v1\n<id>\n<kind>\n<version>\n` then for each file sorted by `path`: `<path>\n<sha256>\n<size>\n`. The signature therefore covers every hash and size; the index itself can stay unsigned (tampering with it can only add entries whose signature fails, which is a denial of service, not code execution).
|
||||
- The app embeds the public keys: `addons/keys.rs` `TRUSTED_KEYS: [(&str, [u8; 32]); N]`, `key_id` selects one, so a rotation is a release that adds `k2` and later drops `k1`. The private key never enters this repo; it lives with the owner. Signing happens in the registry repo with a small tool (`xtask registry-sign` over a directory, task 18).
|
||||
- `revoked` lets a release be pulled; an installed revoked version is marked and the UI offers Remove or Rollback. The index is unsigned, so an attacker can hide `revoked`; stated, accepted for v1, solved by signing the index (open question 2).
|
||||
- **Dependency question for the owner**: verification needs an ed25519 implementation. Options: (a) depend on `ring` directly, already in the lockfile and linked through `rustls`, adds roughly nothing to the binary; (b) `ed25519-dalek`; (c) no signatures, rely on HTTPS plus sha256 plus the host allow-list. **Recommendation: (a)**, written approval needed (`AGENTS.md` rule on new dependencies). Until answered, task 3 is blocked and installs must be refused for entries without a valid signature, so the feature cannot ship half-secure.
|
||||
- A sideloaded or seed file (section 11) is verified with the same signature check; there is no "trust me" path.
|
||||
- Integrity beyond signatures: plugin manifests inside the downloaded files must match the index entry (`id`, `version`, `permissions`, commands), or the install fails.
|
||||
|
||||
## D4. Local layout
|
||||
|
||||
All add-on state lives under the app data dir, never in a vault (so a synced vault cannot ship code):
|
||||
|
||||
```
|
||||
<data dir>/registry/ data dir = craft_settings::paths::data_dir("notescraft"), override NOTESCRAFT_CONFIG_DIR
|
||||
state.json installed table, written with craft-atomic (version 1)
|
||||
index-cache.json last verified index + fetched_at + etag
|
||||
store/<kind>/<id>/<version>/ immutable once state points at it; files exactly as in the entry
|
||||
staging/<random>/ downloads in progress; emptied at every start
|
||||
```
|
||||
|
||||
`state.json`: `{ "version": 1, "installed": { "<kind>:<id>": { "version", "previous": "1.0.0"?, "source": "registry|legacy|seed|sideload", "installed_at", "sha256_of_files": "<hex>" } } }`. At most 512 installed entries.
|
||||
|
||||
Per-vault file `<vault>/.notescraft/registry.json` (written with `Vault::write_private`): `{ "version": 1, "enabled": ["plugin:dice"], "wanted": ["grammar:twig"] }`.
|
||||
- `enabled`: plugins on in this vault (plugins are the only kind that is per-vault; permissions approvals stay in settings as today).
|
||||
- `wanted`: ids this vault has been seen to use (a Twig fence). A new machine opening the vault shows one banner "This vault uses 2 add-ons" with an Install button. Nothing is installed from it automatically.
|
||||
|
||||
Grammars, themes, fonts, packs and snippets are **global** (user level); enabling = installed. Rollback keeps one `previous` version dir per id.
|
||||
|
||||
## D5. Network policy
|
||||
|
||||
No network unless one of three things happens, all user-initiated:
|
||||
1. the user opens Settings > Registry (one conditional GET of `index.json`; not at app start);
|
||||
2. the user presses Install, Update, Check or an install button on a card;
|
||||
3. the setting `registry.update_checks` is on (default **off**): at most one conditional GET per 24 h, started after the window is idle, no identifiers, `User-Agent: NotesCraft/<version>`, no vault data, no cookie.
|
||||
|
||||
Everything else (search in the browser, first-run suggestions, "this vault uses X") reads the cached or seed index and works offline. A failed fetch leaves the cache and state untouched and shows "Offline" (the `offline` error code exists). Same gates as grammars on every URL: https only, host allow-list, no credentials, no IP literal, no redirects (`syntax::validate::check_url`). Transport: `ureq` (already linked, no subprocess); the `Transport` trait stays injectable so tests never touch the network.
|
||||
|
||||
## D6. Install, update, remove, rollback
|
||||
|
||||
All entry operations are one function `addons::install::run(plan)` over an injected `Transport`:
|
||||
|
||||
1. Resolve the entry from the cached or fresh index; refuse if `min_app_version` is newer than the app, if `revoked`, if the kind has no consumer yet, if a file list check fails (D7), or if the signature does not verify.
|
||||
2. Check free disk space (twice the entry size) and the installed-table cap.
|
||||
3. Download each file to `staging/<random>/<path>` through the transport: streaming, hard cap per file and per entry, sha256 computed on the bytes actually received (a lying `Content-Length` does not help), abort on the first mismatch.
|
||||
4. Per-kind content check on the staged bytes (D8). A failure deletes staging; nothing else changed.
|
||||
5. Write the final tree `store/<kind>/<id>/<version>/<path>` with `craft_atomic::write_new` per file (refuses to overwrite). The version directory is unreferenced until step 6, so a crash leaves garbage, not a half-install. Garbage (version dirs not in state, staging) is removed at the next start.
|
||||
6. Flip `state.json` with `craft_atomic::atomic_write` (the only commit point): set `version`, keep the old one in `previous`.
|
||||
7. Tell consumers to reload (grammar set rebuild, theme list, plugin set).
|
||||
|
||||
`update` = install of a newer version of an installed id. Downgrade is never an update; only `rollback`.
|
||||
`remove` deletes the state entry first (commit point), then the store dirs, then reloads consumers; blocks of a removed grammar fall back to plain monospace (existing behaviour), a theme in use falls back to the built-in of the same variant with a notice, a plugin is unloaded.
|
||||
`rollback` swaps `version` and `previous` in `state.json` (the old dir is still there) and reloads.
|
||||
|
||||
## D7. Hostile input (no archives by design)
|
||||
|
||||
Entries are file lists, not archives: **no zip or tar anywhere**, which removes zip-slip, symlink and decompression-bomb classes without a parser. What remains, all in `addons/fileset.rs`, pure and unit-tested:
|
||||
- `path`: relative, `/`-separated, 1..4 components, each `[A-Za-z0-9._-]{1,64}`, no component empty, `.` or `..`, none starting with `.`, no `\`, no `:`, no NUL, no Windows reserved device name (`CON`, `NUL`, `COM1`...) as a component, no case-insensitive duplicates, extension on the allow-list of the kind (plugin `manifest.json` and `plugin.wasm` and optional `README.md`; theme `.toml`; grammar `.sublime-syntax`; font `.ttf` `.otf`; snippet `.md` `.txt` `.toml`; pack: each file under `grammars/`, `themes/`, `templates/` or `icons/` with its own extension list). The joined path is built by the code from validated components, never by string concat of server text.
|
||||
- Sizes: per kind caps from D2, at most 64 files, no zero-size file, sum with `checked_add`.
|
||||
- URLs: `check_url` plus `url` host in the allow-list, length cap, no query string.
|
||||
- Download: streamed with a hard cap; received bytes and sha256 must match the entry; a body shorter or longer is a failure; slow-loris bounded by a total time limit (30 s per file, 5 min per entry).
|
||||
- Never execute: nothing from the store is run except `plugin.wasm` through the existing sandbox (fuel, memory, deadline limits, 3-failure auto-disable) and only after the user approved its permissions.
|
||||
- Writes go to `<data dir>/registry/` only; the reserved vault paths `.obsidian/` and `.notescraft/` rules are unchanged; `.notescraft/registry.json` is written only through `write_private`.
|
||||
- JSON bounds: index and state parse with the existing `serde_json` depth limit, strings capped, unknown `version` is `Err`, nothing silently repaired.
|
||||
|
||||
## D8. Kinds
|
||||
|
||||
| Kind | Installed as | Consumer | Per-kind check |
|
||||
|---|---|---|---|
|
||||
| `grammar` | `store/grammar/<id>/<v>/<id>.sublime-syntax` | `syntax::load::set_installed` fed from state (legacy `AppSettings::grammars` also read) | `syntax::validate::check_grammar` with the entry's `sample` |
|
||||
| `theme` | `store/theme/<id>/<v>/<id>.toml` | `Session::user_themes` scans state too | `Theme::from_toml`; zero hard diagnostics; id equals entry id; no built-in id |
|
||||
| `plugin` | `store/plugin/<id>/<v>/{manifest.json,plugin.wasm}` | the plugin loader reads installed plugins listed in the vault's `enabled`; approvals as today | `Manifest::parse`, manifest equals entry, a dry `Plugin::load` with default limits |
|
||||
| `pack` | several files under `grammars/ themes/ templates/ icons/` in one version dir | each consumer reads the pack's subfolder | every file runs its own kind check |
|
||||
| `font` | `store/font/<id>/<v>/<file>` | the script-on-demand font loader may use it before system fonts; not required for v1 consumers | `ttf-parser`-free check: magic bytes `00 01 00 00`, `OTTO` or `true`, size cap (no new dependency) |
|
||||
| `snippet` | `store/snippet/<id>/<v>/*.md` | templates list (insert from the template picker) | UTF-8, size cap, no NUL |
|
||||
|
||||
Kinds `font` and `snippet` are accepted by the index and the installer so the schema is complete, but a kind with no consumer yet is shown as "Not available in this version" and cannot be installed (`D6` step 1). The first consumers to build are grammar, theme, plugin, pack.
|
||||
|
||||
## D9. Engine commands (all in `crates/engine/src/addons/commands.rs`)
|
||||
|
||||
Group registered by `register_addons(reg)` called from `commands.rs::register_core`. Ids use the user-facing prefix `registry.`.
|
||||
|
||||
| Id | Params | Does | Network | Gate class |
|
||||
|---|---|---|---|---|
|
||||
| `registry.list` | `kind?`, `query?`, `state?` (`installed`, `updates`, `available`), `limit?` | cached or seed index joined with `state.json`; each row: id, kind, title, version, installed version, size, state (`available`, `installed`, `update`, `revoked`, `incompatible`, `unsupported`), permissions | no | READ |
|
||||
| `registry.info` | `kind`, `id` | one entry in full: files, sizes, license, source, signature key id, what a plugin may do | no | READ |
|
||||
| `registry.check` | none | conditional GET of the index, verify, replace the cache, return update counts | yes | DELETE-class (see below) |
|
||||
| `registry.install` | `kind`, `id`, `version?` | D6 | yes (or seed) | DELETE-class |
|
||||
| `registry.update` | `kind?`, `id?` (none = all with updates) | D6 per entry | yes | DELETE-class |
|
||||
| `registry.remove` | `kind`, `id` | D6 | no | DELETE-class |
|
||||
| `registry.rollback` | `kind`, `id` | D6 | no | DELETE-class |
|
||||
| `registry.enable` / `registry.disable` | `id` (plugin) | per-vault `enabled` list; enabling asks the existing permission approval | no | WRITE |
|
||||
| `registry.suggest` | `label` or `kind`+`query` | alias lookup in the cached index; used by the grammar card | no | READ |
|
||||
|
||||
Assistant gate (`crates/engine/src/ai/assistant_gate.rs`): `registry.list`, `registry.info`, `registry.suggest` join `READ_COMMANDS`. `registry.enable|disable` join `WRITE_COMMANDS`. The five that fetch or remove third-party code (`check`, `install`, `update`, `remove`, `rollback`) join `DELETE_COMMANDS`, so an assistant needs both `allow_write` and `allow_delete`, the same treatment as `sync.pair` and `server.connect` (trust-expanding actions). Test: `tests/core_commands.rs` `CORE_IDS` and a gate test per id.
|
||||
|
||||
Errors (`CommandError` codes): `registry_refused` (a gate failed, message names it), `offline`, `not_found`, `incompatible` (app too old), `revoked`, `limit`, `disk_full`, `needs_dependency` (no signature verifier yet). Existing `grammar.*` commands stay as thin aliases that call the same code and keep their codes.
|
||||
|
||||
CLI parity: `notescraft run registry.install --params '{"kind":"grammar","id":"twig"}'` already works through the generic `Run`; task 9 adds `notescraft registry list|info|check|install|update|remove|rollback` as sugar in `apps/notescraft-cli/src/args.rs` and `commands/`, printing id, version, size, state. MCP gets them through the registry automatically.
|
||||
|
||||
## D10. UI (Settings > Registry)
|
||||
|
||||
New Settings category in `views/settings_view/categories.rs`, view folder `views/registry_view/` (library components only: Button, Input, Switch, Modal, ConfirmModal; i18n keys under `[ui.registry]`; theme tokens only).
|
||||
|
||||
- Top row: search Input, kind chips (All, Plugins, Themes, Packs, Grammars, Fonts, Snippets), "Check for updates" Button, "Last checked <relative time>" or "Never", Switch "Check for updates automatically" (off).
|
||||
- List rows: title, kind chip, version, size, short summary, state: `Install` Button, `Installed` check, `Update to 1.2.0` Button (accent), `Needs NotesCraft 0.3`, `Revoked`, `Not available in this version`. Update badge: a count dot on the Settings entry and on the category, only after a check.
|
||||
- Detail drawer: summary, license, source link, files and sizes, **permissions as sentences** ("Read notes and search", "Create and change notes", "Edit the open note") for plugins, signature key id, Remove and Rollback Buttons.
|
||||
- Offline banner when the last fetch failed; the list then shows the cache.
|
||||
- Plugin install ends with the existing permission approval modal, then an Enable Switch for the current vault.
|
||||
- **First-run suggestions**: once, after the first vault opens following install or upgrade, a dismissible card "Add-ons you may want" listing the entries with `suggested: true` from the seed index plus anything `registry.suggest` found for this vault. It never fetches; its Install buttons do (D5 trigger 2).
|
||||
- Code card (`views/note_view/blocks/code_card.rs`): the existing "Add grammar" link becomes "Install Twig grammar (3.9 KB)" when `registry.suggest` finds the label, falling back to today's dialog (paste a link) when it does not.
|
||||
- Snapshot example flag `--registry` for visual checks; the screens follow the PaperCraft tokens (rounded cards on canvas, tone not borders).
|
||||
|
||||
## D11. Migration: nobody loses anything
|
||||
|
||||
Rule: **nothing leaves the binary in the same release that adds its registry entry**, and every removal ships with a path that needs no thought.
|
||||
|
||||
1. Release N adds the registry, adopts legacy grammars (`source: legacy`, shown as installed, no update), keeps everything bundled.
|
||||
2. Release N+1 removes an item from the binary only if all hold: its entry is published and signed; the **seed** copy (below) is in the installer; a migration test exists.
|
||||
3. **Seed**: the installer carries `registry-seed/{index.json, files/...}` next to the binary (macOS `Contents/Resources/`, Windows and Linux beside the exe), a verified signed snapshot of the moved items only (about 0.6 MiB). `install` prefers the seed file when its sha256 equals the entry, so moved items install **offline, instantly**. Seed items are not auto-installed. The DMG size does not change; the binary shrinks.
|
||||
4. **Auto-offer by use**, no network:
|
||||
- a fence label that is no longer bundled: the code card asks `registry.suggest` and shows "Install Twig grammar", one click; with `registry.auto_install_for_vault` on (default **off**, open question 3) it installs from the seed without asking;
|
||||
- first launch of N+1: a background scan (ripgrep crates already in the engine, 3 s budget, once per vault and per registry generation) of fence labels and `$` math in the opened vault records `wanted` in `.notescraft/registry.json` and shows one banner "This vault uses Twig, Blade (2 add-ons, 7 KB): Install";
|
||||
- a selected theme that moved: the app keeps the id, falls back to the built-in of the same variant until installed, and the Settings theme picker shows it with an Install button;
|
||||
- math without the font: formulas fall back to the Inter/JetBrains glyph chain (already the last resort) and the math view shows an Install button.
|
||||
5. Tests (task 19 to 21 each): for every moved id, `registry.suggest` finds it, the seed verifies, `registry.install` from the seed in an offline temp session makes `grammar.info` report installed, and removal returns the old fallback behaviour.
|
||||
|
||||
## D12. First-party content, ordered by measured benefit
|
||||
|
||||
| # | Item | Size | Gain | Needs |
|
||||
|---|---|---|---|---|
|
||||
| P0.1 | `strip = "symbols"` in the release profile (macOS, Windows) | - | **-5.38 MB** | nothing |
|
||||
| P0.2 | share one Inter Regular/SemiBold `const`, PDF mono = NoLigatures | - | -1.05 MiB | brief 29 |
|
||||
| P0.3 | drop eframe `default_fonts`, embed NotoEmoji only | - | about -0.95 MiB | snapshot check |
|
||||
| P0.4 | `opt-level = "s"` | - | -2.8 MB | perf measurement |
|
||||
| P1.1 | wire `load_plugins` (+ ~1 MiB, est) | | enables plugins | task 12 |
|
||||
| P2.1 | pack `grammars-extra` (25 grammars as 25 `grammar` entries) | 69 KB | -69 KB, -35 ms first highlight | tasks 19 |
|
||||
| P2.2 | font `stix-math` | 376 KB | -0.36 MiB | task 20 |
|
||||
| P2.3 | pack `icons-animated` (Lottie) | 148 KB | -0.14 MiB | task 21 |
|
||||
| P2.4 | plugin `dice` (exists), then `citations` | 4.8 KB | 0 binary, first real plugin | task 22 |
|
||||
| P2.5 | themes Frappe, Macchiato, Solarized | under 10 KB | negligible, fills the browser | task 22 |
|
||||
| later | language packs when a second locale exists | 60 KB each | keeps `en`-only core | |
|
||||
|
||||
P2.x together: about 0.6 MiB, 1.5 % of the binary. Say it plainly to users: the registry makes the app extensible; the size win is P0.
|
||||
|
||||
## D13. Tests and acceptance
|
||||
|
||||
- Pure: index parse and every rejected shape; fileset table of hostile paths (`../x`, `a/../../b`, `/abs`, `C:\x`, `a\b`, `.hidden`, `a//b`, names with a space or NUL, Windows reserved names `CON`, `NUL`, `COM1` as a whole component, duplicates differing only in case, a 65-char component, 5 components, an empty path); caps with `u64::MAX` sizes (checked add); signature canonical text; version compare; state round trip; unknown version.
|
||||
- IO (TempDir, fake `Transport`): happy install of each kind; hash mismatch; short body; long body; lying size; transport error mid-file; crash simulation (stop before the state flip) leaves the old state and GC removes the new dir; update keeps `previous`; rollback; remove while in use; seed install offline; index with a revoked version; app too old; invalid signature; unknown key id.
|
||||
- Gate and commands: all ids in `CORE_IDS`; gate classes; read-only session refuses mutating commands.
|
||||
- UI (kittest): rows by state; Install sends exactly one command; offline banner; permission sentences; update badge appears only after a check.
|
||||
- Size acceptance (measured, not promised): after P0.1 to P0.3 the macOS binary is at most 33 MB (from 40.2 MB); the registry module itself adds at most 250 KB (estimate for index, installer, validators, UI) and wiring plugins at most 1.2 MiB; both are re-measured with the commands in `measurements.md` and written into this file and `docs/ARCHITECTURE.md`.
|
||||
- `cargo xtask ci`, `cargo xtask i18n`, `cargo xtask layers`, `cargo xtask tree` green; no `unwrap`, `[i]`, `[a..b]`; every public item documented; files under 400 lines; at most 12 loose `.rs` per directory (`addons/` is grouped into `index/`, `install/`, `commands/` from the start).
|
||||
|
||||
## Open questions for the owner
|
||||
|
||||
1. **Ed25519 dependency**: approve `ring` as a direct dependency of `notescraft-engine` (already in the lockfile through `rustls`), or `ed25519-dalek`, or choose no signatures? Recommendation: `ring`, plus `xtask registry-sign` also using it. Blocks task 3 and every install.
|
||||
2. **Sign the index too** (one extra detached signature, stops hidden `revoked` and replay of an old index)? Recommendation yes, 20 lines, after question 1.
|
||||
3. **Auto-install for a vault that uses a moved item**: your words were "installed automatically if the vault uses it". This conflicts with "no network unless the user opts in". Proposal: seed files make it offline and instant, so auto-installing **from the seed** can be on by default (no network), while anything that needs the network asks. Confirm.
|
||||
4. **Custom registry URL / third-party registries**: not in v1 (host allow-list is fixed to yours). Allow later with a per-registry key?
|
||||
5. **Executables** (pandoc, ffmpeg, OCR engines): not downloaded in v1; they stay "detected on PATH". Confirm, or ask for a separate trust design.
|
||||
6. **Ship seed files in the installer?** Costs about 0.6 MiB of installer size, buys offline migration. Alternative: network only.
|
||||
7. **Release profile**: approve `strip = "symbols"` (crash backtraces lose function names; line tables are already `debug = line-tables-only` in dev only) and, after a perf run, `opt-level = "s"`.
|
||||
8. **Drop the egui `default_fonts` feature** and embed only NotoEmoji (about -0.95 MiB): accept if the snapshot comparison shows no tofu in the UI chrome.
|
||||
9. **Index memory**: the 6500-note vault peak of 494 MB and 220 to 380 MB steady is the real "heavy". Want a separate measurement and spec for it?
|
||||
10. Naming: engine module `addons/` and user-facing "Registry" / `registry.*`: fine?
|
||||
@@ -0,0 +1,61 @@
|
||||
# Tasks: registry and debloat
|
||||
|
||||
One task = one agent (`specs/agents/opencode-runbook.md`). `[ ]` open. Spec: `spec.md` (sections D1 to D13). Numbers: `measurements.md`. Rules: `AGENTS.md`, `docs/CODE-MAP.md`. No new crate: everything new is `crates/engine/src/addons/` (not `registry`, that name is the command registry).
|
||||
**First tool call rule**: after at most three reads, write the first OWNS file in your SECOND tool call: a compiling stub with the GOAL signatures and ONE failing test, then make it pass.
|
||||
**Standing VERIFY**: `export PATH="$HOME/.cargo/bin:$PATH"`; `cargo test -p <pkg>`; `cargo clippy -p <pkg> --all-targets -- -D warnings`; `rustfmt --edition 2024 <owned files>`; UI/i18n tasks add `cargo xtask i18n`; the last task of a phase adds `cargo xtask layers` and `cargo xtask tree`.
|
||||
**Standing DO NOT**: edit Cargo files (tasks marked LEAD do), files outside OWNS, git state, `cargo fmt --all`; add `allow` attributes; use `unwrap/expect/panic/[i]/[a..b]` on input-derived data; add a dependency; write outside `<data dir>/registry/` and `.notescraft/registry.json`.
|
||||
**Blocked on owner**: task 3 and everything that installs (7 onward) wait for open question 1 (ed25519). Tasks 1, 2, 4, 5, 6 and Phase 0 do not.
|
||||
|
||||
## Phase 0 - size fixes without a registry (do these first, they are 25 % of the binary)
|
||||
|
||||
- [ ] 0.1 LEAD: release profile. OWNS: root `Cargo.toml` (`[profile.release]` only), `packaging/macos/*.sh` and `packaging/windows/*` only if they strip separately. GOAL: add `strip = "symbols"`; confirm Linux `package.sh` strip is now redundant but harmless. VERIFY: `CARGO_TARGET_DIR=target/agent-size cargo build --release -p notescraft`, `ls -l` is at most 34.9 MB (measured 34,851,464), the app still starts (`measure.py` in `measurements.md`), `cargo xtask ci`. DO NOT: set `panic = "abort"` (the never-crash policy and tests rely on unwinding).
|
||||
- [ ] 0.2 Share fonts (brief `briefs/29-font-dedupe.md`).
|
||||
- [ ] 0.3 Drop egui `default_fonts`. OWNS: `apps/notescraft/Cargo.toml` (eframe feature list only, LEAD edit), `crates/ui-egui/src/design/fonts/mod.rs`, `assets/fonts/NotoEmoji-Regular.ttf` (copy from `epaint_default_fonts`, with its OFL text), `crates/ui-egui/tests/fonts_emoji.rs`. FILES TO OPEN: `crates/ui-egui/src/design/fonts/mod.rs` (`install`), `~/.cargo/registry/src/*/epaint_default_fonts-*/fonts/`. GOAL: `eframe` without `default_fonts`, `FontDefinitions::empty()` plus Inter, JetBrains Mono, STIX, NotoEmoji (last in every chain), the system fallbacks as today. TESTS: kittest renders `😀 ★ → ✓` in proportional and monospace with no missing-glyph box (compare against the egui replacement glyph); the snapshot of the Settings and Explorer screens differs only by sub-pixel noise. VERIFY: standing + `cargo run -p notescraft-ui-egui --example snapshot -- --vault <v> --out a.png --size 1440x900` before and after, read both PNGs; binary shrinks by at least 0.9 MiB. DO NOT: change any font size or weight.
|
||||
- [ ] 0.4 LEAD: `opt-level = "s"` decision. GOAL: measure on `target/agent-size` with the profile env vars: index build of the 6500-note vault (CPU seconds, `measure.py --large-ok`), `cargo test --release -p notescraft-ui-egui --test perf_code_blocks -- --ignored --nocapture`, a graph timelapse frame time; adopt only if none is worse than 10 %. Write the numbers in `measurements.md` section 1.3. DO NOT: adopt without numbers.
|
||||
- [ ] 0.5 Lazy system fallback fonts. OWNS: `crates/ui-egui/src/design/fonts/mod.rs`, `.../scripts.rs` (add Hebrew and Arabic to the on-demand table if missing), `crates/ui-egui/src/design/fonts/tests.rs`. FILES TO OPEN: same, `note_text` / `apply_pending`. GOAL: remove `SYSTEM_FALLBACKS` eager read at `install`; Hebrew and Arabic load through `note_text` like other scripts, so launch does not read `SFHebrew.ttf` and `SFArabic.ttf`. TESTS: a note with Hebrew shows no box after one frame loop (existing test style in `tests.rs`); a note without them never reads the files (counter in the injected `file_size`). VERIFY: standing. DO NOT: change the Latin chain.
|
||||
|
||||
## Phase 1 - pure registry core (engine, no IO unless stated)
|
||||
|
||||
- [ ] 1. Index model, validation, version compare (brief `briefs/30-registry-index-model.md`).
|
||||
- [ ] 2. Fileset guard: path and size rules (brief `briefs/31-registry-fileset-guard.md`).
|
||||
- [ ] 3. BLOCKED (owner Q1). Signature canonical text and verify.
|
||||
OWNS: `crates/engine/src/addons/verify.rs`, `crates/engine/src/addons/keys.rs`, `crates/engine/tests/addons_verify.rs`. FILES TO OPEN: `spec.md` D3, `crates/engine/src/addons/model.rs` (task 1), `crates/syntax/src/validate.rs` (`sha256_hex`). GOAL: `pub trait Verifier { fn verify(&self, entry: &Entry) -> Result<()>; }`, `pub fn canonical_text(entry: &Entry) -> String`, `pub struct Ed25519 { keys }` implementing it with the dependency the owner approved, `TRUSTED_KEYS` with the owner's public key (placeholder key in tests only), a `FakeVerifier` for other tasks. TESTS: known-answer signature generated by the xtask tool (task 18) verifies; one flipped hash byte fails; unknown `key_id` fails; wrong kind or version in the text fails; files in a different order produce the same text. VERIFY: standing. DO NOT: put a private key anywhere; accept an entry without `signature`.
|
||||
- [ ] 4. State and store layout.
|
||||
OWNS: `crates/engine/src/addons/state.rs`, `crates/engine/src/addons/store.rs`, `crates/engine/tests/addons_state.rs`. FILES TO OPEN: `spec.md` D4, `crates/engine/src/activity/outbox.rs` (versioned JSON pattern), `crates/engine/src/grammars/mod.rs` (`grammar_dir`, settings dir), `craft-libs/crates/atomic/src/lib.rs` (`atomic_write`, `write_new`). GOAL: `State { installed: BTreeMap<String, Installed> }` with `parse/to_json` (version 1, cap 512, unknown version `Err`), `Store::open(data_dir)`, `Store::version_dir(kind, id, version) -> PathBuf` built from validated components, `Store::gc(&State)` removing version dirs not in state and `staging/`, `commit(&State)` via `atomic_write`, `VaultFile { enabled, wanted }` with `parse/to_json` (cap 256 each). TESTS: round trip; corrupt JSON returns `Err` and leaves the file byte-identical; gc keeps `version` and `previous` and deletes the rest; commit is the only writer of `state.json`. VERIFY: standing. DO NOT: touch the vault; delete anything outside `<data dir>/registry/`.
|
||||
- [ ] 5. Transport and bounded download.
|
||||
OWNS: `crates/engine/src/addons/net.rs`, `crates/engine/tests/addons_net.rs`. FILES TO OPEN: `crates/syntax/src/fetch.rs` (`Transport`, `Curl`), `crates/syntax/src/validate.rs` (`check_url`), `crates/engine/src/sync/` (how `ureq` is used), `spec.md` D5, D7. GOAL: `pub trait Transport { fn get(&self, url: &str, max_bytes: u64, if_none_match: Option<&str>) -> Result<Response>; }`, `Response { status, etag, bytes }` (304 allowed), `Ureq` implementation (https only, no redirects, total timeout, reads at most `max_bytes + 1` and fails above), `FakeTransport` for tests, `download_file(&dyn Transport, &File, cap) -> Result<Vec<u8>>` that checks host allow-list, size equals entry size, sha256 equals entry. TESTS: host not allow-listed; `http://`; credentials; IP literal; redirect status; body one byte too long; one byte short; wrong sha256; 304 keeps cache; error maps to `offline`. VERIFY: standing; no real network in tests. DO NOT: spawn `curl`; follow redirects; add a header with an identifier beyond `NotesCraft/<version>`.
|
||||
- [ ] 6. Per-kind content checks.
|
||||
OWNS: `crates/engine/src/addons/kinds.rs`, `crates/engine/tests/addons_kinds.rs`. FILES TO OPEN: `spec.md` D8, `crates/syntax/src/validate.rs` (`check_grammar`), `crates/themes/src/load.rs` (`Theme::from_toml` diagnostics), `crates/plugins/src/manifest.rs`, `crates/plugins/src/lib.rs`. GOAL: `check(kind, &Entry, &[(path, bytes)]) -> Result<()>` dispatching to grammar, theme, plugin (manifest equals entry: id, version, permissions, command ids; wasm magic `\0asm`), font (magic bytes), snippet (UTF-8, no NUL), pack (each file by its subfolder). TESTS: one good and one bad fixture per kind; plugin manifest with extra permission fails; theme reusing a built-in id fails; `.tmLanguage` renamed to `.sublime-syntax` fails. VERIFY: standing. DO NOT: instantiate wasm here (task 12 does the dry load).
|
||||
- [ ] 7. BLOCKED on 3. Install, update, remove, rollback pipeline.
|
||||
OWNS: `crates/engine/src/addons/install.rs`, `crates/engine/tests/addons_install.rs`. FILES TO OPEN: `spec.md` D6, tasks 1 to 6 outputs, `crates/engine/src/grammars/mod.rs` (`save`). GOAL: `install(&Ctx, &Index, kind, id, version) -> Result<InstallReport>`, `update`, `remove`, `rollback` exactly as D6 steps 1 to 7; `Ctx { store, transport: &dyn Transport, verifier: &dyn Verifier, app_version, seed: Option<&SeedDir> }`; seed file used when its sha256 equals the entry. TESTS: the list in `spec.md` D13 IO bullet (hash mismatch, short and long body, crash before the state flip leaves the old state and gc removes the new dir, update keeps `previous`, rollback, remove, seed offline, revoked, app too old, bad signature, cap reached). VERIFY: standing. DO NOT: write anywhere but the store and `state.json`; delete the old version before the new state is committed.
|
||||
- [ ] 8. Commands, gate, command list.
|
||||
OWNS: `crates/engine/src/addons/commands.rs`, `crates/engine/src/addons/mod.rs`, one `mod addons;` in `crates/engine/src/lib.rs`, one `register_addons(reg)` call in `crates/engine/src/commands.rs::register_core`, `crates/engine/src/ai/assistant_gate.rs`, `crates/engine/tests/core_commands.rs` (`CORE_IDS`), `crates/engine/tests/addons_commands.rs`. FILES TO OPEN: `spec.md` D9, `crates/engine/src/bases/commands.rs` (group shape), `crates/engine/src/ai/assistant_gate.rs` (`READ_COMMANDS`, `WRITE_COMMANDS`, `DELETE_COMMANDS`), `docs/CODE-MAP.md`. GOAL: the nine commands of D9, error codes, session fields `addons_transport` and `addons_verifier` injectable like `grammar_transport`. TESTS: list joins state and cache; `registry.install` through the registry with a fake transport; read-only session refuses mutating ids; gate: `list|info|suggest` READ, `enable|disable` WRITE, the five others need `allow_delete`; offline `check` leaves the cache. VERIFY: standing + `--test core_commands --test addons_commands`. DO NOT: touch UI; change the `grammar.*` codes.
|
||||
- [ ] 9. CLI sugar. OWNS: `apps/notescraft-cli/src/args.rs`, `apps/notescraft-cli/src/commands/addons.rs`, `apps/notescraft-cli/tests/registry.rs`. FILES TO OPEN: `apps/notescraft-cli/src/commands.rs`, `apps/notescraft-cli/src/commands/notes.rs`, `apps/notescraft-cli/src/output.rs`. GOAL: `notescraft registry list|info|check|install|update|remove|rollback` mapped to the commands, `--json` like the rest, exit code 3 only for write commands while the app holds the lock (`list` and `info` use `open_read_only`). TESTS: parse table; `list --json` shape; `install` against a fake transport env var is not allowed: test the argument-to-params mapping only. VERIFY: standing, `-p notescraft-cli`. DO NOT: add network code to the CLI.
|
||||
|
||||
## Phase 2 - consumers
|
||||
|
||||
- [ ] 10. Grammar consumer and legacy adoption. OWNS: `crates/engine/src/grammars/mod.rs` (call the shared installer, keep codes), `crates/engine/src/addons/consumers.rs`, `crates/engine/tests/grammars.rs` (extend), `crates/engine/tests/addons_grammars.rs`. FILES TO OPEN: `crates/engine/src/grammars/mod.rs`, `crates/engine/src/vault/session.rs` (`load_grammars`), `crates/syntax/src/load.rs`. GOAL: `Session::load_grammars` feeds `set_installed` from `state.json` grammars plus legacy `AppSettings::grammars` once per generation; legacy entries show as `source: legacy`; `grammar.install` and `grammar.remove` call the registry code when the id is in the index. TESTS: legacy install still works; registry install appears in `grammar.info`; rebuild happens once for N installs in one frame; corrupt grammar file skipped. VERIFY: standing + `--test grammars --test grammars_hostile`. DO NOT: delete legacy files; change `MAX_GRAMMARS`.
|
||||
- [ ] 11. Theme consumer. OWNS: `crates/engine/src/prefs/themes.rs`, `crates/engine/tests/addons_themes.rs`. FILES TO OPEN: `crates/engine/src/prefs/themes.rs`, `crates/themes/src/load.rs`. GOAL: `Session::user_themes` also reads installed themes from the store (still capped by `MAX_USER_THEMES`, still refusing built-in ids); a selected id that is not installed resolves to the built-in of the same variant plus a diagnostic. TESTS: installed theme resolves; missing selected theme falls back with a diagnostic; duplicate id between settings folder and store: store loses. VERIFY: standing. DO NOT: change the built-ins.
|
||||
- [ ] 12. Plugin consumer: wire the wasm host into the app. OWNS: `crates/engine/src/plugin_host/plugins.rs` (scan installed plugins listed in the vault `enabled`), `crates/engine/src/vault/session.rs` (call `load_plugins` after open and after enable/disable; one line), `crates/engine/tests/addons_plugins.rs`. FILES TO OPEN: `crates/engine/src/plugin_host/plugins.rs`, `plugin_run.rs`, `crates/engine/tests/plugins.rs`, `crates/engine/src/builtin/plugin_commands.rs`, `plugins/dice/`. GOAL: `load_plugins` merges vault-local plugins (today's behaviour) with installed ones; commands `plugin.<id>.<command>` register; the approval flow is unchanged; a dry `Plugin::load` runs inside `registry.install` (task 6 left it). TESTS: install dice from a fake transport, enable, run `plugin.dice.roll` in a `TempVault`; disabled plugin not registered; permission not approved refuses; a plugin removed while enabled unloads. VERIFY: standing; then re-measure the binary and write the wasmi delta into `measurements.md` section 1.4. DO NOT: change ABI v1; give a plugin more permissions than approved.
|
||||
- [ ] 13. Pack consumer. OWNS: `crates/engine/src/addons/consumers.rs` (pack arm), `crates/engine/tests/addons_packs.rs`. GOAL: a `pack` entry's `grammars/`, `themes/` files flow to the grammar and theme consumers; `icons/` is read by the Lottie loader only when task 21 lands. TESTS: a pack with two grammars and one theme installs and removes atomically as one unit; one bad file fails the whole pack. DO NOT: install per-file.
|
||||
|
||||
## Phase 3 - UI
|
||||
|
||||
- [ ] 14. Registry browser view. OWNS: `crates/ui-egui/src/views/registry_view/mod.rs`, `.../list.rs`, `.../detail.rs`, `crates/ui-egui/src/text/views/registry.rs`, `[ui.registry]` at the END of `crates/i18n/locales/en.toml`, `crates/ui-egui/tests/registry_view.rs`. FILES TO OPEN: `spec.md` D10, `crates/ui-egui/src/views/settings_view/categories.rs`, `crates/ui-egui/src/views/share_dialog/mod.rs` (modal pattern), `.claude/skills/notescraft-egui-screen/`. GOAL: the view returns `Action`s only (`Check`, `Install(kind,id)`, `Update`, `Remove`, `Rollback`, `Enable(id)`), draws rows by state, the drawer with permission sentences, the offline banner; no engine calls inside the view. TESTS: kittest: each state renders its button; Install emits one action; search filters; offline banner; permissions listed as sentences. VERIFY: standing + `cargo xtask i18n`. DO NOT: use raw egui buttons or inputs; hardcode strings; fetch from the view.
|
||||
- [ ] 15. Shell wiring: category, commands, snapshot flag. OWNS: `crates/ui-egui/src/views/settings_view/categories.rs` (one Registry entry), `crates/ui-egui/src/shell/app/vault/registry.rs`, `crates/ui-egui/examples/snapshot.rs` (`--registry`). GOAL: actions call the engine commands on a worker thread (poll like `grammar_dialog.rs`), results update the view data; `registry.list` on open reads the cache only; Check/Install are the network triggers. TESTS: kittest: opening the category makes zero network calls (counter on the fake transport); Install runs one command. VERIFY: standing + snapshot PNG read. DO NOT: block the UI thread.
|
||||
- [ ] 16. Code card, first-run card, update badge. OWNS: `crates/ui-egui/src/views/note_view/blocks/code_card.rs` (the Add-grammar link only), `crates/ui-egui/src/shell/app/vault/first_run.rs`, `crates/ui-egui/src/text/views/registry.rs` (append). GOAL: D10 bullets 5 and 6; card dismissal remembered in `UiState`. TESTS: code card shows "Install Twig grammar" when `registry.suggest` hits, the old dialog link otherwise; first-run card never fetches; badge only after a check. VERIFY: standing + `cargo xtask i18n`. DO NOT: install without a click, unless `registry.auto_install_for_vault` is on and the file comes from the seed.
|
||||
- [ ] 17. Vault `wanted` scan and banner. OWNS: `crates/engine/src/addons/wanted.rs`, `crates/engine/tests/addons_wanted.rs`, `crates/ui-egui/src/shell/app/vault/wanted_banner.rs`. FILES TO OPEN: `crates/engine/src/vault/session.rs`, the content-search code that uses `ignore`/`grep-*`, `spec.md` D11 step 4. GOAL: `scan_wanted(session, budget_ms) -> Vec<String>` over fence labels and `$`/`$$` in the vault with a 3 s budget, run once per vault and registry generation, written to `.notescraft/registry.json` `wanted`; banner lists not-installed entries with size. TESTS: a vault with a ```` ```twig ```` fence yields `grammar:twig`; budget stops a 6500-note scan early without error; installed items are not listed; no network. VERIFY: standing. DO NOT: read notes over 2 MiB; write outside `.notescraft/registry.json`.
|
||||
|
||||
## Phase 4 - first-party content and release
|
||||
|
||||
- [ ] 18. Registry repo layout and signing tool. OWNS: `xtask/src/registry_sign.rs`, one subcommand line in `xtask/src/main.rs`, `xtask/tests/registry_sign.rs`; outside this repo (separate commit by the lead): `notescraft-registry/README.md` (schema v2, `id` not `name`), `index.json`, `files/`. FILES TO OPEN: `spec.md` D1 to D3, `/Volumes/Storage/Projects/notescraft-registry/README.md`, `xtask/src/main.rs`. GOAL: `cargo xtask registry-sign --dir <registry checkout> --key <file>` rebuilds `index.json` from `files/<kind>/<id>/<version>/` plus a small `entry.toml` per item (title, license, aliases), fills size and sha256, signs; `--check` verifies an existing index. TESTS: sign then verify with the engine verifier; tampered file fails `--check`. VERIFY: standing, `-p xtask`. DO NOT: read the key from the repo; print it.
|
||||
- [ ] 19. Move the 25 grammars out. OWNS: `crates/syntax/src/embedded.rs` (shrinks to an empty list or removed), `crates/syntax/grammars/` (moved to the registry repo and to the seed), `crates/syntax/src/load.rs` (drop the embedded loop), `crates/engine/tests/addons_migration_grammars.rs`. GOAL: only after 7, 10, 16, 17, 18, 23 are done; registry entries `twig`, `blade`, ... one per file; tests per D11 step 5; the TOML and TypeScript grammars stay if the owner wants highlighting out of the box (they are the two most common: decide with the owner, default keep). TESTS: for every moved id: suggest finds it, seed verifies, offline install from the seed makes `is_known` true; removal restores plain text. VERIFY: standing + measure `load::current()` (expect about 75 ms instead of 110 ms) and binary delta (about -69 KB). DO NOT: remove a grammar that has no seed copy.
|
||||
- [ ] 20. STIX Two Math as a font entry. OWNS: `crates/ui-egui/src/design/fonts/mod.rs` (STIX loaded from the store when present), `crates/ui-egui/src/views/math_view.rs` (Install button when absent), `assets/fonts/STIXTwoMath-Regular.ttf` (moves to the seed), tests. GOAL: after 7, 16, 18, 23; math without the font uses the existing Inter/JetBrains chain and shows "Install math font (376 KB)". TESTS: with and without the font; rendering never panics; the migration test of D11. VERIFY: binary delta about -0.36 MiB; snapshot of a note with display math both ways.
|
||||
- [ ] 21. Lottie icons as a pack. OWNS: `crates/ui-egui/src/design/lottie_icon/files.rs` (store lookup, static fallback), `assets/icons-anim/` (moves to the seed), tests. GOAL: after 13, 18, 23; `icons-animated` pack; without it every animated icon draws its static Lucide icon (the skill `notescraft-icon-motion` rules still hold). TESTS: each icon id has a static fallback; motion tests run with the pack installed from a temp store. VERIFY: binary delta about -0.14 MiB. DO NOT: change any animation JSON.
|
||||
- [ ] 22. First plugins and themes in the registry. OWNS (registry repo, plus this repo's tests only): `files/plugin/dice/1.0.0/`, `files/theme/{catppuccin-frappe,catppuccin-macchiato,solarized-light,solarized-dark}/1.0.0/` (themes copied from `crates/themes/src/builtin.rs` and kept built-in until a migration test passes), `plugins/dice/README.md` (point to the registry), `crates/engine/tests/addons_first_party.rs`. GOAL: dice builds with `plugins/dice/build.sh`, its `manifest.json` equals the entry; citations follows when roadmap 10 starts. TESTS: dice installs offline from the seed, rolls, `2d6+3` result in range. DO NOT: remove built-in themes in this task.
|
||||
- [ ] 23. Seed packaging. OWNS: `packaging/macos/*`, `packaging/windows/*`, `packaging/linux/package.sh` (copy `registry-seed/` next to the binary), `crates/engine/src/addons/seed.rs`, `crates/engine/tests/addons_seed.rs`. FILES TO OPEN: `packaging/README.md`, `packaging/macos/`. GOAL: `SeedDir::locate()` finds `Contents/Resources/registry-seed` (macOS), `registry-seed/` beside the exe (Windows, Linux), verified signed snapshot of moved items only. TESTS: locate with a fake exe path; missing seed is fine (network install still works); tampered seed file rejected. DO NOT: put anything unsigned in the seed.
|
||||
- [ ] 24. Docs, numbers, status. OWNS: `ROADMAP.md`, `docs/ARCHITECTURE.md`, `docs/CODE-MAP.md` (addons rows), `SPEC.md` sections 6.1 and 7 (a "Registry" subsection, community store no longer deferred), `specs/registry/measurements.md` (new numbers), `specs/plugins-parity/roadmap.md` (add the rows). VERIFY: `cargo xtask ci`. DO NOT: add features.
|
||||
|
||||
## Order and parallelism
|
||||
|
||||
Phase 0 in parallel with 1, 2, 4, 5, 6 (disjoint folders). 3 starts when the owner answers question 1. 7 after 3 to 6. 8 and 9 after 7. 10 to 13 are disjoint after 8. 14 to 17 after 8 (UI can start on a fake command layer). 18 can start at any time and unblocks 3's known-answer test. 19 to 22 only after their dependencies, one at a time, each with its migration test. The first three agent briefs are `specs/plugins-parity/briefs/29..31`.
|
||||
Reference in New Issue
Block a user