mirror of
https://github.com/bherbruck/solidcraft.git
synced 2026-11-07 20:41:50 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 bherbruck
|
306079e30f | ||
|
bherbruck
|
29ac2ec952 |
@@ -725,10 +725,16 @@ jobs:
|
||||
run: |
|
||||
tag="v$VERSION"
|
||||
sha256sum --check SHA256SUMS.txt
|
||||
# Generated notes start at the newest published release, pre-releases included. Left
|
||||
# to GitHub they start at the Latest stable one, and the list outgrows the 125,000
|
||||
# characters a release body may have.
|
||||
start=()
|
||||
prev="$(gh release list --exclude-drafts --limit 30 --json tagName,publishedAt --jq 'sort_by(.publishedAt) | last | .tagName // empty')"
|
||||
if [ -n "$prev" ]; then start=(--notes-start-tag "$prev"); fi
|
||||
# Stage all packages before publishing. Published tags are never replaced.
|
||||
gh release create "$tag" --draft --prerelease --latest=false \
|
||||
--title "SolidCraft v$VERSION (nightly)" --target "$GITHUB_SHA" \
|
||||
--notes-file ../notes.md --generate-notes -- *
|
||||
--notes-file ../notes.md --generate-notes "${start[@]}" -- *
|
||||
gh release edit "$tag" --draft=false --prerelease --latest=false
|
||||
echo "### Published nightly: $tag (\`${GITHUB_SHA::9}\`)" >>"$GITHUB_STEP_SUMMARY"
|
||||
- name: Create or update the draft release
|
||||
@@ -760,8 +766,13 @@ jobs:
|
||||
--target "$GITHUB_SHA" --prerelease="$PRERELEASE"
|
||||
gh release upload "$tag" --clobber -- *
|
||||
else
|
||||
# As for nightlies: start the generated notes at the newest published release.
|
||||
start=()
|
||||
prev="$(gh release list --exclude-drafts --limit 30 --json tagName,publishedAt --jq 'sort_by(.publishedAt) | last | .tagName // empty')"
|
||||
if [ -n "$prev" ]; then start=(--notes-start-tag "$prev"); fi
|
||||
gh release create "$tag" --draft --title "SolidCraft v$VERSION" \
|
||||
--target "$GITHUB_SHA" --notes-file ../notes.md --generate-notes "${pre[@]}" -- *
|
||||
--target "$GITHUB_SHA" --notes-file ../notes.md --generate-notes "${start[@]}" \
|
||||
"${pre[@]}" -- *
|
||||
fi
|
||||
{
|
||||
echo "Draft **SolidCraft v$VERSION** at \`${GITHUB_SHA::9}\`:"
|
||||
|
||||
Generated
+12
-178
@@ -125,21 +125,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cipher 0.4.4",
|
||||
"cipher",
|
||||
"cpufeatures 0.2.17",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aes"
|
||||
version = "0.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc0fc36d7440d015b45468ba08a21e80e056ebbd014c611c6298bb7503e04bd1"
|
||||
dependencies = [
|
||||
"cipher 0.5.2",
|
||||
"cpubits",
|
||||
"cpufeatures 0.3.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.12"
|
||||
@@ -154,21 +143,6 @@ dependencies = [
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "alloc-no-stdlib"
|
||||
version = "2.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
|
||||
|
||||
[[package]]
|
||||
name = "alloc-stdlib"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195"
|
||||
dependencies = [
|
||||
"alloc-no-stdlib",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "allocator-api2"
|
||||
version = "0.2.21"
|
||||
@@ -511,15 +485,6 @@ dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-padding"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b"
|
||||
dependencies = [
|
||||
"hybrid-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block2"
|
||||
version = "0.5.1"
|
||||
@@ -551,16 +516,6 @@ dependencies = [
|
||||
"piper",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brotli-decompressor"
|
||||
version = "5.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583"
|
||||
dependencies = [
|
||||
"alloc-no-stdlib",
|
||||
"alloc-stdlib",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.20.3"
|
||||
@@ -748,16 +703,7 @@ version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6"
|
||||
dependencies = [
|
||||
"cipher 0.4.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cbc"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896"
|
||||
dependencies = [
|
||||
"cipher 0.5.2",
|
||||
"cipher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -822,17 +768,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common 0.1.7",
|
||||
"inout 0.1.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cipher"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c"
|
||||
dependencies = [
|
||||
"crypto-common 0.2.2",
|
||||
"inout 0.2.6",
|
||||
"inout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -951,12 +887,6 @@ version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48"
|
||||
|
||||
[[package]]
|
||||
name = "cpubits"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae"
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
@@ -1066,9 +996,9 @@ version = "4.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "708b509edf7889e53d7efb0ffadd994cc6c2345ccb62f55cfd6b0682165e4fa6"
|
||||
dependencies = [
|
||||
"aes 0.8.4",
|
||||
"block-padding 0.3.3",
|
||||
"cbc 0.1.2",
|
||||
"aes",
|
||||
"block-padding",
|
||||
"cbc",
|
||||
"dbus",
|
||||
"fastrand",
|
||||
"hkdf",
|
||||
@@ -1165,15 +1095,6 @@ version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "ecb"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26f2a8b3e564eba0877223dc343703ad0385794e882e6d13f3a4dd5c6b1f41ac"
|
||||
dependencies = [
|
||||
"cipher 0.5.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ecolor"
|
||||
version = "0.37.0"
|
||||
@@ -1704,7 +1625,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159"
|
||||
dependencies = [
|
||||
"color_quant",
|
||||
"weezl 0.1.12",
|
||||
"weezl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2147,20 +2068,10 @@ version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||
dependencies = [
|
||||
"block-padding 0.3.3",
|
||||
"block-padding",
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "inout"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a580c13f0ffc335ef42dd3f6b1a41b726051c5e1656d9a3aa6fddb3c5dfbbf9"
|
||||
dependencies = [
|
||||
"block-padding 0.4.2",
|
||||
"hybrid-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.15.0"
|
||||
@@ -2405,43 +2316,6 @@ version = "0.4.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
||||
|
||||
[[package]]
|
||||
name = "lopdf"
|
||||
version = "0.45.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bfffda0fe1ab0157e1a13c14bebd3f28671f2fccb7922f0722ec53926e6922d3"
|
||||
dependencies = [
|
||||
"aes 0.9.4",
|
||||
"bitflags 2.13.2",
|
||||
"brotli-decompressor",
|
||||
"cbc 0.2.1",
|
||||
"ecb",
|
||||
"encoding_rs",
|
||||
"flate2",
|
||||
"getrandom 0.4.3",
|
||||
"indexmap",
|
||||
"itoa",
|
||||
"log",
|
||||
"md-5",
|
||||
"nom",
|
||||
"rand 0.10.3",
|
||||
"rangemap",
|
||||
"sha2 0.11.1",
|
||||
"stringprep",
|
||||
"thiserror 2.0.21",
|
||||
"weezl 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "md-5"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"digest 0.11.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
@@ -3437,7 +3311,6 @@ version = "0.10.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af"
|
||||
dependencies = [
|
||||
"chacha20",
|
||||
"getrandom 0.4.3",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
@@ -3482,12 +3355,6 @@ version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca45419789ae5a7899559e9512e58ca889e41f04f1f2445e9f4b290ceccd1d08"
|
||||
|
||||
[[package]]
|
||||
name = "rangemap"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a611d15b50743feb4c76b7d03edcb0e64f399c26961e4efe6975bc398be6aa3d"
|
||||
|
||||
[[package]]
|
||||
name = "raw-window-handle"
|
||||
version = "0.6.2"
|
||||
@@ -3863,8 +3730,8 @@ version = "4.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e4d35ad99a181be0a60ffcbe85d680d98f87bdc4d7644ade319b87076b9dbfd4"
|
||||
dependencies = [
|
||||
"aes 0.8.4",
|
||||
"cbc 0.1.2",
|
||||
"aes",
|
||||
"cbc",
|
||||
"futures-util",
|
||||
"generic-array",
|
||||
"hkdf",
|
||||
@@ -4427,7 +4294,7 @@ dependencies = [
|
||||
"solidcore-record",
|
||||
"solidcraft-geom",
|
||||
"web-time",
|
||||
"weezl 0.1.12",
|
||||
"weezl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4486,7 +4353,6 @@ dependencies = [
|
||||
"egui-wgpu",
|
||||
"image",
|
||||
"log",
|
||||
"lopdf",
|
||||
"minisign-verify",
|
||||
"nix 0.31.3",
|
||||
"pollster 0.4.0",
|
||||
@@ -4567,17 +4433,6 @@ version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6637bab7722d379c8b41ba849228d680cc12d0a45ba1fa2b48f2a30577a06731"
|
||||
|
||||
[[package]]
|
||||
name = "stringprep"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1"
|
||||
dependencies = [
|
||||
"unicode-bidi",
|
||||
"unicode-normalization",
|
||||
"unicode-properties",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "subtle"
|
||||
version = "2.6.1"
|
||||
@@ -4701,7 +4556,7 @@ dependencies = [
|
||||
"flate2",
|
||||
"half",
|
||||
"quick-error",
|
||||
"weezl 0.1.12",
|
||||
"weezl",
|
||||
"zune-jpeg",
|
||||
]
|
||||
|
||||
@@ -4740,12 +4595,6 @@ dependencies = [
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.53.2"
|
||||
@@ -4923,15 +4772,6 @@ version = "1.0.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization"
|
||||
version = "0.1.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d"
|
||||
dependencies = [
|
||||
"tinyvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-properties"
|
||||
version = "0.1.5"
|
||||
@@ -5313,12 +5153,6 @@ version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
|
||||
|
||||
[[package]]
|
||||
name = "weezl"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4ca08e5ef825b65b056d9efbd95c8750683f0a6d0466d02e96dc2e4e360f3d2"
|
||||
|
||||
[[package]]
|
||||
name = "wgpu"
|
||||
version = "30.0.1"
|
||||
|
||||
@@ -59,7 +59,6 @@ proc-macro2 = { workspace = true }
|
||||
# egui-winit's clipboard.
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
|
||||
arboard = { version = "3.6", default-features = false, features = ["image-data"] }
|
||||
lopdf = { version = "0.45", default-features = false }
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
|
||||
# The offscreen viewport render test opens a device without a window.
|
||||
|
||||
@@ -30,11 +30,6 @@ const MAX_PICTURE_BYTES: u64 = 40 * 1024 * 1024;
|
||||
const MAX_TEXT_BYTES: u64 = 256 * 1024;
|
||||
/// Largest PDF sent as a document; a bigger one goes by reference.
|
||||
const MAX_PDF_BYTES: u64 = 12 * 1024 * 1024;
|
||||
/// Pages whose text is taken from a PDF, and the most characters of it kept.
|
||||
const MAX_PDF_PAGES: usize = 300;
|
||||
const MAX_PDF_CHARS: usize = 200_000;
|
||||
/// A PDF page's content may inflate to at most this (a compressed bomb is refused by the reader).
|
||||
const MAX_PDF_PAGE_BYTES: usize = 32 * 1024 * 1024;
|
||||
/// A picture's longer side is scaled down to this before it is sent.
|
||||
pub const MAX_SIDE: u32 = 2048;
|
||||
/// Largest picture sent, as it came or re-encoded: its base64 (a third larger) stays under 5 MB,
|
||||
@@ -93,11 +88,10 @@ pub enum Content {
|
||||
size: [u32; 2],
|
||||
},
|
||||
Text(Arc<String>),
|
||||
/// A PDF, sent whole as a document to an agent that takes them; to the others, its text.
|
||||
/// A PDF, sent whole as a document to an agent that takes them.
|
||||
Pdf {
|
||||
bytes: Arc<Vec<u8>>,
|
||||
path: PathBuf,
|
||||
text: Option<Arc<PdfText>>,
|
||||
},
|
||||
/// A file the agent is pointed to, not given: what it is, its size and where it is.
|
||||
Reference {
|
||||
@@ -109,60 +103,6 @@ pub enum Content {
|
||||
Imported,
|
||||
}
|
||||
|
||||
/// What was taken out of a PDF: its page count and its text, without layout, pictures or fonts.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct PdfText {
|
||||
pub pages: usize,
|
||||
pub text: String,
|
||||
/// The text stops short: of the pages or of the characters kept.
|
||||
pub cut: bool,
|
||||
}
|
||||
|
||||
/// The text of a PDF, from its pages in order (up to [`MAX_PDF_PAGES`] pages and
|
||||
/// [`MAX_PDF_CHARS`] characters). `None` when it can't be read at all (damaged, or a reader
|
||||
/// panic, which is caught); an encrypted or scanned one gives its page count and no text.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub fn pdf_text(bytes: &[u8]) -> Option<PdfText> {
|
||||
std::panic::catch_unwind(|| {
|
||||
let doc = lopdf::Document::load_mem(bytes).ok()?;
|
||||
let pages: Vec<u32> = doc.get_pages().keys().copied().collect();
|
||||
let mut text = String::new();
|
||||
let mut chars = 0usize;
|
||||
let mut cut = pages.len() > MAX_PDF_PAGES;
|
||||
if !doc.is_encrypted() {
|
||||
for page in pages.iter().take(MAX_PDF_PAGES) {
|
||||
// A page that can't be read is skipped; the others still give their text.
|
||||
let chunks = doc.extract_text_chunks_with_limit(&[*page], MAX_PDF_PAGE_BYTES);
|
||||
let page_text: String = chunks.into_iter().filter_map(Result::ok).collect();
|
||||
let page_text = page_text.trim();
|
||||
if page_text.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if !text.is_empty() {
|
||||
text.push_str("\n\n");
|
||||
}
|
||||
let room = MAX_PDF_CHARS.saturating_sub(chars);
|
||||
let n = page_text.chars().count();
|
||||
if n > room {
|
||||
text.extend(page_text.chars().take(room));
|
||||
cut = true;
|
||||
break;
|
||||
}
|
||||
text.push_str(page_text);
|
||||
chars += n;
|
||||
}
|
||||
}
|
||||
Some(PdfText { pages: pages.len(), text, cut })
|
||||
})
|
||||
.ok()
|
||||
.flatten()
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub fn pdf_text(_bytes: &[u8]) -> Option<PdfText> {
|
||||
None
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Attachment {
|
||||
/// Its own on the message, never reused (set when it is added there; 0 until then): the
|
||||
@@ -282,9 +222,7 @@ pub fn read(path: &Path) -> Result<Attachment, String> {
|
||||
return Ok(Attachment { id: 0, name, content: Content::Reference { path, size: meta.len(), mime: "application/pdf" }, thumb: None });
|
||||
}
|
||||
let bytes = std::fs::read(&path).map_err(|e| failed(&name, e))?;
|
||||
// Read here, off the window thread: for an agent that takes no documents.
|
||||
let text = pdf_text(&bytes).map(Arc::new);
|
||||
Ok(Attachment { id: 0, name, content: Content::Pdf { bytes: Arc::new(bytes), path, text }, thumb: None })
|
||||
Ok(Attachment { id: 0, name, content: Content::Pdf { bytes: Arc::new(bytes), path }, thumb: None })
|
||||
}
|
||||
Kind::Other if matches!(path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).as_deref(), Some("heic" | "heif")) => {
|
||||
Err(refused("ui.agent_chat_file_heic", &name, &[]))
|
||||
@@ -396,24 +334,6 @@ fn thumbnail(img: &image::DynamicImage) -> Option<Arc<egui::ColorImage>> {
|
||||
Some(Arc::new(egui::ColorImage::from_rgba_unmultiplied(size, t.as_raw())))
|
||||
}
|
||||
|
||||
/// Add a file's text to the message under `head`, within the `left` characters shared by the
|
||||
/// files: whole when it fits; cut, saying how much is missing, when it doesn't; named only when
|
||||
/// nothing is left.
|
||||
fn fit(out: &mut String, left: &mut usize, head: &str, text: &str) {
|
||||
let t = text.trim_end();
|
||||
let len = t.chars().count();
|
||||
if *left == 0 {
|
||||
*out += &format!("\n\n{head}: left out, the message is at its size limit.");
|
||||
} else if len <= *left {
|
||||
*out += &format!("\n\n{head}:\n```\n{t}\n```");
|
||||
*left -= len;
|
||||
} else {
|
||||
let kept: String = t.chars().take(*left).collect();
|
||||
*out += &format!("\n\n{head} (cut: {} more characters left out):\n```\n{kept}\n```", len - *left);
|
||||
*left = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// What the message says about its files, after the person's words: the text files' text, and the
|
||||
/// pictures and imported parts by name (the agent sees the pictures themselves as images).
|
||||
/// `pictures_sent` and `documents_sent`: whether this agent takes pictures and PDFs. The text files share `budget` characters
|
||||
@@ -433,8 +353,21 @@ pub fn note(files: &[Attachment], pictures_sent: bool, documents_sent: bool, bud
|
||||
let mut left = budget;
|
||||
for f in files {
|
||||
match &f.content {
|
||||
Content::Text(t) => fit(&mut out, &mut left, &format!("Attached file `{}`", f.name), t),
|
||||
Content::Pdf { path, bytes, .. } if documents_sent => {
|
||||
Content::Text(t) => {
|
||||
let t = t.trim_end();
|
||||
let len = t.chars().count();
|
||||
if left == 0 {
|
||||
out += &format!("\n\nAttached file `{}`: left out, the message is at its size limit.", f.name);
|
||||
} else if len <= left {
|
||||
out += &format!("\n\nAttached file `{}`:\n```\n{t}\n```", f.name);
|
||||
left -= len;
|
||||
} else {
|
||||
let kept: String = t.chars().take(left).collect();
|
||||
out += &format!("\n\nAttached file `{}` (cut: {} more characters left out):\n```\n{kept}\n```", f.name, len - left);
|
||||
left = 0;
|
||||
}
|
||||
}
|
||||
Content::Pdf { path, bytes } if documents_sent => {
|
||||
out += &format!(
|
||||
"\n\nAttached PDF `{}` ({}, at {}): sent with this message as a document.",
|
||||
f.name,
|
||||
@@ -442,29 +375,14 @@ pub fn note(files: &[Attachment], pictures_sent: bool, documents_sent: bool, bud
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
// An agent that takes no documents gets the text, or is pointed to the file.
|
||||
Content::Pdf { path, bytes, text } => {
|
||||
let size = size_text(bytes.len() as u64);
|
||||
match text.as_deref().filter(|t| !t.text.trim().is_empty()) {
|
||||
Some(t) => {
|
||||
let cut = if t.cut { "; cut, the rest is in the file" } else { "" };
|
||||
let head = format!(
|
||||
"Attached PDF `{}` ({} pages, {size}, at {}): its text only, layout, pictures and fonts are dropped{cut}",
|
||||
f.name,
|
||||
t.pages,
|
||||
path.display()
|
||||
);
|
||||
fit(&mut out, &mut left, &head, &t.text);
|
||||
}
|
||||
None => {
|
||||
let why = if text.is_some() { "no text could be taken from it (scanned or protected)" } else { "its text could not be read" };
|
||||
out += &format!(
|
||||
"\n\nAttached file `{}` (PDF, {size}, at {}): {why}; read it with your own tools, it is not in this message.",
|
||||
f.name,
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
// An agent that takes no documents is pointed to the file, as for any other file.
|
||||
Content::Pdf { path, bytes } => {
|
||||
out += &format!(
|
||||
"\n\nAttached file `{}` (PDF, {}, at {}): read it with your own tools; it is not in this message.",
|
||||
f.name,
|
||||
size_text(bytes.len() as u64),
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
Content::Reference { path, size, mime } => {
|
||||
out += &format!(
|
||||
@@ -487,7 +405,7 @@ pub fn parts(files: &[Attachment], pictures: bool, documents: bool) -> Vec<Promp
|
||||
.iter()
|
||||
.filter_map(|f| match &f.content {
|
||||
Content::Picture { mime, bytes, .. } if pictures => Some(PromptPart::Image { mime: (*mime).to_string(), base64: backend::base64(bytes) }),
|
||||
Content::Pdf { bytes, path, .. } if documents => Some(PromptPart::Document {
|
||||
Content::Pdf { bytes, path } if documents => Some(PromptPart::Document {
|
||||
name: f.name.clone(),
|
||||
uri: file_uri(path),
|
||||
mime: "application/pdf".to_string(),
|
||||
@@ -758,107 +676,4 @@ mod tests {
|
||||
assert_eq!(size_text(1536), "2 KB");
|
||||
assert_eq!(size_text(5 * 1024 * 1024 + 512 * 1024), "5.5 MB");
|
||||
}
|
||||
|
||||
/// A PDF with one line of text on each page.
|
||||
fn sample_pdf(pages: &[&str]) -> Vec<u8> {
|
||||
use lopdf::content::{Content, Operation};
|
||||
use lopdf::{Document, Object, Stream, dictionary};
|
||||
let mut doc = Document::with_version("1.5");
|
||||
let pages_id = doc.new_object_id();
|
||||
let font = doc.add_object(dictionary! {"Type" => "Font", "Subtype" => "Type1", "BaseFont" => "Helvetica"});
|
||||
let resources = doc.add_object(dictionary! {"Font" => dictionary! {"F1" => font}});
|
||||
let mut kids = Vec::new();
|
||||
for line in pages {
|
||||
let mut ops = vec![
|
||||
Operation::new("BT", vec![]),
|
||||
Operation::new("Tf", vec!["F1".into(), 12.into()]),
|
||||
Operation::new("Td", vec![72.into(), 700.into()]),
|
||||
];
|
||||
if !line.is_empty() {
|
||||
ops.push(Operation::new("Tj", vec![Object::string_literal(*line)]));
|
||||
}
|
||||
ops.push(Operation::new("ET", vec![]));
|
||||
let content = doc.add_object(Stream::new(dictionary! {}, Content { operations: ops }.encode().unwrap()));
|
||||
kids.push(Object::Reference(doc.add_object(dictionary! {"Type" => "Page", "Parent" => pages_id, "Contents" => content})));
|
||||
}
|
||||
let tree = dictionary! {
|
||||
"Type" => "Pages", "Kids" => kids, "Count" => pages.len() as i64, "Resources" => resources,
|
||||
"MediaBox" => vec![0.into(), 0.into(), 595.into(), 842.into()],
|
||||
};
|
||||
doc.objects.insert(pages_id, Object::Dictionary(tree));
|
||||
let catalog = doc.add_object(dictionary! {"Type" => "Catalog", "Pages" => pages_id});
|
||||
doc.trailer.set("Root", catalog);
|
||||
let mut out = Vec::new();
|
||||
doc.save_to(&mut out).unwrap();
|
||||
out
|
||||
}
|
||||
|
||||
/// An agent that takes no documents gets the PDF's text, with its page count and a note that
|
||||
/// the layout was dropped; one that takes documents gets the PDF and not the text.
|
||||
#[test]
|
||||
fn a_pdf_gives_its_text_to_agents_without_documents() {
|
||||
let d = dir("pdftext");
|
||||
let path = d.join("brief.pdf");
|
||||
std::fs::write(&path, sample_pdf(&["Wall 2 mm", "Hole 5 mm"])).unwrap();
|
||||
let a = read(&path).unwrap();
|
||||
let Content::Pdf { text: Some(t), .. } = &a.content else { panic!("a PDF with its text: {:?}", a.content) };
|
||||
assert_eq!(t.pages, 2);
|
||||
assert!(!t.cut && t.text.contains("Wall 2 mm") && t.text.contains("Hole 5 mm"), "{:?}", t.text);
|
||||
assert!(t.text.find("Wall").unwrap() < t.text.find("Hole").unwrap(), "pages in order");
|
||||
let files = vec![a];
|
||||
let n = note(&files, false, false, usize::MAX);
|
||||
assert!(n.contains("Attached PDF `brief.pdf` (2 pages"), "{n}");
|
||||
assert!(n.contains("layout, pictures and fonts are dropped") && n.contains("Wall 2 mm"), "{n}");
|
||||
let short = note(&files, false, false, 5);
|
||||
assert!(short.contains("more characters left out"), "the text shares the message's budget: {short}");
|
||||
let with = note(&files, false, true, usize::MAX);
|
||||
assert!(with.contains("as a document") && !with.contains("Wall 2 mm"), "{with}");
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// A PDF that is all pictures (a scan), or can't be read, says so and points to the file; the
|
||||
/// text kept is capped.
|
||||
#[test]
|
||||
fn pdfs_without_text_or_too_long_are_said_so() {
|
||||
let blank = pdf_text(&sample_pdf(&[""])).unwrap();
|
||||
assert_eq!((blank.pages, blank.text.as_str()), (1, ""));
|
||||
let a = Attachment {
|
||||
id: 0,
|
||||
name: "scan.pdf".into(),
|
||||
content: Content::Pdf { bytes: Arc::new(vec![1; 10]), path: "/a/scan.pdf".into(), text: Some(Arc::new(blank)) },
|
||||
thumb: None,
|
||||
};
|
||||
assert!(note(&[a], false, false, usize::MAX).contains("no text could be taken from it"));
|
||||
let broken = Attachment {
|
||||
id: 0,
|
||||
name: "x.pdf".into(),
|
||||
content: Content::Pdf { bytes: Arc::new(vec![1; 10]), path: "/a/x.pdf".into(), text: None },
|
||||
thumb: None,
|
||||
};
|
||||
assert!(note(&[broken], false, false, usize::MAX).contains("its text could not be read"));
|
||||
let long = "a".repeat(MAX_PDF_CHARS + 5000);
|
||||
let t = pdf_text(&sample_pdf(&[&long])).unwrap();
|
||||
assert!(t.cut && t.text.chars().count() == MAX_PDF_CHARS, "{} characters", t.text.chars().count());
|
||||
}
|
||||
|
||||
/// Hostile PDFs never panic: truncated and damaged copies of a real one are read or refused.
|
||||
#[test]
|
||||
fn damaged_pdfs_never_panic() {
|
||||
let good = sample_pdf(&["Wall 2 mm", "Hole 5 mm"]);
|
||||
assert!(pdf_text(&good).is_some());
|
||||
assert!(pdf_text(b"").is_none() && pdf_text(b"%PDF-1.7\n").is_none() && pdf_text(b"not a pdf").is_none());
|
||||
for cut in (0..good.len()).step_by(37) {
|
||||
let _ = pdf_text(&good[..cut]);
|
||||
}
|
||||
let mut seed = 0x9e37_79b9_u32;
|
||||
for _ in 0..200 {
|
||||
let mut bad = good.clone();
|
||||
for _ in 0..4 {
|
||||
seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
|
||||
let at = usize::try_from(seed >> 8).unwrap() % bad.len();
|
||||
bad[at] = (seed & 0xff) as u8;
|
||||
}
|
||||
let _ = pdf_text(&bad);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user