1use std::path::{Path, PathBuf};
16
17pub const VERSIONS: usize = 20;
19pub const DAYS: i64 = 30;
21pub const SEALED_DAYS: i64 = 12;
33const _: () = assert!(SEALED_DAYS < crate::blobs::UNUSED_DAYS - 1);
34const SEALED: &str = ".sealed";
36const DAY: i64 = 86_400_000;
37
38#[derive(Debug, Clone, PartialEq, Eq)]
41pub struct Version {
42 pub stamp: String,
43 pub at: i64,
44 pub size: u64,
45 pub sealed: Option<String>,
46}
47
48#[derive(Debug, PartialEq, Eq)]
50pub enum Kept {
51 Copy(PathBuf),
52 Sealed { hash: String, size: u64 },
53}
54
55pub fn root(memory: &Path) -> PathBuf {
57 memory.parent().map_or_else(|| PathBuf::from("history"), |share| share.join("history"))
58}
59
60fn stamp(ms: i64) -> String {
62 let Ok(at) = jiff::Timestamp::from_millisecond(ms) else { return ms.to_string() };
63 format!("{}.{:03}Z", at.strftime("%Y-%m-%dT%H-%M-%S"), ms.rem_euclid(1000))
64}
65
66pub fn kept_at(stamp: &str) -> Option<i64> {
68 let (seconds, ms) = stamp.strip_suffix('Z')?.split_once('.')?;
69 let at = jiff::civil::DateTime::strptime("%Y-%m-%dT%H-%M-%S", seconds).ok()?.to_zoned(jiff::tz::TimeZone::UTC).ok()?;
70 Some(at.timestamp().as_millisecond() + ms.parse::<i64>().ok()?)
71}
72
73fn part_dir(root: &Path, part: &str) -> Option<PathBuf> {
75 plain(part).then(|| root.join(part))
76}
77
78fn plain(piece: &str) -> bool {
79 !piece.is_empty() && piece != "." && piece != ".." && !piece.contains(['/', '\\', ':'])
80}
81
82fn below(root: &Path, part: &str, file: &str) -> Option<PathBuf> {
85 let dir = part_dir(root, part)?;
86 file.split('/').all(plain).then(|| file.split('/').fold(dir, |path, piece| path.join(piece)))
87}
88
89fn new_stamp(dir: &Path, now_ms: i64) -> String {
91 let mut ms = now_ms;
92 while dir.join(stamp(ms)).exists() || dir.join(format!("{}{SEALED}", stamp(ms))).exists() {
93 ms += 1;
94 }
95 stamp(ms)
96}
97
98pub fn keep(root: &Path, part: &str, file: &str, path: &Path, now_ms: i64) -> Result<(), String> {
103 let dir = below(root, part, file).ok_or_else(|| format!("{file}: not a file's name"))?;
104 let size = std::fs::metadata(path).map_or(0, |m| m.len());
105 if !crate::disk::fits(root, size) {
106 return Err(format!("share-no-room:{file}"));
107 }
108 crate::share::private_dirs(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;
109 let target = dir.join(new_stamp(&dir, now_ms));
110 let temporary = crate::share::temporary(&target);
112 slow_copy();
113 crate::share::copy_private(path, &temporary).and_then(|()| std::fs::rename(&temporary, &target)).map_err(|e| {
114 let _ = std::fs::remove_file(&temporary);
115 format!("{}: {e}", target.display())
116 })?;
117 prune_file(&dir, now_ms, false);
118 Ok(())
119}
120
121fn standing(path: &Path) -> Option<(u64, Option<std::time::SystemTime>, u64)> {
124 let meta = std::fs::metadata(path).ok()?;
125 #[cfg(unix)]
126 let inode = std::os::unix::fs::MetadataExt::ino(&meta);
127 #[cfg(not(unix))]
128 let inode = 0;
129 Some((meta.len(), meta.modified().ok(), inode))
130}
131
132pub struct Prepared {
140 dir: PathBuf,
141 temporary: PathBuf,
142 path: PathBuf,
143 copied: Option<(u64, Option<std::time::SystemTime>, u64)>,
144}
145
146pub fn prepare(root: &Path, part: &str, file: &str, path: &Path) -> Result<Prepared, String> {
149 let dir = below(root, part, file).ok_or_else(|| format!("{file}: not a file's name"))?;
150 let size = std::fs::metadata(path).map_or(0, |m| m.len());
151 if !crate::disk::fits(root, size) {
152 return Err(format!("share-no-room:{file}"));
153 }
154 crate::share::private_dirs(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;
155 let copied = standing(path);
156 let temporary = crate::share::temporary(&dir.join("prepared"));
157 slow_copy();
158 if let Err(e) = crate::share::copy_private(path, &temporary) {
159 let _ = std::fs::remove_file(&temporary);
160 return Err(format!("{}: {e}", temporary.display()));
161 }
162 let _ = crate::share::set_time(&temporary, std::time::SystemTime::now());
163 Ok(Prepared { dir, temporary, path: path.to_path_buf(), copied })
164}
165
166impl Prepared {
167 pub fn changed(&self) -> Option<String> {
171 let now = standing(&self.path);
172 if now == self.copied && now.is_some() {
173 return None;
174 }
175 let (Some((size, time, inode)), Some((was_size, was_time, was_inode))) = (now, self.copied) else { return Some("gone or unreadable".into()) };
176 let mut said = Vec::new();
177 if size != was_size {
178 said.push(format!("its size {was_size}→{size}"));
179 }
180 if time != was_time {
181 said.push(format!("its time {was_time:?}→{time:?}"));
182 }
183 if inode != was_inode {
184 said.push(format!("its inode {was_inode}→{inode}"));
185 }
186 Some(said.join(", "))
187 }
188
189 pub fn keep(self, now_ms: i64) -> bool {
193 if self.copied.is_none() || standing(&self.path) != self.copied {
194 return false;
195 }
196 let target = self.dir.join(new_stamp(&self.dir, now_ms));
197 if std::fs::rename(&self.temporary, &target).is_err() {
198 return false;
199 }
200 prune_file(&self.dir, now_ms, false);
201 true
202 }
203}
204
205impl Drop for Prepared {
206 fn drop(&mut self) {
207 let _ = std::fs::remove_file(&self.temporary);
208 }
209}
210
211#[cfg(test)]
212thread_local! {
213 pub(crate) static UNDER_LOCK: std::cell::RefCell<Vec<String>> = const { std::cell::RefCell::new(Vec::new()) };
217}
218
219pub(crate) fn copied_under_lock(why: &str) {
221 #[cfg(test)]
222 UNDER_LOCK.with(|all| all.borrow_mut().push(why.to_string()));
223 let _ = why;
224}
225
226#[cfg(test)]
227thread_local! {
228 pub(crate) static SLOW_COPY: std::cell::Cell<std::time::Duration> = const { std::cell::Cell::new(std::time::Duration::ZERO) };
231}
232
233fn slow_copy() {
234 #[cfg(test)]
235 std::thread::sleep(SLOW_COPY.with(std::cell::Cell::get));
236}
237
238pub fn keep_moved(root: &Path, part: &str, file: &str, path: &Path, now_ms: i64) -> Result<u64, String> {
249 let dir = below(root, part, file).ok_or_else(|| format!("{file}: not a file's name"))?;
250 crate::share::private_dirs(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;
251 let size = std::fs::metadata(path).map_or(0, |m| m.len());
252 #[cfg(unix)]
253 let linked = std::fs::metadata(path).is_ok_and(|m| std::os::unix::fs::MetadataExt::nlink(&m) > 1);
254 #[cfg(not(unix))]
255 let linked = false;
256 let target = dir.join(new_stamp(&dir, now_ms));
257 if linked || std::fs::rename(path, &target).is_err() {
258 return keep(root, part, file, path, now_ms).map(|()| size);
259 }
260 crate::share::make_private(&target);
262 prune_file(&dir, now_ms, false);
263 Ok(0)
264}
265
266pub const ACCOUNTS: &str = "accounts";
272
273pub fn keep_bytes(root: &Path, part: &str, file: &str, bytes: &[u8], now_ms: i64) -> Result<PathBuf, String> {
276 let dir = below(root, part, file).ok_or_else(|| format!("{file}: not a file's name"))?;
277 crate::share::private_dirs(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;
278 let target = dir.join(new_stamp(&dir, now_ms));
279 crate::share::write_atomically(&target, bytes)?;
280 prune_file(&dir, now_ms, false);
281 Ok(target)
282}
283
284pub fn keep_sealed(root: &Path, part: &str, file: &str, hash: &str, size: u64, now_ms: i64) -> Result<(), String> {
287 let dir = below(root, part, file).ok_or_else(|| format!("{file}: not a file's name"))?;
288 crate::share::private_dirs(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;
289 let target = dir.join(format!("{}{SEALED}", new_stamp(&dir, now_ms)));
290 crate::share::write_atomically(&target, format!("{hash} {size}\n").as_bytes())?;
291 prune_file(&dir, now_ms, false);
292 Ok(())
293}
294
295fn prune_file(dir: &Path, now_ms: i64, gone: bool) {
299 for (i, version) in versions_in(dir).into_iter().enumerate() {
300 let old = now_ms - version.at > DAYS * DAY;
301 if (old && (i >= VERSIONS || gone)) || (version.sealed.is_some() && now_ms - version.at > SEALED_DAYS * DAY) {
302 let _ = std::fs::remove_file(dir.join(name_of(&version)));
303 }
304 }
305}
306
307fn name_of(version: &Version) -> String {
308 if version.sealed.is_some() { format!("{}{SEALED}", version.stamp) } else { version.stamp.clone() }
309}
310
311fn versions_in(dir: &Path) -> Vec<Version> {
313 let mut out: Vec<Version> = std::fs::read_dir(dir)
314 .into_iter()
315 .flatten()
316 .filter_map(Result::ok)
317 .filter(|e| e.file_type().is_ok_and(|t| t.is_file()))
318 .filter_map(|e| {
319 let name = e.file_name().to_string_lossy().to_string();
320 match name.strip_suffix(SEALED) {
321 Some(stamp) => {
322 let text = crate::share::read_small(&e.path())?;
324 let (hash, size) = text.trim().split_once(' ')?;
325 Some(Version { at: kept_at(stamp)?, size: size.parse().ok()?, sealed: Some(hash.to_string()), stamp: stamp.to_string() })
326 }
327 None => Some(Version { at: kept_at(&name)?, size: e.metadata().map_or(0, |m| m.len()), sealed: None, stamp: name }),
328 }
329 })
330 .collect();
331 out.sort_by(|a, b| b.at.cmp(&a.at).then_with(|| b.stamp.cmp(&a.stamp)));
332 out
333}
334
335pub fn versions(root: &Path, part: &str, file: &str) -> Vec<Version> {
337 below(root, part, file).map(|dir| versions_in(&dir)).unwrap_or_default()
338}
339
340pub fn files(root: &Path, part: &str) -> Vec<(String, Vec<Version>)> {
343 let mut out = Vec::new();
344 if let Some(dir) = part_dir(root, part) {
345 walk(&dir, &dir, &mut out);
346 }
347 out.sort_by(|a, b| b.1.first().map(|v| v.at).cmp(&a.1.first().map(|v| v.at)).then_with(|| a.0.cmp(&b.0)));
348 out
349}
350
351fn walk(top: &Path, dir: &Path, out: &mut Vec<(String, Vec<Version>)>) {
353 let versions = versions_in(dir);
354 if !versions.is_empty()
355 && let Ok(relative) = dir.strip_prefix(top)
356 {
357 out.push((relative.components().map(|c| c.as_os_str().to_string_lossy()).collect::<Vec<_>>().join("/"), versions));
358 }
359 for entry in std::fs::read_dir(dir).into_iter().flatten().filter_map(Result::ok) {
360 if entry.file_type().is_ok_and(|t| t.is_dir()) {
361 walk(top, &entry.path(), out);
362 }
363 }
364}
365
366pub fn draft_copies(root: &Path) -> Vec<PathBuf> {
371 let mut out = Vec::new();
372 for (file, versions) in files(root, "drafts") {
373 let Some(name) = file.strip_prefix("data/drafts/") else { continue };
374 if name.contains('/') || !name.ends_with(".eml") {
375 continue;
376 }
377 for kept in versions.iter().filter(|v| v.sealed.is_none()) {
378 if let Some(Kept::Copy(path)) = version(root, "drafts", &file, &kept.stamp) {
379 out.push(path);
380 }
381 }
382 }
383 out
384}
385
386pub fn version(root: &Path, part: &str, file: &str, stamp: &str) -> Option<Kept> {
388 kept_at(stamp)?;
389 let dir = below(root, part, file)?;
390 if dir.join(stamp).is_file() {
391 return Some(Kept::Copy(dir.join(stamp)));
392 }
393 versions_in(&dir).into_iter().find(|v| v.stamp == stamp).and_then(|v| v.sealed.map(|hash| Kept::Sealed { hash, size: v.size }))
394}
395
396pub fn prune(root: &Path, now_ms: i64, cap: u64, here: &dyn Fn(&str, &str) -> bool) {
402 let mut copies: Vec<(i64, PathBuf, u64)> = Vec::new();
403 for entry in std::fs::read_dir(root).into_iter().flatten().filter_map(Result::ok).filter(|e| e.file_type().is_ok_and(|t| t.is_dir())) {
404 let part = entry.file_name().to_string_lossy().to_string();
405 let mut listed = Vec::new();
406 walk(&entry.path(), &entry.path(), &mut listed);
407 for (file, _) in listed {
408 let Some(dir) = below(root, &part, &file) else { continue };
409 crate::share::clean_leftovers(&dir);
410 prune_file(&dir, now_ms, part != ACCOUNTS && !here(&part, &file));
414 copies.extend(versions_in(&dir).into_iter().filter(|v| v.sealed.is_none()).map(|v| (v.at, dir.join(&v.stamp), v.size)));
415 }
416 }
417 let mut total: u64 = copies.iter().map(|(_, _, size)| size).sum();
418 copies.sort_by_key(|(at, _, _)| *at);
419 for (_, path, size) in copies {
420 if total <= cap {
421 break;
422 }
423 if std::fs::remove_file(&path).is_ok() {
424 total -= size;
425 }
426 }
427}
428
429pub fn cap(root: &Path) -> u64 {
431 crate::disk::space(root).map_or(1 << 30, |(_, total)| (1u64 << 30).min(total / 20))
432}
433
434#[cfg(test)]
435mod tests {
436 use super::*;
437
438 fn scratch(name: &str) -> PathBuf {
439 let base = std::env::temp_dir().join(format!("sioul-history-{name}-{}", std::process::id()));
440 let _ = std::fs::remove_dir_all(&base);
441 std::fs::create_dir_all(&base).unwrap();
442 base
443 }
444
445 #[test]
446 fn twenty_versions_and_thirty_days() {
447 let base = scratch("kept");
448 let root = base.join("history");
449 let file = base.join("lease.md");
450 let start = 1_790_000_000_000;
451 for n in 0..40 {
453 std::fs::write(&file, format!("version {n}")).unwrap();
454 keep(&root, "notes", "files/notes/admin/lease.md", &file, start + n * DAY).unwrap();
455 }
456 let kept = versions(&root, "notes", "files/notes/admin/lease.md");
457 assert_eq!(kept.len(), 31, "{kept:?}");
458 assert_eq!(std::fs::read_to_string(root.join("notes/files/notes/admin/lease.md").join(&kept[0].stamp)).unwrap(), "version 39");
459 for n in 0..25 {
461 keep(&root, "settings", "config/config.toml", &file, start + n * 100 * DAY).unwrap();
462 }
463 assert_eq!(versions(&root, "settings", "config/config.toml").len(), VERSIONS);
464 let listed = files(&root, "notes");
465 assert_eq!(listed.len(), 1);
466 assert_eq!(listed[0].0, "files/notes/admin/lease.md");
467 assert!(matches!(version(&root, "notes", "files/notes/admin/lease.md", &kept[3].stamp), Some(Kept::Copy(_))));
468 assert!(version(&root, "notes", "files/notes/../../x", &kept[3].stamp).is_none());
470 assert!(version(&root, "../notes", "files/notes/admin/lease.md", &kept[3].stamp).is_none());
471 assert!(version(&root, "notes", "files/notes/admin/lease.md", "../../config.toml").is_none());
472 assert_eq!(kept_at(&stamp(start + 123)), Some(start + 123));
473 let _ = std::fs::remove_dir_all(&base);
474 }
475
476 #[test]
477 fn by_reference_capped_and_gone_files_expire() {
478 let base = scratch("bounds");
479 let root = base.join("history");
480 let file = base.join("scan.pdf");
481 std::fs::write(&file, vec![7u8; 10_000]).unwrap();
482 let start = 1_790_000_000_000;
483 keep_sealed(&root, "papers", "files/papers/scan.pdf", &"a".repeat(64), 10_000, start).unwrap();
485 let kept = versions(&root, "papers", "files/papers/scan.pdf");
486 assert_eq!(kept.len(), 1);
487 assert_eq!((kept[0].sealed.as_deref(), kept[0].size), (Some("a".repeat(64).as_str()), 10_000));
488 assert_eq!(version(&root, "papers", "files/papers/scan.pdf", &kept[0].stamp), Some(Kept::Sealed { hash: "a".repeat(64), size: 10_000 }));
489 keep(&root, "notes", "files/notes/gone.md", &file, start).unwrap();
493 keep(&root, "notes", "files/notes/here.md", &file, start).unwrap();
494 prune(&root, start + SEALED_DAYS * DAY, u64::MAX, &|_, _| true);
495 assert_eq!(versions(&root, "papers", "files/papers/scan.pdf").len(), 1);
496 prune(&root, start + SEALED_DAYS * DAY + 1, u64::MAX, &|_, _| true);
497 assert!(versions(&root, "papers", "files/papers/scan.pdf").is_empty());
498 prune(&root, start + 40 * DAY, u64::MAX, &|_, file| file != "files/notes/gone.md");
499 assert!(versions(&root, "notes", "files/notes/gone.md").is_empty());
500 assert_eq!(versions(&root, "notes", "files/notes/here.md").len(), 1);
501 for n in 0..5 {
503 keep(&root, "notes", "files/notes/big.md", &file, start + 70 * DAY + n).unwrap();
504 }
505 prune(&root, start + 70 * DAY + 10, 25_000, &|_, _| true);
506 let left: u64 = files(&root, "notes").iter().flat_map(|(_, v)| v.iter().map(|v| v.size)).sum();
507 assert!(left <= 25_000, "{left}");
508 assert_eq!(versions(&root, "notes", "files/notes/big.md").first().map(|v| v.at), Some(start + 70 * DAY + 4), "the newest stays");
509 let _ = std::fs::remove_dir_all(&base);
510 }
511
512 #[test]
517 fn the_help_and_the_guides_say_how_long_versions_stay() {
518 let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
519 let read = |relative: &str| std::fs::read_to_string(root.join(relative)).unwrap();
520 let help = |relative: &str| read(relative).lines().find(|l| l.starts_with("share-history-help =")).unwrap().to_string();
521 let en = help("crates/sioul-core/locales/en/sioul.ftl");
522 for said in [format!("last {VERSIONS} versions"), format!("last {DAYS} days"), format!("for {SEALED_DAYS} days")] {
523 assert!(en.contains(&said), "{said}: {en}");
524 }
525 let fr = help("crates/sioul-core/locales/fr/sioul.ftl");
526 for said in [format!("{VERSIONS} dernières versions"), format!("{DAYS} derniers jours"), format!("pendant {SEALED_DAYS} jours")] {
527 assert!(fr.contains(&said), "{said}: {fr}");
528 }
529 for (guide, copies, pointers) in [
531 ("website/docs/guide/notes.md", format!("last {VERSIONS}"), format!("{SEALED_DAYS} days")),
532 ("website/fr/docs/guide/notes.md", format!("{VERSIONS} dernières"), format!("{SEALED_DAYS} jours")),
533 ("website/docs/guide/sharing.md", format!("last {VERSIONS}"), format!("for {SEALED_DAYS} days")),
534 ("website/fr/docs/guide/sharing.md", format!("{VERSIONS} dernières"), format!("pendant {SEALED_DAYS} jours")),
535 ] {
536 let text = read(guide);
537 let lines: Vec<&str> = text.lines().filter(|l| l.contains(&copies)).collect();
538 assert!(!lines.is_empty(), "{guide}");
539 for line in lines {
540 assert!(line.contains(&pointers), "{guide}: {line}");
541 }
542 }
543 }
544
545 #[test]
550 #[ignore]
551 #[cfg(unix)]
552 fn what_a_bulk_s_copies_cost() {
553 let base = scratch("bulk-cost");
554 let (notes, root, links) = (base.join("notes"), base.join("history"), base.join("links"));
555 for dir in [¬es, &links] {
556 std::fs::create_dir_all(dir).unwrap();
557 }
558 let note = "a note of some five hundred bytes, as an agent's run writes it. ".repeat(8);
559 for n in 0..7_000 {
560 std::fs::write(notes.join(format!("{n}.md")), ¬e[..500]).unwrap();
561 std::fs::hard_link(notes.join(format!("{n}.md")), links.join(format!("{n}.md"))).unwrap();
562 }
563 let started = std::time::Instant::now();
564 let mut copied = 0;
565 for n in 0..7_000 {
566 let path = notes.join(format!("{n}.md"));
567 copied += keep_moved(&root, "notes", &format!("files/notes/agents/{n}.md"), &path, 1_800_000_000_000).unwrap();
568 std::fs::remove_file(&path).unwrap();
569 }
570 let copying = started.elapsed();
571 for n in 0..7_000 {
572 std::fs::rename(links.join(format!("{n}.md")), notes.join(format!("{n}.md"))).unwrap();
573 }
574 let started = std::time::Instant::now();
575 for n in 0..7_000 {
576 keep_moved(&root, "notes", &format!("files/notes/agents/{n}.md"), ¬es.join(format!("{n}.md")), 1_800_000_100_000).unwrap();
577 }
578 let moving = started.elapsed();
579 println!("7000 take-outs of 500 bytes: copied ({copied} bytes) in {copying:?}; moved in {moving:?}");
580 assert_eq!(copied, 7_000 * 500);
581 let _ = std::fs::remove_dir_all(&base);
582 }
583}