1use crate::SyncError;
45use crate::dav::{self, DAV};
46use serde::{Deserialize, Serialize};
47use std::collections::{BTreeMap, BTreeSet};
48use std::io::{Read, Seek, SeekFrom, Write};
49use std::path::{Path, PathBuf};
50use std::sync::Mutex;
51use std::time::{Duration, Instant, SystemTime};
52
53#[derive(Debug, Clone, Copy)]
57pub(crate) struct Limits {
58 pub(crate) wait: Duration,
59 pub(crate) small: Duration,
60 pub(crate) large: Duration,
61 pub(crate) pull: Duration,
62 pub(crate) blob: Duration,
64 pub(crate) floor: u64,
67 pub(crate) stall: Duration,
68}
69
70pub(crate) const LIMITS: Limits = Limits { wait: Duration::from_secs(10), small: Duration::from_secs(10), large: Duration::from_secs(60), pull: Duration::from_secs(30), blob: Duration::from_secs(15 * 60), floor: FLOOR, stall: Duration::from_secs(30) };
71
72pub const FLOOR: u64 = 16 << 10;
75
76pub(crate) const SEND_LIMITS: Limits = Limits { wait: Duration::from_secs(20), small: Duration::from_secs(30), large: Duration::from_secs(60), pull: Duration::from_secs(20 * 60), blob: Duration::from_secs(15 * 60), floor: FLOOR, stall: Duration::from_secs(30) };
80
81pub const LOOK_AGAIN: i64 = 6 * 3600;
83const SEAL_AGAIN: i64 = 6 * 3600;
85const LIST_AGAIN: i64 = 10 * 60;
87const BLOBS_AGAIN: i64 = 3600;
92const MIRROR_WANTED_BYTES: u64 = 64 << 20;
95const LARGEST: u64 = 64 << 20;
97const TEXTS_LARGEST: u64 = 4 << 20;
100const URGENT: [(&str, &str); 2] = [("texts", "send"), ("phone-messages", "ask")];
103const LISTING: u64 = 64 << 20;
108const OVERLAP: u64 = 1 << 20;
110
111const XML: &str = "application/xml; charset=utf-8";
112const LIST: &str = r#"<?xml version="1.0" encoding="utf-8"?><d:propfind xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns"><d:prop><d:resourcetype/><d:getetag/><d:getcontentlength/><oc:checksums/></d:prop></d:propfind>"#;
113const STAT: &str = r#"<?xml version="1.0" encoding="utf-8"?><d:propfind xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns"><d:prop><d:resourcetype/><d:getetag/><d:getcontentlength/><d:getlastmodified/><oc:checksums/></d:prop></d:propfind>"#;
115const OC: &str = "http://owncloud.org/ns";
117const PRINCIPAL: &str = r#"<?xml version="1.0" encoding="utf-8"?><d:propfind xmlns:d="DAV:"><d:prop><d:current-user-principal/></d:prop></d:propfind>"#;
118
119pub fn cache_of(memory: &Path) -> PathBuf {
123 memory.with_file_name("remote")
124}
125
126fn state_path(memory: &Path) -> PathBuf {
128 memory.with_file_name("remote.toml")
129}
130
131fn shown(folder: &Path) -> String {
132 folder.display().to_string()
133}
134
135pub fn host_of(url: &str) -> String {
137 let rest = url.split_once("://").map_or(url, |(_, rest)| rest);
138 let authority = rest.split(['/', '?', '#']).next().unwrap_or("");
139 authority.rsplit('@').next().unwrap_or("").to_string()
140}
141
142#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
148pub struct State {
149 #[serde(default, skip_serializing_if = "Option::is_none")]
151 pub on: Option<bool>,
152 #[serde(default, skip_serializing_if = "String::is_empty")]
155 pub given: String,
156 #[serde(default)]
158 pub asked: i64,
159 #[serde(default, skip_serializing_if = "Option::is_none")]
162 pub send: Option<bool>,
163 #[serde(default, skip_serializing_if = "String::is_empty")]
165 pub folder: String,
166 #[serde(default, skip_serializing_if = "String::is_empty")]
168 pub account: String,
169 #[serde(default, skip_serializing_if = "String::is_empty")]
170 pub url: String,
171 #[serde(default)]
173 pub confirmed: i64,
174 #[serde(default)]
176 pub sealed: i64,
177 #[serde(default)]
179 pub looked: i64,
180 #[serde(default)]
182 pub last: i64,
183 #[serde(default)]
184 pub tried: i64,
185 #[serde(default, skip_serializing_if = "String::is_empty")]
188 pub said: String,
189 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
194 pub left: BTreeMap<String, u64>,
195 #[serde(default, skip_serializing_if = "Option::is_none")]
200 pub own_entry: Option<(u64, i64)>,
201 #[serde(default, skip_serializing_if = "String::is_empty")]
205 pub mode: String,
206 #[serde(default, skip_serializing_if = "String::is_empty")]
208 pub place: String,
209 #[serde(default, skip_serializing_if = "String::is_empty")]
216 pub user: String,
217 #[serde(default)]
219 pub pushed: i64,
220 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
222 pub files: BTreeMap<String, Fetched>,
223 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
225 pub folders: BTreeMap<String, Looked>,
226 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
228 pub sent: BTreeMap<String, Sent>,
229 #[serde(default, skip_serializing_if = "is_zero_i64")]
233 pub swept_at: i64,
234 #[serde(default, skip_serializing_if = "is_zero_u32")]
238 pub blobs_tries: u32,
239 #[serde(default, skip_serializing_if = "is_zero_i64")]
240 pub blobs_again: i64,
241 #[serde(default, skip_serializing_if = "String::is_empty")]
245 pub blobs_seen: String,
246}
247
248fn is_zero_u32(n: &u32) -> bool {
249 *n == 0
250}
251
252fn is_zero_i64(n: &i64) -> bool {
253 *n == 0
254}
255
256#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
258pub struct Fetched {
259 #[serde(default)]
260 pub etag: String,
261 #[serde(default)]
262 pub size: u64,
263}
264
265#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
267pub struct Looked {
268 #[serde(default)]
269 pub etag: String,
270 #[serde(default)]
271 pub at: i64,
272}
273
274#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
277pub struct Sent {
278 #[serde(default)]
279 pub size: u64,
280 #[serde(default)]
281 pub modified: u64,
282 #[serde(default)]
283 pub etag: String,
284}
285
286impl State {
287 pub fn load(memory: &Path) -> State {
288 std::fs::read_to_string(state_path(memory)).ok().and_then(|text| toml::from_str(&text).ok()).unwrap_or_default()
289 }
290
291 fn write(&self, memory: &Path) -> Result<(), String> {
292 let text = toml::to_string(self).map_err(|e| e.to_string())?;
293 crate::share::write_atomically(&state_path(memory), format!("# Sioul: the sharing folder fetched from its server too (docs/database.md). Never shared.\n{text}").as_bytes())
294 }
295
296 fn save(&self, memory: &Path) -> Result<(), String> {
299 let path = state_path(memory);
300 sioul_core::filelock::with_lock(&path, || {
301 let there = State::load(memory);
302 State { on: there.on, given: there.given, asked: there.asked, send: there.send, ..self.clone() }.write(memory)
303 })
304 }
305
306 fn save_pulled(&self, memory: &Path, ended: i64, loaded_left: &BTreeMap<String, u64>) -> Result<(), String> {
323 let path = state_path(memory);
324 sioul_core::filelock::with_lock(&path, || {
325 let mut merged = State::load(memory);
326 if (merged.url.as_str(), merged.folder.as_str(), merged.account.as_str()) != (self.url.as_str(), self.folder.as_str(), self.account.as_str()) && !merged.url.is_empty() {
327 return Ok(());
328 }
329 let ours_later = self.tried >= merged.tried || merged.tried > ended;
330 let (later, earlier) = if ours_later { (self.clone(), merged.clone()) } else { (merged.clone(), self.clone()) };
331 merged.url = later.url.clone();
332 merged.folder = later.folder.clone();
333 merged.account = later.account.clone();
334 merged.tried = later.tried;
335 merged.last = [later.last, earlier.last].into_iter().filter(|last| *last <= ended).max().unwrap_or(self.last);
337 merged.said = later.said.clone();
338 merged.confirmed = later.confirmed;
339 merged.sealed = later.sealed.max(earlier.sealed);
340 merged.files = earlier.files.into_iter().chain(later.files).collect();
341 merged.folders = earlier.folders.into_iter().chain(later.folders).collect();
342 let brought: Vec<&String> = loaded_left.keys().filter(|name| !self.left.contains_key(*name)).collect();
343 merged.left.retain(|name, _| !brought.contains(&name));
344 merged.left.extend(self.left.iter().filter(|(name, size)| loaded_left.get(*name) != Some(*size)).map(|(name, size)| (name.clone(), *size)));
346 merged.own_entry = [earlier.own_entry, later.own_entry].into_iter().flatten().max_by_key(|(_, at)| *at);
347 merged.write(memory)
348 })
349 }
350
351 pub fn choose(memory: &Path, change: impl FnOnce(&mut State)) -> Result<State, String> {
353 let path = state_path(memory);
354 sioul_core::filelock::with_lock(&path, || {
355 let mut state = State::load(memory);
356 change(&mut state);
357 state.write(memory)?;
358 Ok(state)
359 })
360 }
361
362 pub fn confirmed_for(&self, folder: &Path) -> bool {
364 self.confirmed > 0 && !self.url.is_empty() && self.folder == shown(folder)
365 }
366
367 pub fn fetching(&self) -> bool {
369 self.on != Some(false)
370 }
371
372 pub fn pulls_well(&self, folder: &Path) -> bool {
379 self.mode != MIRROR && self.fetching() && self.confirmed_for(folder) && self.last > 0 && self.said.is_empty() && self.left.is_empty()
380 }
381
382 pub fn went_through_since(&self, folder: &Path, since: i64, now: i64) -> bool {
388 self.pulls_well(folder) && (since..=now).contains(&self.last)
389 }
390
391 pub fn sending(&self) -> bool {
394 self.fetching() && self.send != Some(false)
395 }
396
397 pub fn host(&self) -> String {
399 host_of(&self.url)
400 }
401
402 pub fn look_due(&self, folder: &Path, now: i64) -> bool {
406 self.folder != shown(folder) || self.asked > self.looked || (!self.confirmed_for(folder) && (self.looked == 0 || now - self.looked >= LOOK_AGAIN))
407 }
408}
409
410type Stamp = [Option<(u64, SystemTime)>; 3];
416static ATTACHED: Mutex<BTreeMap<PathBuf, (Stamp, Option<PathBuf>)>> = Mutex::new(BTreeMap::new());
417
418fn stamp_of(path: &Path) -> Option<(u64, SystemTime)> {
419 std::fs::metadata(path).ok().map(|m| (m.len(), m.modified().unwrap_or(SystemTime::UNIX_EPOCH)))
420}
421
422pub fn attach(folder: &Path, memory: &Path) -> Option<PathBuf> {
428 let cache = cache_of(memory);
429 let stamp = [stamp_of(&state_path(memory)), stamp_of(&folder.join("seal.toml")), stamp_of(&cache.join("seal.toml"))];
430 let mut attached = ATTACHED.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
431 if let Some((was, chosen)) = attached.get(folder)
432 && *was == stamp
433 {
434 return chosen.clone();
435 }
436 let chosen = usable(folder, memory).then_some(cache);
437 attached.insert(folder.to_path_buf(), (stamp, chosen.clone()));
438 chosen
439}
440
441pub fn overlay(folder: &Path) -> Option<PathBuf> {
444 ATTACHED.lock().ok()?.get(folder).and_then(|(_, chosen)| chosen.clone())
445}
446
447fn seal_of(folder: &Path) -> Option<Vec<u8>> {
449 let path = folder.join("seal.toml");
450 let size = std::fs::metadata(&path).ok()?.len();
451 (size <= crate::share::SMALL_FILE).then(|| std::fs::read(&path).ok()).flatten()
452}
453
454fn usable(folder: &Path, memory: &Path) -> bool {
456 let state = State::load(memory);
457 state.mode != MIRROR && state.folder == shown(folder) && seal_of(folder).is_some_and(|here| seal_of(&cache_of(memory)).is_some_and(|fetched| fetched == here))
458}
459
460pub fn forget(memory: &Path, folder: &Path) {
462 let _ = std::fs::remove_dir_all(cache_of(memory));
463 let _ = std::fs::remove_file(state_path(memory));
464 let _ = std::fs::remove_file(crate::blobs::Wanted::path(memory));
465 let _ = std::fs::remove_file(Lane::Main.memory(memory));
466 let _ = std::fs::remove_file(Lane::Urgent.memory(memory));
467 if let Ok(mut attached) = ATTACHED.lock() {
468 attached.remove(folder);
469 }
470}
471
472pub struct Login {
478 pub account: String,
479 pub url: String,
480 pub user: String,
481 pub password: Option<String>,
482}
483
484pub(crate) struct Server {
490 agent: ureq::Agent,
491 authorization: String,
492 started: Instant,
493 limits: Limits,
494 listed_bytes: std::sync::atomic::AtomicU64,
496}
497
498#[derive(Debug, Default)]
500pub(crate) struct Listing {
501 pub(crate) etag: String,
502 pub(crate) items: BTreeMap<String, Item>,
503}
504
505#[derive(Debug, Default, Clone)]
506pub(crate) struct Item {
507 pub(crate) dir: bool,
508 pub(crate) etag: String,
509 pub(crate) size: u64,
510 pub(crate) checksum: String,
512 pub(crate) modified: Option<i64>,
515}
516
517pub(crate) struct Got {
520 start: u64,
521 bytes: Vec<u8>,
522 error: Option<SyncError>,
523}
524
525#[derive(Debug)]
529pub(crate) struct PutFailed {
530 pub(crate) error: SyncError,
531 pub(crate) kind: &'static str,
532 pub(crate) words: String,
533 pub(crate) seconds: u64,
534 pub(crate) sent: u64,
535 pub(crate) total: u64,
536}
537
538struct Counted {
540 inner: std::io::Cursor<Vec<u8>>,
541 moved: std::sync::Arc<std::sync::atomic::AtomicU64>,
542}
543
544impl Read for Counted {
545 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
546 let n = self.inner.read(buf)?;
547 self.moved.fetch_add(n as u64, std::sync::atomic::Ordering::Relaxed);
548 Ok(n)
549 }
550}
551
552fn failed(e: &ureq::Error) -> SyncError {
553 match e {
554 ureq::Error::Tls(_) => SyncError::Tls(e.to_string()),
555 _ if format!("{e:?}").starts_with("Rustls") => SyncError::Tls(e.to_string()),
556 _ => SyncError::Network(e.to_string()),
557 }
558}
559
560fn header(response: &ureq::http::Response<ureq::Body>, name: &str) -> Option<String> {
561 response.headers().get(name).and_then(|v| v.to_str().ok()).map(str::to_string)
562}
563
564impl Server {
565 pub(crate) fn new(login: &Login, limits: Limits) -> Result<Server, SyncError> {
566 let password = login.password.as_deref().ok_or(SyncError::NoPassword)?;
567 Ok(Server {
568 agent: dav::shared_agent(),
569 authorization: format!("Basic {}", sioul_core::lines::base64_encode(format!("{}:{password}", login.user).as_bytes())),
570 started: Instant::now(),
571 limits,
572 listed_bytes: std::sync::atomic::AtomicU64::new(0),
573 })
574 }
575
576 pub(crate) fn request(&self, large: bool, method: &str, url: &str, headers: &[(&str, &str)], body: Vec<u8>) -> Result<ureq::http::Response<ureq::Body>, SyncError> {
577 self.request_with(if large { self.limits.large } else { self.limits.small }, method, url, headers, body)
578 }
579
580 fn request_with(&self, whole: Duration, method: &str, url: &str, headers: &[(&str, &str)], body: Vec<u8>) -> Result<ureq::http::Response<ureq::Body>, SyncError> {
584 dav::allowed(url)?;
585 if self.started.elapsed() > self.limits.pull {
586 return Err(SyncError::Network("out of time".into()));
587 }
588 let mut request = ureq::http::Request::builder().method(method).uri(url).header("Authorization", &self.authorization).header("User-Agent", "Sioul");
589 for (name, value) in headers {
590 request = request.header(*name, *value);
591 }
592 let request = request.body(body).map_err(|e| SyncError::Server(e.to_string()))?;
593 let request = self.agent.configure_request(request).timeout_connect(Some(self.limits.wait)).timeout_global(Some(whole)).build();
594 crate::stalls::ask(self.limits.wait, false);
595 let response = self.agent.run(request).map_err(|e| failed(&e))?;
596 if response.status().as_u16() == 401 {
597 return Err(SyncError::Login(format!("{method}: 401")));
598 }
599 Ok(response)
600 }
601
602 pub(crate) fn list(&self, url: &str) -> Result<Listing, SyncError> {
604 self.list_within(self.limits.small, url)
605 }
606
607 pub(crate) fn list_within(&self, whole: Duration, url: &str) -> Result<Listing, SyncError> {
610 let mut response = self.request_with(whole, "PROPFIND", url, &[("Depth", "1"), ("Content-Type", XML)], LIST.as_bytes().to_vec())?;
611 match response.status().as_u16() {
612 207 => {}
613 404 => return Err(SyncError::NotFound(String::new())),
614 status => return Err(SyncError::Server(format!("PROPFIND: {status}"))),
615 }
616 let reader = response.body_mut().with_config().limit(LISTING).reader();
617 let (listing, bytes) = read_listing(reader, url)?;
618 self.listed_bytes.fetch_add(bytes, std::sync::atomic::Ordering::Relaxed);
619 Ok(listing)
620 }
621
622 pub(crate) fn listed_bytes(&self) -> u64 {
624 self.listed_bytes.load(std::sync::atomic::Ordering::Relaxed)
625 }
626
627 pub(crate) fn stat(&self, url: &str) -> Result<Option<Item>, SyncError> {
630 let mut response = self.request(false, "PROPFIND", url, &[("Depth", "0"), ("Content-Type", XML)], STAT.as_bytes().to_vec())?;
631 match response.status().as_u16() {
632 207 => {}
633 404 => return Ok(None),
634 status => return Err(SyncError::Server(format!("PROPFIND: {status}"))),
635 }
636 let body = response.body_mut().with_config().limit(LISTING).read_to_string().map_err(|e| failed(&e))?;
637 let (responses, _) = dav::multistatus(&body)?;
638 let sums = checksums(&body, url);
639 Ok(responses.first().map(|r| Item {
640 dir: r.is(DAV, "collection"),
641 etag: r.text(DAV, "getetag").unwrap_or_default(),
642 size: r.text(DAV, "getcontentlength").and_then(|t| t.parse().ok()).unwrap_or(0),
643 checksum: sums.get(&dav::path_key(&dav::absolute(url, &r.href))).cloned().unwrap_or_default(),
644 modified: r.text(DAV, "getlastmodified").and_then(|t| jiff::fmt::rfc2822::DateTimeParser::new().parse_timestamp(t.trim()).ok()).map(|t| t.as_second()),
645 }))
646 }
647
648 pub(crate) fn put_file(&self, url: &str, body: Vec<u8>, seen: Option<&str>, modified_s: u64, sha1: &str) -> Result<(u16, Option<String>), PutFailed> {
660 let total = body.len() as u64;
661 let started = Instant::now();
662 let gave_up = |error: SyncError, kind: &'static str, sent: u64| PutFailed { words: error.to_string(), error, kind, seconds: started.elapsed().as_millis().div_ceil(1000) as u64, sent, total };
663 dav::allowed(url).map_err(|e| gave_up(e, "refused", 0))?;
664 if self.started.elapsed() > self.limits.pull {
665 return Err(gave_up(SyncError::Network("out of time".into()), "out-of-time", 0));
666 }
667 let precondition = match seen.filter(|etag| !etag.is_empty()) {
668 Some(etag) => ("If-Match", etag),
669 None => ("If-None-Match", "*"),
670 };
671 let whole = self.limits.large + Duration::from_secs(total / self.limits.floor.max(1));
672 let moved = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0));
673 let reader = Counted { inner: std::io::Cursor::new(body), moved: std::sync::Arc::clone(&moved) };
674 let request = ureq::http::Request::builder()
675 .method("PUT")
676 .uri(url)
677 .header("Authorization", &self.authorization)
678 .header("User-Agent", "Sioul")
679 .header(precondition.0, precondition.1)
680 .header("Content-Type", "application/octet-stream")
681 .header("Content-Length", total.to_string())
682 .header("X-OC-Mtime", modified_s.to_string())
683 .header("OC-Checksum", format!("SHA1:{sha1}"))
684 .body(ureq::SendBody::from_owned_reader(reader))
685 .map_err(|e| gave_up(SyncError::Server(e.to_string()), "server", 0))?;
686 let request = self.agent.configure_request(request).timeout_connect(Some(self.limits.wait)).timeout_global(Some(whole)).build();
690 crate::stalls::ask(self.limits.stall, true);
691 match self.agent.run(request) {
692 Ok(response) if response.status().as_u16() == 401 => Err(gave_up(SyncError::Login("PUT: 401".into()), "login", moved.load(std::sync::atomic::Ordering::Relaxed))),
693 Ok(response) => Ok((response.status().as_u16(), header(&response, "ETag").or_else(|| header(&response, "OC-ETag")))),
694 Err(e) => {
695 let sent = moved.load(std::sync::atomic::Ordering::Relaxed);
696 let kind = match &e {
697 ureq::Error::Timeout(ureq::Timeout::Connect | ureq::Timeout::Resolve) | ureq::Error::ConnectionFailed | ureq::Error::HostNotFound => "unreachable",
698 ureq::Error::Timeout(_) | ureq::Error::Io(_) if started.elapsed() + Duration::from_secs(2) < whole && sent > 0 => "stalled",
700 ureq::Error::Timeout(_) => "timeout",
701 ureq::Error::Tls(_) => "tls",
702 _ if format!("{e:?}").starts_with("Rustls") => "tls",
703 _ => "network",
704 };
705 Err(gave_up(failed(&e), kind, sent))
706 }
707 }
708 }
709
710 pub(crate) fn small(&self, url: &str) -> Result<Option<(Vec<u8>, Option<String>)>, SyncError> {
714 self.small_up_to(url, crate::share::SMALL_FILE)
715 }
716
717 pub(crate) fn small_up_to(&self, url: &str, cap: u64) -> Result<Option<(Vec<u8>, Option<String>)>, SyncError> {
719 let mut response = self.request(false, "GET", url, &[("Accept-Encoding", "identity")], Vec::new())?;
720 match response.status().as_u16() {
721 200 => {}
722 404 | 410 => return Ok(None),
723 status => return Err(SyncError::Server(format!("GET: {status}"))),
724 }
725 let etag = header(&response, "ETag");
726 let length = response.body().content_length();
727 let chunked = header(&response, "Transfer-Encoding").is_some_and(|t| t.to_ascii_lowercase().contains("chunked"));
728 if length.is_some_and(|l| l > cap) {
729 return Ok(None);
730 }
731 let bytes = match response.body_mut().with_config().limit(cap).read_to_vec() {
732 Ok(bytes) => bytes,
733 Err(ureq::Error::BodyExceedsLimit(_)) => return Ok(None),
734 Err(e) => return Err(failed(&e)),
735 };
736 if !(length == Some(bytes.len() as u64) || (length.is_none() && chunked)) {
737 return Err(SyncError::Network("GET: cut".into()));
738 }
739 Ok(Some((bytes, etag)))
740 }
741
742 fn tail(&self, url: &str, from: u64) -> Result<Option<Got>, SyncError> {
746 let range = format!("bytes={from}-");
747 let mut headers = vec![("Accept-Encoding", "identity")];
748 if from > 0 {
749 headers.push(("Range", range.as_str()));
750 }
751 let mut response = self.request(true, "GET", url, &headers, Vec::new())?;
752 let start = match response.status().as_u16() {
753 200 => 0,
754 206 => {
755 let said = header(&response, "Content-Range").unwrap_or_default();
757 said.trim().strip_prefix("bytes").and_then(|r| r.trim().split('-').next()).and_then(|s| s.trim().parse().ok()).ok_or_else(|| SyncError::Server(format!("GET: Content-Range {said}")))?
758 }
759 404 | 410 | 416 => return Ok(None),
760 status => return Err(SyncError::Server(format!("GET: {status}"))),
761 };
762 let length = response.body().content_length();
763 let mut reader = response.body_mut().with_config().limit(LARGEST).reader();
764 let mut bytes = Vec::new();
765 let mut piece = vec![0u8; 64 << 10];
766 let mut error = loop {
767 match reader.read(&mut piece) {
768 Ok(0) => break None,
769 Ok(n) => bytes.extend_from_slice(&piece[..n]),
770 Err(e) => break Some(SyncError::Network(e.to_string())),
771 }
772 };
773 if error.is_none() && length.is_some_and(|l| l != bytes.len() as u64) {
774 error = Some(SyncError::Network("GET: cut".into()));
775 }
776 Ok(Some(Got { start, bytes, error }))
777 }
778
779 pub(crate) fn put(&self, url: &str, body: Vec<u8>, seen: Option<&str>) -> Result<(u16, Option<String>), SyncError> {
783 let precondition = match seen {
784 Some(etag) => ("If-Match", etag),
785 None => ("If-None-Match", "*"),
786 };
787 let response = self.request(true, "PUT", url, &[precondition, ("Content-Type", "application/octet-stream")], body)?;
788 Ok((response.status().as_u16(), header(&response, "ETag").or_else(|| header(&response, "OC-ETag"))))
789 }
790
791 pub(crate) fn mkcol(&self, url: &str) -> Result<u16, SyncError> {
793 Ok(self.request(false, "MKCOL", url, &[], Vec::new())?.status().as_u16())
794 }
795
796 pub(crate) fn delete(&self, url: &str, seen: Option<&str>) -> Result<u16, SyncError> {
798 let headers: Vec<(&str, &str)> = seen.map(|etag| vec![("If-Match", etag)]).unwrap_or_default();
799 Ok(self.request(false, "DELETE", url, &headers, Vec::new())?.status().as_u16())
800 }
801
802 pub(crate) fn etag(&self, url: &str) -> Result<Option<String>, SyncError> {
804 let mut response = self.request(false, "PROPFIND", url, &[("Depth", "0"), ("Content-Type", XML)], LIST.as_bytes().to_vec())?;
805 match response.status().as_u16() {
806 207 => {}
807 404 => return Ok(None),
808 status => return Err(SyncError::Server(format!("PROPFIND: {status}"))),
809 }
810 let body = response.body_mut().with_config().limit(LISTING).read_to_string().map_err(|e| failed(&e))?;
811 let (responses, _) = dav::multistatus(&body)?;
812 Ok(responses.first().map(|r| r.text(DAV, "getetag").unwrap_or_default()))
813 }
814
815 pub(crate) fn download(&self, url: &str, path: &Path, limit: u64) -> Result<Option<u64>, SyncError> {
820 let Some((size, temporary)) = self.download_aside(url, path, limit)? else { return Ok(None) };
821 if let Err(e) = std::fs::rename(&temporary, path) {
822 let _ = std::fs::remove_file(&temporary);
823 return Err(SyncError::Disk(format!("{}: {e}", path.display())));
824 }
825 Ok(Some(size))
826 }
827
828 pub(crate) fn download_aside(&self, url: &str, path: &Path, limit: u64) -> Result<Option<(u64, PathBuf)>, SyncError> {
832 let mut response = self.request_with(self.limits.blob, "GET", url, &[("Accept-Encoding", "identity")], Vec::new())?;
833 match response.status().as_u16() {
834 200 => {}
835 404 | 410 => return Ok(None),
836 status => return Err(SyncError::Server(format!("GET: {status}"))),
837 }
838 let length = response.body().content_length();
839 let chunked = header(&response, "Transfer-Encoding").is_some_and(|t| t.to_ascii_lowercase().contains("chunked"));
840 if length.is_some_and(|l| l > limit) {
841 return Ok(None);
842 }
843 if length.is_none() && !chunked {
844 return Err(SyncError::Network("GET: its end cannot be told".into()));
845 }
846 let temporary = crate::share::temporary(path);
847 let disk = |e: std::io::Error| SyncError::Disk(format!("{}: {e}", path.display()));
848 let written = (|| -> Result<u64, SyncError> {
849 if let Some(parent) = path.parent() {
850 crate::share::private_dirs(parent).map_err(disk)?;
851 }
852 let mut out = crate::share::new_private(&temporary).map_err(disk)?;
853 let mut reader = response.body_mut().with_config().limit(limit).reader();
854 let copied = std::io::copy(&mut reader, &mut out).map_err(|e| SyncError::Network(e.to_string()))?;
855 if length.is_some_and(|l| l != copied) {
856 return Err(SyncError::Network("GET: cut".into()));
857 }
858 out.sync_all().map_err(disk)?;
859 Ok(copied)
860 })();
861 if written.is_err() {
862 let _ = std::fs::remove_file(&temporary);
863 }
864 written.map(|size| Some((size, temporary)))
865 }
866}
867
868pub(crate) fn encode_path(path: &str) -> String {
870 path.split('/')
871 .map(|name| name.bytes().map(|b| if b.is_ascii_alphanumeric() || b"-._~".contains(&b) { (b as char).to_string() } else { format!("%{b:02X}") }).collect::<String>())
872 .collect::<Vec<_>>()
873 .join("/")
874}
875
876fn read_listing(mut reader: impl std::io::Read, url: &str) -> Result<(Listing, u64), SyncError> {
886 let me = dav::path_key(url);
887 let mut listing = Listing::default();
888 let mut itself = false;
889 let (mut buf, mut total, mut piece) = (Vec::new(), 0u64, vec![0u8; 64 << 10]);
890 let mut namespaces: Option<String> = None;
891 loop {
892 let n = reader.read(&mut piece).map_err(|e| SyncError::Network(format!("PROPFIND: {e}")))?;
893 total += n as u64;
894 buf.extend_from_slice(&piece[..n]);
895 if namespaces.is_none() {
896 match first_element(&buf) {
897 Some((declared, end)) => {
898 namespaces = Some(declared);
899 buf.drain(..end);
900 }
901 None if n == 0 => return Err(SyncError::Server("XML: no element".into())),
902 None => continue,
903 }
904 }
905 let declared = namespaces.as_deref().unwrap_or_default();
906 let mut from = 0;
907 while let Some(end) = response_end(&buf[from..]) {
908 let chunk = std::str::from_utf8(&buf[from..from + end]).map_err(|e| SyncError::Server(format!("XML: {e}")))?;
909 let start = chunk.find('<').unwrap_or(0);
911 let xml = format!("<listing{declared}>{}</listing>", &chunk[start..]);
912 let (responses, _) = dav::multistatus(&xml)?;
913 let sums = checksums(&xml, url);
914 for response in responses {
915 let path = dav::path_key(&dav::absolute(url, &response.href));
916 let etag = response.text(DAV, "getetag").unwrap_or_default();
917 if path == me {
918 listing.etag = etag;
919 itself = true;
920 continue;
921 }
922 let Some(name) = path.strip_prefix(&me).and_then(|rest| rest.strip_prefix('/')).filter(|n| !n.is_empty() && !n.contains('/')) else { continue };
923 let size = response.text(DAV, "getcontentlength").and_then(|t| t.parse().ok()).unwrap_or(0);
924 listing.items.insert(name.to_string(), Item { dir: response.is(DAV, "collection"), etag, size, checksum: sums.get(&path).cloned().unwrap_or_default(), modified: None });
925 }
926 from += end;
927 }
928 buf.drain(..from);
929 if n == 0 {
930 if end_tag(&buf, b"multistatus").is_none() {
932 return Err(SyncError::Server("XML: the answer ends before its end".into()));
933 }
934 if !itself {
936 return Err(SyncError::Server("XML: the folder's own answer is missing".into()));
937 }
938 return Ok((listing, total));
939 }
940 }
941}
942
943fn first_element(buf: &[u8]) -> Option<(String, usize)> {
946 let mut at = 0;
947 loop {
948 let open = at + buf[at..].iter().position(|b| *b == b'<')?;
949 let close = open + buf[open..].iter().position(|b| *b == b'>')?;
950 let tag = std::str::from_utf8(&buf[open..=close]).ok()?;
951 if tag.starts_with("<?") || tag.starts_with("<!") {
952 at = close + 1;
953 continue;
954 }
955 let mut declared = String::new();
956 let mut rest = tag;
957 while let Some(found) = rest.find(" xmlns").or_else(|| rest.find("\nxmlns")).or_else(|| rest.find("\txmlns")) {
958 let attribute = &rest[found..];
959 let quote_at = attribute.find(['"', '\''])?;
960 let quote = attribute.as_bytes()[quote_at] as char;
961 let value_end = quote_at + 1 + attribute[quote_at + 1..].find(quote)?;
962 declared.push(' ');
963 declared.push_str(attribute[..=value_end].trim_start());
964 rest = &attribute[value_end + 1..];
965 }
966 return Some((declared, close + 1));
967 }
968}
969
970fn response_end(buf: &[u8]) -> Option<usize> {
973 end_tag(buf, b"response")
974}
975
976fn end_tag(buf: &[u8], name: &[u8]) -> Option<usize> {
980 let mut from = 0;
981 while let Some(found) = buf[from..].windows(name.len()).position(|w| w == name) {
982 let at = from + found;
983 let close = at + name.len() + buf[at + name.len()..].iter().position(|b| !b.is_ascii_whitespace())?;
985 if buf[close] == b'>'
987 && let Some(open) = buf[..at].iter().rposition(|b| *b == b'<')
988 {
989 let between = &buf[open + 1..at];
990 if between.first() == Some(&b'/') && between[1..].iter().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'.' | b':')) && (between.len() == 1 || between.last() == Some(&b':')) {
991 return Some(close + 1);
992 }
993 }
994 from = at + name.len();
995 }
996 None
997}
998
999fn checksums(body: &str, url: &str) -> BTreeMap<String, String> {
1002 let Ok(doc) = roxmltree::Document::parse(body) else { return BTreeMap::new() };
1003 doc.descendants()
1004 .filter(|n| n.has_tag_name((DAV, "response")))
1005 .filter_map(|response| {
1006 let href = response.children().find(|c| c.has_tag_name((DAV, "href")))?.text()?.trim().to_string();
1007 let sums = response.descendants().filter(|n| n.has_tag_name((OC, "checksum"))).filter_map(|n| n.text()).map(str::trim).filter(|t| !t.is_empty()).collect::<Vec<_>>().join(" ");
1008 (!sums.is_empty()).then(|| (dav::path_key(&dav::absolute(url, &href)), sums))
1009 })
1010 .collect()
1011}
1012
1013fn sha1_hex(bytes: &[u8]) -> String {
1015 use sha1::Digest;
1016 sha1::Sha1::digest(bytes).iter().map(|b| format!("{b:02x}")).collect()
1017}
1018
1019fn sha1_of(checksums: &str) -> Option<String> {
1021 checksums.split_whitespace().find_map(|sum| sum.split_once(':').filter(|(kind, _)| kind.eq_ignore_ascii_case("sha1")).map(|(_, hex)| hex.to_ascii_lowercase()))
1022}
1023
1024fn code(error: &SyncError, host: &str) -> String {
1026 let kind = match error {
1027 SyncError::Login(_) | SyncError::AppPassword(_) => "login",
1028 SyncError::Tls(_) => "tls",
1029 SyncError::Network(_) => "network",
1030 SyncError::NoPassword => "no-password",
1031 SyncError::Disk(_) => "disk",
1032 SyncError::Server(d) if d == "quota" => "quota",
1033 _ => "server",
1034 };
1035 format!("{kind}:{host}")
1036}
1037
1038pub(crate) fn files_root(server: &Server, login: &Login) -> Result<Option<String>, SyncError> {
1045 let Some(at) = login.url.find("/remote.php/") else { return Ok(None) };
1046 let base = format!("{}/remote.php/dav/", &login.url[..at]);
1047 let mut response = server.request(false, "PROPFIND", &base, &[("Depth", "0"), ("Content-Type", XML)], PRINCIPAL.as_bytes().to_vec())?;
1048 if response.status().as_u16() != 207 {
1049 return Ok(None);
1050 }
1051 let body = response.body_mut().with_config().limit(LISTING).read_to_string().map_err(|e| failed(&e))?;
1052 let (responses, _) = dav::multistatus(&body)?;
1053 let principal = responses.iter().find_map(|r| r.prop(DAV, "current-user-principal").and_then(|p| p.hrefs.first().cloned()));
1054 let root = principal.and_then(|href| {
1055 let at = href.find("/principals/users/")?;
1056 let user = href[at + "/principals/users/".len()..].trim_end_matches('/');
1057 (!user.is_empty() && !user.contains('/')).then(|| dav::absolute(&base, &format!("{}/files/{user}/", &href[..at])))
1058 });
1059 match root {
1060 Some(root) if dav::origin(&root).is_none() || dav::origin(&root) != dav::origin(&base) => Err(SyncError::Elsewhere(root)),
1061 root => Ok(root),
1062 }
1063}
1064
1065pub fn places(folder: &Path, nextcloud: &[String]) -> Vec<String> {
1074 let text = folder.to_string_lossy().replace('\\', "/");
1075 let text = text.trim_end_matches('/');
1076 let mut out: Vec<String> = Vec::new();
1077 for storage in ["/storage/emulated/0", "/sdcard", "/storage/self/primary", "/mnt/sdcard", "/mnt/user/0/primary"] {
1078 let Some(rest) = text.strip_prefix(storage).and_then(|r| r.strip_prefix('/')) else { continue };
1079 match rest.strip_prefix("Android/media/com.nextcloud.client/nextcloud/").and_then(|r| r.split_once('/')) {
1081 Some((_, inside)) => out.push(inside.to_string()),
1082 None if !rest.starts_with("Android/") => out.push(rest.to_string()),
1083 None => {}
1084 }
1085 }
1086 for config in nextcloud {
1087 out.extend(nextcloud_places(text, config));
1088 }
1089 let names: Vec<&str> = text.rsplit('/').filter(|n| !n.is_empty()).take(3).collect();
1090 for n in 1..=names.len() {
1091 out.push(names[..n].iter().rev().copied().collect::<Vec<_>>().join("/"));
1092 }
1093 let mut seen = BTreeSet::new();
1094 out.into_iter().map(|p| p.trim_matches('/').to_string()).filter(|p| !p.is_empty() && seen.insert(p.clone())).collect()
1095}
1096
1097fn nextcloud_places(folder: &str, config: &str) -> Vec<String> {
1101 let mut local: BTreeMap<&str, &str> = BTreeMap::new();
1102 let mut target: BTreeMap<&str, &str> = BTreeMap::new();
1103 for line in config.lines() {
1104 let Some((key, value)) = line.split_once('=') else { continue };
1105 if let Some(prefix) = key.trim().strip_suffix("localPath") {
1106 local.insert(prefix, value.trim());
1107 } else if let Some(prefix) = key.trim().strip_suffix("targetPath") {
1108 target.insert(prefix, value.trim());
1109 }
1110 }
1111 local
1112 .into_iter()
1113 .filter_map(|(prefix, root)| {
1114 let root = root.trim_end_matches('/');
1115 let rest = folder.strip_prefix(root).filter(|r| !root.is_empty() && (r.is_empty() || r.starts_with('/')))?;
1116 Some(format!("{}/{}", target.get(prefix)?.trim_matches('/'), rest.trim_matches('/')))
1117 })
1118 .collect()
1119}
1120
1121pub fn find(memory: &Path, folder: &Path, logins: &[Login], places: &[String], now: i64) -> State {
1127 find_with(memory, folder, logins, places, now, LIMITS)
1128}
1129
1130pub(crate) fn find_with(memory: &Path, folder: &Path, logins: &[Login], places: &[String], now: i64, limits: Limits) -> State {
1131 let mut state = State::load(memory);
1132 if state.mode == MIRROR {
1134 return state;
1135 }
1136 if state.folder != shown(folder) {
1137 let _ = std::fs::remove_dir_all(cache_of(memory));
1139 state = State { on: state.on, given: state.given.clone(), asked: state.asked, send: state.send, folder: shown(folder), ..State::default() };
1140 }
1141 state.looked = now;
1142 state.confirmed = 0;
1143 let Some(seal) = seal_of(folder) else {
1144 state.said = "no-seal".into();
1145 let _ = state.save(memory);
1146 return state;
1147 };
1148 let given = state.given.trim().to_string();
1149 let mut said: Vec<String> = Vec::new();
1150 let mut looked_on: Vec<String> = Vec::new();
1151 for login in logins {
1152 let host = host_of(&login.url);
1153 if given.contains("://") && host_of(&given) != host {
1155 continue;
1156 }
1157 let server = match Server::new(login, limits) {
1158 Ok(server) => server,
1159 Err(e) => {
1160 said.push(code(&e, &login.account));
1161 continue;
1162 }
1163 };
1164 let urls: Vec<String> = if given.contains("://") {
1165 vec![format!("{}/", given.trim_end_matches('/'))]
1166 } else {
1167 let root = match files_root(&server, login) {
1168 Ok(Some(root)) => root,
1169 Ok(None) => continue,
1170 Err(e) => {
1171 said.push(code(&e, &host));
1172 continue;
1173 }
1174 };
1175 let wanted: Vec<String> = if given.is_empty() { places.to_vec() } else { vec![given.trim_matches('/').to_string()] };
1176 wanted.iter().filter(|p| !p.is_empty()).map(|p| format!("{root}{}/", encode_path(p))).collect()
1177 };
1178 if !looked_on.contains(&host) {
1179 looked_on.push(host.clone());
1180 }
1181 for url in urls {
1182 match server.small(&format!("{url}seal.toml")) {
1183 Ok(Some((bytes, etag))) if bytes == seal => {
1184 confirm(&mut state, memory, login, &url, &seal, etag, now);
1185 let _ = state.save(memory);
1186 return state;
1187 }
1188 Ok(Some(_)) => said.push(format!("seal-differs:{host}")),
1189 Ok(None) => {}
1190 Err(e) => {
1191 said.push(code(&e, &host));
1192 if e.is_lasting() || matches!(e, SyncError::Network(_)) {
1194 break;
1195 }
1196 }
1197 }
1198 }
1199 }
1200 let rank = |code: &String| ["seal-differs:", "login:", "no-password:", "tls:", "network:", "server:", "disk:"].iter().position(|p| code.starts_with(p)).unwrap_or(9);
1202 said.sort_by_key(rank);
1203 state.said = match said.into_iter().next() {
1204 Some(why) => why,
1205 None if looked_on.is_empty() && given.contains("://") => format!("no-account-for:{}", host_of(&given)),
1206 None if looked_on.is_empty() => "no-account".into(),
1207 None => format!("not-found:{}", looked_on.join(", ")),
1208 };
1209 let _ = state.save(memory);
1210 state
1211}
1212
1213fn confirm(state: &mut State, memory: &Path, login: &Login, url: &str, seal: &[u8], etag: Option<String>, now: i64) {
1216 let cache = cache_of(memory);
1217 if seal_of(&cache).as_deref() != Some(seal) {
1218 let _ = std::fs::remove_dir_all(&cache);
1219 state.files.clear();
1220 state.folders.clear();
1221 }
1222 if state.url != url {
1223 state.files.clear();
1224 state.folders.clear();
1225 }
1226 let _ = crate::share::write_atomically(&cache.join("seal.toml"), seal);
1227 state.account = login.account.clone();
1228 state.url = url.to_string();
1229 state.confirmed = now;
1230 state.sealed = now;
1231 state.said.clear();
1232 if let Some(etag) = etag {
1234 state.files.insert("seal.toml".into(), Fetched { etag, size: seal.len() as u64 });
1235 }
1236}
1237
1238#[derive(Debug, Clone, Copy, Default)]
1242pub struct Pace {
1243 pub phone: bool,
1245 pub shown: bool,
1247 pub in_use: bool,
1249 pub urgent: bool,
1251 pub step: i64,
1253}
1254
1255pub fn due(state: &State, now: i64, pace: Pace) -> bool {
1265 let since = now - state.tried;
1266 if state.tried < state.confirmed || since < 0 {
1268 return true;
1269 }
1270 if pace.urgent {
1271 return since >= 20;
1272 }
1273 if !pace.phone || pace.shown || pace.in_use {
1274 return since >= 60;
1275 }
1276 since >= pace.step - 30
1277}
1278
1279fn round_cap() -> u64 {
1281 #[cfg(test)]
1282 if let Some(cap) = tests::ROUND_CAP.with(std::cell::Cell::get) {
1283 return cap;
1284 }
1285 LARGEST
1286}
1287
1288#[derive(Debug, Clone, Copy)]
1290pub struct AlarmTimes {
1291 pub alone: Duration,
1293 pub most: Duration,
1295}
1296
1297pub const ALARM_TIMES: AlarmTimes = AlarmTimes { alone: Duration::from_secs(5), most: Duration::from_secs(25) };
1299
1300#[derive(Debug, Clone, PartialEq, Eq)]
1302pub enum AlarmNews {
1303 Pulled,
1305 SyncApp { heard: BTreeSet<String> },
1313}
1314
1315#[allow(clippy::too_many_arguments)]
1329pub fn alarm_news(memory: &Path, folder: &Path, own: &str, key: [u8; 32], login: Option<Login>, lock: &'static Mutex<()>, since: i64, clock: std::sync::Arc<dyn Fn() -> i64 + Send + Sync>, ask: &mut dyn FnMut() -> Duration) -> AlarmNews {
1330 alarm_news_with(memory, folder, own, key, login, lock, since, ALARM_TIMES, clock, ask, LIMITS)
1331}
1332
1333#[allow(clippy::too_many_arguments)]
1334pub(crate) fn alarm_news_with(memory: &Path, folder: &Path, own: &str, key: [u8; 32], login: Option<Login>, lock: &'static Mutex<()>, since: i64, times: AlarmTimes, clock: std::sync::Arc<dyn Fn() -> i64 + Send + Sync>, ask: &mut dyn FnMut() -> Duration, limits: Limits) -> AlarmNews {
1335 let begun = Instant::now();
1336 let until = begun + times.most;
1337 let before = said_by_each(folder, &key, own);
1339 let mut pulling = {
1340 let (memory, folder, own, clock) = (memory.to_path_buf(), folder.to_path_buf(), own.to_string(), std::sync::Arc::clone(&clock));
1341 Some(std::thread::spawn(move || pull_since_with(&memory, &folder, &own, login.as_ref(), lock, since, begun, until, &*clock, limits)))
1342 };
1343 let mut went = None;
1344 let mut asked: Option<Instant> = None;
1345 loop {
1346 if went.is_none() && pulling.as_ref().is_some_and(std::thread::JoinHandle::is_finished) {
1347 went = pulling.take().map(|pull| pull.join().unwrap_or(false));
1348 }
1349 if went == Some(true) {
1350 return AlarmNews::Pulled;
1351 }
1352 let now = Instant::now();
1353 if asked.is_none() && (went == Some(false) || now >= begun + times.alone) {
1355 asked = Some(now + ask());
1356 }
1357 if asked.is_some_and(|end| now >= end) || now >= until {
1358 if through_since(memory, folder, since, begun, clock()) {
1360 return AlarmNews::Pulled;
1361 }
1362 return AlarmNews::SyncApp { heard: heard_since(&before, &said_by_each(folder, &key, own)) };
1363 }
1364 std::thread::sleep(Duration::from_millis(50));
1365 }
1366}
1367
1368fn through_since(memory: &Path, folder: &Path, since: i64, begun: Instant, now: i64) -> bool {
1374 let state = State::load(memory);
1375 if !state.pulls_well(folder) {
1376 return false;
1377 }
1378 match PULLED.lock().ok().and_then(|all| all.get(memory).copied()) {
1379 Some((at, last)) if last == state.last => at >= begun,
1380 _ => state.went_through_since(folder, since, now),
1381 }
1382}
1383
1384#[allow(clippy::too_many_arguments)]
1389pub(crate) fn pull_since_with(memory: &Path, folder: &Path, own: &str, login: Option<&Login>, lock: &Mutex<()>, since: i64, begun: Instant, until: Instant, clock: &dyn Fn() -> i64, limits: Limits) -> bool {
1390 let through = || through_since(memory, folder, since, begun, clock());
1391 let _one = loop {
1392 match lock.try_lock() {
1393 Ok(held) => break held,
1394 Err(std::sync::TryLockError::Poisoned(held)) => break held.into_inner(),
1395 Err(std::sync::TryLockError::WouldBlock) if Instant::now() < until => std::thread::sleep(Duration::from_millis(50)),
1396 Err(std::sync::TryLockError::WouldBlock) => return through(),
1397 }
1398 };
1399 if through() {
1400 return true;
1401 }
1402 let state = State::load(memory);
1406 if (!state.said.is_empty() && (since..=clock()).contains(&state.tried)) || Instant::now() >= until {
1407 return false;
1408 }
1409 let Some(login) = login else { return false };
1410 let _ = pull_with(memory, folder, own, login, clock(), limits);
1411 through()
1412}
1413
1414type Said = BTreeMap<String, (Option<(i64, i64, i64, i64, i64)>, Option<(i64, i64)>)>;
1419
1420pub(crate) fn said_by_each(folder: &Path, key: &[u8; 32], own: &str) -> Said {
1421 let mut out: Said = BTreeMap::new();
1422 for entry in crate::devices::all(folder, key).0.into_iter().filter(|e| e.id != own) {
1423 out.entry(entry.id.clone()).or_default().0 = Some(crate::devices::written(&entry));
1424 }
1425 for claim in crate::lease::claims(folder, key, "health").into_iter().filter(|c| c.computer != own) {
1426 out.entry(claim.computer.clone()).or_default().1 = Some((claim.renewed, claim.until));
1427 }
1428 out
1429}
1430
1431fn heard_since(before: &Said, now: &Said) -> BTreeSet<String> {
1436 now.iter()
1437 .filter(|(id, (entry, claim))| {
1438 let (was_entry, was_claim) = before.get(*id).copied().unwrap_or_default();
1439 entry.is_some_and(|e| was_entry.is_none_or(|w| e > w)) || claim.is_some_and(|c| was_claim.is_none_or(|w| c > w))
1440 })
1441 .map(|(id, _)| id.clone())
1442 .collect()
1443}
1444
1445#[derive(Debug, Default, Clone, PartialEq, Eq)]
1447pub struct Pulled {
1448 pub listed: usize,
1450 pub fetched: usize,
1452 pub sent: usize,
1454 pub same: usize,
1456 pub longer: usize,
1459 pub taken_out: usize,
1462 pub differ: Vec<String>,
1465 pub paced: usize,
1467 pub held: usize,
1470 pub failure: Option<Failure>,
1472 pub bytes: u64,
1474 pub left: usize,
1476 pub listed_bytes: u64,
1478 pub let_go: usize,
1481 pub freed: u64,
1482 pub listing_failed: bool,
1484 pub problem: Option<String>,
1486}
1487
1488enum Stop {
1490 Failed(SyncError),
1491 Unconfirmed(String),
1492}
1493
1494impl From<SyncError> for Stop {
1495 fn from(e: SyncError) -> Stop {
1496 Stop::Failed(e)
1497 }
1498}
1499
1500pub fn pull(memory: &Path, folder: &Path, own: &str, login: &Login, now: i64) -> Pulled {
1505 pull_with(memory, folder, own, login, now, LIMITS)
1506}
1507
1508pub(crate) fn pull_with(memory: &Path, folder: &Path, own: &str, login: &Login, now: i64, limits: Limits) -> Pulled {
1509 let mut state = State::load(memory);
1510 let mut pulled = Pulled::default();
1511 if state.mode == MIRROR {
1513 pulled.problem = Some("mirror".into());
1514 return pulled;
1515 }
1516 if !state.confirmed_for(folder) || state.account != login.account {
1517 pulled.problem = Some("not-confirmed".into());
1518 return pulled;
1519 }
1520 let cache = cache_of(memory);
1521 if !state.fetching() {
1523 tidy_cache(&state, folder, memory, own);
1524 pulled.problem = Some("off".into());
1525 return pulled;
1526 }
1527 if !usable(folder, memory) {
1530 let _ = std::fs::remove_dir_all(&cache);
1531 state = State { on: state.on, given: state.given.clone(), asked: state.asked, send: state.send, folder: state.folder.clone(), said: "no-seal".into(), ..State::default() };
1532 let _ = state.save(memory);
1533 pulled.problem = Some(state.said);
1534 return pulled;
1535 }
1536 let loaded_left = state.left.clone();
1537 state.tried = now;
1538 let begun = Instant::now();
1539 let host = state.host();
1540 let outcome = Server::new(login, limits).map_err(Stop::Failed).and_then(|server| {
1541 let ran = Puller { server: &server, folder, into: &cache, own, state: &mut state, pulled: &mut pulled, now, own_there: BTreeMap::new(), covered: BTreeSet::new(), lost: false, departed: Departed::load(memory, own), departed_there: Vec::new(), past: None, mine: BTreeSet::new() }.run();
1542 pulled.listed_bytes += server.listed_bytes();
1543 ran
1544 });
1545 match outcome {
1546 Ok(()) => {
1547 state.last = now;
1548 state.said.clear();
1549 }
1550 Err(Stop::Unconfirmed(why)) => {
1551 state.confirmed = 0;
1552 state.said = why;
1553 }
1554 Err(Stop::Failed(e)) => state.said = code(&e, &host),
1555 }
1556 tidy_cache(&state, folder, memory, own);
1557 if state.said.is_empty() {
1558 if let Ok(mut all) = PULLED.lock()
1560 && all.get(memory).is_none_or(|(at, _)| *at < begun)
1561 {
1562 all.insert(memory.to_path_buf(), (begun, state.last));
1563 }
1564 }
1565 let _ = state.save_pulled(memory, now + begun.elapsed().as_secs() as i64 + 1, &loaded_left);
1566 pulled.problem = (!state.said.is_empty()).then(|| state.said.clone());
1567 pulled
1568}
1569
1570static PULLED: Mutex<BTreeMap<PathBuf, (Instant, i64)>> = Mutex::new(BTreeMap::new());
1575
1576pub fn tidy(memory: &Path, folder: &Path, own: &str) {
1585 let state = State::load(memory);
1586 if state.folder == shown(folder) && state.mode != MIRROR {
1587 tidy_cache(&state, folder, memory, own);
1588 }
1589}
1590
1591fn tidy_cache(state: &State, folder: &Path, memory: &Path, own: &str) {
1592 let cache = &cache_of(memory);
1593 for dir in [cache.to_path_buf(), cache.join("devices"), cache.join("blobs")].into_iter().chain(std::fs::read_dir(cache.join("leases")).into_iter().flatten().filter_map(Result::ok).map(|e| e.path())) {
1595 crate::share::clean_leftovers(&dir);
1596 }
1597 for relative in cached_files(cache) {
1598 let (here, fetched) = (folder.join(&relative), cache.join(&relative));
1599 let name = relative.rsplit('/').next().unwrap_or_default();
1600 let gone = match kind_of(&relative) {
1601 None => continue,
1602 Some((_, id)) if id == own => true,
1603 Some((Kind::Round, _)) => std::fs::metadata(&here).is_ok_and(|h| std::fs::metadata(&fetched).is_ok_and(|f| h.len() >= f.len())),
1604 Some(_) => !state.fetching() || std::fs::read(&here).ok().is_some_and(|h| std::fs::read(&fetched).ok().is_some_and(|f| f == h)),
1605 };
1606 if gone && !name.starts_with('.') {
1607 let _ = std::fs::remove_file(&fetched);
1608 }
1609 }
1610 let wanted = crate::blobs::Wanted::load(memory);
1618 let mut aged: Vec<String> = Vec::new();
1619 for entry in std::fs::read_dir(cache.join("blobs")).into_iter().flatten().filter_map(Result::ok) {
1620 let name = entry.file_name().to_string_lossy().to_string();
1621 let Ok(meta) = entry.metadata() else { continue };
1622 if name.starts_with('.') || !meta.is_file() {
1623 continue;
1624 }
1625 let brought = std::fs::metadata(folder.join("blobs").join(&name)).is_ok_and(|here| here.len() == meta.len());
1626 let old = meta.modified().ok().and_then(|t| t.elapsed().ok()).is_some_and(|age| age.as_secs() >= WANTED_KEPT_DAYS * 86_400);
1627 if !state.fetching() || brought || old || !wanted.blobs.contains_key(&name) {
1628 let _ = std::fs::remove_file(entry.path());
1629 if old {
1630 aged.push(name);
1631 }
1632 }
1633 }
1634 if !aged.is_empty() {
1635 let _ = crate::blobs::Wanted::change(memory, |wanted| wanted.blobs.retain(|name, _| !aged.contains(name)));
1636 }
1637}
1638
1639#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1643pub(crate) enum Kind {
1644 Round,
1645 Seen,
1646 Device,
1647 Claim,
1648 Urgent,
1652}
1653
1654pub(crate) fn is_id(id: &str) -> bool {
1655 id.len() == 36 && uuid::Uuid::parse_str(id).is_ok()
1656}
1657
1658pub(crate) fn kind_of(relative: &str) -> Option<(Kind, &str)> {
1659 let parts: Vec<&str> = relative.split('/').collect();
1660 match parts.as_slice() {
1661 [name] => {
1662 if let Some((id, round)) = name.strip_suffix(".jsonl").and_then(|stem| stem.rsplit_once('-')) {
1663 return (is_id(id) && round.parse::<u32>().is_ok()).then_some((Kind::Round, id));
1664 }
1665 name.strip_suffix(".toml").filter(|id| is_id(id)).map(|id| (Kind::Seen, id))
1666 }
1667 ["devices", name] => name.strip_suffix(".device").filter(|id| is_id(id)).map(|id| (Kind::Device, id)),
1668 ["leases", _, name] => name.strip_suffix(".lease").filter(|id| !id.is_empty() && !id.starts_with('.')).map(|id| (Kind::Claim, id)),
1669 ["texts", "send", name] | ["phone-messages", "ask", name] => name.strip_suffix(".jsonl").filter(|id| is_id(id)).map(|id| (Kind::Urgent, id)),
1670 _ => None,
1671 }
1672}
1673
1674fn cached_files(cache: &Path) -> Vec<String> {
1676 let names = |dir: &Path| -> Vec<String> { std::fs::read_dir(dir).into_iter().flatten().filter_map(Result::ok).filter(|e| e.file_type().is_ok_and(|t| t.is_file())).map(|e| e.file_name().to_string_lossy().to_string()).collect() };
1677 let mut out: Vec<String> = names(cache);
1678 out.extend(names(&cache.join("devices")).into_iter().map(|n| format!("devices/{n}")));
1679 for part in std::fs::read_dir(cache.join("leases")).into_iter().flatten().filter_map(Result::ok).filter(|e| e.file_type().is_ok_and(|t| t.is_dir())) {
1680 let part = part.file_name().to_string_lossy().to_string();
1681 out.extend(names(&cache.join("leases").join(&part)).into_iter().map(|n| format!("leases/{part}/{n}")));
1682 }
1683 for (top, sub) in URGENT {
1684 out.extend(names(&cache.join(top).join(sub)).into_iter().map(|n| format!("{top}/{sub}/{n}")));
1685 }
1686 out
1687}
1688
1689const WANTED_FILES: usize = 16;
1693const WANTED_BYTES: u64 = 16 << 20;
1695pub const SMALL_WANTED: u64 = 1 << 20;
1698const WANTED_KEPT_DAYS: u64 = 3;
1701const WANTED_AGAIN: i64 = 5 * 60;
1704
1705pub fn fetch_wanted(memory: &Path, folder: &Path, login: &Login, key: &[u8; 32], now: i64, frugal: bool) -> Pulled {
1720 fetch_wanted_with(memory, folder, login, key, now, frugal, LIMITS)
1721}
1722
1723pub fn wanted_due(memory: &Path, folder: &Path, now: i64) -> bool {
1726 crate::blobs::Wanted::path(memory).exists() && !wanted_now(&crate::blobs::Wanted::load(memory), folder, &cache_of(memory).join("blobs"), now).is_empty()
1727}
1728
1729fn wanted_now<'a>(wanted: &'a crate::blobs::Wanted, folder: &Path, into: &Path, now: i64) -> Vec<(&'a String, &'a crate::blobs::Want)> {
1735 let here = |dir: &Path, name: &str| std::fs::metadata(dir.join(name)).is_ok_and(|m| m.len() > 0);
1736 let fresh = |want: &&crate::blobs::Want| now - want.since < WANTED_KEPT_DAYS as i64 * 86_400;
1737 wanted.blobs.iter().chain(wanted.media.iter().filter(|(_, want)| fresh(want))).filter(|(name, want)| want.again <= now && !here(&folder.join("blobs"), name) && !here(into, name)).collect()
1738}
1739
1740#[allow(clippy::too_many_arguments)]
1741pub(crate) fn fetch_wanted_with(memory: &Path, folder: &Path, login: &Login, key: &[u8; 32], now: i64, frugal: bool, limits: Limits) -> Pulled {
1742 let mut pulled = Pulled::default();
1743 let state = State::load(memory);
1744 if state.mode == MIRROR || !state.fetching() || !state.confirmed_for(folder) || state.account != login.account || !usable(folder, memory) {
1745 pulled.problem = Some("not-confirmed".into());
1746 return pulled;
1747 }
1748 let into = cache_of(memory).join("blobs");
1749 sioul_core::filelock::with_lock(&memory.with_file_name("wanted.step"), || {
1750 let wanted = crate::blobs::Wanted::load(memory);
1751 let mut due = wanted_now(&wanted, folder, &into, now);
1752 if due.is_empty() {
1753 return pulled;
1754 }
1755 due.sort_by_key(|(name, want)| (want.size, want.since, *name));
1756 let mut tried: BTreeMap<String, (u32, i64)> = BTreeMap::new();
1757 let mut large_at = wanted.large_at;
1758 let (mut count, mut budget) = (0, WANTED_BYTES);
1759 let outcome = Server::new(login, limits).and_then(|server| {
1760 for (name, want) in due {
1761 if count >= WANTED_FILES || (count > 0 && want.size > budget) {
1762 break;
1763 }
1764 let large = want.size >= SMALL_WANTED;
1765 if frugal && large && now - large_at < PACE {
1766 pulled.paced += 1;
1767 continue;
1768 }
1769 if frugal && large {
1770 large_at = now;
1771 }
1772 count += 1;
1773 budget = budget.saturating_sub(want.size);
1774 let path = into.join(name);
1775 let failed = |soon: bool| {
1776 let tries = want.tries + 1;
1777 (tries, now + if soon { backoff(tries).min(WANTED_AGAIN) } else { backoff(tries) })
1778 };
1779 match server.download(&format!("{}{}", state.url, encode_path(&format!("blobs/{name}"))), &path, BLOB_LIMIT) {
1780 Ok(Some(size)) => match crate::blobs::check(&path, key, &want.hash) {
1781 Ok(()) => {
1782 pulled.fetched += 1;
1783 pulled.bytes += size;
1784 tried.insert(name.clone(), (0, 0));
1785 }
1786 Err(fault) => {
1788 let _ = std::fs::remove_file(&path);
1789 pulled.held += 1;
1790 tried.insert(name.clone(), failed(!matches!(fault, crate::blobs::Fault::Broken)));
1791 }
1792 },
1793 Ok(None) => {
1795 tried.insert(name.clone(), failed(true));
1796 }
1797 Err(e) => {
1798 tried.insert(name.clone(), failed(true));
1799 return Err(e);
1800 }
1801 }
1802 }
1803 Ok(())
1804 });
1805 if let Err(e) = outcome {
1806 pulled.problem = Some(code(&e, &state.host()));
1807 }
1808 let _ = crate::blobs::Wanted::change(memory, |wanted| {
1809 for (name, (tries, again)) in tried {
1810 for want in [wanted.blobs.get_mut(&name), wanted.media.get_mut(&name)].into_iter().flatten() {
1811 (want.tries, want.again) = (tries, again);
1812 }
1813 }
1814 wanted.large_at = wanted.large_at.max(large_at);
1815 });
1816 pulled
1817 })
1818}
1819
1820struct Puller<'a> {
1822 server: &'a Server,
1823 folder: &'a Path,
1825 into: &'a Path,
1826 own: &'a str,
1827 state: &'a mut State,
1828 pulled: &'a mut Pulled,
1829 now: i64,
1830 own_there: BTreeMap<String, String>,
1832 covered: BTreeSet<String>,
1834 lost: bool,
1836 departed: Departed,
1840 departed_there: Vec<(String, String)>,
1844 past: Option<u32>,
1848 mine: BTreeSet<String>,
1852}
1853
1854impl Puller<'_> {
1855 fn url(&self, relative: &str) -> String {
1856 format!("{}{}", self.state.url, encode_path(relative))
1857 }
1858
1859 fn unchanged(&self, relative: &str, etag: &str) -> bool {
1861 !self.lost && !etag.is_empty() && self.state.folders.get(relative).is_some_and(|l| l.etag == etag && self.now - l.at < LIST_AGAIN)
1862 }
1863
1864 fn looked(&mut self, relative: &str, etag: &str) {
1866 self.state.folders.insert(relative.to_string(), Looked { etag: etag.to_string(), at: self.now });
1867 }
1868
1869 fn run(&mut self) -> Result<(), Stop> {
1870 let root = self.server.list(&self.state.url).map_err(|e| match e {
1871 SyncError::NotFound(_) => Stop::Unconfirmed(format!("seal-gone:{}", self.state.host())),
1872 e => Stop::Failed(e),
1873 })?;
1874 self.pulled.listed += root.items.len();
1875 self.seal(&root)?;
1877 self.lost = self.mirror() && own_latest(self.folder, self.own).is_none();
1880 if self.unchanged("", &root.etag) {
1881 let (folder, into) = (self.folder, self.into);
1883 let here = |name: &str| [folder.join(name), into.join(name)].iter().filter_map(|path| std::fs::metadata(path).ok()).map(|m| m.len()).max().unwrap_or(0);
1884 self.state.left.retain(|name, size| here(name) < *size);
1885 return Ok(());
1886 }
1887 self.state.left.clear();
1889 self.own_listed("", &root);
1890 for (name, item) in root.items.iter().filter(|(_, item)| !item.dir) {
1891 match kind_of(name) {
1892 Some((Kind::Round, id)) if id != self.own && (self.departed.past_round(name) || self.departed.removed_as_known(name, item, &self.state.files)) => self.away(name, item),
1896 Some((Kind::Seen, id)) if id != self.own && self.departed.removed_as_known(name, item, &self.state.files) => self.away(name, item),
1897 Some((Kind::Round, id)) if id != self.own => self.round(name, item)?,
1898 Some((Kind::Seen, id)) if id != self.own => self.small(name, item)?,
1899 Some(_) if self.mirror() => self.restore(name, item)?,
1900 _ => {}
1901 }
1902 }
1903 if let Some(devices) = root.items.get("devices").filter(|item| item.dir)
1904 && !self.unchanged("devices/", &devices.etag)
1905 {
1906 let listing = self.server.list(&self.url("devices/"))?;
1907 self.pulled.listed += listing.items.len();
1908 self.own_listed("devices/", &listing);
1909 if let Some(item) = listing.items.get(&format!("{}.device", self.own)).filter(|item| !item.dir) {
1911 self.state.own_entry = Some((item.size, self.now));
1912 }
1913 for (name, item) in listing.items.iter().filter(|(_, item)| !item.dir) {
1914 let relative = format!("devices/{name}");
1915 match kind_of(&relative) {
1916 Some((_, id)) if id != self.own && self.departed.removed_as_known(&relative, item, &self.state.files) => self.away(&relative, item),
1917 Some((_, id)) if id != self.own => self.small(&relative, item)?,
1918 Some(_) if self.mirror() => self.restore(&relative, item)?,
1919 _ => {}
1920 }
1921 }
1922 self.forget_gone("devices/", &listing);
1923 self.looked("devices/", &listing.etag);
1924 }
1925 if let Some(leases) = root.items.get("leases").filter(|item| item.dir)
1926 && !self.unchanged("leases/", &leases.etag)
1927 {
1928 let parts = self.server.list(&self.url("leases/"))?;
1929 self.pulled.listed += parts.items.len();
1930 for (part, item) in parts.items.iter().filter(|(_, item)| item.dir) {
1931 let dir = format!("leases/{part}/");
1932 if self.unchanged(&dir, &item.etag) {
1933 continue;
1934 }
1935 let listing = self.server.list(&self.url(&dir))?;
1936 self.pulled.listed += listing.items.len();
1937 self.own_listed(&dir, &listing);
1938 for (name, item) in listing.items.iter().filter(|(_, item)| !item.dir) {
1939 let relative = format!("{dir}{name}");
1940 match kind_of(&relative) {
1941 Some((_, id)) if id != self.own && self.departed.removed_as_known(&relative, item, &self.state.files) => self.away(&relative, item),
1942 Some((_, id)) if id != self.own => self.small(&relative, item)?,
1943 Some(_) if self.mirror() => self.restore(&relative, item)?,
1944 _ => {}
1945 }
1946 }
1947 self.forget_gone(&dir, &listing);
1948 self.looked(&dir, &listing.etag);
1949 }
1950 self.looked("leases/", &parts.etag);
1951 }
1952 for (top, sub) in URGENT {
1955 let (dir, files) = (format!("{top}/"), format!("{top}/{sub}/"));
1956 let Some(urgent) = root.items.get(top).filter(|item| item.dir) else { continue };
1957 if self.unchanged(&dir, &urgent.etag) {
1958 continue;
1959 }
1960 match self.server.list(&self.url(&files)) {
1961 Ok(listing) => {
1962 self.pulled.listed += listing.items.len();
1963 self.own_listed(&files, &listing);
1964 for (name, item) in listing.items.iter().filter(|(_, item)| !item.dir) {
1965 let relative = format!("{files}{name}");
1966 if let Some((Kind::Urgent, id)) = kind_of(&relative)
1967 && id != self.own
1968 {
1969 if self.departed.removed_as_known(&relative, item, &self.state.files) {
1970 self.away(&relative, item);
1971 } else {
1972 self.small(&relative, item)?;
1973 }
1974 }
1975 }
1976 self.forget_gone(&files, &listing);
1977 self.looked(&dir, &urgent.etag);
1978 }
1979 Err(SyncError::NotFound(_)) => self.looked(&dir, &urgent.etag),
1980 Err(e) => return Err(Stop::Failed(e)),
1981 }
1982 }
1983 if self.mirror() {
1990 self.state.blobs_seen = root.items.get("blobs").filter(|item| item.dir).map(|item| if item.etag.is_empty() { "-".to_string() } else { item.etag.clone() }).unwrap_or_default();
1991 }
1992 self.forget_gone("", &root);
1993 self.looked("", &root.etag);
1994 Ok(())
1995 }
1996
1997 fn away(&mut self, relative: &str, item: &Item) {
2002 if self.mirror() && !item.etag.is_empty() && !self.folder.join(relative).exists() {
2003 self.departed_there.push((relative.to_string(), item.etag.clone()));
2004 }
2005 }
2006
2007 fn mirror(&self) -> bool {
2009 self.into == self.folder
2010 }
2011
2012 fn own_listed(&mut self, dir: &str, listing: &Listing) {
2015 self.covered.insert(dir.to_string());
2016 for (name, item) in listing.items.iter().filter(|(_, item)| !item.dir) {
2017 let relative = format!("{dir}{name}");
2018 let own = match kind_of(&relative) {
2019 Some((_, id)) => id == self.own,
2020 None => dir == "blobs/" && self.state.sent.contains_key(&relative),
2021 };
2022 if own {
2023 self.own_there.insert(relative, item.etag.clone());
2024 }
2025 }
2026 }
2027
2028 fn seal(&mut self, root: &Listing) -> Result<(), Stop> {
2031 let host = self.state.host();
2032 let Some(item) = root.items.get("seal.toml").filter(|item| !item.dir) else { return Err(Stop::Unconfirmed(format!("seal-gone:{host}"))) };
2033 let known = Fetched { etag: item.etag.clone(), size: item.size };
2034 if !item.etag.is_empty() && self.state.files.get("seal.toml") == Some(&known) && self.now - self.state.sealed < SEAL_AGAIN {
2035 return Ok(());
2036 }
2037 let here = seal_of(self.folder).ok_or_else(|| Stop::Unconfirmed("no-seal".into()))?;
2038 match self.server.small(&self.url("seal.toml"))? {
2039 Some((bytes, _)) if bytes == here => {
2040 self.state.sealed = self.now;
2041 self.state.files.insert("seal.toml".into(), known);
2042 Ok(())
2043 }
2044 Some(_) => Err(Stop::Unconfirmed(format!("seal-differs:{host}"))),
2045 None => Err(Stop::Unconfirmed(format!("seal-gone:{host}"))),
2046 }
2047 }
2048
2049 fn forget_gone(&mut self, dir: &str, listing: &Listing) {
2053 let gone: Vec<String> = self.state.files.keys().filter(|path| path.strip_prefix(dir).is_some_and(|name| !name.contains('/') && !listing.items.contains_key(name))).cloned().collect();
2054 for path in gone {
2055 self.state.files.remove(&path);
2056 let round = matches!(kind_of(&path), Some((Kind::Round, _)));
2059 if path != "seal.toml" && (self.into != self.folder || round) {
2060 let _ = std::fs::remove_file(self.into.join(&path));
2061 }
2062 }
2063 }
2064
2065 fn small(&mut self, relative: &str, item: &Item) -> Result<(), Stop> {
2069 let known = Fetched { etag: item.etag.clone(), size: item.size };
2070 if !item.etag.is_empty() && self.state.files.get(relative) == Some(&known) && (!self.mirror() || self.folder.join(relative).exists()) {
2072 return Ok(());
2073 }
2074 let cap = if matches!(kind_of(relative), Some((Kind::Urgent, _))) { TEXTS_LARGEST } else { crate::share::SMALL_FILE };
2075 if item.size > cap {
2076 return Ok(());
2077 }
2078 let Some((bytes, _)) = self.server.small_up_to(&self.url(relative), cap)? else {
2079 self.state.files.remove(relative);
2080 return Ok(());
2081 };
2082 self.pulled.bytes += bytes.len() as u64;
2083 let (here, into) = (self.folder.join(relative), self.into.join(relative));
2084 if std::fs::read(&here).ok().as_deref() == Some(&bytes[..]) {
2085 if into != here {
2087 let _ = std::fs::remove_file(&into);
2088 }
2089 } else {
2090 crate::share::write_atomically(&into, &bytes).map_err(|e| Stop::Failed(SyncError::Disk(e)))?;
2091 self.pulled.fetched += 1;
2092 }
2093 self.state.files.insert(relative.to_string(), known);
2094 Ok(())
2095 }
2096
2097 fn restore(&mut self, relative: &str, item: &Item) -> Result<(), Stop> {
2102 let here = self.folder.join(relative);
2103 if here.exists() || (self.state.sent.contains_key(relative) && !self.lost) || item.size > LARGEST {
2104 return Ok(());
2105 }
2106 if matches!(kind_of(relative), Some((Kind::Round, _))) && round_of(relative).zip(self.past).is_some_and(|(round, past)| round < past) {
2108 return Ok(());
2109 }
2110 let Some(got) = self.server.tail(&self.url(relative), 0)? else { return Ok(()) };
2111 if let Some(e) = got.error {
2112 return Err(Stop::Failed(e));
2113 }
2114 let keep = if matches!(kind_of(relative), Some((Kind::Round, _))) { got.bytes.iter().rposition(|b| *b == b'\n').map_or(0, |at| at + 1) } else { got.bytes.len() };
2116 crate::share::write_atomically(&here, &got.bytes[..keep]).map_err(|e| Stop::Failed(SyncError::Disk(e)))?;
2117 let modified = std::fs::metadata(&here).map(|m| crate::share::modified_ns(&m)).unwrap_or(0);
2118 let etag = if keep == got.bytes.len() { item.etag.clone() } else { String::new() };
2119 self.state.sent.insert(relative.to_string(), Sent { size: keep as u64, modified, etag });
2120 self.pulled.fetched += 1;
2121 self.pulled.bytes += keep as u64;
2122 Ok(())
2123 }
2124
2125 fn push(&mut self) -> Result<(), Stop> {
2133 let mut stopped: Option<Stop> = None;
2134 for relative in own_files(self.folder, self.own, self.state, &self.mine) {
2135 let Ok(meta) = std::fs::metadata(self.folder.join(&relative)) else { continue };
2136 let (size, modified) = (meta.len(), crate::share::modified_ns(&meta));
2137 let sent = self.state.sent.get(&relative).cloned();
2138 let covered = self.covered.contains(dir_of(&relative));
2139 let there = self.own_there.get(&relative).cloned();
2140 let same_here = sent.as_ref().is_some_and(|s| s.size == size && s.modified == modified);
2141 if same_here
2143 && let (Some(sent), Some(etag)) = (&sent, &there)
2144 && sent.etag.is_empty()
2145 {
2146 self.state.sent.insert(relative.clone(), Sent { etag: etag.clone(), ..sent.clone() });
2147 continue;
2148 }
2149 let same_there = !covered || there.as_deref() == sent.as_ref().map(|s| s.etag.as_str());
2150 if same_here && same_there {
2151 continue;
2152 }
2153 if sent.is_none() && relative.starts_with("blobs/") && self.state.files.contains_key(&relative) {
2157 match self.server.etag(&self.url(&relative)) {
2158 Ok(Some(etag)) => {
2159 self.state.sent.insert(relative.clone(), Sent { size, modified, etag });
2160 continue;
2161 }
2162 Ok(None) => {}
2163 Err(e) => {
2164 stopped = Some(Stop::Failed(e));
2165 break;
2166 }
2167 }
2168 }
2169 let seen = if covered { there } else { sent.map(|s| s.etag).filter(|e| !e.is_empty()) };
2170 if let Err(stop) = self.send(&relative, size, modified, seen) {
2173 stopped = Some(stop);
2174 break;
2175 }
2176 }
2177 let taken = self.take_out_own();
2178 self.take_out_departed();
2179 match stopped {
2180 Some(stop) => Err(stop),
2181 None => taken,
2182 }
2183 }
2184
2185 fn take_out_own(&mut self) -> Result<(), Stop> {
2195 if let Some(latest) = own_latest(self.folder, self.own)
2196 && self.covered.contains("")
2197 {
2198 let past = self.past;
2199 let gone: Vec<(String, String)> = self
2200 .own_there
2201 .iter()
2202 .filter(|(relative, _)| matches!(kind_of(relative), Some((Kind::Round, _))) && !self.folder.join(relative).exists())
2203 .filter(|(relative, _)| round_of(relative).is_some_and(|round| round < latest && (self.state.sent.contains_key(*relative) || past.is_some_and(|past| round < past))))
2204 .map(|(relative, etag)| (relative.clone(), etag.clone()))
2205 .collect();
2206 for (relative, etag) in gone {
2207 self.take_out(&relative, Some(etag))?;
2208 }
2209 }
2210 let swept: Vec<(String, String)> = self.state.sent.iter().filter(|(relative, _)| relative.starts_with("blobs/") && !self.folder.join(relative).exists()).map(|(relative, sent)| (relative.clone(), sent.etag.clone())).collect();
2211 for (relative, etag) in swept {
2212 self.take_out(&relative, Some(etag).filter(|e| !e.is_empty()))?;
2213 }
2214 Ok(())
2215 }
2216
2217 fn take_out_departed(&mut self) {
2225 for (relative, etag) in std::mem::take(&mut self.departed_there) {
2226 if matches!(self.server.delete(&self.url(&relative), Some(&etag)), Ok(200 | 204)) {
2227 self.pulled.taken_out += 1;
2228 }
2229 }
2230 if self.departed.blobs.is_empty() || !self.covered.contains("") || self.now - self.state.swept_at < CHECK_AGAIN {
2231 return;
2232 }
2233 self.state.swept_at = self.now;
2234 for (name, at) in self.departed.blobs.clone() {
2235 let relative = format!("blobs/{name}");
2236 if !self.folder.join(&relative).exists() && take_out_swept(self.server, &self.url(&relative), at) {
2237 self.pulled.taken_out += 1;
2238 }
2239 }
2240 }
2241
2242 fn send(&mut self, relative: &str, size: u64, modified: u64, seen: Option<String>) -> Result<(), Stop> {
2249 let blob = relative.starts_with("blobs/");
2250 let bytes = std::fs::read(self.folder.join(relative)).map_err(|e| Stop::Failed(SyncError::Disk(format!("{relative}: {e}"))))?;
2251 let url = self.url(relative);
2252 let mut seen = if blob { None } else { seen };
2253 let sha1 = sha1_hex(&bytes);
2254 for attempt in 0..3 {
2255 let (status, etag) = self.server.put_file(&url, bytes.clone(), seen.as_deref(), modified / 1_000_000_000, &sha1).map_err(|failed| Stop::Failed(failed.error))?;
2257 match status {
2258 200 | 201 | 204 => {
2259 self.state.sent.insert(relative.to_string(), Sent { size, modified, etag: etag.unwrap_or_default() });
2260 self.pulled.sent += 1;
2261 self.pulled.bytes += bytes.len() as u64;
2262 return Ok(());
2263 }
2264 412 if blob => {
2265 self.state.sent.insert(relative.to_string(), Sent { size, modified, etag: String::new() });
2266 return Ok(());
2267 }
2268 412 => seen = self.server.etag(&url)?,
2269 409 if attempt == 0 => self.make_folders(relative)?,
2270 507 => return Err(Stop::Failed(SyncError::Server("quota".into()))),
2271 status => return Err(Stop::Failed(SyncError::Server(format!("PUT: {status}")))),
2272 }
2273 }
2274 Err(Stop::Failed(SyncError::Server("PUT: 412".into())))
2275 }
2276
2277 fn make_folders(&mut self, relative: &str) -> Result<(), Stop> {
2279 let mut at = String::new();
2280 let names: Vec<&str> = relative.split('/').collect();
2281 for name in &names[..names.len().saturating_sub(1)] {
2282 at = format!("{at}{name}/");
2283 match self.server.mkcol(&self.url(&at))? {
2284 201 | 405 => {}
2285 status => return Err(Stop::Failed(SyncError::Server(format!("MKCOL: {status}")))),
2286 }
2287 }
2288 Ok(())
2289 }
2290
2291 fn take_out(&mut self, relative: &str, seen: Option<String>) -> Result<(), Stop> {
2295 match self.server.delete(&self.url(relative), seen.as_deref())? {
2296 200 | 204 | 404 | 412 => {
2297 self.state.sent.remove(relative);
2298 Ok(())
2299 }
2300 status => Err(Stop::Failed(SyncError::Server(format!("DELETE: {status}")))),
2301 }
2302 }
2303
2304 fn round(&mut self, name: &str, item: &Item) -> Result<(), Stop> {
2310 let (here, into) = (self.folder.join(name), self.into.join(name));
2311 let size = |path: &Path| std::fs::metadata(path).map_or(0, |m| m.len());
2312 let longest = size(&here).max(size(&into));
2313 let known = Fetched { etag: item.etag.clone(), size: item.size };
2314 if item.size <= longest {
2315 self.state.files.insert(name.to_string(), known);
2317 return Ok(());
2318 }
2319 if item.size > round_cap() {
2320 self.state.left.insert(name.to_string(), item.size);
2322 self.pulled.left += 1;
2323 return Ok(());
2324 }
2325 let base = if size(&into) >= size(&here) { into.clone() } else { here.clone() };
2326 let (end, last) = whole_end(&base);
2327 let from = if end - last > OVERLAP { end.saturating_sub(1) } else { last };
2328 let overlap = read_range(&base, from, end).unwrap_or_default();
2329 let url = self.url(name);
2330 let Some(mut got) = self.server.tail(&url, from)? else { return Ok(()) };
2331 let joins = got.start == from && from > 0;
2332 if joins && !got.bytes.starts_with(&overlap) {
2333 if got.error.is_some() && overlap.starts_with(&got.bytes) {
2334 return Err(Stop::Failed(got.error.take().unwrap_or(SyncError::Network(String::new()))));
2336 }
2337 match self.server.tail(&url, 0)? {
2339 Some(whole) => got = whole,
2340 None => return Ok(()),
2341 }
2342 } else if !joins && got.start != 0 {
2343 return Err(Stop::Failed(SyncError::Server(format!("GET: from {}", got.start))));
2344 }
2345 let prefix = if got.start == from && from > 0 { from } else { 0 };
2346 self.pulled.bytes += got.bytes.len() as u64;
2347 let whole = got.bytes.iter().rposition(|b| *b == b'\n').map_or(0, |at| at + 1);
2349 if prefix + whole as u64 > longest {
2350 let temporary = crate::share::temporary(&into);
2351 let written = (|| -> std::io::Result<()> {
2352 if let Some(parent) = into.parent() {
2353 crate::share::private_dirs(parent)?;
2354 }
2355 let mut out = crate::share::new_private(&temporary)?;
2356 if prefix > 0 {
2357 std::io::copy(&mut std::fs::File::open(&base)?.take(prefix), &mut out)?;
2358 }
2359 out.write_all(&got.bytes[..whole])?;
2360 out.sync_all()?;
2361 std::fs::rename(&temporary, &into)
2362 })();
2363 if let Err(e) = written {
2364 let _ = std::fs::remove_file(&temporary);
2365 return Err(Stop::Failed(SyncError::Disk(format!("{name}: {e}"))));
2366 }
2367 self.pulled.fetched += 1;
2368 }
2369 match got.error {
2370 Some(e) => Err(Stop::Failed(e)),
2371 None => {
2372 self.state.files.insert(name.to_string(), known);
2373 Ok(())
2374 }
2375 }
2376 }
2377}
2378
2379fn whole_end(path: &Path) -> (u64, u64) {
2381 let Ok(mut file) = std::fs::File::open(path) else { return (0, 0) };
2382 let size = file.metadata().map_or(0, |m| m.len());
2383 let Some(end) = newline_before(&mut file, size).map(|at| at + 1) else { return (0, 0) };
2384 let last = newline_before(&mut file, end - 1).map_or(0, |at| at + 1);
2385 (end, last)
2386}
2387
2388fn newline_before(file: &mut std::fs::File, before: u64) -> Option<u64> {
2390 let mut end = before;
2391 while end > 0 {
2392 let from = end.saturating_sub(64 << 10);
2393 let mut piece = vec![0u8; usize::try_from(end - from).ok()?];
2394 file.seek(SeekFrom::Start(from)).ok()?;
2395 file.read_exact(&mut piece).ok()?;
2396 if let Some(at) = piece.iter().rposition(|b| *b == b'\n') {
2397 return Some(from + at as u64);
2398 }
2399 end = from;
2400 }
2401 None
2402}
2403
2404fn read_range(path: &Path, from: u64, to: u64) -> Option<Vec<u8>> {
2405 let mut file = std::fs::File::open(path).ok()?;
2406 file.seek(SeekFrom::Start(from)).ok()?;
2407 let mut bytes = vec![0u8; usize::try_from(to.saturating_sub(from)).ok()?];
2408 file.read_exact(&mut bytes).ok()?;
2409 Some(bytes)
2410}
2411
2412pub const MIRROR: &str = "mirror";
2419
2420pub fn mirror_of(memory: &Path) -> PathBuf {
2422 memory.with_file_name("mirror")
2423}
2424
2425const BLOB_LIMIT: u64 = crate::blobs::LARGEST + (1 << 20);
2427
2428pub struct Opened {
2432 pub url: String,
2433 root: String,
2434 pub place: String,
2435 pub seal: Option<Vec<u8>>,
2436}
2437
2438pub fn open(login: &Login, place: &str) -> Result<Opened, String> {
2441 open_with(login, place, LIMITS)
2442}
2443
2444pub(crate) fn open_with(login: &Login, place: &str, limits: Limits) -> Result<Opened, String> {
2445 let host = host_of(&login.url);
2446 let place = place.trim().trim_matches('/').to_string();
2447 if place.is_empty() {
2448 return Err("no-place".into());
2449 }
2450 let server = Server::new(login, limits).map_err(|e| code(&e, &login.account))?;
2451 let root = files_root(&server, login).map_err(|e| code(&e, &host))?.ok_or_else(|| format!("no-files:{host}"))?;
2452 let url = format!("{root}{}/", encode_path(&place));
2453 let seal = server.small(&format!("{url}seal.toml")).map_err(|e| code(&e, &host))?.map(|(bytes, _)| bytes);
2454 Ok(Opened { url, root, place, seal })
2455}
2456
2457pub fn create(login: &Login, opened: &Opened, folder: &Path) -> Result<(), String> {
2461 create_with(login, opened, folder, LIMITS)
2462}
2463
2464pub(crate) fn create_with(login: &Login, opened: &Opened, folder: &Path, limits: Limits) -> Result<(), String> {
2465 let host = host_of(&login.url);
2466 let failed = |e: SyncError| code(&e, &host);
2467 let server = Server::new(login, limits).map_err(failed)?;
2468 let mut at = opened.root.clone();
2469 for name in opened.place.split('/').filter(|n| !n.is_empty()) {
2470 at = format!("{at}{}/", encode_path(name));
2471 if !matches!(server.mkcol(&at).map_err(failed)?, 201 | 405) {
2472 return Err(format!("server:{host}"));
2473 }
2474 }
2475 let seal = seal_of(folder).ok_or_else(|| "no-seal".to_string())?;
2476 match server.put(&format!("{}seal.toml", opened.url), seal, None).map_err(failed)?.0 {
2477 200 | 201 | 204 => Ok(()),
2478 412 => Err("sealed-meanwhile".into()),
2479 _ => Err(format!("server:{host}")),
2480 }
2481}
2482
2483pub fn begin(memory: &Path, folder: &Path, login: &Login, opened: &Opened, now: i64) -> Result<(), String> {
2486 begin_as(memory, folder, login, opened, now, String::new())
2487}
2488
2489pub fn begin_signed_in(memory: &Path, folder: &Path, login: &Login, opened: &Opened, now: i64) -> Result<(), String> {
2495 begin_as(memory, folder, login, opened, now, login.user.clone())
2496}
2497
2498fn begin_as(memory: &Path, folder: &Path, login: &Login, opened: &Opened, now: i64, user: String) -> Result<(), String> {
2499 let _ = std::fs::remove_dir_all(cache_of(memory));
2500 let state = State { mode: MIRROR.into(), place: opened.place.clone(), folder: shown(folder), account: login.account.clone(), url: opened.url.clone(), confirmed: now, sealed: now, user, ..State::default() };
2501 let path = state_path(memory);
2502 sioul_core::filelock::with_lock(&path, || state.write(memory))
2503}
2504
2505pub const SOUGHT: [&str; 2] = ["Documents/Sioul", "Sioul"];
2508
2509const SOUGHT_UNDER: usize = 40;
2511
2512#[derive(Debug, Default, Clone, PartialEq, Eq)]
2518pub struct Sought {
2519 pub found: Vec<String>,
2520 pub complete: bool,
2521}
2522
2523pub fn sought(place: &str) -> bool {
2526 let place = place.trim().trim_matches('/');
2527 SOUGHT.contains(&place) || place.strip_prefix("Documents/").is_some_and(|name| !name.is_empty() && !name.contains('/') && !name.starts_with('.'))
2528}
2529
2530pub fn seek(login: &Login) -> Result<Sought, String> {
2542 seek_with(login, LIMITS)
2543}
2544
2545pub(crate) fn seek_with(login: &Login, limits: Limits) -> Result<Sought, String> {
2546 let host = host_of(&login.url);
2547 let server = Server::new(login, limits).map_err(|e| code(&e, &host))?;
2548 let root = files_root(&server, login).map_err(|e| code(&e, &host))?.ok_or_else(|| format!("no-files:{host}"))?;
2549 let mut sought = Sought { found: Vec::new(), complete: true };
2550 let out_of_time = |e: &SyncError| matches!(e, SyncError::Network(d) if d == "out of time");
2551 let look = |place: &str, sought: &mut Sought| -> Result<(), SyncError> {
2553 match server.small(&format!("{root}{}/seal.toml", encode_path(place))) {
2554 Ok(Some((bytes, _))) if crate::share::is_seal(&bytes) => sought.found.push(place.to_string()),
2555 Ok(_) => {}
2556 Err(e) if e.is_lasting() || out_of_time(&e) => return Err(e),
2557 Err(_) => sought.complete = false,
2558 }
2559 Ok(())
2560 };
2561 let ended = |e: SyncError, mut sought: Sought| -> Result<Sought, String> {
2563 if sought.found.is_empty() && e.is_lasting() {
2564 return Err(code(&e, &host));
2565 }
2566 sought.complete = false;
2567 Ok(sought)
2568 };
2569 for place in SOUGHT {
2570 if let Err(e) = look(place, &mut sought) {
2571 return ended(e, sought);
2572 }
2573 }
2574 let extra: Vec<String> = match server.list(&format!("{root}Documents/")) {
2575 Ok(listing) => listing.items.iter().filter(|(name, item)| item.dir && !name.starts_with('.') && name.as_str() != "Sioul").take(SOUGHT_UNDER).map(|(name, _)| format!("Documents/{name}")).collect(),
2576 Err(SyncError::NotFound(_)) => Vec::new(),
2577 Err(e) if e.is_lasting() || out_of_time(&e) => return ended(e, sought),
2578 Err(_) => {
2579 sought.complete = false;
2580 return Ok(sought);
2581 }
2582 };
2583 for place in extra {
2584 if let Err(e) = look(&place, &mut sought) {
2585 return ended(e, sought);
2586 }
2587 }
2588 Ok(sought)
2589}
2590
2591pub fn step(memory: &Path, folder: &Path, own: &str, login: &Login, now: i64, whole: bool) -> Pulled {
2602 step_with(memory, folder, own, login, now, whole, LIMITS)
2603}
2604
2605pub(crate) fn step_with(memory: &Path, folder: &Path, own: &str, login: &Login, now: i64, whole: bool, limits: Limits) -> Pulled {
2606 sioul_core::filelock::with_lock(&memory.with_file_name("mirror.step"), || {
2607 let mut state = State::load(memory);
2608 let mut pulled = Pulled::default();
2609 if state.mode != MIRROR || !state.confirmed_for(folder) || state.account != login.account {
2610 pulled.problem = Some("not-confirmed".into());
2611 return pulled;
2612 }
2613 let host = state.host();
2614 if whole {
2615 state.tried = now;
2616 }
2617 let mine: BTreeSet<String> = crate::share::own_sealed(memory, own).into_keys().collect();
2618 let outcome = Server::new(login, limits).map_err(Stop::Failed).and_then(|server| {
2619 let mut puller = Puller { server: &server, folder, into: folder, own, state: &mut state, pulled: &mut pulled, now, own_there: BTreeMap::new(), covered: BTreeSet::new(), lost: false, departed: Departed::load(memory, own), departed_there: Vec::new(), past: crate::share::rounds_past(folder, memory, own, now * 1000), mine };
2620 let ran = (|| {
2621 if whole {
2622 puller.run()?;
2623 }
2624 puller.push()
2628 })();
2629 puller.pulled.listed_bytes += server.listed_bytes();
2630 ran
2631 });
2632 match outcome {
2633 Ok(()) => {
2634 if whole {
2635 state.last = now;
2636 }
2637 state.pushed = now;
2638 state.said.clear();
2639 }
2640 Err(Stop::Unconfirmed(why)) => {
2641 state.confirmed = 0;
2642 state.said = why;
2643 }
2644 Err(Stop::Failed(e)) => state.said = code(&e, &host),
2645 }
2646 let _ = state.save(memory);
2647 pulled.problem = (!state.said.is_empty()).then(|| state.said.clone());
2648 pulled
2649 })
2650}
2651
2652pub fn sealed_due(memory: &Path, folder: &Path, own: &str, now: i64) -> bool {
2658 let state = State::load(memory);
2659 if state.mode != MIRROR || !state.confirmed_for(folder) {
2660 return false;
2661 }
2662 if listing_due(&state, now) {
2663 return true;
2664 }
2665 let wanted = crate::blobs::Wanted::load(memory);
2666 (!wanted.blobs.is_empty() || !wanted.media.is_empty()) && !wanted_in_copy(&wanted, folder, &Departed::load(memory, own), now).is_empty()
2667}
2668
2669fn listing_due(state: &State, now: i64) -> bool {
2674 if state.blobs_seen.is_empty() || now < state.blobs_again {
2675 return false;
2676 }
2677 let last = state.folders.get("blobs/");
2678 let changed = last.is_none_or(|looked| looked.etag != state.blobs_seen);
2679 let since = last.map_or(i64::MAX, |looked| if now < looked.at { i64::MAX } else { now - looked.at });
2681 (changed && since >= BLOBS_AGAIN) || since >= CHECK_AGAIN
2682}
2683
2684fn wanted_in_copy(wanted: &crate::blobs::Wanted, folder: &Path, departed: &Departed, now: i64) -> Vec<(String, crate::blobs::Want)> {
2688 let blobs = folder.join("blobs");
2689 let here = |name: &str| std::fs::metadata(blobs.join(name)).is_ok_and(|m| m.len() > 0);
2690 let fresh = |want: &&crate::blobs::Want| now - want.since < WANTED_KEPT_DAYS as i64 * 86_400;
2694 let due: BTreeMap<&String, &crate::blobs::Want> = wanted.blobs.iter().chain(wanted.media.iter().filter(|(_, want)| fresh(want))).filter(|(name, want)| want.again <= now && !here(name) && !departed.swept(name)).collect();
2695 due.into_iter().map(|(name, want)| (name.clone(), want.clone())).collect()
2696}
2697
2698pub fn sealed_step(memory: &Path, folder: &Path, own: &str, login: &Login, key: &[u8; 32], now: i64, frugal: bool) -> Pulled {
2715 sealed_step_with(memory, folder, own, login, key, now, frugal, LIMITS)
2716}
2717
2718#[allow(clippy::too_many_arguments)]
2719pub(crate) fn sealed_step_with(memory: &Path, folder: &Path, own: &str, login: &Login, key: &[u8; 32], now: i64, frugal: bool, limits: Limits) -> Pulled {
2720 sioul_core::filelock::with_lock(&memory.with_file_name("mirror.sealed"), || {
2721 let mut pulled = Pulled::default();
2722 let state = State::load(memory);
2723 if state.mode != MIRROR || !state.confirmed_for(folder) || state.account != login.account {
2724 pulled.problem = Some("not-confirmed".into());
2725 return pulled;
2726 }
2727 let server = match Server::new(login, limits) {
2728 Ok(server) => server,
2729 Err(e) => {
2730 pulled.problem = Some(code(&e, &state.host()));
2731 return pulled;
2732 }
2733 };
2734 let url = |relative: &str| format!("{}{}", state.url, encode_path(relative));
2735 if listing_due(&state, now) {
2736 let listed = server.list_within(limits.large, &url("blobs/"));
2738 let before = |map: Vec<&String>| -> BTreeSet<String> { map.into_iter().filter(|relative| relative.starts_with("blobs/")).cloned().collect() };
2739 let (sent_before, fetched_before) = (before(state.sent.keys().collect()), before(state.files.keys().collect()));
2740 let etag = state.blobs_seen.clone();
2741 in_step(memory, folder, &login.account, |state| match &listed {
2742 Ok(listing) => {
2743 let keep: Option<BTreeSet<String>> = crate::share::needed_sealed(memory, own).map(|needed| needed.into_iter().chain(crate::share::own_sealed(memory, own).into_keys()).collect());
2744 blobs_looked(state, folder, listing, keep.as_ref(), (&sent_before, &fetched_before), &mut pulled);
2745 state.folders.insert("blobs/".into(), Looked { etag, at: now });
2746 (state.blobs_tries, state.blobs_again) = (0, 0);
2747 }
2748 Err(_) => {
2750 state.blobs_tries += 1;
2751 state.blobs_again = now + backoff(state.blobs_tries).max(BLOBS_AGAIN);
2752 pulled.listing_failed = true;
2753 }
2754 });
2755 }
2756 if let Err(e) = fetch_wanted_into_copy(&server, memory, folder, (own, &login.account), (key, frugal), now, &url, &mut pulled) {
2757 pulled.problem = Some(code(&e, &state.host()));
2758 }
2759 pulled.listed_bytes = server.listed_bytes();
2760 pulled
2761 })
2762}
2763
2764fn in_step<T>(memory: &Path, folder: &Path, account: &str, change: impl FnOnce(&mut State) -> T) -> Option<T> {
2768 sioul_core::filelock::with_lock(&memory.with_file_name("mirror.step"), || {
2769 let mut state = State::load(memory);
2770 if state.mode != MIRROR || !state.confirmed_for(folder) || state.account != account {
2771 return None;
2772 }
2773 let out = change(&mut state);
2774 let _ = state.save(memory);
2775 Some(out)
2776 })
2777}
2778
2779fn blobs_looked(state: &mut State, folder: &Path, listing: &Listing, keep: Option<&BTreeSet<String>>, (sent_before, fetched_before): (&BTreeSet<String>, &BTreeSet<String>), pulled: &mut Pulled) {
2789 pulled.listed += listing.items.len();
2790 let listed: BTreeSet<&str> = listing.items.iter().filter(|(_, item)| !item.dir).map(|(name, _)| name.as_str()).collect();
2791 let gone_there = |relative: &str| relative.strip_prefix("blobs/").is_some_and(|name| !listed.contains(name));
2792 state.sent.retain(|relative, _| !(sent_before.contains(relative) && gone_there(relative)));
2793 let Some(keep) = keep else { return };
2794 let gone: Vec<String> = state.files.keys().filter(|relative| fetched_before.contains(*relative) && gone_there(relative) && relative.strip_prefix("blobs/").is_some_and(|name| !keep.contains(name))).cloned().collect();
2795 for relative in gone {
2796 state.files.remove(&relative);
2797 let path = folder.join(&relative);
2798 if let Ok(meta) = std::fs::metadata(&path)
2799 && std::fs::remove_file(&path).is_ok()
2800 {
2801 pulled.let_go += 1;
2802 pulled.freed += meta.len();
2803 }
2804 }
2805}
2806
2807#[allow(clippy::too_many_arguments)]
2818fn fetch_wanted_into_copy(server: &Server, memory: &Path, folder: &Path, (own, account): (&str, &str), (key, frugal): (&[u8; 32], bool), now: i64, url: &dyn Fn(&str) -> String, pulled: &mut Pulled) -> Result<(), SyncError> {
2819 let wanted = crate::blobs::Wanted::load(memory);
2820 let mut due = wanted_in_copy(&wanted, folder, &Departed::load(memory, own), now);
2821 if due.is_empty() {
2822 return Ok(());
2823 }
2824 due.sort_by(|a, b| (a.1.size, a.1.since, &a.0).cmp(&(b.1.size, b.1.since, &b.0)));
2825 let (mut tried, mut budget, mut first, mut large_at) = (BTreeMap::new(), MIRROR_WANTED_BYTES, true, wanted.large_at);
2826 let (mut stopped, mut ready) = (None, Vec::new());
2827 let place = |ready: &mut Vec<(String, PathBuf, u64)>, pulled: &mut Pulled| {
2828 for ((_, temporary, size), placed) in ready.iter().zip(place_fetched(memory, folder, account, ready)) {
2829 if placed {
2830 pulled.fetched += 1;
2831 pulled.bytes += size;
2832 }
2833 let _ = std::fs::remove_file(temporary);
2834 }
2835 ready.clear();
2836 };
2837 for (name, want) in due {
2838 if !first && want.size > budget {
2839 break;
2840 }
2841 let large = want.size >= SMALL_WANTED;
2842 if frugal && large && now - large_at < PACE {
2843 pulled.paced += 1;
2844 continue;
2845 }
2846 if frugal && large {
2847 large_at = now;
2848 }
2849 first = false;
2850 let failed = |soon: bool| {
2851 let tries = want.tries + 1;
2852 let soon = soon && now - want.since < 3600;
2853 (tries, now + if soon { backoff(tries).min(WANTED_AGAIN) } else { backoff(tries) })
2854 };
2855 let relative = format!("blobs/{name}");
2856 match server.download_aside(&url(&relative), &folder.join(&relative), BLOB_LIMIT) {
2857 Ok(Some((size, temporary))) => match crate::blobs::check(&temporary, key, &want.hash) {
2858 Ok(()) => {
2859 budget = budget.saturating_sub(size);
2860 tried.insert(name, (0, 0));
2861 ready.push((relative, temporary, size));
2862 if ready.len() >= PLACED_AT_ONCE {
2863 place(&mut ready, pulled);
2864 }
2865 }
2866 Err(fault) => {
2868 let _ = std::fs::remove_file(&temporary);
2869 pulled.held += 1;
2870 tried.insert(name, failed(!matches!(fault, crate::blobs::Fault::Broken)));
2871 }
2872 },
2873 Ok(None) => {
2875 tried.insert(name, failed(true));
2876 }
2877 Err(SyncError::Network(why)) if why == "out of time" => break,
2879 Err(e) => {
2880 tried.insert(name, failed(true));
2881 stopped = Some(e);
2882 break;
2883 }
2884 }
2885 }
2886 place(&mut ready, pulled);
2887 let _ = crate::blobs::Wanted::change(memory, |wanted| {
2888 for (name, (tries, again)) in tried {
2889 for want in [wanted.blobs.get_mut(&name), wanted.media.get_mut(&name)].into_iter().flatten() {
2890 (want.tries, want.again) = (tries, again);
2891 }
2892 }
2893 wanted.large_at = wanted.large_at.max(large_at);
2894 });
2895 stopped.map_or(Ok(()), Err)
2896}
2897
2898const PLACED_AT_ONCE: usize = 64;
2903
2904fn place_fetched(memory: &Path, folder: &Path, account: &str, ready: &[(String, PathBuf, u64)]) -> Vec<bool> {
2911 if ready.is_empty() {
2912 return Vec::new();
2913 }
2914 sioul_core::filelock::with_lock(&memory.with_file_name("mirror.step"), || {
2915 let mut state = State::load(memory);
2916 if state.mode != MIRROR || !state.confirmed_for(folder) || state.account != account {
2917 return vec![false; ready.len()];
2918 }
2919 let free: Vec<bool> = ready.iter().map(|(relative, _, _)| !folder.join(relative).exists()).collect();
2920 for ((relative, _, size), free) in ready.iter().zip(&free) {
2921 if *free {
2922 state.files.insert(relative.clone(), Fetched { etag: String::new(), size: *size });
2923 }
2924 }
2925 if state.save(memory).is_err() {
2926 return vec![false; ready.len()];
2927 }
2928 let placed: Vec<bool> = ready.iter().zip(&free).map(|((relative, temporary, _), free)| *free && std::fs::rename(temporary, folder.join(relative)).is_ok()).collect();
2929 if placed != free {
2931 for ((relative, _, _), (free, placed)) in ready.iter().zip(free.iter().zip(&placed)) {
2932 if *free && !placed {
2933 state.files.remove(relative);
2934 }
2935 }
2936 let _ = state.save(memory);
2937 }
2938 placed
2939 })
2940}
2941
2942fn dir_of(relative: &str) -> &str {
2944 relative.rfind('/').map_or("", |at| &relative[..=at])
2945}
2946
2947fn own_files(folder: &Path, own: &str, state: &State, mine: &BTreeSet<String>) -> Vec<String> {
2955 let mut out = own_plain(folder, own);
2956 out.extend(file_names(&folder.join("blobs")).into_iter().filter(|n| !state.files.contains_key(&format!("blobs/{n}")) || mine.contains(n)).map(|n| format!("blobs/{n}")));
2958 out
2959}
2960
2961fn file_names(dir: &Path) -> Vec<String> {
2963 std::fs::read_dir(dir).into_iter().flatten().filter_map(Result::ok).filter(|e| e.file_type().is_ok_and(|t| t.is_file())).map(|e| e.file_name().to_string_lossy().to_string()).filter(|n| !n.starts_with('.')).collect()
2964}
2965
2966fn own_plain(folder: &Path, own: &str) -> Vec<String> {
2969 let names = file_names;
2970 let mut out: Vec<String> = Vec::new();
2971 let mut records: Vec<(u8, u32, String)> = names(folder)
2972 .into_iter()
2973 .filter_map(|name| match kind_of(&name) {
2974 Some((Kind::Round, id)) if id == own => Some((0, name.trim_end_matches(".jsonl").rsplit('-').next().and_then(|n| n.parse().ok()).unwrap_or(0), name)),
2975 Some((Kind::Seen, id)) if id == own => Some((1, 0, name)),
2976 _ => None,
2977 })
2978 .collect();
2979 records.sort();
2980 out.extend(records.into_iter().map(|(_, _, name)| name));
2981 let mut parts: Vec<String> = std::fs::read_dir(folder.join("leases")).into_iter().flatten().filter_map(Result::ok).filter(|e| e.file_type().is_ok_and(|t| t.is_dir())).map(|e| e.file_name().to_string_lossy().to_string()).collect();
2982 parts.sort();
2983 out.extend(parts.into_iter().map(|part| format!("leases/{part}/{own}.lease")).filter(|claim| folder.join(claim).is_file()));
2984 let entry = format!("devices/{own}.device");
2985 if folder.join(&entry).is_file() {
2986 out.push(entry);
2987 }
2988 out
2989}
2990
2991fn own_latest(folder: &Path, own: &str) -> Option<u32> {
2993 std::fs::read_dir(folder)
2994 .into_iter()
2995 .flatten()
2996 .filter_map(Result::ok)
2997 .filter_map(|e| {
2998 let name = e.file_name().to_string_lossy().to_string();
2999 match kind_of(&name) {
3000 Some((Kind::Round, id)) if id == own => name.trim_end_matches(".jsonl").rsplit('-').next().and_then(|n| n.parse().ok()),
3001 _ => None,
3002 }
3003 })
3004 .max()
3005}
3006
3007const CHECK_AGAIN: i64 = 6 * 3600;
3013pub const LARGE_FILE: u64 = 256 << 10;
3016const PACE: i64 = 10 * 60;
3017pub const SLOW: u64 = 64 << 10;
3020
3021pub fn backoff(tries: u32) -> i64 {
3024 [60, 120, 300, 900, 1800, 3600][tries.saturating_sub(1).min(5) as usize]
3025}
3026
3027#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3031enum Lane {
3032 Main,
3034 Urgent,
3038}
3039
3040impl Lane {
3041 fn memory(self, memory: &Path) -> PathBuf {
3042 memory.with_file_name(match self {
3043 Lane::Main => "sending.toml",
3044 Lane::Urgent => "sending-urgent.toml",
3045 })
3046 }
3047
3048 fn lock(self, memory: &Path) -> PathBuf {
3049 memory.with_file_name(match self {
3050 Lane::Main => "send.step",
3051 Lane::Urgent => "send.urgent",
3052 })
3053 }
3054}
3055
3056#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
3061pub struct Sending {
3062 #[serde(default, skip_serializing_if = "String::is_empty")]
3064 pub url: String,
3065 #[serde(default)]
3067 pub last: i64,
3068 #[serde(default)]
3069 pub tried: i64,
3070 #[serde(default)]
3072 pub checked: i64,
3073 #[serde(default, skip_serializing_if = "String::is_empty")]
3075 pub said: String,
3076 #[serde(default, skip_serializing_if = "Option::is_none")]
3078 pub failure: Option<Failure>,
3079 #[serde(default)]
3083 pub tries: u32,
3084 #[serde(default)]
3085 pub retry_at: i64,
3086 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
3089 pub held: BTreeMap<String, Held>,
3090 #[serde(default)]
3093 pub rate: u64,
3094 #[serde(default)]
3096 pub large_at: i64,
3097 #[serde(default)]
3099 pub again: Again,
3100 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
3102 pub sent: BTreeMap<String, Sent>,
3103 #[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
3111 pub refused: BTreeSet<String>,
3112}
3113
3114#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
3116pub struct Failure {
3117 #[serde(default)]
3120 pub kind: String,
3121 #[serde(default)]
3123 pub file: String,
3124 #[serde(default)]
3125 pub seconds: u64,
3126 #[serde(default)]
3127 pub sent: u64,
3128 #[serde(default)]
3129 pub total: u64,
3130 #[serde(default, skip_serializing_if = "String::is_empty")]
3132 pub words: String,
3133 #[serde(default)]
3135 pub at: i64,
3136}
3137
3138#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
3140pub struct Held {
3141 #[serde(default)]
3142 pub tries: u32,
3143 #[serde(default)]
3144 pub until: i64,
3145}
3146
3147#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
3151pub struct Again {
3152 #[serde(default)]
3153 pub at: i64,
3154 #[serde(default)]
3155 pub sent: usize,
3156 #[serde(default)]
3157 pub same: usize,
3158 #[serde(default, skip_serializing_if = "String::is_empty")]
3159 pub said: String,
3160}
3161
3162#[cfg(test)]
3163fn sending_path(memory: &Path) -> PathBuf {
3164 Lane::Main.memory(memory)
3165}
3166
3167impl Sending {
3168 pub fn load(memory: &Path) -> Sending {
3169 Sending::load_lane(memory, Lane::Main)
3170 }
3171
3172 pub fn load_urgent(memory: &Path) -> Sending {
3174 Sending::load_lane(memory, Lane::Urgent)
3175 }
3176
3177 fn load_lane(memory: &Path, lane: Lane) -> Sending {
3178 std::fs::read_to_string(lane.memory(memory)).ok().and_then(|text| toml::from_str(&text).ok()).unwrap_or_default()
3179 }
3180
3181 fn save(&self, memory: &Path, lane: Lane) -> Result<(), String> {
3182 let text = toml::to_string(self).map_err(|e| e.to_string())?;
3183 crate::share::write_atomically(&lane.memory(memory), format!("# Sioul: this device's files sent to the sharing folder's server beside the sync app (docs/database.md). Never shared.\n{text}").as_bytes())
3184 }
3185
3186 pub fn note_again(memory: &Path, now: i64, why: &str) {
3189 let mut sending = Sending::load(memory);
3190 sending.again = Again { at: now, sent: 0, same: 0, said: why.to_string() };
3191 let _ = sending.save(memory, Lane::Main);
3192 }
3193
3194 pub fn slow(&self) -> bool {
3196 self.rate > 0 && self.rate < SLOW
3197 }
3198
3199 pub fn next_try(&self) -> i64 {
3202 self.held.values().map(|h| h.until).chain([self.retry_at]).filter(|at| *at > 0).min().unwrap_or(0)
3203 }
3204}
3205
3206#[allow(clippy::too_many_arguments)]
3224pub fn send(memory: &Path, folder: &Path, own: &str, login: &Login, sealed: &BTreeMap<String, u64>, now: i64, again: bool, frugal: bool) -> Pulled {
3225 send_lane(Lane::Main, memory, folder, own, login, sealed, now, again, frugal, SEND_LIMITS)
3226}
3227
3228pub fn send_urgent(memory: &Path, folder: &Path, own: &str, login: &Login, now: i64, media: &BTreeMap<String, u64>) -> Pulled {
3237 send_lane(Lane::Urgent, memory, folder, own, login, media, now, false, false, SEND_LIMITS)
3238}
3239
3240#[cfg(test)]
3241pub(crate) fn send_urgent_with(memory: &Path, folder: &Path, own: &str, login: &Login, now: i64, limits: Limits) -> Pulled {
3242 send_lane(Lane::Urgent, memory, folder, own, login, &BTreeMap::new(), now, false, false, limits)
3243}
3244
3245#[cfg(test)]
3246pub(crate) fn send_urgent_media(memory: &Path, folder: &Path, own: &str, login: &Login, now: i64, media: &BTreeMap<String, u64>, limits: Limits) -> Pulled {
3247 send_lane(Lane::Urgent, memory, folder, own, login, media, now, false, false, limits)
3248}
3249
3250#[cfg(test)]
3251#[allow(clippy::too_many_arguments)]
3252pub(crate) fn send_with(memory: &Path, folder: &Path, own: &str, login: &Login, sealed: &BTreeMap<String, u64>, now: i64, again: bool, limits: Limits) -> Pulled {
3253 send_lane(Lane::Main, memory, folder, own, login, sealed, now, again, false, limits)
3254}
3255
3256#[allow(clippy::too_many_arguments)]
3257fn send_lane(lane: Lane, memory: &Path, folder: &Path, own: &str, login: &Login, sealed: &BTreeMap<String, u64>, now: i64, again: bool, frugal: bool, limits: Limits) -> Pulled {
3258 sioul_core::filelock::with_lock(&lane.lock(memory), || {
3259 let state = State::load(memory);
3260 let mut pulled = Pulled::default();
3261 let refused = if state.mode == MIRROR && lane == Lane::Main {
3262 Some("mirror")
3263 } else if !state.confirmed_for(folder) || state.account != login.account {
3264 Some("not-confirmed")
3265 } else if state.mode != MIRROR && !state.sending() {
3266 Some("off")
3267 } else {
3268 None
3269 };
3270 if let Some(why) = refused {
3271 if again {
3272 Sending::note_again(memory, now, why);
3273 }
3274 pulled.problem = Some(why.into());
3275 return pulled;
3276 }
3277 let mut sending = Sending::load_lane(memory, lane);
3278 if sending.url != state.url {
3279 sending = Sending { url: state.url.clone(), again: sending.again.clone(), ..Sending::default() };
3280 }
3281 let entry = format!("devices/{own}.device");
3287 if lane == Lane::Main
3288 && let Some((size, at)) = state.own_entry
3289 && at > sending.last
3290 && sending.sent.get(&entry).is_some_and(|sent| sent.size != size)
3291 {
3292 sending.sent.remove(&entry);
3293 }
3294 let host = state.host();
3295 if !again && now < sending.retry_at {
3297 pulled.problem = Some(format!("waiting:{host}"));
3298 return pulled;
3299 }
3300 sending.tried = now;
3301 let check = again || sending.checked == 0 || now - sending.checked >= CHECK_AGAIN;
3302 let frugal = frugal || sending.slow();
3303 let files: Vec<String> = match lane {
3304 Lane::Urgent => {
3306 let mut names: Vec<&String> = sealed.keys().collect();
3307 names.sort();
3308 let mut files: Vec<String> = names.into_iter().map(|name| format!("blobs/{name}")).filter(|relative| folder.join(relative).is_file()).collect();
3309 files.extend(own_urgent(folder, own));
3310 files
3311 }
3312 Lane::Main => {
3313 let mut files = own_plain(folder, own);
3314 let mut names: Vec<&String> = sealed.keys().collect();
3315 names.sort();
3316 files.extend(names.into_iter().map(|name| format!("blobs/{name}")).filter(|relative| folder.join(relative).is_file()));
3317 files
3318 }
3319 };
3320 let past = if lane == Lane::Main && check { crate::share::rounds_past(folder, memory, own, now * 1000) } else { None };
3322 let departed = if lane == Lane::Main && check { Departed::load(memory, own) } else { Departed::default() };
3324 let outcome = Server::new(login, limits).map_err(Stop::Failed).and_then(|server| {
3325 let ran = Sender { server: &server, folder, memory, own, url: &state.url, sealed, sending: &mut sending, pulled: &mut pulled, now, frugal, lane, past, departed, known: &state.files, unlisted: BTreeSet::new() }.run(&files, check, again);
3326 pulled.listed_bytes += server.listed_bytes();
3327 ran
3328 });
3329 match outcome {
3330 Ok(()) => {
3331 sending.tries = 0;
3332 sending.retry_at = 0;
3333 if sending.held.is_empty() {
3334 sending.last = now;
3335 sending.said.clear();
3336 sending.failure = None;
3337 } else if let Some(failure) = &sending.failure {
3338 sending.said = format!("network:{host}");
3339 pulled.failure = Some(failure.clone());
3340 }
3341 if check {
3342 sending.checked = now;
3343 }
3344 }
3345 Err(Stop::Unconfirmed(why)) => sending.said = why,
3346 Err(Stop::Failed(e)) => {
3347 sending.tries += 1;
3349 sending.retry_at = now + backoff(sending.tries);
3350 sending.said = code(&e, &host);
3351 if sending.failure.as_ref().is_none_or(|f| f.at != now) {
3352 sending.failure = Some(Failure { kind: kind_of_error(&e).into(), file: String::new(), words: e.to_string(), at: now, ..Failure::default() });
3353 }
3354 pulled.failure = sending.failure.clone();
3355 }
3356 }
3357 if again {
3358 sending.again = Again { at: now, sent: pulled.sent, same: pulled.same, said: sending.said.clone() };
3359 }
3360 if lane == Lane::Urgent {
3362 sending.sent.retain(|relative, _| relative.strip_prefix("blobs/").is_none_or(|name| sealed.contains_key(name)));
3363 }
3364 let _ = sending.save(memory, lane);
3365 pulled.problem = (!sending.said.is_empty()).then(|| sending.said.clone());
3366 pulled
3367 })
3368}
3369
3370fn kind_of_error(e: &SyncError) -> &'static str {
3372 match e {
3373 SyncError::Login(_) | SyncError::AppPassword(_) => "login",
3374 SyncError::Tls(_) => "tls",
3375 SyncError::Server(d) if d == "quota" => "quota",
3376 SyncError::Server(_) => "server",
3377 SyncError::Disk(_) => "disk",
3378 SyncError::Network(d) if d == "out of time" => "out-of-time",
3379 _ => "unreachable",
3380 }
3381}
3382
3383pub fn again_mirror(memory: &Path, folder: &Path, own: &str, login: &Login, now: i64) -> Pulled {
3389 again_mirror_with(memory, folder, own, login, now, LIMITS)
3390}
3391
3392pub(crate) fn again_mirror_with(memory: &Path, folder: &Path, own: &str, login: &Login, now: i64, limits: Limits) -> Pulled {
3393 let forgot = sioul_core::filelock::with_lock(&memory.with_file_name("mirror.step"), || {
3394 let mut state = State::load(memory);
3395 if state.mode != MIRROR {
3396 return false;
3397 }
3398 state.sent.retain(|relative, _| relative.starts_with("blobs/"));
3399 state.folders.clear();
3400 state.save(memory).is_ok()
3401 });
3402 let pulled = if forgot { step_with(memory, folder, own, login, now, true, limits) } else { Pulled { problem: Some("not-confirmed".into()), ..Pulled::default() } };
3403 let mut sending = Sending::load(memory);
3404 sending.again = Again { at: now, sent: pulled.sent, same: 0, said: pulled.problem.clone().unwrap_or_default() };
3405 let _ = sending.save(memory, Lane::Main);
3406 pulled
3407}
3408
3409pub fn place_texts(folder: &Path, own: &str, source: &Path) -> Result<bool, String> {
3416 place_urgent(&texts_file(folder, own), source)
3417}
3418
3419pub fn place_asks(folder: &Path, own: &str, source: &Path) -> Result<bool, String> {
3424 place_urgent(&asks_file(folder, own), source)
3425}
3426
3427fn place_urgent(target: &Path, source: &Path) -> Result<bool, String> {
3429 let target = target.to_path_buf();
3430 let Ok(bytes) = std::fs::read(source) else {
3431 return Ok(std::fs::remove_file(&target).is_ok());
3432 };
3433 if std::fs::read(&target).ok().as_deref() == Some(&bytes[..]) {
3434 return Ok(false);
3435 }
3436 if let Some(parent) = target.parent() {
3437 crate::share::private_dirs(parent).map_err(|e| format!("{}: {e}", parent.display()))?;
3438 }
3439 crate::share::write_atomically(&target, &bytes).map(|()| true)
3440}
3441
3442pub fn texts_file(folder: &Path, device: &str) -> PathBuf {
3444 folder.join("texts").join("send").join(format!("{device}.jsonl"))
3445}
3446
3447pub fn asks_file(folder: &Path, device: &str) -> PathBuf {
3449 folder.join("phone-messages").join("ask").join(format!("{device}.jsonl"))
3450}
3451
3452fn own_urgent(folder: &Path, own: &str) -> Vec<String> {
3454 URGENT.iter().map(|(top, sub)| format!("{top}/{sub}/{own}.jsonl")).filter(|relative| folder.join(relative).is_file()).collect()
3455}
3456
3457fn file_kind(relative: &str) -> &'static str {
3459 match kind_of(relative) {
3460 Some((Kind::Round, _)) => "records",
3461 Some((Kind::Seen, _)) => "notes",
3462 Some((Kind::Device, _)) => "entry",
3463 Some((Kind::Claim, _)) => "claim",
3464 Some((Kind::Urgent, _)) if relative.starts_with("phone-messages/") => "replies",
3465 Some((Kind::Urgent, _)) => "texts",
3466 None => "sealed",
3467 }
3468}
3469
3470#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3473enum Went {
3474 Done,
3475 Left,
3476 Held,
3477}
3478
3479struct Sender<'a> {
3481 server: &'a Server,
3482 folder: &'a Path,
3483 memory: &'a Path,
3485 own: &'a str,
3486 url: &'a str,
3488 sealed: &'a BTreeMap<String, u64>,
3489 sending: &'a mut Sending,
3490 pulled: &'a mut Pulled,
3491 now: i64,
3492 frugal: bool,
3493 lane: Lane,
3494 past: Option<u32>,
3497 departed: Departed,
3500 known: &'a BTreeMap<String, Fetched>,
3503 unlisted: BTreeSet<String>,
3505}
3506
3507#[derive(Debug, Default, Clone)]
3517pub(crate) struct Departed {
3518 rounds: BTreeMap<String, u32>,
3519 blobs: BTreeMap<String, i64>,
3520 devices: BTreeMap<String, crate::share::RemovedDevice>,
3528}
3529
3530impl Departed {
3531 fn load(memory: &Path, own: &str) -> Departed {
3532 Departed { rounds: crate::share::departed_past(memory, own), blobs: crate::share::swept_theirs(memory, own), devices: crate::share::removed_devices(memory, own) }
3533 }
3534
3535 fn removed_as_known(&self, relative: &str, item: &Item, known: &BTreeMap<String, Fetched>) -> bool {
3542 let Some((kind, id)) = kind_of(relative) else { return false };
3543 let Some(removed) = self.devices.get(id) else { return false };
3544 let went = match kind {
3545 Kind::Round => return round_of(relative).is_some_and(|round| removed.holds_round(round, item.size)),
3546 Kind::Device => removed.entry,
3547 Kind::Seen => removed.notes,
3548 Kind::Claim | Kind::Urgent => true,
3549 };
3550 went && !item.etag.is_empty() && known.get(relative).is_some_and(|seen| seen.etag == item.etag)
3551 }
3552
3553 fn past_round(&self, relative: &str) -> bool {
3555 match kind_of(relative) {
3556 Some((Kind::Round, id)) => self.rounds.get(id).is_some_and(|below| round_of(relative).is_some_and(|round| round < *below)),
3557 _ => false,
3558 }
3559 }
3560
3561 fn swept(&self, name: &str) -> bool {
3563 self.blobs.contains_key(name)
3564 }
3565}
3566
3567fn round_of(relative: &str) -> Option<u32> {
3569 relative.trim_end_matches(".jsonl").rsplit('-').next().and_then(|n| n.parse::<u32>().ok())
3570}
3571
3572fn take_out_swept(server: &Server, url: &str, swept_ms: i64) -> bool {
3579 let Ok(Some(there)) = server.stat(url) else { return false };
3580 let old = there.modified.is_some_and(|at| at <= swept_ms / 1000 - crate::blobs::UNUSED_DAYS * 86_400);
3581 if there.dir || there.etag.is_empty() || !old {
3582 return false;
3583 }
3584 matches!(server.delete(url, Some(&there.etag)), Ok(200 | 204))
3585}
3586
3587impl Sender<'_> {
3588 fn url(&self, relative: &str) -> String {
3589 format!("{}{}", self.url, encode_path(relative))
3590 }
3591
3592 fn run(&mut self, files: &[String], check: bool, again: bool) -> Result<(), Stop> {
3597 let listed = if check { Some(self.list_own()?) } else { None };
3598 let unlisted = std::mem::take(&mut self.unlisted);
3601 let mut records_held = false;
3602 for relative in files {
3603 let records = matches!(kind_of(relative), Some((Kind::Round, _)));
3604 if records && records_held {
3605 continue;
3606 }
3607 let there = listed.as_ref().filter(|_| !unlisted.contains(dir_of(relative))).map(|listed| listed.get(relative).cloned());
3608 let look = again && listed.is_some() && unlisted.contains(dir_of(relative));
3610 let went = self.one(relative, there, again, look)?;
3611 records_held |= records && went == Went::Held;
3612 }
3613 if self.lane == Lane::Main {
3614 self.take_out_gone(check)?;
3615 if let (Some(listed), Some(past)) = (&listed, self.past) {
3616 self.take_out_past(listed, past)?;
3617 }
3618 if let Some(listed) = &listed {
3619 self.take_out_departed(listed);
3620 }
3621 }
3622 Ok(())
3623 }
3624
3625 fn list_own(&mut self) -> Result<BTreeMap<String, Item>, Stop> {
3629 let mut out = BTreeMap::new();
3630 let root = self.server.list(self.url).map_err(|e| match e {
3631 SyncError::NotFound(_) => Stop::Unconfirmed(format!("seal-gone:{}", host_of(self.url))),
3632 e => Stop::Failed(e),
3633 })?;
3634 self.pulled.listed += root.items.len();
3635 let mut dirs: Vec<String> = Vec::new();
3636 for (name, item) in &root.items {
3637 if item.dir {
3638 if name == "devices" || (name == "blobs" && !self.sealed.is_empty() && self.lane == Lane::Main) {
3639 dirs.push(format!("{name}/"));
3640 }
3641 if name == "blobs" && !self.sealed.is_empty() && self.lane == Lane::Urgent {
3643 self.unlisted.insert("blobs/".into());
3644 }
3645 if let Some((top, sub)) = URGENT.iter().find(|(top, _)| name == top) {
3646 dirs.push(format!("{top}/{sub}/"));
3647 }
3648 if name == "leases" {
3649 match self.server.list(&self.url("leases/")) {
3650 Ok(parts) => dirs.extend(parts.items.iter().filter(|(_, item)| item.dir).map(|(part, _)| format!("leases/{part}/"))),
3651 Err(SyncError::NotFound(_)) => {}
3652 Err(e) => return Err(Stop::Failed(e)),
3653 }
3654 }
3655 } else {
3656 out.insert(name.clone(), item.clone());
3657 }
3658 }
3659 for dir in dirs {
3660 let listing = if dir == "blobs/" { self.server.list_within(self.server.limits.large, &self.url(&dir)) } else { self.server.list(&self.url(&dir)) };
3662 match listing {
3663 Ok(listing) => {
3664 self.pulled.listed += listing.items.len();
3665 out.extend(listing.items.into_iter().filter(|(_, item)| !item.dir).map(|(name, item)| (format!("{dir}{name}"), item)));
3666 }
3667 Err(SyncError::NotFound(_)) => {}
3668 Err(_) if dir == "blobs/" => {
3674 self.pulled.listing_failed = true;
3675 self.unlisted.insert(dir);
3676 }
3677 Err(e) => return Err(Stop::Failed(e)),
3678 }
3679 }
3680 Ok(out)
3681 }
3682
3683 fn same(&self, relative: &str, there: &Item, bytes: &[u8], sha1: &str) -> Result<bool, Stop> {
3688 if there.dir || there.size != bytes.len() as u64 {
3689 return Ok(false);
3690 }
3691 if relative.starts_with("blobs/") {
3692 return Ok(true);
3693 }
3694 if let Some(theirs) = sha1_of(&there.checksum) {
3695 return Ok(theirs == sha1);
3696 }
3697 if matches!(kind_of(relative), Some((Kind::Round, _))) {
3698 return Ok(true);
3699 }
3700 Ok(self.server.small_up_to(&self.url(relative), TEXTS_LARGEST)?.is_some_and(|(theirs, _)| theirs == bytes))
3701 }
3702
3703 fn one(&mut self, relative: &str, listed: Option<Option<Item>>, again: bool, look: bool) -> Result<Went, Stop> {
3713 let path = self.folder.join(relative);
3714 let Ok(meta) = std::fs::metadata(&path) else { return Ok(Went::Left) };
3715 let (size, modified) = (meta.len(), crate::share::modified_ns(&meta));
3716 let blob = relative.strip_prefix("blobs/");
3717 if blob.is_some_and(|name| self.sealed.get(name) != Some(&size)) {
3719 return Ok(Went::Left);
3720 }
3721 let records = matches!(kind_of(relative), Some((Kind::Round, _)));
3722 let sent = self.sending.sent.get(relative).cloned();
3723 let same_here = sent.as_ref().is_some_and(|s| s.size == size && s.modified == modified);
3724 match &listed {
3725 None if same_here && !look => return Ok(Went::Left),
3726 Some(there) if same_here && !again && there.as_ref().is_some_and(|t| t.etag == sent.as_ref().map_or("", |s| s.etag.as_str())) => return Ok(Went::Left),
3727 _ => {}
3728 }
3729 if !again && self.sending.held.get(relative).is_some_and(|held| self.now < held.until) {
3731 return Ok(Went::Held);
3732 }
3733 if records && sent.as_ref().is_some_and(|s| size < s.size) {
3736 self.pulled.longer += 1;
3737 return Ok(Went::Left);
3738 }
3739 let large = size > LARGE_FILE;
3741 if large && self.frugal && !again && self.now - self.sending.large_at < PACE {
3742 self.pulled.paced += 1;
3743 return Ok(if records { Went::Held } else { Went::Left });
3744 }
3745 let bytes = std::fs::read(&path).map_err(|e| Stop::Failed(SyncError::Disk(format!("{relative}: {e}"))))?;
3746 let size = bytes.len() as u64;
3748 let sha1 = sha1_hex(&bytes);
3749 let url = self.url(relative);
3750 let mut there: Option<Option<Item>> = listed;
3752 if there.is_none() && (look || sent.as_ref().is_none_or(|s| s.etag.is_empty())) {
3753 there = Some(self.server.stat(&url)?);
3754 }
3755 let modified_s = modified / 1_000_000_000;
3756 for attempt in 0..3 {
3757 if let Some(Some(item)) = &there {
3758 if self.same(relative, item, &bytes, &sha1)? {
3759 self.sending.sent.insert(relative.to_string(), Sent { size, modified, etag: item.etag.clone() });
3760 self.sending.held.remove(relative);
3761 self.pulled.same += 1;
3762 return Ok(Went::Done);
3763 }
3764 if self.pulled.differ.len() < 8 {
3766 self.pulled.differ.push(format!("{} {size} here, {} there", file_kind(relative), item.size));
3767 }
3768 if records && item.size > size {
3772 self.sending.sent.insert(relative.to_string(), Sent { size: item.size, modified, etag: item.etag.clone() });
3773 self.pulled.longer += 1;
3774 return Ok(Went::Left);
3775 }
3776 }
3777 let seen: Option<String> = match &there {
3778 Some(Some(item)) => Some(item.etag.clone()),
3779 Some(None) => None,
3780 None => sent.as_ref().map(|s| s.etag.clone()),
3781 };
3782 if large {
3783 self.sending.large_at = self.now;
3784 }
3785 let started = Instant::now();
3786 let (status, etag) = match self.server.put_file(&url, bytes.clone(), seen.as_deref(), modified_s, &sha1) {
3787 Ok(answer) => answer,
3788 Err(failed) => return self.failed(relative, failed),
3789 };
3790 match status {
3791 200 | 201 | 204 => {
3792 self.sending.sent.insert(relative.to_string(), Sent { size, modified, etag: etag.unwrap_or_default() });
3793 self.sending.held.remove(relative);
3794 self.pulled.sent += 1;
3795 self.pulled.bytes += size;
3796 if size >= 64 << 10 {
3797 self.sending.rate = size * 1000 / (started.elapsed().as_millis() as u64).max(1);
3798 }
3799 return Ok(Went::Done);
3800 }
3801 412 | 404 => there = Some(self.server.stat(&url)?),
3803 409 if attempt == 0 => self.make_folders(relative)?,
3804 507 => return Err(Stop::Failed(SyncError::Server("quota".into()))),
3805 status => return Err(Stop::Failed(SyncError::Server(format!("PUT: {status}")))),
3806 }
3807 }
3808 Err(Stop::Failed(SyncError::Server("PUT: 412".into())))
3809 }
3810
3811 fn failed(&mut self, relative: &str, failed: PutFailed) -> Result<Went, Stop> {
3816 self.sending.failure = Some(Failure { kind: failed.kind.into(), file: file_kind(relative).into(), seconds: failed.seconds, sent: failed.sent, total: failed.total, words: failed.words.clone(), at: self.now });
3817 if failed.sent > 0 && failed.seconds > 0 {
3818 self.sending.rate = failed.sent / failed.seconds;
3819 }
3820 if matches!(failed.kind, "timeout" | "stalled" | "network") && failed.sent > 0 {
3821 let held = self.sending.held.entry(relative.to_string()).or_default();
3822 held.tries += 1;
3823 held.until = self.now + backoff(held.tries);
3824 self.pulled.held += 1;
3825 return Ok(Went::Held);
3826 }
3827 Err(Stop::Failed(failed.error))
3828 }
3829
3830 fn make_folders(&mut self, relative: &str) -> Result<(), Stop> {
3832 let mut at = String::new();
3833 let names: Vec<&str> = relative.split('/').collect();
3834 for name in &names[..names.len().saturating_sub(1)] {
3835 at = format!("{at}{name}/");
3836 match self.server.mkcol(&self.url(&at))? {
3837 201 | 405 => {}
3838 status => return Err(Stop::Failed(SyncError::Server(format!("MKCOL: {status}")))),
3839 }
3840 }
3841 Ok(())
3842 }
3843
3844 fn take_out_gone(&mut self, check: bool) -> Result<(), Stop> {
3860 let latest = own_latest(self.folder, self.own);
3861 let gone: Vec<(String, Sent)> = self.sending.sent.iter().filter(|(relative, _)| !self.folder.join(relative).exists()).map(|(relative, sent)| (relative.clone(), sent.clone())).collect();
3862 let (folder, Sending { refused, sent, .. }) = (self.folder, &mut *self.sending);
3864 refused.retain(|relative| sent.contains_key(relative) && !folder.join(relative).exists());
3865 let rounds = gone.iter().any(|(relative, _)| matches!(kind_of(relative), Some((Kind::Round, _))));
3871 let floor = if rounds { crate::share::rounds_past(self.folder, self.memory, self.own, self.now * 1000) } else { None };
3872 for (relative, sent) in gone {
3873 let round = relative.trim_end_matches(".jsonl").rsplit('-').next().and_then(|n| n.parse::<u32>().ok());
3874 let past = matches!(kind_of(&relative), Some((Kind::Round, _))) && round.zip(latest).is_some_and(|(round, latest)| round < latest) && round.zip(floor).is_some_and(|(round, floor)| round < floor);
3875 let swept = relative.strip_prefix("blobs/").is_some_and(|name| !self.sealed.contains_key(name));
3876 if swept && !check && self.sending.refused.contains(&relative) {
3877 continue;
3878 }
3879 let mut settled = true;
3880 if past || swept {
3881 settled = match self.server.delete(&self.url(&relative), Some(&sent.etag).filter(|e| !e.is_empty()).map(String::as_str))? {
3882 200 | 204 => {
3883 self.pulled.taken_out += 1;
3884 true
3885 }
3886 412 => self.again_after_412(&relative, &sent, past),
3887 404 => true,
3889 _ => false,
3891 };
3892 }
3893 if swept && !settled {
3894 self.sending.refused.insert(relative);
3895 } else if past || swept || !matches!(kind_of(&relative), Some((Kind::Round, _))) {
3896 self.sending.sent.remove(&relative);
3897 self.sending.held.remove(&relative);
3898 self.sending.refused.remove(&relative);
3899 }
3900 }
3901 Ok(())
3902 }
3903
3904 fn again_after_412(&mut self, relative: &str, sent: &Sent, records: bool) -> bool {
3924 let url = self.url(relative);
3925 let there = match self.server.stat(&url) {
3926 Ok(Some(there)) => there,
3927 Ok(None) => return true,
3928 Err(_) => return false,
3929 };
3930 let dated = records || there.modified == Some((sent.modified / 1_000_000_000) as i64);
3931 if there.dir || there.etag.is_empty() || there.size != sent.size || !dated {
3932 return true;
3933 }
3934 match self.server.delete(&url, Some(&there.etag)) {
3935 Ok(200 | 204) => {
3936 self.pulled.taken_out += 1;
3937 true
3938 }
3939 Ok(404) => true,
3940 _ => false,
3941 }
3942 }
3943
3944 fn take_out_past(&mut self, listed: &BTreeMap<String, Item>, past: u32) -> Result<(), Stop> {
3954 if own_latest(self.folder, self.own).is_none() {
3955 return Ok(());
3956 }
3957 let old: Vec<(String, String)> = listed
3958 .iter()
3959 .filter(|(relative, item)| !item.dir && !item.etag.is_empty() && matches!(kind_of(relative), Some((Kind::Round, id)) if id == self.own))
3960 .filter(|(relative, _)| relative.trim_end_matches(".jsonl").rsplit('-').next().and_then(|n| n.parse::<u32>().ok()).is_some_and(|round| round < past))
3961 .filter(|(relative, _)| !self.folder.join(relative).exists())
3962 .map(|(relative, item)| (relative.clone(), item.etag.clone()))
3963 .collect();
3964 for (relative, etag) in old {
3965 if matches!(self.server.delete(&self.url(&relative), Some(&etag))?, 200 | 204) {
3966 self.pulled.taken_out += 1;
3967 self.sending.sent.remove(&relative);
3968 self.sending.held.remove(&relative);
3969 }
3970 }
3971 Ok(())
3972 }
3973
3974 fn take_out_departed(&mut self, listed: &BTreeMap<String, Item>) {
3983 if self.departed.rounds.is_empty() && self.departed.blobs.is_empty() && self.departed.devices.is_empty() {
3984 return;
3985 }
3986 let fetched = overlay(self.folder);
3987 let here = |relative: &str| self.folder.join(relative).exists() || fetched.as_ref().is_some_and(|f| f.join(relative).exists());
3988 let rounds: Vec<(String, String)> = listed.iter().filter(|(relative, item)| !item.dir && !item.etag.is_empty() && (self.departed.past_round(relative) || self.departed.removed_as_known(relative, item, self.known)) && !here(relative)).map(|(relative, item)| (relative.clone(), item.etag.clone())).collect();
3989 let blobs: Vec<(String, i64)> = self.departed.blobs.iter().map(|(name, at)| (format!("blobs/{name}"), *at)).filter(|(relative, _)| !here(relative)).collect();
3990 for (relative, etag) in rounds {
3991 if matches!(self.server.delete(&self.url(&relative), Some(&etag)), Ok(200 | 204)) {
3992 self.pulled.taken_out += 1;
3993 }
3994 }
3995 for (relative, at) in blobs {
3996 if take_out_swept(self.server, &self.url(&relative), at) {
3997 self.pulled.taken_out += 1;
3998 }
3999 }
4000 }
4001}
4002
4003#[cfg(test)]
4004pub(crate) mod fake {
4005 use std::collections::hash_map::DefaultHasher;
4016 use std::hash::{Hash, Hasher};
4017 use std::io::{BufRead, BufReader, Read, Write};
4018 use std::net::{TcpListener, TcpStream};
4019 use std::path::{Path, PathBuf};
4020 use std::sync::{Arc, Mutex};
4021 use std::time::Duration;
4022
4023 pub(crate) const USER: &str = "jane";
4024 pub(crate) const PASSWORD: &str = "correct horse battery staple";
4025 const FILES: &str = "/remote.php/dav/files/jane";
4026 pub(crate) const APP_PASSWORD: &str = "stand-in-app-password-";
4028
4029 #[derive(Debug, Clone, Default)]
4032 pub(crate) struct Flow {
4033 pub(crate) token: String,
4034 pub(crate) granted: bool,
4035 pub(crate) answered: bool,
4036 }
4037
4038 #[derive(Debug, Clone, Default)]
4041 pub(crate) struct Signing {
4042 pub(crate) endpoint: String,
4043 pub(crate) server: String,
4044 pub(crate) login: String,
4045 }
4046
4047 #[derive(Debug, Clone, Copy)]
4049 pub(crate) enum Fault {
4050 Status(u16),
4052 Late(Duration),
4054 Cut(usize),
4056 Drop,
4058 }
4059
4060 pub(crate) struct Request {
4062 pub method: String,
4063 pub path: String,
4064 headers: Vec<(String, String)>,
4065 pub body: Vec<u8>,
4066 }
4067
4068 impl Request {
4069 pub fn header(&self, name: &str) -> Option<&str> {
4070 self.headers.iter().find(|(n, _)| n.eq_ignore_ascii_case(name)).map(|(_, v)| v.as_str())
4071 }
4072 }
4073
4074 type Faults = Vec<Box<dyn FnMut(&Request) -> Option<Fault> + Send>>;
4075 pub(crate) type Reply = (u16, Vec<(&'static str, String)>, Vec<u8>);
4076
4077 pub(crate) struct Fake {
4078 pub(crate) files: PathBuf,
4080 pub(crate) base: String,
4081 sums: Mutex<std::collections::BTreeMap<PathBuf, (String, String)>>,
4085 pub(crate) throttle: Mutex<Option<u64>>,
4087 pub(crate) received: std::sync::atomic::AtomicU64,
4089 faults: Mutex<Faults>,
4090 seen: Mutex<Vec<String>>,
4091 pub(crate) turn: Mutex<()>,
4093 pub(crate) flows: Mutex<Vec<Flow>>,
4095 pub(crate) signing: Mutex<Signing>,
4096 pub(crate) app_passwords: Mutex<Vec<String>>,
4098 }
4099
4100 impl Fake {
4101 pub(crate) fn start(files: &Path) -> Arc<Fake> {
4102 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
4103 let base = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port());
4104 let fake = Arc::new(Fake {
4105 files: files.to_path_buf(),
4106 base,
4107 sums: Mutex::new(std::collections::BTreeMap::new()),
4108 throttle: Mutex::new(None),
4109 received: std::sync::atomic::AtomicU64::new(0),
4110 faults: Mutex::new(Vec::new()),
4111 seen: Mutex::new(Vec::new()),
4112 turn: Mutex::new(()),
4113 flows: Mutex::new(Vec::new()),
4114 signing: Mutex::new(Signing::default()),
4115 app_passwords: Mutex::new(Vec::new()),
4116 });
4117 let serving = Arc::clone(&fake);
4118 std::thread::spawn(move || {
4119 for stream in listener.incoming().filter_map(Result::ok) {
4120 let fake = Arc::clone(&serving);
4121 std::thread::spawn(move || fake.exchange(stream));
4122 }
4123 });
4124 fake
4125 }
4126
4127 pub(crate) fn login(&self) -> super::Login {
4129 super::Login { account: "cloud".into(), url: format!("{}/remote.php/dav/calendars/{USER}/", self.base), user: USER.into(), password: Some(PASSWORD.into()) }
4130 }
4131
4132 pub(crate) fn folder_url(&self, path: &str) -> String {
4134 format!("{}{FILES}/{}/", self.base, path.trim_matches('/'))
4135 }
4136
4137 pub(crate) fn fault(&self, fault: impl FnMut(&Request) -> Option<Fault> + Send + 'static) {
4139 self.faults.lock().unwrap().push(Box::new(fault));
4140 }
4141
4142 pub(crate) fn clear_faults(&self) {
4143 self.faults.lock().unwrap().clear();
4144 }
4145
4146 pub(crate) fn seen(&self) -> Vec<String> {
4148 self.seen.lock().unwrap().clone()
4149 }
4150
4151 pub(crate) fn forget_seen(&self) {
4152 self.seen.lock().unwrap().clear();
4153 }
4154
4155 fn exchange(&self, mut stream: TcpStream) {
4156 let throttle = *self.throttle.lock().unwrap();
4157 let Some(request) = read(&stream, throttle, &self.received) else { return };
4158 let fault = self.faults.lock().unwrap().iter_mut().find_map(|f| f(&request));
4159 let (status, headers, body) = match fault {
4160 Some(Fault::Drop) => {
4161 self.note(&request, "dropped");
4162 return;
4163 }
4164 Some(Fault::Status(status)) => (status, Vec::new(), Vec::new()),
4165 Some(Fault::Late(wait)) => {
4166 std::thread::sleep(wait);
4167 self.answer(&request)
4168 }
4169 _ => self.answer(&request),
4170 };
4171 self.note(&request, &status.to_string());
4172 let mut head = format!("HTTP/1.1 {status} Fake\r\nContent-Length: {}\r\nConnection: close\r\n", body.len());
4173 for (name, value) in &headers {
4174 head.push_str(&format!("{name}: {value}\r\n"));
4175 }
4176 head.push_str("\r\n");
4177 if stream.write_all(head.as_bytes()).is_err() {
4178 return;
4179 }
4180 let sent = match fault {
4181 Some(Fault::Cut(n)) => &body[..n.min(body.len())],
4182 _ => &body[..],
4183 };
4184 let _ = stream.write_all(sent);
4185 let _ = stream.flush();
4186 }
4187
4188 fn sum_of(&self, path: &Path) -> Option<String> {
4190 let sums = self.sums.lock().unwrap();
4191 sums.get(path).filter(|(etag, _)| *etag == etag_of(path)).map(|(_, sum)| sum.clone())
4192 }
4193
4194 fn propfind(&self, request: &Request, relative: &str, path: &Path) -> Reply {
4195 let Ok(meta) = std::fs::metadata(path) else { return (404, Vec::new(), Vec::new()) };
4196 let mut out = entry(&href(relative, meta.is_dir()), meta.is_dir(), &etag_of(path), meta.len(), self.sum_of(path), date_of(path));
4197 if meta.is_dir() && request.header("Depth") != Some("0") {
4198 let mut children: Vec<_> = std::fs::read_dir(path).into_iter().flatten().filter_map(Result::ok).collect();
4199 children.sort_by_key(|e| e.file_name());
4200 for child in children {
4201 let name = child.file_name().to_string_lossy().to_string();
4202 let inside = if relative.is_empty() { name } else { format!("{relative}/{name}") };
4203 let dir = child.file_type().is_ok_and(|t| t.is_dir());
4204 out += &entry(&href(&inside, dir), dir, &etag_of(&child.path()), child.metadata().map_or(0, |m| m.len()), self.sum_of(&child.path()), date_of(&child.path()));
4205 }
4206 }
4207 (207, vec![("Content-Type", "application/xml; charset=utf-8".into())], multistatus(&out))
4208 }
4209
4210 fn put(&self, request: &Request, path: &Path) -> Reply {
4214 let reply = put(request, path);
4215 if matches!(reply.0, 201 | 204) {
4216 if let Some(seconds) = request.header("X-OC-Mtime").and_then(|t| t.parse::<u64>().ok())
4217 && let Ok(file) = std::fs::File::options().write(true).open(path)
4218 {
4219 let _ = file.set_modified(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(seconds));
4220 }
4221 let mut sums = self.sums.lock().unwrap();
4222 match request.header("OC-Checksum") {
4223 Some(sum) => sums.insert(path.to_path_buf(), (etag_of(path), sum.to_string())),
4224 None => sums.remove(path),
4225 };
4226 }
4227 reply
4228 }
4229
4230 fn note(&self, request: &Request, what: &str) {
4231 self.seen.lock().unwrap().push(format!("{} {} {what}", request.method, request.path));
4232 }
4233
4234 pub(crate) fn grant_flows(&self) {
4236 for flow in self.flows.lock().unwrap().iter_mut() {
4237 flow.granted = true;
4238 }
4239 }
4240
4241 fn flow_begin(&self) -> Reply {
4243 let mut flows = self.flows.lock().unwrap();
4244 let token = format!("polltoken{}x{}", flows.len(), "A".repeat(100));
4245 flows.push(Flow { token: token.clone(), ..Flow::default() });
4246 let signing = self.signing.lock().unwrap().clone();
4247 let endpoint = if signing.endpoint.is_empty() { format!("{}/login/v2/poll", self.base) } else { signing.endpoint };
4248 let body = serde_json::json!({ "poll": { "token": token, "endpoint": endpoint }, "login": format!("{}/login/v2/flow/pagetoken{}", self.base, flows.len()) });
4249 (200, vec![("Content-Type", "application/json".into())], body.to_string().into_bytes())
4250 }
4251
4252 fn flow_poll(&self, request: &Request) -> Reply {
4254 let body = String::from_utf8_lossy(&request.body).to_string();
4255 let Some(token) = body.strip_prefix("token=") else { return (404, Vec::new(), b"[]".to_vec()) };
4256 let mut flows = self.flows.lock().unwrap();
4257 let Some(flow) = flows.iter_mut().find(|f| f.token == token) else { return (404, Vec::new(), b"[]".to_vec()) };
4258 if !flow.granted || flow.answered {
4259 return (404, Vec::new(), b"[]".to_vec());
4260 }
4261 flow.answered = true;
4262 let mut made = self.app_passwords.lock().unwrap();
4263 let password = format!("{APP_PASSWORD}{}", made.len() + 1);
4264 made.push(password.clone());
4265 let signing = self.signing.lock().unwrap().clone();
4266 let server = if signing.server.is_empty() { self.base.clone() } else { signing.server };
4267 let login = if signing.login.is_empty() { USER.to_string() } else { signing.login };
4268 let body = serde_json::json!({ "server": server, "loginName": login, "appPassword": password });
4269 (200, vec![("Content-Type", "application/json".into())], body.to_string().into_bytes())
4270 }
4271
4272 pub(crate) fn answer(&self, request: &Request) -> Reply {
4273 let _turn = self.turn.lock().unwrap();
4274 if request.method == "POST" && request.path == "/index.php/login/v2" {
4276 return self.flow_begin();
4277 }
4278 if request.method == "POST" && request.path == "/login/v2/poll" {
4279 return self.flow_poll(request);
4280 }
4281 let basic = |login: &str, password: &str| format!("Basic {}", sioul_core::lines::base64_encode(format!("{login}:{password}").as_bytes()));
4282 let given = request.header("Authorization").unwrap_or_default().to_string();
4283 let app = self.app_passwords.lock().unwrap().iter().find(|password| given == basic(USER, password)).cloned();
4284 if given != basic(USER, PASSWORD) && app.is_none() {
4285 return (401, Vec::new(), Vec::new());
4286 }
4287 if request.method == "DELETE" && request.path == "/ocs/v2.php/core/apppassword" {
4289 return match app {
4290 Some(password) => {
4291 self.app_passwords.lock().unwrap().retain(|p| *p != password);
4292 (200, Vec::new(), Vec::new())
4293 }
4294 None => (403, Vec::new(), Vec::new()),
4295 };
4296 }
4297 if request.method == "PROPFIND" && request.path == "/remote.php/dav" {
4298 let principal = format!("<d:response><d:href>/remote.php/dav/</d:href><d:propstat><d:prop><d:current-user-principal><d:href>/remote.php/dav/principals/users/{USER}/</d:href></d:current-user-principal></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>");
4299 return (207, vec![("Content-Type", "application/xml".into())], multistatus(&principal));
4300 }
4301 if request.method == "PROPFIND" && request.path == format!("/remote.php/dav/principals/users/{USER}") {
4303 let homes = format!("<d:response><d:href>/remote.php/dav/principals/users/{USER}/</d:href><d:propstat><d:prop><c:calendar-home-set xmlns:c=\"urn:ietf:params:xml:ns:caldav\"><d:href>/remote.php/dav/calendars/{USER}/</d:href></c:calendar-home-set><a:addressbook-home-set xmlns:a=\"urn:ietf:params:xml:ns:carddav\"><d:href>/remote.php/dav/addressbooks/users/{USER}/</d:href></a:addressbook-home-set></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>");
4304 return (207, vec![("Content-Type", "application/xml".into())], multistatus(&homes));
4305 }
4306 let Some(relative) = request.path.strip_prefix(FILES).map(|r| r.trim_start_matches('/').to_string()) else { return (404, Vec::new(), Vec::new()) };
4307 let path = if relative.is_empty() { self.files.clone() } else { self.files.join(&relative) };
4308 match request.method.as_str() {
4309 "PROPFIND" => self.propfind(request, &relative, &path),
4310 "GET" => get(request, &path),
4311 "PUT" => self.put(request, &path),
4312 "MKCOL" => mkcol(&path),
4313 "DELETE" => delete(request, &path),
4314 _ => (405, Vec::new(), Vec::new()),
4315 }
4316 }
4317 }
4318
4319 fn read(stream: &TcpStream, throttle: Option<u64>, received: &std::sync::atomic::AtomicU64) -> Option<Request> {
4321 let mut reader = BufReader::new(stream);
4322 let mut line = String::new();
4323 reader.read_line(&mut line).ok()?;
4324 let mut words = line.split_whitespace();
4325 let (method, raw) = (words.next()?.to_string(), words.next()?.to_string());
4326 let mut headers: Vec<(String, String)> = Vec::new();
4327 loop {
4328 let mut line = String::new();
4329 reader.read_line(&mut line).ok()?;
4330 let Some((name, value)) = line.trim_end().split_once(':') else { break };
4331 headers.push((name.trim().to_string(), value.trim().to_string()));
4332 }
4333 let length = headers.iter().find(|(n, _)| n.eq_ignore_ascii_case("content-length")).and_then(|(_, v)| v.parse().ok()).unwrap_or(0);
4334 let mut body = vec![0; length];
4335 let piece = throttle.map_or(length.max(1), |rate| usize::try_from(rate / 20).unwrap_or(usize::MAX).max(1));
4336 let mut at = 0;
4337 while at < length {
4338 let end = (at + piece).min(length);
4339 let read = reader.read_exact(&mut body[at..end]);
4340 if method == "PUT" {
4341 received.fetch_add((end - at) as u64, std::sync::atomic::Ordering::Relaxed);
4342 }
4343 read.ok()?;
4344 at = end;
4345 if throttle.is_some() && at < length {
4346 std::thread::sleep(Duration::from_millis(50));
4347 }
4348 }
4349 Some(Request { method, path: crate::dav::path_key(&raw), headers, body })
4350 }
4351
4352 pub(crate) fn multistatus(responses: &str) -> Vec<u8> {
4353 format!(r#"<?xml version="1.0" encoding="utf-8"?><d:multistatus xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns">{responses}</d:multistatus>"#).into_bytes()
4354 }
4355
4356 pub(crate) fn etag_of(path: &Path) -> String {
4358 let mut hasher = DefaultHasher::new();
4359 if path.is_dir() {
4360 let mut names: Vec<_> = std::fs::read_dir(path).into_iter().flatten().filter_map(Result::ok).map(|e| e.file_name()).collect();
4361 names.sort();
4362 for name in names {
4363 name.hash(&mut hasher);
4364 etag_of(&path.join(&name)).hash(&mut hasher);
4365 }
4366 } else {
4367 std::fs::read(path).unwrap_or_default().hash(&mut hasher);
4368 GENERATIONS.lock().unwrap().get(path).copied().unwrap_or(0).hash(&mut hasher);
4369 }
4370 format!("{:016x}", hasher.finish())
4371 }
4372
4373 static GENERATIONS: Mutex<std::collections::BTreeMap<PathBuf, u64>> = Mutex::new(std::collections::BTreeMap::new());
4376
4377 pub(crate) fn bump(path: &Path) {
4379 *GENERATIONS.lock().unwrap().entry(path.to_path_buf()).or_insert(0) += 1;
4380 }
4381
4382 fn entry(href: &str, dir: bool, etag: &str, size: u64, sum: Option<String>, modified: Option<String>) -> String {
4383 let kind = if dir { "<d:resourcetype><d:collection/></d:resourcetype>" } else { "<d:resourcetype/>" };
4384 let length = if dir { String::new() } else { format!("<d:getcontentlength>{size}</d:getcontentlength>") };
4385 let sums = sum.map(|sum| format!("<oc:checksums><oc:checksum>{sum}</oc:checksum></oc:checksums>")).unwrap_or_default();
4386 let modified = modified.map(|at| format!("<d:getlastmodified>{at}</d:getlastmodified>")).unwrap_or_default();
4387 format!("<d:response><d:href>{href}</d:href><d:propstat><d:prop>{kind}<d:getetag>"{etag}"</d:getetag>{length}{modified}{sums}</d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>")
4388 }
4389
4390 fn date_of(path: &Path) -> Option<String> {
4393 let seconds = std::fs::metadata(path).ok()?.modified().ok()?.duration_since(std::time::UNIX_EPOCH).ok()?.as_secs();
4394 jiff::fmt::rfc2822::DateTimePrinter::new().timestamp_to_rfc9110_string(&jiff::Timestamp::from_second(seconds as i64).ok()?).ok()
4395 }
4396
4397 pub(crate) fn href(relative: &str, dir: bool) -> String {
4398 let path = super::encode_path(relative);
4399 match (relative.is_empty(), dir) {
4400 (true, _) => format!("{FILES}/"),
4401 (false, true) => format!("{FILES}/{path}/"),
4402 (false, false) => format!("{FILES}/{path}"),
4403 }
4404 }
4405
4406 fn put(request: &Request, path: &Path) -> Reply {
4409 if !path.parent().is_some_and(Path::is_dir) {
4410 return (409, Vec::new(), Vec::new());
4411 }
4412 let reply = put_once(request, path);
4413 if matches!(reply.0, 201 | 204) {
4414 bump(path);
4415 return (reply.0, vec![("ETag", format!("\"{}\"", etag_of(path)))], Vec::new());
4416 }
4417 reply
4418 }
4419
4420 fn put_once(request: &Request, path: &Path) -> Reply {
4421 let current = path.is_file().then(|| format!("\"{}\"", etag_of(path)));
4422 let refused = match (request.header("If-None-Match"), request.header("If-Match")) {
4423 (Some("*"), _) => current.is_some(),
4424 (_, Some(wanted)) => current.as_deref() != Some(wanted),
4425 _ => false,
4426 };
4427 if refused {
4428 return (412, Vec::new(), Vec::new());
4429 }
4430 std::fs::write(path, &request.body).unwrap();
4431 (if current.is_some() { 204 } else { 201 }, vec![("ETag", format!("\"{}\"", etag_of(path)))], Vec::new())
4432 }
4433
4434 fn mkcol(path: &Path) -> Reply {
4435 if path.exists() {
4436 return (405, Vec::new(), Vec::new());
4437 }
4438 if !path.parent().is_some_and(Path::is_dir) {
4439 return (409, Vec::new(), Vec::new());
4440 }
4441 std::fs::create_dir(path).unwrap();
4442 (201, Vec::new(), Vec::new())
4443 }
4444
4445 fn delete(request: &Request, path: &Path) -> Reply {
4446 if !path.exists() {
4447 return (404, Vec::new(), Vec::new());
4448 }
4449 if request.header("If-Match").is_some_and(|wanted| format!("\"{}\"", etag_of(path)) != wanted) {
4450 return (412, Vec::new(), Vec::new());
4451 }
4452 if path.is_dir() {
4453 std::fs::remove_dir_all(path).unwrap();
4454 } else {
4455 std::fs::remove_file(path).unwrap();
4456 }
4457 (204, Vec::new(), Vec::new())
4458 }
4459
4460 fn get(request: &Request, path: &Path) -> Reply {
4461 let Ok(bytes) = std::fs::read(path) else { return (404, Vec::new(), Vec::new()) };
4462 let etag = format!("\"{}\"", etag_of(path));
4463 let from = request.header("Range").and_then(|r| r.strip_prefix("bytes=")).and_then(|r| r.strip_suffix('-')).and_then(|s| s.parse::<usize>().ok());
4464 match from {
4465 Some(from) if from >= bytes.len() => (416, vec![("Content-Range", format!("bytes */{}", bytes.len()))], Vec::new()),
4466 Some(from) => (206, vec![("ETag", etag), ("Content-Range", format!("bytes {from}-{}/{}", bytes.len() - 1, bytes.len()))], bytes[from..].to_vec()),
4467 None => (200, vec![("ETag", etag)], bytes),
4468 }
4469 }
4470}
4471
4472#[cfg(test)]
4473mod tests {
4474 use super::fake::{Fake, Fault, PASSWORD};
4475 use super::*;
4476 use crate::devices::{COMPUTER, Entry, PHONE};
4477 use crate::share::{Outcome, Roots, Sharing, Store};
4478 use sioul_core::config::Config;
4479 use sioul_core::everywhere::{Press, Switch, change_own};
4480 use sioul_core::health::{Doubt, Peer, doubts_now};
4481 use std::sync::Arc;
4482
4483 const KEY: [u8; 32] = [11u8; 32];
4484 const SEAL: &str = "# Sioul: how your devices' shared records are sealed (docs/database.md).\nversion = 1\nsalt = \"AAAAAAAAAAAAAAAAAAAAAA==\"\nmemory_kib = 65536\npasses = 3\ncheck = \"one\"\n";
4485 const OTHER_SEAL: &str = "# Sioul: how your devices' shared records are sealed (docs/database.md).\nversion = 1\nsalt = \"BBBBBBBBBBBBBBBBBBBBBB==\"\nmemory_kib = 65536\npasses = 3\ncheck = \"two\"\n";
4486 const TEST: Limits = Limits { wait: Duration::from_millis(700), small: Duration::from_secs(3), large: Duration::from_secs(5), pull: Duration::from_secs(30), blob: Duration::from_secs(5), floor: 1 << 30, stall: Duration::from_millis(700) };
4488 thread_local! {
4489 pub(super) static ROUND_CAP: std::cell::Cell<Option<u64>> = const { std::cell::Cell::new(None) };
4491 }
4492 const SLOW_STAND_IN: Limits = Limits { wait: Duration::from_secs(30), small: Duration::from_secs(60), large: Duration::from_secs(60), pull: Duration::from_secs(120), ..TEST };
4496 const DUE: i64 = 1_800_000_000;
4497
4498 fn scratch(name: &str) -> PathBuf {
4499 let dir = std::env::temp_dir().join(format!("sioul-remote-{name}-{}", std::process::id()));
4500 let _ = std::fs::remove_dir_all(&dir);
4501 std::fs::create_dir_all(&dir).unwrap();
4502 dir
4503 }
4504
4505 #[test]
4508 fn the_files_stay_on_the_account_s_server() {
4509 use crate::dav::stand_in::{Reply, Request, Server as StandIn, serve};
4510 struct Noted(std::sync::Mutex<Vec<String>>);
4512 impl StandIn for Noted {
4513 fn misbehaves(&self, _: &Request) -> Option<crate::dav::stand_in::Fault> {
4514 None
4515 }
4516 fn answer(&self, request: &Request) -> Reply {
4517 self.0.lock().unwrap().push(format!("{} {} {}", request.method, request.path, request.header("Authorization").unwrap_or("")));
4518 Reply::new(404, "")
4519 }
4520 }
4521 struct Pointing(String);
4523 impl StandIn for Pointing {
4524 fn misbehaves(&self, _: &Request) -> Option<crate::dav::stand_in::Fault> {
4525 None
4526 }
4527 fn answer(&self, _: &Request) -> Reply {
4528 Reply::new(207, format!(r#"<?xml version="1.0" encoding="utf-8"?><d:multistatus xmlns:d="DAV:"><d:response><d:href>/remote.php/dav/</d:href><d:propstat><d:prop><d:current-user-principal><d:href>{}</d:href></d:current-user-principal></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response></d:multistatus>"#, self.0))
4529 }
4530 }
4531 let noted = Arc::new(Noted(std::sync::Mutex::new(Vec::new())));
4532 let there = serve(noted.clone());
4533 let here = serve(Arc::new(Pointing(format!("{there}/remote.php/dav/principals/users/jane/"))));
4534 let login = Login { account: "cloud".into(), url: format!("{here}/remote.php/dav/calendars/jane/"), user: "jane".into(), password: Some("secret".into()) };
4535 let found = files_root(&Server::new(&login, TEST).unwrap(), &login);
4536 assert!(matches!(&found, Err(SyncError::Elsewhere(root)) if root.starts_with(&there)), "{found:?}");
4537 assert!(noted.0.lock().unwrap().is_empty());
4538 let own = serve(Arc::new(Pointing("/remote.php/dav/principals/users/jane/".into())));
4540 let login = Login { url: format!("{own}/remote.php/dav/calendars/jane/"), ..login };
4541 assert_eq!(files_root(&Server::new(&login, TEST).unwrap(), &login).unwrap(), Some(format!("{own}/remote.php/dav/files/jane/")));
4542 }
4543
4544 fn put(dir: &Path, relative: &str, text: &str) {
4545 let path = dir.join(relative);
4546 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
4547 std::fs::write(path, text).unwrap();
4548 }
4549
4550 fn walk(dir: &Path) -> BTreeMap<String, Vec<u8>> {
4552 let mut out = BTreeMap::new();
4553 let mut stack = vec![dir.to_path_buf()];
4554 while let Some(at) = stack.pop() {
4555 for entry in std::fs::read_dir(&at).into_iter().flatten().filter_map(Result::ok) {
4556 let path = entry.path();
4557 if entry.file_name().to_string_lossy().starts_with('.') {
4558 continue;
4559 }
4560 if path.is_dir() {
4561 stack.push(path);
4562 } else {
4563 let relative = path.strip_prefix(dir).unwrap().to_string_lossy().replace('\\', "/");
4564 out.insert(relative, std::fs::read(&path).unwrap());
4565 }
4566 }
4567 }
4568 out
4569 }
4570
4571 fn copy_dir(from: &Path, to: &Path) {
4572 for (relative, bytes) in walk(from) {
4573 put_bytes(to, &relative, &bytes);
4574 }
4575 }
4576
4577 fn put_bytes(dir: &Path, relative: &str, bytes: &[u8]) {
4578 let path = dir.join(relative);
4579 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
4580 std::fs::write(path, bytes).unwrap();
4581 }
4582
4583 struct Device {
4585 roots: Roots,
4586 id: String,
4587 memory: PathBuf,
4588 kind: &'static str,
4589 }
4590
4591 impl Device {
4592 fn new(base: &Path, name: &str, kind: &'static str) -> Device {
4593 let root = base.join(name);
4594 let roots = Roots { config: root.join("config"), data: root.join("data"), state: root.join("state") };
4595 for dir in [&roots.config, &roots.data, &roots.state] {
4596 std::fs::create_dir_all(dir).unwrap();
4597 }
4598 let memory = roots.state.join("share").join("memory.json");
4599 Device { roots, id: uuid::Uuid::new_v4().to_string(), memory, kind }
4600 }
4601
4602 fn stores(&self) -> Vec<Store> {
4603 crate::share::stores(&Config::default(), &self.roots)
4604 }
4605
4606 fn exchange(&self, folder: &Path, now: i64) -> Outcome {
4608 let outcome = crate::share::exchange(&Sharing { folder, computer: &self.id, key: &KEY, memory: &self.memory, files: false, hurry: None }, &self.stores(), now * 1000).unwrap();
4609 crate::devices::change(&crate::devices::own_path(&self.roots.state), Some((folder, &KEY)), |e| e.exported(now * 1000, outcome.wrote, now)).unwrap();
4610 outcome
4611 }
4612
4613 fn all_parts(&self, notes: &Path) -> Vec<Store> {
4616 let config = Config { notes_root: Some(notes.display().to_string()), ..Config::default() };
4617 crate::share::stores_of(&config, &self.roots, &|part| matches!(part, "notes" | "papers" | "texts") || crate::share::shared_by_default(part, &config))
4618 }
4619
4620 fn exchange_all_parts(&self, folder: &Path, now: i64, notes: &Path) -> Outcome {
4623 self.exchange_parts(folder, now, notes, true)
4624 }
4625
4626 fn exchange_parts(&self, folder: &Path, now: i64, notes: &Path, files: bool) -> Outcome {
4629 let stores = self.all_parts(notes);
4630 let outcome = crate::share::exchange(&Sharing { folder, computer: &self.id, key: &KEY, memory: &self.memory, files, hurry: None }, &stores, now * 1000).unwrap();
4631 crate::devices::change(&crate::devices::own_path(&self.roots.state), Some((folder, &KEY)), |e| {
4633 e.exported(now * 1000, outcome.wrote, now);
4634 e.keeping = crate::blobs::KEEPING;
4635 })
4636 .unwrap();
4637 outcome
4638 }
4639
4640 fn session(&self, folder: &Path, f: impl FnOnce(&mut Entry)) {
4641 let (kind, id) = (self.kind, self.id.clone());
4642 crate::devices::change(&crate::devices::own_path(&self.roots.state), Some((folder, &KEY)), |e| {
4643 e.id = id;
4644 e.kind = kind.into();
4645 e.doses = true;
4646 f(e);
4647 })
4648 .unwrap();
4649 }
4650
4651 fn switch(&self) -> PathBuf {
4652 self.roots.state.join(sioul_core::everywhere::SWITCH_FILE)
4653 }
4654
4655 fn press(&self, on: bool, at: i64) {
4657 change_own(&self.switch(), &self.id, |s| {
4658 s.press(&self.id, on, 0, at * 1000);
4659 true
4660 })
4661 .unwrap();
4662 }
4663
4664 fn dnd(&self) -> Option<Press> {
4665 Switch::read(&self.switch()).unwrap().latest()
4666 }
4667
4668 fn write(&self, relative: &str, text: &str) {
4669 put(&self.roots.state, relative, text);
4670 }
4671
4672 fn read(&self, relative: &str) -> String {
4673 std::fs::read_to_string(self.roots.state.join(relative)).unwrap_or_default()
4674 }
4675
4676 fn sees(&self, folder: &Path, other: &Device, now: i64) -> Peer {
4678 let entry = crate::devices::all(folder, &KEY).0.into_iter().find(|e| e.id == other.id).expect("its entry");
4679 let heard = crate::share::heard(&self.memory, &self.id);
4680 Peer { id: other.id.clone(), name: entry.name.clone(), said: Some(entry.said()), complete: entry.wrote.is_none_or(|wrote| heard.complete(&other.id, wrote)), seen: now, heard: entry.exported, ..Peer::default() }
4681 }
4682
4683 fn kept(&self) -> BTreeMap<String, Vec<u8>> {
4685 let mut out = BTreeMap::new();
4686 for (root, dir) in [("config", &self.roots.config), ("data", &self.roots.data), ("state", &self.roots.state)] {
4687 for (relative, bytes) in walk(dir) {
4688 if !relative.starts_with("share/") {
4689 out.insert(format!("{root}/{relative}"), bytes);
4690 }
4691 }
4692 }
4693 out
4694 }
4695 }
4696
4697 struct World {
4702 base: PathBuf,
4703 fake: Arc<Fake>,
4704 server: PathBuf,
4705 phone_folder: PathBuf,
4706 desk: Device,
4707 phone: Device,
4708 }
4709
4710 impl World {
4711 fn new(name: &str) -> World {
4712 let base = scratch(name);
4713 let files = base.join("cloud");
4714 let server = files.join("Documents").join("Sioul");
4715 let phone_folder = base.join("phone").join("storage").join("Documents").join("Sioul");
4716 put(&server, "seal.toml", SEAL);
4717 put(&phone_folder, "seal.toml", SEAL);
4718 let fake = Fake::start(&files);
4719 let desk = Device::new(&base, "desk", COMPUTER);
4720 let phone = Device::new(&base, "phone-state", PHONE);
4721 let world = World { base, fake, server, phone_folder, desk, phone };
4722 world.desk.session(&world.server, |e| e.start(DUE - 7_200));
4723 world.desk.exchange(&world.server, DUE - 7_100);
4724 world.down();
4725 world.phone.session(&world.phone_folder, |e| e.start(DUE - 7_050));
4726 world.phone.exchange(&world.phone_folder, DUE - 7_000);
4727 world.up();
4728 world.desk.exchange(&world.server, DUE - 6_900);
4729 world.down();
4730 world
4731 }
4732
4733 fn up(&self) {
4735 let _turn = self.fake.turn.lock().unwrap();
4736 for (relative, bytes) in walk(&self.phone_folder).into_iter().filter(|(r, _)| r.contains(&self.phone.id)) {
4737 put_bytes(&self.server, &relative, &bytes);
4738 }
4739 }
4740
4741 fn down(&self) {
4743 for (relative, bytes) in walk(&self.server).into_iter().filter(|(r, _)| !r.contains(&self.phone.id)) {
4744 put_bytes(&self.phone_folder, &relative, &bytes);
4745 }
4746 }
4747
4748 fn find(&self, now: i64) -> State {
4749 find_with(&self.phone.memory, &self.phone_folder, &[self.fake.login()], &places(&self.phone_folder, &[]), now, TEST)
4750 }
4751
4752 fn pull(&self, now: i64) -> Pulled {
4753 pull_with(&self.phone.memory, &self.phone_folder, &self.phone.id, &self.fake.login(), now, TEST)
4754 }
4755
4756 fn cache(&self) -> PathBuf {
4757 cache_of(&self.phone.memory)
4758 }
4759
4760 fn desk_round(&self) -> String {
4762 let mut names: Vec<String> = walk(&self.server).into_keys().filter(|n| n.starts_with(&self.desk.id) && n.ends_with(".jsonl")).collect();
4763 names.sort_by_key(|n| n.trim_end_matches(".jsonl").rsplit('-').next().unwrap().parse::<u32>().unwrap());
4764 names.pop().unwrap()
4765 }
4766
4767 fn snapshot(&self) -> PathBuf {
4769 let aside = self.base.join("aside");
4770 let _ = std::fs::remove_dir_all(&aside);
4771 copy_dir(&self.base.join("phone-state"), &aside.join("phone-state"));
4772 copy_dir(&self.phone_folder, &aside.join("folder"));
4773 aside
4774 }
4775
4776 fn restore(&self, aside: &Path) {
4777 let _ = std::fs::remove_dir_all(self.base.join("phone-state"));
4778 let _ = std::fs::remove_dir_all(&self.phone_folder);
4779 copy_dir(&aside.join("phone-state"), &self.base.join("phone-state"));
4780 copy_dir(&aside.join("folder"), &self.phone_folder);
4781 }
4782
4783 fn others_here(&self) -> BTreeMap<String, Vec<u8>> {
4785 walk(&self.phone_folder).into_iter().filter(|(r, _)| !r.contains(&self.phone.id)).collect()
4786 }
4787 }
4788
4789 impl Drop for World {
4790 fn drop(&mut self) {
4791 let _ = std::fs::remove_dir_all(&self.base);
4792 }
4793 }
4794
4795 fn once(matches: impl Fn(&super::fake::Request) -> bool + Send + 'static, fault: Fault) -> impl FnMut(&super::fake::Request) -> Option<Fault> + Send + 'static {
4797 let mut done = false;
4798 move |request| {
4799 if done || !matches(request) {
4800 return None;
4801 }
4802 done = true;
4803 Some(fault)
4804 }
4805 }
4806
4807 #[test]
4808 fn a_stale_folder_is_caught_up_from_the_server() {
4809 let w = World::new("stale");
4810 w.desk.press(true, DUE + 30);
4814 w.desk.exchange(&w.server, DUE + 150);
4815 w.up();
4816 let alone = w.phone.exchange(&w.phone_folder, DUE + 170);
4817 assert!(alone.problems.is_empty(), "{:?}", alone.problems);
4818 assert!(w.phone.dnd().is_none_or(|p| !p.on), "nothing came yet");
4819 let stale = w.phone.sees(&w.phone_folder, &w.desk, DUE + 180);
4820 assert!(matches!(doubts_now(DUE, DUE + 180, None, &[stale]).as_slice(), [Doubt::Working { .. }]), "the desk's news has not come: a doubt");
4821 let state = w.find(DUE + 180);
4823 assert!(state.confirmed_for(&w.phone_folder), "{state:?}");
4824 assert_eq!(state.url, w.fake.folder_url("Documents/Sioul"));
4825 let before = w.others_here();
4826 let pulled = w.pull(DUE + 181);
4827 assert!(pulled.problem.is_none() && pulled.fetched >= 2, "{pulled:?}");
4828 assert_eq!(w.others_here(), before, "nothing written into the synced folder");
4829 let outcome = w.phone.exchange(&w.phone_folder, DUE + 182);
4830 assert!(outcome.problems.is_empty(), "{:?}", outcome.problems);
4831 let press = w.phone.dnd().expect("the desk's press");
4832 assert!(press.on && press.from == w.desk.id, "{press:?}");
4833 let fresh = w.phone.sees(&w.phone_folder, &w.desk, DUE + 183);
4835 assert!(doubts_now(DUE, DUE + 183, None, std::slice::from_ref(&fresh)).is_empty(), "{fresh:?}");
4836 let direct = crate::devices::all(&w.server, &KEY).0.into_iter().find(|e| e.id == w.desk.id).unwrap();
4837 assert_eq!(fresh.said, Some(direct.said()), "the same as the server's folder says");
4838 assert!(crate::share::others(&w.phone_folder, &w.phone.id).iter().any(|o| o.id == w.desk.id && o.heard >= DUE + 150));
4839 for line in w.fake.seen() {
4841 assert!(line.starts_with("PROPFIND ") || line.starts_with("GET "), "{line}");
4842 assert!(!(line.starts_with("GET ") && line.contains(&w.phone.id)), "{line}");
4843 }
4844 let kept = std::fs::read_to_string(state_path(&w.phone.memory)).unwrap();
4845 assert!(!kept.contains(PASSWORD) && !kept.contains("Basic"), "{kept}");
4846 assert!(walk(&w.cache()).keys().all(|name| !name.contains(&w.phone.id)), "never the phone's own");
4847 }
4848
4849 #[test]
4850 fn with_the_folder_fresh_the_backup_changes_nothing() {
4851 let w = World::new("fresh");
4852 w.desk.press(true, DUE + 30);
4853 w.desk.exchange(&w.server, DUE + 150);
4854 crate::lease::renew(&w.server, &KEY, "health", &w.desk.id, DUE + 150, DUE + 150, false, crate::lease::Rule::FollowsYou, crate::share::written(&w.desk.memory, &w.desk.id)).unwrap();
4855 w.up();
4856 w.down();
4857 let aside = w.snapshot();
4859 let without = w.phone.exchange(&w.phone_folder, DUE + 200);
4860 let kept_without = w.phone.kept();
4861 w.restore(&aside);
4863 assert!(w.find(DUE + 190).confirmed_for(&w.phone_folder));
4864 w.fake.forget_seen();
4865 let pulled = w.pull(DUE + 195);
4866 assert!(pulled.problem.is_none(), "{pulled:?}");
4867 assert_eq!(pulled.fetched, 0, "nothing the folder holds is kept twice");
4868 assert!(w.fake.seen().iter().all(|line| !(line.starts_with("GET ") && line.contains(".jsonl"))), "no records fetched: {:?}", w.fake.seen());
4869 assert_eq!(walk(&w.cache()).into_keys().collect::<Vec<_>>(), ["seal.toml"]);
4870 let with = w.phone.exchange(&w.phone_folder, DUE + 200);
4871 assert_eq!((with.received, with.sent, &with.problems), (without.received, without.sent, &without.problems));
4872 assert_eq!(w.phone.kept(), kept_without, "the same files, byte for byte");
4873 w.fake.forget_seen();
4875 w.pull(DUE + 260);
4876 assert!(w.fake.seen().iter().all(|line| line.starts_with("PROPFIND ")), "{:?}", w.fake.seen());
4877 }
4878
4879 #[test]
4880 fn a_half_downloaded_file_is_never_read_as_whole() {
4881 let w = World::new("half");
4882 let first = "\"first@1800000000\" = 1800000000\n\"second@1800000000\" = 1800000001\n";
4884 w.desk.write("health-state.toml", &format!("[taken]\n{first}"));
4885 w.desk.exchange(&w.server, DUE + 50);
4886 w.down();
4887 let marks: String = (0..40).map(|n| format!("\"dose-{n}@1800000000\" = {}\n", DUE + n)).collect();
4889 w.desk.write("health-state.toml", &format!("[taken]\n{first}{marks}"));
4890 w.desk.exchange(&w.server, DUE + 100);
4891 assert!(w.find(DUE + 110).confirmed_for(&w.phone_folder));
4892 let round = w.desk_round();
4893 let whole = std::fs::read(w.server.join(&round)).unwrap();
4894 let (end, last) = whole_end(&w.phone_folder.join(&round));
4895 assert!(end > 0 && whole.len() as u64 > end, "the phone's copy is behind");
4896 let cut = (whole.len() - last as usize) / 2;
4898 let name = round.clone();
4899 w.fake.fault(once(move |r| r.method == "GET" && r.path.ends_with(&name), Fault::Cut(cut)));
4900 let pulled = w.pull(DUE + 120);
4901 assert!(pulled.problem.as_deref().is_some_and(|p| p.starts_with("network:")), "{pulled:?}");
4902 assert!(w.fake.seen().iter().any(|l| l.starts_with("GET ") && l.contains(&round) && l.ends_with(" 206")), "asked from where the phone's copy ends: {:?}", w.fake.seen());
4903 let cached = w.cache().join(&round);
4904 let arrived = std::fs::read(&cached).unwrap_or_default();
4905 assert!(arrived.is_empty() || (arrived.ends_with(b"\n") && whole.starts_with(&arrived) && arrived.len() < whole.len()), "whole lines only: {} of {}", arrived.len(), whole.len());
4906 let first = w.phone.exchange(&w.phone_folder, DUE + 130);
4907 assert!(first.problems.iter().all(|p| !p.starts_with("share-other")), "{:?}", first.problems);
4908 let entry = format!("devices/{}.device", w.desk.id);
4910 let name = entry.clone();
4911 w.fake.fault(once(move |r| r.method == "GET" && r.path.ends_with(&name), Fault::Cut(40)));
4912 let pulled = w.pull(DUE + 140);
4913 assert!(pulled.problem.as_deref().is_some_and(|p| p.starts_with("network:")), "{pulled:?}");
4914 assert_eq!(std::fs::read(&cached).unwrap(), whole, "the records whole now");
4915 assert!(!w.cache().join(&entry).exists(), "an entry cut is not kept");
4916 let (entries, unread) = crate::devices::all(&w.phone_folder, &KEY);
4917 assert!(entries.iter().any(|e| e.id == w.desk.id) && !unread.contains(&w.desk.id), "the folder's copy read meanwhile");
4918 let second = w.phone.exchange(&w.phone_folder, DUE + 150);
4919 assert!(second.problems.iter().all(|p| !p.starts_with("share-other")), "{:?}", second.problems);
4920 assert!(w.pull(DUE + 160).problem.is_none());
4921 assert!(w.cache().join(&entry).exists());
4922 let record = w.phone.read("health-state.toml");
4924 for n in 0..40 {
4925 assert!(record.contains(&format!("dose-{n}@")), "dose-{n}:\n{record}");
4926 }
4927 let heard = crate::share::heard(&w.phone.memory, &w.phone.id);
4928 assert!(heard.complete(&w.desk.id, crate::share::written(&w.desk.memory, &w.desk.id).unwrap()), "{heard:?}");
4929 assert!(!heard.broken.contains_key(&w.desk.id), "no line lost nor read twice: {heard:?}");
4930 }
4931
4932 #[test]
4933 fn a_seal_that_differs_is_never_used() {
4934 let w = World::new("seal");
4935 put(&w.server, "seal.toml", OTHER_SEAL);
4937 let state = w.find(DUE);
4938 assert!(!state.confirmed_for(&w.phone_folder) && state.said.starts_with("seal-differs:"), "{state:?}");
4939 assert!(attach(&w.phone_folder, &w.phone.memory).is_none());
4940 assert_eq!(w.pull(DUE + 1).problem.as_deref(), Some("not-confirmed"));
4941 assert!(w.fake.seen().iter().filter(|l| l.starts_with("GET ")).all(|l| l.ends_with("seal.toml 200") || l.ends_with("seal.toml 404")), "{:?}", w.fake.seen());
4942 put(&w.server, "seal.toml", SEAL);
4944 State::choose(&w.phone.memory, |s| s.asked = DUE + 2).unwrap();
4945 assert!(w.find(DUE + 3).confirmed_for(&w.phone_folder));
4946 assert!(w.pull(DUE + 4).problem.is_none());
4947 put(&w.server, "seal.toml", OTHER_SEAL);
4948 w.desk.press(true, DUE + 5);
4949 w.desk.exchange(&w.server, DUE + 6);
4950 w.fake.forget_seen();
4951 let pulled = w.pull(DUE + 7);
4952 assert!(pulled.problem.as_deref().is_some_and(|p| p.starts_with("seal-differs:")), "{pulled:?}");
4953 assert!(w.fake.seen().iter().filter(|l| l.starts_with("GET ")).all(|l| l.contains("seal.toml")), "{:?}", w.fake.seen());
4954 assert!(!State::load(&w.phone.memory).confirmed_for(&w.phone_folder));
4955 put(&w.server, "seal.toml", SEAL);
4957 State::choose(&w.phone.memory, |s| s.asked = DUE + 8).unwrap();
4958 assert!(w.find(DUE + 9).confirmed_for(&w.phone_folder));
4959 assert!(w.pull(DUE + 10).problem.is_none());
4960 assert!(attach(&w.phone_folder, &w.phone.memory).is_some());
4961 put(&w.phone_folder, "seal.toml", OTHER_SEAL);
4962 assert!(attach(&w.phone_folder, &w.phone.memory).is_none(), "another sharing's");
4963 assert_eq!(w.pull(DUE + 11).problem.as_deref(), Some("no-seal"));
4964 assert!(!w.cache().exists());
4965 }
4966
4967 #[test]
4968 fn server_errors_and_timeouts_leave_the_exchange_as_before() {
4969 let w = World::new("errors");
4970 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
4971 w.desk.press(true, DUE + 30);
4972 w.desk.exchange(&w.server, DUE + 60);
4973 w.up();
4974 for (n, fault) in [Fault::Status(500), Fault::Status(503), Fault::Late(Duration::from_secs(2)), Fault::Drop, Fault::Cut(20)].into_iter().enumerate() {
4975 w.fake.clear_faults();
4976 w.fake.fault(move |_| Some(fault));
4977 let pulled = w.pull(DUE + 100 + n as i64);
4978 let said = pulled.problem.clone().unwrap_or_default();
4979 assert!(said.starts_with("server:") || said.starts_with("network:"), "{fault:?}: {pulled:?}");
4980 assert_eq!(walk(&w.cache()).into_keys().collect::<Vec<_>>(), ["seal.toml"], "{fault:?}: nothing kept");
4981 assert_eq!(State::load(&w.phone.memory).said, said);
4982 }
4983 w.fake.clear_faults();
4984 let aside = w.snapshot();
4986 let with = w.phone.exchange(&w.phone_folder, DUE + 200);
4987 let kept_with = w.phone.kept();
4988 w.restore(&aside);
4989 forget(&w.phone.memory, &w.phone_folder);
4990 let without = w.phone.exchange(&w.phone_folder, DUE + 200);
4991 assert_eq!((with.received, &with.problems), (without.received, &without.problems));
4992 assert_eq!(w.phone.kept(), kept_with);
4993 }
4994
4995 #[test]
4996 fn what_was_fetched_never_wins_over_a_newer_synced_copy() {
4997 let w = World::new("older");
4998 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
4999 let renew = |now: i64| crate::lease::renew(&w.server, &KEY, "health", &w.desk.id, now, now, false, crate::lease::Rule::FollowsYou, crate::share::written(&w.desk.memory, &w.desk.id)).unwrap();
5000 w.desk.press(true, DUE + 30);
5001 w.desk.exchange(&w.server, DUE + 60);
5002 renew(DUE + 60);
5003 assert!(w.pull(DUE + 70).problem.is_none());
5004 let round = w.desk_round();
5005 assert!(w.cache().join(&round).exists() && w.cache().join(format!("devices/{}.device", w.desk.id)).exists());
5006 w.desk.press(false, DUE + 1_000);
5008 w.desk.exchange(&w.server, DUE + 1_100);
5009 renew(DUE + 1_100);
5010 w.up();
5011 w.down();
5012 let entry = crate::devices::all(&w.phone_folder, &KEY).0.into_iter().find(|e| e.id == w.desk.id).unwrap();
5013 assert_eq!(entry.exported, DUE + 1_100, "the folder's entry");
5014 let claim = crate::lease::claims(&w.phone_folder, &KEY, "health").into_iter().find(|c| c.computer == w.desk.id).unwrap();
5015 assert_eq!(claim.renewed, DUE + 1_100, "the folder's claim");
5016 let heard = crate::share::others(&w.phone_folder, &w.phone.id).into_iter().find(|o| o.id == w.desk.id).unwrap().heard;
5017 assert_eq!(heard, DUE + 1_100, "the folder's notes");
5018 w.phone.exchange(&w.phone_folder, DUE + 1_200);
5019 assert!(!w.phone.dnd().unwrap().on, "the later press, from the folder's longer records");
5020 tidy(&w.phone.memory, &w.phone_folder, &w.phone.id);
5022 assert!(!w.cache().join(&round).exists());
5023 State::choose(&w.phone.memory, |s| s.on = Some(false)).unwrap();
5025 tidy(&w.phone.memory, &w.phone_folder, &w.phone.id);
5026 assert_eq!(walk(&w.cache()).into_keys().collect::<Vec<_>>(), ["seal.toml"]);
5027 w.fake.forget_seen();
5028 assert_eq!(w.pull(DUE + 1_300).problem.as_deref(), Some("off"));
5029 assert!(w.fake.seen().is_empty(), "switched off: not a request");
5030 }
5031
5032 #[test]
5036 fn doses_are_never_known_wrongly_with_a_stuck_sync_app_and_the_backup() {
5037 let w = World::new("stuck");
5038 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
5039 let mut marked: Vec<(String, i64)> = Vec::new();
5040 for minute in 0..90 {
5041 let now = DUE + minute * 60;
5042 if minute % 10 == 0 {
5043 marked.push((format!("dose@{minute}"), minute));
5044 let record: String = marked.iter().map(|(d, m)| format!("\"{d}\" = {m}\n")).collect();
5045 w.desk.write("health-state.toml", &format!("[taken]\n{record}"));
5046 }
5047 w.desk.exchange(&w.server, now);
5048 crate::lease::renew(&w.server, &KEY, "health", &w.desk.id, now, now, false, crate::lease::Rule::FollowsYou, crate::share::written(&w.desk.memory, &w.desk.id)).unwrap();
5049 w.up();
5050 let _ = w.pull(now + 20);
5051 w.phone.exchange(&w.phone_folder, now + 30);
5052 if let Some(claim) = crate::lease::claims(&w.phone_folder, &KEY, "health").into_iter().find(|c| c.computer == w.desk.id)
5053 && let Some(wrote) = claim.wrote
5054 && crate::share::heard(&w.phone.memory, &w.phone.id).complete(&w.desk.id, wrote)
5055 {
5056 let record = w.phone.read("health-state.toml");
5057 let claimed = (claim.renewed - DUE) / 60;
5058 for (dose, at) in marked.iter().filter(|(_, m)| *m < claimed) {
5059 assert!(record.contains(dose.as_str()), "minute {minute}: {dose} ({at}) taken as known, not there:\n{record}");
5060 }
5061 }
5062 }
5063 let record = w.phone.read("health-state.toml");
5064 for (dose, _) in &marked {
5065 assert!(record.contains(dose.as_str()), "{dose} never came:\n{record}");
5066 }
5067 let synced = std::fs::read(w.phone_folder.join(w.desk_round())).map(|b| b.len()).unwrap_or(0);
5069 assert!(synced < std::fs::read(w.server.join(w.desk_round())).unwrap().len());
5070 }
5071
5072 const QUICK: AlarmTimes = AlarmTimes { alone: Duration::from_secs(2), most: Duration::from_secs(5) };
5074
5075 #[test]
5084 fn a_doses_alarm_on_its_pull_knows_what_it_knew_with_the_sync_app_asked() {
5085 static ONE_PULL: Mutex<()> = Mutex::new(());
5086 let w = World::new("alarm-alone");
5087 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
5088 assert!(w.pull(DUE).problem.is_none());
5089 let mut marked: Vec<(String, i64)> = Vec::new();
5090 for minute in 0..16 {
5091 let now = DUE + minute * 60;
5092 if minute % 3 == 0 {
5093 marked.push((format!("dose@{minute}"), now));
5094 let record: String = marked.iter().map(|(d, at)| format!("\"{d}\" = {at}\n")).collect();
5095 w.desk.write("health-state.toml", &format!("[taken]\n{record}"));
5096 }
5097 w.desk.exchange(&w.server, now);
5098 w.up();
5099 let aside = w.snapshot();
5101 w.down();
5102 let _ = w.pull(now + 20);
5103 w.phone.exchange(&w.phone_folder, now + 25);
5104 let asked = (w.phone.read("health-state.toml"), w.phone.sees(&w.phone_folder, &w.desk, now + 30));
5105 w.restore(&aside);
5107 assert!(State::load(&w.phone.memory).pulls_well(&w.phone_folder), "minute {minute}");
5108 let mut ask = || -> Duration { panic!("minute {minute}: the pull went through, the sync app is not asked") };
5109 let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, now + 20, QUICK, Arc::new(move || now + 20), &mut ask, TEST);
5110 assert_eq!(came, AlarmNews::Pulled, "minute {minute}");
5111 w.phone.exchange(&w.phone_folder, now + 25);
5112 let alone = (w.phone.read("health-state.toml"), w.phone.sees(&w.phone_folder, &w.desk, now + 30));
5113 assert_eq!(alone.0, asked.0, "minute {minute}: the same answers here");
5114 for (dose, due) in &marked {
5115 let doubted = |peer: &Peer| !doubts_now(*due, now + 30, None, std::slice::from_ref(peer)).is_empty();
5116 assert_eq!(doubted(&alone.1), doubted(&asked.1), "minute {minute}, {dose}: the same doubt");
5117 assert!(alone.0.contains(dose.as_str()) || doubted(&alone.1), "minute {minute}: {dose}, marked on the desk, known not taken");
5118 }
5119 }
5120 assert!(std::fs::read(w.phone_folder.join(w.desk_round())).map_or(0, |b| b.len()) < std::fs::read(w.server.join(w.desk_round())).unwrap().len());
5122 }
5123
5124 #[test]
5133 fn a_doses_alarm_whose_pull_fails_asks_the_sync_app_and_never_knows_wrongly() {
5134 static ONE_PULL: Mutex<()> = Mutex::new(());
5135 let w = World::new("alarm-fails");
5136 assert!(w.find(DUE - 3_000).confirmed_for(&w.phone_folder));
5137 w.desk.exchange(&w.server, DUE - 3_600);
5138 w.desk.session(&w.server, |e| e.close(DUE - 3_590));
5139 assert!(w.pull(DUE - 3_000).problem.is_none());
5140 w.phone.exchange(&w.phone_folder, DUE - 2_990);
5141 assert!(State::load(&w.phone.memory).pulls_well(&w.phone_folder), "the pull works");
5142 assert!(doubts_now(DUE, DUE + 10, None, &[w.phone.sees(&w.phone_folder, &w.desk, DUE + 10)]).is_empty(), "the desk closed before the dose: known");
5143 w.desk.session(&w.server, |e| e.start(DUE - 60));
5144 w.desk.write("health-state.toml", "[taken]\n\"dose@0\" = 1\n");
5145 w.desk.exchange(&w.server, DUE + 30);
5146 w.fake.fault(|_| Some(Fault::Status(503)));
5148 let mut asked = 0;
5149 let mut ask = || {
5150 asked += 1;
5151 w.down();
5152 Duration::from_millis(200)
5153 };
5154 let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST);
5155 assert!(matches!(came, AlarmNews::SyncApp { ref heard } if heard.contains(&w.desk.id)) && asked == 1, "the pull failed: the sync app asked, once, the desk's entry brought: {came:?}");
5156 w.phone.exchange(&w.phone_folder, DUE + 65);
5157 let taken = w.phone.read("health-state.toml").contains("dose@0");
5158 let doubts = doubts_now(DUE, DUE + 70, None, &[w.phone.sees(&w.phone_folder, &w.desk, DUE + 70)]);
5159 assert!(taken || !doubts.is_empty(), "the dose marked on the desk is never known not taken: {doubts:?}");
5160 assert!(taken, "the sync app brought it");
5161 assert!(!State::load(&w.phone.memory).pulls_well(&w.phone_folder), "the next alarm asks the sync app first, as before");
5162 w.fake.clear_faults();
5164 assert!(w.pull(DUE + 180).problem.is_none());
5165 assert!(State::load(&w.phone.memory).pulls_well(&w.phone_folder));
5166 }
5167
5168 #[test]
5173 fn two_doses_at_once_the_second_alarm_waits_for_the_first_ones_pull() {
5174 static ONE_PULL: Mutex<()> = Mutex::new(());
5175 let w = World::new("alarm-twice");
5176 assert!(w.find(DUE - 3_000).confirmed_for(&w.phone_folder));
5177 w.desk.session(&w.server, |e| e.close(DUE - 3_590));
5178 assert!(w.pull(DUE - 3_000).problem.is_none());
5179 w.phone.exchange(&w.phone_folder, DUE - 2_990);
5180 w.desk.session(&w.server, |e| e.start(DUE - 60));
5182 w.desk.write("health-state.toml", "[taken]\n\"a@0\" = 1\n\"b@0\" = 2\n");
5183 w.desk.exchange(&w.server, DUE + 10);
5184 w.fake.fault(once(|r| r.method == "PROPFIND", Fault::Late(Duration::from_millis(450))));
5186 let outcomes: Vec<(AlarmNews, Duration)> = std::thread::scope(|scope| {
5187 let alarm = |pause: u64| {
5188 let w = &w;
5189 scope.spawn(move || {
5190 std::thread::sleep(Duration::from_millis(pause));
5191 let begun = Instant::now();
5192 let mut ask = || -> Duration { panic!("the pull went through: the sync app is not asked") };
5193 (alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST), begun.elapsed())
5194 })
5195 };
5196 let (first, second) = (alarm(0), alarm(150));
5197 vec![first.join().unwrap(), second.join().unwrap()]
5198 });
5199 assert!(outcomes.iter().all(|(came, _)| *came == AlarmNews::Pulled), "{outcomes:?}");
5200 assert!(outcomes[1].1 >= Duration::from_millis(250), "the second waited for the first one's pull: {outcomes:?}");
5201 w.phone.exchange(&w.phone_folder, DUE + 70);
5202 let record = w.phone.read("health-state.toml");
5203 assert!(record.contains("a@0") && record.contains("b@0"), "{record}");
5204 let seen = w.phone.sees(&w.phone_folder, &w.desk, DUE + 70);
5205 assert!(seen.said.as_ref().is_some_and(|s| s.working), "the desk known in use, from the pull: {seen:?}");
5206 }
5207
5208 #[test]
5213 fn an_alarm_while_the_windows_pull_runs_pulls_again_after_it() {
5214 static ONE_PULL: Mutex<()> = Mutex::new(());
5215 let w = World::new("alarm-window");
5216 assert!(w.find(DUE - 3_000).confirmed_for(&w.phone_folder));
5217 w.desk.session(&w.server, |e| e.close(DUE - 3_590));
5218 assert!(w.pull(DUE - 3_000).problem.is_none());
5219 w.phone.exchange(&w.phone_folder, DUE - 2_990);
5220 w.desk.session(&w.server, |e| e.start(DUE - 60));
5221 w.desk.write("health-state.toml", "[taken]\n\"dose@0\" = 1\n");
5222 w.desk.exchange(&w.server, DUE + 10);
5223 w.fake.fault(once(|r| r.method == "PROPFIND", Fault::Late(Duration::from_millis(450))));
5224 let came = std::thread::scope(|scope| {
5225 let window = scope.spawn(|| {
5227 let _one = ONE_PULL.lock().unwrap();
5228 pull_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), DUE + 50, TEST)
5229 });
5230 std::thread::sleep(Duration::from_millis(200));
5231 let mut ask = || -> Duration { panic!("a pull went through: the sync app is not asked") };
5232 let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 61), &mut ask, TEST);
5233 assert!(window.join().unwrap().problem.is_none());
5234 came
5235 });
5236 assert_eq!(came, AlarmNews::Pulled);
5237 assert_eq!(State::load(&w.phone.memory).last, DUE + 61, "pulled again, after the window's");
5238 w.phone.exchange(&w.phone_folder, DUE + 70);
5239 assert!(w.phone.read("health-state.toml").contains("dose@0"));
5240 }
5241
5242 #[test]
5247 fn an_alarm_whose_pull_does_not_go_through_in_time_asks_the_sync_app() {
5248 static ONE_PULL: Mutex<()> = Mutex::new(());
5249 let w = World::new("alarm-slow");
5250 assert!(w.find(DUE - 3_000).confirmed_for(&w.phone_folder));
5251 w.desk.session(&w.server, |e| e.close(DUE - 3_590));
5252 assert!(w.pull(DUE - 3_000).problem.is_none());
5253 w.phone.exchange(&w.phone_folder, DUE - 2_990);
5254 w.desk.session(&w.server, |e| e.start(DUE - 60));
5255 w.desk.write("health-state.toml", "[taken]\n\"dose@0\" = 1\n");
5256 w.desk.exchange(&w.server, DUE + 10);
5257 w.fake.fault(|_| Some(Fault::Late(Duration::from_secs(6))));
5259 let mut asked = 0;
5260 let mut ask = || {
5261 asked += 1;
5262 w.down();
5263 Duration::from_millis(300)
5264 };
5265 let begun = Instant::now();
5266 let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST);
5267 let waited = begun.elapsed();
5268 assert!(matches!(came, AlarmNews::SyncApp { .. }) && asked == 1, "not through in time: the sync app asked, once: {came:?}");
5269 assert!(waited < QUICK.most, "never past the alarm's wait: {waited:?}");
5270 w.phone.exchange(&w.phone_folder, DUE + 65);
5271 assert!(w.phone.read("health-state.toml").contains("dose@0"), "what the sync app brought is read");
5272 w.fake.clear_faults();
5273 }
5274
5275 fn alarm_scene(name: &str, pulled_at: i64, opens: bool) -> World {
5280 let w = World::new(name);
5281 assert!(w.find(DUE - 3_000).confirmed_for(&w.phone_folder));
5282 w.desk.exchange(&w.server, DUE - 3_600);
5283 w.desk.session(&w.server, |e| e.close(DUE - 3_590));
5284 assert!(w.pull(pulled_at).problem.is_none());
5285 w.phone.exchange(&w.phone_folder, DUE - 2_990);
5286 assert!(State::load(&w.phone.memory).pulls_well(&w.phone_folder));
5287 if opens {
5288 w.desk.session(&w.server, |e| e.start(DUE - 60));
5289 w.desk.write("health-state.toml", "[taken]\n\"dose@0\" = 1\n");
5290 w.desk.exchange(&w.server, DUE + 30);
5291 }
5292 w
5293 }
5294
5295 #[test]
5301 fn an_alarm_never_takes_a_pull_from_before_the_clock_was_set_back() {
5302 static ONE_PULL: Mutex<()> = Mutex::new(());
5303 let w = alarm_scene("alarm-clock", DUE + 600, true);
5304 w.fake.fault(|_| Some(Fault::Status(503)));
5305 let mut asked = 0;
5306 let mut ask = || {
5307 asked += 1;
5308 Duration::from_millis(200)
5309 };
5310 let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST);
5311 assert_eq!((&came, asked), (&AlarmNews::SyncApp { heard: BTreeSet::new() }, 1), "the old pull not taken: pulled again, refused; the sync app asked, nothing came");
5312 w.phone.exchange(&w.phone_folder, DUE + 65);
5313 assert!(!w.phone.read("health-state.toml").contains("dose@0"));
5314 let peer = w.phone.sees(&w.phone_folder, &w.desk, DUE + 70);
5315 assert!(doubts_now(DUE, DUE + 70, None, std::slice::from_ref(&peer)).is_empty(), "what was read before the alarm says the desk closed");
5316 assert!(!sioul_core::health::doubts_unread(DUE, DUE + 70, None, std::slice::from_ref(&peer), DUE + 60, &BTreeSet::new()).is_empty(), "nothing read since the alarm: the doubt said, never known");
5317 let state = State::load(&w.phone.memory);
5318 assert!(!state.pulls_well(&w.phone_folder), "the refused pull is the later one, whatever the clock said: {state:?}");
5319 let pace = Pace { phone: true, shown: false, in_use: false, urgent: false, step: 5 * 60 };
5320 assert!(due(&State { tried: DUE + 600, ..state }, DUE + 120, pace), "tried \"later\" than now: due at once");
5321 w.fake.clear_faults();
5322 }
5323
5324 #[test]
5331 fn an_alarm_that_reads_no_news_says_the_doubt_and_one_that_pulls_does_not() {
5332 static ONE_PULL: Mutex<()> = Mutex::new(());
5333 let w = alarm_scene("alarm-nothing", DUE - 3_000, true);
5334 w.fake.fault(|_| Some(Fault::Status(503)));
5335 let mut ask = || Duration::from_millis(200);
5336 let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST);
5337 assert_eq!(came, AlarmNews::SyncApp { heard: BTreeSet::new() });
5338 w.phone.exchange(&w.phone_folder, DUE + 65);
5339 let peer = w.phone.sees(&w.phone_folder, &w.desk, DUE + 70);
5340 assert!(!w.phone.read("health-state.toml").contains("dose@0"));
5341 let doubts = sioul_core::health::doubts_unread(DUE, DUE + 70, None, std::slice::from_ref(&peer), DUE + 60, &BTreeSet::new());
5342 assert!(matches!(doubts.as_slice(), [Doubt::Unheard { closed: true, until, .. }] if *until == DUE - 3_590), "the desk as last heard, closed: {doubts:?}");
5343 w.fake.clear_faults();
5344 static ANOTHER: Mutex<()> = Mutex::new(());
5346 let quiet = alarm_scene("alarm-pulled", DUE - 3_000, false);
5347 let mut never = || -> Duration { panic!("the pull went through: the sync app is not asked") };
5348 let came = alarm_news_with(&quiet.phone.memory, &quiet.phone_folder, &quiet.phone.id, KEY, Some(quiet.fake.login()), &ANOTHER, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut never, TEST);
5349 assert_eq!(came, AlarmNews::Pulled);
5350 quiet.phone.exchange(&quiet.phone_folder, DUE + 65);
5351 assert!(doubts_now(DUE, DUE + 70, None, &[quiet.phone.sees(&quiet.phone_folder, &quiet.desk, DUE + 70)]).is_empty(), "pulled since the alarm, the desk closed: known");
5352 }
5353
5354 fn alarm_verdict(w: &World, came: &AlarmNews, since: i64, at: i64) -> (bool, bool, Vec<Doubt>) {
5358 w.phone.exchange(&w.phone_folder, at);
5359 let taken = w.phone.read("health-state.toml").contains("dose@0");
5360 let peer = w.phone.sees(&w.phone_folder, &w.desk, at + 5);
5361 let doubts = match came {
5362 AlarmNews::SyncApp { heard } => sioul_core::health::doubts_unread(DUE, at + 5, None, std::slice::from_ref(&peer), since, heard),
5363 AlarmNews::Pulled => doubts_now(DUE, at + 5, None, std::slice::from_ref(&peer)),
5364 };
5365 (!taken && doubts.is_empty(), taken, doubts)
5366 }
5367
5368 #[test]
5374 fn an_alarm_doubts_each_device_whose_own_entry_did_not_come() {
5375 static ONE_PULL: Mutex<()> = Mutex::new(());
5376 let w = alarm_scene("alarm-unrelated", DUE - 3_000, true);
5377 w.fake.fault(|_| Some(Fault::Status(503)));
5378 let notes = format!("{}.toml", w.desk.id);
5379 let mut ask = || {
5380 std::fs::copy(w.server.join(¬es), w.phone_folder.join(¬es)).unwrap();
5382 std::fs::File::options().write(true).open(w.phone_folder.join(¬es)).unwrap().set_modified(std::time::SystemTime::now()).unwrap();
5383 Duration::from_millis(200)
5384 };
5385 let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST);
5386 w.fake.clear_faults();
5387 assert_eq!(came, AlarmNews::SyncApp { heard: BTreeSet::new() }, "the notes are no news of the desk's session");
5388 let (wrong, taken, doubts) = alarm_verdict(&w, &came, DUE + 60, DUE + 65);
5389 assert!(!wrong && !taken && matches!(doubts.as_slice(), [Doubt::Unheard { closed: true, .. }]), "{doubts:?}");
5390 static AGAIN: Mutex<()> = Mutex::new(());
5392 let other = alarm_scene("alarm-entry-came", DUE - 3_000, true);
5393 other.fake.fault(|_| Some(Fault::Status(503)));
5394 let entry = format!("devices/{}.device", other.desk.id);
5395 let mut ask = || {
5396 std::fs::copy(other.server.join(&entry), other.phone_folder.join(&entry)).unwrap();
5397 Duration::from_millis(200)
5398 };
5399 let came = alarm_news_with(&other.phone.memory, &other.phone_folder, &other.phone.id, KEY, Some(other.fake.login()), &AGAIN, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST);
5400 other.fake.clear_faults();
5401 assert!(matches!(came, AlarmNews::SyncApp { ref heard } if heard.contains(&other.desk.id)), "{came:?}");
5402 let (wrong, _, doubts) = alarm_verdict(&other, &came, DUE + 60, DUE + 65);
5403 assert!(!wrong && matches!(doubts.as_slice(), [Doubt::Working { .. }] | [Doubt::Coming { .. }]), "the desk in use, as its entry says: {doubts:?}");
5404 }
5405
5406 #[test]
5410 fn a_clock_set_back_never_hides_the_alarms_own_pull() {
5411 static ONE_PULL: Mutex<()> = Mutex::new(());
5412 let w = alarm_scene("alarm-forward-back", DUE - 3_000, false);
5413 assert!(w.pull(DUE + 600).problem.is_none());
5414 let mut never = || -> Duration { panic!("the alarm's own pull went through: the sync app is not asked") };
5415 let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut never, TEST);
5416 assert_eq!(came, AlarmNews::Pulled);
5417 assert_eq!(State::load(&w.phone.memory).last, DUE + 60, "the time from before the set-back gives way");
5418 let (_, _, doubts) = alarm_verdict(&w, &came, DUE + 60, DUE + 65);
5419 assert!(doubts.is_empty(), "{doubts:?}");
5420 }
5421
5422 #[test]
5426 fn a_round_found_again_grown_stays_left_in_the_merge() {
5427 let w = World::new("left-regrown");
5428 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
5429 assert!(w.pull(DUE + 10).problem.is_none());
5430 let disk = State::load(&w.phone.memory);
5431 State { left: BTreeMap::new(), tried: DUE + 20, last: DUE + 20, ..disk.clone() }.write(&w.phone.memory).unwrap();
5433 let loaded: BTreeMap<String, u64> = [("r-1.jsonl".to_string(), 100)].into();
5435 let ours = State { tried: DUE + 21, last: DUE + 21, left: [("r-1.jsonl".to_string(), 200)].into(), ..disk };
5436 ours.save_pulled(&w.phone.memory, DUE + 25, &loaded).unwrap();
5437 let merged = State::load(&w.phone.memory);
5438 assert_eq!(merged.left.get("r-1.jsonl"), Some(&200), "{:?}", merged.left);
5439 assert!(!merged.pulls_well(&w.phone_folder));
5440 }
5441
5442 #[test]
5448 fn only_a_newer_entry_or_health_claim_is_news_of_a_device() {
5449 static ONE_PULL: Mutex<()> = Mutex::new(());
5450 let w = alarm_scene("alarm-notices", DUE - 3_000, true);
5451 crate::lease::renew(&w.server, &KEY, "notices", &w.desk.id, DUE + 20, DUE + 20, false, crate::lease::Rule::FollowsYou, crate::share::written(&w.desk.memory, &w.desk.id)).unwrap();
5452 w.fake.fault(|_| Some(Fault::Status(503)));
5453 let lease = format!("leases/notices/{}.lease", w.desk.id);
5454 let mut ask = || {
5455 let to = w.phone_folder.join(&lease);
5456 std::fs::create_dir_all(to.parent().unwrap()).unwrap();
5457 std::fs::copy(w.server.join(&lease), &to).unwrap();
5458 Duration::from_millis(200)
5459 };
5460 let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST);
5461 w.fake.clear_faults();
5462 assert_eq!(came, AlarmNews::SyncApp { heard: BTreeSet::new() }, "a notices claim is no news of the session");
5463 let (wrong, _, doubts) = alarm_verdict(&w, &came, DUE + 60, DUE + 65);
5464 assert!(!wrong && !doubts.is_empty(), "{doubts:?}");
5465 static AGAIN: Mutex<()> = Mutex::new(());
5467 let touched = alarm_scene("alarm-touched", DUE - 3_000, true);
5468 touched.fake.fault(|_| Some(Fault::Status(503)));
5469 let entry = touched.phone_folder.join(format!("devices/{}.device", touched.desk.id));
5470 let mut ask = || {
5471 std::fs::File::options().write(true).open(&entry).unwrap().set_modified(std::time::SystemTime::now()).unwrap();
5472 Duration::from_millis(200)
5473 };
5474 let came = alarm_news_with(&touched.phone.memory, &touched.phone_folder, &touched.phone.id, KEY, Some(touched.fake.login()), &AGAIN, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST);
5475 touched.fake.clear_faults();
5476 assert_eq!(came, AlarmNews::SyncApp { heard: BTreeSet::new() }, "the same entry at a new time is no news");
5477 let (wrong, _, doubts) = alarm_verdict(&touched, &came, DUE + 60, DUE + 65);
5478 assert!(!wrong && !doubts.is_empty(), "{doubts:?}");
5479 }
5480
5481 #[test]
5487 fn two_pulls_at_once_keep_what_each_found() {
5488 let w = World::new("two-pulls");
5489 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
5490 w.desk.write("health-state.toml", "[taken]\n\"dose@0\" = 1\n");
5491 w.desk.exchange(&w.server, DUE + 10);
5492 w.fake.fault(once(|r| r.method == "PROPFIND", Fault::Late(Duration::from_millis(450))));
5493 std::thread::scope(|scope| {
5494 let slow = scope.spawn(|| pull_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), DUE + 50, TEST));
5495 std::thread::sleep(Duration::from_millis(100));
5496 ROUND_CAP.with(|cap| cap.set(Some(64)));
5498 let quick = pull_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), DUE + 51, TEST);
5500 ROUND_CAP.with(|cap| cap.set(None));
5501 assert!(quick.problem.is_none() && quick.left >= 1, "{quick:?}");
5502 let slow = slow.join().unwrap();
5503 assert!(slow.problem.is_none(), "{slow:?}");
5504 });
5505 let state = State::load(&w.phone.memory);
5506 assert_eq!((state.tried, state.last), (DUE + 51, DUE + 51), "the later begun says when");
5507 assert!(!state.left.is_empty() && !state.pulls_well(&w.phone_folder), "found left by the other, never erased by the slower one's save: {:?}", state.left);
5508 assert!(w.pull(DUE + 120).problem.is_none());
5510 assert!(State::load(&w.phone.memory).pulls_well(&w.phone_folder));
5511 }
5512
5513 #[test]
5518 fn a_round_too_large_for_the_pull_is_said_and_left_to_the_sync_app() {
5519 let w = World::new("too-large");
5520 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
5521 w.desk.write("health-state.toml", "[taken]\n\"dose@0\" = 1\n");
5522 w.desk.exchange(&w.server, DUE + 10);
5523 ROUND_CAP.with(|cap| cap.set(Some(64)));
5524 let pulled = w.pull(DUE + 20);
5525 assert!(pulled.problem.is_none() && pulled.left >= 1, "{pulled:?}");
5526 let state = State::load(&w.phone.memory);
5527 assert!(!state.left.is_empty() && !state.pulls_well(&w.phone_folder) && !state.went_through_since(&w.phone_folder, DUE + 20, DUE + 20), "{:?}", state.left);
5528 w.down();
5530 let pulled = w.pull(DUE + 100);
5531 ROUND_CAP.with(|cap| cap.set(None));
5532 assert_eq!(pulled.left, 0, "{pulled:?}");
5533 assert!(State::load(&w.phone.memory).pulls_well(&w.phone_folder));
5534 }
5535
5536 #[test]
5542 fn the_phones_entry_put_back_older_on_the_server_is_sent_again() {
5543 let w = World::new("entry-back");
5544 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
5545 let entry = format!("devices/{}.device", w.phone.id);
5546 let older = std::fs::read(w.phone_folder.join(&entry)).unwrap();
5547 w.phone.session(&w.phone_folder, |e| e.close(DUE + 10));
5548 assert!(phone_sends(&w, DUE + 20, false).problem.is_none());
5549 let current = std::fs::read(w.phone_folder.join(&entry)).unwrap();
5550 assert_eq!(std::fs::read(w.server.join(&entry)).unwrap(), current);
5551 assert_ne!(older.len(), current.len());
5552 std::fs::write(w.server.join(&entry), &older).unwrap();
5554 w.desk.exchange(&w.server, DUE + 30);
5555 assert!(w.pull(DUE + 40).problem.is_none());
5556 assert_eq!(State::load(&w.phone.memory).own_entry.map(|(size, _)| size), Some(older.len() as u64));
5557 assert!(phone_sends(&w, DUE + 50, false).problem.is_none());
5558 assert_eq!(std::fs::read(w.server.join(&entry)).unwrap(), current, "sent over the copy put back");
5559 }
5560
5561 #[test]
5567 fn found_at_night_the_first_pull_comes_at_once() {
5568 let w = World::new("night");
5569 w.desk.press(true, DUE + 30);
5570 w.desk.exchange(&w.server, DUE + 60);
5571 w.up();
5572 let in_use = |now: i64| crate::devices::all(&w.phone_folder, &KEY).0.iter().any(|e| e.id == w.desk.id && e.working && !e.left && now - e.exported <= sioul_core::health::FRESH + sioul_core::health::SKEW);
5573 let night = |now: i64| Pace { phone: true, shown: false, in_use: in_use(now), urgent: false, step: 15 * 60 };
5574 assert!(!in_use(DUE + 70), "the stale folder tells the desk not in use");
5575 let state = w.find(DUE + 70);
5576 assert!(state.confirmed_for(&w.phone_folder));
5577 attach(&w.phone_folder, &w.phone.memory);
5578 assert!(due(&State::load(&w.phone.memory), DUE + 80, night(DUE + 80)), "found: pulled at once, whatever the hour");
5579 let pulled = w.pull(DUE + 80);
5580 assert!(pulled.problem.is_none() && pulled.listed > 0 && pulled.fetched > 0, "{pulled:?}");
5581 assert!(in_use(DUE + 90), "the desk known in use, from the server");
5582 w.phone.exchange(&w.phone_folder, DUE + 90);
5583 assert!(w.phone.dnd().is_some_and(|p| p.on), "its do-not-disturb too");
5584 let state = State::load(&w.phone.memory);
5585 assert!(!due(&state, DUE + 100, night(DUE + 100)), "not twice in a minute");
5586 w.desk.exchange(&w.server, DUE + 130);
5587 assert!(w.pull(DUE + 140).problem.is_none(), "once a minute while the desk is in use, at night too");
5588 w.desk.session(&w.server, |e| e.close(DUE + 150));
5590 assert!(w.pull(DUE + 200).problem.is_none());
5591 let state = State::load(&w.phone.memory);
5592 assert!(!in_use(DUE + 260) && !due(&state, DUE + 260, night(DUE + 260)));
5593 assert!(due(&state, DUE + 200 + 15 * 60, night(DUE + 200 + 15 * 60)), "a device opened at night is noticed within the step's fifteen minutes");
5594 }
5595
5596 #[test]
5597 fn the_pace_of_pulls() {
5598 let state = State { confirmed: 1_000, tried: 1_000, ..State::default() };
5599 let pace = |phone, shown, in_use, urgent| Pace { phone, shown, in_use, urgent, step: 5 * 60 };
5600 assert!(due(&state, 1_060, pace(false, false, false, false)) && !due(&state, 1_059, pace(false, false, false, false)));
5602 assert!(due(&state, 1_060, pace(true, true, false, false)));
5603 assert!(due(&state, 1_060, pace(true, false, true, false)));
5604 assert!(!due(&state, 1_120, pace(true, false, false, false)) && due(&state, 1_270, pace(true, false, false, false)));
5606 assert!(due(&state, 1_020, pace(true, false, false, true)) && !due(&state, 1_010, pace(true, false, false, true)));
5608 assert!(due(&State { confirmed: 1_005, ..state }, 1_006, pace(true, false, false, false)));
5610 }
5611
5612 #[test]
5615 fn the_pull_alone_is_trusted_only_while_it_goes_through() {
5616 let folder = Path::new("/storage/emulated/0/Documents/Sioul");
5617 let well = State { folder: shown(folder), url: "https://cloud.example.org/remote.php/dav/files/jane/Documents/Sioul/".into(), confirmed: 1_000, last: 1_200, tried: 1_200, ..State::default() };
5618 assert!(well.pulls_well(folder));
5619 assert!(!State { on: Some(false), ..well.clone() }.pulls_well(folder), "switched off here");
5620 assert!(State { on: Some(true), ..well.clone() }.pulls_well(folder));
5621 assert!(!State { confirmed: 0, ..well.clone() }.pulls_well(folder), "not confirmed");
5622 assert!(!State { last: 0, ..well.clone() }.pulls_well(folder), "never pulled yet");
5623 assert!(!State { said: "network:cloud.example.org".into(), ..well.clone() }.pulls_well(folder), "the last pull failed");
5624 assert!(!State { said: "not-found:cloud.example.org".into(), ..well.clone() }.pulls_well(folder), "looked for again, not found");
5625 assert!(!State { mode: MIRROR.into(), ..well.clone() }.pulls_well(folder), "Sioul keeping the folder: no sync app to ask anyway");
5626 assert!(!well.pulls_well(Path::new("/storage/emulated/0/Documents/Other")), "another folder");
5627 }
5628
5629 #[test]
5635 fn a_steps_requests_go_through_one_connection() {
5636 use std::io::{BufRead, BufReader, Write};
5637 use std::sync::atomic::{AtomicUsize, Ordering};
5638 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
5639 let base = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port());
5640 let (connections, requests) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
5641 let (counted, served) = (Arc::clone(&connections), Arc::clone(&requests));
5642 std::thread::spawn(move || {
5644 for stream in listener.incoming().filter_map(Result::ok) {
5645 counted.fetch_add(1, Ordering::SeqCst);
5646 let served = Arc::clone(&served);
5647 std::thread::spawn(move || {
5648 let mut reader = BufReader::new(stream.try_clone().unwrap());
5649 let mut stream = stream;
5650 loop {
5651 let mut line = String::new();
5652 if reader.read_line(&mut line).unwrap_or(0) == 0 {
5653 return;
5654 }
5655 let mut length = 0;
5656 loop {
5657 let mut header = String::new();
5658 reader.read_line(&mut header).unwrap();
5659 if header.trim().is_empty() {
5660 break;
5661 }
5662 if let Some((name, value)) = header.split_once(':')
5663 && name.eq_ignore_ascii_case("content-length")
5664 {
5665 length = value.trim().parse().unwrap();
5666 }
5667 }
5668 let mut body = vec![0; length];
5669 reader.read_exact(&mut body).unwrap();
5670 served.fetch_add(1, Ordering::SeqCst);
5671 let (status, answer) = match line.split_whitespace().next() {
5672 Some("PROPFIND") => {
5674 let path = line.split_whitespace().nth(1).unwrap_or("/");
5675 ("207 Multi-Status", super::fake::multistatus(&format!("<d:response><d:href>{path}</d:href><d:propstat><d:prop><d:resourcetype><d:collection/></d:resourcetype></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>")))
5676 }
5677 Some("GET") => ("200 OK", b"version = 1\n".to_vec()),
5678 _ => ("201 Created", Vec::new()),
5679 };
5680 let head = format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nETag: \"e\"\r\n\r\n", answer.len());
5681 if stream.write_all(head.as_bytes()).and_then(|()| stream.write_all(&answer)).is_err() {
5682 return;
5683 }
5684 }
5685 });
5686 }
5687 });
5688 let login = Login { account: "cloud".into(), url: format!("{base}/remote.php/dav/"), user: "jane".into(), password: Some(PASSWORD.into()) };
5689 let folder = format!("{base}/remote.php/dav/files/jane/Documents/Sioul/");
5690 let pull = Server::new(&login, TEST).unwrap();
5692 pull.list(&folder).unwrap();
5693 assert!(pull.small(&format!("{folder}seal.toml")).unwrap().is_some());
5694 let send = Server::new(&login, Limits { wait: Duration::from_secs(2), stall: Duration::from_secs(3), ..TEST }).unwrap();
5696 assert!(send.stat(&format!("{folder}devices/")).unwrap().is_none_or(|item| item.etag.is_empty()));
5697 let (status, _) = send.put_file(&format!("{folder}phone.toml"), b"read = 1\n".to_vec(), None, 1, &sha1_hex(b"read = 1\n")).unwrap();
5698 assert_eq!(status, 201);
5699 let again = Server::new(&login, TEST).unwrap();
5701 again.list(&folder).unwrap();
5702 assert_eq!(requests.load(Ordering::SeqCst), 5);
5703 assert_eq!(connections.load(Ordering::SeqCst), 1, "one connection for the whole step");
5704 }
5705
5706 #[test]
5707 fn where_the_folder_may_be_on_the_server() {
5708 let found = |folder: &str, configs: &[&str]| places(Path::new(folder), &configs.iter().map(|c| c.to_string()).collect::<Vec<_>>());
5709 assert_eq!(found("/storage/emulated/0/Documents/Sioul", &[]), ["Documents/Sioul", "Sioul", "0/Documents/Sioul"]);
5711 assert_eq!(found("/sdcard/Documents/Sioul/", &[])[0], "Documents/Sioul");
5712 assert_eq!(found("/storage/emulated/0/Android/media/com.nextcloud.client/nextcloud/jane@murena.io/Shared/Sioul", &[])[0], "Shared/Sioul");
5714 let cfg = "[Accounts]\n0\\Folders\\1\\localPath=/home/me/Nextcloud/\n0\\Folders\\1\\targetPath=/\n0\\Folders\\2\\localPath=/home/me/Work\n0\\Folders\\2\\targetPath=/Projects/Work\n0\\url=https://cloud.example.org\n";
5716 assert_eq!(found("/home/me/Nextcloud/Documents/Sioul", &[cfg])[0], "Documents/Sioul");
5717 assert_eq!(found("/home/me/Work/Sioul", &[cfg])[0], "Projects/Work/Sioul");
5718 assert_eq!(found("/home/me/Workshop/Sioul", &[cfg]), ["Sioul", "Workshop/Sioul", "me/Workshop/Sioul"], "not inside /home/me/Work");
5719 assert_eq!(encode_path("Mes documents/Sioul"), "Mes%20documents/Sioul");
5721 assert_eq!(host_of("https://jane@murena.io/remote.php/dav/"), "murena.io");
5722 assert_eq!(kind_of("devices/0f8fad5b-d9cb-469f-a165-70867728950e.device").map(|k| k.0), Some(Kind::Device));
5723 assert_eq!(kind_of("0f8fad5b-d9cb-469f-a165-70867728950e-12.jsonl").map(|k| k.0), Some(Kind::Round));
5724 assert_eq!(kind_of("0f8fad5b-d9cb-469f-a165-70867728950e-12 (conflicted copy).jsonl"), None);
5725 assert_eq!(kind_of("leases/health/desktop.lease").map(|k| k.0), Some(Kind::Claim));
5726 }
5727
5728 #[test]
5729 fn found_on_the_account_given_by_hand_or_not_at_all() {
5730 let w = World::new("given");
5731 State::choose(&w.phone.memory, |s| s.given = "Documents/Sioul".into()).unwrap();
5733 let state = find_with(&w.phone.memory, &w.phone_folder, &[w.fake.login()], &[], DUE, TEST);
5734 assert!(state.confirmed_for(&w.phone_folder), "{state:?}");
5735 State::choose(&w.phone.memory, |s| s.given = w.fake.folder_url("Documents/Sioul")).unwrap();
5737 assert!(find_with(&w.phone.memory, &w.phone_folder, &[w.fake.login()], &[], DUE + 1, TEST).confirmed_for(&w.phone_folder));
5738 State::choose(&w.phone.memory, |s| s.given = "Elsewhere".into()).unwrap();
5740 let state = find_with(&w.phone.memory, &w.phone_folder, &[w.fake.login()], &places(&w.phone_folder, &[]), DUE + 2, TEST);
5741 assert!(state.said.starts_with("not-found:") && !state.confirmed_for(&w.phone_folder), "{state:?}");
5742 State::choose(&w.phone.memory, |s| s.given.clear()).unwrap();
5743 let wrong = Login { password: Some("wrong".into()), ..w.fake.login() };
5745 assert!(find_with(&w.phone.memory, &w.phone_folder, &[wrong], &places(&w.phone_folder, &[]), DUE + 3, TEST).said.starts_with("login:"));
5746 let none = Login { password: None, ..w.fake.login() };
5747 assert_eq!(find_with(&w.phone.memory, &w.phone_folder, &[none], &[], DUE + 4, TEST).said, "no-password:cloud");
5748 assert_eq!(find_with(&w.phone.memory, &w.phone_folder, &[], &[], DUE + 5, TEST).said, "no-account");
5749 let other = Login { url: format!("{}/dav/calendars/jane/", w.fake.base), ..w.fake.login() };
5751 assert_eq!(find_with(&w.phone.memory, &w.phone_folder, &[other], &places(&w.phone_folder, &[]), DUE + 6, TEST).said, "no-account");
5752 let state = State::load(&w.phone.memory);
5754 assert!(!state.look_due(&w.phone_folder, DUE + 7) && state.look_due(&w.phone_folder, DUE + 6 + LOOK_AGAIN));
5755 State::choose(&w.phone.memory, |s| s.asked = DUE + 8).unwrap();
5756 assert!(State::load(&w.phone.memory).look_due(&w.phone_folder, DUE + 9));
5757 }
5758
5759 #[test]
5760 fn your_choices_are_never_written_over_by_a_pull() {
5761 let w = World::new("choices");
5762 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
5763 let held = State::load(&w.phone.memory);
5765 State::choose(&w.phone.memory, |s| s.on = Some(false)).unwrap();
5766 held.save(&w.phone.memory).unwrap();
5767 assert_eq!(State::load(&w.phone.memory).on, Some(false));
5768 State::choose(&w.phone.memory, |s| s.on = None).unwrap();
5769 assert!(State::load(&w.phone.memory).fetching());
5770 }
5771
5772 struct Pair {
5777 base: PathBuf,
5778 fake: Arc<Fake>,
5779 desk: Device,
5780 desk_folder: PathBuf,
5781 phone: Device,
5782 phone_folder: PathBuf,
5783 }
5784
5785 impl Pair {
5786 fn new(name: &str) -> Pair {
5787 let base = scratch(name);
5788 let files = base.join("cloud");
5789 std::fs::create_dir_all(&files).unwrap();
5790 let fake = Fake::start(&files);
5791 let desk = Device::new(&base, "desk", COMPUTER);
5792 let phone = Device::new(&base, "phone", PHONE);
5793 let (desk_folder, phone_folder) = (mirror_of(&desk.memory), mirror_of(&phone.memory));
5794 let login = fake.login();
5795 let opened = open_with(&login, "Documents/Sioul", TEST).unwrap();
5797 assert!(opened.seal.is_none(), "nothing there yet");
5798 put(&desk_folder, "seal.toml", SEAL);
5799 create_with(&login, &opened, &desk_folder, TEST).unwrap();
5800 begin(&desk.memory, &desk_folder, &login, &opened, DUE - 7_300).unwrap();
5801 assert_eq!(create_with(&login, &opened, &desk_folder, TEST), Err("sealed-meanwhile".to_string()));
5803 let joined = open_with(&login, "/Documents/Sioul/", TEST).unwrap();
5805 assert_eq!(joined.seal.as_deref(), Some(SEAL.as_bytes()));
5806 put_bytes(&phone_folder, "seal.toml", joined.seal.as_deref().unwrap());
5807 begin(&phone.memory, &phone_folder, &login, &joined, DUE - 7_250).unwrap();
5808 let pair = Pair { base, fake, desk, desk_folder, phone, phone_folder };
5809 pair.desk.session(&pair.desk_folder, |e| e.start(DUE - 7_200));
5810 pair.phone.session(&pair.phone_folder, |e| e.start(DUE - 7_200));
5811 pair.turn(true, DUE - 7_100);
5812 pair.turn(false, DUE - 7_090);
5813 pair.turn(true, DUE - 7_080);
5814 pair
5815 }
5816
5817 fn device(&self, desk: bool) -> (&Device, &Path) {
5818 if desk { (&self.desk, &self.desk_folder) } else { (&self.phone, &self.phone_folder) }
5819 }
5820
5821 fn step(&self, desk: bool, now: i64, whole: bool) -> Pulled {
5822 let (device, folder) = self.device(desk);
5823 step_with(&device.memory, folder, &device.id, &self.fake.login(), now, whole, TEST)
5824 }
5825
5826 fn sealed(&self, desk: bool, now: i64) -> Pulled {
5828 let (device, folder) = self.device(desk);
5829 sealed_step_with(&device.memory, folder, &device.id, &self.fake.login(), &KEY, now, false, TEST)
5830 }
5831
5832 fn turn(&self, desk: bool, now: i64) -> Outcome {
5836 let before = self.step(desk, now, true);
5837 assert!(before.problem.is_none(), "{before:?}");
5838 let outcome = self.device(desk).0.exchange(self.device(desk).1, now);
5839 let after = self.step(desk, now, false);
5840 assert!(after.problem.is_none(), "{after:?}");
5841 if self.sealed(desk, now).fetched > 0 {
5842 self.device(desk).0.exchange(self.device(desk).1, now);
5843 let _ = self.step(desk, now, false);
5844 }
5845 outcome
5846 }
5847 }
5848
5849 impl Drop for Pair {
5850 fn drop(&mut self) {
5851 let _ = std::fs::remove_dir_all(&self.base);
5852 }
5853 }
5854
5855 #[test]
5856 fn two_devices_without_a_sync_app_share_through_the_server_alone() {
5857 let p = Pair::new("mirror");
5858 p.desk.press(true, DUE + 10);
5860 p.turn(true, DUE + 20);
5861 p.turn(false, DUE + 30);
5862 let press = p.phone.dnd().expect("the desk's press");
5863 assert!(press.on && press.from == p.desk.id, "{press:?}");
5864 p.phone.write("health-state.toml", "[taken]\n\"levo@1800000000\" = 1800000040\n");
5866 p.turn(false, DUE + 40);
5867 p.turn(true, DUE + 50);
5868 assert!(p.desk.read("health-state.toml").contains("levo@1800000000"));
5869 p.phone.press(false, DUE + 60);
5871 p.turn(false, DUE + 70);
5872 p.turn(true, DUE + 80);
5873 assert!(!p.desk.dnd().unwrap().on);
5874 let phone = p.desk.sees(&p.desk_folder, &p.phone, DUE + 90);
5876 assert!(phone.complete && phone.said.as_ref().is_some_and(|s| s.working && s.exported >= DUE + 70), "{phone:?}");
5877 assert!(doubts_now(DUE - 100, DUE + 90, None, &[phone]).is_empty(), "a dose due before its last export is known");
5878 assert_eq!(pull_with(&p.phone.memory, &p.phone_folder, &p.phone.id, &p.fake.login(), DUE + 95, TEST).problem.as_deref(), Some("mirror"));
5880 assert!(attach(&p.phone_folder, &p.phone.memory).is_none());
5881 }
5882
5883 #[test]
5884 fn a_device_never_writes_another_devices_file() {
5885 let p = Pair::new("own");
5886 for minute in 0..6 {
5887 for desk in [true, false] {
5888 let device = p.device(desk).0;
5889 if minute % 2 == 0 {
5890 device.press(minute % 4 == 0, DUE + minute * 60 + 1);
5891 }
5892 p.fake.forget_seen();
5893 p.turn(desk, DUE + minute * 60 + if desk { 0 } else { 30 });
5894 for line in p.fake.seen() {
5895 let path = line.split(' ').nth(1).unwrap_or_default();
5896 if line.starts_with("PUT ") || line.starts_with("DELETE ") {
5897 assert!(path.contains(&device.id) || path.contains("/blobs/"), "{line}");
5898 } else {
5899 assert!(line.starts_with("GET ") || line.starts_with("PROPFIND ") || line.starts_with("MKCOL "), "{line}");
5900 }
5901 }
5902 }
5903 }
5904 }
5905
5906 #[test]
5907 fn a_refused_or_failed_upload_is_never_lost() {
5908 let p = Pair::new("refused");
5909 p.phone.write("health-state.toml", "[taken]\n\"levo@1800000000\" = 1800000040\n");
5910 for (n, fault) in [Fault::Status(412), Fault::Status(500), Fault::Drop].into_iter().enumerate() {
5911 p.fake.clear_faults();
5912 let id = p.phone.id.clone();
5913 p.fake.fault(move |r| (r.method == "PUT" && r.path.contains(&id) && r.path.ends_with(".jsonl")).then_some(fault));
5914 let now = DUE + 40 + n as i64 * 10;
5915 let _ = p.step(false, now, true);
5917 p.phone.exchange(&p.phone_folder, now);
5918 let pushed = p.step(false, now, false);
5919 assert!(pushed.problem.is_some(), "{fault:?}: {pushed:?}");
5920 assert!(p.phone.read("health-state.toml").contains("levo@"), "nothing lost here");
5921 p.turn(true, now + 5);
5922 assert!(!p.desk.read("health-state.toml").contains("levo@"), "{fault:?}: not there yet");
5923 }
5924 p.fake.clear_faults();
5926 p.turn(false, DUE + 100);
5927 p.turn(true, DUE + 110);
5928 assert!(p.desk.read("health-state.toml").contains("levo@1800000000"));
5929 let heard = crate::share::heard(&p.desk.memory, &p.desk.id);
5930 assert!(!heard.broken.contains_key(&p.phone.id), "{heard:?}");
5931 }
5932
5933 #[test]
5934 fn its_copy_lost_a_device_goes_on_after_its_last_record() {
5935 let p = Pair::new("lost");
5936 p.phone.write("health-state.toml", "[taken]\n\"one@1800000000\" = 1800000010\n");
5937 p.turn(false, DUE + 20);
5938 p.turn(true, DUE + 25);
5939 for entry in std::fs::read_dir(&p.phone_folder).unwrap().filter_map(Result::ok) {
5941 if entry.file_name() != "seal.toml" {
5942 let _ = std::fs::remove_dir_all(entry.path());
5943 let _ = std::fs::remove_file(entry.path());
5944 }
5945 }
5946 p.phone.write("health-state.toml", "[taken]\n\"one@1800000000\" = 1800000010\n\"two@1800000000\" = 1800000030\n");
5947 let outcome = p.turn(false, DUE + 40);
5948 assert!(outcome.problems.iter().all(|p| !p.starts_with("share-own")), "{:?}", outcome.problems);
5949 p.turn(true, DUE + 50);
5950 let record = p.desk.read("health-state.toml");
5951 assert!(record.contains("one@") && record.contains("two@"), "{record}");
5952 let heard = crate::share::heard(&p.desk.memory, &p.desk.id);
5953 assert!(!heard.broken.contains_key(&p.phone.id), "no gap nor cut on the desk: {heard:?}");
5954 assert!(p.phone_folder.join(format!("devices/{}.device", p.desk.id)).exists());
5956 }
5957
5958 #[test]
5959 fn sealed_files_travel_whole_and_go_when_swept() {
5960 let p = Pair::new("blobs");
5961 let source = p.base.join("note.md");
5962 std::fs::write(&source, "a note\n".repeat(1000)).unwrap();
5963 let (hash, _) = crate::blobs::hash_file(&source).unwrap();
5964 crate::blobs::put(&p.desk_folder, &KEY, &source, &hash).unwrap();
5965 let name = crate::blobs::name(&KEY, &hash);
5966 p.turn(true, DUE + 10);
5967 let size = std::fs::metadata(p.desk_folder.join("blobs").join(&name)).unwrap().len();
5969 crate::blobs::Wanted::change(&p.phone.memory, |w| {
5970 w.media.insert(name.clone(), crate::blobs::Want { hash: hash.clone(), size, since: DUE, ..Default::default() });
5971 })
5972 .unwrap();
5973 p.turn(false, DUE + 20);
5974 let sealed = std::fs::read(p.desk_folder.join("blobs").join(&name)).unwrap();
5975 assert_eq!(std::fs::read(p.phone_folder.join("blobs").join(&name)).unwrap(), sealed);
5976 crate::blobs::Wanted::change(&p.phone.memory, |w| w.media.clear()).unwrap();
5977 p.fake.forget_seen();
5979 p.turn(false, DUE + 30);
5980 assert!(p.fake.seen().iter().all(|l| !(l.starts_with("PUT ") && l.contains("/blobs/"))), "{:?}", p.fake.seen());
5981 std::fs::remove_file(p.desk_folder.join("blobs").join(&name)).unwrap();
5983 p.turn(true, DUE + 40);
5984 assert!(!p.fake.files.join("Documents/Sioul/blobs").join(&name).exists());
5985 let _ = p.step(false, DUE + 50 + BLOBS_AGAIN, true);
5989 let files = p.phone.memory.with_file_name("files.json");
5990 let mut said: serde_json::Value = serde_json::from_slice(&std::fs::read(&files).unwrap()).unwrap();
5991 assert!(said.as_object_mut().unwrap().remove("needed").is_some(), "said by each exchange of this Sioul");
5992 std::fs::write(&files, serde_json::to_vec(&said).unwrap()).unwrap();
5993 let sealed = p.sealed(false, DUE + 50 + BLOBS_AGAIN);
5994 assert!(sealed.listed_bytes > 0 && sealed.let_go == 0 && p.phone_folder.join("blobs").join(&name).exists(), "listed, none let go: {sealed:?}");
5995 p.turn(false, DUE + 60 + BLOBS_AGAIN + CHECK_AGAIN);
5996 assert!(!p.phone_folder.join("blobs").join(&name).exists(), "let go on the phone");
5997 }
5998
5999 #[test]
6004 fn a_slow_or_failing_sealed_file_never_holds_back_the_records() {
6005 let p = Pair::new("sealed-last");
6006 let made = |seed: u32| {
6007 let mut x = seed.wrapping_mul(2_654_435_761) | 1;
6008 (0..600_000)
6009 .map(|_| {
6010 x ^= x << 13;
6011 x ^= x >> 17;
6012 x ^= x << 5;
6013 x as u8
6014 })
6015 .collect::<Vec<u8>>()
6016 };
6017 let table = p.desk.roots.data.join("spam").join("table.bin");
6018 std::fs::create_dir_all(table.parent().unwrap()).unwrap();
6019 std::fs::write(&table, made(1)).unwrap();
6020 let mut taken = String::from("[taken]\n");
6021 for (n, fault) in [Fault::Status(500), Fault::Drop, Fault::Late(Duration::from_secs(6))].into_iter().enumerate() {
6023 p.fake.clear_faults();
6024 p.fake.fault(move |r| (r.method == "PUT" && r.path.contains("/blobs/")).then_some(fault));
6025 let now = DUE + 100 + n as i64 * 60;
6026 taken.push_str(&format!("\"levo@{}\" = {}\n", 1_800_000_000 + n, 1_800_000_040 + n));
6028 p.desk.write("health-state.toml", &taken);
6029 let _ = p.step(true, now, true);
6030 p.desk.exchange(&p.desk_folder, now);
6031 let pushed = p.step(true, now, false);
6032 assert!(pushed.problem.is_some(), "{fault:?}: {pushed:?}");
6033 p.turn(false, now + 30);
6035 assert!(p.phone.read("health-state.toml").contains(&format!("levo@{}", 1_800_000_000 + n)), "{fault:?}");
6036 }
6037 p.fake.clear_faults();
6039 p.turn(true, DUE + 400);
6040 p.turn(false, DUE + 410);
6041 let phone_table = p.phone.roots.data.join("spam").join("table.bin");
6042 assert_eq!(std::fs::read(&phone_table).unwrap(), std::fs::read(&table).unwrap());
6043 std::fs::write(&table, made(2)).unwrap();
6047 p.turn(true, DUE + 500);
6048 let _ = p.step(false, DUE + 505, true);
6049 p.phone.write("health-state.toml", &format!("{taken}\"two@1800000000\" = 1800000090\n"));
6050 p.phone.exchange(&p.phone_folder, DUE + 506);
6051 p.fake.fault(|r| (r.method == "GET" && r.path.contains("/blobs/")).then_some(Fault::Late(Duration::from_secs(2))));
6052 let stepped = p.step(false, DUE + 510, false);
6053 assert!(stepped.problem.is_none() && stepped.sent > 0, "{stepped:?}");
6054 let sealed = p.sealed(false, DUE + 510);
6055 assert!(sealed.problem.is_some() && sealed.fetched == 0, "{sealed:?}");
6056 p.fake.clear_faults();
6057 p.turn(true, DUE + 520);
6058 assert!(p.desk.read("health-state.toml").contains("two@1800000000"));
6059 p.turn(false, DUE + 530);
6061 assert_ne!(std::fs::read(&phone_table).unwrap(), std::fs::read(&table).unwrap(), "within its backoff");
6062 p.turn(false, DUE + 580);
6063 assert_eq!(std::fs::read(&phone_table).unwrap(), std::fs::read(&table).unwrap());
6064 }
6065
6066 #[test]
6069 fn a_device_with_a_sync_app_and_one_without_share_one_folder() {
6070 let w = World::new("mixed");
6071 let lone = Device::new(&w.base, "lone", PHONE);
6072 let folder = mirror_of(&lone.memory);
6073 let login = w.fake.login();
6074 let opened = open_with(&login, "Documents/Sioul", TEST).unwrap();
6075 put_bytes(&folder, "seal.toml", opened.seal.as_deref().expect("sealed by the desk"));
6076 begin(&lone.memory, &folder, &login, &opened, DUE).unwrap();
6077 lone.session(&folder, |e| e.start(DUE));
6078 let turn = |now: i64| {
6079 assert!(step_with(&lone.memory, &folder, &lone.id, &login, now, true, TEST).problem.is_none());
6080 lone.exchange(&folder, now);
6081 assert!(step_with(&lone.memory, &folder, &lone.id, &login, now, false, TEST).problem.is_none());
6082 };
6083 turn(DUE + 10);
6084 w.desk.press(true, DUE + 20);
6086 w.desk.exchange(&w.server, DUE + 30);
6087 turn(DUE + 40);
6088 assert!(lone.dnd().is_some_and(|p| p.on && p.from == w.desk.id));
6089 lone.write("health-state.toml", "[taken]\n\"levo@1800000000\" = 1800000050\n");
6091 turn(DUE + 50);
6092 w.desk.exchange(&w.server, DUE + 60);
6093 assert!(w.desk.read("health-state.toml").contains("levo@1800000000"));
6094 let seen = w.desk.sees(&w.server, &lone, DUE + 70);
6095 assert!(seen.complete && seen.said.is_some_and(|s| s.working), "the desk knows how the phone is");
6096 }
6097
6098 #[test]
6099 fn keeping_the_folder_is_inert_until_set_up() {
6100 let w = World::new("inert");
6101 w.fake.forget_seen();
6102 let stepped = step_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), DUE, true, TEST);
6103 assert_eq!(stepped.problem.as_deref(), Some("not-confirmed"));
6104 assert!(w.fake.seen().is_empty(), "not a request");
6105 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
6107 w.fake.forget_seen();
6108 assert_eq!(step_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), DUE + 1, false, TEST).problem.as_deref(), Some("not-confirmed"));
6109 assert!(w.fake.seen().is_empty());
6110 }
6111
6112 fn mtime(path: &Path) -> Option<SystemTime> {
6115 std::fs::metadata(path).and_then(|m| m.modified()).ok()
6116 }
6117
6118 fn set_mtime(path: &Path, at: SystemTime) {
6119 if let Ok(file) = std::fs::File::options().write(true).open(path) {
6120 let _ = file.set_modified(at);
6121 }
6122 }
6123
6124 fn sync_up(local: &Path, server: &Path) {
6126 std::fs::create_dir_all(server.parent().unwrap()).unwrap();
6127 std::fs::write(server, std::fs::read(local).unwrap()).unwrap();
6128 fake::bump(server);
6129 if let Some(at) = mtime(local) {
6130 set_mtime(server, at);
6131 }
6132 }
6133
6134 fn sync_down(server: &Path, local: &Path) {
6136 std::fs::create_dir_all(local.parent().unwrap()).unwrap();
6137 let temporary = local.with_file_name(format!(".{}.download", local.file_name().unwrap().to_string_lossy()));
6138 std::fs::write(&temporary, std::fs::read(server).unwrap()).unwrap();
6139 if let Some(at) = mtime(server) {
6140 set_mtime(&temporary, at);
6141 }
6142 std::fs::rename(&temporary, local).unwrap();
6143 }
6144
6145 fn folder_of(relative: &str) -> &str {
6146 relative.rfind('/').map_or("", |at| &relative[..at])
6147 }
6148
6149 #[derive(Default)]
6160 struct EDrive {
6161 known: BTreeMap<String, (String, Option<SystemTime>)>,
6162 dirs: BTreeMap<String, Option<SystemTime>>,
6163 refused: BTreeMap<String, u32>,
6164 uploads: usize,
6165 refusals: usize,
6166 downloads_over_newer: usize,
6167 }
6168
6169 impl EDrive {
6170 fn scan(&mut self, fake: &Fake, server: &Path, phone: &Path) {
6171 let _turn = fake.turn.lock().unwrap();
6172 let (there, here) = (walk(server), walk(phone));
6173 let mut downloads: BTreeSet<String> = BTreeSet::new();
6174 for (relative, bytes) in there.iter().filter(|(r, _)| *r != "seal.toml") {
6175 let etag = fake::etag_of(&server.join(relative));
6176 if self.known.get(relative).is_some_and(|(known, _)| *known == etag) {
6177 continue;
6178 }
6179 if here.get(relative).map(Vec::len) == Some(bytes.len()) {
6180 let time = self.known.get(relative).and_then(|(_, t)| *t);
6181 self.known.insert(relative.clone(), (etag, time));
6182 } else {
6183 downloads.insert(relative.clone());
6184 }
6185 }
6186 let mut uploads: Vec<String> = Vec::new();
6187 for relative in here.keys().filter(|r| *r != "seal.toml") {
6188 let dir = folder_of(relative).to_string();
6189 let looked = self.dirs.get(&dir).copied().flatten() != mtime(&phone.join(&dir));
6190 let later = self.known.get(relative).is_none_or(|(_, t)| mtime(&phone.join(relative)) > *t);
6191 if looked && later {
6192 downloads.remove(relative);
6193 uploads.push(relative.clone());
6194 }
6195 }
6196 for relative in downloads {
6197 if here.get(&relative).is_some_and(|mine| there[&relative].len() < mine.len()) {
6198 self.downloads_over_newer += 1;
6199 }
6200 sync_down(&server.join(&relative), &phone.join(&relative));
6201 self.known.insert(relative.clone(), (fake::etag_of(&server.join(&relative)), mtime(&phone.join(&relative))));
6202 }
6203 for relative in uploads {
6204 self.upload(server, phone, &relative);
6205 }
6206 self.dirs = here.keys().map(|r| folder_of(r).to_string()).chain([String::new()]).map(|d| (d.clone(), mtime(&phone.join(&d)))).collect();
6207 }
6208
6209 fn upload(&mut self, server: &Path, phone: &Path, relative: &str) {
6211 if self.refused.get(relative).is_some_and(|n| *n >= 4) {
6212 return;
6213 }
6214 let there = server.join(relative);
6215 if let Some((known, _)) = self.known.get(relative)
6216 && there.is_file()
6217 && fake::etag_of(&there) != *known
6218 {
6219 self.refusals += 1;
6220 *self.refused.entry(relative.to_string()).or_insert(0) += 1;
6221 return;
6222 }
6223 sync_up(&phone.join(relative), &there);
6224 self.uploads += 1;
6225 self.known.insert(relative.to_string(), (fake::etag_of(&there), mtime(&phone.join(relative))));
6226 }
6227
6228 fn closed(&mut self, fake: &Fake, server: &Path, phone: &Path, relative: &str) {
6230 let _turn = fake.turn.lock().unwrap();
6231 if phone.join(relative).is_file() {
6232 self.upload(server, phone, relative);
6233 }
6234 }
6235 }
6236
6237 #[derive(Default)]
6244 struct Client {
6245 journal: BTreeMap<String, (String, u64, Option<SystemTime>)>,
6246 conflicts: usize,
6247 uploads: usize,
6248 }
6249
6250 impl Client {
6251 fn stamp(path: &Path) -> Option<(u64, Option<SystemTime>)> {
6252 std::fs::metadata(path).ok().filter(|m| m.is_file()).map(|m| (m.len(), m.modified().ok()))
6253 }
6254
6255 fn sync(&mut self, fake: &Fake, server: &Path, local: &Path) {
6256 let _turn = fake.turn.lock().unwrap();
6257 let names: BTreeSet<String> = walk(server).into_keys().chain(walk(local).into_keys()).filter(|r| r != "seal.toml" && !r.contains("(conflicted copy")).collect();
6258 for relative in names {
6259 let (there, here) = (server.join(&relative), local.join(&relative));
6260 let etag = there.is_file().then(|| fake::etag_of(&there));
6261 let stamp = Self::stamp(&here);
6262 let known = self.journal.get(&relative).cloned();
6263 let local_changed = known.as_ref().map(|(_, size, at)| Some((*size, *at))) != Some(stamp);
6264 let remote_changed = known.as_ref().map(|(e, ..)| e.clone()) != etag;
6265 let remember = |journal: &mut BTreeMap<String, (String, u64, Option<SystemTime>)>| {
6266 let (size, at) = Self::stamp(&here).unwrap_or((0, None));
6267 journal.insert(relative.clone(), (fake::etag_of(&there), size, at));
6268 };
6269 match (stamp.is_some(), etag.is_some()) {
6270 (true, true) if !local_changed && !remote_changed => {}
6271 (true, true) if !remote_changed => {
6272 sync_up(&here, &there);
6273 self.uploads += 1;
6274 remember(&mut self.journal);
6275 }
6276 (true, true) if !local_changed => {
6277 sync_down(&there, &here);
6278 remember(&mut self.journal);
6279 }
6280 (true, true) => {
6281 if std::fs::read(&here).ok() != std::fs::read(&there).ok() {
6282 let name = here.file_name().unwrap().to_string_lossy().to_string();
6283 let (stem, extension) = name.rsplit_once('.').unwrap_or((&name, ""));
6284 std::fs::rename(&here, here.with_file_name(format!("{stem} (conflicted copy 20261008 104200).{extension}"))).unwrap();
6285 sync_down(&there, &here);
6286 self.conflicts += 1;
6287 }
6288 remember(&mut self.journal);
6289 }
6290 (true, false) if known.is_some() && !local_changed => {
6291 let _ = std::fs::remove_file(&here);
6292 self.journal.remove(&relative);
6293 }
6294 (true, false) => {
6295 sync_up(&here, &there);
6296 self.uploads += 1;
6297 remember(&mut self.journal);
6298 }
6299 (false, true) if known.is_some() && !remote_changed => {
6300 let _ = std::fs::remove_file(&there);
6301 self.journal.remove(&relative);
6302 }
6303 (false, true) => {
6304 sync_down(&there, &here);
6305 remember(&mut self.journal);
6306 }
6307 (false, false) => {
6308 self.journal.remove(&relative);
6309 }
6310 }
6311 }
6312 }
6313 }
6314
6315 fn records_there(server: &Path, id: &str) -> BTreeMap<String, usize> {
6317 walk(server).into_iter().filter(|(r, _)| r.starts_with(id) && r.ends_with(".jsonl")).map(|(r, b)| (r, b.len())).collect()
6318 }
6319
6320 fn never_shorter(before: &mut BTreeMap<String, usize>, now: BTreeMap<String, usize>, at: &str) {
6321 for (name, size) in &now {
6322 assert!(before.get(name).is_none_or(|was| size >= was), "{at}: {name} shorter there: {size} after {:?}", before.get(name));
6323 }
6324 before.extend(now);
6325 }
6326
6327 fn phone_sends(w: &World, now: i64, again: bool) -> Pulled {
6329 send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &BTreeMap::new(), now, again, TEST)
6330 }
6331
6332 #[test]
6337 fn beside_a_sync_app_only_this_devices_files_go_up_and_never_twice() {
6338 let w = World::new("beside");
6339 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
6340 w.phone.write("health-state.toml", "[taken]\n\"levo@1800000000\" = 1800000040\n");
6342 w.phone.exchange(&w.phone_folder, DUE + 40);
6343 w.fake.forget_seen();
6344 let sent = phone_sends(&w, DUE + 41, false);
6345 assert!(sent.problem.is_none() && sent.sent >= 2, "{sent:?}");
6346 let mine = |line: &String| line.split(' ').nth(1).is_some_and(|path| path.contains(&w.phone.id));
6347 for line in w.fake.seen() {
6348 if line.starts_with("PUT ") || line.starts_with("DELETE ") {
6349 assert!(mine(&line), "never another device's file: {line}");
6350 }
6351 }
6352 for (relative, bytes) in walk(&w.phone_folder).into_iter().filter(|(r, _)| r.contains(&w.phone.id)) {
6353 assert_eq!(std::fs::read(w.server.join(&relative)).ok(), Some(bytes), "{relative} there as here");
6354 }
6355 let kept = std::fs::read_to_string(sending_path(&w.phone.memory)).unwrap();
6356 assert!(!kept.contains(PASSWORD) && !kept.contains("Basic"), "{kept}");
6357 w.desk.exchange(&w.server, DUE + 45);
6358 assert!(w.desk.read("health-state.toml").contains("levo@1800000000"), "the desk has it, no sync app between");
6359 w.up();
6361 for relative in walk(&w.phone_folder).into_keys().filter(|r| r.contains(&w.phone.id)) {
6362 fake::bump(&w.server.join(relative));
6363 }
6364 w.fake.forget_seen();
6365 let again = phone_sends(&w, DUE + 50, false);
6366 assert_eq!((again.sent, &again.problem), (0, &None));
6367 assert!(w.fake.seen().is_empty(), "{:?}", w.fake.seen());
6368 w.phone.write("health-state.toml", "[taken]\n\"levo@1800000000\" = 1800000040\n\"iron@1800000000\" = 1800000060\n");
6370 w.phone.exchange(&w.phone_folder, DUE + 60);
6371 {
6372 let _turn = w.fake.turn.lock().unwrap();
6373 for relative in walk(&w.phone_folder).into_keys().filter(|r| r.contains(&w.phone.id)) {
6374 sync_up(&w.phone_folder.join(&relative), &w.server.join(&relative));
6375 }
6376 }
6377 w.fake.forget_seen();
6378 let raced = phone_sends(&w, DUE + 61, false);
6379 assert!(raced.problem.is_none() && raced.sent == 0 && raced.same >= 2, "{raced:?} {:?}", w.fake.seen());
6380 assert!(w.fake.seen().iter().all(|l| !(l.starts_with("PUT ") && !l.ends_with(" 412"))), "{:?}", w.fake.seen());
6381 let round = walk(&w.phone_folder).into_keys().find(|r| r.starts_with(&w.phone.id) && r.ends_with(".jsonl")).unwrap();
6383 let longer = std::fs::read(w.server.join(&round)).unwrap();
6384 let first = longer.iter().position(|b| *b == b'\n').unwrap() + 1;
6385 std::fs::write(w.phone_folder.join(&round), &longer[..first]).unwrap();
6386 w.fake.forget_seen();
6387 let cut = phone_sends(&w, DUE + 70, false);
6388 assert!(cut.problem.is_none(), "{cut:?}");
6389 assert_eq!(std::fs::read(w.server.join(&round)).unwrap(), longer, "never shorter there");
6390 assert!(w.fake.seen().iter().all(|l| !(l.starts_with("PUT ") && l.contains(&round))), "{:?}", w.fake.seen());
6391 assert_eq!(phone_sends(&w, DUE + 72, true).longer, 1);
6393 let mut appended = longer[..first].to_vec();
6394 appended.extend_from_slice(&longer[first..first + (longer.len() - first) / 2]);
6395 appended.push(b'\n');
6396 std::fs::write(w.phone_folder.join(&round), &appended).unwrap();
6397 w.fake.forget_seen();
6398 assert!(phone_sends(&w, DUE + 75, false).problem.is_none());
6399 assert_eq!(std::fs::read(w.server.join(&round)).unwrap(), longer, "an append here, still shorter, never sent over the longer copy");
6400 assert!(w.fake.seen().iter().all(|l| !(l.starts_with("PUT ") && l.contains(&round))), "{:?}", w.fake.seen());
6401 State::choose(&w.phone.memory, |s| s.send = Some(false)).unwrap();
6403 w.fake.forget_seen();
6404 assert_eq!(phone_sends(&w, DUE + 80, true).problem.as_deref(), Some("off"));
6405 assert!(w.fake.seen().is_empty());
6406 assert!(State::load(&w.phone.memory).fetching());
6407 }
6408
6409 #[test]
6411 fn sealed_files_go_up_whole_and_only_this_devices() {
6412 let w = World::new("beside-blobs");
6413 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
6414 let source = w.base.join("note.md");
6415 std::fs::write(&source, "a note\n".repeat(500)).unwrap();
6416 let (hash, _) = crate::blobs::hash_file(&source).unwrap();
6417 crate::blobs::put(&w.phone_folder, &KEY, &source, &hash).unwrap();
6418 let name = crate::blobs::name(&KEY, &hash);
6419 let size = std::fs::metadata(w.phone_folder.join("blobs").join(&name)).unwrap().len();
6420 put(&w.phone_folder, "blobs/0123456789abcdef", "another device's");
6422 let half: BTreeMap<String, u64> = [(name.clone(), size + 1)].into();
6424 send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &half, DUE + 10, false, TEST);
6425 assert!(!w.server.join("blobs").join(&name).exists());
6426 let whole: BTreeMap<String, u64> = [(name.clone(), size)].into();
6427 let sent = send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &whole, DUE + 20, false, TEST);
6428 assert!(sent.problem.is_none(), "{sent:?}");
6429 assert_eq!(std::fs::read(w.server.join("blobs").join(&name)).unwrap(), std::fs::read(w.phone_folder.join("blobs").join(&name)).unwrap());
6430 assert!(!w.server.join("blobs/0123456789abcdef").exists(), "never another device's");
6431 std::fs::remove_file(w.phone_folder.join("blobs").join(&name)).unwrap();
6433 send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &BTreeMap::new(), DUE + 30, false, TEST);
6434 assert!(!w.server.join("blobs").join(&name).exists());
6435 }
6436
6437 #[test]
6440 fn send_everything_again_after_the_server_lost_this_devices_files() {
6441 let w = World::new("again");
6442 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
6443 w.phone.write("health-state.toml", "[taken]\n\"levo@1800000000\" = 1800000040\n");
6444 w.phone.exchange(&w.phone_folder, DUE + 40);
6445 assert!(phone_sends(&w, DUE + 41, false).problem.is_none());
6446 let own: Vec<String> = walk(&w.phone_folder).into_keys().filter(|r| r.contains(&w.phone.id)).collect();
6447 assert!(own.len() >= 2, "{own:?}");
6448 for relative in &own {
6450 std::fs::remove_file(w.server.join(relative)).unwrap();
6451 }
6452 assert!(crate::devices::all(&w.server, &KEY).0.iter().all(|e| e.id != w.phone.id));
6453 w.fake.forget_seen();
6455 assert_eq!(phone_sends(&w, DUE + 50, false).sent, 0);
6456 assert!(w.fake.seen().is_empty());
6457 let again = phone_sends(&w, DUE + 60, true);
6459 assert!(again.problem.is_none() && again.sent == own.len(), "{again:?}");
6460 for relative in &own {
6461 assert_eq!(std::fs::read(w.server.join(relative)).ok(), std::fs::read(w.phone_folder.join(relative)).ok(), "{relative}");
6462 }
6463 assert_eq!(Sending::load(&w.phone.memory).again, Again { at: DUE + 60, sent: own.len(), same: 0, said: String::new() });
6464 w.desk.exchange(&w.server, DUE + 70);
6465 assert!(w.desk.read("health-state.toml").contains("levo@1800000000"));
6466 assert!(crate::devices::all(&w.server, &KEY).0.iter().any(|e| e.id == w.phone.id), "its entry back");
6467 w.fake.forget_seen();
6469 let twice = phone_sends(&w, DUE + 80, true);
6470 assert!(twice.problem.is_none() && twice.sent == 0 && twice.same == own.len(), "{twice:?}");
6471 assert!(w.fake.seen().iter().all(|l| !l.starts_with("PUT ")), "{:?}", w.fake.seen());
6472 w.fake.fault(|_| Some(Fault::Status(503)));
6474 let failed = phone_sends(&w, DUE + 90, true);
6475 assert!(failed.problem.as_deref().is_some_and(|p| p.starts_with("server:")), "{failed:?}");
6476 assert!(Sending::load(&w.phone.memory).again.said.starts_with("server:"));
6477 }
6478
6479 #[test]
6486 fn doses_are_never_known_wrongly_with_edrive_sending_too() {
6487 let w = World::new("edrive-sends");
6488 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
6489 let mut edrive = EDrive::default();
6490 edrive.scan(&w.fake, &w.server, &w.phone_folder);
6491 let mut marked: Vec<(String, i64)> = Vec::new();
6492 let mut sizes = BTreeMap::new();
6493 for minute in 0..150 {
6494 let now = DUE + minute * 60;
6495 if (minute % 10 == 0 || [17, 19, 77, 79].contains(&minute)) && minute < 130 {
6497 marked.push((format!("dose@{minute}"), now));
6498 let record: String = marked.iter().map(|(d, at)| format!("\"{d}\" = {at}\n")).collect();
6499 w.phone.write("health-state.toml", &format!("[taken]\n{record}"));
6500 }
6501 w.phone.exchange(&w.phone_folder, now);
6502 let round = walk(&w.phone_folder).into_keys().filter(|r| r.starts_with(&w.phone.id) && r.ends_with(".jsonl")).max_by_key(|r| r.trim_end_matches(".jsonl").rsplit('-').next().unwrap().parse::<u32>().unwrap()).unwrap();
6503 match minute % 6 {
6504 1 => edrive.scan(&w.fake, &w.server, &w.phone_folder),
6506 3 => edrive.closed(&w.fake, &w.server, &w.phone_folder, &round),
6508 _ => {}
6509 }
6510 let sent = phone_sends(&w, now + 1, false);
6511 assert!(sent.problem.is_none(), "minute {minute}: {sent:?}");
6512 if minute % 4 == 0 {
6514 edrive.scan(&w.fake, &w.server, &w.phone_folder);
6515 }
6516 never_shorter(&mut sizes, records_there(&w.server, &w.phone.id), &format!("minute {minute}"));
6517 let outcome = w.desk.exchange(&w.server, now + 20);
6518 assert!(outcome.problems.iter().all(|p| !p.starts_with("share-other-cut") && !p.starts_with("share-other-gap")), "minute {minute}: {:?}", outcome.problems);
6519 if let Some(entry) = crate::devices::all(&w.server, &KEY).0.into_iter().find(|e| e.id == w.phone.id)
6520 && let Some(wrote) = entry.wrote
6521 && crate::share::heard(&w.desk.memory, &w.desk.id).complete(&w.phone.id, wrote)
6522 {
6523 let record = w.desk.read("health-state.toml");
6524 for (dose, at) in marked.iter().filter(|(_, at)| *at <= entry.exported) {
6525 assert!(record.contains(dose.as_str()), "minute {minute}: the desk takes {dose} ({at}) as known, its record lacks it:\n{record}");
6526 }
6527 }
6528 }
6529 let record = w.desk.read("health-state.toml");
6530 for (dose, _) in &marked {
6531 assert!(record.contains(dose.as_str()), "{dose} never came:\n{record}");
6532 }
6533 assert!(walk(&w.server).keys().all(|r| !r.contains("conflicted")));
6534 eprintln!("eDrive beside Sioul: {} uploads, {} refused, {} brought down over a newer copy", edrive.uploads, edrive.refusals, edrive.downloads_over_newer);
6535 assert!(edrive.downloads_over_newer > 0 && edrive.refusals > 0, "the races were run");
6536 }
6537
6538 #[test]
6544 fn doses_are_never_known_wrongly_with_the_nextcloud_client_sending_too() {
6545 let w = World::new("client-sends");
6546 let desk_folder = w.base.join("desk-folder").join("Sioul");
6547 copy_dir(&w.server, &desk_folder);
6548 State::choose(&w.desk.memory, |s| s.given = "Documents/Sioul".into()).unwrap();
6549 assert!(find_with(&w.desk.memory, &desk_folder, &[w.fake.login()], &[], DUE, TEST).confirmed_for(&desk_folder));
6550 let mut client = Client::default();
6551 client.sync(&w.fake, &w.server, &desk_folder);
6552 let phone_down = || {
6553 let _turn = w.fake.turn.lock().unwrap();
6554 for (relative, bytes) in walk(&w.server).into_iter().filter(|(r, _)| !r.contains(&w.phone.id)) {
6555 put_bytes(&w.phone_folder, &relative, &bytes);
6556 }
6557 };
6558 let phone_up = || {
6559 let _turn = w.fake.turn.lock().unwrap();
6560 for relative in walk(&w.phone_folder).into_keys().filter(|r| r.contains(&w.phone.id)) {
6561 sync_up(&w.phone_folder.join(&relative), &w.server.join(&relative));
6562 }
6563 };
6564 let mut marked: Vec<(String, i64)> = Vec::new();
6565 let mut sizes = BTreeMap::new();
6566 let mut cuts = 0;
6567 for minute in 0..150 {
6568 let now = DUE + minute * 60;
6569 if (minute % 10 == 0 || [21, 71].contains(&minute)) && minute < 130 {
6571 marked.push((format!("dose@{minute}"), now));
6572 let record: String = marked.iter().map(|(d, at)| format!("\"{d}\" = {at}\n")).collect();
6573 w.desk.write("health-state.toml", &format!("[taken]\n{record}"));
6574 }
6575 let outcome = w.desk.exchange(&desk_folder, now);
6576 cuts += outcome.problems.iter().filter(|p| *p == "share-own-cut").count();
6577 if minute % 5 == 1 {
6579 client.sync(&w.fake, &w.server, &desk_folder);
6580 }
6581 let sent = send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), now + 1, false, TEST);
6582 assert!(sent.problem.is_none(), "minute {minute}: {sent:?}");
6583 if minute % 3 == 0 {
6584 client.sync(&w.fake, &w.server, &desk_folder);
6585 }
6586 never_shorter(&mut sizes, records_there(&w.server, &w.desk.id), &format!("minute {minute}"));
6587 phone_down();
6588 let read = w.phone.exchange(&w.phone_folder, now + 30);
6589 assert!(read.problems.iter().all(|p| !p.starts_with("share-other-cut") && !p.starts_with("share-other-gap")), "minute {minute}: {:?}", read.problems);
6590 phone_up();
6591 if let Some(entry) = crate::devices::all(&w.phone_folder, &KEY).0.into_iter().find(|e| e.id == w.desk.id)
6592 && let Some(wrote) = entry.wrote
6593 && crate::share::heard(&w.phone.memory, &w.phone.id).complete(&w.desk.id, wrote)
6594 {
6595 let record = w.phone.read("health-state.toml");
6596 for (dose, at) in marked.iter().filter(|(_, at)| *at <= entry.exported) {
6597 assert!(record.contains(dose.as_str()), "minute {minute}: the phone takes {dose} ({at}) as known, its record lacks it:\n{record}");
6598 }
6599 }
6600 }
6601 let record = w.phone.read("health-state.toml");
6602 for (dose, _) in &marked {
6603 assert!(record.contains(dose.as_str()), "{dose} never came:\n{record}");
6604 }
6605 assert!(walk(&w.server).keys().all(|r| !r.contains("conflicted")), "a conflicted copy never goes up");
6606 eprintln!("Nextcloud client beside Sioul: {} uploads, {} conflicted copies here, {} rounds started again", client.uploads, client.conflicts, cuts);
6607 assert!(client.conflicts > 0 && cuts > 0, "the races were run");
6608 }
6609
6610 fn desk_beside(w: &World) -> PathBuf {
6615 let desk_folder = w.base.join("desk-folder").join("Sioul");
6616 copy_dir(&w.server, &desk_folder);
6617 State::choose(&w.desk.memory, |s| s.given = "Documents/Sioul".into()).unwrap();
6618 assert!(find_with(&w.desk.memory, &desk_folder, &[w.fake.login()], &[], DUE, TEST).confirmed_for(&desk_folder));
6619 desk_folder
6620 }
6621
6622 fn a_desk_holding_much(w: &World, folder: &Path, at: i64) {
6624 let many: String = (0..8000).map(|n| format!("sender{n}@example.org\n")).collect();
6625 put(&w.desk.roots.config, "known-senders.txt", &many);
6626 w.desk.exchange(folder, at);
6627 }
6628
6629 fn desk_records(folder: &Path, id: &str) -> Vec<(String, u64)> {
6630 let mut out: Vec<(String, u64)> = walk(folder).into_iter().filter(|(r, _)| r.starts_with(id) && r.ends_with(".jsonl")).map(|(r, b)| (r, b.len() as u64)).collect();
6631 out.sort_by_key(|(r, _)| r.trim_end_matches(".jsonl").rsplit('-').next().unwrap().parse::<u32>().unwrap());
6632 out
6633 }
6634
6635 #[test]
6641 fn a_large_file_goes_through_a_slow_uplink() {
6642 let w = World::new("slow-uplink");
6643 let desk_folder = desk_beside(&w);
6644 a_desk_holding_much(&w, &desk_folder, DUE);
6645 let (round, size) = desk_records(&desk_folder, &w.desk.id).into_iter().find(|(_, s)| *s > 1 << 20).unwrap();
6647 assert!(size > 1 << 20, "a large records file: {size}");
6648 *w.fake.throttle.lock().unwrap() = Some(512 << 10);
6650 let flat = Limits { large: Duration::from_secs(1), floor: 1 << 30, ..TEST };
6651 let tried = send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), DUE + 10, false, flat);
6652 let failure = tried.failure.clone().expect("said in detail");
6653 assert!(failure.kind == "timeout" && failure.file == "records" && failure.sent > 0 && failure.total == size && failure.seconds >= 1, "{failure:?}");
6654 assert!(tried.held == 1, "{tried:?}");
6655 let sending = Sending::load(&w.desk.memory);
6656 assert_eq!(sending.failure.as_ref().map(|f| f.kind.as_str()), Some("timeout"));
6657 assert!(sending.held.get(&round).is_some_and(|h| h.tries == 1 && h.until == DUE + 10 + backoff(1)), "{:?}", sending.held);
6658 assert!(sending.rate > 0, "its rate measured from what went");
6659 let spent = w.fake.received.load(std::sync::atomic::Ordering::Relaxed);
6661 let early = send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), DUE + 40, false, flat);
6662 assert!(early.held == 0 && w.fake.received.load(std::sync::atomic::Ordering::Relaxed) - spent < size / 2, "{early:?}");
6663 assert!(std::fs::metadata(w.server.join(&round)).map_or(0, |m| m.len()) < size);
6664 let scaled = Limits { large: Duration::from_secs(1), floor: 128 << 10, stall: Duration::from_secs(3), ..TEST };
6667 let went = send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), DUE + 10 + backoff(1), false, scaled);
6668 assert!(went.problem.is_none() && went.held == 0, "{went:?}");
6669 assert_eq!(std::fs::read(w.server.join(&round)).unwrap(), std::fs::read(desk_folder.join(&round)).unwrap());
6670 let sending = Sending::load(&w.desk.memory);
6671 assert!(sending.held.is_empty() && sending.failure.is_none() && sending.last == DUE + 10 + backoff(1), "{sending:?}");
6672 }
6673
6674 #[test]
6676 fn a_failed_send_waits_for_its_backoff() {
6677 let w = World::new("backoff");
6678 let desk_folder = desk_beside(&w);
6679 w.desk.write("health-state.toml", "[taken]\n\"levo@1800000000\" = 1800000040\n");
6680 w.desk.exchange(&desk_folder, DUE + 40);
6681 w.fake.fault(|r| (r.method == "PUT").then_some(Fault::Status(503)));
6682 let failed = send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), DUE + 41, false, TEST);
6683 assert!(failed.problem.as_deref().is_some_and(|p| p.starts_with("server:")), "{failed:?}");
6684 assert_eq!(Sending::load(&w.desk.memory).retry_at, DUE + 41 + backoff(1));
6685 w.fake.forget_seen();
6686 let early = send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), DUE + 41 + backoff(1) - 1, false, TEST);
6687 assert!(early.problem.as_deref().is_some_and(|p| p.starts_with("waiting:")) && w.fake.seen().is_empty(), "{early:?} {:?}", w.fake.seen());
6688 let again = send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), DUE + 41 + backoff(1), false, TEST);
6690 assert!(again.problem.is_some());
6691 assert_eq!(Sending::load(&w.desk.memory).retry_at, DUE + 41 + backoff(1) + backoff(2));
6692 assert_eq!((backoff(1), backoff(2), backoff(3), backoff(9)), (60, 120, 300, 3600));
6693 w.fake.clear_faults();
6695 let at = DUE + 41 + backoff(1) + backoff(2);
6696 assert!(send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), at, false, TEST).problem.is_none());
6697 let sending = Sending::load(&w.desk.memory);
6698 assert!(sending.tries == 0 && sending.retry_at == 0 && sending.failure.is_none());
6699 }
6700
6701 #[test]
6705 fn a_text_to_send_reaches_the_phone_while_the_records_crawl() {
6706 let w = World::new("texts-lane");
6707 let desk_folder = desk_beside(&w);
6708 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
6709 a_desk_holding_much(&w, &desk_folder, DUE);
6710 *w.fake.throttle.lock().unwrap() = Some(256 << 10);
6712 let slow = Limits { large: Duration::from_secs(2), floor: 64 << 10, stall: Duration::from_secs(5), pull: Duration::from_secs(120), ..TEST };
6713 let crawling = {
6714 let (memory, folder, id, login) = (w.desk.memory.clone(), desk_folder.clone(), w.desk.id.clone(), w.fake.login());
6715 std::thread::spawn(move || send_with(&memory, &folder, &id, &login, &BTreeMap::new(), DUE + 10, false, slow))
6716 };
6717 std::thread::sleep(Duration::from_millis(500));
6718 let source = w.base.join("desk-texts-send.jsonl");
6720 std::fs::write(&source, "a sealed request line\n").unwrap();
6721 let written = Instant::now();
6722 assert!(place_texts(&desk_folder, &w.desk.id, &source).unwrap());
6723 let urgent = send_urgent_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), DUE + 11, slow);
6724 assert!(urgent.problem.is_none() && urgent.sent == 1, "{urgent:?}");
6725 let pulled = pull_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), DUE + 20, TEST);
6727 assert!(pulled.problem.is_none(), "{pulled:?}");
6728 assert_eq!(std::fs::read(texts_file(&cache_of(&w.phone.memory), &w.desk.id)).unwrap(), b"a sealed request line\n");
6729 assert!(!crawling.is_finished(), "the records still going up");
6730 assert!(written.elapsed() < Duration::from_secs(10), "{:?}", written.elapsed());
6731 let crawled = crawling.join().unwrap();
6732 assert!(crawled.problem.is_none(), "{crawled:?}");
6733 assert!(!own_plain(&desk_folder, &w.desk.id).iter().any(|r| r.starts_with("texts/")));
6735 assert!(!place_texts(&desk_folder, &w.desk.id, &source).unwrap(), "unchanged: nothing to place");
6736 std::fs::remove_file(&source).unwrap();
6737 assert!(place_texts(&desk_folder, &w.desk.id, &source).unwrap() && !texts_file(&desk_folder, &w.desk.id).exists());
6738 }
6739
6740 #[test]
6744 fn a_reply_to_send_goes_on_the_texts_lane() {
6745 let w = World::new("replies-lane");
6746 let desk_folder = desk_beside(&w);
6747 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
6748 let (texts, asks) = (w.base.join("desk-texts-send.jsonl"), w.base.join("desk-replies-ask.jsonl"));
6749 std::fs::write(&texts, "a sealed request line\n").unwrap();
6750 std::fs::write(&asks, "r1:a sealed reply line\n").unwrap();
6751 assert!(place_texts(&desk_folder, &w.desk.id, &texts).unwrap());
6752 assert!(place_asks(&desk_folder, &w.desk.id, &asks).unwrap());
6753 assert_eq!(own_urgent(&desk_folder, &w.desk.id).len(), 2);
6754 let urgent = send_urgent_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), DUE + 11, TEST);
6755 assert!(urgent.problem.is_none() && urgent.sent == 2, "{urgent:?}");
6756 let pulled = pull_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), DUE + 20, TEST);
6757 assert!(pulled.problem.is_none(), "{pulled:?}");
6758 assert_eq!(std::fs::read(asks_file(&cache_of(&w.phone.memory), &w.desk.id)).unwrap(), b"r1:a sealed reply line\n");
6759 assert!(!own_plain(&desk_folder, &w.desk.id).iter().any(|r| r.starts_with("phone-messages/")), "never by the main lane");
6760 assert_eq!(file_kind(&format!("phone-messages/ask/{}.jsonl", w.desk.id)), "replies");
6761 std::fs::remove_file(&asks).unwrap();
6762 assert!(place_asks(&desk_folder, &w.desk.id, &asks).unwrap() && !asks_file(&desk_folder, &w.desk.id).exists());
6763 }
6764
6765 #[test]
6771 fn a_request_s_media_go_up_before_it_on_the_urgent_lane() {
6772 let w = World::new("texts-media-lane");
6773 let desk_folder = desk_beside(&w);
6774 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
6775 let picture = w.base.join("picture.jpg");
6777 std::fs::write(&picture, b"\xFF\xD8\xFF a picture").unwrap();
6778 let (hash, _) = crate::blobs::hash_file(&picture).unwrap();
6779 let sealed = crate::blobs::put(&desk_folder, &KEY, &picture, &hash).unwrap().unwrap();
6780 let name = crate::blobs::name(&KEY, &hash);
6781 let source = w.base.join("desk-texts-send.jsonl");
6782 std::fs::write(&source, "a sealed request line naming it\n").unwrap();
6783 assert!(place_texts(&desk_folder, &w.desk.id, &source).unwrap());
6784 w.fake.forget_seen();
6785 let media = BTreeMap::from([(name.clone(), sealed)]);
6786 let urgent = send_urgent_media(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), DUE + 11, &media, TEST);
6787 assert!(urgent.problem.is_none() && urgent.sent == 2, "{urgent:?}");
6788 let puts: Vec<String> = w.fake.seen().into_iter().filter(|line| line.starts_with("PUT ")).collect();
6789 let at = |what: &str| puts.iter().position(|line| line.contains(what));
6790 assert!(at(&format!("blobs/{name}")).is_some() && at(&format!("blobs/{name}")) < at("texts/send/"), "the medium first: {puts:?}");
6791 assert!(!w.fake.seen().iter().any(|line| line.starts_with("PROPFIND") && line.contains("/blobs")), "{:?}", w.fake.seen());
6792 assert_eq!(send_urgent_media(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), DUE + 12, &media, TEST).sent, 0);
6794 assert!(send_urgent_media(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), DUE + 13, &BTreeMap::new(), TEST).problem.is_none());
6795 assert!(!Sending::load_lane(&w.desk.memory, Lane::Urgent).sent.keys().any(|relative| relative.starts_with("blobs/")));
6796 crate::blobs::Wanted::change(&w.phone.memory, |wanted| {
6798 wanted.media.insert(name.clone(), crate::blobs::Want { hash: hash.clone(), size: sealed, since: DUE, ..Default::default() });
6799 })
6800 .unwrap();
6801 assert!(wanted_due(&w.phone.memory, &w.phone_folder, DUE + 20));
6802 let fetched = fetch_wanted_with(&w.phone.memory, &w.phone_folder, &w.fake.login(), &KEY, DUE + 20, false, TEST);
6803 assert_eq!((fetched.fetched, fetched.problem.as_deref()), (1, None), "{fetched:?}");
6804 let copy = w.cache().join("blobs").join(&name);
6806 assert!(crate::blobs::check(©, &KEY, &hash).is_ok() && !w.phone_folder.join("blobs").join(&name).exists());
6807 }
6808
6809 #[test]
6813 fn bytes_an_hour_of_a_computer_writing_each_minute() {
6814 let w = World::new("an-hour");
6815 let desk_folder = desk_beside(&w);
6816 a_desk_holding_much(&w, &desk_folder, DUE);
6817 assert!(send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), DUE + 1, false, TEST).problem.is_none());
6818 let start = w.fake.received.load(std::sync::atomic::Ordering::Relaxed);
6819 let mut minutes: String = String::new();
6820 for minute in 1..=60 {
6821 let now = DUE + minute * 60;
6822 minutes.push_str(&format!("\"dose@{minute}\" = {now}\n"));
6823 w.desk.write("health-state.toml", &format!("[taken]\n{minutes}"));
6824 w.desk.exchange(&desk_folder, now);
6825 crate::lease::renew(&desk_folder, &KEY, "health", &w.desk.id, now, now, false, crate::lease::Rule::FollowsYou, crate::share::written(&w.desk.memory, &w.desk.id)).unwrap();
6826 let sent = send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), now + 1, false, TEST);
6827 assert!(sent.problem.is_none(), "minute {minute}: {sent:?}");
6828 }
6829 let hour = w.fake.received.load(std::sync::atomic::Ordering::Relaxed) - start;
6830 let largest = desk_records(&desk_folder, &w.desk.id).last().map_or(0, |(_, s)| *s);
6831 eprintln!("A computer writing each minute, sent beside its sync app: {hour} bytes in an hour; its current records' file {largest} bytes");
6832 assert!(hour < 3 << 20, "{hour} bytes an hour");
6833 assert!(largest < 64 << 10, "{largest}");
6834 w.phone.exchange(&w.server, DUE + 3_700);
6836 assert!(w.phone.read("health-state.toml").contains("dose@60"));
6837 }
6838
6839 impl Device {
6842 fn exchange_notes(&self, folder: &Path, notes: &Path, now: i64) -> Outcome {
6844 let config = Config { notes_root: Some(notes.display().to_string()), ..Config::default() };
6845 let stores = crate::share::stores_of(&config, &self.roots, &|part| part == "notes" || crate::share::shared_by_default(part, &config));
6846 crate::share::exchange(&Sharing { folder, computer: &self.id, key: &KEY, memory: &self.memory, files: true, hurry: None }, &stores, now * 1000).unwrap()
6847 }
6848 }
6849
6850 fn notes_world(name: &str) -> (World, PathBuf, PathBuf) {
6854 let w = World::new(name);
6855 let (desk_notes, phone_notes) = (w.base.join("desk-notes"), w.base.join("phone-notes"));
6856 for dir in [&desk_notes, &phone_notes] {
6857 std::fs::create_dir_all(dir).unwrap();
6858 }
6859 w.desk.exchange_notes(&w.server, &desk_notes, DUE + 10);
6860 w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 20);
6861 w.up();
6862 assert!(w.find(DUE + 30).confirmed_for(&w.phone_folder));
6863 (w, desk_notes, phone_notes)
6864 }
6865
6866 fn desk_writes(w: &World, desk_notes: &Path, file: &str, text: &str, now: i64) -> String {
6868 put(desk_notes, file, text);
6869 w.desk.exchange_notes(&w.server, desk_notes, now);
6870 let name = crate::blobs::name(&KEY, &crate::blobs::hash_file(&desk_notes.join(file)).unwrap().0);
6871 assert!(w.server.join("blobs").join(&name).exists());
6872 name
6873 }
6874
6875 fn fetch(w: &World, now: i64, frugal: bool) -> Pulled {
6876 fetch_wanted_with(&w.phone.memory, &w.phone_folder, &w.fake.login(), &KEY, now, frugal, TEST)
6877 }
6878
6879 fn blob_gets(w: &World) -> usize {
6880 w.fake.seen().iter().filter(|l| l.starts_with("GET ") && l.contains("/blobs/")).count()
6881 }
6882
6883 #[test]
6890 fn a_note_waited_for_is_fetched_by_its_name_and_written_within_one_step() {
6891 let (w, desk_notes, phone_notes) = notes_world("wanted");
6892 let name = desk_writes(&w, &desk_notes, "lease.md", "The lease, signed.\n", DUE + 60);
6893 w.fake.forget_seen();
6894 let pulled = w.pull(DUE + 70);
6896 assert!(pulled.problem.is_none(), "{pulled:?}");
6897 let first = w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 71);
6898 assert_eq!(first.waits.get("no-blob"), Some(&1), "{first:?}");
6899 assert_eq!((first.received, first.pending), (0, 1));
6900 assert!(!phone_notes.join("lease.md").exists());
6901 let wanted = crate::blobs::Wanted::load(&w.phone.memory);
6902 assert_eq!(wanted.blobs.keys().cloned().collect::<Vec<_>>(), [name.clone()]);
6903 assert!(wanted.blobs[&name].size == 19 && wanted.blobs[&name].since == DUE + 71, "{wanted:?}");
6904 assert!(wanted_due(&w.phone.memory, &w.phone_folder, DUE + 71));
6905 let fetched = fetch(&w, DUE + 72, false);
6907 assert_eq!((fetched.fetched, fetched.problem.as_deref()), (1, None), "{fetched:?}");
6908 assert!(w.cache().join("blobs").join(&name).exists());
6909 assert!(!w.phone_folder.join("blobs").join(&name).exists(), "never written into the synced folder");
6910 assert!(!wanted_due(&w.phone.memory, &w.phone_folder, DUE + 72), "fetched: no longer due");
6911 let second = w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 73);
6912 assert_eq!((second.received, second.pending), (1, 0), "{second:?}");
6913 assert!(second.waits.is_empty() && second.problems.is_empty(), "{second:?}");
6914 assert_eq!(std::fs::read_to_string(phone_notes.join("lease.md")).unwrap(), "The lease, signed.\n");
6915 assert!(!crate::blobs::Wanted::path(&w.phone.memory).exists(), "nothing waits");
6916 assert_eq!(blob_gets(&w), 1, "{:?}", w.fake.seen());
6918 for line in w.fake.seen() {
6919 assert!(!(line.starts_with("PROPFIND") && line.contains("/blobs")), "{line}");
6920 assert!(line.starts_with("PROPFIND ") || line.starts_with("GET "), "{line}");
6921 }
6922 w.fake.forget_seen();
6924 assert_eq!(fetch(&w, DUE + 74, false), Pulled::default());
6925 assert!(w.fake.seen().is_empty());
6926 tidy(&w.phone.memory, &w.phone_folder, &w.phone.id);
6928 assert!(!w.cache().join("blobs").join(&name).exists());
6929 }
6930
6931 #[test]
6936 fn a_damaged_copy_fetched_is_refused_and_fetched_again() {
6937 let (w, desk_notes, phone_notes) = notes_world("wanted-damaged");
6938 let name = desk_writes(&w, &desk_notes, "lease.md", "The lease, signed.\n", DUE + 60);
6939 assert!(w.pull(DUE + 70).problem.is_none());
6940 assert_eq!(w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 71).waits.get("no-blob"), Some(&1));
6941 let whole = std::fs::read(w.server.join("blobs").join(&name)).unwrap();
6943 let mut damaged = whole.clone();
6944 let middle = damaged.len() / 2;
6945 damaged[middle] ^= 0x10;
6946 put_bytes(&w.cache(), &format!("blobs/{name}"), &damaged);
6947 let outcome = w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 72);
6948 assert_eq!(outcome.waits.get("no-blob"), Some(&1), "{outcome:?}");
6949 assert!(outcome.problems.is_empty(), "the copy fetched is not the folder's: nothing said damaged, {outcome:?}");
6950 assert!(!w.cache().join("blobs").join(&name).exists());
6951 assert!(!phone_notes.join("lease.md").exists());
6952 std::fs::write(w.server.join("blobs").join(&name), &damaged).unwrap();
6954 let fetched = fetch(&w, DUE + 73, false);
6955 assert_eq!((fetched.fetched, fetched.held), (0, 1), "{fetched:?}");
6956 assert!(!w.cache().join("blobs").join(&name).exists());
6957 assert_eq!(crate::blobs::Wanted::load(&w.phone.memory).blobs[&name].again, DUE + 73 + 60);
6958 w.fake.forget_seen();
6959 assert_eq!(fetch(&w, DUE + 100, false).fetched, 0);
6960 assert_eq!(blob_gets(&w), 0, "within its wait: not asked");
6961 std::fs::write(w.server.join("blobs").join(&name), &whole).unwrap();
6963 assert_eq!(fetch(&w, DUE + 140, false).fetched, 1);
6964 let outcome = w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 141);
6965 assert_eq!(outcome.received, 1, "{outcome:?}");
6966 assert_eq!(std::fs::read_to_string(phone_notes.join("lease.md")).unwrap(), "The lease, signed.\n");
6967 }
6968
6969 #[test]
6973 fn a_sealed_file_not_on_the_server_yet_is_asked_again_within_minutes() {
6974 let (w, desk_notes, phone_notes) = notes_world("wanted-late");
6975 let name = desk_writes(&w, &desk_notes, "lease.md", "The lease, signed.\n", DUE + 60);
6976 let aside = w.base.join("aside-blob");
6977 std::fs::rename(w.server.join("blobs").join(&name), &aside).unwrap();
6978 assert!(w.pull(DUE + 70).problem.is_none());
6979 assert_eq!(w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 71).waits.get("no-blob"), Some(&1));
6980 let mut at = DUE + 72;
6981 for _ in 0..8 {
6982 let fetched = fetch(&w, at, false);
6983 assert!(fetched.fetched == 0 && fetched.problem.is_none(), "{fetched:?}");
6984 let again = crate::blobs::Wanted::load(&w.phone.memory).blobs[&name].again;
6985 assert!(again > at && again <= at + WANTED_AGAIN, "{again} after {at}");
6986 at = again;
6987 }
6988 std::fs::rename(&aside, w.server.join("blobs").join(&name)).unwrap();
6989 assert_eq!(fetch(&w, at, false).fetched, 1);
6990 assert_eq!(w.phone.exchange_notes(&w.phone_folder, &phone_notes, at + 1).received, 1);
6991 }
6992
6993 #[test]
6997 fn a_sealed_file_fetched_then_brought_by_the_sync_app_is_kept_once() {
6998 let (w, desk_notes, phone_notes) = notes_world("wanted-brought");
6999 let name = desk_writes(&w, &desk_notes, "lease.md", "The lease, signed.\n", DUE + 60);
7000 assert!(w.pull(DUE + 70).problem.is_none());
7001 assert_eq!(w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 71).waits.get("no-blob"), Some(&1));
7002 assert_eq!(fetch(&w, DUE + 72, false).fetched, 1);
7003 w.down();
7005 let (fetched, brought) = (std::fs::read(w.cache().join("blobs").join(&name)).unwrap(), std::fs::read(w.phone_folder.join("blobs").join(&name)).unwrap());
7006 assert_eq!(fetched, brought, "the same file");
7007 tidy(&w.phone.memory, &w.phone_folder, &w.phone.id);
7008 assert!(!w.cache().join("blobs").join(&name).exists(), "the synced folder holds it: the copy fetched goes");
7009 assert!(crate::blobs::Wanted::path(&w.phone.memory).exists(), "the note still waits");
7010 let outcome = w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 73);
7011 assert_eq!(outcome.received, 1, "{outcome:?}");
7012 assert!(!crate::blobs::Wanted::path(&w.phone.memory).exists());
7013 }
7014
7015 #[test]
7020 fn the_sealed_files_fetched_are_pruned() {
7021 let (w, desk_notes, phone_notes) = notes_world("wanted-pruned");
7022 let name = desk_writes(&w, &desk_notes, "lease.md", "The lease, signed.\n", DUE + 60);
7023 assert!(w.pull(DUE + 70).problem.is_none());
7024 assert_eq!(w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 71).waits.get("no-blob"), Some(&1));
7025 assert_eq!(fetch(&w, DUE + 72, false).fetched, 1);
7026 let blobs = w.cache().join("blobs");
7027 put(&blobs, "0123abcd", "a copy no note waits for");
7028 tidy(&w.phone.memory, &w.phone_folder, &w.phone.id);
7029 assert!(blobs.join(&name).exists(), "waited for: kept");
7030 assert!(!blobs.join("0123abcd").exists());
7031 set_mtime(&blobs.join(&name), SystemTime::now() - Duration::from_secs(4 * 86_400));
7032 tidy(&w.phone.memory, &w.phone_folder, &w.phone.id);
7033 assert!(!blobs.join(&name).exists(), "past a few days");
7034 assert!(!crate::blobs::Wanted::path(&w.phone.memory).exists(), "no longer waited for");
7035 w.fake.forget_seen();
7036 assert_eq!(fetch(&w, DUE + 80, false).fetched, 0);
7037 assert_eq!(blob_gets(&w), 0);
7038 assert_eq!(w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 81).waits.get("no-blob"), Some(&1));
7040 assert_eq!(fetch(&w, DUE + 82, false).fetched, 1);
7041 State::choose(&w.phone.memory, |s| s.on = Some(false)).unwrap();
7043 tidy(&w.phone.memory, &w.phone_folder, &w.phone.id);
7044 assert!(!blobs.join(&name).exists());
7045 assert_eq!(fetch(&w, DUE + 83, false).problem.as_deref(), Some("not-confirmed"));
7046 }
7047
7048 #[test]
7052 fn sealed_files_waited_for_come_bounded_and_paced() {
7053 let (w, _, _) = notes_world("wanted-paced");
7054 let made = |seed: u32, size: usize| {
7055 let mut x = seed.wrapping_mul(2_654_435_761) | 1;
7056 (0..size)
7057 .map(|_| {
7058 x ^= x << 13;
7059 x ^= x >> 17;
7060 x ^= x << 5;
7061 x as u8
7062 })
7063 .collect::<Vec<u8>>()
7064 };
7065 let sealed = |seed: u32, size: usize| {
7067 let source = w.base.join(format!("source-{seed}"));
7068 std::fs::write(&source, made(seed, size)).unwrap();
7069 let (hash, size) = crate::blobs::hash_file(&source).unwrap();
7070 crate::blobs::put(&w.server, &KEY, &source, &hash).unwrap();
7071 let name = crate::blobs::name(&KEY, &hash);
7072 crate::blobs::Wanted::change(&w.phone.memory, |wanted| {
7073 wanted.blobs.insert(name.clone(), crate::blobs::Want { hash, size, since: DUE, ..Default::default() });
7074 })
7075 .unwrap();
7076 name
7077 };
7078 let papers = [sealed(1, 1_500_000), sealed(2, 1_200_000)];
7079 let note = sealed(3, 3_000);
7080 let first = fetch(&w, DUE + 100, true);
7082 assert_eq!((first.fetched, first.paced), (2, 1), "{first:?}");
7083 let blobs = w.cache().join("blobs");
7084 assert!(blobs.join(¬e).exists() && blobs.join(&papers[1]).exists() && !blobs.join(&papers[0]).exists());
7085 w.fake.forget_seen();
7086 assert_eq!((fetch(&w, DUE + 400, true).fetched, blob_gets(&w)), (0, 0), "paced: not asked");
7087 assert_eq!(fetch(&w, DUE + 100 + PACE, true).fetched, 1);
7088 assert!(blobs.join(&papers[0]).exists());
7089 let notes: Vec<String> = (10..10 + WANTED_FILES as u32 + 2).map(|seed| sealed(seed, 500)).collect();
7091 assert_eq!(fetch(&w, DUE + 2_000, true).fetched, WANTED_FILES);
7092 assert_eq!(fetch(&w, DUE + 2_001, true).fetched, 2);
7093 assert!(notes.iter().all(|n| blobs.join(n).exists()));
7094 let large: Vec<String> = (40..43).map(|seed| sealed(seed, 7 << 20)).collect();
7096 assert_eq!(fetch(&w, DUE + 3_000, false).fetched, 2, "14 MiB, then the third past 16");
7097 assert_eq!(fetch(&w, DUE + 3_001, false).fetched, 1);
7098 assert!(large.iter().all(|n| blobs.join(n).exists()));
7099 }
7100
7101 fn rounds_there(server: &Path, id: &str) -> BTreeSet<u32> {
7105 walk(server).into_keys().filter(|r| r.starts_with(id) && r.ends_with(".jsonl")).filter_map(|r| r.trim_end_matches(".jsonl").rsplit('-').next().and_then(|n| n.parse().ok())).collect()
7106 }
7107
7108 fn rounds_stand(w: &World, round: u32, full_round: u32, read: u32, at: i64) {
7112 let mut memory: serde_json::Value = serde_json::from_slice(&std::fs::read(&w.phone.memory).unwrap()).unwrap();
7113 memory["round"] = round.into();
7114 memory["full_round"] = full_round.into();
7115 std::fs::write(&w.phone.memory, serde_json::to_vec(&memory).unwrap()).unwrap();
7116 put(&w.phone_folder, &format!("{}.toml", w.desk.id), &format!("at = {at}\nround = 1\n\n[read]\n\"{}\" = {read}\n", w.phone.id));
7117 }
7118
7119 #[test]
7128 fn what_edrive_sent_again_is_taken_out_once_past() {
7129 let w = World::new("edrive-again");
7130 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
7131 let round = |n: u32| format!("{}-{n}.jsonl", w.phone.id);
7132 for n in 40..=43 {
7133 put(&w.phone_folder, &round(n), &format!("{{\"round\":{n}}}\n").repeat(n as usize));
7134 }
7135 let seal = |text: &str| {
7136 let source = w.base.join(format!("{}.md", text.len()));
7137 std::fs::write(&source, text.repeat(200)).unwrap();
7138 let (hash, _) = crate::blobs::hash_file(&source).unwrap();
7139 crate::blobs::put(&w.phone_folder, &KEY, &source, &hash).unwrap();
7140 let name = crate::blobs::name(&KEY, &hash);
7141 let size = std::fs::metadata(w.phone_folder.join("blobs").join(&name)).unwrap().len();
7142 (name, size)
7143 };
7144 let (mine, mine_size) = seal("a note of the phone's\n");
7145 let (used, used_size) = seal("a note the desk uses again\n");
7146 let sealed: BTreeMap<String, u64> = [(mine.clone(), mine_size), (used.clone(), used_size)].into();
7147 let sent = send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &sealed, DUE + 10, false, TEST);
7148 assert!(sent.problem.is_none() && sent.taken_out == 0, "{sent:?}");
7149 {
7150 let _turn = w.fake.turn.lock().unwrap();
7151 for relative in [round(40), round(41), format!("blobs/{mine}"), format!("blobs/{used}")] {
7153 sync_up(&w.phone_folder.join(&relative), &w.server.join(&relative));
7154 }
7155 let grown = w.server.join(round(41));
7157 let mut bytes = std::fs::read(&grown).unwrap();
7158 bytes.extend_from_slice(b"{\"later\":1}\n");
7159 std::fs::write(&grown, bytes).unwrap();
7160 fake::bump(&grown);
7161 let again = w.server.join("blobs").join(&used);
7163 fake::bump(&again);
7164 set_mtime(&again, SystemTime::now() + Duration::from_secs(3600));
7165 }
7166 for relative in [round(40), round(41), format!("blobs/{mine}"), format!("blobs/{used}")] {
7169 std::fs::remove_file(w.phone_folder.join(relative)).unwrap();
7170 }
7171 rounds_stand(&w, 43, 42, 43, DUE + 15);
7172 w.fake.forget_seen();
7173 let gone = send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &BTreeMap::new(), DUE + 20, false, TEST);
7174 assert!(gone.problem.is_none(), "{gone:?}");
7175 assert_eq!(gone.taken_out, 2, "{:?}", w.fake.seen());
7176 assert!(!w.server.join(round(40)).exists(), "the past round eDrive sent again: taken out");
7177 assert!(!w.server.join("blobs").join(&mine).exists(), "the swept sealed file eDrive sent again: taken out");
7178 assert!(w.server.join("blobs").join(&used).exists(), "sent again with a new time, by a device that uses it: left");
7179 assert!(w.server.join(round(41)).exists(), "longer there: left (a listing looks at it again)");
7180 assert!(w.server.join(round(42)).exists() && w.server.join(round(43)).exists(), "its last rounds stay");
7181 let deletes: Vec<String> = w.fake.seen().into_iter().filter(|l| l.starts_with("DELETE ")).collect();
7182 assert_eq!(deletes.len(), 6, "each refused once (412), two taken out over their ETag there: {deletes:?}");
7183 assert!(deletes.iter().all(|l| l.contains(&w.phone.id) || l.contains(&mine) || l.contains(&used)), "never another device's file: {deletes:?}");
7184 w.fake.forget_seen();
7186 let quiet = send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &BTreeMap::new(), DUE + 30, false, TEST);
7187 assert!(quiet.problem.is_none() && quiet.taken_out == 0, "{quiet:?}");
7188 assert!(w.fake.seen().is_empty(), "{:?}", w.fake.seen());
7189 }
7190
7191 #[test]
7198 fn past_rounds_left_on_the_server_go_at_the_next_listing() {
7199 let w = World::new("past-rounds");
7200 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
7201 let mut edrive = EDrive::default();
7202 edrive.scan(&w.fake, &w.server, &w.phone_folder);
7203 let lines = |from: usize, count: usize| -> String { (from..from + count).map(|n| format!("phone-sender-{n:06}@example.org\n")).collect() };
7204 let mut list = String::new();
7205 let mut left_there: BTreeMap<String, Vec<u8>> = BTreeMap::new();
7206 let mut now = DUE;
7207 for step in 0..24 {
7210 list.push_str(&lines(step * 1_500, 1_500));
7211 put(&w.phone.roots.config, "known-senders.txt", &list);
7212 now += 600;
7213 w.phone.exchange(&w.phone_folder, now);
7214 let sent = phone_sends(&w, now + 1, false);
7215 assert!(sent.problem.is_none(), "step {step}: {sent:?}");
7216 edrive.scan(&w.fake, &w.server, &w.phone_folder);
7217 for (relative, bytes) in walk(&w.server).into_iter().filter(|(r, _)| r.starts_with(&w.phone.id) && r.ends_with(".jsonl")) {
7219 left_there.insert(relative, bytes);
7220 }
7221 w.desk.exchange(&w.server, now + 30);
7222 edrive.scan(&w.fake, &w.server, &w.phone_folder);
7223 }
7224 let past = crate::share::rounds_past(&w.phone_folder, &w.phone.memory, &w.phone.id, (now + 60) * 1000).unwrap();
7225 assert!(past > 3, "several rounds past their time: below {past}");
7226 assert!(rounds_there(&w.server, &w.phone.id).iter().all(|r| *r >= past), "{:?}, below {past}", rounds_there(&w.server, &w.phone.id));
7228 assert!(rounds_there(&w.phone_folder, &w.phone.id).iter().all(|r| *r >= past));
7229 let put_back = |w: &World| {
7230 let _turn = w.fake.turn.lock().unwrap();
7231 for (relative, bytes) in &left_there {
7232 if !w.server.join(relative).exists() {
7233 put_bytes(&w.server, relative, bytes);
7234 fake::bump(&w.server.join(relative));
7235 }
7236 }
7237 };
7238 put_back(&w);
7240 let stranger = uuid::Uuid::new_v4().to_string();
7241 put(&w.server, &format!("{stranger}-1.jsonl"), "{\"n\":1}\n");
7242 let desks: BTreeMap<String, Vec<u8>> = walk(&w.server).into_iter().filter(|(r, _)| r.contains(&w.desk.id)).collect();
7243 let below: Vec<u32> = rounds_there(&w.server, &w.phone.id).into_iter().filter(|r| *r < past).collect();
7244 assert!(below.len() >= 3, "{below:?}");
7245 w.fake.forget_seen();
7247 assert_eq!(phone_sends(&w, now + 120, false).taken_out, 0);
7248 assert!(w.fake.seen().iter().all(|l| !l.starts_with("DELETE ")));
7249 let lagging = uuid::Uuid::new_v4().to_string();
7251 let held = below[below.len() / 2];
7252 let notes = |read: u32, at: i64| format!("at = {at}\nround = 1\n\n[read]\n\"{}\" = {read}\n", w.phone.id);
7253 put(&w.phone_folder, &format!("{lagging}.toml"), ¬es(held, now + 100));
7254 let checked = phone_sends(&w, now + 6 * 3600 + 200, false);
7255 assert!(checked.problem.is_none(), "{checked:?}");
7256 assert_eq!(checked.taken_out, below.iter().filter(|r| **r < held).count(), "{checked:?}");
7257 assert_eq!(rounds_there(&w.server, &w.phone.id).into_iter().filter(|r| *r < past).collect::<Vec<_>>(), below.iter().copied().filter(|r| *r >= held).collect::<Vec<_>>(), "from round {held} on, held back");
7258 put(&w.phone_folder, &format!("{lagging}.toml"), ¬es(past, now + 6 * 3600 + 300));
7260 assert!(phone_sends(&w, now + 12 * 3600 + 400, false).taken_out > 0);
7261 assert!(rounds_there(&w.server, &w.phone.id).iter().all(|r| *r >= past));
7262 put_back(&w);
7264 let aside = w.base.join("rounds-aside");
7265 std::fs::create_dir_all(&aside).unwrap();
7266 let mine: Vec<String> = walk(&w.phone_folder).into_keys().filter(|r| r.starts_with(&w.phone.id) && r.ends_with(".jsonl")).collect();
7267 for relative in &mine {
7268 std::fs::rename(w.phone_folder.join(relative), aside.join(relative)).unwrap();
7269 }
7270 assert_eq!(phone_sends(&w, now + 18 * 3600 + 500, false).taken_out, 0, "none of its records here: nothing taken out");
7271 assert!(rounds_there(&w.server, &w.phone.id).iter().any(|r| *r < past));
7272 for relative in &mine {
7273 std::fs::rename(aside.join(relative), w.phone_folder.join(relative)).unwrap();
7274 }
7275 assert!(phone_sends(&w, now + 24 * 3600 + 600, false).taken_out > 0);
7276 assert!(rounds_there(&w.server, &w.phone.id).iter().all(|r| *r >= past));
7277 assert!(w.server.join(format!("{stranger}-1.jsonl")).exists());
7279 assert_eq!(walk(&w.server).into_iter().filter(|(r, _)| r.contains(&w.desk.id)).collect::<BTreeMap<_, _>>(), desks);
7280 }
7281
7282 #[test]
7287 fn a_failed_look_after_a_refused_delete_holds_nothing_back() {
7288 let w = World::new("look-fails");
7289 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
7290 let round = |n: u32| format!("{}-{n}.jsonl", w.phone.id);
7291 for n in 40..=42 {
7292 put(&w.phone_folder, &round(n), &format!("{{\"round\":{n}}}\n"));
7293 }
7294 assert!(send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &BTreeMap::new(), DUE + 10, false, TEST).problem.is_none());
7295 {
7296 let _turn = w.fake.turn.lock().unwrap();
7297 sync_up(&w.phone_folder.join(round(40)), &w.server.join(round(40)));
7298 }
7299 std::fs::remove_file(w.phone_folder.join(round(40))).unwrap();
7300 rounds_stand(&w, 42, 41, 42, DUE + 15);
7301 put(&w.phone_folder, &round(43), "{\"round\":43}\n");
7302 let looked = round(40);
7303 w.fake.fault(move |r| (r.method == "PROPFIND" && r.path.ends_with(&looked)).then_some(Fault::Status(500)));
7304 let sent = send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &BTreeMap::new(), DUE + 20, false, TEST);
7305 assert!(sent.problem.is_none() && sent.failure.is_none(), "the send goes on: {sent:?}");
7306 assert_eq!((sent.sent, sent.taken_out), (1, 0), "the new round went, the refused one stayed");
7307 let sending = Sending::load(&w.phone.memory);
7308 assert_eq!((sending.tries, sending.retry_at), (0, 0), "no backoff");
7309 assert!(w.server.join(round(40)).exists() && w.server.join(round(43)).exists());
7310 w.fake.clear_faults();
7312 w.fake.forget_seen();
7313 send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &BTreeMap::new(), DUE + 30, false, TEST);
7314 assert!(w.fake.seen().iter().all(|l| !l.contains(&round(40))), "{:?}", w.fake.seen());
7315 let listed = send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &BTreeMap::new(), DUE + 10 + 6 * 3600, false, TEST);
7316 assert_eq!(listed.taken_out, 1, "{listed:?}");
7317 assert!(!w.server.join(round(40)).exists());
7318 }
7319
7320 #[test]
7330 fn a_delete_answered_otherwise_holds_nothing_back() {
7331 for status in [423, 503, 403] {
7332 let w = World::new(&format!("delete-{status}"));
7333 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
7334 let round = |n: u32| format!("{}-{n}.jsonl", w.phone.id);
7335 for n in 40..=42 {
7336 put(&w.phone_folder, &round(n), &format!("{{\"round\":{n}}}\n"));
7337 }
7338 let seal = |text: &str| {
7339 let source = w.base.join(format!("{}.md", text.len()));
7340 std::fs::write(&source, text.repeat(200)).unwrap();
7341 let (hash, _) = crate::blobs::hash_file(&source).unwrap();
7342 crate::blobs::put(&w.phone_folder, &KEY, &source, &hash).unwrap();
7343 let name = crate::blobs::name(&KEY, &hash);
7344 let size = std::fs::metadata(w.phone_folder.join("blobs").join(&name)).unwrap().len();
7345 (name, size)
7346 };
7347 let (refused, refused_size) = seal("a note of the phone's\n");
7348 let (looked, looked_size) = seal("another note of the phone's\n");
7349 let sealed: BTreeMap<String, u64> = [(refused.clone(), refused_size), (looked.clone(), looked_size)].into();
7350 assert!(send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &sealed, DUE + 10, false, TEST).problem.is_none());
7351 let blob = |name: &str| format!("blobs/{name}");
7352 {
7353 let _turn = w.fake.turn.lock().unwrap();
7355 sync_up(&w.phone_folder.join(blob(&looked)), &w.server.join(blob(&looked)));
7356 }
7357 for relative in [round(40), blob(&refused), blob(&looked)] {
7359 std::fs::remove_file(w.phone_folder.join(relative)).unwrap();
7360 }
7361 rounds_stand(&w, 42, 41, 42, DUE + 15);
7362 put(&w.phone_folder, &round(43), "{\"round\":43}\n");
7363 let (deleted, stat) = ([round(40), refused.clone()], looked.clone());
7364 w.fake.fault(move |r| (r.method == "DELETE" && deleted.iter().any(|d| r.path.ends_with(d.as_str()))).then_some(Fault::Status(status)));
7365 w.fake.fault(move |r| (r.method == "PROPFIND" && r.path.ends_with(&stat)).then_some(Fault::Status(status)));
7366 let sent = send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &BTreeMap::new(), DUE + 20, false, TEST);
7367 assert!(sent.problem.is_none() && sent.failure.is_none(), "{status}: the send goes on: {sent:?}");
7368 assert_eq!((sent.sent, sent.taken_out), (1, 0), "{status}: the new round went, the others stayed there");
7369 let sending = Sending::load(&w.phone.memory);
7370 assert_eq!((sending.tries, sending.retry_at), (0, 0), "{status}: no backoff");
7371 assert!(!sending.sent.contains_key(&round(40)), "{status}: the round forgotten here");
7372 assert_eq!(sending.refused, [blob(&refused), blob(&looked)].into_iter().collect(), "{status}: the sealed files kept for the next listing");
7373 assert!([round(40), round(43), blob(&refused), blob(&looked)].iter().all(|r| w.server.join(r).exists()));
7374 w.fake.clear_faults();
7376 w.fake.forget_seen();
7377 let quiet = send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &BTreeMap::new(), DUE + 30, false, TEST);
7378 assert!(quiet.problem.is_none(), "{status}: {quiet:?}");
7379 assert!(w.fake.seen().iter().all(|l| !l.contains(&round(40)) && !l.contains(&refused) && !l.contains(&looked)), "{status}: {:?}", w.fake.seen());
7380 let listed = send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &BTreeMap::new(), DUE + 10 + 6 * 3600, false, TEST);
7382 assert_eq!(listed.taken_out, 3, "{status}: {listed:?}");
7383 assert!([round(40), blob(&refused), blob(&looked)].iter().all(|r| !w.server.join(r).exists()));
7384 let sending = Sending::load(&w.phone.memory);
7385 assert!(sending.refused.is_empty() && !sending.sent.keys().any(|r| r.starts_with("blobs/")), "{status}: {sending:?}");
7386 }
7387 }
7388
7389 #[test]
7395 fn a_round_gone_here_goes_there_only_below_the_readers_floor() {
7396 let w = World::new("readers-floor");
7397 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
7398 let round = |n: u32| format!("{}-{n}.jsonl", w.phone.id);
7399 for n in 40..=43 {
7400 put(&w.phone_folder, &round(n), &format!("{{\"round\":{n}}}\n"));
7401 }
7402 assert!(send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &BTreeMap::new(), DUE + 10, false, TEST).problem.is_none());
7403 rounds_stand(&w, 43, 40, 41, DUE + 15);
7405 std::fs::remove_file(w.phone_folder.join(round(41))).unwrap();
7406 let sent = send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &BTreeMap::new(), DUE + 20, false, TEST);
7407 assert!(sent.problem.is_none() && sent.taken_out == 0, "{sent:?}");
7408 assert!(w.server.join(round(41)).exists(), "below the latest round, not below the readers' floor: kept there");
7409 rounds_stand(&w, 43, 42, 43, DUE + 25);
7411 let sent = send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &BTreeMap::new(), DUE + 30, false, TEST);
7412 assert_eq!(sent.taken_out, 1, "{sent:?}");
7413 assert!(!w.server.join(round(41)).exists());
7414 assert!([40, 42, 43].iter().all(|n| w.server.join(round(*n)).exists()), "never one here");
7415 }
7416
7417 fn noise(length: usize, seed: u32) -> Vec<u8> {
7421 let mut x = seed.wrapping_mul(2_654_435_761).wrapping_add(1);
7422 (0..length)
7423 .map(|_| {
7424 x ^= x << 13;
7425 x ^= x >> 17;
7426 x ^= x << 5;
7427 x as u8
7428 })
7429 .collect()
7430 }
7431
7432 fn content_name(path: &Path) -> String {
7434 crate::blobs::name(&KEY, &crate::blobs::hash_file(path).unwrap().0)
7435 }
7436
7437 #[test]
7451 fn six_weeks_of_edits_what_the_server_holds_settles() {
7452 crate::share::LARGE_SETTLE.with(|s| s.set(crate::share::LARGE_SETTLING));
7453 let w = World::new("six-weeks");
7454 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
7455 let (dn, pn) = (w.base.join("desk-notes"), w.base.join("phone-notes"));
7456 std::fs::create_dir_all(&dn).unwrap();
7457 std::fs::create_dir_all(&pn).unwrap();
7458 let texts = w.phone.roots.data.join(sioul_core::texts::FOLDER).join(sioul_core::texts::LOG).join(format!("{}.jsonl", w.phone.id));
7459 std::fs::create_dir_all(texts.parent().unwrap()).unwrap();
7460 let text_line = |n: usize| format!("{{\"n\":{n},\"padded\":\"{}\"}}\n", "x".repeat(230));
7461 std::fs::write(&texts, (0..3_000).map(text_line).collect::<String>()).unwrap();
7462 let mut texts_made = 3_000;
7463 let long_ago = SystemTime::now() - Duration::from_secs(365 * 86_400);
7467 let age_sealed = |folder: &Path| {
7468 for entry in std::fs::read_dir(folder.join("blobs")).into_iter().flatten().filter_map(Result::ok) {
7469 if !entry.file_name().to_string_lossy().starts_with('.') && entry.metadata().and_then(|m| m.modified()).is_ok_and(|t| t > long_ago + Duration::from_secs(86_400)) {
7470 set_mtime(&entry.path(), long_ago);
7471 }
7472 }
7473 };
7474 let mut edrive = EDrive::default();
7475 let mut replaced: BTreeMap<String, i64> = BTreeMap::new();
7477 let mut sizes: BTreeMap<String, u64> = BTreeMap::new();
7478 let mut takes: Vec<String> = Vec::new();
7479 let (mut last_song, mut last_paper): (Option<String>, Option<String>) = (None, None);
7480 let (mut phone_floor, mut desk_floor, mut taken_out) = (0u32, 0u32, 0usize);
7481 let mut overheads: Vec<u64> = Vec::new();
7482 let mut table: Vec<String> = Vec::new();
7483 let start = DUE + 3_600;
7484 let days = 42i64;
7485 for day in 0..days {
7486 for slot in 0..3i64 {
7487 let now = start + day * 86_400 + slot * 4 * 3_600;
7488 for i in 0..12usize {
7490 let n = (day as usize * 36 + slot as usize * 12 + i) % 50;
7491 let path = dn.join(format!("note-{n:02}.md"));
7492 if path.is_file() {
7493 replaced.insert(content_name(&path), now);
7494 }
7495 std::fs::write(&path, format!("note {n}, day {day}, edit {slot}-{i}\n").repeat(10 + (n * 7) % 90)).unwrap();
7496 }
7497 if day % 6 == 1 {
7499 let song = dn.join("scenes").join("song.mp3");
7500 std::fs::create_dir_all(song.parent().unwrap()).unwrap();
7501 std::fs::write(&song, noise((1 << 20) + 100_000, (day * 10 + slot) as u32)).unwrap();
7502 let name = content_name(&song);
7503 if slot < 2 {
7504 takes.push(name);
7505 } else {
7506 set_mtime(&song, SystemTime::now() - Duration::from_secs(11 * 60));
7507 if let Some(old) = last_song.replace(name) {
7508 replaced.insert(old, now);
7509 }
7510 }
7511 }
7512 if day % 10 == 2 && slot == 1 {
7514 let paper = dn.join(sioul_core::papers::FOLDER).join("scan.pdf");
7515 std::fs::create_dir_all(paper.parent().unwrap()).unwrap();
7516 std::fs::write(&paper, noise(400_000, 1_000 + day as u32)).unwrap();
7517 if let Some(old) = last_paper.replace(content_name(&paper)) {
7518 replaced.insert(old, now);
7519 }
7520 }
7521 w.desk.exchange_all_parts(&w.server, now, &dn);
7522 age_sealed(&w.server);
7523 {
7525 let mut log = std::fs::OpenOptions::new().append(true).open(&texts).unwrap();
7526 for _ in 0..50 {
7527 log.write_all(text_line(texts_made).as_bytes()).unwrap();
7528 texts_made += 1;
7529 }
7530 }
7531 if slot == 1 {
7532 let journal = pn.join("journal.md");
7533 if journal.is_file() {
7534 replaced.insert(content_name(&journal), now);
7535 }
7536 std::fs::write(&journal, format!("the phone's own note, day {day}\n").repeat(20)).unwrap();
7537 }
7538 w.phone.exchange_all_parts(&w.phone_folder, now + 60, &pn);
7539 age_sealed(&w.phone_folder);
7540 phone_floor = crate::share::rounds_past(&w.phone_folder, &w.phone.memory, &w.phone.id, (now + 60) * 1000).unwrap();
7541 let sent = send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &crate::share::own_sealed(&w.phone.memory, &w.phone.id), now + 61, false, TEST);
7542 assert!(sent.problem.is_none(), "day {day}: {sent:?}");
7543 taken_out += sent.taken_out;
7544 edrive.scan(&w.fake, &w.server, &w.phone_folder);
7545 w.desk.exchange_all_parts(&w.server, now + 120, &dn);
7546 age_sealed(&w.server);
7547 desk_floor = crate::share::rounds_past(&w.server, &w.desk.memory, &w.desk.id, (now + 120) * 1000).unwrap();
7548 edrive.scan(&w.fake, &w.server, &w.phone_folder);
7549 }
7550 let end = start + day * 86_400 + 8 * 3_600 + 120;
7552 let there = walk(&w.server);
7553 let total: u64 = there.values().map(|b| b.len() as u64).sum();
7554 let blobs: BTreeMap<String, u64> = there.iter().filter_map(|(r, b)| r.strip_prefix("blobs/").map(|n| (n.to_string(), b.len() as u64))).collect();
7555 for (name, size) in &blobs {
7556 sizes.entry(name.clone()).or_insert(*size);
7557 }
7558 let current: BTreeSet<String> = [&dn, &pn].into_iter().flat_map(|dir| walk(dir).into_keys().map(move |r| content_name(&dir.join(r)))).collect();
7559 let mut overhead = 0;
7560 for (name, size) in blobs.iter().filter(|(n, _)| !current.contains(*n)) {
7561 let at = replaced.get(name).copied().unwrap_or_else(|| panic!("day {day}: a sealed file there that no edit replaced"));
7562 assert!(end - at <= 15 * 86_400, "day {day}: a content replaced {} days ago still there", (end - at) / 86_400);
7563 overhead += size;
7564 }
7565 assert!(takes.iter().all(|t| !blobs.contains_key(t)), "day {day}: a take saved minutes before the next reached the server");
7566 let (phone_rounds, desk_rounds) = (rounds_there(&w.server, &w.phone.id), rounds_there(&w.server, &w.desk.id));
7567 assert!(phone_rounds.iter().all(|r| *r >= phone_floor), "day {day}: the phone's rounds {phone_rounds:?}, below {phone_floor}");
7568 assert!(desk_rounds.iter().all(|r| *r >= desk_floor), "day {day}: the desk's rounds {desk_rounds:?}, below {desk_floor}");
7569 let bytes_of = |id: &str| -> u64 { there.iter().filter(|(r, _)| r.starts_with(id) && r.ends_with(".jsonl")).map(|(_, b)| b.len() as u64).sum() };
7570 let current_bytes: u64 = blobs.iter().filter(|(n, _)| current.contains(*n)).map(|(_, s)| s).sum();
7571 let replaced_bytes: u64 = replaced.keys().filter_map(|n| sizes.get(n)).sum();
7572 table.push(format!(
7573 "day {day:2}: {:>6} KB there; current sealed {:>6} KB; replaced, still there {:>5} KB ({:>3} files); phone's records {:>5} KB in {:>2} rounds ({:?}); desk's {:>4} KB in {} rounds; replaced since day 0 {:>6} KB",
7574 total / 1000,
7575 current_bytes / 1000,
7576 overhead / 1000,
7577 blobs.keys().filter(|n| !current.contains(*n)).count(),
7578 bytes_of(&w.phone.id) / 1000,
7579 phone_rounds.len(),
7580 phone_rounds.first().zip(phone_rounds.last()).map(|(a, b)| (*a, *b)).unwrap_or_default(),
7581 bytes_of(&w.desk.id) / 1000,
7582 desk_rounds.len(),
7583 replaced_bytes / 1000,
7584 ));
7585 overheads.push(overhead);
7586 }
7587 eprintln!("{}", table.join("\n"));
7588 eprintln!("eDrive: {} uploads, {} refused; the phone's sends took out {taken_out} files there", edrive.uploads, edrive.refusals);
7589 let weeks = |from: usize, to: usize| overheads[from..to].iter().copied().max().unwrap_or(0);
7591 assert!(weeks(35, 42) <= weeks(21, 28) + (1 << 20) + 200_000, "settled: {} then {}", weeks(21, 28), weeks(35, 42));
7592 assert!(taken_out > 0 && edrive.uploads > 0, "eDrive sent the phone's files again, and Sioul took them out there");
7593 crate::share::LARGE_SETTLE.with(|s| s.set(0));
7594 }
7595
7596 #[test]
7603 fn the_seal_is_found_at_documents_sioul_and_elsewhere() {
7604 use super::fake::USER;
7605 let base = scratch("seek");
7606 let files = base.join("cloud");
7607 std::fs::create_dir_all(files.join("Documents/Photos")).unwrap();
7608 let fake = Fake::start(&files);
7609 let login = crate::loginflow::Grant::of(&fake.base, USER, PASSWORD).login_for("cloud");
7610 let found = |login: &Login| seek_with(login, TEST).unwrap().found;
7611 assert_eq!(seek_with(&login, TEST).unwrap(), Sought { found: Vec::new(), complete: true }, "none yet, every place looked at");
7612 put(&files, "Documents/Sioul/seal.toml", SEAL);
7613 assert_eq!(found(&login), ["Documents/Sioul"]);
7614 std::fs::remove_dir_all(files.join("Documents/Sioul")).unwrap();
7615 put(&files, "Sioul/seal.toml", SEAL);
7616 put(&files, "Documents/Partagé/seal.toml", OTHER_SEAL);
7617 put(&files, "Documents/Notes/seal.toml", "not a seal");
7618 put(&files, "Documents/.hidden/seal.toml", SEAL);
7619 assert_eq!(seek_with(&login, TEST).unwrap(), Sought { found: vec!["Sioul".into(), "Documents/Partagé".into()], complete: true });
7620 assert!(fake.seen().iter().all(|l| l.starts_with("PROPFIND ") || (l.starts_with("GET ") && l.contains("/seal.toml "))), "{:?}", fake.seen());
7622 std::fs::remove_dir_all(files.join("Documents")).unwrap();
7624 assert_eq!(seek_with(&login, TEST).unwrap(), Sought { found: vec!["Sioul".into()], complete: true });
7625 let refused = Login { password: Some("wrong".into()), ..login };
7627 assert!(seek_with(&refused, TEST).unwrap_err().starts_with("login:"));
7628 assert!(sought("Documents/Sioul") && sought("/Sioul/") && sought("Documents/Partagé"));
7630 assert!(!sought("Documents/A/B") && !sought("Partage/Sioul") && !sought("Documents/.hidden") && !sought("Documents/"));
7631 let _ = std::fs::remove_dir_all(&base);
7632 }
7633
7634 #[test]
7641 fn a_slow_or_failing_look_keeps_what_it_found_and_says_it_is_incomplete() {
7642 use super::fake::USER;
7643 let base = scratch("seek-slow");
7644 let files = base.join("cloud");
7645 for folder in 0..12 {
7646 std::fs::create_dir_all(files.join(format!("Documents/Dossier {folder:02}"))).unwrap();
7647 }
7648 put(&files, "Documents/Sioul/seal.toml", SEAL);
7649 let fake = Fake::start(&files);
7650 let login = crate::loginflow::Grant::of(&fake.base, USER, PASSWORD).login_for("cloud");
7651 let _ = seek_with(&login, TEST).unwrap();
7653 let seen = fake.seen();
7654 let seal_at = seen.iter().position(|l| l.contains("/Documents/Sioul/seal.toml")).unwrap();
7655 let listed_at = seen.iter().position(|l| l.starts_with("PROPFIND ") && l.ends_with("/Documents 207")).unwrap();
7656 assert!(seal_at < listed_at, "{seen:?}");
7657 fake.fault(|r| (r.method == "PROPFIND" && r.path.ends_with("/files/jane/Documents")).then_some(Fault::Status(500)));
7659 assert_eq!(seek_with(&login, TEST).unwrap(), Sought { found: vec!["Documents/Sioul".into()], complete: false });
7660 fake.clear_faults();
7661 let tight = Limits { pull: Duration::from_millis(800), ..TEST };
7663 fake.fault(|r| (r.method == "GET" && r.path.contains("/Dossier ")).then_some(Fault::Late(Duration::from_millis(300))));
7664 assert_eq!(seek_with(&login, tight).unwrap(), Sought { found: vec!["Documents/Sioul".into()], complete: false });
7665 fake.clear_faults();
7666 fake.fault(|r| (r.method == "GET" && r.path.ends_with("/Documents/Sioul/seal.toml")).then_some(Fault::Status(503)));
7668 assert_eq!(seek_with(&login, TEST).unwrap(), Sought { found: Vec::new(), complete: false });
7669 fake.clear_faults();
7670 let _ = std::fs::remove_dir_all(&base);
7671 }
7672
7673 fn keyed_exchange(device: &Device, folder: &Path, key: &[u8; 32], now: i64) -> Outcome {
7675 let outcome = crate::share::exchange(&Sharing { folder, computer: &device.id, key, memory: &device.memory, files: false, hurry: None }, &device.stores(), now * 1000).unwrap();
7676 crate::devices::change(&crate::devices::own_path(&device.roots.state), Some((folder, key)), |e| {
7677 e.id = device.id.clone();
7678 e.kind = device.kind.into();
7679 if e.started == 0 {
7680 e.start(now);
7681 }
7682 e.exported(now * 1000, outcome.wrote, now);
7683 })
7684 .unwrap();
7685 outcome
7686 }
7687
7688 #[test]
7697 fn a_fresh_device_joins_through_the_sign_in_and_receives_the_others_settings() {
7698 use super::fake::USER;
7699 let base = scratch("join");
7700 let files = base.join("cloud");
7701 let server = files.join("Documents").join("Sioul");
7702 std::fs::create_dir_all(&server).unwrap();
7703 let fake = Fake::start(&files);
7704 let passphrase = "a few words you keep";
7706 let key = crate::share::key_for(&server, passphrase).unwrap();
7707 let desk = Device::new(&base, "desk", COMPUTER);
7708 put(&desk.roots.config, "config.toml", "language = \"fr\"\n\n[[account]]\nid = \"agenda\"\nkind = \"dav\"\naddress = \"jane@example.org\"\nhost = \"cloud.example.org\"\nurl = \"https://cloud.example.org/remote.php/dav/calendars/jane/\"\n");
7709 keyed_exchange(&desk, &server, &key, DUE - 600);
7710 let address = crate::loginflow::address_of(&fake.base).unwrap();
7712 let flow = crate::loginflow::begin(&address, "Sioul (Android)").unwrap();
7713 fake.grant_flows();
7714 let grant = crate::loginflow::wait(&flow, &std::sync::atomic::AtomicBool::new(false), Duration::from_secs(10), Duration::from_millis(50)).unwrap();
7715 let login = grant.login_for("cloud-agenda");
7716 assert_eq!(seek_with(&login, TEST).unwrap().found, ["Documents/Sioul"]);
7718 let opened = open_with(&login, "Documents/Sioul", TEST).unwrap();
7719 let phone = Device::new(&base, "phone", PHONE);
7720 let mirror = mirror_of(&phone.memory);
7721 put_bytes(&mirror, "seal.toml", opened.seal.as_deref().expect("sealed there"));
7722 assert_eq!(crate::share::key_for(&mirror, "not the one chosen there").unwrap_err(), crate::share::Refused::WrongPassphrase);
7724 assert_eq!(crate::share::key_for(&mirror, passphrase).unwrap(), key);
7725 begin_signed_in(&phone.memory, &mirror, &login, &opened, DUE - 500).unwrap();
7726 let state = State::load(&phone.memory);
7727 assert_eq!((state.user.as_str(), state.mode.as_str(), state.account.as_str()), (USER, MIRROR, "cloud-agenda"));
7728 assert!(!std::fs::read_to_string(state_path(&phone.memory)).unwrap().contains(grant.password()), "never the app password");
7729 let pulled = step_with(&phone.memory, &mirror, &phone.id, &login, DUE - 400, true, TEST);
7731 assert!(pulled.problem.is_none() && pulled.fetched > 0, "{pulled:?}");
7732 let outcome = keyed_exchange(&phone, &mirror, &key, DUE - 400);
7733 assert!(outcome.received > 0, "{outcome:?}");
7734 let pushed = step_with(&phone.memory, &mirror, &phone.id, &login, DUE - 399, false, TEST);
7735 assert!(pushed.problem.is_none() && pushed.sent > 0, "{pushed:?}");
7736 let config = std::fs::read_to_string(phone.roots.config.join("config.toml")).unwrap_or_default();
7737 assert!(config.contains("language = \"fr\"") && config.contains("id = \"agenda\""), "the desk's settings and accounts: {config}");
7738 keyed_exchange(&desk, &server, &key, DUE - 300);
7740 assert!(crate::devices::all(&server, &key).0.iter().any(|e| e.id == phone.id && e.kind == PHONE));
7741 for (relative, bytes) in walk(&phone.roots.state).into_iter().chain(walk(&phone.roots.config)) {
7743 assert!(!String::from_utf8_lossy(&bytes).contains(grant.password()), "{relative}");
7744 }
7745 let _ = std::fs::remove_dir_all(&base);
7746 }
7747
7748 #[test]
7755 fn a_first_look_cut_short_is_never_taken_for_a_whole_one() {
7756 use super::fake::USER;
7757 let base = scratch("join-late");
7758 let files = base.join("cloud");
7759 let server = files.join("Documents").join("Sioul");
7760 std::fs::create_dir_all(&server).unwrap();
7761 let fake = Fake::start(&files);
7762 let passphrase = "a few words you keep";
7763 let key = crate::share::key_for(&server, passphrase).unwrap();
7764 let desk = Device::new(&base, "desk", COMPUTER);
7765 put(&desk.roots.config, "config.toml", "[[account]]\nid = \"murena\"\nkind = \"dav\"\naddress = \"jane\"\nhost = \"cloud.example.org\"\nurl = \"https://cloud.example.org/remote.php/dav/calendars/jane/\"\n");
7766 keyed_exchange(&desk, &server, &key, DUE - 600);
7767 let grant = crate::loginflow::Grant::of(&fake.base, USER, PASSWORD);
7768 let login = grant.login_for("cloud-agenda");
7769 let opened = open_with(&login, "Documents/Sioul", TEST).unwrap();
7770 let phone = Device::new(&base, "phone", PHONE);
7771 let mirror = mirror_of(&phone.memory);
7772 put_bytes(&mirror, "seal.toml", opened.seal.as_deref().unwrap());
7773 assert_eq!(crate::share::key_for(&mirror, passphrase).unwrap(), key);
7774 begin_signed_in(&phone.memory, &mirror, &login, &opened, DUE - 500).unwrap();
7775 let id = desk.id.clone();
7777 fake.fault(move |r| (r.method == "GET" && r.path.contains(&id) && r.path.ends_with(".jsonl")).then_some(Fault::Late(Duration::from_millis(900))));
7778 let tight = Limits { pull: Duration::from_millis(600), ..TEST };
7779 let cut = step_with(&phone.memory, &mirror, &phone.id, &login, DUE - 400, true, tight);
7780 assert!(cut.problem.is_some(), "{cut:?}");
7781 let state = State::load(&phone.memory);
7782 assert!(!state.said.is_empty() && state.last < DUE - 500, "no whole look since the join: {state:?}");
7783 keyed_exchange(&phone, &mirror, &key, DUE - 400);
7784 assert!(!std::fs::read_to_string(phone.roots.config.join("config.toml")).unwrap_or_default().contains("id = \"murena\""), "the desk's account not here yet");
7785 fake.clear_faults();
7787 let whole = step_with(&phone.memory, &mirror, &phone.id, &login, DUE - 300, true, TEST);
7788 assert!(whole.problem.is_none(), "{whole:?}");
7789 let state = State::load(&phone.memory);
7790 assert!(state.said.is_empty() && state.last >= DUE - 300 && state.left.is_empty(), "{state:?}");
7791 keyed_exchange(&phone, &mirror, &key, DUE - 300);
7792 let config = std::fs::read_to_string(phone.roots.config.join("config.toml")).unwrap_or_default();
7793 assert!(config.contains("id = \"murena\""), "{config}");
7794 let _ = std::fs::remove_dir_all(&base);
7795 }
7796
7797 fn full_round_of(d: &Device) -> u32 {
7799 std::fs::read(&d.memory).ok().and_then(|b| serde_json::from_slice::<serde_json::Value>(&b).ok()).and_then(|m| m["full_round"].as_u64()).unwrap_or(0) as u32
7800 }
7801
7802 fn carried(w: &World, folder: &Path, id: &str) {
7805 let _turn = w.fake.turn.lock().unwrap();
7806 for (relative, bytes) in walk(folder).into_iter().filter(|(r, _)| r.contains(id)) {
7807 put_bytes(&w.server, &relative, &bytes);
7808 }
7809 for (relative, bytes) in walk(&w.server).into_iter().filter(|(r, _)| !r.contains(id)) {
7810 put_bytes(folder, &relative, &bytes);
7811 }
7812 }
7813
7814 fn senders(step: usize) -> String {
7816 (0..(step + 1) * 1_500).map(|n| format!("old-sender-{n:06}@example.org\n")).collect()
7817 }
7818
7819 fn swept_here(d: &Device, names: &[&str], at: i64) {
7822 let mut memory: serde_json::Value = serde_json::from_slice(&std::fs::read(&d.memory).unwrap()).unwrap();
7823 memory["taken"] = names.iter().map(|name| (name.to_string(), serde_json::Value::from(at * 1000))).collect::<serde_json::Map<_, _>>().into();
7824 std::fs::write(&d.memory, serde_json::to_vec(&memory).unwrap()).unwrap();
7825 }
7826
7827 #[test]
7839 fn a_dead_phone_s_past_rounds_go_from_the_server_too() {
7840 let w = World::new("departed-rounds");
7841 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
7842 let old = Device::new(&w.base, "old-state", PHONE);
7843 let folder = w.base.join("old").join("storage").join("Documents").join("Sioul");
7844 put(&folder, "seal.toml", SEAL);
7845 old.session(&folder, |e| e.start(DUE - 6_000));
7846 let mut now = DUE;
7847 let mut step = 0;
7848 while full_round_of(&old) <= 1 {
7849 put(&old.roots.config, "known-senders.txt", &senders(step));
7850 now += 600;
7851 carried(&w, &folder, &old.id);
7852 old.exchange(&folder, now);
7853 carried(&w, &folder, &old.id);
7854 w.desk.exchange(&w.server, now + 10);
7855 w.down();
7856 w.phone.exchange(&w.phone_folder, now + 20);
7857 w.up();
7858 step += 1;
7859 }
7860 let full = full_round_of(&old);
7861 put(&old.roots.config, "known-senders.txt", &senders(step));
7863 carried(&w, &folder, &old.id);
7864 old.exchange(&folder, now + 300);
7865 carried(&w, &folder, &old.id);
7866 w.desk.exchange(&w.server, now + 310);
7867 w.down();
7868 w.phone.exchange(&w.phone_folder, now + 320);
7869 w.up();
7870 let all = rounds_there(&w.server, &old.id);
7871 let kept: BTreeSet<u32> = all.iter().copied().filter(|r| *r >= full).collect();
7872 let past: Vec<String> = all.iter().filter(|r| **r < full).map(|r| format!("{}-{r}.jsonl", old.id)).collect();
7873 assert!(past.len() >= 3 && kept.len() >= 2, "{all:?}, full {full}");
7874 assert!(rounds_there(&folder, &old.id).len() < all.len(), "it took its own out of its folder, never of the server");
7875 crate::share::forget_device(&w.phone_folder, &KEY, &w.phone.memory, &w.phone.id, &old.id).unwrap();
7877 w.phone.exchange(&w.phone_folder, now + 400);
7878 w.up();
7879 assert_eq!(rounds_there(&w.phone_folder, &old.id), kept);
7880 assert_eq!(rounds_there(&w.server, &old.id), all);
7881 assert_eq!(crate::share::departed_past(&w.phone.memory, &w.phone.id).get(&old.id), Some(&full));
7882 w.fake.forget_seen();
7884 let pulled = w.pull(now + 410);
7885 assert!(pulled.problem.is_none(), "{pulled:?}");
7886 assert!(w.fake.seen().iter().any(|l| l.starts_with("PROPFIND ")), "listed there");
7887 assert!(w.fake.seen().iter().all(|l| !(l.starts_with("GET ") && past.iter().any(|r| l.contains(r.as_str())))), "never fetched again: {:?}", w.fake.seen());
7888 assert!(past.iter().all(|r| !w.cache().join(r).exists()));
7889 let (stale, fresh) = ("a".repeat(64), "b".repeat(64));
7891 for (name, at) in [(&stale, SystemTime::now() - Duration::from_secs(100 * 86_400)), (&fresh, std::time::UNIX_EPOCH + Duration::from_secs(now as u64))] {
7892 put(&w.server, &format!("blobs/{name}"), "sealed\n");
7893 set_mtime(&w.server.join("blobs").join(name), at);
7894 }
7895 swept_here(&w.phone, &[&stale, &fresh], now + 400);
7896 let (refused, gone) = (past[0].clone(), past[1].clone());
7898 let r = refused.clone();
7899 w.fake.fault(once(move |q| q.method == "DELETE" && q.path.ends_with(&r), Fault::Status(412)));
7900 let (g, server) = (gone.clone(), w.server.clone());
7901 w.fake.fault(move |q| {
7902 (q.method == "DELETE" && q.path.ends_with(&g)).then(|| {
7903 let _ = std::fs::remove_file(server.join(&g));
7904 Fault::Status(404)
7905 })
7906 });
7907 w.fake.forget_seen();
7908 let sent = phone_sends(&w, now + 420, true);
7909 assert!(sent.problem.is_none(), "the send goes on: {sent:?}");
7910 assert_eq!(sent.taken_out, past.len() - 2 + 1, "every past round but two, and the stale sealed file: {:?}", w.fake.seen());
7911 let left: BTreeSet<u32> = kept.iter().copied().chain(remote_round(&refused)).collect();
7912 assert_eq!(rounds_there(&w.server, &old.id), left, "the refused one left, the one gone already gone");
7913 assert!(!w.server.join("blobs").join(&stale).exists(), "still as old there: taken out");
7914 assert!(w.server.join("blobs").join(&fresh).exists(), "made fresh there since: another device's use, left");
7915 let deletes: Vec<String> = w.fake.seen().into_iter().filter(|l| l.starts_with("DELETE ")).collect();
7916 assert!(deletes.iter().all(|l| past.iter().any(|r| l.contains(r.as_str())) || l.contains(&stale)), "only those names: {deletes:?}");
7917 w.fake.clear_faults();
7919 let later = phone_sends(&w, now + 420 + CHECK_AGAIN, false);
7920 assert!(later.problem.is_none() && later.taken_out == 1, "{later:?}");
7921 assert_eq!(rounds_there(&w.server, &old.id), kept);
7922 let quiet = phone_sends(&w, now + 420 + 2 * CHECK_AGAIN, false);
7923 assert!(quiet.problem.is_none() && quiet.taken_out == 0, "{quiet:?}");
7924 assert_eq!(rounds_there(&w.server, &old.id), kept);
7925 }
7926
7927 fn remote_round(relative: &str) -> Option<u32> {
7929 round_of(relative)
7930 }
7931
7932 #[test]
7939 fn a_dead_phone_s_past_rounds_go_from_the_server_in_the_mirror_too() {
7940 let p = Pair::new("departed-mirror");
7941 let login = p.fake.login();
7942 let old = Device::new(&p.base, "old", PHONE);
7943 let old_folder = mirror_of(&old.memory);
7944 let joined = open_with(&login, "Documents/Sioul", TEST).unwrap();
7945 put_bytes(&old_folder, "seal.toml", joined.seal.as_deref().unwrap());
7946 begin(&old.memory, &old_folder, &login, &joined, DUE - 7_000).unwrap();
7947 old.session(&old_folder, |e| e.start(DUE - 6_900));
7948 let old_turn = |now: i64| {
7949 let before = step_with(&old.memory, &old_folder, &old.id, &login, now, true, TEST);
7950 assert!(before.problem.is_none(), "{before:?}");
7951 old.exchange(&old_folder, now);
7952 let after = step_with(&old.memory, &old_folder, &old.id, &login, now, false, TEST);
7953 assert!(after.problem.is_none(), "{after:?}");
7954 };
7955 old_turn(DUE - 6_800);
7956 p.turn(true, DUE - 6_700);
7957 p.turn(false, DUE - 6_600);
7958 let mut now = DUE;
7960 let mut step = 0;
7961 while full_round_of(&old) <= 1 {
7962 put(&old.roots.config, "known-senders.txt", &senders(step));
7963 now += 600;
7964 old_turn(now);
7965 step += 1;
7966 }
7967 let full = full_round_of(&old);
7968 put(&old.roots.config, "known-senders.txt", &senders(step));
7969 old_turn(now + 300);
7970 let there = || rounds_there(&p.fake.files.join("Documents").join("Sioul"), &old.id);
7971 let all = there();
7972 let kept: BTreeSet<u32> = all.iter().copied().filter(|r| *r >= full).collect();
7973 let past: Vec<String> = all.iter().filter(|r| **r < full).map(|r| format!("{}-{r}.jsonl", old.id)).collect();
7974 assert!(past.len() >= 3 && kept.len() >= 2, "{all:?}, full {full}");
7975 p.turn(true, now + 400);
7977 p.turn(false, now + 500);
7978 p.turn(true, now + 600);
7979 assert!(past.iter().all(|r| p.desk_folder.join(r).exists() && p.phone_folder.join(r).exists()));
7980 crate::share::forget_device(&p.desk_folder, &KEY, &p.desk.memory, &p.desk.id, &old.id).unwrap();
7982 p.turn(true, now + 700);
7983 assert!(past.iter().all(|r| !p.desk_folder.join(r).exists()), "taken out of the desk's copy");
7984 assert_eq!(there(), all, "still there until the desk's next step");
7985 let (stale, fresh) = ("c".repeat(64), "d".repeat(64));
7986 let blobs = p.fake.files.join("Documents").join("Sioul").join("blobs");
7987 for (name, at) in [(&stale, SystemTime::now() - Duration::from_secs(100 * 86_400)), (&fresh, std::time::UNIX_EPOCH + Duration::from_secs(now as u64))] {
7988 put(&blobs, name, "sealed\n");
7989 set_mtime(&blobs.join(name), at);
7990 }
7991 swept_here(&p.desk, &[&stale, &fresh], now + 700);
7992 p.fake.forget_seen();
7994 let step = p.step(true, now + 800, true);
7995 assert!(step.problem.is_none(), "{step:?}");
7996 let seen = p.fake.seen();
7997 assert!(seen.iter().all(|l| !(l.starts_with("GET ") && (past.iter().any(|r| l.contains(r.as_str())) || l.contains(&stale) || l.contains(&fresh)))), "never fetched again: {seen:?}");
7998 assert_eq!(there(), kept);
7999 assert!(!blobs.join(&stale).exists() && blobs.join(&fresh).exists(), "the stale one taken out, the fresh one left");
8000 assert!(!p.desk_folder.join("blobs").join(&fresh).exists(), "never fetched");
8001 let deletes: Vec<String> = seen.into_iter().filter(|l| l.starts_with("DELETE ")).collect();
8002 assert_eq!(deletes.len(), past.len() + 1, "{deletes:?}");
8003 p.turn(false, now + 900);
8005 assert!(past.iter().all(|r| !p.phone_folder.join(r).exists()));
8006 assert!(kept.iter().all(|r| p.phone_folder.join(format!("{}-{r}.jsonl", old.id)).exists() && p.desk_folder.join(format!("{}-{r}.jsonl", old.id)).exists()));
8007 }
8008
8009 #[test]
8017 fn a_removed_device_s_files_go_from_the_server_in_the_mirror() {
8018 let p = Pair::new("removed-mirror");
8019 let login = p.fake.login();
8020 let old = Device::new(&p.base, "old", PHONE);
8021 let old_folder = mirror_of(&old.memory);
8022 let joined = open_with(&login, "Documents/Sioul", TEST).unwrap();
8023 put_bytes(&old_folder, "seal.toml", joined.seal.as_deref().unwrap());
8024 begin(&old.memory, &old_folder, &login, &joined, DUE - 7_000).unwrap();
8025 old.session(&old_folder, |e| e.start(DUE - 6_900));
8026 let old_turn = |now: i64| {
8027 let before = step_with(&old.memory, &old_folder, &old.id, &login, now, true, TEST);
8028 assert!(before.problem.is_none(), "{before:?}");
8029 old.exchange(&old_folder, now);
8030 let after = step_with(&old.memory, &old_folder, &old.id, &login, now, false, TEST);
8031 assert!(after.problem.is_none(), "{after:?}");
8032 };
8033 old_turn(DUE - 6_800);
8034 p.turn(true, DUE - 6_700);
8035 p.turn(false, DUE - 6_600);
8036 let mut now = DUE;
8038 for step in 0..3 {
8039 put(&old.roots.config, "known-senders.txt", &senders(step));
8040 crate::lease::renew(&old_folder, &KEY, "health", &old.id, now, now, false, crate::lease::Rule::FollowsYou, None).unwrap();
8041 now += 600;
8042 old_turn(now);
8043 }
8044 put(&old.roots.config, "safe-senders.txt", "one@example.org\n");
8046 old_turn(now + 50);
8047 p.turn(true, now + 100);
8048 p.turn(false, now + 200);
8049 p.turn(true, now + 300);
8050 let server = p.fake.files.join("Documents").join("Sioul");
8051 let all = rounds_there(&server, &old.id);
8052 let claim = format!("leases/health/{}.lease", old.id);
8053 let (entry, notes) = (format!("devices/{}.device", old.id), format!("{}.toml", old.id));
8054 assert!(all.len() >= 2 && server.join(&claim).exists() && p.desk_folder.join(&claim).exists(), "{all:?}");
8055 assert_eq!(rounds_there(&p.desk_folder, &old.id), all);
8056 crate::share::forget_device(&p.desk_folder, &KEY, &p.desk.memory, &p.desk.id, &old.id).unwrap();
8058 let removed = crate::share::remove_device_files(&p.desk_folder, &KEY, &p.desk.memory, &p.desk.id, &p.desk.stores(), &old.id, (now + 400) * 1000).unwrap();
8059 assert_eq!(removed.rounds, all.len(), "{removed:?}");
8060 assert!(rounds_there(&p.desk_folder, &old.id).is_empty() && !p.desk_folder.join(&claim).exists());
8061 assert!(p.desk_folder.join(&entry).exists() && p.desk_folder.join(¬es).exists(), "forgotten here only: its entry and its note stay");
8062 p.fake.forget_seen();
8064 let step = p.step(true, now + 500, true);
8065 assert!(step.problem.is_none(), "{step:?}");
8066 let seen = p.fake.seen();
8067 assert!(seen.iter().all(|l| !(l.starts_with("GET ") && l.contains(&old.id))), "never fetched again: {seen:?}");
8068 assert!(rounds_there(&server, &old.id).is_empty(), "every round, its last too: {:?}", rounds_there(&server, &old.id));
8069 assert!(!server.join(&claim).exists());
8070 assert!(server.join(&entry).exists() && server.join(¬es).exists());
8071 p.turn(true, now + 600);
8073 p.turn(true, now + 610);
8074 assert!(rounds_there(&p.desk_folder, &old.id).is_empty() && !p.desk_folder.join(&claim).exists());
8075 p.turn(false, now + 700);
8077 assert!(rounds_there(&p.phone_folder, &old.id).is_empty(), "{:?}", rounds_there(&p.phone_folder, &old.id));
8078 crate::lease::renew(&old_folder, &KEY, "health", &old.id, now + 800, now + 800, false, crate::lease::Rule::FollowsYou, None).unwrap();
8080 put(&old.roots.config, "safe-senders.txt", "one@example.org\ntwo@example.org\n");
8081 old.session(&old_folder, |e| e.start(now + 790));
8082 old_turn(now + 800);
8083 assert!(rounds_there(&server, &old.id).contains(all.iter().max().unwrap()), "its last round, longer, there again");
8085 assert!(server.join(&claim).exists());
8086 p.turn(true, now + 900);
8087 assert!(std::fs::read_to_string(p.desk.roots.config.join("safe-senders.txt")).unwrap_or_default().contains("two@example.org"), "its last round written on, fetched at once");
8088 put(&old.roots.config, "known-senders.txt", &senders(9));
8089 old_turn(now + 905);
8090 p.turn(true, now + 910);
8091 assert!(!rounds_there(&p.desk_folder, &old.id).is_empty(), "what it wrote since comes");
8092 assert!(p.desk_folder.join(&claim).exists(), "its claim renewed: fetched");
8093 assert!(crate::share::removed_devices(&p.desk.memory, &p.desk.id).is_empty(), "it counts again: no longer removed");
8094 let senders_here = std::fs::read_to_string(p.desk.roots.config.join("known-senders.txt")).unwrap_or_default();
8095 assert!(senders_here.contains("old-sender-014999@example.org"), "its last change read");
8096 }
8097
8098 #[test]
8104 fn own_past_rounds_left_on_the_server_go_in_the_mirror_too() {
8105 let p = Pair::new("own-past-mirror");
8106 let server = p.fake.files.join("Documents").join("Sioul");
8107 let mut left_there: BTreeMap<String, Vec<u8>> = BTreeMap::new();
8108 let mut now = DUE;
8109 let mut step = 0;
8110 while full_round_of(&p.phone) <= 1 {
8112 put(&p.phone.roots.config, "known-senders.txt", &senders(step));
8113 now += 600;
8114 p.turn(false, now);
8115 for (relative, bytes) in walk(&server).into_iter().filter(|(r, _)| r.starts_with(&p.phone.id) && r.ends_with(".jsonl")) {
8116 left_there.insert(relative, bytes);
8117 }
8118 p.turn(true, now + 30);
8119 step += 1;
8120 }
8121 p.turn(false, now + 60);
8122 p.turn(false, now + 70);
8123 let past = crate::share::rounds_past(&p.phone_folder, &p.phone.memory, &p.phone.id, (now + 80) * 1000).unwrap();
8124 assert!(past > 1, "below {past}");
8125 assert!(rounds_there(&server, &p.phone.id).iter().all(|r| *r >= past), "{:?}", rounds_there(&server, &p.phone.id));
8126 {
8128 let _turn = p.fake.turn.lock().unwrap();
8129 for (relative, bytes) in &left_there {
8130 if !server.join(relative).exists() {
8131 put_bytes(&server, relative, bytes);
8132 fake::bump(&server.join(relative));
8133 }
8134 }
8135 }
8136 let below: Vec<u32> = rounds_there(&server, &p.phone.id).into_iter().filter(|r| *r < past).collect();
8137 assert!(!below.is_empty());
8138 again_mirror_with(&p.phone.memory, &p.phone_folder, &p.phone.id, &p.fake.login(), now + 90, TEST);
8139 p.fake.forget_seen();
8141 let pulled = p.step(false, now + 100, true);
8142 assert!(pulled.problem.is_none(), "{pulled:?}");
8143 let seen = p.fake.seen();
8144 assert!(seen.iter().all(|l| !(l.starts_with("GET ") && below.iter().any(|r| l.ends_with(&format!("{}-{r}.jsonl", p.phone.id))))), "never brought back: {seen:?}");
8145 assert!(rounds_there(&p.phone_folder, &p.phone.id).iter().all(|r| *r >= past));
8146 assert!(rounds_there(&server, &p.phone.id).iter().all(|r| *r >= past), "{:?}, below {past}", rounds_there(&server, &p.phone.id));
8147 put(&p.phone.roots.config, "known-senders.txt", &senders(step + 1));
8149 p.turn(false, now + 200);
8150 p.turn(true, now + 230);
8151 assert!(std::fs::read_to_string(p.desk.roots.config.join("known-senders.txt")).unwrap_or_default().contains(&format!("old-sender-{:06}@example.org", (step + 2) * 1_500 - 1)));
8152 }
8153
8154 fn held_out_of_the_server(p: &Pair, dn: &Path, pn: &Path) -> (String, i64) {
8160 let turn = |desk: bool, now: i64| {
8161 let (device, folder) = p.device(desk);
8162 let before = p.step(desk, now, true);
8163 assert!(before.problem.is_none(), "{before:?}");
8164 device.exchange_all_parts(folder, now, if desk { dn } else { pn });
8165 let after = p.step(desk, now, false);
8166 assert!(after.problem.is_none(), "{after:?}");
8167 if p.sealed(desk, now).fetched > 0 {
8168 device.exchange_all_parts(folder, now, if desk { dn } else { pn });
8169 let _ = p.step(desk, now, false);
8170 }
8171 };
8172 put(dn, "x.md", "# x, a note put back later\n");
8173 put(dn, "y.md", "# y, a note that stays\n");
8174 turn(true, DUE);
8175 turn(false, DUE + 10);
8176 turn(false, DUE + 20);
8177 assert_eq!(std::fs::read_to_string(pn.join("x.md")).unwrap(), "# x, a note put back later\n", "fetched by name, written");
8178 let x = {
8179 let path = p.base.join("x-content");
8180 std::fs::write(&path, "# x, a note put back later\n").unwrap();
8181 crate::blobs::name(&KEY, &crate::blobs::hash_file(&path).unwrap().0)
8182 };
8183 let server = p.fake.files.join("Documents").join("Sioul");
8184 for copy in [p.desk_folder.join("blobs").join(&x), server.join("blobs").join(&x)] {
8185 set_mtime(©, SystemTime::now() - Duration::from_secs(100 * 86_400));
8186 }
8187 std::fs::remove_file(dn.join("x.md")).unwrap();
8188 turn(true, DUE + 100);
8189 turn(true, DUE + 800);
8190 turn(false, DUE + 900);
8191 assert!(!pn.join("x.md").exists(), "taken out on the phone, kept among its earlier versions");
8192 put(&p.desk.roots.config, "known-senders.txt", &senders(1));
8194 turn(true, DUE + 1000);
8195 put(&p.desk.roots.config, "known-senders.txt", &senders(2));
8196 turn(true, DUE + 1100);
8197 turn(false, DUE + 1200);
8198 turn(true, DUE + 1300);
8199 assert!(!server.join("blobs").join(&x).exists(), "freed, off the server");
8200 assert!(p.phone_folder.join("blobs").join(&x).exists(), "still in the phone's copy, as fetched");
8201 (x, DUE + 1300)
8202 }
8203
8204 fn the_desk_gets_x_back(p: &Pair, dn: &Path, pn: &Path, mut now: i64) {
8206 for _ in 0..3 {
8207 now += 100;
8208 let _ = p.step(true, now, true);
8209 p.desk.exchange_all_parts(&p.desk_folder, now, dn);
8210 let _ = p.step(true, now, false);
8211 if p.sealed(true, now).fetched > 0 {
8212 p.desk.exchange_all_parts(&p.desk_folder, now, dn);
8213 let _ = p.step(true, now, false);
8214 }
8215 }
8216 let _ = pn;
8217 assert_eq!(std::fs::read_to_string(dn.join("x.md")).unwrap_or_default(), "# x, a note put back later\n", "put back on the phone: back on the desk");
8218 }
8219
8220 #[test]
8226 fn a_note_put_back_in_the_mirror_reaches_the_others_once_its_content_left() {
8227 let p = Pair::new("mirror-put-back");
8228 let (dn, pn) = (p.base.join("desk-notes"), p.base.join("phone-notes"));
8229 let (x, now) = held_out_of_the_server(&p, &dn, &pn);
8230 let needed = "e".repeat(64);
8232 put(&p.phone_folder, &format!("blobs/{needed}"), "a stale copy this device still needs\n");
8233 {
8234 let mut state = State::load(&p.phone.memory);
8235 state.files.insert(format!("blobs/{needed}"), Fetched { etag: String::new(), size: 38 });
8236 state.save(&p.phone.memory).unwrap();
8237 let files = p.phone.memory.with_file_name("files.json");
8238 let mut said: serde_json::Value = serde_json::from_slice(&std::fs::read(&files).unwrap()).unwrap();
8239 said["needed"].as_array_mut().expect("said by each exchange of this Sioul").push(serde_json::json!(needed));
8240 std::fs::write(&files, serde_json::to_vec(&said).unwrap()).unwrap();
8241 }
8242 let later = now + BLOBS_AGAIN + 100;
8243 let stepped = p.step(false, later, true);
8244 assert!(stepped.problem.is_none() && stepped.let_go == 0, "never in a step: {stepped:?}");
8245 let sealed = p.sealed(false, later);
8246 assert!(sealed.problem.is_none() && sealed.let_go >= 1 && sealed.freed > 0 && sealed.listed_bytes > 0, "{sealed:?}");
8247 assert!(!p.phone_folder.join("blobs").join(&x).exists(), "the server no longer has it: let go");
8248 assert!(p.phone_folder.join("blobs").join(&needed).exists(), "still needed here: kept");
8249 put(&pn, "x.md", "# x, a note put back later\n");
8251 p.phone.exchange_all_parts(&p.phone_folder, later + 10, &pn);
8252 let pushed = p.step(false, later + 10, false);
8253 assert!(pushed.problem.is_none() && pushed.sent > 0, "{pushed:?}");
8254 let server = p.fake.files.join("Documents").join("Sioul");
8255 assert!(server.join("blobs").join(&x).exists(), "sent again");
8256 the_desk_gets_x_back(&p, &dn, &pn, later + 20);
8257 }
8258
8259 #[test]
8263 fn a_note_put_back_in_the_mirror_before_its_listing_reaches_the_others() {
8264 let p = Pair::new("mirror-put-back-early");
8265 let (dn, pn) = (p.base.join("desk-notes"), p.base.join("phone-notes"));
8266 let (x, now) = held_out_of_the_server(&p, &dn, &pn);
8267 put(&pn, "x.md", "# x, a note put back later\n");
8268 p.phone.exchange_all_parts(&p.phone_folder, now + 10, &pn);
8269 let pushed = p.step(false, now + 10, false);
8270 assert!(pushed.problem.is_none(), "{pushed:?}");
8271 let server = p.fake.files.join("Documents").join("Sioul");
8272 assert!(server.join("blobs").join(&x).exists(), "its own now: sent where the server lacked it");
8273 assert!(crate::share::own_sealed(&p.phone.memory, &p.phone.id).contains_key(&x));
8274 the_desk_gets_x_back(&p, &dn, &pn, now + 20);
8275 }
8276
8277 #[test]
8284 fn blobs_are_listed_seldom_whatever_their_number_and_come_by_name() {
8285 let p = Pair::new("blobs-many");
8286 let server = p.fake.files.join("Documents").join("Sioul");
8287 let login = p.fake.login();
8291 let step = |now: i64| step_with(&p.phone.memory, &p.phone_folder, &p.phone.id, &login, now, true, SLOW_STAND_IN);
8292 let sealed_step = |now: i64| sealed_step_with(&p.phone.memory, &p.phone_folder, &p.phone.id, &login, &KEY, now, false, SLOW_STAND_IN);
8293 {
8294 let _turn = p.fake.turn.lock().unwrap();
8295 for n in 0..11_000 {
8296 put(&server, &format!("blobs/{n:064x}"), "s");
8297 }
8298 }
8299 let lists = |seen: &[String]| seen.iter().filter(|l| l.starts_with("PROPFIND ") && l.split(' ').nth(1).is_some_and(|path| path.ends_with("/blobs"))).count();
8300 p.fake.forget_seen();
8302 let stepped = step(DUE + 100_000);
8303 assert!(stepped.problem.is_none() && lists(&p.fake.seen()) == 0, "never in a step: {stepped:?}");
8304 let sealed = sealed_step(DUE + 100_000);
8305 assert!(sealed.problem.is_none() && !sealed.listing_failed, "{sealed:?}");
8306 assert!(sealed.listed >= 11_000 && sealed.listed_bytes > 4 << 20, "past the old cap: {sealed:?}");
8307 assert_eq!(sealed.fetched, 0, "none waited for: none fetched");
8308 assert_eq!(lists(&p.fake.seen()), 1);
8309 let source = p.base.join("medium.bin");
8311 std::fs::write(&source, "a medium\n".repeat(100)).unwrap();
8312 let (hash, _) = crate::blobs::hash_file(&source).unwrap();
8313 let elsewhere = p.base.join("elsewhere");
8314 crate::blobs::put(&elsewhere, &KEY, &source, &hash).unwrap();
8315 let name = crate::blobs::name(&KEY, &hash);
8316 let sealed_bytes = std::fs::read(elsewhere.join("blobs").join(&name)).unwrap();
8317 put_bytes(&server, &format!("blobs/{name}"), &sealed_bytes);
8318 crate::blobs::Wanted::change(&p.phone.memory, |w| {
8319 w.media.insert(name.clone(), crate::blobs::Want { hash: hash.clone(), size: sealed_bytes.len() as u64, since: DUE, ..Default::default() });
8320 })
8321 .unwrap();
8322 p.fake.forget_seen();
8323 let _ = step(DUE + 100_600);
8324 let sealed = sealed_step(DUE + 100_600);
8325 assert!(sealed.problem.is_none(), "{sealed:?}");
8326 assert_eq!(lists(&p.fake.seen()), 0, "within the hour: not listed again");
8327 assert!(p.phone_folder.join("blobs").join(&name).exists() && sealed.fetched == 1, "fetched by its name: {sealed:?}");
8328 assert!(State::load(&p.phone.memory).files.contains_key(&format!("blobs/{name}")), "as fetched: never sent as its own");
8329 let other = "f".repeat(64);
8331 put(&server, &format!("blobs/{other}"), "not what it should be");
8332 crate::blobs::Wanted::change(&p.phone.memory, |w| {
8333 w.media.insert(other.clone(), crate::blobs::Want { hash: hash.clone(), size: 21, since: DUE, ..Default::default() });
8334 })
8335 .unwrap();
8336 let sealed = sealed_step(DUE + 100_700);
8337 assert!(sealed.held == 1 && !p.phone_folder.join("blobs").join(&other).exists(), "{sealed:?}");
8338 p.fake.fault(move |r| (r.method == "PROPFIND" && r.path.ends_with("/blobs")).then_some(Fault::Status(500)));
8340 let _ = step(DUE + 100_000 + BLOBS_AGAIN + 10);
8341 let sealed = sealed_step(DUE + 100_000 + BLOBS_AGAIN + 10);
8342 assert!(sealed.listing_failed && sealed.let_go == 0, "{sealed:?}");
8343 p.fake.forget_seen();
8344 let sealed = sealed_step(DUE + 100_000 + BLOBS_AGAIN + 70);
8345 assert!(!sealed.listing_failed && lists(&p.fake.seen()) == 0, "its backoff: not asked again: {sealed:?}");
8346 p.fake.clear_faults();
8347 let _ = step(DUE + 100_000 + 3 * BLOBS_AGAIN);
8348 let sealed = sealed_step(DUE + 100_000 + 3 * BLOBS_AGAIN);
8349 assert!(sealed.problem.is_none() && !sealed.listing_failed && sealed.listed >= 11_002, "{sealed:?}");
8350 assert!(p.phone_folder.join("blobs").join(&name).exists(), "still listed there: kept");
8351 }
8352
8353 #[test]
8359 fn a_content_fetched_and_named_here_is_sent_only_where_the_server_lacks_it() {
8360 let p = Pair::new("mirror-same-bytes");
8361 let (dn, pn) = (p.base.join("desk-notes"), p.base.join("phone-notes"));
8362 let turn = |desk: bool, now: i64| {
8363 let (device, folder) = p.device(desk);
8364 let notes = if desk { &dn } else { &pn };
8365 assert!(p.step(desk, now, true).problem.is_none());
8366 device.exchange_all_parts(folder, now, notes);
8367 assert!(p.step(desk, now, false).problem.is_none());
8368 if p.sealed(desk, now).fetched > 0 {
8369 device.exchange_all_parts(folder, now, notes);
8370 let _ = p.step(desk, now, false);
8371 }
8372 };
8373 put(&dn, "y.md", "# the same bytes in two notes\n");
8374 turn(true, DUE);
8375 turn(false, DUE + 10);
8376 turn(false, DUE + 20);
8377 assert_eq!(std::fs::read_to_string(pn.join("y.md")).unwrap_or_default(), "# the same bytes in two notes\n");
8378 put(&pn, "z.md", "# the same bytes in two notes\n");
8379 p.phone.exchange_all_parts(&p.phone_folder, DUE + 30, &pn);
8380 p.fake.forget_seen();
8381 let pushed = p.step(false, DUE + 30, false);
8382 assert!(pushed.problem.is_none(), "{pushed:?}");
8383 let seen = p.fake.seen();
8384 assert!(seen.iter().all(|l| !(l.starts_with("PUT ") && l.contains("/blobs/"))), "there already: never sent: {seen:?}");
8385 assert!(seen.iter().any(|l| l.starts_with("PROPFIND ") && l.contains("/blobs/")), "looked at, once: {seen:?}");
8386 p.fake.forget_seen();
8387 let _ = p.step(false, DUE + 40, false);
8388 assert!(p.fake.seen().iter().all(|l| !l.contains("/blobs/")), "{:?}", p.fake.seen());
8389 }
8390
8391 #[test]
8395 fn an_own_sealed_file_lost_on_the_server_goes_up_again() {
8396 let p = Pair::new("own-lost");
8397 let source = p.base.join("note.md");
8398 std::fs::write(&source, "a note\n".repeat(100)).unwrap();
8399 let (hash, _) = crate::blobs::hash_file(&source).unwrap();
8400 crate::blobs::put(&p.desk_folder, &KEY, &source, &hash).unwrap();
8401 let name = crate::blobs::name(&KEY, &hash);
8402 let there = p.fake.files.join("Documents/Sioul/blobs").join(&name);
8403 p.turn(true, DUE + 10);
8404 p.turn(true, DUE + 20);
8405 assert!(there.exists(), "sent");
8406 std::fs::remove_file(&there).unwrap();
8407 p.turn(true, DUE + 30);
8409 assert!(!there.exists());
8410 p.turn(true, DUE + 30 + BLOBS_AGAIN);
8412 p.turn(true, DUE + 40 + BLOBS_AGAIN);
8413 assert!(there.exists(), "sent again");
8414 }
8415
8416 #[test]
8419 fn a_listing_cut_short_is_no_listing() {
8420 let entry = |href: &str, kind: &str| format!("<d:response><d:href>{href}</d:href><d:propstat><d:prop>{kind}<d:getetag>"e"</d:getetag></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>");
8421 let whole = fake::multistatus(&format!("{}{}", entry("/dav/blobs/", "<d:resourcetype><d:collection/></d:resourcetype>"), entry("/dav/blobs/a", "<d:resourcetype/>")));
8422 let (listing, bytes) = read_listing(&whole[..], "https://example.org/dav/blobs/").unwrap();
8423 assert!(listing.items.contains_key("a") && bytes == whole.len() as u64);
8424 let cut = &whole[..whole.len() - "</d:multistatus>".len()];
8425 assert!(read_listing(cut, "https://example.org/dav/blobs/").is_err());
8426 let spaced = String::from_utf8(whole.clone()).unwrap().replace("</d:response>", "</d:response >").replace("</d:multistatus>", "</d:multistatus\r\n>");
8428 let (listing, _) = read_listing(spaced.as_bytes(), "https://example.org/dav/blobs/").unwrap();
8429 assert!(listing.items.contains_key("a"), "{listing:?}");
8430 let without = fake::multistatus(&entry("/dav/blobs/a", "<d:resourcetype/>"));
8432 assert!(read_listing(&without[..], "https://example.org/dav/blobs/").is_err());
8433 }
8434
8435 #[test]
8441 fn a_listing_that_leaves_files_out_never_takes_what_records_name() {
8442 let p = Pair::new("mirror-short-listing");
8443 let (dn, pn) = (p.base.join("desk-notes"), p.base.join("phone-notes"));
8444 let turn = |desk: bool, now: i64| {
8445 let (device, folder) = p.device(desk);
8446 let notes = if desk { &dn } else { &pn };
8447 assert!(p.step(desk, now, true).problem.is_none());
8448 device.exchange_all_parts(folder, now, notes);
8449 assert!(p.step(desk, now, false).problem.is_none());
8450 if p.sealed(desk, now).fetched > 0 {
8451 device.exchange_all_parts(folder, now, notes);
8452 let _ = p.step(desk, now, false);
8453 }
8454 };
8455 put(&dn, "today.md", "# today\n");
8456 for n in 0..12 {
8457 put(&dn, &format!("trip/{n}.md"), &format!("# day {n} of the trip\n"));
8458 }
8459 turn(true, DUE);
8460 turn(false, DUE + 10);
8461 assert_eq!(walk(&pn).len(), 13, "all here");
8462 std::fs::remove_dir_all(pn.join("trip")).unwrap();
8464 let held = p.phone.exchange_all_parts(&p.phone_folder, DUE + 20, &pn);
8465 assert!(held.withheld.as_ref().is_some_and(|w| w.iter().any(|w| w.count == 12)), "{held:?}");
8466 let server_blobs = p.fake.files.join("Documents").join("Sioul").join("blobs");
8468 let aside = p.base.join("blobs-aside");
8469 std::fs::rename(&server_blobs, &aside).unwrap();
8470 put(&server_blobs, &"0".repeat(64), "another device's");
8471 let now = DUE + 30 + BLOBS_AGAIN;
8472 assert!(p.step(false, now, true).problem.is_none());
8473 let sealed = p.sealed(false, now);
8474 assert!(sealed.listed > 0 && sealed.let_go == 0, "every content a record names kept: {sealed:?}");
8475 let back = crate::share::put_back_withheld(&p.phone_folder, &KEY, &p.phone.memory, &p.phone.id, &p.phone.all_parts(&pn), "files/notes/");
8476 assert_eq!(back, Ok(crate::share::PutBack { back: 12, held: 12 }));
8477 std::fs::remove_dir_all(&server_blobs).unwrap();
8478 std::fs::rename(&aside, &server_blobs).unwrap();
8479 }
8480
8481 #[test]
8486 fn the_background_asks_for_new_notes_and_the_window_writes_them_offline() {
8487 let p = Pair::new("mirror-background");
8488 let (dn, pn) = (p.base.join("desk-notes"), p.base.join("phone-notes"));
8489 let desk_turn = |now: i64| {
8490 assert!(p.step(true, now, true).problem.is_none());
8491 p.desk.exchange_all_parts(&p.desk_folder, now, &dn);
8492 assert!(p.step(true, now, false).problem.is_none());
8493 };
8494 put(&dn, "a.md", "# a\n");
8495 desk_turn(DUE);
8496 assert!(p.step(false, DUE + 10, true).problem.is_none());
8498 p.phone.exchange_all_parts(&p.phone_folder, DUE + 10, &pn);
8499 let _ = p.sealed(false, DUE + 10);
8500 p.phone.exchange_all_parts(&p.phone_folder, DUE + 10, &pn);
8501 assert_eq!(std::fs::read_to_string(pn.join("a.md")).unwrap_or_default(), "# a\n");
8502 put(&dn, "b.md", "# b, written while the phone was put away\n");
8504 desk_turn(DUE + 100);
8505 assert!(p.step(false, DUE + 200, true).problem.is_none());
8506 let quick = p.phone.exchange_parts(&p.phone_folder, DUE + 200, &pn, false);
8507 assert!(quick.waits.contains_key("not-read") && !pn.join("b.md").exists(), "{quick:?}");
8508 let sealed = p.sealed(false, DUE + 200);
8509 assert_eq!(sealed.fetched, 1, "asked for, and brought: {sealed:?}");
8510 p.fake.fault(|_| Some(Fault::Drop));
8512 p.phone.exchange_all_parts(&p.phone_folder, DUE + 300, &pn);
8513 assert_eq!(std::fs::read_to_string(pn.join("b.md")).unwrap_or_default(), "# b, written while the phone was put away\n");
8514 }
8515
8516 #[test]
8520 fn sealed_files_that_came_are_placed_together_never_over_one_there() {
8521 let p = Pair::new("mirror-place");
8522 let blobs = p.phone_folder.join("blobs");
8523 std::fs::create_dir_all(&blobs).unwrap();
8524 let ready: Vec<(String, PathBuf, u64)> = (0..3)
8525 .map(|n| {
8526 let came = blobs.join(format!(".came-{n}"));
8527 std::fs::write(&came, format!("sealed {n}")).unwrap();
8528 (format!("blobs/{n:064x}"), came, 8)
8529 })
8530 .collect();
8531 put(&p.phone_folder, &format!("blobs/{:064x}", 1), "sealed here meanwhile");
8532 assert_eq!(place_fetched(&p.phone.memory, &p.phone_folder, &p.fake.login().account, &ready), vec![true, false, true]);
8533 let state = State::load(&p.phone.memory);
8534 assert!(state.files.contains_key(&ready[0].0) && !state.files.contains_key(&ready[1].0) && state.files.contains_key(&ready[2].0));
8535 assert_eq!(std::fs::read_to_string(blobs.join(format!("{:064x}", 1))).unwrap(), "sealed here meanwhile");
8536 }
8537
8538 #[test]
8542 fn send_everything_again_finds_a_sealed_file_lost_while_blobs_cannot_be_listed() {
8543 let w = World::new("again-unlisted");
8544 assert!(w.find(DUE).confirmed_for(&w.phone_folder));
8545 let source = w.base.join("note.md");
8546 std::fs::write(&source, "a note\n".repeat(50)).unwrap();
8547 let (hash, _) = crate::blobs::hash_file(&source).unwrap();
8548 crate::blobs::put(&w.phone_folder, &KEY, &source, &hash).unwrap();
8549 let name = crate::blobs::name(&KEY, &hash);
8550 let size = std::fs::metadata(w.phone_folder.join("blobs").join(&name)).unwrap().len();
8551 let whole: BTreeMap<String, u64> = [(name.clone(), size)].into();
8552 assert!(send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &whole, DUE + 10, false, TEST).problem.is_none());
8553 let there = w.server.join("blobs").join(&name);
8554 assert!(there.exists());
8555 std::fs::remove_file(&there).unwrap();
8556 w.fake.fault(|r| (r.method == "PROPFIND" && r.path.ends_with("/blobs")).then_some(Fault::Status(500)));
8557 let again = send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &whole, DUE + 20, true, TEST);
8558 assert!(again.listing_failed && there.exists(), "asked of the server itself, sent again: {again:?}");
8559 }
8560
8561 #[test]
8565 fn large_sealed_files_are_paced_on_a_frugal_connection() {
8566 let p = Pair::new("mirror-frugal");
8567 crate::blobs::Wanted::change(&p.phone.memory, |w| {
8568 for n in 0..2 {
8569 w.blobs.insert(format!("{n:064x}"), crate::blobs::Want { hash: "h".into(), size: 2 << 20, since: DUE, ..Default::default() });
8570 }
8571 })
8572 .unwrap();
8573 let gets = |p: &Pair| p.fake.seen().iter().filter(|l| l.starts_with("GET ") && l.contains("/blobs/")).count();
8574 let login = p.fake.login();
8575 p.fake.forget_seen();
8576 let frugal = sealed_step_with(&p.phone.memory, &p.phone_folder, &p.phone.id, &login, &KEY, DUE + 10, true, TEST);
8577 assert!(gets(&p) == 1 && frugal.paced == 1, "one large at a time: {frugal:?}");
8578 p.fake.forget_seen();
8579 let _ = sealed_step_with(&p.phone.memory, &p.phone_folder, &p.phone.id, &login, &KEY, DUE + 20, true, TEST);
8580 assert_eq!(gets(&p), 0, "the next within ten minutes waits");
8581 let _ = sealed_step_with(&p.phone.memory, &p.phone_folder, &p.phone.id, &login, &KEY, DUE + 20 + PACE, true, TEST);
8582 assert!(gets(&p) >= 1, "ten minutes on, the next");
8583 }
8584
8585 #[test]
8589 fn a_medium_never_there_is_asked_three_days_then_left() {
8590 let p = Pair::new("mirror-media");
8591 let name = "e".repeat(64);
8592 crate::blobs::Wanted::change(&p.phone.memory, |w| {
8593 w.media.insert(name.clone(), crate::blobs::Want { hash: "h".into(), size: 10, since: DUE, ..Default::default() });
8594 })
8595 .unwrap();
8596 let asked = |p: &Pair| p.fake.seen().iter().filter(|l| l.starts_with("GET ") && l.contains(name.as_str())).count();
8597 p.fake.forget_seen();
8598 let _ = p.sealed(false, DUE + 10);
8599 let _ = p.sealed(false, DUE + 10 + WANTED_AGAIN);
8600 assert_eq!(asked(&p), 2, "twice in its first five minutes");
8601 p.fake.forget_seen();
8603 for minute in 0..60 {
8604 let _ = p.sealed(false, DUE + 86_400 + minute * 60);
8605 }
8606 assert!(asked(&p) <= 4, "less often than every five minutes: {}", asked(&p));
8607 p.fake.forget_seen();
8609 for at in [DUE + 3 * 86_400 + 10, DUE + 3 * 86_400 + 4_000, DUE + 4 * 86_400] {
8610 let _ = p.sealed(false, at);
8611 }
8612 assert_eq!(asked(&p), 0, "left");
8613 crate::blobs::Wanted::change(&p.phone.memory, |w| {
8615 if let Some(want) = w.media.get_mut(&name) {
8616 (want.since, want.again, want.tries) = (DUE + 4 * 86_400 + 60, 0, 0);
8617 }
8618 })
8619 .unwrap();
8620 let _ = p.sealed(false, DUE + 4 * 86_400 + 120);
8621 assert_eq!(asked(&p), 1, "asked again");
8622 }
8623}