diff --git a/CHANGELOG.md b/CHANGELOG.md index eafe18d6..5b4a7156 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,17 @@ breaks something, or changes the observable behavior of existing surfaces broadly; **patch** for fixes, and for additive opt-in capability — a new key, flag, or page that no existing user can encounter without opting into it. +## [Unreleased] + +### Fixed + + +- Daemon logs summarize healthy IPC connection starts and closes once per + minute instead of logging each one. Failures and subscriber lifecycle + events keep their per-connection attribution, including the original + first-write time. Livehost session polling now reuses its healthy + connection and names the reason when it opens a replacement. + ## [0.68.0] - 2026-09-08 Web serving. Files, directories, adapter documentation and the changelog are diff --git a/crates/spt-daemon/src/conn.rs b/crates/spt-daemon/src/conn.rs index 9c51ad4e..163fecb6 100644 --- a/crates/spt-daemon/src/conn.rs +++ b/crates/spt-daemon/src/conn.rs @@ -61,7 +61,7 @@ use std::io; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Condvar, Mutex, MutexGuard, OnceLock}; +use std::sync::{Arc, Condvar, LazyLock, Mutex, MutexGuard, OnceLock}; use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -99,12 +99,107 @@ pub(crate) fn init_log_anchor() { /// to field incident times (wall). // [impl->REQ-CONN-POISON-ATTRIBUTION] pub(crate) fn log_stamp() -> String { - let wall_ms = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_millis()) - .unwrap_or(0); - let mono_ms = MONO_ANCHOR.get_or_init(Instant::now).elapsed().as_millis(); - format!("wall_ms={wall_ms} mono_ms={mono_ms}") + let stamp = LogTime::now(); + format!("wall_ms={} mono_ms={}", stamp.wall_ms, stamp.mono_ms) +} + +/// Retain the original write instant even when its healthy line is aggregated. +#[derive(Clone, Copy)] +struct LogTime { + wall_ms: u128, + mono_ms: u128, +} + +impl LogTime { + fn now() -> Self { + Self { + wall_ms: SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis()) + .unwrap_or(0), + mono_ms: MONO_ANCHOR.get_or_init(Instant::now).elapsed().as_millis(), + } + } +} + +const HEALTHY_LOG_INTERVAL: Duration = Duration::from_secs(60); + +struct HealthyWindow { + since: Instant, + starts: u64, + closes: u64, +} + +/// One process-wide counter window, not one limiter per connection. No thread, +/// queue or per-conn healthy string: the next event after 60s emits the totals. +/// Exceptional and role-transition records bypass this window entirely. +// [impl->REQ-CONN-HEALTHY-LIFECYCLE-BOUNDED] +struct LifecycleLog { + healthy: Mutex, + #[cfg(test)] + capture: Option>>, + #[cfg(test)] + clock: Option>, +} + +impl LifecycleLog { + fn new() -> Self { + Self { + healthy: Mutex::new(HealthyWindow { + since: Instant::now(), + starts: 0, + closes: 0, + }), + #[cfg(test)] + capture: None, + #[cfg(test)] + clock: None, + } + } + + #[cfg(test)] + fn captured(now: Instant) -> Self { + let mut log = Self::new(); + log.healthy.get_mut().unwrap().since = now; + log.capture = Some(Mutex::new(Vec::new())); + log.clock = Some(Mutex::new(now)); + log + } + + fn emit(&self, args: std::fmt::Arguments<'_>) { + #[cfg(test)] + if let Some(lines) = &self.capture { + recover(lines).push(args.to_string()); + return; + } + spt_proto::emit_line_err!("{args}"); + } + + fn healthy(&self, start: bool) { + let mut window = recover(&self.healthy); + // Read time under the counter lock so concurrent callers cannot move + // the window backwards after waiting for a newer caller to flush. + let now = Instant::now(); + #[cfg(test)] + let now = self.clock.as_ref().map(|c| *recover(c)).unwrap_or(now); + if start { + window.starts = window.starts.saturating_add(1); + } else { + window.closes = window.closes.saturating_add(1); + } + let elapsed = now.saturating_duration_since(window.since); + if elapsed < HEALTHY_LOG_INTERVAL { + return; + } + let (starts, closes) = (window.starts, window.closes); + *window = HealthyWindow { since: now, starts: 0, closes: 0 }; + drop(window); + self.emit(format_args!( + "CONN_LIFECYCLE_SUMMARY: scope=broker-ipc {} interval_ms={} starts={starts} closes={closes} \ + [REQ-CONN-HEALTHY-LIFECYCLE-BOUNDED]", + log_stamp(), elapsed.as_millis() + )); + } } /// Attribution-label cap: facts accrete per role attach (a long-lived brain @@ -144,8 +239,9 @@ struct Inner { /// Accreted attribution facts (subscriber role, endpoint/session where /// known — [`BrokerConn::describe`]). Bounded by [`LABEL_CAP`]. label: Mutex, - /// Latch: the once-per-conn `write-start` lifecycle record fired. - first_write_logged: AtomicBool, + /// Once-per-conn original first committed write time; retained on failures. + first_write: OnceLock, + lifecycle_log: Arc, gate: Mutex, gate_cv: Condvar, dog: Mutex, @@ -223,7 +319,13 @@ impl Inner { fn attribution(&self) -> String { let label = recover(&self.label); let facts: &str = if label.is_empty() { "role=unattributed" } else { &label }; - format!("conn={} {} {}", self.id, facts, log_stamp()) + match self.first_write.get() { + Some(start) => format!( + "conn={} {} {} first_write_wall_ms={} first_write_mono_ms={}", + self.id, facts, log_stamp(), start.wall_ms, start.mono_ms + ), + None => format!("conn={} {} {} first_write=none", self.id, facts, log_stamp()), + } } /// Render the once-per-conn retirement record — the F-039 token split @@ -261,18 +363,17 @@ impl Inner { } } - /// One BOUNDED per-conn lifecycle record (leg d, doyle-confirmed - /// UNCONDITIONAL): `write-start` (first write only), `transport-close` - /// (drop), `writer-exit` / `*-attach` / `*-replaced` (emitted by the - /// broker's sink machinery through [`BrokerConn::lifecycle_event`]). The - /// timeout/cancel leg is the retirement record itself. + /// Per-conn exceptional and role-transition records remain unconditional. + /// releases#286 supersedes only healthy start/close emission, which uses + /// the process-wide interval counter instead. Retirement retains the + /// original first-write time even when the start line was aggregated. // [impl->REQ-CONN-POISON-ATTRIBUTION] fn lifecycle(&self, event: &str, extra: &str) { let sep = if extra.is_empty() { "" } else { " " }; - spt_proto::emit_line_err!( + self.lifecycle_log.emit(format_args!( "CONN_LIFECYCLE: {} event={event}{sep}{extra} [REQ-CONN-POISON-ATTRIBUTION]", self.attribution() - ); + )); } /// Abort the connection's in-flight I/O OUT OF BAND (both directions — @@ -287,10 +388,10 @@ impl Inner { // First poison of this conn: loud, once. The token is class-split // (F-039 leg a): deadline → CONN_WRITE_POISONED (the wedge // observable), organic fast-fail → CONN_WRITE_RETIRED. - eprintln!( + self.lifecycle_log.emit(format_args!( "{}", self.render_retirement(self.timed_out.load(Ordering::Acquire), cause) - ); + )); } #[cfg(windows)] // SAFETY: scalar kernel32 call on a handle whose owning half is kept @@ -424,11 +525,17 @@ impl BrokerConn { /// deadline (gate-wait + OS write completion) — the broker passes its /// `brain_write_deadline()` (`SPT_BRAIN_WRITE_DEADLINE_MS` knob). pub(crate) fn new(half: SendHalf, bound: Duration) -> Self { + static LOG: LazyLock> = LazyLock::new(|| Arc::new(LifecycleLog::new())); + Self::with_log(half, bound, Arc::clone(&LOG)) + } + + fn with_log(half: SendHalf, bound: Duration, lifecycle_log: Arc) -> Self { let raw = raw_of(&half); let inner = Arc::new(Inner { id: CONN_ID_SEQ.fetch_add(1, Ordering::Relaxed) + 1, label: Mutex::new(String::new()), - first_write_logged: AtomicBool::new(false), + first_write: OnceLock::new(), + lifecycle_log, gate: Mutex::new(Gate { half: Some(half) }), gate_cv: Condvar::new(), dog: Mutex::new(Dog { @@ -539,12 +646,12 @@ impl BrokerConn { return Ok(ConnWrite::Superseded); } - // Once-per-conn `write-start` lifecycle record (leg d): marks the conn's - // first committed write, so a conn that later retires can be correlated - // to when it went active (fresh-carrier churn shows as open→start→retire - // triplets with fresh ids). - if !inner.first_write_logged.swap(true, Ordering::AcqRel) { - inner.lifecycle("write-start", ""); + // The gate serializes first-write initialization. Save the instant + // BEFORE I/O so a later failure still names when this conn went active. + // [impl->REQ-CONN-HEALTHY-LIFECYCLE-BOUNDED] + if inner.first_write.get().is_none() { + let _ = inner.first_write.set(LogTime::now()); + inner.lifecycle_log.healthy(true); } // ── Check out the half + arm the watchdog, then write OUT of locks. ── @@ -638,17 +745,20 @@ impl BrokerConn { impl Drop for BrokerConn { fn drop(&mut self) { - // Once-per-conn `transport-close` lifecycle record (leg d): the handle - // is about to close — the terminal record of this conn id's life, with - // how it ended (organic EOF vs poisoned vs deadline-poisoned). - self.inner.lifecycle( - "transport-close", - &format!( - "poisoned={} timed_out={}", - self.inner.poisoned.load(Ordering::Acquire), - self.inner.timed_out.load(Ordering::Acquire) - ), - ); + // Only a healthy physical close is counted instead of logged per conn. + // Error/timeout closes keep their full attribution and original start. + // [impl->REQ-CONN-HEALTHY-LIFECYCLE-BOUNDED] + if self.inner.poisoned.load(Ordering::Acquire) { + self.inner.lifecycle( + "transport-close", + &format!( + "poisoned=true timed_out={}", + self.inner.timed_out.load(Ordering::Acquire) + ), + ); + } else { + self.inner.lifecycle_log.healthy(false); + } { let mut d = recover(&self.inner.dog); d.shutdown = true; @@ -688,6 +798,17 @@ mod tests { /// primitive, the client end returned so a test can hold it alive or drop /// it to force an ORGANIC write failure. fn make_conn() -> (BrokerConn, Stream) { + make_conn_with_log(Arc::new(LifecycleLog::new())) + } + + fn make_conn_with_log(log: Arc) -> (BrokerConn, Stream) { + let (conn, client, _recv) = make_duplex_with_log(log); + (conn, client) + } + + fn make_duplex_with_log( + log: Arc, + ) -> (BrokerConn, Stream, interprocess::local_socket::RecvHalf) { static SEQ: AtomicU32 = AtomicU32::new(0); let name = format!( "spt-daemon-connattr-{}-{}.sock", @@ -697,13 +818,78 @@ mod tests { let listener = LocalSocketTransport::bind(&name).expect("bind"); let client = LocalSocketTransport::connect(&name).expect("connect"); let server = listener.accept().expect("accept"); - let (_recv, send) = server.split(); + let (recv, send) = server.split(); ( - BrokerConn::new(send, Duration::from_millis(2000)), + BrokerConn::with_log(send, Duration::from_millis(2000), log), client, + recv, ) } + // [unit->REQ-CONN-HEALTHY-LIFECYCLE-BOUNDED] + #[test] + fn healthy_round_trips_and_reconnections_emit_one_interval_summary() { + let now = Instant::now(); + let log = Arc::new(LifecycleLog::captured(now)); + // The real framed-write/drop path, both a persistent carrier and fresh + // per-poll carriers. An isolated clock/sink avoids global test races. + const N: u64 = 32; + for round in 0..N { + let (conn, mut client, mut recv) = make_duplex_with_log(Arc::clone(&log)); + conn.describe("role=brain"); + for seq in 0..4 { + let request = Envelope::new("poll", serde_json::json!([round, seq])); + write_frame(&mut client, &request).unwrap(); + let received = crate::codec::read_frame(&mut recv).unwrap(); + assert_eq!(received.payload, request.payload); + assert_eq!(conn.write(&request).unwrap(), ConnWrite::Done); + let reply = crate::codec::read_frame(&mut client).unwrap(); + assert_eq!(reply.payload, request.payload); + } + drop(conn); + } + assert!(recover(log.capture.as_ref().unwrap()).is_empty()); + *recover(log.clock.as_ref().unwrap()) = now + HEALTHY_LOG_INTERVAL; + let (conn, mut client) = make_conn_with_log(Arc::clone(&log)); + conn.describe("role=brain"); + conn.write(&Envelope::new("poll", serde_json::json!(N))).unwrap(); + assert_eq!(crate::codec::read_frame(&mut client).unwrap().payload, serde_json::json!(N)); + drop(conn); + let lines = recover(log.capture.as_ref().unwrap()); + assert_eq!(lines.len(), 1, "healthy volume is per interval, not per conn"); + assert!(lines[0].contains(&format!("starts={}", N + 1))); + assert!(lines[0].contains(&format!("closes={N}"))); + } + + // [unit->REQ-CONN-HEALTHY-LIFECYCLE-BOUNDED] + // [unit->REQ-CONN-POISON-ATTRIBUTION] + #[test] + fn failures_and_role_events_bypass_healthy_aggregation_with_original_start() { + let log = Arc::new(LifecycleLog::captured(Instant::now())); + let (conn, mut client) = make_conn_with_log(Arc::clone(&log)); + conn.describe("role=brain controller session=7"); + conn.write(&Envelope::new("poll", serde_json::json!({}))).unwrap(); + crate::codec::read_frame(&mut client).unwrap(); + let start = *conn.inner.first_write.get().expect("first committed write"); + conn.lifecycle_event("writer-exit", "reason=write-failed"); + conn.inner.poison_and_cancel(Some(&io::Error::new(io::ErrorKind::BrokenPipe, "peer gone"))); + // Repeated cancellation must not repeat the retirement record. + conn.inner.poison_and_cancel(None); + let id = conn.id(); + drop(conn); + let lines = recover(log.capture.as_ref().unwrap()); + assert_eq!(lines.len(), 3, "role event, retirement and poisoned close survive"); + for line in lines.iter() { + assert!(line.contains(&format!("conn={id}"))); + assert!(line.contains("role=brain controller session=7")); + assert!(line.contains(&format!("first_write_wall_ms={}", start.wall_ms))); + assert!(line.contains(&format!("first_write_mono_ms={}", start.mono_ms))); + } + assert!(lines[1].contains("CONN_WRITE_RETIRED:")); + assert!(lines[1].contains("BrokenPipe")); + assert!(lines[2].contains("poisoned=true timed_out=false")); + } + // [unit->REQ-CONN-POISON-DIAL-SCOPE] the F-039 leg-(a) token split: the loud // CONN_WRITE_POISONED token is RESERVED for the deadline (timed_out) class; // an organic write failure renders the distinct CONN_WRITE_RETIRED token diff --git a/crates/spt-daemon/src/livehost.rs b/crates/spt-daemon/src/livehost.rs index 63f479d9..151c5e2e 100644 --- a/crates/spt-daemon/src/livehost.rs +++ b/crates/spt-daemon/src/livehost.rs @@ -569,7 +569,7 @@ pub fn resume_restart_orphaned_endpoints( registered: &[(AdapterRecord, Manifest)], adapters_dir: &Path, ) { - let Some(live) = query_live_session_endpoints() else { + let Some(live) = SessionPoll::default().query(&crate::endpoint::broker_socket_name()) else { return; // broker unreachable — skip (never mass-respawn on a hiccup) }; for id in perch::list_self_perch_ids(owlery) { @@ -995,20 +995,57 @@ pub fn reconcile_hosted_liveness(owlery: &Path, live_sessions: &BTreeSet offlined } -/// Query the broker for the set of endpoint ids it currently hosts a session for -/// (`KIND_SESSIONS`) — the B2 pull signal. `None` when the broker is unreachable -/// (the caller then SKIPS the offline pass this tick rather than mass-offlining). -fn query_live_session_endpoints() -> Option> { - let mut brain = Brain::cold_start(&crate::endpoint::broker_socket_name(), now_ms()).ok()?; - let reply = brain.sessions().ok()?; - Some( - reply - .sessions - .into_iter() - .map(|s| s.endpoint) - .filter(|e| !e.is_empty()) - .collect(), - ) +/// The livehost session census is request/reply only, never a subscription. +/// Retain its healthy carrier across 5s polls; discard it on EVERY query error +/// so a failed/partially consumed reply can never contaminate the next census. +/// `None` still means skip reconciliation, not an empty authoritative census. +// [impl->REQ-CONN-HEALTHY-LIFECYCLE-BOUNDED] +#[derive(Default)] +struct SessionPoll { + brain: Option, + retry: bool, +} + +impl SessionPoll { + fn query(&mut self, broker_name: &str) -> Option> { + if self.brain.is_none() { + let reason = if self.retry { "previous-query-error" } else { "initial-query" }; + match Brain::cold_start(broker_name, now_ms()) { + Ok(brain) => { + spt_proto::emit_line_err!( + "BRAIN_CONN_OPEN: caller=livehost-session-query reason={reason} \ + policy=reuse-until-query-error {}", + crate::conn::log_stamp() + ); + self.brain = Some(brain); + } + Err(e) => { + spt_proto::emit_line_err!( + "BRAIN_CONN_OPEN_FAIL: caller=livehost-session-query reason={reason} \ + kind={:?}: {e} {}", e.kind(), crate::conn::log_stamp() + ); + return None; + } + } + } + match self.brain.as_mut().expect("connected session census").sessions() { + Ok(reply) => Some( + reply.sessions.into_iter().map(|s| s.endpoint) + .filter(|e| !e.is_empty()).collect() + ), + Err(e) => { + // No same-tick retry: the caller skips this census exactly as + // before. Next tick starts a fresh physical connection. + self.brain = None; + self.retry = true; + spt_proto::emit_line_err!( + "BRAIN_CONN_RETIRED: caller=livehost-session-query reason=query-error \ + kind={:?}: {e} {}", e.kind(), crate::conn::log_stamp() + ); + None + } + } + } } /// The normalized program basename a perch's adapter would spawn its Psyche as — @@ -1333,11 +1370,12 @@ pub fn spawn_live_host(stop: Arc, reason: StartReason) -> JoinHandle // [impl->REQ-UPDATE-FINISH-ENDPOINT-SURVIVAL] resume_restart_orphaned_endpoints(&owlery, ®istered, &adapters_dir); } + let mut session_poll = SessionPoll::default(); while !stop.load(Ordering::SeqCst) { // TEST-ONLY gate `SPT_LIVEHOST_RECONCILE_DISABLE` (runtime env, DEFAULT // unset = normal production, untouched): skip the reconcile poll+body so an // int rig can prove REQ-UPDATE-TRIAL-DRAIN-DRIVE's core-loop KIND_SESSIONS - // reap-driver in ISOLATION. This loop's own `query_live_session_endpoints()` + // reap-driver in ISOLATION. This loop's own `SessionPoll::query` // → `brain.sessions()` (every LIVE_RECONCILE_INTERVAL_MS) would OTHERWISE // drive the same broker reap and mask the fix (todlando 2026-07-09). Setting // it faithfully REPRODUCES THE FIELD'S livehost-silent update-trial condition @@ -1371,7 +1409,7 @@ pub fn spawn_live_host(stop: Arc, reason: StartReason) -> JoinHandle // controllable==Some(true) gate inside keeps relay/legacy perches exempt. // [impl->REQ-HAZARD-HOSTED-LIVENESS-RECONCILE] // [impl->REQ-HAZARD-LIVEHOST-BOOT-LIVENESS-GATE] - if let Some(live) = query_live_session_endpoints() { + if let Some(live) = session_poll.query(&crate::endpoint::broker_socket_name()) { reconcile_hosted_liveness(&owlery, &live); } reconcile_once(&owlery, ®istered, &adapters_dir, &set, &cfg, reason); @@ -1397,6 +1435,58 @@ mod tests { use crate::test_home::with_home; use std::time::{Duration, Instant}; + // [unit->REQ-CONN-HEALTHY-LIFECYCLE-BOUNDED] + #[test] + fn session_poll_reuses_healthy_carrier_and_reopens_after_query_error() { + use crate::codec::{read_frame, write_frame}; + use crate::frame::{Envelope, Role}; + use crate::transport::{recv_hello, DaemonTransport, LocalSocketTransport}; + use std::sync::atomic::AtomicU32; + + static SEQ: AtomicU32 = AtomicU32::new(0); + let name = format!( + "spt-livehost-poll-{}-{}", std::process::id(), + SEQ.fetch_add(1, Ordering::Relaxed) + ); + let listener = LocalSocketTransport::bind(&name).unwrap(); + const N: usize = 16; + let server = std::thread::spawn(move || { + let mut accepted = 0; + let mut conn = listener.accept().unwrap(); + accepted += 1; + recv_hello(&mut conn, Role::Brain).unwrap(); + for _ in 0..N { + assert_eq!(read_frame(&mut conn).unwrap().kind, crate::msg::KIND_SESSIONS); + write_frame(&mut conn, &Envelope::new( + crate::msg::KIND_SESSIONS_REPLY, serde_json::json!({"sessions": []}) + )).unwrap(); + } + // A malformed reply is a query error even while the transport is + // still alive. It must retire, not leave a desynchronized cache. + assert_eq!(read_frame(&mut conn).unwrap().kind, crate::msg::KIND_SESSIONS); + write_frame(&mut conn, &Envelope::new( + crate::msg::KIND_SESSIONS_REPLY, serde_json::json!({"sessions": false}) + )).unwrap(); + assert!(read_frame(&mut conn).is_err(), "failed query closes old carrier"); + let mut conn = listener.accept().unwrap(); + accepted += 1; + recv_hello(&mut conn, Role::Brain).unwrap(); + assert_eq!(read_frame(&mut conn).unwrap().kind, crate::msg::KIND_SESSIONS); + write_frame(&mut conn, &Envelope::new( + crate::msg::KIND_SESSIONS_REPLY, serde_json::json!({"sessions": []}) + )).unwrap(); + accepted + }); + let mut poll = SessionPoll::default(); + for _ in 0..N { + assert_eq!(poll.query(&name), Some(BTreeSet::new())); + } + assert_eq!(poll.query(&name), None, "failure is not an empty session census"); + assert_eq!(poll.query(&name), Some(BTreeSet::new())); + drop(poll); + assert_eq!(server.join().unwrap(), 2, "one healthy carrier plus one recovery"); + } + // A no-op Psyche summarizer command: spawns + exits 0 (we never depend on its // output — the daemon online-stamp is what hosts it, not the pid). #[cfg(windows)] diff --git a/docs/RCA-FLEET-DAEMON-14444.md b/docs/RCA-FLEET-DAEMON-14444.md index f7e897d1..010613fd 100644 --- a/docs/RCA-FLEET-DAEMON-14444.md +++ b/docs/RCA-FLEET-DAEMON-14444.md @@ -69,3 +69,76 @@ per 2 MB of log, under `[REQ-CONN-POISON-ATTRIBUTION]`, is its own finding. (image path or start time), which is the only way reuse becomes a kill. My own standing rule from the orphan reap is exactly this: re-verify the path AT kill time. 3. **Bound the log volume** independently of the cause. + +## releases#286 — stderr follow-up (2026-09-09) + + + +The historical 303 MB incident above remains the reported baseline, not a +newly reproduced event. A read-only sample of the currently installed +`%LOCALAPPDATA%/spt-core/logs/daemon.stderr.log` ended at byte **4,163,267**: +the last **1,999,958 complete-line bytes** spanned wall stamps +**1788997126331–1788998151874** (1025.543 s). It contained **5127** first writes +(656,158 bytes), **5092** unpoisoned/non-timeout closes (850,622 bytes), +**3** organic retirements, **0** deadline poisons, **107** family-gate lines +and **108** rendezvous-up lines. Healthy start/close records alone consumed +**75.3%** of that sample. A subsequent live append observation over +**307.203 s** added **79,741 bytes** (**259.57 B/s**), with **104** starts, +**104** healthy closes and **30** family gates paired with **30** rendezvous +binds. These are different observation windows, not a claimed constant rate. +No daemon was restarted or reconfigured for either measurement. + +**Cause discrimination.** `role=brain` identifies *every broker IPC client*, +not the supervised coordinator. `run_brain` already keeps its heartbeat +`Brain` alive. In contrast, the pre-fix +`livehost::query_live_session_endpoints` constructed `Brain::cold_start`, +called `sessions`, and dropped the carrier on every invocation from the +five-second reconcile loop. This is a source-confirmed avoidable reopen +cause, consistent with the recurring five-second bare-brain start/close +pairs in the live log, **not an attribution of all 5127 connections to that +caller**. Other short-lived callers include dispatch idle queries and +per-operation dispatch/CLI clients; their lifetimes are not changed here. + +The livehost census now owns one carrier across polls. Its `BRAIN_CONN_OPEN` +breadcrumb names `caller=livehost-session-query` and distinguishes +`reason=initial-query` from `reason=previous-query-error`. Every query error +discards the carrier, returns an unavailable census (never an empty +authoritative census), and permits a fresh connection only on the next +scheduled query. The one-shot boot orphan census remains one-shot. +No read/write deadline, poison semantics, or retry cadence changed. + +**Family gate exonerated.** The sole diagnostic site in +`NetEndpoint::bind_scoped` is already once per real bind attempt, not per IPC +connection. `pairhost::spawn_meet_rotation` binds a new derived identity for +each attached subnet at every 30-second TOTP boundary; the observed +`BIGNET`, `SPT_DEV`, and `SPT_MANTLE` windows account for the three lines per +boundary. These listeners cannot share a stable identity without changing +the pairing protocol. Bind and diagnostic policies are intentionally +unchanged; a process-global "log once" would hide later real binds. + +**Explicit contract amendment.** Issue #286 supersedes only the +`REQ-CONN-POISON-ATTRIBUTION` leg-(d) requirement for unconditional +*healthy start/close lines*. Across the broker process these become +`CONN_LIFECYCLE_SUMMARY` totals, at most once per 60-second monotonic interval, +flushed by the first subsequent lifecycle counter event. Starts count the +first committed write attempt; closes count unpoisoned physical drops. +There is no timer thread and no shutdown flush, so a quiet tail remains +pending until another counter event (and can be lost on process exit). +Poison, organic failure, poisoned close, and all role-transition records +remain unconditional per connection. They retain conn id, role facts, +wall/monotonic stamps **and the original first-write wall/monotonic time**; +a connection that never wrote says `first_write=none`. + +The deterministic regression cells exercise real framed socket exchanges +over both retained and fresh carriers with an isolated fixed log clock/sink, +assert one interval summary for N healthy connections, preserve exceptional +records through the same sink, and drive livehost census reuse plus recovery +from a malformed reply. Focused commands (not executed in the preparation +lane; integration owner runs them after landing): + +```text +cargo test -p spt-daemon --lib conn::tests:: +cargo test -p spt-daemon --lib livehost::tests::session_poll_reuses_healthy_carrier_and_reopens_after_query_error +cargo test -p spt-daemon --test conn_blackhole_lifecycle +traceable-reqs check --json +``` diff --git a/traceable-reqs.toml b/traceable-reqs.toml index 21917ea9..9224b3aa 100644 --- a/traceable-reqs.toml +++ b/traceable-reqs.toml @@ -2569,6 +2569,14 @@ id = "REQ-CONN-POISON-ATTRIBUTION" title = "MSG-IDENTITY W6 / F-039 legs b-d (doyle W6 LOCK 2026-07-10, minted per amendment 3): every broker-conn lifecycle record is ATTRIBUTABLE — the W6 RCA's terminal undecidability (per-line 1:1 CONN_WRITE_POISONED churn = fresh-carrier churn OR stderr interleave artifact) exists because records carry no stable conn identity, no role/endpoint/session context, and no timestamps, and the once-per-conn poison latch hides multiplicity. THREE LEGS. (b) IDENTITY: mint a stable per-physical-conn id (monotonic u64 at conn construction — Arc::ptr_eq is the only identity today and it does not survive a log line) plus subscriber role and endpoint/session where known, stamped on CONN_WRITE_POISONED, CONN_WRITE_RETIRED, logical stall-evict, attach/resume/detach, and write-retirement records (RCA attach sites: presence nethost.rs:379, stream nethost.rs:258, controller broker.rs:891, viewer broker.rs:1073). (c) TIME: daemon stderr correlation records carry wall-clock AND monotonic timestamps (stderrlog has neither; broker+brain share one file — interleave is unresolvable without them). (d) LIFECYCLE (doyle-confirmed UNCONDITIONAL, not debug-gated): one BOUNDED set of per-conn lifecycle events — write start/timeout-cancel/transport close/writer exit/replacement-reattach (hertz RCA fix-shape items 1-3). Constraint (doyle LOCK): the split/attribution must not REDUCE total information, only correct its attribution; NO timeout-value changes; NO suppression-as-fix. Gate: unit — lifecycle records carry conn id + role + timestamps; the id is unique per physical conn and stable across that conn's records. Kin REQ-CONN-POISON-DIAL-SCOPE (leg a, the token split these fields ride on), REQ-CONN-BLACKHOLE-LIFECYCLE-HARNESS (leg e, consumes these records), REQ-HAZARD-SHAREDSEND-NO-BLOCKING-WRITE-UNDER-LOCK (behavior invariant preserved)." required_stages = ["impl", "unit"] # FLIPPED in the W6 build commit carrying the evidence (todlando 2026-07-10). impl = conn id mint + describe/label accretion + log_stamp wall+mono (anchored at both daemon entries) + bounded lifecycle events (write-start, transport-close, writer-exit, attach/replace/detach) + attribution on stall-evict records. unit = conn.rs record-shape assertions (id/role/time present + parity across both retirement tokens, id unique per conn, label bounded). +# releases#286 narrowly supersedes leg (d)'s unconditional healthy start/close +# lines. Failure/poison, role transitions and original first-write time remain +# attributable per physical conn; healthy counts are interval totals instead. +[[requirements]] +id = "REQ-CONN-HEALTHY-LIFECYCLE-BOUNDED" +title = "Healthy broker IPC starts and closes aggregate once per monotonic interval across connections, independently of request volume; poison, failure and role-transition evidence remains per-connection with original first-write time. Livehost session polling reuses its healthy carrier and names why it opens a new one; role=brain alone never identifies the supervised coordinator. Network family diagnostics remain once per real endpoint bind attempt, not IPC churn. Narrow healthy-start/close supersession of REQ-CONN-POISON-ATTRIBUTION leg d (releases#286)." +required_stages = ["doc", "impl", "unit"] + [[requirements]] id = "REQ-CONN-BLACKHOLE-LIFECYCLE-HARNESS" title = "MSG-IDENTITY W6 / F-039 leg e (doyle W6 LOCK 2026-07-10, minted per amendment 3 — hertz's five invariants VERBATIM from his RCA fix-shape item 5): 'Build a deterministic black-holed-controller harness against current v0.30.6 semantics and assert: unrelated sessions continue; the bad physical connection is canceled/closed within the bound; its writer exits; a fresh viewer can attach; no lock or task remains owned by the retired connection.' The harness is the standing conformance rig for the r4 SHAREDSEND fix-class — hertz's RCA discipline: only after a timestamped incident maps to a FAILING lifecycle invariant does an ownership/cancellation defect get fixed (the likely shape being complete physical-connection cancellation and writer-task join/retirement, never a broader timeout increase). Consumes REQ-CONN-POISON-ATTRIBUTION's records (conn id + lifecycle events are what make the five assertions checkable deterministically). Kin REQ-HAZARD-SHAREDSEND-NO-BLOCKING-WRITE-UNDER-LOCK (the invariant class under test — its brain_decouple int stays the Windows-mandatory gate leg), REQ-CONN-POISON-DIAL-SCOPE." @@ -4860,6 +4868,7 @@ name = "message-format-identity" requirements = [ "REQ-CONN-BLACKHOLE-LIFECYCLE-HARNESS", "REQ-CONN-POISON-ATTRIBUTION", + "REQ-CONN-HEALTHY-LIFECYCLE-BOUNDED", "REQ-CONN-POISON-DIAL-SCOPE", "REQ-DAEMON-SERVICE-INSTALL", "REQ-EP-3",