diff --git a/crates/spt-daemon/tests/twohost_web.rs b/crates/spt-daemon/tests/twohost_web.rs index 9f6c19a2..7022c2de 100644 --- a/crates/spt-daemon/tests/twohost_web.rs +++ b/crates/spt-daemon/tests/twohost_web.rs @@ -45,7 +45,7 @@ //! daemons); the two-box climb is the gate's field leg. use std::io::{Read, Write as _}; -use std::net::{IpAddr, SocketAddr, TcpStream}; +use std::net::{IpAddr, SocketAddr, TcpStream, UdpSocket}; use std::path::Path; use std::sync::atomic::{AtomicBool, AtomicU64}; use std::sync::Arc; @@ -120,6 +120,21 @@ const RANGE_OFFSET: u16 = 1; const DENY_OFFSET: u16 = 2; /// The requester-cell offset the helper cell uses — B dials this one A-ward. const HELPER_OFFSET: u16 = 3; +/// The INBOUND PROBE's offset. Clear of the four requester cells (0..=3) and +/// inside the same `port_a + offset` range the rig already uses, so whatever +/// rule lets the ceremony's ports through covers this one too — a probe on a +/// port outside that range would vouch for a path nothing else takes. +const PROBE_OFFSET: u16 = 9; +/// How long A listens and B sends. Ten seconds against the ceremony's 900: the +/// point of the probe is that a blocked link is answered in the time it takes +/// to notice, not in the time it takes to give up. +const PROBE_WINDOW: Duration = Duration::from_secs(10); +const PROBE_MAGIC: &[u8] = b"SPT-TWOHOST-INBOUND-PROBE-v1"; +/// A's liveness beacon, A→B. Distinct from the probe and the ack because a +/// listener that cannot tell them apart is a clock that never starts. +const PROBE_BEACON: &[u8] = b"SPT-TWOHOST-INBOUND-BEACON-v1"; +/// A's receipt, A→B, and B's fast green. +const PROBE_ACK: &[u8] = b"SPT-TWOHOST-INBOUND-ACK-v1"; const HELPER_BYTES: &[u8] = b"# the file the user pointed at\n"; const OPEN_ENDPOINT: &str = "w1-open"; @@ -296,6 +311,336 @@ fn helper_envelope(quoted: &str) -> String { ) } +// ══ THE INBOUND PROBE ═══════════════════════════════════════════════════════ +// A golden run once spent 900 s on each half discovering that B's datagrams +// never reached A: every QUIC dial died on the 10 s bound — 75 of them, cadenced +// at exactly 12.00 s, never one ADMIT. +// +// TWO LAYERS were behind it and EITHER ALONE produces the same silence, which is +// why the first fix looked wrong when it changed nothing: +// 1. the host firewall — BlockInbound on every profile, and its allow rules are +// PER-EXECUTABLE, so no rule covered the test binary and the runner (a +// service) never shows an Allow dialog; +// 2. the TAILNET ACL, asymmetric — a member device may open flows TO a TAGGED +// resource and the reverse is denied, so the receiver permits inbound from a +// list the sender is simply absent from. +// Every face that ever appeared to work rode RETURN traffic of flows the receiver +// had opened; a COLD claim in this direction can never cross. The ceremony cannot +// tell that apart from a pairing fault: at B a datagram dropped by a firewall +// looks exactly like one that arrived and was ignored, so the failure was +// triaged as product, then rig, before it was measured as box. +// +// WHY THIS LIVES IN THIS BINARY rather than in a shell script beside it, which +// would have been the smaller change: the Windows rule is PER-EXECUTABLE. An +// allow rule naming pwsh, python, or a bench script says nothing about +// `twohost_web-.exe`, so a probe written in any of them would today agree +// with reality by coincidence and would go GREEN the moment anyone allowed the +// prober — certifying the exact failure it exists to catch. Only a probe that +// IS the test binary carries the test binary's firewall identity. (It is also +// why the operator rule must be PORT+REMOTE scoped: a program-scoped rule is +// orphaned by the next rebuild's hash, exactly as the dead `_work\spt-core` +// rules were. This probe stays correct under either.) +// +// ── THERE IS NO SHARED CLOCK, SO THERE IS NO FIXED WINDOW ──────────────────── +// +// The step boundary orders the probe BEFORE the ceremony on ONE host. It does +// NOT align the two hosts: after the ladder's last rung each half pays its own +// wrapper and its own `cargo test` invocation, and on the Windows side that is a +// fingerprint scan plus Defender's first touch of a fresh exe — seconds to tens +// of seconds, measured on that box all week. +// +// A FIXED SYMMETRIC WINDOW THEREFORE ADDS A THIRD CAUSE THAT LOOKS EXACTLY LIKE +// THE OTHER TWO: "the peer is not up yet" produces the same silence as a +// firewall drop and an ACL denial, and a failure text naming two layers would be +// confidently wrong about a third. MEASURED, not argued: the first hand-run of +// this probe in the A=kitsubito direction (2026-09-09) redded INBOUND_BLOCKED at +// 10.24 s purely because `cargo` startup on the sender put its first datagram +// after the receiver's 10 s window closed. Same silence, innocent box — the +// defect class this probe exists to document, reproduced inside the probe. +// +// So the rendezvous is DATA, not timing: +// * A listens up to the rig's own budget (`SPT_TWO_HOST_WAIT_SECS`) and stops +// the instant a PROBE datagram arrives. +// * A BEACONS to B's probe port once a second while it listens. A→B is the +// direction that is open under both faults (measured 3/3 from this host on +// 2026-09-08 while the reverse read 0/3) — THE DESIGN DEPENDS ON IT, which is +// why B's timeout for "no beacon at all" is its own fourth outcome below and +// never INBOUND_BLOCKED. +// * B binds its probe port BEFORE it sends anything, waits for the first +// beacon, and only then starts a ten-second clock. B has now excluded "A is +// not up", so its red can name the two layers honestly. +// * A ACKs the first PROBE it receives, to the address it came from. B stops on +// the ACK — the fast green — and the ACK is return traffic of a flow B opened, +// which crosses under either fault, so it proves nothing on its own and is +// not asked to. +// +// THREE DISTINCT MAGICS, because a listener that cannot tell a beacon from an ack +// is a clock that never starts and a red that never fires. +// +// THE ASSERTION LIVES ON A, deliberately: B cannot know whether its datagram +// arrived, so a B-side "I sent it" proves nothing about the path. A's own red +// waits the full budget ONCE, named, instead of 900 s per cell per half. +#[test] +fn two_host_inbound_probe_role_a() { + let Some(rig) = Rig::from_env("a") else { + return; + }; + let port = rig.port_a + PROBE_OFFSET; + let beacon_target = SocketAddr::new(rig.peer_ip, rig.port_b + PROBE_OFFSET); + let socket = UdpSocket::bind(SocketAddr::new(IpAddr::from([0, 0, 0, 0]), port)) + .unwrap_or_else(|e| panic!("INBOUND_PROBE: role A could not bind udp {port}: {e}")); + socket + .set_read_timeout(Some(Duration::from_millis(250))) + .expect("probe socket takes a read timeout"); + // ⚠ THE LISTENING SOCKET NEVER SENDS. THAT IS THE WHOLE DESIGN. + // + // Every outbound datagram from `port` — a beacon, an ack, anything — opens + // stateful firewall / NAT return state for that port, so B's probe then + // arrives as SOLICITED return traffic and crosses under exactly the two + // faults this cell exists to detect. That is this file's own warning — + // "solicited return traffic works under either fault, so an echo reply + // proves nothing about this direction" — applied to the guard rather than to + // the ceremony. + // + // MEASURED, not reasoned (2026-09-09): with the beacon sharing the listening + // socket, the out-of-grant control arm (probe port forced to 7509, outside + // the operator's udp 7460-7499 rule) went GREEN — the probe certified a path + // that a listen-only run had measured as BLOCKED eight minutes earlier. The + // beacon and the ack therefore both leave this ephemeral socket, and B sends + // from an ephemeral port of its own, so no run repeats a 4-tuple a previous + // run opened and none can poison the next. + let beacon_socket = UdpSocket::bind(SocketAddr::new(IpAddr::from([0, 0, 0, 0]), 0)) + .expect("probe beacon binds an ephemeral port"); + println!( + "TWOHOST-WEB probe role A: listening on udp {port} for up to {:?}, beaconing to {beacon_target}", + rig.wait + ); + + let deadline = Instant::now() + rig.wait; + let mut buf = [0u8; 64]; + let mut seen = 0usize; + let mut beacons = 0usize; + let mut last_beacon = Instant::now() - Duration::from_secs(1); + while Instant::now() < deadline { + // One beacon a second, so B can start its own clock from evidence that + // this side is up rather than from a guess about start-up skew. + if last_beacon.elapsed() >= Duration::from_secs(1) { + match beacon_socket.send_to(PROBE_BEACON, beacon_target) { + Ok(_) => beacons += 1, + Err(e) => println!("TWOHOST-WEB probe role A: beacon send error (continuing): {e}"), + } + last_beacon = Instant::now(); + } + match socket.recv_from(&mut buf) { + Ok((n, from)) if buf[..n] == *PROBE_MAGIC => { + seen += 1; + println!("TWOHOST-WEB probe role A: datagram {seen} from {from} after {beacons} beacon(s)"); + // The ack is B's fast green. It is RETURN traffic of a flow B + // opened, so it crosses under either fault — which is exactly why + // nothing is inferred from it beyond "A heard you". + // OUT OF THE BEACON SOCKET, not this one: an ack sent from the + // listening port would write return state for the very tuple the + // next run must find cold, so the guard would poison its own + // next measurement at the end of every run, green or red. + // + // AND ADDRESSED TO B'S BEACON PORT, not to `from`. MEASURED + // 2026-09-09: acking to `from` (B's ephemeral probe port) leaves + // A's ack unsolicited at B — it comes from this ephemeral socket, + // not from the port B dialled, so it matches no return state B + // opened — and a receiver whose inbound rule is a fixed port + // RANGE drops it. B then reported INBOUND_BLOCKED on a link its + // own peer had just certified: the A-side said INBOUND OK and the + // B-side said blocked, in the same run. The rig's port range is + // the only inbound address B can be reached on cold, so the ack + // uses it, and B reads the ack on the socket it already binds. + if let Err(e) = beacon_socket.send_to(PROBE_ACK, beacon_target) { + println!("TWOHOST-WEB probe role A: ack send error (continuing): {e}"); + } + break; + } + // A stray datagram is not this probe's business; keep waiting rather + // than passing on someone else's traffic. + Ok((n, from)) => println!("TWOHOST-WEB probe role A: {n} foreign bytes from {from}"), + Err(_) => {} + } + } + + // ITS OWN OUTCOME, never the ceremony's error: this reds as INBOUND_BLOCKED + // and names the peer, the port, the budget and the fix, so the next box + // regression is answered by the failure text instead of a 900 s investigation. + assert!( + seen > 0, + "INBOUND_BLOCKED: role A received NO probe datagram on udp {port} from {} within {:?} \ + (and sent {beacons} beacon(s) to {beacon_target} in that time). +This is a BOX rule, not a product or rig fault: the ceremony below would spend the full +SPT_TWO_HOST_WAIT_SECS budget dialling a host that cannot receive, and would report it as a pairing +failure. + +THERE ARE TWO LAYERS AND EITHER ONE ALONE PRODUCES THIS EXACT SILENCE. Fixing only the first is the +mistake this text exists to prevent: it is NECESSARY, NOT SUFFICIENT, and the run reds identically +afterwards. + + LAYER 1 - the Windows host firewall on the receiver. BlockInbound on every profile, and its allow + rules are PER-EXECUTABLE, so a rule naming pwsh or python says nothing about this test binary and + the runner (a service) never shows an Allow dialog. Fix, operator, elevated - allow INBOUND UDP + {}-{} from {} only: PORT+REMOTE scoped, never program-scoped, or the next rebuild binary hash + orphans the rule the way the dead _work/spt-core rules were orphaned: + New-NetFirewallRule -DisplayName 'spt twohost rig inbound' -Direction Inbound -Protocol UDP -LocalPort {}-{} -RemoteAddress {} -Action Allow + + LAYER 2 - the TAILNET ACL, which is asymmetric and denies this direction. The sender is a TAGGED + resource owned by another tailnet user; a member device may open flows TO it, and the reverse is + denied, so the receiver permits inbound from a list this sender is simply absent from. Every + helper-stall face that ever appeared to work rode RETURN traffic of flows the receiver opened; a + COLD claim in this direction can never cross. Fix, operator - grant it in the tailnet policy + (asked on releases#272: src tag:eye-tracking-resource -> dst this host, udp 7460-7499). Verify with + `tailscale debug netmap` on the RECEIVER: the sender IP must appear among the PacketFilter + permitted inbound sources. + +THE DISCRIMINATOR, ten seconds, run it before assuming which layer bit: bind the SAME listener and +send to it once over the LAN and once over Tailscale. LAN receives and Tailscale does not = LAYER 2, +the host firewall is not your problem. Neither receives = LAYER 1 (or both). Solicited return traffic +works under either fault, so an echo reply proves nothing about this direction. + +WHAT THIS RED HAS ALREADY EXCLUDED: the peer being slow to start. B only begins its own clock once it +has received a beacon from this host, and this cell listened for the whole budget above. + +Port range above is exactly what THIS rig binds (port_a..probe); releases#272 carries the wider +7460-7499, which covers it. Apply the board's range if they differ - this message and that comment +must never become two different commands.", + rig.peer_ip, + rig.wait, + rig.port_a, + rig.port_a + PROBE_OFFSET, + rig.peer_ip, + rig.port_a, + rig.port_a + PROBE_OFFSET, + rig.peer_ip, + ); + println!("TWOHOST-WEB probe role A: INBOUND OK on udp {port}"); +} + +/// B's half: wait for proof that A is up, then send for exactly ten seconds. +/// +/// The ten seconds are the point — a blocked link is answered in the time it +/// takes to notice, not in the time it takes to give up — but they only start +/// once a beacon has arrived, because a clock started before the peer exists +/// measures start-up skew and calls it a firewall. +#[test] +fn two_host_inbound_probe_role_b() { + let Some(rig) = Rig::from_env("b") else { + return; + }; + let listen = rig.port_b + PROBE_OFFSET; + let target = SocketAddr::new(rig.peer_ip, rig.port_a + PROBE_OFFSET); + // BOUND BEFORE ANYTHING IS SENT: A beacons to this port cold, and a beacon + // that arrives before the bind is a beacon that never happened. + let socket = UdpSocket::bind(SocketAddr::new(IpAddr::from([0, 0, 0, 0]), listen)) + .unwrap_or_else(|e| panic!("INBOUND_PROBE: role B could not bind udp {listen}: {e}")); + socket + .set_read_timeout(Some(Duration::from_millis(250))) + .expect("probe socket takes a read timeout"); + // ⚠ THE PROBES LEAVE A SEPARATE, EPHEMERAL SOCKET — this fixed one only ever + // RECEIVES beacons. + // + // With a fixed source port here, the tuple (B:port_b+9 → A:probe) REPEATS + // from run to run, so any earlier outbound from A to that port leaves return + // state that carries the next run's probe across a link that is actually + // blocked — at either layer, since WFP and Tailscale's filter both track + // outbound UDP flows by 4-tuple. An ephemeral source port makes each run's + // tuple new, so no run can poison the next BY CONSTRUCTION rather than by a + // cold-start convention nobody will remember. The ack comes back to this + // ephemeral port because A replies to `from`; that direction is open anyway + // and nothing is inferred from it. + let probe_socket = UdpSocket::bind(SocketAddr::new(IpAddr::from([0, 0, 0, 0]), 0)) + .expect("probe sender binds an ephemeral port"); + probe_socket + .set_read_timeout(Some(Duration::from_millis(250))) + .expect("probe sender takes a read timeout"); + println!( + "TWOHOST-WEB probe role B: waiting up to {:?} on udp {listen} for A's beacon, then sending to {target} for {PROBE_WINDOW:?}", + rig.wait + ); + + let mut buf = [0u8; 64]; + let beacon_deadline = Instant::now() + rig.wait; + let mut heard_beacon = false; + while Instant::now() < beacon_deadline { + match socket.recv_from(&mut buf) { + Ok((n, from)) if buf[..n] == *PROBE_BEACON => { + println!("TWOHOST-WEB probe role B: beacon from {from} — A is up, starting the clock"); + heard_beacon = true; + break; + } + Ok((n, from)) => println!("TWOHOST-WEB probe role B: {n} foreign bytes from {from}"), + Err(_) => {} + } + } + + // THE FOURTH OUTCOME, and it gets its own name. No beacon means this half + // never established that the other half exists — which is a RENDEZVOUS + // failure, not an inbound block, and calling it INBOUND_BLOCKED would be the + // same conflation this whole probe was written to end, one layer up. + assert!( + heard_beacon, + "PROBE_NO_BEACON: role B heard nothing from A on udp {listen} within {:?}. +This is NOT an inbound-block finding and must not be reported as one: it says only that A's beacons +did not reach this host, so this half never learned that the other half is running. + +Look, in this order: (1) did A's step run at all on the other host, and did it reach its 'listening on +udp' line; (2) is A->B blocked, which is the direction this rig has always found OPEN (measured 3/3 +from A on 2026-09-08 while the reverse read 0/3) — if it has closed, that is a NEW box finding and +the more interesting one; (3) is this host's own udp {listen} bound by something else. +A's own cell reds INBOUND_BLOCKED with the two-layer text if the fault is the reverse direction.", + rig.wait + ); + + // The window that actually measures the link, started from evidence. + let deadline = Instant::now() + PROBE_WINDOW; + let mut sent = 0usize; + let mut acked = false; + while Instant::now() < deadline { + match probe_socket.send_to(PROBE_MAGIC, target) { + Ok(_) => sent += 1, + Err(e) => println!("TWOHOST-WEB probe role B: send error (continuing): {e}"), + } + // The ack arrives on the FIXED socket, not the ephemeral sender: A sends + // it to this host's rig port so it is addressable cold (see A's cell). + match socket.recv_from(&mut buf) { + Ok((n, from)) if buf[..n] == *PROBE_ACK => { + println!("TWOHOST-WEB probe role B: ack from {from} after {sent} datagram(s)"); + acked = true; + break; + } + // A beacon is not an ack. Reading one as the other would stop the + // clock on the evidence that started it. + Ok((n, _)) if buf[..n] == *PROBE_BEACON => {} + Ok((n, from)) => println!("TWOHOST-WEB probe role B: {n} foreign bytes from {from}"), + Err(_) => {} + } + } + println!("TWOHOST-WEB probe role B: {sent} datagrams sent to {target}, acked={acked}"); + assert!(sent > 0, "role B could not send a single probe datagram to {target}"); + + // B's fast RED, and it is the one IR-89 promises: ten seconds, its own name, + // with "A is not up yet" already excluded by the beacon above. + assert!( + acked, + "INBOUND_BLOCKED: role B sent {sent} datagram(s) to {target} in {PROBE_WINDOW:?} and A never +acked — while A's beacons were arriving here the whole time, so A IS UP and the reverse direction is +open. B->A is blocked. + +THERE ARE TWO LAYERS AND EITHER ONE ALONE PRODUCES THIS EXACT SILENCE; the receiver's own cell +carries the full text and the two operator fixes. In short: LAYER 1 is the receiver's per-executable +host firewall (allow INBOUND UDP for the rig's port range from this host, PORT+REMOTE scoped, never +program-scoped), LAYER 2 is the asymmetric tailnet ACL (this sender must appear among the receiver's +PacketFilter permitted inbound sources — check `tailscale debug netmap` on the RECEIVER). Fixing only +one is the mistake: it is NECESSARY, NOT SUFFICIENT, and the run reds identically afterwards. + +Do not spend the ceremony's budget on this: the dial below would fail the same way for the same +reason, and would report it as a pairing failure." + ); +} + // ── Role B: the owner ───────────────────────────────────────────────────── /// Serves for the rig deadline: two registered files and one endpoint-scoped