diff --git a/.github/workflows/golden.yml b/.github/workflows/golden.yml index 2066734b..40f06699 100644 --- a/.github/workflows/golden.yml +++ b/.github/workflows/golden.yml @@ -1106,6 +1106,31 @@ jobs: # this host OWNS the files, hfenduleam fetches them through its own # loopback listener. Its own bin, its own ports (+20), after the ladder. # [int->REQ-WEB-CROSS-NODE-PROXY] + # THE INBOUND PROBE, before the ceremony (releases#272). A golden run spent + # 900 s on EACH half discovering that B's datagrams never reached A: the + # Windows host is BlockInbound on every profile and no rule covered the + # test binary, so every QUIC dial died on its 10 s bound -- 75 of them, + # cadenced at exactly 12.00 s, never one ADMIT. At B a dropped datagram is + # indistinguishable from a delivered one, so it was triaged as product, + # then as rig, before it was measured as box. + # + # SEPARATE INVOCATION, not a filter inside the ceremony run: within one + # `cargo test` the cells are threads and nothing orders them, so only a + # step boundary puts this FIRST. + # + # SAME BINARY, not a shell probe: the Windows rule is PER-EXECUTABLE, so an + # allow rule naming pwsh or python says nothing about twohost_web-.exe + # -- a shell probe would pass the moment the PROBER was allowed and certify + # the failure it exists to catch. The assertion lives on A, which is the + # only side that can see arrival. + - name: Two-host inbound probe — role B (sender) + shell: bash + run: cargo test -p spt-daemon --test twohost_web two_host_inbound_probe -- --nocapture + env: + SPT_TWO_HOST: "1" + SPT_TWO_HOST_ROLE: "b" + SPT_TWO_HOST_SECRET: "rig-twohost-ci" + SPT_TWO_HOST_PEER_IP: "100.68.35.65" # hfenduleam (tailscale) - name: Two-host web serving — role B (owner) shell: bash run: bash .github/bench/wrap.sh twohost-web-b -- cargo test -p spt-daemon --test twohost_web -- --nocapture @@ -1243,6 +1268,31 @@ jobs: # serving cells — three cells, each its own listener + broker, fetching # kitsubito's files through this host's loopback. After the ladder. # [int->REQ-WEB-CROSS-NODE-PROXY] + # THE INBOUND PROBE, before the ceremony (releases#272). A golden run spent + # 900 s on EACH half discovering that B's datagrams never reached A: the + # Windows host is BlockInbound on every profile and no rule covered the + # test binary, so every QUIC dial died on its 10 s bound -- 75 of them, + # cadenced at exactly 12.00 s, never one ADMIT. At B a dropped datagram is + # indistinguishable from a delivered one, so it was triaged as product, + # then as rig, before it was measured as box. + # + # SEPARATE INVOCATION, not a filter inside the ceremony run: within one + # `cargo test` the cells are threads and nothing orders them, so only a + # step boundary puts this FIRST. + # + # SAME BINARY, not a shell probe: the Windows rule is PER-EXECUTABLE, so an + # allow rule naming pwsh or python says nothing about twohost_web-.exe + # -- a shell probe would pass the moment the PROBER was allowed and certify + # the failure it exists to catch. The assertion lives on A, which is the + # only side that can see arrival. + - name: Two-host inbound probe — role A (receipt asserted here) + shell: pwsh + run: cargo test -p spt-daemon --test twohost_web two_host_inbound_probe -- --nocapture + env: + SPT_TWO_HOST: "1" + SPT_TWO_HOST_ROLE: "a" + SPT_TWO_HOST_SECRET: "rig-twohost-ci" + SPT_TWO_HOST_PEER_IP: "100.98.197.12" # kitsubito (tailscale) - name: Two-host web serving — role A (requester) shell: pwsh run: .github/bench/wrap.ps1 -Step "twohost-web-a" -Command "cargo test -p spt-daemon --test twohost_web -- --nocapture" diff --git a/crates/spt-daemon/tests/twohost_web.rs b/crates/spt-daemon/tests/twohost_web.rs index 9f6c19a2..0eb2371c 100644 --- a/crates/spt-daemon/tests/twohost_web.rs +++ b/crates/spt-daemon/tests/twohost_web.rs @@ -45,7 +45,7 @@ //! daemons); the two-box climb is the gate's field leg. use std::io::{Read, Write as _}; -use std::net::{IpAddr, SocketAddr, TcpStream}; +use std::net::{IpAddr, SocketAddr, TcpStream, UdpSocket}; use std::path::Path; use std::sync::atomic::{AtomicBool, AtomicU64}; use std::sync::Arc; @@ -120,6 +120,16 @@ const RANGE_OFFSET: u16 = 1; const DENY_OFFSET: u16 = 2; /// The requester-cell offset the helper cell uses — B dials this one A-ward. const HELPER_OFFSET: u16 = 3; +/// The INBOUND PROBE's offset. Clear of the four requester cells (0..=3) and +/// inside the same `port_a + offset` range the rig already uses, so whatever +/// rule lets the ceremony's ports through covers this one too — a probe on a +/// port outside that range would vouch for a path nothing else takes. +const PROBE_OFFSET: u16 = 9; +/// How long A listens and B sends. Ten seconds against the ceremony's 900: the +/// point of the probe is that a blocked link is answered in the time it takes +/// to notice, not in the time it takes to give up. +const PROBE_WINDOW: Duration = Duration::from_secs(10); +const PROBE_MAGIC: &[u8] = b"SPT-TWOHOST-INBOUND-PROBE-v1"; const HELPER_BYTES: &[u8] = b"# the file the user pointed at\n"; const OPEN_ENDPOINT: &str = "w1-open"; @@ -303,6 +313,149 @@ fn helper_envelope(quoted: &str) -> String { /// production loop; the gate it runs is the production gate. // [int->REQ-WEB-CROSS-NODE-PROXY] // [int->REQ-WEB-ACCESS-SURFACE] +// ══ THE INBOUND PROBE ═══════════════════════════════════════════════════════ +// A golden run once spent 900 s on each half discovering that B's datagrams +// never reached A: every QUIC dial died on the 10 s bound — 75 of them, cadenced +// at exactly 12.00 s, never one ADMIT. +// +// TWO LAYERS were behind it and EITHER ALONE produces the same silence, which is +// why the first fix looked wrong when it changed nothing: +// 1. the host firewall — BlockInbound on every profile, and its allow rules are +// PER-EXECUTABLE, so no rule covered the test binary and the runner (a +// service) never shows an Allow dialog; +// 2. the TAILNET ACL, asymmetric — a member device may open flows TO a TAGGED +// resource and the reverse is denied, so the receiver permits inbound from a +// list the sender is simply absent from. +// Every face that ever appeared to work rode RETURN traffic of flows the receiver +// had opened; a COLD claim in this direction can never cross. The ceremony cannot +// tell that apart from a pairing fault: at B a datagram dropped by a firewall +// looks exactly like one that arrived and was ignored, so the failure was +// triaged as product, then rig, before it was measured as box. +// +// These two cells answer that question FIRST, in ten seconds, in their own +// words. They are run by their own `cargo test` invocation before the ceremony +// step (see golden.yml) — inside one invocation the cells are threads and +// nothing orders them. +// +// WHY THIS LIVES IN THIS BINARY rather than in a shell script beside it, which +// would have been the smaller change: the Windows rule is PER-EXECUTABLE. An +// allow rule naming pwsh, python, or a bench script says nothing about +// `twohost_web-.exe`, so a probe written in any of them would today agree +// with reality by coincidence and would go GREEN the moment anyone allowed the +// prober — certifying the exact failure it exists to catch. Only a probe that +// IS the test binary carries the test binary's firewall identity. (It is also +// why the operator rule must be PORT+REMOTE scoped: a program-scoped rule is +// orphaned by the next rebuild's hash, exactly as the dead `_work\spt-core` +// rules were. This probe stays correct under either.) +// +// THE ASSERTION LIVES ON A, deliberately. B cannot know whether its datagram +// arrived, so a B-side "I sent it" proves nothing about the path. +#[test] +fn two_host_inbound_probe_role_a() { + let Some(rig) = Rig::from_env("a") else { + return; + }; + let port = rig.port_a + PROBE_OFFSET; + let socket = UdpSocket::bind(SocketAddr::new(IpAddr::from([0, 0, 0, 0]), port)) + .unwrap_or_else(|e| panic!("INBOUND_PROBE: role A could not bind udp {port}: {e}")); + socket + .set_read_timeout(Some(Duration::from_millis(250))) + .expect("probe socket takes a read timeout"); + println!("TWOHOST-WEB probe role A: listening on udp {port} for {PROBE_WINDOW:?}"); + + let deadline = Instant::now() + PROBE_WINDOW; + let mut buf = [0u8; 64]; + let mut seen = 0usize; + while Instant::now() < deadline { + match socket.recv_from(&mut buf) { + Ok((n, from)) if buf[..n] == *PROBE_MAGIC => { + seen += 1; + println!("TWOHOST-WEB probe role A: datagram {seen} from {from}"); + break; + } + // A stray datagram is not this probe's business; keep waiting rather + // than passing on someone else's traffic. + Ok((n, from)) => println!("TWOHOST-WEB probe role A: {n} foreign bytes from {from}"), + Err(_) => {} + } + } + + // ITS OWN OUTCOME, never the ceremony's error: this reds as INBOUND_BLOCKED + // and names the peer, the port, the window and the fix, so the next box + // regression is answered by the failure text instead of a 900 s investigation. + assert!( + seen > 0, + "INBOUND_BLOCKED: role A received NO probe datagram on udp {port} from {} within {PROBE_WINDOW:?}. +This is a BOX rule, not a product or rig fault: the ceremony below would spend the full +SPT_TWO_HOST_WAIT_SECS budget dialling a host that cannot receive, and would report it as a pairing +failure. + +THERE ARE TWO LAYERS AND EITHER ONE ALONE PRODUCES THIS EXACT SILENCE. Fixing only the first is the +mistake this text exists to prevent: it is NECESSARY, NOT SUFFICIENT, and the run reds identically +afterwards. + + LAYER 1 - the Windows host firewall on the receiver. BlockInbound on every profile, and its allow + rules are PER-EXECUTABLE, so a rule naming pwsh or python says nothing about this test binary and + the runner (a service) never shows an Allow dialog. Fix, operator, elevated - allow INBOUND UDP + {}-{} from {} only: PORT+REMOTE scoped, never program-scoped, or the next rebuild binary hash + orphans the rule the way the dead _work/spt-core rules were orphaned: + New-NetFirewallRule -DisplayName 'spt twohost rig inbound' -Direction Inbound -Protocol UDP -LocalPort {}-{} -RemoteAddress {} -Action Allow + + LAYER 2 - the TAILNET ACL, which is asymmetric and denies this direction. The sender is a TAGGED + resource owned by another tailnet user; a member device may open flows TO it, and the reverse is + denied, so the receiver permits inbound from a list this sender is simply absent from. Every + helper-stall face that ever appeared to work rode RETURN traffic of flows the receiver opened; a + COLD claim in this direction can never cross. Fix, operator - grant it in the tailnet policy + (asked on releases#272: src tag:eye-tracking-resource -> dst this host, udp 7460-7499). Verify with + `tailscale debug netmap` on the RECEIVER: the sender IP must appear among the PacketFilter + permitted inbound sources. + +THE DISCRIMINATOR, ten seconds, run it before assuming which layer bit: bind the SAME listener and +send to it once over the LAN and once over Tailscale. LAN receives and Tailscale does not = LAYER 2, +the host firewall is not your problem. Neither receives = LAYER 1 (or both). Solicited return traffic +works under either fault, so an echo reply proves nothing about this direction. + +Port range above is exactly what THIS rig binds (port_a..probe); releases#272 carries the wider +7460-7499, which covers it. Apply the board's range if they differ - this message and that comment +must never become two different commands.", + rig.peer_ip, + rig.port_a, + rig.port_a + PROBE_OFFSET, + rig.peer_ip, + rig.port_a, + rig.port_a + PROBE_OFFSET, + rig.peer_ip, + ); + println!("TWOHOST-WEB probe role A: INBOUND OK on udp {port}"); +} + +#[test] +fn two_host_inbound_probe_role_b() { + let Some(rig) = Rig::from_env("b") else { + return; + }; + let target = SocketAddr::new(rig.peer_ip, rig.port_a + PROBE_OFFSET); + let socket = UdpSocket::bind(SocketAddr::new(IpAddr::from([0, 0, 0, 0]), 0)) + .expect("probe sender binds an ephemeral port"); + println!("TWOHOST-WEB probe role B: sending to {target} for {PROBE_WINDOW:?}"); + + // Send for the WHOLE window rather than once: A's job may still be starting, + // and the question is whether ANY datagram crosses, not whether the first one + // happened to. This cell CANNOT FAIL on delivery — only A can see that — so a + // send error is reported and the window continues. + let deadline = Instant::now() + PROBE_WINDOW; + let mut sent = 0usize; + while Instant::now() < deadline { + match socket.send_to(PROBE_MAGIC, target) { + Ok(_) => sent += 1, + Err(e) => println!("TWOHOST-WEB probe role B: send error (continuing): {e}"), + } + thread::sleep(Duration::from_millis(500)); + } + println!("TWOHOST-WEB probe role B: {sent} datagrams sent to {target}"); + assert!(sent > 0, "role B could not send a single probe datagram to {target}"); +} + #[test] fn two_host_web_role_b() { let Some(rig) = Rig::from_env("b") else {