From 1fb8b8ec10109d2327e4db8805f8dbcee7325276 Mon Sep 17 00:00:00 2001 From: John Poole Date: Mon, 8 Jun 2026 13:23:43 -0700 Subject: [PATCH] Meant to include these in the prior commit, adding here & now. --- audits/resource-tier1-rns-1.2.4.md | 141 ++++++++++++++++ tools/verify_all.py | 110 +++++++++++++ tools/verify_resource.py | 253 +++++++++++++++++++++++++++++ 3 files changed, 504 insertions(+) create mode 100644 audits/resource-tier1-rns-1.2.4.md create mode 100644 tools/verify_all.py create mode 100644 tools/verify_resource.py diff --git a/audits/resource-tier1-rns-1.2.4.md b/audits/resource-tier1-rns-1.2.4.md new file mode 100644 index 0000000..61924b4 --- /dev/null +++ b/audits/resource-tier1-rns-1.2.4.md @@ -0,0 +1,141 @@ +# Tier 1 Audit: Resource Fragmentation + +Question: Does `SPEC.md` §10 accurately describe the wire-visible Resource +fragmentation behavior of upstream RNS 1.2.4? + +Evidence baseline: + +- RNS package: `rns==1.2.4` +- Source: installed package paths `RNS/Resource.py`, `RNS/Packet.py`, + `RNS/Link.py`, and `LXMF/LXMessage.py` +- Audit date: 2026-06-08 + +This began as a Tier 1 source-analysis report. The focused Tier 2 checks now +live in `tools/verify_resource.py`; the confirmed F1-F6 corrections were +promoted into `SPEC.md` §10 on 2026-06-08. + +## Confirmed Core Model + +The central §10 model matches RNS 1.2.4: + +- Resource prepends a throwaway 4-byte prefix, encrypts the complete stream + once with the Link, then slices the ciphertext into parts + (`Resource.py:404-428`, `453-472`; `Packet.py:201-204`). +- Advertisement `r` is a distinct 4-byte salt used for the resource hash and + per-part map hashes (`Resource.py:440-443`, `505-506`). +- Resource integrity is `SHA256(uncompressed_segment_plaintext || r)`, and the + expected proof is `SHA256(uncompressed_segment_plaintext || resource_hash)` + (`Resource.py:440-443`, `681-695`, `752-758`). +- RESOURCE_REQ may carry requested part hashes while also requesting a hashmap + continuation; the sender fulfils parts before emitting RESOURCE_HMU + (`Resource.py:994-1027`, `1027-1064`). +- RESOURCE parts are matched only within the receiver's current window, while + sender lookup is bounded by `COLLISION_GUARD_SIZE` + (`Resource.py:863-890`, `999-1010`). + +## Findings Requiring Correction or Clarification + +### F1 — Direct LXMF threshold is 319 bytes, not approximately 360 + +`SPEC.md` §10 introduction says Resource carries an LXMF body larger than +approximately 360 bytes. With default RNS 1.2.4 / LXMF 0.9.7 parameters: + +```text +RNS.Link.MDU = 431 +LXMessage.LXMF_OVERHEAD = 112 +LXMessage.LINK_PACKET_MAX_CONTENT = 319 +``` + +`LXMessage.pack()` selects Resource representation for DIRECT delivery when +`content_size > 319` (`LXMF/LXMessage.py:80-89`, `414-421`). + +Recommended Tier 3 correction: say 319 bytes with default parameters, while +noting that the threshold derives from Link MDU and may vary with protocol +parameters. + +### F2 — Resource is not the only possible way to carry larger application data + +The §10 introduction calls Resource "the only way" to carry payloads exceeding +one Link packet. Resource is the standard RNS mechanism used by LXMF, Link +REQUEST/RESPONSE, and `rncp`, but applications can also stream or sequence data +over Channel or their own Link DATA protocol. + +Recommended Tier 3 correction: replace "the only way" with "the standard RNS +mechanism used by LXMF, REQUEST/RESPONSE, and file-transfer utilities". + +### F3 — Advertisement `d` is total logical-resource size + +For multi-segment resources, advertisement field `d` is `resource.total_size`, +the total uncompressed logical transfer size including metadata, not the +plaintext size of the advertised segment (`Resource.py:281-314`, +`Resource.py:1281-1283`). + +Each segment still has its own: + +- `t`: encrypted transfer size for this segment +- `n`: part count for this segment +- `h`: integrity hash for this segment +- `r`: salt for this segment + +Recommended Tier 3 correction: define `d` as total logical-resource size and +explicitly distinguish it from the current segment's uncompressed plaintext +length, which is not directly advertised. + +### F4 — `RESOURCE_RCL` is not a general receiver-side cancel notification + +`Resource.reject(advertisement_packet)` sends `RESOURCE_RCL` +(`Resource.py:154-163`). A corrupt receiver also calls `reject()` and tears +down the Link (`Resource.py:1081-1084`). + +However, ordinary receiver-side `Resource.cancel()` only removes the incoming +resource locally; it does not send `RESOURCE_RCL` +(`Resource.py:1086-1097`). The current §10.9 wording implies either side can +always notify cancellation on the wire. + +Recommended Tier 3 correction: describe `RESOURCE_RCL` as advertisement +rejection / corrupt-resource rejection. Do not claim ordinary receiver cancel +always emits it. + +### F5 — Resource-part packet storage wording is imprecise + +`SPEC.md` §10.2 says packed wire bytes are stored in `parts[i]`. Upstream stores +pre-packed `RNS.Packet` objects in `parts`; each packet's `data` is the raw +ciphertext slice and its `raw` field is the packed Reticulum packet +(`Resource.py:450-472`). + +Recommended Tier 3 clarification: distinguish the stored Packet object, its +Resource body (`part.data`), and complete Reticulum wire packet (`part.raw`). + +### F6 — Pinned-source mismatch in §10.7 citation + +The exhausted-REQ callout cites RNS 1.2.9 while this repository is pinned to +RNS 1.2.4. The behavior is already present in RNS 1.2.4 +(`Resource.py:994-1064`). + +Recommended Tier 3 correction: cite the pinned 1.2.4 behavior first; retain a +later-version note only when documenting an actual version change. + +## Tier 2 Verifier Scope + +The first focused verifier is implemented in `tools/verify_resource.py` and +avoids a live threaded transfer. It currently covers items 1-6 below: + +1. Construct a Resource with a deterministic fake Link encryption key, fixed + throwaway prefix, and fixed advertisement `r`. +2. Verify whole-stream encryption occurs before slicing. +3. Verify Resource-part Packet bodies are raw slices and are not packet-level + re-encrypted. +4. Verify advertisement dictionary fields, flags, and hashmap first segment. +5. Verify `hash`, `truncated_hash`, `expected_proof`, and map-hash formulas. +6. Verify RESOURCE_REQ parsing for both normal and exhausted-with-parts forms, + including simultaneous part fulfilment and RESOURCE_HMU generation. +7. Verify receiver assembly strips the throwaway prefix, decrypts once, + validates the hash, and emits the expected RESOURCE_PRF bytes. +8. Add a multi-segment fixture proving `d` remains total logical size while + `t`, `n`, `h`, and `r` describe the current segment. +9. Add negative cases: malformed ADV, wrong `r`, corrupt part, invalid HMU + boundary, and oversized decompression. + +Remaining Tier 2 work is a deterministic `test-vectors/resources.json` plus +items 7-9. The confirmed first claim set has already been promoted into +`SPEC.md` and the Resource flow documents. diff --git a/tools/verify_all.py b/tools/verify_all.py new file mode 100644 index 0000000..34ad60e --- /dev/null +++ b/tools/verify_all.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +""" +Run the complete verifier suite against the versions pinned in requirements.txt. + +This is the repository's Tier 2 baseline gate. It refuses to run individual +verifiers when the active Python environment does not contain the exact pinned +RNS and LXMF versions, then runs every tools/verify_*.py script in isolation. + +Exit code 0 means every verifier passed. Exit code 1 means the environment +does not match the pins or at least one verifier failed. +""" + +from __future__ import annotations + +import importlib.metadata +import pathlib +import re +import subprocess +import sys + + +TOOLS_DIR = pathlib.Path(__file__).resolve().parent +REQUIREMENTS_PATH = TOOLS_DIR / "requirements.txt" +PIN_PATTERN = re.compile(r"^(rns|lxmf)==([A-Za-z0-9_.+-]+)$", re.IGNORECASE) + + +def load_pins() -> dict[str, str]: + pins: dict[str, str] = {} + for raw_line in REQUIREMENTS_PATH.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + match = PIN_PATTERN.match(line) + if match: + pins[match.group(1).lower()] = match.group(2) + + missing = {"rns", "lxmf"} - pins.keys() + if missing: + names = ", ".join(sorted(missing)) + raise RuntimeError(f"missing required pins in {REQUIREMENTS_PATH}: {names}") + return pins + + +def installed_versions(names: list[str]) -> dict[str, str]: + versions: dict[str, str] = {} + for name in names: + try: + versions[name] = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + versions[name] = "" + return versions + + +def verifier_paths() -> list[pathlib.Path]: + return sorted( + path + for path in TOOLS_DIR.glob("verify_*.py") + if path.name != pathlib.Path(__file__).name + ) + + +def main() -> int: + try: + pins = load_pins() + except (OSError, RuntimeError) as exc: + print(f"FAIL baseline configuration: {exc}") + return 1 + + installed = installed_versions(sorted(pins)) + mismatches = [ + f"{name}: installed {installed[name]}, pinned {pins[name]}" + for name in sorted(pins) + if installed[name] != pins[name] + ] + + print(f"Python: {sys.executable}") + print("Pinned environment: " + ", ".join( + f"{name}=={pins[name]}" for name in sorted(pins) + )) + + if mismatches: + print("FAIL environment does not match tools/requirements.txt:") + for mismatch in mismatches: + print(f" - {mismatch}") + return 1 + + verifiers = verifier_paths() + if not verifiers: + print("FAIL no verifier scripts found") + return 1 + + failures: list[tuple[str, int]] = [] + for index, path in enumerate(verifiers, start=1): + print(f"\n=== [{index}/{len(verifiers)}] {path.name} ===", flush=True) + result = subprocess.run([sys.executable, str(path)], cwd=TOOLS_DIR.parent) + if result.returncode != 0: + failures.append((path.name, result.returncode)) + + print("\n=== Verification summary ===") + print(f"Passed: {len(verifiers) - len(failures)}") + print(f"Failed: {len(failures)}") + if failures: + for name, returncode in failures: + print(f" - {name}: exit {returncode}") + return 1 + + print("ALL VERIFIERS PASS") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/verify_resource.py b/tools/verify_resource.py new file mode 100644 index 0000000..70db4a3 --- /dev/null +++ b/tools/verify_resource.py @@ -0,0 +1,253 @@ +""" +Verifier for SPEC.md S10 (Resource fragmentation protocol). + +Exercises upstream RNS 1.2.4 Resource construction with a deterministic fake +Link and verifies: + + 1. Default direct-LXMF single-packet content limit is 319 bytes. + 2. Resource encrypts one complete stream, then slices that ciphertext into + RESOURCE packet bodies without packet-level re-encryption. + 3. Resource hash, expected proof, and map-hash formulas. + 4. RESOURCE_ADV fields and flags. + 5. Multi-segment advertisement `d` is total logical-resource size. + 6. An exhausted RESOURCE_REQ may carry part requests; upstream fulfils the + requested parts and emits RESOURCE_HMU. + 7. RESOURCE_RCL is emitted for advertisement rejection, while an ordinary + receiver-side cancel is local-only. + +Exit code 0 on PASS, non-zero on FAIL. +""" + +from __future__ import annotations + +import os +import sys +import time + +import LXMF +import RNS +from LXMF.LXMessage import LXMessage +from RNS.Cryptography.Token import Token +from RNS.Resource import Resource, ResourceAdvertisement + + +FIXED_TOKEN_KEY = bytes(range(64)) + + +def fail(msg: str) -> None: + print(f"FAIL: {msg}") + sys.exit(1) + + +class FakeLink: + """Minimum Link surface needed by Resource and Packet construction.""" + + def __init__(self, mtu: int = RNS.Reticulum.MTU): + self.type = RNS.Destination.LINK + self.status = RNS.Link.ACTIVE + self.mtu = mtu + self.mdu = RNS.Link.MDU + self.hash = bytes.fromhex("00112233445566778899aabbccddeeff") + self.link_id = self.hash + self.rtt = 0.1 + self.traffic_timeout_factor = 1 + self.last_outbound = 0 + self.tx = 0 + self.txbytes = 0 + self._token = Token(FIXED_TOKEN_KEY) + self.cancelled_incoming = [] + self.cancelled_outgoing = [] + + def encrypt(self, data: bytes) -> bytes: + return self._token.encrypt(data) + + def decrypt(self, data: bytes) -> bytes: + return self._token.decrypt(data) + + def cancel_incoming_resource(self, resource) -> None: + self.cancelled_incoming.append(resource) + + def cancel_outgoing_resource(self, resource) -> None: + self.cancelled_outgoing.append(resource) + + def resource_concluded(self, resource) -> None: + pass + + +def verify_default_lxmf_threshold() -> None: + expected = RNS.Link.MDU - LXMessage.LXMF_OVERHEAD + if LXMessage.LINK_PACKET_MAX_CONTENT != expected: + fail( + "S10 direct-LXMF threshold formula mismatch: " + f"{LXMessage.LINK_PACKET_MAX_CONTENT} != {expected}" + ) + if expected != 319: + fail(f"S10 default direct-LXMF threshold changed: got {expected}, want 319") + print("PASS S10 default direct-LXMF Resource threshold: content > 319 bytes") + + +def verify_preparation_and_advertisement() -> None: + link = FakeLink() + plaintext = (b"resource-verifier-" * 70) + bytes(range(64)) + resource = Resource(plaintext, link, advertise=False, auto_compress=False) + + stream = b"".join(part.data for part in resource.parts) + decrypted = link.decrypt(stream) + prefix = decrypted[:Resource.RANDOM_HASH_SIZE] + recovered = decrypted[Resource.RANDOM_HASH_SIZE:] + + if recovered != plaintext: + fail("S10.2 whole-stream decrypt did not recover original plaintext") + if len(prefix) != Resource.RANDOM_HASH_SIZE: + fail("S10.2 throwaway prefix has wrong length") + + for index, part in enumerate(resource.parts): + if part.context != RNS.Packet.RESOURCE: + fail(f"S10.3 part {index} has context {part.context:#x}, want RESOURCE") + if part.raw[19:] != part.data: + fail(f"S10.6 part {index} was packet-level encrypted or altered") + + expected_hash = RNS.Identity.full_hash(plaintext + resource.random_hash) + expected_proof = RNS.Identity.full_hash(plaintext + expected_hash) + if resource.hash != expected_hash: + fail("S10.2 resource hash formula mismatch") + if resource.expected_proof != expected_proof: + fail("S10.2 expected proof formula mismatch") + + for index, part in enumerate(resource.parts): + expected_map_hash = RNS.Identity.full_hash(part.data + resource.random_hash)[:Resource.MAPHASH_LEN] + if part.map_hash != expected_map_hash: + fail(f"S10.2 map-hash formula mismatch for part {index}") + + adv = ResourceAdvertisement.unpack(ResourceAdvertisement(resource).pack()) + if adv.t != len(stream): + fail(f"S10.4 ADV t mismatch: {adv.t} != {len(stream)}") + if adv.d != len(plaintext): + fail(f"S10.4 ADV d mismatch for single segment: {adv.d} != {len(plaintext)}") + if adv.n != len(resource.parts): + fail(f"S10.4 ADV n mismatch: {adv.n} != {len(resource.parts)}") + if adv.h != resource.hash or adv.r != resource.random_hash: + fail("S10.4 ADV hash or random-hash field mismatch") + if not adv.e or adv.c or adv.s or adv.u or adv.p or adv.x: + fail(f"S10.4 ADV flags unexpected: {adv.f:#x}") + + print( + "PASS S10.2/S10.4/S10.6 whole-stream encryption, ciphertext slicing, " + "hash formulas, and ADV fields" + ) + + +def verify_multisegment_total_size() -> None: + link = FakeLink() + logical_size = Resource.MAX_EFFICIENT_SIZE + 257 + resource = Resource(b"M" * logical_size, link, advertise=False, auto_compress=False) + adv = ResourceAdvertisement.unpack(ResourceAdvertisement(resource).pack()) + + if resource.total_segments != 2 or not resource.split: + fail("S10.11 expected a two-segment Resource") + if adv.d != logical_size: + fail(f"S10.4 multi-segment ADV d is {adv.d}, want total logical size {logical_size}") + if resource.uncompressed_size >= logical_size: + fail("S10.4 first-segment plaintext unexpectedly equals total logical size") + + print( + "PASS S10.4/S10.11 multi-segment ADV d is total logical-resource size, " + "not current-segment plaintext size" + ) + + +def verify_exhausted_request_with_parts() -> None: + link = FakeLink() + payload_size = ResourceAdvertisement.HASHMAP_MAX_LEN * (link.mtu - RNS.Reticulum.HEADER_MAXSIZE - RNS.Reticulum.IFAC_MIN_SIZE) + resource = Resource(b"R" * payload_size, link, advertise=False, auto_compress=False) + if len(resource.parts) <= ResourceAdvertisement.HASHMAP_MAX_LEN: + fail("S10.7 fixture did not produce a multi-segment hashmap") + + resource.status = Resource.TRANSFERRING + resource.adv_sent = time.time() + resource.rtt = 0.1 + + requested_part = resource.parts[0] + last_known_index = ResourceAdvertisement.HASHMAP_MAX_LEN - 1 + last_known_hash = resource.parts[last_known_index].map_hash + request_data = ( + bytes([Resource.HASHMAP_IS_EXHAUSTED]) + + last_known_hash + + resource.hash + + requested_part.map_hash + ) + + captured: list[RNS.Packet] = [] + real_outbound = RNS.Transport.outbound + + def fake_outbound(packet): + captured.append(packet) + return True + + RNS.Transport.outbound = staticmethod(fake_outbound) + try: + resource.request(request_data) + finally: + RNS.Transport.outbound = real_outbound + + contexts = [packet.context for packet in captured] + if RNS.Packet.RESOURCE not in contexts: + fail("S10.7 exhausted RESOURCE_REQ did not fulfil bundled part request") + if RNS.Packet.RESOURCE_HMU not in contexts: + fail("S10.7 exhausted RESOURCE_REQ did not emit RESOURCE_HMU") + + print("PASS S10.7 exhausted RESOURCE_REQ fulfils bundled parts and emits RESOURCE_HMU") + + +def verify_receiver_reject_vs_cancel() -> None: + link = FakeLink() + outgoing = Resource(b"reject-me", link, advertise=False, auto_compress=False) + advertisement_plaintext = ResourceAdvertisement(outgoing).pack() + + class AdvertisementPacket: + plaintext = advertisement_plaintext + link = None + + AdvertisementPacket.link = link + + captured: list[RNS.Packet] = [] + real_outbound = RNS.Transport.outbound + + def fake_outbound(packet): + captured.append(packet) + return True + + RNS.Transport.outbound = staticmethod(fake_outbound) + try: + Resource.reject(AdvertisementPacket) + if [packet.context for packet in captured] != [RNS.Packet.RESOURCE_RCL]: + fail("S10.9 advertisement rejection did not emit exactly one RESOURCE_RCL") + + captured.clear() + incoming = Resource(None, link) + incoming.status = Resource.TRANSFERRING + incoming.initiator = False + incoming.callback = None + incoming.cancel() + if captured: + fail("S10.9 ordinary receiver-side cancel unexpectedly emitted a packet") + if incoming not in link.cancelled_incoming: + fail("S10.9 ordinary receiver-side cancel did not remove incoming Resource") + finally: + RNS.Transport.outbound = real_outbound + + print("PASS S10.9 RESOURCE_RCL rejects advertisements; ordinary receiver cancel is local-only") + + +def main() -> None: + print(f"verify_resource.py against RNS {RNS.__version__} / LXMF {LXMF.__version__}") + verify_default_lxmf_threshold() + verify_preparation_and_advertisement() + verify_multisegment_total_size() + verify_exhausted_request_with_parts() + verify_receiver_reject_vs_cancel() + print("ALL PASS") + + +if __name__ == "__main__": + main()