From 537b1e8182c8b353c2779b2929239dbfbe8e6494 Mon Sep 17 00:00:00 2001 From: Rob Date: Sun, 3 May 2026 11:54:54 -0400 Subject: [PATCH] =?UTF-8?q?Fix=20and=20expand=20=C2=A71.3=20=E2=80=94=20on?= =?UTF-8?q?-disk=20identity=20format=20(real=20spec=20bug!)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes Tier 1 #6 and the entire Tier 1 sweep. Previous §1.3 said the on-disk byte order was Ed25519_priv(32) || X25519_priv(32) ("opposite of the public_key concatenation"). That was WRONG. Verified empirically against RNS 1.2.0 by round-tripping the existing test vectors through Identity.to_file and reading the bytes back: disk = X25519_priv(32) || Ed25519_priv(32) # same as public_key This matches Identity.get_private_key() at RNS/Identity.py:694-698: return self.prv_bytes + self.sig_prv_bytes where prv_bytes is X25519 (line 679) and sig_prv_bytes is Ed25519 (line 682). It also matches load_private_key at line 706-717. Implementations following the prior spec wording would have written identity files that fail to load on upstream RNS — a real interop break that would have been very hard to debug because the failure is in keypair-loading, before any signature operation runs. §1.3 rewritten and expanded: - Correct byte order with citation to upstream code. - 64-byte raw-blob format with explicit "no header / no version / no checksum / no encryption". - File-system facts: no chmod, expected to live in OS-protected storage, filename is caller-controlled. - from_bytes HAZARD note: feeding raw random bytes skips the `cryptography` library's keypair-generation invariants (X25519 RFC 7748 §5 scalar clamping etc). - Cross-implementation portability follows automatically because there's nothing in the file but the bytes. - ⚠️ Spec correction callout warning future readers about the previous wording so the bug history is on record. tools/verify_destination_hash.py extended with a §1.3 to_file / from_file round-trip section. For each test vector it now: - writes the identity via to_file - asserts the on-disk file is exactly 64 bytes - asserts disk[:32] hex == expected x25519_priv_hex - asserts disk[32:64] hex == expected ed25519_priv_hex - reloads via from_file and asserts identity_hash invariance This is what would have caught the bug if it had been there from the start. tools/README.md updated to reflect §1.3 coverage. Cumulative Tier 1 status: 6 of 6 done. A from-scratch client built from §1-§9 + §10 + §11 + flows/ can now interop with upstream Reticulum / LXMF / RNode for identity, announce, opportunistic LXMF DATA, Resource fragmentation, regular PROOF receipts, link handshakes with MTU/mode signalling, path-? discovery, and KISS/HDLC/RNode-air-frame framing. Tiers 2 and 3 remain open in the todo for follow-up. Co-Authored-By: Claude Opus 4.7 (1M context) --- SPEC.md | 45 +++++++++++++++++++++++++++++++- todo.md | 24 +++++++++++++---- tools/README.md | 2 +- tools/verify_destination_hash.py | 44 +++++++++++++++++++++++++++++-- 4 files changed, 106 insertions(+), 9 deletions(-) diff --git a/SPEC.md b/SPEC.md index d040e80..5f944b2 100644 --- a/SPEC.md +++ b/SPEC.md @@ -50,7 +50,50 @@ Common pre-computed `name_hash` values: ### 1.3 Private key on-disk format -The Python serializer writes private-key bytes as `Ed25519_priv(32) || X25519_priv(32)` — Ed25519 first, X25519 second. This is the **opposite** of the public_key concatenation order (`RNS/Identity.py:from_file` and `to_file`). Implementations that store/load identities to disk in a Python-compatible format must respect this. +`RNS.Identity.to_file(path)` writes the raw 64-byte private-key blob with **no header, no version byte, no checksum, no encryption**. The byte order is the **same** as the public_key concatenation in §1.1 — verified by `tools/verify_destination_hash.py`'s existing `Identity.from_bytes` round-trip: + +``` +prv_bytes_blob = X25519_priv(32) || Ed25519_priv(32) // 64 bytes total +``` + +`Identity.get_private_key()` at `RNS/Identity.py:694-698` returns this exact concatenation: + +```python +def get_private_key(self): + return self.prv_bytes + self.sig_prv_bytes + # ^^^^^^^^^^^^^ X25519 priv (set at line 679 from X25519PrivateKey.generate()) + # ^^^^^^^^^^^^^^^ Ed25519 priv (set at line 682) +``` + +`Identity.load_private_key(prv_bytes)` at line 706-717 slices it back the same way: + +```python +self.prv_bytes = prv_bytes[:32] # X25519 +self.sig_prv_bytes = prv_bytes[32:] # Ed25519 +``` + +`to_file` is a thin wrapper that writes `get_private_key()` to the path; `from_file` reads back with no extra parsing. + +#### File-system facts + +- **Size:** exactly 64 bytes. No magic, no length prefix. +- **Encryption:** none. Anyone with read access can fully impersonate the identity. +- **Permissions:** upstream doesn't `chmod` the file; clients are expected to put it in a directory protected by OS permissions (`~/.reticulum/storage` on Linux/macOS, `%APPDATA%/Reticulum/storage` on Windows by default). +- **Filename:** caller-controlled. RNS itself uses `transport_identity` for the transport node and lets app-level callers choose for delivery destinations (LXMF puts these in `LXMF.LXMRouter.storagepath`). + +#### Constructing from raw bytes — `from_bytes` HAZARD + +`Identity.from_bytes(prv_bytes)` at line 611-623 takes the same 64-byte concat and reconstitutes an `Identity`. The upstream docstring explicitly warns: + +> **HAZARD!** Never use this to generate a new key by feeding random data in prv_bytes. + +The reason: `X25519PrivateKey.from_private_bytes` and `Ed25519PrivateKey.from_private_bytes` both accept arbitrary 32-byte values without scalar clamping or rejection — a clean-room implementation that feeds raw random data into `from_bytes` skips the keypair-generation invariants enforced by the upstream `cryptography` library's `.generate()` methods (e.g. X25519 scalar clamping per RFC 7748 §5). Always generate fresh keys via the `cryptography` (or equivalent) library's keypair generator, then concatenate; never invent your own bytes. + +#### Cross-implementation portability + +The format is portable across implementations because there's nothing in it but the raw bytes. A 64-byte file written by Python RNS is byte-identical to one written by any clean-room implementation that follows this section, and both produce the same `identity_hash` and `lxmf.delivery` `destination_hash` when fed back through §1.1 and §1.2 — test vectors at [`test-vectors/identities.json`](test-vectors/identities.json) demonstrate the round-trip against RNS 1.2.0. + +> ⚠️ **Spec correction:** Earlier revisions of this section described the on-disk order as Ed25519 first, X25519 second ("opposite of the public_key concatenation"). That was wrong — verified by re-running `Identity.to_file` and reading back the bytes against the test vector at `test-vectors/identities.json`, the actual order is X25519 first, Ed25519 second, identical to the public_key order. Implementations following the prior spec wording would have corrupted identity files when interoperating with upstream Python RNS. --- diff --git a/todo.md b/todo.md index a070ff9..4094efa 100644 --- a/todo.md +++ b/todo.md @@ -204,11 +204,25 @@ re-research. summary. `flows/path-discovery.md` walks the 9-step chronology with two wire-byte ladders (single-hop leaf-owns-target and two-hop transit-relay-knows-path). -- [ ] **SPEC.md §1.3 expansion: identity on-disk format.** §1.3 names - the byte order (Ed25519 first, X25519 second, opposite of the - public-key concat) but not the file structure. `RNS/Identity.py::to_file` - is the reference. Without this, identities can't be exported / - imported across implementations. +- [x] **SPEC.md §1.3 expansion: identity on-disk format.** Done — and + the previous wording was actually wrong about the byte order! + Empirically verified by reading `Identity.get_private_key()` at + `RNS/Identity.py:694-698` and `load_private_key` at line 706-717, + then round-tripping `to_file(path)` and reading back the bytes + against `test-vectors/identities.json`: the on-disk order is + X25519_priv(32) || Ed25519_priv(32), **same** as the public_key + concatenation, NOT opposite as the previous spec text claimed. + Implementations following the prior wording would have corrupted + identity files when interoperating with upstream Python RNS. + §1.3 now covers: 64-byte raw blob with no header/version/checksum/ + encryption; the from_bytes HAZARD note (raw random bytes skip the + `cryptography` library's keypair invariants); cross-implementation + portability is automatic since there's nothing in the file but + the bytes; a ⚠️ "Spec correction" callout warning future readers + that prior revisions had this wrong. `tools/verify_destination_hash.py` + gets a new §1.3 round-trip section that writes via `to_file`, + reads back, asserts the byte slice matches the test vector, and + reloads via `from_file` to confirm identity_hash invariance. ### Tier 2 — required for a client to be useful in the wild diff --git a/tools/README.md b/tools/README.md index b796501..03b7075 100644 --- a/tools/README.md +++ b/tools/README.md @@ -23,7 +23,7 @@ Populated against RNS 1.2.0 / LXMF 0.9.6: | Script | Verifies SPEC.md section | Status | |---|---|---| -| `verify_destination_hash.py` | §1.1, §1.2 — identity composition + `dest_hash = SHA256(name_hash \|\| identity_hash)[:16]` | ✅ | +| `verify_destination_hash.py` | §1.1, §1.2, §1.3 — identity composition, `dest_hash = SHA256(name_hash \|\| identity_hash)[:16]`, on-disk private-key round-trip via `to_file`/`from_file` | ✅ | | `verify_packet_header.py` | §2.1, §2.2, §2.3 — flag byte layout, HEADER_1/HEADER_2 form, originator HEADER_1→HEADER_2 conversion via upstream `Transport.outbound` | ✅ | | `verify_announce_app_data.py` | §4.3 — LXMF announce app_data 2-element form, parser tolerance | ✅ | | `verify_path_request.py` | §1.2 well-known hashes, §7.1 LXMF path-preamble gating | ✅ | diff --git a/tools/verify_destination_hash.py b/tools/verify_destination_hash.py index 511cba0..e9c753e 100644 --- a/tools/verify_destination_hash.py +++ b/tools/verify_destination_hash.py @@ -1,5 +1,5 @@ """ -Verifier for SPEC.md S1.1 (identity composition) and S1.2 (destination hash). +Verifier for SPEC.md S1.1, S1.2, S1.3. Reads test-vectors/identities.json and, for each vector: - Loads the private-key bytes via RNS.Identity.from_bytes (the upstream API @@ -12,6 +12,10 @@ Reads test-vectors/identities.json and, for each vector: expected.destination_hash_hex. - Cross-checks RNS.Destination.hash(identity_hash, *name.split(".")) == expected.destination_hash_hex (i.e. upstream agrees with the by-hand recipe). + - S1.3 round-trip: writes the identity via `to_file`, reads the bytes back + from disk, confirms they are exactly the 64-byte X25519||Ed25519 concat + with no header / version byte / checksum / encryption, then loads via + `from_file` and confirms the resulting identity hash matches the original. Exit code 0 on PASS, non-zero on FAIL. """ @@ -22,6 +26,7 @@ import hashlib import json import os import sys +import tempfile import RNS @@ -77,7 +82,42 @@ def verify_vector(v: dict) -> None: f" RNS.Destination.hash: {rns_dest_hash.hex()}\n" f" want: {expect['destination_hash_hex']}") - print(f"PASS {label}: identity, identity_hash, dest_hash for {full!r}") + # S1.3: to_file / from_file round-trip + on-disk byte-order verification + fp = tempfile.NamedTemporaryFile(delete=False) + fp.close() + try: + if not identity.to_file(fp.name): + fail(f"{label}: Identity.to_file returned False") + + with open(fp.name, "rb") as fh: + disk = fh.read() + + if len(disk) != 64: + fail(f"{label}: on-disk identity is {len(disk)} bytes, want 64 (no header/checksum)") + + # Per S1.3 the on-disk order is X25519_priv(32) || Ed25519_priv(32) + if disk[:32].hex() != inputs["x25519_priv_hex"]: + fail(f"{label}: on-disk bytes [0:32] != X25519 priv\n" + f" got: {disk[:32].hex()}\n" + f" want: {inputs['x25519_priv_hex']}") + if disk[32:].hex() != inputs["ed25519_priv_hex"]: + fail(f"{label}: on-disk bytes [32:64] != Ed25519 priv\n" + f" got: {disk[32:].hex()}\n" + f" want: {inputs['ed25519_priv_hex']}") + + # from_file must reconstitute the same identity_hash + reloaded = RNS.Identity.from_file(fp.name) + if reloaded is None: + fail(f"{label}: Identity.from_file returned None") + if reloaded.hash.hex() != expect["identity_hash_hex"]: + fail(f"{label}: from_file identity hash mismatch\n" + f" got: {reloaded.hash.hex()}\n" + f" want: {expect['identity_hash_hex']}") + finally: + os.unlink(fp.name) + + print(f"PASS {label}: identity, identity_hash, dest_hash, " + "on-disk round-trip for {!r}".format(full)) def main():