diff --git a/.agents/skills/lxmf-update/SKILL.md b/.agents/skills/lxmf-update/SKILL.md new file mode 100644 index 0000000..20396c3 --- /dev/null +++ b/.agents/skills/lxmf-update/SKILL.md @@ -0,0 +1,40 @@ +--- +name: lxmf-update +description: Check upstream LXMF and RNS implementations for changes affecting the LXMF specification, run local verifier tools, and propose spec updates with citations. +--- + +# LXMF update skill + +Use this skill when asked to check whether the LXMF specification is current with upstream LXMF or RNS. + +## Sources of truth + +Prefer, in order: + +1. Upstream LXMF source code at a pinned release or commit. +2. Upstream RNS source code where LXMF behavior depends on Reticulum behavior. +3. Reproducible local traces and test vectors. +4. Upstream prose documentation, only when consistent with source behavior. + +## Required workflow + +1. Determine the currently pinned upstream LXMF and RNS versions from `SOURCES.md`. +2. Check whether newer upstream releases or commits exist. +3. Identify changed files relevant to: + - message packing + - fields and flags + - destination handling + - identity/signature behavior + - encryption behavior + - propagation + - delivery semantics + - storage format if externally observable +4. Run the verifier tools in `tools/`. +5. Compare observed behavior against `SPEC.md`. +6. Produce a report with: + - changed upstream files + - affected specification sections + - proposed edits + - test-vector impact + - unresolved ambiguities +7. Do not modify `SPEC.md` unless explicitly asked. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..caf3250 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +blank_issues_enabled: false +contact_links: + - name: ✨ Feature Request or Idea + url: https://github.com/markqvist/Reticulum/discussions/new?category=ideas + about: Propose and discuss features and ideas + - name: 💬 Questions, Help & Discussion + about: Ask anything, or get help + url: https://github.com/markqvist/Reticulum/discussions/new/choose + - name: 📖 Read the Reticulum Manual + url: https://markqvist.github.io/Reticulum/manual/ + about: The complete documentation for Reticulum diff --git a/.github/ISSUE_TEMPLATE/🐛-bug-report.md b/.github/ISSUE_TEMPLATE/🐛-bug-report.md new file mode 100644 index 0000000..65b492e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/🐛-bug-report.md @@ -0,0 +1,35 @@ +--- +name: "\U0001F41B Bug Report" +about: Report a reproducible bug +title: '' +labels: '' +assignees: '' + +--- + +**Read the Contribution Guidelines** +Before creating a bug report on this issue tracker, you **must** read the [Contribution Guidelines](https://github.com/markqvist/Reticulum/blob/master/Contributing.md). Issues that do not follow the contribution guidelines **will be deleted without comment**. + +- The issue tracker is used by developers of this project. **Do not use it to ask general questions, or for support requests**. +- Ideas and feature requests can be made on the [Discussions](https://github.com/markqvist/Reticulum/discussions). **Only** feature requests accepted by maintainers and developers are tracked and included on the issue tracker. **Do not post feature requests here**. +- After reading the [Contribution Guidelines](https://github.com/markqvist/Reticulum/blob/master/Contributing.md), **delete this section only** (*"Read the Contribution Guidelines"*) from your bug report, **and fill in all the other sections**. + +**Describe the Bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Describe in detail how to reproduce the bug. + +**Expected Behavior** +A clear and concise description of what you expected to happen. + +**Logs & Screenshots** +Please include any relevant log output. If applicable, also add screenshots to help explain your problem. + +**System Information** +- OS and version +- Python version +- Program version + +**Additional context** +Add any other context about the problem here. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..06e38e3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,132 @@ +*.DS_Store +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +LXMF/Utilities/LXMF +RNS +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..d4c9224 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,16 @@ +# AGENTS.md + +## Repository purpose + +This repository attempts to define an implementation-derived specification for LXMF. + +Do not treat upstream prose documentation as authoritative when it conflicts with observed source behavior. Prefer source code, test vectors, and reproducible traces. + +## Required discipline + +- Distinguish normative behavior from observed implementation behavior. +- Cite exact upstream files, functions, classes, constants, and version tags. +- Do not invent protocol rules. +- When behavior is unclear, mark it as "undetermined" rather than guessing. +- Prefer Perl tools for extraction, comparison, and report generation. +- Keep generated examples reproducible from scripts under tools/. diff --git a/ChatGPT_1.md b/ChatGPT_1.md new file mode 100644 index 0000000..c40e197 --- /dev/null +++ b/ChatGPT_1.md @@ -0,0 +1,174 @@ +Yes. For Codex, the closest equivalents are: + +1. **`AGENTS.md`** for persistent project-wide instructions. +2. **Codex Skills using `SKILL.md`** for reusable workflows, very similar in spirit to the Claude Code file you linked. +3. Optionally **`PLANS.md` / ExecPlan-style documents** for long-running specification work. + +The important correction is that Codex does now support **Skills with `SKILL.md`**, not just `AGENTS.md`. OpenAI’s Codex documentation says a skill is a directory containing a required `SKILL.md` file, with optional `scripts/`, `references/`, `assets/`, and related support files. Codex initially sees the skill name, description, and path, then loads the full `SKILL.md` only when it decides that skill applies. ([OpenAI Developers][1]) + +So, yes: you can do essentially the same thing that `thatSFguy` is doing, but I would structure it slightly differently for LXMF. + +## Recommended Codex layout + +For an LXMF specification repository, I would use something like: + +```text +lxmf-specification/ +├── AGENTS.md +├── SPEC.md +├── NOTES.md +├── SOURCES.md +├── tools/ +│ ├── extract_lxmf_constants.pl +│ ├── compare_upstream_lxmf.pl +│ ├── verify_examples.pl +│ └── make_test_vectors.pl +├── examples/ +│ ├── lxmf_message_minimal.hex +│ ├── lxmf_message_signed.hex +│ └── lxmf_propagation_example.hex +├── references/ +│ ├── lxmf_source_map.md +│ ├── rns_dependency_map.md +│ └── terminology.md +└── .agents/ + └── skills/ + └── lxmf-update/ + └── SKILL.md +``` + +## Role of each file + +### `AGENTS.md` + +Use this for always-on repository rules. Codex reads `AGENTS.md` before doing work, and the docs describe it as the mechanism for project-specific persistent guidance. ([OpenAI Developers][2]) + +Example use: + +```markdown +# AGENTS.md + +## Repository purpose + +This repository attempts to define an implementation-derived specification for LXMF. + +Do not treat upstream prose documentation as authoritative when it conflicts with observed source behavior. Prefer source code, test vectors, and reproducible traces. + +## Required discipline + +- Distinguish normative behavior from observed implementation behavior. +- Cite exact upstream files, functions, classes, constants, and version tags. +- Do not invent protocol rules. +- When behavior is unclear, mark it as "undetermined" rather than guessing. +- Prefer Perl tools for extraction, comparison, and report generation. +- Keep generated examples reproducible from scripts under tools/. +``` + +### `.agents/skills/lxmf-update/SKILL.md` + +Use this for the repeatable workflow: “check upstream LXMF/RNS, compare against our spec, update citations, run verifiers, propose diffs.” + +That is the closest Codex analog to the Claude Code `rns-update/SKILL.md` you found. + +A minimal Codex skill might look like: + +```markdown +--- +name: lxmf-update +description: Check upstream LXMF and RNS implementations for changes affecting the LXMF specification, run local verifier tools, and propose spec updates with citations. +--- + +# LXMF update skill + +Use this skill when asked to check whether the LXMF specification is current with upstream LXMF or RNS. + +## Sources of truth + +Prefer, in order: + +1. Upstream LXMF source code at a pinned release or commit. +2. Upstream RNS source code where LXMF behavior depends on Reticulum behavior. +3. Reproducible local traces and test vectors. +4. Upstream prose documentation, only when consistent with source behavior. + +## Required workflow + +1. Determine the currently pinned upstream LXMF and RNS versions from `SOURCES.md`. +2. Check whether newer upstream releases or commits exist. +3. Identify changed files relevant to: + - message packing + - fields and flags + - destination handling + - identity/signature behavior + - encryption behavior + - propagation + - delivery semantics + - storage format if externally observable +4. Run the verifier tools in `tools/`. +5. Compare observed behavior against `SPEC.md`. +6. Produce a report with: + - changed upstream files + - affected specification sections + - proposed edits + - test-vector impact + - unresolved ambiguities +7. Do not modify `SPEC.md` unless explicitly asked. +``` + +Codex supports skills in the CLI, IDE extension, and Codex app, and the documented repository skill location is `.agents/skills` under the current directory or repository root. ([OpenAI Developers][1]) + +### `PLANS.md` + +For your use case, I would also add a planning document, because “derive a protocol specification from implementation” is not a one-shot coding task. OpenAI’s Codex cookbook describes using `PLANS.md` / ExecPlans for long-running, multi-stage work, with `AGENTS.md` telling Codex when to use it. ([OpenAI Developers][3]) + +For LXMF, `PLANS.md` could define the required sections of each investigation: + +```markdown +# LXMF Specification Investigation Plan + +Each investigation must include: + +1. Question being answered. +2. Upstream files inspected. +3. Exact version or commit. +4. Relevant classes/functions/constants. +5. Observed behavior. +6. Whether behavior is normative, inferred, or unresolved. +7. Test vectors needed. +8. SPEC.md sections affected. +9. Open questions. +``` + +## My recommendation + +For your goal, I would not start by asking Codex to “write the LXMF specification.” That is too broad and likely to produce confident prose ahead of evidence. + +I would instead give Codex narrow tasks such as: + +```text +Using the lxmf-update skill, inspect upstream LXMF and determine the exact serialized structure of an LXMessage. Do not edit SPEC.md. Produce a report listing every field, its source-code origin, and whether its behavior is confirmed by a test vector. +``` + +Then: + +```text +Create Perl tooling under tools/ that extracts or verifies the LXMF message packing behavior described in the report. Generate at least one reproducible test vector. +``` + +Then: + +```text +Update SPEC.md section "LXMessage Wire Format" using only behavior confirmed by source inspection or verifier output. Mark all unverified behavior as unresolved. +``` + +## Bottom line + +Yes, you can accomplish the same kind of workflow with Codex. In fact, Codex gives you both mechanisms: + +`AGENTS.md` for standing repo rules, and `SKILL.md` under `.agents/skills/...` for a reusable “LXMF update / spec maintenance” workflow. OpenAI’s Codex docs explicitly describe customization as a combination of `AGENTS.md`, skills, MCP, memories, and subagents; for your use case, `AGENTS.md` plus a `SKILL.md` skill is the right starting point. ([OpenAI Developers][4]) + +[1]: https://developers.openai.com/codex/skills "Agent Skills – Codex | OpenAI Developers" +[2]: https://developers.openai.com/codex/guides/agents-md "Custom instructions with AGENTS.md – Codex | OpenAI Developers" +[3]: https://developers.openai.com/cookbook/articles/codex_exec_plans "Using PLANS.md for multi-hour problem solving" +[4]: https://developers.openai.com/codex/concepts/customization "Customization – Codex | OpenAI Developers" + diff --git a/Codex_1.md b/Codex_1.md new file mode 100644 index 0000000..049676a --- /dev/null +++ b/Codex_1.md @@ -0,0 +1,8 @@ +This directory: /usr/local/src/reticulum/LXMF +contains my GitHub fork (https://github.com/jlpoolen/LXMF) of Mark Qvist's: https://github.com/markqvist/LXMF which, incidentally, is a public mirror of some git server served up through Reticulum -- the source is not provided. + +I created an empty project in my Forgejo server: LXMF-specification at https://salemdata.net/repo/jlpoole/LMXF-specification +and I added my Forgejo instance as a remote repository identified by "forgejo". + +I added some files based on a ChatGPT chat, see ChatGPT_1.md. +So, for your first task: Using the lxmf-update skill, inspect upstream LXMF and determine the exact serialized structure of an LXMessage. Do not edit SPEC.md. Produce a report listing every field, its source-code origin, and whether its behavior is confirmed by a test vector. diff --git a/Codex_Response_1.md b/Codex_Response_1.md new file mode 100644 index 0000000..8b45de7 --- /dev/null +++ b/Codex_Response_1.md @@ -0,0 +1,83 @@ +No, I don’t need `SPEC.md` or `SOURCES.md` for this first report. I used the checked-out LXMF source directly. I did not edit `SPEC.md` or any other file. + +**LXMessage Serialized Structure** + +Source inspected: LXMF commit `fab12ad9bf9f997797034950f289fe41a79dcf5a` + +Core `lxmf_bytes` layout: + +```text +destination_hash || source_hash || signature || msgpack_payload +``` + +Where `msgpack_payload` is: + +```text +[ + timestamp, + title, + content, + fields, + optional_stamp +] +``` + +The fifth element is only present when a normal message stamp is generated and appended. + +| Field | Serialized position | Type / size | Source origin | Test vector confirmed | +|---|---:|---|---|---| +| `destination_hash` | bytes `0..15` | 16 bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:383), length from [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:40) and RNS 128-bit truncated hashes in [Reticulum.py](/usr/local/src/reticulum/Reticulum/RNS/Reticulum.py:146) | No | +| `source_hash` | bytes `16..31` | 16 bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:384) | No | +| `signature` | bytes `32..95` | 64 bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:378), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:385); RNS signature length from [Identity.py](/usr/local/src/reticulum/Reticulum/RNS/Identity.py:59) and [Identity.py](/usr/local/src/reticulum/Reticulum/RNS/Identity.py:80) | No | +| `msgpack_payload` | bytes `96..end` | msgpack list | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362), packed at [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:381) | No | +| `timestamp` | payload index `0` | Python float, normally msgpack float64 on this RNS umsgpack | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:357), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362); float64 behavior from [umsgpack.py](/usr/local/src/reticulum/Reticulum/RNS/vendor/umsgpack.py:1177) and [umsgpack.py](/usr/local/src/reticulum/Reticulum/RNS/vendor/umsgpack.py:325) | No | +| `title` | payload index `1` | bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:130), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:193), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362) | No | +| `content` | payload index `2` | bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:135), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:202), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362) | No | +| `fields` | payload index `3` | dict, default `{}` | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:215), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362) | No | +| `stamp` | payload index `4`, optional | bytes, only if generated | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:371), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:373); extracted on unpack at [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:754) | No | + +**Hash And Signature** + +`message_id` / `hash` is: + +```text +SHA256(destination_hash || source_hash || msgpack([timestamp, title, content, fields])) +``` + +Source: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:364), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:368), RNS SHA-256 at [Identity.py](/usr/local/src/reticulum/Reticulum/RNS/Identity.py:238). + +The signature is over: + +```text +destination_hash || source_hash || msgpack([timestamp, title, content, fields]) || message_id +``` + +Source: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:375), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:378), RNS signing at [Identity.py](/usr/local/src/reticulum/Reticulum/RNS/Identity.py:767). + +Important: the optional normal `stamp` is excluded from both `message_id` and signature. Unpack removes payload index `4` before recomputing hash/signature material: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:754). + +**Outer Representations** + +Direct/link/resource delivery uses full `self.packed` unchanged: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:636), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:654). + +Opportunistic packet delivery omits the destination hash from the packet payload because the RNS packet destination already implies it: + +```text +source_hash || signature || msgpack_payload +``` + +Source: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:633). + +Propagated and paper messages wrap/encrypt everything after the destination hash: + +```text +destination_hash || encrypted(source_hash || signature || msgpack_payload) +``` + +Source: propagated [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:429), paper [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:449). Propagation transit additionally msgpacks `[time.time(), [lxmf_data]]`: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:436). + +**Notes** + +The README is not authoritative here: it describes payload order as timestamp, content, title, fields, but current code serializes timestamp, title, content, fields. + +No test vector confirmation exists in this checkout. There is no `tools/` directory and no tracked `SPEC.md` or `SOURCES.md`, so every “Test vector confirmed” entry above is marked `No`. diff --git a/Codex_Response_2.md b/Codex_Response_2.md new file mode 100644 index 0000000..8b45de7 --- /dev/null +++ b/Codex_Response_2.md @@ -0,0 +1,83 @@ +No, I don’t need `SPEC.md` or `SOURCES.md` for this first report. I used the checked-out LXMF source directly. I did not edit `SPEC.md` or any other file. + +**LXMessage Serialized Structure** + +Source inspected: LXMF commit `fab12ad9bf9f997797034950f289fe41a79dcf5a` + +Core `lxmf_bytes` layout: + +```text +destination_hash || source_hash || signature || msgpack_payload +``` + +Where `msgpack_payload` is: + +```text +[ + timestamp, + title, + content, + fields, + optional_stamp +] +``` + +The fifth element is only present when a normal message stamp is generated and appended. + +| Field | Serialized position | Type / size | Source origin | Test vector confirmed | +|---|---:|---|---|---| +| `destination_hash` | bytes `0..15` | 16 bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:383), length from [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:40) and RNS 128-bit truncated hashes in [Reticulum.py](/usr/local/src/reticulum/Reticulum/RNS/Reticulum.py:146) | No | +| `source_hash` | bytes `16..31` | 16 bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:384) | No | +| `signature` | bytes `32..95` | 64 bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:378), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:385); RNS signature length from [Identity.py](/usr/local/src/reticulum/Reticulum/RNS/Identity.py:59) and [Identity.py](/usr/local/src/reticulum/Reticulum/RNS/Identity.py:80) | No | +| `msgpack_payload` | bytes `96..end` | msgpack list | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362), packed at [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:381) | No | +| `timestamp` | payload index `0` | Python float, normally msgpack float64 on this RNS umsgpack | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:357), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362); float64 behavior from [umsgpack.py](/usr/local/src/reticulum/Reticulum/RNS/vendor/umsgpack.py:1177) and [umsgpack.py](/usr/local/src/reticulum/Reticulum/RNS/vendor/umsgpack.py:325) | No | +| `title` | payload index `1` | bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:130), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:193), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362) | No | +| `content` | payload index `2` | bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:135), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:202), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362) | No | +| `fields` | payload index `3` | dict, default `{}` | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:215), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362) | No | +| `stamp` | payload index `4`, optional | bytes, only if generated | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:371), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:373); extracted on unpack at [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:754) | No | + +**Hash And Signature** + +`message_id` / `hash` is: + +```text +SHA256(destination_hash || source_hash || msgpack([timestamp, title, content, fields])) +``` + +Source: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:364), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:368), RNS SHA-256 at [Identity.py](/usr/local/src/reticulum/Reticulum/RNS/Identity.py:238). + +The signature is over: + +```text +destination_hash || source_hash || msgpack([timestamp, title, content, fields]) || message_id +``` + +Source: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:375), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:378), RNS signing at [Identity.py](/usr/local/src/reticulum/Reticulum/RNS/Identity.py:767). + +Important: the optional normal `stamp` is excluded from both `message_id` and signature. Unpack removes payload index `4` before recomputing hash/signature material: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:754). + +**Outer Representations** + +Direct/link/resource delivery uses full `self.packed` unchanged: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:636), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:654). + +Opportunistic packet delivery omits the destination hash from the packet payload because the RNS packet destination already implies it: + +```text +source_hash || signature || msgpack_payload +``` + +Source: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:633). + +Propagated and paper messages wrap/encrypt everything after the destination hash: + +```text +destination_hash || encrypted(source_hash || signature || msgpack_payload) +``` + +Source: propagated [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:429), paper [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:449). Propagation transit additionally msgpacks `[time.time(), [lxmf_data]]`: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:436). + +**Notes** + +The README is not authoritative here: it describes payload order as timestamp, content, title, fields, but current code serializes timestamp, title, content, fields. + +No test vector confirmation exists in this checkout. There is no `tools/` directory and no tracked `SPEC.md` or `SOURCES.md`, so every “Test vector confirmed” entry above is marked `No`. diff --git a/Codex_Response_3.md b/Codex_Response_3.md new file mode 100644 index 0000000..6362cd8 --- /dev/null +++ b/Codex_Response_3.md @@ -0,0 +1,27 @@ +Task #2 completed in `/usr/local/src/reticulum/LXMF`. + +Created: + +- `tools/make_test_vectors.pl` +- `tools/verify_examples.pl` +- `tools/LXMF/MessagePack.pm` +- `tools/README.md` +- `examples/lxmf_message_minimal.hex` +- `examples/lxmf_message_stamped.hex` +- `examples/lxmf_message_vectors.json` + +The dependency-free Perl tooling verifies: + +- Fixed hash/signature positions +- Payload order and MessagePack types +- Message ID calculation +- Optional stamp exclusion from message ID and signature input + +Validation passed: + +```text +Reticulum umsgpack byte-for-byte cross-check passed +All LXMessage test-vector checks passed. +``` + +The vectors regenerate deterministically. Signature bytes are documented placeholders; Ed25519 validation is not claimed. `SPEC.md` was not created or modified. diff --git a/Codex_Response_4.md b/Codex_Response_4.md new file mode 100644 index 0000000..c1d9211 --- /dev/null +++ b/Codex_Response_4.md @@ -0,0 +1,14 @@ +Created [SPEC.md](/usr/local/src/reticulum/LXMF/SPEC.md) containing the evidence-limited `LXMessage Wire Format` section. + +It documents: + +- Core serialized layout and payload order +- Message ID and signature-input calculations +- Stamp exclusion behavior +- Delivery representations +- Exact source references and evidence commits +- Explicit unresolved behavior + +Verification: `perl tools/verify_examples.pl` passed all checks. + +Only `SPEC.md` is untracked; no other files were changed. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b7ed71d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM python:alpine +LABEL authors="Petr Blaha petr.blaha@cleverdata.cz" +USER root +RUN apk update +RUN apk add sdl2_ttf sdl2 build-base libc-dev pkgconfig gstreamer sdl2_mixer sdl2_image sdl2_pango linux-headers mesa-dev py3-virtualenv + +RUN addgroup -S myuser && adduser -S -G myuser myuser +USER myuser +WORKDIR /home/myuser + +RUN pip install --upgrade pip + + +ENV PATH="/home/myuser/.local/bin:${PATH}" + +################### BEGIN LXMF ########################################### + +COPY --chown=myuser:myuser requirements.txt requirements.txt + +RUN pip install --user -r requirements.txt + + +COPY --chown=myuser:myuser . . + +#Python create virtual environment +RUN virtualenv /home/myuser/LXMF/venv +RUN source /home/myuser/LXMF/venv/bin/activate + +RUN make all + +################### END LXMF ########################################### diff --git a/Dockerfile.howto b/Dockerfile.howto new file mode 100644 index 0000000..bb20e5f --- /dev/null +++ b/Dockerfile.howto @@ -0,0 +1,6 @@ +# Run docker command one by one(all four), it will build LXMF artifact and copy to dist directory. +# No need to build locally and install dependencies +docker build -t lxmfdockerimage . +docker run -d -it --name lxmfdockercontainer lxmfdockerimage /bin/sh +docker cp lxmfdockercontainer:/home/myuser/dist . +docker rm -f lxmfdockercontainer \ No newline at end of file diff --git a/FUNDING.yml b/FUNDING.yml new file mode 100644 index 0000000..d125d55 --- /dev/null +++ b/FUNDING.yml @@ -0,0 +1,3 @@ +liberapay: Reticulum +ko_fi: markqvist +custom: "https://unsigned.io/donate" diff --git a/LICENSE b/LICENSE index 1513e21..f5fb92d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,9 +1,29 @@ -MIT License +Reticulum License -Copyright (c) 2026 jlpoole +Copyright (c) 2020-2025 Mark Qvist -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +- The Software shall not be used in any kind of system which includes amongst + its functions the ability to purposefully do harm to human beings. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +- The Software shall not be used, directly or indirectly, in the creation of + an artificial intelligence, machine learning or language model training + dataset, including but not limited to any use that contributes to the + training or development of such a model or algorithm. + +- The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py new file mode 100644 index 0000000..6239b18 --- /dev/null +++ b/LXMF/Handlers.py @@ -0,0 +1,103 @@ +import time +import threading +import RNS +import RNS.vendor.umsgpack as msgpack + +from .LXMF import APP_NAME, stamp_cost_from_app_data, pn_announce_data_is_valid +from .LXMessage import LXMessage + +class LXMFDeliveryAnnounceHandler: + def __init__(self, lxmrouter): + self.aspect_filter = APP_NAME+".delivery" + self.receive_path_responses = True + self.lxmrouter = lxmrouter + + def received_announce(self, destination_hash, announced_identity, app_data): + try: + stamp_cost = stamp_cost_from_app_data(app_data) + self.lxmrouter.update_stamp_cost(destination_hash, stamp_cost) + + except Exception as e: + RNS.log(f"Could not decode stamp cost from announce data. The contained exception was: {e}", RNS.LOG_DEBUG) + + for lxmessage in self.lxmrouter.pending_outbound: + if destination_hash == lxmessage.destination_hash: + if lxmessage.method == LXMessage.DIRECT or lxmessage.method == LXMessage.OPPORTUNISTIC: + lxmessage.next_delivery_attempt = time.time() + + def outbound_trigger(): + while self.lxmrouter.outbound_processing_lock.locked(): time.sleep(0.1) + self.lxmrouter.process_outbound() + + threading.Thread(target=outbound_trigger, daemon=True).start() + + +class LXMFPropagationAnnounceHandler: + def __init__(self, lxmrouter): + self.aspect_filter = APP_NAME+".propagation" + self.receive_path_responses = True + self.lxmrouter = lxmrouter + + def received_announce(self, destination_hash, announced_identity, app_data, announce_packet_hash, is_path_response): + try: + if type(app_data) == bytes: + if destination_hash == self.lxmrouter.get_outbound_propagation_node(): + if pn_announce_data_is_valid(app_data): + for lxmessage in self.lxmrouter.pending_outbound: + if lxmessage.method == LXMessage.PROPAGATED: + lxmessage.next_delivery_attempt = time.time() + + def outbound_trigger(): + while self.lxmrouter.outbound_processing_lock.locked(): time.sleep(0.1) + self.lxmrouter.process_outbound() + + threading.Thread(target=outbound_trigger, daemon=True).start() + + if self.lxmrouter.propagation_node: + if pn_announce_data_is_valid(app_data): + data = msgpack.unpackb(app_data) + node_timebase = int(data[1]) + propagation_enabled = data[2] + propagation_transfer_limit = int(data[3]) + propagation_sync_limit = int(data[4]) + propagation_stamp_cost = int(data[5][0]) + propagation_stamp_cost_flexibility = int(data[5][1]) + peering_cost = int(data[5][2]) + metadata = data[6] + + if destination_hash in self.lxmrouter.static_peers: + static_peer = self.lxmrouter.peers[destination_hash] + if not is_path_response or static_peer.last_heard == 0: + self.lxmrouter.peer(destination_hash=destination_hash, + timestamp=node_timebase, + propagation_transfer_limit=propagation_transfer_limit, + propagation_sync_limit=propagation_sync_limit, + propagation_stamp_cost=propagation_stamp_cost, + propagation_stamp_cost_flexibility=propagation_stamp_cost_flexibility, + peering_cost=peering_cost, + metadata=metadata) + + else: + if self.lxmrouter.autopeer and not is_path_response: + if propagation_enabled == True: + if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: + self.lxmrouter.peer(destination_hash=destination_hash, + timestamp=node_timebase, + propagation_transfer_limit=propagation_transfer_limit, + propagation_sync_limit=propagation_sync_limit, + propagation_stamp_cost=propagation_stamp_cost, + propagation_stamp_cost_flexibility=propagation_stamp_cost_flexibility, + peering_cost=peering_cost, + metadata=metadata) + + else: + if destination_hash in self.lxmrouter.peers: + RNS.log(f"Peer {self.lxmrouter.peers[destination_hash]} moved outside auto-peering range, breaking peering...") + self.lxmrouter.unpeer(destination_hash, node_timebase) + + elif propagation_enabled == False: + self.lxmrouter.unpeer(destination_hash, node_timebase) + + except Exception as e: + RNS.log("Error while evaluating propagation node announce, ignoring announce.", RNS.LOG_DEBUG) + RNS.log(f"The contained exception was: {str(e)}", RNS.LOG_DEBUG) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py new file mode 100644 index 0000000..d981ae0 --- /dev/null +++ b/LXMF/LXMF.py @@ -0,0 +1,250 @@ +APP_NAME = "lxmf" + +########################################################## +# The following core fields are provided to facilitate # +# interoperability in data exchange between various LXMF # +# clients and systems. # +########################################################## +FIELD_EMBEDDED_LXMS = 0x01 +FIELD_TELEMETRY = 0x02 +FIELD_TELEMETRY_STREAM = 0x03 +FIELD_ICON_APPEARANCE = 0x04 +FIELD_FILE_ATTACHMENTS = 0x05 +FIELD_IMAGE = 0x06 +FIELD_AUDIO = 0x07 +FIELD_THREAD = 0x08 # Bytes, full thread ID hash +FIELD_COMMANDS = 0x09 +FIELD_RESULTS = 0x0A +FIELD_GROUP = 0x0B +FIELD_TICKET = 0x0C +FIELD_EVENT = 0x0D +FIELD_RNR_REFS = 0x0E +FIELD_RENDERER = 0x0F +FIELD_REPLY_TO = 0x30 # Bytes, full LXMessage.hash +FIELD_REPLY_QUOTE = 0x31 # Bytes, quoted content in UTF-8 encoding +FIELD_REACTION = 0x40 # Dict, see "Reaction dict indices" below +FIELD_COMMENT = 0x41 # Dict, see "Comment dict indices" below +FIELD_CONTINUATION = 0x42 # Dict, see "Continuation dict indices" below + +# Unallocated fields between 0x00 and 0x80, both included, +# should be considered reserved for future extensibility +# For experimental and unstable features, it is recommended +# to use fields above 0xFF. + +# For usecases such as including custom data structures, +# embedding or encapsulating other data types or protocols +# that are not native to LXMF, or bridging/tunneling +# external protocols or services over LXMF, the following +# fields are available. A format/type/protocol (or other) +# identifier can be included in the CUSTOM_TYPE field, and +# the embedded payload can be included in the CUSTOM_DATA +# field. It is up to the client application to correctly +# discern and potentially utilise any data embedded using +# this mechanism. +FIELD_CUSTOM_TYPE = 0xFB +FIELD_CUSTOM_DATA = 0xFC +FIELD_CUSTOM_META = 0xFD + +# The non-specific and debug fields are intended for +# development, testing and debugging use. +FIELD_NON_SPECIFIC = 0xFE +FIELD_DEBUG = 0xFF + +########################################################## +# The following section lists field-specific specifiers, # +# modes and identifiers that are native to LXMF. It is # +# optional for any client or system to support any of # +# these, and they are provided as template for easing # +# interoperability without sacrificing expandability # +# and flexibility of the format. # +########################################################## + +# Audio modes for the data structure in FIELD_AUDIO + +# Codec2 Audio Modes +AM_CODEC2_450PWB = 0x01 +AM_CODEC2_450 = 0x02 +AM_CODEC2_700C = 0x03 +AM_CODEC2_1200 = 0x04 +AM_CODEC2_1300 = 0x05 +AM_CODEC2_1400 = 0x06 +AM_CODEC2_1600 = 0x07 +AM_CODEC2_2400 = 0x08 +AM_CODEC2_3200 = 0x09 + +# Opus Audio Modes +AM_OPUS_OGG = 0x10 +AM_OPUS_LBW = 0x11 +AM_OPUS_MBW = 0x12 +AM_OPUS_PTT = 0x13 +AM_OPUS_RT_HDX = 0x14 +AM_OPUS_RT_FDX = 0x15 +AM_OPUS_STANDARD = 0x16 +AM_OPUS_HQ = 0x17 +AM_OPUS_BROADCAST = 0x18 +AM_OPUS_LOSSLESS = 0x19 + +# Custom, unspecified audio mode, the client must +# determine it itself based on the included data. +AM_CUSTOM = 0xFF + +# Message renderer specifications for FIELD_RENDERER. +# The renderer specification is completely optional, +# and only serves as an indication to the receiving +# client on how to render the message contents. It is +# not mandatory to implement, either on sending or +# receiving sides, but is the recommended way to +# signal how to render a message, if non-plaintext +# formatting is used. +RENDERER_PLAIN = 0x00 +RENDERER_MICRON = 0x01 +RENDERER_MARKDOWN = 0x02 +RENDERER_BBCODE = 0x03 + +# Clients choose how to handle reaction content, if at all. +# While reactions are typically a single unicode emoji or +# similar, the exact implementation and sanitization is +# left up to the client. When using the FIELD_REACTION +# field, the contents is a dict with the following keys: +REACTION_TO = 0x00 # Bytes, full LXMessage.hash +REACTION_CONTENT = 0x01 # Bytes, the reaction content in UTF-8 encoding + +# Clients choose how to handle messages intended as comments +# for other message, if at all. The actual comment content +# is carried as the normal LXM content, meaning clients that +# do not support comments will display them as normal messages. +# When using the FIELD_COMMENT field, the contents is a dict +# with the following keys: +COMMENT_FOR = 0x00 # Bytes, full LXMessage.hash + +# Clients choose how to handle messages that continue earlier +# messages, if at all. The actual continuation content is +# carried as the normal LXM content, meaning clients that +# do not support continuations will display them as normal. +# When using the FIELD_CONTINUATION field, the contents is a +# dict with the following keys: +CONTINUATION_OF = 0x00 # Bytes, full LXMessage.hash + +# Optional propagation node metadata fields. These +# fields may be highly unstable in allocation and +# availability until the version 1.0.0 release, so use +# at your own risk until then, and expect changes! +PN_META_VERSION = 0x00 +PN_META_NAME = 0x01 +PN_META_SYNC_STRATUM = 0x02 +PN_META_SYNC_THROTTLE = 0x03 +PN_META_AUTH_BAND = 0x04 +PN_META_UTIL_PRESSURE = 0x05 +PN_META_CUSTOM = 0xFF + +# Supported functionality codes for signalling +# feature and capability support. +SF_COMPRESSION = 0x00 + +########################################################## +# The following helper functions makes it easier to # +# handle and operate on LXMF data in client programs # +########################################################## + +import RNS +import RNS.vendor.umsgpack as msgpack +def display_name_from_app_data(app_data=None): + if app_data == None: return None + elif len(app_data) == 0: return None + else: + # Version 0.5.0+ announce format + if (app_data[0] >= 0x90 and app_data[0] <= 0x9f) or app_data[0] == 0xdc: + peer_data = msgpack.unpackb(app_data) + if type(peer_data) == list: + if len(peer_data) < 1: return None + else: + dn = peer_data[0] + if dn == None: return None + else: + try: + decoded = dn.decode("utf-8").replace("\x00", "").strip() + return decoded + except Exception as e: + RNS.log(f"Could not decode display name in included announce data. The contained exception was: {e}", RNS.LOG_ERROR) + return None + + # Original announce format + else: return app_data.decode("utf-8") + +def stamp_cost_from_app_data(app_data=None): + if app_data == None or app_data == b"": return None + else: + # Version 0.5.0+ announce format + if (app_data[0] >= 0x90 and app_data[0] <= 0x9f) or app_data[0] == 0xdc: + peer_data = msgpack.unpackb(app_data) + if type(peer_data) == list: + if len(peer_data) < 2: return None + else: return peer_data[1] + + # Original announce format + else: return None + +def compression_support_from_app_data(app_data=None): + if app_data == None or app_data == b"": return None + else: + # Version 0.5.0+ announce format + if (app_data[0] >= 0x90 and app_data[0] <= 0x9f) or app_data[0] == 0xdc: + peer_data = msgpack.unpackb(app_data) + if type(peer_data) == list: + if len(peer_data) < 3: return True + else: + if not type(peer_data[2]) == list: return True + else: return SF_COMPRESSION in peer_data[2] + + # Original announce format + else: return True + +def pn_name_from_app_data(app_data=None): + if app_data == None: return None + else: + if pn_announce_data_is_valid(app_data): + data = msgpack.unpackb(app_data) + metadata = data[6] + if not PN_META_NAME in metadata: return None + else: + try: return metadata[PN_META_NAME].decode("utf-8") + except: return None + + return None + +def pn_stamp_cost_from_app_data(app_data=None): + if app_data == None: return None + else: + if pn_announce_data_is_valid(app_data): + data = msgpack.unpackb(app_data) + return data[5][0] + + else: return None + +def pn_announce_data_is_valid(data): + try: + if type(data) != bytes: return False + else: data = msgpack.unpackb(data) + if len(data) < 7: raise ValueError("Invalid announce data: Insufficient peer data, likely from deprecated LXMF version") + else: + try: int(data[1]) + except: raise ValueError("Invalid announce data: Could not decode timebase") + if data[2] != True and data[2] != False: raise ValueError("Invalid announce data: Indeterminate propagation node status") + try: int(data[3]) + except: raise ValueError("Invalid announce data: Could not decode propagation transfer limit") + try: int(data[4]) + except: raise ValueError("Invalid announce data: Could not decode propagation sync limit") + if type(data[5]) != list: raise ValueError("Invalid announce data: Could not decode stamp costs") + try: int(data[5][0]) + except: raise ValueError("Invalid announce data: Could not decode target stamp cost") + try: int(data[5][1]) + except: raise ValueError("Invalid announce data: Could not decode stamp cost flexibility") + try: int(data[5][2]) + except: raise ValueError("Invalid announce data: Could not decode peering cost") + if type(data[6]) != dict: raise ValueError("Invalid announce data: Could not decode metadata") + + except Exception as e: + RNS.log(f"Could not validate propagation node announce data: {e}", RNS.LOG_DEBUG) + return False + + return True \ No newline at end of file diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py new file mode 100644 index 0000000..37515e6 --- /dev/null +++ b/LXMF/LXMPeer.py @@ -0,0 +1,642 @@ +import os +import time +import threading + +import RNS +import RNS.vendor.umsgpack as msgpack +import LXMF.LXStamper as LXStamper + +from collections import deque +from .LXMF import APP_NAME +from .LXMF import PN_META_NAME + +class LXMPeer: + OFFER_REQUEST_PATH = "/offer" + MESSAGE_GET_PATH = "/get" + + IDLE = 0x00 + LINK_ESTABLISHING = 0x01 + LINK_READY = 0x02 + REQUEST_SENT = 0x03 + RESPONSE_RECEIVED = 0x04 + RESOURCE_TRANSFERRING = 0x05 + + ERROR_NO_IDENTITY = 0xf0 + ERROR_NO_ACCESS = 0xf1 + ERROR_INVALID_KEY = 0xf3 + ERROR_INVALID_DATA = 0xf4 + ERROR_INVALID_STAMP = 0xf5 + ERROR_THROTTLED = 0xf6 + ERROR_NOT_FOUND = 0xfd + ERROR_TIMEOUT = 0xfe + + STRATEGY_LAZY = 0x01 + STRATEGY_PERSISTENT = 0x02 + DEFAULT_SYNC_STRATEGY = STRATEGY_PERSISTENT + + # Maximum amount of time a peer can + # be unreachable before it is removed + MAX_UNREACHABLE = 14*24*60*60 + + # Everytime consecutive time a sync + # link fails to establish, add this + # amount off time to wait before the + # next sync is attempted. + SYNC_BACKOFF_STEP = 12*60 + + # How long to wait for an answer to + # peer path requests before deferring + # sync to later. + PATH_REQUEST_GRACE = 7.5 + + @staticmethod + def from_bytes(peer_bytes, router): + dictionary = msgpack.unpackb(peer_bytes) + peer_destination_hash = dictionary["destination_hash"] + peer_peering_timebase = dictionary["peering_timebase"] + peer_alive = dictionary["alive"] + peer_last_heard = dictionary["last_heard"] + + peer = LXMPeer(router, peer_destination_hash) + peer.peering_timebase = peer_peering_timebase + peer.alive = peer_alive + peer.last_heard = peer_last_heard + + if "link_establishment_rate" in dictionary: peer.link_establishment_rate = dictionary["link_establishment_rate"] + else: peer.link_establishment_rate = 0 + + if "sync_transfer_rate" in dictionary: peer.sync_transfer_rate = dictionary["sync_transfer_rate"] + else: peer.sync_transfer_rate = 0 + + if "propagation_transfer_limit" in dictionary: + try: peer.propagation_transfer_limit = float(dictionary["propagation_transfer_limit"]) + except Exception as e: peer.propagation_transfer_limit = None + else: peer.propagation_transfer_limit = None + + if "propagation_sync_limit" in dictionary: + try: peer.propagation_sync_limit = int(dictionary["propagation_sync_limit"]) + except: peer.propagation_sync_limit = peer.propagation_transfer_limit + else: peer.propagation_sync_limit = peer.propagation_transfer_limit + + if "propagation_stamp_cost" in dictionary: + try: peer.propagation_stamp_cost = int(dictionary["propagation_stamp_cost"]) + except: peer.propagation_stamp_cost = None + else: peer.propagation_stamp_cost = None + + if "propagation_stamp_cost_flexibility" in dictionary: + try: peer.propagation_stamp_cost_flexibility = int(dictionary["propagation_stamp_cost_flexibility"]) + except: peer.propagation_stamp_cost_flexibility = None + else: peer.propagation_stamp_cost_flexibility = None + + if "peering_cost" in dictionary: + try: peer.peering_cost = int(dictionary["peering_cost"]) + except: peer.peering_cost = None + else: peer.peering_cost = None + + if "sync_strategy" in dictionary: + try: peer.sync_strategy = int(dictionary["sync_strategy"]) + except: peer.sync_strategy = LXMPeer.DEFAULT_SYNC_STRATEGY + else: peer.sync_strategy = LXMPeer.DEFAULT_SYNC_STRATEGY + + if "offered" in dictionary: peer.offered = dictionary["offered"] + else: peer.offered = 0 + if "outgoing" in dictionary: peer.outgoing = dictionary["outgoing"] + else: peer.outgoing = 0 + if "incoming" in dictionary: peer.incoming = dictionary["incoming"] + else: peer.incoming = 0 + if "rx_bytes" in dictionary: peer.rx_bytes = dictionary["rx_bytes"] + else: peer.rx_bytes = 0 + if "tx_bytes" in dictionary: peer.tx_bytes = dictionary["tx_bytes"] + else: peer.tx_bytes = 0 + if "last_sync_attempt" in dictionary: peer.last_sync_attempt = dictionary["last_sync_attempt"] + else: peer.last_sync_attempt = 0 + if "peering_key" in dictionary: peer.peering_key = dictionary["peering_key"] + else: peer.peering_key = None + if "metadata" in dictionary: peer.metadata = dictionary["metadata"] + else: peer.metadata = None + + hm_count = 0 + for transient_id in dictionary["handled_ids"]: + if transient_id in router.propagation_entries: + peer.add_handled_message(transient_id) + hm_count += 1 + + um_count = 0 + for transient_id in dictionary["unhandled_ids"]: + if transient_id in router.propagation_entries: + peer.add_unhandled_message(transient_id) + um_count += 1 + + peer._hm_count = hm_count + peer._um_count = um_count + peer._hm_counts_synced = True + peer._um_counts_synced = True + + del dictionary + return peer + + def to_bytes(self): + dictionary = {} + dictionary["peering_timebase"] = self.peering_timebase + dictionary["alive"] = self.alive + dictionary["metadata"] = self.metadata + dictionary["last_heard"] = self.last_heard + dictionary["sync_strategy"] = self.sync_strategy + dictionary["peering_key"] = self.peering_key + dictionary["destination_hash"] = self.destination_hash + dictionary["link_establishment_rate"] = self.link_establishment_rate + dictionary["sync_transfer_rate"] = self.sync_transfer_rate + dictionary["propagation_transfer_limit"] = self.propagation_transfer_limit + dictionary["propagation_sync_limit"] = self.propagation_sync_limit + dictionary["propagation_stamp_cost"] = self.propagation_stamp_cost + dictionary["propagation_stamp_cost_flexibility"] = self.propagation_stamp_cost_flexibility + dictionary["peering_cost"] = self.peering_cost + dictionary["last_sync_attempt"] = self.last_sync_attempt + dictionary["offered"] = self.offered + dictionary["outgoing"] = self.outgoing + dictionary["incoming"] = self.incoming + dictionary["rx_bytes"] = self.rx_bytes + dictionary["tx_bytes"] = self.tx_bytes + + handled_ids = [] + for transient_id in self.handled_messages: + handled_ids.append(transient_id) + + unhandled_ids = [] + for transient_id in self.unhandled_messages: + unhandled_ids.append(transient_id) + + dictionary["handled_ids"] = handled_ids + dictionary["unhandled_ids"] = unhandled_ids + + peer_bytes = msgpack.packb(dictionary) + del dictionary + + return peer_bytes + + def __init__(self, router, destination_hash, sync_strategy=DEFAULT_SYNC_STRATEGY): + self.alive = False + self.last_heard = 0 + self.sync_strategy = sync_strategy + self.peering_key = None + self.peering_cost = None + self.metadata = None + + self.next_sync_attempt = 0 + self.last_sync_attempt = 0 + self.sync_backoff = 0 + self.peering_timebase = 0 + self.link_establishment_rate = 0 + self.sync_transfer_rate = 0 + + self.propagation_transfer_limit = None + self.propagation_sync_limit = None + self.propagation_stamp_cost = None + self.propagation_stamp_cost_flexibility = None + self.currently_transferring_messages = None + self.handled_messages_queue = deque() + self.unhandled_messages_queue = deque() + + self.offered = 0 # Messages offered to this peer + self.outgoing = 0 # Messages transferred to this peer + self.incoming = 0 # Messages received from this peer + self.rx_bytes = 0 # Bytes received from this peer + self.tx_bytes = 0 # Bytes sent to this peer + + self._hm_count = 0 + self._um_count = 0 + self._hm_counts_synced = False + self._um_counts_synced = False + + self._peering_key_lock = threading.Lock() + + self.link = None + self.state = LXMPeer.IDLE + + self.last_offer = [] + + self.router = router + self.destination_hash = destination_hash + self.identity = RNS.Identity.recall(destination_hash) + if self.identity != None: + self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + else: + self.destination = None + RNS.log(f"Could not recall identity for LXMF propagation peer {RNS.prettyhexrep(self.destination_hash)}, will retry identity resolution on next sync", RNS.LOG_WARNING) + + def peering_key_ready(self): + if not self.peering_cost: return False + if type(self.peering_key) == list and len(self.peering_key) == 2: + value = self.peering_key[1] + if value >= self.peering_cost: return True + else: + RNS.log(f"Peering key value mismatch for {self}. Current value is {value}, but peer requires {self.peering_cost}. Scheduling regeneration...", RNS.LOG_WARNING) + self.peering_key = None + + return False + + def peering_key_value(self): + if type(self.peering_key) == list and len(self.peering_key) == 2: return self.peering_key[1] + else: return None + + def generate_peering_key(self): + if self.peering_cost == None: return False + with self._peering_key_lock: + if self.peering_key != None: return True + else: + RNS.log(f"Generating peering key for {self}", RNS.LOG_NOTICE) + if self.router.identity == None: + RNS.log(f"Could not update peering key for {self} since the local LXMF router identity is not configured", RNS.LOG_ERROR) + return False + + if self.identity == None: + self.identity = RNS.Identity.recall(destination_hash) + if self.identity == None: + RNS.log(f"Could not update peering key for {self} since its identity could not be recalled", RNS.LOG_ERROR) + return False + + key_material = self.identity.hash+self.router.identity.hash + peering_key, value = LXStamper.generate_stamp(key_material, self.peering_cost, expand_rounds=LXStamper.WORKBLOCK_EXPAND_ROUNDS_PEERING) + if value >= self.peering_cost: + self.peering_key = [peering_key, value] + RNS.log(f"Peering key successfully generated for {self}", RNS.LOG_NOTICE) + return True + + return False + + def sync(self): + RNS.log("Initiating LXMF Propagation Node sync with peer "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_DEBUG) + self.last_sync_attempt = time.time() + + sync_time_reached = time.time() > self.next_sync_attempt + stamp_costs_known = self.propagation_stamp_cost != None and self.propagation_stamp_cost_flexibility != None and self.peering_cost != None + peering_key_ready = self.peering_key_ready() + sync_checks = sync_time_reached and stamp_costs_known and peering_key_ready + + if not sync_checks: + try: + if not sync_time_reached: + postpone_reason = " due to previous failures" + if self.last_sync_attempt > self.last_heard: self.alive = False + elif not stamp_costs_known: + postpone_reason = " since its required stamp costs are not yet known" + elif not peering_key_ready: + postpone_reason = " since a peering key has not been generated yet" + def job(): self.generate_peering_key() + threading.Thread(target=job, daemon=True).start() + + delay = self.next_sync_attempt-time.time() + postpone_delay = f" for {RNS.prettytime(delay)}" if delay > 0 else "" + RNS.log(f"Postponing sync with peer {RNS.prettyhexrep(self.destination_hash)}{postpone_delay}{postpone_reason}", RNS.LOG_DEBUG) + except Exception as e: + RNS.trace_exception(e) + + else: + if not RNS.Transport.has_path(self.destination_hash): + RNS.log("No path to peer "+RNS.prettyhexrep(self.destination_hash)+" exists, requesting...", RNS.LOG_DEBUG) + RNS.Transport.request_path(self.destination_hash) + time.sleep(LXMPeer.PATH_REQUEST_GRACE) + + if not RNS.Transport.has_path(self.destination_hash): + RNS.log("Path request was not answered, retrying sync with peer "+RNS.prettyhexrep(self.destination_hash)+" later", RNS.LOG_DEBUG) + + else: + if self.identity == None: + self.identity = RNS.Identity.recall(destination_hash) + if self.identity != None: + self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + + if self.destination != None: + if len(self.unhandled_messages) == 0: + RNS.log(f"Sync requested for {self}, but no unhandled messages exist for peer. Sync complete.", RNS.LOG_DEBUG) + return + + if len(self.unhandled_messages) > 0: + if self.currently_transferring_messages != None: + RNS.log(f"Sync requested for {self}, but current message transfer index was not clear. Aborting.", RNS.LOG_ERROR) + return + + if self.state == LXMPeer.IDLE: + RNS.log("Establishing link for sync to peer "+RNS.prettyhexrep(self.destination_hash)+"...", RNS.LOG_DEBUG) + self.sync_backoff += LXMPeer.SYNC_BACKOFF_STEP + self.next_sync_attempt = time.time() + self.sync_backoff + self.link = RNS.Link(self.destination, established_callback = self.link_established, closed_callback = self.link_closed) + self.state = LXMPeer.LINK_ESTABLISHING + + else: + if self.state == LXMPeer.LINK_READY: + self.alive = True + self.last_heard = time.time() + self.sync_backoff = 0 + min_accepted_cost = min(0, self.propagation_stamp_cost-self.propagation_stamp_cost_flexibility) + + RNS.log("Synchronisation link to peer "+RNS.prettyhexrep(self.destination_hash)+" established, preparing sync offer...", RNS.LOG_DEBUG) + unhandled_entries = [] + unhandled_ids = [] + purged_ids = [] + low_value_ids = [] + for transient_id in self.unhandled_messages: + if transient_id in self.router.propagation_entries: + if self.router.get_stamp_value(transient_id) < min_accepted_cost: low_value_ids.append(transient_id) + else: + unhandled_entry = [ transient_id, + self.router.get_weight(transient_id), + self.router.get_size(transient_id) ] + + unhandled_entries.append(unhandled_entry) + + else: purged_ids.append(transient_id) + + for transient_id in purged_ids: + RNS.log(f"Dropping unhandled message {RNS.prettyhexrep(transient_id)} for peer {RNS.prettyhexrep(self.destination_hash)} since it no longer exists in the message store.", RNS.LOG_DEBUG) + self.remove_unhandled_message(transient_id) + + for transient_id in low_value_ids: + RNS.log(f"Dropping unhandled message {RNS.prettyhexrep(transient_id)} for peer {RNS.prettyhexrep(self.destination_hash)} since its stamp value is lower than peer requirement of {min_accepted_cost}.", RNS.LOG_DEBUG) + self.remove_unhandled_message(transient_id) + + unhandled_entries.sort(key=lambda e: e[1], reverse=False) + per_message_overhead = 16 # Really only 2 bytes, but set a bit higher for now + cumulative_size = 24 # Initialised to highest reasonable binary structure overhead + RNS.log(f"Syncing to peer with per-message limit {RNS.prettysize(self.propagation_transfer_limit*1000)} and sync limit {RNS.prettysize(self.propagation_sync_limit*1000)}") # TODO: Remove debug + + for unhandled_entry in unhandled_entries: + transient_id = unhandled_entry[0] + weight = unhandled_entry[1] + lxm_size = unhandled_entry[2] + lxm_transfer_size = lxm_size+per_message_overhead + next_size = cumulative_size + lxm_transfer_size + + if self.propagation_transfer_limit != None and lxm_transfer_size > (self.propagation_transfer_limit*1000): + self.remove_unhandled_message(transient_id) + self.add_handled_message(transient_id) + continue + + if self.propagation_sync_limit != None and next_size >= (self.propagation_sync_limit*1000): + continue + + cumulative_size += lxm_transfer_size + unhandled_ids.append(transient_id) + + offer = [self.peering_key[0], unhandled_ids] + + RNS.log(f"Offering {len(unhandled_ids)} messages to peer {RNS.prettyhexrep(self.destination.hash)} ({RNS.prettysize(len(msgpack.packb(unhandled_ids)))})", RNS.LOG_VERBOSE) + self.last_offer = unhandled_ids + self.link.request(LXMPeer.OFFER_REQUEST_PATH, offer, response_callback=self.offer_response, failed_callback=self.request_failed) + self.state = LXMPeer.REQUEST_SENT + + else: + RNS.log(f"Could not request sync to peer {RNS.prettyhexrep(self.destination_hash)} since its identity could not be recalled.", RNS.LOG_ERROR) + + def request_failed(self, request_receipt): + RNS.log(f"Sync request to peer {self.destination} failed", RNS.LOG_DEBUG) + if self.link != None: self.link.teardown() + self.state = LXMPeer.IDLE + + def offer_response(self, request_receipt): + try: + self.state = LXMPeer.RESPONSE_RECEIVED + response = request_receipt.response + + wanted_messages = [] + wanted_message_ids = [] + + if response == LXMPeer.ERROR_NO_IDENTITY: + if self.link != None: + RNS.log("Remote peer indicated that no identification was received, retrying...", RNS.LOG_VERBOSE) + self.link.identify(self.router.identity) + self.state = LXMPeer.LINK_READY + self.sync() + return + + elif response == LXMPeer.ERROR_NO_ACCESS: + RNS.log("Remote indicated that access was denied, breaking peering", RNS.LOG_VERBOSE) + self.router.unpeer(self.destination_hash) + return + + elif response == LXMPeer.ERROR_THROTTLED: + throttle_time = self.router.PN_STAMP_THROTTLE + RNS.log(f"Remote indicated that we're throttled, postponing sync for {RNS.prettytime(throttle_time)}", RNS.LOG_VERBOSE) + self.next_sync_attempt = time.time()+throttle_time + return + + elif response == False: + # Peer already has all advertised messages + for transient_id in self.last_offer: + if transient_id in self.unhandled_messages: + self.add_handled_message(transient_id) + self.remove_unhandled_message(transient_id) + + + elif response == True: + # Peer wants all advertised messages + for transient_id in self.last_offer: + wanted_messages.append(self.router.propagation_entries[transient_id]) + wanted_message_ids.append(transient_id) + + else: + # Peer wants some advertised messages + for transient_id in self.last_offer.copy(): + # If the peer did not want the message, it has + # already received it from another peer. + if not transient_id in response: + self.add_handled_message(transient_id) + self.remove_unhandled_message(transient_id) + + for transient_id in response: + wanted_messages.append(self.router.propagation_entries[transient_id]) + wanted_message_ids.append(transient_id) + + if len(wanted_messages) > 0: + RNS.log(f"Peer {RNS.prettyhexrep(self.destination_hash)} wanted {str(len(wanted_messages))} of the available messages", RNS.LOG_VERBOSE) + + lxm_list = [] + for message_entry in wanted_messages: + file_path = message_entry[1] + if os.path.isfile(file_path): + file = open(file_path, "rb") + lxmf_data = file.read() + file.close() + lxm_list.append(lxmf_data) + + data = msgpack.packb([time.time(), lxm_list]) + RNS.log(f"Total transfer size for this sync is {RNS.prettysize(len(data))}", RNS.LOG_VERBOSE) + resource = RNS.Resource(data, self.link, callback = self.resource_concluded) + self.currently_transferring_messages = wanted_message_ids + self.current_sync_transfer_started = time.time() + self.state = LXMPeer.RESOURCE_TRANSFERRING + + else: + RNS.log(f"Peer {RNS.prettyhexrep(self.destination_hash)} did not request any of the available messages, sync completed", RNS.LOG_VERBOSE) + self.offered += len(self.last_offer) + if self.link != None: + self.link.teardown() + + self.link = None + self.state = LXMPeer.IDLE + + except Exception as e: + RNS.log("Error while handling offer response from peer "+str(self.destination), RNS.LOG_ERROR) + RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) + + if self.link != None: + self.link.teardown() + + self.link = None + self.state = LXMPeer.IDLE + + def resource_concluded(self, resource): + if resource.status == RNS.Resource.COMPLETE: + if self.currently_transferring_messages == None: + RNS.log(f"Sync transfer completed on {self}, but transferred message index was unavailable. Aborting.", RNS.LOG_ERROR) + if self.link != None: self.link.teardown() + self.link = None + self.state = LXMPeer.IDLE + + for transient_id in self.currently_transferring_messages: + self.add_handled_message(transient_id) + self.remove_unhandled_message(transient_id) + + if self.link != None: self.link.teardown() + self.link = None + self.state = LXMPeer.IDLE + + rate_str = "" + if self.current_sync_transfer_started != None: + self.sync_transfer_rate = (resource.get_transfer_size()*8)/(time.time()-self.current_sync_transfer_started) + rate_str = f" at {RNS.prettyspeed(self.sync_transfer_rate)}" + + RNS.log(f"Syncing {len(self.currently_transferring_messages)} messages to peer {RNS.prettyhexrep(self.destination_hash)} completed{rate_str}", RNS.LOG_VERBOSE) + self.alive = True + self.last_heard = time.time() + self.offered += len(self.last_offer) + self.outgoing += len(self.currently_transferring_messages) + self.tx_bytes += resource.get_data_size() + + self.currently_transferring_messages = None + self.current_sync_transfer_started = None + + if self.sync_strategy == self.STRATEGY_PERSISTENT: + if self.unhandled_message_count > 0: self.sync() + + else: + RNS.log("Resource transfer for LXMF peer sync failed to "+str(self.destination), RNS.LOG_VERBOSE) + if self.link != None: self.link.teardown() + self.link = None + self.state = LXMPeer.IDLE + self.currently_transferring_messages = None + self.current_sync_transfer_started = None + + def link_established(self, link): + self.link.identify(self.router.identity) + link_establishment_rate = link.get_establishment_rate() + if link_establishment_rate != None: + self.link_establishment_rate = link_establishment_rate + + self.state = LXMPeer.LINK_READY + self.next_sync_attempt = 0 + self.sync() + + def link_closed(self, link): + self.link = None + self.state = LXMPeer.IDLE + + def queued_items(self): + return len(self.handled_messages_queue) > 0 or len(self.unhandled_messages_queue) > 0 + + def queue_unhandled_message(self, transient_id): + self.unhandled_messages_queue.append(transient_id) + + def queue_handled_message(self, transient_id): + self.handled_messages_queue.append(transient_id) + + def process_queues(self): + if len(self.unhandled_messages_queue) > 0 or len(self.handled_messages_queue) > 0: + handled_messages = self.handled_messages + unhandled_messages = self.unhandled_messages + + while len(self.handled_messages_queue) > 0: + transient_id = self.handled_messages_queue.pop() + if not transient_id in handled_messages: self.add_handled_message(transient_id) + if transient_id in unhandled_messages: self.remove_unhandled_message(transient_id) + + while len(self.unhandled_messages_queue) > 0: + transient_id = self.unhandled_messages_queue.pop() + if not transient_id in handled_messages and not transient_id in unhandled_messages: + self.add_unhandled_message(transient_id) + + del handled_messages, unhandled_messages + + @property + def handled_messages(self): + pes = self.router.propagation_entries.copy() + hm = list(filter(lambda tid: self.destination_hash in pes[tid][4], pes)) + self._hm_count = len(hm); del pes + self._hm_counts_synced = True + return hm + + @property + def unhandled_messages(self): + pes = self.router.propagation_entries.copy() + um = list(filter(lambda tid: self.destination_hash in pes[tid][5], pes)) + self._um_count = len(um); del pes + self._um_counts_synced = True + return um + + @property + def handled_message_count(self): + if not self._hm_counts_synced: self._update_counts() + return self._hm_count + + @property + def unhandled_message_count(self): + if not self._um_counts_synced: self._update_counts() + return self._um_count + + @property + def acceptance_rate(self): + return 0 if self.offered == 0 else (self.outgoing/self.offered) + + def _update_counts(self): + if not self._hm_counts_synced: + hm = self.handled_messages; del hm + + if not self._um_counts_synced: + um = self.unhandled_messages; del um + + def add_handled_message(self, transient_id): + if transient_id in self.router.propagation_entries: + if not self.destination_hash in self.router.propagation_entries[transient_id][4]: + self.router.propagation_entries[transient_id][4].append(self.destination_hash) + self._hm_counts_synced = False + + def add_unhandled_message(self, transient_id): + if transient_id in self.router.propagation_entries: + if not self.destination_hash in self.router.propagation_entries[transient_id][5]: + self.router.propagation_entries[transient_id][5].append(self.destination_hash) + self._um_count += 1 + + def remove_handled_message(self, transient_id): + if transient_id in self.router.propagation_entries: + if self.destination_hash in self.router.propagation_entries[transient_id][4]: + self.router.propagation_entries[transient_id][4].remove(self.destination_hash) + self._hm_counts_synced = False + + def remove_unhandled_message(self, transient_id): + if transient_id in self.router.propagation_entries: + if self.destination_hash in self.router.propagation_entries[transient_id][5]: + self.router.propagation_entries[transient_id][5].remove(self.destination_hash) + self._um_counts_synced = False + + @property + def name(self): + if type(self.metadata) != dict: return None + else: + if not PN_META_NAME in self.metadata: return None + else: + try: return self.metadata[PN_META_NAME].decode("utf-8") + except: return None + + def __str__(self): + if self.destination_hash: return RNS.prettyhexrep(self.destination_hash) + else: return "" \ No newline at end of file diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py new file mode 100644 index 0000000..aca6621 --- /dev/null +++ b/LXMF/LXMRouter.py @@ -0,0 +1,2732 @@ +import os +import sys +import time +import math +import random +import base64 +import atexit +import signal +import threading + +from collections import deque + +import RNS +import RNS.vendor.umsgpack as msgpack + +from .LXMF import APP_NAME +from .LXMF import FIELD_TICKET +from .LXMF import PN_META_NAME +from .LXMF import SF_COMPRESSION +from .LXMF import pn_announce_data_is_valid + +from .LXMPeer import LXMPeer +from .LXMessage import LXMessage +from .Handlers import LXMFDeliveryAnnounceHandler +from .Handlers import LXMFPropagationAnnounceHandler + +import LXMF.LXStamper as LXStamper + +class LXMRouter: + MAX_DELIVERY_ATTEMPTS = 5 + PROCESSING_INTERVAL = 4 + DELIVERY_RETRY_WAIT = 10 + PATH_REQUEST_WAIT = 7 + MAX_PATHLESS_TRIES = 1 + LINK_MAX_INACTIVITY = 10*60 + P_LINK_MAX_INACTIVITY = 3*60 + + MESSAGE_EXPIRY = 30*24*60*60 + STAMP_COST_EXPIRY = 45*24*60*60 + + NODE_ANNOUNCE_DELAY = 20 + + MAX_PEERS = 20 + AUTOPEER = True + AUTOPEER_MAXDEPTH = 4 + FASTEST_N_RANDOM_POOL = 2 + ROTATION_HEADROOM_PCT = 10 + ROTATION_AR_MAX = 0.5 + + PEERING_COST = 18 + MAX_PEERING_COST = 26 + PROPAGATION_COST_MIN = 13 + PROPAGATION_COST_FLEX = 3 + PROPAGATION_COST = 16 + PROPAGATION_LIMIT = 256 + SYNC_LIMIT = PROPAGATION_LIMIT*40 + DELIVERY_LIMIT = 1000 + + PR_PATH_TIMEOUT = 10 + PN_STAMP_THROTTLE = 180 + + PR_IDLE = 0x00 + PR_PATH_REQUESTED = 0x01 + PR_LINK_ESTABLISHING = 0x02 + PR_LINK_ESTABLISHED = 0x03 + PR_REQUEST_SENT = 0x04 + PR_RECEIVING = 0x05 + PR_RESPONSE_RECEIVED = 0x06 + PR_COMPLETE = 0x07 + PR_NO_PATH = 0xf0 + PR_LINK_FAILED = 0xf1 + PR_TRANSFER_FAILED = 0xf2 + PR_NO_IDENTITY_RCVD = 0xf3 + PR_NO_ACCESS = 0xf4 + PR_FAILED = 0xfe + + PR_ALL_MESSAGES = 0x00 + + DUPLICATE_SIGNAL = "lxmf_duplicate" + + STATS_GET_PATH = "/pn/get/stats" + SYNC_REQUEST_PATH = "/pn/peer/sync" + UNPEER_REQUEST_PATH = "/pn/peer/unpeer" + + + ### Developer-facing API ############################## + ####################################################### + + def __init__(self, identity=None, storagepath=None, autopeer=AUTOPEER, autopeer_maxdepth=None, + propagation_limit=PROPAGATION_LIMIT, delivery_limit=DELIVERY_LIMIT, sync_limit=SYNC_LIMIT, + enforce_ratchets=False, enforce_stamps=False, static_peers = [], max_peers=None, + from_static_only=False, sync_strategy=LXMPeer.STRATEGY_PERSISTENT, + propagation_cost=PROPAGATION_COST, propagation_cost_flexibility=PROPAGATION_COST_FLEX, + peering_cost=PEERING_COST, max_peering_cost=MAX_PEERING_COST, name=None): + + random.seed(os.urandom(10)) + + self.pending_inbound = [] + self.pending_outbound = [] + self.direct_links = {} + self.backchannel_links = {} + self.delivery_destinations = {} + + self.prioritised_list = [] + self.ignored_list = [] + self.allowed_list = [] + self.control_allowed_list = [] + self.auth_required = False + self.retain_synced_on_node = False + + self.default_sync_strategy = sync_strategy + self.processing_inbound = False + self.processing_count = 0 + self.name = name + + self.propagation_node = False + self.propagation_node_start_time = None + + if storagepath == None: raise ValueError("LXMF cannot be initialised without a storage path") + else: + self.storagepath = storagepath+"/lxmf" + self.ratchetpath = self.storagepath+"/ratchets" + + self.outbound_propagation_node = None + self.outbound_propagation_link = None + + if delivery_limit == None: delivery_limit = LXMRouter.DELIVERY_LIMIT + if propagation_cost < LXMRouter.PROPAGATION_COST_MIN: propagation_cost = LXMRouter.PROPAGATION_COST_MIN + + self.message_storage_limit = None + self.information_storage_limit = None + self.propagation_per_transfer_limit = propagation_limit + self.propagation_per_sync_limit = sync_limit + self.delivery_per_transfer_limit = delivery_limit + self.propagation_stamp_cost = propagation_cost + self.propagation_stamp_cost_flexibility = propagation_cost_flexibility + self.peering_cost = peering_cost + self.max_peering_cost = max_peering_cost + self.enforce_ratchets = enforce_ratchets + self._enforce_stamps = enforce_stamps + self.pending_deferred_stamps = {} + self.throttled_peers = {} + + if sync_limit == None or self.propagation_per_sync_limit < self.propagation_per_transfer_limit: + self.propagation_per_sync_limit = self.propagation_per_transfer_limit + + self.wants_download_on_path_available_from = None + self.wants_download_on_path_available_to = None + self.propagation_transfer_state = LXMRouter.PR_IDLE + self.propagation_transfer_progress = 0.0 + self.propagation_transfer_last_result = None + self.propagation_transfer_last_duplicates = None + self.propagation_transfer_max_messages = None + self.prioritise_rotating_unreachable_peers = False + self.active_propagation_links = [] + self.validated_peer_links = {} + self.locally_delivered_transient_ids = {} + self.locally_processed_transient_ids = {} + self.outbound_stamp_costs = {} + self.available_tickets = {"outbound": {}, "inbound": {}, "last_deliveries": {}} + + self.outbound_processing_lock = threading.Lock() + self.cost_file_lock = threading.Lock() + self.ticket_file_lock = threading.Lock() + self.stamp_gen_lock = threading.Lock() + self.exit_handler_running = False + + if identity == None: + identity = RNS.Identity() + + self.identity = identity + self.propagation_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation") + self.propagation_destination.set_default_app_data(self.get_propagation_node_app_data) + self.control_destination = None + self.client_propagation_messages_received = 0 + self.client_propagation_messages_served = 0 + self.unpeered_propagation_incoming = 0 + self.unpeered_propagation_rx_bytes = 0 + + if autopeer != None: self.autopeer = autopeer + else: self.autopeer = LXMRouter.AUTOPEER + + if autopeer_maxdepth != None: self.autopeer_maxdepth = autopeer_maxdepth + else: self.autopeer_maxdepth = LXMRouter.AUTOPEER_MAXDEPTH + + if max_peers == None: self.max_peers = LXMRouter.MAX_PEERS + else: + if type(max_peers) == int and max_peers >= 0: self.max_peers = max_peers + else: raise ValueError(f"Invalid value for max_peers: {max_peers}") + + self.from_static_only = from_static_only + if type(static_peers) != list: raise ValueError(f"Invalid type supplied for static peer list: {type(static_peers)}") + else: + for static_peer in static_peers: + if type(static_peer) != bytes: raise ValueError(f"Invalid static peer destination hash: {static_peer}") + else: + if len(static_peer) != RNS.Reticulum.TRUNCATED_HASHLENGTH//8: raise ValueError(f"Invalid static peer destination hash: {static_peer}") + + self.static_peers = static_peers + + self.peers = {} + self.propagation_entries = {} + + self.peer_distribution_queue = deque() + + RNS.Transport.register_announce_handler(LXMFDeliveryAnnounceHandler(self)) + RNS.Transport.register_announce_handler(LXMFPropagationAnnounceHandler(self)) + + self.__delivery_callback = None + + try: + if os.path.isfile(self.storagepath+"/local_deliveries"): + locally_delivered_file = open(self.storagepath+"/local_deliveries", "rb") + data = locally_delivered_file.read() + locally_delivered_file.close() + self.locally_delivered_transient_ids = msgpack.unpackb(data) + if not type(self.locally_delivered_transient_ids) == dict: + RNS.log("Invalid data format for loaded locally delivered transient IDs, recreating...", RNS.LOG_ERROR) + self.locally_delivered_transient_ids = {} + + except Exception as e: + RNS.log(f"Could not load locally delivered message ID cache from storage. The contained exception was: {e}", RNS.LOG_ERROR) + self.locally_delivered_transient_ids = {} + + try: + if os.path.isfile(self.storagepath+"/locally_processed"): + locally_processed_file = open(self.storagepath+"/locally_processed", "rb") + data = locally_processed_file.read() + locally_processed_file.close() + self.locally_processed_transient_ids = msgpack.unpackb(data) + if not type(self.locally_processed_transient_ids) == dict: + RNS.log("Invalid data format for loaded locally processed transient IDs, recreating...", RNS.LOG_ERROR) + self.locally_processed_transient_ids = {} + + + except Exception as e: + RNS.log("Could not load locally processed message ID cache from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + self.locally_processed_transient_ids = {} + + try: + self.clean_transient_id_caches() + + except Exception as e: + RNS.log("Could not clean transient ID caches. The contained exception was : "+str(e), RNS.LOG_ERROR) + self.locally_delivered_transient_ids = {} + self.locally_processed_transient_ids = {} + + try: + if os.path.isfile(self.storagepath+"/outbound_stamp_costs"): + with self.cost_file_lock: + with open(self.storagepath+"/outbound_stamp_costs", "rb") as outbound_stamp_cost_file: + data = outbound_stamp_cost_file.read() + self.outbound_stamp_costs = msgpack.unpackb(data) + if not type(self.outbound_stamp_costs) == dict: + RNS.log("Invalid data format for loaded outbound stamp costs, recreating...", RNS.LOG_ERROR) + self.outbound_stamp_costs = {} + + self.clean_outbound_stamp_costs() + self.save_outbound_stamp_costs() + + except Exception as e: + RNS.log("Could not load outbound stamp costs from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + + try: + if os.path.isfile(self.storagepath+"/available_tickets"): + with self.ticket_file_lock: + with open(self.storagepath+"/available_tickets", "rb") as available_tickets_file: + data = available_tickets_file.read() + self.available_tickets = msgpack.unpackb(data) + if not type(self.available_tickets) == dict: + RNS.log("Invalid data format for loaded available tickets, recreating...", RNS.LOG_ERROR) + self.available_tickets = {"outbound": {}, "inbound": {}, "last_deliveries": {}} + if not "outbound" in self.available_tickets: + RNS.log("Missing outbound entry in loaded available tickets, recreating...", RNS.LOG_ERROR) + self.available_tickets["outbound"] = {} + if not "inbound" in self.available_tickets: + RNS.log("Missing inbound entry in loaded available tickets, recreating...", RNS.LOG_ERROR) + self.available_tickets["inbound"] = {} + if not "last_deliveries" in self.available_tickets: + RNS.log("Missing local_deliveries entry in loaded available tickets, recreating...", RNS.LOG_ERROR) + self.available_tickets["last_deliveries"] = {} + + self.clean_available_tickets() + self.save_available_tickets() + + except Exception as e: + RNS.log("Could not load outbound stamp costs from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + + atexit.register(self.exit_handler) + signal.signal(signal.SIGINT, self.sigint_handler) + signal.signal(signal.SIGTERM, self.sigterm_handler) + + job_thread = threading.Thread(target=self.jobloop) + job_thread.setDaemon(True) + job_thread.start() + + def announce(self, destination_hash, attached_interface=None): + if destination_hash in self.delivery_destinations: + self.delivery_destinations[destination_hash].announce(app_data=self.get_announce_app_data(destination_hash), attached_interface=attached_interface) + + def get_propagation_node_announce_metadata(self): + metadata = {} + if self.name: metadata[PN_META_NAME] = str(self.name).encode("utf-8") + return metadata + + def get_propagation_node_app_data(self): + metadata = self.get_propagation_node_announce_metadata() + node_state = self.propagation_node and not self.from_static_only + stamp_cost = [self.propagation_stamp_cost, self.propagation_stamp_cost_flexibility, self.peering_cost] + announce_data = [ False, # 0: Legacy LXMF PN support + int(time.time()), # 1: Current node timebase + node_state, # 2: Boolean flag signalling propagation node state + self.propagation_per_transfer_limit, # 3: Per-transfer limit for message propagation in kilobytes + self.propagation_per_sync_limit, # 4: Limit for incoming propagation node syncs + stamp_cost, # 5: Propagation stamp cost for this node + metadata ] # 6: Node metadata + + return msgpack.packb(announce_data) + + def announce_propagation_node(self): + def delayed_announce(): + time.sleep(LXMRouter.NODE_ANNOUNCE_DELAY) + self.propagation_destination.announce(app_data=self.get_propagation_node_app_data()) + if len(self.control_allowed_list) > 1: self.control_destination.announce() + + da_thread = threading.Thread(target=delayed_announce) + da_thread.setDaemon(True) + da_thread.start() + + def register_delivery_identity(self, identity, display_name = None, stamp_cost = None): + if len(self.delivery_destinations) != 0: + RNS.log("Currently only one delivery identity is supported per LXMF router instance", RNS.LOG_ERROR) + return None + + if not os.path.isdir(self.ratchetpath): + os.makedirs(self.ratchetpath) + + delivery_destination = RNS.Destination(identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "delivery") + delivery_destination.enable_ratchets(f"{self.ratchetpath}/{RNS.hexrep(delivery_destination.hash, delimit=False)}.ratchets") + delivery_destination.set_packet_callback(self.delivery_packet) + delivery_destination.set_link_established_callback(self.delivery_link_established) + delivery_destination.display_name = display_name + + if self.enforce_ratchets: + delivery_destination.enforce_ratchets() + + if display_name != None: + def get_app_data(): + return self.get_announce_app_data(delivery_destination.hash) + delivery_destination.set_default_app_data(get_app_data) + + self.delivery_destinations[delivery_destination.hash] = delivery_destination + self.set_inbound_stamp_cost(delivery_destination.hash, stamp_cost) + + return delivery_destination + + def register_delivery_callback(self, callback): + self.__delivery_callback = callback + + def set_inbound_stamp_cost(self, destination_hash, stamp_cost): + if destination_hash in self.delivery_destinations: + delivery_destination = self.delivery_destinations[destination_hash] + if stamp_cost == None: + delivery_destination.stamp_cost = None + return True + elif type(stamp_cost) == int: + if stamp_cost < 1: + delivery_destination.stamp_cost = None + elif stamp_cost < 255: + delivery_destination.stamp_cost = stamp_cost + else: + return False + + return True + + return False + + def get_outbound_stamp_cost(self, destination_hash): + if destination_hash in self.outbound_stamp_costs: + stamp_cost = self.outbound_stamp_costs[destination_hash][1] + return stamp_cost + else: + return None + + def set_active_propagation_node(self, destination_hash): + self.set_outbound_propagation_node(destination_hash) + # self.set_inbound_propagation_node(destination_hash) + + def set_outbound_propagation_node(self, destination_hash): + if len(destination_hash) != RNS.Identity.TRUNCATED_HASHLENGTH//8 or type(destination_hash) != bytes: + raise ValueError("Invalid destination hash for outbound propagation node") + else: + if self.outbound_propagation_node != destination_hash: + self.outbound_propagation_node = destination_hash + if self.outbound_propagation_link != None: + if self.outbound_propagation_link.destination.hash != destination_hash: + self.outbound_propagation_link.teardown() + self.outbound_propagation_link = None + + def get_outbound_propagation_node(self): + return self.outbound_propagation_node + + def get_outbound_propagation_cost(self): + target_propagation_cost = None + pn_destination_hash = self.get_outbound_propagation_node() + pn_app_data = RNS.Identity.recall_app_data(pn_destination_hash) + if pn_announce_data_is_valid(pn_app_data): + pn_config = msgpack.unpackb(pn_app_data) + target_propagation_cost = pn_config[5][0] + + if not target_propagation_cost: + RNS.log(f"Could not retrieve cached propagation node config. Requesting path to propagation node to get target propagation cost...", RNS.LOG_DEBUG) + RNS.Transport.request_path(pn_destination_hash) + timeout = time.time() + LXMRouter.PATH_REQUEST_WAIT + while not RNS.Identity.recall_app_data(pn_destination_hash) and time.time() < timeout: + time.sleep(0.5) + + pn_app_data = RNS.Identity.recall_app_data(pn_destination_hash) + if pn_announce_data_is_valid(pn_app_data): + pn_config = msgpack.unpackb(pn_app_data) + target_propagation_cost = pn_config[5][0] + + if not target_propagation_cost: RNS.log("Propagation node stamp cost still unavailable after path request", RNS.LOG_ERROR) + return target_propagation_cost + + def set_inbound_propagation_node(self, destination_hash): + # TODO: Implement + raise NotImplementedError("Inbound/outbound propagation node differentiation is currently not implemented") + + def get_inbound_propagation_node(self): + return self.get_outbound_propagation_node() + + def set_retain_node_lxms(self, retain): + if retain == True: + self.retain_synced_on_node = True + else: + self.retain_synced_on_node = False + + def set_authentication(self, required=None): + if required != None: + self.auth_required = required + + def requires_authentication(self): + return self.auth_required + + def allow(self, identity_hash=None): + if isinstance(identity_hash, bytes) and len(identity_hash) == RNS.Identity.TRUNCATED_HASHLENGTH//8: + if not identity_hash in self.allowed_list: + self.allowed_list.append(identity_hash) + else: + raise ValueError("Allowed identity hash must be "+str(RNS.Identity.TRUNCATED_HASHLENGTH//8)+" bytes") + + def disallow(self, identity_hash=None): + if isinstance(identity_hash, bytes) and len(identity_hash) == RNS.Identity.TRUNCATED_HASHLENGTH//8: + if identity_hash in self.allowed_list: + self.allowed_list.pop(identity_hash) + else: + raise ValueError("Disallowed identity hash must be "+str(RNS.Identity.TRUNCATED_HASHLENGTH//8)+" bytes") + + def allow_control(self, identity_hash=None): + if isinstance(identity_hash, bytes) and len(identity_hash) == RNS.Identity.TRUNCATED_HASHLENGTH//8: + if not identity_hash in self.control_allowed_list: self.control_allowed_list.append(identity_hash) + else: raise ValueError("Allowed identity hash must be "+str(RNS.Identity.TRUNCATED_HASHLENGTH//8)+" bytes") + + def disallow_control(self, identity_hash=None): + if isinstance(identity_hash, bytes) and len(identity_hash) == RNS.Identity.TRUNCATED_HASHLENGTH//8: + if identity_hash in self.control_allowed_list: self.control_allowed_list.pop(identity_hash) + else: raise ValueError("Disallowed identity hash must be "+str(RNS.Identity.TRUNCATED_HASHLENGTH//8)+" bytes") + + def prioritise(self, destination_hash=None): + if isinstance(destination_hash, bytes) and len(destination_hash) == RNS.Reticulum.TRUNCATED_HASHLENGTH//8: + if not destination_hash in self.prioritised_list: + self.prioritised_list.append(destination_hash) + else: + raise ValueError("Prioritised destination hash must be "+str(RNS.Reticulum.TRUNCATED_HASHLENGTH//8)+" bytes") + + def unprioritise(self, identity_hash=None): + if isinstance(destination_hash, bytes) and len(destination_hash) == RNS.Reticulum.TRUNCATED_HASHLENGTH//8: + if destination_hash in self.prioritised_list: + self.prioritised_list.pop(destination_hash) + else: + raise ValueError("Prioritised destination hash must be "+str(RNS.Reticulum.TRUNCATED_HASHLENGTH//8)+" bytes") + + def request_messages_from_propagation_node(self, identity, max_messages = PR_ALL_MESSAGES): + if max_messages == None: + max_messages = LXMRouter.PR_ALL_MESSAGES + + self.propagation_transfer_progress = 0.0 + self.propagation_transfer_max_messages = max_messages + if self.outbound_propagation_node != None: + if self.outbound_propagation_link != None and self.outbound_propagation_link.status == RNS.Link.ACTIVE: + self.propagation_transfer_state = LXMRouter.PR_LINK_ESTABLISHED + RNS.log("Requesting message list from propagation node", RNS.LOG_DEBUG) + self.outbound_propagation_link.identify(identity) + self.outbound_propagation_link.request( + LXMPeer.MESSAGE_GET_PATH, + [None, None], # Set both want and have fields to None to get message list + response_callback=self.message_list_response, + failed_callback=self.message_get_failed + ) + self.propagation_transfer_state = LXMRouter.PR_REQUEST_SENT + else: + if self.outbound_propagation_link == None: + if RNS.Transport.has_path(self.outbound_propagation_node): + self.wants_download_on_path_available_from = None + self.propagation_transfer_state = LXMRouter.PR_LINK_ESTABLISHING + RNS.log("Establishing link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" for message download", RNS.LOG_DEBUG) + propagation_node_identity = RNS.Identity.recall(self.outbound_propagation_node) + propagation_node_destination = RNS.Destination(propagation_node_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + def msg_request_established_callback(link): + self.request_messages_from_propagation_node(identity, self.propagation_transfer_max_messages) + + self.outbound_propagation_link = RNS.Link(propagation_node_destination, established_callback=msg_request_established_callback) + else: + RNS.log("No path known for message download from propagation node "+RNS.prettyhexrep(self.outbound_propagation_node)+". Requesting path...", RNS.LOG_DEBUG) + RNS.Transport.request_path(self.outbound_propagation_node) + self.wants_download_on_path_available_from = self.outbound_propagation_node + self.wants_download_on_path_available_to = identity + self.wants_download_on_path_available_timeout = time.time() + LXMRouter.PR_PATH_TIMEOUT + self.propagation_transfer_state = LXMRouter.PR_PATH_REQUESTED + self.request_messages_path_job() + else: + RNS.log("Waiting for propagation node link to become active", RNS.LOG_EXTREME) + else: + RNS.log("Cannot request LXMF propagation node sync, no default propagation node configured", RNS.LOG_WARNING) + + def cancel_propagation_node_requests(self): + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + self.outbound_propagation_link = None + + self.acknowledge_sync_completion(reset_state=True) + + def enable_propagation(self): + try: + self.messagepath = self.storagepath+"/messagestore" + + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) + + if not os.path.isdir(self.messagepath): + os.makedirs(self.messagepath) + + self.propagation_entries = {} + + st = time.time(); RNS.log("Indexing messagestore...", RNS.LOG_NOTICE) + for filename in os.listdir(self.messagepath): + components = filename.split("_") + if len(components) >= 3: + if float(components[1]) > 0: + if len(components[0]) == RNS.Identity.HASHLENGTH//8*2: + try: + transient_id = bytes.fromhex(components[0]) + received = float(components[1]) + stamp_value = int(components[2]) + filepath = self.messagepath+"/"+filename + msg_size = os.path.getsize(filepath) + file = open(filepath, "rb") + destination_hash = file.read(LXMessage.DESTINATION_LENGTH) + file.close() + + self.propagation_entries[transient_id] = [ + destination_hash, # 0: Destination hash + filepath, # 1: Storage location + received, # 2: Receive timestamp + msg_size, # 3: Message size + [], # 4: Handled peers + [], # 5: Unhandled peers + stamp_value, # 6: Stamp value + ] + + except Exception as e: + RNS.log("Could not read LXM from message store. The contained exception was: "+str(e), RNS.LOG_ERROR) + + et = time.time(); mps = 0 if et-st == 0 else math.floor(len(self.propagation_entries)/(et-st)) + RNS.log(f"Indexed {len(self.propagation_entries)} messages in {RNS.prettytime(et-st)}, {mps} msgs/s", RNS.LOG_NOTICE) + RNS.log("Rebuilding peer synchronisation states...", RNS.LOG_NOTICE) + st = time.time() + + peers_storage_path = self.storagepath+"/peers" + if os.path.isfile(peers_storage_path): + peers_file = open(peers_storage_path, "rb") + peers_data = peers_file.read() + peers_file.close() + + if len(peers_data) > 0: + try: serialised_peers = msgpack.unpackb(peers_data) + except Exception as e: + RNS.log(f"Could not load propagation node peering data from storage. The contained exception was: {e}", RNS.LOG_ERROR) + RNS.log(f"The peering data file located at {peers_storage_path} is likely corrupt.", RNS.LOG_ERROR) + RNS.log(f"You can delete this file and attempt starting the propagation node again, but peer synchronization states will be lost.", RNS.LOG_ERROR) + raise e + + del peers_data + + while len(serialised_peers) > 0: + serialised_peer = serialised_peers.pop() + peer = LXMPeer.from_bytes(serialised_peer, self) + del serialised_peer + if peer.destination_hash in self.static_peers and peer.last_heard == 0: + RNS.Transport.request_path(peer.destination_hash) + if peer.identity != None: + self.peers[peer.destination_hash] = peer + lim_str = ", no transfer limit" + if peer.propagation_transfer_limit != None: + lim_str = ", "+RNS.prettysize(peer.propagation_transfer_limit*1000)+" transfer limit" + RNS.log("Rebuilt peer "+RNS.prettyhexrep(peer.destination_hash)+" with "+str(peer.unhandled_message_count)+" unhandled messages"+lim_str, RNS.LOG_DEBUG) + else: + RNS.log("Peer "+RNS.prettyhexrep(peer.destination_hash)+" could not be loaded, because its identity could not be recalled. Dropping peer.", RNS.LOG_DEBUG) + del peer + + del serialised_peers + + if len(self.static_peers) > 0: + for static_peer in self.static_peers: + if not static_peer in self.peers: + RNS.log(f"Activating static peering with {RNS.prettyhexrep(static_peer)}", RNS.LOG_NOTICE) + self.peers[static_peer] = LXMPeer(self, static_peer, sync_strategy=self.default_sync_strategy) + if self.peers[static_peer].last_heard == 0: + # TODO: Allow path request responses through announce handler + # momentarily here, so peering config can be updated even if + # the static peer is not available to directly send an announce. + RNS.Transport.request_path(static_peer) + + RNS.log(f"Rebuilt synchronisation state for {len(self.peers)} peers in {RNS.prettytime(time.time()-st)}", RNS.LOG_NOTICE) + + try: + if os.path.isfile(self.storagepath+"/node_stats"): + node_stats_file = open(self.storagepath+"/node_stats", "rb") + data = node_stats_file.read() + node_stats_file.close() + node_stats = msgpack.unpackb(data) + + if not type(node_stats) == dict: + RNS.log("Invalid data format for loaded local node stats, node stats will be reset", RNS.LOG_ERROR) + else: + self.client_propagation_messages_received = node_stats["client_propagation_messages_received"] + self.client_propagation_messages_served = node_stats["client_propagation_messages_served"] + self.unpeered_propagation_incoming = node_stats["unpeered_propagation_incoming"] + self.unpeered_propagation_rx_bytes = node_stats["unpeered_propagation_rx_bytes"] + + except Exception as e: + RNS.log("Could not load local node stats. The contained exception was: "+str(e), RNS.LOG_ERROR) + + self.propagation_node = True + self.propagation_node_start_time = time.time() + self.propagation_destination.set_link_established_callback(self.propagation_link_established) + self.propagation_destination.set_packet_callback(self.propagation_packet) + + self.propagation_destination.register_request_handler(LXMPeer.OFFER_REQUEST_PATH, self.offer_request, allow = RNS.Destination.ALLOW_ALL) + self.propagation_destination.register_request_handler(LXMPeer.MESSAGE_GET_PATH, self.message_get_request, allow = RNS.Destination.ALLOW_ALL) + + self.control_allowed_list = [self.identity.hash] + self.control_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") + self.control_destination.register_request_handler(LXMRouter.STATS_GET_PATH, self.stats_get_request, allow = RNS.Destination.ALLOW_LIST, allowed_list=self.control_allowed_list) + self.control_destination.register_request_handler(LXMRouter.SYNC_REQUEST_PATH, self.peer_sync_request, allow = RNS.Destination.ALLOW_LIST, allowed_list=self.control_allowed_list) + self.control_destination.register_request_handler(LXMRouter.UNPEER_REQUEST_PATH, self.peer_unpeer_request, allow = RNS.Destination.ALLOW_LIST, allowed_list=self.control_allowed_list) + + if self.message_storage_limit != None: + limit_str = ", limit is "+RNS.prettysize(self.message_storage_limit) + else: + limit_str = "" + + RNS.log("LXMF Propagation Node message store size is "+RNS.prettysize(self.message_storage_size())+limit_str, RNS.LOG_DEBUG) + + self.announce_propagation_node() + + except Exception as e: + RNS.log("Could not enable propagation node. The contained exception was: "+str(e), RNS.LOG_ERROR) + raise e + RNS.panic() + + def disable_propagation(self): + self.propagation_node = False + self.announce_propagation_node() + + def enforce_stamps(self): + self._enforce_stamps = True + + def ignore_stamps(self): + self._enforce_stamps = False + + def ignore_destination(self, destination_hash): + if not destination_hash in self.ignored_list: + self.ignored_list.append(destination_hash) + + def unignore_destination(self, destination_hash): + if destination_hash in self.ignored_list: + self.ignored_list.remove(destination_hash) + + def set_message_storage_limit(self, kilobytes = None, megabytes = None, gigabytes = None): + limit_bytes = 0 + + if kilobytes != None: + limit_bytes += kilobytes*1000 + + if megabytes != None: + limit_bytes += megabytes*1000*1000 + + if gigabytes != None: + limit_bytes += gigabytes*1000*1000*1000 + + if limit_bytes == 0: + limit_bytes = None + + try: + if limit_bytes == None or int(limit_bytes) > 0: + self.message_storage_limit = int(limit_bytes) + else: + raise ValueError("Cannot set LXMF information storage limit to "+str(limit_bytes)) + + except Exception as e: + raise ValueError("Cannot set LXMF information storage limit to "+str(limit_bytes)) + + def message_storage_limit(self): + return self.message_storage_limit + + def message_storage_size(self): + if self.propagation_node: + return sum(self.propagation_entries[f][3] for f in self.propagation_entries) + else: + return None + + def set_information_storage_limit(self, kilobytes = None, megabytes = None, gigabytes = None): + limit_bytes = 0 + if kilobytes != None: limit_bytes += kilobytes*1000 + if megabytes != None: limit_bytes += megabytes*1000*1000 + if gigabytes != None: limit_bytes += gigabytes*1000*1000*1000 + if limit_bytes == 0: limit_bytes = None + + try: + if limit_bytes == None or int(limit_bytes) > 0: self.information_storage_limit = int(limit_bytes) + else: raise ValueError("Cannot set LXMF information storage limit to "+str(limit_bytes)) + except Exception as e: raise ValueError("Cannot set LXMF information storage limit to "+str(limit_bytes)) + + def information_storage_limit(self): + return self.information_storage_limit + + def information_storage_size(self): + pass + + def delivery_link_available(self, destination_hash): + if destination_hash in self.direct_links or destination_hash in self.backchannel_links: return True + else: return False + + + ### Propagation Node Control ########################## + ####################################################### + + def compile_stats(self): + if not self.propagation_node: return None + else: + peer_stats = {} + for peer_id in self.peers.copy(): + peer = self.peers[peer_id] + peer_stats[peer_id] = { + "type": "static" if peer_id in self.static_peers else "discovered", + "state": peer.state, + "alive": peer.alive, + "name": peer.name, + "last_heard": int(peer.last_heard), + "next_sync_attempt": peer.next_sync_attempt, + "last_sync_attempt": peer.last_sync_attempt, + "sync_backoff": peer.sync_backoff, + "peering_timebase": peer.peering_timebase, + "ler": int(peer.link_establishment_rate), + "str": int(peer.sync_transfer_rate), + "transfer_limit": peer.propagation_transfer_limit, + "sync_limit": peer.propagation_sync_limit, + "target_stamp_cost": peer.propagation_stamp_cost, + "stamp_cost_flexibility": peer.propagation_stamp_cost_flexibility, + "peering_cost": peer.peering_cost, + "peering_key": peer.peering_key_value(), + "network_distance": RNS.Transport.hops_to(peer_id), + "rx_bytes": peer.rx_bytes, + "tx_bytes": peer.tx_bytes, + "acceptance_rate": peer.acceptance_rate, + "messages": { + "offered": peer.offered, + "outgoing": peer.outgoing, + "incoming": peer.incoming, + "unhandled": peer.unhandled_message_count + }, + } + + node_stats = { + "identity_hash": self.identity.hash, + "destination_hash": self.propagation_destination.hash, + "uptime": time.time()-self.propagation_node_start_time, + "delivery_limit": self.delivery_per_transfer_limit, + "propagation_limit": self.propagation_per_transfer_limit, + "sync_limit": self.propagation_per_sync_limit, + "target_stamp_cost": self.propagation_stamp_cost, + "stamp_cost_flexibility": self.propagation_stamp_cost_flexibility, + "peering_cost": self.peering_cost, + "max_peering_cost": self.max_peering_cost, + "autopeer_maxdepth": self.autopeer_maxdepth, + "from_static_only": self.from_static_only, + "messagestore": { + "count": len(self.propagation_entries), + "bytes": self.message_storage_size(), + "limit": self.message_storage_limit, + }, + "clients" : { + "client_propagation_messages_received": self.client_propagation_messages_received, + "client_propagation_messages_served": self.client_propagation_messages_served, + }, + "unpeered_propagation_incoming": self.unpeered_propagation_incoming, + "unpeered_propagation_rx_bytes": self.unpeered_propagation_rx_bytes, + "static_peers": len(self.static_peers), + "discovered_peers": len(self.peers)-len(self.static_peers), + "total_peers": len(self.peers), + "max_peers": self.max_peers, + "peers": peer_stats, + } + + return node_stats + + def stats_get_request(self, path, data, request_id, remote_identity, requested_at): + if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY + elif remote_identity.hash not in self.control_allowed_list: return LXMPeer.ERROR_NO_ACCESS + else: return self.compile_stats() + + def peer_sync_request(self, path, data, request_id, remote_identity, requested_at): + if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY + elif remote_identity.hash not in self.control_allowed_list: return LXMPeer.ERROR_NO_ACCESS + else: + if type(data) != bytes: return LXMPeer.ERROR_INVALID_DATA + elif len(data) != RNS.Identity.TRUNCATED_HASHLENGTH//8: return LXMPeer.ERROR_INVALID_DATA + else: + if not data in self.peers: return LXMPeer.ERROR_NOT_FOUND + else: + self.peers[data].sync() + return True + + def peer_unpeer_request(self, path, data, request_id, remote_identity, requested_at): + if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY + elif remote_identity.hash not in self.control_allowed_list: return LXMPeer.ERROR_NO_ACCESS + else: + if type(data) != bytes: return LXMPeer.ERROR_INVALID_DATA + elif len(data) != RNS.Identity.TRUNCATED_HASHLENGTH//8: return LXMPeer.ERROR_INVALID_DATA + else: + if not data in self.peers: return LXMPeer.ERROR_NOT_FOUND + else: + self.unpeer(data) + return True + + + ### Utility & Maintenance ############################# + ####################################################### + + JOB_OUTBOUND_INTERVAL = 1 + JOB_STAMPS_INTERVAL = 1 + JOB_LINKS_INTERVAL = 1 + JOB_TRANSIENT_INTERVAL = 60 + JOB_STORE_INTERVAL = 120 + JOB_PEERSYNC_INTERVAL = 6 + JOB_PEERINGEST_INTERVAL= JOB_PEERSYNC_INTERVAL + JOB_ROTATE_INTERVAL = 56*JOB_PEERINGEST_INTERVAL + def jobs(self): + if not self.exit_handler_running: + self.processing_count += 1 + + if self.processing_count % LXMRouter.JOB_OUTBOUND_INTERVAL == 0: + self.process_outbound() + + if self.processing_count % LXMRouter.JOB_STAMPS_INTERVAL == 0: + threading.Thread(target=self.process_deferred_stamps, daemon=True).start() + + if self.processing_count % LXMRouter.JOB_LINKS_INTERVAL == 0: + self.clean_links() + + if self.processing_count % LXMRouter.JOB_TRANSIENT_INTERVAL == 0: + self.clean_transient_id_caches() + + if self.processing_count % LXMRouter.JOB_STORE_INTERVAL == 0: + if self.propagation_node == True: self.clean_message_store() + + if self.processing_count % LXMRouter.JOB_PEERINGEST_INTERVAL == 0: + if self.propagation_node == True: self.flush_queues() + + if self.processing_count % LXMRouter.JOB_ROTATE_INTERVAL == 0: + if self.propagation_node == True: self.rotate_peers() + + if self.processing_count % LXMRouter.JOB_PEERSYNC_INTERVAL == 0: + if self.propagation_node == True: self.sync_peers() + self.clean_throttled_peers() + + def jobloop(self): + while (True): + # TODO: Improve this to scheduling, so manual + # triggers can delay next run + try: + self.jobs() + except Exception as e: + RNS.log("An error ocurred while running LXMF Router jobs.", RNS.LOG_ERROR) + RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) + RNS.trace_exception(e) + time.sleep(LXMRouter.PROCESSING_INTERVAL) + + def flush_queues(self): + if len(self.peers) > 0: + self.flush_peer_distribution_queue() + RNS.log("Calculating peer distribution queue mappings...", RNS.LOG_DEBUG); st = time.time() + for peer_id in self.peers.copy(): + if peer_id in self.peers: + peer = self.peers[peer_id] + if peer.queued_items(): + peer.process_queues() + + RNS.log(f"Distribution queue mapping completed in {RNS.prettytime(time.time()-st)}", RNS.LOG_DEBUG) + + def clean_links(self): + closed_links = [] + for link_hash in self.direct_links: + link = self.direct_links[link_hash] + inactive_time = link.no_data_for() + + if inactive_time > LXMRouter.LINK_MAX_INACTIVITY: + link.teardown() + closed_links.append(link_hash) + if link.link_id in self.validated_peer_links: + self.validated_peer_links.pop(link.link_id) + + for link_hash in closed_links: + cleaned_link = self.direct_links.pop(link_hash) + RNS.log("Cleaned link "+str(cleaned_link), RNS.LOG_DEBUG) + + try: + inactive_links = [] + for link in self.active_propagation_links: + if link.no_data_for() > LXMRouter.P_LINK_MAX_INACTIVITY: + inactive_links.append(link) + + for link in inactive_links: + self.active_propagation_links.remove(link) + link.teardown() + + except Exception as e: + RNS.log("An error occurred while cleaning inbound propagation links. The contained exception was: "+str(e), RNS.LOG_ERROR) + + if self.outbound_propagation_link != None and self.outbound_propagation_link.status == RNS.Link.CLOSED: + self.outbound_propagation_link = None + if self.propagation_transfer_state == LXMRouter.PR_COMPLETE: + self.acknowledge_sync_completion() + elif self.propagation_transfer_state < LXMRouter.PR_LINK_ESTABLISHED: + self.acknowledge_sync_completion(failure_state=LXMRouter.PR_LINK_FAILED) + elif self.propagation_transfer_state >= LXMRouter.PR_LINK_ESTABLISHED and self.propagation_transfer_state < LXMRouter.PR_COMPLETE: + self.acknowledge_sync_completion(failure_state=LXMRouter.PR_TRANSFER_FAILED) + else: + RNS.log(f"Unknown propagation transfer state on link cleaning: {self.propagation_transfer_state}", RNS.LOG_DEBUG) + self.acknowledge_sync_completion() + + RNS.log("Cleaned outbound propagation link", RNS.LOG_DEBUG) + + def clean_transient_id_caches(self): + now = time.time() + removed_entries = [] + for transient_id in self.locally_delivered_transient_ids: + timestamp = self.locally_delivered_transient_ids[transient_id] + if now > timestamp+LXMRouter.MESSAGE_EXPIRY*6.0: + removed_entries.append(transient_id) + + for transient_id in removed_entries: + self.locally_delivered_transient_ids.pop(transient_id) + RNS.log("Cleaned "+RNS.prettyhexrep(transient_id)+" from local delivery cache", RNS.LOG_DEBUG) + + removed_entries = [] + for transient_id in self.locally_processed_transient_ids: + timestamp = self.locally_processed_transient_ids[transient_id] + if now > timestamp+LXMRouter.MESSAGE_EXPIRY*6.0: + removed_entries.append(transient_id) + + for transient_id in removed_entries: + self.locally_processed_transient_ids.pop(transient_id) + RNS.log("Cleaned "+RNS.prettyhexrep(transient_id)+" from locally processed cache", RNS.LOG_DEBUG) + + def update_stamp_cost(self, destination_hash, stamp_cost): + RNS.log(f"Updating outbound stamp cost for {RNS.prettyhexrep(destination_hash)} to {stamp_cost}", RNS.LOG_DEBUG) + self.outbound_stamp_costs[destination_hash] = [time.time(), stamp_cost] + + def job(): self.save_outbound_stamp_costs() + threading.Thread(target=self.save_outbound_stamp_costs, daemon=True).start() + + def get_announce_app_data(self, destination_hash): + if destination_hash in self.delivery_destinations: + delivery_destination = self.delivery_destinations[destination_hash] + + display_name = None + if delivery_destination.display_name != None: + display_name = delivery_destination.display_name.encode("utf-8") + + stamp_cost = None + if delivery_destination.stamp_cost != None and type(delivery_destination.stamp_cost) == int: + if delivery_destination.stamp_cost > 0 and delivery_destination.stamp_cost < 255: + stamp_cost = delivery_destination.stamp_cost + + supported_functionality = [SF_COMPRESSION] + peer_data = [display_name, stamp_cost, supported_functionality] + + return msgpack.packb(peer_data) + + def get_size(self, transient_id): + lxm_size = self.propagation_entries[transient_id][3] + return lxm_size + + def get_weight(self, transient_id): + dst_hash = self.propagation_entries[transient_id][0] + lxm_rcvd = self.propagation_entries[transient_id][2] + lxm_size = self.propagation_entries[transient_id][3] + + now = time.time() + age_weight = max(1, (now - lxm_rcvd)/60/60/24/4) + + if dst_hash in self.prioritised_list: priority_weight = 0.1 + else: priority_weight = 1.0 + + return priority_weight * age_weight * lxm_size + + def get_stamp_value(self, transient_id): + if not transient_id in self.propagation_entries: return None + else: return self.propagation_entries[transient_id][6] + + def generate_ticket(self, destination_hash, expiry=LXMessage.TICKET_EXPIRY): + now = time.time() + ticket = None + if destination_hash in self.available_tickets["last_deliveries"]: + last_delivery = self.available_tickets["last_deliveries"][destination_hash] + elapsed = now - last_delivery + if elapsed < LXMessage.TICKET_INTERVAL: + RNS.log(f"A ticket for {RNS.prettyhexrep(destination_hash)} was already delivered {RNS.prettytime(elapsed)} ago, not including another ticket yet", RNS.LOG_DEBUG) + return None + + if destination_hash in self.available_tickets["inbound"]: + for ticket in self.available_tickets["inbound"][destination_hash]: + ticket_entry = self.available_tickets["inbound"][destination_hash][ticket] + expires = ticket_entry[0]; validity_left = expires - now + if validity_left > LXMessage.TICKET_RENEW: + RNS.log(f"Found generated ticket for {RNS.prettyhexrep(destination_hash)} with {RNS.prettytime(validity_left)} of validity left, re-using this one", RNS.LOG_DEBUG) + return [expires, ticket] + + else: + self.available_tickets["inbound"][destination_hash] = {} + + RNS.log(f"No generated tickets for {RNS.prettyhexrep(destination_hash)} with enough validity found, generating a new one", RNS.LOG_DEBUG) + expires = now+expiry + ticket = os.urandom(LXMessage.TICKET_LENGTH) + self.available_tickets["inbound"][destination_hash][ticket] = [expires] + self.save_available_tickets() + + return [expires, ticket] + + def remember_ticket(self, destination_hash, ticket_entry): + expires = ticket_entry[0]-time.time() + RNS.log(f"Remembering ticket for {RNS.prettyhexrep(destination_hash)}, expires in {RNS.prettytime(expires)}", RNS.LOG_DEBUG) + self.available_tickets["outbound"][destination_hash] = [ticket_entry[0], ticket_entry[1]] + + def get_outbound_ticket(self, destination_hash): + if destination_hash in self.available_tickets["outbound"]: + entry = self.available_tickets["outbound"][destination_hash] + if entry[0] > time.time(): + return entry[1] + + return None + + def get_outbound_ticket_expiry(self, destination_hash): + if destination_hash in self.available_tickets["outbound"]: + entry = self.available_tickets["outbound"][destination_hash] + if entry[0] > time.time(): + return entry[0] + + return None + + def get_inbound_tickets(self, destination_hash): + now = time.time() + available_tickets = [] + if destination_hash in self.available_tickets["inbound"]: + for inbound_ticket in self.available_tickets["inbound"][destination_hash]: + if now < self.available_tickets["inbound"][destination_hash][inbound_ticket][0]: + available_tickets.append(inbound_ticket) + + if len(available_tickets) == 0: + return None + else: + return available_tickets + + def clean_throttled_peers(self): + expired_entries = [] + now = time.time() + for peer_hash in self.throttled_peers: + if now > self.throttled_peers[peer_hash]: expired_entries.append(peer_hash) + + for peer_hash in expired_entries: self.throttled_peers.pop(peer_hash) + + def clean_message_store(self): + RNS.log("Cleaning message store", RNS.LOG_VERBOSE) + # Check and remove expired messages + now = time.time() + removed_entries = {} + for transient_id in self.propagation_entries.copy(): + entry = self.propagation_entries[transient_id] + filepath = entry[1] + stamp_value = entry[6] + filename = os.path.split(filepath)[-1] + components = filename.split("_") + + if len(components) == 3 and float(components[1]) > 0 and len(os.path.split(components[0])[1]) == (RNS.Identity.HASHLENGTH//8)*2 and int(components[2]) == stamp_value: + timestamp = float(components[1]) + if now > timestamp+LXMRouter.MESSAGE_EXPIRY: + RNS.log("Purging message "+RNS.prettyhexrep(transient_id)+" due to expiry", RNS.LOG_EXTREME) + removed_entries[transient_id] = filepath + else: + RNS.log("Purging message "+RNS.prettyhexrep(transient_id)+" due to invalid file path", RNS.LOG_WARNING) + removed_entries[transient_id] = filepath + + removed_count = 0 + for transient_id in removed_entries: + try: + filepath = removed_entries[transient_id] + self.propagation_entries.pop(transient_id) + if os.path.isfile(filepath): + os.unlink(filepath) + removed_count += 1 + except Exception as e: + RNS.log("Could not remove "+RNS.prettyhexrep(transient_id)+" from message store. The contained exception was: "+str(e), RNS.LOG_ERROR) + + if removed_count > 0: + RNS.log("Cleaned "+str(removed_count)+" entries from the message store", RNS.LOG_VERBOSE) + + # Check size of message store and cull if needed + try: + message_storage_size = self.message_storage_size() + if message_storage_size != None: + if self.message_storage_limit != None and message_storage_size > self.message_storage_limit: + # Clean the message storage according to priorities + bytes_needed = message_storage_size - self.message_storage_limit + bytes_cleaned = 0 + + weighted_entries = [] + for transient_id in self.propagation_entries.copy(): + weighted_entries.append([ + self.propagation_entries[transient_id], + self.get_weight(transient_id), + transient_id + ]) + + weighted_entries.sort(key=lambda we: we[1], reverse=True) + + i = 0 + while i < len(weighted_entries) and bytes_cleaned < bytes_needed: + try: + w = weighted_entries[i] + entry = w[0] + transient_id = w[2] + filepath = entry[1] + + if os.path.isfile(filepath): + os.unlink(filepath) + + self.propagation_entries.pop(transient_id) + bytes_cleaned += entry[3] + + RNS.log("Removed "+RNS.prettyhexrep(transient_id)+" with weight "+str(w[1])+" to clear up "+RNS.prettysize(entry[3])+", now cleaned "+RNS.prettysize(bytes_cleaned)+" out of "+RNS.prettysize(bytes_needed)+" needed", RNS.LOG_EXTREME) + + except Exception as e: + RNS.log("Error while cleaning LXMF message from message store. The contained exception was: "+str(e), RNS.LOG_ERROR) + + finally: + i += 1 + + RNS.log("LXMF message store size is now "+RNS.prettysize(self.message_storage_size())+" for "+str(len(self.propagation_entries))+" items", RNS.LOG_EXTREME) + + + except Exception as e: + RNS.log("Could not clean the LXMF message store. The contained exception was: "+str(e), RNS.LOG_ERROR) + + def save_locally_delivered_transient_ids(self): + try: + if len(self.locally_delivered_transient_ids) > 0: + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) + + with open(self.storagepath+"/local_deliveries", "wb") as locally_delivered_file: + locally_delivered_file.write(msgpack.packb(self.locally_delivered_transient_ids)) + + except Exception as e: + RNS.log("Could not save locally delivered message ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + + def save_locally_processed_transient_ids(self): + try: + if len(self.locally_processed_transient_ids) > 0: + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) + + with open(self.storagepath+"/locally_processed", "wb") as locally_processed_file: + locally_processed_file.write(msgpack.packb(self.locally_processed_transient_ids)) + + except Exception as e: + RNS.log("Could not save locally processed transient ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + + def save_node_stats(self): + try: + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) + + with open(self.storagepath+"/node_stats", "wb") as stats_file: + node_stats = { + "client_propagation_messages_received": self.client_propagation_messages_received, + "client_propagation_messages_served": self.client_propagation_messages_served, + "unpeered_propagation_incoming": self.unpeered_propagation_incoming, + "unpeered_propagation_rx_bytes": self.unpeered_propagation_rx_bytes, + } + stats_file.write(msgpack.packb(node_stats)) + + except Exception as e: + RNS.log("Could not save local node stats to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + + + def clean_outbound_stamp_costs(self): + try: + expired = [] + for destination_hash in self.outbound_stamp_costs: + entry = self.outbound_stamp_costs[destination_hash] + if time.time() > entry[0] + LXMRouter.STAMP_COST_EXPIRY: + expired.append(destination_hash) + + for destination_hash in expired: + self.outbound_stamp_costs.pop(destination_hash) + + except Exception as e: + RNS.log(f"Error while cleaning outbound stamp costs. The contained exception was: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + def save_outbound_stamp_costs(self): + with self.cost_file_lock: + try: + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) + + outbound_stamp_costs_file = open(self.storagepath+"/outbound_stamp_costs", "wb") + outbound_stamp_costs_file.write(msgpack.packb(self.outbound_stamp_costs.copy())) + outbound_stamp_costs_file.close() + + except Exception as e: + RNS.log("Could not save outbound stamp costs to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + + def clean_available_tickets(self): + try: + # Clean outbound tickets + expired_outbound = [] + for destination_hash in self.available_tickets["outbound"]: + entry = self.available_tickets["outbound"][destination_hash] + if time.time() > entry[0]: + expired_outbound.append(destination_hash) + + for destination_hash in expired_outbound: + self.available_tickets["outbound"].pop(destination_hash) + + # Clean inbound tickets + for destination_hash in self.available_tickets["inbound"]: + expired_inbound = [] + for inbound_ticket in self.available_tickets["inbound"][destination_hash]: + entry = self.available_tickets["inbound"][destination_hash][inbound_ticket] + ticket_expiry = entry[0] + if time.time() > ticket_expiry+LXMessage.TICKET_GRACE: + expired_inbound.append(inbound_ticket) + + for inbound_ticket in expired_inbound: + self.available_tickets["inbound"][destination_hash].pop(inbound_ticket) + + except Exception as e: + RNS.log(f"Error while cleaning available tickets. The contained exception was: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + def save_available_tickets(self): + with self.ticket_file_lock: + try: + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) + + available_tickets_file = open(self.storagepath+"/available_tickets", "wb") + available_tickets_file.write(msgpack.packb(self.available_tickets)) + available_tickets_file.close() + + except Exception as e: + RNS.log("Could not save available tickets to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + + def reload_available_tickets(self): + RNS.log("Reloading available tickets from storage", RNS.LOG_DEBUG) + try: + with self.ticket_file_lock: + with open(self.storagepath+"/available_tickets", "rb") as available_tickets_file: + data = available_tickets_file.read() + self.available_tickets = msgpack.unpackb(data) + if not type(self.available_tickets) == dict: + RNS.log("Invalid data format for loaded available tickets, recreating...", RNS.LOG_ERROR) + self.available_tickets = {"outbound": {}, "inbound": {}, "last_deliveries": {}} + if not "outbound" in self.available_tickets: + RNS.log("Missing outbound entry in loaded available tickets, recreating...", RNS.LOG_ERROR) + self.available_tickets["outbound"] = {} + if not "inbound" in self.available_tickets: + RNS.log("Missing inbound entry in loaded available tickets, recreating...", RNS.LOG_ERROR) + self.available_tickets["inbound"] = {} + if not "last_deliveries" in self.available_tickets: + RNS.log("Missing local_deliveries entry in loaded available tickets, recreating...", RNS.LOG_ERROR) + self.available_tickets["last_deliveries"] = {} + + except Exception as e: + RNS.log(f"An error occurred while reloading available tickets from storage: {e}", RNS.LOG_ERROR) + + def exit_handler(self): + if self.exit_handler_running: + return + + self.exit_handler_running = True + + RNS.log("Tearing down delivery destinations...", RNS.LOG_NOTICE) + for destination_hash in self.delivery_destinations: + delivery_destination = self.delivery_destinations[destination_hash] + delivery_destination.set_packet_callback(None) + delivery_destination.set_link_established_callback(None) + for link in delivery_destination.links: + try: + if link.status == RNS.Link.ACTIVE: + link.teardown() + except Exception as e: + RNS.log("Error while tearing down propagation link: {e}", RNS.LOG_ERROR) + + if self.propagation_node: + RNS.log("Tearing down propagation node destination...", RNS.LOG_NOTICE) + self.propagation_destination.set_link_established_callback(None) + self.propagation_destination.set_packet_callback(None) + self.propagation_destination.deregister_request_handler(LXMPeer.OFFER_REQUEST_PATH) + self.propagation_destination.deregister_request_handler(LXMPeer.MESSAGE_GET_PATH) + self.propagation_destination.deregister_request_handler(LXMRouter.STATS_GET_PATH) + self.propagation_destination.deregister_request_handler(LXMRouter.SYNC_REQUEST_PATH) + self.propagation_destination.deregister_request_handler(LXMRouter.UNPEER_REQUEST_PATH) + for link in self.active_propagation_links: + try: + if link.status == RNS.Link.ACTIVE: + link.teardown() + except Exception as e: + RNS.log("Error while tearing down propagation link: {e}", RNS.LOG_ERROR) + + RNS.log("Persisting LXMF state data to storage...", RNS.LOG_NOTICE) + self.flush_queues() + if self.propagation_node: + try: + st = time.time(); RNS.log(f"Saving {len(self.peers)} peer synchronisation states to storage...", RNS.LOG_NOTICE) + serialised_peers = [] + peer_dict = self.peers.copy() + for peer_id in peer_dict: + peer = self.peers[peer_id] + serialised_peers.append(peer.to_bytes()) + + peers_file = open(self.storagepath+"/peers", "wb") + peers_file.write(msgpack.packb(serialised_peers)) + peers_file.close() + + RNS.log(f"Saved {len(serialised_peers)} peers to storage in {RNS.prettyshorttime(time.time()-st)}", RNS.LOG_NOTICE) + + except Exception as e: + RNS.log("Could not save propagation node peers to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + + self.save_locally_delivered_transient_ids() + self.save_locally_processed_transient_ids() + self.save_node_stats() + + def sigint_handler(self, signal, frame): + if threading.current_thread() != threading.main_thread(): + RNS.log(f"SIGINT on non-main thread {threading.current_thread()}, exiting immediately", RNS.LOG_WARNING) + os._exit(0) + else: + if not self.exit_handler_running: + RNS.log("Received SIGINT, shutting down now!", RNS.LOG_WARNING) + self.exit_handler() + RNS.exit(0) + else: + RNS.log("Received SIGINT, but exit handler is running, keeping process alive until storage persist is complete", RNS.LOG_WARNING) + + def sigterm_handler(self, signal, frame): + if threading.current_thread() != threading.main_thread(): + RNS.log(f"SIGTERM on non-main thread {threading.current_thread()}, exiting immediately", RNS.LOG_WARNING) + os._exit(0) + else: + if not self.exit_handler_running: + RNS.log("Received SIGTERM, shutting down now!", RNS.LOG_WARNING) + self.exit_handler() + RNS.exit(0) + else: + RNS.log("Received SIGTERM, but exit handler is running, keeping process alive until storage persist is complete", RNS.LOG_WARNING) + + def __str__(self): + return "" + + + ### Message Download ################################## + ####################################################### + + def request_messages_path_job(self): + job_thread = threading.Thread(target=self.__request_messages_path_job) + job_thread.setDaemon(True) + job_thread.start() + + def __request_messages_path_job(self): + path_timeout = self.wants_download_on_path_available_timeout + while not RNS.Transport.has_path(self.wants_download_on_path_available_from) and time.time() < path_timeout: + time.sleep(0.1) + + if RNS.Transport.has_path(self.wants_download_on_path_available_from): + self.request_messages_from_propagation_node(self.wants_download_on_path_available_to, self.propagation_transfer_max_messages) + else: + RNS.log("Propagation node path request timed out", RNS.LOG_DEBUG) + self.acknowledge_sync_completion(failure_state=LXMRouter.PR_NO_PATH) + + def identity_allowed(self, identity): + if self.auth_required: + if identity.hash in self.allowed_list: + return True + else: + return False + + else: + return True + + def message_get_request(self, path, data, request_id, remote_identity, requested_at): + if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY + elif not self.identity_allowed(remote_identity): return LXMPeer.ERROR_NO_ACCESS + else: + try: + remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") + + # If both want and have fields are empty, send a list of + # available messages. + if data[0] == None and data[1] == None: + available_messages = [] + for transient_id in self.propagation_entries: + message_entry = self.propagation_entries[transient_id] + if message_entry[0] == remote_destination.hash: + message_size = os.path.getsize(message_entry[1]) + available_entry = [transient_id, message_size] + available_messages.append(available_entry) + + available_messages.sort(key=lambda e: e[1], reverse=False) + + transient_ids = [] + for available_entry in available_messages: transient_ids.append(available_entry[0]) + return transient_ids + + else: + # Process messages the client already have + if data[1] != None and len(data[1]) > 0: + for transient_id in data[1]: + if transient_id in self.propagation_entries and self.propagation_entries[transient_id][0] == remote_destination.hash: + try: + filepath = self.propagation_entries[transient_id][1] + self.propagation_entries.pop(transient_id) + os.unlink(filepath) + # TODO: Remove debug + # RNS.log("Client "+RNS.prettyhexrep(remote_destination.hash)+" purged message "+RNS.prettyhexrep(transient_id)+" at "+str(filepath), RNS.LOG_DEBUG) + + except Exception as e: + RNS.log("Error while processing message purge request from "+RNS.prettyhexrep(remote_destination.hash)+". The contained exception was: "+str(e), RNS.LOG_ERROR) + + + # Process wanted messages + response_messages = [] + if data[0] != None and len(data[0]) > 0: + client_transfer_limit = None + if len(data) >= 3: + try: + client_transfer_limit = float(data[2])*1000 + RNS.log("Client indicates transfer limit of "+RNS.prettysize(client_transfer_limit), RNS.LOG_DEBUG) + except: pass + + per_message_overhead = 16 # Really only 2 bytes, but set a bit higher for now + cumulative_size = 24 # Initialised to highest reasonable binary structure overhead + for transient_id in data[0]: + if transient_id in self.propagation_entries and self.propagation_entries[transient_id][0] == remote_destination.hash: + try: + filepath = self.propagation_entries[transient_id][1] + RNS.log("Client "+RNS.prettyhexrep(remote_destination.hash)+" requested message "+RNS.prettyhexrep(transient_id)+" at "+str(filepath), RNS.LOG_DEBUG) + + message_file = open(filepath, "rb") + lxmf_data = message_file.read() + message_file.close() + + lxm_size = len(lxmf_data) + next_size = cumulative_size + (lxm_size+per_message_overhead) + + if client_transfer_limit != None and next_size > client_transfer_limit: pass + else: + response_messages.append(lxmf_data[:-LXStamper.STAMP_SIZE]) + cumulative_size += (lxm_size+per_message_overhead) + + except Exception as e: + RNS.log("Error while processing message download request from "+RNS.prettyhexrep(remote_destination.hash)+". The contained exception was: "+str(e), RNS.LOG_ERROR) + + self.client_propagation_messages_served += len(response_messages) + return response_messages + + except Exception as e: + RNS.log("Error occurred while generating response for download request, the contained exception was: "+str(e), RNS.LOG_DEBUG) + return None + + def message_list_response(self, request_receipt): + if request_receipt.response == LXMPeer.ERROR_NO_IDENTITY: + RNS.log("Propagation node indicated missing identification on list request, tearing down link.", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + self.propagation_transfer_state = LXMRouter.PR_NO_IDENTITY_RCVD + + elif request_receipt.response == LXMPeer.ERROR_NO_ACCESS: + RNS.log("Propagation node did not allow list request, tearing down link.", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + self.propagation_transfer_state = LXMRouter.PR_NO_ACCESS + + else: + if request_receipt.response != None and isinstance(request_receipt.response, list): + haves = [] + wants = [] + if len(request_receipt.response) > 0: + for transient_id in request_receipt.response: + if self.has_message(transient_id): + if not self.retain_synced_on_node: + haves.append(transient_id) + else: + if self.propagation_transfer_max_messages == LXMRouter.PR_ALL_MESSAGES or len(wants) < self.propagation_transfer_max_messages: + wants.append(transient_id) + + ms = "" if len(wants) == 1 else "s" + RNS.log(f"Requesting {len(wants)} message{ms} from propagation node", RNS.LOG_DEBUG) + request_receipt.link.request( + LXMPeer.MESSAGE_GET_PATH, + [wants, haves, self.delivery_per_transfer_limit], + response_callback=self.message_get_response, + failed_callback=self.message_get_failed, + progress_callback=self.message_get_progress) + + else: + self.propagation_transfer_state = LXMRouter.PR_COMPLETE + self.propagation_transfer_progress = 1.0 + self.propagation_transfer_last_result = 0 + + else: + RNS.log("Invalid message list data received from propagation node", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + + def message_get_response(self, request_receipt): + if request_receipt.response == LXMPeer.ERROR_NO_IDENTITY: + RNS.log("Propagation node indicated missing identification on get request, tearing down link.", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + self.propagation_transfer_state = LXMRouter.PR_NO_IDENTITY_RCVD + + elif request_receipt.response == LXMPeer.ERROR_NO_ACCESS: + RNS.log("Propagation node did not allow get request, tearing down link.", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + self.propagation_transfer_state = LXMRouter.PR_NO_ACCESS + + else: + duplicates = 0 + if request_receipt.response != None and len(request_receipt.response) > 0: + haves = [] + for lxmf_data in request_receipt.response: + result = self.lxmf_propagation(lxmf_data, signal_duplicate=LXMRouter.DUPLICATE_SIGNAL) + if result == LXMRouter.DUPLICATE_SIGNAL: duplicates += 1 + haves.append(RNS.Identity.full_hash(lxmf_data)) + + # Return a list of successfully received messages to the node. + # This deletes the messages on the propagation node. + # TODO: Add option to keep messages on node. + request_receipt.link.request( + LXMPeer.MESSAGE_GET_PATH, + [None, haves], + # response_callback=self.message_syncfinal_response, + failed_callback=self.message_get_failed, + # progress_callback=self.message_get_progress + ) + + self.propagation_transfer_state = LXMRouter.PR_COMPLETE + self.propagation_transfer_progress = 1.0 + self.propagation_transfer_last_duplicates = duplicates + self.propagation_transfer_last_result = len(request_receipt.response) + self.save_locally_delivered_transient_ids() + + def message_get_progress(self, request_receipt): + self.propagation_transfer_state = LXMRouter.PR_RECEIVING + self.propagation_transfer_progress = request_receipt.get_progress() + + def message_get_failed(self, request_receipt): + RNS.log("Message list/get request failed", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + + def acknowledge_sync_completion(self, reset_state=False, failure_state=None): + self.propagation_transfer_last_result = None + if reset_state or self.propagation_transfer_state <= LXMRouter.PR_COMPLETE: + if failure_state == None: + self.propagation_transfer_state = LXMRouter.PR_IDLE + else: + self.propagation_transfer_state = failure_state + + self.propagation_transfer_progress = 0.0 + self.wants_download_on_path_available_from = None + self.wants_download_on_path_available_to = None + + def has_message(self, transient_id): + if transient_id in self.locally_delivered_transient_ids: + return True + else: + return False + + def cancel_outbound(self, message_id, cancel_state=LXMessage.CANCELLED): + try: + if message_id in self.pending_deferred_stamps: + lxm = self.pending_deferred_stamps[message_id] + RNS.log(f"Cancelling deferred stamp generation for {lxm}", RNS.LOG_DEBUG) + lxm.state = cancel_state + LXStamper.cancel_work(message_id) + + lxmessage = None + for lxm in self.pending_outbound: + if lxm.message_id == message_id: + lxmessage = lxm + + if lxmessage != None: + lxmessage.state = cancel_state + if lxmessage in self.pending_outbound: + RNS.log(f"Cancelling {lxmessage} in outbound queue", RNS.LOG_DEBUG) + if lxmessage.representation == LXMessage.RESOURCE: + if lxmessage.resource_representation != None: + lxmessage.resource_representation.cancel() + + self.process_outbound() + + except Exception as e: + RNS.log(f"An error occurred while cancelling {lxmessage}: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + def handle_outbound(self, lxmessage): + destination_hash = lxmessage.get_destination().hash + + if lxmessage.desired_method == LXMessage.PROPAGATED and not self.outbound_propagation_node: + self.fail_message(lxmessage) + raise IOError("Attempt to send propagated message with no outbound propagation node configured") + + if lxmessage.stamp_cost == None: + if destination_hash in self.outbound_stamp_costs: + stamp_cost = self.outbound_stamp_costs[destination_hash][1] + lxmessage.stamp_cost = stamp_cost + RNS.log(f"No stamp cost set on LXM to {RNS.prettyhexrep(destination_hash)}, autoconfigured to {stamp_cost}, as required by latest announce", RNS.LOG_DEBUG) + + lxmessage.state = LXMessage.OUTBOUND + + # If an outbound ticket is available for this + # destination, attach it to the message. + lxmessage.outbound_ticket = self.get_outbound_ticket(destination_hash) + if lxmessage.outbound_ticket != None and lxmessage.defer_stamp: + RNS.log(f"Deferred stamp generation was requested for {lxmessage}, but outbound ticket was applied, processing immediately", RNS.LOG_DEBUG) + lxmessage.defer_stamp = False + + # If requested, include a ticket to allow the + # destination to reply without generating a stamp. + if lxmessage.include_ticket: + ticket = self.generate_ticket(lxmessage.destination_hash) + if ticket: lxmessage.fields[FIELD_TICKET] = ticket + + if not lxmessage.packed: lxmessage.pack() + + unknown_path_requested = False + if not RNS.Transport.has_path(destination_hash) and lxmessage.method == LXMessage.OPPORTUNISTIC: + RNS.log(f"Pre-emptively requesting unknown path for opportunistic {lxmessage}", RNS.LOG_DEBUG) + RNS.Transport.request_path(destination_hash) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + unknown_path_requested = True + + lxmessage.determine_transport_encryption() + + if lxmessage.defer_stamp and lxmessage.stamp_cost == None: + RNS.log(f"Deferred stamp generation was requested for {lxmessage}, but no stamp is required, processing immediately", RNS.LOG_DEBUG) + lxmessage.defer_stamp = False + + if not lxmessage.defer_stamp and not (lxmessage.desired_method == LXMessage.PROPAGATED and lxmessage.defer_propagation_stamp): + while not unknown_path_requested and self.outbound_processing_lock.locked(): time.sleep(0.05) + + self.pending_outbound.append(lxmessage) + if not unknown_path_requested: threading.Thread(target=self.process_outbound, daemon=True).start() + + else: self.pending_deferred_stamps[lxmessage.message_id] = lxmessage + + def get_outbound_progress(self, lxm_hash): + for lxm in self.pending_outbound: + if lxm.hash == lxm_hash: + return lxm.progress + + for lxm_id in self.pending_deferred_stamps: + if self.pending_deferred_stamps[lxm_id].hash == lxm_hash: + return self.pending_deferred_stamps[lxm_id].progress + + return None + + def get_outbound_lxm_stamp_cost(self, lxm_hash): + for lxm in self.pending_outbound: + if lxm.hash == lxm_hash: + if lxm.outbound_ticket: return None + else: return lxm.stamp_cost + + for lxm_id in self.pending_deferred_stamps: + if self.pending_deferred_stamps[lxm_id].hash == lxm_hash: + lxm = self.pending_deferred_stamps[lxm_id] + if lxm.outbound_ticket: return None + else: return lxm.stamp_cost + + return None + + def get_outbound_lxm_propagation_stamp_cost(self, lxm_hash): + for lxm in self.pending_outbound: + if lxm.hash == lxm_hash: + return lxm.propagation_target_cost + + for lxm_id in self.pending_deferred_stamps: + if self.pending_deferred_stamps[lxm_id].hash == lxm_hash: + return self.pending_deferred_stamps[lxm_id].propagation_target_cost + + return None + + + ### Message Routing & Delivery ######################## + ####################################################### + + def lxmf_delivery(self, lxmf_data, destination_type = None, phy_stats = None, ratchet_id = None, method = None, no_stamp_enforcement=False, allow_duplicate=False): + try: + message = LXMessage.unpack_from_bytes(lxmf_data) + + if message.source_blackholed: + RNS.log(f"Dropping LXM from blackholed identity {message.source.identity}", RNS.LOG_DEBUG) + return False + + if ratchet_id and not message.ratchet_id: message.ratchet_id = ratchet_id + if method: message.method = method + + if message.signature_validated and FIELD_TICKET in message.fields: + ticket_entry = message.fields[FIELD_TICKET] + if type(ticket_entry) == list and len(ticket_entry) > 1: + expires = ticket_entry[0] + ticket = ticket_entry[1] + + if time.time() < expires: + if type(ticket) == bytes and len(ticket) == LXMessage.TICKET_LENGTH: + self.remember_ticket(message.source_hash, ticket_entry) + def save_job(): + self.save_available_tickets() + threading.Thread(target=save_job, daemon=True).start() + + required_stamp_cost = self.delivery_destinations[message.destination_hash].stamp_cost + if required_stamp_cost != None: + destination_tickets = self.get_inbound_tickets(message.source_hash) + if message.validate_stamp(required_stamp_cost, tickets=destination_tickets): + message.stamp_valid = True + message.stamp_checked = True + else: + message.stamp_valid = False + message.stamp_checked = True + + if not message.stamp_valid: + if no_stamp_enforcement: + RNS.log(f"Received {message} with invalid stamp, but allowing anyway, since stamp enforcement was temporarily disabled", RNS.LOG_NOTICE) + else: + if self._enforce_stamps: + RNS.log(f"Dropping {message} with invalid stamp", RNS.LOG_NOTICE) + return False + else: + RNS.log(f"Received {message} with invalid stamp, but allowing anyway, since stamp enforcement is disabled", RNS.LOG_NOTICE) + else: + RNS.log(f"Received {message} with valid stamp", RNS.LOG_DEBUG) + + if phy_stats != None: + if "rssi" in phy_stats: message.rssi = phy_stats["rssi"] + if "snr" in phy_stats: message.snr = phy_stats["snr"] + if "q" in phy_stats: message.q = phy_stats["q"] + + # TODO: Update these descriptions to account for ratchets + if destination_type == RNS.Destination.SINGLE: + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + elif destination_type == RNS.Destination.GROUP: + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES + elif destination_type == RNS.Destination.LINK: + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + else: + message.transport_encrypted = False + message.transport_encryption = None + + if message.source_hash in self.ignored_list: + RNS.log(str(self)+" ignored message from "+RNS.prettyhexrep(message.source_hash), RNS.LOG_DEBUG) + return False + + if not allow_duplicate and self.has_message(message.hash): + RNS.log(str(self)+" ignored already received message from "+RNS.prettyhexrep(message.source_hash), RNS.LOG_DEBUG) + return False + else: + self.locally_delivered_transient_ids[message.hash] = time.time() + + if self.__delivery_callback != None and callable(self.__delivery_callback): + try: + self.__delivery_callback(message) + except Exception as e: + RNS.log("An error occurred in the external delivery callback for "+str(message), RNS.LOG_ERROR) + RNS.trace_exception(e) + + return True + + except Exception as e: + RNS.log("Could not assemble LXMF message from received data", RNS.LOG_NOTICE) + RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + return False + + def delivery_packet(self, data, packet): + packet.prove() + try: + method = None + if packet.destination_type != RNS.Destination.LINK: + method = LXMessage.OPPORTUNISTIC + lxmf_data = b"" + lxmf_data += packet.destination.hash + lxmf_data += data + else: + method = LXMessage.DIRECT + lxmf_data = data + + try: + reticulum = RNS.Reticulum.get_instance() + if packet.rssi == None: packet.rssi = reticulum.get_packet_rssi(packet.packet_hash) + if packet.snr == None: packet.snr = reticulum.get_packet_snr(packet.packet_hash) + if packet.q == None: packet.q = reticulum.get_packet_q(packet.packet_hash) + except Exception as e: + RNS.log("Error while retrieving physical link stats for LXMF delivery packet: "+str(e), RNS.LOG_ERROR) + + phy_stats = {"rssi": packet.rssi, "snr": packet.snr, "q": packet.q} + + def job(): self.lxmf_delivery(lxmf_data, packet.destination_type, phy_stats=phy_stats, ratchet_id=packet.ratchet_id, method=method) + threading.Thread(target=job, daemon=True).start() + + except Exception as e: + RNS.log("Exception occurred while parsing incoming LXMF data.", RNS.LOG_ERROR) + RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) + + def delivery_link_established(self, link): + link.track_phy_stats(True) + link.set_packet_callback(self.delivery_packet) + link.set_resource_strategy(RNS.Link.ACCEPT_APP) + link.set_resource_callback(self.delivery_resource_advertised) + link.set_resource_started_callback(self.resource_transfer_began) + link.set_resource_concluded_callback(self.delivery_resource_concluded) + link.set_remote_identified_callback(self.delivery_remote_identified) + + def delivery_link_closed(self, link): + pass + + def resource_transfer_began(self, resource): + RNS.log("Transfer began for LXMF delivery resource "+str(resource), RNS.LOG_DEBUG) + + def delivery_resource_advertised(self, resource): + size = resource.get_data_size() + limit = self.delivery_per_transfer_limit*1000 + if limit != None and size > limit: + RNS.log("Rejecting "+RNS.prettysize(size)+" incoming LXMF delivery resource, since it exceeds the limit of "+RNS.prettysize(limit), RNS.LOG_DEBUG) + return False + else: + return True + + def delivery_resource_concluded(self, resource): + RNS.log("Transfer concluded for LXMF delivery resource "+str(resource), RNS.LOG_DEBUG) + if resource.status == RNS.Resource.COMPLETE: + ratchet_id = None + # Set ratchet ID to link ID if available + if resource.link and hasattr(resource.link, "link_id"): + ratchet_id = resource.link.link_id + phy_stats = {"rssi": resource.link.rssi, "snr": resource.link.snr, "q": resource.link.q} + self.lxmf_delivery(resource.data.read(), resource.link.type, phy_stats=phy_stats, ratchet_id=ratchet_id, method=LXMessage.DIRECT) + + def delivery_remote_identified(self, link, identity): + destination_hash = RNS.Destination.hash_from_name_and_identity("lxmf.delivery", identity) + self.backchannel_links[destination_hash] = link + RNS.log(f"Backchannel became available for {RNS.prettyhexrep(destination_hash)} on delivery link {link}", RNS.LOG_DEBUG) + + + ### Peer Sync & Propagation ########################### + ####################################################### + + def peer(self, destination_hash, timestamp, propagation_transfer_limit, propagation_sync_limit, propagation_stamp_cost, propagation_stamp_cost_flexibility, peering_cost, metadata): + if peering_cost > self.max_peering_cost: + if destination_hash in self.peers: + RNS.log(f"Peer {RNS.prettyhexrep(destination_hash)} increased peering cost beyond local accepted maximum, breaking peering...", RNS.LOG_NOTICE) + self.unpeer(destination_hash, timestamp) + else: + RNS.log(f"Not peering with {RNS.prettyhexrep(destination_hash)}, since its peering cost of {peering_cost} exceeds local maximum of {self.max_peering_cost}", RNS.LOG_NOTICE) + + else: + if destination_hash in self.peers: + peer = self.peers[destination_hash] + if timestamp > peer.peering_timebase: + peer.alive = True + peer.metadata = metadata + peer.sync_backoff = 0 + peer.next_sync_attempt = 0 + peer.peering_timebase = timestamp + peer.last_heard = time.time() + peer.propagation_stamp_cost = propagation_stamp_cost + peer.propagation_stamp_cost_flexibility = propagation_stamp_cost_flexibility + peer.peering_cost = peering_cost + peer.propagation_transfer_limit = propagation_transfer_limit + if propagation_sync_limit != None: peer.propagation_sync_limit = propagation_sync_limit + else: peer.propagation_sync_limit = propagation_transfer_limit + + RNS.log(f"Peering config updated for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_VERBOSE) + + else: + if len(self.peers) >= self.max_peers: RNS.log(f"Max peers reached, not peering with {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) + else: + peer = LXMPeer(self, destination_hash, sync_strategy=self.default_sync_strategy) + peer.alive = True + peer.metadata = metadata + peer.last_heard = time.time() + peer.propagation_stamp_cost = propagation_stamp_cost + peer.propagation_stamp_cost_flexibility = propagation_stamp_cost_flexibility + peer.peering_cost = peering_cost + peer.propagation_transfer_limit = propagation_transfer_limit + if propagation_sync_limit != None: peer.propagation_sync_limit = propagation_sync_limit + else: peer.propagation_sync_limit = propagation_transfer_limit + + self.peers[destination_hash] = peer + RNS.log(f"Peered with {RNS.prettyhexrep(destination_hash)}", RNS.LOG_NOTICE) + + + def unpeer(self, destination_hash, timestamp = None): + if timestamp == None: + timestamp = int(time.time()) + + if destination_hash in self.peers: + peer = self.peers[destination_hash] + + if timestamp >= peer.peering_timebase: + self.peers.pop(destination_hash) + RNS.log("Broke peering with "+str(peer.destination)) + + def rotate_peers(self): + try: + rotation_headroom = max(1, math.floor(self.max_peers*(LXMRouter.ROTATION_HEADROOM_PCT/100.0))) + required_drops = len(self.peers) - (self.max_peers - rotation_headroom) + if required_drops > 0 and len(self.peers) - required_drops > 1: + peers = self.peers.copy() + untested_peers = [] + for peer_id in self.peers: + peer = self.peers[peer_id] + if peer.last_sync_attempt == 0: + untested_peers.append(peer) + + if len(untested_peers) >= rotation_headroom: + RNS.log("Newly added peer threshold reached, postponing peer rotation", RNS.LOG_DEBUG) + return + + fully_synced_peers = {} + for peer_id in peers: + peer = peers[peer_id] + if peer.unhandled_message_count == 0: + fully_synced_peers[peer_id] = peer + + if len(fully_synced_peers) > 0: + peers = fully_synced_peers + ms = "" if len(fully_synced_peers) == 1 else "s" + RNS.log(f"Found {len(fully_synced_peers)} fully synced peer{ms}, using as peer rotation pool basis", RNS.LOG_DEBUG) + + culled_peers = [] + waiting_peers = [] + unresponsive_peers = [] + for peer_id in peers: + peer = peers[peer_id] + if not peer_id in self.static_peers and peer.state == LXMPeer.IDLE: + if peer.alive: + if peer.offered == 0: + # Don't consider for unpeering until at + # least one message has been offered + pass + else: + waiting_peers.append(peer) + else: + unresponsive_peers.append(peer) + + drop_pool = [] + if len(unresponsive_peers) > 0: + drop_pool.extend(unresponsive_peers) + if not self.prioritise_rotating_unreachable_peers: + drop_pool.extend(waiting_peers) + + else: + drop_pool.extend(waiting_peers) + + if len(drop_pool) > 0: + drop_count = min(required_drops, len(drop_pool)) + low_acceptance_rate_peers = sorted( + drop_pool, + key=lambda p: ( 0 if p.offered == 0 else (p.outgoing/p.offered) ), + reverse=False + )[0:drop_count] + + dropped_peers = 0 + for peer in low_acceptance_rate_peers: + ar = 0 if peer.offered == 0 else round((peer.outgoing/peer.offered)*100, 2) + if ar < LXMRouter.ROTATION_AR_MAX*100: + reachable_str = "reachable" if peer.alive else "unreachable" + RNS.log(f"Acceptance rate for {reachable_str} peer {RNS.prettyhexrep(peer.destination_hash)} was: {ar}% ({peer.outgoing}/{peer.offered}, {peer.unhandled_message_count} unhandled messages)", RNS.LOG_DEBUG) + self.unpeer(peer.destination_hash) + dropped_peers += 1 + + ms = "" if dropped_peers == 1 else "s" + RNS.log(f"Dropped {dropped_peers} low acceptance rate peer{ms} to increase peering headroom", RNS.LOG_DEBUG) + + except Exception as e: + RNS.log(f"An error occurred during peer rotation: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + def sync_peers(self): + culled_peers = [] + waiting_peers = [] + unresponsive_peers = [] + peers = self.peers.copy() + for peer_id in peers: + peer = peers[peer_id] + if time.time() > peer.last_heard + LXMPeer.MAX_UNREACHABLE: + if not peer_id in self.static_peers: culled_peers.append(peer_id) + + else: + if peer.state == LXMPeer.IDLE and len(peer.unhandled_messages) > 0: + if peer.alive: waiting_peers.append(peer) + else: + if hasattr(peer, "next_sync_attempt") and time.time() > peer.next_sync_attempt: unresponsive_peers.append(peer) + else: pass # RNS.log("Not adding peer "+str(peer)+" since it is in sync backoff", RNS.LOG_DEBUG) + + peer_pool = [] + if len(waiting_peers) > 0: + fastest_peers = sorted( + waiting_peers, + key=lambda p: p.sync_transfer_rate, + reverse=True + )[0:min(LXMRouter.FASTEST_N_RANDOM_POOL, len(waiting_peers))] + peer_pool.extend(fastest_peers) + + unknown_speed_peers = [p for p in waiting_peers if p.sync_transfer_rate == 0] + if len(unknown_speed_peers) > 0: + peer_pool.extend( + unknown_speed_peers[ + 0:min( + len(unknown_speed_peers), + len(fastest_peers) + )] + ) + + RNS.log("Selecting peer to sync from "+str(len(waiting_peers))+" waiting peers.", RNS.LOG_DEBUG) + + elif len(unresponsive_peers) > 0: + RNS.log("No active peers available, randomly selecting peer to sync from "+str(len(unresponsive_peers))+" unresponsive peers.", RNS.LOG_DEBUG) + peer_pool = unresponsive_peers + + if len(peer_pool) > 0: + selected_index = random.randint(0,len(peer_pool)-1) + selected_peer = peer_pool[selected_index] + RNS.log("Selected waiting peer "+str(selected_index)+": "+RNS.prettyhexrep(selected_peer.destination.hash), RNS.LOG_DEBUG) + selected_peer.sync() + + for peer_id in culled_peers: + RNS.log("Removing peer "+RNS.prettyhexrep(peer_id)+" due to excessive unreachability", RNS.LOG_WARNING) + try: + if peer_id in self.peers: + self.peers.pop(peer_id) + except Exception as e: + RNS.log("Error while removing peer "+RNS.prettyhexrep(peer_id)+". The contained exception was: "+str(e), RNS.LOG_ERROR) + + def propagation_link_established(self, link): + link.set_packet_callback(self.propagation_packet) + link.set_resource_strategy(RNS.Link.ACCEPT_APP) + link.set_resource_callback(self.propagation_resource_advertised) + link.set_resource_started_callback(self.resource_transfer_began) + link.set_resource_concluded_callback(self.propagation_resource_concluded) + self.active_propagation_links.append(link) + + def propagation_resource_advertised(self, resource): + if self.from_static_only: + remote_identity = resource.link.get_remote_identity() + if remote_identity == None: + RNS.log(f"Rejecting propagation resource from unidentified peer", RNS.LOG_DEBUG) + return False + else: + remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + remote_hash = remote_destination.hash + remote_str = RNS.prettyhexrep(remote_hash) + if not remote_hash in self.static_peers: + RNS.log(f"Rejecting propagation resource from {remote_str} not in static peers list", RNS.LOG_DEBUG) + return False + + size = resource.get_data_size() + limit = self.propagation_per_sync_limit*1000 + if limit != None and size > limit: + RNS.log(f"Rejecting {RNS.prettysize(size)} incoming propagation resource, since it exceeds the limit of {RNS.prettysize(limit)}", RNS.LOG_DEBUG) + return False + else: + return True + + def propagation_packet(self, data, packet): + try: + if packet.destination_type != RNS.Destination.LINK: return + else: + data = msgpack.unpackb(data) + remote_timebase = data[0] + messages = data[1] + + min_accepted_cost = max(0, self.propagation_stamp_cost-self.propagation_stamp_cost_flexibility) + validated_messages = LXStamper.validate_pn_stamps(messages, min_accepted_cost) + + for validated_entry in validated_messages: + lxmf_data = validated_entry[1] + stamp_value = validated_entry[2] + stamp_data = validated_entry[3] + self.lxmf_propagation(lxmf_data, stamp_value=stamp_value, stamp_data=stamp_data) + self.client_propagation_messages_received += 1 + + if len(validated_messages) == len(messages): + ms = "" if len(messages) == 1 else "s" + RNS.log(f"Received {len(messages)} propagation message{ms} from client with valid stamp{ms}", RNS.LOG_DEBUG) + packet.prove() + else: + RNS.log("Propagation transfer from client contained messages with invalid stamps", RNS.LOG_NOTICE) + reject_data = msgpack.packb([LXMPeer.ERROR_INVALID_STAMP]) + RNS.Packet(packet.link, reject_data).send() + packet.link.teardown() + + except Exception as e: + RNS.log("Exception occurred while parsing incoming LXMF propagation data.", RNS.LOG_ERROR) + RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) + + def offer_request(self, path, data, request_id, link_id, remote_identity, requested_at): + if remote_identity == None: + return LXMPeer.ERROR_NO_IDENTITY + else: + remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + remote_hash = remote_destination.hash + remote_str = RNS.prettyhexrep(remote_hash) + + if remote_hash in self.throttled_peers: + throttle_remaining = self.throttled_peers[remote_hash]-time.time() + if throttle_remaining > 0: + RNS.log(f"Propagation offer from node {remote_str} rejected, throttled for {RNS.prettytime(throttle_remaining)} more", RNS.LOG_NOTICE) + return LXMPeer.ERROR_THROTTLED + else: self.throttled_peers.pop(remote_hash) + + if self.from_static_only: + if not remote_hash in self.static_peers: + RNS.log(f"Rejecting propagation request from {remote_str} not in static peers list", RNS.LOG_DEBUG) + return LXMPeer.ERROR_NO_ACCESS + + try: + if type(data) != list and len(data) < 2: return LXMPeer.ERROR_INVALID_DATA + + peering_id = self.identity.hash+remote_identity.hash + target_cost = self.peering_cost + peering_key = data[0] + transient_ids = data[1] + wanted_ids = [] + + ts = time.time() + peering_key_valid = LXStamper.validate_peering_key(peering_id, peering_key, target_cost) + td = time.time() - ts + + if not peering_key_valid: + RNS.log(f"Invalid peering key for incoming sync offer", RNS.LOG_DEBUG) + return LXMPeer.ERROR_INVALID_KEY + + else: + RNS.log(f"Peering key validated for incoming offer in {RNS.prettytime(td)}", RNS.LOG_DEBUG) + self.validated_peer_links[link_id] = True + for transient_id in transient_ids: + if not transient_id in self.propagation_entries: wanted_ids.append(transient_id) + + if len(wanted_ids) == 0: return False + elif len(wanted_ids) == len(transient_ids): return True + else: return wanted_ids + + except Exception as e: + RNS.log("Error occurred while generating response for sync request, the contained exception was: "+str(e), RNS.LOG_DEBUG) + RNS.trace_exception(e) + return None + + def propagation_resource_concluded(self, resource): + if resource.status == RNS.Resource.COMPLETE: + try: + data = msgpack.unpackb(resource.data.read()) + + if type(data) == list and len(data) == 2 and type(data[0] == float) and type(data[1]) == list: + # This is a series of propagation messages from a peer or originator + remote_identity = resource.link.get_remote_identity() + remote_timebase = data[0] + messages = data[1] + remote_hash = None + remote_str = "unknown client" + + if remote_identity != None: + remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + remote_hash = remote_destination.hash + remote_app_data = RNS.Identity.recall_app_data(remote_hash) + remote_str = RNS.prettyhexrep(remote_hash) + + if remote_hash in self.peers: remote_str = f"peer {remote_str}" + else: + if pn_announce_data_is_valid(remote_app_data): + # 1: Current node timebase + # 2: Boolean flag signalling propagation node state + # 3: Per-transfer limit for message propagation in kilobytes + # 4: Limit for incoming propagation node syncs + # 5: Propagation stamp costs for this node + # 6: Node metadata + pn_config = msgpack.unpackb(remote_app_data) + if pn_config[2] and self.autopeer and RNS.Transport.hops_to(remote_hash) <= self.autopeer_maxdepth: + remote_timebase = pn_config[1] + remote_transfer_limit = pn_config[3] + remote_sync_limit = pn_config[4] + remote_stamp_cost = pn_config[5][0] + remote_stamp_flex = pn_config[5][1] + remote_peering_cost = pn_config[5][2] + remote_metadata = pn_config[6] + + RNS.log(f"Auto-peering with {remote_str} discovered via incoming sync", RNS.LOG_DEBUG) # TODO: Remove debug + self.peer(remote_hash, remote_timebase, remote_transfer_limit, remote_sync_limit, remote_stamp_cost, remote_stamp_flex, remote_peering_cost, remote_metadata) + + peering_key_valid = False + if remote_identity != None: + if resource.link.link_id in self.validated_peer_links and self.validated_peer_links[resource.link.link_id] == True: + peering_key_valid = True + + if not peering_key_valid and len(messages) > 1: + resource.link.teardown() + RNS.log(f"Received multiple propagation messages from {remote_str} without valid peering key presentation. This is not supposed to happen, ignoring.", RNS.LOG_WARNING) + RNS.log(f"Clients and peers without a valid peering key can only deliver 1 message per transfer.", RNS.LOG_WARNING) + else: + ms = "" if len(messages) == 1 else "s" + RNS.log(f"Received {len(messages)} message{ms} from {remote_str}, validating stamps...", RNS.LOG_VERBOSE) + + min_accepted_cost = max(0, self.propagation_stamp_cost-self.propagation_stamp_cost_flexibility) + validated_messages = LXStamper.validate_pn_stamps(messages, min_accepted_cost) + invalid_stamps = len(messages)-len(validated_messages) + ms = "" if invalid_stamps == 1 else "s" + if len(validated_messages) == len(messages): RNS.log(f"All message stamps validated from {remote_str}", RNS.LOG_VERBOSE) + else: RNS.log(f"Transfer from {remote_str} contained {invalid_stamps} invalid stamp{ms}", RNS.LOG_WARNING) + + for validated_entry in validated_messages: + transient_id = validated_entry[0] + lxmf_data = validated_entry[1] + stamp_value = validated_entry[2] + stamp_data = validated_entry[3] + peer = None + + if remote_hash != None and remote_hash in self.peers: + peer = self.peers[remote_hash] + peer.incoming += 1 + peer.rx_bytes += len(lxmf_data) + else: + if remote_identity != None: + self.unpeered_propagation_incoming += 1 + self.unpeered_propagation_rx_bytes += len(lxmf_data) + else: + self.client_propagation_messages_received += 1 + + self.lxmf_propagation(lxmf_data, from_peer=peer, stamp_value=stamp_value, stamp_data=stamp_data) + if peer != None: peer.queue_handled_message(transient_id) + + invalid_message_count = len(messages) - len(validated_messages) + if invalid_message_count > 0: + resource.link.teardown() + if remote_identity != None: + throttle_time = LXMRouter.PN_STAMP_THROTTLE + self.throttled_peers[remote_hash] = time.time()+throttle_time + ms = "" if invalid_message_count == 1 else "s" + RNS.log(f"Propagation transfer from {remote_str} contained {invalid_message_count} message{ms} with invalid stamps, throttled for {RNS.prettytime(throttle_time)}", RNS.LOG_NOTICE) + + else: + RNS.log("Invalid data structure received at propagation destination, ignoring", RNS.LOG_DEBUG) + + except Exception as e: + RNS.log("Error while unpacking received propagation resource", RNS.LOG_DEBUG) + RNS.trace_exception(e) + + def enqueue_peer_distribution(self, transient_id, from_peer): + self.peer_distribution_queue.append([transient_id, from_peer]) + + def flush_peer_distribution_queue(self): + if len(self.peer_distribution_queue) > 0: + entries = [] + while len(self.peer_distribution_queue) > 0: + entries.append(self.peer_distribution_queue.pop()) + + for peer_id in self.peers.copy(): + if peer_id in self.peers: + peer = self.peers[peer_id] + for entry in entries: + transient_id = entry[0] + from_peer = entry[1] + if peer != from_peer: + peer.queue_unhandled_message(transient_id) + + def lxmf_propagation(self, lxmf_data, signal_local_delivery=None, signal_duplicate=None, allow_duplicate=False, is_paper_message=False, + from_peer=None, stamp_value=None, stamp_data=None): + if is_paper_message: no_stamp_enforcement = True + else: no_stamp_enforcement = False + + try: + if len(lxmf_data) >= LXMessage.LXMF_OVERHEAD: + transient_id = RNS.Identity.full_hash(lxmf_data) + + if (not transient_id in self.propagation_entries and not transient_id in self.locally_processed_transient_ids) or allow_duplicate == True: + received = time.time() + destination_hash = lxmf_data[:LXMessage.DESTINATION_LENGTH] + + self.locally_processed_transient_ids[transient_id] = received + + if destination_hash in self.delivery_destinations: + delivery_destination = self.delivery_destinations[destination_hash] + encrypted_lxmf_data = lxmf_data[LXMessage.DESTINATION_LENGTH:] + decrypted_lxmf_data = delivery_destination.decrypt(encrypted_lxmf_data) + if decrypted_lxmf_data != None: + delivery_data = lxmf_data[:LXMessage.DESTINATION_LENGTH]+decrypted_lxmf_data + self.lxmf_delivery(delivery_data, delivery_destination.type, ratchet_id=delivery_destination.latest_ratchet_id, method=LXMessage.PROPAGATED, no_stamp_enforcement=no_stamp_enforcement, allow_duplicate=allow_duplicate) + self.locally_delivered_transient_ids[transient_id] = time.time() + + if signal_local_delivery != None: + return signal_local_delivery + + else: + if self.propagation_node: + stamped_data = lxmf_data+stamp_data + value_component = f"_{stamp_value}" if stamp_value and stamp_value > 0 else "" + file_path = f"{self.messagepath}/{RNS.hexrep(transient_id, delimit=False)}_{received}{value_component}" + msg_file = open(file_path, "wb") + msg_file.write(stamped_data); msg_file.close() + + RNS.log(f"Received propagated LXMF message {RNS.prettyhexrep(transient_id)} with stamp value {stamp_value}, adding to peer distribution queues...", RNS.LOG_EXTREME) + self.propagation_entries[transient_id] = [destination_hash, file_path, time.time(), len(stamped_data), [], [], stamp_value] + self.enqueue_peer_distribution(transient_id, from_peer) + + else: + # TODO: Add message to sneakernet queues when implemented + RNS.log(f"Received propagated LXMF message {RNS.prettyhexrep(transient_id)}, but this instance is not hosting a propagation node, discarding message.", RNS.LOG_DEBUG) + + return True + + else: + if signal_duplicate != None: + return signal_duplicate + + else: + return False + + return False + + except Exception as e: + RNS.log("Could not assemble propagated LXMF message from received data", RNS.LOG_DEBUG) + RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + RNS.trace_exception(e) + return False + + def ingest_lxm_uri(self, uri, signal_local_delivery=None, signal_duplicate=None, allow_duplicate=False): + try: + if not uri.lower().startswith(LXMessage.URI_SCHEMA+"://"): + RNS.log("Cannot ingest LXM, invalid URI provided.", RNS.LOG_ERROR) + return False + + else: + lxmf_data = base64.urlsafe_b64decode(uri.replace(LXMessage.URI_SCHEMA+"://", "").replace("/", "")+"==") + transient_id = RNS.Identity.full_hash(lxmf_data) + + router_propagation_result = self.lxmf_propagation(lxmf_data, signal_local_delivery=signal_local_delivery, signal_duplicate=signal_duplicate, allow_duplicate=allow_duplicate, is_paper_message=True) + if router_propagation_result != False: + RNS.log("LXM with transient ID "+RNS.prettyhexrep(transient_id)+" was ingested.", RNS.LOG_DEBUG) + return router_propagation_result + else: + RNS.log("No valid LXM could be ingested from the provided URI", RNS.LOG_DEBUG) + return False + + except Exception as e: + RNS.log("Error while decoding URI-encoded LXMF message. The contained exception was: "+str(e), RNS.LOG_ERROR) + return False + + def fail_message(self, lxmessage): + RNS.log(str(lxmessage)+" failed to send", RNS.LOG_DEBUG) + + lxmessage.progress = 0.0 + if lxmessage in self.pending_outbound: self.pending_outbound.remove(lxmessage) + if lxmessage.state != LXMessage.REJECTED: lxmessage.state = LXMessage.FAILED + if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): + lxmessage.failed_callback(lxmessage) + + def process_deferred_stamps(self): + if len(self.pending_deferred_stamps) > 0: + + if self.stamp_gen_lock.locked(): + return + + else: + with self.stamp_gen_lock: + selected_lxm = None + selected_message_id = None + for message_id in self.pending_deferred_stamps: + lxmessage = self.pending_deferred_stamps[message_id] + if selected_lxm == None: + selected_lxm = lxmessage + selected_message_id = message_id + + if selected_lxm != None: + if selected_lxm.state == LXMessage.CANCELLED: + RNS.log(f"Message cancelled during deferred stamp generation for {selected_lxm}.", RNS.LOG_DEBUG) + selected_lxm.stamp_generation_failed = True + self.pending_deferred_stamps.pop(selected_message_id) + if selected_lxm.failed_callback != None and callable(selected_lxm.failed_callback): + selected_lxm.failed_callback(lxmessage) + + return + + if selected_lxm.defer_stamp: + if selected_lxm.stamp == None: stamp_generation_success = False + else: stamp_generation_success = True + else: stamp_generation_success = True + + if selected_lxm.desired_method == LXMessage.PROPAGATED: + if selected_lxm.propagation_stamp == None: propagation_stamp_generation_success = False + else: propagation_stamp_generation_success = True + else: propagation_stamp_generation_success = True + + if stamp_generation_success == False: + RNS.log(f"Starting stamp generation for {selected_lxm}...", RNS.LOG_DEBUG) + generated_stamp = selected_lxm.get_stamp() + if generated_stamp: + selected_lxm.stamp = generated_stamp + selected_lxm.defer_stamp = False + selected_lxm.packed = None + selected_lxm.pack(payload_updated=True) + stamp_generation_success = True + RNS.log(f"Stamp generation completed for {selected_lxm}", RNS.LOG_DEBUG) + else: + if selected_lxm.state == LXMessage.CANCELLED: + RNS.log(f"Message cancelled during deferred stamp generation for {selected_lxm}.", RNS.LOG_DEBUG) + selected_lxm.stamp_generation_failed = True + self.pending_deferred_stamps.pop(selected_message_id) + if selected_lxm.failed_callback != None and callable(selected_lxm.failed_callback): + selected_lxm.failed_callback(lxmessage) + else: + RNS.log(f"Deferred stamp generation did not succeed. Failing {selected_lxm}.", RNS.LOG_ERROR) + selected_lxm.stamp_generation_failed = True + self.pending_deferred_stamps.pop(selected_message_id) + self.fail_message(selected_lxm) + + if propagation_stamp_generation_success == False: + RNS.log(f"Starting propagation stamp generation for {selected_lxm}...", RNS.LOG_DEBUG) + pn_target_cost = self.get_outbound_propagation_cost() + if pn_target_cost == None: + RNS.log("Failed to get propagation node stamp cost, cannot generate propagation stamp", RNS.LOG_ERROR) + selected_lxm.stamp_generation_failed = True + self.pending_deferred_stamps.pop(selected_message_id) + self.fail_message(selected_lxm) + + else: + propagation_stamp = selected_lxm.get_propagation_stamp(target_cost=pn_target_cost) + if propagation_stamp: + selected_lxm.propagation_stamp = propagation_stamp + selected_lxm.defer_propagation_stamp = False + selected_lxm.packed = None + selected_lxm.pack() + propagation_stamp_generation_success = True + RNS.log(f"Propagation stamp generation completed for {selected_lxm}", RNS.LOG_DEBUG) + else: + if selected_lxm.state == LXMessage.CANCELLED: + RNS.log(f"Message cancelled during deferred propagation stamp generation for {selected_lxm}.", RNS.LOG_DEBUG) + selected_lxm.stamp_generation_failed = True + self.pending_deferred_stamps.pop(selected_message_id) + if selected_lxm.failed_callback != None and callable(selected_lxm.failed_callback): + selected_lxm.failed_callback(lxmessage) + else: + RNS.log(f"Deferred propagation stamp generation did not succeed. Failing {selected_lxm}.", RNS.LOG_ERROR) + selected_lxm.stamp_generation_failed = True + self.pending_deferred_stamps.pop(selected_message_id) + self.fail_message(selected_lxm) + + if stamp_generation_success and propagation_stamp_generation_success: + self.pending_deferred_stamps.pop(selected_message_id) + self.pending_outbound.append(selected_lxm) + + def propagation_transfer_signalling_packet(self, data, packet): + try: + unpacked = msgpack.unpackb(data) + if type(unpacked) == list and len(unpacked) >= 1: + signal = unpacked[0] + if signal == LXMPeer.ERROR_INVALID_STAMP: + RNS.log("Message rejected by propagation node", RNS.LOG_ERROR) + if hasattr(packet, "link") and hasattr(packet.link, "for_lxmessage"): + lxm = packet.link.for_lxmessage + RNS.log(f"Invalid propagation stamp on {lxm}", RNS.LOG_ERROR) + self.cancel_outbound(lxm.message_id, cancel_state=LXMessage.REJECTED) + + except Exception as e: + RNS.log(f"An error occurred while processing propagation transfer signalling. The contained exception was: {e}", RNS.LOG_ERROR) + + def process_outbound(self, sender = None): + if self.outbound_processing_lock.locked(): return + with self.outbound_processing_lock: + for lxmessage in self.pending_outbound: + if lxmessage.state == LXMessage.DELIVERED: + RNS.log("Delivery has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) + self.pending_outbound.remove(lxmessage) + try: RNS.Reticulum.get_instance()._retain_destination_data(lxmessage.destination_hash) + except Exception as e: RNS.log(f"An error occurred while marking {RNS.prettyhexrep(lxmessage.destination_hash)} for announce data retainment: {e}", RNS.LOG_ERROR) + + # Udate ticket delivery stats + if lxmessage.include_ticket and FIELD_TICKET in lxmessage.fields: + RNS.log(f"Updating latest ticket delivery for {RNS.prettyhexrep(lxmessage.destination_hash)}", RNS.LOG_DEBUG) + self.available_tickets["last_deliveries"][lxmessage.destination_hash] = time.time() + self.save_available_tickets() + + # Prepare link for backchannel communications + delivery_destination_hash = lxmessage.get_destination().hash + if lxmessage.method == LXMessage.DIRECT and delivery_destination_hash in self.direct_links: + direct_link = self.direct_links[delivery_destination_hash] + if not hasattr(direct_link, "backchannel_identified") or direct_link.backchannel_identified == False: + if direct_link.initiator == True: + source_destination_hash = lxmessage.get_source().hash + if source_destination_hash in self.delivery_destinations: + backchannel_identity = self.delivery_destinations[source_destination_hash].identity + backchannel_desthash = RNS.Destination.hash_from_name_and_identity("lxmf.delivery", backchannel_identity) + direct_link.identify(backchannel_identity) + direct_link.backchannel_identified = True + self.delivery_link_established(direct_link) + RNS.log(f"Performed backchannel identification as {RNS.prettyhexrep(backchannel_desthash)} on {direct_link}", RNS.LOG_DEBUG) + + elif lxmessage.method == LXMessage.PROPAGATED and lxmessage.state == LXMessage.SENT: + RNS.log("Propagation has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) + self.pending_outbound.remove(lxmessage) + + elif lxmessage.state == LXMessage.CANCELLED: + RNS.log("Cancellation requested for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) + self.pending_outbound.remove(lxmessage) + if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): + lxmessage.failed_callback(lxmessage) + + elif lxmessage.state == LXMessage.REJECTED: + RNS.log("Receiver rejected "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) + if lxmessage in self.pending_outbound: self.pending_outbound.remove(lxmessage) + if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): + lxmessage.failed_callback(lxmessage) + + else: + RNS.log("Outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + + if lxmessage.progress == None or lxmessage.progress < 0.01: lxmessage.progress = 0.01 + + # Outbound handling for opportunistic messages + if lxmessage.method == LXMessage.OPPORTUNISTIC: + if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: + if lxmessage.delivery_attempts >= LXMRouter.MAX_PATHLESS_TRIES and not RNS.Transport.has_path(lxmessage.get_destination().hash): + RNS.log(f"Requesting path to {RNS.prettyhexrep(lxmessage.get_destination().hash)} after {lxmessage.delivery_attempts} pathless tries for {lxmessage}", RNS.LOG_DEBUG) + lxmessage.delivery_attempts += 1 + RNS.Transport.request_path(lxmessage.get_destination().hash) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + lxmessage.progress = 0.01 + elif lxmessage.delivery_attempts == LXMRouter.MAX_PATHLESS_TRIES+1 and RNS.Transport.has_path(lxmessage.get_destination().hash): + RNS.log(f"Opportunistic delivery for {lxmessage} still unsuccessful after {lxmessage.delivery_attempts} attempts, trying to rediscover path to {RNS.prettyhexrep(lxmessage.get_destination().hash)}", RNS.LOG_DEBUG) + lxmessage.delivery_attempts += 1 + RNS.Reticulum.get_instance().drop_path(lxmessage.get_destination().hash) + def rediscover_job(): + time.sleep(0.5) + RNS.Transport.request_path(lxmessage.get_destination().hash) + threading.Thread(target=rediscover_job, daemon=True).start() + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + lxmessage.progress = 0.01 + else: + if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: + lxmessage.delivery_attempts += 1 + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + RNS.log("Opportunistic delivery attempt "+str(lxmessage.delivery_attempts)+" for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + lxmessage.send() + else: + RNS.log("Max delivery attempts reached for oppertunistic "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + self.fail_message(lxmessage) + + # Outbound handling for messages transferred + # over a direct link to the final recipient + elif lxmessage.method == LXMessage.DIRECT: + if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: + delivery_destination_hash = lxmessage.get_destination().hash + direct_link = None + + if delivery_destination_hash in self.direct_links: + # An established direct link already exists to + # the destination, so we'll try to use it for + # delivering the message + direct_link = self.direct_links[delivery_destination_hash] + RNS.log(f"Using available direct link {direct_link} to {RNS.prettyhexrep(delivery_destination_hash)}", RNS.LOG_DEBUG) + + elif delivery_destination_hash in self.backchannel_links: + # An established backchannel link exists to + # the destination, so we'll try to use it for + # delivering the message + direct_link = self.backchannel_links[delivery_destination_hash] + RNS.log(f"Using available backchannel link {direct_link} to {RNS.prettyhexrep(delivery_destination_hash)}", RNS.LOG_DEBUG) + + if direct_link != None: + if direct_link.status == RNS.Link.ACTIVE: + if lxmessage.progress == None or lxmessage.progress < 0.05: + lxmessage.progress = 0.05 + if lxmessage.state != LXMessage.SENDING: + RNS.log("Starting transfer of "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" on link "+str(direct_link), RNS.LOG_DEBUG) + lxmessage.set_delivery_destination(direct_link) + lxmessage.send() + else: + if lxmessage.representation == LXMessage.RESOURCE: + RNS.log("The transfer of "+str(lxmessage)+" is in progress ("+str(round(lxmessage.progress*100, 1))+"%)", RNS.LOG_DEBUG) + else: + RNS.log("Waiting for proof for "+str(lxmessage)+" sent as link packet", RNS.LOG_DEBUG) + elif direct_link.status == RNS.Link.CLOSED: + if direct_link.activated_at != None: + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was closed unexpectedly, retrying path request...", RNS.LOG_DEBUG) + RNS.Transport.request_path(lxmessage.get_destination().hash) + else: + if not hasattr(lxmessage, "path_request_retried"): + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was never activated, retrying path request...", RNS.LOG_DEBUG) + RNS.Transport.request_path(lxmessage.get_destination().hash) + lxmessage.path_request_retried = True + else: + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was never activated", RNS.LOG_DEBUG) + + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + + lxmessage.set_delivery_destination(None) + if delivery_destination_hash in self.direct_links: + self.direct_links.pop(delivery_destination_hash) + if delivery_destination_hash in self.backchannel_links: + self.backchannel_links.pop(delivery_destination_hash) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + else: + # Simply wait for the link to become active or close + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" is pending, waiting for link to become active", RNS.LOG_DEBUG) + else: + # No link exists, so we'll try to establish one, but + # only if we've never tried before, or the retry wait + # period has elapsed. + if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: + lxmessage.delivery_attempts += 1 + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + + if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: + if RNS.Transport.has_path(lxmessage.get_destination().hash): + RNS.log("Establishing link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + delivery_link = RNS.Link(lxmessage.get_destination()) + delivery_link.set_link_established_callback(self.process_outbound) + self.direct_links[delivery_destination_hash] = delivery_link + lxmessage.progress = 0.03 + else: + RNS.log("No path known for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+". Requesting path...", RNS.LOG_DEBUG) + RNS.Transport.request_path(lxmessage.get_destination().hash) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + lxmessage.progress = 0.01 + else: + RNS.log("Max delivery attempts reached for direct "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + self.fail_message(lxmessage) + + # Outbound handling for messages transported via + # propagation to a LXMF router network. + elif lxmessage.method == LXMessage.PROPAGATED: + RNS.log("Attempting propagated delivery for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + + if self.outbound_propagation_node == None: + RNS.log("No outbound propagation node specified for propagated "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_ERROR) + self.fail_message(lxmessage) + else: + if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: + + if self.outbound_propagation_link != None: + # A link already exists, so we'll try to use it + # to deliver the message + if self.outbound_propagation_link.status == RNS.Link.ACTIVE: + if lxmessage.state != LXMessage.SENDING: + RNS.log("Starting propagation transfer of "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" via "+RNS.prettyhexrep(self.outbound_propagation_node), RNS.LOG_DEBUG) + lxmessage.set_delivery_destination(self.outbound_propagation_link) + lxmessage.send() + else: + if lxmessage.representation == LXMessage.RESOURCE: + RNS.log("The transfer of "+str(lxmessage)+" is in progress ("+str(round(lxmessage.progress*100, 1))+"%)", RNS.LOG_DEBUG) + else: + RNS.log("Waiting for proof for "+str(lxmessage)+" sent as link packet", RNS.LOG_DEBUG) + elif self.outbound_propagation_link.status == RNS.Link.CLOSED: + RNS.log("The link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" was closed", RNS.LOG_DEBUG) + self.outbound_propagation_link = None + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + else: + # Simply wait for the link to become + # active or close + RNS.log("The propagation link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" is pending, waiting for link to become active", RNS.LOG_DEBUG) + else: + # No link exists, so we'll try to establish one, but + # only if we've never tried before, or the retry wait + # period has elapsed. + if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: + lxmessage.delivery_attempts += 1 + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + + if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: + if RNS.Transport.has_path(self.outbound_propagation_node): + RNS.log("Establishing link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" for propagation attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + propagation_node_identity = RNS.Identity.recall(self.outbound_propagation_node) + propagation_node_destination = RNS.Destination(propagation_node_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + self.outbound_propagation_link = RNS.Link(propagation_node_destination, established_callback=self.process_outbound) + self.outbound_propagation_link.set_packet_callback(self.propagation_transfer_signalling_packet) + self.outbound_propagation_link.for_lxmessage = lxmessage + else: + RNS.log("No path known for propagation attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(self.outbound_propagation_node)+". Requesting path...", RNS.LOG_DEBUG) + RNS.Transport.request_path(self.outbound_propagation_node) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + + else: + RNS.log("Max delivery attempts reached for propagated "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + self.fail_message(lxmessage) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py new file mode 100644 index 0000000..95d2310 --- /dev/null +++ b/LXMF/LXMessage.py @@ -0,0 +1,842 @@ +import RNS +import RNS.vendor.umsgpack as msgpack + +import os +import time +import base64 +import multiprocessing + +import LXMF.LXStamper as LXStamper +from .LXMF import APP_NAME, compression_support_from_app_data +from threading import Lock + + +class LXMessage: + GENERATING = 0x00 + OUTBOUND = 0x01 + SENDING = 0x02 + SENT = 0x04 + DELIVERED = 0x08 + REJECTED = 0xFD + CANCELLED = 0xFE + FAILED = 0xFF + states = [GENERATING, OUTBOUND, SENDING, SENT, DELIVERED, REJECTED, CANCELLED, FAILED] + + UNKNOWN = 0x00 + PACKET = 0x01 + RESOURCE = 0x02 + representations = [UNKNOWN, PACKET, RESOURCE] + + OPPORTUNISTIC = 0x01 + DIRECT = 0x02 + PROPAGATED = 0x03 + PAPER = 0x05 + valid_methods = [OPPORTUNISTIC, DIRECT, PROPAGATED, PAPER] + + SOURCE_UNKNOWN = 0x01 + SIGNATURE_INVALID = 0x02 + unverified_reasons = [SOURCE_UNKNOWN, SIGNATURE_INVALID] + + DESTINATION_LENGTH = RNS.Identity.TRUNCATED_HASHLENGTH//8 + SIGNATURE_LENGTH = RNS.Identity.SIGLENGTH//8 + TICKET_LENGTH = RNS.Identity.TRUNCATED_HASHLENGTH//8 + + # Default ticket expiry is 3 weeks, with an + # additional grace period of 5 days, allowing + # for timekeeping inaccuracies. Tickets will + # automatically renew when there is less than + # 14 days to expiry. + TICKET_EXPIRY = 21*24*60*60 + TICKET_GRACE = 5*24*60*60 + TICKET_RENEW = 14*24*60*60 + TICKET_INTERVAL = 1*24*60*60 + COST_TICKET = 0x100 + + # LXMF overhead is 112 bytes per message: + # 16 bytes for destination hash + # 16 bytes for source hash + # 64 bytes for Ed25519 signature + # 8 bytes for timestamp + # 8 bytes for msgpack structure + TIMESTAMP_SIZE = 8 + STRUCT_OVERHEAD = 8 + LXMF_OVERHEAD = 2*DESTINATION_LENGTH + SIGNATURE_LENGTH + TIMESTAMP_SIZE + STRUCT_OVERHEAD + + # With an MTU of 500, the maximum amount of data + # we can send in a single encrypted packet is + # 391 bytes. + ENCRYPTED_PACKET_MDU = RNS.Packet.ENCRYPTED_MDU + TIMESTAMP_SIZE + + # The max content length we can fit in LXMF message + # inside a single RNS packet is the encrypted MDU, minus + # the LXMF overhead. We can optimise a bit though, by + # inferring the destination hash from the destination + # field of the packet, therefore we also add the length + # of a destination hash to the calculation. With default + # RNS and LXMF parameters, the largest single-packet + # LXMF message we can send is 295 bytes. If a message + # is larger than that, a Reticulum link will be used. + ENCRYPTED_PACKET_MAX_CONTENT = ENCRYPTED_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH + + # Links can carry a larger MDU, due to less overhead per + # packet. The link MDU with default Reticulum parameters + # is 431 bytes. + LINK_PACKET_MDU = RNS.Link.MDU + + # Which means that we can deliver single-packet LXMF + # messages with content of up to 319 bytes over a link. + # If a message is larger than that, LXMF will sequence + # and transfer it as a RNS resource over the link instead. + LINK_PACKET_MAX_CONTENT = LINK_PACKET_MDU - LXMF_OVERHEAD + + # For plain packets without encryption, we can + # fit up to 368 bytes of content. + PLAIN_PACKET_MDU = RNS.Packet.PLAIN_MDU + PLAIN_PACKET_MAX_CONTENT = PLAIN_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH + + # Descriptive strings regarding transport encryption + ENCRYPTION_DESCRIPTION_AES = "AES-128" + ENCRYPTION_DESCRIPTION_EC = "Curve25519" + ENCRYPTION_DESCRIPTION_UNENCRYPTED = "Unencrypted" + + # Constants for QR/URI encoding LXMs + URI_SCHEMA = "lxm" + QR_ERROR_CORRECTION = "ERROR_CORRECT_L" + QR_MAX_STORAGE = 2953 + PAPER_MDU = ((QR_MAX_STORAGE-(len(URI_SCHEMA)+len("://")))*6)//8 + + def __str__(self): + if self.hash != None: + return "" + else: + return "" + + def __init__(self, destination, source, content = "", title = "", fields = None, desired_method = None, destination_hash = None, source_hash = None, stamp_cost=None, include_ticket=False): + + if isinstance(destination, RNS.Destination) or destination == None: + self.__destination = destination + if destination != None: self.destination_hash = destination.hash + else: self.destination_hash = destination_hash + + else: raise ValueError("LXMessage initialised with invalid destination") + + if isinstance(source, RNS.Destination) or source == None: + self.__source = source + if source != None: self.source_hash = source.hash + else: self.source_hash = source_hash + + else: raise ValueError("LXMessage initialised with invalid source") + + if title == None: title = "" + + if type(title) == bytes: self.set_title_from_bytes(title) + else: self.set_title_from_string(title) + + if type(content) == bytes: self.set_content_from_bytes(content) + else: self.set_content_from_string(content) + + self.set_fields(fields) + + self.payload = None + self.timestamp = None + self.signature = None + self.hash = None + self.transient_id = None + self.packed = None + self.auto_compress = True + self.state = LXMessage.GENERATING + self.method = LXMessage.UNKNOWN + self.progress = 0.0 + self.rssi = None + self.snr = None + self.q = None + + self.stamp = None + self.stamp_cost = stamp_cost + self.stamp_value = None + self.stamp_valid = False + self.stamp_checked = False + self.propagation_stamp = None + self.propagation_stamp_value = None + self.propagation_stamp_valid = False + self.propagation_target_cost = None + self.defer_stamp = True + self.defer_propagation_stamp = True + self.outbound_ticket = None + self.include_ticket = include_ticket + + self.propagation_packed = None + self.paper_packed = None + + self.incoming = False + self.source_blackholed = False + self.signature_validated = False + self.unverified_reason = None + self.ratchet_id = None + + self.representation = LXMessage.UNKNOWN + self.desired_method = desired_method + self.delivery_attempts = 0 + self.transport_encrypted = False + self.transport_encryption = None + self.ratchet_id = None + self.packet_representation = None + self.resource_representation = None + self.__delivery_destination = None + self.__delivery_callback = None + self.__pn_encrypted_data = None + self.__persist_lock = Lock() + self.failed_callback = None + + self.deferred_stamp_generating = False + + def set_title_from_string(self, title_string): + self.title = title_string.encode("utf-8") + + def set_title_from_bytes(self, title_bytes): + self.title = title_bytes + + def title_as_string(self): + return self.title.decode("utf-8") + + def set_content_from_string(self, content_string): + self.content = content_string.encode("utf-8") + + def set_content_from_bytes(self, content_bytes): + self.content = content_bytes + + def content_as_string(self): + try: + return self.content.decode("utf-8") + except Exception as e: + RNS.log(f"{self} could not decode message content as string: {e}") + return None + + def set_fields(self, fields): + if isinstance(fields, dict) or fields == None: + self.fields = fields or {} + else: + raise ValueError("LXMessage property \"fields\" can only be dict or None") + + def get_fields(self): + return self.fields + + @property + def destination(self): + return self.__destination + + @destination.setter + def destination(self, destination): + self.set_destination(destination) + + def get_destination(self): + return self.destination + + def set_destination(self, destination): + if self.destination == None: + if isinstance(destination, RNS.Destination): + self.__destination = destination + else: + raise ValueError("Invalid destination set on LXMessage") + else: + raise ValueError("Cannot reassign destination on LXMessage") + + @property + def source(self): + return self.__source + + @source.setter + def source(self, source): + self.set_source(source) + + def get_source(self): + return self.source + + def set_source(self, source): + if self.source == None: + if isinstance(source, RNS.Destination): + self.__source = source + else: + raise ValueError("Invalid source set on LXMessage") + else: + raise ValueError("Cannot reassign source on LXMessage") + + def set_delivery_destination(self, delivery_destination): + self.__delivery_destination = delivery_destination + + def register_delivery_callback(self, callback): + self.__delivery_callback = callback + + def register_failed_callback(self, callback): + self.failed_callback = callback + + def validate_stamp(self, target_cost, tickets=None): + if tickets != None: + for ticket in tickets: + try: + if self.stamp == RNS.Identity.truncated_hash(ticket+self.message_id): + RNS.log(f"Stamp on {self} validated by inbound ticket", RNS.LOG_DEBUG) # TODO: Remove at some point + self.stamp_value = LXMessage.COST_TICKET + return True + except Exception as e: + RNS.log(f"Error while validating ticket: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + if self.stamp == None: + return False + else: + workblock = LXStamper.stamp_workblock(self.message_id) + if LXStamper.stamp_valid(self.stamp, target_cost, workblock): + RNS.log(f"Stamp on {self} validated", RNS.LOG_DEBUG) # TODO: Remove at some point + self.stamp_value = LXStamper.stamp_value(workblock, self.stamp) + return True + else: + return False + + def get_stamp(self, timeout=None): + # If an outbound ticket exists, use this for + # generating a valid stamp. + if self.outbound_ticket != None and type(self.outbound_ticket) == bytes and len(self.outbound_ticket) == LXMessage.TICKET_LENGTH: + generated_stamp = RNS.Identity.truncated_hash(self.outbound_ticket+self.message_id) + self.stamp_value = LXMessage.COST_TICKET + RNS.log(f"Generated stamp with outbound ticket {RNS.hexrep(self.outbound_ticket)} for {self}", RNS.LOG_DEBUG) # TODO: Remove at some point + return generated_stamp + + # If no stamp cost is required, we can just + # return immediately. + elif self.stamp_cost == None: + self.stamp_value = None + return None + + # If a stamp was already generated, return + # it immediately. + elif self.stamp != None: + return self.stamp + + # Otherwise, we will need to generate a + # valid stamp according to the cost that + # the receiver has specified. + else: + generated_stamp, value = LXStamper.generate_stamp(self.message_id, self.stamp_cost) + if generated_stamp: + self.stamp_value = value + self.stamp_valid = True + return generated_stamp + + else: + return None + + def get_propagation_stamp(self, target_cost, timeout=None): + # If a stamp was already generated, return + # it immediately. + if self.propagation_stamp != None: + return self.propagation_stamp + + # Otherwise, we will need to generate a + # valid stamp according to the cost that + # the propagation node has specified. + else: + self.propagation_target_cost = target_cost + if self.propagation_target_cost == None: + raise ValueError("Cannot generate propagation stamp without configured target propagation cost") + + + if not self.transient_id: self.pack() + generated_stamp, value = LXStamper.generate_stamp(self.transient_id, target_cost, expand_rounds=LXStamper.WORKBLOCK_EXPAND_ROUNDS_PN) + if generated_stamp: + self.propagation_stamp = generated_stamp + self.propagation_stamp_value = value + self.propagation_stamp_valid = True + return generated_stamp + + else: + return None + + def pack(self, payload_updated=False): + if not self.packed: + if self.timestamp == None: self.timestamp = time.time() + + self.propagation_packed = None + self.paper_packed = None + + self.payload = [self.timestamp, self.title, self.content, self.fields] + + hashed_part = b"" + hashed_part += self.__destination.hash + hashed_part += self.__source.hash + hashed_part += msgpack.packb(self.payload) + self.hash = RNS.Identity.full_hash(hashed_part) + self.message_id = self.hash + + if not self.defer_stamp: + self.stamp = self.get_stamp() + if self.stamp != None: self.payload.append(self.stamp) + + signed_part = b"" + signed_part += hashed_part + signed_part += self.hash + self.signature = self.__source.sign(signed_part) + self.signature_validated = True + + packed_payload = msgpack.packb(self.payload) + self.packed = b"" + self.packed += self.__destination.hash + self.packed += self.__source.hash + self.packed += self.signature + self.packed += packed_payload + self.packed_size = len(self.packed) + content_size = len(packed_payload)-LXMessage.TIMESTAMP_SIZE-LXMessage.STRUCT_OVERHEAD + + # If no desired delivery method has been defined, + # one will be chosen according to these rules: + if self.desired_method == None: + self.desired_method = LXMessage.DIRECT + + # If opportunistic delivery was requested, check + # that message will fit within packet size limits + if self.desired_method == LXMessage.OPPORTUNISTIC: + if self.__destination.type == RNS.Destination.SINGLE: + if content_size > LXMessage.ENCRYPTED_PACKET_MAX_CONTENT: + RNS.log(f"Opportunistic delivery was requested for {self}, but content of length {content_size} exceeds packet size limit. Falling back to link-based delivery.", RNS.LOG_DEBUG) + self.desired_method = LXMessage.DIRECT + + # Set delivery parameters according to delivery method + if self.desired_method == LXMessage.OPPORTUNISTIC: + if self.__destination.type == RNS.Destination.SINGLE: + single_packet_content_limit = LXMessage.ENCRYPTED_PACKET_MAX_CONTENT + elif self.__destination.type == RNS.Destination.PLAIN: + single_packet_content_limit = LXMessage.PLAIN_PACKET_MAX_CONTENT + + if content_size > single_packet_content_limit: + raise TypeError(f"LXMessage desired opportunistic delivery method, but content of length {content_size} exceeds single-packet content limit of {single_packet_content_limit}.") + else: + self.method = LXMessage.OPPORTUNISTIC + self.representation = LXMessage.PACKET + self.__delivery_destination = self.__destination + + elif self.desired_method == LXMessage.DIRECT: + single_packet_content_limit = LXMessage.LINK_PACKET_MAX_CONTENT + if content_size <= single_packet_content_limit: + self.method = self.desired_method + self.representation = LXMessage.PACKET + else: + self.method = self.desired_method + self.representation = LXMessage.RESOURCE + + elif self.desired_method == LXMessage.PROPAGATED: + single_packet_content_limit = LXMessage.LINK_PACKET_MAX_CONTENT + + if self.__pn_encrypted_data == None or payload_updated: + self.__pn_encrypted_data = self.__destination.encrypt(self.packed[LXMessage.DESTINATION_LENGTH:]) + self.ratchet_id = self.__destination.latest_ratchet_id + + lxmf_data = self.packed[:LXMessage.DESTINATION_LENGTH]+self.__pn_encrypted_data + self.transient_id = RNS.Identity.full_hash(lxmf_data) + if self.propagation_stamp != None: lxmf_data += self.propagation_stamp + self.propagation_packed = msgpack.packb([time.time(), [lxmf_data]]) + + content_size = len(self.propagation_packed) + if content_size <= single_packet_content_limit: + self.method = self.desired_method + self.representation = LXMessage.PACKET + else: + self.method = self.desired_method + self.representation = LXMessage.RESOURCE + + elif self.desired_method == LXMessage.PAPER: + paper_content_limit = LXMessage.PAPER_MDU + + encrypted_data = self.__destination.encrypt(self.packed[LXMessage.DESTINATION_LENGTH:]) + self.ratchet_id = self.__destination.latest_ratchet_id + self.paper_packed = self.packed[:LXMessage.DESTINATION_LENGTH]+encrypted_data + + content_size = len(self.paper_packed) + if content_size <= paper_content_limit: + self.method = self.desired_method + self.representation = LXMessage.PAPER + else: + raise TypeError("LXMessage desired paper delivery method, but content exceeds paper message maximum size.") + + else: + raise ValueError("Attempt to re-pack LXMessage "+str(self)+" that was already packed") + + def send(self): + self.determine_transport_encryption() + self.determine_compression_support() + + if self.method == LXMessage.OPPORTUNISTIC: + lxm_packet = self.__as_packet() + lxm_packet.send().set_delivery_callback(self.__mark_delivered) + self.progress = 0.50 + self.ratchet_id = lxm_packet.ratchet_id + self.state = LXMessage.SENT + + elif self.method == LXMessage.DIRECT: + self.state = LXMessage.SENDING + + if self.representation == LXMessage.PACKET: + lxm_packet = self.__as_packet() + receipt = lxm_packet.send() + self.ratchet_id = self.__delivery_destination.link_id + if receipt: + receipt.set_delivery_callback(self.__mark_delivered) + receipt.set_timeout_callback(self.__link_packet_timed_out) + self.progress = 0.50 + else: + if self.__delivery_destination: + self.__delivery_destination.teardown() + + elif self.representation == LXMessage.RESOURCE: + self.resource_representation = self.__as_resource() + self.ratchet_id = self.__delivery_destination.link_id + self.progress = 0.10 + + elif self.method == LXMessage.PROPAGATED: + self.state = LXMessage.SENDING + + if self.representation == LXMessage.PACKET: + receipt = self.__as_packet().send() + if receipt: + receipt.set_delivery_callback(self.__mark_propagated) + receipt.set_timeout_callback(self.__link_packet_timed_out) + self.progress = 0.50 + else: + self.__delivery_destination.teardown() + + elif self.representation == LXMessage.RESOURCE: + self.resource_representation = self.__as_resource() + self.progress = 0.10 + + def determine_compression_support(self): + app_data = RNS.Identity.recall_app_data(self.destination_hash) + if app_data: self.auto_compress = compression_support_from_app_data(app_data) + else: self.auto_compress = True + + ###### TODO: Remove debug logging + if app_data: RNS.log(f"Set compression support from app data to: {self.auto_compress}", RNS.LOG_DEBUG) + else: RNS.log(f"Defaulting compression support to {self.auto_compress}", RNS.LOG_DEBUG) + ###### + + def determine_transport_encryption(self): + # TODO: These descriptions are old and outdated. + # Update the transport encryption descriptions to + # account for ratchets and other changes. + if self.method == LXMessage.OPPORTUNISTIC: + if self.__destination.type == RNS.Destination.SINGLE: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + elif self.__destination.type == RNS.Destination.GROUP: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES + else: + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + elif self.method == LXMessage.DIRECT: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + elif self.method == LXMessage.PROPAGATED: + if self.__destination.type == RNS.Destination.SINGLE: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + elif self.__destination.type == RNS.Destination.GROUP: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES + else: + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + elif self.method == LXMessage.PAPER: + if self.__destination.type == RNS.Destination.SINGLE: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + elif self.__destination.type == RNS.Destination.GROUP: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES + else: + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + else: + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + + def __mark_delivered(self, receipt = None): + RNS.log("Received delivery notification for "+str(self), RNS.LOG_DEBUG) + self.state = LXMessage.DELIVERED + self.progress = 1.0 + + if self.__delivery_callback != None and callable(self.__delivery_callback): + try: + self.__delivery_callback(self) + except Exception as e: + RNS.log("An error occurred in the external delivery callback for "+str(self), RNS.LOG_ERROR) + RNS.trace_exception(e) + + def __mark_propagated(self, receipt = None): + RNS.log("Received propagation success notification for "+str(self), RNS.LOG_DEBUG) + self.state = LXMessage.SENT + self.progress = 1.0 + + if self.__delivery_callback != None and callable(self.__delivery_callback): + try: + self.__delivery_callback(self) + except Exception as e: + RNS.log("An error occurred in the external delivery callback for "+str(self), RNS.LOG_ERROR) + RNS.trace_exception(e) + + def __mark_paper_generated(self, receipt = None): + RNS.log("Paper message generation succeeded for "+str(self), RNS.LOG_DEBUG) + self.state = LXMessage.PAPER + self.progress = 1.0 + + if self.__delivery_callback != None and callable(self.__delivery_callback): + try: + self.__delivery_callback(self) + except Exception as e: + RNS.log("An error occurred in the external delivery callback for "+str(self), RNS.LOG_ERROR) + RNS.trace_exception(e) + + def __resource_concluded(self, resource): + if resource.status == RNS.Resource.COMPLETE: + self.__mark_delivered() + else: + if resource.status == RNS.Resource.REJECTED: + self.state = LXMessage.REJECTED + + elif self.state != LXMessage.CANCELLED: + resource.link.teardown() + self.state = LXMessage.OUTBOUND + + def __propagation_resource_concluded(self, resource): + if resource.status == RNS.Resource.COMPLETE: + self.__mark_propagated() + else: + if self.state != LXMessage.CANCELLED: + resource.link.teardown() + self.state = LXMessage.OUTBOUND + + def __link_packet_timed_out(self, packet_receipt): + if self.state != LXMessage.CANCELLED: + if packet_receipt: + packet_receipt.destination.teardown() + + self.state = LXMessage.OUTBOUND + + def __update_transfer_progress(self, resource): + self.progress = 0.10 + (resource.get_progress()*0.90) + + def __as_packet(self): + if not self.packed: + self.pack() + + if not self.__delivery_destination: + raise ValueError("Can't synthesize packet for LXMF message before delivery destination is known") + + if self.method == LXMessage.OPPORTUNISTIC: + return RNS.Packet(self.__delivery_destination, self.packed[LXMessage.DESTINATION_LENGTH:]) + elif self.method == LXMessage.DIRECT: + return RNS.Packet(self.__delivery_destination, self.packed) + elif self.method == LXMessage.PROPAGATED: + return RNS.Packet(self.__delivery_destination, self.propagation_packed) + + def __as_resource(self): + if not self.packed: + self.pack() + + if not self.__delivery_destination: + raise ValueError("Can't synthesize resource for LXMF message before delivery destination is known") + + if not self.__delivery_destination.type == RNS.Destination.LINK: + raise TypeError("Tried to synthesize resource for LXMF message on a delivery destination that was not a link") + + if not self.__delivery_destination.status == RNS.Link.ACTIVE: + raise ConnectionError("Tried to synthesize resource for LXMF message on a link that was not active") + + if self.method == LXMessage.DIRECT: + return RNS.Resource(self.packed, self.__delivery_destination, callback = self.__resource_concluded, progress_callback = self.__update_transfer_progress, auto_compress=self.auto_compress) + elif self.method == LXMessage.PROPAGATED: + return RNS.Resource(self.propagation_packed, self.__delivery_destination, callback = self.__propagation_resource_concluded, progress_callback = self.__update_transfer_progress) + else: + return None + + def packed_container(self): + if not self.packed: + self.pack() + + container = { + "state": self.state, + "lxmf_bytes": self.packed, + "transport_encrypted": self.transport_encrypted, + "transport_encryption": self.transport_encryption, + "method": self.method + } + + return msgpack.packb(container) + + def write_to_directory(self, directory_path): + file_name = RNS.hexrep(self.hash, delimit=False) + file_path = directory_path+"/"+file_name + tmp_path = file_path+".tmp."+str(os.getpid() or time.time())+"."+RNS.hexrep(os.urandom(8), delimit=False) + + with self.__persist_lock: + try: + with open(tmp_path, "wb") as file: + file.write(self.packed_container()) + file.flush() + try: os.fsync(file.fileno()) + except OSError as e: RNS.log(f"Error while waiting for persist fsync for {self}: {e}", RNS.LOG_WARNING) + + os.replace(tmp_path, file_path) + return file_path + + except Exception as e: + try: + if os.path.exists(tmp_path): os.unlink(tmp_path) + except Exception as e: RNS.log(f"Error while cleaning temporary file {tmp_path} for {self}: {e}", RNS.LOG_ERROR) + + RNS.log(f"Error while writing LXMF message to file \"{file_path}\". The contained exception was: {e}", RNS.LOG_ERROR) + return None + + def as_uri(self, finalise=True): + if not self.packed: + self.pack() + + if self.desired_method == LXMessage.PAPER and self.paper_packed != None: + # Encode packed LXM with URL-safe base64 and remove padding + encoded_bytes = base64.urlsafe_b64encode(self.paper_packed) + + # Add protocol specifier and return + lxm_uri = LXMessage.URI_SCHEMA+"://"+encoded_bytes.decode("utf-8").replace("=","") + + if finalise: + self.determine_transport_encryption() + self.__mark_paper_generated() + + return lxm_uri + + else: + raise TypeError("Attempt to represent LXM with non-paper delivery method as URI") + + def as_qr(self): + if not self.packed: + self.pack() + + if self.desired_method == LXMessage.PAPER and self.paper_packed != None: + import importlib + if importlib.util.find_spec('qrcode') != None: + import qrcode + + qr = qrcode.make( + error_correction = qrcode.constants.__dict__[LXMessage.QR_ERROR_CORRECTION], + border = 1, + data = self.as_uri(finalise=False), + ) + + self.determine_transport_encryption() + self.__mark_paper_generated() + + return qr + + else: + RNS.log("Generating QR-code representanions of LXMs requires the \"qrcode\" module to be installed.", RNS.LOG_CRITICAL) + RNS.log("You can install it with the command: python3 -m pip install qrcode", RNS.LOG_CRITICAL) + return None + + else: + raise TypeError("Attempt to represent LXM with non-paper delivery method as QR-code") + + @staticmethod + def unpack_from_bytes(lxmf_bytes, original_method = None): + destination_hash = lxmf_bytes[:LXMessage.DESTINATION_LENGTH] + source_hash = lxmf_bytes[LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH] + signature = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH] + packed_payload = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH:] + unpacked_payload = msgpack.unpackb(packed_payload) + + # Extract stamp from payload if included + if len(unpacked_payload) > 4: + stamp = unpacked_payload[4] + unpacked_payload = unpacked_payload[:4] + packed_payload = msgpack.packb(unpacked_payload) + else: + stamp = None + + hashed_part = b"" + destination_hash + source_hash + packed_payload + message_hash = RNS.Identity.full_hash(hashed_part) + signed_part = b"" + hashed_part + message_hash + timestamp = unpacked_payload[0] + title_bytes = unpacked_payload[1] + content_bytes = unpacked_payload[2] + fields = unpacked_payload[3] + + destination_identity = RNS.Identity.recall(destination_hash, _no_use=True) + if destination_identity != None: + destination = RNS.Destination(destination_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") + else: + destination = None + + source_identity = RNS.Identity.recall(source_hash, _no_use=True) + if source_identity != None: + source = RNS.Destination(source_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") + else: + source = None + + message = LXMessage( + destination = destination, + source = source, + content = "", + title = "", + fields = fields, + destination_hash = destination_hash, + source_hash = source_hash, + desired_method = original_method) + + message.hash = message_hash + message.message_id = message.hash + message.signature = signature + message.stamp = stamp + message.incoming = True + message.timestamp = timestamp + message.packed = lxmf_bytes + message.packed_size = len(lxmf_bytes) + message.set_title_from_bytes(title_bytes) + message.set_content_from_bytes(content_bytes) + + try: + if source_identity != None: message.source_blackholed = RNS.Reticulum.get_instance().is_blackholed(source_identity) + except Exception as e: RNS.log(f"Could not determine message source blackhole status: {e}", RNS.LOG_WARNING) + + try: + if source: + if source.identity.validate(signature, signed_part): + message.signature_validated = True + else: + message.signature_validated = False + message.unverified_reason = LXMessage.SIGNATURE_INVALID + else: + signature_validated = False + message.unverified_reason = LXMessage.SOURCE_UNKNOWN + RNS.log("Unpacked LXMF message signature could not be validated, since source identity is unknown", RNS.LOG_DEBUG) + except Exception as e: + message.signature_validated = False + RNS.log("Error while validating LXMF message signature. The contained exception was: "+str(e), RNS.LOG_ERROR) + + return message + + @staticmethod + def unpack_from_file(lxmf_file_handle): + try: + container = msgpack.unpackb(lxmf_file_handle.read()) + lxm = LXMessage.unpack_from_bytes(container["lxmf_bytes"]) + + if "state" in container: + lxm.state = container["state"] + if "transport_encrypted" in container: + lxm.transport_encrypted = container["transport_encrypted"] + if "transport_encryption" in container: + lxm.transport_encryption = container["transport_encryption"] + if "method" in container: + lxm.method = container["method"] + + return lxm + except Exception as e: + RNS.log("Could not unpack LXMessage from file. The contained exception was: "+str(e), RNS.LOG_ERROR) + return None \ No newline at end of file diff --git a/LXMF/LXStamper.py b/LXMF/LXStamper.py new file mode 100644 index 0000000..ea25f08 --- /dev/null +++ b/LXMF/LXStamper.py @@ -0,0 +1,523 @@ +import RNS +import RNS.vendor.umsgpack as msgpack + +import os +import sys +import time +import math +import itertools +import contextlib +import multiprocessing + +WORKBLOCK_EXPAND_ROUNDS = 3000 +WORKBLOCK_EXPAND_ROUNDS_PN = 1000 +WORKBLOCK_EXPAND_ROUNDS_PEERING = 25 +STAMP_SIZE = RNS.Identity.HASHLENGTH//8 +PN_VALIDATION_POOL_MIN_SIZE = 256 +USE_WORKER_MANAGER = False + +active_jobs = {} + +if sys.version_info[0] >= 3 and sys.version_info[1] >= 14: + USE_WORKER_MANAGER = True + +@contextlib.contextmanager +def worker_context(ctx, stamp_cost, workblock, message_id): + stop_event = ctx.Event() + result_queue = ctx.Queue(1) + rounds_queue = ctx.Queue() + + active_jobs[message_id] = [stop_event, result_queue] + job_procs = [] + + try: yield stop_event, result_queue, rounds_queue, job_procs + finally: + if message_id in active_jobs: del active_jobs[message_id] + stop_event.set() + + for p in job_procs: + if p.is_alive(): p.terminate() + + for p in job_procs: p.join(timeout=0.5) + + result_queue.close() + result_queue.join_thread() + rounds_queue.close() + rounds_queue.join_thread() + stop_event.clear() + +def stamp_workblock(material, expand_rounds=WORKBLOCK_EXPAND_ROUNDS): + wb_st = time.time() + workblock = b"" + for n in range(expand_rounds): + workblock += RNS.Cryptography.hkdf(length=256, + derive_from=material, + salt=RNS.Identity.full_hash(material+msgpack.packb(n)), + context=None) + wb_time = time.time() - wb_st + # RNS.log(f"Stamp workblock size {RNS.prettysize(len(workblock))}, generated in {round(wb_time*1000,2)}ms", RNS.LOG_DEBUG) + + return workblock + +def stamp_value(workblock, stamp): + value = 0 + bits = 256 + material = RNS.Identity.full_hash(workblock+stamp) + i = int.from_bytes(material, byteorder="big") + while ((i & (1 << (bits - 1))) == 0): + i = (i << 1) + value += 1 + + return value + +def stamp_valid(stamp, target_cost, workblock): + target = 0b1 << 256-target_cost + result = RNS.Identity.full_hash(workblock+stamp) + if int.from_bytes(result, byteorder="big") > target: return False + else: return True + +def validate_peering_key(peering_id, peering_key, target_cost): + workblock = stamp_workblock(peering_id, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PEERING) + if not stamp_valid(peering_key, target_cost, workblock): return False + else: return True + +def validate_pn_stamp(transient_data, target_cost): + from .LXMessage import LXMessage + if len(transient_data) <= LXMessage.LXMF_OVERHEAD+STAMP_SIZE: return None, None, None, None + else: + lxm_data = transient_data[:-STAMP_SIZE] + stamp = transient_data[-STAMP_SIZE:] + transient_id = RNS.Identity.full_hash(lxm_data) + workblock = stamp_workblock(transient_id, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PN) + + if not stamp_valid(stamp, target_cost, workblock): return None, None, None, None + else: + value = stamp_value(workblock, stamp) + return transient_id, lxm_data, value, stamp + +def validate_pn_stamps_job_simple(transient_list, target_cost): + validated_messages = [] + for transient_data in transient_list: + transient_id, lxm_data, value, stamp_data = validate_pn_stamp(transient_data, target_cost) + if transient_id: validated_messages.append([transient_id, lxm_data, value, stamp_data]) + + return validated_messages + +def validate_pn_stamps_job_multip(transient_list, target_cost): + cores = multiprocessing.cpu_count() + pool_count = min(cores, math.ceil(len(transient_list) / PN_VALIDATION_POOL_MIN_SIZE)) + + RNS.log(f"Validating {len(transient_list)} stamps using {pool_count} processes...", RNS.LOG_VERBOSE) + with multiprocessing.get_context("spawn").Pool(pool_count) as p: + validated_entries = p.starmap(validate_pn_stamp, zip(transient_list, itertools.repeat(target_cost))) + + RNS.log(f"Validation pool completed for {len(transient_list)} stamps", RNS.LOG_VERBOSE) + + return [e for e in validated_entries if e[0] != None] + +def validate_pn_stamps(transient_list, target_cost): + non_mp_platform = RNS.vendor.platformutils.is_android() + if len(transient_list) <= PN_VALIDATION_POOL_MIN_SIZE or non_mp_platform: return validate_pn_stamps_job_simple(transient_list, target_cost) + else: return validate_pn_stamps_job_multip(transient_list, target_cost) + +def generate_stamp(message_id, stamp_cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS): + RNS.log(f"Generating stamp with cost {stamp_cost} for {RNS.prettyhexrep(message_id)}...", RNS.LOG_DEBUG) + workblock = stamp_workblock(message_id, expand_rounds=expand_rounds) + + start_time = time.time() + stamp = None + rounds = 0 + value = 0 + + if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): stamp, rounds = job_simple(stamp_cost, workblock, message_id) + elif RNS.vendor.platformutils.is_android(): stamp, rounds = job_android(stamp_cost, workblock, message_id) + else: + if USE_WORKER_MANAGER: stamp, rounds = job_linux_managed(stamp_cost, workblock, message_id) + else: stamp, rounds = job_linux(stamp_cost, workblock, message_id) + + duration = time.time() - start_time + speed = rounds/duration + if stamp != None: value = stamp_value(workblock, stamp) + + RNS.log(f"Stamp with value {value} generated in {RNS.prettytime(duration)}, {rounds} rounds, {int(speed)} rounds per second", RNS.LOG_DEBUG) + + return stamp, value + +def cancel_work(message_id): + if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): + try: + if message_id in active_jobs: + active_jobs[message_id] = True + + except Exception as e: + RNS.log("Error while terminating stamp generation workers: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + elif RNS.vendor.platformutils.is_android(): + try: + if message_id in active_jobs: + active_jobs[message_id] = True + + except Exception as e: + RNS.log("Error while terminating stamp generation workers: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + else: + try: + if message_id in active_jobs: + stop_event = active_jobs[message_id][0] + result_queue = active_jobs[message_id][1] + stop_event.set() + result_queue.put(None) + active_jobs.pop(message_id) + + except Exception as e: + RNS.log("Error while terminating stamp generation workers: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + +def job_simple(stamp_cost, workblock, message_id): + # A simple, single-process stamp generator. + # should work on any platform, and is used + # as a fall-back, in case of limited multi- + # processing and/or acceleration support. + + platform = RNS.vendor.platformutils.get_platform() + RNS.log(f"Running stamp generation on {platform}, work limited to single CPU core. This will be slower than ideal.", RNS.LOG_WARNING) + + rounds = 0 + pstamp = os.urandom(256//8) + st = time.time() + + active_jobs[message_id] = False; + + def sv(s, c, w): + target = 0b1<<256-c; m = w+s + result = RNS.Identity.full_hash(m) + if int.from_bytes(result, byteorder="big") > target: return False + else: return True + + while not sv(pstamp, stamp_cost, workblock) and not active_jobs[message_id]: + pstamp = os.urandom(256//8); rounds += 1 + if rounds % 2500 == 0: + speed = rounds / (time.time()-st) + RNS.log(f"Stamp generation running. {rounds} rounds completed so far, {int(speed)} rounds per second", RNS.LOG_DEBUG) + + if active_jobs[message_id] == True: + pstamp = None + + active_jobs.pop(message_id) + + return pstamp, rounds + +def job_linux_managed(stamp_cost, workblock, message_id): + ctx = multiprocessing.get_context("fork") + cores = multiprocessing.cpu_count() + jobs = cores if cores <= 12 else int(cores/2) + + allow_kill = True + stamp = None + total_rounds = 0 + + with worker_context(ctx, stamp_cost, workblock, message_id) as (stop_event, result_queue, rounds_queue, job_procs): + RNS.log(f"Starting {jobs} stamp generation workers", RNS.LOG_DEBUG) + + def job(stop_event, sc, wb, worker_id): + terminated = False + rounds = 0 + pstamp = os.urandom(256//8) + + def sv(s, c, w): + target = 0b1<<256-c; m = w+s + result = RNS.Identity.full_hash(m) + if int.from_bytes(result, byteorder="big") > target: return False + else: return True + + while not stop_event.is_set() and not sv(pstamp, sc, wb): + pstamp = os.urandom(256//8); rounds += 1 + + if not stop_event.is_set(): + stop_event.set() + try: result_queue.put_nowait(pstamp) + except: pass + + try: rounds_queue.put_nowait(rounds) + except: pass + + for jpn in range(jobs): + p = ctx.Process(target=job, args=(stop_event, stamp_cost, workblock, jpn), daemon=True) + job_procs.append(p) + p.start() + + try: stamp = result_queue.get(timeout=None) + except Exception as e: + RNS.log(f"Failed to get result from workers: {e}", RNS.LOG_ERROR) + stamp = None + + # Collect any potential spurious + # results from worker queue. + try: + while True: result_queue.get_nowait() + except: pass + + for j in range(jobs): + nrounds = 0 + try: + nrounds = rounds_queue.get(timeout=2) + except Exception as e: + RNS.log(f"Failed to get round stats part {j}: {e}", RNS.LOG_ERROR) + total_rounds += nrounds + + all_exited = False + exit_timeout = time.time() + 5 + while time.time() < exit_timeout: + if not any(p.is_alive() for p in job_procs): + all_exited = True + break + time.sleep(0.1) + + if not all_exited: + RNS.log("Stamp generation IPC timeout, possible worker deadlock. Terminating remaining processes.", RNS.LOG_ERROR) + if allow_kill: + for j in range(jobs): + process = job_procs[j] + process.kill() + else: + return None + + else: + for j in range(jobs): + process = job_procs[j] + process.join() + # RNS.log(f"Joined {j} / {process}", RNS.LOG_DEBUG) # TODO: Remove + + return stamp, total_rounds + + +def job_linux(stamp_cost, workblock, message_id): + allow_kill = True + stamp = None + total_rounds = 0 + cores = multiprocessing.cpu_count() + jobs = cores if cores <= 12 else int(cores/2) + stop_event = multiprocessing.Event() + result_queue = multiprocessing.Queue(1) + rounds_queue = multiprocessing.Queue() + + def job(stop_event, pn, sc, wb): + terminated = False + rounds = 0 + pstamp = os.urandom(256//8) + + def sv(s, c, w): + target = 0b1<<256-c; m = w+s + result = RNS.Identity.full_hash(m) + if int.from_bytes(result, byteorder="big") > target: + return False + else: + return True + + while not stop_event.is_set() and not sv(pstamp, sc, wb): + pstamp = os.urandom(256//8); rounds += 1 + + if not stop_event.is_set(): + stop_event.set() + result_queue.put(pstamp) + rounds_queue.put(rounds) + + job_procs = [] + RNS.log(f"Starting {jobs} stamp generation workers", RNS.LOG_DEBUG) + for jpn in range(jobs): + process = multiprocessing.get_context("fork").Process(target=job, kwargs={"stop_event": stop_event, "pn": jpn, "sc": stamp_cost, "wb": workblock}, daemon=True) + job_procs.append(process) + process.start() + + active_jobs[message_id] = [stop_event, result_queue] + + stamp = result_queue.get() + + # Collect any potential spurious + # results from worker queue. + try: + while True: result_queue.get_nowait() + except: pass + + for j in range(jobs): + nrounds = 0 + try: + nrounds = rounds_queue.get(timeout=2) + except Exception as e: + RNS.log(f"Failed to get round stats part {j}: {e}", RNS.LOG_ERROR) + total_rounds += nrounds + + all_exited = False + exit_timeout = time.time() + 5 + while time.time() < exit_timeout: + if not any(p.is_alive() for p in job_procs): + all_exited = True + break + time.sleep(0.1) + + if not all_exited: + RNS.log("Stamp generation IPC timeout, possible worker deadlock. Terminating remaining processes.", RNS.LOG_ERROR) + if allow_kill: + for j in range(jobs): + process = job_procs[j] + process.kill() + else: + return None + + else: + for j in range(jobs): + process = job_procs[j] + process.join() + # RNS.log(f"Joined {j} / {process}", RNS.LOG_DEBUG) # TODO: Remove + + return stamp, total_rounds + +def job_android(stamp_cost, workblock, message_id): + # Semaphore support is flaky to non-existent on + # Android, so we need to manually dispatch and + # manage workloads here, while periodically + # checking in on the progress. + + stamp = None + start_time = time.time() + total_rounds = 0 + rounds_per_worker = 1000 + + use_nacl = False + try: + import nacl.encoding + import nacl.hash + use_nacl = True + except: + pass + + if use_nacl: + def full_hash(m): + return nacl.hash.sha256(m, encoder=nacl.encoding.RawEncoder) + else: + def full_hash(m): + return RNS.Identity.full_hash(m) + + def sv(s, c, w): + target = 0b1<<256-c + m = w+s + result = full_hash(m) + if int.from_bytes(result, byteorder="big") > target: + return False + else: + return True + + wm = multiprocessing.Manager() + jobs = multiprocessing.cpu_count() + + def job(procnum=None, results_dict=None, wb=None, sc=None, jr=None): + # RNS.log(f"Worker {procnum} starting for {jr} rounds...") # TODO: Remove + try: + rounds = 0 + found_stamp = None + + while True: + pstamp = os.urandom(256//8) + rounds += 1 + if sv(pstamp, sc, wb): + found_stamp = pstamp + break + + if rounds >= jr: + # RNS.log(f"Worker {procnum} found no result in {rounds} rounds") # TODO: Remove + break + + results_dict[procnum] = [found_stamp, rounds] + except Exception as e: + RNS.log(f"Stamp generation worker error: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + active_jobs[message_id] = False; + + RNS.log(f"Dispatching {jobs} workers for stamp generation...", RNS.LOG_DEBUG) # TODO: Remove + + results_dict = wm.dict() + while stamp == None and active_jobs[message_id] == False: + job_procs = [] + try: + for pnum in range(jobs): + pargs = {"procnum":pnum, "results_dict": results_dict, "wb": workblock, "sc":stamp_cost, "jr":rounds_per_worker} + process = multiprocessing.Process(target=job, kwargs=pargs) + job_procs.append(process) + process.start() + + for process in job_procs: + process.join() + + for j in results_dict: + r = results_dict[j] + total_rounds += r[1] + if r[0] != None: + stamp = r[0] + + if stamp == None: + elapsed = time.time() - start_time + speed = total_rounds/elapsed + RNS.log(f"Stamp generation running. {total_rounds} rounds completed so far, {int(speed)} rounds per second", RNS.LOG_DEBUG) + + except Exception as e: + RNS.log(f"Stamp generation job error: {e}") + RNS.trace_exception(e) + + active_jobs.pop(message_id) + + return stamp, total_rounds + +# def stamp_value_linear(workblock, stamp): +# value = 0 +# bits = 256 +# material = RNS.Identity.full_hash(workblock+stamp) +# s = int.from_bytes(material, byteorder="big") +# return s.bit_count() + +if __name__ == "__main__": + import sys + if len(sys.argv) < 2: + RNS.log("No cost argument provided", RNS.LOG_ERROR) + exit(1) + else: + try: cost = int(sys.argv[1]) + except Exception as e: + RNS.log(f"Invalid cost argument provided: {e}", RNS.LOG_ERROR) + exit(1) + + rounds = 1 + if len(sys.argv) > 2: + try: rounds = int(sys.argv[2]) + except Exception as e: + RNS.log(f"Invalid rounds argument provided: {e}", RNS.LOG_ERROR) + exit(1) + + RNS.loglevel = RNS.LOG_DEBUG + + for i in range(rounds): + RNS.log("", RNS.LOG_DEBUG) + RNS.log(f"Round {i+1} of {rounds}", RNS.LOG_DEBUG) + + RNS.log("Testing LXMF stamp generation", RNS.LOG_DEBUG) + message_id = os.urandom(32) + generate_stamp(message_id, cost) + + RNS.log("", RNS.LOG_DEBUG) + RNS.log("Testing propagation stamp generation", RNS.LOG_DEBUG) + message_id = os.urandom(32) + generate_stamp(message_id, cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PN) + + RNS.log("", RNS.LOG_DEBUG) + RNS.log("Testing peering key generation", RNS.LOG_DEBUG) + message_id = os.urandom(32) + generate_stamp(message_id, cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PEERING) + + # transient_list = [] + # st = time.time(); count = 10000 + # for i in range(count): transient_list.append(os.urandom(256)) + # validate_pn_stamps(transient_list, 5) + # dt = time.time()-st; mps = count/dt + # RNS.log(f"Validated {count} PN stamps in {RNS.prettytime(dt)}, {round(mps,1)} m/s", RNS.LOG_DEBUG) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py new file mode 100644 index 0000000..5c797c4 --- /dev/null +++ b/LXMF/Utilities/lxmd.py @@ -0,0 +1,1126 @@ +#!/usr/bin/env python3 + +# Reticulum License +# +# Copyright (c) 2020-2025 Mark Qvist +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# - The Software shall not be used in any kind of system which includes amongst +# its functions the ability to purposefully do harm to human beings. +# +# - The Software shall not be used, directly or indirectly, in the creation of +# an artificial intelligence, machine learning or language model training +# dataset, including but not limited to any use that contributes to the +# training or development of such a model or algorithm. +# +# - The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +DEFFERED_JOBS_DELAY = 10 +JOBS_INTERVAL = 5 + +import RNS +import LXMF +import argparse +import threading +import subprocess +import shlex +import time +import os + +from LXMF._version import __version__ +from LXMF import APP_NAME + +from RNS.vendor.configobj import ConfigObj + +configpath = None +ignoredpath = None +identitypath = None +storagedir = None +lxmdir = None +targetloglevel = None + +identity = None +lxmd_config = None +message_router = None +lxmf_destination = None +active_configuration = {} + +last_peer_announce = None +last_node_announce = None + +def create_default_config(configpath): + lxmd_config = ConfigObj(__default_lxmd_config__.splitlines()) + lxmd_config.filename = configpath + lxmd_config.write() + +def apply_config(): + global active_configuration, targetloglevel + try: + # Load peer settings + if "lxmf" in lxmd_config and "display_name" in lxmd_config["lxmf"]: + active_configuration["display_name"] = lxmd_config["lxmf"]["display_name"] + else: + active_configuration["display_name"] = "Anonymous Peer" + + if "lxmf" in lxmd_config and "announce_at_start" in lxmd_config["lxmf"]: + active_configuration["peer_announce_at_start"] = lxmd_config["lxmf"].as_bool("announce_at_start") + else: + active_configuration["peer_announce_at_start"] = False + + if "lxmf" in lxmd_config and "announce_interval" in lxmd_config["lxmf"]: + active_configuration["peer_announce_interval"] = lxmd_config["lxmf"].as_int("announce_interval")*60 + else: + active_configuration["peer_announce_interval"] = None + + if "lxmf" in lxmd_config and "delivery_transfer_max_accepted_size" in lxmd_config["lxmf"]: + active_configuration["delivery_transfer_max_accepted_size"] = lxmd_config["lxmf"].as_float("delivery_transfer_max_accepted_size") + if active_configuration["delivery_transfer_max_accepted_size"] < 0.38: + active_configuration["delivery_transfer_max_accepted_size"] = 0.38 + else: + active_configuration["delivery_transfer_max_accepted_size"] = 1000 + + if "lxmf" in lxmd_config and "on_inbound" in lxmd_config["lxmf"]: + active_configuration["on_inbound"] = lxmd_config["lxmf"]["on_inbound"] + else: + active_configuration["on_inbound"] = None + + # Load propagation node settings + if "propagation" in lxmd_config and "enable_node" in lxmd_config["propagation"]: + active_configuration["enable_propagation_node"] = lxmd_config["propagation"].as_bool("enable_node") + else: + active_configuration["enable_propagation_node"] = False + + if "propagation" in lxmd_config and "node_name" in lxmd_config["propagation"]: + active_configuration["node_name"] = lxmd_config["propagation"].get("node_name") + else: + active_configuration["node_name"] = None + + if "propagation" in lxmd_config and "auth_required" in lxmd_config["propagation"]: + active_configuration["auth_required"] = lxmd_config["propagation"].as_bool("auth_required") + else: + active_configuration["auth_required"] = False + + if "propagation" in lxmd_config and "announce_at_start" in lxmd_config["propagation"]: + active_configuration["node_announce_at_start"] = lxmd_config["propagation"].as_bool("announce_at_start") + else: + active_configuration["node_announce_at_start"] = False + + if "propagation" in lxmd_config and "autopeer" in lxmd_config["propagation"]: + active_configuration["autopeer"] = lxmd_config["propagation"].as_bool("autopeer") + else: + active_configuration["autopeer"] = True + + if "propagation" in lxmd_config and "autopeer_maxdepth" in lxmd_config["propagation"]: + active_configuration["autopeer_maxdepth"] = lxmd_config["propagation"].as_int("autopeer_maxdepth") + else: + active_configuration["autopeer_maxdepth"] = None + + if "propagation" in lxmd_config and "announce_interval" in lxmd_config["propagation"]: + active_configuration["node_announce_interval"] = lxmd_config["propagation"].as_int("announce_interval")*60 + else: + active_configuration["node_announce_interval"] = None + + if "propagation" in lxmd_config and "message_storage_limit" in lxmd_config["propagation"]: + active_configuration["message_storage_limit"] = lxmd_config["propagation"].as_float("message_storage_limit") + if active_configuration["message_storage_limit"] < 0.005: + active_configuration["message_storage_limit"] = 0.005 + else: + active_configuration["message_storage_limit"] = 500 + + if "propagation" in lxmd_config and "propagation_transfer_max_accepted_size" in lxmd_config["propagation"]: + active_configuration["propagation_transfer_max_accepted_size"] = lxmd_config["propagation"].as_float("propagation_transfer_max_accepted_size") + if active_configuration["propagation_transfer_max_accepted_size"] < 0.38: + active_configuration["propagation_transfer_max_accepted_size"] = 0.38 + else: + active_configuration["propagation_transfer_max_accepted_size"] = 256 + + if "propagation" in lxmd_config and "propagation_message_max_accepted_size" in lxmd_config["propagation"]: + active_configuration["propagation_transfer_max_accepted_size"] = lxmd_config["propagation"].as_float("propagation_message_max_accepted_size") + if active_configuration["propagation_transfer_max_accepted_size"] < 0.38: + active_configuration["propagation_transfer_max_accepted_size"] = 0.38 + else: + active_configuration["propagation_transfer_max_accepted_size"] = 256 + + if "propagation" in lxmd_config and "propagation_sync_max_accepted_size" in lxmd_config["propagation"]: + active_configuration["propagation_sync_max_accepted_size"] = lxmd_config["propagation"].as_float("propagation_sync_max_accepted_size") + if active_configuration["propagation_sync_max_accepted_size"] < 0.38: + active_configuration["propagation_sync_max_accepted_size"] = 0.38 + else: + active_configuration["propagation_sync_max_accepted_size"] = 256*40 + + if "propagation" in lxmd_config and "propagation_stamp_cost_target" in lxmd_config["propagation"]: + active_configuration["propagation_stamp_cost_target"] = lxmd_config["propagation"].as_int("propagation_stamp_cost_target") + if active_configuration["propagation_stamp_cost_target"] < LXMF.LXMRouter.PROPAGATION_COST_MIN: + active_configuration["propagation_stamp_cost_target"] = LXMF.LXMRouter.PROPAGATION_COST_MIN + else: + active_configuration["propagation_stamp_cost_target"] = LXMF.LXMRouter.PROPAGATION_COST + + if "propagation" in lxmd_config and "propagation_stamp_cost_flexibility" in lxmd_config["propagation"]: + active_configuration["propagation_stamp_cost_flexibility"] = lxmd_config["propagation"].as_int("propagation_stamp_cost_flexibility") + if active_configuration["propagation_stamp_cost_flexibility"] < 0: + active_configuration["propagation_stamp_cost_flexibility"] = 0 + else: + active_configuration["propagation_stamp_cost_flexibility"] = LXMF.LXMRouter.PROPAGATION_COST_FLEX + + if "propagation" in lxmd_config and "peering_cost" in lxmd_config["propagation"]: + active_configuration["peering_cost"] = lxmd_config["propagation"].as_int("peering_cost") + if active_configuration["peering_cost"] < 0: + active_configuration["peering_cost"] = 0 + else: + active_configuration["peering_cost"] = LXMF.LXMRouter.PEERING_COST + + if "propagation" in lxmd_config and "remote_peering_cost_max" in lxmd_config["propagation"]: + active_configuration["remote_peering_cost_max"] = lxmd_config["propagation"].as_int("remote_peering_cost_max") + if active_configuration["remote_peering_cost_max"] < 0: + active_configuration["remote_peering_cost_max"] = 0 + else: + active_configuration["remote_peering_cost_max"] = LXMF.LXMRouter.MAX_PEERING_COST + + if "propagation" in lxmd_config and "prioritise_destinations" in lxmd_config["propagation"]: + active_configuration["prioritised_lxmf_destinations"] = lxmd_config["propagation"].as_list("prioritise_destinations") + else: + active_configuration["prioritised_lxmf_destinations"] = [] + + if "propagation" in lxmd_config and "control_allowed" in lxmd_config["propagation"]: + active_configuration["control_allowed_identities"] = lxmd_config["propagation"].as_list("control_allowed") + else: + active_configuration["control_allowed_identities"] = [] + + if "propagation" in lxmd_config and "static_peers" in lxmd_config["propagation"]: + static_peers = lxmd_config["propagation"].as_list("static_peers") + active_configuration["static_peers"] = [] + for static_peer in static_peers: + active_configuration["static_peers"].append(bytes.fromhex(static_peer)) + else: + active_configuration["static_peers"] = [] + + if "propagation" in lxmd_config and "max_peers" in lxmd_config["propagation"]: + active_configuration["max_peers"] = lxmd_config["propagation"].as_int("max_peers") + else: + active_configuration["max_peers"] = None + + if "propagation" in lxmd_config and "from_static_only" in lxmd_config["propagation"]: + active_configuration["from_static_only"] = lxmd_config["propagation"].as_bool("from_static_only") + else: + active_configuration["from_static_only"] = False + + # Load various settings + if "logging" in lxmd_config and "loglevel" in lxmd_config["logging"]: + targetloglevel = lxmd_config["logging"].as_int("loglevel") + + active_configuration["ignored_lxmf_destinations"] = [] + if os.path.isfile(ignoredpath): + try: + fh = open(ignoredpath, "rb") + ignored_input = fh.read() + fh.close() + + ignored_hash_strs = ignored_input.splitlines() + + for hash_str in ignored_hash_strs: + if len(hash_str) == RNS.Identity.TRUNCATED_HASHLENGTH//8*2: + try: + ignored_hash = bytes.fromhex(hash_str.decode("utf-8")) + active_configuration["ignored_lxmf_destinations"].append(ignored_hash) + + except Exception as e: + RNS.log("Could not decode hash from: "+str(hash_str), RNS.LOG_DEBUG) + RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + + except Exception as e: + RNS.log("Error while loading list of ignored destinations: "+str(e), RNS.LOG_ERROR) + + active_configuration["allowed_identities"] = [] + if os.path.isfile(allowedpath): + try: + fh = open(allowedpath, "rb") + allowed_input = fh.read() + fh.close() + + allowed_hash_strs = allowed_input.splitlines() + + for hash_str in allowed_hash_strs: + if len(hash_str) == RNS.Identity.TRUNCATED_HASHLENGTH//8*2: + try: + allowed_hash = bytes.fromhex(hash_str.decode("utf-8")) + active_configuration["allowed_identities"].append(allowed_hash) + + except Exception as e: + RNS.log("Could not decode hash from: "+str(hash_str), RNS.LOG_DEBUG) + RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + + except Exception as e: + RNS.log("Error while loading list of allowed identities: "+str(e), RNS.LOG_ERROR) + + except Exception as e: + RNS.log("Could not apply LXM Daemon configuration. The contained exception was: "+str(e), RNS.LOG_ERROR) + raise e + exit(3) + +def lxmf_delivery(lxm): + global active_configuration, lxmdir + + try: + written_path = lxm.write_to_directory(lxmdir) + RNS.log("Received "+str(lxm)+" written to "+str(written_path), RNS.LOG_DEBUG) + + if active_configuration["on_inbound"]: + RNS.log("Calling external program to handle message", RNS.LOG_DEBUG) + command = active_configuration["on_inbound"] + processing_command = command+" \""+written_path+"\"" + return_code = subprocess.call(shlex.split(processing_command), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + else: + RNS.log("No action defined for inbound messages, ignoring", RNS.LOG_DEBUG) + + except Exception as e: + RNS.log("Error occurred while processing received message "+str(lxm)+". The contained exception was: "+str(e), RNS.LOG_ERROR) + + +def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbound = None, verbosity = 0, quietness = 0, service = False): + global configpath, ignoredpath, identitypath, allowedpath, storagedir, lxmdir + global lxmd_config, active_configuration, targetloglevel + global message_router, lxmf_destination + + if service: + targetlogdest = RNS.LOG_FILE + targetloglevel = None + else: + targetlogdest = RNS.LOG_STDOUT + + # Get configuration + if configdir == None: + if os.path.isdir("/etc/lxmd") and os.path.isfile("/etc/lxmd/config"): + configdir = "/etc/lxmd" + elif os.path.isdir(RNS.Reticulum.userdir+"/.config/lxmd") and os.path.isfile(RNS.Reticulum.userdir+"/.config/lxmd/config"): + configdir = RNS.Reticulum.userdir+"/.config/lxmd" + else: + configdir = RNS.Reticulum.userdir+"/.lxmd" + + configpath = configdir+"/config" + ignoredpath = configdir+"/ignored" + allowedpath = configdir+"/allowed" + identitypath = configdir+"/identity" + storagedir = configdir+"/storage" + lxmdir = storagedir+"/messages" + + if not os.path.isdir(storagedir): os.makedirs(storagedir) + if not os.path.isdir(lxmdir): os.makedirs(lxmdir) + + if not os.path.isfile(configpath): + RNS.log("Could not load config file, creating default configuration file...") + create_default_config(configpath) + RNS.log("Default config file created. Make any necessary changes in "+configpath+" and restart lxmd if needed.") + time.sleep(1.5) + + if os.path.isfile(configpath): + try: + lxmd_config = ConfigObj(configpath) + except Exception as e: + RNS.log("Could not parse the configuration at "+configpath, RNS.LOG_ERROR) + RNS.log("Check your configuration file for errors!", RNS.LOG_ERROR) + RNS.panic() + + apply_config() + RNS.log("Configuration loaded from "+configpath, RNS.LOG_VERBOSE) + + if targetloglevel == None: + targetloglevel = 3 + + if verbosity != 0 or quietness != 0: + targetloglevel = targetloglevel+verbosity-quietness + + # Start Reticulum + RNS.log("Substantiating Reticulum...") + reticulum = RNS.Reticulum(configdir=rnsconfigdir, loglevel=targetloglevel, logdest=targetlogdest) + if targetlogdest == RNS.LOG_FILE: RNS.logfile = configdir+"/logfile" + + # Generate or load primary identity + if os.path.isfile(identitypath): + try: + identity = RNS.Identity.from_file(identitypath) + if identity != None: + RNS.log("Loaded Primary Identity %s" % (str(identity))) + else: + RNS.log("Could not load the Primary Identity from "+identitypath, RNS.LOG_ERROR) + exit(4) + except Exception as e: + RNS.log("Could not load the Primary Identity from "+identitypath, RNS.LOG_ERROR) + RNS.log("The contained exception was: %s" % (str(e)), RNS.LOG_ERROR) + exit(1) + else: + try: + RNS.log("No Primary Identity file found, creating new...") + identity = RNS.Identity() + identity.to_file(identitypath) + RNS.log("Created new Primary Identity %s" % (str(identity))) + except Exception as e: + RNS.log("Could not create and save a new Primary Identity", RNS.LOG_ERROR) + RNS.log("The contained exception was: %s" % (str(e)), RNS.LOG_ERROR) + exit(2) + + # Start LXMF + message_router = LXMF.LXMRouter( + identity = identity, + storagepath = storagedir, + autopeer = active_configuration["autopeer"], + autopeer_maxdepth = active_configuration["autopeer_maxdepth"], + propagation_limit = active_configuration["propagation_transfer_max_accepted_size"], + propagation_cost = active_configuration["propagation_stamp_cost_target"], + propagation_cost_flexibility = active_configuration["propagation_stamp_cost_flexibility"], + peering_cost = active_configuration["peering_cost"], + max_peering_cost = active_configuration["remote_peering_cost_max"], + sync_limit = active_configuration["propagation_sync_max_accepted_size"], + delivery_limit = active_configuration["delivery_transfer_max_accepted_size"], + max_peers = active_configuration["max_peers"], + static_peers = active_configuration["static_peers"], + from_static_only = active_configuration["from_static_only"], + name = active_configuration["node_name"]) + + message_router.register_delivery_callback(lxmf_delivery) + + for destination_hash in active_configuration["ignored_lxmf_destinations"]: + message_router.ignore_destination(destination_hash) + + lxmf_destination = message_router.register_delivery_identity(identity, display_name=active_configuration["display_name"]) + + RNS.Identity.remember( + packet_hash=None, + destination_hash=lxmf_destination.hash, + public_key=identity.get_public_key(), + app_data=None + ) + + # Set up authentication + if active_configuration["auth_required"]: + message_router.set_authentication(required=True) + + if len(active_configuration["allowed_identities"]) == 0: + RNS.log("Clint authentication was enabled, but no identity hashes could be loaded from "+str(allowedpath)+". Nobody will be able to sync messages from this propagation node.", RNS.LOG_WARNING) + + for identity_hash in active_configuration["allowed_identities"]: + message_router.allow(identity_hash) + + RNS.log("LXMF Router ready to receive on "+RNS.prettyhexrep(lxmf_destination.hash)) + + if run_pn or active_configuration["enable_propagation_node"]: + message_router.set_message_storage_limit(megabytes=active_configuration["message_storage_limit"]) + for dest_str in active_configuration["prioritised_lxmf_destinations"]: + try: + dest_hash = bytes.fromhex(dest_str) + if len(dest_hash) == RNS.Reticulum.TRUNCATED_HASHLENGTH//8: message_router.prioritise(dest_hash) + except Exception as e: RNS.log("Cannot prioritise "+str(dest_str)+", it is not a valid destination hash", RNS.LOG_ERROR) + + message_router.enable_propagation() + + for ident_str in active_configuration["control_allowed_identities"]: + try: + identity_hash = bytes.fromhex(ident_str) + if len(identity_hash) == RNS.Reticulum.TRUNCATED_HASHLENGTH//8: message_router.allow_control(identity_hash) + except Exception as e: RNS.log(f"Cannot allow control from {ident_str}, it is not a valid identity hash", RNS.LOG_ERROR) + + RNS.log("LXMF Propagation Node started on "+RNS.prettyhexrep(message_router.propagation_destination.hash)) + + RNS.log("Started lxmd version {version}".format(version=__version__), RNS.LOG_NOTICE) + + threading.Thread(target=deferred_start_jobs, daemon=True).start() + + while True: + time.sleep(1) + +def jobs(): + global active_configuration, last_peer_announce, last_node_announce + global message_router, lxmf_destination + + while True: + try: + if "peer_announce_interval" in active_configuration and active_configuration["peer_announce_interval"] != None: + if time.time() > last_peer_announce + active_configuration["peer_announce_interval"]: + RNS.log("Sending announce for LXMF delivery destination", RNS.LOG_VERBOSE) + message_router.announce(lxmf_destination.hash) + last_peer_announce = time.time() + + if "node_announce_interval" in active_configuration and active_configuration["node_announce_interval"] != None: + if time.time() > last_node_announce + active_configuration["node_announce_interval"]: + RNS.log("Sending announce for LXMF Propagation Node", RNS.LOG_VERBOSE) + message_router.announce_propagation_node() + last_node_announce = time.time() + + except Exception as e: + RNS.log("An error occurred while running periodic jobs. The contained exception was: "+str(e), RNS.LOG_ERROR) + + time.sleep(JOBS_INTERVAL) + +def deferred_start_jobs(): + global active_configuration, last_peer_announce, last_node_announce + global message_router, lxmf_destination + time.sleep(DEFFERED_JOBS_DELAY) + RNS.log("Running deferred start jobs", RNS.LOG_DEBUG) + if active_configuration["peer_announce_at_start"]: + RNS.log("Sending announce for LXMF delivery destination", RNS.LOG_EXTREME) + message_router.announce(lxmf_destination.hash) + + if active_configuration["node_announce_at_start"]: + RNS.log("Sending announce for LXMF Propagation Node", RNS.LOG_EXTREME) + message_router.announce_propagation_node() + + last_peer_announce = time.time() + last_node_announce = time.time() + threading.Thread(target=jobs, daemon=True).start() + +def _request_sync(identity, destination_hash, remote_identity, timeout=15, exit_on_fail=False): + control_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") + + timeout = time.time()+timeout + def check_timeout(): + if time.time() > timeout: + if exit_on_fail: + print("Requesting lxmd peer sync timed out, exiting now") + exit(200) + else: + return LXMF.LXMPeer.LXMPeer.ERROR_TIMEOUT + else: + time.sleep(0.1) + + if not RNS.Transport.has_path(control_destination.hash): + RNS.Transport.request_path(control_destination.hash) + while not RNS.Transport.has_path(control_destination.hash): + tc = check_timeout() + if tc: + return tc + + link = RNS.Link(control_destination) + while not link.status == RNS.Link.ACTIVE: + tc = check_timeout() + if tc: + return tc + + link.identify(identity) + request_receipt = link.request(LXMF.LXMRouter.SYNC_REQUEST_PATH, data=destination_hash, response_callback=None, failed_callback=None) + while not request_receipt.get_status() == RNS.RequestReceipt.READY: + tc = check_timeout() + if tc: + return tc + + link.teardown() + return request_receipt.get_response() + + +def request_sync(target, remote=None, configdir=None, rnsconfigdir=None, verbosity=0, quietness=0, timeout=15, identity_path=None): + global configpath, identitypath, storagedir, lxmdir + global lxmd_config, active_configuration, targetloglevel + + try: + peer_destination_hash = bytes.fromhex(target) + if len(peer_destination_hash) != RNS.Identity.TRUNCATED_HASHLENGTH//8: raise ValueError(f"Destination hash length must be {RNS.Identity.TRUNCATED_HASHLENGTH//8*2} characters") + except Exception as e: + print(f"Invalid peer destination hash: {e}") + exit(203) + remote + _remote_init(configdir, rnsconfigdir, verbosity, quietness, identity_path) + response = _request_sync(identity, peer_destination_hash, remote_identity=_get_target_identity(remote), timeout=timeout, exit_on_fail=True) + + if response == LXMF.LXMPeer.LXMPeer.ERROR_NO_IDENTITY: + print("Remote received no identity") + exit(203) + elif response == LXMF.LXMPeer.LXMPeer.ERROR_NO_ACCESS: + print("Access denied") + exit(204) + elif response == LXMF.LXMPeer.LXMPeer.ERROR_INVALID_DATA: + print("Invalid data received by remote") + exit(205) + elif response == LXMF.LXMPeer.LXMPeer.ERROR_NOT_FOUND: + print("The requested peer was not found") + exit(206) + elif response == None: + print("Empty response received") + exit(207) + else: + print(f"Sync requested for peer {RNS.prettyhexrep(peer_destination_hash)}") + exit(0) + +def _request_unpeer(identity, destination_hash, remote_identity, timeout=15, exit_on_fail=False): + control_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") + + timeout = time.time()+timeout + def check_timeout(): + if time.time() > timeout: + if exit_on_fail: + print("Requesting lxmd peering break timed out, exiting now") + exit(200) + else: return LXMF.LXMPeer.LXMPeer.ERROR_TIMEOUT + else: time.sleep(0.1) + + if not RNS.Transport.has_path(control_destination.hash): + RNS.Transport.request_path(control_destination.hash) + while not RNS.Transport.has_path(control_destination.hash): + tc = check_timeout() + if tc: + return tc + + link = RNS.Link(control_destination) + while not link.status == RNS.Link.ACTIVE: + tc = check_timeout() + if tc: + return tc + + link.identify(identity) + request_receipt = link.request(LXMF.LXMRouter.UNPEER_REQUEST_PATH, data=destination_hash, response_callback=None, failed_callback=None) + while not request_receipt.get_status() == RNS.RequestReceipt.READY: + tc = check_timeout() + if tc: + return tc + + link.teardown() + return request_receipt.get_response() + + +def request_unpeer(target, remote=None, configdir=None, rnsconfigdir=None, verbosity=0, quietness=0, timeout=15, identity_path=None): + global configpath, identitypath, storagedir, lxmdir + global lxmd_config, active_configuration, targetloglevel + + try: + peer_destination_hash = bytes.fromhex(target) + if len(peer_destination_hash) != RNS.Identity.TRUNCATED_HASHLENGTH//8: raise ValueError(f"Destination hash length must be {RNS.Identity.TRUNCATED_HASHLENGTH//8*2} characters") + except Exception as e: + print(f"Invalid peer destination hash: {e}") + exit(203) + remote + _remote_init(configdir, rnsconfigdir, verbosity, quietness, identity_path) + response = _request_unpeer(identity, peer_destination_hash, remote_identity=_get_target_identity(remote), timeout=timeout, exit_on_fail=True) + + if response == LXMF.LXMPeer.LXMPeer.ERROR_NO_IDENTITY: + print("Remote received no identity") + exit(203) + elif response == LXMF.LXMPeer.LXMPeer.ERROR_NO_ACCESS: + print("Access denied") + exit(204) + elif response == LXMF.LXMPeer.LXMPeer.ERROR_INVALID_DATA: + print("Invalid data received by remote") + exit(205) + elif response == LXMF.LXMPeer.LXMPeer.ERROR_NOT_FOUND: + print("The requested peer was not found") + exit(206) + elif response == None: + print("Empty response received") + exit(207) + else: + print(f"Broke peering with {RNS.prettyhexrep(peer_destination_hash)}") + exit(0) + +def query_status(identity, remote_identity=None, timeout=5, exit_on_fail=False): + if remote_identity == None: remote_identity = identity + control_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") + + timeout = time.time()+timeout + def check_timeout(): + if time.time() > timeout: + if exit_on_fail: + print("Getting lxmd statistics timed out, exiting now") + exit(200) + else: return LXMF.LXMPeer.LXMPeer.ERROR_TIMEOUT + else: time.sleep(0.1) + + if not RNS.Transport.has_path(control_destination.hash): + RNS.Transport.request_path(control_destination.hash) + while not RNS.Transport.has_path(control_destination.hash): + tc = check_timeout() + if tc: return tc + + link = RNS.Link(control_destination) + while not link.status == RNS.Link.ACTIVE: + tc = check_timeout() + if tc: return tc + + link.identify(identity) + request_receipt = link.request(LXMF.LXMRouter.STATS_GET_PATH, data=None, response_callback=None, failed_callback=None) + while not request_receipt.get_status() == RNS.RequestReceipt.READY: + tc = check_timeout() + if tc: return tc + + link.teardown() + return request_receipt.get_response() + +def get_status(remote=None, configdir=None, rnsconfigdir=None, verbosity=0, quietness=0, timeout=5, + show_status=False, show_peers=False, identity_path=None): + + global identity + _remote_init(configdir, rnsconfigdir, verbosity, quietness, identity_path) + response = query_status(identity, remote_identity=_get_target_identity(remote), timeout=timeout, exit_on_fail=True) + + if response == LXMF.LXMPeer.LXMPeer.ERROR_NO_IDENTITY: + print("Remote received no identity") + exit(203) + if response == LXMF.LXMPeer.LXMPeer.ERROR_NO_ACCESS: + print("Access denied") + exit(204) + elif response == None: + print("Empty response received") + exit(207) + else: + s = response + mutil = round((s["messagestore"]["bytes"]/s["messagestore"]["limit"])*100, 2) + ms_util = f"{mutil}%" + if s["from_static_only"]: + who_str = "static peers only" + else: + who_str = "all nodes" + + available_peers = 0 + unreachable_peers = 0 + peered_incoming = 0 + peered_outgoing = 0 + peered_rx_bytes = 0 + peered_tx_bytes = 0 + for peer_id in s["peers"]: + p = s["peers"][peer_id] + pm = p["messages"] + peered_incoming += pm["incoming"] + peered_outgoing += pm["outgoing"] + peered_rx_bytes += p["rx_bytes"] + peered_tx_bytes += p["tx_bytes"] + + if p["alive"]: available_peers += 1 + else: unreachable_peers += 1 + + total_incoming = peered_incoming+s["unpeered_propagation_incoming"]+s["clients"]["client_propagation_messages_received"] + total_rx_bytes = peered_rx_bytes+s["unpeered_propagation_rx_bytes"] + if total_incoming != 0: df = round(peered_outgoing/total_incoming, 2) + else: df = 0 + + dhs = RNS.prettyhexrep(s["destination_hash"]); uts = RNS.prettytime(s["uptime"]) + print(f"\nLXMF Propagation Node running on {dhs}, uptime is {uts}") + + if show_status: + msb = RNS.prettysize(s["messagestore"]["bytes"]); msl = RNS.prettysize(s["messagestore"]["limit"]) + ptl = RNS.prettysize(s["propagation_limit"]*1000); psl = RNS.prettysize(s["sync_limit"]*1000); + uprx = RNS.prettysize(s["unpeered_propagation_rx_bytes"]) + mscnt = s["messagestore"]["count"]; stp = s["total_peers"]; smp = s["max_peers"]; sdp = s["discovered_peers"] + ssp = s["static_peers"]; cprr = s["clients"]["client_propagation_messages_received"] + cprs = s["clients"]["client_propagation_messages_served"]; upi = s["unpeered_propagation_incoming"] + psc = s["target_stamp_cost"]; scf = s["stamp_cost_flexibility"] + pc = s["peering_cost"]; pcm = s["max_peering_cost"] + print(f"Messagestore contains {mscnt} messages, {msb} ({ms_util} utilised of {msl})") + print(f"Required propagation stamp cost is {psc}, flexibility is {scf}") + print(f"Peering cost is {pc}, max remote peering cost is {pcm}") + print(f"Accepting propagated messages from {who_str}") + print(f"{ptl} message limit, {psl} sync limit") + print(f"") + print(f"Peers : {stp} total (peer limit is {smp})") + print(f" {sdp} discovered, {ssp} static") + print(f" {available_peers} available, {unreachable_peers} unreachable") + print(f"") + print(f"Traffic : {total_incoming} messages received in total ({RNS.prettysize(total_rx_bytes)})") + print(f" {peered_incoming} messages received from peered nodes ({RNS.prettysize(peered_rx_bytes)})") + print(f" {upi} messages received from unpeered nodes ({uprx})") + print(f" {peered_outgoing} messages transferred to peered nodes ({RNS.prettysize(peered_tx_bytes)})") + print(f" {cprr} propagation messages received directly from clients") + print(f" {cprs} propagation messages served to clients") + print(f" Distribution factor is {df}") + print(f"") + + if show_peers: + if not show_status: + print("") + + for peer_id in s["peers"]: + ind = " " + p = s["peers"][peer_id] + if p["type"] == "static": + t = "Static peer " + elif p["type"] == "discovered": + t = "Discovered peer " + else: + t = "Unknown peer " + a = "Available" if p["alive"] == True else "Unreachable" + h = max(time.time()-p["last_heard"], 0) + hops = p["network_distance"] + hs = "hops unknown" if hops == RNS.Transport.PATHFINDER_M else f"{hops} hop away" if hops == 1 else f"{hops} hops away" + pm = p["messages"]; pk = p["peering_key"] + pc = p["peering_cost"]; psc = p["target_stamp_cost"]; psf = p["stamp_cost_flexibility"] + if pc == None: pc = "unknown" + if psc == None: psc = "unknown" + if psf == None: psf = "unknown" + if pk == None: pk = "Not generated" + else: pk = f"Generated, value is {pk}" + if p["last_sync_attempt"] != 0: + lsa = p["last_sync_attempt"] + ls = f"last synced {RNS.prettytime(max(time.time()-lsa, 0))} ago" + else: + ls = "never synced" + + sstr = RNS.prettyspeed(p["str"]); sler = RNS.prettyspeed(p["ler"]) + stl = RNS.prettysize(p["transfer_limit"]*1000) if p["transfer_limit"] else "Unknown" + ssl = RNS.prettysize(p["sync_limit"]*1000) if p["sync_limit"] else "unknown" + srxb = RNS.prettysize(p["rx_bytes"]); stxb = RNS.prettysize(p["tx_bytes"]); pmo = pm["offered"]; pmout = pm["outgoing"] + pmi = pm["incoming"]; pmuh = pm["unhandled"]; ar = round(p["acceptance_rate"]*100, 2) + if p["name"] == None: nn = "" + else: nn = p["name"].strip().replace("\n", "").replace("\r", "") + if len(nn) > 45: nn = f"{nn[:45]}..." + print(f"{ind}{t}{RNS.prettyhexrep(peer_id)}") + if len(nn): print(f"{ind*2}Name : {nn}") + print(f"{ind*2}Status : {a}, {hs}, last heard {RNS.prettytime(h)} ago") + print(f"{ind*2}Costs : Propagation {psc} (flex {psf}), peering {pc}") + print(f"{ind*2}Sync key : {pk}") + print(f"{ind*2}Speeds : {sstr} STR, {sler} LER") + print(f"{ind*2}Limits : {stl} message limit, {ssl} sync limit") + print(f"{ind*2}Messages : {pmo} offered, {pmout} outgoing, {pmi} incoming, {ar}% acceptance rate") + print(f"{ind*2}Traffic : {srxb} received, {stxb} sent") + ms = "" if pm["unhandled"] == 1 else "s" + print(f"{ind*2}Sync state : {pmuh} unhandled message{ms}, {ls}") + print("") + +def _get_target_identity(remote=None, timeout=5): + global identity + timeout = time.time()+timeout + def check_timeout(): + if time.time() > timeout: + print("Resolving remote identity timed out, exiting now") + exit(200) + else: time.sleep(0.1) + + if remote == None: return identity + else: + try: + destination_hash = bytes.fromhex(remote) + if len(destination_hash) != RNS.Identity.TRUNCATED_HASHLENGTH//8: raise ValueError(f"Destination hash length must be {RNS.Identity.TRUNCATED_HASHLENGTH//8*2} characters") + except Exception as e: + print(f"Invalid remote destination hash: {e}") + exit(203) + + remote_identity = RNS.Identity.recall(destination_hash) + if remote_identity: return remote_identity + else: + if not RNS.Transport.has_path(destination_hash): + RNS.Transport.request_path(destination_hash) + while not RNS.Transport.has_path(destination_hash): + tc = check_timeout() + if tc: return tc + + return RNS.Identity.recall(destination_hash) + +def _remote_init(configdir=None, rnsconfigdir=None, verbosity=0, quietness=0, identity_path=None): + global configpath, identitypath, storagedir, lxmdir, identity + global lxmd_config, active_configuration, targetloglevel + targetlogdest = RNS.LOG_STDOUT + + if identity_path == None: + if configdir == None: + if os.path.isdir("/etc/lxmd") and os.path.isfile("/etc/lxmd/config"): configdir = "/etc/lxmd" + elif os.path.isdir(RNS.Reticulum.userdir+"/.config/lxmd") and os.path.isfile(RNS.Reticulum.userdir+"/.config/lxmd/config"): configdir = RNS.Reticulum.userdir+"/.config/lxmd" + else: configdir = RNS.Reticulum.userdir+"/.lxmd" + + configpath = configdir+"/config" + identitypath = configdir+"/identity" + identity = None + + if not os.path.isdir(configdir): + RNS.log("Specified configuration directory does not exist, exiting now", RNS.LOG_ERROR) + exit(201) + if not os.path.isfile(identitypath): + RNS.log("Identity file not found in specified configuration directory, exiting now", RNS.LOG_ERROR) + exit(202) + else: + identity = RNS.Identity.from_file(identitypath) + if identity == None: + RNS.log("Could not load the Primary Identity from "+identitypath, RNS.LOG_ERROR) + exit(4) + + else: + if not os.path.isfile(identity_path): + RNS.log("Identity file not found in specified configuration directory, exiting now", RNS.LOG_ERROR) + exit(202) + else: + identity = RNS.Identity.from_file(identity_path) + if identity == None: + RNS.log("Could not load the Primary Identity from "+identity_path, RNS.LOG_ERROR) + exit(4) + + if targetloglevel == None: targetloglevel = 3 + if verbosity != 0 or quietness != 0: targetloglevel = targetloglevel+verbosity-quietness + + reticulum = RNS.Reticulum(configdir=rnsconfigdir, loglevel=targetloglevel, logdest=targetlogdest) + +def main(): + try: + parser = argparse.ArgumentParser(description="Lightweight Extensible Messaging Daemon") + parser.add_argument("--config", action="store", default=None, help="path to alternative lxmd config directory", type=str) + parser.add_argument("--rnsconfig", action="store", default=None, help="path to alternative Reticulum config directory", type=str) + parser.add_argument("-p", "--propagation-node", action="store_true", default=False, help="run an LXMF Propagation Node") + parser.add_argument("-i", "--on-inbound", action="store", metavar="PATH", default=None, help="executable to run when a message is received", type=str) + parser.add_argument("-v", "--verbose", action="count", default=0) + parser.add_argument("-q", "--quiet", action="count", default=0) + parser.add_argument("-s", "--service", action="store_true", default=False, help="lxmd is running as a service and should log to file") + parser.add_argument("--status", action="store_true", default=False, help="display node status") + parser.add_argument("--peers", action="store_true", default=False, help="display peered nodes") + parser.add_argument("--sync", action="store", default=None, help="request a sync with the specified peer", type=str) + parser.add_argument("-b", "--break", dest="unpeer", action="store", default=None, help="break peering with the specified peer", type=str) + parser.add_argument("--timeout", action="store", default=None, help="timeout in seconds for query operations", type=float) + parser.add_argument("-r", "--remote", action="store", default=None, help="remote propagation node destination hash", type=str) + parser.add_argument("--identity", action="store", default=None, help="path to identity used for remote requests", type=str) + parser.add_argument("--exampleconfig", action="store_true", default=False, help="print verbose configuration example to stdout and exit") + parser.add_argument("--version", action="version", version="lxmd {version}".format(version=__version__)) + + args = parser.parse_args() + + if args.exampleconfig: + print(__default_lxmd_config__) + exit() + + if args.status or args.peers: + if not args.timeout: args.timeout = 5 + get_status(configdir = args.config, + rnsconfigdir=args.rnsconfig, + verbosity=args.verbose, + quietness=args.quiet, + timeout=args.timeout, + show_status=args.status, + show_peers=args.peers, + identity_path=args.identity, + remote=args.remote) + exit() + + if args.sync: + if not args.timeout: args.timeout = 10 + request_sync(target=args.sync, + configdir = args.config, + rnsconfigdir=args.rnsconfig, + verbosity=args.verbose, + quietness=args.quiet, + timeout=args.timeout, + identity_path=args.identity, + remote=args.remote) + exit() + + if args.unpeer: + if not args.timeout: args.timeout = 10 + request_unpeer(target=args.unpeer, + configdir = args.config, + rnsconfigdir=args.rnsconfig, + verbosity=args.verbose, + quietness=args.quiet, + timeout=args.timeout, + identity_path=args.identity, + remote=args.remote) + exit() + + program_setup(configdir = args.config, + rnsconfigdir=args.rnsconfig, + run_pn=args.propagation_node, + on_inbound=args.on_inbound, + verbosity=args.verbose, + quietness=args.quiet, + service=args.service) + + except KeyboardInterrupt: + print("") + exit() + +__default_lxmd_config__ = """# This is an example LXM Daemon config file. +# You should probably edit it to suit your +# intended usage. + +[propagation] + +# Whether to enable propagation node + +enable_node = no + +# You can specify identity hashes for remotes +# that are allowed to control and query status +# for this propagation node. + +# control_allowed = 7d7e542829b40f32364499b27438dba8, 437229f8e29598b2282b88bad5e44698 + +# An optional name for this node, included +# in announces. + +# node_name = Anonymous Propagation Node + +# Automatic announce interval in minutes. +# 6 hours by default. + +announce_interval = 360 + +# Whether to announce when the node starts. + +announce_at_start = yes + +# Wheter to automatically peer with other +# propagation nodes on the network. + +autopeer = yes + +# The maximum peering depth (in hops) for +# automatically peered nodes. + +autopeer_maxdepth = 6 + +# The maximum amount of storage to use for +# the LXMF Propagation Node message store, +# specified in megabytes. When this limit +# is reached, LXMF will periodically remove +# messages in its message store. By default, +# LXMF prioritises keeping messages that are +# new and small. Large and old messages will +# be removed first. This setting is optional +# and defaults to 500 megabytes. + +# message_storage_limit = 500 + +# The maximum accepted transfer size per in- +# coming propagation message, in kilobytes. +# This sets the upper limit for the size of +# single messages accepted onto this node. + +# propagation_message_max_accepted_size = 256 + +# The maximum accepted transfer size per in- +# coming propagation node sync. +# +# If a node wants to propagate a larger number +# of messages to this node, than what can fit +# within this limit, it will prioritise sending +# the smallest messages first, and try again +# with any remaining messages at a later point. + +# propagation_sync_max_accepted_size = 10240 + +# You can configure the target stamp cost +# required to deliver messages via this node. + +# propagation_stamp_cost_target = 16 + +# If set higher than 0, the stamp cost flexi- +# bility option will make this node accept +# messages with a lower stamp cost than the +# target from other propagation nodes (but +# not from peers directly). This allows the +# network to gradually adjust stamp cost. + +# propagation_stamp_cost_flexibility = 3 + +# The peering_cost option configures the target +# value required for a remote node to peer with +# and deliver messages to this node. + +# peering_cost = 18 + +# You can configure the maximum peering cost +# of remote nodes that this node will peer with. +# Setting this to a higher number will allow +# this node to peer with other nodes requiring +# a higher peering key value, but will require +# more computation time during initial peering +# when generating the peering key. + +# remote_peering_cost_max = 26 + +# You can tell the LXMF message router to +# prioritise storage for one or more +# destinations. If the message store reaches +# the specified limit, LXMF will prioritise +# keeping messages for destinations specified +# with this option. This setting is optional, +# and generally you do not need to use it. + +# prioritise_destinations = 41d20c727598a3fbbdf9106133a3a0ed, d924b81822ca24e68e2effea99bcb8cf + +# You can configure the maximum number of other +# propagation nodes that this node will peer +# with automatically. The default is 20. + +# max_peers = 20 + +# You can configure a list of static propagation +# node peers, that this node will always be +# peered with, by specifying a list of +# destination hashes. + +# static_peers = e17f833c4ddf8890dd3a79a6fea8161d, 5a2d0029b6e5ec87020abaea0d746da4 + +# You can configure the propagation node to +# only accept incoming propagation messages +# from configured static peers. + +# from_static_only = True + +# By default, any destination is allowed to +# connect and download messages, but you can +# optionally restrict this. If you enable +# authentication, you must provide a list of +# allowed identity hashes in the a file named +# "allowed" in the lxmd config directory. + +auth_required = no + + +[lxmf] + +# The LXM Daemon will create an LXMF destination +# that it can receive messages on. This option sets +# the announced display name for this destination. + +display_name = Anonymous Peer + +# It is possible to announce the internal LXMF +# destination when the LXM Daemon starts up. + +announce_at_start = no + +# You can also announce the delivery destination +# at a specified interval. This is not enabled by +# default. + +# announce_interval = 360 + +# The maximum accepted unpacked size for mes- +# sages received directly from other peers, +# specified in kilobytes. Messages larger than +# this will be rejected before the transfer +# begins. + +delivery_transfer_max_accepted_size = 1000 + +# You can configure an external program to be run +# every time a message is received. The program +# will receive as an argument the full path to the +# message saved as a file. The example below will +# simply result in the message getting deleted as +# soon as it has been received. + +# on_inbound = rm + + +[logging] +# Valid log levels are 0 through 7: +# 0: Log only critical information +# 1: Log errors and lower log levels +# 2: Log warnings and lower log levels +# 3: Log notices and lower log levels +# 4: Log info and lower (this is the default) +# 5: Verbose logging +# 6: Debug logging +# 7: Extreme logging + +loglevel = 4 + +""" + +if __name__ == "__main__": + main() diff --git a/LXMF/__init__.py b/LXMF/__init__.py new file mode 100644 index 0000000..0458f8d --- /dev/null +++ b/LXMF/__init__.py @@ -0,0 +1,10 @@ +import os +import glob +from .LXMessage import LXMessage +from .LXMRouter import LXMRouter +from .LXMF import * + +from ._version import __version__ + +modules = glob.glob(os.path.dirname(__file__)+"/*.py") +__all__ = [ os.path.basename(f)[:-3] for f in modules if not f.endswith('__init__.py')] \ No newline at end of file diff --git a/LXMF/_version.py b/LXMF/_version.py new file mode 100644 index 0000000..5c4105c --- /dev/null +++ b/LXMF/_version.py @@ -0,0 +1 @@ +__version__ = "1.0.1" diff --git a/MIRROR.md b/MIRROR.md new file mode 100644 index 0000000..ef2fdd4 --- /dev/null +++ b/MIRROR.md @@ -0,0 +1,33 @@ +This repository is a public mirror. All potential future development is happening elsewhere. + +I am stepping back from all public-facing interaction with this project. Reticulum has always been primarily my work, and continuing in the current public, internet-facing model is no longer sustainable. + +The software remains available for use as-is. Occasional updates may appear at unpredictable intervals, but there will be no support, no responses to issues, no discussions, and no community management in this or any other public venue. If it doesn't work for you, it doesn't work. That is the entire extent of available troubleshooting assistance I can offer you. + +If you've followed this project for a while, you already know what this means. You know who designed, wrote and tested this, and you know how many years of my life it took. You'll also know about both my particular challenges and strengths, and how I believe anything worth building needs to be built and maintained with our own hands. + +Seven months ago, I said I needed to step back, that I was exhausted, and that I needed to recover. I believed a public resolve would be enough to effectuate that, but while striving to get just a few more useful features and protocols out, the unproductive requests and demands also ramped up, and I got pulled back into the same patterns and draining interactions that I'd explicitly said I couldn't sustain anymore. + +So here's what you might have already guessed: I'm done playing the game by rules I can't win at. + +Everything you need is right here, and by any sensible measure, it's done. Anyone who wants to invest the time, skill and persistence can build on it, or completely re-imagine it with different priorities. That was always the point. + +The people who actually contributed - you know who you are, and you know I mean it when I say: Thank you. All of you who've used this to build something real - that was the goal, and you did it without needing me to hold your hand. + +The rest of you: You have what you need. Use it or don't. I am not going to be the person who explains it to you anymore. + +This is not a temporary break. It's not "see you after some rest", but a recognition that the current model is fundamentally incompatible with my life, my health, and my reality. + +If you want to support continued work, you can do so at the donation links listed in this repository. But please understand, that this is not purchasing support or guaranteeing updates. It is support for work that happens on my timeline, according to my capacity, which at the moment is not what it was. + +If you want Reticulum to continue evolving, you have the power to make that happen. The protocol is public domain. The code is open source. Everything you need is right here. I've provided the tools, but building what comes next is not my responsibility anymore. It's yours. + +To the small group of people who has actually been here, and understood what this work was and what it cost - you already know where to find me if it actually matters. + +To everyone else: This is where we part ways. No hard feelings. It's just time. + +--- + +असतो मा सद्गमय +तमसो मा ज्योतिर्गमय +मृत्योर्मा अमृतं गमय diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..cb7f141 --- /dev/null +++ b/Makefile @@ -0,0 +1,35 @@ +all: release + +clean: + @echo Cleaning... + -rm -r ./build + -rm -r ./dist + +remove_symlinks: + @echo Removing symlinks for build... + -rm ./RNS + -rm ./LXMF/Utilities/LXMF + +create_symlinks: + @echo Creating symlinks... + -ln -s ../Reticulum/RNS ./ + -ln -s ../../LXMF ./LXMF/Utilities/LXMF + +build_wheel: + python3 setup.py bdist_wheel + +build_sdist: + python3 setup.py sdist + +build_spkg: remove_symlinks build_sdist create_symlinks + +release: remove_symlinks build_wheel build_spkg create_symlinks + +upload: + @echo Ready to publish release over Reticulum + @read VOID + rngit release rns://7649a50d84610232d1416b41d2896aff/reticulum/lxmf create $$(python setup.py --getversion):dist --name lxmf + +upload-pip: + @echo Uploading to PyPi... + twine upload dist/*.whl dist/*.tar.gz diff --git a/PLANS.md b/PLANS.md new file mode 100644 index 0000000..cbaedc0 --- /dev/null +++ b/PLANS.md @@ -0,0 +1,13 @@ +# LXMF Specification Investigation Plan + +Each investigation must include: + +1. Question being answered. +2. Upstream files inspected. +3. Exact version or commit. +4. Relevant classes/functions/constants. +5. Observed behavior. +6. Whether behavior is normative, inferred, or unresolved. +7. Test vectors needed. +8. SPEC.md sections affected. +9. Open questions. diff --git a/README.md b/README.md index ee643a4..d8c9e47 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,218 @@ -# LMXF-specification +# Lightweight Extensible Message Format -Based on Mark Qvist's LMXF, this project attempts to distill a specification from the Python code which is an implementation of the LMXF "protocol" \ No newline at end of file +LXMF is a simple and flexible messaging format and delivery protocol that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](a8d24177d946de4f1f0a0fe1af9a1338:/page/repo.mu`g=reticulum|r=reticulum) and offers zero-conf message routing, end-to-end encryption and Forward Secrecy, and can be transported over any kind of medium that Reticulum supports. + +LXMF is efficient enough that it can deliver messages over extremely low-bandwidth systems such as packet radio or LoRa. Encrypted LXMF messages can also be encoded as QR-codes or text-based URIs, allowing completely analog *paper message* transport. + +User-facing clients built on LXMF include: + +- [Sideband](https://unsigned.io/sideband) +- [MeshChat](https://github.com/liamcottle/reticulum-meshchat) +- [Nomad Network](https://unsigned.io/nomadnet) + +Community-provided tools and utilities for LXMF include: + +- [LXMFy](https://lxmfy.quad4.io/) +- [LXMF-Bot](https://github.com/randogoth/lxmf-bot) +- [LXMF Messageboard](https://github.com/chengtripp/lxmf_messageboard) +- [LXMEvent](https://github.com/faragher/LXMEvent) +- [RangeMap](https://github.com/faragher/RangeMap) +- [LXMF Tools](https://github.com/SebastianObi/LXMF-Tools) + +## Structure + +LXMF messages are stored in a simple and efficient format, that's easy to parse and write. + +### The format follows this general structure: + +- Destination +- Source +- Ed25519 Signature +- Payload + - Timestamp + - Content + - Title + - Fields + +### And these rules: + +1. A LXMF message is identified by its __message-id__, which is a SHA-256 hash of the __Destination__, __Source__ and __Payload__. The message-id is never included directly in the message, since it can always be inferred from the message itself. + + In some cases the actual message-id cannot be inferred, for example when a Propagation Node is storing an encrypted message for an offline user. In these cases a _transient-id_ is used to identify the message while in storage or transit. + +2. __Destination__, __Source__, __Signature__ and __Payload__ parts are mandatory, as is the __Timestamp__ part of the payload. + - The __Destination__ and __Source__ fields are 16-byte Reticulum destination hashes + - The __Signature__ field is a 64-byte Ed25519 signature of the __Destination__, __Source__, __Payload__ and __message-id__ + - The __Payload__ part is a [msgpacked](https://msgpack.org) list containing four items: + 1. The __Timestamp__ is a double-precision floating point number representing the number of seconds since the UNIX epoch. + 2. The __Content__ is the optional content or body of the message + 3. The __Title__ is an optional title for the message + 4. The __Fields__ is an optional dictionary + +3. The __Content__, __Title__ and __Fields__ parts must be included in the message structure, but can be left empty. + +4. The __Fields__ part can be left empty, or contain a dictionary of any structure or depth. + +## Usage Examples + +LXMF offers flexibility to implement many different messaging schemes, ranging from human communication to machine control and sensor monitoring. Here are a few examples: + +- A messaging system for passing short, simple messages between human users, akin to SMS can be implemented using only the __Content__ field, and leaving all other optional fields empty. + +- For sending full-size mail, an email-like system can be implemented using the __Title__ and __Content__ fields to store "subject" and "body" parts of the message, and optionally the __Fields__ part can be used to store attachments or other metadata. + +- Machine-control messages or sensor readings can be implemented using command structures embedded in the __Fields__ dictionary. + +- Distributed discussion or news-groups, akin to USENET or similar systems, can be implemented using the relevant fields and LXMF Propagation Nodes. Broadcast bulletins can be implemented in a similar fashion. + +## Propagation Nodes + +LXM Propagation Nodes offer a way to store and forward messages to users or endpoints that are not directly reachable at the time of message emission. Propagation Nodes can also provide infrastructure for distributed bulletin, news or discussion boards. + +When Propagation Nodes exist on a Reticulum network, they will by default peer with each other and synchronise messages, automatically creating an encrypted, distributed message store. Users and other endpoints can retrieve messages destined for them from any available Propagation Nodes on the network. + +## The LXM Router + +The LXM Router handles transporting messages over a Reticulum network, managing delivery receipts, outbound and inbound queues, and is the point of API interaction for client programs. The LXM Router also implements functionality for acting as an LXMF Propagation Node. + +Programatically, using the LXM Router to send a message is as simple as: + +```python +import LXMF + +lxm_router = LXMF.LXMRouter() + +message = LXMF.LXMessage(destination, source, "This is a short, simple message.") + +lxm_router.handle_outbound(message) +``` + +The LXM Router then handles the heavy lifting, such as message packing, encryption, delivery confirmation, path lookup, routing, retries and failure notifications. + +## Transport Encryption + +LXMF uses encryption provided by [Reticulum](https://reticulum.network), and thus uses end-to-end encryption by default. The delivery method of a message will influence which transport encryption scheme is used. + +- If a message is delivered over a Reticulum link (which is the default method), the message will be encrypted with ephemeral AES-128 keys derived with ECDH on Curve25519. This mode offers forward secrecy. + +- A message can be delivered opportunistically, embedded in a single Reticulum packet. In this cases the message will be opportunistically routed through the network, and will be encrypted with per-packet AES-128 keys derived with ECDH on Curve25519. + +- If a message is delivered to the Reticulum GROUP destination type, the message will be encrypted using the symmetric AES-128 key of the GROUP destination. + +## Wire Format & Overhead + +Assuming the default Reticulum configuration, the binary wire-format is as follows: + +- 16 bytes destination hash +- 16 bytes source hash +- 64 bytes Ed25519 signature +- Remaining bytes of [msgpack](https://msgpack.org) payload data, in accordance with the structure defined above + +The complete message overhead for LXMF is only 111 bytes, which in return gives you timestamped, digitally signed, infinitely extensible, end-to-end encrypted, zero-conf routed, minimal-infrastructure messaging that's easy to use and build applications with. + +## Code Examples + +Before writing your own programs using LXMF, you need to have a basic understanding of how the [Reticulum](https://reticulum.network) protocol and API works. Please see the [Reticulum Manual](https://reticulum.network/manual/). For a few simple examples of how to send and receive messages with LXMF, please see the [receiver example](./docs/example_receiver.py) and the [sender example](./docs/example_sender.py) included in this repository. + +## Example Paper Message + +You can try out the paper messaging functionality by using the following QR code. It is a paper message sent to the LXMF address `6b3362bd2c1dbf87b66a85f79a8d8c75`. To be able to decrypt and read the message, you will need to import the following Reticulum Identity to an LXMF messaging app: + +`3BPTDTQCRZPKJT3TXAJCMQFMOYWIM3OCLKPWMG4HCF2T4CH3YZHVNHNRDU6QAZWV2KBHMWBNT2C62TQEVC5GLFM4MN25VLZFSK3ADRQ=` + +The [Sideband](https://unsigned.io/sideband) application allows you to do this easily. After you have imported the identity into an app of your choice, you can scan the following QR code and open it in the app, where it will be decrypted and added as a message. + +[Paper message QR code](a8d24177d946de4f1f0a0fe1af9a1338:/page/blob.mu`g=reticulum|r=lxmf|ref=HEAD|path=docs/paper_msg_test.png) + +You can also find the entire message in this link: + +``` +lxm://azNivSwdv4e2aoX3mo2MdTAozuI7BlzrLlHULmnVgpz3dNT9CMPVwgywzCJP8FVogj5j_kU7j7ywuvBNcr45kRTrd19c3iHenmnSDe4VEd6FuGsAiT0Khzl7T81YZHPTDhRNp0FdhDE9AJ7uphw7zKMyqhHHxOxqrYeBeKF66gpPxDceqjsOApvsSwggjcuHBx9OxOBy05XmnJxA1unCKgvNfOFYc1T47luxoY3c0dLOJnJPwZuFRytx2TXlQNZzOJ28yTEygIfkDqEO9mZi5lgev7XZJ0DvgioQxMIyoCm7lBUzfq66zW3SQj6vHHph7bhr36dLOCFgk4fZA6yia2MlTT9KV66Tn2l8mPNDlvuSAJhwDA_xx2PN9zKadCjo9sItkAp8r-Ss1CzoUWZUAyT1oDw7ly6RrzGBG-e3eM3CL6u1juIeFiHby7_3cON-6VTUuk4xR5nwKlFTu5vsYMVXe5H3VahiDSS4Q1aqX7I +``` + +On operating systems that allow for registering custom URI-handlers, you can click the link, and it will be decoded directly in your LXMF client. This works with Sideband on Android. + +## Installation + +If you want to try out LXMF, you can install it with pip: + +```bash +pip install lxmf +``` + +If you are using an operating system that blocks normal user package installation via `pip`, +you can return `pip` to normal behaviour by editing the `~/.config/pip/pip.conf` file, +and adding the following directive in the `[global]` section: + +```text +[global] +break-system-packages = true +``` + +Alternatively, you can use the `pipx` tool to install Reticulum in an isolated environment: + +```bash +pipx install lxmf +``` + +## Daemon Included + +The `lxmf` package comes with the `lxmd` program, a fully functional (but lightweight) LXMF message router and propagation node daemon. After installing the `lxmf` package, you can run `lxmd --help` to learn more about the command-line options: + +```text +$ lxmd --help + +usage: lxmd [-h] [--config CONFIG] [--rnsconfig RNSCONFIG] [-p] [-i PATH] [-v] [-q] [-s] [--exampleconfig] [--version] + +Lightweight Extensible Messaging Daemon + +options: + -h, --help show this help message and exit + --config CONFIG path to alternative lxmd config directory + --rnsconfig RNSCONFIG + path to alternative Reticulum config directory + -p, --propagation-node + run an LXMF Propagation Node + -i PATH, --on-inbound PATH + executable to run when a message is received + -v, --verbose + -q, --quiet + -s, --service lxmd is running as a service and should log to file + --exampleconfig print verbose configuration example to stdout and exit + --version show program's version number and exit +``` + +Or run `lxmd --exampleconfig` to generate a commented example configuration documenting all the available configuration directives. + +## Support LXMF Development +You can help support the continued development of open, free and private communications systems by donating via one of the following channels: + +- Monero: + ``` + 84FpY1QbxHcgdseePYNmhTHcrgMX4nFfBYtz2GKYToqHVVhJp8Eaw1Z1EedRnKD19b3B8NiLCGVxzKV17UMmmeEsCrPyA5w + ``` +- Bitcoin + ``` + bc1pgqgu8h8xvj4jtafslq396v7ju7hkgymyrzyqft4llfslz5vp99psqfk3a6 + ``` +- Ethereum + ``` + 0x91C421DdfB8a30a49A71d63447ddb54cEBe3465E + ``` +- Liberapay: https://liberapay.com/Reticulum/ + +- Ko-Fi: https://ko-fi.com/markqvist + + +## Caveat Emptor + +LXMF is beta software, and should be considered experimental. While it has been built with cryptography best practices very foremost in mind, it _has not_ been externally security audited, and there could very well be privacy-breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch. + +## Development Roadmap + +LXMF is actively being developed, and the following improvements and features are currently planned for implementation: + +- Sneakernet and physical transport functionality +- Content Destinations, and easy to use API for group messaging and discussion threads +- Write and release full API and protocol documentation +- Documenting and possibly expanding LXMF limits and priorities diff --git a/SPEC.md b/SPEC.md new file mode 100644 index 0000000..a6dcc8e --- /dev/null +++ b/SPEC.md @@ -0,0 +1,153 @@ +# LXMF Implementation-Derived Specification + +## LXMessage Wire Format + +This section describes behavior confirmed in the checked-in LXMF implementation +and by the reproducible vectors under `examples/`. It does not assign normative +requirements to independent implementations. + +Evidence baseline: + +- LXMF commit `d483d40d5cb994c6027802af1bf284dfeccc623b` +- Reticulum commit `2646f673ee061135d7c351ef44dce290ccd7e06e` + +### Evidence status + +- **Source-confirmed** means the behavior is directly implemented by the + checked-in LXMF or Reticulum source. +- **Vector-confirmed** means `perl tools/verify_examples.pl` confirms the + behavior for the deterministic minimal and stamped vectors. +- Behavior not covered by either form of evidence is marked **Unresolved**. + +### Core serialized message + +`LXMessage.pack()` constructs `lxmf_bytes` by concatenating: + +```text +destination_hash || source_hash || signature || packed_payload +``` + +| Part | Position | Confirmed representation | Evidence | +|---|---:|---|---| +| `destination_hash` | bytes `0..15` | 16 bytes | Source-confirmed: `LXMF/LXMessage.py:40`, `LXMF/LXMessage.py:383`; Vector-confirmed | +| `source_hash` | bytes `16..31` | 16 bytes | Source-confirmed: `LXMF/LXMessage.py:384`; Vector-confirmed | +| `signature` | bytes `32..95` | 64 bytes | Source-confirmed: `LXMF/LXMessage.py:41`, `LXMF/LXMessage.py:385`; Vector-confirmed position and length only | +| `packed_payload` | bytes `96..end` | MessagePack array | Source-confirmed: `LXMF/LXMessage.py:362`, `LXMF/LXMessage.py:381-386`; Vector-confirmed | + +The 16-byte hash length derives from +`RNS.Identity.TRUNCATED_HASHLENGTH // 8`. The checked-in Reticulum source sets +the truncated hash length to 128 bits. The 64-byte signature length derives +from `RNS.Identity.SIGLENGTH // 8`. See `RNS/Reticulum.py:146-147` and +`RNS/Identity.py:80-83` in the evidence-baseline Reticulum checkout. + +### Payload + +For messages generated by `LXMessage.pack()`, the payload is: + +```text +[ + timestamp, + title, + content, + fields +] +``` + +If a message stamp is available when packing, it is appended: + +```text +[ + timestamp, + title, + content, + fields, + stamp +] +``` + +| Index | Name | Confirmed behavior | Evidence | +|---:|---|---|---| +| `0` | `timestamp` | Set from `time.time()` when not already set, then packed as the first payload value | Source-confirmed: `LXMF/LXMessage.py:357`, `LXMF/LXMessage.py:362`; Vector-confirmed as MessagePack float64 for the deterministic vectors | +| `1` | `title` | Constructor string input is UTF-8 encoded to bytes; byte input is retained | Source-confirmed: `LXMF/LXMessage.py:130-133`, `LXMF/LXMessage.py:193-197`; Vector-confirmed as MessagePack binary | +| `2` | `content` | Constructor string input is UTF-8 encoded to bytes; byte input is retained | Source-confirmed: `LXMF/LXMessage.py:135-136`, `LXMF/LXMessage.py:202-206`; Vector-confirmed as MessagePack binary | +| `3` | `fields` | Constructor input must be a dictionary or `None`; `None` becomes an empty dictionary | Source-confirmed: `LXMF/LXMessage.py:138`, `LXMF/LXMessage.py:215-219`; Vector-confirmed for an empty MessagePack map | +| `4` | `stamp` | Optional value appended after the four base payload values | Source-confirmed: `LXMF/LXMessage.py:371-373`; Vector-confirmed as MessagePack binary in the stamped vector | + +The payload order above is source- and vector-confirmed. It differs from the +order stated in the upstream `README.md`, which lists content before title. + +### Message ID + +The message ID, also stored as `LXMessage.hash`, is: + +```text +SHA-256( + destination_hash || + source_hash || + msgpack([timestamp, title, content, fields]) +) +``` + +The optional stamp is excluded from the MessagePack payload used to calculate +the message ID. This is source-confirmed by both packing and unpacking behavior, +and vector-confirmed by the minimal and stamped vectors producing the same +message ID. See `LXMF/LXMessage.py:362-369` and +`LXMF/LXMessage.py:754-764`. Reticulum defines `full_hash()` as SHA-256 at +`RNS/Identity.py:238-246`. + +The message ID is not included in `lxmf_bytes`. + +### Signature input + +`LXMessage.pack()` requests a signature over: + +```text +destination_hash || +source_hash || +msgpack([timestamp, title, content, fields]) || +message_id +``` + +The optional stamp is excluded from the signature input. The construction of +the signature input is source-confirmed at `LXMF/LXMessage.py:375-378` and +`LXMF/LXMessage.py:762-764`, and vector-confirmed. + +The deterministic vectors contain placeholder signature bytes. They confirm +the signature position and input bytes, but do not confirm signing or signature +validation. + +### Delivery representations + +The core `lxmf_bytes` representation is modified or wrapped for some delivery +methods: + +| Delivery representation | Confirmed serialized data | Evidence | +|---|---|---| +| Direct packet or resource | Full `lxmf_bytes` | Source-confirmed: `LXMF/LXMessage.py:635-636`, `LXMF/LXMessage.py:653-654` | +| Opportunistic packet | `lxmf_bytes` without the leading destination hash | Source-confirmed: `LXMF/LXMessage.py:633-634` | +| Propagated message data | `destination_hash || encrypt(lxmf_bytes after destination_hash)` | Source-confirmed: `LXMF/LXMessage.py:429-436` | +| Propagation transfer wrapper | MessagePack `[time.time(), [propagated_message_data]]` | Source-confirmed: `LXMF/LXMessage.py:436` | +| Paper message data | `destination_hash || encrypt(lxmf_bytes after destination_hash)` | Source-confirmed: `LXMF/LXMessage.py:446-451` | +| Paper URI | URL-safe Base64 of paper message data, without `=` padding, prefixed by `lxm://` | Source-confirmed: `LXMF/LXMessage.py:698-707` | + +These delivery representations are not covered by the current deterministic +test vectors. + +### Unresolved behavior + +The following behavior is not established by the current source inspection and +test-vector coverage: + +- Normative requirements for independent LXMF implementations. +- Cross-implementation Ed25519 signature generation and validation. +- Accepted signature encodings beyond the fixed 64-byte position generated by + the checked-in implementation. +- A universal MessagePack float width for timestamps on every supported + platform. The deterministic vectors confirm float64 only for those vectors. +- Interoperable constraints on the contents, key types, value types, ordering, + and nesting depth of `fields`. +- Interoperable constraints on stamp length and encoding. +- Required handling of malformed, truncated, non-canonical, or payload arrays + containing fewer than four or more than five entries. +- Byte-for-byte vectors for opportunistic, propagated, paper, URI, encrypted, + or persisted-container representations. diff --git a/Zen of Reticulum.md b/Zen of Reticulum.md new file mode 100644 index 0000000..aa286be --- /dev/null +++ b/Zen of Reticulum.md @@ -0,0 +1,415 @@ +# Zen of Reticulum + +## I: The Illusion Of The Center + +For the better part of a generation, we have been taught to visualize the digital world through the lens of hierarchy. The mental maps we carry are dominated by a single, misleading image: **The Cloud**. + +We imagine the network as a vast, ethereal space "up there" or "out there". A centralized repository of services and data to which we, the lowly clients, must connect. We build our software with this assumption hardcoded into our logic: *There is a server. The server has the authority. The server knows the way. I must find the server to function*. + +This is the Client-Server mental model, and it is the primary obstacle to understanding Reticulum. + +### Fallacy Of The Cloud + +The first step in the Zen of Reticulum is to realize that *there is no cloud*. There is only other people's computers. When you build for the cloud, you are building *for* a landlord. You are accepting that your application's existence is conditional on the permission, uptime, and continued goodwill of a central authority. + +In Reticulum, you must shift your thinking from "connecting to" to "being among". Reticulum is not a service you subscribe to - *it is a fabric you inhabit*. There is no "up there". There is only *here* and *there*, and the space between them is peer-to-peer. + +### Decentralization Or Uncentralizability? + +It is common to hear the word "decentralized" thrown around in modern tech circles. But often, this is merely a marketing term for "slightly distributed centralization". A blockchain with a few dominant miners, or a federated protocol with a few giant servers. *In practice*, it's still centralized. It simply has a few centers instead of one. + +Reticulum goes further. It wants **Uncentralizability**. + +This is not a wishful political stance, but a foundational mathematical characteristic of the protocol, onto which everything else has been built. Reticulum assumes that every peer on the network is potentially hostile, and every link is potentially compromised. It is designed with no "privileged" nodes. While some nodes may act as Transport Instances - forwarding traffic for others - they do so *blindly*, and they only know about their immediate surroundings, and nothing more. They route based on cryptographic proofs, not on administrative privilege. They cannot see who is talking to whom, nor can they selectively manipulate traffic without breaking their own ability to route entirely. + +The system is designed to make hierarchy structurally impossible. You cannot hijack an address, because there is no central registry to hijack. You cannot block a user, because there is no central switch to flip. You can offer paths through the network, but you can't force anyone to use them. + +### Death To The Address + +To break free of the center, you must also let go of the concept of the "Address". + +In the IP world, an address is a location. It is a coordinate in a *deeply hierarchical* and static grid. If you move your computer to a different house, your address changes. If your router reboots, your address might change. Your *identity* is bound to your *location*, and therefore, it is fragile, and easily controlled. + +Reticulum abolishes this link between *Identity* and *Location*. + +In Reticulum, an address is not a place; it is a **Hash of an Identity**. It is a cryptographic representation of *who* you are, not *where* you are. Because of this, your address is portable. You can take a laptop from a WiFi cafe in Berlin, to a LoRa mesh in the mountains, to a packet radio link on a boat, and your "address" - your *Destination Hash* - never changes. + +The network does not route to a place; it routes to a *person* (or a machine). When you send a packet, you are not targeting a coordinate in a grid; you are encrypting a message for a specific entity. The network dynamically discovers where that entity currently resides, and it does so in a way where no one really knows where that entity is actually located physically. + +**Consider:** + +- **The Old Way:** *"I am at `192.168.1.5`. Come find me"*. +- **The Zen Way:** *"I am `<327c1b2f87c9353e01769b01090b18f2>`. Wherever I am, my peers can reach me"*. + +Once you stop thinking about servers and start thinking about portable identities, where everyone can always reach everyone else directly, the illusion of the center fades away. You realize there *is* no center holding the network together. No coordinators or bureaucrats required. The network is simply the sum of its peers, communicating directly, sovereignly, and without a master. + + +## II: Physics Of Trust +*Paranoia Is A Great Design Principle* + +If we accept that there is no center - that the network is a chaotic, peer-to-peer mesh - we are forced to confront a terrifying reality: **There is no one guarding the door**. + +In the traditional networking mindset, we rely on the concept of the "trusted core". We assume our local coffee shop WiFi is safe, or that the backbone providers are neutral custodians. We build our security like a castle: strong walls on the outside, soft and trusting on the inside. We use encryption only when we step out into the "wild" internet. + +### Hostile Environments + +The Zen of Reticulum requires you to invert this. You must assume that *every* environment is hostile. This isn't cynicism, just uncaring physics. + +When you transmit information over radio waves, you are shouting into a crowded room. Anyone can listen. When you traverse the internet, your packets pass through routers controlled by strangers, corporations, and state actors. Assuming privacy in this environment without cryptographic protection is not optimism but gross negligence. + +Reticulum is built on the premise that every link is tapped, and every peer is a potential adversary. If your system cannot survive an adversary owning the physical layer, it cannot survive at all. + +But this is the paradox: By assuming the network is hostile, you make it safe. When you accept the dangers for what they are, they become manageable. When you stop trusting the infrastructure and start trusting the math, you eliminate the single point of failure: Human integrity. + +### Encryption Is Not A Feature + +In the world of TCP/IP, encryption is an afterthought. It is a layer we slap on top of the protocol (HTTPS, TLS) to patch the security holes of the original design. It is a "feature" you sometimes *enable* for "sensitive data". This is fundamentally flawed, since all data is sensitive. + +In Reticulum, encryption is **gravity**. + +It is not optional. It is not a plugin. It is the *fundamental force that allows the network to exist*. If you were to strip the encryption from Reticulum, the routing would break. The Transport system uses cryptographic signatures and entropy to verify paths and pass information. If packets were plaintext, intermediate nodes could not prove that a route was valid, nor could endpoints prevent spoofing or tampering. + +In Reticulum, the entropy of the encrypted packet *is* the routing logic. + +To ask for a version of Reticulum without encryption is like asking for a version of the ocean without liquid. You are not asking for a feature change; you're asking for a different physical universe. We design for a universe where information has mass, structure, and integrity. + +### Zero-Trust Architectures + +We must unlearn our reliance on **Institutional Trust**. + +For decades, we have been trained to trust authorities. We trust a website because a chain of Certificate Authorities (companies we don't know) vouches for it. We trust an app because it is in an app store (run by a corporation we don't control). We trust a message because it comes from a phone number assigned by a telecom. Yet, everything in our digital information sphere today is more untrustworthy and risky than a medieval second-hand underwear market. + +Reticulum replaces institutional trust with **Cryptographic Proof**. + +In Reticulum, you do not trust a node because it has a nice hostname or because it is listed in a directory. You trust it because it holds the private key corresponding to the Destination Hash you are communicating with. This trust is binary, mathematical, and **absolute**. Either the signature matches, or it does not. There is no "maybe". + +This shift moves the power from the institution to the individual. You become the ultimate arbiter of your own trust relationships. You decide which keys to accept, which paths to follow, and which identities to recognize. + +**Consider:** + +- **The Old Way:** *"I trust this site because the browser says the lock icon is green"*. +- **The Zen Way:** *"I trust this destination because I have verified its hash fingerprint out-of-band, and the math confirms the signature"*. + +When you internalize the Physics of Trust, you stop looking for protection from firewalls, VPNs, and Terms of Service agreements. You realize that true security comes from the design of the protocol itself. You can stop trusting the cloud, and you start trusting the code - because you can verify it yourself. + + +## III: Merits Of Scarcity +*Every Bit Counts* + +We have grown addicted to abundance. In the modern digital ecosystem, bandwidth is treated as an endless, flat ocean. We stream high-definition video without a thought, we ship entire libraries of code just to render a single button, and we measure performance in gigabits per second. This abundance has hollowed out our craft. When constraints vanish, efficiency dies, and with it, a certain kind of Clarity and Quality. + +Reticulum asks you to step out of the ocean and onto the tightrope. + +### The Bandwidth Fallacy + +The Zen of Reticulum requires the realization that **5 bits per second is a valid speed**. + +To a modern developer, this sounds like paralysis. But there is a profound freedom in limits: When you have a gigabit connection, you can be incredibly sloppy. You can be wasteful. You can push your problems onto the infrastructure. *"It’s slow? Get a faster router"*. + +But on a high-latency, low-bandwidth link (be it a noisy HF radio channel or a tenuous LoRa hop) you cannot push problems anywhere. You must solve them. The network does not negotiate with waste. + +This forces a shift from consumption to interaction. You are no longer, then, consuming a service provided by a fat pipe; you are engaging in a careful negotiation with the physical medium. The medium becomes a partner in the conversation, not just a dumb conduit. You suddenly need to *understand the world to be in it*. + +### Cost Of A Byte + +In a scarce economy, a byte is not just data, but energy, time, and space. + +Every byte you transmit consumes battery life on a solar-powered node. It occupies valuable airtime that could have been used by another peer. It represents a measurable slice of the electromagnetic spectrum. + +When you internalize this, you begin to write code differently. You stop asking, "How much data can I send?" and start asking, "What is the *minimum* amount of information required to convey this intent? How can I best utilize my informational entropy?" + +This is where the elegance of Reticulum shines. The protocol is designed to strip away the non-essential. A link establishment takes three very small packets. A destination hash fits in 16 bytes. The overhead is vanishingly small, leaving almost the entire channel for the message itself. + +**Consider:** + +- **The Old Way:** *"I need to send a status update. I'll send a JSON object with metadata, timestamps, and user profile info (15KB)."* +- **The Zen Way:** *"I need to send a status update. I'll send a single byte representing the state code. The context is already known."* + +This is of course optimization, but more importantly, *it is a form of respect*. Efficiency in a shared medium is an act of stewardship. By taking only what you need from the network, you leave room for others. The network listens to those who speak with purpose. + +### Flow & Time + +Scarcity also teaches us about time. We have become addicted to the *synchronous* now - the instant ping, the real-time stream. But Reticulum embraces *asynchronous* time. + +When links are intermittent and latency is measured in minutes or hours, "real-time" is an illusion. Reticulum doesn't encourage **Store and Forward** as a mere fallback, but as a primary mode of existence. You write a message, it propagates when it can, and it arrives when it arrives. + +This changes the psychological texture of communication. It removes the anxiety of the immediate response. It allows for contemplation. You are not demanding the recipient's attention *right now*; you are placing a gift in their path, to be found when they are ready. + +By designing for delay, you design for resilience. You are no longer building a house of cards that collapses when a single packet drops. You are building a stone arch that distributes the load *over time*. + +### Liberation From Limits + +There is a strange optimism in scarcity. When you are forced to work within strict constraints, you are forced to prioritize. *You* must decide what truly matters. *That* is the real core of agency. + +In the infinite fantasy world of The Cloud, everything is urgent, so nothing is. In the economy of Reticulum, the cost of transmission forces you to weigh the value of your message. Do you really need to send that heart beat? Is that photo essential? + +When you strip away the noise, what remains is *signal*. + +This discipline creates a different kind of developer. It creates a craftsman who understands that the best code is the code you don't have to write. It creates a user who understands that the most powerful message is the one that is *understood*, not the one that is loudest. In the world of Reticulum, you are not a mere consumer of bandwidth; you are an architect of intent. + + +## IV: Sovereignty Through Infrastructure +**Be Your Own Network** + +We live in an era of digital tenancy. We lease our connectivity from ISPs. We rent our storage from cloud providers. We even borrow our identity from social media platforms. We are tenants in a house we did not build, governed by rules we did not write, subject to eviction at the whim of a landlord who has never met us. + +The Zen of Reticulum is the realization that you *can* own the house. + +### A Carrier-Grade Fallacy + +For decades, we have been gaslit into believing that networking is really not just hard, but impossible. It is presented as a dark art reserved for telcos and billionaires, requiring millions of dollars of fiber optics, climate-controlled data centers, and armies of engineers. We are told that building reliable infrastructure is "too complex" for the individual or small organization. + +This is a big, fat lie. + +Physics is simple. A radio wave needs a transmitter and a receiver. A packet needs a path. The "complexity" of the modern internet is largely bureaucratic - a mountain of billing systems, regulatory capture, and legacy cruft designed to keep the gatekeepers in power. + +Reticulum strips away the bureaucracy. It runs on hardware that costs the price of a dinner. It runs on spectrum that is free to use. It demonstrates that a robust, planetary-scale network does not require a Fortune 500 company. It requires only the will to deploy, and the distributed, uncoordinated efforts of many individuals. + +### Personal Infrastructure + +This is where the rubber meets the road. You can read about Reticulum, you can understand the theory, but the insights only arrive when you plug in a radio and run a Transport Node. Suddenly, you are no longer a consumer. You're an operator. + +This shift is subtle but profound. When you run your own infrastructure, the network ceases to be a service that is provided *to* you. It becomes a space that you *inhabit*. You become responsible for the flow of information. You gain an intimate understanding of the medium - the way the weather affects the radio waves, the way the topology changes, the way the packets dance through the ether. + +There is a quiet competence that comes from this. You stop asking "Is the internet down?" and start asking "Is *my* links up?" You stop waiting for a technician and start checking the logs. This is a form of strength. To understand the system that carries your words is to be free from the mystery that keeps you dependent. + +### The Ability To Disconnect + +Why go to the trouble? Why buy the radio, write the config, and leave the Pi running in the corner? + +Because the old, centralized network is fragile. And because most of us doesn't even really want to be there anymore. + +The internet we rely on today is a chain of single points of failure. Cut the undersea cable, and a continent goes dark. Shut down the power grid, and the cloud evaporates. Deprioritize the "wrong" traffic, and the flow of information is strangled. + +Sovereignty is the ability to survive the cut, whether or not that cut was an accident or on purpose. + +When you build your own infrastructure, you build a lifeline. Reticulum is designed to function over media that the traditional internet cannot touch - bare wires, battery-powered radios, ad-hoc WiFi meshes. When the grid fails, or the censors arrive, or the bill goes unpaid, your Reticulum network continues to hum. + +This is not about "dropping out" of society. It is about building a substrate on which an actual *Society* can function. + +**Consider:** + +- **The Old Way:** "My connection is slow. I should call my ISP and complain." +- **The Zen Way:** "The path is noisy. I will adjust the antenna or find a better route." + +By taking ownership of the infrastructure, you take ownership of your voice. You stop shouting into someone else's megaphone and start building your own. The network is no longer something that happens to you; it is something you make happen. + + +# V: Identity and Nomadism +**A Fluid Self** + +In the old world, you are defined by your coordinates. If you are at `34.109.71.5`, you're *here*. If you unplug the cable and walk down the street, you vanish. Your digital self evaporates because it was tethered to the wall. You are a ghost in the endless machinations of gears, levers and transistors, bound to the hardware, and those that own it. + +This creates a subtle, constant anxiety. We are terrified of disconnecting because, in the architecture of the old web, disconnecting is a kind of death. + +The Zen of Reticulum offers a different way to be. + +### Portable Existence + +In Reticulum, your identity is not a location, or a username granted by a service. It is a cryptographic key - a complex, unique mathematical signature that exists independently of the physical world. You can carry it only in your mind, if you want to. + +Think of it less like a street address and more like a name. *A true name*. + +If you travel from Berlin to Tokyo, you do not change your name. You are still you. The people who know you can still recognize you. Reticulum applies this principle to the network layer. Your Destination Hash is **invariant**. It travels with you, stored securely on your device, *immutable as a stone*. + +This changes the relationship between you and the machine. You are not "logged into" the network via a specific gateway. You *are* the endpoint. The network does not connect to a place; *it converges on you*. + +### Roaming Nodes + +This freedom introduces a new concept of time and space: **Nomadism**. + +Because your identity is portable, your connectivity can be fluid. You can be sitting at a desk connected to a fiber backbone one moment, and walking through a field connected only to a long-range LoRa mesh the next. To the rest of the network, nothing has changed. Your friends do not need to update your contact info. The messages they send do not bounce back. The network senses the shift in the medium and reroutes the flow of data automatically. + +You are no longer a stationary node in a fixed grid. You are a wanderer in a fluid medium. + +The interfaces - whether it is WiFi, Ethernet, Packet Radio, or a physical wire - is merely the clothing your node wears. You change it to suit the environment. Underneath, you remain the same. This is the liberation of the protocol. It treats the physical medium as a transient circumstance, not a definition of self. + +**Consider:** + +- **The Old Way:** *"I lost connection. I have to reconnect to the VPN to tell them where I am now."* +- **The Zen Way:** *"I moved. The network subtly bends to accomodate this new reality."* + +### Announcing Presence + +How does the network find a wanderer? It listens. + +In the IP world, we query directories. We ask a server, "Where is Mark?" The server checks its database and gives us a coordinate. This means that someone, somewhere, is keeping track of you. It assumes and *requires* surveillance. + +Reticulum replaces surveillance with **Announces**. + +Instead of asking a central authority where you are, you simply state your presence. You broadcast a cryptographic proof: "I am here, and I am who I say I am". This ripples out through the mesh. Your neighbors hear it, update their path tables, and pass it on. + +This is a quiet, organic process. It is the digital equivalent of lighting lanterns in the dark. You do not need to chase the light; you let the light find you. It respects your autonomy. You choose when to announce, how often to speak, and to whom. You also choose when to disappear - for but a moment or perpetually. + +### Anchor In The Flow + +There is a deep peace in this nomadism. It teaches you that stability does not come from standing still. Stability comes from *internal coherence*. + +By holding your own private key, you hold your own center of gravity. The world around you; the infrastructure, the topography and the availability of links can all shift chaotically. Storms can knock out towers. Cables can be cut. The internet can go down. + +But as long as you possess your key, you possess your identity. The entire infrastructure can be destroyed and rebuilt, and you are still you. Nothing lasts, yet nothing is lost. + +You become a sovereign entity moving through the noise, connected not by the rigidity of cables, but by the fluidity of recognition. The network becomes a place you inhabit, rather than a utility you subscribe to: You are at home in the ether. + + +## VI: Ethics Of The Tool +**Technology With Conscience** + +You have unlearned the center. You have accepted the physics of trust. You have embraced the economy of scarcity and the freedom of unbound nomadism. You are standing in a new space. Now, look at the tool in your hand. + +In the old world, we were taught that technology is neutral. We are told that "guns don't kill people, people do", or that a component is just a component, indifferent to what its combinatorial potential is. This is a convenient lie. It serves only to allow the builders to wash their hands of responsibility. + +But we know better now. We know that **architecture is politics**, and *politics is control*. The way you build a system determines how it will be used. If you build a system optimized for mass surveillance, you *will* get a panopticon. If you build a system optimized for centralized control, you *will* get a dictatorship. If you build a system optimized for extraction, you *will* get a parasite. + +The Zen of Reticulum asserts that a tool is never neutral. + +On the very contrary: A tool is intent, **crystallized**. + +### The Harm Principle + +Why does the Reticulum License forbid the software from being used in systems designed to harm humans? Is it not just a restriction on freedom? + +It is a restriction on *license*, yes, but it is an expansion of *freedom*. + +Building powerful tools without a moral compass is in no way virtuous or commendable, it is plain and simple irresponsibility. + +A tool that can easily be used to oppress is a real danger to the user. If you build a network that can be turned against you by a tyrant, you are not free. You are merely waiting for the leash to tighten. By encoding the "Harm Principle" into the legal DNA of the reference implementation, we are building a safeguard. We are stating, clearly and immutably, that *this tool* is for **life**, not for death. + +This aligns the software with the interests of humanity. It cements that the network cannot be conscripted into a kill-system, a weaponized drone controller, or a torture device without breaking the license and the law. It is a line drawn in the sand - not by a government or external authority, but by the creators of the tool itself. + +**Consider:** + +- **The Old Way:** *"It's just software. How people use it is not my problem."* +- **The Zen Way:** *"This software is a habitat. I will not allow it to be used to build a cage."* + +It is *your* choice whether to align with this - we are not forcing this stance on anyone. If you choose to align with life over death, with creativity over destruction, we grant you an immensely powerful tool, to own and build with as you please. If you do not, we deny it. + +If you do not like this, we most assuredly do not need you here, and you are on your own. + +### Public Domain Protocol + +This leads to a vital distinction: The difference between the *idea* and the *implementation*. + +The protocol - the mathematical rules of how Reticulum works - is dedicated to the Public Domain. It belongs to humanity. **No one can own it**. Anyone can implement it, improve it, or adapt it. This is the core idea of free communication, which itself must be forever free. + +But the functional, deployed *reference implementation* - the Python code, the maintenance, the years of labor - has a conscience. This distinction is the engine of sustainability. It allows the protocol to be universal, while ensuring that the specific labor of the builders is not hijacked to undermine the foundational intent of the project itself. From this document, it should be very clear what this intent is. + +If you want to build a system with Reticulum that manipulates and damages users for profits or targets missiles, you can use the public domain protocol, and start from scratch. But you cannot take our work. You must do your own. This serves as a pillar of accountability. If you want to build a weapon, *you* go and forge the steel yourself, while the world observes. And when the blood is drawn - it is on **your** hands. + +### Preserving Human Agency + +We live in an era of predatory extraction. The open-source commons is being scraped, ingested, and regurgitated by machine learning algorithms, whose corporate owners seek to replace the very humans who built those commons. Our code, our words, and our creativity is being used to train systems that are specifically designed to make us obsolete, without offering anything else in return than serfdom and leashes. + +Reticulum stands against this. + +The license protects the software from being used to feed the beast. It draws a hard line: This tool is for *people*. It is for human-to-human connection. It is not a dataset to be strip-mined for the purpose of building a synthetic overlord, puppeteered by a miniscule conglomerate of controllers. + +This is a radical act of preservation. By protecting the code from AI appropriation, we are protecting space for human agency. We are ensuring that there remains a digital realm where the actors are flesh, blood and soul, where decisions are made by minds, not overlords hiding behind models. + +When you use Reticulum, you are using a tool that respects you. It does not see you as a product to be tracked. It does not see your data as fuel for an algorithm. It sees you as a sovereign, equal peer. + +This changes the foundational premise of using the technology. It restores dignity to the interaction. You are not the user of a service; you are a participant in a mutual covenant. The tool aligns with your autonomy, rather than eroding it. + +In this way, ethics is not a restriction, but a foundation. It is the foundation that helps ensure the network will still belong to you tomorrow. + + +## VII: Design Patterns For Post-IP Systems +**Practical Philosophy for Developers** + +The philosophy is useless if it cannot be hammered into code. The metaphors we have explored - nomadism, scarcity, trust - are not just poetry, but real-world engineering constraints. When you sit down to write software for Reticulum, these concepts must shape the very structure of your application. + +We are now moving from the *why* to the *how*. This is where the abstract becomes concrete, and where you will see the true depth of the patterns we have been weaving. + +### Store & Forward + +The web has trained us to be impatient. We write synchronous code. We fire a request and we wait, blocking the UI, holding our breath. If the response doesn't come in 250 milliseconds, we show a spinner. If it doesn't come in five seconds, we show an error. We treat network connectivity as a binary state: either we are "online" or we are "broken". + +This is brittle. It is a rejection of reality. + +In Reticulum, connectivity is a spectrum, and presence is asynchronous. If at all applicable to your intent, you must design your applications to embrace **Store & Forward**. + +Instead of demanding an immediate answer, your application should act as a patient participant. You create a message for someone or something in the mesh. The network holds it. It carries it from node to node, perhaps over hours or days, waiting for the recipient to appear. When they finally surface, the message is delivered. This requires a shift from "request/response" to "event/handler". How exactly you do this is a challenge for you to solve intelligently within your problem domain, but Reticulum-based systems already exist that does this extremely well, and you can use them for inspiration. + +**Consider:** + +- **The Old Way:** `Connect() -> Send() -> Wait() -> Crash if timeout.` +- **The Zen Way:** `Send() -> Continue living. -> Receive() when it arrives.` + +This changes the user experience profoundly. It removes the anxiety of the loading bar. It creates a sense of continuity. The user is not "waiting for the network"; they are interacting with a persistent log of communication that lives in the network itself. + +### Naming Is Power + +In the IP world, we are slaves to the Domain Name System. We rely on a hierarchy of registrars to map human-readable names to machine-readable addresses. This hierarchy is a choke point. If the registrar revokes your domain, or if the DNS server goes down, you vanish. + +Reticulum dissolves this hierarchy with **Hash-based Identity**. + +In this design pattern, a name is not a string you look up; it is a cryptographic destination you verify. When you design for Reticulum, you stop asking the user for a URL and start asking for a Destination or Identity Hash. + +This feels strange at first. A hash like `<83b7328926fed0d2e6a10a7671f9e237>` looks alien compared to `myfriend.com`. But that alienness is the armor. It **cannot** be spoofed. It **cannot** be censored by a registrar. It is **absolute**. + +Designing for this means shifting your UI metaphors. You are no longer browsing a web of pages; you are managing a ledger of keys. You are building an "Address Book" that is actually a keyring. The names are given by the user, and the power stays with them. That hashes look complex is directly analogous to the strengths of the bonds formed by their use. It forces the user to engage in a moment of verification, an out-of-band handshake, which restores the human element of trust that SSL certificates stripped away. + +### The Interface Is The Medium + +One of the most liberating patterns in Reticulum is **Transport Agnosticism**. + +In traditional networking, your code is often littered with transport logic. "Am I on WiFi? Check bandwidth. Am I on Cellular? Check data plan. Am I on Ethernet?". You are constantly micromanaging the pipe. + +In Reticulum, you write to the API, and the API writes to the medium. You send a packet to a Destination. You do not care if that packet travels over a TCP tunnel, a LoRa radio wave, or a serial wire interface. That is the stack's concern. + +This allows you to write **Universal Applications**. +Imagine a messaging app. You write it once. It works on a laptop connected to fiber. It works on a phone in the city using WiFi. And, without a single line of code changed, it works on a device in the wilderness, talking only to other devices via radio. + +The pattern is simple: **Never code to the hardware. Code to the intent.** + +**Consider:** + +- **The Old Way:** `socket.connect(ip, port)` +- **The Zen Way:** `RNS.Packet(destination, data).send()` + +By abstracting the medium, you make your software immortal to changes in infrastructure. The user might switch from a 4G hotspot to a HF modem tomorrow. Your software doesn't need to know. It simply continues the conversation. + +### Emergent Patterns + +When you combine these patterns - *Store & Forward*, *Hash-based Identity*, and *Transport Agnosticism* - you create software that feels fundamentally different. + +It feels *grounded*. It doesn't flicker when the signal drops. It doesn't panic when the server is down. It has weight. It has persistence. It has *relevance*. + +You are no longer building a "client" that begs a "server" for attention. You are building an autonomous agent that exists within the mesh. It speaks when it needs to, listens when it can, and carries its identity with it wherever it goes. + +This is the culmination of the Zen. The code is not just a set of instructions: It is a behavioral envelope. It is a way of *being* in the network. + + +## VIII: Fabric Of The Independent + +We have stripped away the illusions. We have seen that the center is empty, that trust *must* be hard, that resources are finite, and that we must own our infrastructure. We have seen that tools have ethics and that our identity can move fluidly. + +This is a reclaiming of the commons. For too long, we have allowed the most vital substrate of human society - *our ability to speak to one another* - to be colonized by entities that do not share our interests. We have allowed the architecture of our communication to be designed by accountants rather than architects. + +We are taking it back. Not by petitioning the masters, but by building the new world within, over, under and around the shell of the old. + +### The Work Is Finished + +The heavy lifting is done. + +The protocol is in the public domain, a gift to humanity that can never be taken away. The software is written, tested, and running on devices scattered across the globe. The manual lies open before you. The source code for the reference implementation is now distributed on hundreds of thousands of devices across the planet. No one can delete or destroy it. The hardware is accessible and abundant. + +It was a hard road to get here, but we got here. Now, there is no roadmap committee waiting for approval. There is no venture capital dictating the user experience. There is no CEO to sign off on the next feature release. + +There is only you. + +The barrier to entry is no longer complexity: It is the mere habit of dependency. You were conditioned to wait. Wait for the app update. Wait for the ISP to fix the line. Wait for the platform to allow the post. Wait for the government to change the policies. Wait for the likes. Wait for the revolution to be televised. + +The revolution never was televised. + +It is packetized. + +### Open Sky + +The future of this technology is a construction project. + +It looks like a single node on a windowsill, listening to the static. It looks like a message sent to a neighbor, bypassing the noise of the commercial web. It looks like a community mesh that grows, link by link, hop by hop, carried by hands that care more about connection than profit. + +You have the blueprints. You have the tools. You have the philosophy. The noise of the old world has fallen away, leaving you with the quiet clarity of the open spectrum. + +*Mark, early 2026* \ No newline at end of file diff --git a/docs/example_receiver.py b/docs/example_receiver.py new file mode 100644 index 0000000..20c2efe --- /dev/null +++ b/docs/example_receiver.py @@ -0,0 +1,72 @@ +import RNS +import LXMF +import time + +required_stamp_cost = 8 +enforce_stamps = False + +def delivery_callback(message): + global my_lxmf_destination, router + time_string = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(message.timestamp)) + signature_string = "Signature is invalid, reason undetermined" + if message.signature_validated: + signature_string = "Validated" + else: + if message.unverified_reason == LXMF.LXMessage.SIGNATURE_INVALID: + signature_string = "Invalid signature" + if message.unverified_reason == LXMF.LXMessage.SOURCE_UNKNOWN: + signature_string = "Cannot verify, source is unknown" + + if message.stamp_valid: + stamp_string = "Validated" + else: + stamp_string = "Invalid" + + RNS.log("\t+--- LXMF Delivery ---------------------------------------------") + RNS.log("\t| Source hash : "+RNS.prettyhexrep(message.source_hash)) + RNS.log("\t| Source instance : "+str(message.get_source())) + RNS.log("\t| Destination hash : "+RNS.prettyhexrep(message.destination_hash)) + RNS.log("\t| Destination instance : "+str(message.get_destination())) + RNS.log("\t| Transport Encryption : "+str(message.transport_encryption)) + RNS.log("\t| Timestamp : "+time_string) + RNS.log("\t| Title : "+str(message.title_as_string())) + RNS.log("\t| Content : "+str(message.content_as_string())) + RNS.log("\t| Fields : "+str(message.fields)) + if message.ratchet_id: + RNS.log("\t| Ratchet : "+str(RNS.Identity._get_ratchet_id(message.ratchet_id))) + RNS.log("\t| Message signature : "+signature_string) + RNS.log("\t| Stamp : "+stamp_string) + RNS.log("\t+---------------------------------------------------------------") + + # Optionally, send a reply + # source = my_lxmf_destination + # dest = message.source + # lxm = LXMF.LXMessage(dest, source, "Reply", None, desired_method=LXMF.LXMessage.DIRECT, include_ticket=True) + # router.handle_outbound(lxm) + +r = RNS.Reticulum() + +router = LXMF.LXMRouter(storagepath="./tmp1", enforce_stamps=enforce_stamps) +identity = RNS.Identity() +my_lxmf_destination = router.register_delivery_identity(identity, display_name="Anonymous Peer", stamp_cost=required_stamp_cost) +router.register_delivery_callback(delivery_callback) + +RNS.log("Ready to receive on: "+RNS.prettyhexrep(my_lxmf_destination.hash)) + + +# You can set a propagation node address to test receiving +# messages from a propagation node, instead of directly + +# router.set_outbound_propagation_node(bytes.fromhex("e75d9b6a69f82b48b6077cf2242d7499")) + + +# This loop allows you to execute various actions for testing +# and experimenting with the example scripts. +while True: + input() + RNS.log("Announcing lxmf.delivery destination...") + router.announce(my_lxmf_destination.hash) + + # input() + # RNS.log("Requesting messages from propagation node...") + # router.request_messages_from_propagation_node(identity) diff --git a/docs/example_sender.py b/docs/example_sender.py new file mode 100644 index 0000000..bcb8d36 --- /dev/null +++ b/docs/example_sender.py @@ -0,0 +1,69 @@ +import LXMF +import RNS +import time +import random + +random_names = ["Tom", "Delilah", "Nancey", "Williams", "Neomi", "Curtis", "Alexa", "Theodora", "Ted", "Dinorah", "Nicol", "Drusilla", "Annalisa", "Verlene", "Latesha", "Tina", "Mia", "Brock", "Timothy", "Philip", "Willian", "Reyna", "Simona", "Mimi", "Stanford", "Ferne", "Catalina", "Lucie", "Jaye", "Natasha", "Willetta", "Isabel", "Esperanza", "Ciara", "Eusebio", "William", "Elma", "Angelica", "Coreen", "Melani", "Jonathan", "Maryland", "Caroline", "Gregg", "Ora", "Jacqui", "Letty", "Roselle", "Oralee", "Angla"] +random_titles = ["Long time", "Hi again", "Hi there", "Test message", "", "", "Something different"] +random_msgs = ["If wishes were horses then beggars might fly. Stuff like that. It's enough to drive you crazy.", "'My ident cards were stolen,' Jason said. 'That fivehundred-dollar bill is yours if you can get me to someone who can replace them. If you're going to do it, do it right now; I'm not going to wait.' Wait to be picked up by a pol or a nat, he thought. Caught here in this rundown dingy hotel.", "A six, no matter what the external circumstances, will always prevail. Because that's the way they genetically defined us.", "'Should be there in an hour,' he called back over his shoulder to Chuck. Then he added, in an afterthought, 'Wonder if the computer’s finished its run. It was due about now.'. Chuck didn’t reply, so George swung round in his saddle. He could just see Chuck’s face, a white oval turned toward the sky."] + +def delivery_callback(message): + pass + +r = RNS.Reticulum() +router = LXMF.LXMRouter(storagepath="./tmp2") +router.register_delivery_callback(delivery_callback) +ident = RNS.Identity() +source = router.register_delivery_identity(ident, display_name=random_names[random.randint(0,len(random_names)-1)], stamp_cost=8) +router.announce(source.hash) +RNS.log("Source announced") + +print("Recipient: ", end=" ") +recipient_hexhash = input() +recipient_hash = bytes.fromhex(recipient_hexhash) + +if not RNS.Transport.has_path(recipient_hash): + RNS.log("Destination is not yet known. Requesting path and waiting for announce to arrive...") + RNS.Transport.request_path(recipient_hash) + while not RNS.Transport.has_path(recipient_hash): + time.sleep(0.1) + +# Recall the server identity +recipient_identity = RNS.Identity.recall(recipient_hash) + +dest = RNS.Destination(recipient_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, "lxmf", "delivery") + +# This loop allows you to execute various actions for testing +# and experimenting with the example scripts. +while True: + + # Create a message that will be sent directly to the + # destination over a Reticulum link: + + lxm = LXMF.LXMessage(dest, source, random_msgs[random.randint(0,len(random_msgs)-1)], + random_titles[random.randint(0,len(random_titles)-1)], + desired_method=LXMF.LXMessage.DIRECT, include_ticket=True) + + # Or, create an oppertunistic, single-packet message + # for sending without first establishing a link: + + # lxm = LXMF.LXMessage(dest, source, "This is a test", + # random_titles[random.randint(0,len(random_titles)-1)], + # desired_method=LXMF.LXMessage.OPPORTUNISTIC, include_ticket=True) + + + # Or, try sending the message via a propagation node: + + # router.set_outbound_propagation_node(bytes.fromhex("e75d9b6a69f82b48b6077cf2242d7499")) + # lxm = LXMF.LXMessage(dest, source, random_msgs[random.randint(0,len(random_msgs)-1)], + # random_titles[random.randint(0,len(random_titles)-1)], + # desired_method=LXMF.LXMessage.PROPAGATED) + + # Finally dispatch the message to the LXMF message + # router, which will handle the delivery according + # to the specified message parameters and options: + + router.handle_outbound(lxm) + + # Wait for user input before starting over + input() \ No newline at end of file diff --git a/docs/paper_msg_test.png b/docs/paper_msg_test.png new file mode 100644 index 0000000..ff901c6 Binary files /dev/null and b/docs/paper_msg_test.png differ diff --git a/examples/lxmf_message_minimal.hex b/examples/lxmf_message_minimal.hex new file mode 100644 index 0000000..85f8d89 --- /dev/null +++ b/examples/lxmf_message_minimal.hex @@ -0,0 +1 @@ +000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f94cb41d954fc40100000c40a54657374207469746c65c41744657465726d696e6973746963204c584d4620626f647980 diff --git a/examples/lxmf_message_stamped.hex b/examples/lxmf_message_stamped.hex new file mode 100644 index 0000000..be6fc07 --- /dev/null +++ b/examples/lxmf_message_stamped.hex @@ -0,0 +1 @@ +000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f95cb41d954fc40100000c40a54657374207469746c65c41744657465726d696e6973746963204c584d4620626f647980c420a0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebf diff --git a/examples/lxmf_message_vectors.json b/examples/lxmf_message_vectors.json new file mode 100644 index 0000000..b60d155 --- /dev/null +++ b/examples/lxmf_message_vectors.json @@ -0,0 +1,26 @@ +{ + "content_hex" : "44657465726d696e6973746963204c584d4620626f6479", + "description" : "Deterministic LXMessage packing vectors; signature bytes are placeholders", + "destination_hash_hex" : "000102030405060708090a0b0c0d0e0f", + "fields" : {}, + "format" : "destination_hash || source_hash || signature || msgpack_payload", + "message_id_hex" : "55f497f6f2ea5d858eaff541a9fa77f9d5bdf57be75be4a94df65d1f9a7ffea3", + "minimal_lxmf_hex" : "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f94cb41d954fc40100000c40a54657374207469746c65c41744657465726d696e6973746963204c584d4620626f647980", + "payload_order" : [ + "timestamp", + "title", + "content", + "fields", + "optional_stamp" + ], + "payload_with_stamp_hex" : "95cb41d954fc40100000c40a54657374207469746c65c41744657465726d696e6973746963204c584d4620626f647980c420a0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebf", + "payload_without_stamp_hex" : "94cb41d954fc40100000c40a54657374207469746c65c41744657465726d696e6973746963204c584d4620626f647980", + "signature_hex" : "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f", + "signature_valid" : false, + "signed_part_hex" : "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f94cb41d954fc40100000c40a54657374207469746c65c41744657465726d696e6973746963204c584d4620626f64798055f497f6f2ea5d858eaff541a9fa77f9d5bdf57be75be4a94df65d1f9a7ffea3", + "source_hash_hex" : "101112131415161718191a1b1c1d1e1f", + "stamp_hex" : "a0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebf", + "stamped_lxmf_hex" : "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f95cb41d954fc40100000c40a54657374207469746c65c41744657465726d696e6973746963204c584d4620626f647980c420a0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebf", + "timestamp" : 1700000000.25, + "title_hex" : "54657374207469746c65" +} diff --git a/grep_results.txt b/grep_results.txt new file mode 100644 index 0000000..52ae092 --- /dev/null +++ b/grep_results.txt @@ -0,0 +1,39 @@ +./LXMF/Utilities/lxmd.py:DEFFERED_JOBS_DELAY = 10 +./LXMF/Utilities/lxmd.py:def create_default_config(configpath): +./LXMF/Utilities/lxmd.py:def apply_config(): +./LXMF/Utilities/lxmd.py:def lxmf_delivery(lxm): +./LXMF/Utilities/lxmd.py:def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbound = None, verbosity = 0, quietness = 0, service = False): +./LXMF/Utilities/lxmd.py:def jobs(): +./LXMF/Utilities/lxmd.py:def deferred_start_jobs(): +./LXMF/Utilities/lxmd.py:def _request_sync(identity, destination_hash, remote_identity, timeout=15, exit_on_fail=False): +./LXMF/Utilities/lxmd.py:def request_sync(target, remote=None, configdir=None, rnsconfigdir=None, verbosity=0, quietness=0, timeout=15, identity_path=None): +./LXMF/Utilities/lxmd.py:def _request_unpeer(identity, destination_hash, remote_identity, timeout=15, exit_on_fail=False): +./LXMF/Utilities/lxmd.py:def request_unpeer(target, remote=None, configdir=None, rnsconfigdir=None, verbosity=0, quietness=0, timeout=15, identity_path=None): +./LXMF/Utilities/lxmd.py:def query_status(identity, remote_identity=None, timeout=5, exit_on_fail=False): +./LXMF/Utilities/lxmd.py:def get_status(remote=None, configdir=None, rnsconfigdir=None, verbosity=0, quietness=0, timeout=5, +./LXMF/Utilities/lxmd.py:def _get_target_identity(remote=None, timeout=5): +./LXMF/Utilities/lxmd.py:def _remote_init(configdir=None, rnsconfigdir=None, verbosity=0, quietness=0, identity_path=None): +./LXMF/Utilities/lxmd.py:def main(): +./LXMF/LXMF.py:def display_name_from_app_data(app_data=None): +./LXMF/LXMF.py:def stamp_cost_from_app_data(app_data=None): +./LXMF/LXMF.py:def compression_support_from_app_data(app_data=None): +./LXMF/LXMF.py:def pn_name_from_app_data(app_data=None): +./LXMF/LXMF.py:def pn_stamp_cost_from_app_data(app_data=None): +./LXMF/LXMF.py:def pn_announce_data_is_valid(data): +./LXMF/LXStamper.py:def worker_context(ctx, stamp_cost, workblock, message_id): +./LXMF/LXStamper.py:def stamp_workblock(material, expand_rounds=WORKBLOCK_EXPAND_ROUNDS): +./LXMF/LXStamper.py:def stamp_value(workblock, stamp): +./LXMF/LXStamper.py:def stamp_valid(stamp, target_cost, workblock): +./LXMF/LXStamper.py:def validate_peering_key(peering_id, peering_key, target_cost): +./LXMF/LXStamper.py:def validate_pn_stamp(transient_data, target_cost): +./LXMF/LXStamper.py:def validate_pn_stamps_job_simple(transient_list, target_cost): +./LXMF/LXStamper.py:def validate_pn_stamps_job_multip(transient_list, target_cost): +./LXMF/LXStamper.py:def validate_pn_stamps(transient_list, target_cost): +./LXMF/LXStamper.py:def generate_stamp(message_id, stamp_cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS): +./LXMF/LXStamper.py:def cancel_work(message_id): +./LXMF/LXStamper.py:def job_simple(stamp_cost, workblock, message_id): +./LXMF/LXStamper.py:def job_linux_managed(stamp_cost, workblock, message_id): +./LXMF/LXStamper.py:def job_linux(stamp_cost, workblock, message_id): +./LXMF/LXStamper.py:def job_android(stamp_cost, workblock, message_id): +./docs/example_receiver.py:def delivery_callback(message): +./docs/example_sender.py:def delivery_callback(message): diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f0f3fc8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +qrcode>=7.4.2 +rns>=1.0.0 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..82521ec --- /dev/null +++ b/setup.py @@ -0,0 +1,36 @@ +import sys +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +exec(open("LXMF/_version.py", "r").read()) + +if "--getversion" in sys.argv: + print(__version__, end="") + exit(0) + +setuptools.setup( + name="lxmf", + version=__version__, + author="Mark Qvist", + author_email="mark@unsigned.io", + description="Lightweight Extensible Message Format for Reticulum", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/markqvist/lxmf", + packages=["LXMF", "LXMF.Utilities"], + license="Reticulum License", + license_files = ("LICENSE"), + classifiers=[ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", + ], + entry_points= { + 'console_scripts': [ + 'lxmd=LXMF.Utilities.lxmd:main', + ] + }, + install_requires=["rns>=1.3.5"], + python_requires=">=3.7", +) diff --git a/tools/LXMF/MessagePack.pm b/tools/LXMF/MessagePack.pm new file mode 100644 index 0000000..5ce8c39 --- /dev/null +++ b/tools/LXMF/MessagePack.pm @@ -0,0 +1,181 @@ +package LXMF::MessagePack; + +use strict; +use warnings; +use Exporter 'import'; + +our @EXPORT_OK = qw( + pack_array + pack_bin + pack_float64 + pack_map + parse_item +); + +sub pack_float64 { + my ($value) = @_; + return "\xcb" . pack("d>", $value); +} + +sub pack_bin { + my ($value) = @_; + my $length = length($value); + + return "\xc4" . pack("C", $length) . $value if $length <= 0xff; + return "\xc5" . pack("n", $length) . $value if $length <= 0xffff; + return "\xc6" . pack("N", $length) . $value; +} + +sub pack_array { + my (@items) = @_; + my $length = scalar @items; + my $header; + + if ($length <= 15) { + $header = pack("C", 0x90 | $length); + } + elsif ($length <= 0xffff) { + $header = "\xdc" . pack("n", $length); + } + else { + $header = "\xdd" . pack("N", $length); + } + + return $header . join("", @items); +} + +sub pack_map { + my (@entries) = @_; + die "pack_map requires key/value byte-string pairs\n" if @entries % 2; + + my $length = @entries / 2; + my $header; + + if ($length <= 15) { + $header = pack("C", 0x80 | $length); + } + elsif ($length <= 0xffff) { + $header = "\xde" . pack("n", $length); + } + else { + $header = "\xdf" . pack("N", $length); + } + + return $header . join("", @entries); +} + +sub _take { + my ($bytes, $offset_ref, $length) = @_; + die "truncated MessagePack item\n" if $$offset_ref + $length > length($bytes); + my $value = substr($bytes, $$offset_ref, $length); + $$offset_ref += $length; + return $value; +} + +sub _parse_length { + my ($bytes, $offset_ref, $width) = @_; + my $raw = _take($bytes, $offset_ref, $width); + return unpack("C", $raw) if $width == 1; + return unpack("n", $raw) if $width == 2; + return unpack("N", $raw); +} + +sub parse_item { + my ($bytes, $offset_ref) = @_; + $offset_ref //= \my $offset; + $$offset_ref //= 0; + + my $start = $$offset_ref; + my $code = unpack("C", _take($bytes, $offset_ref, 1)); + my $item = { code => $code }; + + if ($code <= 0x7f) { + @$item{qw(type value)} = ("integer", $code); + } + elsif ($code >= 0xe0) { + @$item{qw(type value)} = ("integer", $code - 256); + } + elsif (($code & 0xf0) == 0x90) { + _parse_array($bytes, $offset_ref, $item, $code & 0x0f); + } + elsif (($code & 0xf0) == 0x80) { + _parse_map($bytes, $offset_ref, $item, $code & 0x0f); + } + elsif (($code & 0xe0) == 0xa0) { + my $length = $code & 0x1f; + @$item{qw(type value)} = ("string", _take($bytes, $offset_ref, $length)); + } + elsif ($code == 0xc0) { + $item->{type} = "nil"; + $item->{value} = undef; + } + elsif ($code == 0xc2 || $code == 0xc3) { + @$item{qw(type value)} = ("boolean", $code == 0xc3 ? 1 : 0); + } + elsif ($code == 0xca) { + @$item{qw(type value)} = ("float32", unpack("f>", _take($bytes, $offset_ref, 4))); + } + elsif ($code == 0xcb) { + @$item{qw(type value)} = ("float64", unpack("d>", _take($bytes, $offset_ref, 8))); + } + elsif ($code >= 0xc4 && $code <= 0xc6) { + my $width = 1 << ($code - 0xc4); + my $length = _parse_length($bytes, $offset_ref, $width); + @$item{qw(type value)} = ("binary", _take($bytes, $offset_ref, $length)); + } + elsif ($code >= 0xcc && $code <= 0xcf) { + my @formats = ("C", "n", "N", "Q>"); + my @widths = (1, 2, 4, 8); + my $index = $code - 0xcc; + @$item{qw(type value)} = ( + "integer", + unpack($formats[$index], _take($bytes, $offset_ref, $widths[$index])), + ); + } + elsif ($code >= 0xd0 && $code <= 0xd3) { + my @formats = ("c", "s>", "l>", "q>"); + my @widths = (1, 2, 4, 8); + my $index = $code - 0xd0; + @$item{qw(type value)} = ( + "integer", + unpack($formats[$index], _take($bytes, $offset_ref, $widths[$index])), + ); + } + elsif ($code == 0xd9 || $code == 0xda || $code == 0xdb) { + my $width = 1 << ($code - 0xd9); + my $length = _parse_length($bytes, $offset_ref, $width); + @$item{qw(type value)} = ("string", _take($bytes, $offset_ref, $length)); + } + elsif ($code == 0xdc || $code == 0xdd) { + my $width = $code == 0xdc ? 2 : 4; + _parse_array($bytes, $offset_ref, $item, _parse_length($bytes, $offset_ref, $width)); + } + elsif ($code == 0xde || $code == 0xdf) { + my $width = $code == 0xde ? 2 : 4; + _parse_map($bytes, $offset_ref, $item, _parse_length($bytes, $offset_ref, $width)); + } + else { + die sprintf("unsupported MessagePack code 0x%02x at offset %d\n", $code, $start); + } + + $item->{start} = $start; + $item->{end} = $$offset_ref; + $item->{raw} = substr($bytes, $start, $$offset_ref - $start); + return $item; +} + +sub _parse_array { + my ($bytes, $offset_ref, $item, $length) = @_; + $item->{type} = "array"; + $item->{items} = [map { parse_item($bytes, $offset_ref) } 1 .. $length]; +} + +sub _parse_map { + my ($bytes, $offset_ref, $item, $length) = @_; + $item->{type} = "map"; + $item->{entries} = [ + map { [parse_item($bytes, $offset_ref), parse_item($bytes, $offset_ref)] } 1 .. $length + ]; +} + +1; diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 0000000..75538fa --- /dev/null +++ b/tools/README.md @@ -0,0 +1,32 @@ +# LXMessage Test-Vector Tooling + +The scripts in this directory verify the implementation-derived LXMessage +packing behavior described in `Code_Response_1.md`. + +They use only Perl core modules. The local `LXMF::MessagePack` module implements +the MessagePack subset needed to generate the vectors and parse their complete +payloads. + +Generate deterministic vectors: + +```sh +perl tools/make_test_vectors.pl +``` + +Verify the vectors: + +```sh +perl tools/verify_examples.pl +``` + +The vectors confirm: + +- the 16-byte destination hash, 16-byte source hash, and 64-byte signature + positions; +- payload order `[timestamp, title, content, fields, optional_stamp]`; +- float64 timestamp and binary title/content/stamp encodings; +- the message ID calculation; +- exclusion of the optional stamp from the message ID and signature input. + +The signature bytes are deterministic placeholders. These vectors do not claim +to verify Ed25519 signing or signature validation. diff --git a/tools/make_test_vectors.pl b/tools/make_test_vectors.pl new file mode 100755 index 0000000..8d4385b --- /dev/null +++ b/tools/make_test_vectors.pl @@ -0,0 +1,76 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use Digest::SHA qw(sha256); +use FindBin qw($Bin); +use File::Path qw(make_path); +use JSON::PP; +use lib "$Bin"; +use LXMF::MessagePack qw(pack_array pack_bin pack_float64 pack_map); + +my $output_dir = "$Bin/../examples"; +make_path($output_dir); + +my $destination_hash = pack("C*", 0x00 .. 0x0f); +my $source_hash = pack("C*", 0x10 .. 0x1f); +my $signature = pack("C*", 0x20 .. 0x5f); +my $stamp = pack("C*", 0xa0 .. 0xbf); + +my $timestamp = 1_700_000_000.25; +my $title = "Test title"; +my $content = "Deterministic LXMF body"; +my $fields = pack_map(); + +my @required_items = ( + pack_float64($timestamp), + pack_bin($title), + pack_bin($content), + $fields, +); + +my $payload = pack_array(@required_items); +my $stamped_payload = pack_array(@required_items, pack_bin($stamp)); +my $message_id = sha256($destination_hash . $source_hash . $payload); +my $signed_part = $destination_hash . $source_hash . $payload . $message_id; + +my $minimal = $destination_hash . $source_hash . $signature . $payload; +my $stamped = $destination_hash . $source_hash . $signature . $stamped_payload; + +write_hex("$output_dir/lxmf_message_minimal.hex", $minimal); +write_hex("$output_dir/lxmf_message_stamped.hex", $stamped); + +my $manifest = { + description => "Deterministic LXMessage packing vectors; signature bytes are placeholders", + format => "destination_hash || source_hash || signature || msgpack_payload", + payload_order => [qw(timestamp title content fields optional_stamp)], + destination_hash_hex => unpack("H*", $destination_hash), + source_hash_hex => unpack("H*", $source_hash), + signature_hex => unpack("H*", $signature), + signature_valid => JSON::PP::false, + timestamp => $timestamp, + title_hex => unpack("H*", $title), + content_hex => unpack("H*", $content), + fields => {}, + stamp_hex => unpack("H*", $stamp), + payload_without_stamp_hex => unpack("H*", $payload), + payload_with_stamp_hex => unpack("H*", $stamped_payload), + message_id_hex => unpack("H*", $message_id), + signed_part_hex => unpack("H*", $signed_part), + minimal_lxmf_hex => unpack("H*", $minimal), + stamped_lxmf_hex => unpack("H*", $stamped), +}; + +open my $json_fh, ">:raw", "$output_dir/lxmf_message_vectors.json" + or die "cannot write vector manifest: $!\n"; +print {$json_fh} JSON::PP->new->canonical->pretty->encode($manifest); +close $json_fh; + +print "Generated deterministic LXMessage vectors in $output_dir\n"; + +sub write_hex { + my ($path, $bytes) = @_; + open my $fh, ">:raw", $path or die "cannot write $path: $!\n"; + print {$fh} unpack("H*", $bytes), "\n"; + close $fh; +} diff --git a/tools/verify_examples.pl b/tools/verify_examples.pl new file mode 100755 index 0000000..0f86e67 --- /dev/null +++ b/tools/verify_examples.pl @@ -0,0 +1,88 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use Digest::SHA qw(sha256); +use FindBin qw($Bin); +use JSON::PP; +use lib "$Bin"; +use LXMF::MessagePack qw(pack_array parse_item); + +my $examples_dir = "$Bin/../examples"; +my $manifest = read_json("$examples_dir/lxmf_message_vectors.json"); +my $minimal = read_hex("$examples_dir/lxmf_message_minimal.hex"); +my $stamped = read_hex("$examples_dir/lxmf_message_stamped.hex"); + +verify_lxmessage("minimal", $minimal, $manifest, 4); +verify_lxmessage("stamped", $stamped, $manifest, 5); + +check(unpack("H*", $minimal) eq $manifest->{minimal_lxmf_hex}, "minimal full bytes match manifest"); +check(unpack("H*", $stamped) eq $manifest->{stamped_lxmf_hex}, "stamped full bytes match manifest"); + +print "All LXMessage test-vector checks passed.\n"; + +sub verify_lxmessage { + my ($name, $bytes, $expected, $expected_count) = @_; + + check(length($bytes) > 96, "$name vector has fixed prefix and payload"); + my $destination_hash = substr($bytes, 0, 16); + my $source_hash = substr($bytes, 16, 16); + my $signature = substr($bytes, 32, 64); + my $packed_payload = substr($bytes, 96); + + check(unpack("H*", $destination_hash) eq $expected->{destination_hash_hex}, "$name destination hash"); + check(unpack("H*", $source_hash) eq $expected->{source_hash_hex}, "$name source hash"); + check(unpack("H*", $signature) eq $expected->{signature_hex}, "$name signature position and length"); + + my $offset = 0; + my $payload = parse_item($packed_payload, \$offset); + check($offset == length($packed_payload), "$name payload consumes all remaining bytes"); + check($payload->{type} eq "array", "$name payload is a MessagePack array"); + check(@{$payload->{items}} == $expected_count, "$name payload has $expected_count entries"); + + my @items = @{$payload->{items}}; + check($items[0]{type} eq "float64", "$name payload[0] timestamp is float64"); + check($items[0]{value} == $expected->{timestamp}, "$name payload[0] timestamp value"); + check($items[1]{type} eq "binary", "$name payload[1] title is binary"); + check(unpack("H*", $items[1]{value}) eq $expected->{title_hex}, "$name payload[1] title value"); + check($items[2]{type} eq "binary", "$name payload[2] content is binary"); + check(unpack("H*", $items[2]{value}) eq $expected->{content_hex}, "$name payload[2] content value"); + check($items[3]{type} eq "map", "$name payload[3] fields is a map"); + check(@{$items[3]{entries}} == 0, "$name payload[3] fields map is empty"); + + my $unstamped_payload = pack_array(map { $_->{raw} } @items[0 .. 3]); + my $message_id = sha256($destination_hash . $source_hash . $unstamped_payload); + my $signed_part = $destination_hash . $source_hash . $unstamped_payload . $message_id; + + check(unpack("H*", $unstamped_payload) eq $expected->{payload_without_stamp_hex}, "$name canonical unstamped payload"); + check(unpack("H*", $message_id) eq $expected->{message_id_hex}, "$name message ID excludes optional stamp"); + check(unpack("H*", $signed_part) eq $expected->{signed_part_hex}, "$name signature input excludes optional stamp"); + + if ($expected_count == 5) { + check($items[4]{type} eq "binary", "$name payload[4] stamp is binary"); + check(unpack("H*", $items[4]{value}) eq $expected->{stamp_hex}, "$name payload[4] stamp value"); + } +} + +sub read_json { + my ($path) = @_; + open my $fh, "<:raw", $path or die "cannot read $path: $!\n"; + local $/; + return JSON::PP->new->decode(<$fh>); +} + +sub read_hex { + my ($path) = @_; + open my $fh, "<:raw", $path or die "cannot read $path: $!\n"; + local $/; + my $hex = <$fh>; + $hex =~ s/\s+//g; + die "$path contains non-hexadecimal data\n" if $hex =~ /[^0-9a-fA-F]/; + return pack("H*", $hex); +} + +sub check { + my ($condition, $description) = @_; + die "not ok - $description\n" unless $condition; + print "ok - $description\n"; +}