From e45d82d4af4338141f327ec5dfbcc439afb72626 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 29 Apr 2020 10:18:15 +0200 Subject: [PATCH 001/436] Initial commit --- .gitignore | 129 +++++++++++++++++++++++++++++++++++++++++++++++++++++ LICENSE | 21 +++++++++ 2 files changed, 150 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b6e4761 --- /dev/null +++ b/.gitignore @@ -0,0 +1,129 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..32af56c --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Mark Qvist + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 1060b8ac27b6ae3d85876289be335e99ac0c02cb Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 29 Apr 2020 10:20:21 +0200 Subject: [PATCH 002/436] Base protocol and message format implemented --- LICENSE | 2 +- LXMF.py | 489 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 490 insertions(+), 1 deletion(-) create mode 100644 LXMF.py diff --git a/LICENSE b/LICENSE index 32af56c..a25bd7a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020 Mark Qvist +Copyright (c) 2020 Mark Qvist / unsigned.io Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/LXMF.py b/LXMF.py new file mode 100644 index 0000000..93c8d5b --- /dev/null +++ b/LXMF.py @@ -0,0 +1,489 @@ +import math +import time +import threading +import RNS +import RNS.vendor.umsgpack as msgpack + +APP_NAME = "lxmf" + +class LXMessage: + DRAFT = 0x00 + OUTBOUND = 0x01 + SENDING = 0x02 + SENT = 0x04 + DELIVERED = 0x08 + states = [DRAFT, OUTBOUND, SENDING, SENT, DELIVERED] + + UNKNOWN = 0x00 + PACKET = 0x01 + RESOURCE = 0x02 + representations = [UNKNOWN, PACKET, RESOURCE] + + OPPORTUNISTIC = 0x01 + DIRECT = 0x02 + PROPAGATED = 0x03 + valid_methods = [OPPORTUNISTIC, DIRECT, PROPAGATED] + + DESTINATION_LENGTH = RNS.Identity.TRUNCATED_HASHLENGTH//8 + SIGNATURE_LENGTH = RNS.Identity.SIGLENGTH//8 + + # LXMF overhead is 163 bytes per message: + # 10 bytes for destination hash + # 10 bytes for source hash + # 128 bytes for RSA signature + # 8 bytes for timestamp + # 7 bytes for msgpack structure + LXMF_OVERHEAD = 2*DESTINATION_LENGTH + SIGNATURE_LENGTH + 8 + 7 + + # With an MTU of 500, the maximum RSA-encrypted + # amount of data we can send in a single packet + # is given by the below calculation; 258 bytes. + RSA_PACKET_MDU = RNS.Packet.RSA_MDU + + # The max content length we can fit in LXMF message + # inside a single RNS packet is the RSA MDU, minus + # the LXMF overhead. We can optimise a bit though, by + # inferring the destination hash from the destination + # field of the packet, therefore we also add the length + # of a destination hash to the calculation. With default + # RNS and LXMF parameters, the largest single-packet + # LXMF message we can send is 105 bytes. If a message + # is larger than that, a Reticulum link will be used. + RSA_PACKET_MAX_CONTENT = RSA_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH + + # Links can carry a significantly larger MDU, due to + # more efficient elliptic curve cryptography. The link + # MDU with default Reticulum parameters is 415 bytes. + LINK_PACKET_MDU = RNS.Link.MDU + + # Which means that we can deliver single-packet LXMF + # messages with content of up to 252 bytes over a link. + # If a message is larger than that, LXMF will sequence + # and transfer it as a RNS resource over the link instead. + LINK_PACKET_MAX_CONTENT = LINK_PACKET_MDU - LXMF_OVERHEAD + + # For plain packets without encryption, we can + # fit up to 324 bytes of content. + PLAIN_PACKET_MDU = RNS.Packet.PLAIN_MDU + PLAIN_PACKET_MAX_CONTENT = PLAIN_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH + + def __init__(self, destination, source, content = "", title = "", fields = None, desired_method = None, destination_hash = None, source_hash = None): + if isinstance(destination, RNS.Destination) or destination == None: + self.__destination = destination + if destination != None: + self.destination_hash = destination.hash + else: + self.destination_hash = destination_hash + else: + raise ValueError("LXMessage initialised with invalid destination") + + if isinstance(source, RNS.Destination) or source == None: + self.__source = source + if source != None: + self.source_hash = source.hash + else: + self.source_hash = source_hash + else: + raise ValueError("LXMessage initialised with invalid source") + + self.set_title_from_string(title) + self.set_content_from_string(content) + + self.set_fields(fields) + + self.payload = None + self.timestamp = None + self.signature = None + self.hash = None + self.packed = None + self.progress = None + self.state = LXMessage.DRAFT + self.method = LXMessage.UNKNOWN + + self.representation = LXMessage.UNKNOWN + self.desired_method = desired_method + self.delivery_attempts = 0 + self.transport_encryption = None + self.packet_representation = None + self.resource_representation = None + self.__delivery_destination = None + + def set_title_from_string(self, title_string): + self.title = title_string.encode("utf-8") + + def set_title_from_bytes(self, title_bytes): + self.title = title_bytes + + def title_as_string(self): + return self.title.decode("utf-8") + + def set_content_from_string(self, content_string): + self.content = content_string.encode("utf-8") + + def set_content_from_bytes(self, content_bytes): + self.content = content_bytes + + def content_as_string(self): + return self.content.decode("utf-8") + + def set_fields(self, fields): + if isinstance(fields, dict) or fields == None: + self.fields = fields + else: + raise ValueError("LXMessage property \"fields\" can only be dict or None") + + def get_fields(self): + return self.__fields + + def set_destination(self, destination): + if self.destination == None: + if isinstance(destination, RNS.Destination): + self.__destination = destination + else: + raise ValueError("Invalid destination set on LXMessage") + else: + raise ValueError("Cannot reassign destination on LXMessage") + + def get_destination(self): + return self.__destination + + def set_source(self, source): + if self.source == None: + if isinstance(source, RNS.Destination): + self.__source = source + else: + raise ValueError("Invalid source set on LXMessage") + else: + raise ValueError("Cannot reassign source on LXMessage") + + def get_source(self): + return self.__source + + def set_delivery_destination(self, delivery_destination): + self.__delivery_destination = delivery_destination + + def pack(self): + self.timestamp = time.time() + self.payload = [self.timestamp, self.title, self.content, self.fields] + + hashed_part = b"" + hashed_part += self.__destination.hash + hashed_part += self.__source.hash + hashed_part += msgpack.packb(self.timestamp) + hashed_part += msgpack.packb(self.payload) + self.hash = RNS.Identity.fullHash(hashed_part) + + signed_part = b"" + signed_part += hashed_part + signed_part += self.hash + self.signature = self.__source.sign(signed_part) + + self.packed = b"" + self.packed += self.__destination.hash + self.packed += self.__source.hash + self.packed += self.signature + packed_payload = msgpack.packb(self.payload) + self.packed += packed_payload + self.packed_size = len(self.packed) + content_size = len(packed_payload) + + # If no desired delivery method has been defined, + # one will be chosen according to these rules: + if self.desired_method == None: + self.desired_method == LXMessage.DIRECT + # TODO: Expand rules to something more intelligent + + if self.desired_method == LXMessage.OPPORTUNISTIC: + if self.__destination.type == RNS.Destination.SINGLE: + single_packet_content_limit = LXMessage.RSA_PACKET_MAX_CONTENT + elif self.__destination.type == RNS.Destination.PLAIN: + single_packet_content_limit = LXMessage.PLAIN_PACKET_MAX_CONTENT + + if content_size > single_packet_content_limit: + raise TypeError("LXMessage desired opportunistic delivery method, but content exceeds single-packet size.") + else: + self.method = LXMessage.OPPORTUNISTIC + self.representation = LXMessage.PACKET + self.__delivery_destination = self.__destination + + elif self.desired_method == LXMessage.DIRECT or self.desired_method == LXMessage.PROPAGATED: + single_packet_content_limit = LXMessage.LINK_PACKET_MAX_CONTENT + if content_size <= single_packet_content_limit: + self.method = self.desired_method + self.representation = LXMessage.PACKET + else: + self.method = self.desired_method + self.representation = LXMessage.RESOURCE + + def send(self): + if self.method == LXMessage.OPPORTUNISTIC: + self.__as_packet().send().delivery_callback(self.__mark_delivered) + self.state = LXMessage.SENT + elif self.method == LXMessage.DIRECT: + self.state = LXMessage.SENDING + self.resource_representation = self.__as_resource() + elif self.method == LXMessage.PROPAGATED: + pass + + def __mark_delivered(self, receipt = None): + RNS.log("Received delivery notification for "+str(self), RNS.LOG_DEBUG) + self.state = LXMessage.DELIVERED + + def __resource_concluded(self, resource): + if resource.status == RNS.Resource.COMPLETE: + self.__mark_delivered() + else: + resource.link.teardown() + self.state = LXMessage.OUTBOUND + + + def __update_transfer_progress(self, resource): + self.progress = resource.progress() + + def __as_packet(self): + if not self.packed: + self.pack() + + if not self.__delivery_destination: + raise ValueError("Can't synthesize packet for LXMF message before delivery destination is known") + + if self.method == LXMessage.OPPORTUNISTIC: + return RNS.Packet(self.__delivery_destination, self.packed[LXMessage.DESTINATION_LENGTH:]) + elif self.method == LXMessage.DIRECT or self.method == LXMessage.PROPAGATED: + return RNS.Packet(self.__delivery_destination, self.packed) + + def __as_resource(self): + if not self.packed: + self.pack() + + if not self.__delivery_destination: + raise ValueError("Can't synthesize resource for LXMF message before delivery destination is known") + + if not self.__delivery_destination.type == RNS.Destination.LINK: + raise TypeError("Tried to synthesize resource for LXMF message on a delivery destination that was not a link") + + if not self.__delivery_destination.status == RNS.Link.ACTIVE: + raise ConnectionError("Tried to synthesize resource for LXMF message on a link that was not active") + + self.progress = 0.0 + return RNS.Resource(self.packed, self.__delivery_destination, callback = self.__resource_concluded, progress_callback = self.__update_transfer_progress) + + @staticmethod + def unpack_from_bytes(lxmf_bytes): + destination_hash = lxmf_bytes[:LXMessage.DESTINATION_LENGTH] + source_hash = lxmf_bytes[LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH] + signature = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH] + packed_payload = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH:] + unpacked_payload = msgpack.unpackb(packed_payload) + destination = RNS.Identity.recall(destination_hash) + source = RNS.Identity.recall(source_hash) + timestamp = unpacked_payload[0] + title_bytes = unpacked_payload[1] + content_bytes = unpacked_payload[2] + fields = unpacked_payload[3] + + message = LXMessage( + destination = destination, + source = source, + content = "", + title = "", + fields = fields, + destination_hash = destination_hash, + source_hash = source_hash) + + message.set_title_from_bytes(title_bytes) + message.set_content_from_bytes(content_bytes) + message.timestamp = timestamp + + return message + + @staticmethod + def unpack_from_file(lxmf_file_handle): + pass + +class LXMRouter: + MAX_DELIVERY_ATTEMPTS = 3 + PROCESSING_INTERVAL = 5 + DELIVERY_RETRY_WAIT = 15 + + def __init__(self): + self.pending_inbound = [] + self.pending_outbound = [] + self.failed_outbound = [] + self.direct_links = {} + self.delivery_destinations = {} + + self.processing_outbound = False + self.processing_inbound = False + + self.identity = RNS.Identity() + self.lxmf_query_destination = RNS.Destination(None, RNS.Destination.IN, RNS.Destination.PLAIN, APP_NAME, "query") + self.propagation_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation") + + self.__delivery_callback = None + + job_thread = threading.Thread(target=self.jobloop) + job_thread.setDaemon(True) + job_thread.start() + + def register_delivery_identity(self, identity, display_name = None): + delivery_destination = RNS.Destination(identity, RNS.Destination.IN, RNS.Destination.SINGLE, "lxmf", "delivery") + delivery_destination.packet_callback(self.delivery_packet) + delivery_destination.link_established_callback(self.delivery_link_established) + delivery_destination.display_name = display_name + + self.delivery_destinations[delivery_destination.hash] = delivery_destination + return delivery_destination + + def register_delivery_callback(self, callback): + self.__delivery_callback = callback + + def handle_outbound(self, lxmessage): + RNS.log("LXM Router received outbound message: "+str(lxmessage)) + while self.processing_outbound: + time.sleep(0.1) + + if not lxmessage.packed: + lxmessage.pack() + + self.pending_outbound.append(lxmessage) + self.process_outbound() + + def lxmf_delivery(self, lxmf_data, destination_type = None): + try: + message = LXMessage.unpack_from_bytes(lxmf_data) + except Exception as e: + RNS.log("Could not assemble LXMF message from received data", RNS.LOG_NOTICE) + RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + + if RNS.Reticulum.should_allow_unencrypted(): + message.transport_encryption = "Consider unencrypted (Disabling encryption was allowed in Reticulum configuration)" + else: + if destination_type == RNS.Destination.SINGLE: + message.transport_encryption = "RSA-"+str(RNS.Identity.KEYSIZE) + elif destination_type == RNS.Destination.GROUP: + message.transport_encryption = "AES-128" + elif destination_type == RNS.Destination.LINK: + message.transport_encryption = "EC-SECP256R1" + else: + message.transport_encryption = None + + if self.__delivery_callback != None: + self.__delivery_callback(message) + + return True + + def delivery_packet(self, data, packet): + try: + if packet.destination.type != RNS.Destination.LINK: + lxmf_data = b"" + lxmf_data += packet.destination.hash + lxmf_data += data + else: + lxmf_data = data + + if self.lxmf_delivery(lxmf_data, packet.destination.type): + packet.prove() + + except Exception as e: + RNS.log("Exception occurred while parsing incoming LXMF data.", RNS.LOG_ERROR) + RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) + + def delivery_link_established(self, link): + link.packet_callback(self.delivery_packet) + link.set_resource_strategy(RNS.Link.ACCEPT_ALL) + link.resource_started_callback(self.resource_transfer_began) + link.resource_concluded_callback(self.resource_transfer_concluded) + + def delivery_link_closed(self, link): + pass + + def resource_transfer_began(self, resource): + RNS.log("Transfer began for resource "+str(resource), RNS.LOG_DEBUG) + + def resource_transfer_concluded(self, resource): + RNS.log("Transfer concluded for resource "+str(resource), RNS.LOG_DEBUG) + if resource.status == RNS.Resource.COMPLETE: + self.lxmf_delivery(resource.data, resource.link.type) + + def jobloop(self): + while (True): + # TODO: Improve this to scheduling, so manual + # triggers can delay next run + self.jobs() + time.sleep(LXMRouter.PROCESSING_INTERVAL) + + def jobs(self): + self.process_outbound() + + def process_outbound(self, sender = None): + if self.processing_outbound: + return + + for lxmessage in self.pending_outbound: + if lxmessage.state == LXMessage.DELIVERED: + RNS.log("Delivery has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) + self.pending_outbound.remove(lxmessage) + else: + RNS.log("Starting outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + # Outbound handling for opportunistic messages + if lxmessage.method == LXMessage.OPPORTUNISTIC: + if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: + if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: + lxmessage.delivery_attempts += 1 + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + RNS.log("Opportunistic delivery attempt "+str(lxmessage.delivery_attempts)+" for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + lxmessage.send() + else: + RNS.log("Max delivery attempts reached for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + self.pending_outbound.remove(lxmessage) + self.failed_outbound.append(lxmessage) + + # Outbound handling for messages transferred + # over a direct link to the final recipient + elif lxmessage.method == LXMessage.DIRECT: + if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: + delivery_destination_hash = lxmessage.get_destination().hash + + if delivery_destination_hash in self.direct_links: + # A link already exists, so we'll try to use it + # to deliver the message + direct_link = self.direct_links[delivery_destination_hash] + if direct_link.status == RNS.Link.ACTIVE: + if lxmessage.state != LXMessage.SENDING: + RNS.log("Starting transfer of "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + lxmessage.set_delivery_destination(direct_link) + lxmessage.send() + else: + RNS.log("The transfer of "+str(lxmessage)+" is in progress ("+str(round(lxmessage.progress*100, 1))+"%)", RNS.LOG_DEBUG) + elif direct_link.status == RNS.Link.CLOSED: + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was closed", RNS.LOG_DEBUG) + lxmessage.set_delivery_destination(None) + self.direct_links.pop(delivery_destination_hash) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + else: + # Simply wait for the link to become + # active or close + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" is pending, waiting for link to become active", RNS.LOG_DEBUG) + else: + # No link exists, so we'll try to establish one, but + # only if we've never tried before, or the retry wait + # period has elapsed. + if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: + lxmessage.delivery_attempts += 1 + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: + RNS.log("Establishing link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + delivery_link = RNS.Link(lxmessage.get_destination()) + delivery_link.link_established_callback(self.process_outbound) + self.direct_links[delivery_destination_hash] = delivery_link + else: + RNS.log("Max delivery attempts reached for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + self.pending_outbound.remove(lxmessage) + self.failed_outbound.append(lxmessage) + + # Outbound handling for messages transported via + # propagation to a LXMF router network. + elif lxmessage.method == LXMessage.PROPAGATED: + RNS.log("Attempting propagated delivery for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + raise NotImplementedError("LXMF propagation is not implemented yet") \ No newline at end of file From ac5c751bb493deff92be4120fa6e4ad541fa45f2 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 29 Apr 2020 10:20:41 +0200 Subject: [PATCH 003/436] Create README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..661f3d0 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# LXMF +Lightweight Extensible Message Format for Reticulum From 4589610b1f1501d092ea468b77795e37e9a8f997 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 29 Apr 2020 12:37:19 +0200 Subject: [PATCH 004/436] Added readme --- LXMF.py | 20 +- README.html | 844 ++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 100 ++++++- 3 files changed, 952 insertions(+), 12 deletions(-) create mode 100644 README.html diff --git a/LXMF.py b/LXMF.py index 93c8d5b..09d5196 100644 --- a/LXMF.py +++ b/LXMF.py @@ -166,17 +166,17 @@ class LXMessage: self.timestamp = time.time() self.payload = [self.timestamp, self.title, self.content, self.fields] - hashed_part = b"" - hashed_part += self.__destination.hash - hashed_part += self.__source.hash - hashed_part += msgpack.packb(self.timestamp) - hashed_part += msgpack.packb(self.payload) - self.hash = RNS.Identity.fullHash(hashed_part) + hashed_part = b"" + hashed_part += self.__destination.hash + hashed_part += self.__source.hash + hashed_part += msgpack.packb(self.payload) + self.hash = RNS.Identity.fullHash(hashed_part) + self.message_id = self.hash - signed_part = b"" - signed_part += hashed_part - signed_part += self.hash - self.signature = self.__source.sign(signed_part) + signed_part = b"" + signed_part += hashed_part + signed_part += self.hash + self.signature = self.__source.sign(signed_part) self.packed = b"" self.packed += self.__destination.hash diff --git a/README.html b/README.html new file mode 100644 index 0000000..2c15a78 --- /dev/null +++ b/README.html @@ -0,0 +1,844 @@ +README

+Lightweight Extensible Message Format

+

LXMF is a simple and flexible messaging format that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of Reticulum and offers zero-conf message routing, end-to-end encryption and Perfect Forward Secrecy by default.

+

+Structure

+

LXMF messages are stored in a simple and efficient format, that's easy to parse and write.

+
+The format follows this general structure:
+
    +
  • Destination
    • +
  • Source
    • +
  • RSA Signature
    • +
  • Payload +
      +
    • Timestamp
      • +
    • Title
      • +
    • Content
      • +
    • Fields
      • +
    +
    • +
+
+And these rules:
+
    +
  1. +

    A LXMF message is identified by it's message-id, which is a SHA-256 hash of the Destination, Source and Payload. The message-id is never included directly in the message, since it can always be inferred from the message itself.

    +

    In some cases the actual message-id cannot be inferred, for example when a Propagation Node is storing an encrypted message for an offline user. In theses cases a transient-id is used to identify the message while in storage or transit.

    +
    2. +
  2. +

    Destination, Source, Signature and Payload parts are mandatory, as is the Timestamp part of the payload.

    +
      +
    • The Destination and Source fields are Reticulum destination hashes
      • +
    • The Signature field is a RSA signature of the Destination, Source, Payload and message-id +
      • +
    • The Payload part is a list containing four items: +
        +
      1. The Timestamp is double-precision floating point number representing the number of seconds since the UNIX epoch.
        2. +
      2. The Title is an optional title for the message
        3. +
      3. The Content is the optional content or body of the message
        4. +
      4. The Fields is an optional dictionary
        5. +
      +
      • +
    +
    3. +
  3. +

    The Title, Content and Fields parts must be included in the message structure, but can be left empty.

    +
    4. +
  4. +

    The Fields part can be left empty, or contain a dictionary of any structure or depth.

    +
    5. +
+

+Usage Examples

+

LXMF offers flexibility to implement many different messaging schemes, ranging from human communication to machine control and sensor monitoring. Here's a few examples:

+
    +
  • +

    A messaging system for passing short, simple messages between human users, akin to SMS can be implemented using only the Content field, and leaving all other optional fields empty.

    +
    • +
  • +

    For sending full-size mail, an email-like system can be implemented using the Title and Content fields to store "subject" and "body" parts of the message, and optionally the Fields part can be used to store attachments or other metadata.

    +
    • +
  • +

    Machine-control messages or sensor readings can be implemented using command structures embedded in the Fields dictionary.

    +
    • +
  • +

    Distributed discussion or news-groups, akin to USENET or similar systems, can be implemented using the relevant fields and LXMF Propagation Nodes. Broadcast bulletins can be implemented in a similar fashion.

    +
    • +
+

+Propagation Nodes

+

LXM Propagation Nodes offer a way to store and forward messages to users or endpoints that are not directly reachable at the time of message emission. Propagation Nodes can also provide infrastructure for distributed bulletin, news or discussion boards.

+

+The LXM Router

+

The LXM Router handles transporting messages over a Reticulum network, managing delivery receipts, outbound and inbound queues, and is the point of API interaction for client programs. The LXM Router also implements functionality for acting as an LXMF Propagation Node.

+

Programatically, using the LXM Router to send a message is as simple as:

+
import LXMF
+
+lxm_router = LXMF.LXMRouter()
+
+message = LXMF.LXMessage(destination, source, "This is a short, simple message.")
+
+lxm_router.handle_outbound(message)
+
+

The LXM Router then handles the heavy lifting, such as message packing, encryption, delivery confirmation, path lookup, routing, retries and failure notifications.

+

+Transport Encryption

+

LXMF uses encryption provided by Reticulum, and thus uses end-to-end encryption by default. The delivery method of a message will influence which transport encryption scheme is used.

+
    +
  • +

    A message can be delivered opportunistically, embedded in a single Reticulum packet. In this cases the message will be opportunistically routed through the network, and use RSA-1024 asymmetric encryption.

    +
    • +
  • +

    If a message is delivered to the Reticulum GROUP destination type, the message will be transported using AES-128 encryption.

    +
    • +
  • +

    If a message is delivered over a Reticulum link (which is the default method), the message will be encrypted with ephemeral ECDH on the SECP256R1 curve. This mode offers Perfect Forward Secrecy.

    +
    • +
+

+Wire Format & Overhead

+

Assuming the default Reticulum configuration, the binary wire-format is as follows:

+
    +
  • 10 bytes destination hash
    • +
  • 10 bytes source hash
    • +
  • 128 bytes RSA signature
    • +
  • Remaining bytes of msgpack payload data, in accordance with the structure defined above
    • +
+

The complete message overhead for LXMF is only 163 bytes, which in return gives you timestamped, digitally signed, infinitely extensible, end-to-end encrypted, zero-conf routed, minimal-infrastructure messaging that's easy to use and build applications with.

+

+Caveat Emptor

+

As with Reticulum, LXMF is alpha software, and should be considered experimental. While it has been built with cryptography best-practices very foremost in mind, it has not been externally security audited, and there could very well be privacy-breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch.

+
\ No newline at end of file diff --git a/README.md b/README.md index 661f3d0..aa395a3 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,98 @@ -# LXMF -Lightweight Extensible Message Format for Reticulum +Lightweight Extensible Message Format +========== + +LXMF is a simple and flexible messaging format that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](https://github.com/markqvist/reticulum) and offers zero-conf message routing, end-to-end encryption and Perfect Forward Secrecy by default. + +## Structure + +LXMF messages are stored in a simple and efficient format, that's easy to parse and write. + +##### The format follows this general structure: + +- Destination +- Source +- RSA Signature +- Payload + - Timestamp + - Title + - Content + - Fields + +##### And these rules: + +1. A LXMF message is identified by it's __message-id__, which is a SHA-256 hash of the __Destination__, __Source__ and __Payload__. The message-id is never included directly in the message, since it can always be inferred from the message itself. + + In some cases the actual message-id cannot be inferred, for example when a Propagation Node is storing an encrypted message for an offline user. In theses cases a _transient-id_ is used to identify the message while in storage or transit. + +2. __Destination__, __Source__, __Signature__ and __Payload__ parts are mandatory, as is the __Timestamp__ part of the payload. + - The __Destination__ and __Source__ fields are Reticulum destination hashes + - The __Signature__ field is a RSA signature of the __Destination__, __Source__, __Payload__ and __message-id__ + - The __Payload__ part is a list containing four items: + 1. The __Timestamp__ is double-precision floating point number representing the number of seconds since the UNIX epoch. + 2. The __Title__ is an optional title for the message + 3. The __Content__ is the optional content or body of the message + 4. The __Fields__ is an optional dictionary + +3. The __Title__, __Content__ and __Fields__ parts must be included in the message structure, but can be left empty. + +4. The __Fields__ part can be left empty, or contain a dictionary of any structure or depth. + +## Usage Examples + +LXMF offers flexibility to implement many different messaging schemes, ranging from human communication to machine control and sensor monitoring. Here's a few examples: + +- A messaging system for passing short, simple messages between human users, akin to SMS can be implemented using only the __Content__ field, and leaving all other optional fields empty. + +- For sending full-size mail, an email-like system can be implemented using the __Title__ and __Content__ fields to store "subject" and "body" parts of the message, and optionally the __Fields__ part can be used to store attachments or other metadata. + +- Machine-control messages or sensor readings can be implemented using command structures embedded in the __Fields__ dictionary. + +- Distributed discussion or news-groups, akin to USENET or similar systems, can be implemented using the relevant fields and LXMF Propagation Nodes. Broadcast bulletins can be implemented in a similar fashion. + +## Propagation Nodes + +LXM Propagation Nodes offer a way to store and forward messages to users or endpoints that are not directly reachable at the time of message emission. Propagation Nodes can also provide infrastructure for distributed bulletin, news or discussion boards. + +## The LXM Router + +The LXM Router handles transporting messages over a Reticulum network, managing delivery receipts, outbound and inbound queues, and is the point of API interaction for client programs. The LXM Router also implements functionality for acting as an LXMF Propagation Node. + +Programatically, using the LXM Router to send a message is as simple as: + +```python +import LXMF + +lxm_router = LXMF.LXMRouter() + +message = LXMF.LXMessage(destination, source, "This is a short, simple message.") + +lxm_router.handle_outbound(message) + +``` + +The LXM Router then handles the heavy lifting, such as message packing, encryption, delivery confirmation, path lookup, routing, retries and failure notifications. + +## Transport Encryption + +LXMF uses encryption provided by [Reticulum](https://github.com/markqvist/reticulum), and thus uses end-to-end encryption by default. The delivery method of a message will influence which transport encryption scheme is used. + +- A message can be delivered opportunistically, embedded in a single Reticulum packet. In this cases the message will be opportunistically routed through the network, and use _RSA-1024_ asymmetric encryption. + +- If a message is delivered to the Reticulum GROUP destination type, the message will be transported using _AES-128_ encryption. + +- If a message is delivered over a Reticulum link (which is the default method), the message will be encrypted with _ephemeral ECDH_ on the _SECP256R1_ curve. This mode offers Perfect Forward Secrecy. + +## Wire Format & Overhead + +Assuming the default Reticulum configuration, the binary wire-format is as follows: + +- 10 bytes destination hash +- 10 bytes source hash +- 128 bytes RSA signature +- Remaining bytes of [msgpack](https://msgpack.org) payload data, in accordance with the structure defined above + +The complete message overhead for LXMF is only 163 bytes, which in return gives you timestamped, digitally signed, infinitely extensible, end-to-end encrypted, zero-conf routed, minimal-infrastructure messaging that's easy to use and build applications with. + +## Caveat Emptor + +As with Reticulum, LXMF is alpha software, and should be considered experimental. While it has been built with cryptography best-practices very foremost in mind, it _has not_ been externally security audited, and there could very well be privacy-breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch. \ No newline at end of file From aecb142229c3b182b1d22a7fda70b99ccd292142 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 29 Apr 2020 12:57:06 +0200 Subject: [PATCH 005/436] Restructured as package --- .gitignore | 1 + LXMF.py => LXMF/LXMF.py | 0 LXMF/__init__.py | 7 +++++++ README.html | 4 ++++ README.md | 10 +++++++++- setup.py | 23 +++++++++++++++++++++++ 6 files changed, 44 insertions(+), 1 deletion(-) rename LXMF.py => LXMF/LXMF.py (100%) create mode 100644 LXMF/__init__.py create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index b6e4761..45efbd4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.DS_Store # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/LXMF.py b/LXMF/LXMF.py similarity index 100% rename from LXMF.py rename to LXMF/LXMF.py diff --git a/LXMF/__init__.py b/LXMF/__init__.py new file mode 100644 index 0000000..a304c21 --- /dev/null +++ b/LXMF/__init__.py @@ -0,0 +1,7 @@ +import os +import glob +from .LXMF import LXMessage +from .LXMF import LXMRouter + +modules = glob.glob(os.path.dirname(__file__)+"/*.py") +__all__ = [ os.path.basename(f)[:-3] for f in modules if not f.endswith('__init__.py')] \ No newline at end of file diff --git a/README.html b/README.html index 2c15a78..2ad08fe 100644 --- a/README.html +++ b/README.html @@ -841,4 +841,8 @@ lxm_router.handle_outbound(message)

Caveat Emptor

As with Reticulum, LXMF is alpha software, and should be considered experimental. While it has been built with cryptography best-practices very foremost in mind, it has not been externally security audited, and there could very well be privacy-breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch.

+

+Installation

+

If you want to try out LXMF, you can install it with pip:

+
pip3 install lxmf
\ No newline at end of file diff --git a/README.md b/README.md index aa395a3..b49a6c5 100644 --- a/README.md +++ b/README.md @@ -95,4 +95,12 @@ The complete message overhead for LXMF is only 163 bytes, which in return gives ## Caveat Emptor -As with Reticulum, LXMF is alpha software, and should be considered experimental. While it has been built with cryptography best-practices very foremost in mind, it _has not_ been externally security audited, and there could very well be privacy-breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch. \ No newline at end of file +As with Reticulum, LXMF is alpha software, and should be considered experimental. While it has been built with cryptography best-practices very foremost in mind, it _has not_ been externally security audited, and there could very well be privacy-breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch. + +## Installation + +If you want to try out LXMF, you can install it with pip: + +```bash +pip3 install lxmf +``` \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..36148ab --- /dev/null +++ b/setup.py @@ -0,0 +1,23 @@ +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="lxmf", + version="0.0.1", + author="Mark Qvist", + author_email="mark@unsigned.io", + description="Lightweight Extensible Message Format for Reticulum", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/markqvist/lxmf", + packages=setuptools.find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + install_requires=['rns'], + python_requires='>=3.6', +) \ No newline at end of file From 64f12b2857a7959728ae599c17fd9bf98d64baef Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 29 Apr 2020 13:00:18 +0200 Subject: [PATCH 006/436] Modified gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 45efbd4..a942c6d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -.DS_Store +*.DS_Store # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] From 4b81a8d239025897d84742fae0966b11152dc257 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 1 May 2020 21:07:24 +0200 Subject: [PATCH 007/436] Update README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b49a6c5..c81a5f0 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ LXMF messages are stored in a simple and efficient format, that's easy to parse - RSA Signature - Payload - Timestamp - - Title - Content + - Title - Fields ##### And these rules: @@ -28,12 +28,12 @@ LXMF messages are stored in a simple and efficient format, that's easy to parse - The __Destination__ and __Source__ fields are Reticulum destination hashes - The __Signature__ field is a RSA signature of the __Destination__, __Source__, __Payload__ and __message-id__ - The __Payload__ part is a list containing four items: - 1. The __Timestamp__ is double-precision floating point number representing the number of seconds since the UNIX epoch. - 2. The __Title__ is an optional title for the message - 3. The __Content__ is the optional content or body of the message + 1. The __Timestamp__ is a double-precision floating point number representing the number of seconds since the UNIX epoch. + 2. The __Content__ is an optional title for the message + 3. The __Title__ is the optional content or body of the message 4. The __Fields__ is an optional dictionary -3. The __Title__, __Content__ and __Fields__ parts must be included in the message structure, but can be left empty. +3. The __Content__, __Title__ and __Fields__ parts must be included in the message structure, but can be left empty. 4. The __Fields__ part can be left empty, or contain a dictionary of any structure or depth. @@ -103,4 +103,4 @@ If you want to try out LXMF, you can install it with pip: ```bash pip3 install lxmf -``` \ No newline at end of file +``` From 91f7b1ebdeea1383dc52799b37274703731b82ef Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 9 Dec 2020 22:09:32 +0100 Subject: [PATCH 008/436] Fixed typo in readme --- README.html | 17 ++++++++--------- README.md | 4 ++-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/README.html b/README.html index 2ad08fe..2f83335 100644 --- a/README.html +++ b/README.html @@ -745,8 +745,8 @@ pre {
  • Payload
    • Timestamp
      • -
    • Title
    • Content
      • +
    • Title
    • Fields
    • @@ -766,16 +766,16 @@ pre {
  • The Payload part is a list containing four items:
      -
    1. The Timestamp is double-precision floating point number representing the number of seconds since the UNIX epoch.
      2. -
    2. The Title is an optional title for the message
      3. +
    3. The Timestamp is a double-precision floating point number representing the number of seconds since the UNIX epoch.
    4. The Content is the optional content or body of the message
      5. +
    5. The Title is an optional title for the message
    6. The Fields is an optional dictionary
  • -

    The Title, Content and Fields parts must be included in the message structure, but can be left empty.

    +

    The Content, Title and Fields parts must be included in the message structure, but can be left empty.

  • The Fields part can be left empty, or contain a dictionary of any structure or depth.

    @@ -805,14 +805,13 @@ pre { The LXM Router

    The LXM Router handles transporting messages over a Reticulum network, managing delivery receipts, outbound and inbound queues, and is the point of API interaction for client programs. The LXM Router also implements functionality for acting as an LXMF Propagation Node.

    Programatically, using the LXM Router to send a message is as simple as:

    -
    import LXMF
+
    import LXMF
 
-lxm_router = LXMF.LXMRouter()
+lxm_router = LXMF.LXMRouter()
 
-message = LXMF.LXMessage(destination, source, "This is a short, simple message.")
+message = LXMF.LXMessage(destination, source, "This is a short, simple message.")
 
-lxm_router.handle_outbound(message)
-
    
+lxm_router.handle_outbound(message)

    The LXM Router then handles the heavy lifting, such as message packing, encryption, delivery confirmation, path lookup, routing, retries and failure notifications.

    Transport Encryption

    diff --git a/README.md b/README.md index c81a5f0..b90ae1a 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,8 @@ LXMF messages are stored in a simple and efficient format, that's easy to parse - The __Signature__ field is a RSA signature of the __Destination__, __Source__, __Payload__ and __message-id__ - The __Payload__ part is a list containing four items: 1. The __Timestamp__ is a double-precision floating point number representing the number of seconds since the UNIX epoch. - 2. The __Content__ is an optional title for the message - 3. The __Title__ is the optional content or body of the message + 2. The __Content__ is the optional content or body of the message + 3. The __Title__ is an optional title for the message 4. The __Fields__ is an optional dictionary 3. The __Content__, __Title__ and __Fields__ parts must be included in the message structure, but can be left empty. From 0c32bbcb680d5cde2e52398692a2e676f0f7388c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 9 Dec 2020 22:23:54 +0100 Subject: [PATCH 009/436] Updated readme --- README.html | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.html b/README.html index 2f83335..2b5ff21 100644 --- a/README.html +++ b/README.html @@ -824,7 +824,7 @@ pre {

    If a message is delivered to the Reticulum GROUP destination type, the message will be transported using AES-128 encryption.

  • -

    If a message is delivered over a Reticulum link (which is the default method), the message will be encrypted with ephemeral ECDH on the SECP256R1 curve. This mode offers Perfect Forward Secrecy.

    +

    If a message is delivered over a Reticulum link (which is the default method), the message will be encrypted with ephemeral keys derived with ECDH on the SECP256R1 curve. This mode offers Perfect Forward Secrecy.

    • diff --git a/README.md b/README.md index b90ae1a..8194833 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ LXMF uses encryption provided by [Reticulum](https://github.com/markqvist/reticu - If a message is delivered to the Reticulum GROUP destination type, the message will be transported using _AES-128_ encryption. -- If a message is delivered over a Reticulum link (which is the default method), the message will be encrypted with _ephemeral ECDH_ on the _SECP256R1_ curve. This mode offers Perfect Forward Secrecy. +- If a message is delivered over a Reticulum link (which is the default method), the message will be encrypted with _ephemeral_ keys derived with _ECDH_ on the _SECP256R1_ curve. This mode offers Perfect Forward Secrecy. ## Wire Format & Overhead From 01b5ba60d06d3dbaea9366445ec6ab6022c5d6d5 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 4 May 2021 10:19:07 +0200 Subject: [PATCH 010/436] Added signature validation on incoming messages --- LXMF/LXMF.py | 199 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 118 insertions(+), 81 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 09d5196..8f92ef6 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -7,22 +7,26 @@ import RNS.vendor.umsgpack as msgpack APP_NAME = "lxmf" class LXMessage: - DRAFT = 0x00 - OUTBOUND = 0x01 - SENDING = 0x02 - SENT = 0x04 - DELIVERED = 0x08 - states = [DRAFT, OUTBOUND, SENDING, SENT, DELIVERED] + DRAFT = 0x00 + OUTBOUND = 0x01 + SENDING = 0x02 + SENT = 0x04 + DELIVERED = 0x08 + states = [DRAFT, OUTBOUND, SENDING, SENT, DELIVERED] - UNKNOWN = 0x00 - PACKET = 0x01 - RESOURCE = 0x02 - representations = [UNKNOWN, PACKET, RESOURCE] + UNKNOWN = 0x00 + PACKET = 0x01 + RESOURCE = 0x02 + representations = [UNKNOWN, PACKET, RESOURCE] - OPPORTUNISTIC = 0x01 - DIRECT = 0x02 - PROPAGATED = 0x03 - valid_methods = [OPPORTUNISTIC, DIRECT, PROPAGATED] + OPPORTUNISTIC = 0x01 + DIRECT = 0x02 + PROPAGATED = 0x03 + valid_methods = [OPPORTUNISTIC, DIRECT, PROPAGATED] + + SOURCE_UNKNOWN = 0x01 + SIGNATURE_INVALID = 0x02 + unverified_reasons = [SOURCE_UNKNOWN, SIGNATURE_INVALID] DESTINATION_LENGTH = RNS.Identity.TRUNCATED_HASHLENGTH//8 SIGNATURE_LENGTH = RNS.Identity.SIGLENGTH//8 @@ -100,6 +104,10 @@ class LXMessage: self.state = LXMessage.DRAFT self.method = LXMessage.UNKNOWN + self.incoming = False + self.signature_validated = False + self.unverified_reason = None + self.representation = LXMessage.UNKNOWN self.desired_method = desired_method self.delivery_attempts = 0 @@ -163,57 +171,62 @@ class LXMessage: self.__delivery_destination = delivery_destination def pack(self): - self.timestamp = time.time() - self.payload = [self.timestamp, self.title, self.content, self.fields] + if not self.packed: + self.timestamp = time.time() + self.payload = [self.timestamp, self.title, self.content, self.fields] - hashed_part = b"" - hashed_part += self.__destination.hash - hashed_part += self.__source.hash - hashed_part += msgpack.packb(self.payload) - self.hash = RNS.Identity.fullHash(hashed_part) - self.message_id = self.hash - - signed_part = b"" - signed_part += hashed_part - signed_part += self.hash - self.signature = self.__source.sign(signed_part) + hashed_part = b"" + hashed_part += self.__destination.hash + hashed_part += self.__source.hash + hashed_part += msgpack.packb(self.payload) + self.hash = RNS.Identity.fullHash(hashed_part) + self.message_id = self.hash + + signed_part = b"" + signed_part += hashed_part + signed_part += self.hash + self.signature = self.__source.sign(signed_part) + self.signature_validated = True - self.packed = b"" - self.packed += self.__destination.hash - self.packed += self.__source.hash - self.packed += self.signature - packed_payload = msgpack.packb(self.payload) - self.packed += packed_payload - self.packed_size = len(self.packed) - content_size = len(packed_payload) + self.packed = b"" + self.packed += self.__destination.hash + self.packed += self.__source.hash + self.packed += self.signature + packed_payload = msgpack.packb(self.payload) + self.packed += packed_payload + self.packed_size = len(self.packed) + content_size = len(packed_payload) - # If no desired delivery method has been defined, - # one will be chosen according to these rules: - if self.desired_method == None: - self.desired_method == LXMessage.DIRECT - # TODO: Expand rules to something more intelligent + # If no desired delivery method has been defined, + # one will be chosen according to these rules: + if self.desired_method == None: + self.desired_method == LXMessage.DIRECT + # TODO: Expand rules to something more intelligent - if self.desired_method == LXMessage.OPPORTUNISTIC: - if self.__destination.type == RNS.Destination.SINGLE: - single_packet_content_limit = LXMessage.RSA_PACKET_MAX_CONTENT - elif self.__destination.type == RNS.Destination.PLAIN: - single_packet_content_limit = LXMessage.PLAIN_PACKET_MAX_CONTENT + if self.desired_method == LXMessage.OPPORTUNISTIC: + if self.__destination.type == RNS.Destination.SINGLE: + single_packet_content_limit = LXMessage.RSA_PACKET_MAX_CONTENT + elif self.__destination.type == RNS.Destination.PLAIN: + single_packet_content_limit = LXMessage.PLAIN_PACKET_MAX_CONTENT - if content_size > single_packet_content_limit: - raise TypeError("LXMessage desired opportunistic delivery method, but content exceeds single-packet size.") - else: - self.method = LXMessage.OPPORTUNISTIC - self.representation = LXMessage.PACKET - self.__delivery_destination = self.__destination + if content_size > single_packet_content_limit: + raise TypeError("LXMessage desired opportunistic delivery method, but content exceeds single-packet size.") + else: + self.method = LXMessage.OPPORTUNISTIC + self.representation = LXMessage.PACKET + self.__delivery_destination = self.__destination + + elif self.desired_method == LXMessage.DIRECT or self.desired_method == LXMessage.PROPAGATED: + single_packet_content_limit = LXMessage.LINK_PACKET_MAX_CONTENT + if content_size <= single_packet_content_limit: + self.method = self.desired_method + self.representation = LXMessage.PACKET + else: + self.method = self.desired_method + self.representation = LXMessage.RESOURCE + else: + raise ValueError("Attempt to re-pack LXMessage "+str(self)+" that was already packed") - elif self.desired_method == LXMessage.DIRECT or self.desired_method == LXMessage.PROPAGATED: - single_packet_content_limit = LXMessage.LINK_PACKET_MAX_CONTENT - if content_size <= single_packet_content_limit: - self.method = self.desired_method - self.representation = LXMessage.PACKET - else: - self.method = self.desired_method - self.representation = LXMessage.RESOURCE def send(self): if self.method == LXMessage.OPPORTUNISTIC: @@ -274,9 +287,13 @@ class LXMessage: source_hash = lxmf_bytes[LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH] signature = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH] packed_payload = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH:] + hashed_part = b"" + destination_hash + source_hash + packed_payload + message_hash = RNS.Identity.fullHash(hashed_part) + signed_part = b"" + hashed_part + message_hash unpacked_payload = msgpack.unpackb(packed_payload) destination = RNS.Identity.recall(destination_hash) - source = RNS.Identity.recall(source_hash) + source_identity = RNS.Identity.recall(source_hash) + source = RNS.Destination(source_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") timestamp = unpacked_payload[0] title_bytes = unpacked_payload[1] content_bytes = unpacked_payload[2] @@ -291,15 +308,31 @@ class LXMessage: destination_hash = destination_hash, source_hash = source_hash) + message.hash = message_hash + message.signature = signature + message.incoming = True + message.timestamp = timestamp + message.packed = lxmf_bytes message.set_title_from_bytes(title_bytes) message.set_content_from_bytes(content_bytes) - message.timestamp = timestamp + + try: + if source: + if source.identity.validate(signature, signed_part): + message.signature_validated = True + else: + message.signature_validated = False + message.unverified_reason = LXMessage.SIGNATURE_INVALID + else: + signature_validated = False + message.unverified_reason = LXMessage.SOURCE_UNKNOWN + RNS.log("LXMF message signature could not be validated, since source identity is unknown") + except Exception as e: + message.signature_validated = False + RNS.log("Error while validating LXMF message signature. The contained exception was: "+str(e), RNS.LOG_ERROR) return message - @staticmethod - def unpack_from_file(lxmf_file_handle): - pass class LXMRouter: MAX_DELIVERY_ATTEMPTS = 3 @@ -352,26 +385,30 @@ class LXMRouter: def lxmf_delivery(self, lxmf_data, destination_type = None): try: message = LXMessage.unpack_from_bytes(lxmf_data) + + if RNS.Reticulum.should_allow_unencrypted(): + message.transport_encryption = "Consider unencrypted (Disabling encryption was allowed in Reticulum configuration)" + else: + if destination_type == RNS.Destination.SINGLE: + message.transport_encryption = "RSA-"+str(RNS.Identity.KEYSIZE) + elif destination_type == RNS.Destination.GROUP: + message.transport_encryption = "AES-128" + elif destination_type == RNS.Destination.LINK: + message.transport_encryption = "EC-SECP256R1" + else: + message.transport_encryption = None + + if self.__delivery_callback != None: + self.__delivery_callback(message) + + return True + except Exception as e: RNS.log("Could not assemble LXMF message from received data", RNS.LOG_NOTICE) RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + raise e + return False - if RNS.Reticulum.should_allow_unencrypted(): - message.transport_encryption = "Consider unencrypted (Disabling encryption was allowed in Reticulum configuration)" - else: - if destination_type == RNS.Destination.SINGLE: - message.transport_encryption = "RSA-"+str(RNS.Identity.KEYSIZE) - elif destination_type == RNS.Destination.GROUP: - message.transport_encryption = "AES-128" - elif destination_type == RNS.Destination.LINK: - message.transport_encryption = "EC-SECP256R1" - else: - message.transport_encryption = None - - if self.__delivery_callback != None: - self.__delivery_callback(message) - - return True def delivery_packet(self, data, packet): try: @@ -404,7 +441,7 @@ class LXMRouter: def resource_transfer_concluded(self, resource): RNS.log("Transfer concluded for resource "+str(resource), RNS.LOG_DEBUG) if resource.status == RNS.Resource.COMPLETE: - self.lxmf_delivery(resource.data, resource.link.type) + self.lxmf_delivery(resource.data.read(), resource.link.type) def jobloop(self): while (True): From b57960227d4c1f5fa0674705e898341f97d5e1f1 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 4 May 2021 11:06:12 +0200 Subject: [PATCH 011/436] File writing and unpacking added --- LXMF/LXMF.py | 76 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 21 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 8f92ef6..11b3422 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -281,23 +281,48 @@ class LXMessage: self.progress = 0.0 return RNS.Resource(self.packed, self.__delivery_destination, callback = self.__resource_concluded, progress_callback = self.__update_transfer_progress) + def write_to_directory(self, directory_path): + file_name = RNS.hexrep(self.hash, delimit=False) + file_path = directory_path+"/"+file_name + + try: + if not self.packed: + self.pack() + + file = open(file_path, "wb") + file.write(self.packed) + file.close() + + except Exception as e: + RNS.log("Error while writing LXMF message to file \""+str(file_path)+"\". The contained exception was: "+str(e), RNS.LOG_ERROR) + + @staticmethod def unpack_from_bytes(lxmf_bytes): - destination_hash = lxmf_bytes[:LXMessage.DESTINATION_LENGTH] - source_hash = lxmf_bytes[LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH] - signature = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH] - packed_payload = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH:] - hashed_part = b"" + destination_hash + source_hash + packed_payload - message_hash = RNS.Identity.fullHash(hashed_part) - signed_part = b"" + hashed_part + message_hash - unpacked_payload = msgpack.unpackb(packed_payload) - destination = RNS.Identity.recall(destination_hash) - source_identity = RNS.Identity.recall(source_hash) - source = RNS.Destination(source_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") - timestamp = unpacked_payload[0] - title_bytes = unpacked_payload[1] - content_bytes = unpacked_payload[2] - fields = unpacked_payload[3] + destination_hash = lxmf_bytes[:LXMessage.DESTINATION_LENGTH] + source_hash = lxmf_bytes[LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH] + signature = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH] + packed_payload = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH:] + hashed_part = b"" + destination_hash + source_hash + packed_payload + message_hash = RNS.Identity.fullHash(hashed_part) + signed_part = b"" + hashed_part + message_hash + unpacked_payload = msgpack.unpackb(packed_payload) + timestamp = unpacked_payload[0] + title_bytes = unpacked_payload[1] + content_bytes = unpacked_payload[2] + fields = unpacked_payload[3] + + destination_identity = RNS.Identity.recall(destination_hash) + if destination_identity != None: + destination = RNS.Destination(destination_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") + else: + destination = None + + source_identity = RNS.Identity.recall(source_hash) + if source_identity != None: + source = RNS.Destination(source_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") + else: + source = None message = LXMessage( destination = destination, @@ -308,11 +333,12 @@ class LXMessage: destination_hash = destination_hash, source_hash = source_hash) - message.hash = message_hash - message.signature = signature - message.incoming = True - message.timestamp = timestamp - message.packed = lxmf_bytes + message.hash = message_hash + message.signature = signature + message.incoming = True + message.timestamp = timestamp + message.packed = lxmf_bytes + message.packed_size = len(lxmf_bytes) message.set_title_from_bytes(title_bytes) message.set_content_from_bytes(content_bytes) @@ -326,12 +352,20 @@ class LXMessage: else: signature_validated = False message.unverified_reason = LXMessage.SOURCE_UNKNOWN - RNS.log("LXMF message signature could not be validated, since source identity is unknown") + RNS.log("Unpacked LXMF message signature could not be validated, since source identity is unknown", RNS.LOG_DEBUG) except Exception as e: message.signature_validated = False RNS.log("Error while validating LXMF message signature. The contained exception was: "+str(e), RNS.LOG_ERROR) return message + + @staticmethod + def unpack_from_file(lxmf_file_handle): + try: + return LXMessage.unpack_from_bytes(lxmf_file_handle.read()) + except Exception as e: + RNS.log("Could not unpack LXMessage from file. The contained exception was: "+str(e), RNS.LOG_ERROR) + return None class LXMRouter: From 3d53ca0124ec969b61a5b89bc5babfb2a394da8b Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 12 May 2021 14:03:35 +0200 Subject: [PATCH 012/436] Outbound procedure fix --- LXMF/LXMF.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 11b3422..b717252 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -71,6 +71,12 @@ class LXMessage: PLAIN_PACKET_MDU = RNS.Packet.PLAIN_MDU PLAIN_PACKET_MAX_CONTENT = PLAIN_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH + def __str__(self): + if self.hash != None: + return "" + else: + return "" + def __init__(self, destination, source, content = "", title = "", fields = None, desired_method = None, destination_hash = None, source_hash = None): if isinstance(destination, RNS.Destination) or destination == None: self.__destination = destination @@ -406,13 +412,14 @@ class LXMRouter: self.__delivery_callback = callback def handle_outbound(self, lxmessage): - RNS.log("LXM Router received outbound message: "+str(lxmessage)) - while self.processing_outbound: - time.sleep(0.1) - if not lxmessage.packed: lxmessage.pack() + RNS.log("LXM Router received outbound message: "+str(lxmessage)) + + while self.processing_outbound: + time.sleep(0.1) + self.pending_outbound.append(lxmessage) self.process_outbound() From 892fa7fc9d0d4eb14d5c3f89da4569571d0bc442 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 13 May 2021 16:55:32 +0200 Subject: [PATCH 013/436] Implemented message state in disk format. Implemented automated link cleaning. --- LXMF/LXMF.py | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index b717252..d3d967e 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -121,6 +121,7 @@ class LXMessage: self.packet_representation = None self.resource_representation = None self.__delivery_destination = None + self.__delivery_callback = None def set_title_from_string(self, title_string): self.title = title_string.encode("utf-8") @@ -176,9 +177,14 @@ class LXMessage: def set_delivery_destination(self, delivery_destination): self.__delivery_destination = delivery_destination + def register_delivery_callback(self, callback): + self.__delivery_callback = callback + def pack(self): if not self.packed: - self.timestamp = time.time() + if self.timestamp == None: + self.timestamp = time.time() + self.payload = [self.timestamp, self.title, self.content, self.fields] hashed_part = b"" @@ -248,6 +254,9 @@ class LXMessage: RNS.log("Received delivery notification for "+str(self), RNS.LOG_DEBUG) self.state = LXMessage.DELIVERED + if self.__delivery_callback != None: + self.__delivery_callback(self) + def __resource_concluded(self, resource): if resource.status == RNS.Resource.COMPLETE: self.__mark_delivered() @@ -295,12 +304,18 @@ class LXMessage: if not self.packed: self.pack() + container = {"state": self.state, "lxmf_bytes": self.packed} + packed_container = msgpack.packb(container) + file = open(file_path, "wb") - file.write(self.packed) + file.write(packed_container) file.close() + return file_path + except Exception as e: RNS.log("Error while writing LXMF message to file \""+str(file_path)+"\". The contained exception was: "+str(e), RNS.LOG_ERROR) + return None @staticmethod @@ -368,7 +383,10 @@ class LXMessage: @staticmethod def unpack_from_file(lxmf_file_handle): try: - return LXMessage.unpack_from_bytes(lxmf_file_handle.read()) + container = msgpack.unpackb(lxmf_file_handle.read()) + lxm = LXMessage.unpack_from_bytes(container["lxmf_bytes"]) + lxm.state = container["state"] + return lxm except Exception as e: RNS.log("Could not unpack LXMessage from file. The contained exception was: "+str(e), RNS.LOG_ERROR) return None @@ -378,6 +396,7 @@ class LXMRouter: MAX_DELIVERY_ATTEMPTS = 3 PROCESSING_INTERVAL = 5 DELIVERY_RETRY_WAIT = 15 + LINK_MAX_INACTIVITY = 30 def __init__(self): self.pending_inbound = [] @@ -411,6 +430,11 @@ class LXMRouter: def register_delivery_callback(self, callback): self.__delivery_callback = callback + def announce(self, destination_hash): + if destination_hash in self.delivery_destinations: + delivery_destination = self.delivery_destinations[destination_hash] + delivery_destination.announce(delivery_destination.display_name.encode("utf-8")) + def handle_outbound(self, lxmessage): if not lxmessage.packed: lxmessage.pack() @@ -493,6 +517,23 @@ class LXMRouter: def jobs(self): self.process_outbound() + self.clean_links() + + def clean_links(self): + closed_links = [] + for link_hash in self.direct_links: + #TODO: Fix + link = self.direct_links[link_hash] + inactive_time = link.inactive_for() + + if inactive_time > LXMRouter.LINK_MAX_INACTIVITY: + link.teardown() + closed_links.append(link_hash) + RNS.log(str(link)+" was inactive for "+str(inactive_time)+" seconds and closed") + + for link_hash in closed_links: + self.direct_links.pop(link_hash) + RNS.log("Removed "+RNS.hexrep(link_hash, delimit=False)+" from direct link list, since it was closed") def process_outbound(self, sender = None): if self.processing_outbound: From 5443040e3621399597e6131b2b4cd9223004d43c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 13 May 2021 16:55:55 +0200 Subject: [PATCH 014/436] Version bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 36148ab..3ba27d2 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="lxmf", - version="0.0.1", + version="0.0.2", author="Mark Qvist", author_email="mark@unsigned.io", description="Lightweight Extensible Message Format for Reticulum", From da0ba60b71d9814e1c21e502b43f9f209ac9f949 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 14 May 2021 12:43:29 +0200 Subject: [PATCH 015/436] Added failed state to LXMF messages --- LXMF/LXMF.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index d3d967e..9a00fa3 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -12,7 +12,8 @@ class LXMessage: SENDING = 0x02 SENT = 0x04 DELIVERED = 0x08 - states = [DRAFT, OUTBOUND, SENDING, SENT, DELIVERED] + FAILED = 0xFF + states = [DRAFT, OUTBOUND, SENDING, SENT, DELIVERED, FAILED] UNKNOWN = 0x00 PACKET = 0x01 @@ -122,6 +123,7 @@ class LXMessage: self.resource_representation = None self.__delivery_destination = None self.__delivery_callback = None + self.__failed_callback = None def set_title_from_string(self, title_string): self.title = title_string.encode("utf-8") @@ -180,6 +182,9 @@ class LXMessage: def register_delivery_callback(self, callback): self.__delivery_callback = callback + def register_failed_callback(self, callback): + self.__failed_callback = callback + def pack(self): if not self.packed: if self.timestamp == None: @@ -248,6 +253,7 @@ class LXMessage: self.state = LXMessage.SENDING self.resource_representation = self.__as_resource() elif self.method == LXMessage.PROPAGATED: + # TODO: Implement propagation pass def __mark_delivered(self, receipt = None): @@ -257,6 +263,13 @@ class LXMessage: if self.__delivery_callback != None: self.__delivery_callback(self) + def __mark_failed(self, receipt = None): + RNS.log(str(self)+" failed to send", RNS.LOG_DEBUG) + self.state = LXMessage.FAILED + + if self.__failed_callback != None: + self.__failed_callback(self) + def __resource_concluded(self, resource): if resource.status == RNS.Resource.COMPLETE: self.__mark_delivered() From e069713226b1a840fd07718b9fc80ef7ce6562d0 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 14 May 2021 13:25:06 +0200 Subject: [PATCH 016/436] Message fail callback --- LXMF/LXMF.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 9a00fa3..a451b6f 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -263,7 +263,7 @@ class LXMessage: if self.__delivery_callback != None: self.__delivery_callback(self) - def __mark_failed(self, receipt = None): + def _LXMRouter__mark_failed(self, receipt = None): RNS.log(str(self)+" failed to send", RNS.LOG_DEBUG) self.state = LXMessage.FAILED @@ -548,6 +548,11 @@ class LXMRouter: self.direct_links.pop(link_hash) RNS.log("Removed "+RNS.hexrep(link_hash, delimit=False)+" from direct link list, since it was closed") + def fail_message(self, lxmessage): + self.pending_outbound.remove(lxmessage) + self.failed_outbound.append(lxmessage) + lxmessage.__mark_failed() + def process_outbound(self, sender = None): if self.processing_outbound: return @@ -560,21 +565,20 @@ class LXMRouter: RNS.log("Starting outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) # Outbound handling for opportunistic messages if lxmessage.method == LXMessage.OPPORTUNISTIC: - if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: + if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: lxmessage.delivery_attempts += 1 lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT RNS.log("Opportunistic delivery attempt "+str(lxmessage.delivery_attempts)+" for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) lxmessage.send() else: - RNS.log("Max delivery attempts reached for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - self.pending_outbound.remove(lxmessage) - self.failed_outbound.append(lxmessage) + RNS.log("Max delivery attempts reached for oppertunistic "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + self.fail_message(lxmessage) # Outbound handling for messages transferred # over a direct link to the final recipient elif lxmessage.method == LXMessage.DIRECT: - if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: + if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: delivery_destination_hash = lxmessage.get_destination().hash if delivery_destination_hash in self.direct_links: @@ -610,9 +614,8 @@ class LXMRouter: delivery_link.link_established_callback(self.process_outbound) self.direct_links[delivery_destination_hash] = delivery_link else: - RNS.log("Max delivery attempts reached for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - self.pending_outbound.remove(lxmessage) - self.failed_outbound.append(lxmessage) + RNS.log("Max delivery attempts reached for direct "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + self.fail_message(lxmessage) # Outbound handling for messages transported via # propagation to a LXMF router network. From b9ac841c6535a2e804b955145d61caaa32ea50a6 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 14 May 2021 14:43:14 +0200 Subject: [PATCH 017/436] Failed outbound message handling --- LXMF/LXMF.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index a451b6f..554ff62 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -123,7 +123,7 @@ class LXMessage: self.resource_representation = None self.__delivery_destination = None self.__delivery_callback = None - self.__failed_callback = None + self.failed_callback = None def set_title_from_string(self, title_string): self.title = title_string.encode("utf-8") @@ -183,7 +183,7 @@ class LXMessage: self.__delivery_callback = callback def register_failed_callback(self, callback): - self.__failed_callback = callback + self.failed_callback = callback def pack(self): if not self.packed: @@ -263,13 +263,6 @@ class LXMessage: if self.__delivery_callback != None: self.__delivery_callback(self) - def _LXMRouter__mark_failed(self, receipt = None): - RNS.log(str(self)+" failed to send", RNS.LOG_DEBUG) - self.state = LXMessage.FAILED - - if self.__failed_callback != None: - self.__failed_callback(self) - def __resource_concluded(self, resource): if resource.status == RNS.Resource.COMPLETE: self.__mark_delivered() @@ -449,6 +442,7 @@ class LXMRouter: delivery_destination.announce(delivery_destination.display_name.encode("utf-8")) def handle_outbound(self, lxmessage): + lxmessage.state = LXMessage.OUTBOUND if not lxmessage.packed: lxmessage.pack() @@ -535,7 +529,6 @@ class LXMRouter: def clean_links(self): closed_links = [] for link_hash in self.direct_links: - #TODO: Fix link = self.direct_links[link_hash] inactive_time = link.inactive_for() @@ -549,9 +542,14 @@ class LXMRouter: RNS.log("Removed "+RNS.hexrep(link_hash, delimit=False)+" from direct link list, since it was closed") def fail_message(self, lxmessage): + RNS.log(str(lxmessage)+" failed to send", RNS.LOG_DEBUG) + self.pending_outbound.remove(lxmessage) self.failed_outbound.append(lxmessage) - lxmessage.__mark_failed() + + lxmessage.state = LXMessage.FAILED + if lxmessage.failed_callback != None: + lxmessage.failed_callback(lxmessage) def process_outbound(self, sender = None): if self.processing_outbound: From 9ca3fa5be254631b6103da888e3c487f39485869 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 14 May 2021 19:34:04 +0200 Subject: [PATCH 018/436] Added transport encryption indicators --- LXMF/LXMF.py | 77 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 66 insertions(+), 11 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 554ff62..508a9de 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -72,6 +72,12 @@ class LXMessage: PLAIN_PACKET_MDU = RNS.Packet.PLAIN_MDU PLAIN_PACKET_MAX_CONTENT = PLAIN_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH + # Descriptive strings regarding transport encryption + ENCRYPTION_DESCRIPTION_RSA = "RSA-"+str(RNS.Identity.KEYSIZE) + ENCRYPTION_DESCRIPTION_AES = "AES-128" + ENCRYPTION_DESCRIPTION_EC = "EC-SECP256R1" + ENCRYPTION_DESCRIPTION_UNENCRYPTED = "Unencrypted" + def __str__(self): if self.hash != None: return "" @@ -118,6 +124,7 @@ class LXMessage: self.representation = LXMessage.UNKNOWN self.desired_method = desired_method self.delivery_attempts = 0 + self.transport_encrypted = False self.transport_encryption = None self.packet_representation = None self.resource_representation = None @@ -244,8 +251,9 @@ class LXMessage: else: raise ValueError("Attempt to re-pack LXMessage "+str(self)+" that was already packed") - def send(self): + self.determine_transport_encryption() + if self.method == LXMessage.OPPORTUNISTIC: self.__as_packet().send().delivery_callback(self.__mark_delivered) self.state = LXMessage.SENT @@ -256,6 +264,38 @@ class LXMessage: # TODO: Implement propagation pass + def determine_transport_encryption(self): + if RNS.Reticulum.should_allow_unencrypted(): + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + else: + if self.method == LXMessage.OPPORTUNISTIC: + if self.destination.type == RNS.Destination.SINGLE: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_RSA + elif destination_type == RNS.Destination.GROUP: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES + else: + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + elif self.method == LXMessage.DIRECT: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + elif self.method == LXMessage.PROPAGATED: + if self.destination.type == RNS.Destination.SINGLE: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_RSA + elif destination_type == RNS.Destination.GROUP: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES + else: + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + else: + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + def __mark_delivered(self, receipt = None): RNS.log("Received delivery notification for "+str(self), RNS.LOG_DEBUG) self.state = LXMessage.DELIVERED @@ -310,7 +350,13 @@ class LXMessage: if not self.packed: self.pack() - container = {"state": self.state, "lxmf_bytes": self.packed} + container = { + "state": self.state, + "lxmf_bytes": self.packed, + "transport_encrypted": self.transport_encrypted, + "transport_encryption": self.transport_encryption + } + packed_container = msgpack.packb(container) file = open(file_path, "wb") @@ -391,7 +437,14 @@ class LXMessage: try: container = msgpack.unpackb(lxmf_file_handle.read()) lxm = LXMessage.unpack_from_bytes(container["lxmf_bytes"]) - lxm.state = container["state"] + + if "state" in container: + lxm.state = container["state"] + if "transport_encrypted" in container: + lxm.transport_encrypted = container["transport_encrypted"] + if "transport_encryption" in container: + lxm.transport_encryption = container["transport_encryption"] + return lxm except Exception as e: RNS.log("Could not unpack LXMessage from file. The contained exception was: "+str(e), RNS.LOG_ERROR) @@ -446,7 +499,7 @@ class LXMRouter: if not lxmessage.packed: lxmessage.pack() - RNS.log("LXM Router received outbound message: "+str(lxmessage)) + lxmessage.determine_transport_encryption() while self.processing_outbound: time.sleep(0.1) @@ -459,15 +512,20 @@ class LXMRouter: message = LXMessage.unpack_from_bytes(lxmf_data) if RNS.Reticulum.should_allow_unencrypted(): - message.transport_encryption = "Consider unencrypted (Disabling encryption was allowed in Reticulum configuration)" + message.transport_encrypted = False + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED else: if destination_type == RNS.Destination.SINGLE: - message.transport_encryption = "RSA-"+str(RNS.Identity.KEYSIZE) + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_RSA elif destination_type == RNS.Destination.GROUP: - message.transport_encryption = "AES-128" + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES elif destination_type == RNS.Destination.LINK: - message.transport_encryption = "EC-SECP256R1" + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC else: + message.transport_encrypted = False message.transport_encryption = None if self.__delivery_callback != None: @@ -478,7 +536,6 @@ class LXMRouter: except Exception as e: RNS.log("Could not assemble LXMF message from received data", RNS.LOG_NOTICE) RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) - raise e return False @@ -535,11 +592,9 @@ class LXMRouter: if inactive_time > LXMRouter.LINK_MAX_INACTIVITY: link.teardown() closed_links.append(link_hash) - RNS.log(str(link)+" was inactive for "+str(inactive_time)+" seconds and closed") for link_hash in closed_links: self.direct_links.pop(link_hash) - RNS.log("Removed "+RNS.hexrep(link_hash, delimit=False)+" from direct link list, since it was closed") def fail_message(self, lxmessage): RNS.log(str(lxmessage)+" failed to send", RNS.LOG_DEBUG) From b991a53a31316bcf82d63d9f8bb9bbd3ac148478 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 15 May 2021 21:09:14 +0200 Subject: [PATCH 019/436] Auto-configure destination default app data to LXMF display name --- LXMF/LXMF.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 508a9de..f4f91c8 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -483,6 +483,9 @@ class LXMRouter: delivery_destination.link_established_callback(self.delivery_link_established) delivery_destination.display_name = display_name + if display_name != None: + delivery_destination.set_default_app_data(display_name.encode("utf-8")) + self.delivery_destinations[delivery_destination.hash] = delivery_destination return delivery_destination From ca55c1fd7a911a8fbb2edf9cebfee6fd4cbe0d34 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 17 May 2021 00:07:28 +0200 Subject: [PATCH 020/436] Updated method names --- LXMF/LXMF.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index f4f91c8..8d3491f 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -203,7 +203,7 @@ class LXMessage: hashed_part += self.__destination.hash hashed_part += self.__source.hash hashed_part += msgpack.packb(self.payload) - self.hash = RNS.Identity.fullHash(hashed_part) + self.hash = RNS.Identity.full_hash(hashed_part) self.message_id = self.hash signed_part = b"" @@ -377,7 +377,7 @@ class LXMessage: signature = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH] packed_payload = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH:] hashed_part = b"" + destination_hash + source_hash + packed_payload - message_hash = RNS.Identity.fullHash(hashed_part) + message_hash = RNS.Identity.full_hash(hashed_part) signed_part = b"" + hashed_part + message_hash unpacked_payload = msgpack.unpackb(packed_payload) timestamp = unpacked_payload[0] From 62d79b627fcb3b37da38153d2795e61559f57384 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 17 May 2021 14:04:27 +0200 Subject: [PATCH 021/436] Reticulum moved to Curve25519 --- LXMF/LXMF.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 8d3491f..c86e195 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -75,7 +75,7 @@ class LXMessage: # Descriptive strings regarding transport encryption ENCRYPTION_DESCRIPTION_RSA = "RSA-"+str(RNS.Identity.KEYSIZE) ENCRYPTION_DESCRIPTION_AES = "AES-128" - ENCRYPTION_DESCRIPTION_EC = "EC-SECP256R1" + ENCRYPTION_DESCRIPTION_EC = "Curve25519+Fernet" ENCRYPTION_DESCRIPTION_UNENCRYPTED = "Unencrypted" def __str__(self): From 4215bd186e507c04c022f8290c05673b23307df7 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 17 May 2021 19:45:46 +0200 Subject: [PATCH 022/436] Updated readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8194833..36ff8e0 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ LXMF uses encryption provided by [Reticulum](https://github.com/markqvist/reticu - If a message is delivered to the Reticulum GROUP destination type, the message will be transported using _AES-128_ encryption. -- If a message is delivered over a Reticulum link (which is the default method), the message will be encrypted with _ephemeral_ keys derived with _ECDH_ on the _SECP256R1_ curve. This mode offers Perfect Forward Secrecy. +- If a message is delivered over a Reticulum link (which is the default method), the message will be encrypted with _ephemeral_ keys derived with _ECDH_ on _Curve25519_. This mode offers Perfect Forward Secrecy. ## Wire Format & Overhead From 5f578f93b308309a79d51c6875962565e2a39f36 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 17 May 2021 21:54:28 +0200 Subject: [PATCH 023/436] Added automatic path requests. Bumped version to 0.0.4. --- LXMF/LXMF.py | 17 +++++++++++------ setup.py | 4 ++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index c86e195..3275f09 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -455,7 +455,7 @@ class LXMRouter: MAX_DELIVERY_ATTEMPTS = 3 PROCESSING_INTERVAL = 5 DELIVERY_RETRY_WAIT = 15 - LINK_MAX_INACTIVITY = 30 + LINK_MAX_INACTIVITY = 10*60 def __init__(self): self.pending_inbound = [] @@ -597,7 +597,8 @@ class LXMRouter: closed_links.append(link_hash) for link_hash in closed_links: - self.direct_links.pop(link_hash) + cleaned_link = self.direct_links.pop(link_hash) + RNS.log("Cleaned link "+str(cleaned_link), RNS.LOG_DEBUG) def fail_message(self, lxmessage): RNS.log(str(lxmessage)+" failed to send", RNS.LOG_DEBUG) @@ -665,10 +666,14 @@ class LXMRouter: lxmessage.delivery_attempts += 1 lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: - RNS.log("Establishing link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - delivery_link = RNS.Link(lxmessage.get_destination()) - delivery_link.link_established_callback(self.process_outbound) - self.direct_links[delivery_destination_hash] = delivery_link + if RNS.Transport.has_path(lxmessage.get_destination().hash): + RNS.log("Establishing link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + delivery_link = RNS.Link(lxmessage.get_destination()) + delivery_link.link_established_callback(self.process_outbound) + self.direct_links[delivery_destination_hash] = delivery_link + else: + RNS.log("No path known for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+". Requesting path...", RNS.LOG_DEBUG) + RNS.Transport.request_path(lxmessage.get_destination().hash) else: RNS.log("Max delivery attempts reached for direct "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) self.fail_message(lxmessage) diff --git a/setup.py b/setup.py index 3ba27d2..7a47b64 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="lxmf", - version="0.0.2", + version="0.0.4", author="Mark Qvist", author_email="mark@unsigned.io", description="Lightweight Extensible Message Format for Reticulum", @@ -18,6 +18,6 @@ setuptools.setup( "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - install_requires=['rns'], + install_requires=['rns>=0.2.0'], python_requires='>=3.6', ) \ No newline at end of file From 75c7e8026978edd1ddd6ae7794068ee07abf098b Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 20 May 2021 22:17:55 +0200 Subject: [PATCH 024/436] Updated LXMF to Reticulum 0.2.1 elliptic curve cryptography --- LXMF/LXMF.py | 47 +++++++++++++++++++++++------------------------ README.md | 2 +- setup.py | 2 +- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 3275f09..b7a07dc 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -32,37 +32,37 @@ class LXMessage: DESTINATION_LENGTH = RNS.Identity.TRUNCATED_HASHLENGTH//8 SIGNATURE_LENGTH = RNS.Identity.SIGLENGTH//8 - # LXMF overhead is 163 bytes per message: + # LXMF overhead is 99 bytes per message: # 10 bytes for destination hash # 10 bytes for source hash - # 128 bytes for RSA signature + # 64 bytes for Ed25519 signature # 8 bytes for timestamp # 7 bytes for msgpack structure LXMF_OVERHEAD = 2*DESTINATION_LENGTH + SIGNATURE_LENGTH + 8 + 7 - # With an MTU of 500, the maximum RSA-encrypted - # amount of data we can send in a single packet - # is given by the below calculation; 258 bytes. - RSA_PACKET_MDU = RNS.Packet.RSA_MDU + # With an MTU of 500, the maximum amount of data + # we can send in a single encrypted packet is + # 383 bytes. + ENCRYPTED_PACKET_MDU = RNS.Packet.ENCRYPTED_MDU # The max content length we can fit in LXMF message - # inside a single RNS packet is the RSA MDU, minus + # inside a single RNS packet is the encrypted MDU, minus # the LXMF overhead. We can optimise a bit though, by # inferring the destination hash from the destination # field of the packet, therefore we also add the length # of a destination hash to the calculation. With default # RNS and LXMF parameters, the largest single-packet - # LXMF message we can send is 105 bytes. If a message + # LXMF message we can send is 294 bytes. If a message # is larger than that, a Reticulum link will be used. - RSA_PACKET_MAX_CONTENT = RSA_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH + ENCRYPTED_PACKET_MAX_CONTENT = ENCRYPTED_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH - # Links can carry a significantly larger MDU, due to - # more efficient elliptic curve cryptography. The link - # MDU with default Reticulum parameters is 415 bytes. + # Links can carry a larger MDU, due to less overhead per + # packet. The link MDU with default Reticulum parameters + # is 415 bytes. LINK_PACKET_MDU = RNS.Link.MDU # Which means that we can deliver single-packet LXMF - # messages with content of up to 252 bytes over a link. + # messages with content of up to 316 bytes over a link. # If a message is larger than that, LXMF will sequence # and transfer it as a RNS resource over the link instead. LINK_PACKET_MAX_CONTENT = LINK_PACKET_MDU - LXMF_OVERHEAD @@ -73,9 +73,8 @@ class LXMessage: PLAIN_PACKET_MAX_CONTENT = PLAIN_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH # Descriptive strings regarding transport encryption - ENCRYPTION_DESCRIPTION_RSA = "RSA-"+str(RNS.Identity.KEYSIZE) ENCRYPTION_DESCRIPTION_AES = "AES-128" - ENCRYPTION_DESCRIPTION_EC = "Curve25519+Fernet" + ENCRYPTION_DESCRIPTION_EC = "Curve25519" ENCRYPTION_DESCRIPTION_UNENCRYPTED = "Unencrypted" def __str__(self): @@ -255,7 +254,7 @@ class LXMessage: self.determine_transport_encryption() if self.method == LXMessage.OPPORTUNISTIC: - self.__as_packet().send().delivery_callback(self.__mark_delivered) + self.__as_packet().send().set_delivery_callback(self.__mark_delivered) self.state = LXMessage.SENT elif self.method == LXMessage.DIRECT: self.state = LXMessage.SENDING @@ -272,7 +271,7 @@ class LXMessage: if self.method == LXMessage.OPPORTUNISTIC: if self.destination.type == RNS.Destination.SINGLE: self.transport_encrypted = True - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_RSA + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC elif destination_type == RNS.Destination.GROUP: self.transport_encrypted = True self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES @@ -285,7 +284,7 @@ class LXMessage: elif self.method == LXMessage.PROPAGATED: if self.destination.type == RNS.Destination.SINGLE: self.transport_encrypted = True - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_RSA + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC elif destination_type == RNS.Destination.GROUP: self.transport_encrypted = True self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES @@ -479,8 +478,8 @@ class LXMRouter: def register_delivery_identity(self, identity, display_name = None): delivery_destination = RNS.Destination(identity, RNS.Destination.IN, RNS.Destination.SINGLE, "lxmf", "delivery") - delivery_destination.packet_callback(self.delivery_packet) - delivery_destination.link_established_callback(self.delivery_link_established) + delivery_destination.set_packet_callback(self.delivery_packet) + delivery_destination.set_link_established_callback(self.delivery_link_established) delivery_destination.display_name = display_name if display_name != None: @@ -559,10 +558,10 @@ class LXMRouter: RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) def delivery_link_established(self, link): - link.packet_callback(self.delivery_packet) + link.set_packet_callback(self.delivery_packet) link.set_resource_strategy(RNS.Link.ACCEPT_ALL) - link.resource_started_callback(self.resource_transfer_began) - link.resource_concluded_callback(self.resource_transfer_concluded) + link.set_resource_started_callback(self.resource_transfer_began) + link.set_resource_concluded_callback(self.resource_transfer_concluded) def delivery_link_closed(self, link): pass @@ -669,7 +668,7 @@ class LXMRouter: if RNS.Transport.has_path(lxmessage.get_destination().hash): RNS.log("Establishing link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) delivery_link = RNS.Link(lxmessage.get_destination()) - delivery_link.link_established_callback(self.process_outbound) + delivery_link.set_link_established_callback(self.process_outbound) self.direct_links[delivery_destination_hash] = delivery_link else: RNS.log("No path known for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+". Requesting path...", RNS.LOG_DEBUG) diff --git a/README.md b/README.md index 36ff8e0..8a68113 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ The complete message overhead for LXMF is only 163 bytes, which in return gives ## Caveat Emptor -As with Reticulum, LXMF is alpha software, and should be considered experimental. While it has been built with cryptography best-practices very foremost in mind, it _has not_ been externally security audited, and there could very well be privacy-breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch. +LXMF is alpha software, and should be considered experimental. While it has been built with cryptography best-practices very foremost in mind, it _has not_ been externally security audited, and there could very well be privacy-breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch. ## Installation diff --git a/setup.py b/setup.py index 7a47b64..c1b2ef0 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="lxmf", - version="0.0.4", + version="0.0.5", author="Mark Qvist", author_email="mark@unsigned.io", description="Lightweight Extensible Message Format for Reticulum", From 81ab9f5a2856cd45f8b4d300cac55387d76b70b4 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 20 May 2021 22:57:46 +0200 Subject: [PATCH 025/436] Updated dependency versions --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c1b2ef0..2828c76 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="lxmf", - version="0.0.5", + version="0.0.6", author="Mark Qvist", author_email="mark@unsigned.io", description="Lightweight Extensible Message Format for Reticulum", @@ -18,6 +18,6 @@ setuptools.setup( "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - install_requires=['rns>=0.2.0'], + install_requires=['rns>=0.2.1'], python_requires='>=3.6', ) \ No newline at end of file From e8413be3dcb2a1562eb2598e809de625ec9f440f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 20 May 2021 23:08:19 +0200 Subject: [PATCH 026/436] Updated readme --- README.html | 847 ---------------------------------------------------- README.md | 10 +- 2 files changed, 5 insertions(+), 852 deletions(-) delete mode 100644 README.html diff --git a/README.html b/README.html deleted file mode 100644 index 2b5ff21..0000000 --- a/README.html +++ /dev/null @@ -1,847 +0,0 @@ -README

    -Lightweight Extensible Message Format

    -

    LXMF is a simple and flexible messaging format that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of Reticulum and offers zero-conf message routing, end-to-end encryption and Perfect Forward Secrecy by default.

    -

    -Structure

    -

    LXMF messages are stored in a simple and efficient format, that's easy to parse and write.

    -
    -The format follows this general structure:
    -
      -
    • Destination
      • -
    • Source
      • -
    • RSA Signature
      • -
    • Payload -
        -
      • Timestamp
        • -
      • Content
        • -
      • Title
        • -
      • Fields
        • -
      -
      • -
    -
    -And these rules:
    -
      -
    1. -

      A LXMF message is identified by it's message-id, which is a SHA-256 hash of the Destination, Source and Payload. The message-id is never included directly in the message, since it can always be inferred from the message itself.

      -

      In some cases the actual message-id cannot be inferred, for example when a Propagation Node is storing an encrypted message for an offline user. In theses cases a transient-id is used to identify the message while in storage or transit.

      -
      2. -
    2. -

      Destination, Source, Signature and Payload parts are mandatory, as is the Timestamp part of the payload.

      -
        -
      • The Destination and Source fields are Reticulum destination hashes
        • -
      • The Signature field is a RSA signature of the Destination, Source, Payload and message-id -
        • -
      • The Payload part is a list containing four items: -
          -
        1. The Timestamp is a double-precision floating point number representing the number of seconds since the UNIX epoch.
          2. -
        2. The Content is the optional content or body of the message
          3. -
        3. The Title is an optional title for the message
          4. -
        4. The Fields is an optional dictionary
          5. -
        -
        • -
      -
      3. -
    3. -

      The Content, Title and Fields parts must be included in the message structure, but can be left empty.

      -
      4. -
    4. -

      The Fields part can be left empty, or contain a dictionary of any structure or depth.

      -
      5. -
    -

    -Usage Examples

    -

    LXMF offers flexibility to implement many different messaging schemes, ranging from human communication to machine control and sensor monitoring. Here's a few examples:

    -
      -
    • -

      A messaging system for passing short, simple messages between human users, akin to SMS can be implemented using only the Content field, and leaving all other optional fields empty.

      -
      • -
    • -

      For sending full-size mail, an email-like system can be implemented using the Title and Content fields to store "subject" and "body" parts of the message, and optionally the Fields part can be used to store attachments or other metadata.

      -
      • -
    • -

      Machine-control messages or sensor readings can be implemented using command structures embedded in the Fields dictionary.

      -
      • -
    • -

      Distributed discussion or news-groups, akin to USENET or similar systems, can be implemented using the relevant fields and LXMF Propagation Nodes. Broadcast bulletins can be implemented in a similar fashion.

      -
      • -
    -

    -Propagation Nodes

    -

    LXM Propagation Nodes offer a way to store and forward messages to users or endpoints that are not directly reachable at the time of message emission. Propagation Nodes can also provide infrastructure for distributed bulletin, news or discussion boards.

    -

    -The LXM Router

    -

    The LXM Router handles transporting messages over a Reticulum network, managing delivery receipts, outbound and inbound queues, and is the point of API interaction for client programs. The LXM Router also implements functionality for acting as an LXMF Propagation Node.

    -

    Programatically, using the LXM Router to send a message is as simple as:

    -
    import LXMF
-
-lxm_router = LXMF.LXMRouter()
-
-message = LXMF.LXMessage(destination, source, "This is a short, simple message.")
-
-lxm_router.handle_outbound(message)
    -

    The LXM Router then handles the heavy lifting, such as message packing, encryption, delivery confirmation, path lookup, routing, retries and failure notifications.

    -

    -Transport Encryption

    -

    LXMF uses encryption provided by Reticulum, and thus uses end-to-end encryption by default. The delivery method of a message will influence which transport encryption scheme is used.

    -
      -
    • -

      A message can be delivered opportunistically, embedded in a single Reticulum packet. In this cases the message will be opportunistically routed through the network, and use RSA-1024 asymmetric encryption.

      -
      • -
    • -

      If a message is delivered to the Reticulum GROUP destination type, the message will be transported using AES-128 encryption.

      -
      • -
    • -

      If a message is delivered over a Reticulum link (which is the default method), the message will be encrypted with ephemeral keys derived with ECDH on the SECP256R1 curve. This mode offers Perfect Forward Secrecy.

      -
      • -
    -

    -Wire Format & Overhead

    -

    Assuming the default Reticulum configuration, the binary wire-format is as follows:

    -
      -
    • 10 bytes destination hash
      • -
    • 10 bytes source hash
      • -
    • 128 bytes RSA signature
      • -
    • Remaining bytes of msgpack payload data, in accordance with the structure defined above
      • -
    -

    The complete message overhead for LXMF is only 163 bytes, which in return gives you timestamped, digitally signed, infinitely extensible, end-to-end encrypted, zero-conf routed, minimal-infrastructure messaging that's easy to use and build applications with.

    -

    -Caveat Emptor

    -

    As with Reticulum, LXMF is alpha software, and should be considered experimental. While it has been built with cryptography best-practices very foremost in mind, it has not been externally security audited, and there could very well be privacy-breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch.

    -

    -Installation

    -

    If you want to try out LXMF, you can install it with pip:

    -
    pip3 install lxmf
    -
    \ No newline at end of file diff --git a/README.md b/README.md index 8a68113..fbcb1cd 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ LXMF messages are stored in a simple and efficient format, that's easy to parse - Destination - Source -- RSA Signature +- Ed25519 Signature - Payload - Timestamp - Content @@ -26,7 +26,7 @@ LXMF messages are stored in a simple and efficient format, that's easy to parse 2. __Destination__, __Source__, __Signature__ and __Payload__ parts are mandatory, as is the __Timestamp__ part of the payload. - The __Destination__ and __Source__ fields are Reticulum destination hashes - - The __Signature__ field is a RSA signature of the __Destination__, __Source__, __Payload__ and __message-id__ + - The __Signature__ field is a Ed25519 signature of the __Destination__, __Source__, __Payload__ and __message-id__ - The __Payload__ part is a list containing four items: 1. The __Timestamp__ is a double-precision floating point number representing the number of seconds since the UNIX epoch. 2. The __Content__ is the optional content or body of the message @@ -76,7 +76,7 @@ The LXM Router then handles the heavy lifting, such as message packing, encrypti LXMF uses encryption provided by [Reticulum](https://github.com/markqvist/reticulum), and thus uses end-to-end encryption by default. The delivery method of a message will influence which transport encryption scheme is used. -- A message can be delivered opportunistically, embedded in a single Reticulum packet. In this cases the message will be opportunistically routed through the network, and use _RSA-1024_ asymmetric encryption. +- A message can be delivered opportunistically, embedded in a single Reticulum packet. In this cases the message will be opportunistically routed through the network, and will be encrypted with _ephemeral_ keys derived with _ECDH_ on _Curve25519_. This mode offers Perfect Forward Secrecy. - If a message is delivered to the Reticulum GROUP destination type, the message will be transported using _AES-128_ encryption. @@ -88,10 +88,10 @@ Assuming the default Reticulum configuration, the binary wire-format is as follo - 10 bytes destination hash - 10 bytes source hash -- 128 bytes RSA signature +- 64 bytes Ed25519 signature - Remaining bytes of [msgpack](https://msgpack.org) payload data, in accordance with the structure defined above -The complete message overhead for LXMF is only 163 bytes, which in return gives you timestamped, digitally signed, infinitely extensible, end-to-end encrypted, zero-conf routed, minimal-infrastructure messaging that's easy to use and build applications with. +The complete message overhead for LXMF is only 99 bytes, which in return gives you timestamped, digitally signed, infinitely extensible, end-to-end encrypted, zero-conf routed, minimal-infrastructure messaging that's easy to use and build applications with. ## Caveat Emptor From e7f2ccc002febb8a5c8ec159576ec945dd087b99 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 20 May 2021 23:09:33 +0200 Subject: [PATCH 027/436] Updated readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fbcb1cd..1f97785 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Lightweight Extensible Message Format ========== -LXMF is a simple and flexible messaging format that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](https://github.com/markqvist/reticulum) and offers zero-conf message routing, end-to-end encryption and Perfect Forward Secrecy by default. +LXMF is a simple and flexible messaging format and delivery protocol that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](https://github.com/markqvist/reticulum) and offers zero-conf message routing, end-to-end encryption and Perfect Forward Secrecy by default. ## Structure From ad541b3db9c3f974113a34608be51ba0865e3fae Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 2 Jul 2021 23:29:15 +0200 Subject: [PATCH 028/436] Vresion bump. Relaxed required python version. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2828c76..1f28770 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="lxmf", - version="0.0.6", + version="0.0.7", author="Mark Qvist", author_email="mark@unsigned.io", description="Lightweight Extensible Message Format for Reticulum", @@ -19,5 +19,5 @@ setuptools.setup( "Operating System :: OS Independent", ], install_requires=['rns>=0.2.1'], - python_requires='>=3.6', + python_requires='>=3.5', ) \ No newline at end of file From 1e8ef437b950386e2d9a2f31fd2306979c64086a Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 28 Aug 2021 15:30:47 +0200 Subject: [PATCH 029/436] Implemented delivery as single packets over links. --- LXMF/LXMF.py | 28 ++++++++++++++++++++++------ README.md | 8 ++++---- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index b7a07dc..44e25af 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -228,7 +228,7 @@ class LXMessage: if self.desired_method == LXMessage.OPPORTUNISTIC: if self.__destination.type == RNS.Destination.SINGLE: - single_packet_content_limit = LXMessage.RSA_PACKET_MAX_CONTENT + single_packet_content_limit = LXMessage.ENCRYPTED_PACKET_MAX_CONTENT elif self.__destination.type == RNS.Destination.PLAIN: single_packet_content_limit = LXMessage.PLAIN_PACKET_MAX_CONTENT @@ -256,9 +256,18 @@ class LXMessage: if self.method == LXMessage.OPPORTUNISTIC: self.__as_packet().send().set_delivery_callback(self.__mark_delivered) self.state = LXMessage.SENT + elif self.method == LXMessage.DIRECT: self.state = LXMessage.SENDING - self.resource_representation = self.__as_resource() + + if self.representation == LXMessage.PACKET: + receipt = self.__as_packet().send() + receipt.set_delivery_callback(self.__mark_delivered) + receipt.set_timeout_callback(self.__link_packet_timed_out) + + elif self.representation == LXMessage.RESOURCE: + self.resource_representation = self.__as_resource() + elif self.method == LXMessage.PROPAGATED: # TODO: Implement propagation pass @@ -309,6 +318,10 @@ class LXMessage: resource.link.teardown() self.state = LXMessage.OUTBOUND + def __link_packet_timed_out(self, packet_receipt): + packet_receipt.destination.teardown() + self.state = LXMessage.OUTBOUND + def __update_transfer_progress(self, resource): self.progress = resource.progress() @@ -543,14 +556,14 @@ class LXMRouter: def delivery_packet(self, data, packet): try: - if packet.destination.type != RNS.Destination.LINK: + if packet.destination_type != RNS.Destination.LINK: lxmf_data = b"" lxmf_data += packet.destination.hash lxmf_data += data else: lxmf_data = data - if self.lxmf_delivery(lxmf_data, packet.destination.type): + if self.lxmf_delivery(lxmf_data, packet.destination_type): packet.prove() except Exception as e: @@ -617,7 +630,7 @@ class LXMRouter: if lxmessage.state == LXMessage.DELIVERED: RNS.log("Delivery has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) self.pending_outbound.remove(lxmessage) - else: + else: RNS.log("Starting outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) # Outbound handling for opportunistic messages if lxmessage.method == LXMessage.OPPORTUNISTIC: @@ -647,7 +660,10 @@ class LXMRouter: lxmessage.set_delivery_destination(direct_link) lxmessage.send() else: - RNS.log("The transfer of "+str(lxmessage)+" is in progress ("+str(round(lxmessage.progress*100, 1))+"%)", RNS.LOG_DEBUG) + if lxmessage.representation == LXMessage.RESOURCE: + RNS.log("The transfer of "+str(lxmessage)+" is in progress ("+str(round(lxmessage.progress*100, 1))+"%)", RNS.LOG_DEBUG) + else: + RNS.log("Waiting for proof for "+str(lxmessage)+" sent as link packet", RNS.LOG_DEBUG) elif direct_link.status == RNS.Link.CLOSED: RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was closed", RNS.LOG_DEBUG) lxmessage.set_delivery_destination(None) diff --git a/README.md b/README.md index 1f97785..c6fffca 100644 --- a/README.md +++ b/README.md @@ -20,14 +20,14 @@ LXMF messages are stored in a simple and efficient format, that's easy to parse ##### And these rules: -1. A LXMF message is identified by it's __message-id__, which is a SHA-256 hash of the __Destination__, __Source__ and __Payload__. The message-id is never included directly in the message, since it can always be inferred from the message itself. +1. A LXMF message is identified by its __message-id__, which is a SHA-256 hash of the __Destination__, __Source__ and __Payload__. The message-id is never included directly in the message, since it can always be inferred from the message itself. In some cases the actual message-id cannot be inferred, for example when a Propagation Node is storing an encrypted message for an offline user. In theses cases a _transient-id_ is used to identify the message while in storage or transit. 2. __Destination__, __Source__, __Signature__ and __Payload__ parts are mandatory, as is the __Timestamp__ part of the payload. - - The __Destination__ and __Source__ fields are Reticulum destination hashes - - The __Signature__ field is a Ed25519 signature of the __Destination__, __Source__, __Payload__ and __message-id__ - - The __Payload__ part is a list containing four items: + - The __Destination__ and __Source__ fields are 10-byte Reticulum destination hashes + - The __Signature__ field is a 64-byte Ed25519 signature of the __Destination__, __Source__, __Payload__ and __message-id__ + - The __Payload__ part is a [msgpacked](https://msgpack.org) list containing four items: 1. The __Timestamp__ is a double-precision floating point number representing the number of seconds since the UNIX epoch. 2. The __Content__ is the optional content or body of the message 3. The __Title__ is an optional title for the message From 1ac9c3ad7243142b66a0af17ca998daccc5a65c0 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 28 Aug 2021 15:56:37 +0200 Subject: [PATCH 030/436] Added announce handler for faster initial link establishment. --- LXMF/LXMF.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 44e25af..cfe0d97 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -463,10 +463,28 @@ class LXMessage: return None +class LXMFDeliveryAnnounceHandler: + def __init__(self, lxmrouter): + self.aspect_filter = APP_NAME+".delivery" + self.lxmrouter = lxmrouter + + def received_announce(self, destination_hash, announced_identity, app_data): + for lxmessage in self.lxmrouter.pending_outbound: + if destination_hash == lxmessage.destination_hash: + if lxmessage.method == LXMessage.DIRECT: + lxmessage.next_delivery_attempt = time.time() + + while self.lxmrouter.processing_outbound: + time.sleep(0.1) + + self.lxmrouter.process_outbound() + + class LXMRouter: MAX_DELIVERY_ATTEMPTS = 3 PROCESSING_INTERVAL = 5 DELIVERY_RETRY_WAIT = 15 + PATH_REQUEST_WAIT = 5 LINK_MAX_INACTIVITY = 10*60 def __init__(self): @@ -483,6 +501,8 @@ class LXMRouter: self.lxmf_query_destination = RNS.Destination(None, RNS.Destination.IN, RNS.Destination.PLAIN, APP_NAME, "query") self.propagation_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation") + RNS.Transport.register_announce_handler(LXMFDeliveryAnnounceHandler(self)) + self.__delivery_callback = None job_thread = threading.Thread(target=self.jobloop) @@ -680,6 +700,7 @@ class LXMRouter: if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: lxmessage.delivery_attempts += 1 lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: if RNS.Transport.has_path(lxmessage.get_destination().hash): RNS.log("Establishing link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) @@ -689,6 +710,7 @@ class LXMRouter: else: RNS.log("No path known for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+". Requesting path...", RNS.LOG_DEBUG) RNS.Transport.request_path(lxmessage.get_destination().hash) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT else: RNS.log("Max delivery attempts reached for direct "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) self.fail_message(lxmessage) From 11a9ea989f491936c6e1abc36c8685232356f84c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 28 Aug 2021 15:57:22 +0200 Subject: [PATCH 031/436] Fixed indentation. --- LXMF/LXMF.py | 1314 +++++++++++++++++++++++++------------------------- 1 file changed, 657 insertions(+), 657 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index cfe0d97..3612902 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -7,460 +7,460 @@ import RNS.vendor.umsgpack as msgpack APP_NAME = "lxmf" class LXMessage: - DRAFT = 0x00 - OUTBOUND = 0x01 - SENDING = 0x02 - SENT = 0x04 - DELIVERED = 0x08 - FAILED = 0xFF - states = [DRAFT, OUTBOUND, SENDING, SENT, DELIVERED, FAILED] - - UNKNOWN = 0x00 - PACKET = 0x01 - RESOURCE = 0x02 - representations = [UNKNOWN, PACKET, RESOURCE] - - OPPORTUNISTIC = 0x01 - DIRECT = 0x02 - PROPAGATED = 0x03 - valid_methods = [OPPORTUNISTIC, DIRECT, PROPAGATED] - - SOURCE_UNKNOWN = 0x01 - SIGNATURE_INVALID = 0x02 - unverified_reasons = [SOURCE_UNKNOWN, SIGNATURE_INVALID] - - DESTINATION_LENGTH = RNS.Identity.TRUNCATED_HASHLENGTH//8 - SIGNATURE_LENGTH = RNS.Identity.SIGLENGTH//8 - - # LXMF overhead is 99 bytes per message: - # 10 bytes for destination hash - # 10 bytes for source hash - # 64 bytes for Ed25519 signature - # 8 bytes for timestamp - # 7 bytes for msgpack structure - LXMF_OVERHEAD = 2*DESTINATION_LENGTH + SIGNATURE_LENGTH + 8 + 7 - - # With an MTU of 500, the maximum amount of data - # we can send in a single encrypted packet is - # 383 bytes. - ENCRYPTED_PACKET_MDU = RNS.Packet.ENCRYPTED_MDU - - # The max content length we can fit in LXMF message - # inside a single RNS packet is the encrypted MDU, minus - # the LXMF overhead. We can optimise a bit though, by - # inferring the destination hash from the destination - # field of the packet, therefore we also add the length - # of a destination hash to the calculation. With default - # RNS and LXMF parameters, the largest single-packet - # LXMF message we can send is 294 bytes. If a message - # is larger than that, a Reticulum link will be used. - ENCRYPTED_PACKET_MAX_CONTENT = ENCRYPTED_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH - - # Links can carry a larger MDU, due to less overhead per - # packet. The link MDU with default Reticulum parameters - # is 415 bytes. - LINK_PACKET_MDU = RNS.Link.MDU - - # Which means that we can deliver single-packet LXMF - # messages with content of up to 316 bytes over a link. - # If a message is larger than that, LXMF will sequence - # and transfer it as a RNS resource over the link instead. - LINK_PACKET_MAX_CONTENT = LINK_PACKET_MDU - LXMF_OVERHEAD - - # For plain packets without encryption, we can - # fit up to 324 bytes of content. - PLAIN_PACKET_MDU = RNS.Packet.PLAIN_MDU - PLAIN_PACKET_MAX_CONTENT = PLAIN_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH - - # Descriptive strings regarding transport encryption - ENCRYPTION_DESCRIPTION_AES = "AES-128" - ENCRYPTION_DESCRIPTION_EC = "Curve25519" - ENCRYPTION_DESCRIPTION_UNENCRYPTED = "Unencrypted" - - def __str__(self): - if self.hash != None: - return "" - else: - return "" - - def __init__(self, destination, source, content = "", title = "", fields = None, desired_method = None, destination_hash = None, source_hash = None): - if isinstance(destination, RNS.Destination) or destination == None: - self.__destination = destination - if destination != None: - self.destination_hash = destination.hash - else: - self.destination_hash = destination_hash - else: - raise ValueError("LXMessage initialised with invalid destination") - - if isinstance(source, RNS.Destination) or source == None: - self.__source = source - if source != None: - self.source_hash = source.hash - else: - self.source_hash = source_hash - else: - raise ValueError("LXMessage initialised with invalid source") - - self.set_title_from_string(title) - self.set_content_from_string(content) - - self.set_fields(fields) - - self.payload = None - self.timestamp = None - self.signature = None - self.hash = None - self.packed = None - self.progress = None - self.state = LXMessage.DRAFT - self.method = LXMessage.UNKNOWN - - self.incoming = False - self.signature_validated = False - self.unverified_reason = None - - self.representation = LXMessage.UNKNOWN - self.desired_method = desired_method - self.delivery_attempts = 0 - self.transport_encrypted = False - self.transport_encryption = None - self.packet_representation = None - self.resource_representation = None - self.__delivery_destination = None - self.__delivery_callback = None - self.failed_callback = None - - def set_title_from_string(self, title_string): - self.title = title_string.encode("utf-8") - - def set_title_from_bytes(self, title_bytes): - self.title = title_bytes - - def title_as_string(self): - return self.title.decode("utf-8") - - def set_content_from_string(self, content_string): - self.content = content_string.encode("utf-8") - - def set_content_from_bytes(self, content_bytes): - self.content = content_bytes - - def content_as_string(self): - return self.content.decode("utf-8") - - def set_fields(self, fields): - if isinstance(fields, dict) or fields == None: - self.fields = fields - else: - raise ValueError("LXMessage property \"fields\" can only be dict or None") - - def get_fields(self): - return self.__fields - - def set_destination(self, destination): - if self.destination == None: - if isinstance(destination, RNS.Destination): - self.__destination = destination - else: - raise ValueError("Invalid destination set on LXMessage") - else: - raise ValueError("Cannot reassign destination on LXMessage") - - def get_destination(self): - return self.__destination - - def set_source(self, source): - if self.source == None: - if isinstance(source, RNS.Destination): - self.__source = source - else: - raise ValueError("Invalid source set on LXMessage") - else: - raise ValueError("Cannot reassign source on LXMessage") - - def get_source(self): - return self.__source - - def set_delivery_destination(self, delivery_destination): - self.__delivery_destination = delivery_destination - - def register_delivery_callback(self, callback): - self.__delivery_callback = callback - - def register_failed_callback(self, callback): - self.failed_callback = callback - - def pack(self): - if not self.packed: - if self.timestamp == None: - self.timestamp = time.time() - - self.payload = [self.timestamp, self.title, self.content, self.fields] - - hashed_part = b"" - hashed_part += self.__destination.hash - hashed_part += self.__source.hash - hashed_part += msgpack.packb(self.payload) - self.hash = RNS.Identity.full_hash(hashed_part) - self.message_id = self.hash - - signed_part = b"" - signed_part += hashed_part - signed_part += self.hash - self.signature = self.__source.sign(signed_part) - self.signature_validated = True - - self.packed = b"" - self.packed += self.__destination.hash - self.packed += self.__source.hash - self.packed += self.signature - packed_payload = msgpack.packb(self.payload) - self.packed += packed_payload - self.packed_size = len(self.packed) - content_size = len(packed_payload) - - # If no desired delivery method has been defined, - # one will be chosen according to these rules: - if self.desired_method == None: - self.desired_method == LXMessage.DIRECT - # TODO: Expand rules to something more intelligent - - if self.desired_method == LXMessage.OPPORTUNISTIC: - if self.__destination.type == RNS.Destination.SINGLE: - single_packet_content_limit = LXMessage.ENCRYPTED_PACKET_MAX_CONTENT - elif self.__destination.type == RNS.Destination.PLAIN: - single_packet_content_limit = LXMessage.PLAIN_PACKET_MAX_CONTENT - - if content_size > single_packet_content_limit: - raise TypeError("LXMessage desired opportunistic delivery method, but content exceeds single-packet size.") - else: - self.method = LXMessage.OPPORTUNISTIC - self.representation = LXMessage.PACKET - self.__delivery_destination = self.__destination - - elif self.desired_method == LXMessage.DIRECT or self.desired_method == LXMessage.PROPAGATED: - single_packet_content_limit = LXMessage.LINK_PACKET_MAX_CONTENT - if content_size <= single_packet_content_limit: - self.method = self.desired_method - self.representation = LXMessage.PACKET - else: - self.method = self.desired_method - self.representation = LXMessage.RESOURCE - else: - raise ValueError("Attempt to re-pack LXMessage "+str(self)+" that was already packed") - - def send(self): - self.determine_transport_encryption() - - if self.method == LXMessage.OPPORTUNISTIC: - self.__as_packet().send().set_delivery_callback(self.__mark_delivered) - self.state = LXMessage.SENT - - elif self.method == LXMessage.DIRECT: - self.state = LXMessage.SENDING - - if self.representation == LXMessage.PACKET: - receipt = self.__as_packet().send() - receipt.set_delivery_callback(self.__mark_delivered) - receipt.set_timeout_callback(self.__link_packet_timed_out) - - elif self.representation == LXMessage.RESOURCE: - self.resource_representation = self.__as_resource() - - elif self.method == LXMessage.PROPAGATED: - # TODO: Implement propagation - pass - - def determine_transport_encryption(self): - if RNS.Reticulum.should_allow_unencrypted(): - self.transport_encrypted = False - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED - else: - if self.method == LXMessage.OPPORTUNISTIC: - if self.destination.type == RNS.Destination.SINGLE: - self.transport_encrypted = True - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC - elif destination_type == RNS.Destination.GROUP: - self.transport_encrypted = True - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES - else: - self.transport_encrypted = False - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED - elif self.method == LXMessage.DIRECT: - self.transport_encrypted = True - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC - elif self.method == LXMessage.PROPAGATED: - if self.destination.type == RNS.Destination.SINGLE: - self.transport_encrypted = True - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC - elif destination_type == RNS.Destination.GROUP: - self.transport_encrypted = True - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES - else: - self.transport_encrypted = False - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED - else: - self.transport_encrypted = False - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED - - def __mark_delivered(self, receipt = None): - RNS.log("Received delivery notification for "+str(self), RNS.LOG_DEBUG) - self.state = LXMessage.DELIVERED - - if self.__delivery_callback != None: - self.__delivery_callback(self) - - def __resource_concluded(self, resource): - if resource.status == RNS.Resource.COMPLETE: - self.__mark_delivered() - else: - resource.link.teardown() - self.state = LXMessage.OUTBOUND - - def __link_packet_timed_out(self, packet_receipt): - packet_receipt.destination.teardown() - self.state = LXMessage.OUTBOUND - - - def __update_transfer_progress(self, resource): - self.progress = resource.progress() - - def __as_packet(self): - if not self.packed: - self.pack() - - if not self.__delivery_destination: - raise ValueError("Can't synthesize packet for LXMF message before delivery destination is known") - - if self.method == LXMessage.OPPORTUNISTIC: - return RNS.Packet(self.__delivery_destination, self.packed[LXMessage.DESTINATION_LENGTH:]) - elif self.method == LXMessage.DIRECT or self.method == LXMessage.PROPAGATED: - return RNS.Packet(self.__delivery_destination, self.packed) - - def __as_resource(self): - if not self.packed: - self.pack() - - if not self.__delivery_destination: - raise ValueError("Can't synthesize resource for LXMF message before delivery destination is known") - - if not self.__delivery_destination.type == RNS.Destination.LINK: - raise TypeError("Tried to synthesize resource for LXMF message on a delivery destination that was not a link") - - if not self.__delivery_destination.status == RNS.Link.ACTIVE: - raise ConnectionError("Tried to synthesize resource for LXMF message on a link that was not active") - - self.progress = 0.0 - return RNS.Resource(self.packed, self.__delivery_destination, callback = self.__resource_concluded, progress_callback = self.__update_transfer_progress) - - def write_to_directory(self, directory_path): - file_name = RNS.hexrep(self.hash, delimit=False) - file_path = directory_path+"/"+file_name - - try: - if not self.packed: - self.pack() - - container = { - "state": self.state, - "lxmf_bytes": self.packed, - "transport_encrypted": self.transport_encrypted, - "transport_encryption": self.transport_encryption - } - - packed_container = msgpack.packb(container) - - file = open(file_path, "wb") - file.write(packed_container) - file.close() - - return file_path - - except Exception as e: - RNS.log("Error while writing LXMF message to file \""+str(file_path)+"\". The contained exception was: "+str(e), RNS.LOG_ERROR) - return None - - - @staticmethod - def unpack_from_bytes(lxmf_bytes): - destination_hash = lxmf_bytes[:LXMessage.DESTINATION_LENGTH] - source_hash = lxmf_bytes[LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH] - signature = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH] - packed_payload = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH:] - hashed_part = b"" + destination_hash + source_hash + packed_payload - message_hash = RNS.Identity.full_hash(hashed_part) - signed_part = b"" + hashed_part + message_hash - unpacked_payload = msgpack.unpackb(packed_payload) - timestamp = unpacked_payload[0] - title_bytes = unpacked_payload[1] - content_bytes = unpacked_payload[2] - fields = unpacked_payload[3] - - destination_identity = RNS.Identity.recall(destination_hash) - if destination_identity != None: - destination = RNS.Destination(destination_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") - else: - destination = None - - source_identity = RNS.Identity.recall(source_hash) - if source_identity != None: - source = RNS.Destination(source_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") - else: - source = None - - message = LXMessage( - destination = destination, - source = source, - content = "", - title = "", - fields = fields, - destination_hash = destination_hash, - source_hash = source_hash) - - message.hash = message_hash - message.signature = signature - message.incoming = True - message.timestamp = timestamp - message.packed = lxmf_bytes - message.packed_size = len(lxmf_bytes) - message.set_title_from_bytes(title_bytes) - message.set_content_from_bytes(content_bytes) - - try: - if source: - if source.identity.validate(signature, signed_part): - message.signature_validated = True - else: - message.signature_validated = False - message.unverified_reason = LXMessage.SIGNATURE_INVALID - else: - signature_validated = False - message.unverified_reason = LXMessage.SOURCE_UNKNOWN - RNS.log("Unpacked LXMF message signature could not be validated, since source identity is unknown", RNS.LOG_DEBUG) - except Exception as e: - message.signature_validated = False - RNS.log("Error while validating LXMF message signature. The contained exception was: "+str(e), RNS.LOG_ERROR) - - return message - - @staticmethod - def unpack_from_file(lxmf_file_handle): - try: - container = msgpack.unpackb(lxmf_file_handle.read()) - lxm = LXMessage.unpack_from_bytes(container["lxmf_bytes"]) - - if "state" in container: - lxm.state = container["state"] - if "transport_encrypted" in container: - lxm.transport_encrypted = container["transport_encrypted"] - if "transport_encryption" in container: - lxm.transport_encryption = container["transport_encryption"] - - return lxm - except Exception as e: - RNS.log("Could not unpack LXMessage from file. The contained exception was: "+str(e), RNS.LOG_ERROR) - return None + DRAFT = 0x00 + OUTBOUND = 0x01 + SENDING = 0x02 + SENT = 0x04 + DELIVERED = 0x08 + FAILED = 0xFF + states = [DRAFT, OUTBOUND, SENDING, SENT, DELIVERED, FAILED] + + UNKNOWN = 0x00 + PACKET = 0x01 + RESOURCE = 0x02 + representations = [UNKNOWN, PACKET, RESOURCE] + + OPPORTUNISTIC = 0x01 + DIRECT = 0x02 + PROPAGATED = 0x03 + valid_methods = [OPPORTUNISTIC, DIRECT, PROPAGATED] + + SOURCE_UNKNOWN = 0x01 + SIGNATURE_INVALID = 0x02 + unverified_reasons = [SOURCE_UNKNOWN, SIGNATURE_INVALID] + + DESTINATION_LENGTH = RNS.Identity.TRUNCATED_HASHLENGTH//8 + SIGNATURE_LENGTH = RNS.Identity.SIGLENGTH//8 + + # LXMF overhead is 99 bytes per message: + # 10 bytes for destination hash + # 10 bytes for source hash + # 64 bytes for Ed25519 signature + # 8 bytes for timestamp + # 7 bytes for msgpack structure + LXMF_OVERHEAD = 2*DESTINATION_LENGTH + SIGNATURE_LENGTH + 8 + 7 + + # With an MTU of 500, the maximum amount of data + # we can send in a single encrypted packet is + # 383 bytes. + ENCRYPTED_PACKET_MDU = RNS.Packet.ENCRYPTED_MDU + + # The max content length we can fit in LXMF message + # inside a single RNS packet is the encrypted MDU, minus + # the LXMF overhead. We can optimise a bit though, by + # inferring the destination hash from the destination + # field of the packet, therefore we also add the length + # of a destination hash to the calculation. With default + # RNS and LXMF parameters, the largest single-packet + # LXMF message we can send is 294 bytes. If a message + # is larger than that, a Reticulum link will be used. + ENCRYPTED_PACKET_MAX_CONTENT = ENCRYPTED_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH + + # Links can carry a larger MDU, due to less overhead per + # packet. The link MDU with default Reticulum parameters + # is 415 bytes. + LINK_PACKET_MDU = RNS.Link.MDU + + # Which means that we can deliver single-packet LXMF + # messages with content of up to 316 bytes over a link. + # If a message is larger than that, LXMF will sequence + # and transfer it as a RNS resource over the link instead. + LINK_PACKET_MAX_CONTENT = LINK_PACKET_MDU - LXMF_OVERHEAD + + # For plain packets without encryption, we can + # fit up to 324 bytes of content. + PLAIN_PACKET_MDU = RNS.Packet.PLAIN_MDU + PLAIN_PACKET_MAX_CONTENT = PLAIN_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH + + # Descriptive strings regarding transport encryption + ENCRYPTION_DESCRIPTION_AES = "AES-128" + ENCRYPTION_DESCRIPTION_EC = "Curve25519" + ENCRYPTION_DESCRIPTION_UNENCRYPTED = "Unencrypted" + + def __str__(self): + if self.hash != None: + return "" + else: + return "" + + def __init__(self, destination, source, content = "", title = "", fields = None, desired_method = None, destination_hash = None, source_hash = None): + if isinstance(destination, RNS.Destination) or destination == None: + self.__destination = destination + if destination != None: + self.destination_hash = destination.hash + else: + self.destination_hash = destination_hash + else: + raise ValueError("LXMessage initialised with invalid destination") + + if isinstance(source, RNS.Destination) or source == None: + self.__source = source + if source != None: + self.source_hash = source.hash + else: + self.source_hash = source_hash + else: + raise ValueError("LXMessage initialised with invalid source") + + self.set_title_from_string(title) + self.set_content_from_string(content) + + self.set_fields(fields) + + self.payload = None + self.timestamp = None + self.signature = None + self.hash = None + self.packed = None + self.progress = None + self.state = LXMessage.DRAFT + self.method = LXMessage.UNKNOWN + + self.incoming = False + self.signature_validated = False + self.unverified_reason = None + + self.representation = LXMessage.UNKNOWN + self.desired_method = desired_method + self.delivery_attempts = 0 + self.transport_encrypted = False + self.transport_encryption = None + self.packet_representation = None + self.resource_representation = None + self.__delivery_destination = None + self.__delivery_callback = None + self.failed_callback = None + + def set_title_from_string(self, title_string): + self.title = title_string.encode("utf-8") + + def set_title_from_bytes(self, title_bytes): + self.title = title_bytes + + def title_as_string(self): + return self.title.decode("utf-8") + + def set_content_from_string(self, content_string): + self.content = content_string.encode("utf-8") + + def set_content_from_bytes(self, content_bytes): + self.content = content_bytes + + def content_as_string(self): + return self.content.decode("utf-8") + + def set_fields(self, fields): + if isinstance(fields, dict) or fields == None: + self.fields = fields + else: + raise ValueError("LXMessage property \"fields\" can only be dict or None") + + def get_fields(self): + return self.__fields + + def set_destination(self, destination): + if self.destination == None: + if isinstance(destination, RNS.Destination): + self.__destination = destination + else: + raise ValueError("Invalid destination set on LXMessage") + else: + raise ValueError("Cannot reassign destination on LXMessage") + + def get_destination(self): + return self.__destination + + def set_source(self, source): + if self.source == None: + if isinstance(source, RNS.Destination): + self.__source = source + else: + raise ValueError("Invalid source set on LXMessage") + else: + raise ValueError("Cannot reassign source on LXMessage") + + def get_source(self): + return self.__source + + def set_delivery_destination(self, delivery_destination): + self.__delivery_destination = delivery_destination + + def register_delivery_callback(self, callback): + self.__delivery_callback = callback + + def register_failed_callback(self, callback): + self.failed_callback = callback + + def pack(self): + if not self.packed: + if self.timestamp == None: + self.timestamp = time.time() + + self.payload = [self.timestamp, self.title, self.content, self.fields] + + hashed_part = b"" + hashed_part += self.__destination.hash + hashed_part += self.__source.hash + hashed_part += msgpack.packb(self.payload) + self.hash = RNS.Identity.full_hash(hashed_part) + self.message_id = self.hash + + signed_part = b"" + signed_part += hashed_part + signed_part += self.hash + self.signature = self.__source.sign(signed_part) + self.signature_validated = True + + self.packed = b"" + self.packed += self.__destination.hash + self.packed += self.__source.hash + self.packed += self.signature + packed_payload = msgpack.packb(self.payload) + self.packed += packed_payload + self.packed_size = len(self.packed) + content_size = len(packed_payload) + + # If no desired delivery method has been defined, + # one will be chosen according to these rules: + if self.desired_method == None: + self.desired_method == LXMessage.DIRECT + # TODO: Expand rules to something more intelligent + + if self.desired_method == LXMessage.OPPORTUNISTIC: + if self.__destination.type == RNS.Destination.SINGLE: + single_packet_content_limit = LXMessage.ENCRYPTED_PACKET_MAX_CONTENT + elif self.__destination.type == RNS.Destination.PLAIN: + single_packet_content_limit = LXMessage.PLAIN_PACKET_MAX_CONTENT + + if content_size > single_packet_content_limit: + raise TypeError("LXMessage desired opportunistic delivery method, but content exceeds single-packet size.") + else: + self.method = LXMessage.OPPORTUNISTIC + self.representation = LXMessage.PACKET + self.__delivery_destination = self.__destination + + elif self.desired_method == LXMessage.DIRECT or self.desired_method == LXMessage.PROPAGATED: + single_packet_content_limit = LXMessage.LINK_PACKET_MAX_CONTENT + if content_size <= single_packet_content_limit: + self.method = self.desired_method + self.representation = LXMessage.PACKET + else: + self.method = self.desired_method + self.representation = LXMessage.RESOURCE + else: + raise ValueError("Attempt to re-pack LXMessage "+str(self)+" that was already packed") + + def send(self): + self.determine_transport_encryption() + + if self.method == LXMessage.OPPORTUNISTIC: + self.__as_packet().send().set_delivery_callback(self.__mark_delivered) + self.state = LXMessage.SENT + + elif self.method == LXMessage.DIRECT: + self.state = LXMessage.SENDING + + if self.representation == LXMessage.PACKET: + receipt = self.__as_packet().send() + receipt.set_delivery_callback(self.__mark_delivered) + receipt.set_timeout_callback(self.__link_packet_timed_out) + + elif self.representation == LXMessage.RESOURCE: + self.resource_representation = self.__as_resource() + + elif self.method == LXMessage.PROPAGATED: + # TODO: Implement propagation + pass + + def determine_transport_encryption(self): + if RNS.Reticulum.should_allow_unencrypted(): + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + else: + if self.method == LXMessage.OPPORTUNISTIC: + if self.destination.type == RNS.Destination.SINGLE: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + elif destination_type == RNS.Destination.GROUP: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES + else: + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + elif self.method == LXMessage.DIRECT: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + elif self.method == LXMessage.PROPAGATED: + if self.destination.type == RNS.Destination.SINGLE: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + elif destination_type == RNS.Destination.GROUP: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES + else: + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + else: + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + + def __mark_delivered(self, receipt = None): + RNS.log("Received delivery notification for "+str(self), RNS.LOG_DEBUG) + self.state = LXMessage.DELIVERED + + if self.__delivery_callback != None: + self.__delivery_callback(self) + + def __resource_concluded(self, resource): + if resource.status == RNS.Resource.COMPLETE: + self.__mark_delivered() + else: + resource.link.teardown() + self.state = LXMessage.OUTBOUND + + def __link_packet_timed_out(self, packet_receipt): + packet_receipt.destination.teardown() + self.state = LXMessage.OUTBOUND + + + def __update_transfer_progress(self, resource): + self.progress = resource.progress() + + def __as_packet(self): + if not self.packed: + self.pack() + + if not self.__delivery_destination: + raise ValueError("Can't synthesize packet for LXMF message before delivery destination is known") + + if self.method == LXMessage.OPPORTUNISTIC: + return RNS.Packet(self.__delivery_destination, self.packed[LXMessage.DESTINATION_LENGTH:]) + elif self.method == LXMessage.DIRECT or self.method == LXMessage.PROPAGATED: + return RNS.Packet(self.__delivery_destination, self.packed) + + def __as_resource(self): + if not self.packed: + self.pack() + + if not self.__delivery_destination: + raise ValueError("Can't synthesize resource for LXMF message before delivery destination is known") + + if not self.__delivery_destination.type == RNS.Destination.LINK: + raise TypeError("Tried to synthesize resource for LXMF message on a delivery destination that was not a link") + + if not self.__delivery_destination.status == RNS.Link.ACTIVE: + raise ConnectionError("Tried to synthesize resource for LXMF message on a link that was not active") + + self.progress = 0.0 + return RNS.Resource(self.packed, self.__delivery_destination, callback = self.__resource_concluded, progress_callback = self.__update_transfer_progress) + + def write_to_directory(self, directory_path): + file_name = RNS.hexrep(self.hash, delimit=False) + file_path = directory_path+"/"+file_name + + try: + if not self.packed: + self.pack() + + container = { + "state": self.state, + "lxmf_bytes": self.packed, + "transport_encrypted": self.transport_encrypted, + "transport_encryption": self.transport_encryption + } + + packed_container = msgpack.packb(container) + + file = open(file_path, "wb") + file.write(packed_container) + file.close() + + return file_path + + except Exception as e: + RNS.log("Error while writing LXMF message to file \""+str(file_path)+"\". The contained exception was: "+str(e), RNS.LOG_ERROR) + return None + + + @staticmethod + def unpack_from_bytes(lxmf_bytes): + destination_hash = lxmf_bytes[:LXMessage.DESTINATION_LENGTH] + source_hash = lxmf_bytes[LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH] + signature = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH] + packed_payload = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH:] + hashed_part = b"" + destination_hash + source_hash + packed_payload + message_hash = RNS.Identity.full_hash(hashed_part) + signed_part = b"" + hashed_part + message_hash + unpacked_payload = msgpack.unpackb(packed_payload) + timestamp = unpacked_payload[0] + title_bytes = unpacked_payload[1] + content_bytes = unpacked_payload[2] + fields = unpacked_payload[3] + + destination_identity = RNS.Identity.recall(destination_hash) + if destination_identity != None: + destination = RNS.Destination(destination_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") + else: + destination = None + + source_identity = RNS.Identity.recall(source_hash) + if source_identity != None: + source = RNS.Destination(source_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") + else: + source = None + + message = LXMessage( + destination = destination, + source = source, + content = "", + title = "", + fields = fields, + destination_hash = destination_hash, + source_hash = source_hash) + + message.hash = message_hash + message.signature = signature + message.incoming = True + message.timestamp = timestamp + message.packed = lxmf_bytes + message.packed_size = len(lxmf_bytes) + message.set_title_from_bytes(title_bytes) + message.set_content_from_bytes(content_bytes) + + try: + if source: + if source.identity.validate(signature, signed_part): + message.signature_validated = True + else: + message.signature_validated = False + message.unverified_reason = LXMessage.SIGNATURE_INVALID + else: + signature_validated = False + message.unverified_reason = LXMessage.SOURCE_UNKNOWN + RNS.log("Unpacked LXMF message signature could not be validated, since source identity is unknown", RNS.LOG_DEBUG) + except Exception as e: + message.signature_validated = False + RNS.log("Error while validating LXMF message signature. The contained exception was: "+str(e), RNS.LOG_ERROR) + + return message + + @staticmethod + def unpack_from_file(lxmf_file_handle): + try: + container = msgpack.unpackb(lxmf_file_handle.read()) + lxm = LXMessage.unpack_from_bytes(container["lxmf_bytes"]) + + if "state" in container: + lxm.state = container["state"] + if "transport_encrypted" in container: + lxm.transport_encrypted = container["transport_encrypted"] + if "transport_encryption" in container: + lxm.transport_encryption = container["transport_encryption"] + + return lxm + except Exception as e: + RNS.log("Could not unpack LXMessage from file. The contained exception was: "+str(e), RNS.LOG_ERROR) + return None class LXMFDeliveryAnnounceHandler: @@ -470,253 +470,253 @@ class LXMFDeliveryAnnounceHandler: def received_announce(self, destination_hash, announced_identity, app_data): for lxmessage in self.lxmrouter.pending_outbound: - if destination_hash == lxmessage.destination_hash: - if lxmessage.method == LXMessage.DIRECT: - lxmessage.next_delivery_attempt = time.time() + if destination_hash == lxmessage.destination_hash: + if lxmessage.method == LXMessage.DIRECT: + lxmessage.next_delivery_attempt = time.time() - while self.lxmrouter.processing_outbound: - time.sleep(0.1) + while self.lxmrouter.processing_outbound: + time.sleep(0.1) - self.lxmrouter.process_outbound() + self.lxmrouter.process_outbound() class LXMRouter: - MAX_DELIVERY_ATTEMPTS = 3 - PROCESSING_INTERVAL = 5 - DELIVERY_RETRY_WAIT = 15 - PATH_REQUEST_WAIT = 5 - LINK_MAX_INACTIVITY = 10*60 + MAX_DELIVERY_ATTEMPTS = 3 + PROCESSING_INTERVAL = 5 + DELIVERY_RETRY_WAIT = 15 + PATH_REQUEST_WAIT = 5 + LINK_MAX_INACTIVITY = 10*60 - def __init__(self): - self.pending_inbound = [] - self.pending_outbound = [] - self.failed_outbound = [] - self.direct_links = {} - self.delivery_destinations = {} + def __init__(self): + self.pending_inbound = [] + self.pending_outbound = [] + self.failed_outbound = [] + self.direct_links = {} + self.delivery_destinations = {} - self.processing_outbound = False - self.processing_inbound = False + self.processing_outbound = False + self.processing_inbound = False - self.identity = RNS.Identity() - self.lxmf_query_destination = RNS.Destination(None, RNS.Destination.IN, RNS.Destination.PLAIN, APP_NAME, "query") - self.propagation_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation") + self.identity = RNS.Identity() + self.lxmf_query_destination = RNS.Destination(None, RNS.Destination.IN, RNS.Destination.PLAIN, APP_NAME, "query") + self.propagation_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation") - RNS.Transport.register_announce_handler(LXMFDeliveryAnnounceHandler(self)) + RNS.Transport.register_announce_handler(LXMFDeliveryAnnounceHandler(self)) - self.__delivery_callback = None + self.__delivery_callback = None - job_thread = threading.Thread(target=self.jobloop) - job_thread.setDaemon(True) - job_thread.start() + job_thread = threading.Thread(target=self.jobloop) + job_thread.setDaemon(True) + job_thread.start() - def register_delivery_identity(self, identity, display_name = None): - delivery_destination = RNS.Destination(identity, RNS.Destination.IN, RNS.Destination.SINGLE, "lxmf", "delivery") - delivery_destination.set_packet_callback(self.delivery_packet) - delivery_destination.set_link_established_callback(self.delivery_link_established) - delivery_destination.display_name = display_name + def register_delivery_identity(self, identity, display_name = None): + delivery_destination = RNS.Destination(identity, RNS.Destination.IN, RNS.Destination.SINGLE, "lxmf", "delivery") + delivery_destination.set_packet_callback(self.delivery_packet) + delivery_destination.set_link_established_callback(self.delivery_link_established) + delivery_destination.display_name = display_name - if display_name != None: - delivery_destination.set_default_app_data(display_name.encode("utf-8")) + if display_name != None: + delivery_destination.set_default_app_data(display_name.encode("utf-8")) - self.delivery_destinations[delivery_destination.hash] = delivery_destination - return delivery_destination + self.delivery_destinations[delivery_destination.hash] = delivery_destination + return delivery_destination - def register_delivery_callback(self, callback): - self.__delivery_callback = callback + def register_delivery_callback(self, callback): + self.__delivery_callback = callback - def announce(self, destination_hash): - if destination_hash in self.delivery_destinations: - delivery_destination = self.delivery_destinations[destination_hash] - delivery_destination.announce(delivery_destination.display_name.encode("utf-8")) + def announce(self, destination_hash): + if destination_hash in self.delivery_destinations: + delivery_destination = self.delivery_destinations[destination_hash] + delivery_destination.announce(delivery_destination.display_name.encode("utf-8")) - def handle_outbound(self, lxmessage): - lxmessage.state = LXMessage.OUTBOUND - if not lxmessage.packed: - lxmessage.pack() + def handle_outbound(self, lxmessage): + lxmessage.state = LXMessage.OUTBOUND + if not lxmessage.packed: + lxmessage.pack() - lxmessage.determine_transport_encryption() + lxmessage.determine_transport_encryption() - while self.processing_outbound: - time.sleep(0.1) + while self.processing_outbound: + time.sleep(0.1) - self.pending_outbound.append(lxmessage) - self.process_outbound() + self.pending_outbound.append(lxmessage) + self.process_outbound() - def lxmf_delivery(self, lxmf_data, destination_type = None): - try: - message = LXMessage.unpack_from_bytes(lxmf_data) + def lxmf_delivery(self, lxmf_data, destination_type = None): + try: + message = LXMessage.unpack_from_bytes(lxmf_data) - if RNS.Reticulum.should_allow_unencrypted(): - message.transport_encrypted = False - message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED - else: - if destination_type == RNS.Destination.SINGLE: - message.transport_encrypted = True - message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_RSA - elif destination_type == RNS.Destination.GROUP: - message.transport_encrypted = True - message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES - elif destination_type == RNS.Destination.LINK: - message.transport_encrypted = True - message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC - else: - message.transport_encrypted = False - message.transport_encryption = None + if RNS.Reticulum.should_allow_unencrypted(): + message.transport_encrypted = False + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + else: + if destination_type == RNS.Destination.SINGLE: + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_RSA + elif destination_type == RNS.Destination.GROUP: + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES + elif destination_type == RNS.Destination.LINK: + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + else: + message.transport_encrypted = False + message.transport_encryption = None - if self.__delivery_callback != None: - self.__delivery_callback(message) + if self.__delivery_callback != None: + self.__delivery_callback(message) - return True + return True - except Exception as e: - RNS.log("Could not assemble LXMF message from received data", RNS.LOG_NOTICE) - RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) - return False + except Exception as e: + RNS.log("Could not assemble LXMF message from received data", RNS.LOG_NOTICE) + RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + return False - def delivery_packet(self, data, packet): - try: - if packet.destination_type != RNS.Destination.LINK: - lxmf_data = b"" - lxmf_data += packet.destination.hash - lxmf_data += data - else: - lxmf_data = data + def delivery_packet(self, data, packet): + try: + if packet.destination_type != RNS.Destination.LINK: + lxmf_data = b"" + lxmf_data += packet.destination.hash + lxmf_data += data + else: + lxmf_data = data - if self.lxmf_delivery(lxmf_data, packet.destination_type): - packet.prove() + if self.lxmf_delivery(lxmf_data, packet.destination_type): + packet.prove() - except Exception as e: - RNS.log("Exception occurred while parsing incoming LXMF data.", RNS.LOG_ERROR) - RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) + except Exception as e: + RNS.log("Exception occurred while parsing incoming LXMF data.", RNS.LOG_ERROR) + RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) - def delivery_link_established(self, link): - link.set_packet_callback(self.delivery_packet) - link.set_resource_strategy(RNS.Link.ACCEPT_ALL) - link.set_resource_started_callback(self.resource_transfer_began) - link.set_resource_concluded_callback(self.resource_transfer_concluded) + def delivery_link_established(self, link): + link.set_packet_callback(self.delivery_packet) + link.set_resource_strategy(RNS.Link.ACCEPT_ALL) + link.set_resource_started_callback(self.resource_transfer_began) + link.set_resource_concluded_callback(self.resource_transfer_concluded) - def delivery_link_closed(self, link): - pass + def delivery_link_closed(self, link): + pass - def resource_transfer_began(self, resource): - RNS.log("Transfer began for resource "+str(resource), RNS.LOG_DEBUG) + def resource_transfer_began(self, resource): + RNS.log("Transfer began for resource "+str(resource), RNS.LOG_DEBUG) - def resource_transfer_concluded(self, resource): - RNS.log("Transfer concluded for resource "+str(resource), RNS.LOG_DEBUG) - if resource.status == RNS.Resource.COMPLETE: - self.lxmf_delivery(resource.data.read(), resource.link.type) + def resource_transfer_concluded(self, resource): + RNS.log("Transfer concluded for resource "+str(resource), RNS.LOG_DEBUG) + if resource.status == RNS.Resource.COMPLETE: + self.lxmf_delivery(resource.data.read(), resource.link.type) - def jobloop(self): - while (True): - # TODO: Improve this to scheduling, so manual - # triggers can delay next run - self.jobs() - time.sleep(LXMRouter.PROCESSING_INTERVAL) + def jobloop(self): + while (True): + # TODO: Improve this to scheduling, so manual + # triggers can delay next run + self.jobs() + time.sleep(LXMRouter.PROCESSING_INTERVAL) - def jobs(self): - self.process_outbound() - self.clean_links() + def jobs(self): + self.process_outbound() + self.clean_links() - def clean_links(self): - closed_links = [] - for link_hash in self.direct_links: - link = self.direct_links[link_hash] - inactive_time = link.inactive_for() + def clean_links(self): + closed_links = [] + for link_hash in self.direct_links: + link = self.direct_links[link_hash] + inactive_time = link.inactive_for() - if inactive_time > LXMRouter.LINK_MAX_INACTIVITY: - link.teardown() - closed_links.append(link_hash) + if inactive_time > LXMRouter.LINK_MAX_INACTIVITY: + link.teardown() + closed_links.append(link_hash) - for link_hash in closed_links: - cleaned_link = self.direct_links.pop(link_hash) - RNS.log("Cleaned link "+str(cleaned_link), RNS.LOG_DEBUG) + for link_hash in closed_links: + cleaned_link = self.direct_links.pop(link_hash) + RNS.log("Cleaned link "+str(cleaned_link), RNS.LOG_DEBUG) - def fail_message(self, lxmessage): - RNS.log(str(lxmessage)+" failed to send", RNS.LOG_DEBUG) + def fail_message(self, lxmessage): + RNS.log(str(lxmessage)+" failed to send", RNS.LOG_DEBUG) - self.pending_outbound.remove(lxmessage) - self.failed_outbound.append(lxmessage) + self.pending_outbound.remove(lxmessage) + self.failed_outbound.append(lxmessage) - lxmessage.state = LXMessage.FAILED - if lxmessage.failed_callback != None: - lxmessage.failed_callback(lxmessage) + lxmessage.state = LXMessage.FAILED + if lxmessage.failed_callback != None: + lxmessage.failed_callback(lxmessage) - def process_outbound(self, sender = None): - if self.processing_outbound: - return + def process_outbound(self, sender = None): + if self.processing_outbound: + return - for lxmessage in self.pending_outbound: - if lxmessage.state == LXMessage.DELIVERED: - RNS.log("Delivery has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) - self.pending_outbound.remove(lxmessage) - else: - RNS.log("Starting outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - # Outbound handling for opportunistic messages - if lxmessage.method == LXMessage.OPPORTUNISTIC: - if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: - if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: - lxmessage.delivery_attempts += 1 - lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT - RNS.log("Opportunistic delivery attempt "+str(lxmessage.delivery_attempts)+" for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - lxmessage.send() - else: - RNS.log("Max delivery attempts reached for oppertunistic "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - self.fail_message(lxmessage) + for lxmessage in self.pending_outbound: + if lxmessage.state == LXMessage.DELIVERED: + RNS.log("Delivery has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) + self.pending_outbound.remove(lxmessage) + else: + RNS.log("Starting outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + # Outbound handling for opportunistic messages + if lxmessage.method == LXMessage.OPPORTUNISTIC: + if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: + if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: + lxmessage.delivery_attempts += 1 + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + RNS.log("Opportunistic delivery attempt "+str(lxmessage.delivery_attempts)+" for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + lxmessage.send() + else: + RNS.log("Max delivery attempts reached for oppertunistic "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + self.fail_message(lxmessage) - # Outbound handling for messages transferred - # over a direct link to the final recipient - elif lxmessage.method == LXMessage.DIRECT: - if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: - delivery_destination_hash = lxmessage.get_destination().hash + # Outbound handling for messages transferred + # over a direct link to the final recipient + elif lxmessage.method == LXMessage.DIRECT: + if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: + delivery_destination_hash = lxmessage.get_destination().hash - if delivery_destination_hash in self.direct_links: - # A link already exists, so we'll try to use it - # to deliver the message - direct_link = self.direct_links[delivery_destination_hash] - if direct_link.status == RNS.Link.ACTIVE: - if lxmessage.state != LXMessage.SENDING: - RNS.log("Starting transfer of "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - lxmessage.set_delivery_destination(direct_link) - lxmessage.send() - else: - if lxmessage.representation == LXMessage.RESOURCE: - RNS.log("The transfer of "+str(lxmessage)+" is in progress ("+str(round(lxmessage.progress*100, 1))+"%)", RNS.LOG_DEBUG) - else: - RNS.log("Waiting for proof for "+str(lxmessage)+" sent as link packet", RNS.LOG_DEBUG) - elif direct_link.status == RNS.Link.CLOSED: - RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was closed", RNS.LOG_DEBUG) - lxmessage.set_delivery_destination(None) - self.direct_links.pop(delivery_destination_hash) - lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT - else: - # Simply wait for the link to become - # active or close - RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" is pending, waiting for link to become active", RNS.LOG_DEBUG) - else: - # No link exists, so we'll try to establish one, but - # only if we've never tried before, or the retry wait - # period has elapsed. - if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: - lxmessage.delivery_attempts += 1 - lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + if delivery_destination_hash in self.direct_links: + # A link already exists, so we'll try to use it + # to deliver the message + direct_link = self.direct_links[delivery_destination_hash] + if direct_link.status == RNS.Link.ACTIVE: + if lxmessage.state != LXMessage.SENDING: + RNS.log("Starting transfer of "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + lxmessage.set_delivery_destination(direct_link) + lxmessage.send() + else: + if lxmessage.representation == LXMessage.RESOURCE: + RNS.log("The transfer of "+str(lxmessage)+" is in progress ("+str(round(lxmessage.progress*100, 1))+"%)", RNS.LOG_DEBUG) + else: + RNS.log("Waiting for proof for "+str(lxmessage)+" sent as link packet", RNS.LOG_DEBUG) + elif direct_link.status == RNS.Link.CLOSED: + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was closed", RNS.LOG_DEBUG) + lxmessage.set_delivery_destination(None) + self.direct_links.pop(delivery_destination_hash) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + else: + # Simply wait for the link to become + # active or close + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" is pending, waiting for link to become active", RNS.LOG_DEBUG) + else: + # No link exists, so we'll try to establish one, but + # only if we've never tried before, or the retry wait + # period has elapsed. + if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: + lxmessage.delivery_attempts += 1 + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT - if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: - if RNS.Transport.has_path(lxmessage.get_destination().hash): - RNS.log("Establishing link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - delivery_link = RNS.Link(lxmessage.get_destination()) - delivery_link.set_link_established_callback(self.process_outbound) - self.direct_links[delivery_destination_hash] = delivery_link - else: - RNS.log("No path known for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+". Requesting path...", RNS.LOG_DEBUG) - RNS.Transport.request_path(lxmessage.get_destination().hash) - lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT - else: - RNS.log("Max delivery attempts reached for direct "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - self.fail_message(lxmessage) + if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: + if RNS.Transport.has_path(lxmessage.get_destination().hash): + RNS.log("Establishing link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + delivery_link = RNS.Link(lxmessage.get_destination()) + delivery_link.set_link_established_callback(self.process_outbound) + self.direct_links[delivery_destination_hash] = delivery_link + else: + RNS.log("No path known for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+". Requesting path...", RNS.LOG_DEBUG) + RNS.Transport.request_path(lxmessage.get_destination().hash) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + else: + RNS.log("Max delivery attempts reached for direct "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + self.fail_message(lxmessage) - # Outbound handling for messages transported via - # propagation to a LXMF router network. - elif lxmessage.method == LXMessage.PROPAGATED: - RNS.log("Attempting propagated delivery for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - raise NotImplementedError("LXMF propagation is not implemented yet") \ No newline at end of file + # Outbound handling for messages transported via + # propagation to a LXMF router network. + elif lxmessage.method == LXMessage.PROPAGATED: + RNS.log("Attempting propagated delivery for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + raise NotImplementedError("LXMF propagation is not implemented yet") \ No newline at end of file From b7de273d16136a47d967afbe89c22e64f9ab4863 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 28 Aug 2021 20:10:31 +0200 Subject: [PATCH 032/436] Updated versions. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 1f28770..37e1e11 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,6 @@ setuptools.setup( "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - install_requires=['rns>=0.2.1'], - python_requires='>=3.5', + install_requires=['rns>=0.2.3'], + python_requires='>=3.6', ) \ No newline at end of file From a1bdcdf7f117f2c2f647806b5581dcda1e7e134c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 29 Aug 2021 14:02:07 +0200 Subject: [PATCH 033/436] Updated version. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 37e1e11..6e39686 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="lxmf", - version="0.0.7", + version="0.0.8", author="Mark Qvist", author_email="mark@unsigned.io", description="Lightweight Extensible Message Format for Reticulum", From aa36f0ef6dd8e69012224b04a83125326a48ec15 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 2 Sep 2021 20:37:14 +0200 Subject: [PATCH 034/436] Updated RNS API call. --- LXMF/LXMF.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 3612902..6a9ac11 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -324,7 +324,7 @@ class LXMessage: def __update_transfer_progress(self, resource): - self.progress = resource.progress() + self.progress = resource.get_progress() def __as_packet(self): if not self.packed: From 3b0c2cac22bfee7071849e34bf5105742cb6596b Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 2 Sep 2021 20:43:00 +0200 Subject: [PATCH 035/436] Updated versions. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 6e39686..bacf3e8 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="lxmf", - version="0.0.8", + version="0.0.9", author="Mark Qvist", author_email="mark@unsigned.io", description="Lightweight Extensible Message Format for Reticulum", @@ -18,6 +18,6 @@ setuptools.setup( "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - install_requires=['rns>=0.2.3'], + install_requires=['rns>=0.2.4'], python_requires='>=3.6', ) \ No newline at end of file From 4d3e529e63799fd65a7f550796cd8ca34da7c0ac Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 3 Sep 2021 23:24:29 +0200 Subject: [PATCH 036/436] Updated according to RNS API. --- LXMF/LXMF.py | 46 +++++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 6a9ac11..72b4dd6 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -273,36 +273,32 @@ class LXMessage: pass def determine_transport_encryption(self): - if RNS.Reticulum.should_allow_unencrypted(): - self.transport_encrypted = False - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED - else: - if self.method == LXMessage.OPPORTUNISTIC: - if self.destination.type == RNS.Destination.SINGLE: - self.transport_encrypted = True - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC - elif destination_type == RNS.Destination.GROUP: - self.transport_encrypted = True - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES - else: - self.transport_encrypted = False - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED - elif self.method == LXMessage.DIRECT: + if self.method == LXMessage.OPPORTUNISTIC: + if self.destination.type == RNS.Destination.SINGLE: self.transport_encrypted = True self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC - elif self.method == LXMessage.PROPAGATED: - if self.destination.type == RNS.Destination.SINGLE: - self.transport_encrypted = True - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC - elif destination_type == RNS.Destination.GROUP: - self.transport_encrypted = True - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES - else: - self.transport_encrypted = False - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + elif destination_type == RNS.Destination.GROUP: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES else: self.transport_encrypted = False self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + elif self.method == LXMessage.DIRECT: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + elif self.method == LXMessage.PROPAGATED: + if self.destination.type == RNS.Destination.SINGLE: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + elif destination_type == RNS.Destination.GROUP: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES + else: + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + else: + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED def __mark_delivered(self, receipt = None): RNS.log("Received delivery notification for "+str(self), RNS.LOG_DEBUG) From 6cb3a23c9db40ca71773ef406b2a7754c6a23c4f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 3 Sep 2021 23:27:21 +0200 Subject: [PATCH 037/436] Updated according to RNS API. --- LXMF/LXMF.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 72b4dd6..c395b25 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -542,22 +542,18 @@ class LXMRouter: try: message = LXMessage.unpack_from_bytes(lxmf_data) - if RNS.Reticulum.should_allow_unencrypted(): - message.transport_encrypted = False - message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + if destination_type == RNS.Destination.SINGLE: + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_RSA + elif destination_type == RNS.Destination.GROUP: + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES + elif destination_type == RNS.Destination.LINK: + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC else: - if destination_type == RNS.Destination.SINGLE: - message.transport_encrypted = True - message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_RSA - elif destination_type == RNS.Destination.GROUP: - message.transport_encrypted = True - message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES - elif destination_type == RNS.Destination.LINK: - message.transport_encrypted = True - message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC - else: - message.transport_encrypted = False - message.transport_encryption = None + message.transport_encrypted = False + message.transport_encryption = None if self.__delivery_callback != None: self.__delivery_callback(message) From 335d9b1adf22257680276c6ccf1989ca59fbc3ac Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 4 Sep 2021 11:03:00 +0200 Subject: [PATCH 038/436] Updated size descriptions. --- LXMF/LXMF.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index c395b25..f33f64f 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -58,17 +58,17 @@ class LXMessage: # Links can carry a larger MDU, due to less overhead per # packet. The link MDU with default Reticulum parameters - # is 415 bytes. + # is 431 bytes. LINK_PACKET_MDU = RNS.Link.MDU # Which means that we can deliver single-packet LXMF - # messages with content of up to 316 bytes over a link. + # messages with content of up to 332 bytes over a link. # If a message is larger than that, LXMF will sequence # and transfer it as a RNS resource over the link instead. LINK_PACKET_MAX_CONTENT = LINK_PACKET_MDU - LXMF_OVERHEAD # For plain packets without encryption, we can - # fit up to 324 bytes of content. + # fit up to 388 bytes of content. PLAIN_PACKET_MDU = RNS.Packet.PLAIN_MDU PLAIN_PACKET_MAX_CONTENT = PLAIN_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH From 7e34b201e1e386acccc338545ccb0ab31e0102a0 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 2 Oct 2021 22:03:43 +0200 Subject: [PATCH 039/436] Updated gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a942c6d..33feb9e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ *.so # Distribution / packaging +RNS .Python build/ develop-eggs/ From 1c5e879abdeada61f85800c7506b621d02b2eaaa Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 2 Oct 2021 22:03:58 +0200 Subject: [PATCH 040/436] Updated version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bacf3e8..1b05948 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="lxmf", - version="0.0.9", + version="0.1.0", author="Mark Qvist", author_email="mark@unsigned.io", description="Lightweight Extensible Message Format for Reticulum", From ad773dcbc5a6ec3d487aeedcfb013dca1fb66ce4 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 2 Oct 2021 22:04:31 +0200 Subject: [PATCH 041/436] General structure for LXMF propagation --- LXMF/LXMF.py | 336 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 328 insertions(+), 8 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index f33f64f..75b77e9 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -1,3 +1,4 @@ +import os import math import time import threading @@ -476,6 +477,167 @@ class LXMFDeliveryAnnounceHandler: self.lxmrouter.process_outbound() +class LXMFPropagationAnnounceHandler: + def __init__(self, lxmrouter): + self.aspect_filter = APP_NAME+".propagation" + self.lxmrouter = lxmrouter + + def received_announce(self, destination_hash, announced_identity, app_data): + try: + data = msgpack.unpackb(app_data) + + if self.lxmrouter.autopeer: + if data[0] == True: + if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: + self.lxmrouter.peer(destination_hash, data[1]) + elif data[0] == False: + self.lxmrouter.unpeer(destination_hash, data[1]) + + except Exception as e: + RNS.log("Error while evaluating propagation node announce, ignoring announce.", RNS.LOG_DEBUG) + RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + + +class LXMPeer: + OFFER_REQUEST_PATH = "o" + + IDLE = 0x00 + LINK_ESTABLISHING = 0x01 + LINK_READY = 0x02 + REQUEST_SENT = 0x03 + RESPONSE_RECEIVED = 0x04 + RESOURCE_TRANSFERRING = 0x05 + + ERROR_NO_IDENTITY = 0xf0 + + @staticmethod + def from_bytes(bytes): + pass + + def to_bytes(self): + pass + + def __init__(self, router, destination_hash): + self.alive = False + self.last_heard = None + self.peering_timebase = 0 + + self.link = None + self.state = LXMPeer.IDLE + + self.unhandled_messages = {} + self.handled_messages = {} + + self.router = router + self.destination_hash = destination_hash + self.identity = RNS.Identity.recall(destination_hash) + self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + + def sync(self, initiator=True): + if len(self.unhandled_messages) > 0: + if self.state == LXMPeer.IDLE: + self.link = RNS.Link(self.destination, established_callback = self.link_established, closed_callback = self.link_closed) + self.state = LXMPeer.LINK_ESTABLISHING + else: + if self.state == LXMPeer.LINK_READY: + unhandled_ids = [] + for transient_id in self.unhandled_messages: + unhandled_ids.append(transient_id) + + # TODO: Remove + RNS.log("Sending sync request to peer") + self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) + + def request_failed(self, request_receipt): + RNS.log("Sync request to peer "+str(self.destination)+" failed", RNS.LOG_DEBUG) + if self.link != None: + self.link.teardown() + + def offer_response(self, request_receipt): + # TODO: Remove + RNS.log("Got offer response from peer") + try: + self.state = LXMPeer.RESPONSE_RECEIVED + response = request_receipt.response + + RNS.log(str(response)) + + wanted_messages = [] + wanted_message_ids = [] + + if response == LXMPeer.ERROR_NO_IDENTITY: + if self.link != None: + self.link.indentify() + self.state = LXMPeer.LINK_READY + + elif response == False: + # Peer already has all advertised messages + + # TODO: Remove + RNS.log("Peer had all advertised messages", RNS.LOG_DEBUG) + + for transient_id in self.unhandled_messages: + message = self.unhandled_messages[transient_id] + self.handled_messages[transient_id] = message + + self.unhandled_messages = {} + + elif response == True: + # Peer wants all advertised messages + for transient_id in self.unhandled_messages: + wanted_messages.append(self.unhandled_messages[transient_id][2]) + wanted_message_ids.append(transient_id) + + else: + # Peer wants some advertised messages + for transient_id in self.unhandled_messages: + # If the peer did not want the message, it has + # already received it from another peer. + if not transient_id in response: + message = self.unhandled_messages.pop(transient_id) + self.handled_messages[transient_id] = message + + for transient_id in response: + wanted_messages.append(self.unhandled_messages[transient_id][2]) + wanted_message_ids.append(transient_id) + + if len(wanted_messages) > 0: + # TODO: Remove + RNS.log("Peer wanted: "+str(wanted_messages), RNS.LOG_DEBUG) + + data = msgpack.packb(wanted_messages) + resource = RNS.Resource(data, self.link, callback = self.resource_concluded) + resource.transferred_messages = wanted_message_ids + self.state = LXMPeer.RESOURCE_TRANSFERRING + + except Exception as e: + RNS.log("Error while handling offer response from peer "+str(self.destination), RNS.LOG_DEBUG) + RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + + + def resource_concluded(self, resource): + if resource.status == RNS.Resource.COMPLETE: + for transient_id in resource.transferred_messages: + message = self.unhandled_messages.pop(transient_id) + self.handled_messages[transient_id] = message + else: + RNS.log("Resource transfer for LXMF peer sync failed to "+str(self.destination), RNS.LOG_DEBUG) + + + + def link_established(self, link): + self.link.identify(self.router.identity) + self.state = LXMPeer.LINK_READY + + def link_closed(self, link): + self.link = None + self.state = LXMPeer.IDLE + + def handle_message(self, transient_id): + if not transient_id in self.handled_messages and not self.unhandled_messages: + self.unhandled_messages[transient_id] = self.router.propagation_entries[transient_id] + + class LXMRouter: MAX_DELIVERY_ATTEMPTS = 3 PROCESSING_INTERVAL = 5 @@ -483,7 +645,21 @@ class LXMRouter: PATH_REQUEST_WAIT = 5 LINK_MAX_INACTIVITY = 10*60 - def __init__(self): + AUTOPEER = True + AUTOPEER_MAXDEPTH = 4 + + # TODO: Remove + def ptest(self): + i1 = RNS.Identity() + i2 = RNS.Identity() + d1 = RNS.Destination(i1, RNS.Destination.IN, RNS.Destination.SINGLE, "test", "delivery") + d2 = RNS.Destination(i2, RNS.Destination.IN, RNS.Destination.SINGLE, "test", "delivery") + + lxm = LXMessage(d1, d2, content="No content here") + + return lxm + + def __init__(self, identity = None, autopeer = AUTOPEER, autopeer_maxdepth = AUTOPEER_MAXDEPTH): self.pending_inbound = [] self.pending_outbound = [] self.failed_outbound = [] @@ -492,12 +668,26 @@ class LXMRouter: self.processing_outbound = False self.processing_inbound = False + self.processing_count = 0 - self.identity = RNS.Identity() + self.propagation_node = False + + if identity == None: + identity = RNS.Identity() + + self.identity = identity self.lxmf_query_destination = RNS.Destination(None, RNS.Destination.IN, RNS.Destination.PLAIN, APP_NAME, "query") self.propagation_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation") + self.autopeer = autopeer + self.autopeer_maxdepth = autopeer_maxdepth + + self.peers = {} + self.propagation_entries = {} + self.propagated_ids = {} + RNS.Transport.register_announce_handler(LXMFDeliveryAnnounceHandler(self)) + RNS.Transport.register_announce_handler(LXMFPropagationAnnounceHandler(self)) self.__delivery_callback = None @@ -544,7 +734,7 @@ class LXMRouter: if destination_type == RNS.Destination.SINGLE: message.transport_encrypted = True - message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_RSA + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC elif destination_type == RNS.Destination.GROUP: message.transport_encrypted = True message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES @@ -586,7 +776,7 @@ class LXMRouter: link.set_packet_callback(self.delivery_packet) link.set_resource_strategy(RNS.Link.ACCEPT_ALL) link.set_resource_started_callback(self.resource_transfer_began) - link.set_resource_concluded_callback(self.resource_transfer_concluded) + link.set_resource_concluded_callback(self.delivery_resource_concluded) def delivery_link_closed(self, link): pass @@ -594,11 +784,133 @@ class LXMRouter: def resource_transfer_began(self, resource): RNS.log("Transfer began for resource "+str(resource), RNS.LOG_DEBUG) - def resource_transfer_concluded(self, resource): - RNS.log("Transfer concluded for resource "+str(resource), RNS.LOG_DEBUG) + def delivery_resource_concluded(self, resource): + RNS.log("Transfer concluded for delivery resource "+str(resource), RNS.LOG_DEBUG) if resource.status == RNS.Resource.COMPLETE: self.lxmf_delivery(resource.data.read(), resource.link.type) + + def enable_propagation(self, storagepath): + try: + self.storagepath = storagepath+"/lxmf" + + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) + + # TODO: Load peers and data + + self.propagation_node = True + self.propagation_destination.set_link_established_callback(self.propagation_link_established) + + self.propagation_destination.register_request_handler(LXMPeer.OFFER_REQUEST_PATH, self.offer_request, allow = RNS.Destination.ALLOW_ALL) + + self.announce_propagation_node() + + except Exception as e: + RNS.log("Could not enable propagation node. The contained exception was: "+str(e), RNS.LOG_ERROR) + RNS.panic() + + def disable_propagation(self): + self.propagation_node = False + self.announce_propagation_node() + + def announce_propagation_node(self): + data = msgpack.packb([self.propagation_node, int(time.time())]) + self.propagation_destination.announce(app_data=data) + + def offer_request(self, path, data, request_id, remote_identity, requested_at): + # TODO: Remove + RNS.log("Offer request") + + if remote_identity == None: + return LXMPeer.ERROR_NO_IDENTITY + else: + try: + # TODO: Remove + RNS.log(str(data)) + + transient_ids = data + wanted_ids = [] + + for transient_id in transient_ids: + if not transient_id in self.propagation_entries: + wanted_ids.append(transient_id) + + if len(wanted_ids) == 0: + return False + + elif len(wanted_ids) == len(transient_ids): + return True + + else: + return wanted_ids + + except Exception as e: + RNS.log("Error occurred while generating response for sync request, the contained exception was: "+str(e), RNS.LOG_DEBUG) + return None + + def propagation_link_established(self, link): + link.set_resource_strategy(RNS.Link.ACCEPT_ALL) + link.set_resource_started_callback(self.resource_transfer_began) + link.set_resource_concluded_callback(self.propagation_resource_concluded) + + + def propagation_resource_concluded(self, resource): + RNS.log("Transfer concluded for delivery resource "+str(resource), RNS.LOG_DEBUG) + if resource.status == RNS.Resource.COMPLETE: + # TODO: The peer this was received from should + # have the transient id added to it's list of + # already handled messages. + self.lxmf_propagation(resource.data.read()) + + + def lxmf_propagation(self, lxmf_data): + try: + if len(lxmf_data) >= LXMessage.LXMF_OVERHEAD: + transient_id = RNS.Identity.full_hash(lxmf_data) + if not transient_id in self.propagation_entries and not transient_id in self.propagated_ids: + received = time.time() + propagation_entry = [transient_id, received, lxmf_data] + + self.propagation_entries[transient_id] = propagation_entry + + for peer_id in self.peers: + peer = self.peers[peer_id] + peer.handle_message(transient_id) + + return True + + return False + + except Exception as e: + RNS.log("Could not assemble propagated LXMF message from received data", RNS.LOG_NOTICE) + RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + return False + + + def peer(self, destination_hash, timestamp): + if destination_hash in self.peers: + peer = self.peers[destination_hash] + peer.alive = True + peer.peering_timebase = timestamp + peer.last_heard = time.time() + else: + peer = LXMPeer(self, destination_hash) + self.peers[destination_hash] = peer + RNS.log(str(self.propagation_destination)+" peered with "+str(peer.destination)) + + def unpeer(self, destination_hash, timestamp = None): + if timestamp == None: + timestamp = int(time.time()) + + if destination_hash in self.peers: + peer = self.peers[destination_hash] + + if timestamp >= peer.peering_timebase: + self.peers.pop(destination_hash) + RNS.log(str(self.propagation_destination)+" broke peering with "+str(peer.destination)) + + def jobloop(self): while (True): # TODO: Improve this to scheduling, so manual @@ -606,9 +918,17 @@ class LXMRouter: self.jobs() time.sleep(LXMRouter.PROCESSING_INTERVAL) + JOB_OUTBOUND_INTERVAL = 1 + JOB_LINKS_INTERVAL = 1 def jobs(self): - self.process_outbound() - self.clean_links() + if self.processing_count % LXMRouter.JOB_OUTBOUND_INTERVAL == 0: + self.process_outbound() + + if self.processing_count % LXMRouter.JOB_LINKS_INTERVAL == 0: + self.clean_links() + + self.processing_count += 1 + def clean_links(self): closed_links = [] From 734305e9c7a0d20b68ab35000db46200131a6f6e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 2 Oct 2021 23:30:38 +0200 Subject: [PATCH 042/436] General structure for LXMF propagation --- LXMF/LXMF.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 75b77e9..9002b7e 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -499,7 +499,7 @@ class LXMFPropagationAnnounceHandler: class LXMPeer: - OFFER_REQUEST_PATH = "o" + OFFER_REQUEST_PATH = "/offer" IDLE = 0x00 LINK_ESTABLISHING = 0x01 @@ -861,7 +861,13 @@ class LXMRouter: # TODO: The peer this was received from should # have the transient id added to it's list of # already handled messages. - self.lxmf_propagation(resource.data.read()) + try: + messages = msgpack.unpackb(resource.data.read()) + for message in messages: + self.lxmf_propagation(lxmessage) + + except Exception as e: + RNS.log("Error while unpacking received propagation messages", RNS.LOG_DEBUG) def lxmf_propagation(self, lxmf_data): From 093d4f1317a920bfdb3f98cc4586e801af25e970 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 3 Oct 2021 10:15:29 +0200 Subject: [PATCH 043/436] Implemented propagation node auto reverse peering --- LXMF/LXMF.py | 41 ++++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 9002b7e..cccad31 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -547,6 +547,9 @@ class LXMPeer: # TODO: Remove RNS.log("Sending sync request to peer") self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) + else: + # TODO: Remove + RNS.log("No unsynced messages") def request_failed(self, request_receipt): RNS.log("Sync request to peer "+str(self.destination)+" failed", RNS.LOG_DEBUG) @@ -577,8 +580,8 @@ class LXMPeer: RNS.log("Peer had all advertised messages", RNS.LOG_DEBUG) for transient_id in self.unhandled_messages: - message = self.unhandled_messages[transient_id] - self.handled_messages[transient_id] = message + message_entry = self.unhandled_messages[transient_id] + self.handled_messages[transient_id] = message_entry self.unhandled_messages = {} @@ -594,8 +597,8 @@ class LXMPeer: # If the peer did not want the message, it has # already received it from another peer. if not transient_id in response: - message = self.unhandled_messages.pop(transient_id) - self.handled_messages[transient_id] = message + message_entry = self.unhandled_messages.pop(transient_id) + self.handled_messages[transient_id] = message_entry for transient_id in response: wanted_messages.append(self.unhandled_messages[transient_id][2]) @@ -605,7 +608,7 @@ class LXMPeer: # TODO: Remove RNS.log("Peer wanted: "+str(wanted_messages), RNS.LOG_DEBUG) - data = msgpack.packb(wanted_messages) + data = msgpack.packb([time.time(), wanted_messages]) resource = RNS.Resource(data, self.link, callback = self.resource_concluded) resource.transferred_messages = wanted_message_ids self.state = LXMPeer.RESOURCE_TRANSFERRING @@ -856,15 +859,32 @@ class LXMRouter: def propagation_resource_concluded(self, resource): - RNS.log("Transfer concluded for delivery resource "+str(resource), RNS.LOG_DEBUG) + RNS.log("Transfer concluded for propagation resource "+str(resource), RNS.LOG_DEBUG) if resource.status == RNS.Resource.COMPLETE: # TODO: The peer this was received from should # have the transient id added to it's list of # already handled messages. try: - messages = msgpack.unpackb(resource.data.read()) - for message in messages: - self.lxmf_propagation(lxmessage) + data = msgpack.unpackb(resource.data.read()) + remote_timebase = data[0] + remote_hash = None + remote_identity = resource.link.get_remote_identity() + + if remote_identity != None: + remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + remote_hash = remote_destination.hash + + if not remote_hash in self.peers: + self.peer(remote_hash, remote_timebase) + + messages = data[1] + for lxmf_data in messages: + if remote_hash in self.peers: + transient_id = RNS.Identity.full_hash(lxmf_data) + peer = self.peers[remote_hash] + peer.handled_messages[transient_id] = [transient_id, remote_timebase, lxmf_data] + + self.lxmf_propagation(lxmf_data) except Exception as e: RNS.log("Error while unpacking received propagation messages", RNS.LOG_DEBUG) @@ -880,6 +900,9 @@ class LXMRouter: self.propagation_entries[transient_id] = propagation_entry + # TODO: Remove? + RNS.log("Received propagated LXMF message "+RNS.prettyhexrep(transient_id)+", adding to peer distribution queues...", RNS.LOG_DEBUG) + for peer_id in self.peers: peer = self.peers[peer_id] peer.handle_message(transient_id) From 354e1932dc0e1f3a97b7d70e05d634170b283c22 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 3 Oct 2021 10:24:24 +0200 Subject: [PATCH 044/436] Sync sequence set up --- LXMF/LXMF.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index cccad31..c69a321 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -547,6 +547,7 @@ class LXMPeer: # TODO: Remove RNS.log("Sending sync request to peer") self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) + self.state = LXMPeer.REQUEST_SENT else: # TODO: Remove RNS.log("No unsynced messages") @@ -570,8 +571,10 @@ class LXMPeer: if response == LXMPeer.ERROR_NO_IDENTITY: if self.link != None: + RNS.log("Remote peer indicated that no identification was received, retrying...", RNS.LOG_DEBUG) self.link.indentify() self.state = LXMPeer.LINK_READY + self.sync() elif response == False: # Peer already has all advertised messages @@ -631,6 +634,7 @@ class LXMPeer: def link_established(self, link): self.link.identify(self.router.identity) self.state = LXMPeer.LINK_READY + self.sync() def link_closed(self, link): self.link = None From 52dc086a26faa2f1e9476b69ad67a6517ddf022b Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 3 Oct 2021 18:49:40 +0200 Subject: [PATCH 045/436] Implemented propagation fundamentals --- LXMF/LXMF.py | 237 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 188 insertions(+), 49 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index c69a321..7502fb6 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -197,6 +197,8 @@ class LXMessage: if self.timestamp == None: self.timestamp = time.time() + self.propagation_packed = None + self.payload = [self.timestamp, self.title, self.content, self.fields] hashed_part = b"" @@ -240,7 +242,7 @@ class LXMessage: self.representation = LXMessage.PACKET self.__delivery_destination = self.__destination - elif self.desired_method == LXMessage.DIRECT or self.desired_method == LXMessage.PROPAGATED: + elif self.desired_method == LXMessage.DIRECT: single_packet_content_limit = LXMessage.LINK_PACKET_MAX_CONTENT if content_size <= single_packet_content_limit: self.method = self.desired_method @@ -248,6 +250,21 @@ class LXMessage: else: self.method = self.desired_method self.representation = LXMessage.RESOURCE + + elif self.desired_method == LXMessage.PROPAGATED: + single_packet_content_limit = LXMessage.LINK_PACKET_MAX_CONTENT + + encrypted_data = self.__destination.encrypt(self.packed[LXMessage.DESTINATION_LENGTH:]) + self.propagation_packed = msgpack.packb([time.time(), [self.packed[:LXMessage.DESTINATION_LENGTH]+encrypted_data]]) + + content_size = len(self.propagation_packed) + if content_size <= single_packet_content_limit: + self.method = self.desired_method + self.representation = LXMessage.PACKET + else: + self.method = self.desired_method + self.representation = LXMessage.RESOURCE + else: raise ValueError("Attempt to re-pack LXMessage "+str(self)+" that was already packed") @@ -270,12 +287,20 @@ class LXMessage: self.resource_representation = self.__as_resource() elif self.method == LXMessage.PROPAGATED: - # TODO: Implement propagation - pass + self.state = LXMessage.SENDING + + if self.representation == LXMessage.PACKET: + receipt = self.__as_packet().send() + receipt.set_delivery_callback(self.__mark_propagated) + receipt.set_timeout_callback(self.__link_packet_timed_out) + + elif self.representation == LXMessage.RESOURCE: + self.resource_representation = self.__as_resource() + def determine_transport_encryption(self): if self.method == LXMessage.OPPORTUNISTIC: - if self.destination.type == RNS.Destination.SINGLE: + if self.__destination.type == RNS.Destination.SINGLE: self.transport_encrypted = True self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC elif destination_type == RNS.Destination.GROUP: @@ -288,10 +313,10 @@ class LXMessage: self.transport_encrypted = True self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC elif self.method == LXMessage.PROPAGATED: - if self.destination.type == RNS.Destination.SINGLE: + if self.__destination.type == RNS.Destination.SINGLE: self.transport_encrypted = True self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC - elif destination_type == RNS.Destination.GROUP: + elif self.__destination.type == RNS.Destination.GROUP: self.transport_encrypted = True self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES else: @@ -308,6 +333,13 @@ class LXMessage: if self.__delivery_callback != None: self.__delivery_callback(self) + def __mark_propagated(self, receipt = None): + RNS.log("Received propagation success notification for "+str(self), RNS.LOG_DEBUG) + self.state = LXMessage.SENT + + if self.__delivery_callback != None: + self.__delivery_callback(self) + def __resource_concluded(self, resource): if resource.status == RNS.Resource.COMPLETE: self.__mark_delivered() @@ -315,6 +347,13 @@ class LXMessage: resource.link.teardown() self.state = LXMessage.OUTBOUND + def __propagation_resource_concluded(self, resource): + if resource.status == RNS.Resource.COMPLETE: + self.__mark_propagated() + else: + resource.link.teardown() + self.state = LXMessage.OUTBOUND + def __link_packet_timed_out(self, packet_receipt): packet_receipt.destination.teardown() self.state = LXMessage.OUTBOUND @@ -332,8 +371,10 @@ class LXMessage: if self.method == LXMessage.OPPORTUNISTIC: return RNS.Packet(self.__delivery_destination, self.packed[LXMessage.DESTINATION_LENGTH:]) - elif self.method == LXMessage.DIRECT or self.method == LXMessage.PROPAGATED: + elif self.method == LXMessage.DIRECT: return RNS.Packet(self.__delivery_destination, self.packed) + elif self.method == LXMessage.PROPAGATED: + return RNS.Packet(self.__delivery_destination, self.propagation_packed) def __as_resource(self): if not self.packed: @@ -349,7 +390,13 @@ class LXMessage: raise ConnectionError("Tried to synthesize resource for LXMF message on a link that was not active") self.progress = 0.0 - return RNS.Resource(self.packed, self.__delivery_destination, callback = self.__resource_concluded, progress_callback = self.__update_transfer_progress) + + if self.method == LXMessage.DIRECT: + return RNS.Resource(self.packed, self.__delivery_destination, callback = self.__resource_concluded, progress_callback = self.__update_transfer_progress) + elif self.method == LXMessage.PROPAGATED: + return RNS.Resource(self.propagation_packed, self.__delivery_destination, callback = self.__propagation_resource_concluded, progress_callback = self.__update_transfer_progress) + else: + return None def write_to_directory(self, directory_path): file_name = RNS.hexrep(self.hash, delimit=False) @@ -484,14 +531,15 @@ class LXMFPropagationAnnounceHandler: def received_announce(self, destination_hash, announced_identity, app_data): try: - data = msgpack.unpackb(app_data) + if type(app_data) == bytes: + data = msgpack.unpackb(app_data) - if self.lxmrouter.autopeer: - if data[0] == True: - if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: - self.lxmrouter.peer(destination_hash, data[1]) - elif data[0] == False: - self.lxmrouter.unpeer(destination_hash, data[1]) + if self.lxmrouter.propagation_node and self.lxmrouter.autopeer: + if data[0] == True: + if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: + self.lxmrouter.peer(destination_hash, data[1]) + elif data[0] == False: + self.lxmrouter.unpeer(destination_hash, data[1]) except Exception as e: RNS.log("Error while evaluating propagation node announce, ignoring announce.", RNS.LOG_DEBUG) @@ -655,17 +703,6 @@ class LXMRouter: AUTOPEER = True AUTOPEER_MAXDEPTH = 4 - # TODO: Remove - def ptest(self): - i1 = RNS.Identity() - i2 = RNS.Identity() - d1 = RNS.Destination(i1, RNS.Destination.IN, RNS.Destination.SINGLE, "test", "delivery") - d2 = RNS.Destination(i2, RNS.Destination.IN, RNS.Destination.SINGLE, "test", "delivery") - - lxm = LXMessage(d1, d2, content="No content here") - - return lxm - def __init__(self, identity = None, autopeer = AUTOPEER, autopeer_maxdepth = AUTOPEER_MAXDEPTH): self.pending_inbound = [] self.pending_outbound = [] @@ -679,6 +716,9 @@ class LXMRouter: self.propagation_node = False + self.outbound_propagation_node = None + self.outbound_propagation_link = None + if identity == None: identity = RNS.Identity() @@ -717,6 +757,19 @@ class LXMRouter: def register_delivery_callback(self, callback): self.__delivery_callback = callback + def set_outbound_propagation_node(self, destination_hash): + if len(destination_hash) != RNS.Identity.TRUNCATED_HASHLENGTH//8 or type(destination_hash) != bytes: + raise ValueError("Invalid destination hash for outbound propagation node") + else: + self.outbound_propagation_node = destination_hash + if self.outbound_propagation_link != None: + if self.outbound_propagation_link.destination.hash != destination_hash: + self.outbound_propagation_link.teardown() + self.outbound_propagation_link = None + + def get_outbound_propagation_node(self): + return self.outbound_propagation_node + def announce(self, destination_hash): if destination_hash in self.delivery_destinations: delivery_destination = self.delivery_destinations[destination_hash] @@ -808,6 +861,7 @@ class LXMRouter: self.propagation_node = True self.propagation_destination.set_link_established_callback(self.propagation_link_established) + self.propagation_destination.set_packet_callback(self.propagation_packet) self.propagation_destination.register_request_handler(LXMPeer.OFFER_REQUEST_PATH, self.offer_request, allow = RNS.Destination.ALLOW_ALL) @@ -857,11 +911,31 @@ class LXMRouter: return None def propagation_link_established(self, link): + link.set_packet_callback(self.propagation_packet) link.set_resource_strategy(RNS.Link.ACCEPT_ALL) link.set_resource_started_callback(self.resource_transfer_began) link.set_resource_concluded_callback(self.propagation_resource_concluded) + def propagation_packet(self, data, packet): + try: + if packet.destination_type != RNS.Destination.LINK: + pass + else: + data = msgpack.unpackb(data) + remote_timebase = data[0] + + messages = data[1] + for lxmf_data in messages: + self.lxmf_propagation(lxmf_data) + + packet.prove() + + except Exception as e: + RNS.log("Exception occurred while parsing incoming LXMF propagation data.", RNS.LOG_ERROR) + RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) + + def propagation_resource_concluded(self, resource): RNS.log("Transfer concluded for propagation resource "+str(resource), RNS.LOG_DEBUG) if resource.status == RNS.Resource.COMPLETE: @@ -870,28 +944,33 @@ class LXMRouter: # already handled messages. try: data = msgpack.unpackb(resource.data.read()) - remote_timebase = data[0] - remote_hash = None - remote_identity = resource.link.get_remote_identity() - if remote_identity != None: - remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") - remote_hash = remote_destination.hash + if type(data) == list and len(data) == 2 and type(data[0] == float) and type(data[1]) == list: + # This is a series of propagation messages from a peer or originator + remote_timebase = data[0] + remote_hash = None + remote_identity = resource.link.get_remote_identity() - if not remote_hash in self.peers: - self.peer(remote_hash, remote_timebase) + if remote_identity != None: + remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + remote_hash = remote_destination.hash - messages = data[1] - for lxmf_data in messages: - if remote_hash in self.peers: - transient_id = RNS.Identity.full_hash(lxmf_data) - peer = self.peers[remote_hash] - peer.handled_messages[transient_id] = [transient_id, remote_timebase, lxmf_data] + if not remote_hash in self.peers: + self.peer(remote_hash, remote_timebase) - self.lxmf_propagation(lxmf_data) + messages = data[1] + for lxmf_data in messages: + if remote_hash != None and remote_hash in self.peers: + transient_id = RNS.Identity.full_hash(lxmf_data) + peer = self.peers[remote_hash] + peer.handled_messages[transient_id] = [transient_id, remote_timebase, lxmf_data] + + self.lxmf_propagation(lxmf_data) + else: + RNS.log("Invalid data structure received at propagation destination, ignoring", RNS.LOG_DEBUG) except Exception as e: - RNS.log("Error while unpacking received propagation messages", RNS.LOG_DEBUG) + RNS.log("Error while unpacking received propagation resource", RNS.LOG_DEBUG) def lxmf_propagation(self, lxmf_data): @@ -901,15 +980,23 @@ class LXMRouter: if not transient_id in self.propagation_entries and not transient_id in self.propagated_ids: received = time.time() propagation_entry = [transient_id, received, lxmf_data] + destination_hash = lxmf_data[:LXMessage.DESTINATION_LENGTH] - self.propagation_entries[transient_id] = propagation_entry + if destination_hash in self.delivery_destinations: + delivery_destination = self.delivery_destinations[destination_hash] + encrypted_lxmf_data = lxmf_data[LXMessage.DESTINATION_LENGTH:] + decrypted_lxmf_data = delivery_destination.decrypt(encrypted_lxmf_data) + delivery_data = lxmf_data[:LXMessage.DESTINATION_LENGTH]+decrypted_lxmf_data + self.lxmf_delivery(delivery_data, delivery_destination.type) - # TODO: Remove? - RNS.log("Received propagated LXMF message "+RNS.prettyhexrep(transient_id)+", adding to peer distribution queues...", RNS.LOG_DEBUG) + else: + if self.propagation_node: + self.propagation_entries[transient_id] = propagation_entry - for peer_id in self.peers: - peer = self.peers[peer_id] - peer.handle_message(transient_id) + RNS.log("Received propagated LXMF message "+RNS.prettyhexrep(transient_id)+", adding to peer distribution queues...", RNS.LOG_DEBUG) + for peer_id in self.peers: + peer = self.peers[peer_id] + peer.handle_message(transient_id) return True @@ -995,6 +1082,9 @@ class LXMRouter: if lxmessage.state == LXMessage.DELIVERED: RNS.log("Delivery has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) self.pending_outbound.remove(lxmessage) + elif lxmessage.state == LXMessage.SENT: + RNS.log("Propagation has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) + self.pending_outbound.remove(lxmessage) else: RNS.log("Starting outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) # Outbound handling for opportunistic messages @@ -1064,4 +1154,53 @@ class LXMRouter: # propagation to a LXMF router network. elif lxmessage.method == LXMessage.PROPAGATED: RNS.log("Attempting propagated delivery for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - raise NotImplementedError("LXMF propagation is not implemented yet") \ No newline at end of file + + if self.outbound_propagation_node == None: + RNS.log("No outbound propagation node specified for propagated "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_ERROR) + self.fail_message(lxmessage) + else: + if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: + + if self.outbound_propagation_link != None: + # A link already exists, so we'll try to use it + # to deliver the message + if self.outbound_propagation_link.status == RNS.Link.ACTIVE: + if lxmessage.state != LXMessage.SENDING: + RNS.log("Starting propagation transfer of "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" via "+RNS.prettyhexrep(self.outbound_propagation_node), RNS.LOG_DEBUG) + lxmessage.set_delivery_destination(self.outbound_propagation_link) + lxmessage.send() + else: + if lxmessage.representation == LXMessage.RESOURCE: + RNS.log("The transfer of "+str(lxmessage)+" is in progress ("+str(round(lxmessage.progress*100, 1))+"%)", RNS.LOG_DEBUG) + else: + RNS.log("Waiting for proof for "+str(lxmessage)+" sent as link packet", RNS.LOG_DEBUG) + elif self.outbound_propagation_link.status == RNS.Link.CLOSED: + RNS.log("The link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" was closed", RNS.LOG_DEBUG) + self.outbound_propagation_link = None + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + else: + # Simply wait for the link to become + # active or close + RNS.log("The propagation link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" is pending, waiting for link to become active: "+str(self.outbound_propagation_link.status), RNS.LOG_DEBUG) + else: + # No link exists, so we'll try to establish one, but + # only if we've never tried before, or the retry wait + # period has elapsed. + if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: + lxmessage.delivery_attempts += 1 + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + + if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: + if RNS.Transport.has_path(self.outbound_propagation_node): + RNS.log("Establishing link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" for propagation attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + propagation_node_identity = RNS.Identity.recall(self.outbound_propagation_node) + propagation_node_destination = RNS.Destination(propagation_node_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + self.outbound_propagation_link = RNS.Link(propagation_node_destination, established_callback=self.process_outbound) + else: + RNS.log("No path known for propagation attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(self.outbound_propagation_node)+". Requesting path...", RNS.LOG_DEBUG) + RNS.Transport.request_path(self.outbound_propagation_node) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + + else: + RNS.log("Max delivery attempts reached for propagated "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + self.fail_message(lxmessage) \ No newline at end of file From f5cb49b46b757bbfd56b82f56b3f9e512ff44b9e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 3 Oct 2021 21:00:28 +0200 Subject: [PATCH 046/436] Propagation node state save and restore --- LXMF/LXMF.py | 132 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 120 insertions(+), 12 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 7502fb6..8d43e31 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -1,6 +1,7 @@ import os import math import time +import atexit import threading import RNS import RNS.vendor.umsgpack as msgpack @@ -291,8 +292,11 @@ class LXMessage: if self.representation == LXMessage.PACKET: receipt = self.__as_packet().send() - receipt.set_delivery_callback(self.__mark_propagated) - receipt.set_timeout_callback(self.__link_packet_timed_out) + if receipt: + receipt.set_delivery_callback(self.__mark_propagated) + receipt.set_timeout_callback(self.__link_packet_timed_out) + else: + self.__delivery_destination.teardown() elif self.representation == LXMessage.RESOURCE: self.resource_representation = self.__as_resource() @@ -355,7 +359,9 @@ class LXMessage: self.state = LXMessage.OUTBOUND def __link_packet_timed_out(self, packet_receipt): - packet_receipt.destination.teardown() + if packet_receipt: + packet_receipt.destination.teardown() + self.state = LXMessage.OUTBOUND @@ -559,11 +565,43 @@ class LXMPeer: ERROR_NO_IDENTITY = 0xf0 @staticmethod - def from_bytes(bytes): - pass + def from_bytes(peer_bytes, router): + dictionary = msgpack.unpackb(peer_bytes) + + peer = LXMPeer(router, dictionary["destination_hash"]) + peer.peering_timebase = dictionary["peering_timebase"] + peer.alive = dictionary["alive"] + peer.last_heard = dictionary["last_heard"] + + for transient_id in dictionary["handled_ids"]: + if transient_id in router.propagation_entries: + peer.handled_messages[transient_id] = router.propagation_entries[transient_id] + + for transient_id in dictionary["unhandled_ids"]: + if transient_id in router.propagation_entries: + peer.unhandled_messages[transient_id] = router.propagation_entries[transient_id] + + return peer def to_bytes(self): - pass + dictionary = {} + dictionary["peering_timebase"] = self.peering_timebase + dictionary["alive"] = self.alive + dictionary["last_heard"] = self.last_heard + dictionary["destination_hash"] = self.destination_hash + + handled_ids = [] + for transient_id in self.handled_messages: + handled_ids.append(transient_id) + + unhandled_ids = [] + for transient_id in self.unhandled_messages: + unhandled_ids.append(transient_id) + + dictionary["handled_ids"] = handled_ids + dictionary["unhandled_ids"] = unhandled_ids + + return msgpack.packb(dictionary) def __init__(self, router, destination_hash): self.alive = False @@ -639,7 +677,7 @@ class LXMPeer: elif response == True: # Peer wants all advertised messages for transient_id in self.unhandled_messages: - wanted_messages.append(self.unhandled_messages[transient_id][2]) + wanted_messages.append(self.unhandled_messages[transient_id]) wanted_message_ids.append(transient_id) else: @@ -652,14 +690,24 @@ class LXMPeer: self.handled_messages[transient_id] = message_entry for transient_id in response: - wanted_messages.append(self.unhandled_messages[transient_id][2]) + wanted_messages.append(self.unhandled_messages[transient_id]) wanted_message_ids.append(transient_id) if len(wanted_messages) > 0: # TODO: Remove RNS.log("Peer wanted: "+str(wanted_messages), RNS.LOG_DEBUG) - data = msgpack.packb([time.time(), wanted_messages]) + lxm_list = [] + + for file_path in wanted_messages: + # TODO: Remove + RNS.log("Loading "+str(file_path)+" for transfer") + file = open(file_path, "rb") + lxmf_data = file.read() + file.close() + lxm_list.append(lxmf_data) + + data = msgpack.packb([time.time(), lxm_list]) resource = RNS.Resource(data, self.link, callback = self.resource_concluded) resource.transferred_messages = wanted_message_ids self.state = LXMPeer.RESOURCE_TRANSFERRING @@ -738,10 +786,33 @@ class LXMRouter: self.__delivery_callback = None + atexit.register(self.exit_handler) + job_thread = threading.Thread(target=self.jobloop) job_thread.setDaemon(True) job_thread.start() + def exit_handler(self): + try: + serialised_peers = [] + for peer_id in self.peers: + peer = self.peers[peer_id] + serialised_peers.append(peer.to_bytes()) + # TODO: Remove + RNS.log("Saving peer "+str(peer)) + + peers_file = open(self.storagepath+"/peers", "wb") + peers_file.write(msgpack.packb(serialised_peers)) + peers_file.close() + + # TODO: Remove + RNS.log("Saved peers") + + + except Exception as e: + RNS.log("Could not save propagation node peers to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + + def register_delivery_identity(self, identity, display_name = None): delivery_destination = RNS.Destination(identity, RNS.Destination.IN, RNS.Destination.SINGLE, "lxmf", "delivery") delivery_destination.set_packet_callback(self.delivery_packet) @@ -853,11 +924,43 @@ class LXMRouter: def enable_propagation(self, storagepath): try: self.storagepath = storagepath+"/lxmf" + self.messagepath = self.storagepath+"/messagestore" if not os.path.isdir(self.storagepath): os.makedirs(self.storagepath) - # TODO: Load peers and data + if not os.path.isdir(self.messagepath): + os.makedirs(self.messagepath) + + self.propagation_entries = {} + for filename in os.listdir(self.messagepath): + components = filename.split("_") + if len(components) == 2: + if float(components[1]) > 0: + if len(components[0]) == RNS.Identity.HASHLENGTH//8*2: + transient_id = bytes.fromhex(components[0]) + received = components[1] + + filepath = self.messagepath+"/"+filename + file = open(filepath, "rb") + destination_hash = file.read(LXMessage.DESTINATION_LENGTH) + file.close() + + self.propagation_entries[transient_id] = [destination_hash, filepath] + + # TODO: Remove + RNS.log("Registered msg "+RNS.prettyhexrep(transient_id)+" at "+filepath+" for "+RNS.prettyhexrep(destination_hash)) + + if os.path.isfile(self.storagepath+"/peers"): + peers_file = open(self.storagepath+"/peers", "rb") + serialised_peers = msgpack.unpackb(peers_file.read()) + + for serialised_peer in serialised_peers: + peer = LXMPeer.from_bytes(serialised_peer, self) + self.peers[peer.destination_hash] = peer + # TODO: Remove + RNS.log("Loaded peer "+RNS.prettyhexrep(peer.destination_hash)) + self.propagation_node = True self.propagation_destination.set_link_established_callback(self.propagation_link_established) @@ -991,7 +1094,12 @@ class LXMRouter: else: if self.propagation_node: - self.propagation_entries[transient_id] = propagation_entry + file_path = self.messagepath+"/"+RNS.hexrep(transient_id, delimit=False)+"_"+str(received) + msg_file = open(file_path, "wb") + msg_file.write(lxmf_data) + msg_file.close() + + self.propagation_entries[transient_id] = [destination_hash, file_path] RNS.log("Received propagated LXMF message "+RNS.prettyhexrep(transient_id)+", adding to peer distribution queues...", RNS.LOG_DEBUG) for peer_id in self.peers: @@ -1003,7 +1111,7 @@ class LXMRouter: return False except Exception as e: - RNS.log("Could not assemble propagated LXMF message from received data", RNS.LOG_NOTICE) + RNS.log("Could not assemble propagated LXMF message from received data", RNS.LOG_DEBUG) RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) return False From b675e1c47eb08b1fc6d967c0b89ffcb30abfe44d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 7 Oct 2021 18:12:54 +0200 Subject: [PATCH 047/436] Implemented LXMF propagation client request handlers --- LXMF/LXMF.py | 287 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 280 insertions(+), 7 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 8d43e31..5af55e1 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -1,6 +1,7 @@ import os import math import time +import random import atexit import threading import RNS @@ -554,6 +555,7 @@ class LXMFPropagationAnnounceHandler: class LXMPeer: OFFER_REQUEST_PATH = "/offer" + MESSAGE_GET_PATH = "/get" IDLE = 0x00 LINK_ESTABLISHING = 0x01 @@ -699,7 +701,8 @@ class LXMPeer: lxm_list = [] - for file_path in wanted_messages: + for message_entry in wanted_messages: + file_path = message_entry[1] # TODO: Remove RNS.log("Loading "+str(file_path)+" for transfer") file = open(file_path, "rb") @@ -737,8 +740,13 @@ class LXMPeer: self.state = LXMPeer.IDLE def handle_message(self, transient_id): - if not transient_id in self.handled_messages and not self.unhandled_messages: + if not transient_id in self.handled_messages and not transient_id in self.unhandled_messages: + # TODO: Remove + RNS.log("The message "+RNS.prettyhexrep(transient_id)+" was added to distribution queue for "+RNS.prettyhexrep(self.destination_hash)) self.unhandled_messages[transient_id] = self.router.propagation_entries[transient_id] + else: + # TODO: Remove + RNS.log("The message "+RNS.prettyhexrep(transient_id)+" was NOT added to distribution queue for "+RNS.prettyhexrep(self.destination_hash)) class LXMRouter: @@ -751,7 +759,21 @@ class LXMRouter: AUTOPEER = True AUTOPEER_MAXDEPTH = 4 + PR_PATH_TIMEOUT = 10 + + PR_IDLE = 0x00 + PR_PATH_REQUESTED = 0x01 + PR_LINK_ESTABLISHING = 0x02 + PR_LINK_ESTABLISHED = 0x03 + PR_REQUEST_SENT = 0x04 + PR_RESPONSE_RECEIVED = 0x05 + PR_COMPLETE = 0x06 + + PR_ALL_MESSAGES = 0x00 + def __init__(self, identity = None, autopeer = AUTOPEER, autopeer_maxdepth = AUTOPEER_MAXDEPTH): + random.seed(os.urandom(10)) + self.pending_inbound = [] self.pending_outbound = [] self.failed_outbound = [] @@ -767,6 +789,12 @@ class LXMRouter: self.outbound_propagation_node = None self.outbound_propagation_link = None + self.wants_download_on_path_available_from = None + self.wants_download_on_path_available_to = None + self.propagation_transfer_state = LXMRouter.PR_IDLE + self.propagation_transfer_progress = 0.0 + self.propagation_transfer_last_result = None + if identity == None: identity = RNS.Identity() @@ -841,6 +869,140 @@ class LXMRouter: def get_outbound_propagation_node(self): return self.outbound_propagation_node + def cancel_propagation_node_requests(self): + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + self.outbound_propagation_link = None + + self.acknowledge_sync_completion() + + def acknowledge_sync_completion(self): + self.propagation_transfer_state = LXMRouter.PR_IDLE + self.propagation_transfer_progress = 0.0 + self.propagation_transfer_last_result = None + self.wants_download_on_path_available_from = None + self.wants_download_on_path_available_to = None + + def request_messages_from_propagation_node(self, identity, max_messages = PR_ALL_MESSAGES): + self.propagation_transfer_progress = 0.0 + if self.outbound_propagation_link != None and self.outbound_propagation_link.status == RNS.Link.ACTIVE: + self.propagation_transfer_state = LXMRouter.PR_LINK_ESTABLISHED + self.outbound_propagation_link.identify(identity) + self.outbound_propagation_link.request( + LXMPeer.MESSAGE_GET_PATH, + [None, None], + response_callback=self.message_list_response, + failed_callback=self.message_get_failed + ) + else: + if self.outbound_propagation_link == None: + if RNS.Transport.has_path(self.outbound_propagation_node): + self.wants_download_on_path_available_from = None + self.propagation_transfer_state = LXMRouter.PR_LINK_ESTABLISHING + RNS.log("Establishing link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" for message download", RNS.LOG_DEBUG) + propagation_node_identity = RNS.Identity.recall(self.outbound_propagation_node) + propagation_node_destination = RNS.Destination(propagation_node_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + def msg_request_established_callback(link): + self.request_messages_from_propagation_node(identity) + + self.outbound_propagation_link = RNS.Link(propagation_node_destination, established_callback=msg_request_established_callback) + else: + RNS.log("No path known for message download from propagation node "+RNS.prettyhexrep(self.outbound_propagation_node)+". Requesting path...", RNS.LOG_DEBUG) + RNS.Transport.request_path(self.outbound_propagation_node) + self.wants_download_on_path_available_from = self.outbound_propagation_node + self.wants_download_on_path_available_to = identity + self.wants_download_on_path_available_timeout = time.time() + LXMRouter.PR_PATH_TIMEOUT + self.propagation_transfer_state = LXMRouter.PR_PATH_REQUESTED + self.request_messages_path_job() + else: + # TODO: Remove + RNS.log("Waiting for propagation node link to become active", RNS.LOG_DEBUG) + + + def request_messages_path_job(self): + job_thread = threading.Thread(target=self.__request_messages_path_job) + job_thread.setDaemon(True) + job_thread.start() + + def __request_messages_path_job(self): + while not RNS.Transport.has_path(self.wants_download_on_path_available_from) and time.time() < self.wants_download_on_path_available_timeout: + time.sleep(0.1) + + if RNS.Transport.has_path(self.wants_download_on_path_available_from): + self.request_messages_from_propagation_node(self.wants_download_on_path_available_to) + else: + RNS.log("Propagation node path request timed out", RNS.LOG_DEBUG) + self.acknowledge_sync_completion() + + + def has_message(self, transient_id): + return False + + def message_get_failed(self, request_receipt): + # TODO: Remove or change + RNS.log("Message list/get request failed", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + + def message_list_response(self, request_receipt): + # TODO: Remove + RNS.log("Got message list response from propagation node") + if request_receipt.response == LXMPeer.ERROR_NO_IDENTITY: + RNS.log("Propagation node indicated missing identification on reuquest", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + else: + # TODO: Remove + if request_receipt.response != None: + RNS.log("Received message list from node:", RNS.LOG_DEBUG) + haves = [] + wants = [] + if len(request_receipt.response) > 0: + for transient_id in request_receipt.response: + RNS.log(" "+RNS.prettyhexrep(transient_id), RNS.LOG_DEBUG) + if self.has_message(transient_id): + haves.append(transient_id) + else: + wants.append(transient_id) + + request_receipt.link.request( + LXMPeer.MESSAGE_GET_PATH, + [wants, haves], + response_callback=self.message_get_response, + failed_callback=self.message_get_failed, + progress_callback=self.message_get_progress + ) + else: + # TODO: Remove + RNS.log("No messages on node", RNS.LOG_DEBUG) + self.propagation_transfer_state = LXMRouter.PR_COMPLETE + self.propagation_transfer_progress = 1.0 + self.propagation_transfer_last_result = 0 + + def message_get_progress(self, request_receipt): + self.propagation_transfer_progress = request_receipt.get_progress() + RNS.log("Progress: "+str(self.propagation_transfer_progress)) + + def message_get_response(self, request_receipt): + # TODO: Remove + RNS.log("Got message download response from propagation node") + if request_receipt.response == LXMPeer.ERROR_NO_IDENTITY: + RNS.log("Propagation node indicated missing identification on reuquest", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + else: + if request_receipt.response != None and len(request_receipt.response) > 0: + for lxmf_data in request_receipt.response: + self.lxmf_propagation(lxmf_data) + else: + RNS.log("No LXMF data received in message download response", RNS.LOG_DEBUG) + + # TODO: Remove + RNS.log("No messages on node", RNS.LOG_DEBUG) + self.propagation_transfer_state = LXMRouter.PR_COMPLETE + self.propagation_transfer_progress = 1.0 + self.propagation_transfer_last_result = len(request_receipt.response) + def announce(self, destination_hash): if destination_hash in self.delivery_destinations: delivery_destination = self.delivery_destinations[destination_hash] @@ -957,9 +1119,14 @@ class LXMRouter: for serialised_peer in serialised_peers: peer = LXMPeer.from_bytes(serialised_peer, self) - self.peers[peer.destination_hash] = peer - # TODO: Remove - RNS.log("Loaded peer "+RNS.prettyhexrep(peer.destination_hash)) + if peer.identity != None: + self.peers[peer.destination_hash] = peer + + # TODO: Remove + RNS.log("Loaded peer "+RNS.prettyhexrep(peer.destination_hash)+" with "+str(len(peer.unhandled_messages))+" unhandled messages") + else: + RNS.log("Peer "+RNS.prettyhexrep(peer.destination_hash)+" could not be loaded, because its identity could not be recalled. Dropping peer.") + self.propagation_node = True @@ -967,6 +1134,7 @@ class LXMRouter: self.propagation_destination.set_packet_callback(self.propagation_packet) self.propagation_destination.register_request_handler(LXMPeer.OFFER_REQUEST_PATH, self.offer_request, allow = RNS.Destination.ALLOW_ALL) + self.propagation_destination.register_request_handler(LXMPeer.MESSAGE_GET_PATH, self.message_get_request, allow = RNS.Destination.ALLOW_ALL) self.announce_propagation_node() @@ -1013,6 +1181,82 @@ class LXMRouter: RNS.log("Error occurred while generating response for sync request, the contained exception was: "+str(e), RNS.LOG_DEBUG) return None + def message_get_request(self, path, data, request_id, remote_identity, requested_at): + # TODO: Remove + RNS.log("Message get request") + + if remote_identity == None: + return LXMPeer.ERROR_NO_IDENTITY + else: + try: + # TODO: Remove + RNS.log(str(data)) + + remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") + + # If both want and have fields are empty, send a list of + # available messages. + if data[0] == None and data[1] == None: + available_messages = [] + for transient_id in self.propagation_entries: + message_entry = self.propagation_entries[transient_id] + if message_entry[0] == remote_destination.hash: + message_size = os.path.getsize(message_entry[1]) + available_entry = [transient_id, message_size] + available_messages.append(available_entry) + + available_messages.sort(key=lambda e: e[1], reverse=False) + + # TODO: Remove + for am in available_messages: + RNS.log("Msg size: "+str(am[1])) + ############## + + transient_ids = [] + for available_entry in available_messages: + transient_ids.append(available_entry[0]) + + return transient_ids + + else: + # Process messages the client already have + if data[1] != None and len(data[1]) > 0: + for transient_id in data[1]: + if transient_id in self.propagation_entries and self.propagation_entries[transient_id][0] == remote_destination.hash: + try: + filepath = self.propagation_entries[transient_id][1] + self.propagation_entries.remove(transient_id) + os.unlink(filepath) + RNS.log("Client "+RNS.prettyhexrep(remote_destination.hash)+" purged message "+RNS.prettyhexrep(transient_id)+" at "+str(filepath), RNS.LOG_DEBUG) + + except Exception as e: + RNS.log("Error while processing message purge request from "+RNS.prettyhexrep(remote_destination.hash)+". The contained exception was: "+str(e), RNS.LOG_ERROR) + + + # Process wanted messages + response_messages = [] + if data[0] != None and len(data[0]) > 0: + for transient_id in data[0]: + if transient_id in self.propagation_entries and self.propagation_entries[transient_id][0] == remote_destination.hash: + try: + filepath = self.propagation_entries[transient_id][1] + RNS.log("Client "+RNS.prettyhexrep(remote_destination.hash)+" requested message "+RNS.prettyhexrep(transient_id)+" at "+str(filepath), RNS.LOG_DEBUG) + + message_file = open(filepath, "rb") + lxmf_data = message_file.read() + response_messages.append(lxmf_data) + message_file.close() + + except Exception as e: + RNS.log("Error while processing message download request from "+RNS.prettyhexrep(remote_destination.hash)+". The contained exception was: "+str(e), RNS.LOG_ERROR) + + return response_messages + + + except Exception as e: + RNS.log("Error occurred while generating response for download request, the contained exception was: "+str(e), RNS.LOG_DEBUG) + return None + def propagation_link_established(self, link): link.set_packet_callback(self.propagation_packet) link.set_resource_strategy(RNS.Link.ACCEPT_ALL) @@ -1040,7 +1284,7 @@ class LXMRouter: def propagation_resource_concluded(self, resource): - RNS.log("Transfer concluded for propagation resource "+str(resource), RNS.LOG_DEBUG) + RNS.log("Transfer concluded for incoming propagation resource "+str(resource), RNS.LOG_DEBUG) if resource.status == RNS.Resource.COMPLETE: # TODO: The peer this was received from should # have the transient id added to it's list of @@ -1124,8 +1368,10 @@ class LXMRouter: peer.last_heard = time.time() else: peer = LXMPeer(self, destination_hash) + peer.alive = True + peer.last_heard = time.time() self.peers[destination_hash] = peer - RNS.log(str(self.propagation_destination)+" peered with "+str(peer.destination)) + RNS.log("Peered with "+str(peer.destination)) def unpeer(self, destination_hash, timestamp = None): if timestamp == None: @@ -1148,6 +1394,7 @@ class LXMRouter: JOB_OUTBOUND_INTERVAL = 1 JOB_LINKS_INTERVAL = 1 + JOB_PEERSYNC_INTERVAL = 12 def jobs(self): if self.processing_count % LXMRouter.JOB_OUTBOUND_INTERVAL == 0: self.process_outbound() @@ -1155,6 +1402,9 @@ class LXMRouter: if self.processing_count % LXMRouter.JOB_LINKS_INTERVAL == 0: self.clean_links() + if self.processing_count % LXMRouter.JOB_PEERSYNC_INTERVAL == 0: + self.sync_peers() + self.processing_count += 1 @@ -1172,6 +1422,29 @@ class LXMRouter: cleaned_link = self.direct_links.pop(link_hash) RNS.log("Cleaned link "+str(cleaned_link), RNS.LOG_DEBUG) + if self.outbound_propagation_link != None and self.outbound_propagation_link.status == RNS.Link.CLOSED: + self.outbound_propagation_link = None + self.acknowledge_sync_completion() + RNS.log("Cleaned outbound propagation link", RNS.LOG_DEBUG) + + + def sync_peers(self): + waiting_peers = [] + for peer_id in self.peers: + peer = self.peers[peer_id] + if peer.state == LXMPeer.IDLE and len(peer.unhandled_messages) > 0: + waiting_peers.append(peer) + + if len(waiting_peers) > 0: + # TODO: Remove + RNS.log("Randomly selecting peer to sync from "+str(len(waiting_peers))+" waiting peers.", RNS.LOG_DEBUG) + selected_index = random.randint(0,len(waiting_peers)-1) + selected_peer = waiting_peers[selected_index] + # TODO: Remove + RNS.log("Selected waiting peer "+str(selected_index)+": "+RNS.prettyhexrep(selected_peer.destination.hash), RNS.LOG_DEBUG) + selected_peer.sync() + + def fail_message(self, lxmessage): RNS.log(str(lxmessage)+" failed to send", RNS.LOG_DEBUG) From 9a75b9c1ecc96c8df051efb95a26842e5b39dabe Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 7 Oct 2021 20:52:53 +0200 Subject: [PATCH 048/436] Implemented LXMF propagation peer sync and client downloads --- LXMF/LXMF.py | 86 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 27 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 5af55e1..69cd36a 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -417,7 +417,8 @@ class LXMessage: "state": self.state, "lxmf_bytes": self.packed, "transport_encrypted": self.transport_encrypted, - "transport_encryption": self.transport_encryption + "transport_encryption": self.transport_encryption, + "method": self.method } packed_container = msgpack.packb(container) @@ -507,6 +508,8 @@ class LXMessage: lxm.transport_encrypted = container["transport_encrypted"] if "transport_encryption" in container: lxm.transport_encryption = container["transport_encryption"] + if "method" in container: + lxm.method = container["method"] return lxm except Exception as e: @@ -644,6 +647,8 @@ class LXMPeer: RNS.log("Sync request to peer "+str(self.destination)+" failed", RNS.LOG_DEBUG) if self.link != None: self.link.teardown() + else: + self.state = LXMPeer.IDLE def offer_response(self, request_receipt): # TODO: Remove @@ -703,17 +708,23 @@ class LXMPeer: for message_entry in wanted_messages: file_path = message_entry[1] - # TODO: Remove - RNS.log("Loading "+str(file_path)+" for transfer") - file = open(file_path, "rb") - lxmf_data = file.read() - file.close() - lxm_list.append(lxmf_data) + if os.path.isfile(file_path): + # TODO: Remove + RNS.log("Loading "+str(file_path)+" for transfer") + file = open(file_path, "rb") + lxmf_data = file.read() + file.close() + lxm_list.append(lxmf_data) + else: + # TODO: Remove + RNS.log("The requested message "+str(file_path)+" does not exist") data = msgpack.packb([time.time(), lxm_list]) resource = RNS.Resource(data, self.link, callback = self.resource_concluded) resource.transferred_messages = wanted_message_ids self.state = LXMPeer.RESOURCE_TRANSFERRING + else: + self.state = LXMPeer.IDLE except Exception as e: RNS.log("Error while handling offer response from peer "+str(self.destination), RNS.LOG_DEBUG) @@ -725,8 +736,13 @@ class LXMPeer: for transient_id in resource.transferred_messages: message = self.unhandled_messages.pop(transient_id) self.handled_messages[transient_id] = message + self.state = LXMPeer.IDLE else: RNS.log("Resource transfer for LXMF peer sync failed to "+str(self.destination), RNS.LOG_DEBUG) + if self.link != None: + self.link.teardown() + else: + self.state = LXMPeer.IDLE @@ -794,6 +810,7 @@ class LXMRouter: self.propagation_transfer_state = LXMRouter.PR_IDLE self.propagation_transfer_progress = 0.0 self.propagation_transfer_last_result = None + self.locally_delivered_transient_ids = {} if identity == None: identity = RNS.Identity() @@ -821,24 +838,25 @@ class LXMRouter: job_thread.start() def exit_handler(self): - try: - serialised_peers = [] - for peer_id in self.peers: - peer = self.peers[peer_id] - serialised_peers.append(peer.to_bytes()) + if self.propagation_node: + try: + serialised_peers = [] + for peer_id in self.peers: + peer = self.peers[peer_id] + serialised_peers.append(peer.to_bytes()) + # TODO: Remove + RNS.log("Saving peer "+str(peer)) + + peers_file = open(self.storagepath+"/peers", "wb") + peers_file.write(msgpack.packb(serialised_peers)) + peers_file.close() + # TODO: Remove - RNS.log("Saving peer "+str(peer)) - - peers_file = open(self.storagepath+"/peers", "wb") - peers_file.write(msgpack.packb(serialised_peers)) - peers_file.close() - - # TODO: Remove - RNS.log("Saved peers") + RNS.log("Saved peers") - except Exception as e: - RNS.log("Could not save propagation node peers to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + except Exception as e: + RNS.log("Could not save propagation node peers to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) def register_delivery_identity(self, identity, display_name = None): @@ -936,7 +954,10 @@ class LXMRouter: def has_message(self, transient_id): - return False + if transient_id in self.locally_delivered_transient_ids: + return True + else: + return False def message_get_failed(self, request_receipt): # TODO: Remove or change @@ -992,8 +1013,21 @@ class LXMRouter: self.outbound_propagation_link.teardown() else: if request_receipt.response != None and len(request_receipt.response) > 0: + haves = [] for lxmf_data in request_receipt.response: self.lxmf_propagation(lxmf_data) + haves.append(RNS.Identity.full_hash(lxmf_data)) + + # Return a list of successfully received messages to the node + # TODO: Remove + RNS.log("Telling node to clear "+str(len(haves))+" messages") + request_receipt.link.request( + LXMPeer.MESSAGE_GET_PATH, + [None, haves], + # response_callback=self.message_syncfinal_response, + failed_callback=self.message_get_failed, + # progress_callback=self.message_get_progress + ) else: RNS.log("No LXMF data received in message download response", RNS.LOG_DEBUG) @@ -1189,9 +1223,6 @@ class LXMRouter: return LXMPeer.ERROR_NO_IDENTITY else: try: - # TODO: Remove - RNS.log(str(data)) - remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") # If both want and have fields are empty, send a list of @@ -1225,7 +1256,7 @@ class LXMRouter: if transient_id in self.propagation_entries and self.propagation_entries[transient_id][0] == remote_destination.hash: try: filepath = self.propagation_entries[transient_id][1] - self.propagation_entries.remove(transient_id) + self.propagation_entries.pop(transient_id) os.unlink(filepath) RNS.log("Client "+RNS.prettyhexrep(remote_destination.hash)+" purged message "+RNS.prettyhexrep(transient_id)+" at "+str(filepath), RNS.LOG_DEBUG) @@ -1335,6 +1366,7 @@ class LXMRouter: decrypted_lxmf_data = delivery_destination.decrypt(encrypted_lxmf_data) delivery_data = lxmf_data[:LXMessage.DESTINATION_LENGTH]+decrypted_lxmf_data self.lxmf_delivery(delivery_data, delivery_destination.type) + self.locally_delivered_transient_ids[transient_id] = time.time() else: if self.propagation_node: From 156109bf0beeaeb9a2adb98cae2668322d3250a0 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 7 Oct 2021 21:03:27 +0200 Subject: [PATCH 049/436] Cleanup --- LXMF/LXMF.py | 93 ++++++++++------------------------------------------ 1 file changed, 17 insertions(+), 76 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 69cd36a..9d86f9c 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -635,13 +635,9 @@ class LXMPeer: for transient_id in self.unhandled_messages: unhandled_ids.append(transient_id) - # TODO: Remove - RNS.log("Sending sync request to peer") + RNS.log("Sending sync request to peer "+str(self.destination), RNS.LOG_DEBUG) self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) self.state = LXMPeer.REQUEST_SENT - else: - # TODO: Remove - RNS.log("No unsynced messages") def request_failed(self, request_receipt): RNS.log("Sync request to peer "+str(self.destination)+" failed", RNS.LOG_DEBUG) @@ -651,8 +647,6 @@ class LXMPeer: self.state = LXMPeer.IDLE def offer_response(self, request_receipt): - # TODO: Remove - RNS.log("Got offer response from peer") try: self.state = LXMPeer.RESPONSE_RECEIVED response = request_receipt.response @@ -671,10 +665,6 @@ class LXMPeer: elif response == False: # Peer already has all advertised messages - - # TODO: Remove - RNS.log("Peer had all advertised messages", RNS.LOG_DEBUG) - for transient_id in self.unhandled_messages: message_entry = self.unhandled_messages[transient_id] self.handled_messages[transient_id] = message_entry @@ -701,23 +691,17 @@ class LXMPeer: wanted_message_ids.append(transient_id) if len(wanted_messages) > 0: - # TODO: Remove - RNS.log("Peer wanted: "+str(wanted_messages), RNS.LOG_DEBUG) + RNS.log("Peer wanted "+str(len(wanted_messages))+" of the available messages", RNS.LOG_DEBUG) lxm_list = [] for message_entry in wanted_messages: file_path = message_entry[1] if os.path.isfile(file_path): - # TODO: Remove - RNS.log("Loading "+str(file_path)+" for transfer") file = open(file_path, "rb") lxmf_data = file.read() file.close() lxm_list.append(lxmf_data) - else: - # TODO: Remove - RNS.log("The requested message "+str(file_path)+" does not exist") data = msgpack.packb([time.time(), lxm_list]) resource = RNS.Resource(data, self.link, callback = self.resource_concluded) @@ -727,8 +711,8 @@ class LXMPeer: self.state = LXMPeer.IDLE except Exception as e: - RNS.log("Error while handling offer response from peer "+str(self.destination), RNS.LOG_DEBUG) - RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + RNS.log("Error while handling offer response from peer "+str(self.destination), RNS.LOG_ERROR) + RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) def resource_concluded(self, resource): @@ -757,12 +741,12 @@ class LXMPeer: def handle_message(self, transient_id): if not transient_id in self.handled_messages and not transient_id in self.unhandled_messages: - # TODO: Remove - RNS.log("The message "+RNS.prettyhexrep(transient_id)+" was added to distribution queue for "+RNS.prettyhexrep(self.destination_hash)) + # TODO: Remove at some point + RNS.log("The message "+RNS.prettyhexrep(transient_id)+" was added to distribution queue for "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_EXTREME) self.unhandled_messages[transient_id] = self.router.propagation_entries[transient_id] else: - # TODO: Remove - RNS.log("The message "+RNS.prettyhexrep(transient_id)+" was NOT added to distribution queue for "+RNS.prettyhexrep(self.destination_hash)) + # TODO: Remove at some point + RNS.log("The message "+RNS.prettyhexrep(transient_id)+" was NOT added to distribution queue for "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_EXTREME) class LXMRouter: @@ -844,16 +828,12 @@ class LXMRouter: for peer_id in self.peers: peer = self.peers[peer_id] serialised_peers.append(peer.to_bytes()) - # TODO: Remove - RNS.log("Saving peer "+str(peer)) peers_file = open(self.storagepath+"/peers", "wb") peers_file.write(msgpack.packb(serialised_peers)) peers_file.close() - # TODO: Remove - RNS.log("Saved peers") - + RNS.log("Saved "+str(len(serialised_peers))+" peers to storage", RNS.LOG_DEBUG) except Exception as e: RNS.log("Could not save propagation node peers to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) @@ -933,8 +913,8 @@ class LXMRouter: self.propagation_transfer_state = LXMRouter.PR_PATH_REQUESTED self.request_messages_path_job() else: - # TODO: Remove - RNS.log("Waiting for propagation node link to become active", RNS.LOG_DEBUG) + # TODO: Remove at some point + RNS.log("Waiting for propagation node link to become active", RNS.LOG_EXTREME) def request_messages_path_job(self): @@ -960,27 +940,21 @@ class LXMRouter: return False def message_get_failed(self, request_receipt): - # TODO: Remove or change RNS.log("Message list/get request failed", RNS.LOG_DEBUG) if self.outbound_propagation_link != None: self.outbound_propagation_link.teardown() def message_list_response(self, request_receipt): - # TODO: Remove - RNS.log("Got message list response from propagation node") if request_receipt.response == LXMPeer.ERROR_NO_IDENTITY: - RNS.log("Propagation node indicated missing identification on reuquest", RNS.LOG_DEBUG) + RNS.log("Propagation node indicated missing identification on list request, tearing down link.", RNS.LOG_DEBUG) if self.outbound_propagation_link != None: self.outbound_propagation_link.teardown() else: - # TODO: Remove if request_receipt.response != None: - RNS.log("Received message list from node:", RNS.LOG_DEBUG) haves = [] wants = [] if len(request_receipt.response) > 0: for transient_id in request_receipt.response: - RNS.log(" "+RNS.prettyhexrep(transient_id), RNS.LOG_DEBUG) if self.has_message(transient_id): haves.append(transient_id) else: @@ -994,21 +968,16 @@ class LXMRouter: progress_callback=self.message_get_progress ) else: - # TODO: Remove - RNS.log("No messages on node", RNS.LOG_DEBUG) self.propagation_transfer_state = LXMRouter.PR_COMPLETE self.propagation_transfer_progress = 1.0 self.propagation_transfer_last_result = 0 def message_get_progress(self, request_receipt): self.propagation_transfer_progress = request_receipt.get_progress() - RNS.log("Progress: "+str(self.propagation_transfer_progress)) def message_get_response(self, request_receipt): - # TODO: Remove - RNS.log("Got message download response from propagation node") if request_receipt.response == LXMPeer.ERROR_NO_IDENTITY: - RNS.log("Propagation node indicated missing identification on reuquest", RNS.LOG_DEBUG) + RNS.log("Propagation node indicated missing identification on get request, tearing down link.", RNS.LOG_DEBUG) if self.outbound_propagation_link != None: self.outbound_propagation_link.teardown() else: @@ -1019,8 +988,6 @@ class LXMRouter: haves.append(RNS.Identity.full_hash(lxmf_data)) # Return a list of successfully received messages to the node - # TODO: Remove - RNS.log("Telling node to clear "+str(len(haves))+" messages") request_receipt.link.request( LXMPeer.MESSAGE_GET_PATH, [None, haves], @@ -1028,11 +995,7 @@ class LXMRouter: failed_callback=self.message_get_failed, # progress_callback=self.message_get_progress ) - else: - RNS.log("No LXMF data received in message download response", RNS.LOG_DEBUG) - # TODO: Remove - RNS.log("No messages on node", RNS.LOG_DEBUG) self.propagation_transfer_state = LXMRouter.PR_COMPLETE self.propagation_transfer_progress = 1.0 self.propagation_transfer_last_result = len(request_receipt.response) @@ -1143,9 +1106,6 @@ class LXMRouter: file.close() self.propagation_entries[transient_id] = [destination_hash, filepath] - - # TODO: Remove - RNS.log("Registered msg "+RNS.prettyhexrep(transient_id)+" at "+filepath+" for "+RNS.prettyhexrep(destination_hash)) if os.path.isfile(self.storagepath+"/peers"): peers_file = open(self.storagepath+"/peers", "rb") @@ -1155,12 +1115,9 @@ class LXMRouter: peer = LXMPeer.from_bytes(serialised_peer, self) if peer.identity != None: self.peers[peer.destination_hash] = peer - - # TODO: Remove - RNS.log("Loaded peer "+RNS.prettyhexrep(peer.destination_hash)+" with "+str(len(peer.unhandled_messages))+" unhandled messages") + RNS.log("Loaded peer "+RNS.prettyhexrep(peer.destination_hash)+" with "+str(len(peer.unhandled_messages))+" unhandled messages", RNS.LOG_DEBUG) else: - RNS.log("Peer "+RNS.prettyhexrep(peer.destination_hash)+" could not be loaded, because its identity could not be recalled. Dropping peer.") - + RNS.log("Peer "+RNS.prettyhexrep(peer.destination_hash)+" could not be loaded, because its identity could not be recalled. Dropping peer.", RNS.LOG_DEBUG) self.propagation_node = True @@ -1185,16 +1142,10 @@ class LXMRouter: self.propagation_destination.announce(app_data=data) def offer_request(self, path, data, request_id, remote_identity, requested_at): - # TODO: Remove - RNS.log("Offer request") - if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY else: try: - # TODO: Remove - RNS.log(str(data)) - transient_ids = data wanted_ids = [] @@ -1216,9 +1167,6 @@ class LXMRouter: return None def message_get_request(self, path, data, request_id, remote_identity, requested_at): - # TODO: Remove - RNS.log("Message get request") - if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY else: @@ -1238,11 +1186,6 @@ class LXMRouter: available_messages.sort(key=lambda e: e[1], reverse=False) - # TODO: Remove - for am in available_messages: - RNS.log("Msg size: "+str(am[1])) - ############## - transient_ids = [] for available_entry in available_messages: transient_ids.append(available_entry[0]) @@ -1468,12 +1411,10 @@ class LXMRouter: waiting_peers.append(peer) if len(waiting_peers) > 0: - # TODO: Remove - RNS.log("Randomly selecting peer to sync from "+str(len(waiting_peers))+" waiting peers.", RNS.LOG_DEBUG) + RNS.log("Randomly selecting peer to sync from "+str(len(waiting_peers))+" waiting peers.", RNS.LOG_EXTREME) selected_index = random.randint(0,len(waiting_peers)-1) selected_peer = waiting_peers[selected_index] - # TODO: Remove - RNS.log("Selected waiting peer "+str(selected_index)+": "+RNS.prettyhexrep(selected_peer.destination.hash), RNS.LOG_DEBUG) + RNS.log("Selected waiting peer "+str(selected_index)+": "+RNS.prettyhexrep(selected_peer.destination.hash), RNS.LOG_EXTREME) selected_peer.sync() From 92b4aaaaee668676fbde53634ca48d04ee80da3f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 8 Oct 2021 08:51:59 +0200 Subject: [PATCH 050/436] Updated storage configuration --- LXMF/LXMF.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 9d86f9c..4f8ddde 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -771,7 +771,7 @@ class LXMRouter: PR_ALL_MESSAGES = 0x00 - def __init__(self, identity = None, autopeer = AUTOPEER, autopeer_maxdepth = AUTOPEER_MAXDEPTH): + def __init__(self, identity = None, storagepath = None, autopeer = AUTOPEER, autopeer_maxdepth = AUTOPEER_MAXDEPTH): random.seed(os.urandom(10)) self.pending_inbound = [] @@ -786,6 +786,11 @@ class LXMRouter: self.propagation_node = False + if storagepath == None: + raise ValueError("LXMF cannot be initialised without a storage path") + else: + self.storagepath = storagepath+"/lxmf" + self.outbound_propagation_node = None self.outbound_propagation_link = None @@ -815,6 +820,16 @@ class LXMRouter: self.__delivery_callback = None + try: + if os.path.isfile(self.storagepath+"/local_deliveries"): + locally_delivered_file = open(self.storagepath+"/local_deliveries", "rb") + data = locally_delivered_file.read() + self.locally_delivered_transient_ids = msgpack.unpackb(data) + locally_delivered_file.close() + + except Exception as e: + RNS.log("Could not load locally delivered message ID cache from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + atexit.register(self.exit_handler) job_thread = threading.Thread(target=self.jobloop) @@ -838,6 +853,17 @@ class LXMRouter: except Exception as e: RNS.log("Could not save propagation node peers to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + try: + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) + + locally_delivered_file = open(self.storagepath+"/local_deliveries", "wb") + locally_delivered_file.write(msgpack.packb(self.locally_delivered_transient_ids)) + locally_delivered_file.close() + + except Exception as e: + RNS.log("Could not save locally delivered message ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + def register_delivery_identity(self, identity, display_name = None): delivery_destination = RNS.Destination(identity, RNS.Destination.IN, RNS.Destination.SINGLE, "lxmf", "delivery") @@ -1080,9 +1106,8 @@ class LXMRouter: self.lxmf_delivery(resource.data.read(), resource.link.type) - def enable_propagation(self, storagepath): + def enable_propagation(self): try: - self.storagepath = storagepath+"/lxmf" self.messagepath = self.storagepath+"/messagestore" if not os.path.isdir(self.storagepath): From 937ae70a85c0aa451039ec6d702abf54bf35b8f9 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 8 Oct 2021 09:13:02 +0200 Subject: [PATCH 051/436] Updated sync request handling --- LXMF/LXMF.py | 67 +++++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 4f8ddde..532449b 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -908,39 +908,42 @@ class LXMRouter: self.wants_download_on_path_available_to = None def request_messages_from_propagation_node(self, identity, max_messages = PR_ALL_MESSAGES): - self.propagation_transfer_progress = 0.0 - if self.outbound_propagation_link != None and self.outbound_propagation_link.status == RNS.Link.ACTIVE: - self.propagation_transfer_state = LXMRouter.PR_LINK_ESTABLISHED - self.outbound_propagation_link.identify(identity) - self.outbound_propagation_link.request( - LXMPeer.MESSAGE_GET_PATH, - [None, None], - response_callback=self.message_list_response, - failed_callback=self.message_get_failed - ) - else: - if self.outbound_propagation_link == None: - if RNS.Transport.has_path(self.outbound_propagation_node): - self.wants_download_on_path_available_from = None - self.propagation_transfer_state = LXMRouter.PR_LINK_ESTABLISHING - RNS.log("Establishing link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" for message download", RNS.LOG_DEBUG) - propagation_node_identity = RNS.Identity.recall(self.outbound_propagation_node) - propagation_node_destination = RNS.Destination(propagation_node_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") - def msg_request_established_callback(link): - self.request_messages_from_propagation_node(identity) - - self.outbound_propagation_link = RNS.Link(propagation_node_destination, established_callback=msg_request_established_callback) - else: - RNS.log("No path known for message download from propagation node "+RNS.prettyhexrep(self.outbound_propagation_node)+". Requesting path...", RNS.LOG_DEBUG) - RNS.Transport.request_path(self.outbound_propagation_node) - self.wants_download_on_path_available_from = self.outbound_propagation_node - self.wants_download_on_path_available_to = identity - self.wants_download_on_path_available_timeout = time.time() + LXMRouter.PR_PATH_TIMEOUT - self.propagation_transfer_state = LXMRouter.PR_PATH_REQUESTED - self.request_messages_path_job() + if self.outbound_propagation_node != None: + self.propagation_transfer_progress = 0.0 + if self.outbound_propagation_link != None and self.outbound_propagation_link.status == RNS.Link.ACTIVE: + self.propagation_transfer_state = LXMRouter.PR_LINK_ESTABLISHED + self.outbound_propagation_link.identify(identity) + self.outbound_propagation_link.request( + LXMPeer.MESSAGE_GET_PATH, + [None, None], + response_callback=self.message_list_response, + failed_callback=self.message_get_failed + ) else: - # TODO: Remove at some point - RNS.log("Waiting for propagation node link to become active", RNS.LOG_EXTREME) + if self.outbound_propagation_link == None: + if RNS.Transport.has_path(self.outbound_propagation_node): + self.wants_download_on_path_available_from = None + self.propagation_transfer_state = LXMRouter.PR_LINK_ESTABLISHING + RNS.log("Establishing link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" for message download", RNS.LOG_DEBUG) + propagation_node_identity = RNS.Identity.recall(self.outbound_propagation_node) + propagation_node_destination = RNS.Destination(propagation_node_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + def msg_request_established_callback(link): + self.request_messages_from_propagation_node(identity) + + self.outbound_propagation_link = RNS.Link(propagation_node_destination, established_callback=msg_request_established_callback) + else: + RNS.log("No path known for message download from propagation node "+RNS.prettyhexrep(self.outbound_propagation_node)+". Requesting path...", RNS.LOG_DEBUG) + RNS.Transport.request_path(self.outbound_propagation_node) + self.wants_download_on_path_available_from = self.outbound_propagation_node + self.wants_download_on_path_available_to = identity + self.wants_download_on_path_available_timeout = time.time() + LXMRouter.PR_PATH_TIMEOUT + self.propagation_transfer_state = LXMRouter.PR_PATH_REQUESTED + self.request_messages_path_job() + else: + # TODO: Remove at some point + RNS.log("Waiting for propagation node link to become active", RNS.LOG_EXTREME) + else: + RNS.log("Cannot request LXMF propagation node sync, no default propagation node configured", RNS.LOG_WARNING) def request_messages_path_job(self): From 25bcba84d2f270a4b62e948d8c5bdc6c01e11397 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 8 Oct 2021 11:57:52 +0200 Subject: [PATCH 052/436] Added max messages per sync option --- LXMF/LXMF.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 532449b..c2d7f8d 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -625,19 +625,26 @@ class LXMPeer: self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") def sync(self, initiator=True): - if len(self.unhandled_messages) > 0: - if self.state == LXMPeer.IDLE: - self.link = RNS.Link(self.destination, established_callback = self.link_established, closed_callback = self.link_closed) - self.state = LXMPeer.LINK_ESTABLISHING - else: - if self.state == LXMPeer.LINK_READY: - unhandled_ids = [] - for transient_id in self.unhandled_messages: - unhandled_ids.append(transient_id) + if self.identity == None: + self.identity = RNS.Identity.recall(destination_hash) + self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") - RNS.log("Sending sync request to peer "+str(self.destination), RNS.LOG_DEBUG) - self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) - self.state = LXMPeer.REQUEST_SENT + if self.identity != None: + if len(self.unhandled_messages) > 0: + if self.state == LXMPeer.IDLE: + self.link = RNS.Link(self.destination, established_callback = self.link_established, closed_callback = self.link_closed) + self.state = LXMPeer.LINK_ESTABLISHING + else: + if self.state == LXMPeer.LINK_READY: + unhandled_ids = [] + for transient_id in self.unhandled_messages: + unhandled_ids.append(transient_id) + + RNS.log("Sending sync request to peer "+str(self.destination), RNS.LOG_DEBUG) + self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) + self.state = LXMPeer.REQUEST_SENT + else: + RNS.log("Could not request sync to peer "+RNS.prettyhexrep(self.destination_hash)+" since its identity could not be recalled.", RNS.LOG_ERROR) def request_failed(self, request_receipt): RNS.log("Sync request to peer "+str(self.destination)+" failed", RNS.LOG_DEBUG) @@ -799,6 +806,7 @@ class LXMRouter: self.propagation_transfer_state = LXMRouter.PR_IDLE self.propagation_transfer_progress = 0.0 self.propagation_transfer_last_result = None + self.propagation_transfer_max_messages = None self.locally_delivered_transient_ids = {} if identity == None: @@ -908,6 +916,10 @@ class LXMRouter: self.wants_download_on_path_available_to = None def request_messages_from_propagation_node(self, identity, max_messages = PR_ALL_MESSAGES): + if max_messages == None: + max_messages = LXMRouter.PR_ALL_MESSAGES + + self.propagation_transfer_max_messages = max_messages if self.outbound_propagation_node != None: self.propagation_transfer_progress = 0.0 if self.outbound_propagation_link != None and self.outbound_propagation_link.status == RNS.Link.ACTIVE: @@ -987,7 +999,8 @@ class LXMRouter: if self.has_message(transient_id): haves.append(transient_id) else: - wants.append(transient_id) + if self.propagation_transfer_max_messages == LXMRouter.PR_ALL_MESSAGES or len(wants) < self.propagation_transfer_max_messages: + wants.append(transient_id) request_receipt.link.request( LXMPeer.MESSAGE_GET_PATH, From 0c34dae1af0bb35de433d8b9bef93459d6342711 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 8 Oct 2021 13:03:14 +0200 Subject: [PATCH 053/436] Implemented message store cleaning jobs --- LXMF/LXMF.py | 80 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 73 insertions(+), 7 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index c2d7f8d..dd0c86e 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -637,8 +637,16 @@ class LXMPeer: else: if self.state == LXMPeer.LINK_READY: unhandled_ids = [] + purged_ids = [] for transient_id in self.unhandled_messages: - unhandled_ids.append(transient_id) + if transient_id in self.router.propagation_entries: + unhandled_ids.append(transient_id) + else: + purged_ids.append(transient_id) + + for transient_id in purged_ids: + RNS.log("Dropping unhandled message "+RNS.prettyhexrep(transient_id)+" for peer "+RNS.prettyhexrep(self.destination_hash)+" since it no longer exists in the message store.", RNS.LOG_WARNING) + self.unhandled_messages.pop(transient_id) RNS.log("Sending sync request to peer "+str(self.destination), RNS.LOG_DEBUG) self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) @@ -763,6 +771,8 @@ class LXMRouter: PATH_REQUEST_WAIT = 5 LINK_MAX_INACTIVITY = 10*60 + MESSAGE_EXPIRY = 30*24*60*60 + AUTOPEER = True AUTOPEER_MAXDEPTH = 4 @@ -834,6 +844,7 @@ class LXMRouter: data = locally_delivered_file.read() self.locally_delivered_transient_ids = msgpack.unpackb(data) locally_delivered_file.close() + self.clean_transient_id_cache() except Exception as e: RNS.log("Could not load locally delivered message ID cache from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) @@ -918,7 +929,7 @@ class LXMRouter: def request_messages_from_propagation_node(self, identity, max_messages = PR_ALL_MESSAGES): if max_messages == None: max_messages = LXMRouter.PR_ALL_MESSAGES - + self.propagation_transfer_max_messages = max_messages if self.outbound_propagation_node != None: self.propagation_transfer_progress = 0.0 @@ -1408,21 +1419,29 @@ class LXMRouter: self.jobs() time.sleep(LXMRouter.PROCESSING_INTERVAL) - JOB_OUTBOUND_INTERVAL = 1 - JOB_LINKS_INTERVAL = 1 - JOB_PEERSYNC_INTERVAL = 12 + JOB_OUTBOUND_INTERVAL = 1 + JOB_LINKS_INTERVAL = 1 + JOB_TRANSIENT_INTERVAL = 60 + JOB_STORE_INTERVAL = 120 + JOB_PEERSYNC_INTERVAL = 12 def jobs(self): + self.processing_count += 1 + if self.processing_count % LXMRouter.JOB_OUTBOUND_INTERVAL == 0: self.process_outbound() if self.processing_count % LXMRouter.JOB_LINKS_INTERVAL == 0: self.clean_links() + if self.processing_count % LXMRouter.JOB_TRANSIENT_INTERVAL == 0: + self.clean_transient_id_cache() + + if self.processing_count % LXMRouter.JOB_STORE_INTERVAL == 0: + self.clean_message_store() + if self.processing_count % LXMRouter.JOB_PEERSYNC_INTERVAL == 0: self.sync_peers() - self.processing_count += 1 - def clean_links(self): closed_links = [] @@ -1443,6 +1462,53 @@ class LXMRouter: self.acknowledge_sync_completion() RNS.log("Cleaned outbound propagation link", RNS.LOG_DEBUG) + def clean_transient_id_cache(self): + # TODO: Remove + RNS.log("Cleaning transient id cache") + + now = time.time() + removed_entries = [] + for transient_id in self.locally_delivered_transient_ids: + timestamp = self.locally_delivered_transient_ids[transient_id] + if now > timestamp+LXMRouter.MESSAGE_EXPIRY*1.1: + removed_entries.append(transient_id) + + for transient_id in removed_entries: + self.locally_delivered_transient_ids.pop(transient_id) + RNS.log("Cleaned "+RNS.prettyhexrep(transient_id)+" from local delivery cache", RNS.LOG_DEBUG) + + + def clean_message_store(self): + now = time.time() + removed_entries = {} + for transient_id in self.propagation_entries: + entry = self.propagation_entries[transient_id] + filepath = entry[1] + components = filepath.split("_") + + if len(components) == 2 and float(components[1]) > 0 and len(os.path.split(components[0])[1]) == (RNS.Identity.HASHLENGTH//8)*2: + timestamp = float(components[1]) + if now > timestamp+LXMRouter.MESSAGE_EXPIRY: + RNS.log("Purging message "+RNS.prettyhexrep(transient_id)+" due to expiry", RNS.LOG_DEBUG) + removed_entries[transient_id] = filepath + else: + RNS.log("Purging message "+RNS.prettyhexrep(transient_id)+" due to invalid file path", RNS.LOG_WARNING) + removed_entries[transient_id] = filepath + + removed_count = 0 + for transient_id in removed_entries: + try: + filepath = removed_entries[transient_id] + self.propagation_entries.pop(transient_id) + if os.path.isfile(filepath): + os.unlink(filepath) + removed_count += 1 + except Exception as e: + RNS.log("Could not remove "+RNS.prettyhexrep(transient_id)+" from message store. The contained exception was: "+str(e), RNS.LOG_ERROR) + + if removed_count > 0: + RNS.log("Cleaned "+str(removed_count)+" entries from the message store", RNS.LOG_DEBUG) + def sync_peers(self): waiting_peers = [] From b2596bd56178309906af08531220be6f2acf7601 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 8 Oct 2021 13:40:22 +0200 Subject: [PATCH 054/436] Cleanup --- LXMF/LXMF.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index dd0c86e..e69cf46 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -666,8 +666,6 @@ class LXMPeer: self.state = LXMPeer.RESPONSE_RECEIVED response = request_receipt.response - RNS.log(str(response)) - wanted_messages = [] wanted_message_ids = [] @@ -1463,9 +1461,6 @@ class LXMRouter: RNS.log("Cleaned outbound propagation link", RNS.LOG_DEBUG) def clean_transient_id_cache(self): - # TODO: Remove - RNS.log("Cleaning transient id cache") - now = time.time() removed_entries = [] for transient_id in self.locally_delivered_transient_ids: From 71a9f16bde5b13de669923668fd6378d5b9a6883 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 8 Oct 2021 13:55:16 +0200 Subject: [PATCH 055/436] Fixed sync limit on path response re-trigger --- LXMF/LXMF.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index e69cf46..5bc15cc 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -949,7 +949,7 @@ class LXMRouter: propagation_node_identity = RNS.Identity.recall(self.outbound_propagation_node) propagation_node_destination = RNS.Destination(propagation_node_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") def msg_request_established_callback(link): - self.request_messages_from_propagation_node(identity) + self.request_messages_from_propagation_node(identity, self.propagation_transfer_max_messages) self.outbound_propagation_link = RNS.Link(propagation_node_destination, established_callback=msg_request_established_callback) else: @@ -977,7 +977,7 @@ class LXMRouter: time.sleep(0.1) if RNS.Transport.has_path(self.wants_download_on_path_available_from): - self.request_messages_from_propagation_node(self.wants_download_on_path_available_to) + self.request_messages_from_propagation_node(self.wants_download_on_path_available_to, self.propagation_transfer_max_messages) else: RNS.log("Propagation node path request timed out", RNS.LOG_DEBUG) self.acknowledge_sync_completion() From f391b95597c131645609db90a6c3084f20ee2d8e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 8 Oct 2021 15:11:17 +0200 Subject: [PATCH 056/436] Updated loglevels --- LXMF/LXMF.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 5bc15cc..04c1bcd 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -645,7 +645,7 @@ class LXMPeer: purged_ids.append(transient_id) for transient_id in purged_ids: - RNS.log("Dropping unhandled message "+RNS.prettyhexrep(transient_id)+" for peer "+RNS.prettyhexrep(self.destination_hash)+" since it no longer exists in the message store.", RNS.LOG_WARNING) + RNS.log("Dropping unhandled message "+RNS.prettyhexrep(transient_id)+" for peer "+RNS.prettyhexrep(self.destination_hash)+" since it no longer exists in the message store.", RNS.LOG_DEBUG) self.unhandled_messages.pop(transient_id) RNS.log("Sending sync request to peer "+str(self.destination), RNS.LOG_DEBUG) @@ -1513,10 +1513,10 @@ class LXMRouter: waiting_peers.append(peer) if len(waiting_peers) > 0: - RNS.log("Randomly selecting peer to sync from "+str(len(waiting_peers))+" waiting peers.", RNS.LOG_EXTREME) + RNS.log("Randomly selecting peer to sync from "+str(len(waiting_peers))+" waiting peers.", RNS.LOG_DEBUG) selected_index = random.randint(0,len(waiting_peers)-1) selected_peer = waiting_peers[selected_index] - RNS.log("Selected waiting peer "+str(selected_index)+": "+RNS.prettyhexrep(selected_peer.destination.hash), RNS.LOG_EXTREME) + RNS.log("Selected waiting peer "+str(selected_index)+": "+RNS.prettyhexrep(selected_peer.destination.hash), RNS.LOG_DEBUG) selected_peer.sync() From 2a6c602e9b55fa31067dc2bfcfa0cc55024a330a Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 8 Oct 2021 15:25:24 +0200 Subject: [PATCH 057/436] Updated logging --- LXMF/LXMF.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 04c1bcd..1dc19c7 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -625,17 +625,27 @@ class LXMPeer: self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") def sync(self, initiator=True): + RNS.log("Attempting sync to peer "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_DEBUG) + + if RNS.Transport.has_path(self.destination_hash): + RNS.log("Path to peer "+RNS.prettyhexrep(self.destination_hash)+" exist over "+str(RNS.Transport.hops_to(self.destination_hash))+" via "+str(RNS.Transport.next_hop_interface(self.destination_hash)), RNS.LOG_DEBUG) + else: + RNS.log("Attempting sync to peer "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_DEBUG) + if self.identity == None: + RNS.log("Attempting to recall identity for peer "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_DEBUG) self.identity = RNS.Identity.recall(destination_hash) self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") if self.identity != None: if len(self.unhandled_messages) > 0: if self.state == LXMPeer.IDLE: + RNS.log("Establishing link for sync to peer "+RNS.prettyhexrep(self.destination_hash)+"...", RNS.LOG_DEBUG) self.link = RNS.Link(self.destination, established_callback = self.link_established, closed_callback = self.link_closed) self.state = LXMPeer.LINK_ESTABLISHING else: if self.state == LXMPeer.LINK_READY: + RNS.log("Sync link to peer "+RNS.prettyhexrep(self.destination_hash)+" established, preparing request...", RNS.LOG_DEBUG) unhandled_ids = [] purged_ids = [] for transient_id in self.unhandled_messages: From 831e91a87adfe596a143f9122461cfa4d5913f68 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 8 Oct 2021 17:05:11 +0200 Subject: [PATCH 058/436] Improved peer sync sequence --- LXMF/LXMF.py | 92 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 39 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 1dc19c7..1f3dc82 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -627,42 +627,47 @@ class LXMPeer: def sync(self, initiator=True): RNS.log("Attempting sync to peer "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_DEBUG) - if RNS.Transport.has_path(self.destination_hash): - RNS.log("Path to peer "+RNS.prettyhexrep(self.destination_hash)+" exist over "+str(RNS.Transport.hops_to(self.destination_hash))+" via "+str(RNS.Transport.next_hop_interface(self.destination_hash)), RNS.LOG_DEBUG) + if not RNS.Transport.has_path(self.destination_hash): + RNS.log("No path to peer "+RNS.prettyhexrep(self.destination_hash)+" exists, requesting...", RNS.LOG_DEBUG) + RNS.Transport.request_path(self.destination_hash) + RNS.log("Path requested, retrying sync later", RNS.LOG_DEBUG) else: - RNS.log("Attempting sync to peer "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_DEBUG) + RNS.log("Path to peer "+RNS.prettyhexrep(self.destination_hash)+" exist over "+str(RNS.Transport.hops_to(self.destination_hash))+" hops via "+str(RNS.Transport.next_hop_interface(self.destination_hash)), RNS.LOG_DEBUG) - if self.identity == None: - RNS.log("Attempting to recall identity for peer "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_DEBUG) - self.identity = RNS.Identity.recall(destination_hash) - self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + if self.identity == None: + RNS.log("Attempting to recall identity for peer "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_DEBUG) + self.identity = RNS.Identity.recall(destination_hash) + self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") - if self.identity != None: - if len(self.unhandled_messages) > 0: - if self.state == LXMPeer.IDLE: - RNS.log("Establishing link for sync to peer "+RNS.prettyhexrep(self.destination_hash)+"...", RNS.LOG_DEBUG) - self.link = RNS.Link(self.destination, established_callback = self.link_established, closed_callback = self.link_closed) - self.state = LXMPeer.LINK_ESTABLISHING - else: - if self.state == LXMPeer.LINK_READY: - RNS.log("Sync link to peer "+RNS.prettyhexrep(self.destination_hash)+" established, preparing request...", RNS.LOG_DEBUG) - unhandled_ids = [] - purged_ids = [] - for transient_id in self.unhandled_messages: - if transient_id in self.router.propagation_entries: - unhandled_ids.append(transient_id) - else: - purged_ids.append(transient_id) + if self.identity != None: + if len(self.unhandled_messages) > 0: + if self.state == LXMPeer.IDLE: + RNS.log("Establishing link for sync to peer "+RNS.prettyhexrep(self.destination_hash)+"...", RNS.LOG_DEBUG) + self.link = RNS.Link(self.destination, established_callback = self.link_established, closed_callback = self.link_closed) + self.state = LXMPeer.LINK_ESTABLISHING - for transient_id in purged_ids: - RNS.log("Dropping unhandled message "+RNS.prettyhexrep(transient_id)+" for peer "+RNS.prettyhexrep(self.destination_hash)+" since it no longer exists in the message store.", RNS.LOG_DEBUG) - self.unhandled_messages.pop(transient_id) + # TODO: Remove + RNS.log("Establishment timeout is "+str(self.link.establishment_timeout)) + else: + if self.state == LXMPeer.LINK_READY: + RNS.log("Sync link to peer "+RNS.prettyhexrep(self.destination_hash)+" established, preparing request...", RNS.LOG_DEBUG) + unhandled_ids = [] + purged_ids = [] + for transient_id in self.unhandled_messages: + if transient_id in self.router.propagation_entries: + unhandled_ids.append(transient_id) + else: + purged_ids.append(transient_id) - RNS.log("Sending sync request to peer "+str(self.destination), RNS.LOG_DEBUG) - self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) - self.state = LXMPeer.REQUEST_SENT - else: - RNS.log("Could not request sync to peer "+RNS.prettyhexrep(self.destination_hash)+" since its identity could not be recalled.", RNS.LOG_ERROR) + for transient_id in purged_ids: + RNS.log("Dropping unhandled message "+RNS.prettyhexrep(transient_id)+" for peer "+RNS.prettyhexrep(self.destination_hash)+" since it no longer exists in the message store.", RNS.LOG_DEBUG) + self.unhandled_messages.pop(transient_id) + + RNS.log("Sending sync request to peer "+str(self.destination), RNS.LOG_DEBUG) + self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) + self.state = LXMPeer.REQUEST_SENT + else: + RNS.log("Could not request sync to peer "+RNS.prettyhexrep(self.destination_hash)+" since its identity could not be recalled.", RNS.LOG_ERROR) def request_failed(self, request_receipt): RNS.log("Sync request to peer "+str(self.destination)+" failed", RNS.LOG_DEBUG) @@ -744,6 +749,8 @@ class LXMPeer: message = self.unhandled_messages.pop(transient_id) self.handled_messages[transient_id] = message self.state = LXMPeer.IDLE + self.link.teardown() + RNS.log("Sync to peer "+RNS.prettyhexrep(self.destination_hash)+" completed", RNS.LOG_DEBUG) else: RNS.log("Resource transfer for LXMF peer sync failed to "+str(self.destination), RNS.LOG_DEBUG) if self.link != None: @@ -759,6 +766,9 @@ class LXMPeer: self.sync() def link_closed(self, link): + # TODO: Remove + RNS.log("The sync link to peer "+RNS.prettyhexrep(self.destination_hash)+" was closed: "+str(link.teardown_reason), RNS.LOG_DEBUG) + self.link = None self.state = LXMPeer.IDLE @@ -1169,15 +1179,18 @@ class LXMRouter: if os.path.isfile(self.storagepath+"/peers"): peers_file = open(self.storagepath+"/peers", "rb") - serialised_peers = msgpack.unpackb(peers_file.read()) + peers_data = peers_file.read() - for serialised_peer in serialised_peers: - peer = LXMPeer.from_bytes(serialised_peer, self) - if peer.identity != None: - self.peers[peer.destination_hash] = peer - RNS.log("Loaded peer "+RNS.prettyhexrep(peer.destination_hash)+" with "+str(len(peer.unhandled_messages))+" unhandled messages", RNS.LOG_DEBUG) - else: - RNS.log("Peer "+RNS.prettyhexrep(peer.destination_hash)+" could not be loaded, because its identity could not be recalled. Dropping peer.", RNS.LOG_DEBUG) + if len(peers_data) > 0: + serialised_peers = msgpack.unpackb(peers_data) + + for serialised_peer in serialised_peers: + peer = LXMPeer.from_bytes(serialised_peer, self) + if peer.identity != None: + self.peers[peer.destination_hash] = peer + RNS.log("Loaded peer "+RNS.prettyhexrep(peer.destination_hash)+" with "+str(len(peer.unhandled_messages))+" unhandled messages", RNS.LOG_DEBUG) + else: + RNS.log("Peer "+RNS.prettyhexrep(peer.destination_hash)+" could not be loaded, because its identity could not be recalled. Dropping peer.", RNS.LOG_DEBUG) self.propagation_node = True @@ -1191,6 +1204,7 @@ class LXMRouter: except Exception as e: RNS.log("Could not enable propagation node. The contained exception was: "+str(e), RNS.LOG_ERROR) + raise e RNS.panic() def disable_propagation(self): From fbbda8153ef0507b43d5b62433bdbbfe61baa162 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 8 Oct 2021 17:34:27 +0200 Subject: [PATCH 059/436] Fixed invalid mutation in peer sync --- LXMF/LXMF.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 1f3dc82..b23189e 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -707,7 +707,8 @@ class LXMPeer: else: # Peer wants some advertised messages - for transient_id in self.unhandled_messages: + peer_had_messages = [] + for transient_id in self.unhandled_messages.copy(): # If the peer did not want the message, it has # already received it from another peer. if not transient_id in response: @@ -742,6 +743,12 @@ class LXMPeer: RNS.log("Error while handling offer response from peer "+str(self.destination), RNS.LOG_ERROR) RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) + if self.link != None: + self.link.teardown() + + self.link = None + self.state = LXMPeer.IDLE + def resource_concluded(self, resource): if resource.status == RNS.Resource.COMPLETE: From ca18d067f290c0980f024c48302c97e8aaa48b72 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 8 Oct 2021 17:42:55 +0200 Subject: [PATCH 060/436] Updated readme --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c6fffca..de09fc0 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Lightweight Extensible Message Format ========== -LXMF is a simple and flexible messaging format and delivery protocol that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](https://github.com/markqvist/reticulum) and offers zero-conf message routing, end-to-end encryption and Perfect Forward Secrecy by default. +LXMF is a simple and flexible messaging format and delivery protocol that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](https://github.com/markqvist/reticulum) and offers zero-conf message routing, end-to-end encryption and Forward Secrecy by default. ## Structure @@ -53,6 +53,8 @@ LXMF offers flexibility to implement many different messaging schemes, ranging f LXM Propagation Nodes offer a way to store and forward messages to users or endpoints that are not directly reachable at the time of message emission. Propagation Nodes can also provide infrastructure for distributed bulletin, news or discussion boards. +When Propagation Nodes exist on a Reticulum network, they will by default peer with each other and synchronise messages, automatically creating an encrypted, distributed message store. Users and other endpoints can retrieve messages destined for them from any available Propagation Nodes on the network. + ## The LXM Router The LXM Router handles transporting messages over a Reticulum network, managing delivery receipts, outbound and inbound queues, and is the point of API interaction for client programs. The LXM Router also implements functionality for acting as an LXMF Propagation Node. @@ -95,7 +97,7 @@ The complete message overhead for LXMF is only 99 bytes, which in return gives y ## Caveat Emptor -LXMF is alpha software, and should be considered experimental. While it has been built with cryptography best-practices very foremost in mind, it _has not_ been externally security audited, and there could very well be privacy-breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch. +LXMF is beta software, and should be considered experimental. While it has been built with cryptography best-practices very foremost in mind, it _has not_ been externally security audited, and there could very well be privacy-breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch. ## Installation From d2bc7fc32e41a3d428175327458fa77a0814009e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 8 Oct 2021 17:59:33 +0200 Subject: [PATCH 061/436] Cleanup --- LXMF/LXMF.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index b23189e..c7d852d 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -646,8 +646,6 @@ class LXMPeer: self.link = RNS.Link(self.destination, established_callback = self.link_established, closed_callback = self.link_closed) self.state = LXMPeer.LINK_ESTABLISHING - # TODO: Remove - RNS.log("Establishment timeout is "+str(self.link.establishment_timeout)) else: if self.state == LXMPeer.LINK_READY: RNS.log("Sync link to peer "+RNS.prettyhexrep(self.destination_hash)+" established, preparing request...", RNS.LOG_DEBUG) @@ -773,9 +771,6 @@ class LXMPeer: self.sync() def link_closed(self, link): - # TODO: Remove - RNS.log("The sync link to peer "+RNS.prettyhexrep(self.destination_hash)+" was closed: "+str(link.teardown_reason), RNS.LOG_DEBUG) - self.link = None self.state = LXMPeer.IDLE @@ -784,10 +779,6 @@ class LXMPeer: # TODO: Remove at some point RNS.log("The message "+RNS.prettyhexrep(transient_id)+" was added to distribution queue for "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_EXTREME) self.unhandled_messages[transient_id] = self.router.propagation_entries[transient_id] - else: - # TODO: Remove at some point - RNS.log("The message "+RNS.prettyhexrep(transient_id)+" was NOT added to distribution queue for "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_EXTREME) - class LXMRouter: MAX_DELIVERY_ATTEMPTS = 3 From 934a208965cc52c9aa3065c5cce862c8e9b86242 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 9 Oct 2021 23:24:35 +0200 Subject: [PATCH 062/436] Implemented auto unpeering unreachable nodes --- LXMF/LXMF.py | 29 ++++++++++++++++++++++++++--- setup.py | 4 ++-- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index c7d852d..52d01d6 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -569,6 +569,10 @@ class LXMPeer: ERROR_NO_IDENTITY = 0xf0 + # Maximum amount of time a peer can + # be unreachable before it is removed + MAX_UNREACHABLE = 4*24*60*60 + @staticmethod def from_bytes(peer_bytes, router): dictionary = msgpack.unpackb(peer_bytes) @@ -610,7 +614,7 @@ class LXMPeer: def __init__(self, router, destination_hash): self.alive = False - self.last_heard = None + self.last_heard = 0 self.peering_timebase = 0 self.link = None @@ -648,6 +652,9 @@ class LXMPeer: else: if self.state == LXMPeer.LINK_READY: + self.alive = True + self.last_heard = time.time() + RNS.log("Sync link to peer "+RNS.prettyhexrep(self.destination_hash)+" established, preparing request...", RNS.LOG_DEBUG) unhandled_ids = [] purged_ids = [] @@ -756,6 +763,8 @@ class LXMPeer: self.state = LXMPeer.IDLE self.link.teardown() RNS.log("Sync to peer "+RNS.prettyhexrep(self.destination_hash)+" completed", RNS.LOG_DEBUG) + self.alive = True + self.last_heard = time.time() else: RNS.log("Resource transfer for LXMF peer sync failed to "+str(self.destination), RNS.LOG_DEBUG) if self.link != None: @@ -780,6 +789,12 @@ class LXMPeer: RNS.log("The message "+RNS.prettyhexrep(transient_id)+" was added to distribution queue for "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_EXTREME) self.unhandled_messages[transient_id] = self.router.propagation_entries[transient_id] + def __str__(self): + if self.destination_hash: + return RNS.prettyhexrep(self.destination_hash) + else: + return "" + class LXMRouter: MAX_DELIVERY_ATTEMPTS = 3 PROCESSING_INTERVAL = 5 @@ -1528,11 +1543,15 @@ class LXMRouter: def sync_peers(self): + culled_peers = [] waiting_peers = [] for peer_id in self.peers: peer = self.peers[peer_id] - if peer.state == LXMPeer.IDLE and len(peer.unhandled_messages) > 0: - waiting_peers.append(peer) + if time.time() > peer.last_heard + LXMPeer.MAX_UNREACHABLE: + culled_peers.append(peer_id) + else: + if peer.state == LXMPeer.IDLE and len(peer.unhandled_messages) > 0: + waiting_peers.append(peer) if len(waiting_peers) > 0: RNS.log("Randomly selecting peer to sync from "+str(len(waiting_peers))+" waiting peers.", RNS.LOG_DEBUG) @@ -1541,6 +1560,10 @@ class LXMRouter: RNS.log("Selected waiting peer "+str(selected_index)+": "+RNS.prettyhexrep(selected_peer.destination.hash), RNS.LOG_DEBUG) selected_peer.sync() + for peer in culled_peers: + RNS.log("Removing peer "+RNS.prettyhexrep(peer)+" due to excessive unreachability", RNS.LOG_WARNING) + self.peers.pop(peer_id) + def fail_message(self, lxmessage): RNS.log(str(lxmessage)+" failed to send", RNS.LOG_DEBUG) diff --git a/setup.py b/setup.py index 1b05948..7c38ec0 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="lxmf", - version="0.1.0", + version="0.1.1", author="Mark Qvist", author_email="mark@unsigned.io", description="Lightweight Extensible Message Format for Reticulum", @@ -18,6 +18,6 @@ setuptools.setup( "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - install_requires=['rns>=0.2.4'], + install_requires=['rns>=0.2.8'], python_requires='>=3.6', ) \ No newline at end of file From 512a34e2c3df862b911eed7328fd42392a1dd28c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 10 Oct 2021 00:14:04 +0200 Subject: [PATCH 063/436] Updated log statement --- LXMF/LXMF.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 52d01d6..a7bb8a6 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -1444,7 +1444,7 @@ class LXMRouter: if timestamp >= peer.peering_timebase: self.peers.pop(destination_hash) - RNS.log(str(self.propagation_destination)+" broke peering with "+str(peer.destination)) + RNS.log("Broke peering with "+str(peer.destination)) def jobloop(self): From 29f60899679b751adafaffb95be73825a6bb1971 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 12 Oct 2021 21:11:20 +0200 Subject: [PATCH 064/436] Updated sync sequence status codes --- LXMF/LXMF.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index a7bb8a6..f199d01 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -814,8 +814,9 @@ class LXMRouter: PR_LINK_ESTABLISHING = 0x02 PR_LINK_ESTABLISHED = 0x03 PR_REQUEST_SENT = 0x04 - PR_RESPONSE_RECEIVED = 0x05 - PR_COMPLETE = 0x06 + PR_RECEIVING = 0x05 + PR_RESPONSE_RECEIVED = 0x06 + PR_COMPLETE = 0x07 PR_ALL_MESSAGES = 0x00 @@ -973,6 +974,7 @@ class LXMRouter: response_callback=self.message_list_response, failed_callback=self.message_get_failed ) + self.propagation_transfer_state = LXMRouter.PR_REQUEST_SENT else: if self.outbound_propagation_link == None: if RNS.Transport.has_path(self.outbound_propagation_node): @@ -1057,6 +1059,7 @@ class LXMRouter: self.propagation_transfer_last_result = 0 def message_get_progress(self, request_receipt): + self.propagation_transfer_state = LXMRouter.PR_RECEIVING self.propagation_transfer_progress = request_receipt.get_progress() def message_get_response(self, request_receipt): From 776e72d0badfde655aa613fe9914e836102d03b0 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 15 Oct 2021 19:40:22 +0200 Subject: [PATCH 065/436] Updated version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7c38ec0..20bc98b 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="lxmf", - version="0.1.1", + version="0.1.2", author="Mark Qvist", author_email="mark@unsigned.io", description="Lightweight Extensible Message Format for Reticulum", From 38bbf194219c4b1ac9393ea281de42551a4d2d8b Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 1 Dec 2021 19:28:19 +0100 Subject: [PATCH 066/436] Error handling --- LXMF/LXMF.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index f199d01..040a551 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -1565,7 +1565,11 @@ class LXMRouter: for peer in culled_peers: RNS.log("Removing peer "+RNS.prettyhexrep(peer)+" due to excessive unreachability", RNS.LOG_WARNING) - self.peers.pop(peer_id) + try: + # TODO: Check this and add "if in" clause + self.peers.pop(peer_id) + except Exception as e: + RNS.log("Error while removing peer "+RNS.prettyhexrep(peer_id)+". The contained exception was: "+str(e), RNS.LOG_ERROR) def fail_message(self, lxmessage): @@ -1707,4 +1711,4 @@ class LXMRouter: else: RNS.log("Max delivery attempts reached for propagated "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - self.fail_message(lxmessage) \ No newline at end of file + self.fail_message(lxmessage) From 1ca2493c36ce802904c0a0fd47e327b0c09d2659 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 1 Dec 2021 19:28:27 +0100 Subject: [PATCH 067/436] Added makefile --- Makefile | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8d3e67a --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +all: release + +clean: + @echo Cleaning... + -rm -r ./build + -rm -r ./dist + +remove_symlinks: + @echo Removing symlinks for build... + -rm ./RNS + +create_symlinks: + @echo Creating symlinks... + -ln -s ../Reticulum/RNS ./ + +build_wheel: + python3 setup.py sdist bdist_wheel + +release: remove_symlinks build_wheel create_symlinks + +upload: + @echo Uploading to PyPi... + twine upload dist/* From 6aa793252f4ed1e3f3647c0d6f8b6a5844ad2f9c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 1 Dec 2021 20:16:36 +0100 Subject: [PATCH 068/436] Updated version --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 20bc98b..134758f 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="lxmf", - version="0.1.2", + version="0.1.3", author="Mark Qvist", author_email="mark@unsigned.io", description="Lightweight Extensible Message Format for Reticulum", @@ -20,4 +20,4 @@ setuptools.setup( ], install_requires=['rns>=0.2.8'], python_requires='>=3.6', -) \ No newline at end of file +) From 38b6f7651c6dc6f0f18379ed01e23925cc7b172f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 10 Dec 2021 18:36:18 +0100 Subject: [PATCH 069/436] Peer cleanup fix --- LXMF/LXMF.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 040a551..e0b7b06 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -1566,8 +1566,9 @@ class LXMRouter: for peer in culled_peers: RNS.log("Removing peer "+RNS.prettyhexrep(peer)+" due to excessive unreachability", RNS.LOG_WARNING) try: - # TODO: Check this and add "if in" clause - self.peers.pop(peer_id) + # TODO: Check this + if peer_id in self.peers: + self.peers.pop(peer_id) except Exception as e: RNS.log("Error while removing peer "+RNS.prettyhexrep(peer_id)+". The contained exception was: "+str(e), RNS.LOG_ERROR) From 5e8b2d87afa7e821dadfc22cd3bf60f52cfdd393 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 10 Dec 2021 20:08:25 +0100 Subject: [PATCH 070/436] Updated dependency version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 134758f..aec1b5e 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,6 @@ setuptools.setup( "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - install_requires=['rns>=0.2.8'], + install_requires=['rns>=0.3.0'], python_requires='>=3.6', ) From 137db1e6a30846cb07b62ed81f9e7cc61da7479f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 11 Dec 2021 19:56:35 +0100 Subject: [PATCH 071/436] Added path request retry on link initial link failure --- LXMF/LXMF.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index e0b7b06..f1e2e56 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -796,14 +796,16 @@ class LXMPeer: return "" class LXMRouter: - MAX_DELIVERY_ATTEMPTS = 3 + MAX_DELIVERY_ATTEMPTS = 4 PROCESSING_INTERVAL = 5 - DELIVERY_RETRY_WAIT = 15 + DELIVERY_RETRY_WAIT = 12 PATH_REQUEST_WAIT = 5 LINK_MAX_INACTIVITY = 10*60 MESSAGE_EXPIRY = 30*24*60*60 + NODE_ANNOUNCE_DELAY = 20 + AUTOPEER = True AUTOPEER_MAXDEPTH = 4 @@ -1228,8 +1230,14 @@ class LXMRouter: self.announce_propagation_node() def announce_propagation_node(self): - data = msgpack.packb([self.propagation_node, int(time.time())]) - self.propagation_destination.announce(app_data=data) + def delayed_announce(): + time.sleep(LXMRouter.NODE_ANNOUNCE_DELAY) + data = msgpack.packb([self.propagation_node, int(time.time())]) + self.propagation_destination.announce(app_data=data) + + da_thread = threading.Thread(target=delayed_announce) + da_thread.setDaemon(True) + da_thread.start() def offer_request(self, path, data, request_id, remote_identity, requested_at): if remote_identity == None: @@ -1629,7 +1637,18 @@ class LXMRouter: else: RNS.log("Waiting for proof for "+str(lxmessage)+" sent as link packet", RNS.LOG_DEBUG) elif direct_link.status == RNS.Link.CLOSED: - RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was closed", RNS.LOG_DEBUG) + if direct_link.activated_at != None: + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was closed", RNS.LOG_DEBUG) + else: + if not hasattr(lxmessage, "path_request_retried"): + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was never activated, retrying path request...", RNS.LOG_DEBUG) + RNS.Transport.request_path(lxmessage.get_destination().hash) + lxmessage.path_request_retried = True + else: + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was never activated", RNS.LOG_DEBUG) + + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + lxmessage.set_delivery_destination(None) self.direct_links.pop(delivery_destination_hash) lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT From 704930f91aca0bf40ef362e848c34c8c90a4411e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 25 Mar 2022 20:24:36 +0100 Subject: [PATCH 072/436] Updated version --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index aec1b5e..a6d3121 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="lxmf", - version="0.1.3", + version="0.1.4", author="Mark Qvist", author_email="mark@unsigned.io", description="Lightweight Extensible Message Format for Reticulum", @@ -18,6 +18,6 @@ setuptools.setup( "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - install_requires=['rns>=0.3.0'], + install_requires=['rns>=0.3.3'], python_requires='>=3.6', ) From 38cc471eafc288cd81a642cafd893dc43ae8e4f2 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 25 Mar 2022 20:25:00 +0100 Subject: [PATCH 073/436] Improved error handling --- LXMF/LXMF.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index f1e2e56..cb38c82 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -405,26 +405,28 @@ class LXMessage: else: return None + def packed_container(self): + if not self.packed: + self.pack() + + container = { + "state": self.state, + "lxmf_bytes": self.packed, + "transport_encrypted": self.transport_encrypted, + "transport_encryption": self.transport_encryption, + "method": self.method + } + + return msgpack.packb(container) + + def write_to_directory(self, directory_path): file_name = RNS.hexrep(self.hash, delimit=False) file_path = directory_path+"/"+file_name try: - if not self.packed: - self.pack() - - container = { - "state": self.state, - "lxmf_bytes": self.packed, - "transport_encrypted": self.transport_encrypted, - "transport_encryption": self.transport_encryption, - "method": self.method - } - - packed_container = msgpack.packb(container) - file = open(file_path, "wb") - file.write(packed_container) + file.write(self.packed_container()) file.close() return file_path From 853e70dcbeb854acc8e27135e435ee4005673096 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 28 Mar 2022 15:19:43 +0200 Subject: [PATCH 074/436] Dependency bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a6d3121..969ff26 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,6 @@ setuptools.setup( "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - install_requires=['rns>=0.3.3'], + install_requires=['rns>=0.3.4'], python_requires='>=3.6', ) From 505176b6eef9e6e4915a87b8bc104da9811f1e4b Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 13 Apr 2022 11:10:21 +0200 Subject: [PATCH 075/436] Fixed delivery method default. Thanks Stephan ;) --- LXMF/LXMF.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index cb38c82..b47195e 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -228,7 +228,7 @@ class LXMessage: # If no desired delivery method has been defined, # one will be chosen according to these rules: if self.desired_method == None: - self.desired_method == LXMessage.DIRECT + self.desired_method = LXMessage.DIRECT # TODO: Expand rules to something more intelligent if self.desired_method == LXMessage.OPPORTUNISTIC: From 2d1828513faa1a7e7c0abf98cbcc2db14f2b4bf6 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 13 Apr 2022 11:14:41 +0200 Subject: [PATCH 076/436] Fixed destination type reference --- LXMF/LXMF.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index b47195e..c547ff8 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -308,7 +308,7 @@ class LXMessage: if self.__destination.type == RNS.Destination.SINGLE: self.transport_encrypted = True self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC - elif destination_type == RNS.Destination.GROUP: + elif self.__destination.type == RNS.Destination.GROUP: self.transport_encrypted = True self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES else: From ed4effdab700094d3096dc36a61708e285a6b5f8 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 13 Apr 2022 11:33:27 +0200 Subject: [PATCH 077/436] Fixed invalid name in peer culling. Thanks Stephan :) --- LXMF/LXMF.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index c547ff8..9b467f3 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -1573,10 +1573,9 @@ class LXMRouter: RNS.log("Selected waiting peer "+str(selected_index)+": "+RNS.prettyhexrep(selected_peer.destination.hash), RNS.LOG_DEBUG) selected_peer.sync() - for peer in culled_peers: - RNS.log("Removing peer "+RNS.prettyhexrep(peer)+" due to excessive unreachability", RNS.LOG_WARNING) + for peer_id in culled_peers: + RNS.log("Removing peer "+RNS.prettyhexrep(peer_id)+" due to excessive unreachability", RNS.LOG_WARNING) try: - # TODO: Check this if peer_id in self.peers: self.peers.pop(peer_id) except Exception as e: From bd3ace0885cfdbad187c2ea6ca9979a4c0cf18cd Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 15 Apr 2022 11:30:08 +0200 Subject: [PATCH 078/436] Updated version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 969ff26..cc5de3e 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="lxmf", - version="0.1.4", + version="0.1.5", author="Mark Qvist", author_email="mark@unsigned.io", description="Lightweight Extensible Message Format for Reticulum", From 323241a4b827e980895b26988e9100977260fa5c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 27 Apr 2022 20:46:26 +0200 Subject: [PATCH 079/436] Fixed unhandled behaviour when link interface intermittently becomes unavailable --- LXMF/LXMF.py | 8 ++++++-- setup.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 9b467f3..6bb5714 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -282,8 +282,12 @@ class LXMessage: if self.representation == LXMessage.PACKET: receipt = self.__as_packet().send() - receipt.set_delivery_callback(self.__mark_delivered) - receipt.set_timeout_callback(self.__link_packet_timed_out) + if receipt: + receipt.set_delivery_callback(self.__mark_delivered) + receipt.set_timeout_callback(self.__link_packet_timed_out) + else: + if self.__delivery_destination: + self.__delivery_destination.teardown() elif self.representation == LXMessage.RESOURCE: self.resource_representation = self.__as_resource() diff --git a/setup.py b/setup.py index cc5de3e..e218750 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="lxmf", - version="0.1.5", + version="0.1.6", author="Mark Qvist", author_email="mark@unsigned.io", description="Lightweight Extensible Message Format for Reticulum", From ece55d10d6eada12728a24d925be1ff65b99c45e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 17 May 2022 13:25:00 +0200 Subject: [PATCH 080/436] Improved error handling --- LXMF/LXMF.py | 23 ++++++++++++++++------- setup.py | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 6bb5714..9a486f7 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -339,15 +339,21 @@ class LXMessage: RNS.log("Received delivery notification for "+str(self), RNS.LOG_DEBUG) self.state = LXMessage.DELIVERED - if self.__delivery_callback != None: - self.__delivery_callback(self) + if self.__delivery_callback != None and callable(self.__delivery_callback): + try: + self.__delivery_callback(self) + except Exception as e: + RNS.log("An error occurred in the external delivery callback for "+str(message), RNS.LOG_ERROR) def __mark_propagated(self, receipt = None): RNS.log("Received propagation success notification for "+str(self), RNS.LOG_DEBUG) self.state = LXMessage.SENT - if self.__delivery_callback != None: - self.__delivery_callback(self) + if self.__delivery_callback != None and callable(self.__delivery_callback): + try: + self.__delivery_callback(self) + except Exception as e: + RNS.log("An error occurred in the external delivery callback for "+str(message), RNS.LOG_ERROR) def __resource_concluded(self, resource): if resource.status == RNS.Resource.COMPLETE: @@ -1130,8 +1136,11 @@ class LXMRouter: message.transport_encrypted = False message.transport_encryption = None - if self.__delivery_callback != None: - self.__delivery_callback(message) + if self.__delivery_callback != None and callable(self.__delivery_callback): + try: + self.__delivery_callback(message) + except Exception as e: + RNS.log("An error occurred in the external delivery callback for "+str(message), RNS.LOG_ERROR) return True @@ -1593,7 +1602,7 @@ class LXMRouter: self.failed_outbound.append(lxmessage) lxmessage.state = LXMessage.FAILED - if lxmessage.failed_callback != None: + if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): lxmessage.failed_callback(lxmessage) def process_outbound(self, sender = None): diff --git a/setup.py b/setup.py index e218750..8a090d1 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,6 @@ setuptools.setup( "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - install_requires=['rns>=0.3.4'], + install_requires=['rns>=0.3.6'], python_requires='>=3.6', ) From b2aba3f6db59d016afa994604b3d4a85c70ca718 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 17 May 2022 19:43:51 +0200 Subject: [PATCH 081/436] Added ignore functionality --- LXMF/LXMF.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 9a486f7..a1d2b0f 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -843,6 +843,8 @@ class LXMRouter: self.direct_links = {} self.delivery_destinations = {} + self.ignored_list = [] + self.processing_outbound = False self.processing_inbound = False self.processing_count = 0 @@ -1136,6 +1138,10 @@ class LXMRouter: message.transport_encrypted = False message.transport_encryption = None + if message.source_hash in self.ignored_list: + RNS.log(str(self)+" ignored message from "+RNS.prettyhexrep(message.source_hash), RNS.LOG_DEBUG) + return False + if self.__delivery_callback != None and callable(self.__delivery_callback): try: self.__delivery_callback(message) @@ -1595,6 +1601,14 @@ class LXMRouter: RNS.log("Error while removing peer "+RNS.prettyhexrep(peer_id)+". The contained exception was: "+str(e), RNS.LOG_ERROR) + def ignore_destination(self, destination_hash): + if not destination_hash in self.ignored_list: + self.ignored_list.append(destination_hash) + + def unignore_destination(self, destination_hash): + if destination_hash in self.ignored_list: + self.ignored_list.remove(destination_hash) + def fail_message(self, lxmessage): RNS.log(str(lxmessage)+" failed to send", RNS.LOG_DEBUG) @@ -1605,6 +1619,9 @@ class LXMRouter: if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): lxmessage.failed_callback(lxmessage) + def __str__(self): + return "" + def process_outbound(self, sender = None): if self.processing_outbound: return From 33e41caf4ab1039694dc198ee3df44bfe0f9adbd Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 23 May 2022 10:30:27 +0200 Subject: [PATCH 082/436] Updated log output --- LXMF/LXMF.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index a1d2b0f..188941e 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -640,18 +640,17 @@ class LXMPeer: self.identity = RNS.Identity.recall(destination_hash) self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") - def sync(self, initiator=True): - RNS.log("Attempting sync to peer "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_DEBUG) + def sync(self): + RNS.log("Initiating LXMF Propagation Node sync with peer "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_DEBUG) if not RNS.Transport.has_path(self.destination_hash): RNS.log("No path to peer "+RNS.prettyhexrep(self.destination_hash)+" exists, requesting...", RNS.LOG_DEBUG) RNS.Transport.request_path(self.destination_hash) RNS.log("Path requested, retrying sync later", RNS.LOG_DEBUG) else: - RNS.log("Path to peer "+RNS.prettyhexrep(self.destination_hash)+" exist over "+str(RNS.Transport.hops_to(self.destination_hash))+" hops via "+str(RNS.Transport.next_hop_interface(self.destination_hash)), RNS.LOG_DEBUG) + # RNS.log("Path to peer "+RNS.prettyhexrep(self.destination_hash)+" exists over "+str(RNS.Transport.hops_to(self.destination_hash))+" hops via "+str(RNS.Transport.next_hop_interface(self.destination_hash)), RNS.LOG_DEBUG) if self.identity == None: - RNS.log("Attempting to recall identity for peer "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_DEBUG) self.identity = RNS.Identity.recall(destination_hash) self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") @@ -667,7 +666,7 @@ class LXMPeer: self.alive = True self.last_heard = time.time() - RNS.log("Sync link to peer "+RNS.prettyhexrep(self.destination_hash)+" established, preparing request...", RNS.LOG_DEBUG) + RNS.log("Synchronisation link to peer "+RNS.prettyhexrep(self.destination_hash)+" established, preparing request...", RNS.LOG_DEBUG) unhandled_ids = [] purged_ids = [] for transient_id in self.unhandled_messages: From 0a84d9da76077c77b622f1d7e0a9b6cc1840ec98 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 17 Jun 2022 08:22:18 +0200 Subject: [PATCH 083/436] Cleanup --- LXMF/LXMF.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 188941e..ef4dcc8 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -647,9 +647,8 @@ class LXMPeer: RNS.log("No path to peer "+RNS.prettyhexrep(self.destination_hash)+" exists, requesting...", RNS.LOG_DEBUG) RNS.Transport.request_path(self.destination_hash) RNS.log("Path requested, retrying sync later", RNS.LOG_DEBUG) + else: - # RNS.log("Path to peer "+RNS.prettyhexrep(self.destination_hash)+" exists over "+str(RNS.Transport.hops_to(self.destination_hash))+" hops via "+str(RNS.Transport.next_hop_interface(self.destination_hash)), RNS.LOG_DEBUG) - if self.identity == None: self.identity = RNS.Identity.recall(destination_hash) self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") @@ -1089,7 +1088,9 @@ class LXMRouter: self.lxmf_propagation(lxmf_data) haves.append(RNS.Identity.full_hash(lxmf_data)) - # Return a list of successfully received messages to the node + # Return a list of successfully received messages to the node. + # This deletes the messages on the propagation node. + # TODO: Add option to keep messages on node. request_receipt.link.request( LXMPeer.MESSAGE_GET_PATH, [None, haves], From 039df2bd46020d2a6071726ce3cdecea38eb5f10 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 17 Jun 2022 08:54:04 +0200 Subject: [PATCH 084/436] Restructured module --- LXMF/Handlers.py | 45 ++ LXMF/LXMF.py | 1766 +-------------------------------------------- LXMF/LXMPeer.py | 244 +++++++ LXMF/LXMRouter.py | 974 +++++++++++++++++++++++++ LXMF/LXMessage.py | 525 ++++++++++++++ LXMF/__init__.py | 4 +- 6 files changed, 1791 insertions(+), 1767 deletions(-) create mode 100644 LXMF/Handlers.py create mode 100644 LXMF/LXMPeer.py create mode 100644 LXMF/LXMRouter.py create mode 100644 LXMF/LXMessage.py diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py new file mode 100644 index 0000000..32e045e --- /dev/null +++ b/LXMF/Handlers.py @@ -0,0 +1,45 @@ +import time +import RNS +import RNS.vendor.umsgpack as msgpack + +from .LXMF import APP_NAME + +from .LXMessage import LXMessage + +class LXMFDeliveryAnnounceHandler: + def __init__(self, lxmrouter): + self.aspect_filter = APP_NAME+".delivery" + self.lxmrouter = lxmrouter + + def received_announce(self, destination_hash, announced_identity, app_data): + for lxmessage in self.lxmrouter.pending_outbound: + if destination_hash == lxmessage.destination_hash: + if lxmessage.method == LXMessage.DIRECT: + lxmessage.next_delivery_attempt = time.time() + + while self.lxmrouter.processing_outbound: + time.sleep(0.1) + + self.lxmrouter.process_outbound() + + +class LXMFPropagationAnnounceHandler: + def __init__(self, lxmrouter): + self.aspect_filter = APP_NAME+".propagation" + self.lxmrouter = lxmrouter + + def received_announce(self, destination_hash, announced_identity, app_data): + try: + if type(app_data) == bytes: + data = msgpack.unpackb(app_data) + + if self.lxmrouter.propagation_node and self.lxmrouter.autopeer: + if data[0] == True: + if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: + self.lxmrouter.peer(destination_hash, data[1]) + elif data[0] == False: + self.lxmrouter.unpeer(destination_hash, data[1]) + + except Exception as e: + RNS.log("Error while evaluating propagation node announce, ignoring announce.", RNS.LOG_DEBUG) + RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index ef4dcc8..d19b423 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -1,1765 +1 @@ -import os -import math -import time -import random -import atexit -import threading -import RNS -import RNS.vendor.umsgpack as msgpack - -APP_NAME = "lxmf" - -class LXMessage: - DRAFT = 0x00 - OUTBOUND = 0x01 - SENDING = 0x02 - SENT = 0x04 - DELIVERED = 0x08 - FAILED = 0xFF - states = [DRAFT, OUTBOUND, SENDING, SENT, DELIVERED, FAILED] - - UNKNOWN = 0x00 - PACKET = 0x01 - RESOURCE = 0x02 - representations = [UNKNOWN, PACKET, RESOURCE] - - OPPORTUNISTIC = 0x01 - DIRECT = 0x02 - PROPAGATED = 0x03 - valid_methods = [OPPORTUNISTIC, DIRECT, PROPAGATED] - - SOURCE_UNKNOWN = 0x01 - SIGNATURE_INVALID = 0x02 - unverified_reasons = [SOURCE_UNKNOWN, SIGNATURE_INVALID] - - DESTINATION_LENGTH = RNS.Identity.TRUNCATED_HASHLENGTH//8 - SIGNATURE_LENGTH = RNS.Identity.SIGLENGTH//8 - - # LXMF overhead is 99 bytes per message: - # 10 bytes for destination hash - # 10 bytes for source hash - # 64 bytes for Ed25519 signature - # 8 bytes for timestamp - # 7 bytes for msgpack structure - LXMF_OVERHEAD = 2*DESTINATION_LENGTH + SIGNATURE_LENGTH + 8 + 7 - - # With an MTU of 500, the maximum amount of data - # we can send in a single encrypted packet is - # 383 bytes. - ENCRYPTED_PACKET_MDU = RNS.Packet.ENCRYPTED_MDU - - # The max content length we can fit in LXMF message - # inside a single RNS packet is the encrypted MDU, minus - # the LXMF overhead. We can optimise a bit though, by - # inferring the destination hash from the destination - # field of the packet, therefore we also add the length - # of a destination hash to the calculation. With default - # RNS and LXMF parameters, the largest single-packet - # LXMF message we can send is 294 bytes. If a message - # is larger than that, a Reticulum link will be used. - ENCRYPTED_PACKET_MAX_CONTENT = ENCRYPTED_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH - - # Links can carry a larger MDU, due to less overhead per - # packet. The link MDU with default Reticulum parameters - # is 431 bytes. - LINK_PACKET_MDU = RNS.Link.MDU - - # Which means that we can deliver single-packet LXMF - # messages with content of up to 332 bytes over a link. - # If a message is larger than that, LXMF will sequence - # and transfer it as a RNS resource over the link instead. - LINK_PACKET_MAX_CONTENT = LINK_PACKET_MDU - LXMF_OVERHEAD - - # For plain packets without encryption, we can - # fit up to 388 bytes of content. - PLAIN_PACKET_MDU = RNS.Packet.PLAIN_MDU - PLAIN_PACKET_MAX_CONTENT = PLAIN_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH - - # Descriptive strings regarding transport encryption - ENCRYPTION_DESCRIPTION_AES = "AES-128" - ENCRYPTION_DESCRIPTION_EC = "Curve25519" - ENCRYPTION_DESCRIPTION_UNENCRYPTED = "Unencrypted" - - def __str__(self): - if self.hash != None: - return "" - else: - return "" - - def __init__(self, destination, source, content = "", title = "", fields = None, desired_method = None, destination_hash = None, source_hash = None): - if isinstance(destination, RNS.Destination) or destination == None: - self.__destination = destination - if destination != None: - self.destination_hash = destination.hash - else: - self.destination_hash = destination_hash - else: - raise ValueError("LXMessage initialised with invalid destination") - - if isinstance(source, RNS.Destination) or source == None: - self.__source = source - if source != None: - self.source_hash = source.hash - else: - self.source_hash = source_hash - else: - raise ValueError("LXMessage initialised with invalid source") - - self.set_title_from_string(title) - self.set_content_from_string(content) - - self.set_fields(fields) - - self.payload = None - self.timestamp = None - self.signature = None - self.hash = None - self.packed = None - self.progress = None - self.state = LXMessage.DRAFT - self.method = LXMessage.UNKNOWN - - self.incoming = False - self.signature_validated = False - self.unverified_reason = None - - self.representation = LXMessage.UNKNOWN - self.desired_method = desired_method - self.delivery_attempts = 0 - self.transport_encrypted = False - self.transport_encryption = None - self.packet_representation = None - self.resource_representation = None - self.__delivery_destination = None - self.__delivery_callback = None - self.failed_callback = None - - def set_title_from_string(self, title_string): - self.title = title_string.encode("utf-8") - - def set_title_from_bytes(self, title_bytes): - self.title = title_bytes - - def title_as_string(self): - return self.title.decode("utf-8") - - def set_content_from_string(self, content_string): - self.content = content_string.encode("utf-8") - - def set_content_from_bytes(self, content_bytes): - self.content = content_bytes - - def content_as_string(self): - return self.content.decode("utf-8") - - def set_fields(self, fields): - if isinstance(fields, dict) or fields == None: - self.fields = fields - else: - raise ValueError("LXMessage property \"fields\" can only be dict or None") - - def get_fields(self): - return self.__fields - - def set_destination(self, destination): - if self.destination == None: - if isinstance(destination, RNS.Destination): - self.__destination = destination - else: - raise ValueError("Invalid destination set on LXMessage") - else: - raise ValueError("Cannot reassign destination on LXMessage") - - def get_destination(self): - return self.__destination - - def set_source(self, source): - if self.source == None: - if isinstance(source, RNS.Destination): - self.__source = source - else: - raise ValueError("Invalid source set on LXMessage") - else: - raise ValueError("Cannot reassign source on LXMessage") - - def get_source(self): - return self.__source - - def set_delivery_destination(self, delivery_destination): - self.__delivery_destination = delivery_destination - - def register_delivery_callback(self, callback): - self.__delivery_callback = callback - - def register_failed_callback(self, callback): - self.failed_callback = callback - - def pack(self): - if not self.packed: - if self.timestamp == None: - self.timestamp = time.time() - - self.propagation_packed = None - - self.payload = [self.timestamp, self.title, self.content, self.fields] - - hashed_part = b"" - hashed_part += self.__destination.hash - hashed_part += self.__source.hash - hashed_part += msgpack.packb(self.payload) - self.hash = RNS.Identity.full_hash(hashed_part) - self.message_id = self.hash - - signed_part = b"" - signed_part += hashed_part - signed_part += self.hash - self.signature = self.__source.sign(signed_part) - self.signature_validated = True - - self.packed = b"" - self.packed += self.__destination.hash - self.packed += self.__source.hash - self.packed += self.signature - packed_payload = msgpack.packb(self.payload) - self.packed += packed_payload - self.packed_size = len(self.packed) - content_size = len(packed_payload) - - # If no desired delivery method has been defined, - # one will be chosen according to these rules: - if self.desired_method == None: - self.desired_method = LXMessage.DIRECT - # TODO: Expand rules to something more intelligent - - if self.desired_method == LXMessage.OPPORTUNISTIC: - if self.__destination.type == RNS.Destination.SINGLE: - single_packet_content_limit = LXMessage.ENCRYPTED_PACKET_MAX_CONTENT - elif self.__destination.type == RNS.Destination.PLAIN: - single_packet_content_limit = LXMessage.PLAIN_PACKET_MAX_CONTENT - - if content_size > single_packet_content_limit: - raise TypeError("LXMessage desired opportunistic delivery method, but content exceeds single-packet size.") - else: - self.method = LXMessage.OPPORTUNISTIC - self.representation = LXMessage.PACKET - self.__delivery_destination = self.__destination - - elif self.desired_method == LXMessage.DIRECT: - single_packet_content_limit = LXMessage.LINK_PACKET_MAX_CONTENT - if content_size <= single_packet_content_limit: - self.method = self.desired_method - self.representation = LXMessage.PACKET - else: - self.method = self.desired_method - self.representation = LXMessage.RESOURCE - - elif self.desired_method == LXMessage.PROPAGATED: - single_packet_content_limit = LXMessage.LINK_PACKET_MAX_CONTENT - - encrypted_data = self.__destination.encrypt(self.packed[LXMessage.DESTINATION_LENGTH:]) - self.propagation_packed = msgpack.packb([time.time(), [self.packed[:LXMessage.DESTINATION_LENGTH]+encrypted_data]]) - - content_size = len(self.propagation_packed) - if content_size <= single_packet_content_limit: - self.method = self.desired_method - self.representation = LXMessage.PACKET - else: - self.method = self.desired_method - self.representation = LXMessage.RESOURCE - - else: - raise ValueError("Attempt to re-pack LXMessage "+str(self)+" that was already packed") - - def send(self): - self.determine_transport_encryption() - - if self.method == LXMessage.OPPORTUNISTIC: - self.__as_packet().send().set_delivery_callback(self.__mark_delivered) - self.state = LXMessage.SENT - - elif self.method == LXMessage.DIRECT: - self.state = LXMessage.SENDING - - if self.representation == LXMessage.PACKET: - receipt = self.__as_packet().send() - if receipt: - receipt.set_delivery_callback(self.__mark_delivered) - receipt.set_timeout_callback(self.__link_packet_timed_out) - else: - if self.__delivery_destination: - self.__delivery_destination.teardown() - - elif self.representation == LXMessage.RESOURCE: - self.resource_representation = self.__as_resource() - - elif self.method == LXMessage.PROPAGATED: - self.state = LXMessage.SENDING - - if self.representation == LXMessage.PACKET: - receipt = self.__as_packet().send() - if receipt: - receipt.set_delivery_callback(self.__mark_propagated) - receipt.set_timeout_callback(self.__link_packet_timed_out) - else: - self.__delivery_destination.teardown() - - elif self.representation == LXMessage.RESOURCE: - self.resource_representation = self.__as_resource() - - - def determine_transport_encryption(self): - if self.method == LXMessage.OPPORTUNISTIC: - if self.__destination.type == RNS.Destination.SINGLE: - self.transport_encrypted = True - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC - elif self.__destination.type == RNS.Destination.GROUP: - self.transport_encrypted = True - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES - else: - self.transport_encrypted = False - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED - elif self.method == LXMessage.DIRECT: - self.transport_encrypted = True - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC - elif self.method == LXMessage.PROPAGATED: - if self.__destination.type == RNS.Destination.SINGLE: - self.transport_encrypted = True - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC - elif self.__destination.type == RNS.Destination.GROUP: - self.transport_encrypted = True - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES - else: - self.transport_encrypted = False - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED - else: - self.transport_encrypted = False - self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED - - def __mark_delivered(self, receipt = None): - RNS.log("Received delivery notification for "+str(self), RNS.LOG_DEBUG) - self.state = LXMessage.DELIVERED - - if self.__delivery_callback != None and callable(self.__delivery_callback): - try: - self.__delivery_callback(self) - except Exception as e: - RNS.log("An error occurred in the external delivery callback for "+str(message), RNS.LOG_ERROR) - - def __mark_propagated(self, receipt = None): - RNS.log("Received propagation success notification for "+str(self), RNS.LOG_DEBUG) - self.state = LXMessage.SENT - - if self.__delivery_callback != None and callable(self.__delivery_callback): - try: - self.__delivery_callback(self) - except Exception as e: - RNS.log("An error occurred in the external delivery callback for "+str(message), RNS.LOG_ERROR) - - def __resource_concluded(self, resource): - if resource.status == RNS.Resource.COMPLETE: - self.__mark_delivered() - else: - resource.link.teardown() - self.state = LXMessage.OUTBOUND - - def __propagation_resource_concluded(self, resource): - if resource.status == RNS.Resource.COMPLETE: - self.__mark_propagated() - else: - resource.link.teardown() - self.state = LXMessage.OUTBOUND - - def __link_packet_timed_out(self, packet_receipt): - if packet_receipt: - packet_receipt.destination.teardown() - - self.state = LXMessage.OUTBOUND - - - def __update_transfer_progress(self, resource): - self.progress = resource.get_progress() - - def __as_packet(self): - if not self.packed: - self.pack() - - if not self.__delivery_destination: - raise ValueError("Can't synthesize packet for LXMF message before delivery destination is known") - - if self.method == LXMessage.OPPORTUNISTIC: - return RNS.Packet(self.__delivery_destination, self.packed[LXMessage.DESTINATION_LENGTH:]) - elif self.method == LXMessage.DIRECT: - return RNS.Packet(self.__delivery_destination, self.packed) - elif self.method == LXMessage.PROPAGATED: - return RNS.Packet(self.__delivery_destination, self.propagation_packed) - - def __as_resource(self): - if not self.packed: - self.pack() - - if not self.__delivery_destination: - raise ValueError("Can't synthesize resource for LXMF message before delivery destination is known") - - if not self.__delivery_destination.type == RNS.Destination.LINK: - raise TypeError("Tried to synthesize resource for LXMF message on a delivery destination that was not a link") - - if not self.__delivery_destination.status == RNS.Link.ACTIVE: - raise ConnectionError("Tried to synthesize resource for LXMF message on a link that was not active") - - self.progress = 0.0 - - if self.method == LXMessage.DIRECT: - return RNS.Resource(self.packed, self.__delivery_destination, callback = self.__resource_concluded, progress_callback = self.__update_transfer_progress) - elif self.method == LXMessage.PROPAGATED: - return RNS.Resource(self.propagation_packed, self.__delivery_destination, callback = self.__propagation_resource_concluded, progress_callback = self.__update_transfer_progress) - else: - return None - - def packed_container(self): - if not self.packed: - self.pack() - - container = { - "state": self.state, - "lxmf_bytes": self.packed, - "transport_encrypted": self.transport_encrypted, - "transport_encryption": self.transport_encryption, - "method": self.method - } - - return msgpack.packb(container) - - - def write_to_directory(self, directory_path): - file_name = RNS.hexrep(self.hash, delimit=False) - file_path = directory_path+"/"+file_name - - try: - file = open(file_path, "wb") - file.write(self.packed_container()) - file.close() - - return file_path - - except Exception as e: - RNS.log("Error while writing LXMF message to file \""+str(file_path)+"\". The contained exception was: "+str(e), RNS.LOG_ERROR) - return None - - - @staticmethod - def unpack_from_bytes(lxmf_bytes): - destination_hash = lxmf_bytes[:LXMessage.DESTINATION_LENGTH] - source_hash = lxmf_bytes[LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH] - signature = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH] - packed_payload = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH:] - hashed_part = b"" + destination_hash + source_hash + packed_payload - message_hash = RNS.Identity.full_hash(hashed_part) - signed_part = b"" + hashed_part + message_hash - unpacked_payload = msgpack.unpackb(packed_payload) - timestamp = unpacked_payload[0] - title_bytes = unpacked_payload[1] - content_bytes = unpacked_payload[2] - fields = unpacked_payload[3] - - destination_identity = RNS.Identity.recall(destination_hash) - if destination_identity != None: - destination = RNS.Destination(destination_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") - else: - destination = None - - source_identity = RNS.Identity.recall(source_hash) - if source_identity != None: - source = RNS.Destination(source_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") - else: - source = None - - message = LXMessage( - destination = destination, - source = source, - content = "", - title = "", - fields = fields, - destination_hash = destination_hash, - source_hash = source_hash) - - message.hash = message_hash - message.signature = signature - message.incoming = True - message.timestamp = timestamp - message.packed = lxmf_bytes - message.packed_size = len(lxmf_bytes) - message.set_title_from_bytes(title_bytes) - message.set_content_from_bytes(content_bytes) - - try: - if source: - if source.identity.validate(signature, signed_part): - message.signature_validated = True - else: - message.signature_validated = False - message.unverified_reason = LXMessage.SIGNATURE_INVALID - else: - signature_validated = False - message.unverified_reason = LXMessage.SOURCE_UNKNOWN - RNS.log("Unpacked LXMF message signature could not be validated, since source identity is unknown", RNS.LOG_DEBUG) - except Exception as e: - message.signature_validated = False - RNS.log("Error while validating LXMF message signature. The contained exception was: "+str(e), RNS.LOG_ERROR) - - return message - - @staticmethod - def unpack_from_file(lxmf_file_handle): - try: - container = msgpack.unpackb(lxmf_file_handle.read()) - lxm = LXMessage.unpack_from_bytes(container["lxmf_bytes"]) - - if "state" in container: - lxm.state = container["state"] - if "transport_encrypted" in container: - lxm.transport_encrypted = container["transport_encrypted"] - if "transport_encryption" in container: - lxm.transport_encryption = container["transport_encryption"] - if "method" in container: - lxm.method = container["method"] - - return lxm - except Exception as e: - RNS.log("Could not unpack LXMessage from file. The contained exception was: "+str(e), RNS.LOG_ERROR) - return None - - -class LXMFDeliveryAnnounceHandler: - def __init__(self, lxmrouter): - self.aspect_filter = APP_NAME+".delivery" - self.lxmrouter = lxmrouter - - def received_announce(self, destination_hash, announced_identity, app_data): - for lxmessage in self.lxmrouter.pending_outbound: - if destination_hash == lxmessage.destination_hash: - if lxmessage.method == LXMessage.DIRECT: - lxmessage.next_delivery_attempt = time.time() - - while self.lxmrouter.processing_outbound: - time.sleep(0.1) - - self.lxmrouter.process_outbound() - - -class LXMFPropagationAnnounceHandler: - def __init__(self, lxmrouter): - self.aspect_filter = APP_NAME+".propagation" - self.lxmrouter = lxmrouter - - def received_announce(self, destination_hash, announced_identity, app_data): - try: - if type(app_data) == bytes: - data = msgpack.unpackb(app_data) - - if self.lxmrouter.propagation_node and self.lxmrouter.autopeer: - if data[0] == True: - if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: - self.lxmrouter.peer(destination_hash, data[1]) - elif data[0] == False: - self.lxmrouter.unpeer(destination_hash, data[1]) - - except Exception as e: - RNS.log("Error while evaluating propagation node announce, ignoring announce.", RNS.LOG_DEBUG) - RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) - - -class LXMPeer: - OFFER_REQUEST_PATH = "/offer" - MESSAGE_GET_PATH = "/get" - - IDLE = 0x00 - LINK_ESTABLISHING = 0x01 - LINK_READY = 0x02 - REQUEST_SENT = 0x03 - RESPONSE_RECEIVED = 0x04 - RESOURCE_TRANSFERRING = 0x05 - - ERROR_NO_IDENTITY = 0xf0 - - # Maximum amount of time a peer can - # be unreachable before it is removed - MAX_UNREACHABLE = 4*24*60*60 - - @staticmethod - def from_bytes(peer_bytes, router): - dictionary = msgpack.unpackb(peer_bytes) - - peer = LXMPeer(router, dictionary["destination_hash"]) - peer.peering_timebase = dictionary["peering_timebase"] - peer.alive = dictionary["alive"] - peer.last_heard = dictionary["last_heard"] - - for transient_id in dictionary["handled_ids"]: - if transient_id in router.propagation_entries: - peer.handled_messages[transient_id] = router.propagation_entries[transient_id] - - for transient_id in dictionary["unhandled_ids"]: - if transient_id in router.propagation_entries: - peer.unhandled_messages[transient_id] = router.propagation_entries[transient_id] - - return peer - - def to_bytes(self): - dictionary = {} - dictionary["peering_timebase"] = self.peering_timebase - dictionary["alive"] = self.alive - dictionary["last_heard"] = self.last_heard - dictionary["destination_hash"] = self.destination_hash - - handled_ids = [] - for transient_id in self.handled_messages: - handled_ids.append(transient_id) - - unhandled_ids = [] - for transient_id in self.unhandled_messages: - unhandled_ids.append(transient_id) - - dictionary["handled_ids"] = handled_ids - dictionary["unhandled_ids"] = unhandled_ids - - return msgpack.packb(dictionary) - - def __init__(self, router, destination_hash): - self.alive = False - self.last_heard = 0 - self.peering_timebase = 0 - - self.link = None - self.state = LXMPeer.IDLE - - self.unhandled_messages = {} - self.handled_messages = {} - - self.router = router - self.destination_hash = destination_hash - self.identity = RNS.Identity.recall(destination_hash) - self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") - - def sync(self): - RNS.log("Initiating LXMF Propagation Node sync with peer "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_DEBUG) - - if not RNS.Transport.has_path(self.destination_hash): - RNS.log("No path to peer "+RNS.prettyhexrep(self.destination_hash)+" exists, requesting...", RNS.LOG_DEBUG) - RNS.Transport.request_path(self.destination_hash) - RNS.log("Path requested, retrying sync later", RNS.LOG_DEBUG) - - else: - if self.identity == None: - self.identity = RNS.Identity.recall(destination_hash) - self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") - - if self.identity != None: - if len(self.unhandled_messages) > 0: - if self.state == LXMPeer.IDLE: - RNS.log("Establishing link for sync to peer "+RNS.prettyhexrep(self.destination_hash)+"...", RNS.LOG_DEBUG) - self.link = RNS.Link(self.destination, established_callback = self.link_established, closed_callback = self.link_closed) - self.state = LXMPeer.LINK_ESTABLISHING - - else: - if self.state == LXMPeer.LINK_READY: - self.alive = True - self.last_heard = time.time() - - RNS.log("Synchronisation link to peer "+RNS.prettyhexrep(self.destination_hash)+" established, preparing request...", RNS.LOG_DEBUG) - unhandled_ids = [] - purged_ids = [] - for transient_id in self.unhandled_messages: - if transient_id in self.router.propagation_entries: - unhandled_ids.append(transient_id) - else: - purged_ids.append(transient_id) - - for transient_id in purged_ids: - RNS.log("Dropping unhandled message "+RNS.prettyhexrep(transient_id)+" for peer "+RNS.prettyhexrep(self.destination_hash)+" since it no longer exists in the message store.", RNS.LOG_DEBUG) - self.unhandled_messages.pop(transient_id) - - RNS.log("Sending sync request to peer "+str(self.destination), RNS.LOG_DEBUG) - self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) - self.state = LXMPeer.REQUEST_SENT - else: - RNS.log("Could not request sync to peer "+RNS.prettyhexrep(self.destination_hash)+" since its identity could not be recalled.", RNS.LOG_ERROR) - - def request_failed(self, request_receipt): - RNS.log("Sync request to peer "+str(self.destination)+" failed", RNS.LOG_DEBUG) - if self.link != None: - self.link.teardown() - else: - self.state = LXMPeer.IDLE - - def offer_response(self, request_receipt): - try: - self.state = LXMPeer.RESPONSE_RECEIVED - response = request_receipt.response - - wanted_messages = [] - wanted_message_ids = [] - - if response == LXMPeer.ERROR_NO_IDENTITY: - if self.link != None: - RNS.log("Remote peer indicated that no identification was received, retrying...", RNS.LOG_DEBUG) - self.link.indentify() - self.state = LXMPeer.LINK_READY - self.sync() - - elif response == False: - # Peer already has all advertised messages - for transient_id in self.unhandled_messages: - message_entry = self.unhandled_messages[transient_id] - self.handled_messages[transient_id] = message_entry - - self.unhandled_messages = {} - - elif response == True: - # Peer wants all advertised messages - for transient_id in self.unhandled_messages: - wanted_messages.append(self.unhandled_messages[transient_id]) - wanted_message_ids.append(transient_id) - - else: - # Peer wants some advertised messages - peer_had_messages = [] - for transient_id in self.unhandled_messages.copy(): - # If the peer did not want the message, it has - # already received it from another peer. - if not transient_id in response: - message_entry = self.unhandled_messages.pop(transient_id) - self.handled_messages[transient_id] = message_entry - - for transient_id in response: - wanted_messages.append(self.unhandled_messages[transient_id]) - wanted_message_ids.append(transient_id) - - if len(wanted_messages) > 0: - RNS.log("Peer wanted "+str(len(wanted_messages))+" of the available messages", RNS.LOG_DEBUG) - - lxm_list = [] - - for message_entry in wanted_messages: - file_path = message_entry[1] - if os.path.isfile(file_path): - file = open(file_path, "rb") - lxmf_data = file.read() - file.close() - lxm_list.append(lxmf_data) - - data = msgpack.packb([time.time(), lxm_list]) - resource = RNS.Resource(data, self.link, callback = self.resource_concluded) - resource.transferred_messages = wanted_message_ids - self.state = LXMPeer.RESOURCE_TRANSFERRING - else: - self.state = LXMPeer.IDLE - - except Exception as e: - RNS.log("Error while handling offer response from peer "+str(self.destination), RNS.LOG_ERROR) - RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) - - if self.link != None: - self.link.teardown() - - self.link = None - self.state = LXMPeer.IDLE - - - def resource_concluded(self, resource): - if resource.status == RNS.Resource.COMPLETE: - for transient_id in resource.transferred_messages: - message = self.unhandled_messages.pop(transient_id) - self.handled_messages[transient_id] = message - self.state = LXMPeer.IDLE - self.link.teardown() - RNS.log("Sync to peer "+RNS.prettyhexrep(self.destination_hash)+" completed", RNS.LOG_DEBUG) - self.alive = True - self.last_heard = time.time() - else: - RNS.log("Resource transfer for LXMF peer sync failed to "+str(self.destination), RNS.LOG_DEBUG) - if self.link != None: - self.link.teardown() - else: - self.state = LXMPeer.IDLE - - - - def link_established(self, link): - self.link.identify(self.router.identity) - self.state = LXMPeer.LINK_READY - self.sync() - - def link_closed(self, link): - self.link = None - self.state = LXMPeer.IDLE - - def handle_message(self, transient_id): - if not transient_id in self.handled_messages and not transient_id in self.unhandled_messages: - # TODO: Remove at some point - RNS.log("The message "+RNS.prettyhexrep(transient_id)+" was added to distribution queue for "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_EXTREME) - self.unhandled_messages[transient_id] = self.router.propagation_entries[transient_id] - - def __str__(self): - if self.destination_hash: - return RNS.prettyhexrep(self.destination_hash) - else: - return "" - -class LXMRouter: - MAX_DELIVERY_ATTEMPTS = 4 - PROCESSING_INTERVAL = 5 - DELIVERY_RETRY_WAIT = 12 - PATH_REQUEST_WAIT = 5 - LINK_MAX_INACTIVITY = 10*60 - - MESSAGE_EXPIRY = 30*24*60*60 - - NODE_ANNOUNCE_DELAY = 20 - - AUTOPEER = True - AUTOPEER_MAXDEPTH = 4 - - PR_PATH_TIMEOUT = 10 - - PR_IDLE = 0x00 - PR_PATH_REQUESTED = 0x01 - PR_LINK_ESTABLISHING = 0x02 - PR_LINK_ESTABLISHED = 0x03 - PR_REQUEST_SENT = 0x04 - PR_RECEIVING = 0x05 - PR_RESPONSE_RECEIVED = 0x06 - PR_COMPLETE = 0x07 - - PR_ALL_MESSAGES = 0x00 - - def __init__(self, identity = None, storagepath = None, autopeer = AUTOPEER, autopeer_maxdepth = AUTOPEER_MAXDEPTH): - random.seed(os.urandom(10)) - - self.pending_inbound = [] - self.pending_outbound = [] - self.failed_outbound = [] - self.direct_links = {} - self.delivery_destinations = {} - - self.ignored_list = [] - - self.processing_outbound = False - self.processing_inbound = False - self.processing_count = 0 - - self.propagation_node = False - - if storagepath == None: - raise ValueError("LXMF cannot be initialised without a storage path") - else: - self.storagepath = storagepath+"/lxmf" - - self.outbound_propagation_node = None - self.outbound_propagation_link = None - - self.wants_download_on_path_available_from = None - self.wants_download_on_path_available_to = None - self.propagation_transfer_state = LXMRouter.PR_IDLE - self.propagation_transfer_progress = 0.0 - self.propagation_transfer_last_result = None - self.propagation_transfer_max_messages = None - self.locally_delivered_transient_ids = {} - - if identity == None: - identity = RNS.Identity() - - self.identity = identity - self.lxmf_query_destination = RNS.Destination(None, RNS.Destination.IN, RNS.Destination.PLAIN, APP_NAME, "query") - self.propagation_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation") - - self.autopeer = autopeer - self.autopeer_maxdepth = autopeer_maxdepth - - self.peers = {} - self.propagation_entries = {} - self.propagated_ids = {} - - RNS.Transport.register_announce_handler(LXMFDeliveryAnnounceHandler(self)) - RNS.Transport.register_announce_handler(LXMFPropagationAnnounceHandler(self)) - - self.__delivery_callback = None - - try: - if os.path.isfile(self.storagepath+"/local_deliveries"): - locally_delivered_file = open(self.storagepath+"/local_deliveries", "rb") - data = locally_delivered_file.read() - self.locally_delivered_transient_ids = msgpack.unpackb(data) - locally_delivered_file.close() - self.clean_transient_id_cache() - - except Exception as e: - RNS.log("Could not load locally delivered message ID cache from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) - - atexit.register(self.exit_handler) - - job_thread = threading.Thread(target=self.jobloop) - job_thread.setDaemon(True) - job_thread.start() - - def exit_handler(self): - if self.propagation_node: - try: - serialised_peers = [] - for peer_id in self.peers: - peer = self.peers[peer_id] - serialised_peers.append(peer.to_bytes()) - - peers_file = open(self.storagepath+"/peers", "wb") - peers_file.write(msgpack.packb(serialised_peers)) - peers_file.close() - - RNS.log("Saved "+str(len(serialised_peers))+" peers to storage", RNS.LOG_DEBUG) - - except Exception as e: - RNS.log("Could not save propagation node peers to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) - - try: - if not os.path.isdir(self.storagepath): - os.makedirs(self.storagepath) - - locally_delivered_file = open(self.storagepath+"/local_deliveries", "wb") - locally_delivered_file.write(msgpack.packb(self.locally_delivered_transient_ids)) - locally_delivered_file.close() - - except Exception as e: - RNS.log("Could not save locally delivered message ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) - - - def register_delivery_identity(self, identity, display_name = None): - delivery_destination = RNS.Destination(identity, RNS.Destination.IN, RNS.Destination.SINGLE, "lxmf", "delivery") - delivery_destination.set_packet_callback(self.delivery_packet) - delivery_destination.set_link_established_callback(self.delivery_link_established) - delivery_destination.display_name = display_name - - if display_name != None: - delivery_destination.set_default_app_data(display_name.encode("utf-8")) - - self.delivery_destinations[delivery_destination.hash] = delivery_destination - return delivery_destination - - def register_delivery_callback(self, callback): - self.__delivery_callback = callback - - def set_outbound_propagation_node(self, destination_hash): - if len(destination_hash) != RNS.Identity.TRUNCATED_HASHLENGTH//8 or type(destination_hash) != bytes: - raise ValueError("Invalid destination hash for outbound propagation node") - else: - self.outbound_propagation_node = destination_hash - if self.outbound_propagation_link != None: - if self.outbound_propagation_link.destination.hash != destination_hash: - self.outbound_propagation_link.teardown() - self.outbound_propagation_link = None - - def get_outbound_propagation_node(self): - return self.outbound_propagation_node - - def cancel_propagation_node_requests(self): - if self.outbound_propagation_link != None: - self.outbound_propagation_link.teardown() - self.outbound_propagation_link = None - - self.acknowledge_sync_completion() - - def acknowledge_sync_completion(self): - self.propagation_transfer_state = LXMRouter.PR_IDLE - self.propagation_transfer_progress = 0.0 - self.propagation_transfer_last_result = None - self.wants_download_on_path_available_from = None - self.wants_download_on_path_available_to = None - - def request_messages_from_propagation_node(self, identity, max_messages = PR_ALL_MESSAGES): - if max_messages == None: - max_messages = LXMRouter.PR_ALL_MESSAGES - - self.propagation_transfer_max_messages = max_messages - if self.outbound_propagation_node != None: - self.propagation_transfer_progress = 0.0 - if self.outbound_propagation_link != None and self.outbound_propagation_link.status == RNS.Link.ACTIVE: - self.propagation_transfer_state = LXMRouter.PR_LINK_ESTABLISHED - self.outbound_propagation_link.identify(identity) - self.outbound_propagation_link.request( - LXMPeer.MESSAGE_GET_PATH, - [None, None], - response_callback=self.message_list_response, - failed_callback=self.message_get_failed - ) - self.propagation_transfer_state = LXMRouter.PR_REQUEST_SENT - else: - if self.outbound_propagation_link == None: - if RNS.Transport.has_path(self.outbound_propagation_node): - self.wants_download_on_path_available_from = None - self.propagation_transfer_state = LXMRouter.PR_LINK_ESTABLISHING - RNS.log("Establishing link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" for message download", RNS.LOG_DEBUG) - propagation_node_identity = RNS.Identity.recall(self.outbound_propagation_node) - propagation_node_destination = RNS.Destination(propagation_node_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") - def msg_request_established_callback(link): - self.request_messages_from_propagation_node(identity, self.propagation_transfer_max_messages) - - self.outbound_propagation_link = RNS.Link(propagation_node_destination, established_callback=msg_request_established_callback) - else: - RNS.log("No path known for message download from propagation node "+RNS.prettyhexrep(self.outbound_propagation_node)+". Requesting path...", RNS.LOG_DEBUG) - RNS.Transport.request_path(self.outbound_propagation_node) - self.wants_download_on_path_available_from = self.outbound_propagation_node - self.wants_download_on_path_available_to = identity - self.wants_download_on_path_available_timeout = time.time() + LXMRouter.PR_PATH_TIMEOUT - self.propagation_transfer_state = LXMRouter.PR_PATH_REQUESTED - self.request_messages_path_job() - else: - # TODO: Remove at some point - RNS.log("Waiting for propagation node link to become active", RNS.LOG_EXTREME) - else: - RNS.log("Cannot request LXMF propagation node sync, no default propagation node configured", RNS.LOG_WARNING) - - - def request_messages_path_job(self): - job_thread = threading.Thread(target=self.__request_messages_path_job) - job_thread.setDaemon(True) - job_thread.start() - - def __request_messages_path_job(self): - while not RNS.Transport.has_path(self.wants_download_on_path_available_from) and time.time() < self.wants_download_on_path_available_timeout: - time.sleep(0.1) - - if RNS.Transport.has_path(self.wants_download_on_path_available_from): - self.request_messages_from_propagation_node(self.wants_download_on_path_available_to, self.propagation_transfer_max_messages) - else: - RNS.log("Propagation node path request timed out", RNS.LOG_DEBUG) - self.acknowledge_sync_completion() - - - def has_message(self, transient_id): - if transient_id in self.locally_delivered_transient_ids: - return True - else: - return False - - def message_get_failed(self, request_receipt): - RNS.log("Message list/get request failed", RNS.LOG_DEBUG) - if self.outbound_propagation_link != None: - self.outbound_propagation_link.teardown() - - def message_list_response(self, request_receipt): - if request_receipt.response == LXMPeer.ERROR_NO_IDENTITY: - RNS.log("Propagation node indicated missing identification on list request, tearing down link.", RNS.LOG_DEBUG) - if self.outbound_propagation_link != None: - self.outbound_propagation_link.teardown() - else: - if request_receipt.response != None: - haves = [] - wants = [] - if len(request_receipt.response) > 0: - for transient_id in request_receipt.response: - if self.has_message(transient_id): - haves.append(transient_id) - else: - if self.propagation_transfer_max_messages == LXMRouter.PR_ALL_MESSAGES or len(wants) < self.propagation_transfer_max_messages: - wants.append(transient_id) - - request_receipt.link.request( - LXMPeer.MESSAGE_GET_PATH, - [wants, haves], - response_callback=self.message_get_response, - failed_callback=self.message_get_failed, - progress_callback=self.message_get_progress - ) - else: - self.propagation_transfer_state = LXMRouter.PR_COMPLETE - self.propagation_transfer_progress = 1.0 - self.propagation_transfer_last_result = 0 - - def message_get_progress(self, request_receipt): - self.propagation_transfer_state = LXMRouter.PR_RECEIVING - self.propagation_transfer_progress = request_receipt.get_progress() - - def message_get_response(self, request_receipt): - if request_receipt.response == LXMPeer.ERROR_NO_IDENTITY: - RNS.log("Propagation node indicated missing identification on get request, tearing down link.", RNS.LOG_DEBUG) - if self.outbound_propagation_link != None: - self.outbound_propagation_link.teardown() - else: - if request_receipt.response != None and len(request_receipt.response) > 0: - haves = [] - for lxmf_data in request_receipt.response: - self.lxmf_propagation(lxmf_data) - haves.append(RNS.Identity.full_hash(lxmf_data)) - - # Return a list of successfully received messages to the node. - # This deletes the messages on the propagation node. - # TODO: Add option to keep messages on node. - request_receipt.link.request( - LXMPeer.MESSAGE_GET_PATH, - [None, haves], - # response_callback=self.message_syncfinal_response, - failed_callback=self.message_get_failed, - # progress_callback=self.message_get_progress - ) - - self.propagation_transfer_state = LXMRouter.PR_COMPLETE - self.propagation_transfer_progress = 1.0 - self.propagation_transfer_last_result = len(request_receipt.response) - - def announce(self, destination_hash): - if destination_hash in self.delivery_destinations: - delivery_destination = self.delivery_destinations[destination_hash] - delivery_destination.announce(delivery_destination.display_name.encode("utf-8")) - - def handle_outbound(self, lxmessage): - lxmessage.state = LXMessage.OUTBOUND - if not lxmessage.packed: - lxmessage.pack() - - lxmessage.determine_transport_encryption() - - while self.processing_outbound: - time.sleep(0.1) - - self.pending_outbound.append(lxmessage) - self.process_outbound() - - def lxmf_delivery(self, lxmf_data, destination_type = None): - try: - message = LXMessage.unpack_from_bytes(lxmf_data) - - if destination_type == RNS.Destination.SINGLE: - message.transport_encrypted = True - message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC - elif destination_type == RNS.Destination.GROUP: - message.transport_encrypted = True - message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES - elif destination_type == RNS.Destination.LINK: - message.transport_encrypted = True - message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC - else: - message.transport_encrypted = False - message.transport_encryption = None - - if message.source_hash in self.ignored_list: - RNS.log(str(self)+" ignored message from "+RNS.prettyhexrep(message.source_hash), RNS.LOG_DEBUG) - return False - - if self.__delivery_callback != None and callable(self.__delivery_callback): - try: - self.__delivery_callback(message) - except Exception as e: - RNS.log("An error occurred in the external delivery callback for "+str(message), RNS.LOG_ERROR) - - return True - - except Exception as e: - RNS.log("Could not assemble LXMF message from received data", RNS.LOG_NOTICE) - RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) - return False - - - def delivery_packet(self, data, packet): - try: - if packet.destination_type != RNS.Destination.LINK: - lxmf_data = b"" - lxmf_data += packet.destination.hash - lxmf_data += data - else: - lxmf_data = data - - if self.lxmf_delivery(lxmf_data, packet.destination_type): - packet.prove() - - except Exception as e: - RNS.log("Exception occurred while parsing incoming LXMF data.", RNS.LOG_ERROR) - RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) - - def delivery_link_established(self, link): - link.set_packet_callback(self.delivery_packet) - link.set_resource_strategy(RNS.Link.ACCEPT_ALL) - link.set_resource_started_callback(self.resource_transfer_began) - link.set_resource_concluded_callback(self.delivery_resource_concluded) - - def delivery_link_closed(self, link): - pass - - def resource_transfer_began(self, resource): - RNS.log("Transfer began for resource "+str(resource), RNS.LOG_DEBUG) - - def delivery_resource_concluded(self, resource): - RNS.log("Transfer concluded for delivery resource "+str(resource), RNS.LOG_DEBUG) - if resource.status == RNS.Resource.COMPLETE: - self.lxmf_delivery(resource.data.read(), resource.link.type) - - - def enable_propagation(self): - try: - self.messagepath = self.storagepath+"/messagestore" - - if not os.path.isdir(self.storagepath): - os.makedirs(self.storagepath) - - if not os.path.isdir(self.messagepath): - os.makedirs(self.messagepath) - - self.propagation_entries = {} - for filename in os.listdir(self.messagepath): - components = filename.split("_") - if len(components) == 2: - if float(components[1]) > 0: - if len(components[0]) == RNS.Identity.HASHLENGTH//8*2: - transient_id = bytes.fromhex(components[0]) - received = components[1] - - filepath = self.messagepath+"/"+filename - file = open(filepath, "rb") - destination_hash = file.read(LXMessage.DESTINATION_LENGTH) - file.close() - - self.propagation_entries[transient_id] = [destination_hash, filepath] - - if os.path.isfile(self.storagepath+"/peers"): - peers_file = open(self.storagepath+"/peers", "rb") - peers_data = peers_file.read() - - if len(peers_data) > 0: - serialised_peers = msgpack.unpackb(peers_data) - - for serialised_peer in serialised_peers: - peer = LXMPeer.from_bytes(serialised_peer, self) - if peer.identity != None: - self.peers[peer.destination_hash] = peer - RNS.log("Loaded peer "+RNS.prettyhexrep(peer.destination_hash)+" with "+str(len(peer.unhandled_messages))+" unhandled messages", RNS.LOG_DEBUG) - else: - RNS.log("Peer "+RNS.prettyhexrep(peer.destination_hash)+" could not be loaded, because its identity could not be recalled. Dropping peer.", RNS.LOG_DEBUG) - - - self.propagation_node = True - self.propagation_destination.set_link_established_callback(self.propagation_link_established) - self.propagation_destination.set_packet_callback(self.propagation_packet) - - self.propagation_destination.register_request_handler(LXMPeer.OFFER_REQUEST_PATH, self.offer_request, allow = RNS.Destination.ALLOW_ALL) - self.propagation_destination.register_request_handler(LXMPeer.MESSAGE_GET_PATH, self.message_get_request, allow = RNS.Destination.ALLOW_ALL) - - self.announce_propagation_node() - - except Exception as e: - RNS.log("Could not enable propagation node. The contained exception was: "+str(e), RNS.LOG_ERROR) - raise e - RNS.panic() - - def disable_propagation(self): - self.propagation_node = False - self.announce_propagation_node() - - def announce_propagation_node(self): - def delayed_announce(): - time.sleep(LXMRouter.NODE_ANNOUNCE_DELAY) - data = msgpack.packb([self.propagation_node, int(time.time())]) - self.propagation_destination.announce(app_data=data) - - da_thread = threading.Thread(target=delayed_announce) - da_thread.setDaemon(True) - da_thread.start() - - def offer_request(self, path, data, request_id, remote_identity, requested_at): - if remote_identity == None: - return LXMPeer.ERROR_NO_IDENTITY - else: - try: - transient_ids = data - wanted_ids = [] - - for transient_id in transient_ids: - if not transient_id in self.propagation_entries: - wanted_ids.append(transient_id) - - if len(wanted_ids) == 0: - return False - - elif len(wanted_ids) == len(transient_ids): - return True - - else: - return wanted_ids - - except Exception as e: - RNS.log("Error occurred while generating response for sync request, the contained exception was: "+str(e), RNS.LOG_DEBUG) - return None - - def message_get_request(self, path, data, request_id, remote_identity, requested_at): - if remote_identity == None: - return LXMPeer.ERROR_NO_IDENTITY - else: - try: - remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") - - # If both want and have fields are empty, send a list of - # available messages. - if data[0] == None and data[1] == None: - available_messages = [] - for transient_id in self.propagation_entries: - message_entry = self.propagation_entries[transient_id] - if message_entry[0] == remote_destination.hash: - message_size = os.path.getsize(message_entry[1]) - available_entry = [transient_id, message_size] - available_messages.append(available_entry) - - available_messages.sort(key=lambda e: e[1], reverse=False) - - transient_ids = [] - for available_entry in available_messages: - transient_ids.append(available_entry[0]) - - return transient_ids - - else: - # Process messages the client already have - if data[1] != None and len(data[1]) > 0: - for transient_id in data[1]: - if transient_id in self.propagation_entries and self.propagation_entries[transient_id][0] == remote_destination.hash: - try: - filepath = self.propagation_entries[transient_id][1] - self.propagation_entries.pop(transient_id) - os.unlink(filepath) - RNS.log("Client "+RNS.prettyhexrep(remote_destination.hash)+" purged message "+RNS.prettyhexrep(transient_id)+" at "+str(filepath), RNS.LOG_DEBUG) - - except Exception as e: - RNS.log("Error while processing message purge request from "+RNS.prettyhexrep(remote_destination.hash)+". The contained exception was: "+str(e), RNS.LOG_ERROR) - - - # Process wanted messages - response_messages = [] - if data[0] != None and len(data[0]) > 0: - for transient_id in data[0]: - if transient_id in self.propagation_entries and self.propagation_entries[transient_id][0] == remote_destination.hash: - try: - filepath = self.propagation_entries[transient_id][1] - RNS.log("Client "+RNS.prettyhexrep(remote_destination.hash)+" requested message "+RNS.prettyhexrep(transient_id)+" at "+str(filepath), RNS.LOG_DEBUG) - - message_file = open(filepath, "rb") - lxmf_data = message_file.read() - response_messages.append(lxmf_data) - message_file.close() - - except Exception as e: - RNS.log("Error while processing message download request from "+RNS.prettyhexrep(remote_destination.hash)+". The contained exception was: "+str(e), RNS.LOG_ERROR) - - return response_messages - - - except Exception as e: - RNS.log("Error occurred while generating response for download request, the contained exception was: "+str(e), RNS.LOG_DEBUG) - return None - - def propagation_link_established(self, link): - link.set_packet_callback(self.propagation_packet) - link.set_resource_strategy(RNS.Link.ACCEPT_ALL) - link.set_resource_started_callback(self.resource_transfer_began) - link.set_resource_concluded_callback(self.propagation_resource_concluded) - - - def propagation_packet(self, data, packet): - try: - if packet.destination_type != RNS.Destination.LINK: - pass - else: - data = msgpack.unpackb(data) - remote_timebase = data[0] - - messages = data[1] - for lxmf_data in messages: - self.lxmf_propagation(lxmf_data) - - packet.prove() - - except Exception as e: - RNS.log("Exception occurred while parsing incoming LXMF propagation data.", RNS.LOG_ERROR) - RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) - - - def propagation_resource_concluded(self, resource): - RNS.log("Transfer concluded for incoming propagation resource "+str(resource), RNS.LOG_DEBUG) - if resource.status == RNS.Resource.COMPLETE: - # TODO: The peer this was received from should - # have the transient id added to it's list of - # already handled messages. - try: - data = msgpack.unpackb(resource.data.read()) - - if type(data) == list and len(data) == 2 and type(data[0] == float) and type(data[1]) == list: - # This is a series of propagation messages from a peer or originator - remote_timebase = data[0] - remote_hash = None - remote_identity = resource.link.get_remote_identity() - - if remote_identity != None: - remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") - remote_hash = remote_destination.hash - - if not remote_hash in self.peers: - self.peer(remote_hash, remote_timebase) - - messages = data[1] - for lxmf_data in messages: - if remote_hash != None and remote_hash in self.peers: - transient_id = RNS.Identity.full_hash(lxmf_data) - peer = self.peers[remote_hash] - peer.handled_messages[transient_id] = [transient_id, remote_timebase, lxmf_data] - - self.lxmf_propagation(lxmf_data) - else: - RNS.log("Invalid data structure received at propagation destination, ignoring", RNS.LOG_DEBUG) - - except Exception as e: - RNS.log("Error while unpacking received propagation resource", RNS.LOG_DEBUG) - - - def lxmf_propagation(self, lxmf_data): - try: - if len(lxmf_data) >= LXMessage.LXMF_OVERHEAD: - transient_id = RNS.Identity.full_hash(lxmf_data) - if not transient_id in self.propagation_entries and not transient_id in self.propagated_ids: - received = time.time() - propagation_entry = [transient_id, received, lxmf_data] - destination_hash = lxmf_data[:LXMessage.DESTINATION_LENGTH] - - if destination_hash in self.delivery_destinations: - delivery_destination = self.delivery_destinations[destination_hash] - encrypted_lxmf_data = lxmf_data[LXMessage.DESTINATION_LENGTH:] - decrypted_lxmf_data = delivery_destination.decrypt(encrypted_lxmf_data) - delivery_data = lxmf_data[:LXMessage.DESTINATION_LENGTH]+decrypted_lxmf_data - self.lxmf_delivery(delivery_data, delivery_destination.type) - self.locally_delivered_transient_ids[transient_id] = time.time() - - else: - if self.propagation_node: - file_path = self.messagepath+"/"+RNS.hexrep(transient_id, delimit=False)+"_"+str(received) - msg_file = open(file_path, "wb") - msg_file.write(lxmf_data) - msg_file.close() - - self.propagation_entries[transient_id] = [destination_hash, file_path] - - RNS.log("Received propagated LXMF message "+RNS.prettyhexrep(transient_id)+", adding to peer distribution queues...", RNS.LOG_DEBUG) - for peer_id in self.peers: - peer = self.peers[peer_id] - peer.handle_message(transient_id) - - return True - - return False - - except Exception as e: - RNS.log("Could not assemble propagated LXMF message from received data", RNS.LOG_DEBUG) - RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) - return False - - - def peer(self, destination_hash, timestamp): - if destination_hash in self.peers: - peer = self.peers[destination_hash] - peer.alive = True - peer.peering_timebase = timestamp - peer.last_heard = time.time() - else: - peer = LXMPeer(self, destination_hash) - peer.alive = True - peer.last_heard = time.time() - self.peers[destination_hash] = peer - RNS.log("Peered with "+str(peer.destination)) - - def unpeer(self, destination_hash, timestamp = None): - if timestamp == None: - timestamp = int(time.time()) - - if destination_hash in self.peers: - peer = self.peers[destination_hash] - - if timestamp >= peer.peering_timebase: - self.peers.pop(destination_hash) - RNS.log("Broke peering with "+str(peer.destination)) - - - def jobloop(self): - while (True): - # TODO: Improve this to scheduling, so manual - # triggers can delay next run - self.jobs() - time.sleep(LXMRouter.PROCESSING_INTERVAL) - - JOB_OUTBOUND_INTERVAL = 1 - JOB_LINKS_INTERVAL = 1 - JOB_TRANSIENT_INTERVAL = 60 - JOB_STORE_INTERVAL = 120 - JOB_PEERSYNC_INTERVAL = 12 - def jobs(self): - self.processing_count += 1 - - if self.processing_count % LXMRouter.JOB_OUTBOUND_INTERVAL == 0: - self.process_outbound() - - if self.processing_count % LXMRouter.JOB_LINKS_INTERVAL == 0: - self.clean_links() - - if self.processing_count % LXMRouter.JOB_TRANSIENT_INTERVAL == 0: - self.clean_transient_id_cache() - - if self.processing_count % LXMRouter.JOB_STORE_INTERVAL == 0: - self.clean_message_store() - - if self.processing_count % LXMRouter.JOB_PEERSYNC_INTERVAL == 0: - self.sync_peers() - - - def clean_links(self): - closed_links = [] - for link_hash in self.direct_links: - link = self.direct_links[link_hash] - inactive_time = link.inactive_for() - - if inactive_time > LXMRouter.LINK_MAX_INACTIVITY: - link.teardown() - closed_links.append(link_hash) - - for link_hash in closed_links: - cleaned_link = self.direct_links.pop(link_hash) - RNS.log("Cleaned link "+str(cleaned_link), RNS.LOG_DEBUG) - - if self.outbound_propagation_link != None and self.outbound_propagation_link.status == RNS.Link.CLOSED: - self.outbound_propagation_link = None - self.acknowledge_sync_completion() - RNS.log("Cleaned outbound propagation link", RNS.LOG_DEBUG) - - def clean_transient_id_cache(self): - now = time.time() - removed_entries = [] - for transient_id in self.locally_delivered_transient_ids: - timestamp = self.locally_delivered_transient_ids[transient_id] - if now > timestamp+LXMRouter.MESSAGE_EXPIRY*1.1: - removed_entries.append(transient_id) - - for transient_id in removed_entries: - self.locally_delivered_transient_ids.pop(transient_id) - RNS.log("Cleaned "+RNS.prettyhexrep(transient_id)+" from local delivery cache", RNS.LOG_DEBUG) - - - def clean_message_store(self): - now = time.time() - removed_entries = {} - for transient_id in self.propagation_entries: - entry = self.propagation_entries[transient_id] - filepath = entry[1] - components = filepath.split("_") - - if len(components) == 2 and float(components[1]) > 0 and len(os.path.split(components[0])[1]) == (RNS.Identity.HASHLENGTH//8)*2: - timestamp = float(components[1]) - if now > timestamp+LXMRouter.MESSAGE_EXPIRY: - RNS.log("Purging message "+RNS.prettyhexrep(transient_id)+" due to expiry", RNS.LOG_DEBUG) - removed_entries[transient_id] = filepath - else: - RNS.log("Purging message "+RNS.prettyhexrep(transient_id)+" due to invalid file path", RNS.LOG_WARNING) - removed_entries[transient_id] = filepath - - removed_count = 0 - for transient_id in removed_entries: - try: - filepath = removed_entries[transient_id] - self.propagation_entries.pop(transient_id) - if os.path.isfile(filepath): - os.unlink(filepath) - removed_count += 1 - except Exception as e: - RNS.log("Could not remove "+RNS.prettyhexrep(transient_id)+" from message store. The contained exception was: "+str(e), RNS.LOG_ERROR) - - if removed_count > 0: - RNS.log("Cleaned "+str(removed_count)+" entries from the message store", RNS.LOG_DEBUG) - - - def sync_peers(self): - culled_peers = [] - waiting_peers = [] - for peer_id in self.peers: - peer = self.peers[peer_id] - if time.time() > peer.last_heard + LXMPeer.MAX_UNREACHABLE: - culled_peers.append(peer_id) - else: - if peer.state == LXMPeer.IDLE and len(peer.unhandled_messages) > 0: - waiting_peers.append(peer) - - if len(waiting_peers) > 0: - RNS.log("Randomly selecting peer to sync from "+str(len(waiting_peers))+" waiting peers.", RNS.LOG_DEBUG) - selected_index = random.randint(0,len(waiting_peers)-1) - selected_peer = waiting_peers[selected_index] - RNS.log("Selected waiting peer "+str(selected_index)+": "+RNS.prettyhexrep(selected_peer.destination.hash), RNS.LOG_DEBUG) - selected_peer.sync() - - for peer_id in culled_peers: - RNS.log("Removing peer "+RNS.prettyhexrep(peer_id)+" due to excessive unreachability", RNS.LOG_WARNING) - try: - if peer_id in self.peers: - self.peers.pop(peer_id) - except Exception as e: - RNS.log("Error while removing peer "+RNS.prettyhexrep(peer_id)+". The contained exception was: "+str(e), RNS.LOG_ERROR) - - - def ignore_destination(self, destination_hash): - if not destination_hash in self.ignored_list: - self.ignored_list.append(destination_hash) - - def unignore_destination(self, destination_hash): - if destination_hash in self.ignored_list: - self.ignored_list.remove(destination_hash) - - def fail_message(self, lxmessage): - RNS.log(str(lxmessage)+" failed to send", RNS.LOG_DEBUG) - - self.pending_outbound.remove(lxmessage) - self.failed_outbound.append(lxmessage) - - lxmessage.state = LXMessage.FAILED - if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): - lxmessage.failed_callback(lxmessage) - - def __str__(self): - return "" - - def process_outbound(self, sender = None): - if self.processing_outbound: - return - - for lxmessage in self.pending_outbound: - if lxmessage.state == LXMessage.DELIVERED: - RNS.log("Delivery has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) - self.pending_outbound.remove(lxmessage) - elif lxmessage.state == LXMessage.SENT: - RNS.log("Propagation has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) - self.pending_outbound.remove(lxmessage) - else: - RNS.log("Starting outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - # Outbound handling for opportunistic messages - if lxmessage.method == LXMessage.OPPORTUNISTIC: - if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: - if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: - lxmessage.delivery_attempts += 1 - lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT - RNS.log("Opportunistic delivery attempt "+str(lxmessage.delivery_attempts)+" for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - lxmessage.send() - else: - RNS.log("Max delivery attempts reached for oppertunistic "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - self.fail_message(lxmessage) - - # Outbound handling for messages transferred - # over a direct link to the final recipient - elif lxmessage.method == LXMessage.DIRECT: - if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: - delivery_destination_hash = lxmessage.get_destination().hash - - if delivery_destination_hash in self.direct_links: - # A link already exists, so we'll try to use it - # to deliver the message - direct_link = self.direct_links[delivery_destination_hash] - if direct_link.status == RNS.Link.ACTIVE: - if lxmessage.state != LXMessage.SENDING: - RNS.log("Starting transfer of "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - lxmessage.set_delivery_destination(direct_link) - lxmessage.send() - else: - if lxmessage.representation == LXMessage.RESOURCE: - RNS.log("The transfer of "+str(lxmessage)+" is in progress ("+str(round(lxmessage.progress*100, 1))+"%)", RNS.LOG_DEBUG) - else: - RNS.log("Waiting for proof for "+str(lxmessage)+" sent as link packet", RNS.LOG_DEBUG) - elif direct_link.status == RNS.Link.CLOSED: - if direct_link.activated_at != None: - RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was closed", RNS.LOG_DEBUG) - else: - if not hasattr(lxmessage, "path_request_retried"): - RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was never activated, retrying path request...", RNS.LOG_DEBUG) - RNS.Transport.request_path(lxmessage.get_destination().hash) - lxmessage.path_request_retried = True - else: - RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was never activated", RNS.LOG_DEBUG) - - lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT - - lxmessage.set_delivery_destination(None) - self.direct_links.pop(delivery_destination_hash) - lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT - else: - # Simply wait for the link to become - # active or close - RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" is pending, waiting for link to become active", RNS.LOG_DEBUG) - else: - # No link exists, so we'll try to establish one, but - # only if we've never tried before, or the retry wait - # period has elapsed. - if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: - lxmessage.delivery_attempts += 1 - lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT - - if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: - if RNS.Transport.has_path(lxmessage.get_destination().hash): - RNS.log("Establishing link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - delivery_link = RNS.Link(lxmessage.get_destination()) - delivery_link.set_link_established_callback(self.process_outbound) - self.direct_links[delivery_destination_hash] = delivery_link - else: - RNS.log("No path known for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+". Requesting path...", RNS.LOG_DEBUG) - RNS.Transport.request_path(lxmessage.get_destination().hash) - lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT - else: - RNS.log("Max delivery attempts reached for direct "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - self.fail_message(lxmessage) - - # Outbound handling for messages transported via - # propagation to a LXMF router network. - elif lxmessage.method == LXMessage.PROPAGATED: - RNS.log("Attempting propagated delivery for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - - if self.outbound_propagation_node == None: - RNS.log("No outbound propagation node specified for propagated "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_ERROR) - self.fail_message(lxmessage) - else: - if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: - - if self.outbound_propagation_link != None: - # A link already exists, so we'll try to use it - # to deliver the message - if self.outbound_propagation_link.status == RNS.Link.ACTIVE: - if lxmessage.state != LXMessage.SENDING: - RNS.log("Starting propagation transfer of "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" via "+RNS.prettyhexrep(self.outbound_propagation_node), RNS.LOG_DEBUG) - lxmessage.set_delivery_destination(self.outbound_propagation_link) - lxmessage.send() - else: - if lxmessage.representation == LXMessage.RESOURCE: - RNS.log("The transfer of "+str(lxmessage)+" is in progress ("+str(round(lxmessage.progress*100, 1))+"%)", RNS.LOG_DEBUG) - else: - RNS.log("Waiting for proof for "+str(lxmessage)+" sent as link packet", RNS.LOG_DEBUG) - elif self.outbound_propagation_link.status == RNS.Link.CLOSED: - RNS.log("The link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" was closed", RNS.LOG_DEBUG) - self.outbound_propagation_link = None - lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT - else: - # Simply wait for the link to become - # active or close - RNS.log("The propagation link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" is pending, waiting for link to become active: "+str(self.outbound_propagation_link.status), RNS.LOG_DEBUG) - else: - # No link exists, so we'll try to establish one, but - # only if we've never tried before, or the retry wait - # period has elapsed. - if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: - lxmessage.delivery_attempts += 1 - lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT - - if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: - if RNS.Transport.has_path(self.outbound_propagation_node): - RNS.log("Establishing link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" for propagation attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - propagation_node_identity = RNS.Identity.recall(self.outbound_propagation_node) - propagation_node_destination = RNS.Destination(propagation_node_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") - self.outbound_propagation_link = RNS.Link(propagation_node_destination, established_callback=self.process_outbound) - else: - RNS.log("No path known for propagation attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(self.outbound_propagation_node)+". Requesting path...", RNS.LOG_DEBUG) - RNS.Transport.request_path(self.outbound_propagation_node) - lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT - - else: - RNS.log("Max delivery attempts reached for propagated "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - self.fail_message(lxmessage) +APP_NAME = "lxmf" \ No newline at end of file diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py new file mode 100644 index 0000000..aee5263 --- /dev/null +++ b/LXMF/LXMPeer.py @@ -0,0 +1,244 @@ +import os +import time + +import RNS +import RNS.vendor.umsgpack as msgpack + +from .LXMF import APP_NAME + +class LXMPeer: + OFFER_REQUEST_PATH = "/offer" + MESSAGE_GET_PATH = "/get" + + IDLE = 0x00 + LINK_ESTABLISHING = 0x01 + LINK_READY = 0x02 + REQUEST_SENT = 0x03 + RESPONSE_RECEIVED = 0x04 + RESOURCE_TRANSFERRING = 0x05 + + ERROR_NO_IDENTITY = 0xf0 + + # Maximum amount of time a peer can + # be unreachable before it is removed + MAX_UNREACHABLE = 4*24*60*60 + + @staticmethod + def from_bytes(peer_bytes, router): + dictionary = msgpack.unpackb(peer_bytes) + + peer = LXMPeer(router, dictionary["destination_hash"]) + peer.peering_timebase = dictionary["peering_timebase"] + peer.alive = dictionary["alive"] + peer.last_heard = dictionary["last_heard"] + + for transient_id in dictionary["handled_ids"]: + if transient_id in router.propagation_entries: + peer.handled_messages[transient_id] = router.propagation_entries[transient_id] + + for transient_id in dictionary["unhandled_ids"]: + if transient_id in router.propagation_entries: + peer.unhandled_messages[transient_id] = router.propagation_entries[transient_id] + + return peer + + def to_bytes(self): + dictionary = {} + dictionary["peering_timebase"] = self.peering_timebase + dictionary["alive"] = self.alive + dictionary["last_heard"] = self.last_heard + dictionary["destination_hash"] = self.destination_hash + + handled_ids = [] + for transient_id in self.handled_messages: + handled_ids.append(transient_id) + + unhandled_ids = [] + for transient_id in self.unhandled_messages: + unhandled_ids.append(transient_id) + + dictionary["handled_ids"] = handled_ids + dictionary["unhandled_ids"] = unhandled_ids + + return msgpack.packb(dictionary) + + def __init__(self, router, destination_hash): + self.alive = False + self.last_heard = 0 + self.peering_timebase = 0 + + self.link = None + self.state = LXMPeer.IDLE + + self.unhandled_messages = {} + self.handled_messages = {} + + self.router = router + self.destination_hash = destination_hash + self.identity = RNS.Identity.recall(destination_hash) + self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + + def sync(self): + RNS.log("Initiating LXMF Propagation Node sync with peer "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_DEBUG) + + if not RNS.Transport.has_path(self.destination_hash): + RNS.log("No path to peer "+RNS.prettyhexrep(self.destination_hash)+" exists, requesting...", RNS.LOG_DEBUG) + RNS.Transport.request_path(self.destination_hash) + RNS.log("Path requested, retrying sync later", RNS.LOG_DEBUG) + + else: + if self.identity == None: + self.identity = RNS.Identity.recall(destination_hash) + self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + + if self.identity != None: + if len(self.unhandled_messages) > 0: + if self.state == LXMPeer.IDLE: + RNS.log("Establishing link for sync to peer "+RNS.prettyhexrep(self.destination_hash)+"...", RNS.LOG_DEBUG) + self.link = RNS.Link(self.destination, established_callback = self.link_established, closed_callback = self.link_closed) + self.state = LXMPeer.LINK_ESTABLISHING + + else: + if self.state == LXMPeer.LINK_READY: + self.alive = True + self.last_heard = time.time() + + RNS.log("Synchronisation link to peer "+RNS.prettyhexrep(self.destination_hash)+" established, preparing request...", RNS.LOG_DEBUG) + unhandled_ids = [] + purged_ids = [] + for transient_id in self.unhandled_messages: + if transient_id in self.router.propagation_entries: + unhandled_ids.append(transient_id) + else: + purged_ids.append(transient_id) + + for transient_id in purged_ids: + RNS.log("Dropping unhandled message "+RNS.prettyhexrep(transient_id)+" for peer "+RNS.prettyhexrep(self.destination_hash)+" since it no longer exists in the message store.", RNS.LOG_DEBUG) + self.unhandled_messages.pop(transient_id) + + RNS.log("Sending sync request to peer "+str(self.destination), RNS.LOG_DEBUG) + self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) + self.state = LXMPeer.REQUEST_SENT + else: + RNS.log("Could not request sync to peer "+RNS.prettyhexrep(self.destination_hash)+" since its identity could not be recalled.", RNS.LOG_ERROR) + + def request_failed(self, request_receipt): + RNS.log("Sync request to peer "+str(self.destination)+" failed", RNS.LOG_DEBUG) + if self.link != None: + self.link.teardown() + else: + self.state = LXMPeer.IDLE + + def offer_response(self, request_receipt): + try: + self.state = LXMPeer.RESPONSE_RECEIVED + response = request_receipt.response + + wanted_messages = [] + wanted_message_ids = [] + + if response == LXMPeer.ERROR_NO_IDENTITY: + if self.link != None: + RNS.log("Remote peer indicated that no identification was received, retrying...", RNS.LOG_DEBUG) + self.link.indentify() + self.state = LXMPeer.LINK_READY + self.sync() + + elif response == False: + # Peer already has all advertised messages + for transient_id in self.unhandled_messages: + message_entry = self.unhandled_messages[transient_id] + self.handled_messages[transient_id] = message_entry + + self.unhandled_messages = {} + + elif response == True: + # Peer wants all advertised messages + for transient_id in self.unhandled_messages: + wanted_messages.append(self.unhandled_messages[transient_id]) + wanted_message_ids.append(transient_id) + + else: + # Peer wants some advertised messages + peer_had_messages = [] + for transient_id in self.unhandled_messages.copy(): + # If the peer did not want the message, it has + # already received it from another peer. + if not transient_id in response: + message_entry = self.unhandled_messages.pop(transient_id) + self.handled_messages[transient_id] = message_entry + + for transient_id in response: + wanted_messages.append(self.unhandled_messages[transient_id]) + wanted_message_ids.append(transient_id) + + if len(wanted_messages) > 0: + RNS.log("Peer wanted "+str(len(wanted_messages))+" of the available messages", RNS.LOG_DEBUG) + + lxm_list = [] + + for message_entry in wanted_messages: + file_path = message_entry[1] + if os.path.isfile(file_path): + file = open(file_path, "rb") + lxmf_data = file.read() + file.close() + lxm_list.append(lxmf_data) + + data = msgpack.packb([time.time(), lxm_list]) + resource = RNS.Resource(data, self.link, callback = self.resource_concluded) + resource.transferred_messages = wanted_message_ids + self.state = LXMPeer.RESOURCE_TRANSFERRING + else: + self.state = LXMPeer.IDLE + + except Exception as e: + RNS.log("Error while handling offer response from peer "+str(self.destination), RNS.LOG_ERROR) + RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) + + if self.link != None: + self.link.teardown() + + self.link = None + self.state = LXMPeer.IDLE + + + def resource_concluded(self, resource): + if resource.status == RNS.Resource.COMPLETE: + for transient_id in resource.transferred_messages: + message = self.unhandled_messages.pop(transient_id) + self.handled_messages[transient_id] = message + self.state = LXMPeer.IDLE + self.link.teardown() + RNS.log("Sync to peer "+RNS.prettyhexrep(self.destination_hash)+" completed", RNS.LOG_DEBUG) + self.alive = True + self.last_heard = time.time() + else: + RNS.log("Resource transfer for LXMF peer sync failed to "+str(self.destination), RNS.LOG_DEBUG) + if self.link != None: + self.link.teardown() + else: + self.state = LXMPeer.IDLE + + + + def link_established(self, link): + self.link.identify(self.router.identity) + self.state = LXMPeer.LINK_READY + self.sync() + + def link_closed(self, link): + self.link = None + self.state = LXMPeer.IDLE + + def handle_message(self, transient_id): + if not transient_id in self.handled_messages and not transient_id in self.unhandled_messages: + # TODO: Remove at some point + RNS.log("The message "+RNS.prettyhexrep(transient_id)+" was added to distribution queue for "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_EXTREME) + self.unhandled_messages[transient_id] = self.router.propagation_entries[transient_id] + + def __str__(self): + if self.destination_hash: + return RNS.prettyhexrep(self.destination_hash) + else: + return "" \ No newline at end of file diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py new file mode 100644 index 0000000..0718478 --- /dev/null +++ b/LXMF/LXMRouter.py @@ -0,0 +1,974 @@ +import os +import time +import random +import atexit +import threading + +import RNS +import RNS.vendor.umsgpack as msgpack + +from .LXMF import APP_NAME + +from .LXMPeer import LXMPeer +from .LXMessage import LXMessage +from .Handlers import LXMFDeliveryAnnounceHandler +from .Handlers import LXMFPropagationAnnounceHandler + +class LXMRouter: + MAX_DELIVERY_ATTEMPTS = 4 + PROCESSING_INTERVAL = 5 + DELIVERY_RETRY_WAIT = 12 + PATH_REQUEST_WAIT = 5 + LINK_MAX_INACTIVITY = 10*60 + + MESSAGE_EXPIRY = 30*24*60*60 + + NODE_ANNOUNCE_DELAY = 20 + + AUTOPEER = True + AUTOPEER_MAXDEPTH = 4 + + PR_PATH_TIMEOUT = 10 + + PR_IDLE = 0x00 + PR_PATH_REQUESTED = 0x01 + PR_LINK_ESTABLISHING = 0x02 + PR_LINK_ESTABLISHED = 0x03 + PR_REQUEST_SENT = 0x04 + PR_RECEIVING = 0x05 + PR_RESPONSE_RECEIVED = 0x06 + PR_COMPLETE = 0x07 + + PR_ALL_MESSAGES = 0x00 + + def __init__(self, identity = None, storagepath = None, autopeer = AUTOPEER, autopeer_maxdepth = AUTOPEER_MAXDEPTH): + random.seed(os.urandom(10)) + + self.pending_inbound = [] + self.pending_outbound = [] + self.failed_outbound = [] + self.direct_links = {} + self.delivery_destinations = {} + + self.ignored_list = [] + + self.processing_outbound = False + self.processing_inbound = False + self.processing_count = 0 + + self.propagation_node = False + + if storagepath == None: + raise ValueError("LXMF cannot be initialised without a storage path") + else: + self.storagepath = storagepath+"/lxmf" + + self.outbound_propagation_node = None + self.outbound_propagation_link = None + + self.wants_download_on_path_available_from = None + self.wants_download_on_path_available_to = None + self.propagation_transfer_state = LXMRouter.PR_IDLE + self.propagation_transfer_progress = 0.0 + self.propagation_transfer_last_result = None + self.propagation_transfer_max_messages = None + self.locally_delivered_transient_ids = {} + + if identity == None: + identity = RNS.Identity() + + self.identity = identity + self.lxmf_query_destination = RNS.Destination(None, RNS.Destination.IN, RNS.Destination.PLAIN, APP_NAME, "query") + self.propagation_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation") + + self.autopeer = autopeer + self.autopeer_maxdepth = autopeer_maxdepth + + self.peers = {} + self.propagation_entries = {} + self.propagated_ids = {} + + RNS.Transport.register_announce_handler(LXMFDeliveryAnnounceHandler(self)) + RNS.Transport.register_announce_handler(LXMFPropagationAnnounceHandler(self)) + + self.__delivery_callback = None + + try: + if os.path.isfile(self.storagepath+"/local_deliveries"): + locally_delivered_file = open(self.storagepath+"/local_deliveries", "rb") + data = locally_delivered_file.read() + self.locally_delivered_transient_ids = msgpack.unpackb(data) + locally_delivered_file.close() + self.clean_transient_id_cache() + + except Exception as e: + RNS.log("Could not load locally delivered message ID cache from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + + atexit.register(self.exit_handler) + + job_thread = threading.Thread(target=self.jobloop) + job_thread.setDaemon(True) + job_thread.start() + + def exit_handler(self): + if self.propagation_node: + try: + serialised_peers = [] + for peer_id in self.peers: + peer = self.peers[peer_id] + serialised_peers.append(peer.to_bytes()) + + peers_file = open(self.storagepath+"/peers", "wb") + peers_file.write(msgpack.packb(serialised_peers)) + peers_file.close() + + RNS.log("Saved "+str(len(serialised_peers))+" peers to storage", RNS.LOG_DEBUG) + + except Exception as e: + RNS.log("Could not save propagation node peers to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + + try: + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) + + locally_delivered_file = open(self.storagepath+"/local_deliveries", "wb") + locally_delivered_file.write(msgpack.packb(self.locally_delivered_transient_ids)) + locally_delivered_file.close() + + except Exception as e: + RNS.log("Could not save locally delivered message ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + + + def register_delivery_identity(self, identity, display_name = None): + delivery_destination = RNS.Destination(identity, RNS.Destination.IN, RNS.Destination.SINGLE, "lxmf", "delivery") + delivery_destination.set_packet_callback(self.delivery_packet) + delivery_destination.set_link_established_callback(self.delivery_link_established) + delivery_destination.display_name = display_name + + if display_name != None: + delivery_destination.set_default_app_data(display_name.encode("utf-8")) + + self.delivery_destinations[delivery_destination.hash] = delivery_destination + return delivery_destination + + def register_delivery_callback(self, callback): + self.__delivery_callback = callback + + def set_outbound_propagation_node(self, destination_hash): + if len(destination_hash) != RNS.Identity.TRUNCATED_HASHLENGTH//8 or type(destination_hash) != bytes: + raise ValueError("Invalid destination hash for outbound propagation node") + else: + self.outbound_propagation_node = destination_hash + if self.outbound_propagation_link != None: + if self.outbound_propagation_link.destination.hash != destination_hash: + self.outbound_propagation_link.teardown() + self.outbound_propagation_link = None + + def get_outbound_propagation_node(self): + return self.outbound_propagation_node + + def cancel_propagation_node_requests(self): + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + self.outbound_propagation_link = None + + self.acknowledge_sync_completion() + + def acknowledge_sync_completion(self): + self.propagation_transfer_state = LXMRouter.PR_IDLE + self.propagation_transfer_progress = 0.0 + self.propagation_transfer_last_result = None + self.wants_download_on_path_available_from = None + self.wants_download_on_path_available_to = None + + def request_messages_from_propagation_node(self, identity, max_messages = PR_ALL_MESSAGES): + if max_messages == None: + max_messages = LXMRouter.PR_ALL_MESSAGES + + self.propagation_transfer_max_messages = max_messages + if self.outbound_propagation_node != None: + self.propagation_transfer_progress = 0.0 + if self.outbound_propagation_link != None and self.outbound_propagation_link.status == RNS.Link.ACTIVE: + self.propagation_transfer_state = LXMRouter.PR_LINK_ESTABLISHED + self.outbound_propagation_link.identify(identity) + self.outbound_propagation_link.request( + LXMPeer.MESSAGE_GET_PATH, + [None, None], + response_callback=self.message_list_response, + failed_callback=self.message_get_failed + ) + self.propagation_transfer_state = LXMRouter.PR_REQUEST_SENT + else: + if self.outbound_propagation_link == None: + if RNS.Transport.has_path(self.outbound_propagation_node): + self.wants_download_on_path_available_from = None + self.propagation_transfer_state = LXMRouter.PR_LINK_ESTABLISHING + RNS.log("Establishing link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" for message download", RNS.LOG_DEBUG) + propagation_node_identity = RNS.Identity.recall(self.outbound_propagation_node) + propagation_node_destination = RNS.Destination(propagation_node_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + def msg_request_established_callback(link): + self.request_messages_from_propagation_node(identity, self.propagation_transfer_max_messages) + + self.outbound_propagation_link = RNS.Link(propagation_node_destination, established_callback=msg_request_established_callback) + else: + RNS.log("No path known for message download from propagation node "+RNS.prettyhexrep(self.outbound_propagation_node)+". Requesting path...", RNS.LOG_DEBUG) + RNS.Transport.request_path(self.outbound_propagation_node) + self.wants_download_on_path_available_from = self.outbound_propagation_node + self.wants_download_on_path_available_to = identity + self.wants_download_on_path_available_timeout = time.time() + LXMRouter.PR_PATH_TIMEOUT + self.propagation_transfer_state = LXMRouter.PR_PATH_REQUESTED + self.request_messages_path_job() + else: + # TODO: Remove at some point + RNS.log("Waiting for propagation node link to become active", RNS.LOG_EXTREME) + else: + RNS.log("Cannot request LXMF propagation node sync, no default propagation node configured", RNS.LOG_WARNING) + + + def request_messages_path_job(self): + job_thread = threading.Thread(target=self.__request_messages_path_job) + job_thread.setDaemon(True) + job_thread.start() + + def __request_messages_path_job(self): + while not RNS.Transport.has_path(self.wants_download_on_path_available_from) and time.time() < self.wants_download_on_path_available_timeout: + time.sleep(0.1) + + if RNS.Transport.has_path(self.wants_download_on_path_available_from): + self.request_messages_from_propagation_node(self.wants_download_on_path_available_to, self.propagation_transfer_max_messages) + else: + RNS.log("Propagation node path request timed out", RNS.LOG_DEBUG) + self.acknowledge_sync_completion() + + + def has_message(self, transient_id): + if transient_id in self.locally_delivered_transient_ids: + return True + else: + return False + + def message_get_failed(self, request_receipt): + RNS.log("Message list/get request failed", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + + def message_list_response(self, request_receipt): + if request_receipt.response == LXMPeer.ERROR_NO_IDENTITY: + RNS.log("Propagation node indicated missing identification on list request, tearing down link.", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + else: + if request_receipt.response != None: + haves = [] + wants = [] + if len(request_receipt.response) > 0: + for transient_id in request_receipt.response: + if self.has_message(transient_id): + haves.append(transient_id) + else: + if self.propagation_transfer_max_messages == LXMRouter.PR_ALL_MESSAGES or len(wants) < self.propagation_transfer_max_messages: + wants.append(transient_id) + + request_receipt.link.request( + LXMPeer.MESSAGE_GET_PATH, + [wants, haves], + response_callback=self.message_get_response, + failed_callback=self.message_get_failed, + progress_callback=self.message_get_progress + ) + else: + self.propagation_transfer_state = LXMRouter.PR_COMPLETE + self.propagation_transfer_progress = 1.0 + self.propagation_transfer_last_result = 0 + + def message_get_progress(self, request_receipt): + self.propagation_transfer_state = LXMRouter.PR_RECEIVING + self.propagation_transfer_progress = request_receipt.get_progress() + + def message_get_response(self, request_receipt): + if request_receipt.response == LXMPeer.ERROR_NO_IDENTITY: + RNS.log("Propagation node indicated missing identification on get request, tearing down link.", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + else: + if request_receipt.response != None and len(request_receipt.response) > 0: + haves = [] + for lxmf_data in request_receipt.response: + self.lxmf_propagation(lxmf_data) + haves.append(RNS.Identity.full_hash(lxmf_data)) + + # Return a list of successfully received messages to the node. + # This deletes the messages on the propagation node. + # TODO: Add option to keep messages on node. + request_receipt.link.request( + LXMPeer.MESSAGE_GET_PATH, + [None, haves], + # response_callback=self.message_syncfinal_response, + failed_callback=self.message_get_failed, + # progress_callback=self.message_get_progress + ) + + self.propagation_transfer_state = LXMRouter.PR_COMPLETE + self.propagation_transfer_progress = 1.0 + self.propagation_transfer_last_result = len(request_receipt.response) + + def announce(self, destination_hash): + if destination_hash in self.delivery_destinations: + delivery_destination = self.delivery_destinations[destination_hash] + delivery_destination.announce(delivery_destination.display_name.encode("utf-8")) + + def handle_outbound(self, lxmessage): + lxmessage.state = LXMessage.OUTBOUND + if not lxmessage.packed: + lxmessage.pack() + + lxmessage.determine_transport_encryption() + + while self.processing_outbound: + time.sleep(0.1) + + self.pending_outbound.append(lxmessage) + self.process_outbound() + + def lxmf_delivery(self, lxmf_data, destination_type = None): + try: + message = LXMessage.unpack_from_bytes(lxmf_data) + + if destination_type == RNS.Destination.SINGLE: + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + elif destination_type == RNS.Destination.GROUP: + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES + elif destination_type == RNS.Destination.LINK: + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + else: + message.transport_encrypted = False + message.transport_encryption = None + + if message.source_hash in self.ignored_list: + RNS.log(str(self)+" ignored message from "+RNS.prettyhexrep(message.source_hash), RNS.LOG_DEBUG) + return False + + if self.__delivery_callback != None and callable(self.__delivery_callback): + try: + self.__delivery_callback(message) + except Exception as e: + RNS.log("An error occurred in the external delivery callback for "+str(message), RNS.LOG_ERROR) + + return True + + except Exception as e: + RNS.log("Could not assemble LXMF message from received data", RNS.LOG_NOTICE) + RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + return False + + + def delivery_packet(self, data, packet): + try: + if packet.destination_type != RNS.Destination.LINK: + lxmf_data = b"" + lxmf_data += packet.destination.hash + lxmf_data += data + else: + lxmf_data = data + + if self.lxmf_delivery(lxmf_data, packet.destination_type): + packet.prove() + + except Exception as e: + RNS.log("Exception occurred while parsing incoming LXMF data.", RNS.LOG_ERROR) + RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) + + def delivery_link_established(self, link): + link.set_packet_callback(self.delivery_packet) + link.set_resource_strategy(RNS.Link.ACCEPT_ALL) + link.set_resource_started_callback(self.resource_transfer_began) + link.set_resource_concluded_callback(self.delivery_resource_concluded) + + def delivery_link_closed(self, link): + pass + + def resource_transfer_began(self, resource): + RNS.log("Transfer began for resource "+str(resource), RNS.LOG_DEBUG) + + def delivery_resource_concluded(self, resource): + RNS.log("Transfer concluded for delivery resource "+str(resource), RNS.LOG_DEBUG) + if resource.status == RNS.Resource.COMPLETE: + self.lxmf_delivery(resource.data.read(), resource.link.type) + + + def enable_propagation(self): + try: + self.messagepath = self.storagepath+"/messagestore" + + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) + + if not os.path.isdir(self.messagepath): + os.makedirs(self.messagepath) + + self.propagation_entries = {} + for filename in os.listdir(self.messagepath): + components = filename.split("_") + if len(components) == 2: + if float(components[1]) > 0: + if len(components[0]) == RNS.Identity.HASHLENGTH//8*2: + transient_id = bytes.fromhex(components[0]) + received = components[1] + + filepath = self.messagepath+"/"+filename + file = open(filepath, "rb") + destination_hash = file.read(LXMessage.DESTINATION_LENGTH) + file.close() + + self.propagation_entries[transient_id] = [destination_hash, filepath] + + if os.path.isfile(self.storagepath+"/peers"): + peers_file = open(self.storagepath+"/peers", "rb") + peers_data = peers_file.read() + + if len(peers_data) > 0: + serialised_peers = msgpack.unpackb(peers_data) + + for serialised_peer in serialised_peers: + peer = LXMPeer.from_bytes(serialised_peer, self) + if peer.identity != None: + self.peers[peer.destination_hash] = peer + RNS.log("Loaded peer "+RNS.prettyhexrep(peer.destination_hash)+" with "+str(len(peer.unhandled_messages))+" unhandled messages", RNS.LOG_DEBUG) + else: + RNS.log("Peer "+RNS.prettyhexrep(peer.destination_hash)+" could not be loaded, because its identity could not be recalled. Dropping peer.", RNS.LOG_DEBUG) + + + self.propagation_node = True + self.propagation_destination.set_link_established_callback(self.propagation_link_established) + self.propagation_destination.set_packet_callback(self.propagation_packet) + + self.propagation_destination.register_request_handler(LXMPeer.OFFER_REQUEST_PATH, self.offer_request, allow = RNS.Destination.ALLOW_ALL) + self.propagation_destination.register_request_handler(LXMPeer.MESSAGE_GET_PATH, self.message_get_request, allow = RNS.Destination.ALLOW_ALL) + + self.announce_propagation_node() + + except Exception as e: + RNS.log("Could not enable propagation node. The contained exception was: "+str(e), RNS.LOG_ERROR) + raise e + RNS.panic() + + def disable_propagation(self): + self.propagation_node = False + self.announce_propagation_node() + + def announce_propagation_node(self): + def delayed_announce(): + time.sleep(LXMRouter.NODE_ANNOUNCE_DELAY) + data = msgpack.packb([self.propagation_node, int(time.time())]) + self.propagation_destination.announce(app_data=data) + + da_thread = threading.Thread(target=delayed_announce) + da_thread.setDaemon(True) + da_thread.start() + + def offer_request(self, path, data, request_id, remote_identity, requested_at): + if remote_identity == None: + return LXMPeer.ERROR_NO_IDENTITY + else: + try: + transient_ids = data + wanted_ids = [] + + for transient_id in transient_ids: + if not transient_id in self.propagation_entries: + wanted_ids.append(transient_id) + + if len(wanted_ids) == 0: + return False + + elif len(wanted_ids) == len(transient_ids): + return True + + else: + return wanted_ids + + except Exception as e: + RNS.log("Error occurred while generating response for sync request, the contained exception was: "+str(e), RNS.LOG_DEBUG) + return None + + def message_get_request(self, path, data, request_id, remote_identity, requested_at): + if remote_identity == None: + return LXMPeer.ERROR_NO_IDENTITY + else: + try: + remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") + + # If both want and have fields are empty, send a list of + # available messages. + if data[0] == None and data[1] == None: + available_messages = [] + for transient_id in self.propagation_entries: + message_entry = self.propagation_entries[transient_id] + if message_entry[0] == remote_destination.hash: + message_size = os.path.getsize(message_entry[1]) + available_entry = [transient_id, message_size] + available_messages.append(available_entry) + + available_messages.sort(key=lambda e: e[1], reverse=False) + + transient_ids = [] + for available_entry in available_messages: + transient_ids.append(available_entry[0]) + + return transient_ids + + else: + # Process messages the client already have + if data[1] != None and len(data[1]) > 0: + for transient_id in data[1]: + if transient_id in self.propagation_entries and self.propagation_entries[transient_id][0] == remote_destination.hash: + try: + filepath = self.propagation_entries[transient_id][1] + self.propagation_entries.pop(transient_id) + os.unlink(filepath) + RNS.log("Client "+RNS.prettyhexrep(remote_destination.hash)+" purged message "+RNS.prettyhexrep(transient_id)+" at "+str(filepath), RNS.LOG_DEBUG) + + except Exception as e: + RNS.log("Error while processing message purge request from "+RNS.prettyhexrep(remote_destination.hash)+". The contained exception was: "+str(e), RNS.LOG_ERROR) + + + # Process wanted messages + response_messages = [] + if data[0] != None and len(data[0]) > 0: + for transient_id in data[0]: + if transient_id in self.propagation_entries and self.propagation_entries[transient_id][0] == remote_destination.hash: + try: + filepath = self.propagation_entries[transient_id][1] + RNS.log("Client "+RNS.prettyhexrep(remote_destination.hash)+" requested message "+RNS.prettyhexrep(transient_id)+" at "+str(filepath), RNS.LOG_DEBUG) + + message_file = open(filepath, "rb") + lxmf_data = message_file.read() + response_messages.append(lxmf_data) + message_file.close() + + except Exception as e: + RNS.log("Error while processing message download request from "+RNS.prettyhexrep(remote_destination.hash)+". The contained exception was: "+str(e), RNS.LOG_ERROR) + + return response_messages + + + except Exception as e: + RNS.log("Error occurred while generating response for download request, the contained exception was: "+str(e), RNS.LOG_DEBUG) + return None + + def propagation_link_established(self, link): + link.set_packet_callback(self.propagation_packet) + link.set_resource_strategy(RNS.Link.ACCEPT_ALL) + link.set_resource_started_callback(self.resource_transfer_began) + link.set_resource_concluded_callback(self.propagation_resource_concluded) + + + def propagation_packet(self, data, packet): + try: + if packet.destination_type != RNS.Destination.LINK: + pass + else: + data = msgpack.unpackb(data) + remote_timebase = data[0] + + messages = data[1] + for lxmf_data in messages: + self.lxmf_propagation(lxmf_data) + + packet.prove() + + except Exception as e: + RNS.log("Exception occurred while parsing incoming LXMF propagation data.", RNS.LOG_ERROR) + RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) + + + def propagation_resource_concluded(self, resource): + RNS.log("Transfer concluded for incoming propagation resource "+str(resource), RNS.LOG_DEBUG) + if resource.status == RNS.Resource.COMPLETE: + # TODO: The peer this was received from should + # have the transient id added to it's list of + # already handled messages. + try: + data = msgpack.unpackb(resource.data.read()) + + if type(data) == list and len(data) == 2 and type(data[0] == float) and type(data[1]) == list: + # This is a series of propagation messages from a peer or originator + remote_timebase = data[0] + remote_hash = None + remote_identity = resource.link.get_remote_identity() + + if remote_identity != None: + remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + remote_hash = remote_destination.hash + + if not remote_hash in self.peers: + self.peer(remote_hash, remote_timebase) + + messages = data[1] + for lxmf_data in messages: + if remote_hash != None and remote_hash in self.peers: + transient_id = RNS.Identity.full_hash(lxmf_data) + peer = self.peers[remote_hash] + peer.handled_messages[transient_id] = [transient_id, remote_timebase, lxmf_data] + + self.lxmf_propagation(lxmf_data) + else: + RNS.log("Invalid data structure received at propagation destination, ignoring", RNS.LOG_DEBUG) + + except Exception as e: + RNS.log("Error while unpacking received propagation resource", RNS.LOG_DEBUG) + + + def lxmf_propagation(self, lxmf_data): + try: + if len(lxmf_data) >= LXMessage.LXMF_OVERHEAD: + transient_id = RNS.Identity.full_hash(lxmf_data) + if not transient_id in self.propagation_entries and not transient_id in self.propagated_ids: + received = time.time() + propagation_entry = [transient_id, received, lxmf_data] + destination_hash = lxmf_data[:LXMessage.DESTINATION_LENGTH] + + if destination_hash in self.delivery_destinations: + delivery_destination = self.delivery_destinations[destination_hash] + encrypted_lxmf_data = lxmf_data[LXMessage.DESTINATION_LENGTH:] + decrypted_lxmf_data = delivery_destination.decrypt(encrypted_lxmf_data) + delivery_data = lxmf_data[:LXMessage.DESTINATION_LENGTH]+decrypted_lxmf_data + self.lxmf_delivery(delivery_data, delivery_destination.type) + self.locally_delivered_transient_ids[transient_id] = time.time() + + else: + if self.propagation_node: + file_path = self.messagepath+"/"+RNS.hexrep(transient_id, delimit=False)+"_"+str(received) + msg_file = open(file_path, "wb") + msg_file.write(lxmf_data) + msg_file.close() + + self.propagation_entries[transient_id] = [destination_hash, file_path] + + RNS.log("Received propagated LXMF message "+RNS.prettyhexrep(transient_id)+", adding to peer distribution queues...", RNS.LOG_DEBUG) + for peer_id in self.peers: + peer = self.peers[peer_id] + peer.handle_message(transient_id) + + return True + + return False + + except Exception as e: + RNS.log("Could not assemble propagated LXMF message from received data", RNS.LOG_DEBUG) + RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + return False + + + def peer(self, destination_hash, timestamp): + if destination_hash in self.peers: + peer = self.peers[destination_hash] + peer.alive = True + peer.peering_timebase = timestamp + peer.last_heard = time.time() + else: + peer = LXMPeer(self, destination_hash) + peer.alive = True + peer.last_heard = time.time() + self.peers[destination_hash] = peer + RNS.log("Peered with "+str(peer.destination)) + + def unpeer(self, destination_hash, timestamp = None): + if timestamp == None: + timestamp = int(time.time()) + + if destination_hash in self.peers: + peer = self.peers[destination_hash] + + if timestamp >= peer.peering_timebase: + self.peers.pop(destination_hash) + RNS.log("Broke peering with "+str(peer.destination)) + + + def jobloop(self): + while (True): + # TODO: Improve this to scheduling, so manual + # triggers can delay next run + self.jobs() + time.sleep(LXMRouter.PROCESSING_INTERVAL) + + JOB_OUTBOUND_INTERVAL = 1 + JOB_LINKS_INTERVAL = 1 + JOB_TRANSIENT_INTERVAL = 60 + JOB_STORE_INTERVAL = 120 + JOB_PEERSYNC_INTERVAL = 12 + def jobs(self): + self.processing_count += 1 + + if self.processing_count % LXMRouter.JOB_OUTBOUND_INTERVAL == 0: + self.process_outbound() + + if self.processing_count % LXMRouter.JOB_LINKS_INTERVAL == 0: + self.clean_links() + + if self.processing_count % LXMRouter.JOB_TRANSIENT_INTERVAL == 0: + self.clean_transient_id_cache() + + if self.processing_count % LXMRouter.JOB_STORE_INTERVAL == 0: + self.clean_message_store() + + if self.processing_count % LXMRouter.JOB_PEERSYNC_INTERVAL == 0: + self.sync_peers() + + + def clean_links(self): + closed_links = [] + for link_hash in self.direct_links: + link = self.direct_links[link_hash] + inactive_time = link.inactive_for() + + if inactive_time > LXMRouter.LINK_MAX_INACTIVITY: + link.teardown() + closed_links.append(link_hash) + + for link_hash in closed_links: + cleaned_link = self.direct_links.pop(link_hash) + RNS.log("Cleaned link "+str(cleaned_link), RNS.LOG_DEBUG) + + if self.outbound_propagation_link != None and self.outbound_propagation_link.status == RNS.Link.CLOSED: + self.outbound_propagation_link = None + self.acknowledge_sync_completion() + RNS.log("Cleaned outbound propagation link", RNS.LOG_DEBUG) + + def clean_transient_id_cache(self): + now = time.time() + removed_entries = [] + for transient_id in self.locally_delivered_transient_ids: + timestamp = self.locally_delivered_transient_ids[transient_id] + if now > timestamp+LXMRouter.MESSAGE_EXPIRY*1.1: + removed_entries.append(transient_id) + + for transient_id in removed_entries: + self.locally_delivered_transient_ids.pop(transient_id) + RNS.log("Cleaned "+RNS.prettyhexrep(transient_id)+" from local delivery cache", RNS.LOG_DEBUG) + + + def clean_message_store(self): + now = time.time() + removed_entries = {} + for transient_id in self.propagation_entries: + entry = self.propagation_entries[transient_id] + filepath = entry[1] + components = filepath.split("_") + + if len(components) == 2 and float(components[1]) > 0 and len(os.path.split(components[0])[1]) == (RNS.Identity.HASHLENGTH//8)*2: + timestamp = float(components[1]) + if now > timestamp+LXMRouter.MESSAGE_EXPIRY: + RNS.log("Purging message "+RNS.prettyhexrep(transient_id)+" due to expiry", RNS.LOG_DEBUG) + removed_entries[transient_id] = filepath + else: + RNS.log("Purging message "+RNS.prettyhexrep(transient_id)+" due to invalid file path", RNS.LOG_WARNING) + removed_entries[transient_id] = filepath + + removed_count = 0 + for transient_id in removed_entries: + try: + filepath = removed_entries[transient_id] + self.propagation_entries.pop(transient_id) + if os.path.isfile(filepath): + os.unlink(filepath) + removed_count += 1 + except Exception as e: + RNS.log("Could not remove "+RNS.prettyhexrep(transient_id)+" from message store. The contained exception was: "+str(e), RNS.LOG_ERROR) + + if removed_count > 0: + RNS.log("Cleaned "+str(removed_count)+" entries from the message store", RNS.LOG_DEBUG) + + + def sync_peers(self): + culled_peers = [] + waiting_peers = [] + for peer_id in self.peers: + peer = self.peers[peer_id] + if time.time() > peer.last_heard + LXMPeer.MAX_UNREACHABLE: + culled_peers.append(peer_id) + else: + if peer.state == LXMPeer.IDLE and len(peer.unhandled_messages) > 0: + waiting_peers.append(peer) + + if len(waiting_peers) > 0: + RNS.log("Randomly selecting peer to sync from "+str(len(waiting_peers))+" waiting peers.", RNS.LOG_DEBUG) + selected_index = random.randint(0,len(waiting_peers)-1) + selected_peer = waiting_peers[selected_index] + RNS.log("Selected waiting peer "+str(selected_index)+": "+RNS.prettyhexrep(selected_peer.destination.hash), RNS.LOG_DEBUG) + selected_peer.sync() + + for peer_id in culled_peers: + RNS.log("Removing peer "+RNS.prettyhexrep(peer_id)+" due to excessive unreachability", RNS.LOG_WARNING) + try: + if peer_id in self.peers: + self.peers.pop(peer_id) + except Exception as e: + RNS.log("Error while removing peer "+RNS.prettyhexrep(peer_id)+". The contained exception was: "+str(e), RNS.LOG_ERROR) + + + def ignore_destination(self, destination_hash): + if not destination_hash in self.ignored_list: + self.ignored_list.append(destination_hash) + + def unignore_destination(self, destination_hash): + if destination_hash in self.ignored_list: + self.ignored_list.remove(destination_hash) + + def fail_message(self, lxmessage): + RNS.log(str(lxmessage)+" failed to send", RNS.LOG_DEBUG) + + self.pending_outbound.remove(lxmessage) + self.failed_outbound.append(lxmessage) + + lxmessage.state = LXMessage.FAILED + if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): + lxmessage.failed_callback(lxmessage) + + def __str__(self): + return "" + + def process_outbound(self, sender = None): + if self.processing_outbound: + return + + for lxmessage in self.pending_outbound: + if lxmessage.state == LXMessage.DELIVERED: + RNS.log("Delivery has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) + self.pending_outbound.remove(lxmessage) + elif lxmessage.state == LXMessage.SENT: + RNS.log("Propagation has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) + self.pending_outbound.remove(lxmessage) + else: + RNS.log("Starting outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + # Outbound handling for opportunistic messages + if lxmessage.method == LXMessage.OPPORTUNISTIC: + if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: + if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: + lxmessage.delivery_attempts += 1 + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + RNS.log("Opportunistic delivery attempt "+str(lxmessage.delivery_attempts)+" for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + lxmessage.send() + else: + RNS.log("Max delivery attempts reached for oppertunistic "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + self.fail_message(lxmessage) + + # Outbound handling for messages transferred + # over a direct link to the final recipient + elif lxmessage.method == LXMessage.DIRECT: + if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: + delivery_destination_hash = lxmessage.get_destination().hash + + if delivery_destination_hash in self.direct_links: + # A link already exists, so we'll try to use it + # to deliver the message + direct_link = self.direct_links[delivery_destination_hash] + if direct_link.status == RNS.Link.ACTIVE: + if lxmessage.state != LXMessage.SENDING: + RNS.log("Starting transfer of "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + lxmessage.set_delivery_destination(direct_link) + lxmessage.send() + else: + if lxmessage.representation == LXMessage.RESOURCE: + RNS.log("The transfer of "+str(lxmessage)+" is in progress ("+str(round(lxmessage.progress*100, 1))+"%)", RNS.LOG_DEBUG) + else: + RNS.log("Waiting for proof for "+str(lxmessage)+" sent as link packet", RNS.LOG_DEBUG) + elif direct_link.status == RNS.Link.CLOSED: + if direct_link.activated_at != None: + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was closed", RNS.LOG_DEBUG) + else: + if not hasattr(lxmessage, "path_request_retried"): + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was never activated, retrying path request...", RNS.LOG_DEBUG) + RNS.Transport.request_path(lxmessage.get_destination().hash) + lxmessage.path_request_retried = True + else: + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was never activated", RNS.LOG_DEBUG) + + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + + lxmessage.set_delivery_destination(None) + self.direct_links.pop(delivery_destination_hash) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + else: + # Simply wait for the link to become + # active or close + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" is pending, waiting for link to become active", RNS.LOG_DEBUG) + else: + # No link exists, so we'll try to establish one, but + # only if we've never tried before, or the retry wait + # period has elapsed. + if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: + lxmessage.delivery_attempts += 1 + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + + if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: + if RNS.Transport.has_path(lxmessage.get_destination().hash): + RNS.log("Establishing link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + delivery_link = RNS.Link(lxmessage.get_destination()) + delivery_link.set_link_established_callback(self.process_outbound) + self.direct_links[delivery_destination_hash] = delivery_link + else: + RNS.log("No path known for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+". Requesting path...", RNS.LOG_DEBUG) + RNS.Transport.request_path(lxmessage.get_destination().hash) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + else: + RNS.log("Max delivery attempts reached for direct "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + self.fail_message(lxmessage) + + # Outbound handling for messages transported via + # propagation to a LXMF router network. + elif lxmessage.method == LXMessage.PROPAGATED: + RNS.log("Attempting propagated delivery for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + + if self.outbound_propagation_node == None: + RNS.log("No outbound propagation node specified for propagated "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_ERROR) + self.fail_message(lxmessage) + else: + if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: + + if self.outbound_propagation_link != None: + # A link already exists, so we'll try to use it + # to deliver the message + if self.outbound_propagation_link.status == RNS.Link.ACTIVE: + if lxmessage.state != LXMessage.SENDING: + RNS.log("Starting propagation transfer of "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" via "+RNS.prettyhexrep(self.outbound_propagation_node), RNS.LOG_DEBUG) + lxmessage.set_delivery_destination(self.outbound_propagation_link) + lxmessage.send() + else: + if lxmessage.representation == LXMessage.RESOURCE: + RNS.log("The transfer of "+str(lxmessage)+" is in progress ("+str(round(lxmessage.progress*100, 1))+"%)", RNS.LOG_DEBUG) + else: + RNS.log("Waiting for proof for "+str(lxmessage)+" sent as link packet", RNS.LOG_DEBUG) + elif self.outbound_propagation_link.status == RNS.Link.CLOSED: + RNS.log("The link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" was closed", RNS.LOG_DEBUG) + self.outbound_propagation_link = None + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + else: + # Simply wait for the link to become + # active or close + RNS.log("The propagation link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" is pending, waiting for link to become active: "+str(self.outbound_propagation_link.status), RNS.LOG_DEBUG) + else: + # No link exists, so we'll try to establish one, but + # only if we've never tried before, or the retry wait + # period has elapsed. + if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: + lxmessage.delivery_attempts += 1 + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + + if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: + if RNS.Transport.has_path(self.outbound_propagation_node): + RNS.log("Establishing link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" for propagation attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + propagation_node_identity = RNS.Identity.recall(self.outbound_propagation_node) + propagation_node_destination = RNS.Destination(propagation_node_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + self.outbound_propagation_link = RNS.Link(propagation_node_destination, established_callback=self.process_outbound) + else: + RNS.log("No path known for propagation attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(self.outbound_propagation_node)+". Requesting path...", RNS.LOG_DEBUG) + RNS.Transport.request_path(self.outbound_propagation_node) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + + else: + RNS.log("Max delivery attempts reached for propagated "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + self.fail_message(lxmessage) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py new file mode 100644 index 0000000..6f27d0a --- /dev/null +++ b/LXMF/LXMessage.py @@ -0,0 +1,525 @@ +import RNS +import RNS.vendor.umsgpack as msgpack + +import time + +from .LXMF import APP_NAME + +class LXMessage: + DRAFT = 0x00 + OUTBOUND = 0x01 + SENDING = 0x02 + SENT = 0x04 + DELIVERED = 0x08 + FAILED = 0xFF + states = [DRAFT, OUTBOUND, SENDING, SENT, DELIVERED, FAILED] + + UNKNOWN = 0x00 + PACKET = 0x01 + RESOURCE = 0x02 + representations = [UNKNOWN, PACKET, RESOURCE] + + OPPORTUNISTIC = 0x01 + DIRECT = 0x02 + PROPAGATED = 0x03 + valid_methods = [OPPORTUNISTIC, DIRECT, PROPAGATED] + + SOURCE_UNKNOWN = 0x01 + SIGNATURE_INVALID = 0x02 + unverified_reasons = [SOURCE_UNKNOWN, SIGNATURE_INVALID] + + DESTINATION_LENGTH = RNS.Identity.TRUNCATED_HASHLENGTH//8 + SIGNATURE_LENGTH = RNS.Identity.SIGLENGTH//8 + + # LXMF overhead is 99 bytes per message: + # 10 bytes for destination hash + # 10 bytes for source hash + # 64 bytes for Ed25519 signature + # 8 bytes for timestamp + # 7 bytes for msgpack structure + LXMF_OVERHEAD = 2*DESTINATION_LENGTH + SIGNATURE_LENGTH + 8 + 7 + + # With an MTU of 500, the maximum amount of data + # we can send in a single encrypted packet is + # 383 bytes. + ENCRYPTED_PACKET_MDU = RNS.Packet.ENCRYPTED_MDU + + # The max content length we can fit in LXMF message + # inside a single RNS packet is the encrypted MDU, minus + # the LXMF overhead. We can optimise a bit though, by + # inferring the destination hash from the destination + # field of the packet, therefore we also add the length + # of a destination hash to the calculation. With default + # RNS and LXMF parameters, the largest single-packet + # LXMF message we can send is 294 bytes. If a message + # is larger than that, a Reticulum link will be used. + ENCRYPTED_PACKET_MAX_CONTENT = ENCRYPTED_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH + + # Links can carry a larger MDU, due to less overhead per + # packet. The link MDU with default Reticulum parameters + # is 431 bytes. + LINK_PACKET_MDU = RNS.Link.MDU + + # Which means that we can deliver single-packet LXMF + # messages with content of up to 332 bytes over a link. + # If a message is larger than that, LXMF will sequence + # and transfer it as a RNS resource over the link instead. + LINK_PACKET_MAX_CONTENT = LINK_PACKET_MDU - LXMF_OVERHEAD + + # For plain packets without encryption, we can + # fit up to 388 bytes of content. + PLAIN_PACKET_MDU = RNS.Packet.PLAIN_MDU + PLAIN_PACKET_MAX_CONTENT = PLAIN_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH + + # Descriptive strings regarding transport encryption + ENCRYPTION_DESCRIPTION_AES = "AES-128" + ENCRYPTION_DESCRIPTION_EC = "Curve25519" + ENCRYPTION_DESCRIPTION_UNENCRYPTED = "Unencrypted" + + def __str__(self): + if self.hash != None: + return "" + else: + return "" + + def __init__(self, destination, source, content = "", title = "", fields = None, desired_method = None, destination_hash = None, source_hash = None): + if isinstance(destination, RNS.Destination) or destination == None: + self.__destination = destination + if destination != None: + self.destination_hash = destination.hash + else: + self.destination_hash = destination_hash + else: + raise ValueError("LXMessage initialised with invalid destination") + + if isinstance(source, RNS.Destination) or source == None: + self.__source = source + if source != None: + self.source_hash = source.hash + else: + self.source_hash = source_hash + else: + raise ValueError("LXMessage initialised with invalid source") + + self.set_title_from_string(title) + self.set_content_from_string(content) + + self.set_fields(fields) + + self.payload = None + self.timestamp = None + self.signature = None + self.hash = None + self.packed = None + self.progress = None + self.state = LXMessage.DRAFT + self.method = LXMessage.UNKNOWN + + self.incoming = False + self.signature_validated = False + self.unverified_reason = None + + self.representation = LXMessage.UNKNOWN + self.desired_method = desired_method + self.delivery_attempts = 0 + self.transport_encrypted = False + self.transport_encryption = None + self.packet_representation = None + self.resource_representation = None + self.__delivery_destination = None + self.__delivery_callback = None + self.failed_callback = None + + def set_title_from_string(self, title_string): + self.title = title_string.encode("utf-8") + + def set_title_from_bytes(self, title_bytes): + self.title = title_bytes + + def title_as_string(self): + return self.title.decode("utf-8") + + def set_content_from_string(self, content_string): + self.content = content_string.encode("utf-8") + + def set_content_from_bytes(self, content_bytes): + self.content = content_bytes + + def content_as_string(self): + return self.content.decode("utf-8") + + def set_fields(self, fields): + if isinstance(fields, dict) or fields == None: + self.fields = fields + else: + raise ValueError("LXMessage property \"fields\" can only be dict or None") + + def get_fields(self): + return self.__fields + + def set_destination(self, destination): + if self.destination == None: + if isinstance(destination, RNS.Destination): + self.__destination = destination + else: + raise ValueError("Invalid destination set on LXMessage") + else: + raise ValueError("Cannot reassign destination on LXMessage") + + def get_destination(self): + return self.__destination + + def set_source(self, source): + if self.source == None: + if isinstance(source, RNS.Destination): + self.__source = source + else: + raise ValueError("Invalid source set on LXMessage") + else: + raise ValueError("Cannot reassign source on LXMessage") + + def get_source(self): + return self.__source + + def set_delivery_destination(self, delivery_destination): + self.__delivery_destination = delivery_destination + + def register_delivery_callback(self, callback): + self.__delivery_callback = callback + + def register_failed_callback(self, callback): + self.failed_callback = callback + + def pack(self): + if not self.packed: + if self.timestamp == None: + self.timestamp = time.time() + + self.propagation_packed = None + + self.payload = [self.timestamp, self.title, self.content, self.fields] + + hashed_part = b"" + hashed_part += self.__destination.hash + hashed_part += self.__source.hash + hashed_part += msgpack.packb(self.payload) + self.hash = RNS.Identity.full_hash(hashed_part) + self.message_id = self.hash + + signed_part = b"" + signed_part += hashed_part + signed_part += self.hash + self.signature = self.__source.sign(signed_part) + self.signature_validated = True + + self.packed = b"" + self.packed += self.__destination.hash + self.packed += self.__source.hash + self.packed += self.signature + packed_payload = msgpack.packb(self.payload) + self.packed += packed_payload + self.packed_size = len(self.packed) + content_size = len(packed_payload) + + # If no desired delivery method has been defined, + # one will be chosen according to these rules: + if self.desired_method == None: + self.desired_method = LXMessage.DIRECT + # TODO: Expand rules to something more intelligent + + if self.desired_method == LXMessage.OPPORTUNISTIC: + if self.__destination.type == RNS.Destination.SINGLE: + single_packet_content_limit = LXMessage.ENCRYPTED_PACKET_MAX_CONTENT + elif self.__destination.type == RNS.Destination.PLAIN: + single_packet_content_limit = LXMessage.PLAIN_PACKET_MAX_CONTENT + + if content_size > single_packet_content_limit: + raise TypeError("LXMessage desired opportunistic delivery method, but content exceeds single-packet size.") + else: + self.method = LXMessage.OPPORTUNISTIC + self.representation = LXMessage.PACKET + self.__delivery_destination = self.__destination + + elif self.desired_method == LXMessage.DIRECT: + single_packet_content_limit = LXMessage.LINK_PACKET_MAX_CONTENT + if content_size <= single_packet_content_limit: + self.method = self.desired_method + self.representation = LXMessage.PACKET + else: + self.method = self.desired_method + self.representation = LXMessage.RESOURCE + + elif self.desired_method == LXMessage.PROPAGATED: + single_packet_content_limit = LXMessage.LINK_PACKET_MAX_CONTENT + + encrypted_data = self.__destination.encrypt(self.packed[LXMessage.DESTINATION_LENGTH:]) + self.propagation_packed = msgpack.packb([time.time(), [self.packed[:LXMessage.DESTINATION_LENGTH]+encrypted_data]]) + + content_size = len(self.propagation_packed) + if content_size <= single_packet_content_limit: + self.method = self.desired_method + self.representation = LXMessage.PACKET + else: + self.method = self.desired_method + self.representation = LXMessage.RESOURCE + + else: + raise ValueError("Attempt to re-pack LXMessage "+str(self)+" that was already packed") + + def send(self): + self.determine_transport_encryption() + + if self.method == LXMessage.OPPORTUNISTIC: + self.__as_packet().send().set_delivery_callback(self.__mark_delivered) + self.state = LXMessage.SENT + + elif self.method == LXMessage.DIRECT: + self.state = LXMessage.SENDING + + if self.representation == LXMessage.PACKET: + receipt = self.__as_packet().send() + if receipt: + receipt.set_delivery_callback(self.__mark_delivered) + receipt.set_timeout_callback(self.__link_packet_timed_out) + else: + if self.__delivery_destination: + self.__delivery_destination.teardown() + + elif self.representation == LXMessage.RESOURCE: + self.resource_representation = self.__as_resource() + + elif self.method == LXMessage.PROPAGATED: + self.state = LXMessage.SENDING + + if self.representation == LXMessage.PACKET: + receipt = self.__as_packet().send() + if receipt: + receipt.set_delivery_callback(self.__mark_propagated) + receipt.set_timeout_callback(self.__link_packet_timed_out) + else: + self.__delivery_destination.teardown() + + elif self.representation == LXMessage.RESOURCE: + self.resource_representation = self.__as_resource() + + + def determine_transport_encryption(self): + if self.method == LXMessage.OPPORTUNISTIC: + if self.__destination.type == RNS.Destination.SINGLE: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + elif self.__destination.type == RNS.Destination.GROUP: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES + else: + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + elif self.method == LXMessage.DIRECT: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + elif self.method == LXMessage.PROPAGATED: + if self.__destination.type == RNS.Destination.SINGLE: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + elif self.__destination.type == RNS.Destination.GROUP: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES + else: + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + else: + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + + def __mark_delivered(self, receipt = None): + RNS.log("Received delivery notification for "+str(self), RNS.LOG_DEBUG) + self.state = LXMessage.DELIVERED + + if self.__delivery_callback != None and callable(self.__delivery_callback): + try: + self.__delivery_callback(self) + except Exception as e: + RNS.log("An error occurred in the external delivery callback for "+str(message), RNS.LOG_ERROR) + + def __mark_propagated(self, receipt = None): + RNS.log("Received propagation success notification for "+str(self), RNS.LOG_DEBUG) + self.state = LXMessage.SENT + + if self.__delivery_callback != None and callable(self.__delivery_callback): + try: + self.__delivery_callback(self) + except Exception as e: + RNS.log("An error occurred in the external delivery callback for "+str(message), RNS.LOG_ERROR) + + def __resource_concluded(self, resource): + if resource.status == RNS.Resource.COMPLETE: + self.__mark_delivered() + else: + resource.link.teardown() + self.state = LXMessage.OUTBOUND + + def __propagation_resource_concluded(self, resource): + if resource.status == RNS.Resource.COMPLETE: + self.__mark_propagated() + else: + resource.link.teardown() + self.state = LXMessage.OUTBOUND + + def __link_packet_timed_out(self, packet_receipt): + if packet_receipt: + packet_receipt.destination.teardown() + + self.state = LXMessage.OUTBOUND + + + def __update_transfer_progress(self, resource): + self.progress = resource.get_progress() + + def __as_packet(self): + if not self.packed: + self.pack() + + if not self.__delivery_destination: + raise ValueError("Can't synthesize packet for LXMF message before delivery destination is known") + + if self.method == LXMessage.OPPORTUNISTIC: + return RNS.Packet(self.__delivery_destination, self.packed[LXMessage.DESTINATION_LENGTH:]) + elif self.method == LXMessage.DIRECT: + return RNS.Packet(self.__delivery_destination, self.packed) + elif self.method == LXMessage.PROPAGATED: + return RNS.Packet(self.__delivery_destination, self.propagation_packed) + + def __as_resource(self): + if not self.packed: + self.pack() + + if not self.__delivery_destination: + raise ValueError("Can't synthesize resource for LXMF message before delivery destination is known") + + if not self.__delivery_destination.type == RNS.Destination.LINK: + raise TypeError("Tried to synthesize resource for LXMF message on a delivery destination that was not a link") + + if not self.__delivery_destination.status == RNS.Link.ACTIVE: + raise ConnectionError("Tried to synthesize resource for LXMF message on a link that was not active") + + self.progress = 0.0 + + if self.method == LXMessage.DIRECT: + return RNS.Resource(self.packed, self.__delivery_destination, callback = self.__resource_concluded, progress_callback = self.__update_transfer_progress) + elif self.method == LXMessage.PROPAGATED: + return RNS.Resource(self.propagation_packed, self.__delivery_destination, callback = self.__propagation_resource_concluded, progress_callback = self.__update_transfer_progress) + else: + return None + + def packed_container(self): + if not self.packed: + self.pack() + + container = { + "state": self.state, + "lxmf_bytes": self.packed, + "transport_encrypted": self.transport_encrypted, + "transport_encryption": self.transport_encryption, + "method": self.method + } + + return msgpack.packb(container) + + + def write_to_directory(self, directory_path): + file_name = RNS.hexrep(self.hash, delimit=False) + file_path = directory_path+"/"+file_name + + try: + file = open(file_path, "wb") + file.write(self.packed_container()) + file.close() + + return file_path + + except Exception as e: + RNS.log("Error while writing LXMF message to file \""+str(file_path)+"\". The contained exception was: "+str(e), RNS.LOG_ERROR) + return None + + + @staticmethod + def unpack_from_bytes(lxmf_bytes): + destination_hash = lxmf_bytes[:LXMessage.DESTINATION_LENGTH] + source_hash = lxmf_bytes[LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH] + signature = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH] + packed_payload = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH:] + hashed_part = b"" + destination_hash + source_hash + packed_payload + message_hash = RNS.Identity.full_hash(hashed_part) + signed_part = b"" + hashed_part + message_hash + unpacked_payload = msgpack.unpackb(packed_payload) + timestamp = unpacked_payload[0] + title_bytes = unpacked_payload[1] + content_bytes = unpacked_payload[2] + fields = unpacked_payload[3] + + destination_identity = RNS.Identity.recall(destination_hash) + if destination_identity != None: + destination = RNS.Destination(destination_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") + else: + destination = None + + source_identity = RNS.Identity.recall(source_hash) + if source_identity != None: + source = RNS.Destination(source_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") + else: + source = None + + message = LXMessage( + destination = destination, + source = source, + content = "", + title = "", + fields = fields, + destination_hash = destination_hash, + source_hash = source_hash) + + message.hash = message_hash + message.signature = signature + message.incoming = True + message.timestamp = timestamp + message.packed = lxmf_bytes + message.packed_size = len(lxmf_bytes) + message.set_title_from_bytes(title_bytes) + message.set_content_from_bytes(content_bytes) + + try: + if source: + if source.identity.validate(signature, signed_part): + message.signature_validated = True + else: + message.signature_validated = False + message.unverified_reason = LXMessage.SIGNATURE_INVALID + else: + signature_validated = False + message.unverified_reason = LXMessage.SOURCE_UNKNOWN + RNS.log("Unpacked LXMF message signature could not be validated, since source identity is unknown", RNS.LOG_DEBUG) + except Exception as e: + message.signature_validated = False + RNS.log("Error while validating LXMF message signature. The contained exception was: "+str(e), RNS.LOG_ERROR) + + return message + + @staticmethod + def unpack_from_file(lxmf_file_handle): + try: + container = msgpack.unpackb(lxmf_file_handle.read()) + lxm = LXMessage.unpack_from_bytes(container["lxmf_bytes"]) + + if "state" in container: + lxm.state = container["state"] + if "transport_encrypted" in container: + lxm.transport_encrypted = container["transport_encrypted"] + if "transport_encryption" in container: + lxm.transport_encryption = container["transport_encryption"] + if "method" in container: + lxm.method = container["method"] + + return lxm + except Exception as e: + RNS.log("Could not unpack LXMessage from file. The contained exception was: "+str(e), RNS.LOG_ERROR) + return None \ No newline at end of file diff --git a/LXMF/__init__.py b/LXMF/__init__.py index a304c21..e0924fe 100644 --- a/LXMF/__init__.py +++ b/LXMF/__init__.py @@ -1,7 +1,7 @@ import os import glob -from .LXMF import LXMessage -from .LXMF import LXMRouter +from .LXMessage import LXMessage +from .LXMRouter import LXMRouter modules = glob.glob(os.path.dirname(__file__)+"/*.py") __all__ = [ os.path.basename(f)[:-3] for f in modules if not f.endswith('__init__.py')] \ No newline at end of file From ca84f4c8feaa88ad2f40d2ac8995b396842fc206 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 17 Jun 2022 09:15:56 +0200 Subject: [PATCH 085/436] Restructured LXMRouter for legibility --- LXMF/LXMRouter.py | 800 +++++++++++++++++++++++----------------------- 1 file changed, 403 insertions(+), 397 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 0718478..0cfdfa7 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -41,6 +41,10 @@ class LXMRouter: PR_ALL_MESSAGES = 0x00 + + ### Developer-facing API ############################## + ####################################################### + def __init__(self, identity = None, storagepath = None, autopeer = AUTOPEER, autopeer_maxdepth = AUTOPEER_MAXDEPTH): random.seed(os.urandom(10)) @@ -110,34 +114,20 @@ class LXMRouter: job_thread.setDaemon(True) job_thread.start() - def exit_handler(self): - if self.propagation_node: - try: - serialised_peers = [] - for peer_id in self.peers: - peer = self.peers[peer_id] - serialised_peers.append(peer.to_bytes()) + def announce(self, destination_hash): + if destination_hash in self.delivery_destinations: + delivery_destination = self.delivery_destinations[destination_hash] + delivery_destination.announce(delivery_destination.display_name.encode("utf-8")) - peers_file = open(self.storagepath+"/peers", "wb") - peers_file.write(msgpack.packb(serialised_peers)) - peers_file.close() - - RNS.log("Saved "+str(len(serialised_peers))+" peers to storage", RNS.LOG_DEBUG) - - except Exception as e: - RNS.log("Could not save propagation node peers to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) - - try: - if not os.path.isdir(self.storagepath): - os.makedirs(self.storagepath) - - locally_delivered_file = open(self.storagepath+"/local_deliveries", "wb") - locally_delivered_file.write(msgpack.packb(self.locally_delivered_transient_ids)) - locally_delivered_file.close() - - except Exception as e: - RNS.log("Could not save locally delivered message ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + def announce_propagation_node(self): + def delayed_announce(): + time.sleep(LXMRouter.NODE_ANNOUNCE_DELAY) + data = msgpack.packb([self.propagation_node, int(time.time())]) + self.propagation_destination.announce(app_data=data) + da_thread = threading.Thread(target=delayed_announce) + da_thread.setDaemon(True) + da_thread.start() def register_delivery_identity(self, identity, display_name = None): delivery_destination = RNS.Destination(identity, RNS.Destination.IN, RNS.Destination.SINGLE, "lxmf", "delivery") @@ -167,20 +157,6 @@ class LXMRouter: def get_outbound_propagation_node(self): return self.outbound_propagation_node - def cancel_propagation_node_requests(self): - if self.outbound_propagation_link != None: - self.outbound_propagation_link.teardown() - self.outbound_propagation_link = None - - self.acknowledge_sync_completion() - - def acknowledge_sync_completion(self): - self.propagation_transfer_state = LXMRouter.PR_IDLE - self.propagation_transfer_progress = 0.0 - self.propagation_transfer_last_result = None - self.wants_download_on_path_available_from = None - self.wants_download_on_path_available_to = None - def request_messages_from_propagation_node(self, identity, max_messages = PR_ALL_MESSAGES): if max_messages == None: max_messages = LXMRouter.PR_ALL_MESSAGES @@ -224,181 +200,13 @@ class LXMRouter: else: RNS.log("Cannot request LXMF propagation node sync, no default propagation node configured", RNS.LOG_WARNING) - - def request_messages_path_job(self): - job_thread = threading.Thread(target=self.__request_messages_path_job) - job_thread.setDaemon(True) - job_thread.start() - - def __request_messages_path_job(self): - while not RNS.Transport.has_path(self.wants_download_on_path_available_from) and time.time() < self.wants_download_on_path_available_timeout: - time.sleep(0.1) - - if RNS.Transport.has_path(self.wants_download_on_path_available_from): - self.request_messages_from_propagation_node(self.wants_download_on_path_available_to, self.propagation_transfer_max_messages) - else: - RNS.log("Propagation node path request timed out", RNS.LOG_DEBUG) - self.acknowledge_sync_completion() - - - def has_message(self, transient_id): - if transient_id in self.locally_delivered_transient_ids: - return True - else: - return False - - def message_get_failed(self, request_receipt): - RNS.log("Message list/get request failed", RNS.LOG_DEBUG) + def cancel_propagation_node_requests(self): if self.outbound_propagation_link != None: self.outbound_propagation_link.teardown() + self.outbound_propagation_link = None - def message_list_response(self, request_receipt): - if request_receipt.response == LXMPeer.ERROR_NO_IDENTITY: - RNS.log("Propagation node indicated missing identification on list request, tearing down link.", RNS.LOG_DEBUG) - if self.outbound_propagation_link != None: - self.outbound_propagation_link.teardown() - else: - if request_receipt.response != None: - haves = [] - wants = [] - if len(request_receipt.response) > 0: - for transient_id in request_receipt.response: - if self.has_message(transient_id): - haves.append(transient_id) - else: - if self.propagation_transfer_max_messages == LXMRouter.PR_ALL_MESSAGES or len(wants) < self.propagation_transfer_max_messages: - wants.append(transient_id) + self.acknowledge_sync_completion() - request_receipt.link.request( - LXMPeer.MESSAGE_GET_PATH, - [wants, haves], - response_callback=self.message_get_response, - failed_callback=self.message_get_failed, - progress_callback=self.message_get_progress - ) - else: - self.propagation_transfer_state = LXMRouter.PR_COMPLETE - self.propagation_transfer_progress = 1.0 - self.propagation_transfer_last_result = 0 - - def message_get_progress(self, request_receipt): - self.propagation_transfer_state = LXMRouter.PR_RECEIVING - self.propagation_transfer_progress = request_receipt.get_progress() - - def message_get_response(self, request_receipt): - if request_receipt.response == LXMPeer.ERROR_NO_IDENTITY: - RNS.log("Propagation node indicated missing identification on get request, tearing down link.", RNS.LOG_DEBUG) - if self.outbound_propagation_link != None: - self.outbound_propagation_link.teardown() - else: - if request_receipt.response != None and len(request_receipt.response) > 0: - haves = [] - for lxmf_data in request_receipt.response: - self.lxmf_propagation(lxmf_data) - haves.append(RNS.Identity.full_hash(lxmf_data)) - - # Return a list of successfully received messages to the node. - # This deletes the messages on the propagation node. - # TODO: Add option to keep messages on node. - request_receipt.link.request( - LXMPeer.MESSAGE_GET_PATH, - [None, haves], - # response_callback=self.message_syncfinal_response, - failed_callback=self.message_get_failed, - # progress_callback=self.message_get_progress - ) - - self.propagation_transfer_state = LXMRouter.PR_COMPLETE - self.propagation_transfer_progress = 1.0 - self.propagation_transfer_last_result = len(request_receipt.response) - - def announce(self, destination_hash): - if destination_hash in self.delivery_destinations: - delivery_destination = self.delivery_destinations[destination_hash] - delivery_destination.announce(delivery_destination.display_name.encode("utf-8")) - - def handle_outbound(self, lxmessage): - lxmessage.state = LXMessage.OUTBOUND - if not lxmessage.packed: - lxmessage.pack() - - lxmessage.determine_transport_encryption() - - while self.processing_outbound: - time.sleep(0.1) - - self.pending_outbound.append(lxmessage) - self.process_outbound() - - def lxmf_delivery(self, lxmf_data, destination_type = None): - try: - message = LXMessage.unpack_from_bytes(lxmf_data) - - if destination_type == RNS.Destination.SINGLE: - message.transport_encrypted = True - message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC - elif destination_type == RNS.Destination.GROUP: - message.transport_encrypted = True - message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES - elif destination_type == RNS.Destination.LINK: - message.transport_encrypted = True - message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC - else: - message.transport_encrypted = False - message.transport_encryption = None - - if message.source_hash in self.ignored_list: - RNS.log(str(self)+" ignored message from "+RNS.prettyhexrep(message.source_hash), RNS.LOG_DEBUG) - return False - - if self.__delivery_callback != None and callable(self.__delivery_callback): - try: - self.__delivery_callback(message) - except Exception as e: - RNS.log("An error occurred in the external delivery callback for "+str(message), RNS.LOG_ERROR) - - return True - - except Exception as e: - RNS.log("Could not assemble LXMF message from received data", RNS.LOG_NOTICE) - RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) - return False - - - def delivery_packet(self, data, packet): - try: - if packet.destination_type != RNS.Destination.LINK: - lxmf_data = b"" - lxmf_data += packet.destination.hash - lxmf_data += data - else: - lxmf_data = data - - if self.lxmf_delivery(lxmf_data, packet.destination_type): - packet.prove() - - except Exception as e: - RNS.log("Exception occurred while parsing incoming LXMF data.", RNS.LOG_ERROR) - RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) - - def delivery_link_established(self, link): - link.set_packet_callback(self.delivery_packet) - link.set_resource_strategy(RNS.Link.ACCEPT_ALL) - link.set_resource_started_callback(self.resource_transfer_began) - link.set_resource_concluded_callback(self.delivery_resource_concluded) - - def delivery_link_closed(self, link): - pass - - def resource_transfer_began(self, resource): - RNS.log("Transfer began for resource "+str(resource), RNS.LOG_DEBUG) - - def delivery_resource_concluded(self, resource): - RNS.log("Transfer concluded for delivery resource "+str(resource), RNS.LOG_DEBUG) - if resource.status == RNS.Resource.COMPLETE: - self.lxmf_delivery(resource.data.read(), resource.link.type) - - def enable_propagation(self): try: self.messagepath = self.storagepath+"/messagestore" @@ -459,41 +267,160 @@ class LXMRouter: self.propagation_node = False self.announce_propagation_node() - def announce_propagation_node(self): - def delayed_announce(): - time.sleep(LXMRouter.NODE_ANNOUNCE_DELAY) - data = msgpack.packb([self.propagation_node, int(time.time())]) - self.propagation_destination.announce(app_data=data) + def ignore_destination(self, destination_hash): + if not destination_hash in self.ignored_list: + self.ignored_list.append(destination_hash) - da_thread = threading.Thread(target=delayed_announce) - da_thread.setDaemon(True) - da_thread.start() + def unignore_destination(self, destination_hash): + if destination_hash in self.ignored_list: + self.ignored_list.remove(destination_hash) - def offer_request(self, path, data, request_id, remote_identity, requested_at): - if remote_identity == None: - return LXMPeer.ERROR_NO_IDENTITY - else: + + ### Utility & Maintenance ############################# + ####################################################### + + def jobloop(self): + while (True): + # TODO: Improve this to scheduling, so manual + # triggers can delay next run + self.jobs() + time.sleep(LXMRouter.PROCESSING_INTERVAL) + + JOB_OUTBOUND_INTERVAL = 1 + JOB_LINKS_INTERVAL = 1 + JOB_TRANSIENT_INTERVAL = 60 + JOB_STORE_INTERVAL = 120 + JOB_PEERSYNC_INTERVAL = 12 + def jobs(self): + self.processing_count += 1 + + if self.processing_count % LXMRouter.JOB_OUTBOUND_INTERVAL == 0: + self.process_outbound() + + if self.processing_count % LXMRouter.JOB_LINKS_INTERVAL == 0: + self.clean_links() + + if self.processing_count % LXMRouter.JOB_TRANSIENT_INTERVAL == 0: + self.clean_transient_id_cache() + + if self.processing_count % LXMRouter.JOB_STORE_INTERVAL == 0: + self.clean_message_store() + + if self.processing_count % LXMRouter.JOB_PEERSYNC_INTERVAL == 0: + self.sync_peers() + + def clean_links(self): + closed_links = [] + for link_hash in self.direct_links: + link = self.direct_links[link_hash] + inactive_time = link.inactive_for() + + if inactive_time > LXMRouter.LINK_MAX_INACTIVITY: + link.teardown() + closed_links.append(link_hash) + + for link_hash in closed_links: + cleaned_link = self.direct_links.pop(link_hash) + RNS.log("Cleaned link "+str(cleaned_link), RNS.LOG_DEBUG) + + if self.outbound_propagation_link != None and self.outbound_propagation_link.status == RNS.Link.CLOSED: + self.outbound_propagation_link = None + self.acknowledge_sync_completion() + RNS.log("Cleaned outbound propagation link", RNS.LOG_DEBUG) + + def clean_transient_id_cache(self): + now = time.time() + removed_entries = [] + for transient_id in self.locally_delivered_transient_ids: + timestamp = self.locally_delivered_transient_ids[transient_id] + if now > timestamp+LXMRouter.MESSAGE_EXPIRY*1.1: + removed_entries.append(transient_id) + + for transient_id in removed_entries: + self.locally_delivered_transient_ids.pop(transient_id) + RNS.log("Cleaned "+RNS.prettyhexrep(transient_id)+" from local delivery cache", RNS.LOG_DEBUG) + + def clean_message_store(self): + now = time.time() + removed_entries = {} + for transient_id in self.propagation_entries: + entry = self.propagation_entries[transient_id] + filepath = entry[1] + components = filepath.split("_") + + if len(components) == 2 and float(components[1]) > 0 and len(os.path.split(components[0])[1]) == (RNS.Identity.HASHLENGTH//8)*2: + timestamp = float(components[1]) + if now > timestamp+LXMRouter.MESSAGE_EXPIRY: + RNS.log("Purging message "+RNS.prettyhexrep(transient_id)+" due to expiry", RNS.LOG_DEBUG) + removed_entries[transient_id] = filepath + else: + RNS.log("Purging message "+RNS.prettyhexrep(transient_id)+" due to invalid file path", RNS.LOG_WARNING) + removed_entries[transient_id] = filepath + + removed_count = 0 + for transient_id in removed_entries: try: - transient_ids = data - wanted_ids = [] + filepath = removed_entries[transient_id] + self.propagation_entries.pop(transient_id) + if os.path.isfile(filepath): + os.unlink(filepath) + removed_count += 1 + except Exception as e: + RNS.log("Could not remove "+RNS.prettyhexrep(transient_id)+" from message store. The contained exception was: "+str(e), RNS.LOG_ERROR) - for transient_id in transient_ids: - if not transient_id in self.propagation_entries: - wanted_ids.append(transient_id) + if removed_count > 0: + RNS.log("Cleaned "+str(removed_count)+" entries from the message store", RNS.LOG_DEBUG) - if len(wanted_ids) == 0: - return False + def exit_handler(self): + if self.propagation_node: + try: + serialised_peers = [] + for peer_id in self.peers: + peer = self.peers[peer_id] + serialised_peers.append(peer.to_bytes()) - elif len(wanted_ids) == len(transient_ids): - return True + peers_file = open(self.storagepath+"/peers", "wb") + peers_file.write(msgpack.packb(serialised_peers)) + peers_file.close() - else: - return wanted_ids + RNS.log("Saved "+str(len(serialised_peers))+" peers to storage", RNS.LOG_DEBUG) except Exception as e: - RNS.log("Error occurred while generating response for sync request, the contained exception was: "+str(e), RNS.LOG_DEBUG) - return None + RNS.log("Could not save propagation node peers to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + try: + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) + + locally_delivered_file = open(self.storagepath+"/local_deliveries", "wb") + locally_delivered_file.write(msgpack.packb(self.locally_delivered_transient_ids)) + locally_delivered_file.close() + + except Exception as e: + RNS.log("Could not save locally delivered message ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + + def __str__(self): + return "" + + + ### Message Download ################################## + ####################################################### + + def request_messages_path_job(self): + job_thread = threading.Thread(target=self.__request_messages_path_job) + job_thread.setDaemon(True) + job_thread.start() + + def __request_messages_path_job(self): + while not RNS.Transport.has_path(self.wants_download_on_path_available_from) and time.time() < self.wants_download_on_path_available_timeout: + time.sleep(0.1) + + if RNS.Transport.has_path(self.wants_download_on_path_available_from): + self.request_messages_from_propagation_node(self.wants_download_on_path_available_to, self.propagation_transfer_max_messages) + else: + RNS.log("Propagation node path request timed out", RNS.LOG_DEBUG) + self.acknowledge_sync_completion() + def message_get_request(self, path, data, request_id, remote_identity, requested_at): if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY @@ -559,13 +486,228 @@ class LXMRouter: RNS.log("Error occurred while generating response for download request, the contained exception was: "+str(e), RNS.LOG_DEBUG) return None + def message_list_response(self, request_receipt): + if request_receipt.response == LXMPeer.ERROR_NO_IDENTITY: + RNS.log("Propagation node indicated missing identification on list request, tearing down link.", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + else: + if request_receipt.response != None: + haves = [] + wants = [] + if len(request_receipt.response) > 0: + for transient_id in request_receipt.response: + if self.has_message(transient_id): + haves.append(transient_id) + else: + if self.propagation_transfer_max_messages == LXMRouter.PR_ALL_MESSAGES or len(wants) < self.propagation_transfer_max_messages: + wants.append(transient_id) + + request_receipt.link.request( + LXMPeer.MESSAGE_GET_PATH, + [wants, haves], + response_callback=self.message_get_response, + failed_callback=self.message_get_failed, + progress_callback=self.message_get_progress + ) + else: + self.propagation_transfer_state = LXMRouter.PR_COMPLETE + self.propagation_transfer_progress = 1.0 + self.propagation_transfer_last_result = 0 + + def message_get_response(self, request_receipt): + if request_receipt.response == LXMPeer.ERROR_NO_IDENTITY: + RNS.log("Propagation node indicated missing identification on get request, tearing down link.", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + else: + if request_receipt.response != None and len(request_receipt.response) > 0: + haves = [] + for lxmf_data in request_receipt.response: + self.lxmf_propagation(lxmf_data) + haves.append(RNS.Identity.full_hash(lxmf_data)) + + # Return a list of successfully received messages to the node. + # This deletes the messages on the propagation node. + # TODO: Add option to keep messages on node. + request_receipt.link.request( + LXMPeer.MESSAGE_GET_PATH, + [None, haves], + # response_callback=self.message_syncfinal_response, + failed_callback=self.message_get_failed, + # progress_callback=self.message_get_progress + ) + + self.propagation_transfer_state = LXMRouter.PR_COMPLETE + self.propagation_transfer_progress = 1.0 + self.propagation_transfer_last_result = len(request_receipt.response) + + def message_get_progress(self, request_receipt): + self.propagation_transfer_state = LXMRouter.PR_RECEIVING + self.propagation_transfer_progress = request_receipt.get_progress() + + def message_get_failed(self, request_receipt): + RNS.log("Message list/get request failed", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + + def acknowledge_sync_completion(self): + self.propagation_transfer_state = LXMRouter.PR_IDLE + self.propagation_transfer_progress = 0.0 + self.propagation_transfer_last_result = None + self.wants_download_on_path_available_from = None + self.wants_download_on_path_available_to = None + + def has_message(self, transient_id): + if transient_id in self.locally_delivered_transient_ids: + return True + else: + return False + + + ### Message Routing & Delivery ######################## + ####################################################### + + def handle_outbound(self, lxmessage): + lxmessage.state = LXMessage.OUTBOUND + if not lxmessage.packed: + lxmessage.pack() + + lxmessage.determine_transport_encryption() + + while self.processing_outbound: + time.sleep(0.1) + + self.pending_outbound.append(lxmessage) + self.process_outbound() + + def lxmf_delivery(self, lxmf_data, destination_type = None): + try: + message = LXMessage.unpack_from_bytes(lxmf_data) + + if destination_type == RNS.Destination.SINGLE: + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + elif destination_type == RNS.Destination.GROUP: + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES + elif destination_type == RNS.Destination.LINK: + message.transport_encrypted = True + message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + else: + message.transport_encrypted = False + message.transport_encryption = None + + if message.source_hash in self.ignored_list: + RNS.log(str(self)+" ignored message from "+RNS.prettyhexrep(message.source_hash), RNS.LOG_DEBUG) + return False + + if self.__delivery_callback != None and callable(self.__delivery_callback): + try: + self.__delivery_callback(message) + except Exception as e: + RNS.log("An error occurred in the external delivery callback for "+str(message), RNS.LOG_ERROR) + + return True + + except Exception as e: + RNS.log("Could not assemble LXMF message from received data", RNS.LOG_NOTICE) + RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + return False + + def delivery_packet(self, data, packet): + try: + if packet.destination_type != RNS.Destination.LINK: + lxmf_data = b"" + lxmf_data += packet.destination.hash + lxmf_data += data + else: + lxmf_data = data + + if self.lxmf_delivery(lxmf_data, packet.destination_type): + packet.prove() + + except Exception as e: + RNS.log("Exception occurred while parsing incoming LXMF data.", RNS.LOG_ERROR) + RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) + + def delivery_link_established(self, link): + link.set_packet_callback(self.delivery_packet) + link.set_resource_strategy(RNS.Link.ACCEPT_ALL) + link.set_resource_started_callback(self.resource_transfer_began) + link.set_resource_concluded_callback(self.delivery_resource_concluded) + + def delivery_link_closed(self, link): + pass + + def resource_transfer_began(self, resource): + RNS.log("Transfer began for LXMF delivery resource "+str(resource), RNS.LOG_DEBUG) + + def delivery_resource_concluded(self, resource): + RNS.log("Transfer concluded for LXMF delivery resource "+str(resource), RNS.LOG_DEBUG) + if resource.status == RNS.Resource.COMPLETE: + self.lxmf_delivery(resource.data.read(), resource.link.type) + + + ### Peer Sync & Propagation ########################### + ####################################################### + + def peer(self, destination_hash, timestamp): + if destination_hash in self.peers: + peer = self.peers[destination_hash] + peer.alive = True + peer.peering_timebase = timestamp + peer.last_heard = time.time() + else: + peer = LXMPeer(self, destination_hash) + peer.alive = True + peer.last_heard = time.time() + self.peers[destination_hash] = peer + RNS.log("Peered with "+str(peer.destination)) + + def unpeer(self, destination_hash, timestamp = None): + if timestamp == None: + timestamp = int(time.time()) + + if destination_hash in self.peers: + peer = self.peers[destination_hash] + + if timestamp >= peer.peering_timebase: + self.peers.pop(destination_hash) + RNS.log("Broke peering with "+str(peer.destination)) + + def sync_peers(self): + culled_peers = [] + waiting_peers = [] + for peer_id in self.peers: + peer = self.peers[peer_id] + if time.time() > peer.last_heard + LXMPeer.MAX_UNREACHABLE: + culled_peers.append(peer_id) + else: + if peer.state == LXMPeer.IDLE and len(peer.unhandled_messages) > 0: + waiting_peers.append(peer) + + if len(waiting_peers) > 0: + RNS.log("Randomly selecting peer to sync from "+str(len(waiting_peers))+" waiting peers.", RNS.LOG_DEBUG) + selected_index = random.randint(0,len(waiting_peers)-1) + selected_peer = waiting_peers[selected_index] + RNS.log("Selected waiting peer "+str(selected_index)+": "+RNS.prettyhexrep(selected_peer.destination.hash), RNS.LOG_DEBUG) + selected_peer.sync() + + for peer_id in culled_peers: + RNS.log("Removing peer "+RNS.prettyhexrep(peer_id)+" due to excessive unreachability", RNS.LOG_WARNING) + try: + if peer_id in self.peers: + self.peers.pop(peer_id) + except Exception as e: + RNS.log("Error while removing peer "+RNS.prettyhexrep(peer_id)+". The contained exception was: "+str(e), RNS.LOG_ERROR) + def propagation_link_established(self, link): link.set_packet_callback(self.propagation_packet) link.set_resource_strategy(RNS.Link.ACCEPT_ALL) link.set_resource_started_callback(self.resource_transfer_began) link.set_resource_concluded_callback(self.propagation_resource_concluded) - def propagation_packet(self, data, packet): try: if packet.destination_type != RNS.Destination.LINK: @@ -584,6 +726,30 @@ class LXMRouter: RNS.log("Exception occurred while parsing incoming LXMF propagation data.", RNS.LOG_ERROR) RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) + def offer_request(self, path, data, request_id, remote_identity, requested_at): + if remote_identity == None: + return LXMPeer.ERROR_NO_IDENTITY + else: + try: + transient_ids = data + wanted_ids = [] + + for transient_id in transient_ids: + if not transient_id in self.propagation_entries: + wanted_ids.append(transient_id) + + if len(wanted_ids) == 0: + return False + + elif len(wanted_ids) == len(transient_ids): + return True + + else: + return wanted_ids + + except Exception as e: + RNS.log("Error occurred while generating response for sync request, the contained exception was: "+str(e), RNS.LOG_DEBUG) + return None def propagation_resource_concluded(self, resource): RNS.log("Transfer concluded for incoming propagation resource "+str(resource), RNS.LOG_DEBUG) @@ -621,7 +787,6 @@ class LXMRouter: except Exception as e: RNS.log("Error while unpacking received propagation resource", RNS.LOG_DEBUG) - def lxmf_propagation(self, lxmf_data): try: if len(lxmf_data) >= LXMessage.LXMF_OVERHEAD: @@ -662,162 +827,6 @@ class LXMRouter: RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) return False - - def peer(self, destination_hash, timestamp): - if destination_hash in self.peers: - peer = self.peers[destination_hash] - peer.alive = True - peer.peering_timebase = timestamp - peer.last_heard = time.time() - else: - peer = LXMPeer(self, destination_hash) - peer.alive = True - peer.last_heard = time.time() - self.peers[destination_hash] = peer - RNS.log("Peered with "+str(peer.destination)) - - def unpeer(self, destination_hash, timestamp = None): - if timestamp == None: - timestamp = int(time.time()) - - if destination_hash in self.peers: - peer = self.peers[destination_hash] - - if timestamp >= peer.peering_timebase: - self.peers.pop(destination_hash) - RNS.log("Broke peering with "+str(peer.destination)) - - - def jobloop(self): - while (True): - # TODO: Improve this to scheduling, so manual - # triggers can delay next run - self.jobs() - time.sleep(LXMRouter.PROCESSING_INTERVAL) - - JOB_OUTBOUND_INTERVAL = 1 - JOB_LINKS_INTERVAL = 1 - JOB_TRANSIENT_INTERVAL = 60 - JOB_STORE_INTERVAL = 120 - JOB_PEERSYNC_INTERVAL = 12 - def jobs(self): - self.processing_count += 1 - - if self.processing_count % LXMRouter.JOB_OUTBOUND_INTERVAL == 0: - self.process_outbound() - - if self.processing_count % LXMRouter.JOB_LINKS_INTERVAL == 0: - self.clean_links() - - if self.processing_count % LXMRouter.JOB_TRANSIENT_INTERVAL == 0: - self.clean_transient_id_cache() - - if self.processing_count % LXMRouter.JOB_STORE_INTERVAL == 0: - self.clean_message_store() - - if self.processing_count % LXMRouter.JOB_PEERSYNC_INTERVAL == 0: - self.sync_peers() - - - def clean_links(self): - closed_links = [] - for link_hash in self.direct_links: - link = self.direct_links[link_hash] - inactive_time = link.inactive_for() - - if inactive_time > LXMRouter.LINK_MAX_INACTIVITY: - link.teardown() - closed_links.append(link_hash) - - for link_hash in closed_links: - cleaned_link = self.direct_links.pop(link_hash) - RNS.log("Cleaned link "+str(cleaned_link), RNS.LOG_DEBUG) - - if self.outbound_propagation_link != None and self.outbound_propagation_link.status == RNS.Link.CLOSED: - self.outbound_propagation_link = None - self.acknowledge_sync_completion() - RNS.log("Cleaned outbound propagation link", RNS.LOG_DEBUG) - - def clean_transient_id_cache(self): - now = time.time() - removed_entries = [] - for transient_id in self.locally_delivered_transient_ids: - timestamp = self.locally_delivered_transient_ids[transient_id] - if now > timestamp+LXMRouter.MESSAGE_EXPIRY*1.1: - removed_entries.append(transient_id) - - for transient_id in removed_entries: - self.locally_delivered_transient_ids.pop(transient_id) - RNS.log("Cleaned "+RNS.prettyhexrep(transient_id)+" from local delivery cache", RNS.LOG_DEBUG) - - - def clean_message_store(self): - now = time.time() - removed_entries = {} - for transient_id in self.propagation_entries: - entry = self.propagation_entries[transient_id] - filepath = entry[1] - components = filepath.split("_") - - if len(components) == 2 and float(components[1]) > 0 and len(os.path.split(components[0])[1]) == (RNS.Identity.HASHLENGTH//8)*2: - timestamp = float(components[1]) - if now > timestamp+LXMRouter.MESSAGE_EXPIRY: - RNS.log("Purging message "+RNS.prettyhexrep(transient_id)+" due to expiry", RNS.LOG_DEBUG) - removed_entries[transient_id] = filepath - else: - RNS.log("Purging message "+RNS.prettyhexrep(transient_id)+" due to invalid file path", RNS.LOG_WARNING) - removed_entries[transient_id] = filepath - - removed_count = 0 - for transient_id in removed_entries: - try: - filepath = removed_entries[transient_id] - self.propagation_entries.pop(transient_id) - if os.path.isfile(filepath): - os.unlink(filepath) - removed_count += 1 - except Exception as e: - RNS.log("Could not remove "+RNS.prettyhexrep(transient_id)+" from message store. The contained exception was: "+str(e), RNS.LOG_ERROR) - - if removed_count > 0: - RNS.log("Cleaned "+str(removed_count)+" entries from the message store", RNS.LOG_DEBUG) - - - def sync_peers(self): - culled_peers = [] - waiting_peers = [] - for peer_id in self.peers: - peer = self.peers[peer_id] - if time.time() > peer.last_heard + LXMPeer.MAX_UNREACHABLE: - culled_peers.append(peer_id) - else: - if peer.state == LXMPeer.IDLE and len(peer.unhandled_messages) > 0: - waiting_peers.append(peer) - - if len(waiting_peers) > 0: - RNS.log("Randomly selecting peer to sync from "+str(len(waiting_peers))+" waiting peers.", RNS.LOG_DEBUG) - selected_index = random.randint(0,len(waiting_peers)-1) - selected_peer = waiting_peers[selected_index] - RNS.log("Selected waiting peer "+str(selected_index)+": "+RNS.prettyhexrep(selected_peer.destination.hash), RNS.LOG_DEBUG) - selected_peer.sync() - - for peer_id in culled_peers: - RNS.log("Removing peer "+RNS.prettyhexrep(peer_id)+" due to excessive unreachability", RNS.LOG_WARNING) - try: - if peer_id in self.peers: - self.peers.pop(peer_id) - except Exception as e: - RNS.log("Error while removing peer "+RNS.prettyhexrep(peer_id)+". The contained exception was: "+str(e), RNS.LOG_ERROR) - - - def ignore_destination(self, destination_hash): - if not destination_hash in self.ignored_list: - self.ignored_list.append(destination_hash) - - def unignore_destination(self, destination_hash): - if destination_hash in self.ignored_list: - self.ignored_list.remove(destination_hash) - def fail_message(self, lxmessage): RNS.log(str(lxmessage)+" failed to send", RNS.LOG_DEBUG) @@ -828,9 +837,6 @@ class LXMRouter: if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): lxmessage.failed_callback(lxmessage) - def __str__(self): - return "" - def process_outbound(self, sender = None): if self.processing_outbound: return From 3dfb05a25cfc3a429f11c53fd7fe578b37fe9d41 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 17 Jun 2022 12:37:00 +0200 Subject: [PATCH 086/436] Implemented storage limit and weight-based message store culling --- LXMF/LXMRouter.py | 161 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 146 insertions(+), 15 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 0cfdfa7..932e4cf 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -70,6 +70,9 @@ class LXMRouter: self.outbound_propagation_node = None self.outbound_propagation_link = None + self.message_storage_limit = None + self.information_storage_limit = None + self.wants_download_on_path_available_from = None self.wants_download_on_path_available_to = None self.propagation_transfer_state = LXMRouter.PR_IDLE @@ -223,15 +226,25 @@ class LXMRouter: if len(components) == 2: if float(components[1]) > 0: if len(components[0]) == RNS.Identity.HASHLENGTH//8*2: - transient_id = bytes.fromhex(components[0]) - received = components[1] + try: + transient_id = bytes.fromhex(components[0]) + received = float(components[1]) - filepath = self.messagepath+"/"+filename - file = open(filepath, "rb") - destination_hash = file.read(LXMessage.DESTINATION_LENGTH) - file.close() + filepath = self.messagepath+"/"+filename + msg_size = os.path.getsize(filepath) + file = open(filepath, "rb") + destination_hash = file.read(LXMessage.DESTINATION_LENGTH) + file.close() - self.propagation_entries[transient_id] = [destination_hash, filepath] + self.propagation_entries[transient_id] = [ + destination_hash, + filepath, + received, + msg_size, + ] + + except Exception as e: + RNS.log("Could not read LXM from message store. The contained exception was: "+str(e), RNS.LOG_ERROR) if os.path.isfile(self.storagepath+"/peers"): peers_file = open(self.storagepath+"/peers", "rb") @@ -256,6 +269,8 @@ class LXMRouter: self.propagation_destination.register_request_handler(LXMPeer.OFFER_REQUEST_PATH, self.offer_request, allow = RNS.Destination.ALLOW_ALL) self.propagation_destination.register_request_handler(LXMPeer.MESSAGE_GET_PATH, self.message_get_request, allow = RNS.Destination.ALLOW_ALL) + RNS.log("LXMF Propagation Node message store size is "+RNS.prettysize(self.message_storage_size()), RNS.LOG_DEBUG) + self.announce_propagation_node() except Exception as e: @@ -275,17 +290,73 @@ class LXMRouter: if destination_hash in self.ignored_list: self.ignored_list.remove(destination_hash) + def set_message_storage_limit(self, kilobytes = None, megabytes = None, gigabytes = None): + limit_bytes = 0 + + if kilobytes != None: + limit_bytes += gigabytes*1000 + + if megabytes != None: + limit_bytes += gigabytes*1000*1000 + + if gigabytes != None: + limit_bytes += gigabytes*1000*1000*1000 + + if limit_bytes == 0: + limit_bytes = None + + try: + if limit_bytes == None or int(limit_bytes) > 0: + self.message_storage_limit = int(limit_bytes) + else: + raise ValueError("Cannot set LXMF information storage limit to "+str(limit_bytes)) + + except Exception as e: + raise ValueError("Cannot set LXMF information storage limit to "+str(limit_bytes)) + + def message_storage_limit(self): + return self.message_storage_limit + + def message_storage_size(self): + if self.propagation_node: + return sum(self.propagation_entries[f][3] for f in self.propagation_entries) + else: + return None + + def set_information_storage_limit(self, kilobytes = None, megabytes = None, gigabytes = None): + limit_bytes = 0 + + if kilobytes != None: + limit_bytes += gigabytes*1000 + + if megabytes != None: + limit_bytes += gigabytes*1000*1000 + + if gigabytes != None: + limit_bytes += gigabytes*1000*1000*1000 + + if limit_bytes == 0: + limit_bytes = None + + try: + if limit_bytes == None or int(limit_bytes) > 0: + self.information_storage_limit = int(limit_bytes) + else: + raise ValueError("Cannot set LXMF information storage limit to "+str(limit_bytes)) + + except Exception as e: + raise ValueError("Cannot set LXMF information storage limit to "+str(limit_bytes)) + + def information_storage_limit(self): + return self.information_storage_limit + + def information_storage_size(self): + pass + ### Utility & Maintenance ############################# ####################################################### - def jobloop(self): - while (True): - # TODO: Improve this to scheduling, so manual - # triggers can delay next run - self.jobs() - time.sleep(LXMRouter.PROCESSING_INTERVAL) - JOB_OUTBOUND_INTERVAL = 1 JOB_LINKS_INTERVAL = 1 JOB_TRANSIENT_INTERVAL = 60 @@ -309,6 +380,13 @@ class LXMRouter: if self.processing_count % LXMRouter.JOB_PEERSYNC_INTERVAL == 0: self.sync_peers() + def jobloop(self): + while (True): + # TODO: Improve this to scheduling, so manual + # triggers can delay next run + self.jobs() + time.sleep(LXMRouter.PROCESSING_INTERVAL) + def clean_links(self): closed_links = [] for link_hash in self.direct_links: @@ -341,6 +419,7 @@ class LXMRouter: RNS.log("Cleaned "+RNS.prettyhexrep(transient_id)+" from local delivery cache", RNS.LOG_DEBUG) def clean_message_store(self): + # Check and remove expired messages now = time.time() removed_entries = {} for transient_id in self.propagation_entries: @@ -371,6 +450,58 @@ class LXMRouter: if removed_count > 0: RNS.log("Cleaned "+str(removed_count)+" entries from the message store", RNS.LOG_DEBUG) + # Check size of message store and cull if needed + try: + message_storage_size = self.message_storage_size() + if message_storage_size != None: + if self.message_storage_limit != None and message_storage_size > self.message_storage_limit: + # Clean the message storage according to priorities + bytes_needed = message_storage_size - self.message_storage_limit + bytes_cleaned = 0 + + now = time.time() + weighted_entries = [] + for transient_id in self.propagation_entries: + entry = self.propagation_entries[transient_id] + + lxm_rcvd = entry[2] + lxm_size = entry[3] + age_weight = max(1, (now - lxm_rcvd)/60/60/24/4) + + weight = age_weight * lxm_size + weighted_entries.append([entry, weight, transient_id]) + + weighted_entries.sort(key=lambda we: we[1], reverse=True) + + i = 0 + while i < len(weighted_entries) and bytes_cleaned < bytes_needed: + try: + w = weighted_entries[i] + entry = w[0] + transient_id = w[2] + filepath = entry[1] + + if os.path.isfile(filepath): + os.unlink(filepath) + + self.propagation_entries.pop(transient_id) + bytes_cleaned += entry[3] + + RNS.log("Removed "+RNS.prettyhexrep(transient_id)+" with weight "+str(w[1])+" to clear up "+RNS.prettysize(entry[3])+", now cleaned "+RNS.prettysize(bytes_cleaned)+" out of "+RNS.prettysize(bytes_needed)+" needed", RNS.LOG_EXTREME) + + except Exception as e: + RNS.log("Error while cleaning LXMF message from message store. The contained exception was: "+str(e), RNS.LOG_ERROR) + + finally: + i += 1 + + RNS.log("LXMF message store size is now "+RNS.prettysize(self.message_storage_size()), RNS.LOG_EXTREME) + RNS.log("PE len "+str(len(self.propagation_entries))) + + + except Exception as e: + RNS.log("Could not clean the LXMF message store. The contained exception was: "+str(e), RNS.LOG_ERROR) + def exit_handler(self): if self.propagation_node: try: @@ -811,7 +942,7 @@ class LXMRouter: msg_file.write(lxmf_data) msg_file.close() - self.propagation_entries[transient_id] = [destination_hash, file_path] + self.propagation_entries[transient_id] = [destination_hash, file_path, time.time(), len(lxmf_data)] RNS.log("Received propagated LXMF message "+RNS.prettyhexrep(transient_id)+", adding to peer distribution queues...", RNS.LOG_DEBUG) for peer_id in self.peers: From 8be54bcc5f7f3ae3ecf63bf91d7bdfcc16933121 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 17 Jun 2022 13:42:44 +0200 Subject: [PATCH 087/436] Implemented peer authentication --- LXMF/LXMPeer.py | 1 + LXMF/LXMRouter.py | 56 +++++++++++++++++++++++++++++++++++++++++++---- setup.py | 2 +- 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index aee5263..7cc5c20 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -18,6 +18,7 @@ class LXMPeer: RESOURCE_TRANSFERRING = 0x05 ERROR_NO_IDENTITY = 0xf0 + ERROR_NO_ACCESS = 0xf1 # Maximum amount of time a peer can # be unreachable before it is removed diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 932e4cf..ff51254 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -38,6 +38,8 @@ class LXMRouter: PR_RECEIVING = 0x05 PR_RESPONSE_RECEIVED = 0x06 PR_COMPLETE = 0x07 + PR_NO_IDENTITY_RCVD = 0xf0 + PR_NO_ACCESS = 0xf1 PR_ALL_MESSAGES = 0x00 @@ -55,6 +57,8 @@ class LXMRouter: self.delivery_destinations = {} self.ignored_list = [] + self.allowed_list = [] + self.auth_required = False self.processing_outbound = False self.processing_inbound = False @@ -160,6 +164,26 @@ class LXMRouter: def get_outbound_propagation_node(self): return self.outbound_propagation_node + def set_authentication(self, required=None): + if required != None: + self.auth_required = required + + def requires_authentication(self): + return self.auth_required + + def allow(self, identity_hash=None): + if isinstance(identity_hash, bytes) and len(identity_hash) == RNS.Identity.TRUNCATED_HASHLENGTH//8: + self.allowed_list.append(identity_hash) + else: + raise ValueError("Allowed identity hash must be "+str(RNS.Identity.TRUNCATED_HASHLENGTH//8)+" bytes") + + def disallow(self, identity_hash=None): + if isinstance(identity_hash, bytes) and len(identity_hash) == RNS.Identity.TRUNCATED_HASHLENGTH//8: + if identity_hash in self.allowed_list: + self.allowed_list.pop(identity_hash) + else: + raise ValueError("Disallowed identity hash must be "+str(RNS.Identity.TRUNCATED_HASHLENGTH//8)+" bytes") + def request_messages_from_propagation_node(self, identity, max_messages = PR_ALL_MESSAGES): if max_messages == None: max_messages = LXMRouter.PR_ALL_MESSAGES @@ -208,7 +232,7 @@ class LXMRouter: self.outbound_propagation_link.teardown() self.outbound_propagation_link = None - self.acknowledge_sync_completion() + self.acknowledge_sync_completion(reset_state=True) def enable_propagation(self): try: @@ -552,9 +576,23 @@ class LXMRouter: RNS.log("Propagation node path request timed out", RNS.LOG_DEBUG) self.acknowledge_sync_completion() + def identity_allowed(self, identity): + if self.auth_required: + if identity.hash in self.allowed_list: + return True + else: + return False + + else: + return True + def message_get_request(self, path, data, request_id, remote_identity, requested_at): if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY + + elif not self.identity_allowed(remote_identity): + return LXMPeer.ERROR_NO_ACCESS + else: try: remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") @@ -622,8 +660,16 @@ class LXMRouter: RNS.log("Propagation node indicated missing identification on list request, tearing down link.", RNS.LOG_DEBUG) if self.outbound_propagation_link != None: self.outbound_propagation_link.teardown() + self.propagation_transfer_state = LXMRouter.PR_NO_IDENTITY_RCVD + + elif request_receipt.response == LXMPeer.ERROR_NO_ACCESS: + RNS.log("Propagation node did not allow list request, tearing down link.", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + self.propagation_transfer_state = LXMRouter.PR_NO_ACCESS + else: - if request_receipt.response != None: + if request_receipt.response != None and isinstance(request_receipt.response, list): haves = [] wants = [] if len(request_receipt.response) > 0: @@ -682,8 +728,10 @@ class LXMRouter: if self.outbound_propagation_link != None: self.outbound_propagation_link.teardown() - def acknowledge_sync_completion(self): - self.propagation_transfer_state = LXMRouter.PR_IDLE + def acknowledge_sync_completion(self, reset_state=False): + if reset_state or self.propagation_transfer_state <= LXMRouter.PR_COMPLETE: + self.propagation_transfer_state = LXMRouter.PR_IDLE + self.propagation_transfer_progress = 0.0 self.propagation_transfer_last_result = None self.wants_download_on_path_available_from = None diff --git a/setup.py b/setup.py index 8a090d1..54c22ed 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="lxmf", - version="0.1.6", + version="0.1.7", author="Mark Qvist", author_email="mark@unsigned.io", description="Lightweight Extensible Message Format for Reticulum", From e5204423b03dcaba0c6ab52f58fc207b201dafd7 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 17 Jun 2022 14:07:40 +0200 Subject: [PATCH 088/436] Implemented message store prioritisation --- LXMF/LXMRouter.py | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index ff51254..356d00b 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -56,6 +56,7 @@ class LXMRouter: self.direct_links = {} self.delivery_destinations = {} + self.prioritised_list = [] self.ignored_list = [] self.allowed_list = [] self.auth_required = False @@ -173,7 +174,8 @@ class LXMRouter: def allow(self, identity_hash=None): if isinstance(identity_hash, bytes) and len(identity_hash) == RNS.Identity.TRUNCATED_HASHLENGTH//8: - self.allowed_list.append(identity_hash) + if not identity_hash in self.allowed_list: + self.allowed_list.append(identity_hash) else: raise ValueError("Allowed identity hash must be "+str(RNS.Identity.TRUNCATED_HASHLENGTH//8)+" bytes") @@ -184,6 +186,20 @@ class LXMRouter: else: raise ValueError("Disallowed identity hash must be "+str(RNS.Identity.TRUNCATED_HASHLENGTH//8)+" bytes") + def prioritise(self, destination_hash=None): + if isinstance(destination_hash, bytes) and len(destination_hash) == RNS.Reticulum.TRUNCATED_HASHLENGTH//8: + if not destination_hash in self.prioritised_list: + self.prioritised_list.append(destination_hash) + else: + raise ValueError("Prioritised destination hash must be "+str(RNS.Reticulum.TRUNCATED_HASHLENGTH//8)+" bytes") + + def unprioritise(self, identity_hash=None): + if isinstance(destination_hash, bytes) and len(destination_hash) == RNS.Reticulum.TRUNCATED_HASHLENGTH//8: + if destination_hash in self.prioritised_list: + self.prioritised_list.pop(destination_hash) + else: + raise ValueError("Prioritised destination hash must be "+str(RNS.Reticulum.TRUNCATED_HASHLENGTH//8)+" bytes") + def request_messages_from_propagation_node(self, identity, max_messages = PR_ALL_MESSAGES): if max_messages == None: max_messages = LXMRouter.PR_ALL_MESSAGES @@ -318,10 +334,10 @@ class LXMRouter: limit_bytes = 0 if kilobytes != None: - limit_bytes += gigabytes*1000 + limit_bytes += kilobytes*1000 if megabytes != None: - limit_bytes += gigabytes*1000*1000 + limit_bytes += megabytes*1000*1000 if gigabytes != None: limit_bytes += gigabytes*1000*1000*1000 @@ -351,10 +367,10 @@ class LXMRouter: limit_bytes = 0 if kilobytes != None: - limit_bytes += gigabytes*1000 + limit_bytes += kilobytes*1000 if megabytes != None: - limit_bytes += gigabytes*1000*1000 + limit_bytes += megabytes*1000*1000 if gigabytes != None: limit_bytes += gigabytes*1000*1000*1000 @@ -488,11 +504,16 @@ class LXMRouter: for transient_id in self.propagation_entries: entry = self.propagation_entries[transient_id] + dst_hash = entry[0] lxm_rcvd = entry[2] lxm_size = entry[3] age_weight = max(1, (now - lxm_rcvd)/60/60/24/4) + if dst_hash in self.prioritised_list: + priority_weight = 0.1 + else: + priority_weight = 1.0 - weight = age_weight * lxm_size + weight = priority_weight * age_weight * lxm_size weighted_entries.append([entry, weight, transient_id]) weighted_entries.sort(key=lambda we: we[1], reverse=True) From 5bb187a502da6366729bd078bbcaa2ede646b189 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 17 Jun 2022 14:36:37 +0200 Subject: [PATCH 089/436] Update dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 54c22ed..449ec5f 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,6 @@ setuptools.setup( "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - install_requires=['rns>=0.3.6'], + install_requires=['rns>=0.3.8'], python_requires='>=3.6', ) From afe529fa0ad01495a67aca2060b240e15b6e9b4f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 17 Jun 2022 15:17:27 +0200 Subject: [PATCH 090/436] Updated log output --- LXMF/LXMRouter.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 356d00b..fa9ea51 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -309,7 +309,12 @@ class LXMRouter: self.propagation_destination.register_request_handler(LXMPeer.OFFER_REQUEST_PATH, self.offer_request, allow = RNS.Destination.ALLOW_ALL) self.propagation_destination.register_request_handler(LXMPeer.MESSAGE_GET_PATH, self.message_get_request, allow = RNS.Destination.ALLOW_ALL) - RNS.log("LXMF Propagation Node message store size is "+RNS.prettysize(self.message_storage_size()), RNS.LOG_DEBUG) + if self.message_storage_limit != None: + limit_str = ", limit is "+RNS.prettysize(self.message_storage_limit) + else: + limit_str = "" + + RNS.log("LXMF Propagation Node message store size is "+RNS.prettysize(self.message_storage_size())+limit_str, RNS.LOG_DEBUG) self.announce_propagation_node() From ff48661f60df3deab26029b3d45ff2e2a96cae5e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 14 Sep 2022 16:30:42 +0200 Subject: [PATCH 091/436] Added version file --- LXMF/__init__.py | 2 ++ LXMF/_version.py | 1 + setup.py | 4 +++- 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 LXMF/_version.py diff --git a/LXMF/__init__.py b/LXMF/__init__.py index e0924fe..dc0abbb 100644 --- a/LXMF/__init__.py +++ b/LXMF/__init__.py @@ -3,5 +3,7 @@ import glob from .LXMessage import LXMessage from .LXMRouter import LXMRouter +from ._version import __version__ + modules = glob.glob(os.path.dirname(__file__)+"/*.py") __all__ = [ os.path.basename(f)[:-3] for f in modules if not f.endswith('__init__.py')] \ No newline at end of file diff --git a/LXMF/_version.py b/LXMF/_version.py new file mode 100644 index 0000000..283b03a --- /dev/null +++ b/LXMF/_version.py @@ -0,0 +1 @@ +__version__ = "0.1.7" \ No newline at end of file diff --git a/setup.py b/setup.py index 449ec5f..5695c2a 100644 --- a/setup.py +++ b/setup.py @@ -3,9 +3,11 @@ import setuptools with open("README.md", "r") as fh: long_description = fh.read() +exec(open("LXMF/_version.py", "r").read()) + setuptools.setup( name="lxmf", - version="0.1.7", + version=__version__, author="Mark Qvist", author_email="mark@unsigned.io", description="Lightweight Extensible Message Format for Reticulum", From ce91cab7efafc00618c3b42942f1ac67daf62658 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 30 Sep 2022 00:49:47 +0200 Subject: [PATCH 092/436] Updated version and dependencies --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 283b03a..a23ef3f 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.1.7" \ No newline at end of file +__version__ = "0.1.8" \ No newline at end of file diff --git a/setup.py b/setup.py index 5695c2a..0402bb8 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,6 @@ setuptools.setup( "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - install_requires=['rns>=0.3.8'], + install_requires=['rns>=0.3.12'], python_requires='>=3.6', ) From a26dfe3b7b421e95d3b1e75fc59e8a1e9da9e895 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 4 Oct 2022 09:32:38 +0200 Subject: [PATCH 093/436] Version and dependency update --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index a23ef3f..da942bc 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.1.8" \ No newline at end of file +__version__ = "0.1.9" \ No newline at end of file diff --git a/setup.py b/setup.py index 0402bb8..3a09309 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,6 @@ setuptools.setup( "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - install_requires=['rns>=0.3.12'], + install_requires=['rns>=0.3.13'], python_requires='>=3.6', ) From 323a96852501cc0896bf6869c3d3439814328fcb Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 6 Oct 2022 23:38:10 +0200 Subject: [PATCH 094/436] Update version and dependencies --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index da942bc..49f34f4 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.1.9" \ No newline at end of file +__version__ = "0.2.0" \ No newline at end of file diff --git a/setup.py b/setup.py index 3a09309..59f67fe 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,6 @@ setuptools.setup( "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - install_requires=['rns>=0.3.13'], + install_requires=['rns>=0.3.14'], python_requires='>=3.6', ) From 6b28cd869851d7e3c9f98d01ddda561f8627a034 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 20 Oct 2022 14:59:25 +0200 Subject: [PATCH 095/436] Updated dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 59f67fe..d0968af 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,6 @@ setuptools.setup( "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - install_requires=['rns>=0.3.14'], + install_requires=['rns>=0.3.15'], python_requires='>=3.6', ) From 920beaf6f5cdcab6407433bcafbee3b5636a4a38 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 20 Oct 2022 20:57:14 +0200 Subject: [PATCH 096/436] Updated dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d0968af..991b5ba 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,6 @@ setuptools.setup( "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - install_requires=['rns>=0.3.15'], + install_requires=['rns>=0.3.16'], python_requires='>=3.6', ) From 2f43d660d5385236b4c711af3a35cf8b6f837470 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 20 Oct 2022 20:57:47 +0200 Subject: [PATCH 097/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 49f34f4..3ced358 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.2.0" \ No newline at end of file +__version__ = "0.2.1" From a1dae1fc15d77bcd67df74e245b6cc1baf46e5a9 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 22 Oct 2022 14:35:24 +0200 Subject: [PATCH 098/436] Updated versions --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 3ced358..b5fdc75 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.2.1" +__version__ = "0.2.2" diff --git a/setup.py b/setup.py index 991b5ba..7c799c1 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,6 @@ setuptools.setup( "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - install_requires=['rns>=0.3.16'], + install_requires=['rns>=0.3.17'], python_requires='>=3.6', ) From 8cdf12e866aab570b83881653e1fe69dd7976f4e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 22 Oct 2022 14:35:44 +0200 Subject: [PATCH 099/436] Added basic LXMF daemon utility --- LXMF/Utilities/lxmd.py | 264 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 264 insertions(+) create mode 100644 LXMF/Utilities/lxmd.py diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py new file mode 100644 index 0000000..b1267d7 --- /dev/null +++ b/LXMF/Utilities/lxmd.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 + +# MIT License +# +# Copyright (c) 2016-2022 Mark Qvist / unsigned.io +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import RNS +import LXMF +import argparse +import time +import os + +from LXMF._version import __version__ + +from RNS.vendor.configobj import ConfigObj +identity = None +lxmd_config = None +message_router = None +lxmf_destination = None +active_configuration = {} + +def create_default_config(configpath): + lxmd_config = ConfigObj(__default_lxmd_config__.splitlines()) + lxmd_config.filename = configpath + lxmd_config.write() + +def apply_config(): + # TODO: Apply configuration + active_configuration["display_name"] = "Anonymous Peer" + active_configuration["enable_propagation_node"] = True + active_configuration["message_storage_limit"] = 2000 + active_configuration["prioritised_lxmf_destinations"] = [] + active_configuration["ignored_lxmf_destinations"] = [] + +def lxmf_delivery(lxm): + # TODO: Implement delivery callback + pass + +def program_setup(configdir, rnsconfigdir, run_pn, verbosity = 0, quietness = 0, service = False): + targetloglevel = 3+verbosity-quietness + + if service: + targetlogdest = RNS.LOG_FILE + targetloglevel = None + else: + targetlogdest = RNS.LOG_STDOUT + + # Get configuration + if configdir == None: + if os.path.isdir("/etc/lxmd") and os.path.isfile("/etc/lxmd/config"): + configdir = "/etc/lxmd" + elif os.path.isdir(RNS.Reticulum.userdir+"/.config/lxmd") and os.path.isfile(Reticulum.userdir+"/.config/lxmd/config"): + configdir = RNS.Reticulum.userdir+"/.config/lxmd" + else: + configdir = RNS.Reticulum.userdir+"/.lxmd" + + configpath = configdir+"/config" + identitypath = configdir+"/identity" + storagedir = configdir+"/storage" + + if not os.path.isdir(storagedir): + os.makedirs(storagedir) + + if os.path.isfile(configpath): + try: + lxmd_config = ConfigObj(configpath) + except Exception as e: + RNS.log("Could not parse the configuration at "+configpath, RNS.LOG_ERROR) + RNS.log("Check your configuration file for errors!", RNS.LOG_ERROR) + RNS.panic() + else: + RNS.log("Could not load config file, creating default configuration file...") + create_default_config(configpath) + RNS.log("Default config file created. Make any necessary changes in "+configpath+" and restart Reticulum if needed.") + time.sleep(1.5) + + apply_config() + RNS.log("Configuration loaded from "+configpath, RNS.LOG_VERBOSE) + + # Start Reticulum + RNS.log("Substantiating Reticulum...") + reticulum = RNS.Reticulum(configdir=rnsconfigdir, loglevel=targetloglevel, logdest=targetlogdest) + + # Generate or load primary identity + if os.path.isfile(identitypath): + try: + identity = RNS.Identity.from_file(identitypath) + if identity != None: + RNS.log("Loaded Primary Identity %s" % (str(identity))) + else: + RNS.log("Could not load the Primary Identity from "+identitypath, RNS.LOG_ERROR) + nomadnet.panic() + except Exception as e: + RNS.log("Could not load the Primary Identity from "+identitypath, RNS.LOG_ERROR) + RNS.log("The contained exception was: %s" % (str(e)), RNS.LOG_ERROR) + exit(1) + else: + try: + RNS.log("No Primary Identity file found, creating new...") + identity = RNS.Identity() + identity.to_file(identitypath) + RNS.log("Created new Primary Identity %s" % (str(identity))) + except Exception as e: + RNS.log("Could not create and save a new Primary Identity", RNS.LOG_ERROR) + RNS.log("The contained exception was: %s" % (str(e)), RNS.LOG_ERROR) + exit(2) + + # Start LXMF + message_router = LXMF.LXMRouter(identity = identity, storagepath = storagedir, autopeer = True) + message_router.register_delivery_callback(lxmf_delivery) + + for destination_hash in active_configuration["ignored_lxmf_destinations"]: + message_router.ignore_destination(destination_hash) + + lxmf_destination = message_router.register_delivery_identity(identity, display_name=active_configuration["display_name"]) + + RNS.Identity.remember( + packet_hash=None, + destination_hash=lxmf_destination.hash, + public_key=identity.get_public_key(), + app_data=None + ) + + RNS.log("LXMF Router ready to receive on "+RNS.prettyhexrep(lxmf_destination.hash)) + + if active_configuration["enable_propagation_node"]: + message_router.set_message_storage_limit(megabytes=active_configuration["message_storage_limit"]) + for dest_str in active_configuration["prioritised_lxmf_destinations"]: + try: + dest_hash = bytes.fromhex(dest_str) + if len(dest_hash) == RNS.Reticulum.TRUNCATED_HASHLENGTH//8: + message_router.prioritise(dest_hash) + + except Exception as e: + RNS.log("Cannot prioritise "+str(dest_str)+", it is not a valid destination hash", RNS.LOG_ERROR) + + message_router.enable_propagation() + + RNS.log("LXMF Propagation Node started on "+RNS.prettyhexrep(message_router.propagation_destination.hash)) + + RNS.log("Started lxmd version {version}".format(version=__version__), RNS.LOG_NOTICE) + + while True: + time.sleep(1) + +def main(): + try: + parser = argparse.ArgumentParser(description="Lightweight Extensible Messaging Daemon") + parser.add_argument("--config", action="store", default=None, help="path to alternative lxmd config directory", type=str) + parser.add_argument("--rnsconfig", action="store", default=None, help="path to alternative Reticulum config directory", type=str) + parser.add_argument("-p", "--propagation-node", action="store_true", default=False, help="run an LXMF Propagation Node") + parser.add_argument("-i", "--on-inbound", action="store", metavar="PATH", default=None, help="executable to run when a message is received", type=str) + parser.add_argument("-v", "--verbose", action="count", default=0) + parser.add_argument("-q", "--quiet", action="count", default=0) + parser.add_argument("-s", "--service", action="store_true", default=False, help="lxmd is running as a service and should log to file") + parser.add_argument("--exampleconfig", action="store_true", default=False, help="print verbose configuration example to stdout and exit") + parser.add_argument("--version", action="version", version="lxmd {version}".format(version=__version__)) + + args = parser.parse_args() + + if args.exampleconfig: + print(__default_lxmd_config__) + exit() + + program_setup( + configdir = args.config, + rnsconfigdir=args.rnsconfig, + run_pn=args.propagation_node, + verbosity=args.verbose, + quietness=args.quiet, + service=args.service + ) + + except KeyboardInterrupt: + print("") + exit() + +__default_lxmd_config__ = """# This is an example LXM Daemon config file. +# You should probably edit it to suit your +# intended usage. + +[propagation] + +# Whether to enable propagation node +enable_node = no + +# Automatic announce interval in minutes. +# 6 hours by default. +announce_interval = 360 + +# Whether to announce when the node starts. +announce_at_start = yes + +# Wheter to automatically peer with other +# propagation nodes on the network. +autopeer = yes + +# The maximum amount of storage to use for +# the LXMF Propagation Node message store, +# specified in megabytes. When this limit +# is reached, LXMF will periodically remove +# messages in its message store. By default, +# LXMF prioritises keeping messages that are +# new and small. Large and old messages will +# be removed first. This setting is optional +# and defaults to 2 gigabytes. +# message_storage_limit = 2000 + +# You can tell the LXMF message router to +# prioritise storage for one or more +# destinations. If the message store reaches +# the specified limit, LXMF will prioritise +# keeping messages for destinations specified +# with this option. This setting is optional, +# and generally you do not need to use it. +# prioritise_destinations = 41d20c727598a3fbbdf9106133a3a0ed, d924b81822ca24e68e2effea99bcb8cf + + +[lxmf] + +# The LXM Daemon will create an LXMF destination +# that it can receive messages on. This option sets +# the announced display name for this destination. +display_name = Anonymous Peer + +# It is possible to announce the internal LXMF +# destination when the LXM Daemon starts up. +announce_at_start = no + + +[logging] +# Valid log levels are 0 through 7: +# 0: Log only critical information +# 1: Log errors and lower log levels +# 2: Log warnings and lower log levels +# 3: Log notices and lower log levels +# 4: Log info and lower (this is the default) +# 5: Verbose logging +# 6: Debug logging +# 7: Extreme logging +loglevel = 4 + +""" + +if __name__ == "__main__": + main() From 8bb21688e5f803805bac2ed1f21e65c77a8359b0 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 22 Oct 2022 20:12:54 +0200 Subject: [PATCH 100/436] Added config parsing and announce jobs to lxmd --- LXMF/Utilities/lxmd.py | 175 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 164 insertions(+), 11 deletions(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index b1267d7..010ba7a 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -22,39 +22,137 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +DEFFERED_JOBS_DELAY = 10 +JOBS_INTERVAL = 5 + import RNS import LXMF import argparse +import threading import time import os from LXMF._version import __version__ from RNS.vendor.configobj import ConfigObj + +configpath = None +ignoredpath = None +identitypath = None +storagedir = None +targetloglevel = None + identity = None lxmd_config = None message_router = None lxmf_destination = None active_configuration = {} +last_peer_announce = None +last_node_announce = None + def create_default_config(configpath): lxmd_config = ConfigObj(__default_lxmd_config__.splitlines()) lxmd_config.filename = configpath lxmd_config.write() def apply_config(): - # TODO: Apply configuration - active_configuration["display_name"] = "Anonymous Peer" - active_configuration["enable_propagation_node"] = True - active_configuration["message_storage_limit"] = 2000 - active_configuration["prioritised_lxmf_destinations"] = [] - active_configuration["ignored_lxmf_destinations"] = [] + global active_configuration, targetloglevel + try: + # Load peer settings + if "lxmf" in lxmd_config and "display_name" in lxmd_config["lxmf"]: + active_configuration["display_name"] = lxmd_config["lxmf"]["display_name"] + else: + active_configuration["display_name"] = "Anonymous Peer" + + if "lxmf" in lxmd_config and "announce_at_start" in lxmd_config["lxmf"]: + active_configuration["peer_announce_at_start"] = lxmd_config["lxmf"].as_bool("announce_at_start") + else: + active_configuration["peer_announce_at_start"] = False + + if "lxmf" in lxmd_config and "announce_interval" in lxmd_config["lxmf"]: + active_configuration["peer_announce_interval"] = lxmd_config["lxmf"].as_int("announce_interval")*60 + else: + active_configuration["peer_announce_interval"] = None + + if "lxmf" in lxmd_config and "on_inbound" in lxmd_config["lxmf"]: + active_configuration["on_inbound"] = lxmd_config["lxmf"]["on_inbound"] + else: + active_configuration["on_inbound"] = None + + # Load propagation node settings + if "propagation" in lxmd_config and "enable_node" in lxmd_config["propagation"]: + active_configuration["enable_propagation_node"] = lxmd_config["propagation"].as_bool("enable_node") + else: + active_configuration["enable_propagation_node"] = False + + if "propagation" in lxmd_config and "announce_at_start" in lxmd_config["propagation"]: + active_configuration["node_announce_at_start"] = lxmd_config["propagation"].as_bool("announce_at_start") + else: + active_configuration["node_announce_at_start"] = False + + if "propagation" in lxmd_config and "announce_interval" in lxmd_config["propagation"]: + active_configuration["node_announce_interval"] = lxmd_config["propagation"].as_int("announce_interval")*60 + else: + active_configuration["node_announce_interval"] = None + + if "propagation" in lxmd_config and "message_storage_limit" in lxmd_config["propagation"]: + active_configuration["message_storage_limit"] = lxmd_config["propagation"].as_float("message_storage_limit") + if active_configuration["message_storage_limit"] < 0.005: + active_configuration["message_storage_limit"] = 0.005 + else: + active_configuration["message_storage_limit"] = 2000 + + if "propagation" in lxmd_config and "prioritise_destinations" in lxmd_config["propagation"]: + active_configuration["prioritised_lxmf_destinations"] = lxmd_config["propagation"].as_list("prioritise_destinations") + else: + active_configuration["prioritised_lxmf_destinations"] = [] + + # Load various settings + if "logging" in lxmd_config and "loglevel" in lxmd_config["logging"]: + targetloglevel = lxmd_config["logging"].as_int("loglevel") + + active_configuration["ignored_lxmf_destinations"] = [] + if os.path.isfile(ignoredpath): + try: + fh = open(ignoredpath, "rb") + ignored_input = fh.read() + fh.close() + + ignored_hash_strs = ignored_input.splitlines() + + for hash_str in ignored_hash_strs: + if len(hash_str) == RNS.Identity.TRUNCATED_HASHLENGTH//8*2: + try: + ignored_hash = bytes.fromhex(hash_str.decode("utf-8")) + active_configuration["ignored_lxmf_destinations"].append(ignored_hash) + + except Exception as e: + RNS.log("Could not decode RNS Identity hash from: "+str(hash_str), RNS.LOG_DEBUG) + RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + + except Exception as e: + RNS.log("Error while loading list of ignored destinations: "+str(e), RNS.LOG_ERROR) + + except Exception as e: + RNS.log("Could not apply LXM Daemon configuration. The contained exception was: "+str(e), RNS.LOG_ERROR) + raise e + exit(3) def lxmf_delivery(lxm): - # TODO: Implement delivery callback - pass + global active_configuration + RNS.log("Received "+str(lxm), RNS.LOG_DEBUG) + if active_configuration["on_inbound"]: + RNS.log("Calling external program to handle message", RNS.LOG_DEBUG) + else: + RNS.log("No action defined for inbound messages, ignoring", RNS.LOG_EXTREME) + + +def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbound = None, verbosity = 0, quietness = 0, service = False): + global configpath, ignoredpath, identitypath, storagedir + global lxmd_config, active_configuration, targetloglevel + global message_router, lxmf_destination -def program_setup(configdir, rnsconfigdir, run_pn, verbosity = 0, quietness = 0, service = False): targetloglevel = 3+verbosity-quietness if service: @@ -73,6 +171,7 @@ def program_setup(configdir, rnsconfigdir, run_pn, verbosity = 0, quietness = 0, configdir = RNS.Reticulum.userdir+"/.lxmd" configpath = configdir+"/config" + ignoredpath = configdir+"/ignored" identitypath = configdir+"/identity" storagedir = configdir+"/storage" @@ -107,7 +206,7 @@ def program_setup(configdir, rnsconfigdir, run_pn, verbosity = 0, quietness = 0, RNS.log("Loaded Primary Identity %s" % (str(identity))) else: RNS.log("Could not load the Primary Identity from "+identitypath, RNS.LOG_ERROR) - nomadnet.panic() + exit(4) except Exception as e: RNS.log("Could not load the Primary Identity from "+identitypath, RNS.LOG_ERROR) RNS.log("The contained exception was: %s" % (str(e)), RNS.LOG_ERROR) @@ -141,7 +240,7 @@ def program_setup(configdir, rnsconfigdir, run_pn, verbosity = 0, quietness = 0, RNS.log("LXMF Router ready to receive on "+RNS.prettyhexrep(lxmf_destination.hash)) - if active_configuration["enable_propagation_node"]: + if run_pn or active_configuration["enable_propagation_node"]: message_router.set_message_storage_limit(megabytes=active_configuration["message_storage_limit"]) for dest_str in active_configuration["prioritised_lxmf_destinations"]: try: @@ -158,9 +257,49 @@ def program_setup(configdir, rnsconfigdir, run_pn, verbosity = 0, quietness = 0, RNS.log("Started lxmd version {version}".format(version=__version__), RNS.LOG_NOTICE) + threading.Thread(target=deferred_start_jobs, daemon=True).start() + while True: time.sleep(1) +def jobs(): + global active_configuration, last_peer_announce, last_node_announce + global message_router, lxmf_destination + + while True: + try: + if time.time() > last_peer_announce + active_configuration["peer_announce_interval"]: + RNS.log("Sending announce for LXMF delivery destination", RNS.LOG_EXTREME) + message_router.announce(lxmf_destination.hash) + last_peer_announce = time.time() + + if time.time() > last_node_announce + active_configuration["node_announce_interval"]: + RNS.log("Sending announce for LXMF Propagation Node", RNS.LOG_EXTREME) + message_router.announce_propagation_node() + last_node_announce = time.time() + + except Exception as e: + RNS.log("An error occurred while running periodic jobs. The contained exception was: "+str(e), RNS.LOG_ERROR) + + time.sleep(JOBS_INTERVAL) + +def deferred_start_jobs(): + global active_configuration, last_peer_announce, last_node_announce + global message_router, lxmf_destination + time.sleep(DEFFERED_JOBS_DELAY) + RNS.log("Running deferred start jobs") + if active_configuration["peer_announce_at_start"]: + RNS.log("Sending announce for LXMF delivery destination", RNS.LOG_EXTREME) + message_router.announce(lxmf_destination.hash) + + if active_configuration["node_announce_at_start"]: + RNS.log("Sending announce for LXMF Propagation Node", RNS.LOG_EXTREME) + message_router.announce_propagation_node() + + last_peer_announce = time.time() + last_node_announce = time.time() + threading.Thread(target=jobs, daemon=True).start() + def main(): try: parser = argparse.ArgumentParser(description="Lightweight Extensible Messaging Daemon") @@ -184,6 +323,7 @@ def main(): configdir = args.config, rnsconfigdir=args.rnsconfig, run_pn=args.propagation_node, + on_inbound=args.on_inbound, verbosity=args.verbose, quietness=args.quiet, service=args.service @@ -245,6 +385,19 @@ display_name = Anonymous Peer # destination when the LXM Daemon starts up. announce_at_start = no +# You can also announce the delivery destination +# at a specified interval. This is not enabled by +# default. +# announce_interval = 360 + +# You can configure an external program to be run +# every time a message is received. The program +# will receive as an argument the full path to the +# message saved as a file. The example below will +# simply result in the message getting deleted as +# soon as it has been received. +# on_inbound = rm + [logging] # Valid log levels are 0 through 7: From 2181584f69773e53fecea90c89013af91f724362 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 22 Oct 2022 20:26:24 +0200 Subject: [PATCH 101/436] Added lxmd entry point --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7c799c1..b044de6 100644 --- a/setup.py +++ b/setup.py @@ -14,12 +14,17 @@ setuptools.setup( long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/markqvist/lxmf", - packages=setuptools.find_packages(), + packages=["LXMF", "LXMF.Utilities"], classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], + entry_points= { + 'console_scripts': [ + 'lxmd=LXMF.Utilities.lxmd:main', + ] + }, install_requires=['rns>=0.3.17'], python_requires='>=3.6', ) From 1c4928a29b2fb7b55b7430892d60ce53eae7adde Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 22 Oct 2022 21:07:27 +0200 Subject: [PATCH 102/436] Added external processing of received messages to lxmd --- LXMF/Utilities/lxmd.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 010ba7a..147e131 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -29,6 +29,8 @@ import RNS import LXMF import argparse import threading +import subprocess +import shlex import time import os @@ -40,6 +42,7 @@ configpath = None ignoredpath = None identitypath = None storagedir = None +lxmdir = None targetloglevel = None identity = None @@ -140,16 +143,27 @@ def apply_config(): exit(3) def lxmf_delivery(lxm): - global active_configuration - RNS.log("Received "+str(lxm), RNS.LOG_DEBUG) - if active_configuration["on_inbound"]: - RNS.log("Calling external program to handle message", RNS.LOG_DEBUG) - else: - RNS.log("No action defined for inbound messages, ignoring", RNS.LOG_EXTREME) + global active_configuration, lxmdir + + try: + written_path = lxm.write_to_directory(lxmdir) + RNS.log("Received "+str(lxm)+" written to "+str(written_path), RNS.LOG_DEBUG) + + if active_configuration["on_inbound"]: + RNS.log("Calling external program to handle message", RNS.LOG_DEBUG) + command = active_configuration["on_inbound"] + processing_command = command+" \""+written_path+"\"" + return_code = subprocess.call(shlex.split(processing_command), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + else: + RNS.log("No action defined for inbound messages, ignoring", RNS.LOG_DEBUG) + + except Exception as e: + RNS.log("Error occurred while processing received message "+str(lxm)+". The contained exception was: "+str(e), RNS.LOG_ERROR) def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbound = None, verbosity = 0, quietness = 0, service = False): - global configpath, ignoredpath, identitypath, storagedir + global configpath, ignoredpath, identitypath, storagedir, lxmdir global lxmd_config, active_configuration, targetloglevel global message_router, lxmf_destination @@ -174,10 +188,14 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo ignoredpath = configdir+"/ignored" identitypath = configdir+"/identity" storagedir = configdir+"/storage" + lxmdir = storagedir+"/messages" if not os.path.isdir(storagedir): os.makedirs(storagedir) + if not os.path.isdir(lxmdir): + os.makedirs(lxmdir) + if os.path.isfile(configpath): try: lxmd_config = ConfigObj(configpath) From 2d81b47b6ab0fbd0a57284413d8b2f4309c46f02 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 22 Oct 2022 21:25:15 +0200 Subject: [PATCH 103/436] Added peering settings to lxmd --- LXMF/LXMRouter.py | 3 ++- LXMF/Utilities/lxmd.py | 21 ++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index fa9ea51..376f118 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -976,7 +976,8 @@ class LXMRouter: remote_hash = remote_destination.hash if not remote_hash in self.peers: - self.peer(remote_hash, remote_timebase) + if self.autopeer and RNS.Transport.hops_to(remote_hash) <= self.autopeer_maxdepth: + self.peer(remote_hash, remote_timebase) messages = data[1] for lxmf_data in messages: diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 147e131..a089e16 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -94,6 +94,16 @@ def apply_config(): else: active_configuration["node_announce_at_start"] = False + if "propagation" in lxmd_config and "autopeer" in lxmd_config["propagation"]: + active_configuration["autopeer"] = lxmd_config["propagation"].as_bool("autopeer") + else: + active_configuration["autopeer"] = True + + if "propagation" in lxmd_config and "autopeer_maxdepth" in lxmd_config["propagation"]: + active_configuration["autopeer_maxdepth"] = lxmd_config["propagation"].as_int("autopeer_maxdepth") + else: + active_configuration["autopeer_maxdepth"] = None + if "propagation" in lxmd_config and "announce_interval" in lxmd_config["propagation"]: active_configuration["node_announce_interval"] = lxmd_config["propagation"].as_int("announce_interval")*60 else: @@ -241,7 +251,12 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo exit(2) # Start LXMF - message_router = LXMF.LXMRouter(identity = identity, storagepath = storagedir, autopeer = True) + message_router = LXMF.LXMRouter( + identity = identity, + storagepath = storagedir, + autopeer = active_configuration["autopeer"], + autopeer_maxdepth = active_configuration["autopeer_maxdepth"], + ) message_router.register_delivery_callback(lxmf_delivery) for destination_hash in active_configuration["ignored_lxmf_destinations"]: @@ -371,6 +386,10 @@ announce_at_start = yes # propagation nodes on the network. autopeer = yes +# The maximum peering depth (in hops) for +# automatically peered nodes. +autopeer_maxdepth = 4 + # The maximum amount of storage to use for # the LXMF Propagation Node message store, # specified in megabytes. When this limit From adbbaa532ba34f57a7e9c4eda936e269ad794e38 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 22 Oct 2022 22:12:14 +0200 Subject: [PATCH 104/436] Added autopeer and authentication options to lxmd config --- LXMF/LXMRouter.py | 13 +++++++--- LXMF/Utilities/lxmd.py | 55 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 60 insertions(+), 8 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 376f118..cd387e4 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -47,7 +47,7 @@ class LXMRouter: ### Developer-facing API ############################## ####################################################### - def __init__(self, identity = None, storagepath = None, autopeer = AUTOPEER, autopeer_maxdepth = AUTOPEER_MAXDEPTH): + def __init__(self, identity = None, storagepath = None, autopeer = AUTOPEER, autopeer_maxdepth = None): random.seed(os.urandom(10)) self.pending_inbound = [] @@ -93,8 +93,15 @@ class LXMRouter: self.lxmf_query_destination = RNS.Destination(None, RNS.Destination.IN, RNS.Destination.PLAIN, APP_NAME, "query") self.propagation_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation") - self.autopeer = autopeer - self.autopeer_maxdepth = autopeer_maxdepth + if autopeer != None: + self.autopeer = autopeer + else: + self.autopeer = LXMRouter.AUTOPEER + + if autopeer_maxdepth != None: + self.autopeer_maxdepth = autopeer_maxdepth + else: + self.autopeer_maxdepth = LXMRouter.AUTOPEER_MAXDEPTH self.peers = {} self.propagation_entries = {} diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index a089e16..d033424 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -89,6 +89,11 @@ def apply_config(): else: active_configuration["enable_propagation_node"] = False + if "propagation" in lxmd_config and "auth_required" in lxmd_config["propagation"]: + active_configuration["auth_required"] = lxmd_config["propagation"].as_bool("auth_required") + else: + active_configuration["auth_required"] = False + if "propagation" in lxmd_config and "announce_at_start" in lxmd_config["propagation"]: active_configuration["node_announce_at_start"] = lxmd_config["propagation"].as_bool("announce_at_start") else: @@ -141,12 +146,34 @@ def apply_config(): active_configuration["ignored_lxmf_destinations"].append(ignored_hash) except Exception as e: - RNS.log("Could not decode RNS Identity hash from: "+str(hash_str), RNS.LOG_DEBUG) + RNS.log("Could not decode hash from: "+str(hash_str), RNS.LOG_DEBUG) RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) except Exception as e: RNS.log("Error while loading list of ignored destinations: "+str(e), RNS.LOG_ERROR) + active_configuration["allowed_identities"] = [] + if os.path.isfile(allowedpath): + try: + fh = open(allowedpath, "rb") + allowed_input = fh.read() + fh.close() + + allowed_hash_strs = allowed_input.splitlines() + + for hash_str in allowed_hash_strs: + if len(hash_str) == RNS.Identity.TRUNCATED_HASHLENGTH//8*2: + try: + allowed_hash = bytes.fromhex(hash_str.decode("utf-8")) + active_configuration["allowed_identities"].append(allowed_hash) + + except Exception as e: + RNS.log("Could not decode hash from: "+str(hash_str), RNS.LOG_DEBUG) + RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + + except Exception as e: + RNS.log("Error while loading list of allowed identities: "+str(e), RNS.LOG_ERROR) + except Exception as e: RNS.log("Could not apply LXM Daemon configuration. The contained exception was: "+str(e), RNS.LOG_ERROR) raise e @@ -173,7 +200,7 @@ def lxmf_delivery(lxm): def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbound = None, verbosity = 0, quietness = 0, service = False): - global configpath, ignoredpath, identitypath, storagedir, lxmdir + global configpath, ignoredpath, identitypath, allowedpath, storagedir, lxmdir global lxmd_config, active_configuration, targetloglevel global message_router, lxmf_destination @@ -195,7 +222,8 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo configdir = RNS.Reticulum.userdir+"/.lxmd" configpath = configdir+"/config" - ignoredpath = configdir+"/ignored" + ignoredpath = configdir+"/ignored" + allowedpath = configdir+"/allowed" identitypath = configdir+"/identity" storagedir = configdir+"/storage" lxmdir = storagedir+"/messages" @@ -249,7 +277,7 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo RNS.log("Could not create and save a new Primary Identity", RNS.LOG_ERROR) RNS.log("The contained exception was: %s" % (str(e)), RNS.LOG_ERROR) exit(2) - + # Start LXMF message_router = LXMF.LXMRouter( identity = identity, @@ -271,6 +299,12 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo app_data=None ) + # Set up authentication + if active_configuration["auth_required"]: + message_router.set_authentication(required=True) + for identity_hash in active_configuration["allowed_identities"]: + message_router.allow(identity_hash) + RNS.log("LXMF Router ready to receive on "+RNS.prettyhexrep(lxmf_destination.hash)) if run_pn or active_configuration["enable_propagation_node"]: @@ -288,6 +322,9 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo RNS.log("LXMF Propagation Node started on "+RNS.prettyhexrep(message_router.propagation_destination.hash)) + if len(active_configuration["allowed_identities"]) == 0: + RNS.log("Clint authentication was enabled, but no identity hashes could be loaded from "+str(allowedpath)+". Nobody will be able to sync messages from this propagation node.", RNS.LOG_WARNING) + RNS.log("Started lxmd version {version}".format(version=__version__), RNS.LOG_NOTICE) threading.Thread(target=deferred_start_jobs, daemon=True).start() @@ -410,6 +447,14 @@ autopeer_maxdepth = 4 # and generally you do not need to use it. # prioritise_destinations = 41d20c727598a3fbbdf9106133a3a0ed, d924b81822ca24e68e2effea99bcb8cf +# By default, any destination is allowed to +# connect and download messages, but you can +# optionally restrict this. If you enable +# authentication, you must provide a list of +# allowed identity hashes in the a file named +# "allowed" in the lxmd config directory. +auth_required = no + [lxmf] @@ -433,7 +478,7 @@ announce_at_start = no # message saved as a file. The example below will # simply result in the message getting deleted as # soon as it has been received. -# on_inbound = rm +# on_inbound = rm [logging] From cacddfdf30818f9bdb4581c346ec7148b0eead43 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 22 Oct 2022 22:35:07 +0200 Subject: [PATCH 105/436] Save locally delivered transient id cache on sync completion --- LXMF/LXMRouter.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index cd387e4..e7b90af 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -463,7 +463,7 @@ class LXMRouter: removed_entries = [] for transient_id in self.locally_delivered_transient_ids: timestamp = self.locally_delivered_transient_ids[transient_id] - if now > timestamp+LXMRouter.MESSAGE_EXPIRY*1.1: + if now > timestamp+LXMRouter.MESSAGE_EXPIRY*1.25: removed_entries.append(transient_id) for transient_id in removed_entries: @@ -559,6 +559,18 @@ class LXMRouter: except Exception as e: RNS.log("Could not clean the LXMF message store. The contained exception was: "+str(e), RNS.LOG_ERROR) + def save_locally_delivered_transient_ids(self): + try: + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) + + locally_delivered_file = open(self.storagepath+"/local_deliveries", "wb") + locally_delivered_file.write(msgpack.packb(self.locally_delivered_transient_ids)) + locally_delivered_file.close() + + except Exception as e: + RNS.log("Could not save locally delivered message ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + def exit_handler(self): if self.propagation_node: try: @@ -576,16 +588,7 @@ class LXMRouter: except Exception as e: RNS.log("Could not save propagation node peers to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) - try: - if not os.path.isdir(self.storagepath): - os.makedirs(self.storagepath) - - locally_delivered_file = open(self.storagepath+"/local_deliveries", "wb") - locally_delivered_file.write(msgpack.packb(self.locally_delivered_transient_ids)) - locally_delivered_file.close() - - except Exception as e: - RNS.log("Could not save locally delivered message ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + self.save_locally_delivered_transient_ids() def __str__(self): return "" @@ -730,6 +733,14 @@ class LXMRouter: RNS.log("Propagation node indicated missing identification on get request, tearing down link.", RNS.LOG_DEBUG) if self.outbound_propagation_link != None: self.outbound_propagation_link.teardown() + self.propagation_transfer_state = LXMRouter.PR_NO_IDENTITY_RCVD + + elif request_receipt.response == LXMPeer.ERROR_NO_ACCESS: + RNS.log("Propagation node did not allow get request, tearing down link.", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + self.propagation_transfer_state = LXMRouter.PR_NO_ACCESS + else: if request_receipt.response != None and len(request_receipt.response) > 0: haves = [] @@ -751,6 +762,7 @@ class LXMRouter: self.propagation_transfer_state = LXMRouter.PR_COMPLETE self.propagation_transfer_progress = 1.0 self.propagation_transfer_last_result = len(request_receipt.response) + self.save_locally_delivered_transient_ids() def message_get_progress(self, request_receipt): self.propagation_transfer_state = LXMRouter.PR_RECEIVING From 564cd9b18c6ecbafaeb0f1c21026bcf1083fa7cb Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 22 Oct 2022 22:43:28 +0200 Subject: [PATCH 106/436] Updated build scripts --- .gitignore | 1 + Makefile | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 33feb9e..06e38e3 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ *.so # Distribution / packaging +LXMF/Utilities/LXMF RNS .Python build/ diff --git a/Makefile b/Makefile index 8d3e67a..c0b53da 100644 --- a/Makefile +++ b/Makefile @@ -8,10 +8,12 @@ clean: remove_symlinks: @echo Removing symlinks for build... -rm ./RNS + -rm ./LXMF/Utilities/LXMF create_symlinks: @echo Creating symlinks... -ln -s ../Reticulum/RNS ./ + -ln -s ../../LXMF ./LXMF/Utilities/LXMF build_wheel: python3 setup.py sdist bdist_wheel From ab68c15d2d3bea57402fb44878dae1d7cea4e987 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 3 Nov 2022 12:18:54 +0100 Subject: [PATCH 107/436] Updated version and dependencies --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index b5fdc75..d31c31e 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.2.2" +__version__ = "0.2.3" diff --git a/setup.py b/setup.py index b044de6..e0a255d 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.3.17'], + install_requires=['rns>=0.4.0'], python_requires='>=3.6', ) From 730a611f7a81f3a2e22f19d527827a245bca626c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 3 Nov 2022 23:17:50 +0100 Subject: [PATCH 108/436] Updated dependencies --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index d31c31e..788da1f 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.2.3" +__version__ = "0.2.4" diff --git a/setup.py b/setup.py index e0a255d..49dd05a 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.4.0'], + install_requires=['rns>=0.4.1'], python_requires='>=3.6', ) From 58c61a2557573e3ba959f089c212c92d1827b2cd Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 10 Nov 2022 16:20:14 +0100 Subject: [PATCH 109/436] Updated readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index de09fc0..80996d2 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,12 @@ Lightweight Extensible Message Format LXMF is a simple and flexible messaging format and delivery protocol that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](https://github.com/markqvist/reticulum) and offers zero-conf message routing, end-to-end encryption and Forward Secrecy by default. +User-facing clients that use LXMF include: + +- [Sideband](https://github.com/markqvist/sideband) +- [Nomad Network](https://github.com/markqvist/nomadnet) +- [Nexus Messenger](https://github.com/HarlekinSimplex/nexus_messenger) + ## Structure LXMF messages are stored in a simple and efficient format, that's easy to parse and write. From f7ee4c1d86e42c7f07f5c0dc4cf1e31f69323947 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 10 Nov 2022 16:21:33 +0100 Subject: [PATCH 110/436] Updated readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 80996d2..897d9a5 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Lightweight Extensible Message Format LXMF is a simple and flexible messaging format and delivery protocol that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](https://github.com/markqvist/reticulum) and offers zero-conf message routing, end-to-end encryption and Forward Secrecy by default. -User-facing clients that use LXMF include: +User-facing clients build on LXMF include: - [Sideband](https://github.com/markqvist/sideband) - [Nomad Network](https://github.com/markqvist/nomadnet) From f75ea842d9e1a35251f497c7619b9bf3233f17fb Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 10 Nov 2022 16:22:33 +0100 Subject: [PATCH 111/436] Updated readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 897d9a5..55b78ba 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Lightweight Extensible Message Format LXMF is a simple and flexible messaging format and delivery protocol that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](https://github.com/markqvist/reticulum) and offers zero-conf message routing, end-to-end encryption and Forward Secrecy by default. -User-facing clients build on LXMF include: +User-facing clients built on LXMF include: - [Sideband](https://github.com/markqvist/sideband) - [Nomad Network](https://github.com/markqvist/nomadnet) From dad21bcca759421adc167cce51da43b66be5cd27 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 17 Nov 2022 12:55:04 +0100 Subject: [PATCH 112/436] Added roadmap --- README.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 55b78ba..11a8f7f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -Lightweight Extensible Message Format -========== +# Lightweight Extensible Message Format LXMF is a simple and flexible messaging format and delivery protocol that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](https://github.com/markqvist/reticulum) and offers zero-conf message routing, end-to-end encryption and Forward Secrecy by default. @@ -13,7 +12,7 @@ User-facing clients built on LXMF include: LXMF messages are stored in a simple and efficient format, that's easy to parse and write. -##### The format follows this general structure: +### The format follows this general structure: - Destination - Source @@ -24,7 +23,7 @@ LXMF messages are stored in a simple and efficient format, that's easy to parse - Title - Fields -##### And these rules: +### And these rules: 1. A LXMF message is identified by its __message-id__, which is a SHA-256 hash of the __Destination__, __Source__ and __Payload__. The message-id is never included directly in the message, since it can always be inferred from the message itself. @@ -112,3 +111,16 @@ If you want to try out LXMF, you can install it with pip: ```bash pip3 install lxmf ``` + +## Development Roadmap + +LXMF is actively being developed, and the following improvements and features are currently planned for implementation: + +- Write and release full API and protocol documentation +- Update examples in readme to actually work +- Content Destinations, and easy to use API for group messaging and discussion threads +- Sneakernet and physical transport functionality +- LXMF paper- and QR-encoded messages +- Documenting and possibly expanding LXMF limits and priorities +- Sync affinity based on link speeds and distances, for more intelligently choosing what peers to sync next +- Markets on LXMF From bb7bbaf12448611941dd867c997c08ac04947bf1 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 17 Nov 2022 12:57:33 +0100 Subject: [PATCH 113/436] Updated readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 11a8f7f..006dc5f 100644 --- a/README.md +++ b/README.md @@ -122,5 +122,5 @@ LXMF is actively being developed, and the following improvements and features ar - Sneakernet and physical transport functionality - LXMF paper- and QR-encoded messages - Documenting and possibly expanding LXMF limits and priorities -- Sync affinity based on link speeds and distances, for more intelligently choosing what peers to sync next +- Sync affinity based on link speeds and distances, for more intelligently choosing peer sync order - Markets on LXMF From 6b983872fa99067f8f776e8fae637b40b241538f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 17 Nov 2022 13:16:22 +0100 Subject: [PATCH 114/436] Updated readme --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 006dc5f..a9fa98a 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # Lightweight Extensible Message Format -LXMF is a simple and flexible messaging format and delivery protocol that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](https://github.com/markqvist/reticulum) and offers zero-conf message routing, end-to-end encryption and Forward Secrecy by default. +LXMF is a simple and flexible messaging format and delivery protocol that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](https://reticulum.network) and offers zero-conf message routing, end-to-end encryption and Forward Secrecy by default. User-facing clients built on LXMF include: -- [Sideband](https://github.com/markqvist/sideband) -- [Nomad Network](https://github.com/markqvist/nomadnet) +- [Sideband](https://unsigned.io/sideband) +- [Nomad Network](https://unsigned.io/nomadnet) - [Nexus Messenger](https://github.com/HarlekinSimplex/nexus_messenger) ## Structure @@ -81,7 +81,7 @@ The LXM Router then handles the heavy lifting, such as message packing, encrypti ## Transport Encryption -LXMF uses encryption provided by [Reticulum](https://github.com/markqvist/reticulum), and thus uses end-to-end encryption by default. The delivery method of a message will influence which transport encryption scheme is used. +LXMF uses encryption provided by [Reticulum](https://reticulum.network), and thus uses end-to-end encryption by default. The delivery method of a message will influence which transport encryption scheme is used. - A message can be delivered opportunistically, embedded in a single Reticulum packet. In this cases the message will be opportunistically routed through the network, and will be encrypted with _ephemeral_ keys derived with _ECDH_ on _Curve25519_. This mode offers Perfect Forward Secrecy. @@ -93,8 +93,8 @@ LXMF uses encryption provided by [Reticulum](https://github.com/markqvist/reticu Assuming the default Reticulum configuration, the binary wire-format is as follows: -- 10 bytes destination hash -- 10 bytes source hash +- 16 bytes destination hash +- 16 bytes source hash - 64 bytes Ed25519 signature - Remaining bytes of [msgpack](https://msgpack.org) payload data, in accordance with the structure defined above From 3d31620160db104e5e02854097f025a46c9e62b2 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 19 Nov 2022 17:54:29 +0100 Subject: [PATCH 115/436] Added paper message generator --- LXMF/LXMessage.py | 94 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 6f27d0a..d5dbb12 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -2,6 +2,7 @@ import RNS import RNS.vendor.umsgpack as msgpack import time +import base64 from .LXMF import APP_NAME @@ -22,7 +23,8 @@ class LXMessage: OPPORTUNISTIC = 0x01 DIRECT = 0x02 PROPAGATED = 0x03 - valid_methods = [OPPORTUNISTIC, DIRECT, PROPAGATED] + PAPER = 0x05 + valid_methods = [OPPORTUNISTIC, DIRECT, PROPAGATED, PAPER] SOURCE_UNKNOWN = 0x01 SIGNATURE_INVALID = 0x02 @@ -76,6 +78,12 @@ class LXMessage: ENCRYPTION_DESCRIPTION_EC = "Curve25519" ENCRYPTION_DESCRIPTION_UNENCRYPTED = "Unencrypted" + # Constants for QR/URL encoding LXMs + URL_PROTO_SPECIFIER = "lxm" + QR_ERROR_CORRECTION = "ERROR_CORRECT_L" + QR_MAX_STORAGE = 2953 + PAPER_MDU = ((QR_MAX_STORAGE-(len(URL_PROTO_SPECIFIER)+len("://")))*6)//8 + def __str__(self): if self.hash != None: return "" @@ -196,6 +204,7 @@ class LXMessage: self.timestamp = time.time() self.propagation_packed = None + self.paper_packed = None self.payload = [self.timestamp, self.title, self.content, self.fields] @@ -263,6 +272,19 @@ class LXMessage: self.method = self.desired_method self.representation = LXMessage.RESOURCE + elif self.desired_method == LXMessage.PAPER: + paper_content_limit = LXMessage.PAPER_MDU + + encrypted_data = self.__destination.encrypt(self.packed[LXMessage.DESTINATION_LENGTH:]) + self.paper_packed = msgpack.packb(self.packed[:LXMessage.DESTINATION_LENGTH]+encrypted_data) + + content_size = len(self.paper_packed) + if content_size <= paper_content_limit: + self.method = self.desired_method + self.representation = LXMessage.PAPER + else: + raise TypeError("LXMessage desired paper delivery method, but content exceeds paper message maximum size.") + else: raise ValueError("Attempt to re-pack LXMessage "+str(self)+" that was already packed") @@ -327,6 +349,16 @@ class LXMessage: else: self.transport_encrypted = False self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED + elif self.method == LXMessage.PAPER: + if self.__destination.type == RNS.Destination.SINGLE: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC + elif self.__destination.type == RNS.Destination.GROUP: + self.transport_encrypted = True + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_AES + else: + self.transport_encrypted = False + self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED else: self.transport_encrypted = False self.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_UNENCRYPTED @@ -351,6 +383,16 @@ class LXMessage: except Exception as e: RNS.log("An error occurred in the external delivery callback for "+str(message), RNS.LOG_ERROR) + def __mark_paper_generated(self, receipt = None): + RNS.log("Paper message generation succeeded for "+str(self), RNS.LOG_DEBUG) + self.state = LXMessage.PAPER + + if self.__delivery_callback != None and callable(self.__delivery_callback): + try: + self.__delivery_callback(self) + except Exception as e: + RNS.log("An error occurred in the external delivery callback for "+str(message), RNS.LOG_ERROR) + def __resource_concluded(self, resource): if resource.status == RNS.Resource.COMPLETE: self.__mark_delivered() @@ -441,6 +483,56 @@ class LXMessage: RNS.log("Error while writing LXMF message to file \""+str(file_path)+"\". The contained exception was: "+str(e), RNS.LOG_ERROR) return None + def as_url(self, finalise=True): + self.determine_transport_encryption() + if not self.packed: + self.pack() + + if self.desired_method == LXMessage.PAPER and self.paper_packed != None: + # Encode packed LXM with URL-safe base64 and remove padding + encoded_bytes = base64.urlsafe_b64encode(self.paper_packed) + + # Add protocol specifier and return + lxm_url = LXMessage.URL_PROTO_SPECIFIER+"://"+encoded_bytes.decode("utf-8").replace("=","") + + # TODO: Remove + # RNS.log(str(len(lxm_url))+" byte LXM URL: "+str(lxm_url), RNS.LOG_EXTREME) + + if finalise: + self.__mark_paper_generated() + + return lxm_url + + else: + raise TypeError("Attempt to represent LXM with non-paper delivery method as URL") + + def as_qr(self): + self.determine_transport_encryption() + if not self.packed: + self.pack() + + if self.desired_method == LXMessage.PAPER and self.paper_packed != None: + import importlib + if importlib.util.find_spec('qrcode') != None: + import qrcode + + qr = qrcode.make( + error_correction = qrcode.constants.__dict__[LXMessage.QR_ERROR_CORRECTION], + border = 1, + data = self.as_url(finalise=False), + ) + + self.__mark_paper_generated() + + return qr + + else: + RNS.log("Generating QR-code representanions of LXMs requires the \"qrcode\" module to be installed.", RNS.LOG_CRITICAL) + RNS.log("You can install it with the command: python3 -m pip install qrcode", RNS.LOG_CRITICAL) + return None + + else: + raise TypeError("Attempt to represent LXM with non-paper delivery method as QR-code") @staticmethod def unpack_from_bytes(lxmf_bytes): From edbb887d81f5f7845475a96fd44a06673aa8addc Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 19 Nov 2022 20:07:00 +0100 Subject: [PATCH 116/436] Implemented paper message handling --- LXMF/LXMRouter.py | 82 +++++++++++++++++++++++++++++++++++++++++++---- LXMF/LXMessage.py | 2 +- LXMF/_version.py | 2 +- 3 files changed, 78 insertions(+), 8 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index e7b90af..9be992c 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1,6 +1,7 @@ import os import time import random +import base64 import atexit import threading @@ -85,6 +86,7 @@ class LXMRouter: self.propagation_transfer_last_result = None self.propagation_transfer_max_messages = None self.locally_delivered_transient_ids = {} + self.locally_processed_transient_ids = {} if identity == None: identity = RNS.Identity() @@ -105,7 +107,6 @@ class LXMRouter: self.peers = {} self.propagation_entries = {} - self.propagated_ids = {} RNS.Transport.register_announce_handler(LXMFDeliveryAnnounceHandler(self)) RNS.Transport.register_announce_handler(LXMFPropagationAnnounceHandler(self)) @@ -118,7 +119,14 @@ class LXMRouter: data = locally_delivered_file.read() self.locally_delivered_transient_ids = msgpack.unpackb(data) locally_delivered_file.close() - self.clean_transient_id_cache() + + if os.path.isfile(self.storagepath+"/locally_processed"): + locally_processed_file = open(self.storagepath+"/locally_processed", "rb") + data = locally_processed_file.read() + self.locally_processed_transient_ids = msgpack.unpackb(data) + locally_processed_file.close() + + self.clean_transient_id_caches() except Exception as e: RNS.log("Could not load locally delivered message ID cache from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) @@ -424,7 +432,7 @@ class LXMRouter: self.clean_links() if self.processing_count % LXMRouter.JOB_TRANSIENT_INTERVAL == 0: - self.clean_transient_id_cache() + self.clean_transient_id_caches() if self.processing_count % LXMRouter.JOB_STORE_INTERVAL == 0: self.clean_message_store() @@ -458,7 +466,7 @@ class LXMRouter: self.acknowledge_sync_completion() RNS.log("Cleaned outbound propagation link", RNS.LOG_DEBUG) - def clean_transient_id_cache(self): + def clean_transient_id_caches(self): now = time.time() removed_entries = [] for transient_id in self.locally_delivered_transient_ids: @@ -470,6 +478,16 @@ class LXMRouter: self.locally_delivered_transient_ids.pop(transient_id) RNS.log("Cleaned "+RNS.prettyhexrep(transient_id)+" from local delivery cache", RNS.LOG_DEBUG) + removed_entries = [] + for transient_id in self.locally_processed_transient_ids: + timestamp = self.locally_processed_transient_ids[transient_id] + if now > timestamp+LXMRouter.MESSAGE_EXPIRY*1.25: + removed_entries.append(transient_id) + + for transient_id in removed_entries: + self.locally_processed_transient_ids.pop(transient_id) + RNS.log("Cleaned "+RNS.prettyhexrep(transient_id)+" from locally processed cache", RNS.LOG_DEBUG) + def clean_message_store(self): # Check and remove expired messages now = time.time() @@ -571,6 +589,18 @@ class LXMRouter: except Exception as e: RNS.log("Could not save locally delivered message ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + def save_locally_processed_transient_ids(self): + try: + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) + + locally_processed_file = open(self.storagepath+"/locally_processed", "wb") + locally_processed_file.write(msgpack.packb(self.locally_processed_transient_ids)) + locally_processed_file.close() + + except Exception as e: + RNS.log("Could not save locally processed message ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + def exit_handler(self): if self.propagation_node: try: @@ -589,6 +619,7 @@ class LXMRouter: RNS.log("Could not save propagation node peers to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) self.save_locally_delivered_transient_ids() + self.save_locally_processed_transient_ids() def __str__(self): return "" @@ -1012,15 +1043,18 @@ class LXMRouter: except Exception as e: RNS.log("Error while unpacking received propagation resource", RNS.LOG_DEBUG) - def lxmf_propagation(self, lxmf_data): + def lxmf_propagation(self, lxmf_data, signal_local_delivery=None, signal_duplicate=None): try: if len(lxmf_data) >= LXMessage.LXMF_OVERHEAD: transient_id = RNS.Identity.full_hash(lxmf_data) - if not transient_id in self.propagation_entries and not transient_id in self.propagated_ids: + + if not transient_id in self.propagation_entries and not transient_id in self.locally_processed_transient_ids: received = time.time() propagation_entry = [transient_id, received, lxmf_data] destination_hash = lxmf_data[:LXMessage.DESTINATION_LENGTH] + self.locally_processed_transient_ids[transient_id] = received + if destination_hash in self.delivery_destinations: delivery_destination = self.delivery_destinations[destination_hash] encrypted_lxmf_data = lxmf_data[LXMessage.DESTINATION_LENGTH:] @@ -1029,6 +1063,9 @@ class LXMRouter: self.lxmf_delivery(delivery_data, delivery_destination.type) self.locally_delivered_transient_ids[transient_id] = time.time() + if signal_local_delivery != None: + return signal_local_delivery + else: if self.propagation_node: file_path = self.messagepath+"/"+RNS.hexrep(transient_id, delimit=False)+"_"+str(received) @@ -1043,8 +1080,19 @@ class LXMRouter: peer = self.peers[peer_id] peer.handle_message(transient_id) + else: + # TODO: Add message to sneakernet queues when implemented + RNS.log("Received propagated LXMF message "+RNS.prettyhexrep(transient_id)+", but this instance is not hosting a propagation node, discarding message.", RNS.LOG_DEBUG) + return True + else: + if signal_duplicate != None: + return signal_duplicate + + else: + return False + return False except Exception as e: @@ -1052,6 +1100,28 @@ class LXMRouter: RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) return False + def ingest_lxm_url(self, url, signal_local_delivery=None, signal_duplicate=None): + try: + if not url.lower().startswith("lxm://"): + RNS.log("Cannot ingest LXM, invalid URL provided.", RNS.LOG_ERROR) + return False + + else: + lxmf_data = base64.urlsafe_b64decode(url.replace(LXMessage.URL_PROTO_SPECIFIER+"://", "").replace("/", "")+"==") + transient_id = RNS.Identity.full_hash(lxmf_data) + + router_propagation_result = self.lxmf_propagation(lxmf_data, signal_local_delivery=signal_local_delivery, signal_duplicate=signal_duplicate) + if router_propagation_result != False: + RNS.log("LXM with transient ID "+RNS.prettyhexrep(transient_id)+" was ingested.", RNS.LOG_DEBUG) + return router_propagation_result + else: + RNS.log("No valid LXM could be ingested from the provided URL", RNS.LOG_DEBUG) + return False + + except Exception as e: + RNS.log("Error while decoding URL-encoded LXMF message. The contained exception was: "+str(e), RNS.LOG_ERROR) + return False + def fail_message(self, lxmessage): RNS.log(str(lxmessage)+" failed to send", RNS.LOG_DEBUG) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index d5dbb12..6e688a6 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -276,7 +276,7 @@ class LXMessage: paper_content_limit = LXMessage.PAPER_MDU encrypted_data = self.__destination.encrypt(self.packed[LXMessage.DESTINATION_LENGTH:]) - self.paper_packed = msgpack.packb(self.packed[:LXMessage.DESTINATION_LENGTH]+encrypted_data) + self.paper_packed = self.packed[:LXMessage.DESTINATION_LENGTH]+encrypted_data content_size = len(self.paper_packed) if content_size <= paper_content_limit: diff --git a/LXMF/_version.py b/LXMF/_version.py index 788da1f..fe404ae 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.2.4" +__version__ = "0.2.5" From abe2e782830945477e62708e3843d638aa58473b Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 22 Nov 2022 19:53:25 +0100 Subject: [PATCH 117/436] Improved paper message packing --- LXMF/LXMRouter.py | 12 ++++++------ LXMF/LXMessage.py | 27 ++++++++++++++------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 9be992c..d1322ca 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1100,14 +1100,14 @@ class LXMRouter: RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) return False - def ingest_lxm_url(self, url, signal_local_delivery=None, signal_duplicate=None): + def ingest_lxm_uri(self, uri, signal_local_delivery=None, signal_duplicate=None): try: - if not url.lower().startswith("lxm://"): - RNS.log("Cannot ingest LXM, invalid URL provided.", RNS.LOG_ERROR) + if not uri.lower().startswith(LXMessage.URI_SCHEMA+"://"): + RNS.log("Cannot ingest LXM, invalid URI provided.", RNS.LOG_ERROR) return False else: - lxmf_data = base64.urlsafe_b64decode(url.replace(LXMessage.URL_PROTO_SPECIFIER+"://", "").replace("/", "")+"==") + lxmf_data = base64.urlsafe_b64decode(uri.replace(LXMessage.URI_SCHEMA+"://", "").replace("/", "")+"==") transient_id = RNS.Identity.full_hash(lxmf_data) router_propagation_result = self.lxmf_propagation(lxmf_data, signal_local_delivery=signal_local_delivery, signal_duplicate=signal_duplicate) @@ -1115,11 +1115,11 @@ class LXMRouter: RNS.log("LXM with transient ID "+RNS.prettyhexrep(transient_id)+" was ingested.", RNS.LOG_DEBUG) return router_propagation_result else: - RNS.log("No valid LXM could be ingested from the provided URL", RNS.LOG_DEBUG) + RNS.log("No valid LXM could be ingested from the provided URI", RNS.LOG_DEBUG) return False except Exception as e: - RNS.log("Error while decoding URL-encoded LXMF message. The contained exception was: "+str(e), RNS.LOG_ERROR) + RNS.log("Error while decoding URI-encoded LXMF message. The contained exception was: "+str(e), RNS.LOG_ERROR) return False def fail_message(self, lxmessage): diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 6e688a6..3c1188c 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -78,11 +78,11 @@ class LXMessage: ENCRYPTION_DESCRIPTION_EC = "Curve25519" ENCRYPTION_DESCRIPTION_UNENCRYPTED = "Unencrypted" - # Constants for QR/URL encoding LXMs - URL_PROTO_SPECIFIER = "lxm" + # Constants for QR/URI encoding LXMs + URI_SCHEMA = "lxm" QR_ERROR_CORRECTION = "ERROR_CORRECT_L" QR_MAX_STORAGE = 2953 - PAPER_MDU = ((QR_MAX_STORAGE-(len(URL_PROTO_SPECIFIER)+len("://")))*6)//8 + PAPER_MDU = ((QR_MAX_STORAGE-(len(URI_SCHEMA)+len("://")))*6)//8 def __str__(self): if self.hash != None: @@ -123,6 +123,9 @@ class LXMessage: self.state = LXMessage.DRAFT self.method = LXMessage.UNKNOWN + self.propagation_packed = None + self.paper_packed = None + self.incoming = False self.signature_validated = False self.unverified_reason = None @@ -483,7 +486,7 @@ class LXMessage: RNS.log("Error while writing LXMF message to file \""+str(file_path)+"\". The contained exception was: "+str(e), RNS.LOG_ERROR) return None - def as_url(self, finalise=True): + def as_uri(self, finalise=True): self.determine_transport_encryption() if not self.packed: self.pack() @@ -493,18 +496,15 @@ class LXMessage: encoded_bytes = base64.urlsafe_b64encode(self.paper_packed) # Add protocol specifier and return - lxm_url = LXMessage.URL_PROTO_SPECIFIER+"://"+encoded_bytes.decode("utf-8").replace("=","") - - # TODO: Remove - # RNS.log(str(len(lxm_url))+" byte LXM URL: "+str(lxm_url), RNS.LOG_EXTREME) + lxm_uri = LXMessage.URI_SCHEMA+"://"+encoded_bytes.decode("utf-8").replace("=","") if finalise: self.__mark_paper_generated() - return lxm_url + return lxm_uri else: - raise TypeError("Attempt to represent LXM with non-paper delivery method as URL") + raise TypeError("Attempt to represent LXM with non-paper delivery method as URI") def as_qr(self): self.determine_transport_encryption() @@ -519,7 +519,7 @@ class LXMessage: qr = qrcode.make( error_correction = qrcode.constants.__dict__[LXMessage.QR_ERROR_CORRECTION], border = 1, - data = self.as_url(finalise=False), + data = self.as_uri(finalise=False), ) self.__mark_paper_generated() @@ -535,7 +535,7 @@ class LXMessage: raise TypeError("Attempt to represent LXM with non-paper delivery method as QR-code") @staticmethod - def unpack_from_bytes(lxmf_bytes): + def unpack_from_bytes(lxmf_bytes, original_method = None): destination_hash = lxmf_bytes[:LXMessage.DESTINATION_LENGTH] source_hash = lxmf_bytes[LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH] signature = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH] @@ -568,7 +568,8 @@ class LXMessage: title = "", fields = fields, destination_hash = destination_hash, - source_hash = source_hash) + source_hash = source_hash, + desired_method = original_method) message.hash = message_hash message.signature = signature From 54005c148a3b2f74d5435cc0ccad2fda7426a9f6 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 24 Nov 2022 17:59:50 +0100 Subject: [PATCH 118/436] Updated versions --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index fe404ae..01ef120 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.2.5" +__version__ = "0.2.6" diff --git a/setup.py b/setup.py index 49dd05a..2199715 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.4.1'], + install_requires=['rns>=0.4.2'], python_requires='>=3.6', ) From 9fa6a144d06ca39b9a0947408bc3db9e22d7d53d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 24 Nov 2022 18:04:51 +0100 Subject: [PATCH 119/436] Updated readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a9fa98a..57fa4b3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Lightweight Extensible Message Format -LXMF is a simple and flexible messaging format and delivery protocol that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](https://reticulum.network) and offers zero-conf message routing, end-to-end encryption and Forward Secrecy by default. +LXMF is a simple and flexible messaging format and delivery protocol that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](https://reticulum.network) and offers zero-conf message routing, end-to-end encryption and Forward Secrecy, and can be transported over any kind of medium that Reticulum supports, even extremely low-bandwidth systems such as packet radio or LoRa. Encrypted LXMF messages can also be encoded as QR-codes or text-based URIs, allowing completely analog *paper message* transport. User-facing clients built on LXMF include: From a1c897a8ca4fa27b376dfdd98050da05de2f169a Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 24 Nov 2022 18:05:58 +0100 Subject: [PATCH 120/436] Updated readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 57fa4b3..f5abd2f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # Lightweight Extensible Message Format -LXMF is a simple and flexible messaging format and delivery protocol that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](https://reticulum.network) and offers zero-conf message routing, end-to-end encryption and Forward Secrecy, and can be transported over any kind of medium that Reticulum supports, even extremely low-bandwidth systems such as packet radio or LoRa. Encrypted LXMF messages can also be encoded as QR-codes or text-based URIs, allowing completely analog *paper message* transport. +LXMF is a simple and flexible messaging format and delivery protocol that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](https://reticulum.network) and offers zero-conf message routing, end-to-end encryption and Forward Secrecy, and can be transported over any kind of medium that Reticulum supports. + +LXMF is efficient enough that it can deliver messages over extremely low-bandwidth systems such as packet radio or LoRa. Encrypted LXMF messages can also be encoded as QR-codes or text-based URIs, allowing completely analog *paper message* transport. User-facing clients built on LXMF include: From 8f425e6dde7301696a4e2d86d7b438187b20f1af Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 24 Nov 2022 18:17:21 +0100 Subject: [PATCH 121/436] Added paper message info --- README.md | 12 ++++++++++++ docs/paper_msg_test.png | Bin 0 -> 2675 bytes 2 files changed, 12 insertions(+) create mode 100644 docs/paper_msg_test.png diff --git a/README.md b/README.md index f5abd2f..8cfcff0 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,18 @@ Assuming the default Reticulum configuration, the binary wire-format is as follo The complete message overhead for LXMF is only 99 bytes, which in return gives you timestamped, digitally signed, infinitely extensible, end-to-end encrypted, zero-conf routed, minimal-infrastructure messaging that's easy to use and build applications with. +## Example Paper Message + +You can try out the paper messaging functionality by using the following QR-code. It is a paper message sent to the LXMF address `6b3362bd2c1dbf87b66a85f79a8d8c75`. To be able to decrypt and read the message, you will need to import the following Reticulum Identity to an LXMF messaging app: + +`3BPTDTQCRZPKJT3TXAJCMQFMOYWIM3OCLKPWMG4HCF2T4CH3YZHVNHNRDU6QAZWV2KBHMWBNT2C62TQEVC5GLFM4MN25VLZFSK3ADRQ=` + +The [Sideband](https://unsigned.io/sideband) application allows you to do this easily. After the you have imported the identity into an app of your choice, you can scan the following QR-code and open it in the app, where it will be decrypted and added as a message. + +![Paper Message QR-code](./docs/paper_msg_test.png) + +You can also find the entire message [embedded in this link](lxm://azNivSwdv4e2aoX3mo2MdTAozuI7BlzrLlHULmnVgpz3dNT9CMPVwgywzCJP8FVogj5j_kU7j7ywuvBNcr45kRTrd19c3iHenmnSDe4VEd6FuGsAiT0Khzl7T81YZHPTDhRNp0FdhDE9AJ7uphw7zKMyqhHHxOxqrYeBeKF66gpPxDceqjsOApvsSwggjcuHBx9OxOBy05XmnJxA1unCKgvNfOFYc1T47luxoY3c0dLOJnJPwZuFRytx2TXlQNZzOJ28yTEygIfkDqEO9mZi5lgev7XZJ0DvgioQxMIyoCm7lBUzfq66zW3SQj6vHHph7bhr36dLOCFgk4fZA6yia2MlTT9KV66Tn2l8mPNDlvuSAJhwDA_xx2PN9zKadCjo9sItkAp8r-Ss1CzoUWZUAyT1oDw7ly6RrzGBG-e3eM3CL6u1juIeFiHby7_3cON-6VTUuk4xR5nwKlFTu5vsYMVXe5H3VahiDSS4Q1aqX7I). On operating systems that allow for registering custom URI-handlers, you can click the link, and it will be decoded directly in your LXMF client. This works with Sideband on Android. + ## Caveat Emptor LXMF is beta software, and should be considered experimental. While it has been built with cryptography best-practices very foremost in mind, it _has not_ been externally security audited, and there could very well be privacy-breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch. diff --git a/docs/paper_msg_test.png b/docs/paper_msg_test.png new file mode 100644 index 0000000000000000000000000000000000000000..ff901c67c86deab42b6ad81fe2314863c20556e7 GIT binary patch literal 2675 zcmXw5c|4T+8yyC@jO}KZvp@SK*-$8#2Nqq zN^%}9_6vYXmj=5aaKYT~XFR!+)OLnwIHtlaw#A2j9*{Z9%{;^&CFWbu2 zkK;Y7SaFZS_59jtW$*S1h8{Jy%J=)HW0Ou1Xvvz`EZ5`Znys$Bi{c8dtn6SV+o&DL zi{h{(ig9D#&D?ZpPi%S+E_%9ES)9(|K>$~`JlcSOc}D4336%w{$FdneLe!I*@A78l zI(#z)(?dF{mcJi}8iT-5NdJup6xB5exL;gw88&sQ#gA{#dM{C8nBf^8#6!^9U0q?EURyI~N#yc}%HR~C@ zH!Jx^TI8yzU7}uLd~pODyT*lf*LD74EwRk&DRR>h7yncqyLm<_R@&7420cY>0v?%* zZ!(=|K7n|=!zi9E`ntEJba2>Xcx90+-BG}F&sU>x`SRwz`kkfv3?uG9!F@FjPNd#= z+fC63x7dXSN^`1*L9}2^eANE#CW6XzkR#nYDqF z|DIiM<0{srVZ|HA-&siJKFzxO(Rq9`&B14cesuO~cw^&y+QwVAM(#Q=J{8QvcnGy< zRFBq-mF{*wdF26wmEg5FPe3=epI^;j?UnB$yEr39^GOu}PwHdUXrA**VP&KF1K8!K z`w`Jx|KVy?MG`aU`-@c)?a>E4a}l1M?Uup2W~GNr_xU|m+UNF=w*6)s+BE652Q@MN z$SU-42*cl zuLqj8YjKg+zi!m$ePMDgEL@-L#@psSxA?tdiNQENw-}W4GZ4{+8;sGM176&XI=Ez5 z{(KNCAeEV~y1jU)#WNV(y>nxuKt@Rxmv>^M%6$f@HcD}N#rN2 z+pd#oGpb+2<<-w{2xyi$CzANmaa#;X<=Qf#TxOv=iUzTXP9IJ&4++YVf%X^>YVk@% z4t43;eMmc~vOu^S#- zFnev26_N)ert7{4+9c{6fwmOu{BnuEyP3P;+~^hkX9(6<3qelfoBeU)WF>6^q9z|o zN=~9si8A4RCQh2r>jUz7t-GAI^Tg2Nao@70Mzl#mKhz0kvcubh-!}d4JUi?fM2nb& z4I|E-EQFsu{TWa=&Uj#2Up)9zpV%PnxzsPMT&vjK`azZ^Ty$H(cN!4W>xVD4J{8H< zk*Ry&6RaOO!UR+b+b>ue#G=qUmfMHY4-T1PRA*=b<_1e*eq63_P`lA0U$9|Db*s~( zOI(eV0F-;$xfMKL>)5Tbul!;0uN~B9$_@o@;-2sSnBhCia$Sq7nd_5^S(}q9imZpI z9<1n)o-~m-{)p7U``K@K>6Ho*4BP&IUQ69rplUn9LmN2_(|sueFOg2~B!Mb;w6h`w z&mE4O&)?_J%k8yb54oo+R9P5*f(a{+=MqmtZ(gF!2jrv9b8z~I*!0oX2FS0NW!&^yzbzH(gqQczPU$?o)On0@y`!>X>Yz8+EVh>y`U#xC|klq)w!YKk}g*-;GsN2DG5A)>dBE)TZE2G0Vvn z)ai`VA3}4_>q)8BdbC3Z9+g`HYz-54IU0J#Ty@ zP7I&}?9E7QyYnG9??aDulrSH8xbJ*nSk(vo0*lK3tpW7a{IJZYVO)DHt9w}@`9A5E z0nN{2rxc#;#+e2U>8U0q>K7&V$&e86v@lw|^+T5|Y|zD@oNN_^_rH+#F>osLicgdg zc*`&`i0KHsGh{==a^fo#0m@M`8cmCZvQ9cb@M+3@%nSuIc#NW*j)Dc0qOQAm%hfJZ zn??bZjs^u7chc{41AnKSKFm+Ah3mhpLxbxWL!M_ai^$1T%TnKvslu{twtJlC@#4ol zPHAy{h7DQdPbd!(6sf%@7pA})ePqy}pX=WnwcFx~9D0-P#r1lVjTqg%u*1OVSYYwv zZkuSb@{rFG{g$RsmOt00ii-~K4j)EOz2F!PHkKe>UA(&`96!wBC)ookd0f}oxiNDx z`BZf7C|Ea&mrg??hO51WdsaVdbN?E%#@2oih=8|5IH3A@wcZaxZ1sb21c~oztCWr- z>4SR>{?0V-&T$^B4kd&S+bYu`c3K<@#%SMN$WhY*tWWC$MydfY)C6_U5A|jLAvZmi z`_=f~%t|&_IeY)P8Po9kEOXZ$3>5(VZC|+)t*m}fco28EzInyhA^U}_1g~0sz|P7u x7ibClNir2fEw&s%;KpHj>oEE9qlI9ml~w&da>!RWfU`}SU$HbP`@{XOe*o Date: Thu, 24 Nov 2022 18:19:01 +0100 Subject: [PATCH 122/436] Added paper message info --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8cfcff0..ddff8b6 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ You can try out the paper messaging functionality by using the following QR-code The [Sideband](https://unsigned.io/sideband) application allows you to do this easily. After the you have imported the identity into an app of your choice, you can scan the following QR-code and open it in the app, where it will be decrypted and added as a message. -![Paper Message QR-code](./docs/paper_msg_test.png) +
    You can also find the entire message [embedded in this link](lxm://azNivSwdv4e2aoX3mo2MdTAozuI7BlzrLlHULmnVgpz3dNT9CMPVwgywzCJP8FVogj5j_kU7j7ywuvBNcr45kRTrd19c3iHenmnSDe4VEd6FuGsAiT0Khzl7T81YZHPTDhRNp0FdhDE9AJ7uphw7zKMyqhHHxOxqrYeBeKF66gpPxDceqjsOApvsSwggjcuHBx9OxOBy05XmnJxA1unCKgvNfOFYc1T47luxoY3c0dLOJnJPwZuFRytx2TXlQNZzOJ28yTEygIfkDqEO9mZi5lgev7XZJ0DvgioQxMIyoCm7lBUzfq66zW3SQj6vHHph7bhr36dLOCFgk4fZA6yia2MlTT9KV66Tn2l8mPNDlvuSAJhwDA_xx2PN9zKadCjo9sItkAp8r-Ss1CzoUWZUAyT1oDw7ly6RrzGBG-e3eM3CL6u1juIeFiHby7_3cON-6VTUuk4xR5nwKlFTu5vsYMVXe5H3VahiDSS4Q1aqX7I). On operating systems that allow for registering custom URI-handlers, you can click the link, and it will be decoded directly in your LXMF client. This works with Sideband on Android. From 7ee7c60e77263a4ec266c4a5c7c701ee9c7f46f6 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 24 Nov 2022 18:19:58 +0100 Subject: [PATCH 123/436] Added paper message info --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ddff8b6..7f55ac8 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ You can try out the paper messaging functionality by using the following QR-code The [Sideband](https://unsigned.io/sideband) application allows you to do this easily. After the you have imported the identity into an app of your choice, you can scan the following QR-code and open it in the app, where it will be decrypted and added as a message. -
    +

    You can also find the entire message [embedded in this link](lxm://azNivSwdv4e2aoX3mo2MdTAozuI7BlzrLlHULmnVgpz3dNT9CMPVwgywzCJP8FVogj5j_kU7j7ywuvBNcr45kRTrd19c3iHenmnSDe4VEd6FuGsAiT0Khzl7T81YZHPTDhRNp0FdhDE9AJ7uphw7zKMyqhHHxOxqrYeBeKF66gpPxDceqjsOApvsSwggjcuHBx9OxOBy05XmnJxA1unCKgvNfOFYc1T47luxoY3c0dLOJnJPwZuFRytx2TXlQNZzOJ28yTEygIfkDqEO9mZi5lgev7XZJ0DvgioQxMIyoCm7lBUzfq66zW3SQj6vHHph7bhr36dLOCFgk4fZA6yia2MlTT9KV66Tn2l8mPNDlvuSAJhwDA_xx2PN9zKadCjo9sItkAp8r-Ss1CzoUWZUAyT1oDw7ly6RrzGBG-e3eM3CL6u1juIeFiHby7_3cON-6VTUuk4xR5nwKlFTu5vsYMVXe5H3VahiDSS4Q1aqX7I). On operating systems that allow for registering custom URI-handlers, you can click the link, and it will be decoded directly in your LXMF client. This works with Sideband on Android. From aa3886720f839c0e4b1f320f4b233f3f976dfb82 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 24 Nov 2022 18:21:12 +0100 Subject: [PATCH 124/436] Added paper message info --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7f55ac8..3840de9 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ The [Sideband](https://unsigned.io/sideband) application allows you to do this e

    -You can also find the entire message [embedded in this link](lxm://azNivSwdv4e2aoX3mo2MdTAozuI7BlzrLlHULmnVgpz3dNT9CMPVwgywzCJP8FVogj5j_kU7j7ywuvBNcr45kRTrd19c3iHenmnSDe4VEd6FuGsAiT0Khzl7T81YZHPTDhRNp0FdhDE9AJ7uphw7zKMyqhHHxOxqrYeBeKF66gpPxDceqjsOApvsSwggjcuHBx9OxOBy05XmnJxA1unCKgvNfOFYc1T47luxoY3c0dLOJnJPwZuFRytx2TXlQNZzOJ28yTEygIfkDqEO9mZi5lgev7XZJ0DvgioQxMIyoCm7lBUzfq66zW3SQj6vHHph7bhr36dLOCFgk4fZA6yia2MlTT9KV66Tn2l8mPNDlvuSAJhwDA_xx2PN9zKadCjo9sItkAp8r-Ss1CzoUWZUAyT1oDw7ly6RrzGBG-e3eM3CL6u1juIeFiHby7_3cON-6VTUuk4xR5nwKlFTu5vsYMVXe5H3VahiDSS4Q1aqX7I). On operating systems that allow for registering custom URI-handlers, you can click the link, and it will be decoded directly in your LXMF client. This works with Sideband on Android. +You can also find the entire message [in this link](lxm://azNivSwdv4e2aoX3mo2MdTAozuI7BlzrLlHULmnVgpz3dNT9CMPVwgywzCJP8FVogj5j_kU7j7ywuvBNcr45kRTrd19c3iHenmnSDe4VEd6FuGsAiT0Khzl7T81YZHPTDhRNp0FdhDE9AJ7uphw7zKMyqhHHxOxqrYeBeKF66gpPxDceqjsOApvsSwggjcuHBx9OxOBy05XmnJxA1unCKgvNfOFYc1T47luxoY3c0dLOJnJPwZuFRytx2TXlQNZzOJ28yTEygIfkDqEO9mZi5lgev7XZJ0DvgioQxMIyoCm7lBUzfq66zW3SQj6vHHph7bhr36dLOCFgk4fZA6yia2MlTT9KV66Tn2l8mPNDlvuSAJhwDA_xx2PN9zKadCjo9sItkAp8r-Ss1CzoUWZUAyT1oDw7ly6RrzGBG-e3eM3CL6u1juIeFiHby7_3cON-6VTUuk4xR5nwKlFTu5vsYMVXe5H3VahiDSS4Q1aqX7I). On operating systems that allow for registering custom URI-handlers, you can click the link, and it will be decoded directly in your LXMF client. This works with Sideband on Android. ## Caveat Emptor From 683e9ee41225e86de6f9af46fc3b3ee769ca58bb Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 24 Nov 2022 18:22:51 +0100 Subject: [PATCH 125/436] Added paper message info --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3840de9..779c01d 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,11 @@ The [Sideband](https://unsigned.io/sideband) application allows you to do this e

    -You can also find the entire message [in this link](lxm://azNivSwdv4e2aoX3mo2MdTAozuI7BlzrLlHULmnVgpz3dNT9CMPVwgywzCJP8FVogj5j_kU7j7ywuvBNcr45kRTrd19c3iHenmnSDe4VEd6FuGsAiT0Khzl7T81YZHPTDhRNp0FdhDE9AJ7uphw7zKMyqhHHxOxqrYeBeKF66gpPxDceqjsOApvsSwggjcuHBx9OxOBy05XmnJxA1unCKgvNfOFYc1T47luxoY3c0dLOJnJPwZuFRytx2TXlQNZzOJ28yTEygIfkDqEO9mZi5lgev7XZJ0DvgioQxMIyoCm7lBUzfq66zW3SQj6vHHph7bhr36dLOCFgk4fZA6yia2MlTT9KV66Tn2l8mPNDlvuSAJhwDA_xx2PN9zKadCjo9sItkAp8r-Ss1CzoUWZUAyT1oDw7ly6RrzGBG-e3eM3CL6u1juIeFiHby7_3cON-6VTUuk4xR5nwKlFTu5vsYMVXe5H3VahiDSS4Q1aqX7I). On operating systems that allow for registering custom URI-handlers, you can click the link, and it will be decoded directly in your LXMF client. This works with Sideband on Android. +You can also find the entire message in this link: + +`lxm://azNivSwdv4e2aoX3mo2MdTAozuI7BlzrLlHULmnVgpz3dNT9CMPVwgywzCJP8FVogj5j_kU7j7ywuvBNcr45kRTrd19c3iHenmnSDe4VEd6FuGsAiT0Khzl7T81YZHPTDhRNp0FdhDE9AJ7uphw7zKMyqhHHxOxqrYeBeKF66gpPxDceqjsOApvsSwggjcuHBx9OxOBy05XmnJxA1unCKgvNfOFYc1T47luxoY3c0dLOJnJPwZuFRytx2TXlQNZzOJ28yTEygIfkDqEO9mZi5lgev7XZJ0DvgioQxMIyoCm7lBUzfq66zW3SQj6vHHph7bhr36dLOCFgk4fZA6yia2MlTT9KV66Tn2l8mPNDlvuSAJhwDA_xx2PN9zKadCjo9sItkAp8r-Ss1CzoUWZUAyT1oDw7ly6RrzGBG-e3eM3CL6u1juIeFiHby7_3cON-6VTUuk4xR5nwKlFTu5vsYMVXe5H3VahiDSS4Q1aqX7I` + +On operating systems that allow for registering custom URI-handlers, you can click the link, and it will be decoded directly in your LXMF client. This works with Sideband on Android. ## Caveat Emptor From b10755e34fe8db83548f913dc603f6a8ab188370 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 24 Nov 2022 18:34:44 +0100 Subject: [PATCH 126/436] Updated readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 779c01d..b7310d8 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,6 @@ LXMF is actively being developed, and the following improvements and features ar - Update examples in readme to actually work - Content Destinations, and easy to use API for group messaging and discussion threads - Sneakernet and physical transport functionality -- LXMF paper- and QR-encoded messages - Documenting and possibly expanding LXMF limits and priorities - Sync affinity based on link speeds and distances, for more intelligently choosing peer sync order - Markets on LXMF From 1440a0b162dac20eb8cbe17543ab82bed8792a73 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 20 Dec 2022 23:58:09 +0100 Subject: [PATCH 127/436] Added sync backoff for unresponsive peers. Improved sync peer selection. --- LXMF/LXMPeer.py | 105 +++++++++++++++++++++++++++++----------------- LXMF/LXMRouter.py | 27 ++++++++++-- LXMF/_version.py | 2 +- 3 files changed, 91 insertions(+), 43 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 7cc5c20..585b2a3 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -24,6 +24,17 @@ class LXMPeer: # be unreachable before it is removed MAX_UNREACHABLE = 4*24*60*60 + # Everytime consecutive time a sync + # link fails to establish, add this + # amount off time to wait before the + # next sync is attempted. + SYNC_BACKOFF_STEP = 12*60 + + # How long to wait for an answer to + # peer path requests before deferring + # sync to later. + PATH_REQUEST_GRACE = 7.5 + @staticmethod def from_bytes(peer_bytes, router): dictionary = msgpack.unpackb(peer_bytes) @@ -66,6 +77,9 @@ class LXMPeer: def __init__(self, router, destination_hash): self.alive = False self.last_heard = 0 + self.next_sync_attempt = 0 + self.last_sync_attempt = 0 + self.sync_backoff = 0 self.peering_timebase = 0 self.link = None @@ -80,48 +94,61 @@ class LXMPeer: self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") def sync(self): + RNS.log("Initiating LXMF Propagation Node sync with peer "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_DEBUG) + self.last_sync_attempt = time.time() - if not RNS.Transport.has_path(self.destination_hash): - RNS.log("No path to peer "+RNS.prettyhexrep(self.destination_hash)+" exists, requesting...", RNS.LOG_DEBUG) - RNS.Transport.request_path(self.destination_hash) - RNS.log("Path requested, retrying sync later", RNS.LOG_DEBUG) - - else: - if self.identity == None: - self.identity = RNS.Identity.recall(destination_hash) - self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + if time.time() > self.next_sync_attempt: + if not RNS.Transport.has_path(self.destination_hash): + RNS.log("No path to peer "+RNS.prettyhexrep(self.destination_hash)+" exists, requesting...", RNS.LOG_DEBUG) + RNS.Transport.request_path(self.destination_hash) + time.sleep(LXMPeer.PATH_REQUEST_GRACE) - if self.identity != None: - if len(self.unhandled_messages) > 0: - if self.state == LXMPeer.IDLE: - RNS.log("Establishing link for sync to peer "+RNS.prettyhexrep(self.destination_hash)+"...", RNS.LOG_DEBUG) - self.link = RNS.Link(self.destination, established_callback = self.link_established, closed_callback = self.link_closed) - self.state = LXMPeer.LINK_ESTABLISHING - - else: - if self.state == LXMPeer.LINK_READY: - self.alive = True - self.last_heard = time.time() - - RNS.log("Synchronisation link to peer "+RNS.prettyhexrep(self.destination_hash)+" established, preparing request...", RNS.LOG_DEBUG) - unhandled_ids = [] - purged_ids = [] - for transient_id in self.unhandled_messages: - if transient_id in self.router.propagation_entries: - unhandled_ids.append(transient_id) - else: - purged_ids.append(transient_id) - - for transient_id in purged_ids: - RNS.log("Dropping unhandled message "+RNS.prettyhexrep(transient_id)+" for peer "+RNS.prettyhexrep(self.destination_hash)+" since it no longer exists in the message store.", RNS.LOG_DEBUG) - self.unhandled_messages.pop(transient_id) - - RNS.log("Sending sync request to peer "+str(self.destination), RNS.LOG_DEBUG) - self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) - self.state = LXMPeer.REQUEST_SENT + if not RNS.Transport.has_path(self.destination_hash): + RNS.log("Path request was not answered, retrying sync with peer "+RNS.prettyhexrep(self.destination_hash)+" later", RNS.LOG_DEBUG) + else: - RNS.log("Could not request sync to peer "+RNS.prettyhexrep(self.destination_hash)+" since its identity could not be recalled.", RNS.LOG_ERROR) + if self.identity == None: + self.identity = RNS.Identity.recall(destination_hash) + self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + + if self.identity != None: + if len(self.unhandled_messages) > 0: + if self.state == LXMPeer.IDLE: + RNS.log("Establishing link for sync to peer "+RNS.prettyhexrep(self.destination_hash)+"...", RNS.LOG_DEBUG) + self.sync_backoff += LXMPeer.SYNC_BACKOFF_STEP + self.next_sync_attempt = time.time() + self.sync_backoff + self.link = RNS.Link(self.destination, established_callback = self.link_established, closed_callback = self.link_closed) + self.state = LXMPeer.LINK_ESTABLISHING + + else: + if self.state == LXMPeer.LINK_READY: + self.alive = True + self.last_heard = time.time() + self.sync_backoff = 0 + + RNS.log("Synchronisation link to peer "+RNS.prettyhexrep(self.destination_hash)+" established, preparing request...", RNS.LOG_DEBUG) + unhandled_ids = [] + purged_ids = [] + for transient_id in self.unhandled_messages: + if transient_id in self.router.propagation_entries: + unhandled_ids.append(transient_id) + else: + purged_ids.append(transient_id) + + for transient_id in purged_ids: + RNS.log("Dropping unhandled message "+RNS.prettyhexrep(transient_id)+" for peer "+RNS.prettyhexrep(self.destination_hash)+" since it no longer exists in the message store.", RNS.LOG_DEBUG) + self.unhandled_messages.pop(transient_id) + + RNS.log("Sending sync request to peer "+str(self.destination), RNS.LOG_DEBUG) + self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) + self.state = LXMPeer.REQUEST_SENT + else: + RNS.log("Could not request sync to peer "+RNS.prettyhexrep(self.destination_hash)+" since its identity could not be recalled.", RNS.LOG_ERROR) + else: + RNS.log("Postponing sync with peer "+RNS.prettyhexrep(self.destination_hash)+" for "+RNS.prettytime(self.next_sync_attempt-time.time())+" due to previous failures", RNS.LOG_DEBUG) + if self.last_sync_attempt > self.last_heard: + self.alive = False def request_failed(self, request_receipt): RNS.log("Sync request to peer "+str(self.destination)+" failed", RNS.LOG_DEBUG) @@ -191,6 +218,7 @@ class LXMPeer: resource.transferred_messages = wanted_message_ids self.state = LXMPeer.RESOURCE_TRANSFERRING else: + RNS.log("Peer "+RNS.prettyhexrep(self.destination_hash)+" did not request any of the available messages, sync completed", RNS.LOG_DEBUG) self.state = LXMPeer.IDLE except Exception as e: @@ -226,6 +254,7 @@ class LXMPeer: def link_established(self, link): self.link.identify(self.router.identity) self.state = LXMPeer.LINK_READY + self.next_sync_attempt = 0 self.sync() def link_closed(self, link): diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index d1322ca..19df21d 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -910,7 +910,11 @@ class LXMRouter: def peer(self, destination_hash, timestamp): if destination_hash in self.peers: peer = self.peers[destination_hash] - peer.alive = True + if timestamp > peer.peering_timebase: + peer.alive = True + peer.sync_backoff = 0 + peer.next_sync_attempt = 0 + peer.peering_timebase = timestamp peer.last_heard = time.time() else: @@ -934,18 +938,33 @@ class LXMRouter: def sync_peers(self): culled_peers = [] waiting_peers = [] + unresponsive_peers = [] for peer_id in self.peers: peer = self.peers[peer_id] if time.time() > peer.last_heard + LXMPeer.MAX_UNREACHABLE: culled_peers.append(peer_id) else: if peer.state == LXMPeer.IDLE and len(peer.unhandled_messages) > 0: - waiting_peers.append(peer) + if peer.alive: + waiting_peers.append(peer) + else: + if hasattr(peer, "next_sync_attempt") and time.time() > peer.next_sync_attempt: + unresponsive_peers.append(peer) + else: + pass + # RNS.log("Not adding peer "+str(peer)+" since it is in sync backoff", RNS.LOG_DEBUG) + peer_pool = [] if len(waiting_peers) > 0: RNS.log("Randomly selecting peer to sync from "+str(len(waiting_peers))+" waiting peers.", RNS.LOG_DEBUG) - selected_index = random.randint(0,len(waiting_peers)-1) - selected_peer = waiting_peers[selected_index] + peer_pool = waiting_peers + elif len(unresponsive_peers) > 0: + RNS.log("No active peers available, randomly selecting peer to sync from "+str(len(unresponsive_peers))+" unresponsive peers.", RNS.LOG_DEBUG) + peer_pool = unresponsive_peers + + if len(peer_pool) > 0: + selected_index = random.randint(0,len(peer_pool)-1) + selected_peer = peer_pool[selected_index] RNS.log("Selected waiting peer "+str(selected_index)+": "+RNS.prettyhexrep(selected_peer.destination.hash), RNS.LOG_DEBUG) selected_peer.sync() diff --git a/LXMF/_version.py b/LXMF/_version.py index 01ef120..6cd38b7 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.2.6" +__version__ = "0.2.7" From ffb1bdff178f3957d1a4e9ad92ef23272fd6f8b5 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 22 Dec 2022 21:23:29 +0100 Subject: [PATCH 128/436] Updated processing timings --- LXMF/LXMRouter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 19df21d..73c15b6 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -17,8 +17,8 @@ from .Handlers import LXMFPropagationAnnounceHandler class LXMRouter: MAX_DELIVERY_ATTEMPTS = 4 - PROCESSING_INTERVAL = 5 - DELIVERY_RETRY_WAIT = 12 + PROCESSING_INTERVAL = 4 + DELIVERY_RETRY_WAIT = 9 PATH_REQUEST_WAIT = 5 LINK_MAX_INACTIVITY = 10*60 From ae6b5d9a6daf97a78dfd20e0118415d8f76556f8 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 23 Dec 2022 23:49:01 +0100 Subject: [PATCH 129/436] Updated delivery timing --- LXMF/LXMRouter.py | 4 ++-- LXMF/_version.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 73c15b6..d571eab 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -16,9 +16,9 @@ from .Handlers import LXMFDeliveryAnnounceHandler from .Handlers import LXMFPropagationAnnounceHandler class LXMRouter: - MAX_DELIVERY_ATTEMPTS = 4 + MAX_DELIVERY_ATTEMPTS = 5 PROCESSING_INTERVAL = 4 - DELIVERY_RETRY_WAIT = 9 + DELIVERY_RETRY_WAIT = 7 PATH_REQUEST_WAIT = 5 LINK_MAX_INACTIVITY = 10*60 diff --git a/LXMF/_version.py b/LXMF/_version.py index 6cd38b7..c49a95c 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.2.7" +__version__ = "0.2.8" From 390cac84e3863627542ed3c723ad5d9d74b5f305 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 23 Dec 2022 23:49:20 +0100 Subject: [PATCH 130/436] Updated RNS dependency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2199715..4e53ada 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.4.2'], + install_requires=['rns>=0.4.6'], python_requires='>=3.6', ) From 8c94118e0739735e5d27f3e012819687fe8ab926 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 14 Jan 2023 21:10:54 +0100 Subject: [PATCH 131/436] Updated dependencies and version --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index c49a95c..75cf783 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.2.8" +__version__ = "0.2.9" diff --git a/setup.py b/setup.py index 4e53ada..39ed781 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.4.6'], + install_requires=['rns>=0.4.7'], python_requires='>=3.6', ) From 5b81ecce04b678fff4827531791713bfcfab2418 Mon Sep 17 00:00:00 2001 From: Dionysis Grigoropoulos Date: Thu, 26 Jan 2023 02:56:20 +0200 Subject: [PATCH 132/436] Update documentation to match reality --- LXMF/LXMessage.py | 12 ++++++------ README.md | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 3c1188c..6ab264f 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -33,9 +33,9 @@ class LXMessage: DESTINATION_LENGTH = RNS.Identity.TRUNCATED_HASHLENGTH//8 SIGNATURE_LENGTH = RNS.Identity.SIGLENGTH//8 - # LXMF overhead is 99 bytes per message: - # 10 bytes for destination hash - # 10 bytes for source hash + # LXMF overhead is 111 bytes per message: + # 16 bytes for destination hash + # 16 bytes for source hash # 64 bytes for Ed25519 signature # 8 bytes for timestamp # 7 bytes for msgpack structure @@ -53,7 +53,7 @@ class LXMessage: # field of the packet, therefore we also add the length # of a destination hash to the calculation. With default # RNS and LXMF parameters, the largest single-packet - # LXMF message we can send is 294 bytes. If a message + # LXMF message we can send is 288 bytes. If a message # is larger than that, a Reticulum link will be used. ENCRYPTED_PACKET_MAX_CONTENT = ENCRYPTED_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH @@ -63,13 +63,13 @@ class LXMessage: LINK_PACKET_MDU = RNS.Link.MDU # Which means that we can deliver single-packet LXMF - # messages with content of up to 332 bytes over a link. + # messages with content of up to 320 bytes over a link. # If a message is larger than that, LXMF will sequence # and transfer it as a RNS resource over the link instead. LINK_PACKET_MAX_CONTENT = LINK_PACKET_MDU - LXMF_OVERHEAD # For plain packets without encryption, we can - # fit up to 388 bytes of content. + # fit up to 369 bytes of content. PLAIN_PACKET_MDU = RNS.Packet.PLAIN_MDU PLAIN_PACKET_MAX_CONTENT = PLAIN_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH diff --git a/README.md b/README.md index b7310d8..ca0ba1d 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ LXMF messages are stored in a simple and efficient format, that's easy to parse In some cases the actual message-id cannot be inferred, for example when a Propagation Node is storing an encrypted message for an offline user. In theses cases a _transient-id_ is used to identify the message while in storage or transit. 2. __Destination__, __Source__, __Signature__ and __Payload__ parts are mandatory, as is the __Timestamp__ part of the payload. - - The __Destination__ and __Source__ fields are 10-byte Reticulum destination hashes + - The __Destination__ and __Source__ fields are 16-byte Reticulum destination hashes - The __Signature__ field is a 64-byte Ed25519 signature of the __Destination__, __Source__, __Payload__ and __message-id__ - The __Payload__ part is a [msgpacked](https://msgpack.org) list containing four items: 1. The __Timestamp__ is a double-precision floating point number representing the number of seconds since the UNIX epoch. @@ -100,7 +100,7 @@ Assuming the default Reticulum configuration, the binary wire-format is as follo - 64 bytes Ed25519 signature - Remaining bytes of [msgpack](https://msgpack.org) payload data, in accordance with the structure defined above -The complete message overhead for LXMF is only 99 bytes, which in return gives you timestamped, digitally signed, infinitely extensible, end-to-end encrypted, zero-conf routed, minimal-infrastructure messaging that's easy to use and build applications with. +The complete message overhead for LXMF is only 111 bytes, which in return gives you timestamped, digitally signed, infinitely extensible, end-to-end encrypted, zero-conf routed, minimal-infrastructure messaging that's easy to use and build applications with. ## Example Paper Message From 6f80c6941de0a899ad8346efbd2f0e33f62eda39 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 27 Jan 2023 16:36:15 +0100 Subject: [PATCH 133/436] Fixed a erroneous warning message about missing authentication identities. Fixed a missing check for announce intervals being unset in the configuration. Fixes #12. --- LXMF/Utilities/lxmd.py | 25 ++++++++++++++----------- LXMF/_version.py | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index d033424..69f05eb 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -302,6 +302,10 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo # Set up authentication if active_configuration["auth_required"]: message_router.set_authentication(required=True) + + if len(active_configuration["allowed_identities"]) == 0: + RNS.log("Clint authentication was enabled, but no identity hashes could be loaded from "+str(allowedpath)+". Nobody will be able to sync messages from this propagation node.", RNS.LOG_WARNING) + for identity_hash in active_configuration["allowed_identities"]: message_router.allow(identity_hash) @@ -322,9 +326,6 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo RNS.log("LXMF Propagation Node started on "+RNS.prettyhexrep(message_router.propagation_destination.hash)) - if len(active_configuration["allowed_identities"]) == 0: - RNS.log("Clint authentication was enabled, but no identity hashes could be loaded from "+str(allowedpath)+". Nobody will be able to sync messages from this propagation node.", RNS.LOG_WARNING) - RNS.log("Started lxmd version {version}".format(version=__version__), RNS.LOG_NOTICE) threading.Thread(target=deferred_start_jobs, daemon=True).start() @@ -338,15 +339,17 @@ def jobs(): while True: try: - if time.time() > last_peer_announce + active_configuration["peer_announce_interval"]: - RNS.log("Sending announce for LXMF delivery destination", RNS.LOG_EXTREME) - message_router.announce(lxmf_destination.hash) - last_peer_announce = time.time() + if "peer_announce_interval" in active_configuration and active_configuration["peer_announce_interval"] != None: + if time.time() > last_peer_announce + active_configuration["peer_announce_interval"]: + RNS.log("Sending announce for LXMF delivery destination", RNS.LOG_EXTREME) + message_router.announce(lxmf_destination.hash) + last_peer_announce = time.time() - if time.time() > last_node_announce + active_configuration["node_announce_interval"]: - RNS.log("Sending announce for LXMF Propagation Node", RNS.LOG_EXTREME) - message_router.announce_propagation_node() - last_node_announce = time.time() + if "node_announce_interval" in active_configuration and active_configuration["node_announce_interval"] != None: + if time.time() > last_node_announce + active_configuration["node_announce_interval"]: + RNS.log("Sending announce for LXMF Propagation Node", RNS.LOG_EXTREME) + message_router.announce_propagation_node() + last_node_announce = time.time() except Exception as e: RNS.log("An error occurred while running periodic jobs. The contained exception was: "+str(e), RNS.LOG_ERROR) diff --git a/LXMF/_version.py b/LXMF/_version.py index 75cf783..493f741 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.2.9" +__version__ = "0.3.0" From a64f3a58f7fca84cc69c411d270301c465b9d7d0 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 27 Jan 2023 17:00:29 +0100 Subject: [PATCH 134/436] Fixed verbosity and quitness being overridden by config file values --- LXMF/Utilities/lxmd.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 69f05eb..3dc34df 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -204,8 +204,6 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo global lxmd_config, active_configuration, targetloglevel global message_router, lxmf_destination - targetloglevel = 3+verbosity-quietness - if service: targetlogdest = RNS.LOG_FILE targetloglevel = None @@ -234,6 +232,12 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo if not os.path.isdir(lxmdir): os.makedirs(lxmdir) + if not os.path.isfile(configpath): + RNS.log("Could not load config file, creating default configuration file...") + create_default_config(configpath) + RNS.log("Default config file created. Make any necessary changes in "+configpath+" and restart lxmd if needed.") + time.sleep(1.5) + if os.path.isfile(configpath): try: lxmd_config = ConfigObj(configpath) @@ -241,15 +245,16 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo RNS.log("Could not parse the configuration at "+configpath, RNS.LOG_ERROR) RNS.log("Check your configuration file for errors!", RNS.LOG_ERROR) RNS.panic() - else: - RNS.log("Could not load config file, creating default configuration file...") - create_default_config(configpath) - RNS.log("Default config file created. Make any necessary changes in "+configpath+" and restart Reticulum if needed.") - time.sleep(1.5) - + apply_config() RNS.log("Configuration loaded from "+configpath, RNS.LOG_VERBOSE) + if targetloglevel == None: + targetloglevel = 3 + + if verbosity != 0 or quietness != 0: + targetloglevel = targetloglevel+verbosity-quietness + # Start Reticulum RNS.log("Substantiating Reticulum...") reticulum = RNS.Reticulum(configdir=rnsconfigdir, loglevel=targetloglevel, logdest=targetlogdest) From afe6815637cba4e9525bed5b9f8a420511a04393 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 3 Feb 2023 17:56:30 +0100 Subject: [PATCH 135/436] Fixed hardcoded APP_NAME reference --- LXMF/LXMRouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index d571eab..901c933 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -153,7 +153,7 @@ class LXMRouter: da_thread.start() def register_delivery_identity(self, identity, display_name = None): - delivery_destination = RNS.Destination(identity, RNS.Destination.IN, RNS.Destination.SINGLE, "lxmf", "delivery") + delivery_destination = RNS.Destination(identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "delivery") delivery_destination.set_packet_callback(self.delivery_packet) delivery_destination.set_link_established_callback(self.delivery_link_established) delivery_destination.display_name = display_name From 7a40bd44e8a863a1fd91ec76d807f37e6944dae6 Mon Sep 17 00:00:00 2001 From: Jim Tittsler Date: Fri, 10 Feb 2023 16:07:16 +0900 Subject: [PATCH 136/436] Fix typos --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ca0ba1d..ff2e7c7 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ LXMF messages are stored in a simple and efficient format, that's easy to parse 1. A LXMF message is identified by its __message-id__, which is a SHA-256 hash of the __Destination__, __Source__ and __Payload__. The message-id is never included directly in the message, since it can always be inferred from the message itself. - In some cases the actual message-id cannot be inferred, for example when a Propagation Node is storing an encrypted message for an offline user. In theses cases a _transient-id_ is used to identify the message while in storage or transit. + In some cases the actual message-id cannot be inferred, for example when a Propagation Node is storing an encrypted message for an offline user. In these cases a _transient-id_ is used to identify the message while in storage or transit. 2. __Destination__, __Source__, __Signature__ and __Payload__ parts are mandatory, as is the __Timestamp__ part of the payload. - The __Destination__ and __Source__ fields are 16-byte Reticulum destination hashes @@ -46,7 +46,7 @@ LXMF messages are stored in a simple and efficient format, that's easy to parse ## Usage Examples -LXMF offers flexibility to implement many different messaging schemes, ranging from human communication to machine control and sensor monitoring. Here's a few examples: +LXMF offers flexibility to implement many different messaging schemes, ranging from human communication to machine control and sensor monitoring. Here are a few examples: - A messaging system for passing short, simple messages between human users, akin to SMS can be implemented using only the __Content__ field, and leaving all other optional fields empty. @@ -104,11 +104,11 @@ The complete message overhead for LXMF is only 111 bytes, which in return gives ## Example Paper Message -You can try out the paper messaging functionality by using the following QR-code. It is a paper message sent to the LXMF address `6b3362bd2c1dbf87b66a85f79a8d8c75`. To be able to decrypt and read the message, you will need to import the following Reticulum Identity to an LXMF messaging app: +You can try out the paper messaging functionality by using the following QR code. It is a paper message sent to the LXMF address `6b3362bd2c1dbf87b66a85f79a8d8c75`. To be able to decrypt and read the message, you will need to import the following Reticulum Identity to an LXMF messaging app: `3BPTDTQCRZPKJT3TXAJCMQFMOYWIM3OCLKPWMG4HCF2T4CH3YZHVNHNRDU6QAZWV2KBHMWBNT2C62TQEVC5GLFM4MN25VLZFSK3ADRQ=` -The [Sideband](https://unsigned.io/sideband) application allows you to do this easily. After the you have imported the identity into an app of your choice, you can scan the following QR-code and open it in the app, where it will be decrypted and added as a message. +The [Sideband](https://unsigned.io/sideband) application allows you to do this easily. After you have imported the identity into an app of your choice, you can scan the following QR code and open it in the app, where it will be decrypted and added as a message.

    @@ -120,7 +120,7 @@ On operating systems that allow for registering custom URI-handlers, you can cli ## Caveat Emptor -LXMF is beta software, and should be considered experimental. While it has been built with cryptography best-practices very foremost in mind, it _has not_ been externally security audited, and there could very well be privacy-breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch. +LXMF is beta software, and should be considered experimental. While it has been built with cryptography best practices very foremost in mind, it _has not_ been externally security audited, and there could very well be privacy-breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch. ## Installation From 0ba3ea98cb23b50362c8648cf8b52716145d058f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 17 Feb 2023 12:29:00 +0100 Subject: [PATCH 137/436] Added metric for link establishment rate to peers --- LXMF/LXMPeer.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 585b2a3..63da3a1 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -43,6 +43,10 @@ class LXMPeer: peer.peering_timebase = dictionary["peering_timebase"] peer.alive = dictionary["alive"] peer.last_heard = dictionary["last_heard"] + if "link_establishment_rate" in dictionary: + peer.link_establishment_rate = dictionary["link_establishment_rate"] + else: + peer.link_establishment_rate = 0 for transient_id in dictionary["handled_ids"]: if transient_id in router.propagation_entries: @@ -60,6 +64,7 @@ class LXMPeer: dictionary["alive"] = self.alive dictionary["last_heard"] = self.last_heard dictionary["destination_hash"] = self.destination_hash + dictionary["link_establishment_rate"] = self.link_establishment_rate handled_ids = [] for transient_id in self.handled_messages: @@ -81,6 +86,7 @@ class LXMPeer: self.last_sync_attempt = 0 self.sync_backoff = 0 self.peering_timebase = 0 + self.link_establishment_rate = 0 self.link = None self.state = LXMPeer.IDLE @@ -94,7 +100,6 @@ class LXMPeer: self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") def sync(self): - RNS.log("Initiating LXMF Propagation Node sync with peer "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_DEBUG) self.last_sync_attempt = time.time() @@ -253,6 +258,10 @@ class LXMPeer: def link_established(self, link): self.link.identify(self.router.identity) + link_establishment_rate = link.get_establishment_rate() + if link_establishment_rate != None: + self.link_establishment_rate = link_establishment_rate + self.state = LXMPeer.LINK_READY self.next_sync_attempt = 0 self.sync() From 447a8a920c1f5ea8415a6294f1efd8d4ab98e8a6 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 17 Feb 2023 17:10:31 +0100 Subject: [PATCH 138/436] Implemented mechanisms to tend propagation node sync graph towards using faster links first --- LXMF/LXMRouter.py | 27 +++++++++++++++++++++++++-- LXMF/_version.py | 2 +- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 901c933..4b0e64b 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -28,6 +28,7 @@ class LXMRouter: AUTOPEER = True AUTOPEER_MAXDEPTH = 4 + FASTEST_N_RANDOM_POOL = 2 PR_PATH_TIMEOUT = 10 @@ -917,6 +918,7 @@ class LXMRouter: peer.peering_timebase = timestamp peer.last_heard = time.time() + else: peer = LXMPeer(self, destination_hash) peer.alive = True @@ -956,8 +958,29 @@ class LXMRouter: peer_pool = [] if len(waiting_peers) > 0: - RNS.log("Randomly selecting peer to sync from "+str(len(waiting_peers))+" waiting peers.", RNS.LOG_DEBUG) - peer_pool = waiting_peers + fastest_peers = sorted( + waiting_peers, + key=lambda p: p.link_establishment_rate, + reverse=True + )[0:min(FASTEST_N_RANDOM_POOL, len(waiting_peers))] + peer_pool.extend(fastest_peers) + + unknown_speed_peers = [p for p in waiting_peers if p.link_establishment_rate == 0] + if len(unknown_speed_peers) > 0: + peer_pool.extend( + unknown_speed_peers[ + 0:min( + len(unknown_speed_peers), + len(fastest_peers) + )] + ) + + # TODO: Remove + for p in peer_pool: + RNS.log("Peer: "+RNS.prettyhexrep(p.destination.hash)+" "+RNS.prettysize(p.link_establishment_rate/8, "b")+"ps", RNS.LOG_DEBUG) + + RNS.log("Selecting peer to sync from "+str(len(waiting_peers))+" waiting peers.", RNS.LOG_DEBUG) + elif len(unresponsive_peers) > 0: RNS.log("No active peers available, randomly selecting peer to sync from "+str(len(unresponsive_peers))+" unresponsive peers.", RNS.LOG_DEBUG) peer_pool = unresponsive_peers diff --git a/LXMF/_version.py b/LXMF/_version.py index 493f741..260c070 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.3.0" +__version__ = "0.3.1" From ad8fb74d9ef123e53903bb43604c27951a9d0429 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 17 Feb 2023 19:36:20 +0100 Subject: [PATCH 139/436] Cleanup --- LXMF/LXMRouter.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 4b0e64b..bfcd428 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -445,7 +445,12 @@ class LXMRouter: while (True): # TODO: Improve this to scheduling, so manual # triggers can delay next run - self.jobs() + + try: + self.jobs() + except Exception as e: + RNS.log("An error ocurred while running LXMF Router jobs.", RNS.LOG_ERROR) + RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) time.sleep(LXMRouter.PROCESSING_INTERVAL) def clean_links(self): @@ -962,7 +967,7 @@ class LXMRouter: waiting_peers, key=lambda p: p.link_establishment_rate, reverse=True - )[0:min(FASTEST_N_RANDOM_POOL, len(waiting_peers))] + )[0:min(LXMRouter.FASTEST_N_RANDOM_POOL, len(waiting_peers))] peer_pool.extend(fastest_peers) unknown_speed_peers = [p for p in waiting_peers if p.link_establishment_rate == 0] @@ -975,10 +980,6 @@ class LXMRouter: )] ) - # TODO: Remove - for p in peer_pool: - RNS.log("Peer: "+RNS.prettyhexrep(p.destination.hash)+" "+RNS.prettysize(p.link_establishment_rate/8, "b")+"ps", RNS.LOG_DEBUG) - RNS.log("Selecting peer to sync from "+str(len(waiting_peers))+" waiting peers.", RNS.LOG_DEBUG) elif len(unresponsive_peers) > 0: From 51a4d19595eea375845ee8794ecddde45ae66a70 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 17 Feb 2023 22:19:12 +0100 Subject: [PATCH 140/436] Updated dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 39ed781..6b0c2e0 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.4.7'], + install_requires=['rns>=0.4.9'], python_requires='>=3.6', ) From 5abf01a88fa2e80fd8a20c96dafa961b6fa3a882 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 13 Jun 2023 19:56:44 +0200 Subject: [PATCH 141/436] Updated version and dependencies --- LXMF/_version.py | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 260c070..f9aa3e1 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.3.1" +__version__ = "0.3.2" diff --git a/setup.py b/setup.py index 6b0c2e0..5aaf611 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.4.9'], - python_requires='>=3.6', + install_requires=['rns>=0.5.5'], + python_requires='>=3.7', ) From 7220a4d6f77bbd5cfe5c0b8ad55d0bca010c31ff Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 15 Aug 2023 11:07:24 +0200 Subject: [PATCH 142/436] Updated install section --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ff2e7c7..de0d07f 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,22 @@ LXMF is beta software, and should be considered experimental. While it has been If you want to try out LXMF, you can install it with pip: ```bash -pip3 install lxmf +pip install lxmf +``` + +If you are using an operating system that blocks normal user package installation via `pip`, +you can return `pip` to normal behaviour by adding editing the `~/.config/pip/pip.conf` file, +and adding the following directive in the `[global]` section: + +```text +[global] +break-system-packages = true +``` + +Alternatively, you can use the `pipx` tool to install Reticulum in an isolated environment: + +```bash +pipx install lxmf ``` ## Development Roadmap From b690880332ab050f11d563a4ffb91ce0a5159fdb Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 15 Aug 2023 11:08:38 +0200 Subject: [PATCH 143/436] Updated dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5aaf611..d4fb2e1 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.5.5'], + install_requires=['rns>=0.5.7'], python_requires='>=3.7', ) From 55cf93447573c27566b98f787ed89886795a4535 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 15 Aug 2023 11:27:02 +0200 Subject: [PATCH 144/436] Updated install section --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index de0d07f..a8831e2 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ pip install lxmf ``` If you are using an operating system that blocks normal user package installation via `pip`, -you can return `pip` to normal behaviour by adding editing the `~/.config/pip/pip.conf` file, +you can return `pip` to normal behaviour by editing the `~/.config/pip/pip.conf` file, and adding the following directive in the `[global]` section: ```text From 9f051aea17f2d48c0c5b673e20da4f8463231756 Mon Sep 17 00:00:00 2001 From: Petr Blaha Date: Thu, 7 Sep 2023 08:39:57 +0200 Subject: [PATCH 145/436] dockerize LXMF --- Dockerfile | 31 +++++++++++++++++++++++++++++++ Dockerfile.howto | 6 ++++++ requirements.txt | 3 +++ 3 files changed, 40 insertions(+) create mode 100644 Dockerfile create mode 100644 Dockerfile.howto create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..818f3b3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM python:alpine +LABEL authors="Petr Blaha petr.blaha@cleverdata.cz" +USER root +RUN apk update +RUN apk add sdl2_ttf sdl2 build-base libc-dev pkgconfig gstreamer sdl2_mixer sdl2_image sdl2_pango linux-headers mesa-dev py3-virtualenv + +RUN addgroup -S myuser && adduser -S -G myuser myuser +USER myuser +WORKDIR /home/myuser + +RUN pip install --upgrade pip + + +ENV PATH="/home/myuser/.local/bin:${PATH}" + +################### BEGIN Sideband ########################################### + +COPY --chown=myuser:myuser requirements.txt requirements.txt + +RUN pip install --user -r requirements.txt + + +COPY --chown=myuser:myuser . . + +#Python create virtual environment +RUN virtualenv /home/myuser/LXMF/venv +RUN source /home/myuser/LXMF/venv/bin/activate + +RUN make all + +################### END Sideband ########################################### diff --git a/Dockerfile.howto b/Dockerfile.howto new file mode 100644 index 0000000..bb20e5f --- /dev/null +++ b/Dockerfile.howto @@ -0,0 +1,6 @@ +# Run docker command one by one(all four), it will build LXMF artifact and copy to dist directory. +# No need to build locally and install dependencies +docker build -t lxmfdockerimage . +docker run -d -it --name lxmfdockercontainer lxmfdockerimage /bin/sh +docker cp lxmfdockercontainer:/home/myuser/dist . +docker rm -f lxmfdockercontainer \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..48086e6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +qrcode==7.4.2 +rns==0.5.7 +setuptools==68.0.0 From 820c92d38b8c690ab2336e872793c0746a8cfc40 Mon Sep 17 00:00:00 2001 From: Petr Blaha Date: Thu, 7 Sep 2023 08:51:49 +0200 Subject: [PATCH 146/436] dockerize LXMF --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 818f3b3..b7ed71d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ RUN pip install --upgrade pip ENV PATH="/home/myuser/.local/bin:${PATH}" -################### BEGIN Sideband ########################################### +################### BEGIN LXMF ########################################### COPY --chown=myuser:myuser requirements.txt requirements.txt @@ -28,4 +28,4 @@ RUN source /home/myuser/LXMF/venv/bin/activate RUN make all -################### END Sideband ########################################### +################### END LXMF ########################################### From 672054177c2577649565ca906323e7b4aef0ee53 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 19 Sep 2023 11:36:13 +0200 Subject: [PATCH 147/436] Updated version and dependencies --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index f9aa3e1..e19434e 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.3.2" +__version__ = "0.3.3" diff --git a/setup.py b/setup.py index d4fb2e1..961fd34 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.5.7'], + install_requires=['rns>=0.5.9'], python_requires='>=3.7', ) From 0ac4660510da8ffa0476d3114abe39212a4de59c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 7 Oct 2023 16:43:33 +0200 Subject: [PATCH 148/436] Updated version and dependencien --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index e19434e..334b899 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.3.3" +__version__ = "0.3.4" diff --git a/setup.py b/setup.py index 961fd34..e9c0072 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.5.9'], + install_requires=['rns>=0.6.2'], python_requires='>=3.7', ) From b894c5a2a61c8e597f43c6543d2b4c753520a814 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 15 Oct 2023 20:27:35 +0200 Subject: [PATCH 149/436] Fixed paper message generation sequence --- LXMF/LXMessage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 6ab264f..6f8a15a 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -487,7 +487,6 @@ class LXMessage: return None def as_uri(self, finalise=True): - self.determine_transport_encryption() if not self.packed: self.pack() @@ -499,6 +498,7 @@ class LXMessage: lxm_uri = LXMessage.URI_SCHEMA+"://"+encoded_bytes.decode("utf-8").replace("=","") if finalise: + self.determine_transport_encryption() self.__mark_paper_generated() return lxm_uri @@ -507,7 +507,6 @@ class LXMessage: raise TypeError("Attempt to represent LXM with non-paper delivery method as URI") def as_qr(self): - self.determine_transport_encryption() if not self.packed: self.pack() @@ -522,6 +521,7 @@ class LXMessage: data = self.as_uri(finalise=False), ) + self.determine_transport_encryption() self.__mark_paper_generated() return qr From bc8863c3b16b6a48fad54e6472acaa503f9f7968 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 15 Oct 2023 20:28:55 +0200 Subject: [PATCH 150/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 334b899..a8d4557 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.3.4" +__version__ = "0.3.5" From 537123896bf648b2cf28f69973ab13bb42ce4636 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 16 Oct 2023 01:50:01 +0200 Subject: [PATCH 151/436] Increased max unreachable time to 14 days --- LXMF/LXMPeer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 63da3a1..611a874 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -22,7 +22,7 @@ class LXMPeer: # Maximum amount of time a peer can # be unreachable before it is removed - MAX_UNREACHABLE = 4*24*60*60 + MAX_UNREACHABLE = 14*24*60*60 # Everytime consecutive time a sync # link fails to establish, add this From f83600430ab484a161e3560ebce9be1363cfe2ad Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 16 Oct 2023 01:50:37 +0200 Subject: [PATCH 152/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index a8d4557..d7b30e1 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.3.5" +__version__ = "0.3.6" From 0dc1e13f9df076d4578ad72e6c0aed14b3a3f23b Mon Sep 17 00:00:00 2001 From: markqvist Date: Tue, 17 Oct 2023 02:14:07 +0200 Subject: [PATCH 153/436] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a8831e2..bd62246 100644 --- a/README.md +++ b/README.md @@ -85,11 +85,11 @@ The LXM Router then handles the heavy lifting, such as message packing, encrypti LXMF uses encryption provided by [Reticulum](https://reticulum.network), and thus uses end-to-end encryption by default. The delivery method of a message will influence which transport encryption scheme is used. -- A message can be delivered opportunistically, embedded in a single Reticulum packet. In this cases the message will be opportunistically routed through the network, and will be encrypted with _ephemeral_ keys derived with _ECDH_ on _Curve25519_. This mode offers Perfect Forward Secrecy. +- If a message is delivered over a Reticulum link (which is the default method), the message will be encrypted with ephemeral AES-128 keys derived with ECDH on Curve25519. This mode offers forward secrecy. -- If a message is delivered to the Reticulum GROUP destination type, the message will be transported using _AES-128_ encryption. +- A message can be delivered opportunistically, embedded in a single Reticulum packet. In this cases the message will be opportunistically routed through the network, and will be encrypted with per-packet AES-128 keys derived with ECDH on Curve25519. -- If a message is delivered over a Reticulum link (which is the default method), the message will be encrypted with _ephemeral_ keys derived with _ECDH_ on _Curve25519_. This mode offers Perfect Forward Secrecy. +- If a message is delivered to the Reticulum GROUP destination type, the message will be encrypted using the symmetric AES-128 key of the GROUP destination. ## Wire Format & Overhead From a095ba7e405f34f3dd6195f1d24a59fbb3e13d1b Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 23 Oct 2023 01:34:20 +0200 Subject: [PATCH 154/436] Added physical layer link statistics to reception --- LXMF/LXMF.py | 14 +++++++++++++- LXMF/LXMRouter.py | 13 ++++++++++--- LXMF/LXMessage.py | 3 +++ LXMF/__init__.py | 1 + 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index d19b423..6e8d479 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -1 +1,13 @@ -APP_NAME = "lxmf" \ No newline at end of file +APP_NAME = "lxmf" + +# WARNING! These specifications are floating and not +# yet final! Consider highly experiemental, and expect +# them to change in the future! You have been warned :) + +FIELD_EMBEDDED_LXMS = 0x01 +FIELD_TELEMETRY = 0x02 +FIELD_ICON_APPEARANCE = 0x03 +FIELD_FILE_ATTACHMENTS = 0x04 +FIELD_IMAGE = 0x05 +FIELD_AUDIO = 0x06 +FIELD_THREAD = 0x07 \ No newline at end of file diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index bfcd428..a8963d2 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -842,10 +842,15 @@ class LXMRouter: self.pending_outbound.append(lxmessage) self.process_outbound() - def lxmf_delivery(self, lxmf_data, destination_type = None): + def lxmf_delivery(self, lxmf_data, destination_type = None, phy_stats = None): try: message = LXMessage.unpack_from_bytes(lxmf_data) + if phy_stats != None: + if "rssi" in phy_stats: message.rssi = phy_stats["rssi"] + if "snr" in phy_stats: message.snr = phy_stats["snr"] + if "q" in phy_stats: message.q = phy_stats["q"] + if destination_type == RNS.Destination.SINGLE: message.transport_encrypted = True message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC @@ -885,7 +890,8 @@ class LXMRouter: else: lxmf_data = data - if self.lxmf_delivery(lxmf_data, packet.destination_type): + phy_stats = {"rssi": packet.rssi, "snr": packet.snr, "q": packet.q} + if self.lxmf_delivery(lxmf_data, packet.destination_type, phy_stats=phy_stats): packet.prove() except Exception as e: @@ -907,7 +913,8 @@ class LXMRouter: def delivery_resource_concluded(self, resource): RNS.log("Transfer concluded for LXMF delivery resource "+str(resource), RNS.LOG_DEBUG) if resource.status == RNS.Resource.COMPLETE: - self.lxmf_delivery(resource.data.read(), resource.link.type) + phy_stats = {"rssi": resource.link.rssi, "snr": resource.link.snr, "q": resource.link.q} + self.lxmf_delivery(resource.data.read(), resource.link.type, phy_stats=phy_stats) ### Peer Sync & Propagation ########################### diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 6f8a15a..ee83721 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -122,6 +122,9 @@ class LXMessage: self.progress = None self.state = LXMessage.DRAFT self.method = LXMessage.UNKNOWN + self.rssi = None + self.snr = None + self.q = None self.propagation_packed = None self.paper_packed = None diff --git a/LXMF/__init__.py b/LXMF/__init__.py index dc0abbb..0458f8d 100644 --- a/LXMF/__init__.py +++ b/LXMF/__init__.py @@ -2,6 +2,7 @@ import os import glob from .LXMessage import LXMessage from .LXMRouter import LXMRouter +from .LXMF import * from ._version import __version__ From f307f33bf8e8a085f5c429ee09342116239d8758 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 24 Oct 2023 01:47:45 +0200 Subject: [PATCH 155/436] Updated version and dependencies --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index d7b30e1..8879c6c 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.3.6" +__version__ = "0.3.7" diff --git a/setup.py b/setup.py index e9c0072..183736c 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.6.2'], + install_requires=['rns>=0.6.3'], python_requires='>=3.7', ) From 56cb637bfa03b7a06a3f3360d1048c831447f94b Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 28 Oct 2023 00:06:56 +0200 Subject: [PATCH 156/436] Improved propagation node sync feedback. Added physical layer reception statistics. --- LXMF/LXMRouter.py | 41 +++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index a8963d2..fb7f3b0 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -40,8 +40,12 @@ class LXMRouter: PR_RECEIVING = 0x05 PR_RESPONSE_RECEIVED = 0x06 PR_COMPLETE = 0x07 - PR_NO_IDENTITY_RCVD = 0xf0 - PR_NO_ACCESS = 0xf1 + PR_NO_PATH = 0xf0 + PR_LINK_FAILED = 0xf1 + PR_TRANSFER_FAILED = 0xf2 + PR_NO_IDENTITY_RCVD = 0xf3 + PR_NO_ACCESS = 0xf4 + PR_FAILED = 0xfe PR_ALL_MESSAGES = 0x00 @@ -220,9 +224,9 @@ class LXMRouter: if max_messages == None: max_messages = LXMRouter.PR_ALL_MESSAGES + self.propagation_transfer_progress = 0.0 self.propagation_transfer_max_messages = max_messages if self.outbound_propagation_node != None: - self.propagation_transfer_progress = 0.0 if self.outbound_propagation_link != None and self.outbound_propagation_link.status == RNS.Link.ACTIVE: self.propagation_transfer_state = LXMRouter.PR_LINK_ESTABLISHED self.outbound_propagation_link.identify(identity) @@ -469,7 +473,16 @@ class LXMRouter: if self.outbound_propagation_link != None and self.outbound_propagation_link.status == RNS.Link.CLOSED: self.outbound_propagation_link = None - self.acknowledge_sync_completion() + if self.propagation_transfer_state == LXMRouter.PR_COMPLETE: + self.acknowledge_sync_completion() + elif self.propagation_transfer_state < LXMRouter.PR_LINK_ESTABLISHED: + self.acknowledge_sync_completion(failure_state=LXMRouter.PR_LINK_FAILED) + elif self.propagation_transfer_state >= LXMRouter.PR_LINK_ESTABLISHED and self.propagation_transfer_state < LXMRouter.PR_COMPLETE: + self.acknowledge_sync_completion(failure_state=LXMRouter.PR_TRANSFER_FAILED) + else: + RNS.log(f"Unknown propagation transfer state on link cleaning: {self.propagation_transfer_state}", RNS.LOG_DEBUG) + self.acknowledge_sync_completion() + RNS.log("Cleaned outbound propagation link", RNS.LOG_DEBUG) def clean_transient_id_caches(self): @@ -647,7 +660,7 @@ class LXMRouter: self.request_messages_from_propagation_node(self.wants_download_on_path_available_to, self.propagation_transfer_max_messages) else: RNS.log("Propagation node path request timed out", RNS.LOG_DEBUG) - self.acknowledge_sync_completion() + self.acknowledge_sync_completion(failure_state=LXMRouter.PR_NO_PATH) def identity_allowed(self, identity): if self.auth_required: @@ -810,12 +823,15 @@ class LXMRouter: if self.outbound_propagation_link != None: self.outbound_propagation_link.teardown() - def acknowledge_sync_completion(self, reset_state=False): + def acknowledge_sync_completion(self, reset_state=False, failure_state=None): + self.propagation_transfer_last_result = None if reset_state or self.propagation_transfer_state <= LXMRouter.PR_COMPLETE: - self.propagation_transfer_state = LXMRouter.PR_IDLE + if failure_state == None: + self.propagation_transfer_state = LXMRouter.PR_IDLE + else: + self.propagation_transfer_state = failure_state self.propagation_transfer_progress = 0.0 - self.propagation_transfer_last_result = None self.wants_download_on_path_available_from = None self.wants_download_on_path_available_to = None @@ -890,6 +906,14 @@ class LXMRouter: else: lxmf_data = data + try: + reticulum = RNS.Reticulum.get_instance() + if packet.rssi == None: packet.rssi = reticulum.get_packet_rssi(packet.packet_hash) + if packet.snr == None: packet.snr = reticulum.get_packet_snr(packet.packet_hash) + if packet.q == None: packet.q = reticulum.get_packet_q(packet.packet_hash) + except Exception as e: + RNS.log("Error while retrieving physical link stats for LXMF delivery packet: "+str(e), RNS.LOG_ERROR) + phy_stats = {"rssi": packet.rssi, "snr": packet.snr, "q": packet.q} if self.lxmf_delivery(lxmf_data, packet.destination_type, phy_stats=phy_stats): packet.prove() @@ -899,6 +923,7 @@ class LXMRouter: RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) def delivery_link_established(self, link): + link.track_phy_stats(True) link.set_packet_callback(self.delivery_packet) link.set_resource_strategy(RNS.Link.ACCEPT_ALL) link.set_resource_started_callback(self.resource_transfer_began) From da6fb3d0f33b118149cc97692f5f2f2059a8a68a Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 30 Oct 2023 02:26:24 +0100 Subject: [PATCH 157/436] Added field specifiers --- LXMF/LXMF.py | 15 +++++++++------ LXMF/LXMessage.py | 3 +-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 6e8d479..5225d88 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -1,13 +1,16 @@ APP_NAME = "lxmf" -# WARNING! These specifications are floating and not +# WARNING! These field specifiers are floating and not # yet final! Consider highly experiemental, and expect # them to change in the future! You have been warned :) FIELD_EMBEDDED_LXMS = 0x01 FIELD_TELEMETRY = 0x02 -FIELD_ICON_APPEARANCE = 0x03 -FIELD_FILE_ATTACHMENTS = 0x04 -FIELD_IMAGE = 0x05 -FIELD_AUDIO = 0x06 -FIELD_THREAD = 0x07 \ No newline at end of file +FIELD_TELEMETRY_STREAM = 0x03 +FIELD_ICON_APPEARANCE = 0x04 +FIELD_FILE_ATTACHMENTS = 0x05 +FIELD_IMAGE = 0x06 +FIELD_AUDIO = 0x07 +FIELD_THREAD = 0x08 +FIELD_COMMANDS = 0x09 +FIELD_RESULTS = 0x0A \ No newline at end of file diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index ee83721..5c59202 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -111,7 +111,6 @@ class LXMessage: self.set_title_from_string(title) self.set_content_from_string(content) - self.set_fields(fields) self.payload = None @@ -164,7 +163,7 @@ class LXMessage: def set_fields(self, fields): if isinstance(fields, dict) or fields == None: - self.fields = fields + self.fields = fields or {} else: raise ValueError("LXMessage property \"fields\" can only be dict or None") From 4d356bcda897c10b76c9d4d8e5df2133402415b3 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 2 Nov 2023 04:36:51 +0100 Subject: [PATCH 158/436] Tweaked timing --- LXMF/LXMRouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index fb7f3b0..3d3e936 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -19,7 +19,7 @@ class LXMRouter: MAX_DELIVERY_ATTEMPTS = 5 PROCESSING_INTERVAL = 4 DELIVERY_RETRY_WAIT = 7 - PATH_REQUEST_WAIT = 5 + PATH_REQUEST_WAIT = 7 LINK_MAX_INACTIVITY = 10*60 MESSAGE_EXPIRY = 30*24*60*60 From 10be1383e5cb6fac565c5f679310536e5c326115 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 2 Nov 2023 12:48:24 +0100 Subject: [PATCH 159/436] Updated version and dependencies --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 8879c6c..4ad67eb 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.3.7" +__version__ = "0.3.8" diff --git a/setup.py b/setup.py index 183736c..ccd2c25 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.6.3'], + install_requires=['rns>=0.6.4'], python_requires='>=3.7', ) From 2812a07165239931a1ad7ded56901625de32534d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 2 Nov 2023 18:29:46 +0100 Subject: [PATCH 160/436] Cleanup --- LXMF/LXMRouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 3d3e936..06e9abf 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1221,7 +1221,7 @@ class LXMRouter: else: RNS.log("Starting outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) # Outbound handling for opportunistic messages - if lxmessage.method == LXMessage.OPPORTUNISTIC: + if lxmessage.method == LXMessage.OPPORTUNISTIC: if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: lxmessage.delivery_attempts += 1 From 1ee0c83168a050265e06ddae60e63c28edb75d9d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 8 Nov 2023 20:41:51 +0100 Subject: [PATCH 161/436] Updated version and dependencies --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 4ad67eb..771bc6e 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.3.8" +__version__ = "0.3.9" diff --git a/setup.py b/setup.py index ccd2c25..5c17e3b 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.6.4'], + install_requires=['rns>=0.6.7'], python_requires='>=3.7', ) From 754ae969e10d6ee1002dd31bf130573ef30b1e4d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 3 Jan 2024 12:45:18 +0100 Subject: [PATCH 162/436] Added code examples. Closes #8. --- README.md | 4 ++++ docs/example_receiver.py | 37 +++++++++++++++++++++++++++++++++++++ docs/example_sender.py | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 docs/example_receiver.py create mode 100644 docs/example_sender.py diff --git a/README.md b/README.md index bd62246..11451a8 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,10 @@ Assuming the default Reticulum configuration, the binary wire-format is as follo The complete message overhead for LXMF is only 111 bytes, which in return gives you timestamped, digitally signed, infinitely extensible, end-to-end encrypted, zero-conf routed, minimal-infrastructure messaging that's easy to use and build applications with. +## Code Examples + +Before writing your own programs using LXMF, you need to have a basic understanding of how the [Reticulum](https://reticulum.network) protocol and API works. Please see the [Reticulum Manual](https://reticulum.network/manual/). For a few simple examples of how to send and receive messages with LXMF, please see the [receiver example](./docs/example_receiver.py) and the [sender example](./docs/example_sender.py) included in this repository. + ## Example Paper Message You can try out the paper messaging functionality by using the following QR code. It is a paper message sent to the LXMF address `6b3362bd2c1dbf87b66a85f79a8d8c75`. To be able to decrypt and read the message, you will need to import the following Reticulum Identity to an LXMF messaging app: diff --git a/docs/example_receiver.py b/docs/example_receiver.py new file mode 100644 index 0000000..a637cd4 --- /dev/null +++ b/docs/example_receiver.py @@ -0,0 +1,37 @@ +import RNS +import LXMF +import time + +def delivery_callback(message): + time_string = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(message.timestamp)) + signature_string = "Signature is invalid, reason undetermined" + if message.signature_validated: + signature_string = "Validated" + else: + if message.unverified_reason == LXMF.LXMessage.SIGNATURE_INVALID: + signature_string = "Invalid signature" + if message.unverified_reason == LXMF.LXMessage.SOURCE_UNKNOWN: + signature_string = "Cannot verify, source is unknown" + + RNS.log("\t+--- LXMF Delivery ---------------------------------------------") + RNS.log("\t| Source hash : "+RNS.prettyhexrep(message.source_hash)) + RNS.log("\t| Source instance : "+str(message.get_source())) + RNS.log("\t| Destination hash : "+RNS.prettyhexrep(message.destination_hash)) + RNS.log("\t| Destination instance : "+str(message.get_destination())) + RNS.log("\t| Transport Encryption : "+str(message.transport_encryption)) + RNS.log("\t| Timestamp : "+time_string) + RNS.log("\t| Title : "+message.title_as_string()) + RNS.log("\t| Content : "+message.content_as_string()) + RNS.log("\t| Fields : "+str(message.fields)) + RNS.log("\t| Message signature : "+signature_string) + RNS.log("\t+---------------------------------------------------------------") + +r = RNS.Reticulum() + +router = LXMF.LXMRouter(storagepath="./tmp1") +identity = RNS.Identity() +my_lxmf_destination = router.register_delivery_identity(identity) +router.register_delivery_callback(delivery_callback) + +RNS.log("Ready to receive on: "+RNS.prettyhexrep(my_lxmf_destination.hash)) +input() \ No newline at end of file diff --git a/docs/example_sender.py b/docs/example_sender.py new file mode 100644 index 0000000..ab15a16 --- /dev/null +++ b/docs/example_sender.py @@ -0,0 +1,39 @@ +import LXMF +import RNS +import time +import random + +random_names = ["Tom", "Delilah", "Nancey", "Williams", "Neomi", "Curtis", "Alexa", "Theodora", "Ted", "Dinorah", "Nicol", "Drusilla", "Annalisa", "Verlene", "Latesha", "Tina", "Mia", "Brock", "Timothy", "Philip", "Willian", "Reyna", "Simona", "Mimi", "Stanford", "Ferne", "Catalina", "Lucie", "Jaye", "Natasha", "Willetta", "Isabel", "Esperanza", "Ciara", "Eusebio", "William", "Elma", "Angelica", "Coreen", "Melani", "Jonathan", "Maryland", "Caroline", "Gregg", "Ora", "Jacqui", "Letty", "Roselle", "Oralee", "Angla"] +random_titles = ["Long time", "Hi again", "Re: Hi there", "Test message", "", "", "Something different"] +random_msgs = ["If wishes were horses then beggars might fly. Stuff like that. It's enough to drive you crazy.", "'My ident cards were stolen,' Jason said. 'That fivehundred-dollar bill is yours if you can get me to someone who can replace them. If you're going to do it, do it right now; I'm not going to wait.' Wait to be picked up by a pol or a nat, he thought. Caught here in this rundown dingy hotel.", "A six, no matter what the external circumstances, will always prevail. Because that's the way they genetically defined us.", "'Should be there in an hour,' he called back over his shoulder to Chuck. Then he added, in an afterthought, 'Wonder if the computer’s finished its run. It was due about now.'. Chuck didn’t reply, so George swung round in his saddle. He could just see Chuck’s face, a white oval turned toward the sky."] + +def delivery_callback(message): + pass + +r = RNS.Reticulum() +router = LXMF.LXMRouter(storagepath="./tmp2") +router.register_delivery_callback(delivery_callback) +ident = RNS.Identity() +source = router.register_delivery_identity(ident, display_name=random_names[random.randint(0,len(random_names)-1)]) +router.announce(source.hash) +RNS.log("Source announced") + +print("Recipient: ", end=" ") +recipient_hexhash = input() +recipient_hash = bytes.fromhex(recipient_hexhash) + +if not RNS.Transport.has_path(recipient_hash): + RNS.log("Destination is not yet known. Requesting path and waiting for announce to arrive...") + RNS.Transport.request_path(recipient_hash) + while not RNS.Transport.has_path(recipient_hash): + time.sleep(0.1) + +# Recall the server identity +recipient_identity = RNS.Identity.recall(recipient_hash) + +dest = RNS.Destination(recipient_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, "lxmf", "delivery") + +while True: + lxm = LXMF.LXMessage(dest, source, random_msgs[random.randint(0,len(random_msgs)-1)], random_titles[random.randint(0,len(random_titles)-1)], desired_method=LXMF.LXMessage.DIRECT) + router.handle_outbound(lxm) + input() \ No newline at end of file From 6ed16916d645a84e7aa623c904fcfbeccf1940ef Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 7 Jan 2024 16:51:30 +0100 Subject: [PATCH 163/436] Fixed get fields method --- LXMF/LXMessage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 5c59202..0fe1ccf 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -168,7 +168,7 @@ class LXMessage: raise ValueError("LXMessage property \"fields\" can only be dict or None") def get_fields(self): - return self.__fields + return self.fields def set_destination(self, destination): if self.destination == None: From 87413b93555624084a40e1eb97449870486a2595 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 15 Jan 2024 20:29:29 +0100 Subject: [PATCH 164/436] Updated readme --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 11451a8..2f5c888 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,14 @@ User-facing clients built on LXMF include: - [Sideband](https://unsigned.io/sideband) - [Nomad Network](https://unsigned.io/nomadnet) -- [Nexus Messenger](https://github.com/HarlekinSimplex/nexus_messenger) + +Community-provided tools and utilities for LXMF include: + +- [LXMF-Bot](https://github.com/randogoth/lxmf-bot) +- [LXMF Messageboard](https://github.com/chengtripp/lxmf_messageboard) +- [LXMEvent](https://github.com/faragher/LXMEvent) +- [RangeMap](https://github.com/faragher/RangeMap) +- [LXMF Tools](https://github.com/SebastianObi/LXMF-Tools) ## Structure From 7a1bcc134bfe09e7b4ceac3f401c13fee7505354 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 15 Jan 2024 20:29:39 +0100 Subject: [PATCH 165/436] Updated dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5c17e3b..4f5654e 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.6.7'], + install_requires=['rns>=0.7.0'], python_requires='>=3.7', ) From f0e9fa66598b31e3ae4bcc9f188bb63bc298ba5e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 16 Jan 2024 13:26:33 +0100 Subject: [PATCH 166/436] Corrected inconsistent property, getter and setter use. Fixed #16. --- LXMF/LXMessage.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 0fe1ccf..3cf8122 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -170,6 +170,17 @@ class LXMessage: def get_fields(self): return self.fields + @property + def destination(self): + return self.__destination + + @destination.setter + def destination(self, destination): + self.set_destination(destination) + + def get_destination(self): + return self.destination + def set_destination(self, destination): if self.destination == None: if isinstance(destination, RNS.Destination): @@ -179,8 +190,16 @@ class LXMessage: else: raise ValueError("Cannot reassign destination on LXMessage") - def get_destination(self): - return self.__destination + @property + def source(self): + return self.__source + + @source.setter + def source(self, source): + self.set_source(source) + + def get_source(self): + return self.source def set_source(self, source): if self.source == None: @@ -191,9 +210,6 @@ class LXMessage: else: raise ValueError("Cannot reassign source on LXMessage") - def get_source(self): - return self.__source - def set_delivery_destination(self, delivery_destination): self.__delivery_destination = delivery_destination From af2dd5670778e99cb13ba8346288378d0ca044f3 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 29 Feb 2024 18:27:19 +0100 Subject: [PATCH 167/436] Added issue templates --- .github/ISSUE_TEMPLATE/config.yml | 11 ++++++++ .github/ISSUE_TEMPLATE/🐛-bug-report.md | 35 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/🐛-bug-report.md diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..caf3250 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +blank_issues_enabled: false +contact_links: + - name: ✨ Feature Request or Idea + url: https://github.com/markqvist/Reticulum/discussions/new?category=ideas + about: Propose and discuss features and ideas + - name: 💬 Questions, Help & Discussion + about: Ask anything, or get help + url: https://github.com/markqvist/Reticulum/discussions/new/choose + - name: 📖 Read the Reticulum Manual + url: https://markqvist.github.io/Reticulum/manual/ + about: The complete documentation for Reticulum diff --git a/.github/ISSUE_TEMPLATE/🐛-bug-report.md b/.github/ISSUE_TEMPLATE/🐛-bug-report.md new file mode 100644 index 0000000..77ad6c2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/🐛-bug-report.md @@ -0,0 +1,35 @@ +--- +name: "\U0001F41B Bug Report" +about: Report a reproducible bug +title: '' +labels: '' +assignees: '' + +--- + +**Read the Contribution Guidelines** +Before creating a bug report on this issue tracker, you **must** read the [Contribution Guidelines](https://github.com/markqvist/Reticulum/blob/master/Contributing.md). Issues that do not follow the contribution guidelines **will be deleted without comment**. + +- The issue tracker is used by developers of this project. **Do not use it to ask general questions, or for support requests**. +- Ideas and feature requests can be made on the [Discussions](https://github.com/markqvist/Reticulum/discussions). **Only** feature requests accepted by maintainers and developers are tracked and included on the issue tracker. **Do not post feature requests here**. +- After reading the [Contribution Guidelines](https://github.com/markqvist/Reticulum/blob/master/Contributing.md), delete this section from your bug report. + +**Describe the Bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Describe in detail how to reproduce the bug. + +**Expected Behavior** +A clear and concise description of what you expected to happen. + +**Logs & Screenshots** +Please include any relevant log output. If applicable, also add screenshots to help explain your problem. + +**System Information** +- OS and version +- Python version +- Program version + +**Additional context** +Add any other context about the problem here. From 35dc77152828cc537f3a9484617ebc6df32b519a Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 29 Feb 2024 20:12:54 +0100 Subject: [PATCH 168/436] Don't keep links alive after sync --- LXMF/LXMPeer.py | 19 ++++++++++++------- LXMF/LXMRouter.py | 3 ++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 611a874..17c0344 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -224,6 +224,10 @@ class LXMPeer: self.state = LXMPeer.RESOURCE_TRANSFERRING else: RNS.log("Peer "+RNS.prettyhexrep(self.destination_hash)+" did not request any of the available messages, sync completed", RNS.LOG_DEBUG) + if self.link != None: + self.link.teardown() + + self.link = None self.state = LXMPeer.IDLE except Exception as e: @@ -236,25 +240,28 @@ class LXMPeer: self.link = None self.state = LXMPeer.IDLE - def resource_concluded(self, resource): if resource.status == RNS.Resource.COMPLETE: for transient_id in resource.transferred_messages: message = self.unhandled_messages.pop(transient_id) self.handled_messages[transient_id] = message + + if self.link != None: + self.link.teardown() + + self.link = None self.state = LXMPeer.IDLE - self.link.teardown() + RNS.log("Sync to peer "+RNS.prettyhexrep(self.destination_hash)+" completed", RNS.LOG_DEBUG) self.alive = True self.last_heard = time.time() + else: RNS.log("Resource transfer for LXMF peer sync failed to "+str(self.destination), RNS.LOG_DEBUG) if self.link != None: self.link.teardown() - else: - self.state = LXMPeer.IDLE - + self.state = LXMPeer.IDLE def link_established(self, link): self.link.identify(self.router.identity) @@ -272,8 +279,6 @@ class LXMPeer: def handle_message(self, transient_id): if not transient_id in self.handled_messages and not transient_id in self.unhandled_messages: - # TODO: Remove at some point - RNS.log("The message "+RNS.prettyhexrep(transient_id)+" was added to distribution queue for "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_EXTREME) self.unhandled_messages[transient_id] = self.router.propagation_entries[transient_id] def __str__(self): diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 06e9abf..e9a4836 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -653,7 +653,8 @@ class LXMRouter: job_thread.start() def __request_messages_path_job(self): - while not RNS.Transport.has_path(self.wants_download_on_path_available_from) and time.time() < self.wants_download_on_path_available_timeout: + path_timeout = self.wants_download_on_path_available_timeout + while not RNS.Transport.has_path(self.wants_download_on_path_available_from) and time.time() < path_timeout: time.sleep(0.1) if RNS.Transport.has_path(self.wants_download_on_path_available_from): From 7aea4ea209911641c476e9f9903af6a6df7fee86 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 29 Feb 2024 20:15:03 +0100 Subject: [PATCH 169/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 771bc6e..6a9beea 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.3.9" +__version__ = "0.4.0" From 696c78ecea794d1f5728b49c00b97c6d89117667 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 29 Feb 2024 23:02:16 +0100 Subject: [PATCH 170/436] Improved propagation node sync and memory consumption --- LXMF/LXMPeer.py | 8 ++++++-- LXMF/LXMRouter.py | 27 +++++++++++++++++++++------ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 17c0344..97fb747 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -148,8 +148,10 @@ class LXMPeer: RNS.log("Sending sync request to peer "+str(self.destination), RNS.LOG_DEBUG) self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) self.state = LXMPeer.REQUEST_SENT + else: RNS.log("Could not request sync to peer "+RNS.prettyhexrep(self.destination_hash)+" since its identity could not be recalled.", RNS.LOG_ERROR) + else: RNS.log("Postponing sync with peer "+RNS.prettyhexrep(self.destination_hash)+" for "+RNS.prettytime(self.next_sync_attempt-time.time())+" due to previous failures", RNS.LOG_DEBUG) if self.last_sync_attempt > self.last_heard: @@ -159,8 +161,8 @@ class LXMPeer: RNS.log("Sync request to peer "+str(self.destination)+" failed", RNS.LOG_DEBUG) if self.link != None: self.link.teardown() - else: - self.state = LXMPeer.IDLE + + self.state = LXMPeer.IDLE def offer_response(self, request_receipt): try: @@ -222,6 +224,7 @@ class LXMPeer: resource = RNS.Resource(data, self.link, callback = self.resource_concluded) resource.transferred_messages = wanted_message_ids self.state = LXMPeer.RESOURCE_TRANSFERRING + else: RNS.log("Peer "+RNS.prettyhexrep(self.destination_hash)+" did not request any of the available messages, sync completed", RNS.LOG_DEBUG) if self.link != None: @@ -261,6 +264,7 @@ class LXMPeer: if self.link != None: self.link.teardown() + self.link = None self.state = LXMPeer.IDLE def link_established(self, link): diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index e9a4836..04263a0 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -21,6 +21,7 @@ class LXMRouter: DELIVERY_RETRY_WAIT = 7 PATH_REQUEST_WAIT = 7 LINK_MAX_INACTIVITY = 10*60 + P_LINK_MAX_INACTIVITY = 3*60 MESSAGE_EXPIRY = 30*24*60*60 @@ -90,6 +91,7 @@ class LXMRouter: self.propagation_transfer_progress = 0.0 self.propagation_transfer_last_result = None self.propagation_transfer_max_messages = None + self.active_propagation_links = [] self.locally_delivered_transient_ids = {} self.locally_processed_transient_ids = {} @@ -471,6 +473,19 @@ class LXMRouter: cleaned_link = self.direct_links.pop(link_hash) RNS.log("Cleaned link "+str(cleaned_link), RNS.LOG_DEBUG) + try: + inactive_links = [] + for link in self.active_propagation_links: + if link.no_data_for() > LXMRouter.P_LINK_MAX_INACTIVITY: + inactive_links.append(link) + + for link in inactive_links: + self.active_propagation_links.remove(link) + link.teardown() + + except Exception as e: + RNS.log("An error occurred while cleaning inbound propagation links. The contained exception was: "+str(e), RNS.LOG_ERROR) + if self.outbound_propagation_link != None and self.outbound_propagation_link.status == RNS.Link.CLOSED: self.outbound_propagation_link = None if self.propagation_transfer_state == LXMRouter.PR_COMPLETE: @@ -953,9 +968,8 @@ class LXMRouter: peer.alive = True peer.sync_backoff = 0 peer.next_sync_attempt = 0 - - peer.peering_timebase = timestamp - peer.last_heard = time.time() + peer.peering_timebase = timestamp + peer.last_heard = time.time() else: peer = LXMPeer(self, destination_hash) @@ -1038,6 +1052,7 @@ class LXMRouter: link.set_resource_strategy(RNS.Link.ACCEPT_ALL) link.set_resource_started_callback(self.resource_transfer_began) link.set_resource_concluded_callback(self.propagation_resource_concluded) + self.active_propagation_links.append(link) def propagation_packet(self, data, packet): try: @@ -1057,7 +1072,7 @@ class LXMRouter: RNS.log("Exception occurred while parsing incoming LXMF propagation data.", RNS.LOG_ERROR) RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) - def offer_request(self, path, data, request_id, remote_identity, requested_at): + def offer_request(self, path, data, request_id, link_id, remote_identity, requested_at): if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY else: @@ -1086,7 +1101,7 @@ class LXMRouter: RNS.log("Transfer concluded for incoming propagation resource "+str(resource), RNS.LOG_DEBUG) if resource.status == RNS.Resource.COMPLETE: # TODO: The peer this was received from should - # have the transient id added to it's list of + # have the transient id added to its list of # already handled messages. try: data = msgpack.unpackb(resource.data.read()) @@ -1326,7 +1341,7 @@ class LXMRouter: else: # Simply wait for the link to become # active or close - RNS.log("The propagation link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" is pending, waiting for link to become active: "+str(self.outbound_propagation_link.status), RNS.LOG_DEBUG) + RNS.log("The propagation link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" is pending, waiting for link to become active", RNS.LOG_DEBUG) else: # No link exists, so we'll try to establish one, but # only if we've never tried before, or the retry wait From b3bc8e684eeab86b795f6697ae513119eb0a112c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 29 Feb 2024 23:03:02 +0100 Subject: [PATCH 171/436] Updated dependency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4f5654e..b66db9d 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.7.0'], + install_requires=['rns>=0.7.1'], python_requires='>=3.7', ) From bb1b9e4163f72af95f197a5b98bf0760752120c7 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 1 Mar 2024 00:33:42 +0100 Subject: [PATCH 172/436] Updated dependency version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b66db9d..26ce0d7 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.7.1'], + install_requires=['rns>=0.7.2'], python_requires='>=3.7', ) From c7489dc0fa74b91e9b0cd4f2c6a2183ba8f8e031 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 1 Mar 2024 22:33:53 +0100 Subject: [PATCH 173/436] Added transfer limit handling to announce handler --- LXMF/Handlers.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index 32e045e..571a59f 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -34,11 +34,17 @@ class LXMFPropagationAnnounceHandler: data = msgpack.unpackb(app_data) if self.lxmrouter.propagation_node and self.lxmrouter.autopeer: + node_timebase = data[1] + propagation_transfer_limit = None + if len(data) >= 3: + propagation_transfer_limit = data[2] + if data[0] == True: if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: - self.lxmrouter.peer(destination_hash, data[1]) + self.lxmrouter.peer(destination_hash, node_timebase, propagation_transfer_limit) + elif data[0] == False: - self.lxmrouter.unpeer(destination_hash, data[1]) + self.lxmrouter.unpeer(destination_hash, node_timebase) except Exception as e: RNS.log("Error while evaluating propagation node announce, ignoring announce.", RNS.LOG_DEBUG) From 64050d39bfa6c0dafe15c6b0fd2bd2f0bbd40549 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 1 Mar 2024 22:37:54 +0100 Subject: [PATCH 174/436] Added propagation transfer limit options --- LXMF/Handlers.py | 5 ++- LXMF/LXMPeer.py | 54 +++++++++++++++++------ LXMF/LXMRouter.py | 99 +++++++++++++++++++++++++++--------------- LXMF/Utilities/lxmd.py | 50 +++++++++++++++++++++ 4 files changed, 160 insertions(+), 48 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index 571a59f..c09bf1c 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -37,7 +37,10 @@ class LXMFPropagationAnnounceHandler: node_timebase = data[1] propagation_transfer_limit = None if len(data) >= 3: - propagation_transfer_limit = data[2] + try: + propagation_transfer_limit = float(data[2]) + except: + propagation_transfer_limit = None if data[0] == True: if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 97fb747..5ee8986 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -48,6 +48,14 @@ class LXMPeer: else: peer.link_establishment_rate = 0 + if "propagation_transfer_limit" in dictionary: + try: + peer.propagation_transfer_limit = float(dictionary["propagation_transfer_limit"]) + except Exception as e: + peer.propagation_transfer_limit = None + else: + peer.propagation_transfer_limit = None + for transient_id in dictionary["handled_ids"]: if transient_id in router.propagation_entries: peer.handled_messages[transient_id] = router.propagation_entries[transient_id] @@ -65,6 +73,7 @@ class LXMPeer: dictionary["last_heard"] = self.last_heard dictionary["destination_hash"] = self.destination_hash dictionary["link_establishment_rate"] = self.link_establishment_rate + dictionary["propagation_transfer_limit"] = self.propagation_transfer_limit handled_ids = [] for transient_id in self.handled_messages: @@ -87,12 +96,14 @@ class LXMPeer: self.sync_backoff = 0 self.peering_timebase = 0 self.link_establishment_rate = 0 + self.propagation_transfer_limit = None self.link = None self.state = LXMPeer.IDLE self.unhandled_messages = {} self.handled_messages = {} + self.last_offer = [] self.router = router self.destination_hash = destination_hash @@ -133,11 +144,17 @@ class LXMPeer: self.sync_backoff = 0 RNS.log("Synchronisation link to peer "+RNS.prettyhexrep(self.destination_hash)+" established, preparing request...", RNS.LOG_DEBUG) + unhandled_entries = [] unhandled_ids = [] purged_ids = [] for transient_id in self.unhandled_messages: if transient_id in self.router.propagation_entries: - unhandled_ids.append(transient_id) + unhandled_entry = [ + transient_id, + self.router.get_weight(transient_id), + self.router.get_size(transient_id), + ] + unhandled_entries.append(unhandled_entry) else: purged_ids.append(transient_id) @@ -145,8 +162,21 @@ class LXMPeer: RNS.log("Dropping unhandled message "+RNS.prettyhexrep(transient_id)+" for peer "+RNS.prettyhexrep(self.destination_hash)+" since it no longer exists in the message store.", RNS.LOG_DEBUG) self.unhandled_messages.pop(transient_id) + unhandled_entries.sort(key=lambda e: e[1], reverse=False) + cumulative_size = 0 + for unhandled_entry in unhandled_entries: + transient_id = unhandled_entry[0] + weight = unhandled_entry[1] + lxm_size = unhandled_entry[2] + if self.propagation_transfer_limit != None and cumulative_size + lxm_size > (self.propagation_transfer_limit*1000): + pass + else: + cumulative_size += lxm_size + unhandled_ids.append(transient_id) + RNS.log("Sending sync request to peer "+str(self.destination), RNS.LOG_DEBUG) - self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) + self.last_offer = unhandled_ids + self.link.request(LXMPeer.OFFER_REQUEST_PATH, self.last_offer, response_callback=self.offer_response, failed_callback=self.request_failed) self.state = LXMPeer.REQUEST_SENT else: @@ -175,33 +205,31 @@ class LXMPeer: if response == LXMPeer.ERROR_NO_IDENTITY: if self.link != None: RNS.log("Remote peer indicated that no identification was received, retrying...", RNS.LOG_DEBUG) - self.link.indentify() + self.link.identify() self.state = LXMPeer.LINK_READY self.sync() elif response == False: # Peer already has all advertised messages - for transient_id in self.unhandled_messages: - message_entry = self.unhandled_messages[transient_id] - self.handled_messages[transient_id] = message_entry - - self.unhandled_messages = {} + for transient_id in self.last_offer: + if transient_id in self.unhandled_messages: + self.handled_messages[transient_id] = self.unhandled_messages.pop(transient_id) + elif response == True: # Peer wants all advertised messages - for transient_id in self.unhandled_messages: + for transient_id in self.last_offer: wanted_messages.append(self.unhandled_messages[transient_id]) wanted_message_ids.append(transient_id) else: # Peer wants some advertised messages - peer_had_messages = [] - for transient_id in self.unhandled_messages.copy(): + for transient_id in self.last_offer.copy(): # If the peer did not want the message, it has # already received it from another peer. if not transient_id in response: - message_entry = self.unhandled_messages.pop(transient_id) - self.handled_messages[transient_id] = message_entry + if transient_id in self.unhandled_messages: + self.handled_messages[transient_id] = self.unhandled_messages.pop(transient_id) for transient_id in response: wanted_messages.append(self.unhandled_messages[transient_id]) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 04263a0..5141bc6 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -31,30 +31,35 @@ class LXMRouter: AUTOPEER_MAXDEPTH = 4 FASTEST_N_RANDOM_POOL = 2 - PR_PATH_TIMEOUT = 10 + PROPAGATION_LIMIT = 256 + DELIVERY_LIMIT = 1024 - PR_IDLE = 0x00 - PR_PATH_REQUESTED = 0x01 - PR_LINK_ESTABLISHING = 0x02 - PR_LINK_ESTABLISHED = 0x03 - PR_REQUEST_SENT = 0x04 - PR_RECEIVING = 0x05 - PR_RESPONSE_RECEIVED = 0x06 - PR_COMPLETE = 0x07 - PR_NO_PATH = 0xf0 - PR_LINK_FAILED = 0xf1 - PR_TRANSFER_FAILED = 0xf2 - PR_NO_IDENTITY_RCVD = 0xf3 - PR_NO_ACCESS = 0xf4 - PR_FAILED = 0xfe + PR_PATH_TIMEOUT = 10 - PR_ALL_MESSAGES = 0x00 + PR_IDLE = 0x00 + PR_PATH_REQUESTED = 0x01 + PR_LINK_ESTABLISHING = 0x02 + PR_LINK_ESTABLISHED = 0x03 + PR_REQUEST_SENT = 0x04 + PR_RECEIVING = 0x05 + PR_RESPONSE_RECEIVED = 0x06 + PR_COMPLETE = 0x07 + PR_NO_PATH = 0xf0 + PR_LINK_FAILED = 0xf1 + PR_TRANSFER_FAILED = 0xf2 + PR_NO_IDENTITY_RCVD = 0xf3 + PR_NO_ACCESS = 0xf4 + PR_FAILED = 0xfe + + PR_ALL_MESSAGES = 0x00 ### Developer-facing API ############################## ####################################################### - def __init__(self, identity = None, storagepath = None, autopeer = AUTOPEER, autopeer_maxdepth = None): + def __init__(self, identity = None, storagepath = None, autopeer = AUTOPEER, autopeer_maxdepth = None, + propagation_limit = PROPAGATION_LIMIT, delivery_limit = DELIVERY_LIMIT): + random.seed(os.urandom(10)) self.pending_inbound = [] @@ -84,6 +89,8 @@ class LXMRouter: self.message_storage_limit = None self.information_storage_limit = None + self.propagation_per_transfer_limit = propagation_limit + self.delivery_per_transfer_limit = delivery_limit self.wants_download_on_path_available_from = None self.wants_download_on_path_available_to = None @@ -152,7 +159,13 @@ class LXMRouter: def announce_propagation_node(self): def delayed_announce(): time.sleep(LXMRouter.NODE_ANNOUNCE_DELAY) - data = msgpack.packb([self.propagation_node, int(time.time())]) + announce_data = [ + self.propagation_node, # Boolean flag signalling propagation node state + int(time.time()), # Current node timebase + self.propagation_per_transfer_limit, # Per-transfer limit for message propagation in kilobytes + ] + + data = msgpack.packb(announce_data) self.propagation_destination.announce(app_data=data) da_thread = threading.Thread(target=delayed_announce) @@ -319,7 +332,10 @@ class LXMRouter: peer = LXMPeer.from_bytes(serialised_peer, self) if peer.identity != None: self.peers[peer.destination_hash] = peer - RNS.log("Loaded peer "+RNS.prettyhexrep(peer.destination_hash)+" with "+str(len(peer.unhandled_messages))+" unhandled messages", RNS.LOG_DEBUG) + lim_str = ", no transfer limit" + if peer.propagation_transfer_limit != None: + lim_str = ", "+RNS.prettysize(peer.propagation_transfer_limit*1000)+" transfer limit" + RNS.log("Loaded peer "+RNS.prettyhexrep(peer.destination_hash)+" with "+str(len(peer.unhandled_messages))+" unhandled messages"+lim_str, RNS.LOG_DEBUG) else: RNS.log("Peer "+RNS.prettyhexrep(peer.destination_hash)+" could not be loaded, because its identity could not be recalled. Dropping peer.", RNS.LOG_DEBUG) @@ -522,6 +538,28 @@ class LXMRouter: self.locally_processed_transient_ids.pop(transient_id) RNS.log("Cleaned "+RNS.prettyhexrep(transient_id)+" from locally processed cache", RNS.LOG_DEBUG) + def get_weight(self, transient_id): + dst_hash = self.propagation_entries[transient_id][0] + lxm_rcvd = self.propagation_entries[transient_id][2] + lxm_size = self.propagation_entries[transient_id][3] + + now = time.time() + age_weight = max(1, (now - lxm_rcvd)/60/60/24/4) + + if dst_hash in self.prioritised_list: + priority_weight = 0.1 + else: + priority_weight = 1.0 + + weight = priority_weight * age_weight * lxm_size + + return weight + + def get_size(self, transient_id): + lxm_size = self.propagation_entries[transient_id][3] + return lxm_size + + def clean_message_store(self): # Check and remove expired messages now = time.time() @@ -563,22 +601,13 @@ class LXMRouter: bytes_needed = message_storage_size - self.message_storage_limit bytes_cleaned = 0 - now = time.time() weighted_entries = [] for transient_id in self.propagation_entries: - entry = self.propagation_entries[transient_id] - - dst_hash = entry[0] - lxm_rcvd = entry[2] - lxm_size = entry[3] - age_weight = max(1, (now - lxm_rcvd)/60/60/24/4) - if dst_hash in self.prioritised_list: - priority_weight = 0.1 - else: - priority_weight = 1.0 - - weight = priority_weight * age_weight * lxm_size - weighted_entries.append([entry, weight, transient_id]) + weighted_entries.append([ + self.propagation_entries[transient_id], + self.get_weight(transient_id), + transient_id + ]) weighted_entries.sort(key=lambda we: we[1], reverse=True) @@ -961,7 +990,7 @@ class LXMRouter: ### Peer Sync & Propagation ########################### ####################################################### - def peer(self, destination_hash, timestamp): + def peer(self, destination_hash, timestamp, propagation_transfer_limit): if destination_hash in self.peers: peer = self.peers[destination_hash] if timestamp > peer.peering_timebase: @@ -970,11 +999,13 @@ class LXMRouter: peer.next_sync_attempt = 0 peer.peering_timebase = timestamp peer.last_heard = time.time() + peer.propagation_transfer_limit = propagation_transfer_limit else: peer = LXMPeer(self, destination_hash) peer.alive = True peer.last_heard = time.time() + peer.propagation_transfer_limit = propagation_transfer_limit self.peers[destination_hash] = peer RNS.log("Peered with "+str(peer.destination)) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 3dc34df..3a76dbb 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -77,6 +77,13 @@ def apply_config(): active_configuration["peer_announce_interval"] = lxmd_config["lxmf"].as_int("announce_interval")*60 else: active_configuration["peer_announce_interval"] = None + + if "lxmf" in lxmd_config and "delivery_transfer_max_accepted_size" in lxmd_config["lxmf"]: + active_configuration["delivery_transfer_max_accepted_size"] = lxmd_config["lxmf"].as_float("delivery_transfer_max_accepted_size") + if active_configuration["delivery_transfer_max_accepted_size"] < 0.38: + active_configuration["delivery_transfer_max_accepted_size"] = 0.38 + else: + active_configuration["delivery_transfer_max_accepted_size"] = 1024 if "lxmf" in lxmd_config and "on_inbound" in lxmd_config["lxmf"]: active_configuration["on_inbound"] = lxmd_config["lxmf"]["on_inbound"] @@ -121,6 +128,13 @@ def apply_config(): else: active_configuration["message_storage_limit"] = 2000 + if "propagation" in lxmd_config and "propagation_transfer_max_accepted_size" in lxmd_config["propagation"]: + active_configuration["propagation_transfer_max_accepted_size"] = lxmd_config["propagation"].as_float("propagation_transfer_max_accepted_size") + if active_configuration["propagation_transfer_max_accepted_size"] < 0.38: + active_configuration["propagation_transfer_max_accepted_size"] = 0.38 + else: + active_configuration["propagation_transfer_max_accepted_size"] = 256 + if "propagation" in lxmd_config and "prioritise_destinations" in lxmd_config["propagation"]: active_configuration["prioritised_lxmf_destinations"] = lxmd_config["propagation"].as_list("prioritise_destinations") else: @@ -289,6 +303,8 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo storagepath = storagedir, autopeer = active_configuration["autopeer"], autopeer_maxdepth = active_configuration["autopeer_maxdepth"], + propagation_limit = active_configuration["propagation_transfer_max_accepted_size"], + delivery_limit = active_configuration["delivery_transfer_max_accepted_size"], ) message_router.register_delivery_callback(lxmf_delivery) @@ -418,23 +434,41 @@ __default_lxmd_config__ = """# This is an example LXM Daemon config file. [propagation] # Whether to enable propagation node + enable_node = no # Automatic announce interval in minutes. # 6 hours by default. + announce_interval = 360 # Whether to announce when the node starts. + announce_at_start = yes # Wheter to automatically peer with other # propagation nodes on the network. + autopeer = yes # The maximum peering depth (in hops) for # automatically peered nodes. + autopeer_maxdepth = 4 +# The maximum accepted transfer size per in- +# coming propagation transfer, in kilobytes. +# This also sets the upper limit for the size +# of single messages accepted onto this node. +# +# If a node wants to propagate a larger number +# of messages to this node, than what can fit +# within this limit, it will prioritise sending +# the smallest messages first, and try again +# with any remaining messages at a later point. + +propagation_transfer_max_accepted_size = 256 + # The maximum amount of storage to use for # the LXMF Propagation Node message store, # specified in megabytes. When this limit @@ -444,6 +478,7 @@ autopeer_maxdepth = 4 # new and small. Large and old messages will # be removed first. This setting is optional # and defaults to 2 gigabytes. + # message_storage_limit = 2000 # You can tell the LXMF message router to @@ -453,6 +488,7 @@ autopeer_maxdepth = 4 # keeping messages for destinations specified # with this option. This setting is optional, # and generally you do not need to use it. + # prioritise_destinations = 41d20c727598a3fbbdf9106133a3a0ed, d924b81822ca24e68e2effea99bcb8cf # By default, any destination is allowed to @@ -461,6 +497,7 @@ autopeer_maxdepth = 4 # authentication, you must provide a list of # allowed identity hashes in the a file named # "allowed" in the lxmd config directory. + auth_required = no @@ -469,23 +506,35 @@ auth_required = no # The LXM Daemon will create an LXMF destination # that it can receive messages on. This option sets # the announced display name for this destination. + display_name = Anonymous Peer # It is possible to announce the internal LXMF # destination when the LXM Daemon starts up. + announce_at_start = no # You can also announce the delivery destination # at a specified interval. This is not enabled by # default. + # announce_interval = 360 +# The maximum accepted unpacked size for mes- +# sages received directly from other peers, +# specified in kilobytes. Messages larger than +# this will be rejected before the transfer +# begins. + +delivery_transfer_max_accepted_size = 1024 + # You can configure an external program to be run # every time a message is received. The program # will receive as an argument the full path to the # message saved as a file. The example below will # simply result in the message getting deleted as # soon as it has been received. + # on_inbound = rm @@ -499,6 +548,7 @@ announce_at_start = no # 5: Verbose logging # 6: Debug logging # 7: Extreme logging + loglevel = 4 """ From 7ecd3c0d5e70e98f8202db77d0da22ebd99b7482 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 1 Mar 2024 23:26:27 +0100 Subject: [PATCH 175/436] Added incoming resource transfer size limits --- LXMF/LXMRouter.py | 26 +++++++++++++++++++++++--- LXMF/Utilities/lxmd.py | 4 ++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 5141bc6..cedc2ea 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -32,7 +32,7 @@ class LXMRouter: FASTEST_N_RANDOM_POOL = 2 PROPAGATION_LIMIT = 256 - DELIVERY_LIMIT = 1024 + DELIVERY_LIMIT = 1000 PR_PATH_TIMEOUT = 10 @@ -970,7 +970,8 @@ class LXMRouter: def delivery_link_established(self, link): link.track_phy_stats(True) link.set_packet_callback(self.delivery_packet) - link.set_resource_strategy(RNS.Link.ACCEPT_ALL) + link.set_resource_strategy(RNS.Link.ACCEPT_APP) + link.set_resource_callback(self.delivery_resource_advertised) link.set_resource_started_callback(self.resource_transfer_began) link.set_resource_concluded_callback(self.delivery_resource_concluded) @@ -980,6 +981,15 @@ class LXMRouter: def resource_transfer_began(self, resource): RNS.log("Transfer began for LXMF delivery resource "+str(resource), RNS.LOG_DEBUG) + def delivery_resource_advertised(self, resource): + size = resource.get_data_size() + limit = self.delivery_per_transfer_limit*1000 + if limit != None and size > limit: + RNS.log("Rejecting "+RNS.prettysize(size)+" incoming LXMF delivery resource, since it exceeds the limit of "+RNS.prettysize(limit), RNS.LOG_DEBUG) + return False + else: + return True + def delivery_resource_concluded(self, resource): RNS.log("Transfer concluded for LXMF delivery resource "+str(resource), RNS.LOG_DEBUG) if resource.status == RNS.Resource.COMPLETE: @@ -1080,11 +1090,21 @@ class LXMRouter: def propagation_link_established(self, link): link.set_packet_callback(self.propagation_packet) - link.set_resource_strategy(RNS.Link.ACCEPT_ALL) + link.set_resource_strategy(RNS.Link.ACCEPT_APP) + link.set_resource_callback(self.propagation_resource_advertised) link.set_resource_started_callback(self.resource_transfer_began) link.set_resource_concluded_callback(self.propagation_resource_concluded) self.active_propagation_links.append(link) + def propagation_resource_advertised(self, resource): + size = resource.get_data_size() + limit = self.propagation_per_transfer_limit*1000 + if limit != None and size > limit: + RNS.log("Rejecting "+RNS.prettysize(size)+" incoming LXMF propagation resource, since it exceeds the limit of "+RNS.prettysize(limit), RNS.LOG_DEBUG) + return False + else: + return True + def propagation_packet(self, data, packet): try: if packet.destination_type != RNS.Destination.LINK: diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 3a76dbb..38e71b1 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -83,7 +83,7 @@ def apply_config(): if active_configuration["delivery_transfer_max_accepted_size"] < 0.38: active_configuration["delivery_transfer_max_accepted_size"] = 0.38 else: - active_configuration["delivery_transfer_max_accepted_size"] = 1024 + active_configuration["delivery_transfer_max_accepted_size"] = 1000 if "lxmf" in lxmd_config and "on_inbound" in lxmd_config["lxmf"]: active_configuration["on_inbound"] = lxmd_config["lxmf"]["on_inbound"] @@ -526,7 +526,7 @@ announce_at_start = no # this will be rejected before the transfer # begins. -delivery_transfer_max_accepted_size = 1024 +delivery_transfer_max_accepted_size = 1000 # You can configure an external program to be run # every time a message is received. The program From 22493005dc33531d1df0ff93de6dfddb0ebb5144 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 1 Mar 2024 23:48:12 +0100 Subject: [PATCH 176/436] Overhead calculation for propagation transfer limits --- LXMF/LXMPeer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 5ee8986..b1a49af 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -163,7 +163,8 @@ class LXMPeer: self.unhandled_messages.pop(transient_id) unhandled_entries.sort(key=lambda e: e[1], reverse=False) - cumulative_size = 0 + per_message_overhead = 16 # Really only 2 bytes, but set a bit higher for now + cumulative_size = 24 # Initialised to highest reasonable binary structure overhead for unhandled_entry in unhandled_entries: transient_id = unhandled_entry[0] weight = unhandled_entry[1] @@ -171,7 +172,7 @@ class LXMPeer: if self.propagation_transfer_limit != None and cumulative_size + lxm_size > (self.propagation_transfer_limit*1000): pass else: - cumulative_size += lxm_size + cumulative_size += (lxm_size+per_message_overhead) unhandled_ids.append(transient_id) RNS.log("Sending sync request to peer "+str(self.destination), RNS.LOG_DEBUG) From 1d651a9b532bacfc11efe51d002aa8f39ce7e80d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 2 Mar 2024 09:09:51 +0100 Subject: [PATCH 177/436] Added transfer limit awareness to message sync. Added ability to retain messages on node. --- LXMF/LXMPeer.py | 3 ++- LXMF/LXMRouter.py | 36 ++++++++++++++++++++++++++++++------ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index b1a49af..a0b61a5 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -169,7 +169,8 @@ class LXMPeer: transient_id = unhandled_entry[0] weight = unhandled_entry[1] lxm_size = unhandled_entry[2] - if self.propagation_transfer_limit != None and cumulative_size + lxm_size > (self.propagation_transfer_limit*1000): + next_size = cumulative_size + (lxm_size+per_message_overhead) + if self.propagation_transfer_limit != None and next_size > (self.propagation_transfer_limit*1000): pass else: cumulative_size += (lxm_size+per_message_overhead) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index cedc2ea..d3a4b7d 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -72,6 +72,7 @@ class LXMRouter: self.ignored_list = [] self.allowed_list = [] self.auth_required = False + self.retain_synced_on_node = False self.processing_outbound = False self.processing_inbound = False @@ -200,6 +201,12 @@ class LXMRouter: def get_outbound_propagation_node(self): return self.outbound_propagation_node + def set_retain_node_lxms(self, retain): + if retain == True: + self.retain_synced_on_node = True + else: + self.retain_synced_on_node = False + def set_authentication(self, required=None): if required != None: self.auth_required = required @@ -247,7 +254,7 @@ class LXMRouter: self.outbound_propagation_link.identify(identity) self.outbound_propagation_link.request( LXMPeer.MESSAGE_GET_PATH, - [None, None], + [None, None], # Set both want and have fields to None to get message list response_callback=self.message_list_response, failed_callback=self.message_get_failed ) @@ -633,8 +640,7 @@ class LXMRouter: finally: i += 1 - RNS.log("LXMF message store size is now "+RNS.prettysize(self.message_storage_size()), RNS.LOG_EXTREME) - RNS.log("PE len "+str(len(self.propagation_entries))) + RNS.log("LXMF message store size is now "+RNS.prettysize(self.message_storage_size())+" for "+str(len(self.propagation_entries))+" items", RNS.LOG_EXTREME) except Exception as e: @@ -765,6 +771,16 @@ class LXMRouter: # Process wanted messages response_messages = [] if data[0] != None and len(data[0]) > 0: + client_transfer_limit = None + if len(data) >= 3: + try: + client_transfer_limit = float(data[2])*1000 + RNS.log("Client indicates transfer limit of "+RNS.prettysize(client_transfer_limit), RNS.LOG_DEBUG) + except: + pass + + per_message_overhead = 16 # Really only 2 bytes, but set a bit higher for now + cumulative_size = 24 # Initialised to highest reasonable binary structure overhead for transient_id in data[0]: if transient_id in self.propagation_entries and self.propagation_entries[transient_id][0] == remote_destination.hash: try: @@ -773,9 +789,17 @@ class LXMRouter: message_file = open(filepath, "rb") lxmf_data = message_file.read() - response_messages.append(lxmf_data) message_file.close() + lxm_size = len(lxmf_data) + next_size = cumulative_size + (lxm_size+per_message_overhead) + + if client_transfer_limit != None and next_size > client_transfer_limit: + pass + else: + response_messages.append(lxmf_data) + cumulative_size += (lxm_size+per_message_overhead) + except Exception as e: RNS.log("Error while processing message download request from "+RNS.prettyhexrep(remote_destination.hash)+". The contained exception was: "+str(e), RNS.LOG_ERROR) @@ -805,7 +829,7 @@ class LXMRouter: wants = [] if len(request_receipt.response) > 0: for transient_id in request_receipt.response: - if self.has_message(transient_id): + if not self.retain_synced_on_node and self.has_message(transient_id): haves.append(transient_id) else: if self.propagation_transfer_max_messages == LXMRouter.PR_ALL_MESSAGES or len(wants) < self.propagation_transfer_max_messages: @@ -813,7 +837,7 @@ class LXMRouter: request_receipt.link.request( LXMPeer.MESSAGE_GET_PATH, - [wants, haves], + [wants, haves, self.delivery_per_transfer_limit], response_callback=self.message_get_response, failed_callback=self.message_get_failed, progress_callback=self.message_get_progress From 1e9fe969fda85fb313f848403309b873c2ce9c09 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 2 Mar 2024 09:17:34 +0100 Subject: [PATCH 178/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 6a9beea..3d26edf 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.4.0" +__version__ = "0.4.1" From 994bb07efdf9d53e75700177431474d3d41e3c3a Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 17 Mar 2024 00:35:45 +0100 Subject: [PATCH 179/436] Updated timing --- LXMF/LXMRouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index d3a4b7d..991a893 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -18,7 +18,7 @@ from .Handlers import LXMFPropagationAnnounceHandler class LXMRouter: MAX_DELIVERY_ATTEMPTS = 5 PROCESSING_INTERVAL = 4 - DELIVERY_RETRY_WAIT = 7 + DELIVERY_RETRY_WAIT = 10 PATH_REQUEST_WAIT = 7 LINK_MAX_INACTIVITY = 10*60 P_LINK_MAX_INACTIVITY = 3*60 From 739349c2a13e7a254d038781b045dda7f8fb9d49 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 17 Mar 2024 00:35:54 +0100 Subject: [PATCH 180/436] Updated field codes --- LXMF/LXMF.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 5225d88..8febabe 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -13,4 +13,5 @@ FIELD_IMAGE = 0x06 FIELD_AUDIO = 0x07 FIELD_THREAD = 0x08 FIELD_COMMANDS = 0x09 -FIELD_RESULTS = 0x0A \ No newline at end of file +FIELD_RESULTS = 0x0A +FIELD_GROUP = 0x0B \ No newline at end of file From 21af6a4e5c8b0469bdcdda34dec224add46d4f91 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 17 Mar 2024 00:40:15 +0100 Subject: [PATCH 181/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 3d26edf..df12433 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.4.1" +__version__ = "0.4.2" From 3fbe2e94da2c348b2cc9e44803f72275d891397d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 19 Mar 2024 09:47:37 +0100 Subject: [PATCH 182/436] Append incoming to locally delivered transient IDs and check on arrival --- LXMF/LXMRouter.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 991a893..f249b0c 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -953,6 +953,12 @@ class LXMRouter: RNS.log(str(self)+" ignored message from "+RNS.prettyhexrep(message.source_hash), RNS.LOG_DEBUG) return False + if self.has_message(message.hash): + RNS.log(str(self)+" ignored already received message from "+RNS.prettyhexrep(message.source_hash), RNS.LOG_DEBUG) + return False + else: + self.locally_delivered_transient_ids[message.hash] = time.time() + if self.__delivery_callback != None and callable(self.__delivery_callback): try: self.__delivery_callback(message) From d2b2ef54e89dd0df36d89f84c9d7732ca65796d7 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 19 Mar 2024 11:03:12 +0100 Subject: [PATCH 183/436] Added outbound transfer progress monitoring --- LXMF/LXMRouter.py | 22 ++++++++++++++++------ LXMF/LXMessage.py | 13 +++++++++---- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index f249b0c..c89c4c8 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -909,10 +909,6 @@ class LXMRouter: return True else: return False - - - ### Message Routing & Delivery ######################## - ####################################################### def handle_outbound(self, lxmessage): lxmessage.state = LXMessage.OUTBOUND @@ -927,6 +923,17 @@ class LXMRouter: self.pending_outbound.append(lxmessage) self.process_outbound() + def get_outbound_progress(self, lxm_hash): + for lxm in self.pending_outbound: + if lxm.hash == lxm_hash: + return lxm.progress + + return None + + + ### Message Routing & Delivery ######################## + ####################################################### + def lxmf_delivery(self, lxmf_data, destination_type = None, phy_stats = None): try: message = LXMessage.unpack_from_bytes(lxmf_data) @@ -1340,6 +1347,8 @@ class LXMRouter: # to deliver the message direct_link = self.direct_links[delivery_destination_hash] if direct_link.status == RNS.Link.ACTIVE: + if lxmessage.progress == None or lxmessage.progress < 0.05: + lxmessage.progress = 0.05 if lxmessage.state != LXMessage.SENDING: RNS.log("Starting transfer of "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) lxmessage.set_delivery_destination(direct_link) @@ -1366,8 +1375,7 @@ class LXMRouter: self.direct_links.pop(delivery_destination_hash) lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT else: - # Simply wait for the link to become - # active or close + # Simply wait for the link to become active or close RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" is pending, waiting for link to become active", RNS.LOG_DEBUG) else: # No link exists, so we'll try to establish one, but @@ -1383,10 +1391,12 @@ class LXMRouter: delivery_link = RNS.Link(lxmessage.get_destination()) delivery_link.set_link_established_callback(self.process_outbound) self.direct_links[delivery_destination_hash] = delivery_link + lxmessage.progress = 0.03 else: RNS.log("No path known for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+". Requesting path...", RNS.LOG_DEBUG) RNS.Transport.request_path(lxmessage.get_destination().hash) lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + lxmessage.progress = 0.01 else: RNS.log("Max delivery attempts reached for direct "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) self.fail_message(lxmessage) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 3cf8122..5b7721c 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -118,7 +118,7 @@ class LXMessage: self.signature = None self.hash = None self.packed = None - self.progress = None + self.progress = 0.0 self.state = LXMessage.DRAFT self.method = LXMessage.UNKNOWN self.rssi = None @@ -324,12 +324,14 @@ class LXMessage: if receipt: receipt.set_delivery_callback(self.__mark_delivered) receipt.set_timeout_callback(self.__link_packet_timed_out) + self.progress = 0.50 else: if self.__delivery_destination: self.__delivery_destination.teardown() elif self.representation == LXMessage.RESOURCE: self.resource_representation = self.__as_resource() + self.progress = 0.10 elif self.method == LXMessage.PROPAGATED: self.state = LXMessage.SENDING @@ -339,11 +341,13 @@ class LXMessage: if receipt: receipt.set_delivery_callback(self.__mark_propagated) receipt.set_timeout_callback(self.__link_packet_timed_out) + self.progress = 0.50 else: self.__delivery_destination.teardown() elif self.representation == LXMessage.RESOURCE: self.resource_representation = self.__as_resource() + self.progress = 0.10 def determine_transport_encryption(self): @@ -387,6 +391,7 @@ class LXMessage: def __mark_delivered(self, receipt = None): RNS.log("Received delivery notification for "+str(self), RNS.LOG_DEBUG) self.state = LXMessage.DELIVERED + self.progress = 1.0 if self.__delivery_callback != None and callable(self.__delivery_callback): try: @@ -397,6 +402,7 @@ class LXMessage: def __mark_propagated(self, receipt = None): RNS.log("Received propagation success notification for "+str(self), RNS.LOG_DEBUG) self.state = LXMessage.SENT + self.progress = 1.0 if self.__delivery_callback != None and callable(self.__delivery_callback): try: @@ -407,6 +413,7 @@ class LXMessage: def __mark_paper_generated(self, receipt = None): RNS.log("Paper message generation succeeded for "+str(self), RNS.LOG_DEBUG) self.state = LXMessage.PAPER + self.progress = 1.0 if self.__delivery_callback != None and callable(self.__delivery_callback): try: @@ -436,7 +443,7 @@ class LXMessage: def __update_transfer_progress(self, resource): - self.progress = resource.get_progress() + self.progress = 0.10 + (resource.get_progress()*0.90) def __as_packet(self): if not self.packed: @@ -465,8 +472,6 @@ class LXMessage: if not self.__delivery_destination.status == RNS.Link.ACTIVE: raise ConnectionError("Tried to synthesize resource for LXMF message on a link that was not active") - self.progress = 0.0 - if self.method == LXMessage.DIRECT: return RNS.Resource(self.packed, self.__delivery_destination, callback = self.__resource_concluded, progress_callback = self.__update_transfer_progress) elif self.method == LXMessage.PROPAGATED: From 27ffea3ea9dafee7fa2cc2f0ddb674eddde8a79c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 19 Mar 2024 13:45:55 +0100 Subject: [PATCH 184/436] Improved message transfer feedback --- LXMF/LXMRouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index c89c4c8..f7701bf 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1396,7 +1396,7 @@ class LXMRouter: RNS.log("No path known for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+". Requesting path...", RNS.LOG_DEBUG) RNS.Transport.request_path(lxmessage.get_destination().hash) lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT - lxmessage.progress = 0.01 + lxmessage.progress = 0.00 else: RNS.log("Max delivery attempts reached for direct "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) self.fail_message(lxmessage) From 45e39917b5374b8667605fb6a2dae450297f9fbc Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 19 Mar 2024 13:54:09 +0100 Subject: [PATCH 185/436] Updated dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 26ce0d7..43817c8 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.7.2'], + install_requires=['rns>=0.7.3'], python_requires='>=3.7', ) From 4b13d7188c537c00e623531ef6b980dc2dfc8811 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 22 Mar 2024 21:11:32 +0100 Subject: [PATCH 186/436] Send packet proof before delivery processing --- LXMF/LXMRouter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index f7701bf..db19916 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -997,8 +997,9 @@ class LXMRouter: RNS.log("Error while retrieving physical link stats for LXMF delivery packet: "+str(e), RNS.LOG_ERROR) phy_stats = {"rssi": packet.rssi, "snr": packet.snr, "q": packet.q} - if self.lxmf_delivery(lxmf_data, packet.destination_type, phy_stats=phy_stats): - packet.prove() + + packet.prove() + self.lxmf_delivery(lxmf_data, packet.destination_type, phy_stats=phy_stats) except Exception as e: RNS.log("Exception occurred while parsing incoming LXMF data.", RNS.LOG_ERROR) From f6cc9fd1cf108c8cf0c7d3ee2dab96896805bd56 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 23 Mar 2024 19:53:05 +0100 Subject: [PATCH 187/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index df12433..f6b7e26 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.4.2" +__version__ = "0.4.3" From 2b3fa796a1574785df784cc57e6ae9af4f2f6077 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 26 Mar 2024 14:55:59 +0100 Subject: [PATCH 188/436] Updated packet proof logic --- LXMF/LXMRouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index db19916..30788c9 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -980,6 +980,7 @@ class LXMRouter: return False def delivery_packet(self, data, packet): + packet.prove() try: if packet.destination_type != RNS.Destination.LINK: lxmf_data = b"" @@ -998,7 +999,6 @@ class LXMRouter: phy_stats = {"rssi": packet.rssi, "snr": packet.snr, "q": packet.q} - packet.prove() self.lxmf_delivery(lxmf_data, packet.destination_type, phy_stats=phy_stats) except Exception as e: From d767d40c5fdf3a9fa6b0738d20c9eeb4bf0ffa2d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 5 May 2024 20:09:05 +0200 Subject: [PATCH 189/436] Updated dependency version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 43817c8..da4639d 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.7.3'], + install_requires=['rns>=0.7.4'], python_requires='>=3.7', ) From 741b66aab203b3dfbc1454ee869d97999db61cfc Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 5 May 2024 20:11:30 +0200 Subject: [PATCH 190/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index f6b7e26..cd1ee63 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.4.3" +__version__ = "0.4.4" From a810be316fc53bbbca6e0085e32381bdca465141 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 2 Jun 2024 09:19:47 +0200 Subject: [PATCH 191/436] Added audio modes --- LXMF/LXMF.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 8febabe..fa3e2d2 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -14,4 +14,28 @@ FIELD_AUDIO = 0x07 FIELD_THREAD = 0x08 FIELD_COMMANDS = 0x09 FIELD_RESULTS = 0x0A -FIELD_GROUP = 0x0B \ No newline at end of file +FIELD_GROUP = 0x0B + +# Audio modes for the data structure in FIELD_AUDIO + +# Codec2 Audio Modes +AM_CODEC2_450PWB = 0x01 +AM_CODEC2_450 = 0x02 +AM_CODEC2_700C = 0x03 +AM_CODEC2_1200 = 0x04 +AM_CODEC2_1300 = 0x05 +AM_CODEC2_1400 = 0x06 +AM_CODEC2_1600 = 0x07 +AM_CODEC2_2400 = 0x08 +AM_CODEC2_3200 = 0x09 + +# Opus Audio Modes +AM_OPUS_LBW = 0x10 +AM_OPUS_MBW = 0x11 +AM_OPUS_PTT = 0x12 +AM_OPUS_RT_HDX = 0x13 +AM_OPUS_RT_FDX = 0x14 +AM_OPUS_STANDARD = 0x15 +AM_OPUS_HQ = 0x16 +AM_OPUS_BROADCAST = 0x17 +AM_OPUS_LOSSLESS = 0x18 From a35e522e75e6afca4b185a56d989773cafaec52d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 2 Jun 2024 09:25:33 +0200 Subject: [PATCH 192/436] Added custom audio mode to spec --- LXMF/LXMF.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index fa3e2d2..21567a5 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -39,3 +39,7 @@ AM_OPUS_STANDARD = 0x15 AM_OPUS_HQ = 0x16 AM_OPUS_BROADCAST = 0x17 AM_OPUS_LOSSLESS = 0x18 + +# Custom, unspecified audio mode, the +# client must determined it itself +AM_CUSTOM = 0xFF \ No newline at end of file From e5a960b2df023eb0918f30680a3012fb32542cc5 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 4 Jun 2024 18:31:18 +0200 Subject: [PATCH 193/436] Updated OPUS audio modes --- LXMF/LXMF.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 21567a5..a8bec6b 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -30,15 +30,16 @@ AM_CODEC2_2400 = 0x08 AM_CODEC2_3200 = 0x09 # Opus Audio Modes -AM_OPUS_LBW = 0x10 -AM_OPUS_MBW = 0x11 -AM_OPUS_PTT = 0x12 -AM_OPUS_RT_HDX = 0x13 -AM_OPUS_RT_FDX = 0x14 -AM_OPUS_STANDARD = 0x15 -AM_OPUS_HQ = 0x16 -AM_OPUS_BROADCAST = 0x17 -AM_OPUS_LOSSLESS = 0x18 +AM_OPUS_OGG = 0x10 +AM_OPUS_LBW = 0x11 +AM_OPUS_MBW = 0x12 +AM_OPUS_PTT = 0x13 +AM_OPUS_RT_HDX = 0x14 +AM_OPUS_RT_FDX = 0x15 +AM_OPUS_STANDARD = 0x16 +AM_OPUS_HQ = 0x17 +AM_OPUS_BROADCAST = 0x18 +AM_OPUS_LOSSLESS = 0x19 # Custom, unspecified audio mode, the # client must determined it itself From 4520507869d1725ef8cc161167fda2b7ebd4e3c5 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 4 Jun 2024 18:43:46 +0200 Subject: [PATCH 194/436] Updated readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2f5c888..fb7c4dc 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ LXMF is efficient enough that it can deliver messages over extremely low-bandwid User-facing clients built on LXMF include: - [Sideband](https://unsigned.io/sideband) +- [Reticulum MeshChat](https://github.com/liamcottle/reticulum-meshchat) - [Nomad Network](https://unsigned.io/nomadnet) Community-provided tools and utilities for LXMF include: From b4ba9d628b04c69f068b2877ff69fc49b7d843f0 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 5 Jun 2024 00:32:59 +0200 Subject: [PATCH 195/436] Updated readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fb7c4dc..5b7c9bd 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ LXMF is efficient enough that it can deliver messages over extremely low-bandwid User-facing clients built on LXMF include: - [Sideband](https://unsigned.io/sideband) -- [Reticulum MeshChat](https://github.com/liamcottle/reticulum-meshchat) +- [MeshChat](https://github.com/liamcottle/reticulum-meshchat) - [Nomad Network](https://unsigned.io/nomadnet) Community-provided tools and utilities for LXMF include: From 8b8008478da6148e0be8ba1fbc89cbefaede3e05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 18:35:25 +0000 Subject: [PATCH 196/436] Bump setuptools from 68.0.0 to 70.0.0 Bumps [setuptools](https://github.com/pypa/setuptools) from 68.0.0 to 70.0.0. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst) - [Commits](https://github.com/pypa/setuptools/compare/v68.0.0...v70.0.0) --- updated-dependencies: - dependency-name: setuptools dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 48086e6..a322487 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ qrcode==7.4.2 rns==0.5.7 -setuptools==68.0.0 +setuptools==70.0.0 From 8e686b10e8c635c86384ebb9cd205dc13ba79ea7 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 30 Aug 2024 00:11:11 +0200 Subject: [PATCH 197/436] Updated version and dependencies --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index cd1ee63..98a433b 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.4.4" +__version__ = "0.4.5" diff --git a/setup.py b/setup.py index da4639d..7d283d4 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.7.4'], + install_requires=['rns>=0.7.6'], python_requires='>=3.7', ) From 0be569ccd6ba355ac4b81409d195fe687eef08fa Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 30 Aug 2024 00:11:32 +0200 Subject: [PATCH 198/436] Updated field specifiers and descriptions --- LXMF/LXMF.py | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index a8bec6b..0edb6ad 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -1,9 +1,10 @@ APP_NAME = "lxmf" -# WARNING! These field specifiers are floating and not -# yet final! Consider highly experiemental, and expect -# them to change in the future! You have been warned :) - +########################################################## +# The following core fields are provided to facilitate # +# interoperability in data exchange between various LXMF # +# clients and systems. # +########################################################## FIELD_EMBEDDED_LXMS = 0x01 FIELD_TELEMETRY = 0x02 FIELD_TELEMETRY_STREAM = 0x03 @@ -16,6 +17,34 @@ FIELD_COMMANDS = 0x09 FIELD_RESULTS = 0x0A FIELD_GROUP = 0x0B +# For usecases such as including custom data structures, +# embedding or encapsulating other data types or protocols +# that are not native to LXMF, or bridging/tunneling +# external protocols or services over LXMF, the following +# fields are available. A format/type/protocol (or other) +# identifier can be included in the CUSTOM_TYPE field, and +# the embedded payload can be included in the CUSTOM_DATA +# field. It is up to the client application to correctly +# discern and potentially utilise any data embedded using +# this mechanism. +FIELD_CUSTOM_TYPE = 0xFB +FIELD_CUSTOM_DATA = 0xFC +FIELD_CUSTOM_META = 0xFD + +# The non-specific and debug fields are intended for +# development, testing and debugging use. +FIELD_NON_SPECIFIC = 0xFE +FIELD_DEBUG = 0xFF + +########################################################## +# The following section lists field-specific specifiers, # +# modes and identifiers that are native to LXMF. It is # +# optional for any client or system to support any of # +# these, and they are provided as template for easing # +# interoperability without sacrificing expandability # +# and flexibility of the format. # +########################################################## + # Audio modes for the data structure in FIELD_AUDIO # Codec2 Audio Modes @@ -41,6 +70,6 @@ AM_OPUS_HQ = 0x17 AM_OPUS_BROADCAST = 0x18 AM_OPUS_LOSSLESS = 0x19 -# Custom, unspecified audio mode, the -# client must determined it itself +# Custom, unspecified audio mode, the client must +# determine it itself based on the included data. AM_CUSTOM = 0xFF \ No newline at end of file From 68892091ec488337b38e34e0c45756b905008c2d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 30 Aug 2024 00:23:09 +0200 Subject: [PATCH 199/436] Updated readme --- README.md | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5b7c9bd..65018f4 100644 --- a/README.md +++ b/README.md @@ -130,10 +130,6 @@ You can also find the entire message in Date: Wed, 4 Sep 2024 18:58:25 +0200 Subject: [PATCH 203/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 98a433b..3d18726 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.4.5" +__version__ = "0.5.0" From 775ac7ff689b50cf34ee7c06d2c34a937b11d4e6 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 5 Sep 2024 14:56:49 +0200 Subject: [PATCH 204/436] Allow ratchet enforcement and fixed opportunistic delivery queue handling --- LXMF/LXMRouter.py | 18 +++++++++++------- LXMF/LXMessage.py | 4 +++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 3dea0cd..f2daa2f 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -58,7 +58,7 @@ class LXMRouter: ####################################################### def __init__(self, identity = None, storagepath = None, autopeer = AUTOPEER, autopeer_maxdepth = None, - propagation_limit = PROPAGATION_LIMIT, delivery_limit = DELIVERY_LIMIT): + propagation_limit = PROPAGATION_LIMIT, delivery_limit = DELIVERY_LIMIT, enforce_ratchets = False): random.seed(os.urandom(10)) @@ -93,6 +93,7 @@ class LXMRouter: self.information_storage_limit = None self.propagation_per_transfer_limit = propagation_limit self.delivery_per_transfer_limit = delivery_limit + self.enforce_ratchets = enforce_ratchets self.wants_download_on_path_available_from = None self.wants_download_on_path_available_to = None @@ -183,6 +184,8 @@ class LXMRouter: delivery_destination.set_packet_callback(self.delivery_packet) delivery_destination.set_link_established_callback(self.delivery_link_established) delivery_destination.display_name = display_name + if self.enforce_ratchets: + delivery_destination.enforce_ratchets() if display_name != None: delivery_destination.set_default_app_data(display_name.encode("utf-8")) @@ -1245,12 +1248,13 @@ class LXMRouter: delivery_destination = self.delivery_destinations[destination_hash] encrypted_lxmf_data = lxmf_data[LXMessage.DESTINATION_LENGTH:] decrypted_lxmf_data = delivery_destination.decrypt(encrypted_lxmf_data) - delivery_data = lxmf_data[:LXMessage.DESTINATION_LENGTH]+decrypted_lxmf_data - self.lxmf_delivery(delivery_data, delivery_destination.type) - self.locally_delivered_transient_ids[transient_id] = time.time() + if decrypted_lxmf_data != None: + delivery_data = lxmf_data[:LXMessage.DESTINATION_LENGTH]+decrypted_lxmf_data + self.lxmf_delivery(delivery_data, delivery_destination.type) + self.locally_delivered_transient_ids[transient_id] = time.time() - if signal_local_delivery != None: - return signal_local_delivery + if signal_local_delivery != None: + return signal_local_delivery else: if self.propagation_node: @@ -1326,7 +1330,7 @@ class LXMRouter: if lxmessage.state == LXMessage.DELIVERED: RNS.log("Delivery has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) self.pending_outbound.remove(lxmessage) - elif lxmessage.state == LXMessage.SENT: + elif lxmessage.method == LXMessage.PROPAGATED and lxmessage.state == LXMessage.SENT: RNS.log("Propagation has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) self.pending_outbound.remove(lxmessage) else: diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index e8dd1db..7442091 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -351,7 +351,9 @@ class LXMessage: def determine_transport_encryption(self): - # TODO: Update transport encryption descriptions to account for ratchets + # TODO: These descriptions are old and outdated. + # Update the transport encryption descriptions to + # account for ratchets and other changes. if self.method == LXMessage.OPPORTUNISTIC: if self.__destination.type == RNS.Destination.SINGLE: self.transport_encrypted = True From 95ba8cba607f96b2e0d4bc19d525d380cab1e860 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 5 Sep 2024 14:57:20 +0200 Subject: [PATCH 205/436] Updated dependencies --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index a322487..8117b1f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ qrcode==7.4.2 -rns==0.5.7 +rns==0.7.7 setuptools==70.0.0 diff --git a/setup.py b/setup.py index 7d283d4..52489a1 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.7.6'], + install_requires=['rns>=0.7.7'], python_requires='>=3.7', ) From 5366f895b23498b72f5e1a514fa751433039ed30 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 6 Sep 2024 00:51:36 +0200 Subject: [PATCH 206/436] Added stamp generation to LXMessage --- LXMF/LXMessage.py | 62 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 7442091..c6eeb3f 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -1,6 +1,7 @@ import RNS import RNS.vendor.umsgpack as msgpack +import os import time import base64 @@ -90,7 +91,9 @@ class LXMessage: else: return "" - def __init__(self, destination, source, content = "", title = "", fields = None, desired_method = None, destination_hash = None, source_hash = None): + def __init__(self, destination, source, content = "", title = "", fields = None, desired_method = None, + destination_hash = None, source_hash = None, stamp_cost=None): + if isinstance(destination, RNS.Destination) or destination == None: self.__destination = destination if destination != None: @@ -118,9 +121,11 @@ class LXMessage: self.signature = None self.hash = None self.packed = None - self.progress = 0.0 + self.stamp = None + self.stamp_cost = stamp_cost self.state = LXMessage.DRAFT self.method = LXMessage.UNKNOWN + self.progress = 0.0 self.rssi = None self.snr = None self.q = None @@ -219,6 +224,41 @@ class LXMessage: def register_failed_callback(self, callback): self.failed_callback = callback + def validate_stamp(self, target_cost): + if self.stamp == None: + return False + else: + target = 0b1 << 256-target_cost + if int.from_bytes(RNS.Identity.full_hash(self.message_id+self.stamp)) > target: + return False + else: + return True + + def get_stamp(self, timeout=None): + if self.stamp_cost == None: + return None + + elif self.stamp != None: + # TODO: Check that message hash cannot actually + # change under any circumstances before handoff + return self.stamp + + else: + RNS.log(f"Generating stamp for {self}...", RNS.LOG_DEBUG) + start_time = time.time() + stamp = os.urandom(256//8); target = 0b1 << 256-self.stamp_cost; rounds = 1 + while int.from_bytes(RNS.Identity.full_hash(self.message_id+stamp)) > target: + if timeout != None and rounds % 10000 == 0: + if time.time() > start_time + timeout: + RNS.log(f"Stamp generation for {self} timed out", RNS.LOG_ERROR) + return None + + stamp = os.urandom(256//8) + rounds += 1 + + RNS.log(f"Stamp generated in {RNS.prettytime(time.time() - start_time)}", RNS.LOG_DEBUG) + return stamp + def pack(self): if not self.packed: if self.timestamp == None: @@ -235,6 +275,9 @@ class LXMessage: hashed_part += msgpack.packb(self.payload) self.hash = RNS.Identity.full_hash(hashed_part) self.message_id = self.hash + self.stamp = self.get_stamp() + if self.stamp != None: + self.payload.append(self.stamp) signed_part = b"" signed_part += hashed_part @@ -242,11 +285,11 @@ class LXMessage: self.signature = self.__source.sign(signed_part) self.signature_validated = True + packed_payload = msgpack.packb(self.payload) self.packed = b"" self.packed += self.__destination.hash self.packed += self.__source.hash self.packed += self.signature - packed_payload = msgpack.packb(self.payload) self.packed += packed_payload self.packed_size = len(self.packed) content_size = len(packed_payload) @@ -566,10 +609,19 @@ class LXMessage: source_hash = lxmf_bytes[LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH] signature = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH:2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH] packed_payload = lxmf_bytes[2*LXMessage.DESTINATION_LENGTH+LXMessage.SIGNATURE_LENGTH:] + unpacked_payload = msgpack.unpackb(packed_payload) + + # Extract stamp from payload if included + if len(unpacked_payload) > 4: + stamp = unpacked_payload[4] + unpacked_payload = unpacked_payload[:4] + packed_payload = msgpack.packb(unpacked_payload) + else: + stamp = None + hashed_part = b"" + destination_hash + source_hash + packed_payload message_hash = RNS.Identity.full_hash(hashed_part) signed_part = b"" + hashed_part + message_hash - unpacked_payload = msgpack.unpackb(packed_payload) timestamp = unpacked_payload[0] title_bytes = unpacked_payload[1] content_bytes = unpacked_payload[2] @@ -598,7 +650,9 @@ class LXMessage: desired_method = original_method) message.hash = message_hash + message.message_id = message.hash message.signature = signature + message.stamp = stamp message.incoming = True message.timestamp = timestamp message.packed = lxmf_bytes From 4dca031441cd52300acb4f95be406e3470e04f17 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 6 Sep 2024 00:57:06 +0200 Subject: [PATCH 207/436] Updated fields --- LXMF/LXMF.py | 1 + 1 file changed, 1 insertion(+) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 0edb6ad..8d79bf9 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -16,6 +16,7 @@ FIELD_THREAD = 0x08 FIELD_COMMANDS = 0x09 FIELD_RESULTS = 0x0A FIELD_GROUP = 0x0B +FIELD_TICKET = 0x0C # For usecases such as including custom data structures, # embedding or encapsulating other data types or protocols From fe14f8744d5c61ba1e1d20f7aeddcb33603f38dc Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 6 Sep 2024 16:49:01 +0200 Subject: [PATCH 208/436] Added base code for stamp generation and validation --- LXMF/LXMessage.py | 94 ++++++++++++++++++++++++++++++++++------ docs/example_receiver.py | 7 +++ 2 files changed, 88 insertions(+), 13 deletions(-) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index c6eeb3f..a79f600 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -4,6 +4,8 @@ import RNS.vendor.umsgpack as msgpack import os import time import base64 +import signal +import multiprocessing from .LXMF import APP_NAME @@ -224,15 +226,40 @@ class LXMessage: def register_failed_callback(self, callback): self.failed_callback = callback + @staticmethod + def stamp_workblock(message_id): + wb_st = time.time() + expand_rounds = 3000 + workblock = b"" + for n in range(expand_rounds): + workblock += RNS.Cryptography.hkdf( + length=256, + derive_from=message_id, + salt=RNS.Identity.full_hash(message_id+msgpack.packb(n)), + context=None, + ) + wb_time = time.time() - wb_st + RNS.log(f"Stamp workblock size {RNS.prettysize(len(workblock))}, generated in {round(wb_time*1000,2)}ms", RNS.LOG_DEBUG) + + return workblock + + @staticmethod + def stamp_valid(stamp, target_cost, workblock): + target = 0b1 << 256-target_cost + result = RNS.Identity.full_hash(workblock+stamp) + if int.from_bytes(result, byteorder="big") > target: + return False + else: + return True + def validate_stamp(self, target_cost): if self.stamp == None: return False else: - target = 0b1 << 256-target_cost - if int.from_bytes(RNS.Identity.full_hash(self.message_id+self.stamp)) > target: - return False - else: + if LXMessage.stamp_valid(self.stamp, target_cost, LXMessage.stamp_workblock(self.message_id)): return True + else: + return False def get_stamp(self, timeout=None): if self.stamp_cost == None: @@ -244,19 +271,60 @@ class LXMessage: return self.stamp else: - RNS.log(f"Generating stamp for {self}...", RNS.LOG_DEBUG) + RNS.log(f"Generating stamp with cost {self.stamp_cost} for {self}...", RNS.LOG_DEBUG) + workblock = LXMessage.stamp_workblock(self.message_id) start_time = time.time() - stamp = os.urandom(256//8); target = 0b1 << 256-self.stamp_cost; rounds = 1 - while int.from_bytes(RNS.Identity.full_hash(self.message_id+stamp)) > target: - if timeout != None and rounds % 10000 == 0: - if time.time() > start_time + timeout: - RNS.log(f"Stamp generation for {self} timed out", RNS.LOG_ERROR) - return None + total_rounds = 0 + + stop_event = multiprocessing.Event() + result_queue = multiprocessing.Queue(maxsize=1) + rounds_queue = multiprocessing.Queue() + def job(stop_event): + terminated = False + rounds = 0 stamp = os.urandom(256//8) - rounds += 1 + while not LXMessage.stamp_valid(stamp, self.stamp_cost, workblock): + if stop_event.is_set(): + break + + if timeout != None and rounds % 10000 == 0: + if time.time() > start_time + timeout: + RNS.log(f"Stamp generation for {self} timed out", RNS.LOG_ERROR) + return None + + stamp = os.urandom(256//8) + rounds += 1 + + rounds_queue.put(rounds) + if not stop_event.is_set(): + result_queue.put(stamp) + + job_procs = [] + jobs = multiprocessing.cpu_count() + for _ in range(jobs): + process = multiprocessing.Process(target=job, kwargs={"stop_event": stop_event},) + job_procs.append(process) + process.start() + + stamp = result_queue.get() + stop_event.set() + + for j in range(jobs): + process = job_procs[j] + process.join() + total_rounds += rounds_queue.get() + + duration = time.time() - start_time + rounds = total_rounds + + # TODO: Remove stats output + RNS.log(f"Stamp generated in {RNS.prettytime(duration)} / {rounds} rounds", RNS.LOG_DEBUG) + RNS.log(f"Rounds per second {int(rounds/duration)}", RNS.LOG_DEBUG) + RNS.log(f"Stamp: {RNS.hexrep(stamp)}", RNS.LOG_DEBUG) + RNS.log(f"Resulting hash: {RNS.hexrep(RNS.Identity.full_hash(workblock+stamp))}", RNS.LOG_DEBUG) + ########################### - RNS.log(f"Stamp generated in {RNS.prettytime(time.time() - start_time)}", RNS.LOG_DEBUG) return stamp def pack(self): diff --git a/docs/example_receiver.py b/docs/example_receiver.py index 75c628e..e6ea117 100644 --- a/docs/example_receiver.py +++ b/docs/example_receiver.py @@ -13,6 +13,12 @@ def delivery_callback(message): if message.unverified_reason == LXMF.LXMessage.SOURCE_UNKNOWN: signature_string = "Cannot verify, source is unknown" + stamp_cost = 12 + if message.validate_stamp(stamp_cost): + stamp_string = "Valid" + else: + stamp_string = "Not valid" + RNS.log("\t+--- LXMF Delivery ---------------------------------------------") RNS.log("\t| Source hash : "+RNS.prettyhexrep(message.source_hash)) RNS.log("\t| Source instance : "+str(message.get_source())) @@ -24,6 +30,7 @@ def delivery_callback(message): RNS.log("\t| Content : "+message.content_as_string()) RNS.log("\t| Fields : "+str(message.fields)) RNS.log("\t| Message signature : "+signature_string) + RNS.log("\t| Stamp : "+stamp_string) RNS.log("\t+---------------------------------------------------------------") r = RNS.Reticulum() From fc99010a3db7867ab2d52b79487f670218ded27a Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 6 Sep 2024 17:37:24 +0200 Subject: [PATCH 209/436] Updated example --- docs/example_receiver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/example_receiver.py b/docs/example_receiver.py index e6ea117..02ea0c3 100644 --- a/docs/example_receiver.py +++ b/docs/example_receiver.py @@ -13,8 +13,8 @@ def delivery_callback(message): if message.unverified_reason == LXMF.LXMessage.SOURCE_UNKNOWN: signature_string = "Cannot verify, source is unknown" - stamp_cost = 12 - if message.validate_stamp(stamp_cost): + required_stamp_cost = 12 + if message.validate_stamp(required_stamp_cost): stamp_string = "Valid" else: stamp_string = "Not valid" From 537f1823b637ca37be60a9754fbf44f3f65fea5f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 6 Sep 2024 19:54:28 +0200 Subject: [PATCH 210/436] Added utility functions --- LXMF/LXMF.py | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 8d79bf9..0fe4a2c 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -73,4 +73,45 @@ AM_OPUS_LOSSLESS = 0x19 # Custom, unspecified audio mode, the client must # determine it itself based on the included data. -AM_CUSTOM = 0xFF \ No newline at end of file +AM_CUSTOM = 0xFF + + +########################################################## +# The following helper functions makes it easier to # +# handle and operate on LXMF data in client programs # +########################################################## + +import RNS.vendor.umsgpack as msgpack +def display_name_from_app_data(app_data=None): + if app_data == None: + return None + else: + # Version 0.5.0+ announce format + if (app_data[0] >= 0x90 and app_data[0] <= 0x9f) or app_data[0] == 0xdc: + peer_data = msgpack.unpackb(app_data) + if type(peer_data) == list: + if len(peer_data) < 1: + return None + else: + return peer_data[0].decode("utf-8") + + # Original announce format + else: + return app_data.decode("utf-8") + +def stamp_cost_from_app_data(app_data=None): + if app_data == None: + return None + else: + # Version 0.5.0+ announce format + if (app_data[0] >= 0x90 and app_data[0] <= 0x9f) or app_data[0] == 0xdc: + peer_data = msgpack.unpackb(app_data) + if type(peer_data) == list: + if len(peer_data) < 2: + return None + else: + return peer_data[1] + + # Original announce format + else: + return None \ No newline at end of file From 015dcc563146fd601afd9f9a5c5e2dfa5e3a5399 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 6 Sep 2024 19:55:18 +0200 Subject: [PATCH 211/436] Extract announced stamp cost in handlers --- LXMF/Handlers.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index c09bf1c..a33d13d 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -2,13 +2,14 @@ import time import RNS import RNS.vendor.umsgpack as msgpack -from .LXMF import APP_NAME +from .LXMF import APP_NAME, stamp_cost_from_app_data from .LXMessage import LXMessage class LXMFDeliveryAnnounceHandler: def __init__(self, lxmrouter): self.aspect_filter = APP_NAME+".delivery" + self.receive_path_responses = True self.lxmrouter = lxmrouter def received_announce(self, destination_hash, announced_identity, app_data): @@ -22,10 +23,18 @@ class LXMFDeliveryAnnounceHandler: self.lxmrouter.process_outbound() + try: + stamp_cost = stamp_cost_from_app_data(app_data) + if stamp_cost != None: + self.lxmrouter.update_stamp_cost(destination_hash, stamp_cost) + except Exception as e: + RNS.log(f"An error occurred while trying to decode announced stamp cost. The contained exception was: {e}", RNS.LOG_ERROR) + class LXMFPropagationAnnounceHandler: def __init__(self, lxmrouter): self.aspect_filter = APP_NAME+".propagation" + self.receive_path_responses = False self.lxmrouter = lxmrouter def received_announce(self, destination_hash, announced_identity, app_data): From dcb0a18cd748f0800d692e7aab5589a7b309ae77 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 6 Sep 2024 20:34:09 +0200 Subject: [PATCH 212/436] Added automatic signalling of stamp costs, stamp generation and validation --- LXMF/LXMRouter.py | 115 ++++++++++++++++++++++++++++++++++++++++++++-- LXMF/LXMessage.py | 1 + 2 files changed, 111 insertions(+), 5 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index f2daa2f..5e3bf36 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -24,6 +24,7 @@ class LXMRouter: P_LINK_MAX_INACTIVITY = 3*60 MESSAGE_EXPIRY = 30*24*60*60 + STAMP_COST_EXPIRY = 45*24*60*60 NODE_ANNOUNCE_DELAY = 20 @@ -58,7 +59,8 @@ class LXMRouter: ####################################################### def __init__(self, identity = None, storagepath = None, autopeer = AUTOPEER, autopeer_maxdepth = None, - propagation_limit = PROPAGATION_LIMIT, delivery_limit = DELIVERY_LIMIT, enforce_ratchets = False): + propagation_limit = PROPAGATION_LIMIT, delivery_limit = DELIVERY_LIMIT, enforce_ratchets = False, + enforce_stamps = False): random.seed(os.urandom(10)) @@ -94,6 +96,7 @@ class LXMRouter: self.propagation_per_transfer_limit = propagation_limit self.delivery_per_transfer_limit = delivery_limit self.enforce_ratchets = enforce_ratchets + self._enforce_stamps = enforce_stamps self.wants_download_on_path_available_from = None self.wants_download_on_path_available_to = None @@ -104,6 +107,9 @@ class LXMRouter: self.active_propagation_links = [] self.locally_delivered_transient_ids = {} self.locally_processed_transient_ids = {} + self.outbound_stamp_costs = {} + + self.cost_file_lock = threading.Lock() if identity == None: identity = RNS.Identity() @@ -148,6 +154,19 @@ class LXMRouter: except Exception as e: RNS.log("Could not load locally delivered message ID cache from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + try: + if os.path.isfile(self.storagepath+"/outbound_stamp_costs"): + with self.cost_file_lock: + with open(self.storagepath+"/outbound_stamp_costs", "rb") as outbound_stamp_cost_file: + data = outbound_stamp_cost_file.read() + self.outbound_stamp_costs = msgpack.unpackb(data) + + self.clean_outbound_stamp_costs() + self.save_outbound_stamp_costs() + + except Exception as e: + RNS.log("Could not load outbound stamp costs from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + atexit.register(self.exit_handler) job_thread = threading.Thread(target=self.jobloop) @@ -156,8 +175,7 @@ class LXMRouter: def announce(self, destination_hash): if destination_hash in self.delivery_destinations: - delivery_destination = self.delivery_destinations[destination_hash] - delivery_destination.announce(delivery_destination.display_name.encode("utf-8")) + self.delivery_destinations[destination_hash].announce(app_data=self.get_announce_app_data(destination_hash)) def announce_propagation_node(self): def delayed_announce(): @@ -175,7 +193,7 @@ class LXMRouter: da_thread.setDaemon(True) da_thread.start() - def register_delivery_identity(self, identity, display_name = None): + def register_delivery_identity(self, identity, display_name = None, stamp_cost = None): if not os.path.isdir(self.ratchetpath): os.makedirs(self.ratchetpath) @@ -184,11 +202,16 @@ class LXMRouter: delivery_destination.set_packet_callback(self.delivery_packet) delivery_destination.set_link_established_callback(self.delivery_link_established) delivery_destination.display_name = display_name + delivery_destination.stamp_cost = stamp_cost + if self.enforce_ratchets: delivery_destination.enforce_ratchets() if display_name != None: - delivery_destination.set_default_app_data(display_name.encode("utf-8")) + def get_app_data(): + return self.get_announce_app_data(delivery_destination) + + delivery_destination.set_default_app_data(get_app_data) self.delivery_destinations[delivery_destination.hash] = delivery_destination return delivery_destination @@ -380,6 +403,12 @@ class LXMRouter: self.propagation_node = False self.announce_propagation_node() + def enforce_stamps(self): + self._enforce_stamps = True + + def ignore_stamps(self): + self._enforce_stamps = False + def ignore_destination(self, destination_hash): if not destination_hash in self.ignored_list: self.ignored_list.append(destination_hash) @@ -553,6 +582,31 @@ class LXMRouter: self.locally_processed_transient_ids.pop(transient_id) RNS.log("Cleaned "+RNS.prettyhexrep(transient_id)+" from locally processed cache", RNS.LOG_DEBUG) + def update_stamp_cost(self, destination_hash, stamp_cost): + RNS.log(f"Updating outbound stamp cost for {RNS.prettyhexrep(destination_hash)} to {stamp_cost}", RNS.LOG_DEBUG) + self.outbound_stamp_costs[destination_hash] = [time.time(), stamp_cost] + + def job(): + self.save_outbound_stamp_costs() + threading.Thread(target=self.save_outbound_stamp_costs, daemon=True).start() + + def get_announce_app_data(self, destination_hash): + if destination_hash in self.delivery_destinations: + delivery_destination = self.delivery_destinations[destination_hash] + + display_name = None + if delivery_destination.display_name != None: + display_name = delivery_destination.display_name.encode("utf-8") + + stamp_cost = None + if delivery_destination.stamp_cost != None and type(delivery_destination.stamp_cost) == int: + if delivery_destination.stamp_cost > 0 and delivery_destination.stamp_cost < 255: + stamp_cost = delivery_destination.stamp_cost + + peer_data = [display_name, stamp_cost] + + return msgpack.packb(peer_data) + def get_weight(self, transient_id): dst_hash = self.propagation_entries[transient_id][0] lxm_rcvd = self.propagation_entries[transient_id][2] @@ -678,6 +732,36 @@ class LXMRouter: except Exception as e: RNS.log("Could not save locally processed message ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + def clean_outbound_stamp_costs(self): + try: + expired = [] + for destination_hash in self.outbound_stamp_costs: + entry = self.outbound_stamp_costs[destination_hash] + if time.time() > entry[0] + LXMRouter.STAMP_COST_EXPIRY: + expired.append(destination_hash) + + for destination_hash in expired: + RNS.log(f"Cleaning expired stamp cost for {destination_hash}") # TODO: Remove + self.outbound_stamp_costs.pop(destination_hash) + + except Exception as e: + RNS.log(f"Error while cleaning outbound stamp costs. The contained exception was: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + def save_outbound_stamp_costs(self): + with self.cost_file_lock: + try: + RNS.log("Saving outbound stamp costs...", RNS.LOG_DEBUG) # TODO: Remove + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) + + locally_processed_file = open(self.storagepath+"/outbound_stamp_costs", "wb") + locally_processed_file.write(msgpack.packb(self.outbound_stamp_costs)) + locally_processed_file.close() + + except Exception as e: + RNS.log("Could not save locally processed message ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + def exit_handler(self): if self.propagation_node: try: @@ -919,6 +1003,13 @@ class LXMRouter: return False def handle_outbound(self, lxmessage): + if lxmessage.stamp_cost == None: + destination_hash = lxmessage.get_destination().hash + if destination_hash in self.outbound_stamp_costs: + stamp_cost = self.outbound_stamp_costs[destination_hash][1] + lxmessage.stamp_cost = stamp_cost + RNS.log(f"No stamp cost set on LXM to {RNS.prettyhexrep(destination_hash)}, autoconfigured to {stamp_cost}, as required by latest announce", RNS.LOG_DEBUG) + lxmessage.state = LXMessage.OUTBOUND if not lxmessage.packed: lxmessage.pack() @@ -946,6 +1037,20 @@ class LXMRouter: try: message = LXMessage.unpack_from_bytes(lxmf_data) + required_stamp_cost = self.delivery_destinations[message.destination_hash].stamp_cost + if required_stamp_cost != None: + if message.validate_stamp(required_stamp_cost): + message.stamp_valid = True + else: + message.stamp_valid = False + + if not message.stamp_valid: + if self._enforce_stamps: + RNS.log(f"Dropping {message} with invalid stamp", RNS.LOG_NOTICE) + return False + else: + RNS.log(f"Received {message} with invalid stamp, but allowing anyway, since stamp enforcement is disabled", RNS.LOG_NOTICE) + if phy_stats != None: if "rssi" in phy_stats: message.rssi = phy_stats["rssi"] if "snr" in phy_stats: message.snr = phy_stats["snr"] diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index a79f600..76a8661 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -125,6 +125,7 @@ class LXMessage: self.packed = None self.stamp = None self.stamp_cost = stamp_cost + self.stamp_valid = False self.state = LXMessage.DRAFT self.method = LXMessage.UNKNOWN self.progress = 0.0 From 4b5e27a5e22620440cd1ae26e749d34c16fe9806 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 6 Sep 2024 20:34:45 +0200 Subject: [PATCH 213/436] Updated examples --- docs/example_receiver.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/example_receiver.py b/docs/example_receiver.py index 02ea0c3..8effd3e 100644 --- a/docs/example_receiver.py +++ b/docs/example_receiver.py @@ -2,6 +2,9 @@ import RNS import LXMF import time +required_stamp_cost = 8 +enforce_stamps = False + def delivery_callback(message): time_string = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(message.timestamp)) signature_string = "Signature is invalid, reason undetermined" @@ -13,11 +16,10 @@ def delivery_callback(message): if message.unverified_reason == LXMF.LXMessage.SOURCE_UNKNOWN: signature_string = "Cannot verify, source is unknown" - required_stamp_cost = 12 - if message.validate_stamp(required_stamp_cost): - stamp_string = "Valid" + if message.stamp_valid: + stamp_string = "Validated" else: - stamp_string = "Not valid" + stamp_string = "Invalid" RNS.log("\t+--- LXMF Delivery ---------------------------------------------") RNS.log("\t| Source hash : "+RNS.prettyhexrep(message.source_hash)) @@ -35,9 +37,9 @@ def delivery_callback(message): r = RNS.Reticulum() -router = LXMF.LXMRouter(storagepath="./tmp1") +router = LXMF.LXMRouter(storagepath="./tmp1", enforce_stamps=enforce_stamps) identity = RNS.Identity() -my_lxmf_destination = router.register_delivery_identity(identity) +my_lxmf_destination = router.register_delivery_identity(identity, stamp_cost=required_stamp_cost) router.register_delivery_callback(delivery_callback) RNS.log("Ready to receive on: "+RNS.prettyhexrep(my_lxmf_destination.hash)) @@ -54,7 +56,7 @@ RNS.log("Ready to receive on: "+RNS.prettyhexrep(my_lxmf_destination.hash)) while True: input() RNS.log("Announcing lxmf.delivery destination...") - my_lxmf_destination.announce() + router.announce(my_lxmf_destination.hash) # input() # RNS.log("Requesting messages from propagation node...") From 0d76eee6cd25dafd3b0f7e15d28b346cea7d97e5 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 7 Sep 2024 11:35:17 +0200 Subject: [PATCH 214/436] Stamp cost API functions and multi-process stamp generation on Android --- LXMF/LXMF.py | 13 +++- LXMF/LXMRouter.py | 59 +++++++++++++- LXMF/LXMessage.py | 164 +++++++++++++++++++++++++++++---------- docs/example_receiver.py | 2 +- 4 files changed, 189 insertions(+), 49 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 0fe4a2c..3618912 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -85,6 +85,8 @@ import RNS.vendor.umsgpack as msgpack def display_name_from_app_data(app_data=None): if app_data == None: return None + elif len(app_data) == 0: + return None else: # Version 0.5.0+ announce format if (app_data[0] >= 0x90 and app_data[0] <= 0x9f) or app_data[0] == 0xdc: @@ -93,7 +95,16 @@ def display_name_from_app_data(app_data=None): if len(peer_data) < 1: return None else: - return peer_data[0].decode("utf-8") + dn = peer_data[0] + if dn == None: + return None + else: + try: + decoded = dn.decode("utf-8") + return decoded + except: + RNS.log("Could not decode display name in included announce data. The contained exception was: {e}", RNS.LOG_ERROR) + return None # Original announce format else: diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 5e3bf36..e0d61fe 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -173,9 +173,9 @@ class LXMRouter: job_thread.setDaemon(True) job_thread.start() - def announce(self, destination_hash): + def announce(self, destination_hash, attached_interface=None): if destination_hash in self.delivery_destinations: - self.delivery_destinations[destination_hash].announce(app_data=self.get_announce_app_data(destination_hash)) + self.delivery_destinations[destination_hash].announce(app_data=self.get_announce_app_data(destination_hash), attached_interface=attached_interface) def announce_propagation_node(self): def delayed_announce(): @@ -202,7 +202,6 @@ class LXMRouter: delivery_destination.set_packet_callback(self.delivery_packet) delivery_destination.set_link_established_callback(self.delivery_link_established) delivery_destination.display_name = display_name - delivery_destination.stamp_cost = stamp_cost if self.enforce_ratchets: delivery_destination.enforce_ratchets() @@ -214,11 +213,38 @@ class LXMRouter: delivery_destination.set_default_app_data(get_app_data) self.delivery_destinations[delivery_destination.hash] = delivery_destination + self.set_inbound_stamp_cost(delivery_destination.hash, stamp_cost) + return delivery_destination def register_delivery_callback(self, callback): self.__delivery_callback = callback + def set_inbound_stamp_cost(self, destination_hash, stamp_cost): + if destination_hash in self.delivery_destinations: + delivery_destination = self.delivery_destinations[destination_hash] + if stamp_cost == None: + delivery_destination.stamp_cost = None + return True + elif type(stamp_cost) == int: + if stamp_cost < 1: + delivery_destination.stamp_cost = None + elif stamp_cost < 255: + delivery_destination.stamp_cost = stamp_cost + else: + return False + + return True + + return False + + def get_outbound_stamp_cost(self, destination_hash): + if destination_hash in self.outbound_stamp_costs: + stamp_cost = self.outbound_stamp_costs[destination_hash][1] + return stamp_cost + else: + return None + def set_outbound_propagation_node(self, destination_hash): if len(destination_hash) != RNS.Identity.TRUNCATED_HASHLENGTH//8 or type(destination_hash) != bytes: raise ValueError("Invalid destination hash for outbound propagation node") @@ -1020,7 +1046,13 @@ class LXMRouter: time.sleep(0.1) self.pending_outbound.append(lxmessage) - self.process_outbound() + + if lxmessage.defer_stamp and lxmessage.stamp_cost == None: + RNS.log(f"Deferred stamp generation was requested for {lxmessage}, but no stamp is required, processing immediately", RNS.LOG_DEBUG) + lxmessage.defer_stamp = False + + if not lxmessage.defer_stamp: + self.process_outbound() def get_outbound_progress(self, lxm_hash): for lxm in self.pending_outbound: @@ -1029,6 +1061,13 @@ class LXMRouter: return None + def get_outbound_lxm_stamp_cost(self, lxm_hash): + for lxm in self.pending_outbound: + if lxm.hash == lxm_hash: + return lxm.stamp_cost + + return None + ### Message Routing & Delivery ######################## ####################################################### @@ -1440,6 +1479,18 @@ class LXMRouter: self.pending_outbound.remove(lxmessage) else: RNS.log("Starting outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + + # Handle potentially deferred stamp generation + if lxmessage.defer_stamp and lxmessage.stamp == None: + RNS.log(f"Generating deferred stamp for {lxmessage} now", RNS.LOG_DEBUG) + lxmessage.stamp = lxmessage.get_stamp() + lxmessage.defer_stamp = False + lxmessage.packed = None + lxmessage.pack() + + if lxmessage.progress == None or lxmessage.progress < 0.01: + lxmessage.progress = 0.01 + # Outbound handling for opportunistic messages if lxmessage.method == LXMessage.OPPORTUNISTIC: if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 76a8661..3b4956d 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -10,13 +10,13 @@ import multiprocessing from .LXMF import APP_NAME class LXMessage: - DRAFT = 0x00 + GENERATING = 0x00 OUTBOUND = 0x01 SENDING = 0x02 SENT = 0x04 DELIVERED = 0x08 FAILED = 0xFF - states = [DRAFT, OUTBOUND, SENDING, SENT, DELIVERED, FAILED] + states = [GENERATING, OUTBOUND, SENDING, SENT, DELIVERED, FAILED] UNKNOWN = 0x00 PACKET = 0x01 @@ -126,7 +126,8 @@ class LXMessage: self.stamp = None self.stamp_cost = stamp_cost self.stamp_valid = False - self.state = LXMessage.DRAFT + self.defer_stamp = False + self.state = LXMessage.GENERATING self.method = LXMessage.UNKNOWN self.progress = 0.0 self.rssi = None @@ -277,53 +278,128 @@ class LXMessage: start_time = time.time() total_rounds = 0 - stop_event = multiprocessing.Event() - result_queue = multiprocessing.Queue(maxsize=1) - rounds_queue = multiprocessing.Queue() - def job(stop_event): - terminated = False - rounds = 0 - - stamp = os.urandom(256//8) - while not LXMessage.stamp_valid(stamp, self.stamp_cost, workblock): - if stop_event.is_set(): - break - - if timeout != None and rounds % 10000 == 0: - if time.time() > start_time + timeout: - RNS.log(f"Stamp generation for {self} timed out", RNS.LOG_ERROR) - return None + if not RNS.vendor.platformutils.is_android(): + stop_event = multiprocessing.Event() + result_queue = multiprocessing.Queue(maxsize=1) + rounds_queue = multiprocessing.Queue() + def job(stop_event): + terminated = False + rounds = 0 stamp = os.urandom(256//8) - rounds += 1 + while not LXMessage.stamp_valid(stamp, self.stamp_cost, workblock): + if stop_event.is_set(): + break - rounds_queue.put(rounds) - if not stop_event.is_set(): - result_queue.put(stamp) + if timeout != None and rounds % 10000 == 0: + if time.time() > start_time + timeout: + RNS.log(f"Stamp generation for {self} timed out", RNS.LOG_ERROR) + return None - job_procs = [] - jobs = multiprocessing.cpu_count() - for _ in range(jobs): - process = multiprocessing.Process(target=job, kwargs={"stop_event": stop_event},) - job_procs.append(process) - process.start() + stamp = os.urandom(256//8) + rounds += 1 - stamp = result_queue.get() - stop_event.set() + rounds_queue.put(rounds) + if not stop_event.is_set(): + result_queue.put(stamp) - for j in range(jobs): - process = job_procs[j] - process.join() - total_rounds += rounds_queue.get() + job_procs = [] + jobs = multiprocessing.cpu_count() + for _ in range(jobs): + process = multiprocessing.Process(target=job, kwargs={"stop_event": stop_event},) + job_procs.append(process) + process.start() - duration = time.time() - start_time - rounds = total_rounds + stamp = result_queue.get() + stop_event.set() + + for j in range(jobs): + process = job_procs[j] + process.join() + total_rounds += rounds_queue.get() + + duration = time.time() - start_time + rounds = total_rounds + + else: + # Semaphore support is flaky to non-existent on + # Android, so we need to manually dispatch and + # manage workloads here, while periodically + # checking in on the progress. + + use_nacl = False + try: + import nacl.encoding + import nacl.hash + use_nacl = True + except: + pass + + def full_hash(m): + if use_nacl: + return nacl.hash.sha256(m, encoder=nacl.encoding.RawEncoder) + else: + return RNS.Identity.full_hash(m) + + def sv(s, c, w): + target = 0b1<<256-c + m = w+s + result = full_hash(m) + if int.from_bytes(result, byteorder="big") > target: + return False + else: + return True + + stamp = None + wm = multiprocessing.Manager() + jobs = multiprocessing.cpu_count() + + RNS.log(f"Dispatching {jobs} workers for stamp generation...") # TODO: Remove + + results_dict = wm.dict() + while stamp == None: + job_procs = [] + + def job(procnum=None, results_dict=None, wb=None): + RNS.log(f"Worker {procnum} starting...") # TODO: Remove + rounds = 0 + + stamp = os.urandom(256//8) + while not sv(stamp, self.stamp_cost, wb): + if rounds >= 500: + stamp = None + RNS.log(f"Worker {procnum} found no result in {rounds} rounds") # TODO: Remove + break + + stamp = os.urandom(256//8) + rounds += 1 + + results_dict[procnum] = [stamp, rounds] + + for pnum in range(jobs): + process = multiprocessing.Process(target=job, kwargs={"procnum":pnum, "results_dict": results_dict, "wb": workblock},) + job_procs.append(process) + process.start() + + for process in job_procs: + process.join() + + for j in results_dict: + r = results_dict[j] + RNS.log(f"Result from {r}: {r[1]} rounds, stamp: {r[0]}") # TODO: Remove + total_rounds += r[1] + if r[0] != None: + stamp = r[0] + RNS.log(f"Found stamp: {stamp}") # TODO: Remove + + duration = time.time() - start_time + rounds = total_rounds # TODO: Remove stats output RNS.log(f"Stamp generated in {RNS.prettytime(duration)} / {rounds} rounds", RNS.LOG_DEBUG) - RNS.log(f"Rounds per second {int(rounds/duration)}", RNS.LOG_DEBUG) - RNS.log(f"Stamp: {RNS.hexrep(stamp)}", RNS.LOG_DEBUG) - RNS.log(f"Resulting hash: {RNS.hexrep(RNS.Identity.full_hash(workblock+stamp))}", RNS.LOG_DEBUG) + # RNS.log(f"Rounds per second {int(rounds/duration)}", RNS.LOG_DEBUG) + # RNS.log(f"Stamp: {RNS.hexrep(stamp)}", RNS.LOG_DEBUG) + # RNS.log(f"Resulting hash: {RNS.hexrep(RNS.Identity.full_hash(workblock+stamp))}", RNS.LOG_DEBUG) ########################### return stamp @@ -344,9 +420,11 @@ class LXMessage: hashed_part += msgpack.packb(self.payload) self.hash = RNS.Identity.full_hash(hashed_part) self.message_id = self.hash - self.stamp = self.get_stamp() - if self.stamp != None: - self.payload.append(self.stamp) + + if not self.defer_stamp: + self.stamp = self.get_stamp() + if self.stamp != None: + self.payload.append(self.stamp) signed_part = b"" signed_part += hashed_part diff --git a/docs/example_receiver.py b/docs/example_receiver.py index 8effd3e..64f914a 100644 --- a/docs/example_receiver.py +++ b/docs/example_receiver.py @@ -39,7 +39,7 @@ r = RNS.Reticulum() router = LXMF.LXMRouter(storagepath="./tmp1", enforce_stamps=enforce_stamps) identity = RNS.Identity() -my_lxmf_destination = router.register_delivery_identity(identity, stamp_cost=required_stamp_cost) +my_lxmf_destination = router.register_delivery_identity(identity, display_name="Anonymous Peer", stamp_cost=required_stamp_cost) router.register_delivery_callback(delivery_callback) RNS.log("Ready to receive on: "+RNS.prettyhexrep(my_lxmf_destination.hash)) From 74cbd114736fbe3f4f651b0e15aa640ec3324000 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 7 Sep 2024 11:37:39 +0200 Subject: [PATCH 215/436] Cleanup --- LXMF/LXMRouter.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index e0d61fe..56ad297 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -58,10 +58,7 @@ class LXMRouter: ### Developer-facing API ############################## ####################################################### - def __init__(self, identity = None, storagepath = None, autopeer = AUTOPEER, autopeer_maxdepth = None, - propagation_limit = PROPAGATION_LIMIT, delivery_limit = DELIVERY_LIMIT, enforce_ratchets = False, - enforce_stamps = False): - + def __init__(self, identity = None, storagepath = None, autopeer = AUTOPEER, autopeer_maxdepth = None, propagation_limit = PROPAGATION_LIMIT, delivery_limit = DELIVERY_LIMIT, enforce_ratchets = False, enforce_stamps = False): random.seed(os.urandom(10)) self.pending_inbound = [] From 62f5a9eeadffccbd6e751dbc7da949f1ddc7f54c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 7 Sep 2024 15:17:48 +0200 Subject: [PATCH 216/436] Implemented ticket exchanges. Implemented stamp generation and validation by tickets. --- LXMF/LXMRouter.py | 175 +++++++++++++++++++++++++++++++++++++-- LXMF/LXMessage.py | 98 ++++++++++++++++------ docs/example_receiver.py | 7 ++ docs/example_sender.py | 6 +- 4 files changed, 251 insertions(+), 35 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 56ad297..2ff5d75 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -9,6 +9,7 @@ import RNS import RNS.vendor.umsgpack as msgpack from .LXMF import APP_NAME +from .LXMF import FIELD_TICKET from .LXMPeer import LXMPeer from .LXMessage import LXMessage @@ -105,8 +106,10 @@ class LXMRouter: self.locally_delivered_transient_ids = {} self.locally_processed_transient_ids = {} self.outbound_stamp_costs = {} + self.available_tickets = {"outbound": {}, "inbound": {}, "last_deliveries": {}} self.cost_file_lock = threading.Lock() + self.ticket_file_lock = threading.Lock() if identity == None: identity = RNS.Identity() @@ -164,6 +167,31 @@ class LXMRouter: except Exception as e: RNS.log("Could not load outbound stamp costs from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + try: + if os.path.isfile(self.storagepath+"/available_tickets"): + with self.ticket_file_lock: + with open(self.storagepath+"/available_tickets", "rb") as available_tickets_file: + data = available_tickets_file.read() + self.available_tickets = msgpack.unpackb(data) + if not type(self.available_tickets) == dict: + RNS.log("Invalid data format for loaded available tickets, recreating...", RNS.LOG_ERROR) + self.available_tickets = {"outbound": {}, "inbound": {}, "last_deliveries": {}} + if not "outbound" in self.available_tickets: + RNS.log("Missing outbound entry in loaded available tickets, recreating...", RNS.LOG_ERROR) + self.available_tickets["outbound"] = {} + if not "inbound" in self.available_tickets: + RNS.log("Missing inbound entry in loaded available tickets, recreating...", RNS.LOG_ERROR) + self.available_tickets["inbound"] = {} + if not "last_deliveries" in self.available_tickets: + RNS.log("Missing local_deliveries entry in loaded available tickets, recreating...", RNS.LOG_ERROR) + self.available_tickets["last_deliveries"] = {} + + self.clean_available_tickets() + self.save_available_tickets() + + except Exception as e: + RNS.log("Could not load outbound stamp costs from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + atexit.register(self.exit_handler) job_thread = threading.Thread(target=self.jobloop) @@ -191,6 +219,10 @@ class LXMRouter: da_thread.start() def register_delivery_identity(self, identity, display_name = None, stamp_cost = None): + if len(self.delivery_destinations) != 0: + RNS.log("Currently only one delivery identity is supported per LXMF router instance", RNS.LOG_ERROR) + return None + if not os.path.isdir(self.ratchetpath): os.makedirs(self.ratchetpath) @@ -647,6 +679,61 @@ class LXMRouter: return weight + def generate_ticket(self, destination_hash, expiry=LXMessage.TICKET_EXPIRY): + now = time.time() + ticket = None + if destination_hash in self.available_tickets["last_deliveries"]: + last_delivery = self.available_tickets["last_deliveries"][destination_hash] + elapsed = now - last_delivery + if elapsed < LXMessage.TICKET_INTERVAL: + RNS.log(f"A ticket for {RNS.prettyhexrep(destination_hash)} was already delivered {RNS.prettytime(elapsed)} ago, not including another ticket yet", RNS.LOG_DEBUG) + return None + + if destination_hash in self.available_tickets["inbound"]: + for ticket in self.available_tickets["inbound"][destination_hash]: + ticket_entry = self.available_tickets["inbound"][destination_hash][ticket] + expires = ticket_entry[0]; validity_left = expires - now + if validity_left > LXMessage.TICKET_RENEW: + RNS.log(f"Found generated ticket for {RNS.prettyhexrep(destination_hash)} with {RNS.prettytime(validity_left)} of validity left, re-using this one", RNS.LOG_DEBUG) + return [expires, ticket] + + else: + self.available_tickets["inbound"][destination_hash] = {} + + RNS.log(f"No generated tickets for {RNS.prettyhexrep(destination_hash)} with enough validity found, generating a new one", RNS.LOG_DEBUG) + expires = now+expiry + ticket = os.urandom(LXMessage.TICKET_LENGTH) + self.available_tickets["inbound"][destination_hash][ticket] = [expires] + self.save_available_tickets() + + return [expires, ticket] + + def remember_ticket(self, destination_hash, ticket_entry): + expires = ticket_entry[0]-time.time() + RNS.log(f"Remembering ticket for {RNS.prettyhexrep(destination_hash)}, expires in {RNS.prettytime(expires)}", RNS.LOG_DEBUG) + self.available_tickets["outbound"][destination_hash] = [ticket_entry[0], ticket_entry[1]] + + def get_outbound_ticket(self, destination_hash): + if destination_hash in self.available_tickets["outbound"]: + entry = self.available_tickets["outbound"][destination_hash] + if entry[0] > time.time(): + return entry[1] + + return None + + def get_inbound_tickets(self, destination_hash): + now = time.time() + available_tickets = [] + if destination_hash in self.available_tickets["inbound"]: + for inbound_ticket in self.available_tickets["inbound"][destination_hash]: + if now < self.available_tickets["inbound"][destination_hash][inbound_ticket][0]: + available_tickets.append(inbound_ticket) + + if len(available_tickets) == 0: + return None + else: + return available_tickets + def get_size(self, transient_id): lxm_size = self.propagation_entries[transient_id][3] return lxm_size @@ -778,13 +865,57 @@ class LXMRouter: if not os.path.isdir(self.storagepath): os.makedirs(self.storagepath) - locally_processed_file = open(self.storagepath+"/outbound_stamp_costs", "wb") - locally_processed_file.write(msgpack.packb(self.outbound_stamp_costs)) - locally_processed_file.close() + outbound_stamp_costs_file = open(self.storagepath+"/outbound_stamp_costs", "wb") + outbound_stamp_costs_file.write(msgpack.packb(self.outbound_stamp_costs)) + outbound_stamp_costs_file.close() except Exception as e: RNS.log("Could not save locally processed message ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + def clean_available_tickets(self): + try: + # Clean outbound tickets + expired_outbound = [] + for destination_hash in self.available_tickets["outbound"]: + entry = self.available_tickets["outbound"][destination_hash] + if time.time() > entry[0]: + expired_outbound.append(destination_hash) + + for destination_hash in expired_outbound: + RNS.log(f"Cleaning expired outbound ticket for {destination_hash}") # TODO: Remove + self.available_tickets["outbound"].pop(destination_hash) + + # Clean inbound tickets + for destination_hash in self.available_tickets["inbound"]: + expired_inbound = [] + for inbound_ticket in self.available_tickets["inbound"][destination_hash]: + entry = self.available_tickets["inbound"][destination_hash][inbound_ticket] + ticket_expiry = entry[0] + if time.time() > ticket_expiry+LXMessage.TICKET_GRACE: + expired_inbound.append(inbound_ticket) + + for inbound_ticket in expired_inbound: + RNS.log(f"Cleaning expired inbound ticket for {destination_hash}") # TODO: Remove + self.available_tickets["inbound"][destination_hash].pop(destination_hash) + + except Exception as e: + RNS.log(f"Error while cleaning outbound stamp costs. The contained exception was: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + def save_available_tickets(self): + with self.ticket_file_lock: + try: + RNS.log("Saving available tickets...", RNS.LOG_DEBUG) # TODO: Remove + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) + + available_tickets_file = open(self.storagepath+"/available_tickets", "wb") + available_tickets_file.write(msgpack.packb(self.available_tickets)) + available_tickets_file.close() + + except Exception as e: + RNS.log("Could not save available tickets to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + def exit_handler(self): if self.propagation_node: try: @@ -1026,14 +1157,29 @@ class LXMRouter: return False def handle_outbound(self, lxmessage): + destination_hash = lxmessage.get_destination().hash if lxmessage.stamp_cost == None: - destination_hash = lxmessage.get_destination().hash if destination_hash in self.outbound_stamp_costs: stamp_cost = self.outbound_stamp_costs[destination_hash][1] lxmessage.stamp_cost = stamp_cost RNS.log(f"No stamp cost set on LXM to {RNS.prettyhexrep(destination_hash)}, autoconfigured to {stamp_cost}, as required by latest announce", RNS.LOG_DEBUG) lxmessage.state = LXMessage.OUTBOUND + + # If an outbound ticket is available for this + # destination, attach it to the message. + lxmessage.outbound_ticket = self.get_outbound_ticket(destination_hash) + if lxmessage.outbound_ticket != None and lxmessage.defer_stamp: + RNS.log(f"Deferred stamp generation was requested for {lxmessage}, but outbound ticket was applied, processing immediately", RNS.LOG_DEBUG) + lxmessage.defer_stamp = False + + # If requested, include a ticket to allow the + # destination to reply without generating a stamp. + if lxmessage.include_ticket: + ticket = self.generate_ticket(lxmessage.destination_hash) + if ticket: + lxmessage.fields[FIELD_TICKET] = ticket + if not lxmessage.packed: lxmessage.pack() @@ -1073,9 +1219,23 @@ class LXMRouter: try: message = LXMessage.unpack_from_bytes(lxmf_data) + if message.signature_validated and FIELD_TICKET in message.fields: + ticket_entry = message.fields[FIELD_TICKET] + if type(ticket_entry) == list and len(ticket_entry) > 1: + expires = ticket_entry[0] + ticket = ticket_entry[1] + + if time.time() < expires: + if type(ticket) == bytes and len(ticket) == LXMessage.TICKET_LENGTH: + self.remember_ticket(message.source_hash, ticket_entry) + def save_job(): + self.save_available_tickets() + threading.Thread(target=save_job, daemon=True).start() + required_stamp_cost = self.delivery_destinations[message.destination_hash].stamp_cost if required_stamp_cost != None: - if message.validate_stamp(required_stamp_cost): + destination_tickets = self.get_inbound_tickets(message.source_hash) + if message.validate_stamp(required_stamp_cost, tickets=destination_tickets): message.stamp_valid = True else: message.stamp_valid = False @@ -1471,6 +1631,11 @@ class LXMRouter: if lxmessage.state == LXMessage.DELIVERED: RNS.log("Delivery has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) self.pending_outbound.remove(lxmessage) + if lxmessage.include_ticket and FIELD_TICKET in lxmessage.fields: + RNS.log(f"Updating latest ticket delivery for {RNS.prettyhexrep(lxmessage.destination_hash)}", RNS.LOG_DEBUG) + self.available_tickets["last_deliveries"][lxmessage.destination_hash] = time.time() + self.save_available_tickets() + elif lxmessage.method == LXMessage.PROPAGATED and lxmessage.state == LXMessage.SENT: RNS.log("Propagation has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) self.pending_outbound.remove(lxmessage) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 3b4956d..14ff299 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -35,6 +35,17 @@ class LXMessage: DESTINATION_LENGTH = RNS.Identity.TRUNCATED_HASHLENGTH//8 SIGNATURE_LENGTH = RNS.Identity.SIGLENGTH//8 + TICKET_LENGTH = RNS.Identity.TRUNCATED_HASHLENGTH//8 + + # Default ticket expiry is 3 weeks, with an + # additional grace period of 5 days, allowing + # for timekeeping inaccuracies. Tickets will + # automatically renew when there is less than + # 14 days to expiry. + TICKET_EXPIRY = 21*24*60*60 + TICKET_GRACE = 5*24*60*60 + TICKET_RENEW = 14*24*60*60 + TICKET_INTERVAL = 3*24*60*60 # LXMF overhead is 111 bytes per message: # 16 bytes for destination hash @@ -93,8 +104,7 @@ class LXMessage: else: return "" - def __init__(self, destination, source, content = "", title = "", fields = None, desired_method = None, - destination_hash = None, source_hash = None, stamp_cost=None): + def __init__(self, destination, source, content = "", title = "", fields = None, desired_method = None, destination_hash = None, source_hash = None, stamp_cost=None, include_ticket=False): if isinstance(destination, RNS.Destination) or destination == None: self.__destination = destination @@ -114,25 +124,31 @@ class LXMessage: else: raise ValueError("LXMessage initialised with invalid source") + if title == None: + title = "" + self.set_title_from_string(title) self.set_content_from_string(content) self.set_fields(fields) - self.payload = None - self.timestamp = None - self.signature = None - self.hash = None - self.packed = None - self.stamp = None - self.stamp_cost = stamp_cost - self.stamp_valid = False - self.defer_stamp = False - self.state = LXMessage.GENERATING - self.method = LXMessage.UNKNOWN - self.progress = 0.0 - self.rssi = None - self.snr = None - self.q = None + self.payload = None + self.timestamp = None + self.signature = None + self.hash = None + self.packed = None + self.state = LXMessage.GENERATING + self.method = LXMessage.UNKNOWN + self.progress = 0.0 + self.rssi = None + self.snr = None + self.q = None + + self.stamp = None + self.stamp_cost = stamp_cost + self.stamp_valid = False + self.defer_stamp = False + self.outbound_ticket = None + self.include_ticket = include_ticket self.propagation_packed = None self.paper_packed = None @@ -254,7 +270,13 @@ class LXMessage: else: return True - def validate_stamp(self, target_cost): + def validate_stamp(self, target_cost, tickets=None): + if tickets != None: + for ticket in tickets: + if self.stamp == RNS.Identity.truncated_hash(ticket+self.message_id): + RNS.log(f"Stamp on {self} validated by inbound ticket", RNS.LOG_DEBUG) # TODO: Remove at some point + return True + if self.stamp == None: return False else: @@ -264,14 +286,25 @@ class LXMessage: return False def get_stamp(self, timeout=None): - if self.stamp_cost == None: + # If an outbound ticket exists, use this for + # generating a valid stamp. + if self.outbound_ticket != None and type(self.outbound_ticket) == bytes and len(self.outbound_ticket) == LXMessage.TICKET_LENGTH: + RNS.log(f"Generating stamp with outbound ticket for {self}", RNS.LOG_DEBUG) # TODO: Remove at some point + return RNS.Identity.truncated_hash(self.outbound_ticket+self.message_id) + + # If no stamp cost is required, we can just + # return immediately. + elif self.stamp_cost == None: return None + # If a stamp was already generated, return + # it immediately. elif self.stamp != None: - # TODO: Check that message hash cannot actually - # change under any circumstances before handoff return self.stamp + # Otherwise, we will need to generate a + # valid stamp according to the cost that + # the receiver has specified. else: RNS.log(f"Generating stamp with cost {self.stamp_cost} for {self}...", RNS.LOG_DEBUG) workblock = LXMessage.stamp_workblock(self.message_id) @@ -279,6 +312,7 @@ class LXMessage: total_rounds = 0 if not RNS.vendor.platformutils.is_android(): + RNS.log("Preparing IPC semaphores", RNS.LOG_DEBUG) # TODO: Remove stop_event = multiprocessing.Event() result_queue = multiprocessing.Queue(maxsize=1) rounds_queue = multiprocessing.Queue() @@ -305,14 +339,17 @@ class LXMessage: job_procs = [] jobs = multiprocessing.cpu_count() + RNS.log("Starting workers", RNS.LOG_DEBUG) # TODO: Remove for _ in range(jobs): process = multiprocessing.Process(target=job, kwargs={"stop_event": stop_event},) job_procs.append(process) process.start() + RNS.log("Awaiting results on queue", RNS.LOG_DEBUG) # TODO: Remove stamp = result_queue.get() stop_event.set() + RNS.log("Joining worker processes", RNS.LOG_DEBUG) # TODO: Remove for j in range(jobs): process = job_procs[j] process.join() @@ -354,21 +391,21 @@ class LXMessage: wm = multiprocessing.Manager() jobs = multiprocessing.cpu_count() - RNS.log(f"Dispatching {jobs} workers for stamp generation...") # TODO: Remove + # RNS.log(f"Dispatching {jobs} workers for stamp generation...") # TODO: Remove results_dict = wm.dict() while stamp == None: job_procs = [] def job(procnum=None, results_dict=None, wb=None): - RNS.log(f"Worker {procnum} starting...") # TODO: Remove + # RNS.log(f"Worker {procnum} starting...") # TODO: Remove rounds = 0 stamp = os.urandom(256//8) while not sv(stamp, self.stamp_cost, wb): if rounds >= 500: stamp = None - RNS.log(f"Worker {procnum} found no result in {rounds} rounds") # TODO: Remove + # RNS.log(f"Worker {procnum} found no result in {rounds} rounds") # TODO: Remove break stamp = os.urandom(256//8) @@ -386,17 +423,24 @@ class LXMessage: for j in results_dict: r = results_dict[j] - RNS.log(f"Result from {r}: {r[1]} rounds, stamp: {r[0]}") # TODO: Remove + # RNS.log(f"Result from {r}: {r[1]} rounds, stamp: {r[0]}") # TODO: Remove total_rounds += r[1] if r[0] != None: stamp = r[0] - RNS.log(f"Found stamp: {stamp}") # TODO: Remove + # RNS.log(f"Found stamp: {stamp}") # TODO: Remove + + if stamp == None: + elapsed = time.time() - start_time + speed = total_rounds/elapsed + RNS.log(f"Stamp generation for {self} running. {total_rounds} rounds completed so far, {int(speed)} rounds per second", RNS.LOG_DEBUG) duration = time.time() - start_time rounds = total_rounds + + speed = total_rounds/duration # TODO: Remove stats output - RNS.log(f"Stamp generated in {RNS.prettytime(duration)} / {rounds} rounds", RNS.LOG_DEBUG) + RNS.log(f"Stamp generated in {RNS.prettytime(duration)}, {rounds} rounds, {int(speed)} rounds per second", RNS.LOG_DEBUG) # RNS.log(f"Rounds per second {int(rounds/duration)}", RNS.LOG_DEBUG) # RNS.log(f"Stamp: {RNS.hexrep(stamp)}", RNS.LOG_DEBUG) # RNS.log(f"Resulting hash: {RNS.hexrep(RNS.Identity.full_hash(workblock+stamp))}", RNS.LOG_DEBUG) diff --git a/docs/example_receiver.py b/docs/example_receiver.py index 64f914a..7d3aa20 100644 --- a/docs/example_receiver.py +++ b/docs/example_receiver.py @@ -6,6 +6,7 @@ required_stamp_cost = 8 enforce_stamps = False def delivery_callback(message): + global my_lxmf_destination, router time_string = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(message.timestamp)) signature_string = "Signature is invalid, reason undetermined" if message.signature_validated: @@ -35,6 +36,12 @@ def delivery_callback(message): RNS.log("\t| Stamp : "+stamp_string) RNS.log("\t+---------------------------------------------------------------") + # Optionally, send a reply + # source = my_lxmf_destination + # dest = message.source + # lxm = LXMF.LXMessage(dest, source, "Reply", None, desired_method=LXMF.LXMessage.DIRECT, include_ticket=True) + # router.handle_outbound(lxm) + r = RNS.Reticulum() router = LXMF.LXMRouter(storagepath="./tmp1", enforce_stamps=enforce_stamps) diff --git a/docs/example_sender.py b/docs/example_sender.py index 3dc6494..bcb8d36 100644 --- a/docs/example_sender.py +++ b/docs/example_sender.py @@ -14,7 +14,7 @@ r = RNS.Reticulum() router = LXMF.LXMRouter(storagepath="./tmp2") router.register_delivery_callback(delivery_callback) ident = RNS.Identity() -source = router.register_delivery_identity(ident, display_name=random_names[random.randint(0,len(random_names)-1)]) +source = router.register_delivery_identity(ident, display_name=random_names[random.randint(0,len(random_names)-1)], stamp_cost=8) router.announce(source.hash) RNS.log("Source announced") @@ -42,14 +42,14 @@ while True: lxm = LXMF.LXMessage(dest, source, random_msgs[random.randint(0,len(random_msgs)-1)], random_titles[random.randint(0,len(random_titles)-1)], - desired_method=LXMF.LXMessage.DIRECT) + desired_method=LXMF.LXMessage.DIRECT, include_ticket=True) # Or, create an oppertunistic, single-packet message # for sending without first establishing a link: # lxm = LXMF.LXMessage(dest, source, "This is a test", # random_titles[random.randint(0,len(random_titles)-1)], - # desired_method=LXMF.LXMessage.OPPORTUNISTIC) + # desired_method=LXMF.LXMessage.OPPORTUNISTIC, include_ticket=True) # Or, try sending the message via a propagation node: From 40eb014c917c33927384c59ddd9b66533d839580 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 7 Sep 2024 22:40:13 +0200 Subject: [PATCH 217/436] Implemented deferred multiprocessor stamp generation in the background --- LXMF/LXMRouter.py | 101 ++++++++++++++++++++---- LXMF/LXMessage.py | 190 +++++++++++++++++++++++++++------------------- 2 files changed, 200 insertions(+), 91 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 2ff5d75..9a82a13 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -95,6 +95,7 @@ class LXMRouter: self.delivery_per_transfer_limit = delivery_limit self.enforce_ratchets = enforce_ratchets self._enforce_stamps = enforce_stamps + self.pending_deferred_stamps = {} self.wants_download_on_path_available_from = None self.wants_download_on_path_available_to = None @@ -110,6 +111,7 @@ class LXMRouter: self.cost_file_lock = threading.Lock() self.ticket_file_lock = threading.Lock() + self.stamp_gen_lock = threading.Lock() if identity == None: identity = RNS.Identity() @@ -237,8 +239,7 @@ class LXMRouter: if display_name != None: def get_app_data(): - return self.get_announce_app_data(delivery_destination) - + return self.get_announce_app_data(delivery_destination.hash) delivery_destination.set_default_app_data(get_app_data) self.delivery_destinations[delivery_destination.hash] = delivery_destination @@ -540,6 +541,7 @@ class LXMRouter: ####################################################### JOB_OUTBOUND_INTERVAL = 1 + JOB_STAMPS_INTERVAL = 1 JOB_LINKS_INTERVAL = 1 JOB_TRANSIENT_INTERVAL = 60 JOB_STORE_INTERVAL = 120 @@ -550,6 +552,9 @@ class LXMRouter: if self.processing_count % LXMRouter.JOB_OUTBOUND_INTERVAL == 0: self.process_outbound() + if self.processing_count % LXMRouter.JOB_STAMPS_INTERVAL == 0: + threading.Thread(target=self.process_deferred_stamps, daemon=True).start() + if self.processing_count % LXMRouter.JOB_LINKS_INTERVAL == 0: self.clean_links() @@ -721,6 +726,14 @@ class LXMRouter: return None + def get_outbound_ticket_expiry(self, destination_hash): + if destination_hash in self.available_tickets["outbound"]: + entry = self.available_tickets["outbound"][destination_hash] + if entry[0] > time.time(): + return entry[0] + + return None + def get_inbound_tickets(self, destination_hash): now = time.time() available_tickets = [] @@ -916,6 +929,29 @@ class LXMRouter: except Exception as e: RNS.log("Could not save available tickets to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + def reload_available_tickets(self): + RNS.log("Reloading available tickets from storage", RNS.LOG_DEBUG) + try: + with self.ticket_file_lock: + with open(self.storagepath+"/available_tickets", "rb") as available_tickets_file: + data = available_tickets_file.read() + self.available_tickets = msgpack.unpackb(data) + if not type(self.available_tickets) == dict: + RNS.log("Invalid data format for loaded available tickets, recreating...", RNS.LOG_ERROR) + self.available_tickets = {"outbound": {}, "inbound": {}, "last_deliveries": {}} + if not "outbound" in self.available_tickets: + RNS.log("Missing outbound entry in loaded available tickets, recreating...", RNS.LOG_ERROR) + self.available_tickets["outbound"] = {} + if not "inbound" in self.available_tickets: + RNS.log("Missing inbound entry in loaded available tickets, recreating...", RNS.LOG_ERROR) + self.available_tickets["inbound"] = {} + if not "last_deliveries" in self.available_tickets: + RNS.log("Missing local_deliveries entry in loaded available tickets, recreating...", RNS.LOG_ERROR) + self.available_tickets["last_deliveries"] = {} + + except Exception as e: + RNS.log(f"An error occurred while reloading available tickets from storage: {e}", RNS.LOG_ERROR) + def exit_handler(self): if self.propagation_node: try: @@ -1188,19 +1224,24 @@ class LXMRouter: while self.processing_outbound: time.sleep(0.1) - self.pending_outbound.append(lxmessage) - if lxmessage.defer_stamp and lxmessage.stamp_cost == None: RNS.log(f"Deferred stamp generation was requested for {lxmessage}, but no stamp is required, processing immediately", RNS.LOG_DEBUG) lxmessage.defer_stamp = False if not lxmessage.defer_stamp: + self.pending_outbound.append(lxmessage) self.process_outbound() + else: + self.pending_deferred_stamps[lxmessage.message_id] = lxmessage def get_outbound_progress(self, lxm_hash): for lxm in self.pending_outbound: if lxm.hash == lxm_hash: return lxm.progress + + for lxm_id in self.pending_deferred_stamps: + if self.pending_deferred_stamps[lxm_id].hash == lxm_hash: + return self.pending_deferred_stamps[lxm_id].progress return None @@ -1208,6 +1249,10 @@ class LXMRouter: for lxm in self.pending_outbound: if lxm.hash == lxm_hash: return lxm.stamp_cost + + for lxm_id in self.pending_deferred_stamps: + if self.pending_deferred_stamps[lxm_id].hash == lxm_hash: + return self.pending_deferred_stamps[lxm_id].stamp_cost return None @@ -1616,13 +1661,51 @@ class LXMRouter: def fail_message(self, lxmessage): RNS.log(str(lxmessage)+" failed to send", RNS.LOG_DEBUG) - self.pending_outbound.remove(lxmessage) + if lxmessage in self.pending_outbound: + self.pending_outbound.remove(lxmessage) + self.failed_outbound.append(lxmessage) lxmessage.state = LXMessage.FAILED if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): lxmessage.failed_callback(lxmessage) + def process_deferred_stamps(self): + if len(self.pending_deferred_stamps) > 0: + RNS.log(f"Processing deferred stamps...", RNS.LOG_DEBUG) # TODO: Remove + + if self.stamp_gen_lock.locked(): + RNS.log(f"A stamp is already generating, returning...", RNS.LOG_DEBUG) # TODO: Remove + return + + else: + with self.stamp_gen_lock: + selected_lxm = None + selected_message_id = None + for message_id in self.pending_deferred_stamps: + lxmessage = self.pending_deferred_stamps[message_id] + if selected_lxm == None: + selected_lxm = lxmessage + selected_message_id = message_id + + if selected_lxm != None: + RNS.log(f"Starting stamp generation for {selected_lxm}...", RNS.LOG_DEBUG) + generated_stamp = selected_lxm.get_stamp() + if generated_stamp: + selected_lxm.stamp = generated_stamp + selected_lxm.defer_stamp = False + selected_lxm.packed = None + selected_lxm.pack() + self.pending_deferred_stamps.pop(selected_message_id) + self.pending_outbound.append(selected_lxm) + RNS.log(f"Stamp generation completed for {selected_lxm}", RNS.LOG_DEBUG) + else: + RNS.log(f"Deferred stamp generation did not succeed. Failing {selected_lxm}.", RNS.LOG_ERROR) + selected_lxm.stamp_generation_failed = True + self.pending_deferred_stamps.pop(selected_message_id) + self.fail_message(selected_lxm) + + def process_outbound(self, sender = None): if self.processing_outbound: return @@ -1641,14 +1724,6 @@ class LXMRouter: self.pending_outbound.remove(lxmessage) else: RNS.log("Starting outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - - # Handle potentially deferred stamp generation - if lxmessage.defer_stamp and lxmessage.stamp == None: - RNS.log(f"Generating deferred stamp for {lxmessage} now", RNS.LOG_DEBUG) - lxmessage.stamp = lxmessage.get_stamp() - lxmessage.defer_stamp = False - lxmessage.packed = None - lxmessage.pack() if lxmessage.progress == None or lxmessage.progress < 0.01: lxmessage.progress = 0.01 diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 14ff299..86b637d 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -45,7 +45,7 @@ class LXMessage: TICKET_EXPIRY = 21*24*60*60 TICKET_GRACE = 5*24*60*60 TICKET_RENEW = 14*24*60*60 - TICKET_INTERVAL = 3*24*60*60 + TICKET_INTERVAL = 1*24*60*60 # LXMF overhead is 111 bytes per message: # 16 bytes for destination hash @@ -131,24 +131,24 @@ class LXMessage: self.set_content_from_string(content) self.set_fields(fields) - self.payload = None - self.timestamp = None - self.signature = None - self.hash = None - self.packed = None - self.state = LXMessage.GENERATING - self.method = LXMessage.UNKNOWN - self.progress = 0.0 - self.rssi = None - self.snr = None - self.q = None + self.payload = None + self.timestamp = None + self.signature = None + self.hash = None + self.packed = None + self.state = LXMessage.GENERATING + self.method = LXMessage.UNKNOWN + self.progress = 0.0 + self.rssi = None + self.snr = None + self.q = None - self.stamp = None - self.stamp_cost = stamp_cost - self.stamp_valid = False - self.defer_stamp = False - self.outbound_ticket = None - self.include_ticket = include_ticket + self.stamp = None + self.stamp_cost = stamp_cost + self.stamp_valid = False + self.defer_stamp = True + self.outbound_ticket = None + self.include_ticket = include_ticket self.propagation_packed = None self.paper_packed = None @@ -166,7 +166,9 @@ class LXMessage: self.resource_representation = None self.__delivery_destination = None self.__delivery_callback = None - self.failed_callback = None + self.failed_callback = None + + self.deferred_stamp_generating = False def set_title_from_string(self, title_string): self.title = title_string.encode("utf-8") @@ -312,50 +314,79 @@ class LXMessage: total_rounds = 0 if not RNS.vendor.platformutils.is_android(): - RNS.log("Preparing IPC semaphores", RNS.LOG_DEBUG) # TODO: Remove + mp_debug = True + + jobs = multiprocessing.cpu_count() stop_event = multiprocessing.Event() - result_queue = multiprocessing.Queue(maxsize=1) + result_queue = multiprocessing.Queue(1) rounds_queue = multiprocessing.Queue() - def job(stop_event): + + def job(stop_event, pn, sc, wb): terminated = False rounds = 0 + pstamp = os.urandom(256//8) - stamp = os.urandom(256//8) - while not LXMessage.stamp_valid(stamp, self.stamp_cost, workblock): - if stop_event.is_set(): - break + def sv(s, c, w): + target = 0b1<<256-c; m = w+s + result = RNS.Identity.full_hash(m) + if int.from_bytes(result, byteorder="big") > target: + return False + else: + return True - if timeout != None and rounds % 10000 == 0: - if time.time() > start_time + timeout: - RNS.log(f"Stamp generation for {self} timed out", RNS.LOG_ERROR) - return None + while not stop_event.is_set() and not sv(pstamp, sc, wb): + pstamp = os.urandom(256//8); rounds += 1 - stamp = os.urandom(256//8) - rounds += 1 - - rounds_queue.put(rounds) if not stop_event.is_set(): - result_queue.put(stamp) - + stop_event.set() + result_queue.put(pstamp) + rounds_queue.put(rounds) + job_procs = [] - jobs = multiprocessing.cpu_count() - RNS.log("Starting workers", RNS.LOG_DEBUG) # TODO: Remove - for _ in range(jobs): - process = multiprocessing.Process(target=job, kwargs={"stop_event": stop_event},) + RNS.log(f"Starting {jobs} workers", RNS.LOG_DEBUG) # TODO: Remove + for jpn in range(jobs): + process = multiprocessing.Process(target=job, kwargs={"stop_event": stop_event, "pn": jpn, "sc": self.stamp_cost, "wb": workblock},) job_procs.append(process) process.start() - RNS.log("Awaiting results on queue", RNS.LOG_DEBUG) # TODO: Remove stamp = result_queue.get() - stop_event.set() - - RNS.log("Joining worker processes", RNS.LOG_DEBUG) # TODO: Remove - for j in range(jobs): - process = job_procs[j] - process.join() - total_rounds += rounds_queue.get() - + RNS.log("Got stamp result from worker", RNS.LOG_DEBUG) # TODO: Remove duration = time.time() - start_time + + spurious_results = 0 + try: + while True: + result_queue.get_nowait() + spurious_results += 1 + except: + pass + + for j in range(jobs): + nrounds = 0 + try: + nrounds = rounds_queue.get(timeout=2) + except Exception as e: + RNS.log(f"Failed to get round stats part {j}: {e}", RNS.LOG_ERROR) # TODO: Remove + total_rounds += nrounds + + all_exited = False + exit_timeout = time.time() + 5 + while time.time() < exit_timeout: + if not any(p.is_alive() for p in job_procs): + all_exited = True + break + time.sleep(0.1) + + if not all_exited: + RNS.log("Stamp generation IPC timeout, possible worker deadlock", RNS.LOG_ERROR) + return None + + else: + for j in range(jobs): + process = job_procs[j] + process.join() + # RNS.log(f"Joined {j} / {process}", RNS.LOG_DEBUG) # TODO: Remove + rounds = total_rounds else: @@ -365,17 +396,21 @@ class LXMessage: # checking in on the progress. use_nacl = False - try: - import nacl.encoding - import nacl.hash - use_nacl = True - except: - pass + rounds_per_worker = 1000 + if RNS.vendor.platformutils.is_android(): + rounds_per_worker = 500 + try: + import nacl.encoding + import nacl.hash + use_nacl = True + except: + pass - def full_hash(m): - if use_nacl: + if use_nacl: + def full_hash(m): return nacl.hash.sha256(m, encoder=nacl.encoding.RawEncoder) - else: + else: + def full_hash(m): return RNS.Identity.full_hash(m) def sv(s, c, w): @@ -391,30 +426,35 @@ class LXMessage: wm = multiprocessing.Manager() jobs = multiprocessing.cpu_count() - # RNS.log(f"Dispatching {jobs} workers for stamp generation...") # TODO: Remove + RNS.log(f"Dispatching {jobs} workers for stamp generation...", RNS.LOG_DEBUG) # TODO: Remove results_dict = wm.dict() while stamp == None: job_procs = [] - def job(procnum=None, results_dict=None, wb=None): - # RNS.log(f"Worker {procnum} starting...") # TODO: Remove + def job(procnum=None, results_dict=None, wb=None, sc=None, jr=None): + RNS.log(f"Worker {procnum} starting for {jr} rounds...") # TODO: Remove rounds = 0 + found_stamp = None + found_time = None - stamp = os.urandom(256//8) - while not sv(stamp, self.stamp_cost, wb): - if rounds >= 500: - stamp = None + while True: + pstamp = os.urandom(256//8) + rounds += 1 + if sv(pstamp, sc, wb): + found_stamp = pstamp + found_time = time.time() + break + + if rounds >= jr: # RNS.log(f"Worker {procnum} found no result in {rounds} rounds") # TODO: Remove break - stamp = os.urandom(256//8) - rounds += 1 - - results_dict[procnum] = [stamp, rounds] + results_dict[procnum] = [found_stamp, rounds, found_time] for pnum in range(jobs): - process = multiprocessing.Process(target=job, kwargs={"procnum":pnum, "results_dict": results_dict, "wb": workblock},) + pargs = {"procnum":pnum, "results_dict": results_dict, "wb": workblock, "sc":self.stamp_cost, "jr":rounds_per_worker} + process = multiprocessing.Process(target=job, kwargs=pargs) job_procs.append(process) process.start() @@ -423,14 +463,13 @@ class LXMessage: for j in results_dict: r = results_dict[j] - # RNS.log(f"Result from {r}: {r[1]} rounds, stamp: {r[0]}") # TODO: Remove total_rounds += r[1] if r[0] != None: stamp = r[0] - # RNS.log(f"Found stamp: {stamp}") # TODO: Remove + found_time = r[2] if stamp == None: - elapsed = time.time() - start_time + elapsed = found_time - start_time speed = total_rounds/elapsed RNS.log(f"Stamp generation for {self} running. {total_rounds} rounds completed so far, {int(speed)} rounds per second", RNS.LOG_DEBUG) @@ -439,12 +478,7 @@ class LXMessage: speed = total_rounds/duration - # TODO: Remove stats output RNS.log(f"Stamp generated in {RNS.prettytime(duration)}, {rounds} rounds, {int(speed)} rounds per second", RNS.LOG_DEBUG) - # RNS.log(f"Rounds per second {int(rounds/duration)}", RNS.LOG_DEBUG) - # RNS.log(f"Stamp: {RNS.hexrep(stamp)}", RNS.LOG_DEBUG) - # RNS.log(f"Resulting hash: {RNS.hexrep(RNS.Identity.full_hash(workblock+stamp))}", RNS.LOG_DEBUG) - ########################### return stamp From 6d83b019e1a8d17fb164d52945df5f6344c2b1a8 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 8 Sep 2024 01:22:00 +0200 Subject: [PATCH 218/436] Added stamp value property on inbound --- LXMF/Handlers.py | 4 ++-- LXMF/LXMRouter.py | 2 ++ LXMF/LXMessage.py | 34 +++++++++++++++++++++++++++++----- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index a33d13d..eb10a76 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -25,8 +25,8 @@ class LXMFDeliveryAnnounceHandler: try: stamp_cost = stamp_cost_from_app_data(app_data) - if stamp_cost != None: - self.lxmrouter.update_stamp_cost(destination_hash, stamp_cost) + self.lxmrouter.update_stamp_cost(destination_hash, stamp_cost) + except Exception as e: RNS.log(f"An error occurred while trying to decode announced stamp cost. The contained exception was: {e}", RNS.LOG_ERROR) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 9a82a13..49793cb 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1282,8 +1282,10 @@ class LXMRouter: destination_tickets = self.get_inbound_tickets(message.source_hash) if message.validate_stamp(required_stamp_cost, tickets=destination_tickets): message.stamp_valid = True + message.stamp_checked = True else: message.stamp_valid = False + message.stamp_checked = True if not message.stamp_valid: if self._enforce_stamps: diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 86b637d..adbac64 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -46,6 +46,7 @@ class LXMessage: TICKET_GRACE = 5*24*60*60 TICKET_RENEW = 14*24*60*60 TICKET_INTERVAL = 1*24*60*60 + COST_TICKET = 0x100 # LXMF overhead is 111 bytes per message: # 16 bytes for destination hash @@ -145,7 +146,9 @@ class LXMessage: self.stamp = None self.stamp_cost = stamp_cost + self.stamp_value = None self.stamp_valid = False + self.stamp_checked = False self.defer_stamp = True self.outbound_ticket = None self.include_ticket = include_ticket @@ -162,6 +165,7 @@ class LXMessage: self.delivery_attempts = 0 self.transport_encrypted = False self.transport_encryption = None + self.ratchet_id = None self.packet_representation = None self.resource_representation = None self.__delivery_destination = None @@ -272,17 +276,31 @@ class LXMessage: else: return True + @staticmethod + def stamp_value(material): + bits = 256 + value = 0 + i = int.from_bytes(material) + while ((i & (1 << (bits - 1))) == 0): + i = (i << 1) + value += 1 + + return value + def validate_stamp(self, target_cost, tickets=None): if tickets != None: for ticket in tickets: if self.stamp == RNS.Identity.truncated_hash(ticket+self.message_id): RNS.log(f"Stamp on {self} validated by inbound ticket", RNS.LOG_DEBUG) # TODO: Remove at some point + self.stamp_value = LXMessage.COST_TICKET return True if self.stamp == None: return False else: - if LXMessage.stamp_valid(self.stamp, target_cost, LXMessage.stamp_workblock(self.message_id)): + workblock = LXMessage.stamp_workblock(self.message_id) + if LXMessage.stamp_valid(self.stamp, target_cost, workblock): + self.stamp_value = LXMessage.stamp_value(RNS.Identity.full_hash(workblock+self.stamp)) return True else: return False @@ -314,8 +332,8 @@ class LXMessage: total_rounds = 0 if not RNS.vendor.platformutils.is_android(): - mp_debug = True - + allow_kill = True + stamp = None jobs = multiprocessing.cpu_count() stop_event = multiprocessing.Event() result_queue = multiprocessing.Queue(1) @@ -345,7 +363,7 @@ class LXMessage: job_procs = [] RNS.log(f"Starting {jobs} workers", RNS.LOG_DEBUG) # TODO: Remove for jpn in range(jobs): - process = multiprocessing.Process(target=job, kwargs={"stop_event": stop_event, "pn": jpn, "sc": self.stamp_cost, "wb": workblock},) + process = multiprocessing.Process(target=job, kwargs={"stop_event": stop_event, "pn": jpn, "sc": self.stamp_cost, "wb": workblock}, daemon=True) job_procs.append(process) process.start() @@ -379,7 +397,13 @@ class LXMessage: if not all_exited: RNS.log("Stamp generation IPC timeout, possible worker deadlock", RNS.LOG_ERROR) - return None + if allow_kill: + for j in range(jobs): + process = job_procs[j] + process.kill() + return stamp + else: + return None else: for j in range(jobs): From a20f380284a728fa704fd16bc4504200eb386ece Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 8 Sep 2024 14:52:54 +0200 Subject: [PATCH 219/436] Get message ratchet IDs --- LXMF/LXMRouter.py | 21 +++++++++++++++++---- LXMF/LXMessage.py | 26 +++++++++++++++++++++----- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 49793cb..659c47b 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1260,9 +1260,14 @@ class LXMRouter: ### Message Routing & Delivery ######################## ####################################################### - def lxmf_delivery(self, lxmf_data, destination_type = None, phy_stats = None): + def lxmf_delivery(self, lxmf_data, destination_type = None, phy_stats = None, ratchet_id = None, method = None): try: message = LXMessage.unpack_from_bytes(lxmf_data) + if ratchet_id and not message.ratchet_id: + message.ratchet_id = ratchet_id + + if method: + message.method = method if message.signature_validated and FIELD_TICKET in message.fields: ticket_entry = message.fields[FIELD_TICKET] @@ -1299,6 +1304,7 @@ class LXMRouter: if "snr" in phy_stats: message.snr = phy_stats["snr"] if "q" in phy_stats: message.q = phy_stats["q"] + # TODO: Update these descriptions to account for ratchets if destination_type == RNS.Destination.SINGLE: message.transport_encrypted = True message.transport_encryption = LXMessage.ENCRYPTION_DESCRIPTION_EC @@ -1339,11 +1345,14 @@ class LXMRouter: def delivery_packet(self, data, packet): packet.prove() try: + method = None if packet.destination_type != RNS.Destination.LINK: + method = LXMessage.OPPORTUNISTIC lxmf_data = b"" lxmf_data += packet.destination.hash lxmf_data += data else: + method = LXMessage.DIRECT lxmf_data = data try: @@ -1356,7 +1365,7 @@ class LXMRouter: phy_stats = {"rssi": packet.rssi, "snr": packet.snr, "q": packet.q} - self.lxmf_delivery(lxmf_data, packet.destination_type, phy_stats=phy_stats) + self.lxmf_delivery(lxmf_data, packet.destination_type, phy_stats=phy_stats, ratchet_id=packet.ratchet_id, method=method) except Exception as e: RNS.log("Exception occurred while parsing incoming LXMF data.", RNS.LOG_ERROR) @@ -1388,8 +1397,12 @@ class LXMRouter: def delivery_resource_concluded(self, resource): RNS.log("Transfer concluded for LXMF delivery resource "+str(resource), RNS.LOG_DEBUG) if resource.status == RNS.Resource.COMPLETE: + ratchet_id = None + # Set ratchet ID to link ID if available + if resource.link and hasattr(resource.link, "link_id"): + ratchet_id = resource.link.link_id phy_stats = {"rssi": resource.link.rssi, "snr": resource.link.snr, "q": resource.link.q} - self.lxmf_delivery(resource.data.read(), resource.link.type, phy_stats=phy_stats) + self.lxmf_delivery(resource.data.read(), resource.link.type, phy_stats=phy_stats, ratchet_id=ratchet_id, method=LXMessage.DIRECT) ### Peer Sync & Propagation ########################### @@ -1598,7 +1611,7 @@ class LXMRouter: decrypted_lxmf_data = delivery_destination.decrypt(encrypted_lxmf_data) if decrypted_lxmf_data != None: delivery_data = lxmf_data[:LXMessage.DESTINATION_LENGTH]+decrypted_lxmf_data - self.lxmf_delivery(delivery_data, delivery_destination.type) + self.lxmf_delivery(delivery_data, delivery_destination.type, ratchet_id=delivery_destination.latest_ratchet_id, method=LXMessage.PROPAGATED) self.locally_delivered_transient_ids[transient_id] = time.time() if signal_local_delivery != None: diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index adbac64..a41560f 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -159,6 +159,7 @@ class LXMessage: self.incoming = False self.signature_validated = False self.unverified_reason = None + self.ratchet_id = None self.representation = LXMessage.UNKNOWN self.desired_method = desired_method @@ -310,11 +311,13 @@ class LXMessage: # generating a valid stamp. if self.outbound_ticket != None and type(self.outbound_ticket) == bytes and len(self.outbound_ticket) == LXMessage.TICKET_LENGTH: RNS.log(f"Generating stamp with outbound ticket for {self}", RNS.LOG_DEBUG) # TODO: Remove at some point + self.stamp_value = LXMessage.COST_TICKET return RNS.Identity.truncated_hash(self.outbound_ticket+self.message_id) # If no stamp cost is required, we can just # return immediately. elif self.stamp_cost == None: + self.stamp_value = None return None # If a stamp was already generated, return @@ -504,6 +507,9 @@ class LXMessage: RNS.log(f"Stamp generated in {RNS.prettytime(duration)}, {rounds} rounds, {int(speed)} rounds per second", RNS.LOG_DEBUG) + self.stamp_value = LXMessage.stamp_value(RNS.Identity.full_hash(workblock+stamp)) + self.stamp_valid = True + return stamp def pack(self): @@ -575,6 +581,7 @@ class LXMessage: single_packet_content_limit = LXMessage.LINK_PACKET_MAX_CONTENT encrypted_data = self.__destination.encrypt(self.packed[LXMessage.DESTINATION_LENGTH:]) + self.ratchet_id = self.__destination.latest_ratchet_id self.propagation_packed = msgpack.packb([time.time(), [self.packed[:LXMessage.DESTINATION_LENGTH]+encrypted_data]]) content_size = len(self.propagation_packed) @@ -589,6 +596,7 @@ class LXMessage: paper_content_limit = LXMessage.PAPER_MDU encrypted_data = self.__destination.encrypt(self.packed[LXMessage.DESTINATION_LENGTH:]) + self.ratchet_id = self.__destination.latest_ratchet_id self.paper_packed = self.packed[:LXMessage.DESTINATION_LENGTH]+encrypted_data content_size = len(self.paper_packed) @@ -605,14 +613,18 @@ class LXMessage: self.determine_transport_encryption() if self.method == LXMessage.OPPORTUNISTIC: - self.__as_packet().send().set_delivery_callback(self.__mark_delivered) + lxm_packet = self.__as_packet() + lxm_packet.send().set_delivery_callback(self.__mark_delivered) + self.ratchet_id = lxm_packet.ratchet_id self.state = LXMessage.SENT elif self.method == LXMessage.DIRECT: self.state = LXMessage.SENDING if self.representation == LXMessage.PACKET: - receipt = self.__as_packet().send() + lxm_packet = self.__as_packet() + receipt = lxm_packet.send() + self.ratchet_id = self.__delivery_destination.link_id if receipt: receipt.set_delivery_callback(self.__mark_delivered) receipt.set_timeout_callback(self.__link_packet_timed_out) @@ -623,6 +635,7 @@ class LXMessage: elif self.representation == LXMessage.RESOURCE: self.resource_representation = self.__as_resource() + self.ratchet_id = self.__delivery_destination.link_id self.progress = 0.10 elif self.method == LXMessage.PROPAGATED: @@ -692,7 +705,8 @@ class LXMessage: try: self.__delivery_callback(self) except Exception as e: - RNS.log("An error occurred in the external delivery callback for "+str(message), RNS.LOG_ERROR) + RNS.log("An error occurred in the external delivery callback for "+str(self), RNS.LOG_ERROR) + RNS.trace_exception(e) def __mark_propagated(self, receipt = None): RNS.log("Received propagation success notification for "+str(self), RNS.LOG_DEBUG) @@ -703,7 +717,8 @@ class LXMessage: try: self.__delivery_callback(self) except Exception as e: - RNS.log("An error occurred in the external delivery callback for "+str(message), RNS.LOG_ERROR) + RNS.log("An error occurred in the external delivery callback for "+str(self), RNS.LOG_ERROR) + RNS.trace_exception(e) def __mark_paper_generated(self, receipt = None): RNS.log("Paper message generation succeeded for "+str(self), RNS.LOG_DEBUG) @@ -714,7 +729,8 @@ class LXMessage: try: self.__delivery_callback(self) except Exception as e: - RNS.log("An error occurred in the external delivery callback for "+str(message), RNS.LOG_ERROR) + RNS.log("An error occurred in the external delivery callback for "+str(self), RNS.LOG_ERROR) + RNS.trace_exception(e) def __resource_concluded(self, resource): if resource.status == RNS.Resource.COMPLETE: From ae0952455b881d7b9a58498f863665fd3893be8a Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 8 Sep 2024 17:54:11 +0200 Subject: [PATCH 220/436] Ignore stamp requirements for paper messages --- LXMF/LXMRouter.py | 34 ++++++++++++++++++++-------------- LXMF/LXMessage.py | 1 + 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 659c47b..260911c 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -853,7 +853,7 @@ class LXMRouter: locally_processed_file.close() except Exception as e: - RNS.log("Could not save locally processed message ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + RNS.log("Could not save locally processed transient ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) def clean_outbound_stamp_costs(self): try: @@ -864,7 +864,6 @@ class LXMRouter: expired.append(destination_hash) for destination_hash in expired: - RNS.log(f"Cleaning expired stamp cost for {destination_hash}") # TODO: Remove self.outbound_stamp_costs.pop(destination_hash) except Exception as e: @@ -874,7 +873,6 @@ class LXMRouter: def save_outbound_stamp_costs(self): with self.cost_file_lock: try: - RNS.log("Saving outbound stamp costs...", RNS.LOG_DEBUG) # TODO: Remove if not os.path.isdir(self.storagepath): os.makedirs(self.storagepath) @@ -883,7 +881,7 @@ class LXMRouter: outbound_stamp_costs_file.close() except Exception as e: - RNS.log("Could not save locally processed message ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + RNS.log("Could not save outbound stamp costs to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) def clean_available_tickets(self): try: @@ -912,7 +910,7 @@ class LXMRouter: self.available_tickets["inbound"][destination_hash].pop(destination_hash) except Exception as e: - RNS.log(f"Error while cleaning outbound stamp costs. The contained exception was: {e}", RNS.LOG_ERROR) + RNS.log(f"Error while cleaning available tickets. The contained exception was: {e}", RNS.LOG_ERROR) RNS.trace_exception(e) def save_available_tickets(self): @@ -1260,7 +1258,7 @@ class LXMRouter: ### Message Routing & Delivery ######################## ####################################################### - def lxmf_delivery(self, lxmf_data, destination_type = None, phy_stats = None, ratchet_id = None, method = None): + def lxmf_delivery(self, lxmf_data, destination_type = None, phy_stats = None, ratchet_id = None, method = None, no_stamp_enforcement=False): try: message = LXMessage.unpack_from_bytes(lxmf_data) if ratchet_id and not message.ratchet_id: @@ -1293,11 +1291,16 @@ class LXMRouter: message.stamp_checked = True if not message.stamp_valid: - if self._enforce_stamps: - RNS.log(f"Dropping {message} with invalid stamp", RNS.LOG_NOTICE) - return False + if no_stamp_enforcement: + RNS.log(f"Received {message} with invalid stamp, but allowing anyway, since stamp enforcement was temporarily disabled", RNS.LOG_NOTICE) else: - RNS.log(f"Received {message} with invalid stamp, but allowing anyway, since stamp enforcement is disabled", RNS.LOG_NOTICE) + if self._enforce_stamps: + RNS.log(f"Dropping {message} with invalid stamp", RNS.LOG_NOTICE) + return False + else: + RNS.log(f"Received {message} with invalid stamp, but allowing anyway, since stamp enforcement is disabled", RNS.LOG_NOTICE) + else: + RNS.log(f"Received {message} valid stamp", RNS.LOG_DEBUG) if phy_stats != None: if "rssi" in phy_stats: message.rssi = phy_stats["rssi"] @@ -1593,7 +1596,11 @@ class LXMRouter: except Exception as e: RNS.log("Error while unpacking received propagation resource", RNS.LOG_DEBUG) - def lxmf_propagation(self, lxmf_data, signal_local_delivery=None, signal_duplicate=None): + def lxmf_propagation(self, lxmf_data, signal_local_delivery=None, signal_duplicate=None, is_paper_message=False): + no_stamp_enforcement = False + if is_paper_message: + no_stamp_enforcement = True + try: if len(lxmf_data) >= LXMessage.LXMF_OVERHEAD: transient_id = RNS.Identity.full_hash(lxmf_data) @@ -1611,7 +1618,7 @@ class LXMRouter: decrypted_lxmf_data = delivery_destination.decrypt(encrypted_lxmf_data) if decrypted_lxmf_data != None: delivery_data = lxmf_data[:LXMessage.DESTINATION_LENGTH]+decrypted_lxmf_data - self.lxmf_delivery(delivery_data, delivery_destination.type, ratchet_id=delivery_destination.latest_ratchet_id, method=LXMessage.PROPAGATED) + self.lxmf_delivery(delivery_data, delivery_destination.type, ratchet_id=delivery_destination.latest_ratchet_id, method=LXMessage.PROPAGATED, no_stamp_enforcement=no_stamp_enforcement) self.locally_delivered_transient_ids[transient_id] = time.time() if signal_local_delivery != None: @@ -1661,7 +1668,7 @@ class LXMRouter: lxmf_data = base64.urlsafe_b64decode(uri.replace(LXMessage.URI_SCHEMA+"://", "").replace("/", "")+"==") transient_id = RNS.Identity.full_hash(lxmf_data) - router_propagation_result = self.lxmf_propagation(lxmf_data, signal_local_delivery=signal_local_delivery, signal_duplicate=signal_duplicate) + router_propagation_result = self.lxmf_propagation(lxmf_data, signal_local_delivery=signal_local_delivery, signal_duplicate=signal_duplicate, is_paper_message=True) if router_propagation_result != False: RNS.log("LXM with transient ID "+RNS.prettyhexrep(transient_id)+" was ingested.", RNS.LOG_DEBUG) return router_propagation_result @@ -1687,7 +1694,6 @@ class LXMRouter: def process_deferred_stamps(self): if len(self.pending_deferred_stamps) > 0: - RNS.log(f"Processing deferred stamps...", RNS.LOG_DEBUG) # TODO: Remove if self.stamp_gen_lock.locked(): RNS.log(f"A stamp is already generating, returning...", RNS.LOG_DEBUG) # TODO: Remove diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index a41560f..3564260 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -301,6 +301,7 @@ class LXMessage: else: workblock = LXMessage.stamp_workblock(self.message_id) if LXMessage.stamp_valid(self.stamp, target_cost, workblock): + RNS.log(f"Stamp on {self} validated", RNS.LOG_DEBUG) # TODO: Remove at some point self.stamp_value = LXMessage.stamp_value(RNS.Identity.full_hash(workblock+self.stamp)) return True else: From 813467243603517c2938763eaad937a05a07d8bc Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 9 Sep 2024 02:12:27 +0200 Subject: [PATCH 221/436] Cleanup --- LXMF/LXMRouter.py | 1 - LXMF/LXMessage.py | 205 ++++++++++++++++++++++----------------- docs/example_receiver.py | 2 + 3 files changed, 118 insertions(+), 90 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 260911c..199ff63 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1696,7 +1696,6 @@ class LXMRouter: if len(self.pending_deferred_stamps) > 0: if self.stamp_gen_lock.locked(): - RNS.log(f"A stamp is already generating, returning...", RNS.LOG_DEBUG) # TODO: Remove return else: diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 3564260..b70c66f 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -335,7 +335,121 @@ class LXMessage: start_time = time.time() total_rounds = 0 - if not RNS.vendor.platformutils.is_android(): + if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): + platform = RNS.vendor.platformutils.get_platform() + RNS.log(f"Running stamp generation on {platform}, work limited to single CPU core. This will be slow.", RNS.LOG_WARNING) + rounds = 0 + pstamp = os.urandom(256//8) + + def sv(s, c, w): + target = 0b1<<256-c; m = w+s + result = RNS.Identity.full_hash(m) + if int.from_bytes(result, byteorder="big") > target: + return False + else: + return True + + while not sv(pstamp, self.stamp_cost, workblock): + pstamp = os.urandom(256//8); rounds += 1 + + stamp = pstamp + duration = time.time() - start_time + + elif RNS.vendor.platformutils.is_android(): + # Semaphore support is flaky to non-existent on + # Android, so we need to manually dispatch and + # manage workloads here, while periodically + # checking in on the progress. + + use_nacl = False + rounds_per_worker = 1000 + try: + import nacl.encoding + import nacl.hash + use_nacl = True + except: + pass + + if use_nacl: + def full_hash(m): + return nacl.hash.sha256(m, encoder=nacl.encoding.RawEncoder) + else: + def full_hash(m): + return RNS.Identity.full_hash(m) + + def sv(s, c, w): + target = 0b1<<256-c + m = w+s + result = full_hash(m) + if int.from_bytes(result, byteorder="big") > target: + return False + else: + return True + + stamp = None + wm = multiprocessing.Manager() + jobs = multiprocessing.cpu_count() + + RNS.log(f"Dispatching {jobs} workers for stamp generation...", RNS.LOG_DEBUG) # TODO: Remove + + results_dict = wm.dict() + while stamp == None: + job_procs = [] + + def job(procnum=None, results_dict=None, wb=None, sc=None, jr=None): + # RNS.log(f"Worker {procnum} starting for {jr} rounds...") # TODO: Remove + try: + rounds = 0 + found_stamp = None + found_time = None + + while True: + pstamp = os.urandom(256//8) + rounds += 1 + if sv(pstamp, sc, wb): + found_stamp = pstamp + found_time = time.time() + break + + if rounds >= jr: + # RNS.log(f"Worker {procnum} found no result in {rounds} rounds") # TODO: Remove + break + + results_dict[procnum] = [found_stamp, rounds, found_time] + except Exception as e: + RNS.log("WORKER ERROR") + RNS.trace_exception(e) + + try: + for pnum in range(jobs): + pargs = {"procnum":pnum, "results_dict": results_dict, "wb": workblock, "sc":self.stamp_cost, "jr":rounds_per_worker} + process = multiprocessing.Process(target=job, kwargs=pargs) + job_procs.append(process) + process.start() + + for process in job_procs: + process.join() + + for j in results_dict: + r = results_dict[j] + total_rounds += r[1] + if r[0] != None: + stamp = r[0] + found_time = r[2] + + if stamp == None: + elapsed = time.time() - start_time + speed = total_rounds/elapsed + RNS.log(f"Stamp generation for {self} running. {total_rounds} rounds completed so far, {int(speed)} rounds per second", RNS.LOG_DEBUG) + + except Exception as e: + RNS.log("ERROR") + RNS.trace_exception(e) + + duration = time.time() - start_time + rounds = total_rounds + + else: allow_kill = True stamp = None jobs = multiprocessing.cpu_count() @@ -417,94 +531,7 @@ class LXMessage: rounds = total_rounds - else: - # Semaphore support is flaky to non-existent on - # Android, so we need to manually dispatch and - # manage workloads here, while periodically - # checking in on the progress. - - use_nacl = False - rounds_per_worker = 1000 - if RNS.vendor.platformutils.is_android(): - rounds_per_worker = 500 - try: - import nacl.encoding - import nacl.hash - use_nacl = True - except: - pass - - if use_nacl: - def full_hash(m): - return nacl.hash.sha256(m, encoder=nacl.encoding.RawEncoder) - else: - def full_hash(m): - return RNS.Identity.full_hash(m) - - def sv(s, c, w): - target = 0b1<<256-c - m = w+s - result = full_hash(m) - if int.from_bytes(result, byteorder="big") > target: - return False - else: - return True - - stamp = None - wm = multiprocessing.Manager() - jobs = multiprocessing.cpu_count() - - RNS.log(f"Dispatching {jobs} workers for stamp generation...", RNS.LOG_DEBUG) # TODO: Remove - - results_dict = wm.dict() - while stamp == None: - job_procs = [] - - def job(procnum=None, results_dict=None, wb=None, sc=None, jr=None): - RNS.log(f"Worker {procnum} starting for {jr} rounds...") # TODO: Remove - rounds = 0 - found_stamp = None - found_time = None - - while True: - pstamp = os.urandom(256//8) - rounds += 1 - if sv(pstamp, sc, wb): - found_stamp = pstamp - found_time = time.time() - break - - if rounds >= jr: - # RNS.log(f"Worker {procnum} found no result in {rounds} rounds") # TODO: Remove - break - - results_dict[procnum] = [found_stamp, rounds, found_time] - - for pnum in range(jobs): - pargs = {"procnum":pnum, "results_dict": results_dict, "wb": workblock, "sc":self.stamp_cost, "jr":rounds_per_worker} - process = multiprocessing.Process(target=job, kwargs=pargs) - job_procs.append(process) - process.start() - - for process in job_procs: - process.join() - - for j in results_dict: - r = results_dict[j] - total_rounds += r[1] - if r[0] != None: - stamp = r[0] - found_time = r[2] - - if stamp == None: - elapsed = found_time - start_time - speed = total_rounds/elapsed - RNS.log(f"Stamp generation for {self} running. {total_rounds} rounds completed so far, {int(speed)} rounds per second", RNS.LOG_DEBUG) - - duration = time.time() - start_time - rounds = total_rounds - - speed = total_rounds/duration + speed = rounds/duration RNS.log(f"Stamp generated in {RNS.prettytime(duration)}, {rounds} rounds, {int(speed)} rounds per second", RNS.LOG_DEBUG) diff --git a/docs/example_receiver.py b/docs/example_receiver.py index 7d3aa20..999f6a3 100644 --- a/docs/example_receiver.py +++ b/docs/example_receiver.py @@ -32,6 +32,8 @@ def delivery_callback(message): RNS.log("\t| Title : "+message.title_as_string()) RNS.log("\t| Content : "+message.content_as_string()) RNS.log("\t| Fields : "+str(message.fields)) + if message.ratchet_id: + RNS.log("\t| Ratchet : "+str(RNS.Identity._get_ratchet_id(message.ratchet_id))) RNS.log("\t| Message signature : "+signature_string) RNS.log("\t| Stamp : "+stamp_string) RNS.log("\t+---------------------------------------------------------------") From 058186dfdc87b24dd84f2f96bd837b374880305d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 9 Sep 2024 13:15:37 +0200 Subject: [PATCH 222/436] Cleanup --- LXMF/LXMessage.py | 250 ++----------------------------------------- LXMF/LXStamper.py | 263 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 274 insertions(+), 239 deletions(-) create mode 100644 LXMF/LXStamper.py diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index b70c66f..5a8a26f 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -4,11 +4,12 @@ import RNS.vendor.umsgpack as msgpack import os import time import base64 -import signal import multiprocessing +import LXMF.LXStamper as LXStamper from .LXMF import APP_NAME + class LXMessage: GENERATING = 0x00 OUTBOUND = 0x01 @@ -251,23 +252,6 @@ class LXMessage: def register_failed_callback(self, callback): self.failed_callback = callback - @staticmethod - def stamp_workblock(message_id): - wb_st = time.time() - expand_rounds = 3000 - workblock = b"" - for n in range(expand_rounds): - workblock += RNS.Cryptography.hkdf( - length=256, - derive_from=message_id, - salt=RNS.Identity.full_hash(message_id+msgpack.packb(n)), - context=None, - ) - wb_time = time.time() - wb_st - RNS.log(f"Stamp workblock size {RNS.prettysize(len(workblock))}, generated in {round(wb_time*1000,2)}ms", RNS.LOG_DEBUG) - - return workblock - @staticmethod def stamp_valid(stamp, target_cost, workblock): target = 0b1 << 256-target_cost @@ -277,17 +261,6 @@ class LXMessage: else: return True - @staticmethod - def stamp_value(material): - bits = 256 - value = 0 - i = int.from_bytes(material) - while ((i & (1 << (bits - 1))) == 0): - i = (i << 1) - value += 1 - - return value - def validate_stamp(self, target_cost, tickets=None): if tickets != None: for ticket in tickets: @@ -299,10 +272,10 @@ class LXMessage: if self.stamp == None: return False else: - workblock = LXMessage.stamp_workblock(self.message_id) + workblock = LXStamper.stamp_workblock(self.message_id) if LXMessage.stamp_valid(self.stamp, target_cost, workblock): RNS.log(f"Stamp on {self} validated", RNS.LOG_DEBUG) # TODO: Remove at some point - self.stamp_value = LXMessage.stamp_value(RNS.Identity.full_hash(workblock+self.stamp)) + self.stamp_value = LXStamper.stamp_value(workblock, self.stamp) return True else: return False @@ -330,215 +303,14 @@ class LXMessage: # valid stamp according to the cost that # the receiver has specified. else: - RNS.log(f"Generating stamp with cost {self.stamp_cost} for {self}...", RNS.LOG_DEBUG) - workblock = LXMessage.stamp_workblock(self.message_id) - start_time = time.time() - total_rounds = 0 - - if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): - platform = RNS.vendor.platformutils.get_platform() - RNS.log(f"Running stamp generation on {platform}, work limited to single CPU core. This will be slow.", RNS.LOG_WARNING) - rounds = 0 - pstamp = os.urandom(256//8) - - def sv(s, c, w): - target = 0b1<<256-c; m = w+s - result = RNS.Identity.full_hash(m) - if int.from_bytes(result, byteorder="big") > target: - return False - else: - return True - - while not sv(pstamp, self.stamp_cost, workblock): - pstamp = os.urandom(256//8); rounds += 1 - - stamp = pstamp - duration = time.time() - start_time - - elif RNS.vendor.platformutils.is_android(): - # Semaphore support is flaky to non-existent on - # Android, so we need to manually dispatch and - # manage workloads here, while periodically - # checking in on the progress. - - use_nacl = False - rounds_per_worker = 1000 - try: - import nacl.encoding - import nacl.hash - use_nacl = True - except: - pass - - if use_nacl: - def full_hash(m): - return nacl.hash.sha256(m, encoder=nacl.encoding.RawEncoder) - else: - def full_hash(m): - return RNS.Identity.full_hash(m) - - def sv(s, c, w): - target = 0b1<<256-c - m = w+s - result = full_hash(m) - if int.from_bytes(result, byteorder="big") > target: - return False - else: - return True - - stamp = None - wm = multiprocessing.Manager() - jobs = multiprocessing.cpu_count() - - RNS.log(f"Dispatching {jobs} workers for stamp generation...", RNS.LOG_DEBUG) # TODO: Remove - - results_dict = wm.dict() - while stamp == None: - job_procs = [] - - def job(procnum=None, results_dict=None, wb=None, sc=None, jr=None): - # RNS.log(f"Worker {procnum} starting for {jr} rounds...") # TODO: Remove - try: - rounds = 0 - found_stamp = None - found_time = None - - while True: - pstamp = os.urandom(256//8) - rounds += 1 - if sv(pstamp, sc, wb): - found_stamp = pstamp - found_time = time.time() - break - - if rounds >= jr: - # RNS.log(f"Worker {procnum} found no result in {rounds} rounds") # TODO: Remove - break - - results_dict[procnum] = [found_stamp, rounds, found_time] - except Exception as e: - RNS.log("WORKER ERROR") - RNS.trace_exception(e) - - try: - for pnum in range(jobs): - pargs = {"procnum":pnum, "results_dict": results_dict, "wb": workblock, "sc":self.stamp_cost, "jr":rounds_per_worker} - process = multiprocessing.Process(target=job, kwargs=pargs) - job_procs.append(process) - process.start() - - for process in job_procs: - process.join() - - for j in results_dict: - r = results_dict[j] - total_rounds += r[1] - if r[0] != None: - stamp = r[0] - found_time = r[2] - - if stamp == None: - elapsed = time.time() - start_time - speed = total_rounds/elapsed - RNS.log(f"Stamp generation for {self} running. {total_rounds} rounds completed so far, {int(speed)} rounds per second", RNS.LOG_DEBUG) - - except Exception as e: - RNS.log("ERROR") - RNS.trace_exception(e) - - duration = time.time() - start_time - rounds = total_rounds - - else: - allow_kill = True - stamp = None - jobs = multiprocessing.cpu_count() - stop_event = multiprocessing.Event() - result_queue = multiprocessing.Queue(1) - rounds_queue = multiprocessing.Queue() - - def job(stop_event, pn, sc, wb): - terminated = False - rounds = 0 - pstamp = os.urandom(256//8) - - def sv(s, c, w): - target = 0b1<<256-c; m = w+s - result = RNS.Identity.full_hash(m) - if int.from_bytes(result, byteorder="big") > target: - return False - else: - return True - - while not stop_event.is_set() and not sv(pstamp, sc, wb): - pstamp = os.urandom(256//8); rounds += 1 - - if not stop_event.is_set(): - stop_event.set() - result_queue.put(pstamp) - rounds_queue.put(rounds) - - job_procs = [] - RNS.log(f"Starting {jobs} workers", RNS.LOG_DEBUG) # TODO: Remove - for jpn in range(jobs): - process = multiprocessing.Process(target=job, kwargs={"stop_event": stop_event, "pn": jpn, "sc": self.stamp_cost, "wb": workblock}, daemon=True) - job_procs.append(process) - process.start() - - stamp = result_queue.get() - RNS.log("Got stamp result from worker", RNS.LOG_DEBUG) # TODO: Remove - duration = time.time() - start_time - - spurious_results = 0 - try: - while True: - result_queue.get_nowait() - spurious_results += 1 - except: - pass - - for j in range(jobs): - nrounds = 0 - try: - nrounds = rounds_queue.get(timeout=2) - except Exception as e: - RNS.log(f"Failed to get round stats part {j}: {e}", RNS.LOG_ERROR) # TODO: Remove - total_rounds += nrounds - - all_exited = False - exit_timeout = time.time() + 5 - while time.time() < exit_timeout: - if not any(p.is_alive() for p in job_procs): - all_exited = True - break - time.sleep(0.1) - - if not all_exited: - RNS.log("Stamp generation IPC timeout, possible worker deadlock", RNS.LOG_ERROR) - if allow_kill: - for j in range(jobs): - process = job_procs[j] - process.kill() - return stamp - else: - return None - - else: - for j in range(jobs): - process = job_procs[j] - process.join() - # RNS.log(f"Joined {j} / {process}", RNS.LOG_DEBUG) # TODO: Remove - - rounds = total_rounds + generated_stamp, value = LXStamper.generate_stamp(self.message_id, self.stamp_cost) + if generated_stamp: + self.stamp_value = value + self.stamp_valid = True + return generated_stamp - speed = rounds/duration - - RNS.log(f"Stamp generated in {RNS.prettytime(duration)}, {rounds} rounds, {int(speed)} rounds per second", RNS.LOG_DEBUG) - - self.stamp_value = LXMessage.stamp_value(RNS.Identity.full_hash(workblock+stamp)) - self.stamp_valid = True - - return stamp + else: + return None def pack(self): if not self.packed: diff --git a/LXMF/LXStamper.py b/LXMF/LXStamper.py new file mode 100644 index 0000000..9bd58de --- /dev/null +++ b/LXMF/LXStamper.py @@ -0,0 +1,263 @@ +import RNS +import RNS.vendor.umsgpack as msgpack + +import os +import time +import multiprocessing + +WORKBLOCK_EXPAND_ROUNDS = 3000 + +def stamp_workblock(message_id): + wb_st = time.time() + expand_rounds = WORKBLOCK_EXPAND_ROUNDS + workblock = b"" + for n in range(expand_rounds): + workblock += RNS.Cryptography.hkdf( + length=256, + derive_from=message_id, + salt=RNS.Identity.full_hash(message_id+msgpack.packb(n)), + context=None, + ) + wb_time = time.time() - wb_st + RNS.log(f"Stamp workblock size {RNS.prettysize(len(workblock))}, generated in {round(wb_time*1000,2)}ms", RNS.LOG_DEBUG) + + return workblock + +def stamp_value(workblock, stamp): + value = 0 + bits = 256 + material = RNS.Identity.full_hash(workblock+stamp) + i = int.from_bytes(material) + while ((i & (1 << (bits - 1))) == 0): + i = (i << 1) + value += 1 + + return value + +def generate_stamp(message_id, stamp_cost): + RNS.log(f"Generating stamp with cost {stamp_cost} for {RNS.prettyhexrep(message_id)}...", RNS.LOG_DEBUG) + workblock = stamp_workblock(message_id) + + start_time = time.time() + stamp = None + rounds = 0 + value = 0 + + if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): + stamp, rounds = job_simple(stamp_cost, workblock) + + elif RNS.vendor.platformutils.is_android(): + stamp, rounds = job_android(stamp_cost, workblock) + + else: + stamp, rounds = job_linux(stamp_cost, workblock) + + duration = time.time() - start_time + speed = rounds/duration + value = stamp_value(workblock, stamp) + + RNS.log(f"Stamp with value {value} generated in {RNS.prettytime(duration)}, {rounds} rounds, {int(speed)} rounds per second", RNS.LOG_DEBUG) + + return stamp, value + +def job_simple(stamp_cost, workblock): + # A simple, single-process stamp generator. + # should work on any platform, and is used + # as a fall-back, in case of limited multi- + # processing and/or acceleration support. + + platform = RNS.vendor.platformutils.get_platform() + RNS.log(f"Running stamp generation on {platform}, work limited to single CPU core. This will be slower than ideal.", RNS.LOG_WARNING) + + rounds = 0 + pstamp = os.urandom(256//8) + st = time.time() + + def sv(s, c, w): + target = 0b1<<256-c; m = w+s + result = RNS.Identity.full_hash(m) + if int.from_bytes(result, byteorder="big") > target: + return False + else: + return True + + while not sv(pstamp, stamp_cost, workblock): + pstamp = os.urandom(256//8); rounds += 1 + if rounds % 2500 == 0: + speed = rounds / (time.time()-st) + RNS.log(f"Stamp generation running. {rounds} rounds completed so far, {int(speed)} rounds per second", RNS.LOG_DEBUG) + + return pstamp, rounds + +def job_linux(stamp_cost, workblock): + allow_kill = True + stamp = None + total_rounds = 0 + jobs = multiprocessing.cpu_count() + stop_event = multiprocessing.Event() + result_queue = multiprocessing.Queue(1) + rounds_queue = multiprocessing.Queue() + + def job(stop_event, pn, sc, wb): + terminated = False + rounds = 0 + pstamp = os.urandom(256//8) + + def sv(s, c, w): + target = 0b1<<256-c; m = w+s + result = RNS.Identity.full_hash(m) + if int.from_bytes(result, byteorder="big") > target: + return False + else: + return True + + while not stop_event.is_set() and not sv(pstamp, sc, wb): + pstamp = os.urandom(256//8); rounds += 1 + + if not stop_event.is_set(): + stop_event.set() + result_queue.put(pstamp) + rounds_queue.put(rounds) + + job_procs = [] + RNS.log(f"Starting {jobs} stamp generation workers", RNS.LOG_DEBUG) + for jpn in range(jobs): + process = multiprocessing.Process(target=job, kwargs={"stop_event": stop_event, "pn": jpn, "sc": stamp_cost, "wb": workblock}, daemon=True) + job_procs.append(process) + process.start() + + stamp = result_queue.get() + RNS.log("Got stamp result from worker", RNS.LOG_DEBUG) # TODO: Remove + + # Collect any potential spurious + # results from worker queue. + try: + while True: + result_queue.get_nowait() + except: + pass + + for j in range(jobs): + nrounds = 0 + try: + nrounds = rounds_queue.get(timeout=2) + except Exception as e: + RNS.log(f"Failed to get round stats part {j}: {e}", RNS.LOG_ERROR) + total_rounds += nrounds + + all_exited = False + exit_timeout = time.time() + 5 + while time.time() < exit_timeout: + if not any(p.is_alive() for p in job_procs): + all_exited = True + break + time.sleep(0.1) + + if not all_exited: + RNS.log("Stamp generation IPC timeout, possible worker deadlock. Terminating remaining processes.", RNS.LOG_ERROR) + if allow_kill: + for j in range(jobs): + process = job_procs[j] + process.kill() + else: + return None + + else: + for j in range(jobs): + process = job_procs[j] + process.join() + # RNS.log(f"Joined {j} / {process}", RNS.LOG_DEBUG) # TODO: Remove + + return stamp, total_rounds + +def job_android(stamp_cost, workblock): + # Semaphore support is flaky to non-existent on + # Android, so we need to manually dispatch and + # manage workloads here, while periodically + # checking in on the progress. + + stamp = None + start_time = time.time() + total_rounds = 0 + rounds_per_worker = 1000 + + use_nacl = False + try: + import nacl.encoding + import nacl.hash + use_nacl = True + except: + pass + + if use_nacl: + def full_hash(m): + return nacl.hash.sha256(m, encoder=nacl.encoding.RawEncoder) + else: + def full_hash(m): + return RNS.Identity.full_hash(m) + + def sv(s, c, w): + target = 0b1<<256-c + m = w+s + result = full_hash(m) + if int.from_bytes(result, byteorder="big") > target: + return False + else: + return True + + wm = multiprocessing.Manager() + jobs = multiprocessing.cpu_count() + + def job(procnum=None, results_dict=None, wb=None, sc=None, jr=None): + # RNS.log(f"Worker {procnum} starting for {jr} rounds...") # TODO: Remove + try: + rounds = 0 + found_stamp = None + + while True: + pstamp = os.urandom(256//8) + rounds += 1 + if sv(pstamp, sc, wb): + found_stamp = pstamp + break + + if rounds >= jr: + # RNS.log(f"Worker {procnum} found no result in {rounds} rounds") # TODO: Remove + break + + results_dict[procnum] = [found_stamp, rounds] + except Exception as e: + RNS.log(f"Stamp generation worker error: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + RNS.log(f"Dispatching {jobs} workers for stamp generation...", RNS.LOG_DEBUG) # TODO: Remove + + results_dict = wm.dict() + while stamp == None: + job_procs = [] + try: + for pnum in range(jobs): + pargs = {"procnum":pnum, "results_dict": results_dict, "wb": workblock, "sc":stamp_cost, "jr":rounds_per_worker} + process = multiprocessing.Process(target=job, kwargs=pargs) + job_procs.append(process) + process.start() + + for process in job_procs: + process.join() + + for j in results_dict: + r = results_dict[j] + total_rounds += r[1] + if r[0] != None: + stamp = r[0] + + if stamp == None: + elapsed = time.time() - start_time + speed = total_rounds/elapsed + RNS.log(f"Stamp generation running. {total_rounds} rounds completed so far, {int(speed)} rounds per second", RNS.LOG_DEBUG) + + except Exception as e: + RNS.log(f"Stamp generation job error: {e}") + RNS.trace_exception(e) + + return stamp, total_rounds \ No newline at end of file From ea49d22bceca5bae19207cd612f816b6a5f2ec04 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 9 Sep 2024 13:20:54 +0200 Subject: [PATCH 223/436] Add test entrypoint to LXStamper --- LXMF/LXStamper.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/LXMF/LXStamper.py b/LXMF/LXStamper.py index 9bd58de..28cf9cd 100644 --- a/LXMF/LXStamper.py +++ b/LXMF/LXStamper.py @@ -260,4 +260,21 @@ def job_android(stamp_cost, workblock): RNS.log(f"Stamp generation job error: {e}") RNS.trace_exception(e) - return stamp, total_rounds \ No newline at end of file + return stamp, total_rounds + +if __name__ == "__main__": + import sys + if len(sys.argv) < 2: + RNS.log("No cost argument provided", RNS.LOG_ERROR) + exit(1) + else: + try: + cost = int(sys.argv[1]) + except Exception as e: + RNS.log(f"Invalid cost argument provided: {e}", RNS.LOG_ERROR) + exit(1) + + RNS.loglevel = RNS.LOG_DEBUG + RNS.log("Testing LXMF stamp generation", RNS.LOG_DEBUG) + message_id = os.urandom(32) + generate_stamp(message_id, cost) \ No newline at end of file From 98eea1171ee618c572f83295ae1683b916709466 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 9 Sep 2024 15:24:36 +0200 Subject: [PATCH 224/436] Added event field --- LXMF/LXMF.py | 1 + 1 file changed, 1 insertion(+) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 3618912..146d8ec 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -17,6 +17,7 @@ FIELD_COMMANDS = 0x09 FIELD_RESULTS = 0x0A FIELD_GROUP = 0x0B FIELD_TICKET = 0x0C +FIELD_EVENT = 0x0D # For usecases such as including custom data structures, # embedding or encapsulating other data types or protocols From 453772859d4e806577839ac82a985e54ad15857d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 10 Sep 2024 20:17:09 +0200 Subject: [PATCH 225/436] Removed deprecated query destination --- LXMF/LXMRouter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 199ff63..28e3854 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -117,7 +117,6 @@ class LXMRouter: identity = RNS.Identity() self.identity = identity - self.lxmf_query_destination = RNS.Destination(None, RNS.Destination.IN, RNS.Destination.PLAIN, APP_NAME, "query") self.propagation_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation") if autopeer != None: From 717240c8d404dcad7e09eab45cf94ea3ab6cb6b7 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 11 Sep 2024 02:09:43 +0200 Subject: [PATCH 226/436] Cleanup --- LXMF/LXMRouter.py | 2 +- LXMF/LXMessage.py | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 28e3854..d26f155 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1299,7 +1299,7 @@ class LXMRouter: else: RNS.log(f"Received {message} with invalid stamp, but allowing anyway, since stamp enforcement is disabled", RNS.LOG_NOTICE) else: - RNS.log(f"Received {message} valid stamp", RNS.LOG_DEBUG) + RNS.log(f"Received {message} with valid stamp", RNS.LOG_DEBUG) if phy_stats != None: if "rssi" in phy_stats: message.rssi = phy_stats["rssi"] diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 5a8a26f..08e1077 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -264,10 +264,14 @@ class LXMessage: def validate_stamp(self, target_cost, tickets=None): if tickets != None: for ticket in tickets: - if self.stamp == RNS.Identity.truncated_hash(ticket+self.message_id): - RNS.log(f"Stamp on {self} validated by inbound ticket", RNS.LOG_DEBUG) # TODO: Remove at some point - self.stamp_value = LXMessage.COST_TICKET - return True + try: + if self.stamp == RNS.Identity.truncated_hash(ticket+self.message_id): + RNS.log(f"Stamp on {self} validated by inbound ticket", RNS.LOG_DEBUG) # TODO: Remove at some point + self.stamp_value = LXMessage.COST_TICKET + return True + except Exception as e: + RNS.log(f"Error while validating ticket: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) if self.stamp == None: return False @@ -284,9 +288,10 @@ class LXMessage: # If an outbound ticket exists, use this for # generating a valid stamp. if self.outbound_ticket != None and type(self.outbound_ticket) == bytes and len(self.outbound_ticket) == LXMessage.TICKET_LENGTH: - RNS.log(f"Generating stamp with outbound ticket for {self}", RNS.LOG_DEBUG) # TODO: Remove at some point + generated_stamp = RNS.Identity.truncated_hash(self.outbound_ticket+self.message_id) self.stamp_value = LXMessage.COST_TICKET - return RNS.Identity.truncated_hash(self.outbound_ticket+self.message_id) + RNS.log(f"Generated stamp with outbound ticket {RNS.hexrep(self.outbound_ticket)} for {self}", RNS.LOG_DEBUG) # TODO: Remove at some point + return generated_stamp # If no stamp cost is required, we can just # return immediately. From 0fc15e6054605d9afb8b72a6e59272b716fc72e4 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 11 Sep 2024 11:48:36 +0200 Subject: [PATCH 227/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 3d18726..dd9b22c 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.5.0" +__version__ = "0.5.1" From d8e2e2a45f590eb5c4e650dc10ba02444f713619 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 12 Sep 2024 10:11:49 +0200 Subject: [PATCH 228/436] Fix stamp cost extractor function --- LXMF/LXMF.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 146d8ec..23b10b3 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -112,7 +112,7 @@ def display_name_from_app_data(app_data=None): return app_data.decode("utf-8") def stamp_cost_from_app_data(app_data=None): - if app_data == None: + if app_data == None or app_data == b"": return None else: # Version 0.5.0+ announce format From bbf1eda3b0b42d8b0f4b48b16270d9ac10b4e7f3 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 12 Sep 2024 17:57:26 +0200 Subject: [PATCH 229/436] Added backchannel delivery --- LXMF/LXMRouter.py | 49 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index d26f155..7f529b1 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -66,6 +66,7 @@ class LXMRouter: self.pending_outbound = [] self.failed_outbound = [] self.direct_links = {} + self.backchannel_links = {} self.delivery_destinations = {} self.prioritised_list = [] @@ -1380,6 +1381,7 @@ class LXMRouter: link.set_resource_callback(self.delivery_resource_advertised) link.set_resource_started_callback(self.resource_transfer_began) link.set_resource_concluded_callback(self.delivery_resource_concluded) + link.set_remote_identified_callback(self.delivery_remote_identified) def delivery_link_closed(self, link): pass @@ -1406,6 +1408,11 @@ class LXMRouter: phy_stats = {"rssi": resource.link.rssi, "snr": resource.link.snr, "q": resource.link.q} self.lxmf_delivery(resource.data.read(), resource.link.type, phy_stats=phy_stats, ratchet_id=ratchet_id, method=LXMessage.DIRECT) + def delivery_remote_identified(self, link, identity): + destination_hash = RNS.Destination.hash_from_name_and_identity("lxmf.delivery", identity) + self.backchannel_links[destination_hash] = link + RNS.log(f"Backchannel became available for {RNS.prettyhexrep(destination_hash)} on delivery link {link}", RNS.LOG_DEBUG) + ### Peer Sync & Propagation ########################### ####################################################### @@ -1733,11 +1740,26 @@ class LXMRouter: if lxmessage.state == LXMessage.DELIVERED: RNS.log("Delivery has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) self.pending_outbound.remove(lxmessage) + + # Udate ticket delivery stats if lxmessage.include_ticket and FIELD_TICKET in lxmessage.fields: RNS.log(f"Updating latest ticket delivery for {RNS.prettyhexrep(lxmessage.destination_hash)}", RNS.LOG_DEBUG) self.available_tickets["last_deliveries"][lxmessage.destination_hash] = time.time() self.save_available_tickets() + # Prepare link for backchannel communications + delivery_destination_hash = lxmessage.get_destination().hash + if delivery_destination_hash in self.direct_links: + direct_link = self.direct_links[delivery_destination_hash] + if not hasattr(direct_link, "backchannel_identified") or direct_link.backchannel_identified == False: + if direct_link.initiator == True: + source_destination_hash = lxmessage.get_source().hash + if source_destination_hash in self.delivery_destinations: + backchannel_identity = self.delivery_destinations[source_destination_hash].identity + direct_link.identify(backchannel_identity) + self.delivery_link_established(direct_link) + RNS.log(f"Performed backchannel identification as {backchannel_identity} on {direct_link}", RNS.LOG_DEBUG) + elif lxmessage.method == LXMessage.PROPAGATED and lxmessage.state == LXMessage.SENT: RNS.log("Propagation has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) self.pending_outbound.remove(lxmessage) @@ -1764,16 +1786,28 @@ class LXMRouter: elif lxmessage.method == LXMessage.DIRECT: if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: delivery_destination_hash = lxmessage.get_destination().hash - + direct_link = None + if delivery_destination_hash in self.direct_links: - # A link already exists, so we'll try to use it - # to deliver the message + # An established direct link already exists to + # the destination, so we'll try to use it for + # delivering the message direct_link = self.direct_links[delivery_destination_hash] + RNS.log(f"Using available direct link {direct_link} to {RNS.prettyhexrep(delivery_destination_hash)}", RNS.LOG_DEBUG) + + elif delivery_destination_hash in self.backchannel_links: + # An established backchannel link exists to + # the destination, so we'll try to use it for + # delivering the message + direct_link = self.backchannel_links[delivery_destination_hash] + RNS.log(f"Using available backchannel link {direct_link} to {RNS.prettyhexrep(delivery_destination_hash)}", RNS.LOG_DEBUG) + + if direct_link != None: if direct_link.status == RNS.Link.ACTIVE: if lxmessage.progress == None or lxmessage.progress < 0.05: lxmessage.progress = 0.05 if lxmessage.state != LXMessage.SENDING: - RNS.log("Starting transfer of "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + RNS.log("Starting transfer of "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" on link "+str(direct_link), RNS.LOG_DEBUG) lxmessage.set_delivery_destination(direct_link) lxmessage.send() else: @@ -1783,7 +1817,7 @@ class LXMRouter: RNS.log("Waiting for proof for "+str(lxmessage)+" sent as link packet", RNS.LOG_DEBUG) elif direct_link.status == RNS.Link.CLOSED: if direct_link.activated_at != None: - RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was closed", RNS.LOG_DEBUG) + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was closed, reason: "+str(direct_link.teardown_reason), RNS.LOG_DEBUG) else: if not hasattr(lxmessage, "path_request_retried"): RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was never activated, retrying path request...", RNS.LOG_DEBUG) @@ -1795,7 +1829,10 @@ class LXMRouter: lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT lxmessage.set_delivery_destination(None) - self.direct_links.pop(delivery_destination_hash) + if delivery_destination_hash in self.direct_links: + self.direct_links.pop(delivery_destination_hash) + if delivery_destination_hash in self.backchannel_links: + self.backchannel_links.pop(delivery_destination_hash) lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT else: # Simply wait for the link to become active or close From 9193aa5e023a8c4fdd466cfe8a1b7ab2d4c173ed Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 12 Sep 2024 18:24:14 +0200 Subject: [PATCH 230/436] Improved logging --- LXMF/LXMRouter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 7f529b1..533785f 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1756,9 +1756,10 @@ class LXMRouter: source_destination_hash = lxmessage.get_source().hash if source_destination_hash in self.delivery_destinations: backchannel_identity = self.delivery_destinations[source_destination_hash].identity + backchannel_desthash = RNS.Destination.hash_from_name_and_identity("lxmf.delivery", backchannel_identity) direct_link.identify(backchannel_identity) self.delivery_link_established(direct_link) - RNS.log(f"Performed backchannel identification as {backchannel_identity} on {direct_link}", RNS.LOG_DEBUG) + RNS.log(f"Performed backchannel identification as {RNS.prettyhexrep(backchannel_desthash)} on {direct_link}", RNS.LOG_DEBUG) elif lxmessage.method == LXMessage.PROPAGATED and lxmessage.state == LXMessage.SENT: RNS.log("Propagation has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) @@ -1817,7 +1818,7 @@ class LXMRouter: RNS.log("Waiting for proof for "+str(lxmessage)+" sent as link packet", RNS.LOG_DEBUG) elif direct_link.status == RNS.Link.CLOSED: if direct_link.activated_at != None: - RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was closed, reason: "+str(direct_link.teardown_reason), RNS.LOG_DEBUG) + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was closed", RNS.LOG_DEBUG) else: if not hasattr(lxmessage, "path_request_retried"): RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was never activated, retrying path request...", RNS.LOG_DEBUG) From 19696d206d34eddfa72542da6fd974e15403c9ef Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 12 Sep 2024 21:28:50 +0200 Subject: [PATCH 231/436] Actually set property --- LXMF/LXMRouter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 533785f..b7d327f 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1758,6 +1758,7 @@ class LXMRouter: backchannel_identity = self.delivery_destinations[source_destination_hash].identity backchannel_desthash = RNS.Destination.hash_from_name_and_identity("lxmf.delivery", backchannel_identity) direct_link.identify(backchannel_identity) + direct_link.backchannel_identified = True self.delivery_link_established(direct_link) RNS.log(f"Performed backchannel identification as {RNS.prettyhexrep(backchannel_desthash)} on {direct_link}", RNS.LOG_DEBUG) From 0e2f0fb09098ffc3450a25b5e236bebb09842302 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 16 Sep 2024 17:49:22 +0200 Subject: [PATCH 232/436] Updated example --- docs/example_receiver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/example_receiver.py b/docs/example_receiver.py index 999f6a3..9bf1c61 100644 --- a/docs/example_receiver.py +++ b/docs/example_receiver.py @@ -29,8 +29,8 @@ def delivery_callback(message): RNS.log("\t| Destination instance : "+str(message.get_destination())) RNS.log("\t| Transport Encryption : "+str(message.transport_encryption)) RNS.log("\t| Timestamp : "+time_string) - RNS.log("\t| Title : "+message.title_as_string()) - RNS.log("\t| Content : "+message.content_as_string()) + RNS.log("\t| Title : "+str(message.title_as_string())) + RNS.log("\t| Content : "+str(message.content_as_string())) RNS.log("\t| Fields : "+str(message.fields)) if message.ratchet_id: RNS.log("\t| Ratchet : "+str(RNS.Identity._get_ratchet_id(message.ratchet_id))) From 7789e0bc26ac926ce5f4ea4f769236e525d945a4 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 16 Sep 2024 17:49:54 +0200 Subject: [PATCH 233/436] Improved opportunistic delivery logic and performance --- LXMF/LXMRouter.py | 26 ++++++++++++++++++-------- LXMF/LXMessage.py | 40 +++++++++++++++++++++++++++------------- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index b7d327f..0a603ed 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -21,6 +21,7 @@ class LXMRouter: PROCESSING_INTERVAL = 4 DELIVERY_RETRY_WAIT = 10 PATH_REQUEST_WAIT = 7 + MAX_PATHLESS_TRIES = 2 LINK_MAX_INACTIVITY = 10*60 P_LINK_MAX_INACTIVITY = 3*60 @@ -367,7 +368,6 @@ class LXMRouter: self.propagation_transfer_state = LXMRouter.PR_PATH_REQUESTED self.request_messages_path_job() else: - # TODO: Remove at some point RNS.log("Waiting for propagation node link to become active", RNS.LOG_EXTREME) else: RNS.log("Cannot request LXMF propagation node sync, no default propagation node configured", RNS.LOG_WARNING) @@ -893,7 +893,6 @@ class LXMRouter: expired_outbound.append(destination_hash) for destination_hash in expired_outbound: - RNS.log(f"Cleaning expired outbound ticket for {destination_hash}") # TODO: Remove self.available_tickets["outbound"].pop(destination_hash) # Clean inbound tickets @@ -906,7 +905,6 @@ class LXMRouter: expired_inbound.append(inbound_ticket) for inbound_ticket in expired_inbound: - RNS.log(f"Cleaning expired inbound ticket for {destination_hash}") # TODO: Remove self.available_tickets["inbound"][destination_hash].pop(destination_hash) except Exception as e: @@ -916,7 +914,6 @@ class LXMRouter: def save_available_tickets(self): with self.ticket_file_lock: try: - RNS.log("Saving available tickets...", RNS.LOG_DEBUG) # TODO: Remove if not os.path.isdir(self.storagepath): os.makedirs(self.storagepath) @@ -1774,11 +1771,24 @@ class LXMRouter: # Outbound handling for opportunistic messages if lxmessage.method == LXMessage.OPPORTUNISTIC: if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: - if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: + if lxmessage.delivery_attempts >= LXMRouter.MAX_PATHLESS_TRIES and not RNS.Transport.has_path(lxmessage.get_destination().hash): + RNS.log(f"Requesting path to {RNS.prettyhexrep(lxmessage.get_destination().hash)} after {lxmessage.delivery_attempts} pathless tries for {lxmessage}", RNS.LOG_DEBUG) lxmessage.delivery_attempts += 1 - lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT - RNS.log("Opportunistic delivery attempt "+str(lxmessage.delivery_attempts)+" for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - lxmessage.send() + RNS.Transport.request_path(lxmessage.get_destination().hash) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + lxmessage.progress = 0.00 + elif lxmessage.delivery_attempts == LXMRouter.MAX_PATHLESS_TRIES+2 and RNS.Transport.has_path(lxmessage.get_destination().hash): + RNS.log(f"Opportunistic delivery for {lxmessage} still unsuccessful after {lxmessage.delivery_attempts} attempts, trying to update path to {RNS.prettyhexrep(lxmessage.get_destination().hash)}", RNS.LOG_DEBUG) + lxmessage.delivery_attempts += 1 + RNS.Transport.request_path(lxmessage.get_destination().hash) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + lxmessage.progress = 0.00 + else: + if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: + lxmessage.delivery_attempts += 1 + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + RNS.log("Opportunistic delivery attempt "+str(lxmessage.delivery_attempts)+" for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + lxmessage.send() else: RNS.log("Max delivery attempts reached for oppertunistic "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) self.fail_message(lxmessage) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 08e1077..21334a8 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -49,18 +49,20 @@ class LXMessage: TICKET_INTERVAL = 1*24*60*60 COST_TICKET = 0x100 - # LXMF overhead is 111 bytes per message: + # LXMF overhead is 112 bytes per message: # 16 bytes for destination hash # 16 bytes for source hash # 64 bytes for Ed25519 signature # 8 bytes for timestamp - # 7 bytes for msgpack structure - LXMF_OVERHEAD = 2*DESTINATION_LENGTH + SIGNATURE_LENGTH + 8 + 7 + # 8 bytes for msgpack structure + TIMESTAMP_SIZE = 8 + STRUCT_OVERHEAD = 8 + LXMF_OVERHEAD = 2*DESTINATION_LENGTH + SIGNATURE_LENGTH + TIMESTAMP_SIZE + STRUCT_OVERHEAD # With an MTU of 500, the maximum amount of data # we can send in a single encrypted packet is - # 383 bytes. - ENCRYPTED_PACKET_MDU = RNS.Packet.ENCRYPTED_MDU + # 391 bytes. + ENCRYPTED_PACKET_MDU = RNS.Packet.ENCRYPTED_MDU + TIMESTAMP_SIZE # The max content length we can fit in LXMF message # inside a single RNS packet is the encrypted MDU, minus @@ -69,7 +71,7 @@ class LXMessage: # field of the packet, therefore we also add the length # of a destination hash to the calculation. With default # RNS and LXMF parameters, the largest single-packet - # LXMF message we can send is 288 bytes. If a message + # LXMF message we can send is 295 bytes. If a message # is larger than that, a Reticulum link will be used. ENCRYPTED_PACKET_MAX_CONTENT = ENCRYPTED_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH @@ -79,13 +81,13 @@ class LXMessage: LINK_PACKET_MDU = RNS.Link.MDU # Which means that we can deliver single-packet LXMF - # messages with content of up to 320 bytes over a link. + # messages with content of up to 319 bytes over a link. # If a message is larger than that, LXMF will sequence # and transfer it as a RNS resource over the link instead. LINK_PACKET_MAX_CONTENT = LINK_PACKET_MDU - LXMF_OVERHEAD # For plain packets without encryption, we can - # fit up to 369 bytes of content. + # fit up to 368 bytes of content. PLAIN_PACKET_MDU = RNS.Packet.PLAIN_MDU PLAIN_PACKET_MAX_CONTENT = PLAIN_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH @@ -129,8 +131,16 @@ class LXMessage: if title == None: title = "" - self.set_title_from_string(title) - self.set_content_from_string(content) + if type(title) == bytes: + self.set_title_from_bytes(title) + else: + self.set_title_from_string(title) + + if type(content) == bytes: + self.set_content_from_bytes(content) + else: + self.set_content_from_string(content) + self.set_fields(fields) self.payload = None @@ -192,7 +202,11 @@ class LXMessage: self.content = content_bytes def content_as_string(self): - return self.content.decode("utf-8") + try: + return self.content.decode("utf-8") + except Exception as e: + RNS.log(f"{self} could not decode message content as string: {e}") + return None def set_fields(self, fields): if isinstance(fields, dict) or fields == None: @@ -352,7 +366,7 @@ class LXMessage: self.packed += self.signature self.packed += packed_payload self.packed_size = len(self.packed) - content_size = len(packed_payload) + content_size = len(packed_payload)-LXMessage.TIMESTAMP_SIZE-LXMessage.STRUCT_OVERHEAD # If no desired delivery method has been defined, # one will be chosen according to these rules: @@ -367,7 +381,7 @@ class LXMessage: single_packet_content_limit = LXMessage.PLAIN_PACKET_MAX_CONTENT if content_size > single_packet_content_limit: - raise TypeError("LXMessage desired opportunistic delivery method, but content exceeds single-packet size.") + raise TypeError(f"LXMessage desired opportunistic delivery method, but content of length {content_size} exceeds single-packet content limit of {single_packet_content_limit}.") else: self.method = LXMessage.OPPORTUNISTIC self.representation = LXMessage.PACKET From b5d6ed3d9b45a7959498dc81873f6d7904b7ccf5 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 16 Sep 2024 20:08:26 +0200 Subject: [PATCH 234/436] Automatically reconfigure to direct delivery if opportunistic message content is too large --- LXMF/LXMRouter.py | 8 +++++++- LXMF/LXMessage.py | 11 ++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 0a603ed..aa0578a 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -536,6 +536,12 @@ class LXMRouter: def information_storage_size(self): pass + def delivery_link_available(self, destination_hash): + if destination_hash in self.direct_links or destination_hash in self.backchannel_links: + return True + else: + return False + ### Utility & Maintenance ############################# ####################################################### @@ -1746,7 +1752,7 @@ class LXMRouter: # Prepare link for backchannel communications delivery_destination_hash = lxmessage.get_destination().hash - if delivery_destination_hash in self.direct_links: + if lxmessage.method == LXMessage.DIRECT and delivery_destination_hash in self.direct_links: direct_link = self.direct_links[delivery_destination_hash] if not hasattr(direct_link, "backchannel_identified") or direct_link.backchannel_identified == False: if direct_link.initiator == True: diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 21334a8..253085b 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -372,8 +372,16 @@ class LXMessage: # one will be chosen according to these rules: if self.desired_method == None: self.desired_method = LXMessage.DIRECT - # TODO: Expand rules to something more intelligent + + # If opportunistic delivery was requested, check + # that message will fit within packet size limits + if self.desired_method == LXMessage.OPPORTUNISTIC: + if self.__destination.type == RNS.Destination.SINGLE: + if content_size > LXMessage.ENCRYPTED_PACKET_MAX_CONTENT: + RNS.log(f"Opportunistic delivery was requested for {self}, but content exceeds packet size limit. Falling back to link-based delivery.", RNS.LOG_DEBUG) + self.desired_method = LXMessage.DIRECT + # Set delivery parameters according to delivery method if self.desired_method == LXMessage.OPPORTUNISTIC: if self.__destination.type == RNS.Destination.SINGLE: single_packet_content_limit = LXMessage.ENCRYPTED_PACKET_MAX_CONTENT @@ -434,6 +442,7 @@ class LXMessage: if self.method == LXMessage.OPPORTUNISTIC: lxm_packet = self.__as_packet() lxm_packet.send().set_delivery_callback(self.__mark_delivered) + self.progress = 0.50 self.ratchet_id = lxm_packet.ratchet_id self.state = LXMessage.SENT From efc15bde269efdf7e0de7b7b526e1387b8a6b733 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 16 Sep 2024 20:09:57 +0200 Subject: [PATCH 235/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index dd9b22c..7225152 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.5.1" +__version__ = "0.5.2" From 67d21deff9d4038e443891b4bc441135af4b8d8d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 16 Sep 2024 20:11:53 +0200 Subject: [PATCH 236/436] Updated dependencies --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8117b1f..6b7926a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ qrcode==7.4.2 -rns==0.7.7 +rns==0.7.8 setuptools==70.0.0 diff --git a/setup.py b/setup.py index 52489a1..da87c8e 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.7.7'], + install_requires=['rns>=0.7.8'], python_requires='>=3.7', ) From cf6fc4a5105689ba1932b0dc309e6e2e1bf82958 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 17 Sep 2024 12:56:56 +0200 Subject: [PATCH 237/436] Update max pathless tries --- LXMF/LXMRouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index aa0578a..2a05f5b 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -21,7 +21,7 @@ class LXMRouter: PROCESSING_INTERVAL = 4 DELIVERY_RETRY_WAIT = 10 PATH_REQUEST_WAIT = 7 - MAX_PATHLESS_TRIES = 2 + MAX_PATHLESS_TRIES = 1 LINK_MAX_INACTIVITY = 10*60 P_LINK_MAX_INACTIVITY = 3*60 From e1a56be5c1e440c6eca869ad3b02ada940741fd0 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 17 Sep 2024 13:18:16 +0200 Subject: [PATCH 238/436] Pre-emptively request unknown paths for opportunistic messages --- LXMF/LXMRouter.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 2a05f5b..6aafce3 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -276,6 +276,10 @@ class LXMRouter: else: return None + def set_active_propagation_node(self, destination_hash): + self.set_outbound_propagation_node(destination_hash) + # self.set_inbound_propagation_node(destination_hash) + def set_outbound_propagation_node(self, destination_hash): if len(destination_hash) != RNS.Identity.TRUNCATED_HASHLENGTH//8 or type(destination_hash) != bytes: raise ValueError("Invalid destination hash for outbound propagation node") @@ -289,6 +293,13 @@ class LXMRouter: def get_outbound_propagation_node(self): return self.outbound_propagation_node + def set_inbound_propagation_node(self, destination_hash): + # TODO: Implement + raise NotImplementedError("Inbound/outbound propagation node differentiation is currently not implemented") + + def get_inbound_propagation_node(self): + return self.get_outbound_propagation_node() + def set_retain_node_lxms(self, retain): if retain == True: self.retain_synced_on_node = True @@ -1195,6 +1206,7 @@ class LXMRouter: def handle_outbound(self, lxmessage): destination_hash = lxmessage.get_destination().hash + if lxmessage.stamp_cost == None: if destination_hash in self.outbound_stamp_costs: stamp_cost = self.outbound_stamp_costs[destination_hash][1] @@ -1220,10 +1232,13 @@ class LXMRouter: if not lxmessage.packed: lxmessage.pack() - lxmessage.determine_transport_encryption() + unknown_path_requested = False + if not RNS.Transport.has_path(destination_hash) and lxmessage.method == LXMessage.OPPORTUNISTIC: + RNS.log(f"Pre-emptively requesting unknown path for opportunistic {lxmessage}", RNS.LOG_DEBUG) + RNS.Transport.request_path(destination_hash) + unknown_path_requested = True - while self.processing_outbound: - time.sleep(0.1) + lxmessage.determine_transport_encryption() if lxmessage.defer_stamp and lxmessage.stamp_cost == None: RNS.log(f"Deferred stamp generation was requested for {lxmessage}, but no stamp is required, processing immediately", RNS.LOG_DEBUG) @@ -1231,7 +1246,10 @@ class LXMRouter: if not lxmessage.defer_stamp: self.pending_outbound.append(lxmessage) - self.process_outbound() + if not unknown_path_requested: + while self.processing_outbound: + time.sleep(0.05) + self.process_outbound() else: self.pending_deferred_stamps[lxmessage.message_id] = lxmessage @@ -1783,7 +1801,7 @@ class LXMRouter: RNS.Transport.request_path(lxmessage.get_destination().hash) lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT lxmessage.progress = 0.00 - elif lxmessage.delivery_attempts == LXMRouter.MAX_PATHLESS_TRIES+2 and RNS.Transport.has_path(lxmessage.get_destination().hash): + elif lxmessage.delivery_attempts == LXMRouter.MAX_PATHLESS_TRIES+3 and RNS.Transport.has_path(lxmessage.get_destination().hash): RNS.log(f"Opportunistic delivery for {lxmessage} still unsuccessful after {lxmessage.delivery_attempts} attempts, trying to update path to {RNS.prettyhexrep(lxmessage.get_destination().hash)}", RNS.LOG_DEBUG) lxmessage.delivery_attempts += 1 RNS.Transport.request_path(lxmessage.get_destination().hash) From ef1c3331adaabf23fa3e4cdf552f9d0ee06b23d7 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 17 Sep 2024 13:29:36 +0200 Subject: [PATCH 239/436] Improved outbound handling --- LXMF/LXMRouter.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 6aafce3..34cfbc8 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1236,6 +1236,7 @@ class LXMRouter: if not RNS.Transport.has_path(destination_hash) and lxmessage.method == LXMessage.OPPORTUNISTIC: RNS.log(f"Pre-emptively requesting unknown path for opportunistic {lxmessage}", RNS.LOG_DEBUG) RNS.Transport.request_path(destination_hash) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT unknown_path_requested = True lxmessage.determine_transport_encryption() @@ -1245,11 +1246,13 @@ class LXMRouter: lxmessage.defer_stamp = False if not lxmessage.defer_stamp: + while not unknown_path_requested and self.processing_outbound: + time.sleep(0.05) + self.pending_outbound.append(lxmessage) if not unknown_path_requested: - while self.processing_outbound: - time.sleep(0.05) self.process_outbound() + else: self.pending_deferred_stamps[lxmessage.message_id] = lxmessage @@ -1786,6 +1789,7 @@ class LXMRouter: elif lxmessage.method == LXMessage.PROPAGATED and lxmessage.state == LXMessage.SENT: RNS.log("Propagation has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) self.pending_outbound.remove(lxmessage) + else: RNS.log("Starting outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) From a461fd415afcf80ff15c2a1d1cbd61bb5ccf6d04 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 19 Sep 2024 23:00:24 +0200 Subject: [PATCH 240/436] Rediscover path during opportunistic delivery if path fails --- LXMF/LXMRouter.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 34cfbc8..e3480cb 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1805,10 +1805,14 @@ class LXMRouter: RNS.Transport.request_path(lxmessage.get_destination().hash) lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT lxmessage.progress = 0.00 - elif lxmessage.delivery_attempts == LXMRouter.MAX_PATHLESS_TRIES+3 and RNS.Transport.has_path(lxmessage.get_destination().hash): - RNS.log(f"Opportunistic delivery for {lxmessage} still unsuccessful after {lxmessage.delivery_attempts} attempts, trying to update path to {RNS.prettyhexrep(lxmessage.get_destination().hash)}", RNS.LOG_DEBUG) + elif lxmessage.delivery_attempts == LXMRouter.MAX_PATHLESS_TRIES+1 and RNS.Transport.has_path(lxmessage.get_destination().hash): + RNS.log(f"Opportunistic delivery for {lxmessage} still unsuccessful after {lxmessage.delivery_attempts} attempts, trying to rediscover path to {RNS.prettyhexrep(lxmessage.get_destination().hash)}", RNS.LOG_DEBUG) lxmessage.delivery_attempts += 1 - RNS.Transport.request_path(lxmessage.get_destination().hash) + RNS.Reticulum.get_instance().drop_path(lxmessage.get_destination().hash) + def rediscover_job(): + time.sleep(0.5) + RNS.Transport.request_path(lxmessage.get_destination().hash) + threading.Thread(target=rediscover_job, daemon=True).start() lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT lxmessage.progress = 0.00 else: From 05f144ae506568c822284f73c3809097e22bdbc2 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 19 Sep 2024 23:07:58 +0200 Subject: [PATCH 241/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 7225152..43a1e95 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.5.2" +__version__ = "0.5.3" From ac2c9c3a9b0964f4f212663de4fc0362c02e1d13 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 30 Sep 2024 19:27:40 +0200 Subject: [PATCH 242/436] Improved opportunistic delivery performance --- LXMF/Handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index eb10a76..5ce5571 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -15,7 +15,7 @@ class LXMFDeliveryAnnounceHandler: def received_announce(self, destination_hash, announced_identity, app_data): for lxmessage in self.lxmrouter.pending_outbound: if destination_hash == lxmessage.destination_hash: - if lxmessage.method == LXMessage.DIRECT: + if lxmessage.method == LXMessage.DIRECT or lxmessage.method == LXMessage.OPPORTUNISTIC: lxmessage.next_delivery_attempt = time.time() while self.lxmrouter.processing_outbound: From 0e0d01a0b23896cb60423478cb37b68094dbf6a0 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 30 Sep 2024 19:28:13 +0200 Subject: [PATCH 243/436] Updated versions --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 43a1e95..6b27eee 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.5.3" +__version__ = "0.5.4" diff --git a/setup.py b/setup.py index da87c8e..9832796 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.7.8'], + install_requires=['rns>=0.8.1'], python_requires='>=3.7', ) From eefb1c8349827185350075ea54f123f2e12b3b11 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 2 Oct 2024 02:06:00 +0200 Subject: [PATCH 244/436] Updated versions --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9832796..109d799 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.8.1'], + install_requires=['rns>=0.8.0'], python_requires='>=3.7', ) From 19f0fa77242cfdf2f3a05b1fc52cd4ad4ccdf68d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 4 Oct 2024 11:22:39 +0200 Subject: [PATCH 245/436] Don't try creating LXMF peer destination until identity is resolved. Fixes #19. --- LXMF/LXMPeer.py | 10 +++++++--- LXMF/_version.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index a0b61a5..0962b26 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -108,7 +108,10 @@ class LXMPeer: self.router = router self.destination_hash = destination_hash self.identity = RNS.Identity.recall(destination_hash) - self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + if self.identity != None: + self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + else: + RNS.log(f"Could not recall identity for LXMF propagation peer {RNS.prettyhexrep(self.destination_hash)}, will retry identity resolution on next sync", RNS.LOG_WARNING) def sync(self): RNS.log("Initiating LXMF Propagation Node sync with peer "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_DEBUG) @@ -126,9 +129,10 @@ class LXMPeer: else: if self.identity == None: self.identity = RNS.Identity.recall(destination_hash) - self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + if self.identity != None: + self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") - if self.identity != None: + if self.destination != None: if len(self.unhandled_messages) > 0: if self.state == LXMPeer.IDLE: RNS.log("Establishing link for sync to peer "+RNS.prettyhexrep(self.destination_hash)+"...", RNS.LOG_DEBUG) diff --git a/LXMF/_version.py b/LXMF/_version.py index 6b27eee..86716a7 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.5.4" +__version__ = "0.5.5" From 19d8909b104263227922310cf08a9fbc7affe0c2 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 5 Oct 2024 15:26:18 +0200 Subject: [PATCH 246/436] Fixed inbound ticket cleaning bug --- LXMF/LXMRouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index e3480cb..4157741 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -922,7 +922,7 @@ class LXMRouter: expired_inbound.append(inbound_ticket) for inbound_ticket in expired_inbound: - self.available_tickets["inbound"][destination_hash].pop(destination_hash) + self.available_tickets["inbound"][destination_hash].pop(inbound_ticket) except Exception as e: RNS.log(f"Error while cleaning available tickets. The contained exception was: {e}", RNS.LOG_ERROR) From 0178fb0d4f822fe2172d117d001ef96f2ce226a7 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 5 Oct 2024 22:40:14 +0200 Subject: [PATCH 247/436] Don't reset propagation node address and links if already set to the same destination --- LXMF/LXMRouter.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 4157741..7968854 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -284,11 +284,12 @@ class LXMRouter: if len(destination_hash) != RNS.Identity.TRUNCATED_HASHLENGTH//8 or type(destination_hash) != bytes: raise ValueError("Invalid destination hash for outbound propagation node") else: - self.outbound_propagation_node = destination_hash - if self.outbound_propagation_link != None: - if self.outbound_propagation_link.destination.hash != destination_hash: - self.outbound_propagation_link.teardown() - self.outbound_propagation_link = None + if self.outbound_propagation_node != destination_hash: + self.outbound_propagation_node = destination_hash + if self.outbound_propagation_link != None: + if self.outbound_propagation_link.destination.hash != destination_hash: + self.outbound_propagation_link.teardown() + self.outbound_propagation_link = None def get_outbound_propagation_node(self): return self.outbound_propagation_node From 672d7542386375bfb1c015162336973d31b6f62e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 6 Oct 2024 11:13:38 +0200 Subject: [PATCH 248/436] Updated dependency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 109d799..1fb47e9 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.8.0'], + install_requires=['rns>=0.8.2'], python_requires='>=3.7', ) From 0cb771439fa5b4f0d6d632f46e02e54952e7b3f5 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 11 Oct 2024 23:40:27 +0200 Subject: [PATCH 249/436] Fixed incorrect progress values on path waiting --- LXMF/LXMRouter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 7968854..068dc44 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1805,7 +1805,7 @@ class LXMRouter: lxmessage.delivery_attempts += 1 RNS.Transport.request_path(lxmessage.get_destination().hash) lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT - lxmessage.progress = 0.00 + lxmessage.progress = 0.01 elif lxmessage.delivery_attempts == LXMRouter.MAX_PATHLESS_TRIES+1 and RNS.Transport.has_path(lxmessage.get_destination().hash): RNS.log(f"Opportunistic delivery for {lxmessage} still unsuccessful after {lxmessage.delivery_attempts} attempts, trying to rediscover path to {RNS.prettyhexrep(lxmessage.get_destination().hash)}", RNS.LOG_DEBUG) lxmessage.delivery_attempts += 1 @@ -1815,7 +1815,7 @@ class LXMRouter: RNS.Transport.request_path(lxmessage.get_destination().hash) threading.Thread(target=rediscover_job, daemon=True).start() lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT - lxmessage.progress = 0.00 + lxmessage.progress = 0.01 else: if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: lxmessage.delivery_attempts += 1 @@ -1901,7 +1901,7 @@ class LXMRouter: RNS.log("No path known for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+". Requesting path...", RNS.LOG_DEBUG) RNS.Transport.request_path(lxmessage.get_destination().hash) lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT - lxmessage.progress = 0.00 + lxmessage.progress = 0.01 else: RNS.log("Max delivery attempts reached for direct "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) self.fail_message(lxmessage) From aa406d15521b4a15a58d6df1efc2ceb06856ccd7 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 11 Oct 2024 23:45:24 +0200 Subject: [PATCH 250/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 86716a7..a779a44 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.5.5" +__version__ = "0.5.6" From 36f0c17c8b73d707bb4351b9f1d379417763b2c4 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 13 Oct 2024 13:05:52 +0200 Subject: [PATCH 251/436] Added RNR_REFS field --- LXMF/LXMF.py | 1 + 1 file changed, 1 insertion(+) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 23b10b3..c0d3039 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -18,6 +18,7 @@ FIELD_RESULTS = 0x0A FIELD_GROUP = 0x0B FIELD_TICKET = 0x0C FIELD_EVENT = 0x0D +FIELD_RNR_REFS = 0x0E # For usecases such as including custom data structures, # embedding or encapsulating other data types or protocols From c9272c9218de9d12047f595c494c377142211248 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 13 Oct 2024 13:08:10 +0200 Subject: [PATCH 252/436] Fixed missing byteorder argument in stamp value calculation. Fixes #21. --- LXMF/LXStamper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXStamper.py b/LXMF/LXStamper.py index 28cf9cd..2023ec0 100644 --- a/LXMF/LXStamper.py +++ b/LXMF/LXStamper.py @@ -27,7 +27,7 @@ def stamp_value(workblock, stamp): value = 0 bits = 256 material = RNS.Identity.full_hash(workblock+stamp) - i = int.from_bytes(material) + i = int.from_bytes(material, byteorder="big") while ((i & (1 << (bits - 1))) == 0): i = (i << 1) value += 1 From 9ff76c0473e9d4107e079f266dd08144bb74c7c8 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 13 Oct 2024 14:01:10 +0200 Subject: [PATCH 253/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index a779a44..1cc82e6 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.5.6" +__version__ = "0.5.7" From 61331b58d72ae1f88adcc6b7085b260d95dffb14 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 23 Nov 2024 12:47:31 +0100 Subject: [PATCH 254/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 1cc82e6..fc0a843 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.5.7" +__version__ = "0.5.8" From b172c7fcd40c7b470fa0d5140be4e7f22b89ada8 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 23 Nov 2024 12:49:01 +0100 Subject: [PATCH 255/436] Added PN announce data validation to announce handler --- LXMF/Handlers.py | 32 ++++++++++++++++---------------- LXMF/LXMF.py | 28 +++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index 5ce5571..7420ea5 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -2,8 +2,7 @@ import time import RNS import RNS.vendor.umsgpack as msgpack -from .LXMF import APP_NAME, stamp_cost_from_app_data - +from .LXMF import APP_NAME, stamp_cost_from_app_data, pn_announce_data_is_valid from .LXMessage import LXMessage class LXMFDeliveryAnnounceHandler: @@ -40,23 +39,24 @@ class LXMFPropagationAnnounceHandler: def received_announce(self, destination_hash, announced_identity, app_data): try: if type(app_data) == bytes: - data = msgpack.unpackb(app_data) - if self.lxmrouter.propagation_node and self.lxmrouter.autopeer: - node_timebase = data[1] - propagation_transfer_limit = None - if len(data) >= 3: - try: - propagation_transfer_limit = float(data[2]) - except: - propagation_transfer_limit = None + data = msgpack.unpackb(app_data) - if data[0] == True: - if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: - self.lxmrouter.peer(destination_hash, node_timebase, propagation_transfer_limit) + if pn_announce_data_is_valid(data): + node_timebase = data[1] + propagation_transfer_limit = None + if len(data) >= 3: + try: + propagation_transfer_limit = float(data[2]) + except: + propagation_transfer_limit = None - elif data[0] == False: - self.lxmrouter.unpeer(destination_hash, node_timebase) + if data[0] == True: + if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: + self.lxmrouter.peer(destination_hash, node_timebase, propagation_transfer_limit) + + elif data[0] == False: + self.lxmrouter.unpeer(destination_hash, node_timebase) except Exception as e: RNS.log("Error while evaluating propagation node announce, ignoring announce.", RNS.LOG_DEBUG) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index c0d3039..184e530 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -83,6 +83,7 @@ AM_CUSTOM = 0xFF # handle and operate on LXMF data in client programs # ########################################################## +import RNS import RNS.vendor.umsgpack as msgpack def display_name_from_app_data(app_data=None): if app_data == None: @@ -104,8 +105,8 @@ def display_name_from_app_data(app_data=None): try: decoded = dn.decode("utf-8") return decoded - except: - RNS.log("Could not decode display name in included announce data. The contained exception was: {e}", RNS.LOG_ERROR) + except Exception as e: + RNS.log(f"Could not decode display name in included announce data. The contained exception was: {e}", RNS.LOG_ERROR) return None # Original announce format @@ -127,4 +128,25 @@ def stamp_cost_from_app_data(app_data=None): # Original announce format else: - return None \ No newline at end of file + return None + +def pn_announce_data_is_valid(data): + try: + if type(data) == bytes: + data = msgpack.unpackb(data) + + if len(data) < 3: + raise ValueError("Invalid announce data: Insufficient peer data") + else: + if data[0] != True and data[0] != False: + raise ValueError("Invalid announce data: Indeterminate propagation node status") + try: + int(data[1]) + except: + raise ValueError("Invalid announce data: Could not decode peer timebase") + + except Exception as e: + RNS.log(f"Could not validate propagation node announce data: {e}", RNS.LOG_DEBUG) + return False + + return True \ No newline at end of file From c21da895b6510952e1ae3771c13c84b5b3b04ba4 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 23 Nov 2024 13:20:24 +0100 Subject: [PATCH 256/436] Improved duplicate message detection when syncing from multiple different PNs --- LXMF/LXMRouter.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 068dc44..743cfa8 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -351,6 +351,7 @@ class LXMRouter: if self.outbound_propagation_node != None: if self.outbound_propagation_link != None and self.outbound_propagation_link.status == RNS.Link.ACTIVE: self.propagation_transfer_state = LXMRouter.PR_LINK_ESTABLISHED + RNS.log("Requesting message list from propagation node", RNS.LOG_DEBUG) self.outbound_propagation_link.identify(identity) self.outbound_propagation_link.request( LXMPeer.MESSAGE_GET_PATH, @@ -643,7 +644,7 @@ class LXMRouter: removed_entries = [] for transient_id in self.locally_delivered_transient_ids: timestamp = self.locally_delivered_transient_ids[transient_id] - if now > timestamp+LXMRouter.MESSAGE_EXPIRY*1.25: + if now > timestamp+LXMRouter.MESSAGE_EXPIRY*6.0: removed_entries.append(transient_id) for transient_id in removed_entries: @@ -653,7 +654,7 @@ class LXMRouter: removed_entries = [] for transient_id in self.locally_processed_transient_ids: timestamp = self.locally_processed_transient_ids[transient_id] - if now > timestamp+LXMRouter.MESSAGE_EXPIRY*1.25: + if now > timestamp+LXMRouter.MESSAGE_EXPIRY*6.0: removed_entries.append(transient_id) for transient_id in removed_entries: @@ -854,9 +855,8 @@ class LXMRouter: if not os.path.isdir(self.storagepath): os.makedirs(self.storagepath) - locally_delivered_file = open(self.storagepath+"/local_deliveries", "wb") - locally_delivered_file.write(msgpack.packb(self.locally_delivered_transient_ids)) - locally_delivered_file.close() + with open(self.storagepath+"/local_deliveries", "wb") as locally_delivered_file: + locally_delivered_file.write(msgpack.packb(self.locally_delivered_transient_ids)) except Exception as e: RNS.log("Could not save locally delivered message ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) @@ -866,9 +866,8 @@ class LXMRouter: if not os.path.isdir(self.storagepath): os.makedirs(self.storagepath) - locally_processed_file = open(self.storagepath+"/locally_processed", "wb") - locally_processed_file.write(msgpack.packb(self.locally_processed_transient_ids)) - locally_processed_file.close() + with open(self.storagepath+"/locally_processed", "wb") as locally_processed_file: + locally_processed_file.write(msgpack.packb(self.locally_processed_transient_ids)) except Exception as e: RNS.log("Could not save locally processed transient ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) @@ -1124,24 +1123,32 @@ class LXMRouter: wants = [] if len(request_receipt.response) > 0: for transient_id in request_receipt.response: - if not self.retain_synced_on_node and self.has_message(transient_id): - haves.append(transient_id) + if self.has_message(transient_id): + if not self.retain_synced_on_node: + haves.append(transient_id) else: if self.propagation_transfer_max_messages == LXMRouter.PR_ALL_MESSAGES or len(wants) < self.propagation_transfer_max_messages: wants.append(transient_id) + ms = "" if len(wants) == 1 else "s" + RNS.log(f"Requesting {len(wants)} message{ms} from propagation node", RNS.LOG_DEBUG) request_receipt.link.request( LXMPeer.MESSAGE_GET_PATH, [wants, haves, self.delivery_per_transfer_limit], response_callback=self.message_get_response, failed_callback=self.message_get_failed, - progress_callback=self.message_get_progress - ) + progress_callback=self.message_get_progress) + else: self.propagation_transfer_state = LXMRouter.PR_COMPLETE self.propagation_transfer_progress = 1.0 self.propagation_transfer_last_result = 0 + else: + RNS.log("Invalid message list data received from propagation node", RNS.LOG_DEBUG) + if self.outbound_propagation_link != None: + self.outbound_propagation_link.teardown() + def message_get_response(self, request_receipt): if request_receipt.response == LXMPeer.ERROR_NO_IDENTITY: RNS.log("Propagation node indicated missing identification on get request, tearing down link.", RNS.LOG_DEBUG) From 575fbc9ffe820feb5a92acc0c00b7a8582f603a0 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 23 Nov 2024 13:20:43 +0100 Subject: [PATCH 257/436] Updated dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1fb47e9..bb56614 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.8.2'], + install_requires=['rns>=0.8.6'], python_requires='>=3.7', ) From 1a43d93da2d35e78e9e40fd6e84ac6c6bf237fa3 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 9 Dec 2024 18:16:12 +0100 Subject: [PATCH 258/436] Added message renderer field --- LXMF/LXMF.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 184e530..db0edb7 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -19,6 +19,7 @@ FIELD_GROUP = 0x0B FIELD_TICKET = 0x0C FIELD_EVENT = 0x0D FIELD_RNR_REFS = 0x0E +FIELD_RENDERER = 0x0F # For usecases such as including custom data structures, # embedding or encapsulating other data types or protocols @@ -77,6 +78,18 @@ AM_OPUS_LOSSLESS = 0x19 # determine it itself based on the included data. AM_CUSTOM = 0xFF +# Message renderer specifications for FIELD_RENDERER. +# The renderer specification is completely optional, +# and only serves as an indication to the receiving +# client on how to render the message contents. It is +# not mandatory to implement, either on sending or +# receiving sides, but is the recommended way to +# signal how to render a message, if non-plaintext +# formatting is used. +RENDERER_PLAIN = 0x00 +RENDERER_MICRON = 0x01 +RENDERER_MARKDOWN = 0x02 +RENDERER_BBCODE = 0x03 ########################################################## # The following helper functions makes it easier to # From c426c93cc5d63a3dae18ad2264b1299a7ad9e46c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 9 Dec 2024 22:10:17 +0100 Subject: [PATCH 259/436] Updated versions --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index fc0a843..40e294f 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.5.8" +__version__ = "0.5.9" diff --git a/setup.py b/setup.py index bb56614..6ee237c 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.8.6'], + install_requires=['rns>=0.8.7'], python_requires='>=3.7', ) From 96dddf1b3a0328f22b59ac7d4fcc8193d9884a44 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 23 Dec 2024 12:36:53 +0100 Subject: [PATCH 260/436] Added handling of corrupted transient ID cache files --- LXMF/LXMRouter.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 743cfa8..fa168ef 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -143,19 +143,31 @@ class LXMRouter: if os.path.isfile(self.storagepath+"/local_deliveries"): locally_delivered_file = open(self.storagepath+"/local_deliveries", "rb") data = locally_delivered_file.read() - self.locally_delivered_transient_ids = msgpack.unpackb(data) locally_delivered_file.close() - - if os.path.isfile(self.storagepath+"/locally_processed"): - locally_processed_file = open(self.storagepath+"/locally_processed", "rb") - data = locally_processed_file.read() - self.locally_processed_transient_ids = msgpack.unpackb(data) - locally_processed_file.close() - - self.clean_transient_id_caches() + self.locally_delivered_transient_ids = msgpack.unpackb(data) except Exception as e: RNS.log("Could not load locally delivered message ID cache from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + self.locally_delivered_transient_ids = {} + + try: + if os.path.isfile(self.storagepath+"/locally_processed"): + locally_processed_file = open(self.storagepath+"/locally_processed", "rb") + data = locally_processed_file.read() + locally_processed_file.close() + self.locally_processed_transient_ids = msgpack.unpackb(data) + + except Exception as e: + RNS.log("Could not load locally processed message ID cache from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + self.locally_processed_transient_ids = {} + + try: + self.clean_transient_id_caches() + + except Exception as e: + RNS.log("Could not clean transient ID caches. The contained exception was : "+str(e), RNS.LOG_ERROR) + self.locally_delivered_transient_ids = {} + self.locally_processed_transient_ids = {} try: if os.path.isfile(self.storagepath+"/outbound_stamp_costs"): From a6a42eff80cd469f734dbfd24991e5b0713e0666 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 13 Jan 2025 14:35:14 +0100 Subject: [PATCH 261/436] Add sync transfer rate to peer stats --- LXMF/LXMPeer.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 0962b26..a88f6da 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -47,6 +47,11 @@ class LXMPeer: peer.link_establishment_rate = dictionary["link_establishment_rate"] else: peer.link_establishment_rate = 0 + + if "sync_transfer_rate" in dictionary: + peer.sync_transfer_rate = dictionary["sync_transfer_rate"] + else: + peer.sync_transfer_rate = 0 if "propagation_transfer_limit" in dictionary: try: @@ -73,6 +78,7 @@ class LXMPeer: dictionary["last_heard"] = self.last_heard dictionary["destination_hash"] = self.destination_hash dictionary["link_establishment_rate"] = self.link_establishment_rate + dictionary["sync_transfer_rate"] = self.sync_transfer_rate dictionary["propagation_transfer_limit"] = self.propagation_transfer_limit handled_ids = [] @@ -96,6 +102,7 @@ class LXMPeer: self.sync_backoff = 0 self.peering_timebase = 0 self.link_establishment_rate = 0 + self.sync_transfer_rate = 0 self.propagation_transfer_limit = None self.link = None @@ -257,6 +264,7 @@ class LXMPeer: data = msgpack.packb([time.time(), lxm_list]) resource = RNS.Resource(data, self.link, callback = self.resource_concluded) resource.transferred_messages = wanted_message_ids + resource.sync_transfer_started = time.time() self.state = LXMPeer.RESOURCE_TRANSFERRING else: @@ -289,7 +297,12 @@ class LXMPeer: self.link = None self.state = LXMPeer.IDLE - RNS.log("Sync to peer "+RNS.prettyhexrep(self.destination_hash)+" completed", RNS.LOG_DEBUG) + rate_str = "" + if hasattr(resource, "sync_transfer_started") and resource.sync_transfer_started: + self.sync_transfer_rate = (resource.get_transfer_size()*8)/(time.time()-resource.sync_transfer_started) + rate_str = f" at {RNS.prettyspeed(self.sync_transfer_rate)}" + + RNS.log("Sync to peer "+RNS.prettyhexrep(self.destination_hash)+" completed"+rate_str, RNS.LOG_DEBUG) self.alive = True self.last_heard = time.time() From 976305b791a1db8b11996fe8e6dd81cfdc4be863 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 13 Jan 2025 14:37:51 +0100 Subject: [PATCH 262/436] Sort waiting peers by sync transfer rate --- LXMF/LXMRouter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index fa168ef..5410a54 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1514,12 +1514,12 @@ class LXMRouter: if len(waiting_peers) > 0: fastest_peers = sorted( waiting_peers, - key=lambda p: p.link_establishment_rate, + key=lambda p: p.sync_transfer_rate, reverse=True )[0:min(LXMRouter.FASTEST_N_RANDOM_POOL, len(waiting_peers))] peer_pool.extend(fastest_peers) - unknown_speed_peers = [p for p in waiting_peers if p.link_establishment_rate == 0] + unknown_speed_peers = [p for p in waiting_peers if p.sync_transfer_rate == 0] if len(unknown_speed_peers) > 0: peer_pool.extend( unknown_speed_peers[ From 2d175a331fb259d0c0f01dd50907f58c28ccdf0d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 13 Jan 2025 15:26:27 +0100 Subject: [PATCH 263/436] Updated dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6ee237c..6d25b47 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.8.7'], + install_requires=['rns>=0.9.0'], python_requires='>=3.7', ) From d97c4f292e43dfe49b91fb457dab13ded15e9947 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 14 Jan 2025 21:32:10 +0100 Subject: [PATCH 264/436] Fixed missing checks for file corruption --- LXMF/LXMRouter.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 5410a54..64677a8 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -145,6 +145,9 @@ class LXMRouter: data = locally_delivered_file.read() locally_delivered_file.close() self.locally_delivered_transient_ids = msgpack.unpackb(data) + if not type(self.locally_delivered_transient_ids) == dict: + RNS.log("Invalid data format for loaded locally delivered transient IDs, recreating...", RNS.LOG_ERROR) + self.locally_delivered_transient_ids = {} except Exception as e: RNS.log("Could not load locally delivered message ID cache from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) @@ -156,6 +159,10 @@ class LXMRouter: data = locally_processed_file.read() locally_processed_file.close() self.locally_processed_transient_ids = msgpack.unpackb(data) + if not type(self.locally_processed_transient_ids) == dict: + RNS.log("Invalid data format for loaded locally processed transient IDs, recreating...", RNS.LOG_ERROR) + self.locally_processed_transient_ids = {} + except Exception as e: RNS.log("Could not load locally processed message ID cache from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) @@ -175,6 +182,9 @@ class LXMRouter: with open(self.storagepath+"/outbound_stamp_costs", "rb") as outbound_stamp_cost_file: data = outbound_stamp_cost_file.read() self.outbound_stamp_costs = msgpack.unpackb(data) + if not type(self.outbound_stamp_costs) == dict: + RNS.log("Invalid data format for loaded outbound stamp costs, recreating...", RNS.LOG_ERROR) + self.outbound_stamp_costs = {} self.clean_outbound_stamp_costs() self.save_outbound_stamp_costs() From a67695411697e1899f417f23156115283014928f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 18 Jan 2025 19:13:43 +0100 Subject: [PATCH 265/436] Added ability to cancel outbound messages --- LXMF/LXMRouter.py | 45 ++++++++++++++++++++++++++++++++++++++++----- LXMF/LXMessage.py | 23 +++++++++++++---------- 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 64677a8..5c71f93 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1234,6 +1234,30 @@ class LXMRouter: else: return False + def cancel_outbound(self, message_id): + try: + lxmessage = None + for lxm in self.pending_outbound: + if lxm.message_id == message_id: + lxmessage = lxm + + if message_id in self.pending_deferred_stamps: + RNS.log(f"Cancelling deferred stamp generation for {lxmessage}", RNS.LOG_DEBUG) + + if lxmessage != None: + lxmessage.state = LXMessage.CANCELLED + if lxmessage in self.pending_outbound: + RNS.log(f"Cancelling {lxmessage} in outbound queue", RNS.LOG_DEBUG) + if lxmessage.representation == LXMessage.RESOURCE: + if lxmessage.resource_representation != None: + lxmessage.resource_representation.cancel() + + self.process_outbound() + + except Exception as e: + RNS.log(f"An error occurred while cancelling {lxmessage}: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + def handle_outbound(self, lxmessage): destination_hash = lxmessage.get_destination().hash @@ -1780,10 +1804,15 @@ class LXMRouter: self.pending_outbound.append(selected_lxm) RNS.log(f"Stamp generation completed for {selected_lxm}", RNS.LOG_DEBUG) else: - RNS.log(f"Deferred stamp generation did not succeed. Failing {selected_lxm}.", RNS.LOG_ERROR) - selected_lxm.stamp_generation_failed = True - self.pending_deferred_stamps.pop(selected_message_id) - self.fail_message(selected_lxm) + if selected_lxm.state == LXMessage.CANCELLED: + RNS.log(f"Message cancelled during deferred stamp generation for {selected_lxm}.", RNS.LOG_ERROR) + selected_lxm.stamp_generation_failed = True + self.pending_deferred_stamps.pop(selected_message_id) + else: + RNS.log(f"Deferred stamp generation did not succeed. Failing {selected_lxm}.", RNS.LOG_ERROR) + selected_lxm.stamp_generation_failed = True + self.pending_deferred_stamps.pop(selected_message_id) + self.fail_message(selected_lxm) def process_outbound(self, sender = None): @@ -1820,8 +1849,14 @@ class LXMRouter: RNS.log("Propagation has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) self.pending_outbound.remove(lxmessage) + elif lxmessage.state == LXMessage.CANCELLED: + RNS.log("Cancellation requested for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) + self.pending_outbound.remove(lxmessage) + if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): + lxmessage.failed_callback(lxmessage) + else: - RNS.log("Starting outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + RNS.log("Outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) if lxmessage.progress == None or lxmessage.progress < 0.01: lxmessage.progress = 0.01 diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 253085b..316e798 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -16,8 +16,9 @@ class LXMessage: SENDING = 0x02 SENT = 0x04 DELIVERED = 0x08 + CANCELLED = 0xFE FAILED = 0xFF - states = [GENERATING, OUTBOUND, SENDING, SENT, DELIVERED, FAILED] + states = [GENERATING, OUTBOUND, SENDING, SENT, DELIVERED, CANCELLED, FAILED] UNKNOWN = 0x00 PACKET = 0x01 @@ -564,22 +565,24 @@ class LXMessage: if resource.status == RNS.Resource.COMPLETE: self.__mark_delivered() else: - resource.link.teardown() - self.state = LXMessage.OUTBOUND + if self.state != LXMessage.CANCELLED: + resource.link.teardown() + self.state = LXMessage.OUTBOUND def __propagation_resource_concluded(self, resource): if resource.status == RNS.Resource.COMPLETE: self.__mark_propagated() else: - resource.link.teardown() - self.state = LXMessage.OUTBOUND + if self.state != LXMessage.CANCELLED: + resource.link.teardown() + self.state = LXMessage.OUTBOUND def __link_packet_timed_out(self, packet_receipt): - if packet_receipt: - packet_receipt.destination.teardown() - - self.state = LXMessage.OUTBOUND - + if self.state != LXMessage.CANCELLED: + if packet_receipt: + packet_receipt.destination.teardown() + + self.state = LXMessage.OUTBOUND def __update_transfer_progress(self, resource): self.progress = 0.10 + (resource.get_progress()*0.90) From d6b1b9c94dc245b3255f3d5b5830ede1b5f97ef7 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 18 Jan 2025 20:11:31 +0100 Subject: [PATCH 266/436] Added ability to cancel stamp generation --- LXMF/LXMRouter.py | 24 ++++++++++++--- LXMF/LXStamper.py | 66 ++++++++++++++++++++++++++++++++++------ docs/example_receiver.py | 2 +- 3 files changed, 78 insertions(+), 14 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 5c71f93..2ec4e8e 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -16,6 +16,8 @@ from .LXMessage import LXMessage from .Handlers import LXMFDeliveryAnnounceHandler from .Handlers import LXMFPropagationAnnounceHandler +import LXMF.LXStamper as LXStamper + class LXMRouter: MAX_DELIVERY_ATTEMPTS = 5 PROCESSING_INTERVAL = 4 @@ -1236,14 +1238,17 @@ class LXMRouter: def cancel_outbound(self, message_id): try: + if message_id in self.pending_deferred_stamps: + lxm = self.pending_deferred_stamps[message_id] + RNS.log(f"Cancelling deferred stamp generation for {lxm}", RNS.LOG_DEBUG) + lxm.state = LXMessage.CANCELLED + LXStamper.cancel_work(message_id) + lxmessage = None for lxm in self.pending_outbound: if lxm.message_id == message_id: lxmessage = lxm - if message_id in self.pending_deferred_stamps: - RNS.log(f"Cancelling deferred stamp generation for {lxmessage}", RNS.LOG_DEBUG) - if lxmessage != None: lxmessage.state = LXMessage.CANCELLED if lxmessage in self.pending_outbound: @@ -1793,6 +1798,15 @@ class LXMRouter: selected_message_id = message_id if selected_lxm != None: + if selected_lxm.state == LXMessage.CANCELLED: + RNS.log(f"Message cancelled during deferred stamp generation for {selected_lxm}.", RNS.LOG_DEBUG) + selected_lxm.stamp_generation_failed = True + self.pending_deferred_stamps.pop(selected_message_id) + if selected_lxm.failed_callback != None and callable(selected_lxm.failed_callback): + selected_lxm.failed_callback(lxmessage) + + return + RNS.log(f"Starting stamp generation for {selected_lxm}...", RNS.LOG_DEBUG) generated_stamp = selected_lxm.get_stamp() if generated_stamp: @@ -1805,9 +1819,11 @@ class LXMRouter: RNS.log(f"Stamp generation completed for {selected_lxm}", RNS.LOG_DEBUG) else: if selected_lxm.state == LXMessage.CANCELLED: - RNS.log(f"Message cancelled during deferred stamp generation for {selected_lxm}.", RNS.LOG_ERROR) + RNS.log(f"Message cancelled during deferred stamp generation for {selected_lxm}.", RNS.LOG_DEBUG) selected_lxm.stamp_generation_failed = True self.pending_deferred_stamps.pop(selected_message_id) + if selected_lxm.failed_callback != None and callable(selected_lxm.failed_callback): + selected_lxm.failed_callback(lxmessage) else: RNS.log(f"Deferred stamp generation did not succeed. Failing {selected_lxm}.", RNS.LOG_ERROR) selected_lxm.stamp_generation_failed = True diff --git a/LXMF/LXStamper.py b/LXMF/LXStamper.py index 2023ec0..bcfa95b 100644 --- a/LXMF/LXStamper.py +++ b/LXMF/LXStamper.py @@ -7,6 +7,8 @@ import multiprocessing WORKBLOCK_EXPAND_ROUNDS = 3000 +active_jobs = {} + def stamp_workblock(message_id): wb_st = time.time() expand_rounds = WORKBLOCK_EXPAND_ROUNDS @@ -44,23 +46,56 @@ def generate_stamp(message_id, stamp_cost): value = 0 if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): - stamp, rounds = job_simple(stamp_cost, workblock) + stamp, rounds = job_simple(stamp_cost, workblock, message_id) elif RNS.vendor.platformutils.is_android(): - stamp, rounds = job_android(stamp_cost, workblock) + stamp, rounds = job_android(stamp_cost, workblock, message_id) else: - stamp, rounds = job_linux(stamp_cost, workblock) + stamp, rounds = job_linux(stamp_cost, workblock, message_id) duration = time.time() - start_time speed = rounds/duration - value = stamp_value(workblock, stamp) + if stamp != None: + value = stamp_value(workblock, stamp) RNS.log(f"Stamp with value {value} generated in {RNS.prettytime(duration)}, {rounds} rounds, {int(speed)} rounds per second", RNS.LOG_DEBUG) return stamp, value -def job_simple(stamp_cost, workblock): +def cancel_work(message_id): + if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): + try: + if message_id in active_jobs: + active_jobs[message_id] = True + + except Exception as e: + RNS.log("Error while terminating stamp generation workers: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + elif RNS.vendor.platformutils.is_android(): + try: + if message_id in active_jobs: + active_jobs[message_id] = True + + except Exception as e: + RNS.log("Error while terminating stamp generation workers: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + else: + try: + if message_id in active_jobs: + stop_event = active_jobs[message_id][0] + result_queue = active_jobs[message_id][1] + stop_event.set() + result_queue.put(None) + active_jobs.pop(message_id) + + except Exception as e: + RNS.log("Error while terminating stamp generation workers: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + +def job_simple(stamp_cost, workblock, message_id): # A simple, single-process stamp generator. # should work on any platform, and is used # as a fall-back, in case of limited multi- @@ -73,6 +108,8 @@ def job_simple(stamp_cost, workblock): pstamp = os.urandom(256//8) st = time.time() + active_jobs[message_id] = False; + def sv(s, c, w): target = 0b1<<256-c; m = w+s result = RNS.Identity.full_hash(m) @@ -81,15 +118,20 @@ def job_simple(stamp_cost, workblock): else: return True - while not sv(pstamp, stamp_cost, workblock): + while not sv(pstamp, stamp_cost, workblock) and not active_jobs[message_id]: pstamp = os.urandom(256//8); rounds += 1 if rounds % 2500 == 0: speed = rounds / (time.time()-st) RNS.log(f"Stamp generation running. {rounds} rounds completed so far, {int(speed)} rounds per second", RNS.LOG_DEBUG) + if active_jobs[message_id] == True: + pstamp = None + + active_jobs.pop(message_id) + return pstamp, rounds -def job_linux(stamp_cost, workblock): +def job_linux(stamp_cost, workblock, message_id): allow_kill = True stamp = None total_rounds = 0 @@ -126,6 +168,8 @@ def job_linux(stamp_cost, workblock): job_procs.append(process) process.start() + active_jobs[message_id] = [stop_event, result_queue] + stamp = result_queue.get() RNS.log("Got stamp result from worker", RNS.LOG_DEBUG) # TODO: Remove @@ -170,7 +214,7 @@ def job_linux(stamp_cost, workblock): return stamp, total_rounds -def job_android(stamp_cost, workblock): +def job_android(stamp_cost, workblock, message_id): # Semaphore support is flaky to non-existent on # Android, so we need to manually dispatch and # manage workloads here, while periodically @@ -230,10 +274,12 @@ def job_android(stamp_cost, workblock): RNS.log(f"Stamp generation worker error: {e}", RNS.LOG_ERROR) RNS.trace_exception(e) + active_jobs[message_id] = False; + RNS.log(f"Dispatching {jobs} workers for stamp generation...", RNS.LOG_DEBUG) # TODO: Remove results_dict = wm.dict() - while stamp == None: + while stamp == None and active_jobs[message_id] == False: job_procs = [] try: for pnum in range(jobs): @@ -260,6 +306,8 @@ def job_android(stamp_cost, workblock): RNS.log(f"Stamp generation job error: {e}") RNS.trace_exception(e) + active_jobs.pop(message_id) + return stamp, total_rounds if __name__ == "__main__": diff --git a/docs/example_receiver.py b/docs/example_receiver.py index 9bf1c61..20c2efe 100644 --- a/docs/example_receiver.py +++ b/docs/example_receiver.py @@ -69,4 +69,4 @@ while True: # input() # RNS.log("Requesting messages from propagation node...") - # router.request_messages_from_propagation_node(identity) \ No newline at end of file + # router.request_messages_from_propagation_node(identity) From 3948c9a1875417179b1d0ec4c1298ea0230a422f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 18 Jan 2025 21:36:08 +0100 Subject: [PATCH 267/436] Added message reject on too large transfer size --- LXMF/LXMRouter.py | 10 +++++++++- LXMF/LXMessage.py | 8 ++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 2ec4e8e..79678c6 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1777,7 +1777,9 @@ class LXMRouter: self.failed_outbound.append(lxmessage) - lxmessage.state = LXMessage.FAILED + if lxmessage.state != LXMessage.REJECTED: + lxmessage.state = LXMessage.FAILED + if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): lxmessage.failed_callback(lxmessage) @@ -1871,6 +1873,12 @@ class LXMRouter: if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): lxmessage.failed_callback(lxmessage) + elif lxmessage.state == LXMessage.REJECTED: + RNS.log("Receiver rejected "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) + self.pending_outbound.remove(lxmessage) + if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): + lxmessage.failed_callback(lxmessage) + else: RNS.log("Outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 316e798..2342708 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -16,9 +16,10 @@ class LXMessage: SENDING = 0x02 SENT = 0x04 DELIVERED = 0x08 + REJECTED = 0xFD CANCELLED = 0xFE FAILED = 0xFF - states = [GENERATING, OUTBOUND, SENDING, SENT, DELIVERED, CANCELLED, FAILED] + states = [GENERATING, OUTBOUND, SENDING, SENT, DELIVERED, REJECTED, CANCELLED, FAILED] UNKNOWN = 0x00 PACKET = 0x01 @@ -565,7 +566,10 @@ class LXMessage: if resource.status == RNS.Resource.COMPLETE: self.__mark_delivered() else: - if self.state != LXMessage.CANCELLED: + if resource.status == RNS.Resource.REJECTED: + self.state = LXMessage.REJECTED + + elif self.state != LXMessage.CANCELLED: resource.link.teardown() self.state = LXMessage.OUTBOUND From 7bd3cf986d490eef2d1b0ec15e42e62e84b37a0d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 18 Jan 2025 21:39:39 +0100 Subject: [PATCH 268/436] Updated versions --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 40e294f..906d362 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.5.9" +__version__ = "0.6.0" diff --git a/setup.py b/setup.py index 6d25b47..cabf20a 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.9.0'], + install_requires=['rns>=0.9.1'], python_requires='>=3.7', ) From 356cb6412fbfda050dd37d8a680bb6b13351b52f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 21 Jan 2025 10:46:59 +0100 Subject: [PATCH 269/436] Optimise structure overhead --- LXMF/LXMPeer.py | 30 +++++++++++++++++------------- LXMF/LXMRouter.py | 14 +++++++++++++- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index a88f6da..2b10987 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -63,12 +63,13 @@ class LXMPeer: for transient_id in dictionary["handled_ids"]: if transient_id in router.propagation_entries: - peer.handled_messages[transient_id] = router.propagation_entries[transient_id] + peer.handled_messages.append(transient_id) for transient_id in dictionary["unhandled_ids"]: if transient_id in router.propagation_entries: - peer.unhandled_messages[transient_id] = router.propagation_entries[transient_id] + peer.unhandled_messages.append(transient_id) + del dictionary return peer def to_bytes(self): @@ -108,8 +109,8 @@ class LXMPeer: self.link = None self.state = LXMPeer.IDLE - self.unhandled_messages = {} - self.handled_messages = {} + self.unhandled_messages = [] + self.handled_messages = [] self.last_offer = [] self.router = router @@ -118,6 +119,7 @@ class LXMPeer: if self.identity != None: self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") else: + self.destination = None RNS.log(f"Could not recall identity for LXMF propagation peer {RNS.prettyhexrep(self.destination_hash)}, will retry identity resolution on next sync", RNS.LOG_WARNING) def sync(self): @@ -171,7 +173,7 @@ class LXMPeer: for transient_id in purged_ids: RNS.log("Dropping unhandled message "+RNS.prettyhexrep(transient_id)+" for peer "+RNS.prettyhexrep(self.destination_hash)+" since it no longer exists in the message store.", RNS.LOG_DEBUG) - self.unhandled_messages.pop(transient_id) + self.unhandled_messages.remove(transient_id) unhandled_entries.sort(key=lambda e: e[1], reverse=False) per_message_overhead = 16 # Really only 2 bytes, but set a bit higher for now @@ -189,7 +191,7 @@ class LXMPeer: RNS.log("Sending sync request to peer "+str(self.destination), RNS.LOG_DEBUG) self.last_offer = unhandled_ids - self.link.request(LXMPeer.OFFER_REQUEST_PATH, self.last_offer, response_callback=self.offer_response, failed_callback=self.request_failed) + self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) self.state = LXMPeer.REQUEST_SENT else: @@ -226,13 +228,14 @@ class LXMPeer: # Peer already has all advertised messages for transient_id in self.last_offer: if transient_id in self.unhandled_messages: - self.handled_messages[transient_id] = self.unhandled_messages.pop(transient_id) + self.handled_messages.append(transient_id) + self.unhandled_messages.remove(transient_id) elif response == True: # Peer wants all advertised messages for transient_id in self.last_offer: - wanted_messages.append(self.unhandled_messages[transient_id]) + wanted_messages.append(self.router.propagation_entries[transient_id]) wanted_message_ids.append(transient_id) else: @@ -242,10 +245,11 @@ class LXMPeer: # already received it from another peer. if not transient_id in response: if transient_id in self.unhandled_messages: - self.handled_messages[transient_id] = self.unhandled_messages.pop(transient_id) + self.handled_messages.append(transient_id) + self.unhandled_messages.remove(transient_id) for transient_id in response: - wanted_messages.append(self.unhandled_messages[transient_id]) + wanted_messages.append(self.router.propagation_entries[transient_id]) wanted_message_ids.append(transient_id) if len(wanted_messages) > 0: @@ -288,8 +292,8 @@ class LXMPeer: def resource_concluded(self, resource): if resource.status == RNS.Resource.COMPLETE: for transient_id in resource.transferred_messages: - message = self.unhandled_messages.pop(transient_id) - self.handled_messages[transient_id] = message + self.handled_messages.append(transient_id) + self.unhandled_messages.remove(transient_id) if self.link != None: self.link.teardown() @@ -330,7 +334,7 @@ class LXMPeer: def handle_message(self, transient_id): if not transient_id in self.handled_messages and not transient_id in self.unhandled_messages: - self.unhandled_messages[transient_id] = self.router.propagation_entries[transient_id] + self.unhandled_messages.append(transient_id) def __str__(self): if self.destination_hash: diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 79678c6..a19f401 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1,5 +1,6 @@ import os import time +import math import random import base64 import atexit @@ -427,6 +428,8 @@ class LXMRouter: os.makedirs(self.messagepath) self.propagation_entries = {} + + st = time.time(); RNS.log("Indexing messagestore...", RNS.LOG_NOTICE) for filename in os.listdir(self.messagepath): components = filename.split("_") if len(components) == 2: @@ -452,9 +455,13 @@ class LXMRouter: except Exception as e: RNS.log("Could not read LXM from message store. The contained exception was: "+str(e), RNS.LOG_ERROR) + et = time.time(); RNS.log(f"Indexed {len(self.propagation_entries)} messages in {RNS.prettytime(et-st)}, {math.floor(len(self.propagation_entries)/(et-st))} msgs/s", RNS.LOG_NOTICE) + st = time.time(); RNS.log("Loading propagation node peers...", RNS.LOG_NOTICE) + if os.path.isfile(self.storagepath+"/peers"): peers_file = open(self.storagepath+"/peers", "rb") peers_data = peers_file.read() + peers_file.close() if len(peers_data) > 0: serialised_peers = msgpack.unpackb(peers_data) @@ -468,8 +475,13 @@ class LXMRouter: lim_str = ", "+RNS.prettysize(peer.propagation_transfer_limit*1000)+" transfer limit" RNS.log("Loaded peer "+RNS.prettyhexrep(peer.destination_hash)+" with "+str(len(peer.unhandled_messages))+" unhandled messages"+lim_str, RNS.LOG_DEBUG) else: + del peer RNS.log("Peer "+RNS.prettyhexrep(peer.destination_hash)+" could not be loaded, because its identity could not be recalled. Dropping peer.", RNS.LOG_DEBUG) + del serialised_peers + del peers_data + + RNS.log(f"Loaded {len(self.peers)} peers in {RNS.prettytime(time.time()-st)}", RNS.LOG_NOTICE) self.propagation_node = True self.propagation_destination.set_link_established_callback(self.propagation_link_established) @@ -1676,7 +1688,7 @@ class LXMRouter: if remote_hash != None and remote_hash in self.peers: transient_id = RNS.Identity.full_hash(lxmf_data) peer = self.peers[remote_hash] - peer.handled_messages[transient_id] = [transient_id, remote_timebase, lxmf_data] + peer.handled_messages.append(transient_id) self.lxmf_propagation(lxmf_data) else: From 7701f326d99b20bfed3d64c3a80809e02755a06f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 21 Jan 2025 16:33:39 +0100 Subject: [PATCH 270/436] Memory optimisations --- LXMF/LXMPeer.py | 120 +++++++++++++++++++++++++++++++------ LXMF/LXMRouter.py | 149 +++++++++++++++++++++++++++++++++------------- 2 files changed, 209 insertions(+), 60 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 2b10987..f4c522c 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -38,11 +38,16 @@ class LXMPeer: @staticmethod def from_bytes(peer_bytes, router): dictionary = msgpack.unpackb(peer_bytes) + peer_destination_hash = dictionary["destination_hash"] + peer_peering_timebase = dictionary["peering_timebase"] + peer_alive = dictionary["alive"] + peer_last_heard = dictionary["last_heard"] + + peer = LXMPeer(router, peer_destination_hash) + peer.peering_timebase = peer_peering_timebase + peer.alive = peer_alive + peer.last_heard = peer_last_heard - peer = LXMPeer(router, dictionary["destination_hash"]) - peer.peering_timebase = dictionary["peering_timebase"] - peer.alive = dictionary["alive"] - peer.last_heard = dictionary["last_heard"] if "link_establishment_rate" in dictionary: peer.link_establishment_rate = dictionary["link_establishment_rate"] else: @@ -61,13 +66,22 @@ class LXMPeer: else: peer.propagation_transfer_limit = None + hm_count = 0 for transient_id in dictionary["handled_ids"]: if transient_id in router.propagation_entries: - peer.handled_messages.append(transient_id) + peer.add_handled_message(transient_id) + hm_count += 1 + um_count = 0 for transient_id in dictionary["unhandled_ids"]: if transient_id in router.propagation_entries: - peer.unhandled_messages.append(transient_id) + peer.add_unhandled_message(transient_id) + um_count += 1 + + peer._hm_count = hm_count + peer._um_count = um_count + peer._hm_counts_synced = True + peer._um_counts_synced = True del dictionary return peer @@ -93,7 +107,10 @@ class LXMPeer: dictionary["handled_ids"] = handled_ids dictionary["unhandled_ids"] = unhandled_ids - return msgpack.packb(dictionary) + peer_bytes = msgpack.packb(dictionary) + del dictionary + + return peer_bytes def __init__(self, router, destination_hash): self.alive = False @@ -106,11 +123,14 @@ class LXMPeer: self.sync_transfer_rate = 0 self.propagation_transfer_limit = None + self._hm_count = 0 + self._um_count = 0 + self._hm_counts_synced = False + self._um_counts_synced = False + self.link = None self.state = LXMPeer.IDLE - self.unhandled_messages = [] - self.handled_messages = [] self.last_offer = [] self.router = router @@ -173,7 +193,7 @@ class LXMPeer: for transient_id in purged_ids: RNS.log("Dropping unhandled message "+RNS.prettyhexrep(transient_id)+" for peer "+RNS.prettyhexrep(self.destination_hash)+" since it no longer exists in the message store.", RNS.LOG_DEBUG) - self.unhandled_messages.remove(transient_id) + self.remove_unhandled_message(transient_id) unhandled_entries.sort(key=lambda e: e[1], reverse=False) per_message_overhead = 16 # Really only 2 bytes, but set a bit higher for now @@ -228,8 +248,8 @@ class LXMPeer: # Peer already has all advertised messages for transient_id in self.last_offer: if transient_id in self.unhandled_messages: - self.handled_messages.append(transient_id) - self.unhandled_messages.remove(transient_id) + self.add_handled_message(transient_id) + self.remove_unhandled_message(transient_id) elif response == True: @@ -244,9 +264,8 @@ class LXMPeer: # If the peer did not want the message, it has # already received it from another peer. if not transient_id in response: - if transient_id in self.unhandled_messages: - self.handled_messages.append(transient_id) - self.unhandled_messages.remove(transient_id) + self.add_handled_message(transient_id) + self.remove_unhandled_message(transient_id) for transient_id in response: wanted_messages.append(self.router.propagation_entries[transient_id]) @@ -292,8 +311,8 @@ class LXMPeer: def resource_concluded(self, resource): if resource.status == RNS.Resource.COMPLETE: for transient_id in resource.transferred_messages: - self.handled_messages.append(transient_id) - self.unhandled_messages.remove(transient_id) + self.add_handled_message(transient_id) + self.remove_unhandled_message(transient_id) if self.link != None: self.link.teardown() @@ -332,9 +351,72 @@ class LXMPeer: self.link = None self.state = LXMPeer.IDLE - def handle_message(self, transient_id): + def new_propagation_message(self, transient_id): if not transient_id in self.handled_messages and not transient_id in self.unhandled_messages: - self.unhandled_messages.append(transient_id) + self.add_unhandled_message(transient_id) + + @property + def handled_messages(self): + pes = self.router.propagation_entries.copy() + hm = list(filter(lambda tid: self.destination_hash in self.router.propagation_entries[tid][4], pes)) + self._hm_count = len(hm); del pes + return hm + + @property + def unhandled_messages(self): + pes = self.router.propagation_entries.copy() + um = list(filter(lambda tid: self.destination_hash in self.router.propagation_entries[tid][5], pes)) + self._um_count = len(um); del pes + return um + + @property + def handled_message_count(self): + if not self._hm_counts_synced: + self._update_counts() + + return self._hm_count + + @property + def unhandled_message_count(self): + if not self._um_counts_synced: + self._update_counts() + + return self._um_count + + def _update_counts(self): + if not self._hm_counts_synced: + RNS.log("UPDATE HM COUNTS") + hm = self.handled_messages; del hm + self._hm_counts_synced = True + + if not self._um_counts_synced: + RNS.log("UPDATE UM COUNTS") + um = self.unhandled_messages; del um + self._um_counts_synced = True + + def add_handled_message(self, transient_id): + if transient_id in self.router.propagation_entries: + if not self.destination_hash in self.router.propagation_entries[transient_id][4]: + self.router.propagation_entries[transient_id][4].append(self.destination_hash) + self._hm_counts_synced = False + + def add_unhandled_message(self, transient_id): + if transient_id in self.router.propagation_entries: + if not self.destination_hash in self.router.propagation_entries[transient_id][5]: + self.router.propagation_entries[transient_id][5].append(self.destination_hash) + self._um_count += 1 + + def remove_handled_message(self, transient_id): + if transient_id in self.router.propagation_entries: + if self.destination_hash in self.router.propagation_entries[transient_id][4]: + self.router.propagation_entries[transient_id][4].remove(self.destination_hash) + self._hm_counts_synced = False + + def remove_unhandled_message(self, transient_id): + if transient_id in self.router.propagation_entries: + if self.destination_hash in self.router.propagation_entries[transient_id][5]: + self.router.propagation_entries[transient_id][5].remove(self.destination_hash) + self._um_counts_synced = False def __str__(self): if self.destination_hash: diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index a19f401..9163824 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1,9 +1,11 @@ import os +import sys import time import math import random import base64 import atexit +import signal import threading import RNS @@ -94,6 +96,9 @@ class LXMRouter: self.outbound_propagation_node = None self.outbound_propagation_link = None + if delivery_limit == None: + delivery_limit = LXMRouter.DELIVERY_LIMIT + self.message_storage_limit = None self.information_storage_limit = None self.propagation_per_transfer_limit = propagation_limit @@ -117,6 +122,7 @@ class LXMRouter: self.cost_file_lock = threading.Lock() self.ticket_file_lock = threading.Lock() self.stamp_gen_lock = threading.Lock() + self.exit_handler_running = False if identity == None: identity = RNS.Identity() @@ -221,6 +227,8 @@ class LXMRouter: RNS.log("Could not load outbound stamp costs from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) atexit.register(self.exit_handler) + signal.signal(signal.SIGINT, self.sigint_handler) + signal.signal(signal.SIGTERM, self.sigterm_handler) job_thread = threading.Thread(target=self.jobloop) job_thread.setDaemon(True) @@ -446,17 +454,19 @@ class LXMRouter: file.close() self.propagation_entries[transient_id] = [ - destination_hash, - filepath, - received, - msg_size, + destination_hash, # 0: Destination hash + filepath, # 1: Storage location + received, # 2: Receive timestamp + msg_size, # 3: Message size + [], # 4: Handled peers + [], # 5: Unhandled peers ] except Exception as e: RNS.log("Could not read LXM from message store. The contained exception was: "+str(e), RNS.LOG_ERROR) et = time.time(); RNS.log(f"Indexed {len(self.propagation_entries)} messages in {RNS.prettytime(et-st)}, {math.floor(len(self.propagation_entries)/(et-st))} msgs/s", RNS.LOG_NOTICE) - st = time.time(); RNS.log("Loading propagation node peers...", RNS.LOG_NOTICE) + st = time.time(); RNS.log("Rebuilding peer synchronisation states...", RNS.LOG_NOTICE) if os.path.isfile(self.storagepath+"/peers"): peers_file = open(self.storagepath+"/peers", "rb") @@ -465,23 +475,25 @@ class LXMRouter: if len(peers_data) > 0: serialised_peers = msgpack.unpackb(peers_data) + del peers_data - for serialised_peer in serialised_peers: + while len(serialised_peers) > 0: + serialised_peer = serialised_peers.pop() peer = LXMPeer.from_bytes(serialised_peer, self) + del serialised_peer if peer.identity != None: self.peers[peer.destination_hash] = peer lim_str = ", no transfer limit" if peer.propagation_transfer_limit != None: lim_str = ", "+RNS.prettysize(peer.propagation_transfer_limit*1000)+" transfer limit" - RNS.log("Loaded peer "+RNS.prettyhexrep(peer.destination_hash)+" with "+str(len(peer.unhandled_messages))+" unhandled messages"+lim_str, RNS.LOG_DEBUG) + RNS.log("Rebuilt peer "+RNS.prettyhexrep(peer.destination_hash)+" with "+str(peer.unhandled_message_count)+" unhandled messages"+lim_str, RNS.LOG_DEBUG) else: - del peer RNS.log("Peer "+RNS.prettyhexrep(peer.destination_hash)+" could not be loaded, because its identity could not be recalled. Dropping peer.", RNS.LOG_DEBUG) + del peer del serialised_peers - del peers_data - RNS.log(f"Loaded {len(self.peers)} peers in {RNS.prettytime(time.time()-st)}", RNS.LOG_NOTICE) + RNS.log(f"Rebuilt synchronisation state for {len(self.peers)} peers in {RNS.prettytime(time.time()-st)}", RNS.LOG_NOTICE) self.propagation_node = True self.propagation_destination.set_link_established_callback(self.propagation_link_established) @@ -602,36 +614,37 @@ class LXMRouter: JOB_STORE_INTERVAL = 120 JOB_PEERSYNC_INTERVAL = 12 def jobs(self): - self.processing_count += 1 + if not self.exit_handler_running: + self.processing_count += 1 - if self.processing_count % LXMRouter.JOB_OUTBOUND_INTERVAL == 0: - self.process_outbound() + if self.processing_count % LXMRouter.JOB_OUTBOUND_INTERVAL == 0: + self.process_outbound() - if self.processing_count % LXMRouter.JOB_STAMPS_INTERVAL == 0: - threading.Thread(target=self.process_deferred_stamps, daemon=True).start() + if self.processing_count % LXMRouter.JOB_STAMPS_INTERVAL == 0: + threading.Thread(target=self.process_deferred_stamps, daemon=True).start() - if self.processing_count % LXMRouter.JOB_LINKS_INTERVAL == 0: - self.clean_links() + if self.processing_count % LXMRouter.JOB_LINKS_INTERVAL == 0: + self.clean_links() - if self.processing_count % LXMRouter.JOB_TRANSIENT_INTERVAL == 0: - self.clean_transient_id_caches() + if self.processing_count % LXMRouter.JOB_TRANSIENT_INTERVAL == 0: + self.clean_transient_id_caches() - if self.processing_count % LXMRouter.JOB_STORE_INTERVAL == 0: - self.clean_message_store() + if self.processing_count % LXMRouter.JOB_STORE_INTERVAL == 0: + self.clean_message_store() - if self.processing_count % LXMRouter.JOB_PEERSYNC_INTERVAL == 0: - self.sync_peers() + if self.processing_count % LXMRouter.JOB_PEERSYNC_INTERVAL == 0: + self.sync_peers() def jobloop(self): while (True): # TODO: Improve this to scheduling, so manual # triggers can delay next run - try: self.jobs() except Exception as e: RNS.log("An error ocurred while running LXMF Router jobs.", RNS.LOG_ERROR) RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) + RNS.trace_exception(e) time.sleep(LXMRouter.PROCESSING_INTERVAL) def clean_links(self): @@ -888,22 +901,24 @@ class LXMRouter: def save_locally_delivered_transient_ids(self): try: - if not os.path.isdir(self.storagepath): - os.makedirs(self.storagepath) + if len(self.locally_delivered_transient_ids) > 0: + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) - with open(self.storagepath+"/local_deliveries", "wb") as locally_delivered_file: - locally_delivered_file.write(msgpack.packb(self.locally_delivered_transient_ids)) + with open(self.storagepath+"/local_deliveries", "wb") as locally_delivered_file: + locally_delivered_file.write(msgpack.packb(self.locally_delivered_transient_ids)) except Exception as e: RNS.log("Could not save locally delivered message ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) def save_locally_processed_transient_ids(self): try: - if not os.path.isdir(self.storagepath): - os.makedirs(self.storagepath) + if len(self.locally_processed_transient_ids) > 0: + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) - with open(self.storagepath+"/locally_processed", "wb") as locally_processed_file: - locally_processed_file.write(msgpack.packb(self.locally_processed_transient_ids)) + with open(self.storagepath+"/locally_processed", "wb") as locally_processed_file: + locally_processed_file.write(msgpack.packb(self.locally_processed_transient_ids)) except Exception as e: RNS.log("Could not save locally processed transient ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) @@ -1001,10 +1016,43 @@ class LXMRouter: RNS.log(f"An error occurred while reloading available tickets from storage: {e}", RNS.LOG_ERROR) def exit_handler(self): + if self.exit_handler_running: + return + + self.exit_handler_running = True + + RNS.log("Tearing down delivery destinations...", RNS.LOG_NOTICE) + for destination_hash in self.delivery_destinations: + delivery_destination = self.delivery_destinations[destination_hash] + delivery_destination.set_packet_callback(None) + delivery_destination.set_link_established_callback(None) + for link in delivery_destination.links: + try: + if link.status == RNS.Link.ACTIVE: + link.teardown() + except Exception as e: + RNS.log("Error while tearing down propagation link: {e}", RNS.LOG_ERROR) + + if self.propagation_node: + RNS.log("Tearing down propagation node destination...", RNS.LOG_NOTICE) + self.propagation_destination.set_link_established_callback(None) + self.propagation_destination.set_packet_callback(None) + self.propagation_destination.deregister_request_handler(LXMPeer.OFFER_REQUEST_PATH) + self.propagation_destination.deregister_request_handler(LXMPeer.MESSAGE_GET_PATH) + for link in self.active_propagation_links: + try: + if link.status == RNS.Link.ACTIVE: + link.teardown() + except Exception as e: + RNS.log("Error while tearing down propagation link: {e}", RNS.LOG_ERROR) + + RNS.log("Persisting LXMF state data to storage...", RNS.LOG_NOTICE) if self.propagation_node: try: + st = time.time(); RNS.log("Saving peer synchronisation states to storage...", RNS.LOG_NOTICE) serialised_peers = [] - for peer_id in self.peers: + peer_dict = self.peers.copy() + for peer_id in peer_dict: peer = self.peers[peer_id] serialised_peers.append(peer.to_bytes()) @@ -1012,7 +1060,7 @@ class LXMRouter: peers_file.write(msgpack.packb(serialised_peers)) peers_file.close() - RNS.log("Saved "+str(len(serialised_peers))+" peers to storage", RNS.LOG_DEBUG) + RNS.log(f"Saved {len(serialised_peers)} peers to storage in {RNS.prettytime(time.time()-st)}", RNS.LOG_NOTICE) except Exception as e: RNS.log("Could not save propagation node peers to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) @@ -1020,6 +1068,20 @@ class LXMRouter: self.save_locally_delivered_transient_ids() self.save_locally_processed_transient_ids() + def sigint_handler(self, signal, frame): + if not self.exit_handler_running: + RNS.log("Received SIGINT, shutting down now!", RNS.LOG_WARNING) + sys.exit(0) + else: + RNS.log("Received SIGINT, but exit handler is running, keeping process alive until storage persist is complete", RNS.LOG_WARNING) + + def sigterm_handler(self, signal, frame): + if not self.exit_handler_running: + RNS.log("Received SIGTERM, shutting down now!", RNS.LOG_WARNING) + sys.exit(0) + else: + RNS.log("Received SIGTERM, but exit handler is running, keeping process alive until storage persist is complete", RNS.LOG_WARNING) + def __str__(self): return "" @@ -1685,19 +1747,23 @@ class LXMRouter: messages = data[1] for lxmf_data in messages: + peer = None + transient_id = RNS.Identity.full_hash(lxmf_data) if remote_hash != None and remote_hash in self.peers: - transient_id = RNS.Identity.full_hash(lxmf_data) peer = self.peers[remote_hash] - peer.handled_messages.append(transient_id) - self.lxmf_propagation(lxmf_data) + self.lxmf_propagation(lxmf_data, from_peer=peer) + if peer != None: + peer.add_handled_message(transient_id) + else: RNS.log("Invalid data structure received at propagation destination, ignoring", RNS.LOG_DEBUG) except Exception as e: RNS.log("Error while unpacking received propagation resource", RNS.LOG_DEBUG) + RNS.trace_exception(e) - def lxmf_propagation(self, lxmf_data, signal_local_delivery=None, signal_duplicate=None, is_paper_message=False): + def lxmf_propagation(self, lxmf_data, signal_local_delivery=None, signal_duplicate=None, is_paper_message=False, from_peer=None): no_stamp_enforcement = False if is_paper_message: no_stamp_enforcement = True @@ -1708,7 +1774,6 @@ class LXMRouter: if not transient_id in self.propagation_entries and not transient_id in self.locally_processed_transient_ids: received = time.time() - propagation_entry = [transient_id, received, lxmf_data] destination_hash = lxmf_data[:LXMessage.DESTINATION_LENGTH] self.locally_processed_transient_ids[transient_id] = received @@ -1732,12 +1797,13 @@ class LXMRouter: msg_file.write(lxmf_data) msg_file.close() - self.propagation_entries[transient_id] = [destination_hash, file_path, time.time(), len(lxmf_data)] + self.propagation_entries[transient_id] = [destination_hash, file_path, time.time(), len(lxmf_data), [], []] RNS.log("Received propagated LXMF message "+RNS.prettyhexrep(transient_id)+", adding to peer distribution queues...", RNS.LOG_DEBUG) for peer_id in self.peers: peer = self.peers[peer_id] - peer.handle_message(transient_id) + if peer != from_peer: + peer.new_propagation_message(transient_id) else: # TODO: Add message to sneakernet queues when implemented @@ -1757,6 +1823,7 @@ class LXMRouter: except Exception as e: RNS.log("Could not assemble propagated LXMF message from received data", RNS.LOG_DEBUG) RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + RNS.trace_exception(e) return False def ingest_lxm_uri(self, uri, signal_local_delivery=None, signal_duplicate=None): From 44d1d992f8a9be4b81c5f6b302f6f48b1e46e161 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 21 Jan 2025 16:34:00 +0100 Subject: [PATCH 271/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 906d362..43c4ab0 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.6.0" +__version__ = "0.6.1" From bfed126a7c17fd90551204afd0bbab3fac1441f9 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 21 Jan 2025 16:44:24 +0100 Subject: [PATCH 272/436] Memory optimisations --- LXMF/LXMPeer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index f4c522c..d133027 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -358,14 +358,14 @@ class LXMPeer: @property def handled_messages(self): pes = self.router.propagation_entries.copy() - hm = list(filter(lambda tid: self.destination_hash in self.router.propagation_entries[tid][4], pes)) + hm = list(filter(lambda tid: self.destination_hash in pes[tid][4], pes)) self._hm_count = len(hm); del pes return hm @property def unhandled_messages(self): pes = self.router.propagation_entries.copy() - um = list(filter(lambda tid: self.destination_hash in self.router.propagation_entries[tid][5], pes)) + um = list(filter(lambda tid: self.destination_hash in pes[tid][5], pes)) self._um_count = len(um); del pes return um From 1c9c74410790188db976dbecdff3b994d33ac5d9 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 21 Jan 2025 16:51:25 +0100 Subject: [PATCH 273/436] Memory optimisations --- LXMF/LXMPeer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index d133027..add54da 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -360,6 +360,7 @@ class LXMPeer: pes = self.router.propagation_entries.copy() hm = list(filter(lambda tid: self.destination_hash in pes[tid][4], pes)) self._hm_count = len(hm); del pes + self._hm_counts_synced = True return hm @property @@ -367,6 +368,7 @@ class LXMPeer: pes = self.router.propagation_entries.copy() um = list(filter(lambda tid: self.destination_hash in pes[tid][5], pes)) self._um_count = len(um); del pes + self._um_counts_synced = True return um @property @@ -387,12 +389,10 @@ class LXMPeer: if not self._hm_counts_synced: RNS.log("UPDATE HM COUNTS") hm = self.handled_messages; del hm - self._hm_counts_synced = True if not self._um_counts_synced: RNS.log("UPDATE UM COUNTS") um = self.unhandled_messages; del um - self._um_counts_synced = True def add_handled_message(self, transient_id): if transient_id in self.router.propagation_entries: From 1430b1ce90b989e9627d07841b5634e6f3a1f8e1 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 21 Jan 2025 20:20:39 +0100 Subject: [PATCH 274/436] Enqueue and batch process distribution queue mappings --- LXMF/LXMPeer.py | 40 ++++++++++++++++++++++++++++++----- LXMF/LXMRouter.py | 53 +++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 79 insertions(+), 14 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index add54da..74a40c7 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -4,6 +4,7 @@ import time import RNS import RNS.vendor.umsgpack as msgpack +from collections import deque from .LXMF import APP_NAME class LXMPeer: @@ -122,6 +123,8 @@ class LXMPeer: self.link_establishment_rate = 0 self.sync_transfer_rate = 0 self.propagation_transfer_limit = None + self.handled_messages_queue = deque() + self.unhandled_messages_queue = deque() self._hm_count = 0 self._um_count = 0 @@ -351,9 +354,38 @@ class LXMPeer: self.link = None self.state = LXMPeer.IDLE - def new_propagation_message(self, transient_id): - if not transient_id in self.handled_messages and not transient_id in self.unhandled_messages: - self.add_unhandled_message(transient_id) + def queued_items(self): + return len(self.handled_messages_queue) > 0 or len(self.unhandled_messages_queue) > 0 + + def queue_unhandled_message(self, transient_id): + self.unhandled_messages_queue.append(transient_id) + + def queue_handled_message(self, transient_id): + self.handled_messages_queue.append(transient_id) + + def process_queues(self): + if len(self.unhandled_messages_queue) > 0 or len(self.handled_messages_queue) > 0: + # TODO: Remove debug + # st = time.time(); lu = len(self.unhandled_messages_queue); lh = len(self.handled_messages_queue) + + handled_messages = self.handled_messages + unhandled_messages = self.unhandled_messages + + while len(self.handled_messages_queue) > 0: + transient_id = self.handled_messages_queue.pop() + if not transient_id in handled_messages: + self.add_handled_message(transient_id) + if transient_id in unhandled_messages: + self.remove_unhandled_message(transient_id) + + while len(self.unhandled_messages_queue) > 0: + transient_id = self.unhandled_messages_queue.pop() + if not transient_id in handled_messages and not transient_id in unhandled_messages: + self.add_unhandled_message(transient_id) + + del handled_messages, unhandled_messages + # TODO: Remove debug + # RNS.log(f"{self} processed {lh}/{lu} in {RNS.prettytime(time.time()-st)}") @property def handled_messages(self): @@ -387,11 +419,9 @@ class LXMPeer: def _update_counts(self): if not self._hm_counts_synced: - RNS.log("UPDATE HM COUNTS") hm = self.handled_messages; del hm if not self._um_counts_synced: - RNS.log("UPDATE UM COUNTS") um = self.unhandled_messages; del um def add_handled_message(self, transient_id): diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 9163824..1e62914 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -8,6 +8,8 @@ import atexit import signal import threading +from collections import deque + import RNS import RNS.vendor.umsgpack as msgpack @@ -143,6 +145,8 @@ class LXMRouter: self.peers = {} self.propagation_entries = {} + self.peer_distribution_queue = deque() + RNS.Transport.register_announce_handler(LXMFDeliveryAnnounceHandler(self)) RNS.Transport.register_announce_handler(LXMFPropagationAnnounceHandler(self)) @@ -613,6 +617,7 @@ class LXMRouter: JOB_TRANSIENT_INTERVAL = 60 JOB_STORE_INTERVAL = 120 JOB_PEERSYNC_INTERVAL = 12 + JOB_PEERINGEST_INTERVAL= JOB_PEERSYNC_INTERVAL def jobs(self): if not self.exit_handler_running: self.processing_count += 1 @@ -632,6 +637,9 @@ class LXMRouter: if self.processing_count % LXMRouter.JOB_STORE_INTERVAL == 0: self.clean_message_store() + if self.processing_count % LXMRouter.JOB_PEERINGEST_INTERVAL == 0: + self.flush_queues() + if self.processing_count % LXMRouter.JOB_PEERSYNC_INTERVAL == 0: self.sync_peers() @@ -647,6 +655,17 @@ class LXMRouter: RNS.trace_exception(e) time.sleep(LXMRouter.PROCESSING_INTERVAL) + def flush_queues(self): + self.flush_peer_distribution_queue() + RNS.log("Calculating peer distribution queue mappings...", RNS.LOG_DEBUG); st = time.time() + for peer_id in self.peers.copy(): + if peer_id in self.peers: + peer = self.peers[peer_id] + if peer.queued_items(): + peer.process_queues() + + RNS.log(f"Distribution queue mapping completed in {RNS.prettytime(time.time()-st)}", RNS.LOG_DEBUG) + def clean_links(self): closed_links = [] for link_hash in self.direct_links: @@ -1047,6 +1066,7 @@ class LXMRouter: RNS.log("Error while tearing down propagation link: {e}", RNS.LOG_ERROR) RNS.log("Persisting LXMF state data to storage...", RNS.LOG_NOTICE) + self.flush_queues() if self.propagation_node: try: st = time.time(); RNS.log("Saving peer synchronisation states to storage...", RNS.LOG_NOTICE) @@ -1608,8 +1628,9 @@ class LXMRouter: culled_peers = [] waiting_peers = [] unresponsive_peers = [] - for peer_id in self.peers: - peer = self.peers[peer_id] + peers = self.peers.copy() + for peer_id in peers: + peer = peers[peer_id] if time.time() > peer.last_heard + LXMPeer.MAX_UNREACHABLE: culled_peers.append(peer_id) else: @@ -1754,7 +1775,7 @@ class LXMRouter: self.lxmf_propagation(lxmf_data, from_peer=peer) if peer != None: - peer.add_handled_message(transient_id) + peer.queue_handled_message(transient_id) else: RNS.log("Invalid data structure received at propagation destination, ignoring", RNS.LOG_DEBUG) @@ -1763,6 +1784,24 @@ class LXMRouter: RNS.log("Error while unpacking received propagation resource", RNS.LOG_DEBUG) RNS.trace_exception(e) + def enqueue_peer_distribution(self, transient_id, from_peer): + self.peer_distribution_queue.append([transient_id, from_peer]) + + def flush_peer_distribution_queue(self): + if len(self.peer_distribution_queue) > 0: + entries = [] + while len(self.peer_distribution_queue) > 0: + entries.append(self.peer_distribution_queue.pop()) + + for peer_id in self.peers.copy(): + if peer_id in self.peers: + peer = self.peers[peer_id] + for entry in entries: + transient_id = entry[0] + from_peer = entry[1] + if peer != from_peer: + peer.queue_unhandled_message(transient_id) + def lxmf_propagation(self, lxmf_data, signal_local_delivery=None, signal_duplicate=None, is_paper_message=False, from_peer=None): no_stamp_enforcement = False if is_paper_message: @@ -1797,13 +1836,9 @@ class LXMRouter: msg_file.write(lxmf_data) msg_file.close() - self.propagation_entries[transient_id] = [destination_hash, file_path, time.time(), len(lxmf_data), [], []] - RNS.log("Received propagated LXMF message "+RNS.prettyhexrep(transient_id)+", adding to peer distribution queues...", RNS.LOG_DEBUG) - for peer_id in self.peers: - peer = self.peers[peer_id] - if peer != from_peer: - peer.new_propagation_message(transient_id) + self.propagation_entries[transient_id] = [destination_hash, file_path, time.time(), len(lxmf_data), [], []] + self.enqueue_peer_distribution(transient_id, from_peer) else: # TODO: Add message to sneakernet queues when implemented From c2a08ef35588ccd512a7ea7c9898c83e5fd2864e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 21 Jan 2025 20:44:11 +0100 Subject: [PATCH 275/436] Enqueue and batch process distribution queue mappings --- LXMF/LXMRouter.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 1e62914..ee1dca8 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -656,15 +656,16 @@ class LXMRouter: time.sleep(LXMRouter.PROCESSING_INTERVAL) def flush_queues(self): - self.flush_peer_distribution_queue() - RNS.log("Calculating peer distribution queue mappings...", RNS.LOG_DEBUG); st = time.time() - for peer_id in self.peers.copy(): - if peer_id in self.peers: - peer = self.peers[peer_id] - if peer.queued_items(): - peer.process_queues() + if len(self.peers) > 0: + self.flush_peer_distribution_queue() + RNS.log("Calculating peer distribution queue mappings...", RNS.LOG_DEBUG); st = time.time() + for peer_id in self.peers.copy(): + if peer_id in self.peers: + peer = self.peers[peer_id] + if peer.queued_items(): + peer.process_queues() - RNS.log(f"Distribution queue mapping completed in {RNS.prettytime(time.time()-st)}", RNS.LOG_DEBUG) + RNS.log(f"Distribution queue mapping completed in {RNS.prettytime(time.time()-st)}", RNS.LOG_DEBUG) def clean_links(self): closed_links = [] From e69da2ed2a29b33af0acda059aa9a624b475a6e7 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 22 Jan 2025 01:37:09 +0100 Subject: [PATCH 276/436] Added static peers and peering limit --- LXMF/Handlers.py | 14 ++++-- LXMF/LXMPeer.py | 59 ++++++++++++++++++++--- LXMF/LXMRouter.py | 107 +++++++++++++++++++++++++++++++++++------ LXMF/Utilities/lxmd.py | 29 +++++++++-- 4 files changed, 179 insertions(+), 30 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index 7420ea5..22c6cd3 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -51,12 +51,16 @@ class LXMFPropagationAnnounceHandler: except: propagation_transfer_limit = None - if data[0] == True: - if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: - self.lxmrouter.peer(destination_hash, node_timebase, propagation_transfer_limit) + if destination_hash in self.lxmrouter.static_peers: + self.lxmrouter.peer(destination_hash, node_timebase, propagation_transfer_limit) - elif data[0] == False: - self.lxmrouter.unpeer(destination_hash, node_timebase) + else: + if data[0] == True: + if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: + self.lxmrouter.peer(destination_hash, node_timebase, propagation_transfer_limit) + + elif data[0] == False: + self.lxmrouter.unpeer(destination_hash, node_timebase) except Exception as e: RNS.log("Error while evaluating propagation node announce, ignoring announce.", RNS.LOG_DEBUG) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 74a40c7..ec0cfe2 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -66,6 +66,31 @@ class LXMPeer: peer.propagation_transfer_limit = None else: peer.propagation_transfer_limit = None + + if "offered" in dictionary: + peer.offered = dictionary["offered"] + else: + peer.offered = 0 + + if "outgoing" in dictionary: + peer.outgoing = dictionary["outgoing"] + else: + peer.outgoing = 0 + + if "incoming" in dictionary: + peer.incoming = dictionary["incoming"] + else: + peer.incoming = 0 + + if "rx_bytes" in dictionary: + peer.rx_bytes = dictionary["rx_bytes"] + else: + peer.rx_bytes = 0 + + if "tx_bytes" in dictionary: + peer.tx_bytes = dictionary["tx_bytes"] + else: + peer.tx_bytes = 0 hm_count = 0 for transient_id in dictionary["handled_ids"]: @@ -96,6 +121,11 @@ class LXMPeer: dictionary["link_establishment_rate"] = self.link_establishment_rate dictionary["sync_transfer_rate"] = self.sync_transfer_rate dictionary["propagation_transfer_limit"] = self.propagation_transfer_limit + dictionary["offered"] = self.offered + dictionary["outgoing"] = self.outgoing + dictionary["incoming"] = self.incoming + dictionary["rx_bytes"] = self.rx_bytes + dictionary["tx_bytes"] = self.tx_bytes handled_ids = [] for transient_id in self.handled_messages: @@ -126,6 +156,12 @@ class LXMPeer: self.handled_messages_queue = deque() self.unhandled_messages_queue = deque() + self.offered = 0 # Messages offered to this peer + self.outgoing = 0 # Messages transferred to this peer + self.incoming = 0 # Messages received from this peer + self.rx_bytes = 0 # Bytes received from this peer + self.tx_bytes = 0 # Bytes sent to this peer + self._hm_count = 0 self._um_count = 0 self._hm_counts_synced = False @@ -212,7 +248,7 @@ class LXMPeer: cumulative_size += (lxm_size+per_message_overhead) unhandled_ids.append(transient_id) - RNS.log("Sending sync request to peer "+str(self.destination), RNS.LOG_DEBUG) + RNS.log(f"Offering {len(unhandled_ids)} messages to peer {RNS.prettyhexrep(self.destination.hash)}", RNS.LOG_VERBOSE) self.last_offer = unhandled_ids self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) self.state = LXMPeer.REQUEST_SENT @@ -242,10 +278,16 @@ class LXMPeer: if response == LXMPeer.ERROR_NO_IDENTITY: if self.link != None: - RNS.log("Remote peer indicated that no identification was received, retrying...", RNS.LOG_DEBUG) + RNS.log("Remote peer indicated that no identification was received, retrying...", RNS.LOG_VERBOSE) self.link.identify() self.state = LXMPeer.LINK_READY self.sync() + return + + elif response == LXMPeer.ERROR_NO_ACCESS: + RNS.log("Remote indicated that access was denied, breaking peering", RNS.LOG_VERBOSE) + self.router.unpeer(self.destination_hash) + return elif response == False: # Peer already has all advertised messages @@ -275,10 +317,9 @@ class LXMPeer: wanted_message_ids.append(transient_id) if len(wanted_messages) > 0: - RNS.log("Peer wanted "+str(len(wanted_messages))+" of the available messages", RNS.LOG_DEBUG) + RNS.log("Peer wanted "+str(len(wanted_messages))+" of the available messages", RNS.LOG_VERBOSE) lxm_list = [] - for message_entry in wanted_messages: file_path = message_entry[1] if os.path.isfile(file_path): @@ -294,7 +335,8 @@ class LXMPeer: self.state = LXMPeer.RESOURCE_TRANSFERRING else: - RNS.log("Peer "+RNS.prettyhexrep(self.destination_hash)+" did not request any of the available messages, sync completed", RNS.LOG_DEBUG) + RNS.log("Peer "+RNS.prettyhexrep(self.destination_hash)+" did not request any of the available messages, sync completed", RNS.LOG_VERBOSE) + self.offered += len(self.last_offer) if self.link != None: self.link.teardown() @@ -328,12 +370,15 @@ class LXMPeer: self.sync_transfer_rate = (resource.get_transfer_size()*8)/(time.time()-resource.sync_transfer_started) rate_str = f" at {RNS.prettyspeed(self.sync_transfer_rate)}" - RNS.log("Sync to peer "+RNS.prettyhexrep(self.destination_hash)+" completed"+rate_str, RNS.LOG_DEBUG) + RNS.log(f"Syncing {len(resource.transferred_messages)} messages to peer {RNS.prettyhexrep(self.destination_hash)} completed{rate_str}", RNS.LOG_VERBOSE) self.alive = True self.last_heard = time.time() + self.offered += len(self.last_offer) + self.outgoing += len(resource.transferred_messages) + self.tx_bytes += resource.get_data_size() else: - RNS.log("Resource transfer for LXMF peer sync failed to "+str(self.destination), RNS.LOG_DEBUG) + RNS.log("Resource transfer for LXMF peer sync failed to "+str(self.destination), RNS.LOG_VERBOSE) if self.link != None: self.link.teardown() diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index ee1dca8..bfe863d 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -37,6 +37,7 @@ class LXMRouter: NODE_ANNOUNCE_DELAY = 20 + MAX_PEERS = 50 AUTOPEER = True AUTOPEER_MAXDEPTH = 4 FASTEST_N_RANDOM_POOL = 2 @@ -67,7 +68,10 @@ class LXMRouter: ### Developer-facing API ############################## ####################################################### - def __init__(self, identity = None, storagepath = None, autopeer = AUTOPEER, autopeer_maxdepth = None, propagation_limit = PROPAGATION_LIMIT, delivery_limit = DELIVERY_LIMIT, enforce_ratchets = False, enforce_stamps = False): + def __init__(self, identity=None, storagepath=None, autopeer=AUTOPEER, autopeer_maxdepth=None, + propagation_limit=PROPAGATION_LIMIT, delivery_limit=DELIVERY_LIMIT, enforce_ratchets=False, + enforce_stamps=False, static_peers = [], max_peers=None, from_static_only=False): + random.seed(os.urandom(10)) self.pending_inbound = [] @@ -142,6 +146,27 @@ class LXMRouter: else: self.autopeer_maxdepth = LXMRouter.AUTOPEER_MAXDEPTH + if max_peers == None: + self.max_peers = LXMRouter.MAX_PEERS + else: + if type(max_peers) == int and max_peers >= 0: + self.max_peers = max_peers + else: + raise ValueError(f"Invalid value for max_peers: {max_peers}") + + self.from_static_only = from_static_only + if type(static_peers) != list: + raise ValueError(f"Invalid type supplied for static peer list: {type(static_peers)}") + else: + for static_peer in static_peers: + if type(static_peer) != bytes: + raise ValueError(f"Invalid static peer destination hash: {static_peer}") + else: + if len(static_peer) != RNS.Reticulum.TRUNCATED_HASHLENGTH//8: + raise ValueError(f"Invalid static peer destination hash: {static_peer}") + + self.static_peers = static_peers + self.peers = {} self.propagation_entries = {} @@ -245,8 +270,9 @@ class LXMRouter: def announce_propagation_node(self): def delayed_announce(): time.sleep(LXMRouter.NODE_ANNOUNCE_DELAY) + node_state = self.propagation_node and not self.from_static_only announce_data = [ - self.propagation_node, # Boolean flag signalling propagation node state + node_state, # Boolean flag signalling propagation node state int(time.time()), # Current node timebase self.propagation_per_transfer_limit, # Per-transfer limit for message propagation in kilobytes ] @@ -485,6 +511,11 @@ class LXMRouter: serialised_peer = serialised_peers.pop() peer = LXMPeer.from_bytes(serialised_peer, self) del serialised_peer + if peer.destination_hash in self.static_peers and peer.last_heard == 0: + # TODO: Allow path request responses through announce handler + # momentarily here, so peering config can be updated even if + # the static peer is not available to directly send an announce. + RNS.Transport.request_path(peer.destination_hash) if peer.identity != None: self.peers[peer.destination_hash] = peer lim_str = ", no transfer limit" @@ -497,6 +528,17 @@ class LXMRouter: del serialised_peers + if len(self.static_peers) > 0: + for static_peer in self.static_peers: + if not static_peer in self.peers: + RNS.log(f"Activating static peering with {RNS.prettyhexrep(static_peer)}", RNS.LOG_NOTICE) + self.peers[static_peer] = LXMPeer(self, static_peer) + if self.peers[static_peer].last_heard == 0: + # TODO: Allow path request responses through announce handler + # momentarily here, so peering config can be updated even if + # the static peer is not available to directly send an announce. + RNS.Transport.request_path(static_peer) + RNS.log(f"Rebuilt synchronisation state for {len(self.peers)} peers in {RNS.prettytime(time.time()-st)}", RNS.LOG_NOTICE) self.propagation_node = True @@ -643,6 +685,11 @@ class LXMRouter: if self.processing_count % LXMRouter.JOB_PEERSYNC_INTERVAL == 0: self.sync_peers() + # def syncstats(self): + # for peer_id in self.peers: + # p = self.peers[peer_id] + # RNS.log(f"{RNS.prettyhexrep(peer_id)} O={p.offered} S={p.outgoing} I={p.incoming} TX={RNS.prettysize(p.tx_bytes)} RX={RNS.prettysize(p.rx_bytes)}") + def jobloop(self): while (True): # TODO: Improve this to scheduling, so manual @@ -1070,7 +1117,7 @@ class LXMRouter: self.flush_queues() if self.propagation_node: try: - st = time.time(); RNS.log("Saving peer synchronisation states to storage...", RNS.LOG_NOTICE) + st = time.time(); RNS.log(f"Saving {len(self.peers)} peer synchronisation states to storage...", RNS.LOG_NOTICE) serialised_peers = [] peer_dict = self.peers.copy() for peer_id in peer_dict: @@ -1081,7 +1128,7 @@ class LXMRouter: peers_file.write(msgpack.packb(serialised_peers)) peers_file.close() - RNS.log(f"Saved {len(serialised_peers)} peers to storage in {RNS.prettytime(time.time()-st)}", RNS.LOG_NOTICE) + RNS.log(f"Saved {len(serialised_peers)} peers to storage in {RNS.prettyshorttime(time.time()-st)}", RNS.LOG_NOTICE) except Exception as e: RNS.log("Could not save propagation node peers to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) @@ -1605,14 +1652,18 @@ class LXMRouter: peer.peering_timebase = timestamp peer.last_heard = time.time() peer.propagation_transfer_limit = propagation_transfer_limit + RNS.log(f"Peering config updated for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_VERBOSE) else: - peer = LXMPeer(self, destination_hash) - peer.alive = True - peer.last_heard = time.time() - peer.propagation_transfer_limit = propagation_transfer_limit - self.peers[destination_hash] = peer - RNS.log("Peered with "+str(peer.destination)) + if len(self.peers) < self.max_peers: + peer = LXMPeer(self, destination_hash) + peer.alive = True + peer.last_heard = time.time() + peer.propagation_transfer_limit = propagation_transfer_limit + self.peers[destination_hash] = peer + RNS.log(f"Peered with {RNS.prettyhexrep(destination_hash)}", RNS.LOG_NOTICE) + else: + RNS.log(f"Max peers reached, not peering with {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) def unpeer(self, destination_hash, timestamp = None): if timestamp == None: @@ -1633,7 +1684,8 @@ class LXMRouter: for peer_id in peers: peer = peers[peer_id] if time.time() > peer.last_heard + LXMPeer.MAX_UNREACHABLE: - culled_peers.append(peer_id) + if not peer_id in self.static_peers: + culled_peers.append(peer_id) else: if peer.state == LXMPeer.IDLE and len(peer.unhandled_messages) > 0: if peer.alive: @@ -1693,10 +1745,23 @@ class LXMRouter: self.active_propagation_links.append(link) def propagation_resource_advertised(self, resource): + if self.from_static_only: + remote_identity = resource.link.get_remote_identity() + if remote_identity == None: + RNS.log(f"Rejecting propagation resource from unidentified peer", RNS.LOG_DEBUG) + return False + else: + remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + remote_hash = remote_destination.hash + remote_str = RNS.prettyhexrep(remote_hash) + if not remote_hash in self.static_peers: + RNS.log(f"Rejecting propagation resource from {remote_str} not in static peers list", RNS.LOG_DEBUG) + return False + size = resource.get_data_size() limit = self.propagation_per_transfer_limit*1000 if limit != None and size > limit: - RNS.log("Rejecting "+RNS.prettysize(size)+" incoming LXMF propagation resource, since it exceeds the limit of "+RNS.prettysize(limit), RNS.LOG_DEBUG) + RNS.log(f"Rejecting {RNS.prettysize(size)} incoming propagation resource, since it exceeds the limit of {RNS.prettysize(limit)}", RNS.LOG_DEBUG) return False else: return True @@ -1723,6 +1788,14 @@ class LXMRouter: if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY else: + if self.from_static_only: + remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + remote_hash = remote_destination.hash + remote_str = RNS.prettyhexrep(remote_hash) + if not remote_hash in self.static_peers: + RNS.log(f"Rejecting propagation request from {remote_str} not in static peers list", RNS.LOG_DEBUG) + return LXMPeer.ERROR_NO_ACCESS + try: transient_ids = data wanted_ids = [] @@ -1745,7 +1818,6 @@ class LXMRouter: return None def propagation_resource_concluded(self, resource): - RNS.log("Transfer concluded for incoming propagation resource "+str(resource), RNS.LOG_DEBUG) if resource.status == RNS.Resource.COMPLETE: # TODO: The peer this was received from should # have the transient id added to its list of @@ -1757,22 +1829,29 @@ class LXMRouter: # This is a series of propagation messages from a peer or originator remote_timebase = data[0] remote_hash = None + remote_str = "unknown peer" remote_identity = resource.link.get_remote_identity() if remote_identity != None: remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") remote_hash = remote_destination.hash + remote_str = RNS.prettyhexrep(remote_hash) if not remote_hash in self.peers: if self.autopeer and RNS.Transport.hops_to(remote_hash) <= self.autopeer_maxdepth: self.peer(remote_hash, remote_timebase) + else: + remote_str = f"peer {remote_str}" messages = data[1] + RNS.log(f"Received {len(messages)} messages from {remote_str}", RNS.LOG_VERBOSE) for lxmf_data in messages: peer = None transient_id = RNS.Identity.full_hash(lxmf_data) if remote_hash != None and remote_hash in self.peers: peer = self.peers[remote_hash] + peer.incoming += 1 + peer.rx_bytes += len(lxmf_data) self.lxmf_propagation(lxmf_data, from_peer=peer) if peer != None: @@ -1837,7 +1916,7 @@ class LXMRouter: msg_file.write(lxmf_data) msg_file.close() - RNS.log("Received propagated LXMF message "+RNS.prettyhexrep(transient_id)+", adding to peer distribution queues...", RNS.LOG_DEBUG) + RNS.log("Received propagated LXMF message "+RNS.prettyhexrep(transient_id)+", adding to peer distribution queues...", RNS.LOG_EXTREME) self.propagation_entries[transient_id] = [destination_hash, file_path, time.time(), len(lxmf_data), [], []] self.enqueue_peer_distribution(transient_id, from_peer) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 38e71b1..0c87a73 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -140,6 +140,24 @@ def apply_config(): else: active_configuration["prioritised_lxmf_destinations"] = [] + if "propagation" in lxmd_config and "static_peers" in lxmd_config["propagation"]: + static_peers = lxmd_config["propagation"].as_list("static_peers") + active_configuration["static_peers"] = [] + for static_peer in static_peers: + active_configuration["static_peers"].append(bytes.fromhex(static_peer)) + else: + active_configuration["static_peers"] = [] + + if "propagation" in lxmd_config and "max_peers" in lxmd_config["propagation"]: + active_configuration["max_peers"] = lxmd_config["propagation"].as_int("max_peers") + else: + active_configuration["max_peers"] = None + + if "propagation" in lxmd_config and "from_static_only" in lxmd_config["propagation"]: + active_configuration["from_static_only"] = lxmd_config["propagation"].as_bool("from_static_only") + else: + active_configuration["from_static_only"] = False + # Load various settings if "logging" in lxmd_config and "loglevel" in lxmd_config["logging"]: targetloglevel = lxmd_config["logging"].as_int("loglevel") @@ -305,7 +323,10 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo autopeer_maxdepth = active_configuration["autopeer_maxdepth"], propagation_limit = active_configuration["propagation_transfer_max_accepted_size"], delivery_limit = active_configuration["delivery_transfer_max_accepted_size"], - ) + max_peers = active_configuration["max_peers"], + static_peers = active_configuration["static_peers"], + from_static_only = active_configuration["from_static_only"]) + message_router.register_delivery_callback(lxmf_delivery) for destination_hash in active_configuration["ignored_lxmf_destinations"]: @@ -362,13 +383,13 @@ def jobs(): try: if "peer_announce_interval" in active_configuration and active_configuration["peer_announce_interval"] != None: if time.time() > last_peer_announce + active_configuration["peer_announce_interval"]: - RNS.log("Sending announce for LXMF delivery destination", RNS.LOG_EXTREME) + RNS.log("Sending announce for LXMF delivery destination", RNS.LOG_VERBOSE) message_router.announce(lxmf_destination.hash) last_peer_announce = time.time() if "node_announce_interval" in active_configuration and active_configuration["node_announce_interval"] != None: if time.time() > last_node_announce + active_configuration["node_announce_interval"]: - RNS.log("Sending announce for LXMF Propagation Node", RNS.LOG_EXTREME) + RNS.log("Sending announce for LXMF Propagation Node", RNS.LOG_VERBOSE) message_router.announce_propagation_node() last_node_announce = time.time() @@ -381,7 +402,7 @@ def deferred_start_jobs(): global active_configuration, last_peer_announce, last_node_announce global message_router, lxmf_destination time.sleep(DEFFERED_JOBS_DELAY) - RNS.log("Running deferred start jobs") + RNS.log("Running deferred start jobs", RNS.LOG_DEBUG) if active_configuration["peer_announce_at_start"]: RNS.log("Sending announce for LXMF delivery destination", RNS.LOG_EXTREME) message_router.announce(lxmf_destination.hash) From 68257a441ff1029054378185b09f4b61020e9d3e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 22 Jan 2025 09:44:03 +0100 Subject: [PATCH 277/436] Set transfer limit on reverse auto-peer --- LXMF/LXMRouter.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index bfe863d..5465356 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1839,7 +1839,12 @@ class LXMRouter: if not remote_hash in self.peers: if self.autopeer and RNS.Transport.hops_to(remote_hash) <= self.autopeer_maxdepth: - self.peer(remote_hash, remote_timebase) + # TODO: Query cache for an announce and get propagation + # transfer limit from that. For now, initialise it to a + # sane default value, and wait for an announce to arrive + # that will update the peering config to the actual limit. + propagation_transfer_limit = LXMRouter.PROPAGATION_LIMIT//4 + self.peer(remote_hash, remote_timebase, propagation_transfer_limit) else: remote_str = f"peer {remote_str}" From 61b1ecce276631a4ec2c1165c33b5195e46e946d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 22 Jan 2025 10:10:57 +0100 Subject: [PATCH 278/436] Updated readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index faced95..ed7e4f0 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ User-facing clients built on LXMF include: Community-provided tools and utilities for LXMF include: +- [LXMFy](https://lxmfy.quad4.io/) - [LXMF-Bot](https://github.com/randogoth/lxmf-bot) - [LXMF Messageboard](https://github.com/chengtripp/lxmf_messageboard) - [LXMEvent](https://github.com/faragher/LXMEvent) From 2c71cea7a0d2fc0a3ab5bbd26883befb5a0dd9fc Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 23 Jan 2025 14:13:08 +0100 Subject: [PATCH 279/436] Added local node stats request handler --- LXMF/LXMRouter.py | 134 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 131 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 5465356..22ef3ac 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -64,6 +64,8 @@ class LXMRouter: PR_ALL_MESSAGES = 0x00 + STATS_GET_PATH = "/pn/get/stats" + ### Developer-facing API ############################## ####################################################### @@ -92,6 +94,7 @@ class LXMRouter: self.processing_count = 0 self.propagation_node = False + self.propagation_node_start_time = None if storagepath == None: raise ValueError("LXMF cannot be initialised without a storage path") @@ -135,6 +138,11 @@ class LXMRouter: self.identity = identity self.propagation_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation") + self.control_destination = None + self.client_propagation_messages_received = 0 + self.client_propagation_messages_served = 0 + self.unpeered_propagation_incoming = 0 + self.unpeered_propagation_rx_bytes = 0 if autopeer != None: self.autopeer = autopeer @@ -541,13 +549,35 @@ class LXMRouter: RNS.log(f"Rebuilt synchronisation state for {len(self.peers)} peers in {RNS.prettytime(time.time()-st)}", RNS.LOG_NOTICE) + try: + if os.path.isfile(self.storagepath+"/node_stats"): + node_stats_file = open(self.storagepath+"/node_stats", "rb") + data = node_stats_file.read() + node_stats_file.close() + node_stats = msgpack.unpackb(data) + + if not type(node_stats) == dict: + RNS.log("Invalid data format for loaded local node stats, node stats will be reset", RNS.LOG_ERROR) + else: + self.client_propagation_messages_received = node_stats["client_propagation_messages_received"] + self.client_propagation_messages_served = node_stats["client_propagation_messages_served"] + self.unpeered_propagation_incoming = node_stats["unpeered_propagation_incoming"] + self.unpeered_propagation_rx_bytes = node_stats["unpeered_propagation_rx_bytes"] + + except Exception as e: + RNS.log("Could not load local node stats. The contained exception was: "+str(e), RNS.LOG_ERROR) + self.propagation_node = True + self.propagation_node_start_time = time.time() self.propagation_destination.set_link_established_callback(self.propagation_link_established) self.propagation_destination.set_packet_callback(self.propagation_packet) self.propagation_destination.register_request_handler(LXMPeer.OFFER_REQUEST_PATH, self.offer_request, allow = RNS.Destination.ALLOW_ALL) self.propagation_destination.register_request_handler(LXMPeer.MESSAGE_GET_PATH, self.message_get_request, allow = RNS.Destination.ALLOW_ALL) + self.control_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") + self.control_destination.register_request_handler(LXMRouter.STATS_GET_PATH, self.stats_get_request, allow = RNS.Destination.ALLOW_LIST, allowed_list=[self.identity.hash]) + if self.message_storage_limit != None: limit_str = ", limit is "+RNS.prettysize(self.message_storage_limit) else: @@ -650,6 +680,76 @@ class LXMRouter: return False + ### Propagation Node Control ########################## + ####################################################### + + def compile_stats(self): + if not self.propagation_node: + return None + else: + peer_stats = {} + for peer_id in self.peers.copy(): + peer = self.peers[peer_id] + peer_stats[peer_id] = { + "type": "static" if peer_id in self.static_peers else "discovered", + "state": peer.state, + "alive": peer.alive, + "last_heard": int(peer.last_heard), + "next_sync_attempt": peer.next_sync_attempt, + "last_sync_attempt": peer.last_sync_attempt, + "sync_backoff": peer.sync_backoff, + "peering_timebase": peer.peering_timebase, + "ler": int(peer.link_establishment_rate), + "str": int(peer.sync_transfer_rate), + "transfer_limit": peer.propagation_transfer_limit, + "network_distance": RNS.Transport.hops_to(peer_id), + "rx_bytes": peer.rx_bytes, + "tx_bytes": peer.tx_bytes, + "messages": { + "offered": peer.offered, + "outgoing": peer.outgoing, + "incoming": peer.incoming, + }, + } + + node_stats = { + "identity_hash": self.identity.hash, + "destination_hash": self.propagation_destination.hash, + "uptime": time.time()-self.propagation_node_start_time, + "delivery_limit": self.delivery_per_transfer_limit, + "propagation_limit": self.propagation_per_transfer_limit, + "autopeer_maxdepth": self.autopeer_maxdepth, + "from_static_only": self.from_static_only, + "messagestore": { + "count": len(self.propagation_entries), + "bytes": self.message_storage_size(), + "limit": self.message_storage_limit, + }, + "clients" : { + "client_propagation_messages_received": self.client_propagation_messages_received, + "client_propagation_messages_served": self.client_propagation_messages_served, + }, + "unpeered_propagation_incoming": self.unpeered_propagation_incoming, + "unpeered_propagation_rx_bytes": self.unpeered_propagation_rx_bytes, + "static_peers": len(self.static_peers), + "discovered_peers": len(self.peers)-len(self.static_peers), + "total_peers": len(self.peers), + "max_peers": self.max_peers, + "peers": peer_stats, + } + + return node_stats + + def stats_get_request(self, path, data, request_id, remote_identity, requested_at): + RNS.log("Stats request", RNS.LOG_DEBUG) # TODO: Remove debug + if remote_identity == None: + return LXMPeer.ERROR_NO_IDENTITY + elif remote_identity.hash != self.identity.hash: + return LXMPeer.ERROR_NO_ACCESS + else: + return self.compile_stats() + + ### Utility & Maintenance ############################# ####################################################### @@ -970,7 +1070,7 @@ class LXMRouter: try: if len(self.locally_delivered_transient_ids) > 0: if not os.path.isdir(self.storagepath): - os.makedirs(self.storagepath) + os.makedirs(self.storagepath) with open(self.storagepath+"/local_deliveries", "wb") as locally_delivered_file: locally_delivered_file.write(msgpack.packb(self.locally_delivered_transient_ids)) @@ -982,7 +1082,7 @@ class LXMRouter: try: if len(self.locally_processed_transient_ids) > 0: if not os.path.isdir(self.storagepath): - os.makedirs(self.storagepath) + os.makedirs(self.storagepath) with open(self.storagepath+"/locally_processed", "wb") as locally_processed_file: locally_processed_file.write(msgpack.packb(self.locally_processed_transient_ids)) @@ -990,6 +1090,24 @@ class LXMRouter: except Exception as e: RNS.log("Could not save locally processed transient ID cache to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + def save_node_stats(self): + try: + if not os.path.isdir(self.storagepath): + os.makedirs(self.storagepath) + + with open(self.storagepath+"/node_stats", "wb") as stats_file: + node_stats = { + "client_propagation_messages_received": self.client_propagation_messages_received, + "client_propagation_messages_served": self.client_propagation_messages_served, + "unpeered_propagation_incoming": self.unpeered_propagation_incoming, + "unpeered_propagation_rx_bytes": self.unpeered_propagation_rx_bytes, + } + stats_file.write(msgpack.packb(node_stats)) + + except Exception as e: + RNS.log("Could not save local node stats to storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + + def clean_outbound_stamp_costs(self): try: expired = [] @@ -1106,6 +1224,7 @@ class LXMRouter: self.propagation_destination.set_packet_callback(None) self.propagation_destination.deregister_request_handler(LXMPeer.OFFER_REQUEST_PATH) self.propagation_destination.deregister_request_handler(LXMPeer.MESSAGE_GET_PATH) + self.propagation_destination.deregister_request_handler(LXMRouter.STATS_GET_PATH) for link in self.active_propagation_links: try: if link.status == RNS.Link.ACTIVE: @@ -1135,6 +1254,7 @@ class LXMRouter: self.save_locally_delivered_transient_ids() self.save_locally_processed_transient_ids() + self.save_node_stats() def sigint_handler(self, signal, frame): if not self.exit_handler_running: @@ -1263,6 +1383,7 @@ class LXMRouter: except Exception as e: RNS.log("Error while processing message download request from "+RNS.prettyhexrep(remote_destination.hash)+". The contained exception was: "+str(e), RNS.LOG_ERROR) + self.client_propagation_messages_served += len(response_messages) return response_messages @@ -1777,6 +1898,7 @@ class LXMRouter: messages = data[1] for lxmf_data in messages: self.lxmf_propagation(lxmf_data) + self.client_propagation_messages_received += 1 packet.prove() @@ -1849,7 +1971,7 @@ class LXMRouter: remote_str = f"peer {remote_str}" messages = data[1] - RNS.log(f"Received {len(messages)} messages from {remote_str}", RNS.LOG_VERBOSE) + RNS.log(f"Received {len(messages)} message{"" if len(messages) == 1 else "s"} from {remote_str}", RNS.LOG_VERBOSE) for lxmf_data in messages: peer = None transient_id = RNS.Identity.full_hash(lxmf_data) @@ -1857,6 +1979,12 @@ class LXMRouter: peer = self.peers[remote_hash] peer.incoming += 1 peer.rx_bytes += len(lxmf_data) + else: + if remote_identity != None: + self.unpeered_propagation_incoming += 1 + self.unpeered_propagation_rx_bytes += len(lxmf_data) + else: + self.client_propagation_messages_received += 1 self.lxmf_propagation(lxmf_data, from_peer=peer) if peer != None: From f683e038910e45cf9be83b4dc01465ce8c8877ff Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 23 Jan 2025 14:15:12 +0100 Subject: [PATCH 280/436] Added lxmd status getter --- LXMF/Utilities/lxmd.py | 96 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 87 insertions(+), 9 deletions(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 0c87a73..1bc1d12 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -35,6 +35,7 @@ import time import os from LXMF._version import __version__ +from LXMF import APP_NAME from RNS.vendor.configobj import ConfigObj @@ -415,6 +416,75 @@ def deferred_start_jobs(): last_node_announce = time.time() threading.Thread(target=jobs, daemon=True).start() +def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = 0, timeout=5): + global configpath, identitypath, storagedir, lxmdir + global lxmd_config, active_configuration, targetloglevel + targetlogdest = RNS.LOG_STDOUT + + if configdir == None: + if os.path.isdir("/etc/lxmd") and os.path.isfile("/etc/lxmd/config"): + configdir = "/etc/lxmd" + elif os.path.isdir(RNS.Reticulum.userdir+"/.config/lxmd") and os.path.isfile(Reticulum.userdir+"/.config/lxmd/config"): + configdir = RNS.Reticulum.userdir+"/.config/lxmd" + else: + configdir = RNS.Reticulum.userdir+"/.lxmd" + + configpath = configdir+"/config" + identitypath = configdir+"/identity" + identity = None + + if not os.path.isdir(configdir): + RNS.log("Specified configuration directory does not exist, exiting now", RNS.LOG_ERROR) + exit(201) + if not os.path.isfile(identitypath): + RNS.log("Identity file not found in specified configuration directory, exiting now", RNS.LOG_ERROR) + exit(202) + else: + identity = RNS.Identity.from_file(identitypath) + if identity == None: + RNS.log("Could not load the Primary Identity from "+identitypath, RNS.LOG_ERROR) + exit(4) + + if targetloglevel == None: + targetloglevel = 3 + if verbosity != 0 or quietness != 0: + targetloglevel = targetloglevel+verbosity-quietness + + reticulum = RNS.Reticulum(configdir=rnsconfigdir, loglevel=targetloglevel, logdest=targetlogdest) + control_destination = RNS.Destination(identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") + + timeout = time.time()+timeout + def check_timeout(): + if time.time() > timeout: + RNS.log("Getting lxmd statistics timed out, exiting now", RNS.LOG_ERROR) + exit(200) + else: + time.sleep(0.1) + + if not RNS.Transport.has_path(control_destination.hash): + RNS.Transport.request_path(control_destination.hash) + while not RNS.Transport.has_path(control_destination.hash): + check_timeout() + + link = RNS.Link(control_destination) + while not link.status == RNS.Link.ACTIVE: + check_timeout() + + link.identify(identity) + request_receipt = link.request(LXMF.LXMRouter.STATS_GET_PATH, data=None, response_callback=None, failed_callback=None) + while not request_receipt.get_status() == RNS.RequestReceipt.READY: + check_timeout() + + response = request_receipt.get_response() + if response == LXMF.LXMPeer.LXMPeer.ERROR_NO_IDENTITY: + RNS.log("Remote received no identity") + exit(203) + if response == LXMF.LXMPeer.LXMPeer.ERROR_NO_ACCESS: + RNS.log("Access denied") + exit(204) + else: + # TODO: Output stats + def main(): try: parser = argparse.ArgumentParser(description="Lightweight Extensible Messaging Daemon") @@ -425,6 +495,8 @@ def main(): parser.add_argument("-v", "--verbose", action="count", default=0) parser.add_argument("-q", "--quiet", action="count", default=0) parser.add_argument("-s", "--service", action="store_true", default=False, help="lxmd is running as a service and should log to file") + parser.add_argument("--status", action="store_true", default=False, help="display node status") + parser.add_argument("--timeout", action="store", default=5, help="timeout in seconds for query operations", type=float) parser.add_argument("--exampleconfig", action="store_true", default=False, help="print verbose configuration example to stdout and exit") parser.add_argument("--version", action="version", version="lxmd {version}".format(version=__version__)) @@ -434,15 +506,21 @@ def main(): print(__default_lxmd_config__) exit() - program_setup( - configdir = args.config, - rnsconfigdir=args.rnsconfig, - run_pn=args.propagation_node, - on_inbound=args.on_inbound, - verbosity=args.verbose, - quietness=args.quiet, - service=args.service - ) + if args.status: + get_status(configdir = args.config, + rnsconfigdir=args.rnsconfig, + verbosity=args.verbose, + quietness=args.quiet, + timeout=args.timeout) + exit() + + program_setup(configdir = args.config, + rnsconfigdir=args.rnsconfig, + run_pn=args.propagation_node, + on_inbound=args.on_inbound, + verbosity=args.verbose, + quietness=args.quiet, + service=args.service) except KeyboardInterrupt: print("") From 460645cea2abc0a72b8f5d6444184286c4c676e8 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 23 Jan 2025 14:15:31 +0100 Subject: [PATCH 281/436] Added lxmd status getter --- LXMF/Utilities/lxmd.py | 1 + 1 file changed, 1 insertion(+) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 1bc1d12..d8b24d3 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -484,6 +484,7 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = exit(204) else: # TODO: Output stats + pass def main(): try: From e3be7e0cfdb529dece6e51165b67f697c70724b3 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 23 Jan 2025 16:27:01 +0100 Subject: [PATCH 282/436] Persist last sync attempt --- LXMF/LXMPeer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index ec0cfe2..61602c3 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -91,6 +91,11 @@ class LXMPeer: peer.tx_bytes = dictionary["tx_bytes"] else: peer.tx_bytes = 0 + + if "last_sync_attempt" in dictionary: + peer.last_sync_attempt = dictionary["last_sync_attempt"] + else: + peer.last_sync_attempt = 0 hm_count = 0 for transient_id in dictionary["handled_ids"]: @@ -121,6 +126,7 @@ class LXMPeer: dictionary["link_establishment_rate"] = self.link_establishment_rate dictionary["sync_transfer_rate"] = self.sync_transfer_rate dictionary["propagation_transfer_limit"] = self.propagation_transfer_limit + dictionary["last_sync_attempt"] = self.last_sync_attempt dictionary["offered"] = self.offered dictionary["outgoing"] = self.outgoing dictionary["incoming"] = self.incoming From a198e96064fa47af3f8e1dc8db225fbb39f77f80 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 23 Jan 2025 16:27:23 +0100 Subject: [PATCH 283/436] Include unhandled message count in stats --- LXMF/LXMRouter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 22ef3ac..8e824e4 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -709,6 +709,7 @@ class LXMRouter: "offered": peer.offered, "outgoing": peer.outgoing, "incoming": peer.incoming, + "unhandled": peer.unhandled_message_count }, } From 35dd70c59e09a7fa093f7e24b60065317e7507c9 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 23 Jan 2025 16:27:48 +0100 Subject: [PATCH 284/436] Format status and peers output --- LXMF/Utilities/lxmd.py | 140 +++++++++++++++++++++++++++++++++-------- 1 file changed, 115 insertions(+), 25 deletions(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index d8b24d3..dd070fc 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -416,34 +416,45 @@ def deferred_start_jobs(): last_node_announce = time.time() threading.Thread(target=jobs, daemon=True).start() -def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = 0, timeout=5): +def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = 0, timeout=5, show_status=False, show_peers=False, identity_path=None): global configpath, identitypath, storagedir, lxmdir global lxmd_config, active_configuration, targetloglevel targetlogdest = RNS.LOG_STDOUT - if configdir == None: - if os.path.isdir("/etc/lxmd") and os.path.isfile("/etc/lxmd/config"): - configdir = "/etc/lxmd" - elif os.path.isdir(RNS.Reticulum.userdir+"/.config/lxmd") and os.path.isfile(Reticulum.userdir+"/.config/lxmd/config"): - configdir = RNS.Reticulum.userdir+"/.config/lxmd" + if identity_path == None: + if configdir == None: + if os.path.isdir("/etc/lxmd") and os.path.isfile("/etc/lxmd/config"): + configdir = "/etc/lxmd" + elif os.path.isdir(RNS.Reticulum.userdir+"/.config/lxmd") and os.path.isfile(Reticulum.userdir+"/.config/lxmd/config"): + configdir = RNS.Reticulum.userdir+"/.config/lxmd" + else: + configdir = RNS.Reticulum.userdir+"/.lxmd" + + configpath = configdir+"/config" + identitypath = configdir+"/identity" + identity = None + + if not os.path.isdir(configdir): + RNS.log("Specified configuration directory does not exist, exiting now", RNS.LOG_ERROR) + exit(201) + if not os.path.isfile(identitypath): + RNS.log("Identity file not found in specified configuration directory, exiting now", RNS.LOG_ERROR) + exit(202) else: - configdir = RNS.Reticulum.userdir+"/.lxmd" + identity = RNS.Identity.from_file(identitypath) + if identity == None: + RNS.log("Could not load the Primary Identity from "+identitypath, RNS.LOG_ERROR) + exit(4) - configpath = configdir+"/config" - identitypath = configdir+"/identity" - identity = None - - if not os.path.isdir(configdir): - RNS.log("Specified configuration directory does not exist, exiting now", RNS.LOG_ERROR) - exit(201) - if not os.path.isfile(identitypath): - RNS.log("Identity file not found in specified configuration directory, exiting now", RNS.LOG_ERROR) - exit(202) else: - identity = RNS.Identity.from_file(identitypath) - if identity == None: - RNS.log("Could not load the Primary Identity from "+identitypath, RNS.LOG_ERROR) - exit(4) + if not os.path.isfile(identity_path): + RNS.log("Identity file not found in specified configuration directory, exiting now", RNS.LOG_ERROR) + exit(202) + else: + identity = RNS.Identity.from_file(identity_path) + if identity == None: + RNS.log("Could not load the Primary Identity from "+identity_path, RNS.LOG_ERROR) + exit(4) if targetloglevel == None: targetloglevel = 3 @@ -483,8 +494,82 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = RNS.log("Access denied") exit(204) else: - # TODO: Output stats - pass + s = response + ms_util = f"{round((s["messagestore"]["bytes"]/s["messagestore"]["limit"])*100, 2)}%" + if s["from_static_only"]: + who_str = "static peers only" + else: + who_str = "all nodes" + + available_peers = 0 + unreachable_peers = 0 + peered_incoming = 0 + peered_outgoing = 0 + peered_rx_bytes = 0 + peered_tx_bytes = 0 + for peer_id in s["peers"]: + p = s["peers"][peer_id] + pm = p["messages"] + peered_incoming += pm["incoming"] + peered_outgoing += pm["outgoing"] + peered_rx_bytes += p["rx_bytes"] + peered_tx_bytes += p["tx_bytes"] + if p["alive"]: + available_peers += 1 + else: + unreachable_peers += 1 + + total_incoming = peered_incoming+s["unpeered_propagation_incoming"]+s["clients"]["client_propagation_messages_received"] + total_rx_bytes = peered_rx_bytes+s["unpeered_propagation_rx_bytes"] + df = round(peered_outgoing/total_incoming, 2) + + print(f"\nLXMF Propagation Node running on {RNS.prettyhexrep(s["destination_hash"])}, uptime is {RNS.prettytime(s["uptime"])}") + + if show_status: + print(f"Messagestore contains {s["messagestore"]["count"]} messages, {RNS.prettysize(s["messagestore"]["bytes"])} of {RNS.prettysize(s["messagestore"]["limit"])} ({ms_util} utilised)") + print(f"Accepting propagated messages from {who_str}, {RNS.prettysize(s["propagation_limit"]*1000)} per-transfer limit") + print(f"") + print(f"Peers : {s["total_peers"]} total (peer limit is {s["max_peers"]})") + print(f" {s["discovered_peers"]} discovered, {s["static_peers"]} static") + print(f" {available_peers} available, {unreachable_peers} unreachable") + print(f"") + print(f"Traffic : {s["unpeered_propagation_incoming"]} messages received from unpeered nodes ({RNS.prettysize(s["unpeered_propagation_rx_bytes"])})") + print(f" {peered_incoming} messages received from peered nodes ({RNS.prettysize(peered_rx_bytes)})") + print(f" {total_incoming} messages received from peered nodes ({RNS.prettysize(total_rx_bytes)})") + print(f" {peered_outgoing} messages transferred to peered nodes ({RNS.prettysize(peered_tx_bytes)})") + print(f" {s["clients"]["client_propagation_messages_received"]} messages received from clients") + print(f" {s["clients"]["client_propagation_messages_served"]} messages served to clients") + print(f" Distribution factor is {df}") + print(f"") + + if show_peers: + for peer_id in s["peers"]: + ind = " " + p = s["peers"][peer_id] + if p["type"] == "static": + t = "Static peer " + elif p["type"] == "discovered": + t = "Discovered peer " + else: + t = "Unknown peer " + a = "Available" if p["alive"] == True else "Unreachable" + h = max(time.time()-p["last_heard"], 0) + hops = p["network_distance"] + hs = f"{hops} hop away" if hops == 1 else f"{hops} hops away" + pm = p["messages"] + if p["last_sync_attempt"] != 0: + ls = f"last synced {RNS.prettytime(max(time.time()-p["last_sync_attempt"], 0))} ago" + else: + ls = "never synced" + + print(f"{ind}{t}{RNS.prettyhexrep(peer_id)}") + print(f"{ind*2}Status : {a}, {hs}, last heard {RNS.prettytime(h)} ago") + print(f"{ind*2}Speeds : {RNS.prettyspeed(p["str"])} STR, {RNS.prettyspeed(p["ler"])} LER, {RNS.prettysize(p["transfer_limit"]*1000)} transfer limit") + print(f"{ind*2}Messages : {pm["offered"]} offered, {pm["outgoing"]} outgoing, {pm["incoming"]} incoming") + print(f"{ind*2}Traffic : {RNS.prettysize(p["rx_bytes"])} received, {RNS.prettysize(p["tx_bytes"])} sent") + print(f"{ind*2}Sync state : {pm["unhandled"]} unhandled message{"" if pm["unhandled"] == 1 else "s"}, {ls}") + print("") + def main(): try: @@ -497,7 +582,9 @@ def main(): parser.add_argument("-q", "--quiet", action="count", default=0) parser.add_argument("-s", "--service", action="store_true", default=False, help="lxmd is running as a service and should log to file") parser.add_argument("--status", action="store_true", default=False, help="display node status") + parser.add_argument("--peers", action="store_true", default=False, help="display peered nodes") parser.add_argument("--timeout", action="store", default=5, help="timeout in seconds for query operations", type=float) + parser.add_argument("--identity", action="store", default=None, help="path to identity used for query request", type=str) parser.add_argument("--exampleconfig", action="store_true", default=False, help="print verbose configuration example to stdout and exit") parser.add_argument("--version", action="version", version="lxmd {version}".format(version=__version__)) @@ -507,12 +594,15 @@ def main(): print(__default_lxmd_config__) exit() - if args.status: + if args.status or args.peers: get_status(configdir = args.config, rnsconfigdir=args.rnsconfig, verbosity=args.verbose, quietness=args.quiet, - timeout=args.timeout) + timeout=args.timeout, + show_status=args.status, + show_peers=args.peers, + identity_path=args.identity) exit() program_setup(configdir = args.config, From a87458d25f794d84c68f0c4212fedc91bcd7e7fb Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 23 Jan 2025 16:28:11 +0100 Subject: [PATCH 285/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 43c4ab0..22049ab 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.6.1" +__version__ = "0.6.2" From fe59b265c50ce87ca33e183b2b154b3eaaf163c0 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 23 Jan 2025 16:54:12 +0100 Subject: [PATCH 286/436] Fixed fstrings not working on Python < 3.12 --- LXMF/LXMRouter.py | 3 ++- LXMF/Utilities/lxmd.py | 40 ++++++++++++++++++++++++++-------------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 8e824e4..0358428 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1972,7 +1972,8 @@ class LXMRouter: remote_str = f"peer {remote_str}" messages = data[1] - RNS.log(f"Received {len(messages)} message{"" if len(messages) == 1 else "s"} from {remote_str}", RNS.LOG_VERBOSE) + ms = "" if len(messages) == 1 else "s" + RNS.log(f"Received {len(messages)} message{ms} from {remote_str}", RNS.LOG_VERBOSE) for lxmf_data in messages: peer = None transient_id = RNS.Identity.full_hash(lxmf_data) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index dd070fc..54e0b54 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -495,7 +495,8 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = exit(204) else: s = response - ms_util = f"{round((s["messagestore"]["bytes"]/s["messagestore"]["limit"])*100, 2)}%" + mutil = round((s["messagestore"]["bytes"]/s["messagestore"]["limit"])*100, 2) + ms_util = f"{mutil}%" if s["from_static_only"]: who_str = "static peers only" else: @@ -523,22 +524,28 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = total_rx_bytes = peered_rx_bytes+s["unpeered_propagation_rx_bytes"] df = round(peered_outgoing/total_incoming, 2) - print(f"\nLXMF Propagation Node running on {RNS.prettyhexrep(s["destination_hash"])}, uptime is {RNS.prettytime(s["uptime"])}") + dhs = RNS.prettyhexrep(s["destination_hash"]); uts = RNS.prettytime(s["uptime"]) + print(f"\nLXMF Propagation Node running on {dhs}, uptime is {uts}") if show_status: - print(f"Messagestore contains {s["messagestore"]["count"]} messages, {RNS.prettysize(s["messagestore"]["bytes"])} of {RNS.prettysize(s["messagestore"]["limit"])} ({ms_util} utilised)") - print(f"Accepting propagated messages from {who_str}, {RNS.prettysize(s["propagation_limit"]*1000)} per-transfer limit") + msb = RNS.prettysize(s["messagestore"]["bytes"]); msl = RNS.prettysize(s["messagestore"]["limit"]) + ptl = RNS.prettysize(s["propagation_limit"]*1000); uprx = RNS.prettysize(s["unpeered_propagation_rx_bytes"]) + mscnt = s["messagestore"]["count"]; stp = s["total_peers"]; smp = s["max_peers"]; sdp = s["discovered_peers"] + ssp = s["static_peers"]; cprr = s["clients"]["client_propagation_messages_received"] + cprs = s["clients"]["client_propagation_messages_served"]; upi = s["unpeered_propagation_incoming"] + print(f"Messagestore contains {mscnt} messages, {msb} ({ms_util} utilised of {msl})") + print(f"Accepting propagated messages from {who_str}, {ptl} per-transfer limit") print(f"") - print(f"Peers : {s["total_peers"]} total (peer limit is {s["max_peers"]})") - print(f" {s["discovered_peers"]} discovered, {s["static_peers"]} static") + print(f"Peers : {stp} total (peer limit is {smp})") + print(f" {sdp} discovered, {ssp} static") print(f" {available_peers} available, {unreachable_peers} unreachable") print(f"") - print(f"Traffic : {s["unpeered_propagation_incoming"]} messages received from unpeered nodes ({RNS.prettysize(s["unpeered_propagation_rx_bytes"])})") + print(f"Traffic : {upi} messages received from unpeered nodes ({uprx})") print(f" {peered_incoming} messages received from peered nodes ({RNS.prettysize(peered_rx_bytes)})") print(f" {total_incoming} messages received from peered nodes ({RNS.prettysize(total_rx_bytes)})") print(f" {peered_outgoing} messages transferred to peered nodes ({RNS.prettysize(peered_tx_bytes)})") - print(f" {s["clients"]["client_propagation_messages_received"]} messages received from clients") - print(f" {s["clients"]["client_propagation_messages_served"]} messages served to clients") + print(f" {cprr} messages received from clients") + print(f" {cprs} messages served to clients") print(f" Distribution factor is {df}") print(f"") @@ -558,16 +565,21 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = hs = f"{hops} hop away" if hops == 1 else f"{hops} hops away" pm = p["messages"] if p["last_sync_attempt"] != 0: - ls = f"last synced {RNS.prettytime(max(time.time()-p["last_sync_attempt"], 0))} ago" + lsa = p["last_sync_attempt"] + ls = f"last synced {RNS.prettytime(max(time.time()-lsa, 0))} ago" else: ls = "never synced" + sstr = RNS.prettyspeed(p["str"]); sler = RNS.prettyspeed(p["ler"]); stl = RNS.prettysize(p["transfer_limit"]*1000) + srxb = RNS.prettysize(p["rx_bytes"]); stxb = RNS.prettysize(p["tx_bytes"]); pmo = pm["offered"]; pmout = pm["outgoing"] + pmi = pm["incoming"]; pmuh = pm["unhandled"] print(f"{ind}{t}{RNS.prettyhexrep(peer_id)}") print(f"{ind*2}Status : {a}, {hs}, last heard {RNS.prettytime(h)} ago") - print(f"{ind*2}Speeds : {RNS.prettyspeed(p["str"])} STR, {RNS.prettyspeed(p["ler"])} LER, {RNS.prettysize(p["transfer_limit"]*1000)} transfer limit") - print(f"{ind*2}Messages : {pm["offered"]} offered, {pm["outgoing"]} outgoing, {pm["incoming"]} incoming") - print(f"{ind*2}Traffic : {RNS.prettysize(p["rx_bytes"])} received, {RNS.prettysize(p["tx_bytes"])} sent") - print(f"{ind*2}Sync state : {pm["unhandled"]} unhandled message{"" if pm["unhandled"] == 1 else "s"}, {ls}") + print(f"{ind*2}Speeds : {sstr} STR, {sler} LER, {stl} transfer limit") + print(f"{ind*2}Messages : {pmo} offered, {pmout} outgoing, {pmi} incoming") + print(f"{ind*2}Traffic : {srxb} received, {stxb} sent") + ms = "" if pm["unhandled"] == 1 else "s" + print(f"{ind*2}Sync state : {pmuh} unhandled message{ms}, {ls}") print("") From 70186cf8d9fc780eba6ce39494964c31b2519a57 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 23 Jan 2025 17:07:20 +0100 Subject: [PATCH 287/436] Fixed typo --- LXMF/Utilities/lxmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 54e0b54..bb29661 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -542,7 +542,7 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = print(f"") print(f"Traffic : {upi} messages received from unpeered nodes ({uprx})") print(f" {peered_incoming} messages received from peered nodes ({RNS.prettysize(peered_rx_bytes)})") - print(f" {total_incoming} messages received from peered nodes ({RNS.prettysize(total_rx_bytes)})") + print(f" {total_incoming} messages received from all nodes ({RNS.prettysize(total_rx_bytes)})") print(f" {peered_outgoing} messages transferred to peered nodes ({RNS.prettysize(peered_tx_bytes)})") print(f" {cprr} messages received from clients") print(f" {cprs} messages served to clients") From a3e3868f9258ed1f0b930e85a8993234440d448d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 23 Jan 2025 17:09:40 +0100 Subject: [PATCH 288/436] Changed formatting --- LXMF/Utilities/lxmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index bb29661..415ecb6 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -542,7 +542,7 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = print(f"") print(f"Traffic : {upi} messages received from unpeered nodes ({uprx})") print(f" {peered_incoming} messages received from peered nodes ({RNS.prettysize(peered_rx_bytes)})") - print(f" {total_incoming} messages received from all nodes ({RNS.prettysize(total_rx_bytes)})") + print(f" {total_incoming} messages received in total ({RNS.prettysize(total_rx_bytes)})") print(f" {peered_outgoing} messages transferred to peered nodes ({RNS.prettysize(peered_tx_bytes)})") print(f" {cprr} messages received from clients") print(f" {cprs} messages served to clients") From fb4bf9b0b9307cb872e97619a685e8adf44a467e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 23 Jan 2025 17:36:30 +0100 Subject: [PATCH 289/436] Cleanup --- LXMF/LXMRouter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 0358428..bd63e17 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -742,7 +742,6 @@ class LXMRouter: return node_stats def stats_get_request(self, path, data, request_id, remote_identity, requested_at): - RNS.log("Stats request", RNS.LOG_DEBUG) # TODO: Remove debug if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY elif remote_identity.hash != self.identity.hash: From cdea838a6c38f0b9a4f7d983b48361565bbc835f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 23 Jan 2025 17:43:24 +0100 Subject: [PATCH 290/436] Updated status output --- LXMF/Utilities/lxmd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 415ecb6..b52d5ae 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -544,8 +544,8 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = print(f" {peered_incoming} messages received from peered nodes ({RNS.prettysize(peered_rx_bytes)})") print(f" {total_incoming} messages received in total ({RNS.prettysize(total_rx_bytes)})") print(f" {peered_outgoing} messages transferred to peered nodes ({RNS.prettysize(peered_tx_bytes)})") - print(f" {cprr} messages received from clients") - print(f" {cprs} messages served to clients") + print(f" {cprr} propagation messages received diretly from clients") + print(f" {cprs} propagation messages served to clients") print(f" Distribution factor is {df}") print(f"") From aa57b16cf562d8f9409e877f959d9751f8c5b300 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 24 Jan 2025 00:09:36 +0100 Subject: [PATCH 291/436] Fixed #23 --- LXMF/LXMRouter.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index bd63e17..b0a4cc8 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -777,13 +777,16 @@ class LXMRouter: self.clean_transient_id_caches() if self.processing_count % LXMRouter.JOB_STORE_INTERVAL == 0: - self.clean_message_store() + if self.propagation_node == True: + self.clean_message_store() if self.processing_count % LXMRouter.JOB_PEERINGEST_INTERVAL == 0: - self.flush_queues() + if self.propagation_node == True: + self.flush_queues() if self.processing_count % LXMRouter.JOB_PEERSYNC_INTERVAL == 0: - self.sync_peers() + if self.propagation_node == True: + self.sync_peers() # def syncstats(self): # for peer_id in self.peers: @@ -986,12 +989,12 @@ class LXMRouter: lxm_size = self.propagation_entries[transient_id][3] return lxm_size - def clean_message_store(self): + RNS.log("Cleaning message store", RNS.LOG_VERBOSE) # Check and remove expired messages now = time.time() removed_entries = {} - for transient_id in self.propagation_entries: + for transient_id in self.propagation_entries.copy(): entry = self.propagation_entries[transient_id] filepath = entry[1] components = filepath.split("_") @@ -999,7 +1002,7 @@ class LXMRouter: if len(components) == 2 and float(components[1]) > 0 and len(os.path.split(components[0])[1]) == (RNS.Identity.HASHLENGTH//8)*2: timestamp = float(components[1]) if now > timestamp+LXMRouter.MESSAGE_EXPIRY: - RNS.log("Purging message "+RNS.prettyhexrep(transient_id)+" due to expiry", RNS.LOG_DEBUG) + RNS.log("Purging message "+RNS.prettyhexrep(transient_id)+" due to expiry", RNS.LOG_EXTREME) removed_entries[transient_id] = filepath else: RNS.log("Purging message "+RNS.prettyhexrep(transient_id)+" due to invalid file path", RNS.LOG_WARNING) @@ -1017,7 +1020,7 @@ class LXMRouter: RNS.log("Could not remove "+RNS.prettyhexrep(transient_id)+" from message store. The contained exception was: "+str(e), RNS.LOG_ERROR) if removed_count > 0: - RNS.log("Cleaned "+str(removed_count)+" entries from the message store", RNS.LOG_DEBUG) + RNS.log("Cleaned "+str(removed_count)+" entries from the message store", RNS.LOG_VERBOSE) # Check size of message store and cull if needed try: @@ -1029,7 +1032,7 @@ class LXMRouter: bytes_cleaned = 0 weighted_entries = [] - for transient_id in self.propagation_entries: + for transient_id in self.propagation_entries.copy(): weighted_entries.append([ self.propagation_entries[transient_id], self.get_weight(transient_id), From a8cc5f41cf92a7e35b80bc2f6b55292ad4cf170d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 24 Jan 2025 00:21:37 +0100 Subject: [PATCH 292/436] Fixed typo --- LXMF/Utilities/lxmd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index b52d5ae..2f21108 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -540,11 +540,11 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = print(f" {sdp} discovered, {ssp} static") print(f" {available_peers} available, {unreachable_peers} unreachable") print(f"") - print(f"Traffic : {upi} messages received from unpeered nodes ({uprx})") + print(f"Traffic : {total_incoming} messages received in total ({RNS.prettysize(total_rx_bytes)})") print(f" {peered_incoming} messages received from peered nodes ({RNS.prettysize(peered_rx_bytes)})") - print(f" {total_incoming} messages received in total ({RNS.prettysize(total_rx_bytes)})") + print(f" {upi} messages received from unpeered nodes ({uprx})") print(f" {peered_outgoing} messages transferred to peered nodes ({RNS.prettysize(peered_tx_bytes)})") - print(f" {cprr} propagation messages received diretly from clients") + print(f" {cprr} propagation messages received directly from clients") print(f" {cprs} propagation messages served to clients") print(f" Distribution factor is {df}") print(f"") From 6d2eb4f97375dc2c637dd531d94a11738d2cb7ed Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 24 Jan 2025 00:26:47 +0100 Subject: [PATCH 293/436] Updated default config --- LXMF/Utilities/lxmd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 2f21108..7f54835 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -127,7 +127,7 @@ def apply_config(): if active_configuration["message_storage_limit"] < 0.005: active_configuration["message_storage_limit"] = 0.005 else: - active_configuration["message_storage_limit"] = 2000 + active_configuration["message_storage_limit"] = 500 if "propagation" in lxmd_config and "propagation_transfer_max_accepted_size" in lxmd_config["propagation"]: active_configuration["propagation_transfer_max_accepted_size"] = lxmd_config["propagation"].as_float("propagation_transfer_max_accepted_size") @@ -679,9 +679,9 @@ propagation_transfer_max_accepted_size = 256 # LXMF prioritises keeping messages that are # new and small. Large and old messages will # be removed first. This setting is optional -# and defaults to 2 gigabytes. +# and defaults to 500 megabytes. -# message_storage_limit = 2000 +# message_storage_limit = 500 # You can tell the LXMF message router to # prioritise storage for one or more From 962d9c90d1c468e95589b15ccaf2384a379dea35 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 24 Jan 2025 13:50:56 +0100 Subject: [PATCH 294/436] Added wanted inbound peers to PN announce data --- LXMF/Handlers.py | 9 +++++++-- LXMF/LXMRouter.py | 10 ++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index 22c6cd3..ea8960d 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -45,6 +45,11 @@ class LXMFPropagationAnnounceHandler: if pn_announce_data_is_valid(data): node_timebase = data[1] propagation_transfer_limit = None + if len(data) >= 4: + try: + wanted_inbound_peers = int(data[3]) + except: + wanted_inbound_peers = None if len(data) >= 3: try: propagation_transfer_limit = float(data[2]) @@ -52,12 +57,12 @@ class LXMFPropagationAnnounceHandler: propagation_transfer_limit = None if destination_hash in self.lxmrouter.static_peers: - self.lxmrouter.peer(destination_hash, node_timebase, propagation_transfer_limit) + self.lxmrouter.peer(destination_hash, node_timebase, propagation_transfer_limit, wanted_inbound_peers) else: if data[0] == True: if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: - self.lxmrouter.peer(destination_hash, node_timebase, propagation_transfer_limit) + self.lxmrouter.peer(destination_hash, node_timebase, propagation_transfer_limit, wanted_inbound_peers) elif data[0] == False: self.lxmrouter.unpeer(destination_hash, node_timebase) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index b0a4cc8..9eccedc 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -283,6 +283,7 @@ class LXMRouter: node_state, # Boolean flag signalling propagation node state int(time.time()), # Current node timebase self.propagation_per_transfer_limit, # Per-transfer limit for message propagation in kilobytes + self.get_wanted_inbound_peers(), # How many more inbound peers this node wants ] data = msgpack.packb(announce_data) @@ -888,6 +889,10 @@ class LXMRouter: self.save_outbound_stamp_costs() threading.Thread(target=self.save_outbound_stamp_costs, daemon=True).start() + def get_wanted_inbound_peers(self): + # TODO: Implement + return None + def get_announce_app_data(self, destination_hash): if destination_hash in self.delivery_destinations: delivery_destination = self.delivery_destinations[destination_hash] @@ -1766,7 +1771,7 @@ class LXMRouter: ### Peer Sync & Propagation ########################### ####################################################### - def peer(self, destination_hash, timestamp, propagation_transfer_limit): + def peer(self, destination_hash, timestamp, propagation_transfer_limit, wanted_inbound_peers = None): if destination_hash in self.peers: peer = self.peers[destination_hash] if timestamp > peer.peering_timebase: @@ -1969,7 +1974,8 @@ class LXMRouter: # sane default value, and wait for an announce to arrive # that will update the peering config to the actual limit. propagation_transfer_limit = LXMRouter.PROPAGATION_LIMIT//4 - self.peer(remote_hash, remote_timebase, propagation_transfer_limit) + wanted_inbound_peers = None + self.peer(remote_hash, remote_timebase, propagation_transfer_limit, wanted_inbound_peers) else: remote_str = f"peer {remote_str}" From cec903a4dcc878f14f8cd8be6a9abc54868cbea6 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 24 Jan 2025 14:05:12 +0100 Subject: [PATCH 295/436] Added status query API function --- LXMF/Handlers.py | 1 + LXMF/LXMPeer.py | 1 + LXMF/Utilities/lxmd.py | 60 ++++++++++++++++++++++++------------------ 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index ea8960d..26a5df6 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -45,6 +45,7 @@ class LXMFPropagationAnnounceHandler: if pn_announce_data_is_valid(data): node_timebase = data[1] propagation_transfer_limit = None + wanted_inbound_peers = None if len(data) >= 4: try: wanted_inbound_peers = int(data[3]) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 61602c3..5036528 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -20,6 +20,7 @@ class LXMPeer: ERROR_NO_IDENTITY = 0xf0 ERROR_NO_ACCESS = 0xf1 + ERROR_TIMEOUT = 0xfe # Maximum amount of time a peer can # be unreachable before it is removed diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 7f54835..a06d60c 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -416,6 +416,36 @@ def deferred_start_jobs(): last_node_announce = time.time() threading.Thread(target=jobs, daemon=True).start() +def query_status(identity, timeout=5, exit_on_fail=False): + control_destination = RNS.Destination(identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") + + timeout = time.time()+timeout + def check_timeout(): + if time.time() > timeout: + RNS.log("Getting lxmd statistics timed out, exiting now", RNS.LOG_ERROR) + if exit_on_fail: + exit(200) + else: + return LXMF.LXMPeer.LXMPeer.ERROR_TIMEOUT + else: + time.sleep(0.1) + + if not RNS.Transport.has_path(control_destination.hash): + RNS.Transport.request_path(control_destination.hash) + while not RNS.Transport.has_path(control_destination.hash): + check_timeout() + + link = RNS.Link(control_destination) + while not link.status == RNS.Link.ACTIVE: + check_timeout() + + link.identify(identity) + request_receipt = link.request(LXMF.LXMRouter.STATS_GET_PATH, data=None, response_callback=None, failed_callback=None) + while not request_receipt.get_status() == RNS.RequestReceipt.READY: + check_timeout() + + return request_receipt.get_response() + def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = 0, timeout=5, show_status=False, show_peers=False, identity_path=None): global configpath, identitypath, storagedir, lxmdir global lxmd_config, active_configuration, targetloglevel @@ -462,31 +492,8 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = targetloglevel = targetloglevel+verbosity-quietness reticulum = RNS.Reticulum(configdir=rnsconfigdir, loglevel=targetloglevel, logdest=targetlogdest) - control_destination = RNS.Destination(identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") + response = query_status(identity, timeout=timeout, exit_on_fail=True) - timeout = time.time()+timeout - def check_timeout(): - if time.time() > timeout: - RNS.log("Getting lxmd statistics timed out, exiting now", RNS.LOG_ERROR) - exit(200) - else: - time.sleep(0.1) - - if not RNS.Transport.has_path(control_destination.hash): - RNS.Transport.request_path(control_destination.hash) - while not RNS.Transport.has_path(control_destination.hash): - check_timeout() - - link = RNS.Link(control_destination) - while not link.status == RNS.Link.ACTIVE: - check_timeout() - - link.identify(identity) - request_receipt = link.request(LXMF.LXMRouter.STATS_GET_PATH, data=None, response_callback=None, failed_callback=None) - while not request_receipt.get_status() == RNS.RequestReceipt.READY: - check_timeout() - - response = request_receipt.get_response() if response == LXMF.LXMPeer.LXMPeer.ERROR_NO_IDENTITY: RNS.log("Remote received no identity") exit(203) @@ -550,6 +557,9 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = print(f"") if show_peers: + if not show_status: + print("") + for peer_id in s["peers"]: ind = " " p = s["peers"][peer_id] @@ -562,7 +572,7 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = a = "Available" if p["alive"] == True else "Unreachable" h = max(time.time()-p["last_heard"], 0) hops = p["network_distance"] - hs = f"{hops} hop away" if hops == 1 else f"{hops} hops away" + hs = "hops unknown" if hops == RNS.Transport.PATHFINDER_M else f"{hops} hop away" if hops == 1 else f"{hops} hops away" pm = p["messages"] if p["last_sync_attempt"] != 0: lsa = p["last_sync_attempt"] From 26a10cce8f8a572553084c69603ca6605f2672fd Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 26 Jan 2025 01:13:11 +0100 Subject: [PATCH 296/436] Status query return code --- LXMF/Utilities/lxmd.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index a06d60c..73d0eb0 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -422,8 +422,8 @@ def query_status(identity, timeout=5, exit_on_fail=False): timeout = time.time()+timeout def check_timeout(): if time.time() > timeout: - RNS.log("Getting lxmd statistics timed out, exiting now", RNS.LOG_ERROR) if exit_on_fail: + RNS.log("Getting lxmd statistics timed out, exiting now", RNS.LOG_ERROR) exit(200) else: return LXMF.LXMPeer.LXMPeer.ERROR_TIMEOUT @@ -433,16 +433,22 @@ def query_status(identity, timeout=5, exit_on_fail=False): if not RNS.Transport.has_path(control_destination.hash): RNS.Transport.request_path(control_destination.hash) while not RNS.Transport.has_path(control_destination.hash): - check_timeout() + tc = check_timeout() + if tc: + return tc link = RNS.Link(control_destination) while not link.status == RNS.Link.ACTIVE: - check_timeout() + tc = check_timeout() + if tc: + return tc link.identify(identity) request_receipt = link.request(LXMF.LXMRouter.STATS_GET_PATH, data=None, response_callback=None, failed_callback=None) while not request_receipt.get_status() == RNS.RequestReceipt.READY: - check_timeout() + tc = check_timeout() + if tc: + return tc return request_receipt.get_response() From e0163e100a5541ed9abf4c57bb38960739ca23ea Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 27 Jan 2025 10:26:11 +0100 Subject: [PATCH 297/436] Updated issue template --- .github/ISSUE_TEMPLATE/🐛-bug-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/🐛-bug-report.md b/.github/ISSUE_TEMPLATE/🐛-bug-report.md index 77ad6c2..65b492e 100644 --- a/.github/ISSUE_TEMPLATE/🐛-bug-report.md +++ b/.github/ISSUE_TEMPLATE/🐛-bug-report.md @@ -12,7 +12,7 @@ Before creating a bug report on this issue tracker, you **must** read the [Contr - The issue tracker is used by developers of this project. **Do not use it to ask general questions, or for support requests**. - Ideas and feature requests can be made on the [Discussions](https://github.com/markqvist/Reticulum/discussions). **Only** feature requests accepted by maintainers and developers are tracked and included on the issue tracker. **Do not post feature requests here**. -- After reading the [Contribution Guidelines](https://github.com/markqvist/Reticulum/blob/master/Contributing.md), delete this section from your bug report. +- After reading the [Contribution Guidelines](https://github.com/markqvist/Reticulum/blob/master/Contributing.md), **delete this section only** (*"Read the Contribution Guidelines"*) from your bug report, **and fill in all the other sections**. **Describe the Bug** A clear and concise description of what the bug is. From 886ac69a8284e8ca3c3c0a4e2106f7160e8c7f62 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 27 Jan 2025 12:04:05 +0100 Subject: [PATCH 298/436] Tear down control link after use --- LXMF/Utilities/lxmd.py | 1 + 1 file changed, 1 insertion(+) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 73d0eb0..f9a2ef6 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -450,6 +450,7 @@ def query_status(identity, timeout=5, exit_on_fail=False): if tc: return tc + link.teardown() return request_receipt.get_response() def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = 0, timeout=5, show_status=False, show_peers=False, identity_path=None): From e0e901291e20515d12abe105fef30010db7fb1f1 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 27 Jan 2025 12:04:16 +0100 Subject: [PATCH 299/436] Updated logging --- LXMF/LXMessage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 2342708..515ab11 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -380,7 +380,7 @@ class LXMessage: if self.desired_method == LXMessage.OPPORTUNISTIC: if self.__destination.type == RNS.Destination.SINGLE: if content_size > LXMessage.ENCRYPTED_PACKET_MAX_CONTENT: - RNS.log(f"Opportunistic delivery was requested for {self}, but content exceeds packet size limit. Falling back to link-based delivery.", RNS.LOG_DEBUG) + RNS.log(f"Opportunistic delivery was requested for {self}, but content of length {content_size} exceeds packet size limit. Falling back to link-based delivery.", RNS.LOG_DEBUG) self.desired_method = LXMessage.DIRECT # Set delivery parameters according to delivery method From f1d060a92ef9ea9b0a0f3402ff46fc9d91fddd5c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 29 Jan 2025 01:26:36 +0100 Subject: [PATCH 300/436] Added peer rotation --- LXMF/LXMPeer.py | 4 +++ LXMF/LXMRouter.py | 68 +++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 5036528..e2f951a 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -469,6 +469,10 @@ class LXMPeer: return self._um_count + @property + def acceptance_rate(self): + return 0 if self.offered == 0 else (self.outgoing/self.offered) + def _update_counts(self): if not self._hm_counts_synced: hm = self.handled_messages; del hm diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 9eccedc..4bbd24c 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -41,6 +41,7 @@ class LXMRouter: AUTOPEER = True AUTOPEER_MAXDEPTH = 4 FASTEST_N_RANDOM_POOL = 2 + ROTATION_HEADROOM_PCT = 10 PROPAGATION_LIMIT = 256 DELIVERY_LIMIT = 1000 @@ -122,6 +123,7 @@ class LXMRouter: self.propagation_transfer_progress = 0.0 self.propagation_transfer_last_result = None self.propagation_transfer_max_messages = None + self.prioritise_rotating_unreachable_peers = False self.active_propagation_links = [] self.locally_delivered_transient_ids = {} self.locally_processed_transient_ids = {} @@ -783,17 +785,13 @@ class LXMRouter: if self.processing_count % LXMRouter.JOB_PEERINGEST_INTERVAL == 0: if self.propagation_node == True: + self.rotate_peers() self.flush_queues() if self.processing_count % LXMRouter.JOB_PEERSYNC_INTERVAL == 0: if self.propagation_node == True: self.sync_peers() - # def syncstats(self): - # for peer_id in self.peers: - # p = self.peers[peer_id] - # RNS.log(f"{RNS.prettyhexrep(peer_id)} O={p.offered} S={p.outgoing} I={p.incoming} TX={RNS.prettysize(p.tx_bytes)} RX={RNS.prettysize(p.rx_bytes)}") - def jobloop(self): while (True): # TODO: Improve this to scheduling, so manual @@ -1805,6 +1803,66 @@ class LXMRouter: self.peers.pop(destination_hash) RNS.log("Broke peering with "+str(peer.destination)) + def rotate_peers(self): + try: + rotation_headroom = max(1, math.floor(self.max_peers*(LXMRouter.ROTATION_HEADROOM_PCT/100.0))) + required_drops = len(self.peers) - (self.max_peers - rotation_headroom) + if required_drops > 0 and len(self.peers) - required_drops > 1: + peers = self.peers.copy() + untested_peers = [] + for peer_id in self.peers: + peer = self.peers[peer_id] + if peer.last_sync_attempt == 0: + untested_peers.append(peer) + + if len(untested_peers) >= rotation_headroom: + RNS.log("Newly added peer threshold reached, postponing peer rotation", RNS.LOG_DEBUG) + return + + culled_peers = [] + waiting_peers = [] + unresponsive_peers = [] + for peer_id in peers: + peer = peers[peer_id] + if not peer_id in self.static_peers and peer.state == LXMPeer.IDLE: + if peer.alive: + if peer.offered == 0: + # Don't consider for unpeering until at + # least one message has been offered + pass + else: + waiting_peers.append(peer) + else: + unresponsive_peers.append(peer) + + drop_pool = [] + if len(unresponsive_peers) > 0: + drop_pool.extend(unresponsive_peers) + if not self.prioritise_rotating_unreachable_peers: + drop_pool.extend(waiting_peers) + + else: + drop_pool.extend(waiting_peers) + + if len(drop_pool) > 0: + drop_count = min(required_drops, len(drop_pool)) + low_acceptance_rate_peers = sorted( + drop_pool, + key=lambda p: ( 0 if p.offered == 0 else (p.outgoing/p.offered) ), + reverse=False + )[0:drop_count] + + ms = "" if len(low_acceptance_rate_peers) == 1 else "s" + RNS.log(f"Dropping {len(low_acceptance_rate_peers)} lowest acceptance rate peer{ms} to increase peering headroom", RNS.LOG_DEBUG) + for peer in low_acceptance_rate_peers: + ar = 0 if peer.offered == 0 else round((peer.outgoing/peer.offered)*100, 2) + RNS.log(f"Acceptance rate for {RNS.prettyhexrep(peer.destination_hash)} was: {ar}% ({peer.outgoing} / {peer.offered})", RNS.LOG_DEBUG) + self.unpeer(peer.destination_hash) + + except Exception as e: + RNS.log(f"An error occurred during peer rotation: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + def sync_peers(self): culled_peers = [] waiting_peers = [] From 40fc75f5593aee19563ed9403170c7b1c938e7fd Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 29 Jan 2025 14:24:09 +0100 Subject: [PATCH 301/436] Refined peer rotation algorithm --- LXMF/LXMRouter.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 4bbd24c..1d8f50e 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -763,6 +763,7 @@ class LXMRouter: JOB_STORE_INTERVAL = 120 JOB_PEERSYNC_INTERVAL = 12 JOB_PEERINGEST_INTERVAL= JOB_PEERSYNC_INTERVAL + JOB_ROTATE_INTERVAL = 675 def jobs(self): if not self.exit_handler_running: self.processing_count += 1 @@ -785,9 +786,12 @@ class LXMRouter: if self.processing_count % LXMRouter.JOB_PEERINGEST_INTERVAL == 0: if self.propagation_node == True: - self.rotate_peers() self.flush_queues() + if self.processing_count % LXMRouter.JOB_ROTATE_INTERVAL == 0: + if self.propagation_node == True: + self.rotate_peers() + if self.processing_count % LXMRouter.JOB_PEERSYNC_INTERVAL == 0: if self.propagation_node == True: self.sync_peers() @@ -1819,6 +1823,17 @@ class LXMRouter: RNS.log("Newly added peer threshold reached, postponing peer rotation", RNS.LOG_DEBUG) return + fully_synced_peers = {} + for peer_id in peers: + peer = peers[peer_id] + if peer.unhandled_message_count == 0: + fully_synced_peers[peer_id] = peer + + if len(fully_synced_peers) > 0: + peers = fully_synced_peers + ms = "" if len(fully_synced_peers) == 1 else "s" + RNS.log(f"Found {len(fully_synced_peers)} fully synced peer{ms}, using as peer rotation pool basis", RNS.LOG_DEBUG) + culled_peers = [] waiting_peers = [] unresponsive_peers = [] @@ -1856,7 +1871,8 @@ class LXMRouter: RNS.log(f"Dropping {len(low_acceptance_rate_peers)} lowest acceptance rate peer{ms} to increase peering headroom", RNS.LOG_DEBUG) for peer in low_acceptance_rate_peers: ar = 0 if peer.offered == 0 else round((peer.outgoing/peer.offered)*100, 2) - RNS.log(f"Acceptance rate for {RNS.prettyhexrep(peer.destination_hash)} was: {ar}% ({peer.outgoing} / {peer.offered})", RNS.LOG_DEBUG) + reachable_str = "reachable" if peer.alive else "unreachable" + RNS.log(f"Acceptance rate for {reachable_str} peer {RNS.prettyhexrep(peer.destination_hash)} was: {ar}% ({peer.outgoing}/{peer.offered}, {peer.unhandled_message_count} unhandled messages)", RNS.LOG_DEBUG) self.unpeer(peer.destination_hash) except Exception as e: From 40d0b9a5deca5fb054946dfcf37d2442bdac4469 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 29 Jan 2025 21:21:51 +0100 Subject: [PATCH 302/436] Added acceptance rate threshold to peer rotation --- LXMF/LXMRouter.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 1d8f50e..df340d2 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -42,6 +42,7 @@ class LXMRouter: AUTOPEER_MAXDEPTH = 4 FASTEST_N_RANDOM_POOL = 2 ROTATION_HEADROOM_PCT = 10 + ROTATION_AR_MAX = 0.5 PROPAGATION_LIMIT = 256 DELIVERY_LIMIT = 1000 @@ -1867,13 +1868,16 @@ class LXMRouter: reverse=False )[0:drop_count] - ms = "" if len(low_acceptance_rate_peers) == 1 else "s" - RNS.log(f"Dropping {len(low_acceptance_rate_peers)} lowest acceptance rate peer{ms} to increase peering headroom", RNS.LOG_DEBUG) + dropped_peers = 0 for peer in low_acceptance_rate_peers: ar = 0 if peer.offered == 0 else round((peer.outgoing/peer.offered)*100, 2) - reachable_str = "reachable" if peer.alive else "unreachable" - RNS.log(f"Acceptance rate for {reachable_str} peer {RNS.prettyhexrep(peer.destination_hash)} was: {ar}% ({peer.outgoing}/{peer.offered}, {peer.unhandled_message_count} unhandled messages)", RNS.LOG_DEBUG) - self.unpeer(peer.destination_hash) + if ar < LXMRouter.ROTATION_AR_MAX*100: + reachable_str = "reachable" if peer.alive else "unreachable" + RNS.log(f"Acceptance rate for {reachable_str} peer {RNS.prettyhexrep(peer.destination_hash)} was: {ar}% ({peer.outgoing}/{peer.offered}, {peer.unhandled_message_count} unhandled messages)", RNS.LOG_DEBUG) + self.unpeer(peer.destination_hash) + + ms = "" if dropped_peers == 1 else "s" + RNS.log(f"Dropped {dropped_peers} low acceptance rate peer{ms} to increase peering headroom", RNS.LOG_DEBUG) except Exception as e: RNS.log(f"An error occurred during peer rotation: {e}", RNS.LOG_ERROR) From b7b67536400e768658dcc9cf63406ccf9baba468 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Jan 2025 00:37:50 +0100 Subject: [PATCH 303/436] Fixed potential division by zero. Fixes #25. --- LXMF/LXMRouter.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index df340d2..7bb44a5 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -507,8 +507,10 @@ class LXMRouter: except Exception as e: RNS.log("Could not read LXM from message store. The contained exception was: "+str(e), RNS.LOG_ERROR) - et = time.time(); RNS.log(f"Indexed {len(self.propagation_entries)} messages in {RNS.prettytime(et-st)}, {math.floor(len(self.propagation_entries)/(et-st))} msgs/s", RNS.LOG_NOTICE) - st = time.time(); RNS.log("Rebuilding peer synchronisation states...", RNS.LOG_NOTICE) + et = time.time(); mps = 0 if et-st == 0 else math.floor(len(self.propagation_entries)/(et-st)) + RNS.log(f"Indexed {len(self.propagation_entries)} messages in {RNS.prettytime(et-st)}, {mps} msgs/s", RNS.LOG_NOTICE) + RNS.log("Rebuilding peer synchronisation states...", RNS.LOG_NOTICE) + st = time.time(); if os.path.isfile(self.storagepath+"/peers"): peers_file = open(self.storagepath+"/peers", "rb") @@ -1875,6 +1877,7 @@ class LXMRouter: reachable_str = "reachable" if peer.alive else "unreachable" RNS.log(f"Acceptance rate for {reachable_str} peer {RNS.prettyhexrep(peer.destination_hash)} was: {ar}% ({peer.outgoing}/{peer.offered}, {peer.unhandled_message_count} unhandled messages)", RNS.LOG_DEBUG) self.unpeer(peer.destination_hash) + dropped_peers += 1 ms = "" if dropped_peers == 1 else "s" RNS.log(f"Dropped {dropped_peers} low acceptance rate peer{ms} to increase peering headroom", RNS.LOG_DEBUG) From 9eca747757933d283922923c3b598d68a32f7902 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Jan 2025 10:46:31 +0100 Subject: [PATCH 304/436] Updated peer rotation timing to align with distribution queue mapping --- LXMF/LXMRouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 7bb44a5..a364a12 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -766,7 +766,7 @@ class LXMRouter: JOB_STORE_INTERVAL = 120 JOB_PEERSYNC_INTERVAL = 12 JOB_PEERINGEST_INTERVAL= JOB_PEERSYNC_INTERVAL - JOB_ROTATE_INTERVAL = 675 + JOB_ROTATE_INTERVAL = 56*JOB_PEERINGEST_INTERVAL def jobs(self): if not self.exit_handler_running: self.processing_count += 1 From f42ccfc4e93b9a85ed32a6ebc3b5f3ed21a24b49 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Jan 2025 11:23:18 +0100 Subject: [PATCH 305/436] Automatically clean messages exceeding propagation transfer limit for peer from unhandled message queues --- LXMF/LXMPeer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index e2f951a..a777b86 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -250,7 +250,10 @@ class LXMPeer: lxm_size = unhandled_entry[2] next_size = cumulative_size + (lxm_size+per_message_overhead) if self.propagation_transfer_limit != None and next_size > (self.propagation_transfer_limit*1000): - pass + if lxm_size+per_message_overhead > self.propagation_transfer_limit: + RNS.log(f"Message {RNS.prettyhexrep(transient_id)} exceeds transfer limit for {self}, considering handled", RNS.LOG_DEBUG) + self.remove_unhandled_message(transient_id) + self.add_handled_message(transient_id) else: cumulative_size += (lxm_size+per_message_overhead) unhandled_ids.append(transient_id) From b94a712bb626e83755fb54694a2e6a30690957f8 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Jan 2025 11:30:45 +0100 Subject: [PATCH 306/436] Automatically clean messages exceeding propagation transfer limit for peer from unhandled message queues --- LXMF/LXMPeer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index a777b86..a2b6b18 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -250,7 +250,7 @@ class LXMPeer: lxm_size = unhandled_entry[2] next_size = cumulative_size + (lxm_size+per_message_overhead) if self.propagation_transfer_limit != None and next_size > (self.propagation_transfer_limit*1000): - if lxm_size+per_message_overhead > self.propagation_transfer_limit: + if lxm_size+per_message_overhead > (self.propagation_transfer_limit*1000): RNS.log(f"Message {RNS.prettyhexrep(transient_id)} exceeds transfer limit for {self}, considering handled", RNS.LOG_DEBUG) self.remove_unhandled_message(transient_id) self.add_handled_message(transient_id) From 7b4780cfb7537ccc114c9a0d99da7dc6928eb113 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Jan 2025 11:36:11 +0100 Subject: [PATCH 307/436] Automatically clean messages exceeding propagation transfer limit for peer from unhandled message queues --- LXMF/LXMPeer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index a2b6b18..c1294bd 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -251,9 +251,9 @@ class LXMPeer: next_size = cumulative_size + (lxm_size+per_message_overhead) if self.propagation_transfer_limit != None and next_size > (self.propagation_transfer_limit*1000): if lxm_size+per_message_overhead > (self.propagation_transfer_limit*1000): - RNS.log(f"Message {RNS.prettyhexrep(transient_id)} exceeds transfer limit for {self}, considering handled", RNS.LOG_DEBUG) self.remove_unhandled_message(transient_id) self.add_handled_message(transient_id) + RNS.log(f"Message {RNS.prettyhexrep(transient_id)} exceeds transfer limit for {self}, considering handled", RNS.LOG_DEBUG) else: cumulative_size += (lxm_size+per_message_overhead) unhandled_ids.append(transient_id) From c0a8f3be498514611ccb7e44925bf78afd5d71ac Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Jan 2025 15:04:21 +0100 Subject: [PATCH 308/436] Cleanup --- LXMF/Handlers.py | 10 ++++++---- LXMF/LXMRouter.py | 5 +++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index 26a5df6..9876f4c 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -47,10 +47,12 @@ class LXMFPropagationAnnounceHandler: propagation_transfer_limit = None wanted_inbound_peers = None if len(data) >= 4: - try: - wanted_inbound_peers = int(data[3]) - except: - wanted_inbound_peers = None + # TODO: Rethink, probably not necessary anymore + # try: + # wanted_inbound_peers = int(data[3]) + # except: + # wanted_inbound_peers = None + if len(data) >= 3: try: propagation_transfer_limit = float(data[2]) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index a364a12..070dc71 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -286,7 +286,7 @@ class LXMRouter: node_state, # Boolean flag signalling propagation node state int(time.time()), # Current node timebase self.propagation_per_transfer_limit, # Per-transfer limit for message propagation in kilobytes - self.get_wanted_inbound_peers(), # How many more inbound peers this node wants + None, # How many more inbound peers this node wants ] data = msgpack.packb(announce_data) @@ -895,7 +895,8 @@ class LXMRouter: threading.Thread(target=self.save_outbound_stamp_costs, daemon=True).start() def get_wanted_inbound_peers(self): - # TODO: Implement + # TODO: Implement/rethink. + # Probably not necessary anymore. return None def get_announce_app_data(self, destination_hash): From a6cf585109a354554fb223394db3405ed0b9510c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Jan 2025 15:11:26 +0100 Subject: [PATCH 309/436] Cleanup --- LXMF/Handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index 9876f4c..0705074 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -52,6 +52,7 @@ class LXMFPropagationAnnounceHandler: # wanted_inbound_peers = int(data[3]) # except: # wanted_inbound_peers = None + pass if len(data) >= 3: try: From d5540b927fd96d171a096e85efac64a3de921d37 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 31 Jan 2025 13:38:56 +0100 Subject: [PATCH 310/436] Added allow_duplicate option to message ingest API --- LXMF/LXMRouter.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 070dc71..d631841 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1618,7 +1618,7 @@ class LXMRouter: ### Message Routing & Delivery ######################## ####################################################### - def lxmf_delivery(self, lxmf_data, destination_type = None, phy_stats = None, ratchet_id = None, method = None, no_stamp_enforcement=False): + def lxmf_delivery(self, lxmf_data, destination_type = None, phy_stats = None, ratchet_id = None, method = None, no_stamp_enforcement=False, allow_duplicate=False): try: message = LXMessage.unpack_from_bytes(lxmf_data) if ratchet_id and not message.ratchet_id: @@ -1685,7 +1685,7 @@ class LXMRouter: RNS.log(str(self)+" ignored message from "+RNS.prettyhexrep(message.source_hash), RNS.LOG_DEBUG) return False - if self.has_message(message.hash): + if not allow_duplicate and self.has_message(message.hash): RNS.log(str(self)+" ignored already received message from "+RNS.prettyhexrep(message.source_hash), RNS.LOG_DEBUG) return False else: @@ -2107,7 +2107,7 @@ class LXMRouter: if peer != from_peer: peer.queue_unhandled_message(transient_id) - def lxmf_propagation(self, lxmf_data, signal_local_delivery=None, signal_duplicate=None, is_paper_message=False, from_peer=None): + def lxmf_propagation(self, lxmf_data, signal_local_delivery=None, signal_duplicate=None, allow_duplicate=False, is_paper_message=False, from_peer=None): no_stamp_enforcement = False if is_paper_message: no_stamp_enforcement = True @@ -2116,9 +2116,10 @@ class LXMRouter: if len(lxmf_data) >= LXMessage.LXMF_OVERHEAD: transient_id = RNS.Identity.full_hash(lxmf_data) - if not transient_id in self.propagation_entries and not transient_id in self.locally_processed_transient_ids: + if (not transient_id in self.propagation_entries and not transient_id in self.locally_processed_transient_ids) or allow_duplicate == True: received = time.time() destination_hash = lxmf_data[:LXMessage.DESTINATION_LENGTH] + RNS.log("GOT MESSAGE FOR "+RNS.prettyhexrep(destination_hash)) self.locally_processed_transient_ids[transient_id] = received @@ -2128,7 +2129,7 @@ class LXMRouter: decrypted_lxmf_data = delivery_destination.decrypt(encrypted_lxmf_data) if decrypted_lxmf_data != None: delivery_data = lxmf_data[:LXMessage.DESTINATION_LENGTH]+decrypted_lxmf_data - self.lxmf_delivery(delivery_data, delivery_destination.type, ratchet_id=delivery_destination.latest_ratchet_id, method=LXMessage.PROPAGATED, no_stamp_enforcement=no_stamp_enforcement) + self.lxmf_delivery(delivery_data, delivery_destination.type, ratchet_id=delivery_destination.latest_ratchet_id, method=LXMessage.PROPAGATED, no_stamp_enforcement=no_stamp_enforcement, allow_duplicate=allow_duplicate) self.locally_delivered_transient_ids[transient_id] = time.time() if signal_local_delivery != None: @@ -2166,7 +2167,7 @@ class LXMRouter: RNS.trace_exception(e) return False - def ingest_lxm_uri(self, uri, signal_local_delivery=None, signal_duplicate=None): + def ingest_lxm_uri(self, uri, signal_local_delivery=None, signal_duplicate=None, allow_duplicate=False): try: if not uri.lower().startswith(LXMessage.URI_SCHEMA+"://"): RNS.log("Cannot ingest LXM, invalid URI provided.", RNS.LOG_ERROR) @@ -2176,7 +2177,7 @@ class LXMRouter: lxmf_data = base64.urlsafe_b64decode(uri.replace(LXMessage.URI_SCHEMA+"://", "").replace("/", "")+"==") transient_id = RNS.Identity.full_hash(lxmf_data) - router_propagation_result = self.lxmf_propagation(lxmf_data, signal_local_delivery=signal_local_delivery, signal_duplicate=signal_duplicate, is_paper_message=True) + router_propagation_result = self.lxmf_propagation(lxmf_data, signal_local_delivery=signal_local_delivery, signal_duplicate=signal_duplicate, allow_duplicate=allow_duplicate, is_paper_message=True) if router_propagation_result != False: RNS.log("LXM with transient ID "+RNS.prettyhexrep(transient_id)+" was ingested.", RNS.LOG_DEBUG) return router_propagation_result From 1ef46650738f4ddc67579080d2eab60c9affcfa8 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 18 Feb 2025 20:05:19 +0100 Subject: [PATCH 311/436] Cleanup --- LXMF/LXMRouter.py | 1 - requirements.txt | 5 ++--- setup.py | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index d631841..d502ee6 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -2119,7 +2119,6 @@ class LXMRouter: if (not transient_id in self.propagation_entries and not transient_id in self.locally_processed_transient_ids) or allow_duplicate == True: received = time.time() destination_hash = lxmf_data[:LXMessage.DESTINATION_LENGTH] - RNS.log("GOT MESSAGE FOR "+RNS.prettyhexrep(destination_hash)) self.locally_processed_transient_ids[transient_id] = received diff --git a/requirements.txt b/requirements.txt index 6b7926a..2f4f642 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ -qrcode==7.4.2 -rns==0.7.8 -setuptools==70.0.0 +qrcode>=7.4.2 +rns>=0.9.1 diff --git a/setup.py b/setup.py index cabf20a..e01e9eb 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=['rns>=0.9.1'], - python_requires='>=3.7', + install_requires=["rns>=0.9.2"], + python_requires=">=3.7", ) From 570d2c68467e0614609df4dc89d295793e2a4878 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 7 Mar 2025 11:05:50 +0100 Subject: [PATCH 312/436] Added configuration options to default config file --- LXMF/Utilities/lxmd.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index f9a2ef6..48885b2 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -710,6 +710,25 @@ propagation_transfer_max_accepted_size = 256 # prioritise_destinations = 41d20c727598a3fbbdf9106133a3a0ed, d924b81822ca24e68e2effea99bcb8cf +# You can configure the maximum number of other +# propagation nodes that this node will peer +# with automatically. The default is 50. + +# max_peers = 25 + +# You can configure a list of static propagation +# node peers, that this node will always be +# peered with, by specifying a list of +# destination hashes. + +# static_peers = e17f833c4ddf8890dd3a79a6fea8161d, 5a2d0029b6e5ec87020abaea0d746da4 + +# You can configure the propagation node to +# only accept incoming propagation messages +# from configured static peers. + +# from_static_only = True + # By default, any destination is allowed to # connect and download messages, but you can # optionally restrict this. If you enable From 336792c07a47ca60dcf2df1295b8065f0b458a22 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 13 Mar 2025 19:45:15 +0100 Subject: [PATCH 313/436] Updated dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e01e9eb..19737d6 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=0.9.2"], + install_requires=["rns>=0.9.3"], python_requires=">=3.7", ) From 326c0eed8f767e298eb3f09340a41314c254ddda Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 13 Mar 2025 19:46:11 +0100 Subject: [PATCH 314/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 22049ab..63af887 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.6.2" +__version__ = "0.6.3" From e6021b8fed0ffcffcc6a7b9bbdb7d0d1a282ba8f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 15 Apr 2025 20:21:16 +0200 Subject: [PATCH 315/436] Updated license --- LICENSE | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/LICENSE b/LICENSE index a25bd7a..eda392f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ -MIT License +Reticulum License -Copyright (c) 2020 Mark Qvist / unsigned.io +Copyright (c) 2025 Mark Qvist / unsigned.io Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -9,8 +9,16 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +- The Software shall not be used in any kind of system which includes amongst + its functions the ability to purposefully do harm to human beings. + +- The Software shall not be used, directly or indirectly, in the creation of + an artificial intelligence, machine learning or language model training + dataset, including but not limited to any use that contributes to the + training or development of such a model or algorithm. + +- The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, From 1bdcf6ad534c3d34e4966d23bb04a129fd5ebc4f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 15 Apr 2025 20:21:54 +0200 Subject: [PATCH 316/436] Updated license --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index eda392f..f5fb92d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ Reticulum License -Copyright (c) 2025 Mark Qvist / unsigned.io +Copyright (c) 2020-2025 Mark Qvist Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 005d71707c1f99a721e78196e2ecfad1df1c11fa Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 17 Apr 2025 13:31:00 +0200 Subject: [PATCH 317/436] Cleanup --- LXMF/Handlers.py | 14 +++++++------- LXMF/LXMRouter.py | 3 +-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index 0705074..5671170 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -1,4 +1,5 @@ import time +import threading import RNS import RNS.vendor.umsgpack as msgpack @@ -17,10 +18,11 @@ class LXMFDeliveryAnnounceHandler: if lxmessage.method == LXMessage.DIRECT or lxmessage.method == LXMessage.OPPORTUNISTIC: lxmessage.next_delivery_attempt = time.time() - while self.lxmrouter.processing_outbound: - time.sleep(0.1) + def outbound_trigger(): + while self.lxmrouter.processing_outbound: time.sleep(0.1) + self.lxmrouter.process_outbound() - self.lxmrouter.process_outbound() + threading.Thread(target=outbound_trigger, daemon=True).start() try: stamp_cost = stamp_cost_from_app_data(app_data) @@ -55,10 +57,8 @@ class LXMFPropagationAnnounceHandler: pass if len(data) >= 3: - try: - propagation_transfer_limit = float(data[2]) - except: - propagation_transfer_limit = None + try: propagation_transfer_limit = float(data[2]) + except: propagation_transfer_limit = None if destination_hash in self.lxmrouter.static_peers: self.lxmrouter.peer(destination_hash, node_timebase, propagation_transfer_limit, wanted_inbound_peers) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index d502ee6..8da1476 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -2301,8 +2301,7 @@ class LXMRouter: else: RNS.log("Outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - if lxmessage.progress == None or lxmessage.progress < 0.01: - lxmessage.progress = 0.01 + if lxmessage.progress == None or lxmessage.progress < 0.01: lxmessage.progress = 0.01 # Outbound handling for opportunistic messages if lxmessage.method == LXMessage.OPPORTUNISTIC: From 37e99910ec8fb8f1b6df05567aa5ba2009ed9edc Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 12 May 2025 11:58:24 +0200 Subject: [PATCH 318/436] Updated version and RNS dependency version --- LXMF/_version.py | 2 +- setup.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 63af887..49e0fc1 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.6.3" +__version__ = "0.7.0" diff --git a/setup.py b/setup.py index 19737d6..724705f 100644 --- a/setup.py +++ b/setup.py @@ -15,9 +15,10 @@ setuptools.setup( long_description_content_type="text/markdown", url="https://github.com/markqvist/lxmf", packages=["LXMF", "LXMF.Utilities"], + license="Reticulum License", + license_files = ("LICENSE"), classifiers=[ "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], entry_points= { @@ -25,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=0.9.3"], + install_requires=["rns>=0.9.5"], python_requires=">=3.7", ) From 499fe4cc5381ffc641cbc04b7d27cf931cae3bb5 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 15 May 2025 20:27:19 +0200 Subject: [PATCH 319/436] Use no_data_for instead of inactive_for for cleaning links --- LXMF/LXMRouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 8da1476..5b7a5c2 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -827,7 +827,7 @@ class LXMRouter: closed_links = [] for link_hash in self.direct_links: link = self.direct_links[link_hash] - inactive_time = link.inactive_for() + inactive_time = link.no_data_for() if inactive_time > LXMRouter.LINK_MAX_INACTIVITY: link.teardown() From a9622e3a332fd36ffca2b045e42d22c07779266c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 15 May 2025 20:30:12 +0200 Subject: [PATCH 320/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 49e0fc1..a5f830a 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.7.0" +__version__ = "0.7.1" From c2207d1eb79db474a618b2ffd40d6be3ffc2a00a Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 17 May 2025 10:27:21 +0200 Subject: [PATCH 321/436] Added funding --- FUNDING.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 FUNDING.yml diff --git a/FUNDING.yml b/FUNDING.yml new file mode 100644 index 0000000..d125d55 --- /dev/null +++ b/FUNDING.yml @@ -0,0 +1,3 @@ +liberapay: Reticulum +ko_fi: markqvist +custom: "https://unsigned.io/donate" From 787cd069dc98ebab80afdef726a841c45f38566f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 26 May 2025 20:57:46 +0200 Subject: [PATCH 322/436] Fixed division by zero. Closes #30. --- LXMF/Utilities/lxmd.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 48885b2..a4ccaf5 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -529,14 +529,14 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = peered_outgoing += pm["outgoing"] peered_rx_bytes += p["rx_bytes"] peered_tx_bytes += p["tx_bytes"] - if p["alive"]: - available_peers += 1 - else: - unreachable_peers += 1 + + if p["alive"]: available_peers += 1 + else: unreachable_peers += 1 total_incoming = peered_incoming+s["unpeered_propagation_incoming"]+s["clients"]["client_propagation_messages_received"] total_rx_bytes = peered_rx_bytes+s["unpeered_propagation_rx_bytes"] - df = round(peered_outgoing/total_incoming, 2) + if total_incoming != 0: df = round(peered_outgoing/total_incoming, 2) + else: df = 0 dhs = RNS.prettyhexrep(s["destination_hash"]); uts = RNS.prettytime(s["uptime"]) print(f"\nLXMF Propagation Node running on {dhs}, uptime is {uts}") From 416ccf294f37265708f7a703dcdf2daa751fca67 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 13 Jul 2025 13:24:46 +0200 Subject: [PATCH 323/436] Retry path request on unexpectedly closed link --- LXMF/LXMRouter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 5b7a5c2..fe16b05 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -2368,7 +2368,8 @@ class LXMRouter: RNS.log("Waiting for proof for "+str(lxmessage)+" sent as link packet", RNS.LOG_DEBUG) elif direct_link.status == RNS.Link.CLOSED: if direct_link.activated_at != None: - RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was closed", RNS.LOG_DEBUG) + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was closed unexpectedly, retrying path request...", RNS.LOG_DEBUG) + RNS.Transport.request_path(lxmessage.get_destination().hash) else: if not hasattr(lxmessage, "path_request_retried"): RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was never activated, retrying path request...", RNS.LOG_DEBUG) From 5b9f12159326de0f4aee6045f6bd4fa8f9199b2b Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 13 Jul 2025 13:24:52 +0200 Subject: [PATCH 324/436] Updated version and RNS dependency version --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index a5f830a..777f190 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.7.1" +__version__ = "0.8.0" diff --git a/setup.py b/setup.py index 724705f..568ade3 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=0.9.5"], + install_requires=["rns>=1.0.0"], python_requires=">=3.7", ) From 85d8f4f58361c8d523a34626513c1e83d3f3b74a Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 13 Jul 2025 13:42:01 +0200 Subject: [PATCH 325/436] Updated requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2f4f642..f0f3fc8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ qrcode>=7.4.2 -rns>=0.9.1 +rns>=1.0.0 From 61b75526d281a54bbc95a24b0f23b92c1905d951 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 29 Oct 2025 23:24:45 +0100 Subject: [PATCH 326/436] Added separate propagation node per-message and sync transfer limits. Added persistent PN sync strategy. Added concurrent PN peer sync. --- LXMF/Handlers.py | 35 +++++++++----- LXMF/LXMPeer.py | 104 +++++++++++++++++++++++++++++------------ LXMF/LXMRouter.py | 52 +++++++++++++-------- LXMF/Utilities/lxmd.py | 28 +++++++++-- LXMF/_version.py | 2 +- 5 files changed, 153 insertions(+), 68 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index 5671170..e1dba8e 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -41,19 +41,21 @@ class LXMFPropagationAnnounceHandler: def received_announce(self, destination_hash, announced_identity, app_data): try: if type(app_data) == bytes: - if self.lxmrouter.propagation_node and self.lxmrouter.autopeer: + if self.lxmrouter.propagation_node: data = msgpack.unpackb(app_data) - if pn_announce_data_is_valid(data): node_timebase = data[1] propagation_transfer_limit = None + propagation_sync_limit = None wanted_inbound_peers = None + if len(data) >= 5: + try: propagation_sync_limit = int(data[4]) + except Exception as e: propagation_sync_limit = None + if len(data) >= 4: # TODO: Rethink, probably not necessary anymore - # try: - # wanted_inbound_peers = int(data[3]) - # except: - # wanted_inbound_peers = None + # try: wanted_inbound_peers = int(data[3]) + # except: wanted_inbound_peers = None pass if len(data) >= 3: @@ -61,15 +63,24 @@ class LXMFPropagationAnnounceHandler: except: propagation_transfer_limit = None if destination_hash in self.lxmrouter.static_peers: - self.lxmrouter.peer(destination_hash, node_timebase, propagation_transfer_limit, wanted_inbound_peers) + self.lxmrouter.peer(destination_hash=destination_hash, + timestamp=node_timebase, + propagation_transfer_limit=propagation_transfer_limit, + propagation_sync_limit=propagation_sync_limit, + wanted_inbound_peers=wanted_inbound_peers) else: - if data[0] == True: - if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: - self.lxmrouter.peer(destination_hash, node_timebase, propagation_transfer_limit, wanted_inbound_peers) + if self.lxmrouter.autopeer: + if data[0] == True: + if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: + self.lxmrouter.peer(destination_hash=destination_hash, + timestamp=node_timebase, + propagation_transfer_limit=propagation_transfer_limit, + propagation_sync_limit=propagation_sync_limit, + wanted_inbound_peers=wanted_inbound_peers) - elif data[0] == False: - self.lxmrouter.unpeer(destination_hash, node_timebase) + elif data[0] == False: + self.lxmrouter.unpeer(destination_hash, node_timebase) except Exception as e: RNS.log("Error while evaluating propagation node announce, ignoring announce.", RNS.LOG_DEBUG) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index c1294bd..41ea69a 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -11,16 +11,20 @@ class LXMPeer: OFFER_REQUEST_PATH = "/offer" MESSAGE_GET_PATH = "/get" - IDLE = 0x00 - LINK_ESTABLISHING = 0x01 - LINK_READY = 0x02 - REQUEST_SENT = 0x03 - RESPONSE_RECEIVED = 0x04 + IDLE = 0x00 + LINK_ESTABLISHING = 0x01 + LINK_READY = 0x02 + REQUEST_SENT = 0x03 + RESPONSE_RECEIVED = 0x04 RESOURCE_TRANSFERRING = 0x05 - ERROR_NO_IDENTITY = 0xf0 - ERROR_NO_ACCESS = 0xf1 - ERROR_TIMEOUT = 0xfe + ERROR_NO_IDENTITY = 0xf0 + ERROR_NO_ACCESS = 0xf1 + ERROR_TIMEOUT = 0xfe + + STRATEGY_LAZY = 0x01 + STRATEGY_PERSISTENT = 0x02 + DEFAULT_SYNC_STRATEGY = STRATEGY_PERSISTENT # Maximum amount of time a peer can # be unreachable before it is removed @@ -67,6 +71,16 @@ class LXMPeer: peer.propagation_transfer_limit = None else: peer.propagation_transfer_limit = None + + if "propagation_sync_limit" in dictionary: + try: peer.propagation_sync_limit = int(dictionary["propagation_sync_limit"]) + except: peer.propagation_sync_limit = peer.propagation_transfer_limit + else: peer.propagation_sync_limit = peer.propagation_transfer_limit + + if "sync_strategy" in dictionary: + try: peer.sync_strategy = int(dictionary["sync_strategy"]) + except: peer.sync_strategy = LXMPeer.DEFAULT_SYNC_STRATEGY + else: peer.sync_strategy = LXMPeer.DEFAULT_SYNC_STRATEGY if "offered" in dictionary: peer.offered = dictionary["offered"] @@ -127,6 +141,8 @@ class LXMPeer: dictionary["link_establishment_rate"] = self.link_establishment_rate dictionary["sync_transfer_rate"] = self.sync_transfer_rate dictionary["propagation_transfer_limit"] = self.propagation_transfer_limit + dictionary["propagation_sync_limit"] = self.propagation_sync_limit + dictionary["sync_strategy"] = self.sync_strategy dictionary["last_sync_attempt"] = self.last_sync_attempt dictionary["offered"] = self.offered dictionary["outgoing"] = self.outgoing @@ -150,9 +166,11 @@ class LXMPeer: return peer_bytes - def __init__(self, router, destination_hash): + def __init__(self, router, destination_hash, sync_strategy=DEFAULT_SYNC_STRATEGY): self.alive = False self.last_heard = 0 + self.sync_strategy = sync_strategy + self.next_sync_attempt = 0 self.last_sync_attempt = 0 self.sync_backoff = 0 @@ -160,6 +178,8 @@ class LXMPeer: self.link_establishment_rate = 0 self.sync_transfer_rate = 0 self.propagation_transfer_limit = None + self.propagation_sync_limit = None + self.currently_transferring_messages = None self.handled_messages_queue = deque() self.unhandled_messages_queue = deque() @@ -209,6 +229,10 @@ class LXMPeer: if self.destination != None: if len(self.unhandled_messages) > 0: + if self.currently_transferring_messages != None: + RNS.log(f"Sync requested for {self}, but current message transfer index was not clear. Aborting.", RNS.LOG_ERROR) + return + if self.state == LXMPeer.IDLE: RNS.log("Establishing link for sync to peer "+RNS.prettyhexrep(self.destination_hash)+"...", RNS.LOG_DEBUG) self.sync_backoff += LXMPeer.SYNC_BACKOFF_STEP @@ -244,21 +268,26 @@ class LXMPeer: unhandled_entries.sort(key=lambda e: e[1], reverse=False) per_message_overhead = 16 # Really only 2 bytes, but set a bit higher for now cumulative_size = 24 # Initialised to highest reasonable binary structure overhead + RNS.log(f"Syncing to peer with per-message limit {RNS.prettysize(self.propagation_transfer_limit*1000)} and sync limit {RNS.prettysize(self.propagation_sync_limit*1000)}") # TODO: Remove debug for unhandled_entry in unhandled_entries: transient_id = unhandled_entry[0] weight = unhandled_entry[1] lxm_size = unhandled_entry[2] - next_size = cumulative_size + (lxm_size+per_message_overhead) - if self.propagation_transfer_limit != None and next_size > (self.propagation_transfer_limit*1000): - if lxm_size+per_message_overhead > (self.propagation_transfer_limit*1000): - self.remove_unhandled_message(transient_id) - self.add_handled_message(transient_id) - RNS.log(f"Message {RNS.prettyhexrep(transient_id)} exceeds transfer limit for {self}, considering handled", RNS.LOG_DEBUG) - else: - cumulative_size += (lxm_size+per_message_overhead) - unhandled_ids.append(transient_id) + lxm_transfer_size = lxm_size+per_message_overhead + next_size = cumulative_size + lxm_transfer_size - RNS.log(f"Offering {len(unhandled_ids)} messages to peer {RNS.prettyhexrep(self.destination.hash)}", RNS.LOG_VERBOSE) + if self.propagation_transfer_limit != None and lxm_transfer_size > (self.propagation_transfer_limit*1000): + self.remove_unhandled_message(transient_id) + self.add_handled_message(transient_id) + continue + + if self.propagation_sync_limit != None and next_size >= (self.propagation_sync_limit*1000): + continue + + cumulative_size += lxm_transfer_size + unhandled_ids.append(transient_id) + + RNS.log(f"Offering {len(unhandled_ids)} messages to peer {RNS.prettyhexrep(self.destination.hash)} ({RNS.prettysize(len(msgpack.packb(unhandled_ids)))})", RNS.LOG_VERBOSE) self.last_offer = unhandled_ids self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) self.state = LXMPeer.REQUEST_SENT @@ -327,7 +356,7 @@ class LXMPeer: wanted_message_ids.append(transient_id) if len(wanted_messages) > 0: - RNS.log("Peer wanted "+str(len(wanted_messages))+" of the available messages", RNS.LOG_VERBOSE) + RNS.log(f"Peer {RNS.prettyhexrep(self.destination_hash)} wanted {str(len(wanted_messages))} of the available messages", RNS.LOG_VERBOSE) lxm_list = [] for message_entry in wanted_messages: @@ -339,13 +368,14 @@ class LXMPeer: lxm_list.append(lxmf_data) data = msgpack.packb([time.time(), lxm_list]) + RNS.log(f"Total transfer size for this sync is {RNS.prettysize(len(data))}", RNS.LOG_VERBOSE) resource = RNS.Resource(data, self.link, callback = self.resource_concluded) - resource.transferred_messages = wanted_message_ids - resource.sync_transfer_started = time.time() + self.currently_transferring_messages = wanted_message_ids + self.current_sync_transfer_started = time.time() self.state = LXMPeer.RESOURCE_TRANSFERRING else: - RNS.log("Peer "+RNS.prettyhexrep(self.destination_hash)+" did not request any of the available messages, sync completed", RNS.LOG_VERBOSE) + RNS.log(f"Peer {RNS.prettyhexrep(self.destination_hash)} did not request any of the available messages, sync completed", RNS.LOG_VERBOSE) self.offered += len(self.last_offer) if self.link != None: self.link.teardown() @@ -365,7 +395,13 @@ class LXMPeer: def resource_concluded(self, resource): if resource.status == RNS.Resource.COMPLETE: - for transient_id in resource.transferred_messages: + if self.currently_transferring_messages == None: + RNS.log(f"Sync transfer completed on {self}, but transferred message index was unavailable. Aborting.", RNS.LOG_ERROR) + if self.link != None: self.link.teardown() + self.link = None + self.state = LXMPeer.IDLE + + for transient_id in self.currently_transferring_messages: self.add_handled_message(transient_id) self.remove_unhandled_message(transient_id) @@ -376,24 +412,30 @@ class LXMPeer: self.state = LXMPeer.IDLE rate_str = "" - if hasattr(resource, "sync_transfer_started") and resource.sync_transfer_started: - self.sync_transfer_rate = (resource.get_transfer_size()*8)/(time.time()-resource.sync_transfer_started) + if self.current_sync_transfer_started != None: + self.sync_transfer_rate = (resource.get_transfer_size()*8)/(time.time()-self.current_sync_transfer_started) rate_str = f" at {RNS.prettyspeed(self.sync_transfer_rate)}" - RNS.log(f"Syncing {len(resource.transferred_messages)} messages to peer {RNS.prettyhexrep(self.destination_hash)} completed{rate_str}", RNS.LOG_VERBOSE) + RNS.log(f"Syncing {len(self.currently_transferring_messages)} messages to peer {RNS.prettyhexrep(self.destination_hash)} completed{rate_str}", RNS.LOG_VERBOSE) self.alive = True self.last_heard = time.time() self.offered += len(self.last_offer) - self.outgoing += len(resource.transferred_messages) + self.outgoing += len(self.currently_transferring_messages) self.tx_bytes += resource.get_data_size() + + self.currently_transferring_messages = None + self.current_sync_transfer_started = None + + if self.sync_strategy == self.STRATEGY_PERSISTENT: + if self.unhandled_message_count > 0: self.sync() else: RNS.log("Resource transfer for LXMF peer sync failed to "+str(self.destination), RNS.LOG_VERBOSE) - if self.link != None: - self.link.teardown() - + if self.link != None: self.link.teardown() self.link = None self.state = LXMPeer.IDLE + self.currently_transferring_messages = None + self.current_sync_transfer_started = None def link_established(self, link): self.link.identify(self.router.identity) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index fe16b05..0ad75b7 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -45,6 +45,7 @@ class LXMRouter: ROTATION_AR_MAX = 0.5 PROPAGATION_LIMIT = 256 + SYNC_LIMIT = PROPAGATION_LIMIT*40 DELIVERY_LIMIT = 1000 PR_PATH_TIMEOUT = 10 @@ -73,8 +74,9 @@ class LXMRouter: ####################################################### def __init__(self, identity=None, storagepath=None, autopeer=AUTOPEER, autopeer_maxdepth=None, - propagation_limit=PROPAGATION_LIMIT, delivery_limit=DELIVERY_LIMIT, enforce_ratchets=False, - enforce_stamps=False, static_peers = [], max_peers=None, from_static_only=False): + propagation_limit=PROPAGATION_LIMIT, delivery_limit=DELIVERY_LIMIT, sync_limit=SYNC_LIMIT, + enforce_ratchets=False, enforce_stamps=False, static_peers = [], max_peers=None, + from_static_only=False, sync_strategy=LXMPeer.STRATEGY_PERSISTENT): random.seed(os.urandom(10)) @@ -91,9 +93,10 @@ class LXMRouter: self.auth_required = False self.retain_synced_on_node = False - self.processing_outbound = False - self.processing_inbound = False - self.processing_count = 0 + self.default_sync_strategy = sync_strategy + self.processing_outbound = False + self.processing_inbound = False + self.processing_count = 0 self.propagation_node = False self.propagation_node_start_time = None @@ -107,17 +110,20 @@ class LXMRouter: self.outbound_propagation_node = None self.outbound_propagation_link = None - if delivery_limit == None: - delivery_limit = LXMRouter.DELIVERY_LIMIT + if delivery_limit == None: delivery_limit = LXMRouter.DELIVERY_LIMIT self.message_storage_limit = None self.information_storage_limit = None self.propagation_per_transfer_limit = propagation_limit + self.propagation_per_sync_limit = sync_limit self.delivery_per_transfer_limit = delivery_limit self.enforce_ratchets = enforce_ratchets self._enforce_stamps = enforce_stamps self.pending_deferred_stamps = {} + if sync_limit == None or self.propagation_per_sync_limit < self.propagation_per_transfer_limit: + self.propagation_per_sync_limit = self.propagation_per_transfer_limit + self.wants_download_on_path_available_from = None self.wants_download_on_path_available_to = None self.propagation_transfer_state = LXMRouter.PR_IDLE @@ -287,6 +293,7 @@ class LXMRouter: int(time.time()), # Current node timebase self.propagation_per_transfer_limit, # Per-transfer limit for message propagation in kilobytes None, # How many more inbound peers this node wants + self.propagation_per_sync_limit, # Limit for incoming propagation node syncs ] data = msgpack.packb(announce_data) @@ -546,7 +553,7 @@ class LXMRouter: for static_peer in self.static_peers: if not static_peer in self.peers: RNS.log(f"Activating static peering with {RNS.prettyhexrep(static_peer)}", RNS.LOG_NOTICE) - self.peers[static_peer] = LXMPeer(self, static_peer) + self.peers[static_peer] = LXMPeer(self, static_peer, sync_strategy=self.default_sync_strategy) if self.peers[static_peer].last_heard == 0: # TODO: Allow path request responses through announce handler # momentarily here, so peering config can be updated even if @@ -708,6 +715,7 @@ class LXMRouter: "ler": int(peer.link_establishment_rate), "str": int(peer.sync_transfer_rate), "transfer_limit": peer.propagation_transfer_limit, + "sync_limit": peer.propagation_sync_limit, "network_distance": RNS.Transport.hops_to(peer_id), "rx_bytes": peer.rx_bytes, "tx_bytes": peer.tx_bytes, @@ -725,6 +733,7 @@ class LXMRouter: "uptime": time.time()-self.propagation_node_start_time, "delivery_limit": self.delivery_per_transfer_limit, "propagation_limit": self.propagation_per_transfer_limit, + "sync_limit": self.propagation_per_sync_limit, "autopeer_maxdepth": self.autopeer_maxdepth, "from_static_only": self.from_static_only, "messagestore": { @@ -1777,7 +1786,7 @@ class LXMRouter: ### Peer Sync & Propagation ########################### ####################################################### - def peer(self, destination_hash, timestamp, propagation_transfer_limit, wanted_inbound_peers = None): + def peer(self, destination_hash, timestamp, propagation_transfer_limit, propagation_sync_limit, wanted_inbound_peers = None): if destination_hash in self.peers: peer = self.peers[destination_hash] if timestamp > peer.peering_timebase: @@ -1787,16 +1796,23 @@ class LXMRouter: peer.peering_timebase = timestamp peer.last_heard = time.time() peer.propagation_transfer_limit = propagation_transfer_limit + if propagation_sync_limit != None: peer.propagation_sync_limit = propagation_sync_limit + else: peer.propagation_sync_limit = propagation_transfer_limit + RNS.log(f"Peering config updated for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_VERBOSE) else: if len(self.peers) < self.max_peers: - peer = LXMPeer(self, destination_hash) + peer = LXMPeer(self, destination_hash, sync_strategy=self.default_sync_strategy) peer.alive = True peer.last_heard = time.time() peer.propagation_transfer_limit = propagation_transfer_limit + if propagation_sync_limit != None: peer.propagation_sync_limit = propagation_sync_limit + else: peer.propagation_sync_limit = propagation_transfer_limit + self.peers[destination_hash] = peer RNS.log(f"Peered with {RNS.prettyhexrep(destination_hash)}", RNS.LOG_NOTICE) + else: RNS.log(f"Max peers reached, not peering with {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) @@ -1895,18 +1911,14 @@ class LXMRouter: for peer_id in peers: peer = peers[peer_id] if time.time() > peer.last_heard + LXMPeer.MAX_UNREACHABLE: - if not peer_id in self.static_peers: - culled_peers.append(peer_id) + if not peer_id in self.static_peers: culled_peers.append(peer_id) + else: if peer.state == LXMPeer.IDLE and len(peer.unhandled_messages) > 0: - if peer.alive: - waiting_peers.append(peer) + if peer.alive: waiting_peers.append(peer) else: - if hasattr(peer, "next_sync_attempt") and time.time() > peer.next_sync_attempt: - unresponsive_peers.append(peer) - else: - pass - # RNS.log("Not adding peer "+str(peer)+" since it is in sync backoff", RNS.LOG_DEBUG) + if hasattr(peer, "next_sync_attempt") and time.time() > peer.next_sync_attempt: unresponsive_peers.append(peer) + else: pass # RNS.log("Not adding peer "+str(peer)+" since it is in sync backoff", RNS.LOG_DEBUG) peer_pool = [] if len(waiting_peers) > 0: @@ -1970,7 +1982,7 @@ class LXMRouter: return False size = resource.get_data_size() - limit = self.propagation_per_transfer_limit*1000 + limit = self.propagation_per_sync_limit*1000 if limit != None and size > limit: RNS.log(f"Rejecting {RNS.prettysize(size)} incoming propagation resource, since it exceeds the limit of {RNS.prettysize(limit)}", RNS.LOG_DEBUG) return False diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index a4ccaf5..e49bd7a 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -135,6 +135,20 @@ def apply_config(): active_configuration["propagation_transfer_max_accepted_size"] = 0.38 else: active_configuration["propagation_transfer_max_accepted_size"] = 256 + + if "propagation" in lxmd_config and "propagation_message_max_accepted_size" in lxmd_config["propagation"]: + active_configuration["propagation_transfer_max_accepted_size"] = lxmd_config["propagation"].as_float("propagation_message_max_accepted_size") + if active_configuration["propagation_transfer_max_accepted_size"] < 0.38: + active_configuration["propagation_transfer_max_accepted_size"] = 0.38 + else: + active_configuration["propagation_transfer_max_accepted_size"] = 256 + + if "propagation" in lxmd_config and "propagation_sync_max_accepted_size" in lxmd_config["propagation"]: + active_configuration["propagation_sync_max_accepted_size"] = lxmd_config["propagation"].as_float("propagation_sync_max_accepted_size") + if active_configuration["propagation_sync_max_accepted_size"] < 0.38: + active_configuration["propagation_sync_max_accepted_size"] = 0.38 + else: + active_configuration["propagation_sync_max_accepted_size"] = 256*40 if "propagation" in lxmd_config and "prioritise_destinations" in lxmd_config["propagation"]: active_configuration["prioritised_lxmf_destinations"] = lxmd_config["propagation"].as_list("prioritise_destinations") @@ -323,6 +337,7 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo autopeer = active_configuration["autopeer"], autopeer_maxdepth = active_configuration["autopeer_maxdepth"], propagation_limit = active_configuration["propagation_transfer_max_accepted_size"], + sync_limit = active_configuration["propagation_sync_max_accepted_size"], delivery_limit = active_configuration["delivery_transfer_max_accepted_size"], max_peers = active_configuration["max_peers"], static_peers = active_configuration["static_peers"], @@ -676,9 +691,14 @@ autopeer = yes autopeer_maxdepth = 4 # The maximum accepted transfer size per in- -# coming propagation transfer, in kilobytes. -# This also sets the upper limit for the size -# of single messages accepted onto this node. +# coming propagation message, in kilobytes. +# This sets the upper limit for the size of +# single messages accepted onto this node. + +propagation_message_max_accepted_size = 256 + +# The maximum accepted transfer size per in- +# coming propagation node sync. # # If a node wants to propagate a larger number # of messages to this node, than what can fit @@ -686,7 +706,7 @@ autopeer_maxdepth = 4 # the smallest messages first, and try again # with any remaining messages at a later point. -propagation_transfer_max_accepted_size = 256 +propagation_sync_max_accepted_size = 256 # The maximum amount of storage to use for # the LXMF Propagation Node message store, diff --git a/LXMF/_version.py b/LXMF/_version.py index 777f190..8088f75 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.8.0" +__version__ = "0.8.1" From 98347d3ad9d4d30979fa52564632a73ee8262872 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 29 Oct 2025 23:25:15 +0100 Subject: [PATCH 327/436] Increased PN peer sync frequency --- LXMF/LXMRouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 0ad75b7..392a76f 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -773,7 +773,7 @@ class LXMRouter: JOB_LINKS_INTERVAL = 1 JOB_TRANSIENT_INTERVAL = 60 JOB_STORE_INTERVAL = 120 - JOB_PEERSYNC_INTERVAL = 12 + JOB_PEERSYNC_INTERVAL = 6 JOB_PEERINGEST_INTERVAL= JOB_PEERSYNC_INTERVAL JOB_ROTATE_INTERVAL = 56*JOB_PEERINGEST_INTERVAL def jobs(self): From 16dfbc22cd0441937d0e2e2ceebf1f2cb6dd281b Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Oct 2025 12:38:49 +0100 Subject: [PATCH 328/436] Propagation stamp validation via pool dispatch --- LXMF/LXMessage.py | 11 +-------- LXMF/LXStamper.py | 58 +++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 14 deletions(-) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 515ab11..4739f30 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -268,15 +268,6 @@ class LXMessage: def register_failed_callback(self, callback): self.failed_callback = callback - @staticmethod - def stamp_valid(stamp, target_cost, workblock): - target = 0b1 << 256-target_cost - result = RNS.Identity.full_hash(workblock+stamp) - if int.from_bytes(result, byteorder="big") > target: - return False - else: - return True - def validate_stamp(self, target_cost, tickets=None): if tickets != None: for ticket in tickets: @@ -293,7 +284,7 @@ class LXMessage: return False else: workblock = LXStamper.stamp_workblock(self.message_id) - if LXMessage.stamp_valid(self.stamp, target_cost, workblock): + if LXStamper.stamp_valid(self.stamp, target_cost, workblock): RNS.log(f"Stamp on {self} validated", RNS.LOG_DEBUG) # TODO: Remove at some point self.stamp_value = LXStamper.stamp_value(workblock, self.stamp) return True diff --git a/LXMF/LXStamper.py b/LXMF/LXStamper.py index bcfa95b..a9ca7d6 100644 --- a/LXMF/LXStamper.py +++ b/LXMF/LXStamper.py @@ -3,15 +3,18 @@ import RNS.vendor.umsgpack as msgpack import os import time +import math import multiprocessing -WORKBLOCK_EXPAND_ROUNDS = 3000 +WORKBLOCK_EXPAND_ROUNDS = 3000 +WORKBLOCK_EXPAND_ROUNDS_PN = 1000 +STAMP_SIZE = RNS.Identity.HASHLENGTH +PN_VALIDATION_POOL_MIN_SIZE = 256 active_jobs = {} -def stamp_workblock(message_id): +def stamp_workblock(message_id, expand_rounds=WORKBLOCK_EXPAND_ROUNDS): wb_st = time.time() - expand_rounds = WORKBLOCK_EXPAND_ROUNDS workblock = b"" for n in range(expand_rounds): workblock += RNS.Cryptography.hkdf( @@ -21,7 +24,7 @@ def stamp_workblock(message_id): context=None, ) wb_time = time.time() - wb_st - RNS.log(f"Stamp workblock size {RNS.prettysize(len(workblock))}, generated in {round(wb_time*1000,2)}ms", RNS.LOG_DEBUG) + # RNS.log(f"Stamp workblock size {RNS.prettysize(len(workblock))}, generated in {round(wb_time*1000,2)}ms", RNS.LOG_DEBUG) return workblock @@ -36,6 +39,53 @@ def stamp_value(workblock, stamp): return value +def stamp_valid(stamp, target_cost, workblock): + target = 0b1 << 256-target_cost + result = RNS.Identity.full_hash(workblock+stamp) + if int.from_bytes(result, byteorder="big") > target: return False + else: return True + +def validate_pn_stamp(transient_id, stamp): + target_cost = 8 + workblock = stamp_workblock(transient_id, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PN) + if stamp_valid(stamp, target_cost, workblock): + RNS.log(f"Stamp on {RNS.prettyhexrep(transient_id)} validated", RNS.LOG_DEBUG) + value = stamp_value(workblock, stamp) + return True + + return False + +def validate_pn_stamps_job_simple(transient_stamps): + for entry in transient_stamps: + # Get transient ID and stamp for validation + transient_id = transient_stamps[0] + stamp = transient_stamps[1] + + # Store validation result back into list + transient_stamps[2] = validate_pn_stamp(transient_id, stamp) + + return transient_stamps + +def _validate_single_pn_stamp_entry(entry): + transient_id = entry[0] + stamp = entry[1] + entry[2] = validate_pn_stamp(transient_id, stamp) + return entry + +def validate_pn_stamps_job_multip(transient_stamps): + cores = multiprocessing.cpu_count() + pool_count = min(cores, math.ceil(len(transient_stamps) / PN_VALIDATION_POOL_MIN_SIZE)) + + RNS.log(f"Validating {len(transient_stamps)} stamps using {pool_count} processes...") + with multiprocessing.Pool(pool_count) as p: validated_entries = p.map(_validate_single_pn_stamp_entry, transient_stamps) + + return validated_entries + +def validate_pn_stamps(transient_stamps): + non_mp_platform = RNS.vendor.platformutils.is_android() + if len(transient_stamps) <= PN_VALIDATION_POOL_MIN_SIZE or non_mp_platform: validate_pn_stamps_job_simple(transient_stamps) + else: validate_pn_stamps_job_multip(transient_stamps) + def generate_stamp(message_id, stamp_cost): RNS.log(f"Generating stamp with cost {stamp_cost} for {RNS.prettyhexrep(message_id)}...", RNS.LOG_DEBUG) workblock = stamp_workblock(message_id) From e17263d25a7c4022f0c57f1eefb54b989c139ec5 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Oct 2025 12:41:18 +0100 Subject: [PATCH 329/436] Cleanup --- LXMF/LXMRouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 392a76f..30abaa6 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -517,7 +517,7 @@ class LXMRouter: et = time.time(); mps = 0 if et-st == 0 else math.floor(len(self.propagation_entries)/(et-st)) RNS.log(f"Indexed {len(self.propagation_entries)} messages in {RNS.prettytime(et-st)}, {mps} msgs/s", RNS.LOG_NOTICE) RNS.log("Rebuilding peer synchronisation states...", RNS.LOG_NOTICE) - st = time.time(); + st = time.time() if os.path.isfile(self.storagepath+"/peers"): peers_file = open(self.storagepath+"/peers", "rb") From 6cf785227101928a9f073be8a38ea6c86caf6be5 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Oct 2025 13:14:59 +0100 Subject: [PATCH 330/436] Cleanup --- LXMF/LXMF.py | 40 +++++++++++++--------------------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index db0edb7..93e283f 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -99,21 +99,17 @@ RENDERER_BBCODE = 0x03 import RNS import RNS.vendor.umsgpack as msgpack def display_name_from_app_data(app_data=None): - if app_data == None: - return None - elif len(app_data) == 0: - return None + if app_data == None: return None + elif len(app_data) == 0: return None else: # Version 0.5.0+ announce format if (app_data[0] >= 0x90 and app_data[0] <= 0x9f) or app_data[0] == 0xdc: peer_data = msgpack.unpackb(app_data) if type(peer_data) == list: - if len(peer_data) < 1: - return None + if len(peer_data) < 1: return None else: dn = peer_data[0] - if dn == None: - return None + if dn == None: return None else: try: decoded = dn.decode("utf-8") @@ -127,36 +123,26 @@ def display_name_from_app_data(app_data=None): return app_data.decode("utf-8") def stamp_cost_from_app_data(app_data=None): - if app_data == None or app_data == b"": - return None + if app_data == None or app_data == b"": return None else: # Version 0.5.0+ announce format if (app_data[0] >= 0x90 and app_data[0] <= 0x9f) or app_data[0] == 0xdc: peer_data = msgpack.unpackb(app_data) if type(peer_data) == list: - if len(peer_data) < 2: - return None - else: - return peer_data[1] + if len(peer_data) < 2: return None + else: return peer_data[1] # Original announce format - else: - return None + else: return None def pn_announce_data_is_valid(data): try: - if type(data) == bytes: - data = msgpack.unpackb(data) - - if len(data) < 3: - raise ValueError("Invalid announce data: Insufficient peer data") + if type(data) == bytes: data = msgpack.unpackb(data) + if len(data) < 3: raise ValueError("Invalid announce data: Insufficient peer data") else: - if data[0] != True and data[0] != False: - raise ValueError("Invalid announce data: Indeterminate propagation node status") - try: - int(data[1]) - except: - raise ValueError("Invalid announce data: Could not decode peer timebase") + if data[0] != True and data[0] != False: raise ValueError("Invalid announce data: Indeterminate propagation node status") + try: int(data[1]) + except: raise ValueError("Invalid announce data: Could not decode peer timebase") except Exception as e: RNS.log(f"Could not validate propagation node announce data: {e}", RNS.LOG_DEBUG) From b572723a5e293df59cb6e6608d96571a547088ba Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Oct 2025 13:49:26 +0100 Subject: [PATCH 331/436] Cleanup --- LXMF/LXMPeer.py | 66 ++++++++++++++++--------------------------------- 1 file changed, 21 insertions(+), 45 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 41ea69a..3a9fc00 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -54,63 +54,39 @@ class LXMPeer: peer.alive = peer_alive peer.last_heard = peer_last_heard - if "link_establishment_rate" in dictionary: - peer.link_establishment_rate = dictionary["link_establishment_rate"] - else: - peer.link_establishment_rate = 0 + if "link_establishment_rate" in dictionary: peer.link_establishment_rate = dictionary["link_establishment_rate"] + else: peer.link_establishment_rate = 0 - if "sync_transfer_rate" in dictionary: - peer.sync_transfer_rate = dictionary["sync_transfer_rate"] - else: - peer.sync_transfer_rate = 0 + if "sync_transfer_rate" in dictionary: peer.sync_transfer_rate = dictionary["sync_transfer_rate"] + else: peer.sync_transfer_rate = 0 if "propagation_transfer_limit" in dictionary: - try: - peer.propagation_transfer_limit = float(dictionary["propagation_transfer_limit"]) - except Exception as e: - peer.propagation_transfer_limit = None - else: - peer.propagation_transfer_limit = None + try: peer.propagation_transfer_limit = float(dictionary["propagation_transfer_limit"]) + except Exception as e: peer.propagation_transfer_limit = None + else: peer.propagation_transfer_limit = None if "propagation_sync_limit" in dictionary: - try: peer.propagation_sync_limit = int(dictionary["propagation_sync_limit"]) + try: peer.propagation_sync_limit = int(dictionary["propagation_sync_limit"]) except: peer.propagation_sync_limit = peer.propagation_transfer_limit - else: peer.propagation_sync_limit = peer.propagation_transfer_limit + else: peer.propagation_sync_limit = peer.propagation_transfer_limit if "sync_strategy" in dictionary: try: peer.sync_strategy = int(dictionary["sync_strategy"]) except: peer.sync_strategy = LXMPeer.DEFAULT_SYNC_STRATEGY else: peer.sync_strategy = LXMPeer.DEFAULT_SYNC_STRATEGY - if "offered" in dictionary: - peer.offered = dictionary["offered"] - else: - peer.offered = 0 - - if "outgoing" in dictionary: - peer.outgoing = dictionary["outgoing"] - else: - peer.outgoing = 0 - - if "incoming" in dictionary: - peer.incoming = dictionary["incoming"] - else: - peer.incoming = 0 - - if "rx_bytes" in dictionary: - peer.rx_bytes = dictionary["rx_bytes"] - else: - peer.rx_bytes = 0 - - if "tx_bytes" in dictionary: - peer.tx_bytes = dictionary["tx_bytes"] - else: - peer.tx_bytes = 0 - - if "last_sync_attempt" in dictionary: - peer.last_sync_attempt = dictionary["last_sync_attempt"] - else: - peer.last_sync_attempt = 0 + if "offered" in dictionary: peer.offered = dictionary["offered"] + else: peer.offered = 0 + if "outgoing" in dictionary: peer.outgoing = dictionary["outgoing"] + else: peer.outgoing = 0 + if "incoming" in dictionary: peer.incoming = dictionary["incoming"] + else: peer.incoming = 0 + if "rx_bytes" in dictionary: peer.rx_bytes = dictionary["rx_bytes"] + else: peer.rx_bytes = 0 + if "tx_bytes" in dictionary: peer.tx_bytes = dictionary["tx_bytes"] + else: peer.tx_bytes = 0 + if "last_sync_attempt" in dictionary: peer.last_sync_attempt = dictionary["last_sync_attempt"] + else: peer.last_sync_attempt = 0 hm_count = 0 for transient_id in dictionary["handled_ids"]: From 9beeafb0c87c22bf2f5647fdbf49cab65865c3d9 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Oct 2025 14:08:39 +0100 Subject: [PATCH 332/436] Propagation node stamp cost handling --- LXMF/LXMPeer.py | 27 +++++++++++---- LXMF/LXMRouter.py | 57 +++++++++++++++---------------- LXMF/Utilities/lxmd.py | 76 ++++++++++++++++++++++++++++++------------ 3 files changed, 102 insertions(+), 58 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 3a9fc00..9f2519d 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -70,6 +70,16 @@ class LXMPeer: except: peer.propagation_sync_limit = peer.propagation_transfer_limit else: peer.propagation_sync_limit = peer.propagation_transfer_limit + if "propagation_stamp_cost" in dictionary: + try: peer.propagation_stamp_cost = int(dictionary["propagation_stamp_cost"]) + except: peer.propagation_stamp_cost = None + else: peer.propagation_stamp_cost = None + + if "propagation_stamp_cost_flexibility" in dictionary: + try: peer.propagation_stamp_cost_flexibility = int(dictionary["propagation_stamp_cost_flexibility"]) + except: peer.propagation_stamp_cost_flexibility = None + else: peer.propagation_stamp_cost_flexibility = None + if "sync_strategy" in dictionary: try: peer.sync_strategy = int(dictionary["sync_strategy"]) except: peer.sync_strategy = LXMPeer.DEFAULT_SYNC_STRATEGY @@ -118,9 +128,11 @@ class LXMPeer: dictionary["sync_transfer_rate"] = self.sync_transfer_rate dictionary["propagation_transfer_limit"] = self.propagation_transfer_limit dictionary["propagation_sync_limit"] = self.propagation_sync_limit + dictionary["propagation_stamp_cost"] = self.propagation_stamp_cost + dictionary["propagation_stamp_cost_flexibility"] = self.propagation_stamp_cost_flexibility dictionary["sync_strategy"] = self.sync_strategy dictionary["last_sync_attempt"] = self.last_sync_attempt - dictionary["offered"] = self.offered + dictionary["offered"] = self.offered dictionary["outgoing"] = self.outgoing dictionary["incoming"] = self.incoming dictionary["rx_bytes"] = self.rx_bytes @@ -153,11 +165,14 @@ class LXMPeer: self.peering_timebase = 0 self.link_establishment_rate = 0 self.sync_transfer_rate = 0 - self.propagation_transfer_limit = None - self.propagation_sync_limit = None - self.currently_transferring_messages = None - self.handled_messages_queue = deque() - self.unhandled_messages_queue = deque() + + self.propagation_transfer_limit = None + self.propagation_sync_limit = None + self.propagation_stamp_cost = None + self.propagation_stamp_cost_flexibility = None + self.currently_transferring_messages = None + self.handled_messages_queue = deque() + self.unhandled_messages_queue = deque() self.offered = 0 # Messages offered to this peer self.outgoing = 0 # Messages transferred to this peer diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 30abaa6..5c02cf0 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -44,6 +44,9 @@ class LXMRouter: ROTATION_HEADROOM_PCT = 10 ROTATION_AR_MAX = 0.5 + PROPAGATION_COST = 12 + PROPAGATION_COST_MIN = 10 + PROPAGATION_COST_FLEX = 3 PROPAGATION_LIMIT = 256 SYNC_LIMIT = PROPAGATION_LIMIT*40 DELIVERY_LIMIT = 1000 @@ -76,7 +79,8 @@ class LXMRouter: def __init__(self, identity=None, storagepath=None, autopeer=AUTOPEER, autopeer_maxdepth=None, propagation_limit=PROPAGATION_LIMIT, delivery_limit=DELIVERY_LIMIT, sync_limit=SYNC_LIMIT, enforce_ratchets=False, enforce_stamps=False, static_peers = [], max_peers=None, - from_static_only=False, sync_strategy=LXMPeer.STRATEGY_PERSISTENT): + from_static_only=False, sync_strategy=LXMPeer.STRATEGY_PERSISTENT, + propagation_cost=PROPAGATION_COST, propagation_cost_flexibility=PROPAGATION_COST_FLEX): random.seed(os.urandom(10)) @@ -101,8 +105,7 @@ class LXMRouter: self.propagation_node = False self.propagation_node_start_time = None - if storagepath == None: - raise ValueError("LXMF cannot be initialised without a storage path") + if storagepath == None: raise ValueError("LXMF cannot be initialised without a storage path") else: self.storagepath = storagepath+"/lxmf" self.ratchetpath = self.storagepath+"/ratchets" @@ -117,6 +120,8 @@ class LXMRouter: self.propagation_per_transfer_limit = propagation_limit self.propagation_per_sync_limit = sync_limit self.delivery_per_transfer_limit = delivery_limit + self.propagation_stamp_cost = propagation_cost + self.propagation_stamp_cost_flexibility = propagation_cost_flexibility self.enforce_ratchets = enforce_ratchets self._enforce_stamps = enforce_stamps self.pending_deferred_stamps = {} @@ -153,34 +158,24 @@ class LXMRouter: self.unpeered_propagation_incoming = 0 self.unpeered_propagation_rx_bytes = 0 - if autopeer != None: - self.autopeer = autopeer - else: - self.autopeer = LXMRouter.AUTOPEER + if autopeer != None: self.autopeer = autopeer + else: self.autopeer = LXMRouter.AUTOPEER - if autopeer_maxdepth != None: - self.autopeer_maxdepth = autopeer_maxdepth - else: - self.autopeer_maxdepth = LXMRouter.AUTOPEER_MAXDEPTH + if autopeer_maxdepth != None: self.autopeer_maxdepth = autopeer_maxdepth + else: self.autopeer_maxdepth = LXMRouter.AUTOPEER_MAXDEPTH - if max_peers == None: - self.max_peers = LXMRouter.MAX_PEERS + if max_peers == None: self.max_peers = LXMRouter.MAX_PEERS else: - if type(max_peers) == int and max_peers >= 0: - self.max_peers = max_peers - else: - raise ValueError(f"Invalid value for max_peers: {max_peers}") + if type(max_peers) == int and max_peers >= 0: self.max_peers = max_peers + else: raise ValueError(f"Invalid value for max_peers: {max_peers}") self.from_static_only = from_static_only - if type(static_peers) != list: - raise ValueError(f"Invalid type supplied for static peer list: {type(static_peers)}") + if type(static_peers) != list: raise ValueError(f"Invalid type supplied for static peer list: {type(static_peers)}") else: for static_peer in static_peers: - if type(static_peer) != bytes: - raise ValueError(f"Invalid static peer destination hash: {static_peer}") + if type(static_peer) != bytes: raise ValueError(f"Invalid static peer destination hash: {static_peer}") else: - if len(static_peer) != RNS.Reticulum.TRUNCATED_HASHLENGTH//8: - raise ValueError(f"Invalid static peer destination hash: {static_peer}") + if len(static_peer) != RNS.Reticulum.TRUNCATED_HASHLENGTH//8: raise ValueError(f"Invalid static peer destination hash: {static_peer}") self.static_peers = static_peers @@ -288,11 +283,12 @@ class LXMRouter: def delayed_announce(): time.sleep(LXMRouter.NODE_ANNOUNCE_DELAY) node_state = self.propagation_node and not self.from_static_only + stamp_cost = [self.propagation_stamp_cost, self.propagation_stamp_cost_flexibility] announce_data = [ node_state, # Boolean flag signalling propagation node state int(time.time()), # Current node timebase self.propagation_per_transfer_limit, # Per-transfer limit for message propagation in kilobytes - None, # How many more inbound peers this node wants + stamp_cost, # Propagation stamp cost for this node self.propagation_per_sync_limit, # Limit for incoming propagation node syncs ] @@ -716,6 +712,8 @@ class LXMRouter: "str": int(peer.sync_transfer_rate), "transfer_limit": peer.propagation_transfer_limit, "sync_limit": peer.propagation_sync_limit, + "target_stamp_cost": peer.propagation_stamp_cost, + "stamp_cost_flexibility": peer.propagation_stamp_cost_flexibility, "network_distance": RNS.Transport.hops_to(peer_id), "rx_bytes": peer.rx_bytes, "tx_bytes": peer.tx_bytes, @@ -734,6 +732,8 @@ class LXMRouter: "delivery_limit": self.delivery_per_transfer_limit, "propagation_limit": self.propagation_per_transfer_limit, "sync_limit": self.propagation_per_sync_limit, + "target_stamp_cost": self.propagation_stamp_cost, + "stamp_cost_flexibility": self.propagation_stamp_cost_flexibility, "autopeer_maxdepth": self.autopeer_maxdepth, "from_static_only": self.from_static_only, "messagestore": { @@ -757,12 +757,9 @@ class LXMRouter: return node_stats def stats_get_request(self, path, data, request_id, remote_identity, requested_at): - if remote_identity == None: - return LXMPeer.ERROR_NO_IDENTITY - elif remote_identity.hash != self.identity.hash: - return LXMPeer.ERROR_NO_ACCESS - else: - return self.compile_stats() + if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY + elif remote_identity.hash != self.identity.hash: return LXMPeer.ERROR_NO_ACCESS + else: return self.compile_stats() ### Utility & Maintenance ############################# diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index e49bd7a..03d1282 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -150,6 +150,20 @@ def apply_config(): else: active_configuration["propagation_sync_max_accepted_size"] = 256*40 + if "propagation" in lxmd_config and "propagation_stamp_cost_target" in lxmd_config["propagation"]: + active_configuration["propagation_stamp_cost_target"] = lxmd_config["propagation"].as_int("propagation_stamp_cost_target") + if active_configuration["propagation_stamp_cost_target"] < LXMF.LXMRouter.PROPAGATION_COST_MIN: + active_configuration["propagation_stamp_cost_target"] = LXMF.LXMRouter.PROPAGATION_COST_MIN + else: + active_configuration["propagation_stamp_cost_target"] = LXMF.LXMRouter.PROPAGATION_COST + + if "propagation" in lxmd_config and "propagation_stamp_cost_flexibility" in lxmd_config["propagation"]: + active_configuration["propagation_stamp_cost_flexibility"] = lxmd_config["propagation"].as_int("propagation_stamp_cost_flexibility") + if active_configuration["propagation_stamp_cost_flexibility"] < 0: + active_configuration["propagation_stamp_cost_flexibility"] = 0 + else: + active_configuration["propagation_stamp_cost_flexibility"] = LXMF.LXMRouter.PROPAGATION_COST_FLEX + if "propagation" in lxmd_config and "prioritise_destinations" in lxmd_config["propagation"]: active_configuration["prioritised_lxmf_destinations"] = lxmd_config["propagation"].as_list("prioritise_destinations") else: @@ -337,6 +351,8 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo autopeer = active_configuration["autopeer"], autopeer_maxdepth = active_configuration["autopeer_maxdepth"], propagation_limit = active_configuration["propagation_transfer_max_accepted_size"], + propagation_cost = active_configuration["propagation_stamp_cost_target"], + propagation_cost_flexibility = active_configuration["propagation_stamp_cost_flexibility"], sync_limit = active_configuration["propagation_sync_max_accepted_size"], delivery_limit = active_configuration["delivery_transfer_max_accepted_size"], max_peers = active_configuration["max_peers"], @@ -557,13 +573,15 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = print(f"\nLXMF Propagation Node running on {dhs}, uptime is {uts}") if show_status: - msb = RNS.prettysize(s["messagestore"]["bytes"]); msl = RNS.prettysize(s["messagestore"]["limit"]) - ptl = RNS.prettysize(s["propagation_limit"]*1000); uprx = RNS.prettysize(s["unpeered_propagation_rx_bytes"]) + msb = RNS.prettysize(s["messagestore"]["bytes"]); msl = RNS.prettysize(s["messagestore"]["limit"]) + ptl = RNS.prettysize(s["propagation_limit"]*1000); uprx = RNS.prettysize(s["unpeered_propagation_rx_bytes"]) mscnt = s["messagestore"]["count"]; stp = s["total_peers"]; smp = s["max_peers"]; sdp = s["discovered_peers"] - ssp = s["static_peers"]; cprr = s["clients"]["client_propagation_messages_received"] - cprs = s["clients"]["client_propagation_messages_served"]; upi = s["unpeered_propagation_incoming"] + ssp = s["static_peers"]; cprr = s["clients"]["client_propagation_messages_received"] + cprs = s["clients"]["client_propagation_messages_served"]; upi = s["unpeered_propagation_incoming"] + psc = s["target_stamp_cost"]; scf = s["stamp_cost_flexibility"] print(f"Messagestore contains {mscnt} messages, {msb} ({ms_util} utilised of {msl})") print(f"Accepting propagated messages from {who_str}, {ptl} per-transfer limit") + print(f"Required propagation stamp cost is {psc}, flexibility is {scf}") print(f"") print(f"Peers : {stp} total (peer limit is {smp})") print(f" {sdp} discovered, {ssp} static") @@ -690,24 +708,6 @@ autopeer = yes autopeer_maxdepth = 4 -# The maximum accepted transfer size per in- -# coming propagation message, in kilobytes. -# This sets the upper limit for the size of -# single messages accepted onto this node. - -propagation_message_max_accepted_size = 256 - -# The maximum accepted transfer size per in- -# coming propagation node sync. -# -# If a node wants to propagate a larger number -# of messages to this node, than what can fit -# within this limit, it will prioritise sending -# the smallest messages first, and try again -# with any remaining messages at a later point. - -propagation_sync_max_accepted_size = 256 - # The maximum amount of storage to use for # the LXMF Propagation Node message store, # specified in megabytes. When this limit @@ -720,6 +720,38 @@ propagation_sync_max_accepted_size = 256 # message_storage_limit = 500 +# The maximum accepted transfer size per in- +# coming propagation message, in kilobytes. +# This sets the upper limit for the size of +# single messages accepted onto this node. + +# propagation_message_max_accepted_size = 256 + +# The maximum accepted transfer size per in- +# coming propagation node sync. +# +# If a node wants to propagate a larger number +# of messages to this node, than what can fit +# within this limit, it will prioritise sending +# the smallest messages first, and try again +# with any remaining messages at a later point. + +# propagation_sync_max_accepted_size = 10240 + +# You can configure the target stamp cost +# required to deliver messages via this node. + +# propagation_stamp_cost_target = 12 + +# If set higher than 0, the stamp cost flexi- +# bility option will make this node accept +# messages with a lower stamp cost than the +# target from other propagation nodes (but +# not from peers directly). This allows the +# network to gradually adjust stamp cost. + +# propagation_stamp_cost_flexibility = 3 + # You can tell the LXMF message router to # prioritise storage for one or more # destinations. If the message store reaches From 8f54d40abf150a4efe3c3480f0982aa6abd291e7 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Oct 2025 14:09:54 +0100 Subject: [PATCH 333/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 8088f75..3e2f46a 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.8.1" +__version__ = "0.9.0" From 3f91e44a6d6ffec26e7c74e5c434b854e60b1cee Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Oct 2025 14:10:14 +0100 Subject: [PATCH 334/436] Updated RNS version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 568ade3..16d8d3c 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=1.0.0"], + install_requires=["rns>=1.0.1"], python_requires=">=3.7", ) From 0f2d3b06c2bb02d0a096c896b21f6f52100b3eb8 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Oct 2025 15:18:09 +0100 Subject: [PATCH 335/436] Also receive PN path response announces, but only update peering data on live announces --- LXMF/Handlers.py | 85 ++++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index e1dba8e..793ba16 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -35,53 +35,54 @@ class LXMFDeliveryAnnounceHandler: class LXMFPropagationAnnounceHandler: def __init__(self, lxmrouter): self.aspect_filter = APP_NAME+".propagation" - self.receive_path_responses = False + self.receive_path_responses = True self.lxmrouter = lxmrouter - def received_announce(self, destination_hash, announced_identity, app_data): - try: - if type(app_data) == bytes: - if self.lxmrouter.propagation_node: - data = msgpack.unpackb(app_data) - if pn_announce_data_is_valid(data): - node_timebase = data[1] - propagation_transfer_limit = None - propagation_sync_limit = None - wanted_inbound_peers = None - if len(data) >= 5: - try: propagation_sync_limit = int(data[4]) - except Exception as e: propagation_sync_limit = None + def received_announce(self, destination_hash, announced_identity, app_data, announce_packet_hash, is_path_response): + if not is_path_response: + try: + if type(app_data) == bytes: + if self.lxmrouter.propagation_node: + data = msgpack.unpackb(app_data) + if pn_announce_data_is_valid(data): + node_timebase = data[1] + propagation_transfer_limit = None + propagation_sync_limit = None + wanted_inbound_peers = None + if len(data) >= 5: + try: propagation_sync_limit = int(data[4]) + except Exception as e: propagation_sync_limit = None - if len(data) >= 4: - # TODO: Rethink, probably not necessary anymore - # try: wanted_inbound_peers = int(data[3]) - # except: wanted_inbound_peers = None - pass + if len(data) >= 4: + # TODO: Rethink, probably not necessary anymore + # try: wanted_inbound_peers = int(data[3]) + # except: wanted_inbound_peers = None + pass - if len(data) >= 3: - try: propagation_transfer_limit = float(data[2]) - except: propagation_transfer_limit = None + if len(data) >= 3: + try: propagation_transfer_limit = float(data[2]) + except: propagation_transfer_limit = None - if destination_hash in self.lxmrouter.static_peers: - self.lxmrouter.peer(destination_hash=destination_hash, - timestamp=node_timebase, - propagation_transfer_limit=propagation_transfer_limit, - propagation_sync_limit=propagation_sync_limit, - wanted_inbound_peers=wanted_inbound_peers) + if destination_hash in self.lxmrouter.static_peers: + self.lxmrouter.peer(destination_hash=destination_hash, + timestamp=node_timebase, + propagation_transfer_limit=propagation_transfer_limit, + propagation_sync_limit=propagation_sync_limit, + wanted_inbound_peers=wanted_inbound_peers) - else: - if self.lxmrouter.autopeer: - if data[0] == True: - if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: - self.lxmrouter.peer(destination_hash=destination_hash, - timestamp=node_timebase, - propagation_transfer_limit=propagation_transfer_limit, - propagation_sync_limit=propagation_sync_limit, - wanted_inbound_peers=wanted_inbound_peers) + else: + if self.lxmrouter.autopeer: + if data[0] == True: + if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: + self.lxmrouter.peer(destination_hash=destination_hash, + timestamp=node_timebase, + propagation_transfer_limit=propagation_transfer_limit, + propagation_sync_limit=propagation_sync_limit, + wanted_inbound_peers=wanted_inbound_peers) - elif data[0] == False: - self.lxmrouter.unpeer(destination_hash, node_timebase) + elif data[0] == False: + self.lxmrouter.unpeer(destination_hash, node_timebase) - except Exception as e: - RNS.log("Error while evaluating propagation node announce, ignoring announce.", RNS.LOG_DEBUG) - RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + except Exception as e: + RNS.log("Error while evaluating propagation node announce, ignoring announce.", RNS.LOG_DEBUG) + RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) From bd6fe9f9d1e2095fd77eedad1cc406418198dc1e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Oct 2025 15:39:00 +0100 Subject: [PATCH 336/436] Handle propagation node stamp cost peering signalling --- LXMF/Handlers.py | 33 +++++++++++++-------------------- LXMF/LXMF.py | 11 ++++++++++- LXMF/LXMRouter.py | 8 ++++++-- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index 793ba16..41b3200 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -45,42 +45,35 @@ class LXMFPropagationAnnounceHandler: if self.lxmrouter.propagation_node: data = msgpack.unpackb(app_data) if pn_announce_data_is_valid(data): - node_timebase = data[1] - propagation_transfer_limit = None - propagation_sync_limit = None - wanted_inbound_peers = None - if len(data) >= 5: - try: propagation_sync_limit = int(data[4]) - except Exception as e: propagation_sync_limit = None - - if len(data) >= 4: - # TODO: Rethink, probably not necessary anymore - # try: wanted_inbound_peers = int(data[3]) - # except: wanted_inbound_peers = None - pass - - if len(data) >= 3: - try: propagation_transfer_limit = float(data[2]) - except: propagation_transfer_limit = None - + pn_active = data[0] + node_timebase = int(data[1]) + propagation_transfer_limit = int(data[2]) + propagation_sync_limit = int(data[3]) + propagation_stamp_cost = int(data[4][0]) + propagation_stamp_cost_flexibility = int(data[4][1]) + if destination_hash in self.lxmrouter.static_peers: self.lxmrouter.peer(destination_hash=destination_hash, timestamp=node_timebase, propagation_transfer_limit=propagation_transfer_limit, propagation_sync_limit=propagation_sync_limit, + propagation_stamp_cost=propagation_stamp_cost, + propagation_stamp_cost_flexibility=propagation_stamp_cost_flexibility, wanted_inbound_peers=wanted_inbound_peers) else: if self.lxmrouter.autopeer: - if data[0] == True: + if pn_active == True: if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: self.lxmrouter.peer(destination_hash=destination_hash, timestamp=node_timebase, propagation_transfer_limit=propagation_transfer_limit, propagation_sync_limit=propagation_sync_limit, + propagation_stamp_cost=propagation_stamp_cost, + propagation_stamp_cost_flexibility=propagation_stamp_cost_flexibility, wanted_inbound_peers=wanted_inbound_peers) - elif data[0] == False: + elif pn_active == False: self.lxmrouter.unpeer(destination_hash, node_timebase) except Exception as e: diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 93e283f..8d1aaf7 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -138,11 +138,20 @@ def stamp_cost_from_app_data(app_data=None): def pn_announce_data_is_valid(data): try: if type(data) == bytes: data = msgpack.unpackb(data) - if len(data) < 3: raise ValueError("Invalid announce data: Insufficient peer data") + if len(data) < 5: raise ValueError("Invalid announce data: Insufficient peer data") else: if data[0] != True and data[0] != False: raise ValueError("Invalid announce data: Indeterminate propagation node status") try: int(data[1]) except: raise ValueError("Invalid announce data: Could not decode peer timebase") + try: int(data[2]) + except: raise ValueError("Invalid announce data: Could not decode peer propagation transfer limit") + try: int(data[3]) + except: raise ValueError("Invalid announce data: Could not decode peer propagation sync limit") + if type(data[4]) != list: raise ValueError("Invalid announce data: Could not decode peer stamp costs") + try: int(data[4][0]) + except: raise ValueError("Invalid announce data: Could not decode peer target stamp cost") + try: int(data[4][1]) + except: raise ValueError("Invalid announce data: Could not decode peer stamp cost flexibility") except Exception as e: RNS.log(f"Could not validate propagation node announce data: {e}", RNS.LOG_DEBUG) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 5c02cf0..eef9fee 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -288,8 +288,8 @@ class LXMRouter: node_state, # Boolean flag signalling propagation node state int(time.time()), # Current node timebase self.propagation_per_transfer_limit, # Per-transfer limit for message propagation in kilobytes - stamp_cost, # Propagation stamp cost for this node self.propagation_per_sync_limit, # Limit for incoming propagation node syncs + stamp_cost, # Propagation stamp cost for this node ] data = msgpack.packb(announce_data) @@ -1783,7 +1783,7 @@ class LXMRouter: ### Peer Sync & Propagation ########################### ####################################################### - def peer(self, destination_hash, timestamp, propagation_transfer_limit, propagation_sync_limit, wanted_inbound_peers = None): + def peer(self, destination_hash, timestamp, propagation_transfer_limit, propagation_sync_limit, propagation_stamp_cost, propagation_stamp_cost_flexibility): if destination_hash in self.peers: peer = self.peers[destination_hash] if timestamp > peer.peering_timebase: @@ -1792,6 +1792,8 @@ class LXMRouter: peer.next_sync_attempt = 0 peer.peering_timebase = timestamp peer.last_heard = time.time() + peer.propagation_stamp_cost = propagation_stamp_cost + peer.propagation_stamp_cost_flexibility = propagation_stamp_cost_flexibility peer.propagation_transfer_limit = propagation_transfer_limit if propagation_sync_limit != None: peer.propagation_sync_limit = propagation_sync_limit else: peer.propagation_sync_limit = propagation_transfer_limit @@ -1803,6 +1805,8 @@ class LXMRouter: peer = LXMPeer(self, destination_hash, sync_strategy=self.default_sync_strategy) peer.alive = True peer.last_heard = time.time() + peer.propagation_stamp_cost = propagation_stamp_cost + peer.propagation_stamp_cost_flexibility = propagation_stamp_cost_flexibility peer.propagation_transfer_limit = propagation_transfer_limit if propagation_sync_limit != None: peer.propagation_sync_limit = propagation_sync_limit else: peer.propagation_sync_limit = propagation_transfer_limit From aca5bf9c14d9b0aac3156ba4138abea9790d6347 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Oct 2025 16:06:53 +0100 Subject: [PATCH 337/436] Cleanup --- LXMF/Handlers.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index 41b3200..b5f2659 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -58,8 +58,7 @@ class LXMFPropagationAnnounceHandler: propagation_transfer_limit=propagation_transfer_limit, propagation_sync_limit=propagation_sync_limit, propagation_stamp_cost=propagation_stamp_cost, - propagation_stamp_cost_flexibility=propagation_stamp_cost_flexibility, - wanted_inbound_peers=wanted_inbound_peers) + propagation_stamp_cost_flexibility=propagation_stamp_cost_flexibility) else: if self.lxmrouter.autopeer: @@ -70,8 +69,7 @@ class LXMFPropagationAnnounceHandler: propagation_transfer_limit=propagation_transfer_limit, propagation_sync_limit=propagation_sync_limit, propagation_stamp_cost=propagation_stamp_cost, - propagation_stamp_cost_flexibility=propagation_stamp_cost_flexibility, - wanted_inbound_peers=wanted_inbound_peers) + propagation_stamp_cost_flexibility=propagation_stamp_cost_flexibility) elif pn_active == False: self.lxmrouter.unpeer(destination_hash, node_timebase) From d8b25e092f0ba8693964fa75a6a500ba220b3257 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Oct 2025 16:43:26 +0100 Subject: [PATCH 338/436] Added metadate structure to propagation node announces --- LXMF/Handlers.py | 23 +++++++++++++---------- LXMF/LXMF.py | 26 +++++++++++++++++++------- LXMF/LXMRouter.py | 6 ++++-- LXMF/LXStamper.py | 2 +- 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index b5f2659..40c416d 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -45,12 +45,13 @@ class LXMFPropagationAnnounceHandler: if self.lxmrouter.propagation_node: data = msgpack.unpackb(app_data) if pn_announce_data_is_valid(data): - pn_active = data[0] - node_timebase = int(data[1]) - propagation_transfer_limit = int(data[2]) - propagation_sync_limit = int(data[3]) - propagation_stamp_cost = int(data[4][0]) - propagation_stamp_cost_flexibility = int(data[4][1]) + metadata = data[0] + propagation_enabled = data[1] + node_timebase = int(data[2]) + propagation_transfer_limit = int(data[3]) + propagation_sync_limit = int(data[4]) + propagation_stamp_cost = int(data[5][0]) + propagation_stamp_cost_flexibility = int(data[5][1]) if destination_hash in self.lxmrouter.static_peers: self.lxmrouter.peer(destination_hash=destination_hash, @@ -58,20 +59,22 @@ class LXMFPropagationAnnounceHandler: propagation_transfer_limit=propagation_transfer_limit, propagation_sync_limit=propagation_sync_limit, propagation_stamp_cost=propagation_stamp_cost, - propagation_stamp_cost_flexibility=propagation_stamp_cost_flexibility) + propagation_stamp_cost_flexibility=propagation_stamp_cost_flexibility, + metadata=metadata) else: if self.lxmrouter.autopeer: - if pn_active == True: + if propagation_enabled == True: if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: self.lxmrouter.peer(destination_hash=destination_hash, timestamp=node_timebase, propagation_transfer_limit=propagation_transfer_limit, propagation_sync_limit=propagation_sync_limit, propagation_stamp_cost=propagation_stamp_cost, - propagation_stamp_cost_flexibility=propagation_stamp_cost_flexibility) + propagation_stamp_cost_flexibility=propagation_stamp_cost_flexibility, + metadata=metadata) - elif pn_active == False: + elif propagation_enabled == False: self.lxmrouter.unpeer(destination_hash, node_timebase) except Exception as e: diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 8d1aaf7..8342d33 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -91,6 +91,17 @@ RENDERER_MICRON = 0x01 RENDERER_MARKDOWN = 0x02 RENDERER_BBCODE = 0x03 +# Optional propagation node metadata fields. These +# fields may be highly unstable in allocation and +# availability until the version 1.0.0 release, so use +# at your own risk until then, and expect changes! +PN_META_VERSION = 0x00 +PN_META_NAME = 0x01 +PN_META_SYNC_STRATUM = 0x02 +PN_META_SYNC_THROTTLE = 0x03 +PN_META_AUTH_BAND = 0x04 +PN_META_UTIL_PRESSURE = 0x05 + ########################################################## # The following helper functions makes it easier to # # handle and operate on LXMF data in client programs # @@ -138,19 +149,20 @@ def stamp_cost_from_app_data(app_data=None): def pn_announce_data_is_valid(data): try: if type(data) == bytes: data = msgpack.unpackb(data) - if len(data) < 5: raise ValueError("Invalid announce data: Insufficient peer data") + if len(data) < 6: raise ValueError("Invalid announce data: Insufficient peer data") else: - if data[0] != True and data[0] != False: raise ValueError("Invalid announce data: Indeterminate propagation node status") - try: int(data[1]) - except: raise ValueError("Invalid announce data: Could not decode peer timebase") + if type(data[0]) != dict: raise ValueError("Invalid announce data: Could not decode peer metadata") + if data[1] != True and data[1] != False: raise ValueError("Invalid announce data: Indeterminate propagation node status") try: int(data[2]) - except: raise ValueError("Invalid announce data: Could not decode peer propagation transfer limit") + except: raise ValueError("Invalid announce data: Could not decode peer timebase") try: int(data[3]) + except: raise ValueError("Invalid announce data: Could not decode peer propagation transfer limit") + try: int(data[4]) except: raise ValueError("Invalid announce data: Could not decode peer propagation sync limit") if type(data[4]) != list: raise ValueError("Invalid announce data: Could not decode peer stamp costs") - try: int(data[4][0]) + try: int(data[5][0]) except: raise ValueError("Invalid announce data: Could not decode peer target stamp cost") - try: int(data[4][1]) + try: int(data[5][1]) except: raise ValueError("Invalid announce data: Could not decode peer stamp cost flexibility") except Exception as e: diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index eef9fee..4f31920 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -282,9 +282,11 @@ class LXMRouter: def announce_propagation_node(self): def delayed_announce(): time.sleep(LXMRouter.NODE_ANNOUNCE_DELAY) - node_state = self.propagation_node and not self.from_static_only - stamp_cost = [self.propagation_stamp_cost, self.propagation_stamp_cost_flexibility] + node_state = self.propagation_node and not self.from_static_only + stamp_cost = [self.propagation_stamp_cost, self.propagation_stamp_cost_flexibility] + metadata = {} announce_data = [ + metadata, # Node metadata node_state, # Boolean flag signalling propagation node state int(time.time()), # Current node timebase self.propagation_per_transfer_limit, # Per-transfer limit for message propagation in kilobytes diff --git a/LXMF/LXStamper.py b/LXMF/LXStamper.py index a9ca7d6..2db0598 100644 --- a/LXMF/LXStamper.py +++ b/LXMF/LXStamper.py @@ -76,7 +76,7 @@ def validate_pn_stamps_job_multip(transient_stamps): cores = multiprocessing.cpu_count() pool_count = min(cores, math.ceil(len(transient_stamps) / PN_VALIDATION_POOL_MIN_SIZE)) - RNS.log(f"Validating {len(transient_stamps)} stamps using {pool_count} processes...") + RNS.log(f"Validating {len(transient_stamps)} stamps using {pool_count} processes...", RNS.LOG_VERBOSE) with multiprocessing.Pool(pool_count) as p: validated_entries = p.map(_validate_single_pn_stamp_entry, transient_stamps) return validated_entries From c28d3b1432bb2783ca31207fcb9be72345f5998c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Oct 2025 16:44:15 +0100 Subject: [PATCH 339/436] Added metadate structure to propagation node announces --- LXMF/LXMF.py | 1 + 1 file changed, 1 insertion(+) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 8342d33..a57e31a 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -101,6 +101,7 @@ PN_META_SYNC_STRATUM = 0x02 PN_META_SYNC_THROTTLE = 0x03 PN_META_AUTH_BAND = 0x04 PN_META_UTIL_PRESSURE = 0x05 +PN_META_CUSTOM = 0xFF ########################################################## # The following helper functions makes it easier to # From 81a6d503a3d897550d0ae78d05e11a87e71f99d7 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Oct 2025 16:55:44 +0100 Subject: [PATCH 340/436] Automatic legacy PN de-peering --- LXMF/Handlers.py | 6 +++--- LXMF/LXMF.py | 8 ++++---- LXMF/LXMRouter.py | 15 +++++++-------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index 40c416d..5daedd1 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -45,13 +45,13 @@ class LXMFPropagationAnnounceHandler: if self.lxmrouter.propagation_node: data = msgpack.unpackb(app_data) if pn_announce_data_is_valid(data): - metadata = data[0] - propagation_enabled = data[1] - node_timebase = int(data[2]) + node_timebase = int(data[1]) + propagation_enabled = data[2] propagation_transfer_limit = int(data[3]) propagation_sync_limit = int(data[4]) propagation_stamp_cost = int(data[5][0]) propagation_stamp_cost_flexibility = int(data[5][1]) + metadata = data[6] if destination_hash in self.lxmrouter.static_peers: self.lxmrouter.peer(destination_hash=destination_hash, diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index a57e31a..6369d69 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -150,12 +150,11 @@ def stamp_cost_from_app_data(app_data=None): def pn_announce_data_is_valid(data): try: if type(data) == bytes: data = msgpack.unpackb(data) - if len(data) < 6: raise ValueError("Invalid announce data: Insufficient peer data") + if len(data) < 7: raise ValueError("Invalid announce data: Insufficient peer data") else: - if type(data[0]) != dict: raise ValueError("Invalid announce data: Could not decode peer metadata") - if data[1] != True and data[1] != False: raise ValueError("Invalid announce data: Indeterminate propagation node status") - try: int(data[2]) + try: int(data[1]) except: raise ValueError("Invalid announce data: Could not decode peer timebase") + if data[2] != True and data[2] != False: raise ValueError("Invalid announce data: Indeterminate propagation node status") try: int(data[3]) except: raise ValueError("Invalid announce data: Could not decode peer propagation transfer limit") try: int(data[4]) @@ -165,6 +164,7 @@ def pn_announce_data_is_valid(data): except: raise ValueError("Invalid announce data: Could not decode peer target stamp cost") try: int(data[5][1]) except: raise ValueError("Invalid announce data: Could not decode peer stamp cost flexibility") + if type(data[6]) != dict: raise ValueError("Invalid announce data: Could not decode peer metadata") except Exception as e: RNS.log(f"Could not validate propagation node announce data: {e}", RNS.LOG_DEBUG) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 4f31920..816d949 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -285,14 +285,13 @@ class LXMRouter: node_state = self.propagation_node and not self.from_static_only stamp_cost = [self.propagation_stamp_cost, self.propagation_stamp_cost_flexibility] metadata = {} - announce_data = [ - metadata, # Node metadata - node_state, # Boolean flag signalling propagation node state - int(time.time()), # Current node timebase - self.propagation_per_transfer_limit, # Per-transfer limit for message propagation in kilobytes - self.propagation_per_sync_limit, # Limit for incoming propagation node syncs - stamp_cost, # Propagation stamp cost for this node - ] + announce_data = [ False, # Legacy LXMF PN support + int(time.time()), # Current node timebase + node_state, # Boolean flag signalling propagation node state + self.propagation_per_transfer_limit, # Per-transfer limit for message propagation in kilobytes + self.propagation_per_sync_limit, # Limit for incoming propagation node syncs + stamp_cost, # Propagation stamp cost for this node + metadata ] # Node metadata data = msgpack.packb(announce_data) self.propagation_destination.announce(app_data=data) From 6446db4f1111a93d779879669d99a568efbf014c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Oct 2025 17:03:05 +0100 Subject: [PATCH 341/436] Break peering when peer moves outside auto-peering range --- LXMF/Handlers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index 5daedd1..c51231c 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -74,6 +74,11 @@ class LXMFPropagationAnnounceHandler: propagation_stamp_cost_flexibility=propagation_stamp_cost_flexibility, metadata=metadata) + else: + if destination_hash in self.lxmrouter.peers: + RNS.log(f"Peer {self.lxmrouter.peers[destination_hash]} moved outside auto-peering range, breaking peering...") + self.lxmrouter.unpeer(destination_hash, node_timebase) + elif propagation_enabled == False: self.lxmrouter.unpeer(destination_hash, node_timebase) From a62ffa12b1c3af3c3b69b999fc9b3f6ed25b4768 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Oct 2025 19:45:40 +0100 Subject: [PATCH 342/436] Cleanup --- LXMF/Handlers.py | 14 ++++---- LXMF/LXMPeer.py | 94 ++++++++++++++++++++---------------------------- 2 files changed, 45 insertions(+), 63 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index c51231c..fc980c3 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -8,13 +8,13 @@ from .LXMessage import LXMessage class LXMFDeliveryAnnounceHandler: def __init__(self, lxmrouter): - self.aspect_filter = APP_NAME+".delivery" + self.aspect_filter = APP_NAME+".delivery" self.receive_path_responses = True - self.lxmrouter = lxmrouter + self.lxmrouter = lxmrouter def received_announce(self, destination_hash, announced_identity, app_data): for lxmessage in self.lxmrouter.pending_outbound: - if destination_hash == lxmessage.destination_hash: + if destination_hash == lxmessage.destination_hash: if lxmessage.method == LXMessage.DIRECT or lxmessage.method == LXMessage.OPPORTUNISTIC: lxmessage.next_delivery_attempt = time.time() @@ -34,9 +34,9 @@ class LXMFDeliveryAnnounceHandler: class LXMFPropagationAnnounceHandler: def __init__(self, lxmrouter): - self.aspect_filter = APP_NAME+".propagation" + self.aspect_filter = APP_NAME+".propagation" self.receive_path_responses = True - self.lxmrouter = lxmrouter + self.lxmrouter = lxmrouter def received_announce(self, destination_hash, announced_identity, app_data, announce_packet_hash, is_path_response): if not is_path_response: @@ -51,7 +51,7 @@ class LXMFPropagationAnnounceHandler: propagation_sync_limit = int(data[4]) propagation_stamp_cost = int(data[5][0]) propagation_stamp_cost_flexibility = int(data[5][1]) - metadata = data[6] + metadata = data[6] if destination_hash in self.lxmrouter.static_peers: self.lxmrouter.peer(destination_hash=destination_hash, @@ -84,4 +84,4 @@ class LXMFPropagationAnnounceHandler: except Exception as e: RNS.log("Error while evaluating propagation node announce, ignoring announce.", RNS.LOG_DEBUG) - RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG) + RNS.log(f"The contained exception was: {str(e)}", RNS.LOG_DEBUG) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 9f2519d..e86ae84 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -174,11 +174,11 @@ class LXMPeer: self.handled_messages_queue = deque() self.unhandled_messages_queue = deque() - self.offered = 0 # Messages offered to this peer - self.outgoing = 0 # Messages transferred to this peer - self.incoming = 0 # Messages received from this peer - self.rx_bytes = 0 # Bytes received from this peer - self.tx_bytes = 0 # Bytes sent to this peer + self.offered = 0 # Messages offered to this peer + self.outgoing = 0 # Messages transferred to this peer + self.incoming = 0 # Messages received from this peer + self.rx_bytes = 0 # Bytes received from this peer + self.tx_bytes = 0 # Bytes sent to this peer self._hm_count = 0 self._um_count = 0 @@ -243,29 +243,29 @@ class LXMPeer: purged_ids = [] for transient_id in self.unhandled_messages: if transient_id in self.router.propagation_entries: - unhandled_entry = [ - transient_id, - self.router.get_weight(transient_id), - self.router.get_size(transient_id), - ] + unhandled_entry = [ transient_id, + self.router.get_weight(transient_id), + self.router.get_size(transient_id) ] + unhandled_entries.append(unhandled_entry) - else: - purged_ids.append(transient_id) + + else: purged_ids.append(transient_id) for transient_id in purged_ids: RNS.log("Dropping unhandled message "+RNS.prettyhexrep(transient_id)+" for peer "+RNS.prettyhexrep(self.destination_hash)+" since it no longer exists in the message store.", RNS.LOG_DEBUG) self.remove_unhandled_message(transient_id) unhandled_entries.sort(key=lambda e: e[1], reverse=False) - per_message_overhead = 16 # Really only 2 bytes, but set a bit higher for now - cumulative_size = 24 # Initialised to highest reasonable binary structure overhead + per_message_overhead = 16 # Really only 2 bytes, but set a bit higher for now + cumulative_size = 24 # Initialised to highest reasonable binary structure overhead RNS.log(f"Syncing to peer with per-message limit {RNS.prettysize(self.propagation_transfer_limit*1000)} and sync limit {RNS.prettysize(self.propagation_sync_limit*1000)}") # TODO: Remove debug + for unhandled_entry in unhandled_entries: - transient_id = unhandled_entry[0] - weight = unhandled_entry[1] - lxm_size = unhandled_entry[2] + transient_id = unhandled_entry[0] + weight = unhandled_entry[1] + lxm_size = unhandled_entry[2] lxm_transfer_size = lxm_size+per_message_overhead - next_size = cumulative_size + lxm_transfer_size + next_size = cumulative_size + lxm_transfer_size if self.propagation_transfer_limit != None and lxm_transfer_size > (self.propagation_transfer_limit*1000): self.remove_unhandled_message(transient_id) @@ -288,14 +288,11 @@ class LXMPeer: else: RNS.log("Postponing sync with peer "+RNS.prettyhexrep(self.destination_hash)+" for "+RNS.prettytime(self.next_sync_attempt-time.time())+" due to previous failures", RNS.LOG_DEBUG) - if self.last_sync_attempt > self.last_heard: - self.alive = False + if self.last_sync_attempt > self.last_heard: self.alive = False def request_failed(self, request_receipt): - RNS.log("Sync request to peer "+str(self.destination)+" failed", RNS.LOG_DEBUG) - if self.link != None: - self.link.teardown() - + RNS.log(f"Sync request to peer {self.destination} failed", RNS.LOG_DEBUG) + if self.link != None: self.link.teardown() self.state = LXMPeer.IDLE def offer_response(self, request_receipt): @@ -389,18 +386,16 @@ class LXMPeer: if self.currently_transferring_messages == None: RNS.log(f"Sync transfer completed on {self}, but transferred message index was unavailable. Aborting.", RNS.LOG_ERROR) if self.link != None: self.link.teardown() - self.link = None + self.link = None self.state = LXMPeer.IDLE for transient_id in self.currently_transferring_messages: self.add_handled_message(transient_id) self.remove_unhandled_message(transient_id) - if self.link != None: - self.link.teardown() - - self.link = None - self.state = LXMPeer.IDLE + if self.link != None: self.link.teardown() + self.link = None + self.state = LXMPeer.IDLE rate_str = "" if self.current_sync_transfer_started != None: @@ -408,14 +403,14 @@ class LXMPeer: rate_str = f" at {RNS.prettyspeed(self.sync_transfer_rate)}" RNS.log(f"Syncing {len(self.currently_transferring_messages)} messages to peer {RNS.prettyhexrep(self.destination_hash)} completed{rate_str}", RNS.LOG_VERBOSE) - self.alive = True + self.alive = True self.last_heard = time.time() self.offered += len(self.last_offer) self.outgoing += len(self.currently_transferring_messages) self.tx_bytes += resource.get_data_size() self.currently_transferring_messages = None - self.current_sync_transfer_started = None + self.current_sync_transfer_started = None if self.sync_strategy == self.STRATEGY_PERSISTENT: if self.unhandled_message_count > 0: self.sync() @@ -423,10 +418,10 @@ class LXMPeer: else: RNS.log("Resource transfer for LXMF peer sync failed to "+str(self.destination), RNS.LOG_VERBOSE) if self.link != None: self.link.teardown() - self.link = None - self.state = LXMPeer.IDLE + self.link = None + self.state = LXMPeer.IDLE self.currently_transferring_messages = None - self.current_sync_transfer_started = None + self.current_sync_transfer_started = None def link_established(self, link): self.link.identify(self.router.identity) @@ -439,7 +434,7 @@ class LXMPeer: self.sync() def link_closed(self, link): - self.link = None + self.link = None self.state = LXMPeer.IDLE def queued_items(self): @@ -452,19 +447,14 @@ class LXMPeer: self.handled_messages_queue.append(transient_id) def process_queues(self): - if len(self.unhandled_messages_queue) > 0 or len(self.handled_messages_queue) > 0: - # TODO: Remove debug - # st = time.time(); lu = len(self.unhandled_messages_queue); lh = len(self.handled_messages_queue) - + if len(self.unhandled_messages_queue) > 0 or len(self.handled_messages_queue) > 0: handled_messages = self.handled_messages unhandled_messages = self.unhandled_messages while len(self.handled_messages_queue) > 0: transient_id = self.handled_messages_queue.pop() - if not transient_id in handled_messages: - self.add_handled_message(transient_id) - if transient_id in unhandled_messages: - self.remove_unhandled_message(transient_id) + if not transient_id in handled_messages: self.add_handled_message(transient_id) + if transient_id in unhandled_messages: self.remove_unhandled_message(transient_id) while len(self.unhandled_messages_queue) > 0: transient_id = self.unhandled_messages_queue.pop() @@ -472,8 +462,6 @@ class LXMPeer: self.add_unhandled_message(transient_id) del handled_messages, unhandled_messages - # TODO: Remove debug - # RNS.log(f"{self} processed {lh}/{lu} in {RNS.prettytime(time.time()-st)}") @property def handled_messages(self): @@ -493,16 +481,12 @@ class LXMPeer: @property def handled_message_count(self): - if not self._hm_counts_synced: - self._update_counts() - + if not self._hm_counts_synced: self._update_counts() return self._hm_count @property def unhandled_message_count(self): - if not self._um_counts_synced: - self._update_counts() - + if not self._um_counts_synced: self._update_counts() return self._um_count @property @@ -541,7 +525,5 @@ class LXMPeer: self._um_counts_synced = False def __str__(self): - if self.destination_hash: - return RNS.prettyhexrep(self.destination_hash) - else: - return "" \ No newline at end of file + if self.destination_hash: return RNS.prettyhexrep(self.destination_hash) + else: return "" \ No newline at end of file From c84aea745a682e1f834ee7a6f6a5d8023792fa6d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 30 Oct 2025 21:19:38 +0100 Subject: [PATCH 343/436] Improved auto-peering on inbound PN sync. Added support for persisting and loading transient message stamp status. Implemented getting transient message stamp value. --- LXMF/LXMF.py | 3 +- LXMF/LXMPeer.py | 2 +- LXMF/LXMRouter.py | 143 ++++++++++++++++++++++++++-------------------- LXMF/LXStamper.py | 10 ++-- 4 files changed, 89 insertions(+), 69 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 6369d69..b608ceb 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -149,7 +149,8 @@ def stamp_cost_from_app_data(app_data=None): def pn_announce_data_is_valid(data): try: - if type(data) == bytes: data = msgpack.unpackb(data) + if type(data) != bytes: return False + else: data = msgpack.unpackb(data) if len(data) < 7: raise ValueError("Invalid announce data: Insufficient peer data") else: try: int(data[1]) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index e86ae84..e767313 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -259,7 +259,7 @@ class LXMPeer: per_message_overhead = 16 # Really only 2 bytes, but set a bit higher for now cumulative_size = 24 # Initialised to highest reasonable binary structure overhead RNS.log(f"Syncing to peer with per-message limit {RNS.prettysize(self.propagation_transfer_limit*1000)} and sync limit {RNS.prettysize(self.propagation_sync_limit*1000)}") # TODO: Remove debug - + for unhandled_entry in unhandled_entries: transient_id = unhandled_entry[0] weight = unhandled_entry[1] diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 816d949..ace090a 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -15,6 +15,7 @@ import RNS.vendor.umsgpack as msgpack from .LXMF import APP_NAME from .LXMF import FIELD_TICKET +from .LXMF import pn_announce_data_is_valid from .LXMPeer import LXMPeer from .LXMessage import LXMessage @@ -285,13 +286,13 @@ class LXMRouter: node_state = self.propagation_node and not self.from_static_only stamp_cost = [self.propagation_stamp_cost, self.propagation_stamp_cost_flexibility] metadata = {} - announce_data = [ False, # Legacy LXMF PN support - int(time.time()), # Current node timebase - node_state, # Boolean flag signalling propagation node state - self.propagation_per_transfer_limit, # Per-transfer limit for message propagation in kilobytes - self.propagation_per_sync_limit, # Limit for incoming propagation node syncs - stamp_cost, # Propagation stamp cost for this node - metadata ] # Node metadata + announce_data = [ False, # 0: Legacy LXMF PN support + int(time.time()), # 1: Current node timebase + node_state, # 2: Boolean flag signalling propagation node state + self.propagation_per_transfer_limit, # 3: Per-transfer limit for message propagation in kilobytes + self.propagation_per_sync_limit, # 4: Limit for incoming propagation node syncs + stamp_cost, # 5: Propagation stamp cost for this node + metadata ] # 6: Node metadata data = msgpack.packb(announce_data) self.propagation_destination.announce(app_data=data) @@ -486,19 +487,21 @@ class LXMRouter: st = time.time(); RNS.log("Indexing messagestore...", RNS.LOG_NOTICE) for filename in os.listdir(self.messagepath): components = filename.split("_") - if len(components) == 2: + if len(components) >= 2: if float(components[1]) > 0: if len(components[0]) == RNS.Identity.HASHLENGTH//8*2: try: - transient_id = bytes.fromhex(components[0]) - received = float(components[1]) - - filepath = self.messagepath+"/"+filename - msg_size = os.path.getsize(filepath) - file = open(filepath, "rb") - destination_hash = file.read(LXMessage.DESTINATION_LENGTH) + transient_id = bytes.fromhex(components[0]) + received = float(components[1]) + filepath = self.messagepath+"/"+filename + msg_size = os.path.getsize(filepath) + file = open(filepath, "rb") + destination_hash = file.read(LXMessage.DESTINATION_LENGTH) file.close() + if len(components) >= 3: stamp_value = int(components[2]) + else: stamp_value = None + self.propagation_entries[transient_id] = [ destination_hash, # 0: Destination hash filepath, # 1: Storage location @@ -506,6 +509,7 @@ class LXMRouter: msg_size, # 3: Message size [], # 4: Handled peers [], # 5: Unhandled peers + stamp_value, # 6: Stamp value ] except Exception as e: @@ -923,22 +927,26 @@ class LXMRouter: return msgpack.packb(peer_data) - def get_weight(self, transient_id): - dst_hash = self.propagation_entries[transient_id][0] - lxm_rcvd = self.propagation_entries[transient_id][2] + def get_size(self, transient_id): lxm_size = self.propagation_entries[transient_id][3] + return lxm_size - now = time.time() + def get_weight(self, transient_id): + dst_hash = self.propagation_entries[transient_id][0] + lxm_rcvd = self.propagation_entries[transient_id][2] + lxm_size = self.propagation_entries[transient_id][3] + + now = time.time() age_weight = max(1, (now - lxm_rcvd)/60/60/24/4) - if dst_hash in self.prioritised_list: - priority_weight = 0.1 - else: - priority_weight = 1.0 + if dst_hash in self.prioritised_list: priority_weight = 0.1 + else: priority_weight = 1.0 - weight = priority_weight * age_weight * lxm_size + return priority_weight * age_weight * lxm_size - return weight + def get_stamp_value(self, transient_id): + if not transient_id in self.propagation_entries: return None + else: return self.propagation_entries[transient_id][6] def generate_ticket(self, destination_hash, expiry=LXMessage.TICKET_EXPIRY): now = time.time() @@ -1003,10 +1011,6 @@ class LXMRouter: else: return available_tickets - def get_size(self, transient_id): - lxm_size = self.propagation_entries[transient_id][3] - return lxm_size - def clean_message_store(self): RNS.log("Cleaning message store", RNS.LOG_VERBOSE) # Check and remove expired messages @@ -1993,18 +1997,22 @@ class LXMRouter: def propagation_packet(self, data, packet): try: - if packet.destination_type != RNS.Destination.LINK: - pass + if packet.destination_type != RNS.Destination.LINK: return else: - data = msgpack.unpackb(data) + data = msgpack.unpackb(data) remote_timebase = data[0] + messages = data[1] + + ####################################### + # TODO: Check propagation stamps here # + ####################################### + stamps_valid = False - messages = data[1] for lxmf_data in messages: self.lxmf_propagation(lxmf_data) self.client_propagation_messages_received += 1 - packet.prove() + if stamps_valid: packet.prove() except Exception as e: RNS.log("Exception occurred while parsing incoming LXMF propagation data.", RNS.LOG_ERROR) @@ -2053,34 +2061,49 @@ class LXMRouter: if type(data) == list and len(data) == 2 and type(data[0] == float) and type(data[1]) == list: # This is a series of propagation messages from a peer or originator - remote_timebase = data[0] - remote_hash = None - remote_str = "unknown peer" remote_identity = resource.link.get_remote_identity() + remote_timebase = data[0] + messages = data[1] + remote_hash = None + remote_str = "unknown peer" if remote_identity != None: remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") - remote_hash = remote_destination.hash - remote_str = RNS.prettyhexrep(remote_hash) + remote_hash = remote_destination.hash + remote_app_data = RNS.Identity.recall_app_data(remote_hash) + remote_str = RNS.prettyhexrep(remote_hash) - if not remote_hash in self.peers: - if self.autopeer and RNS.Transport.hops_to(remote_hash) <= self.autopeer_maxdepth: - # TODO: Query cache for an announce and get propagation - # transfer limit from that. For now, initialise it to a - # sane default value, and wait for an announce to arrive - # that will update the peering config to the actual limit. - propagation_transfer_limit = LXMRouter.PROPAGATION_LIMIT//4 - wanted_inbound_peers = None - self.peer(remote_hash, remote_timebase, propagation_transfer_limit, wanted_inbound_peers) + if remote_hash in self.peers: remote_str = f"peer {remote_str}" else: - remote_str = f"peer {remote_str}" + if pn_announce_data_is_valid(remote_app_data): + # 1: Current node timebase + # 2: Boolean flag signalling propagation node state + # 3: Per-transfer limit for message propagation in kilobytes + # 4: Limit for incoming propagation node syncs + # 5: Propagation stamp cost for this node + # 6: Node metadata + if remote_app_data[2] and self.autopeer and RNS.Transport.hops_to(remote_hash) <= self.autopeer_maxdepth: + remote_timebase = remote_app_data[1] + remote_transfer_limit = remote_app_data[3] + remote_sync_limit = remote_app_data[4] + remote_stamp_cost = remote_app_data[5][0] + remote_stamp_flex = remote_app_data[5][1] + remote_metadata = remote_app_data[6] + + RNS.log(f"Auto-peering with {remote_str} discovered via incoming sync", RNS.LOG_DEBUG) # TODO: Remove debug + self.peer(remote_hash, remote_timebase, remote_transfer_limit, remote_sync_limit, remote_stamp_cost, remote_stamp_flex, remote_metadata) - messages = data[1] ms = "" if len(messages) == 1 else "s" RNS.log(f"Received {len(messages)} message{ms} from {remote_str}", RNS.LOG_VERBOSE) + + ####################################### + # TODO: Check propagation stamps here # + ####################################### + for lxmf_data in messages: peer = None transient_id = RNS.Identity.full_hash(lxmf_data) + if remote_hash != None and remote_hash in self.peers: peer = self.peers[remote_hash] peer.incoming += 1 @@ -2093,8 +2116,7 @@ class LXMRouter: self.client_propagation_messages_received += 1 self.lxmf_propagation(lxmf_data, from_peer=peer) - if peer != None: - peer.queue_handled_message(transient_id) + if peer != None: peer.queue_handled_message(transient_id) else: RNS.log("Invalid data structure received at propagation destination, ignoring", RNS.LOG_DEBUG) @@ -2121,10 +2143,9 @@ class LXMRouter: if peer != from_peer: peer.queue_unhandled_message(transient_id) - def lxmf_propagation(self, lxmf_data, signal_local_delivery=None, signal_duplicate=None, allow_duplicate=False, is_paper_message=False, from_peer=None): - no_stamp_enforcement = False - if is_paper_message: - no_stamp_enforcement = True + def lxmf_propagation(self, lxmf_data, signal_local_delivery=None, signal_duplicate=None, allow_duplicate=False, is_paper_message=False, from_peer=None, stamp_value=None): + if is_paper_message: no_stamp_enforcement = True + else: no_stamp_enforcement = False try: if len(lxmf_data) >= LXMessage.LXMF_OVERHEAD: @@ -2150,13 +2171,13 @@ class LXMRouter: else: if self.propagation_node: - file_path = self.messagepath+"/"+RNS.hexrep(transient_id, delimit=False)+"_"+str(received) - msg_file = open(file_path, "wb") - msg_file.write(lxmf_data) - msg_file.close() + value_component = f"_{stamp_value}" if stamp_value and stamp_value > 0 else "" + file_path = f"{self.messagepath}/{RNS.hexrep(transient_id, delimit=False)}_{received}{value_component}" + msg_file = open(file_path, "wb") + msg_file.write(lxmf_data); msg_file.close() RNS.log("Received propagated LXMF message "+RNS.prettyhexrep(transient_id)+", adding to peer distribution queues...", RNS.LOG_EXTREME) - self.propagation_entries[transient_id] = [destination_hash, file_path, time.time(), len(lxmf_data), [], []] + self.propagation_entries[transient_id] = [destination_hash, file_path, time.time(), len(lxmf_data), [], [], stamp_value] self.enqueue_peer_distribution(transient_id, from_peer) else: diff --git a/LXMF/LXStamper.py b/LXMF/LXStamper.py index 2db0598..ecf75a3 100644 --- a/LXMF/LXStamper.py +++ b/LXMF/LXStamper.py @@ -17,12 +17,10 @@ def stamp_workblock(message_id, expand_rounds=WORKBLOCK_EXPAND_ROUNDS): wb_st = time.time() workblock = b"" for n in range(expand_rounds): - workblock += RNS.Cryptography.hkdf( - length=256, - derive_from=message_id, - salt=RNS.Identity.full_hash(message_id+msgpack.packb(n)), - context=None, - ) + workblock += RNS.Cryptography.hkdf(length=256, + derive_from=message_id, + salt=RNS.Identity.full_hash(message_id+msgpack.packb(n)), + context=None) wb_time = time.time() - wb_st # RNS.log(f"Stamp workblock size {RNS.prettysize(len(workblock))}, generated in {round(wb_time*1000,2)}ms", RNS.LOG_DEBUG) From 60bf99d15108330ae443bb56417984b9c471a432 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 31 Oct 2025 00:25:46 +0100 Subject: [PATCH 344/436] Cleanup --- LXMF/LXMRouter.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index ace090a..fd6130d 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -905,11 +905,6 @@ class LXMRouter: self.save_outbound_stamp_costs() threading.Thread(target=self.save_outbound_stamp_costs, daemon=True).start() - def get_wanted_inbound_peers(self): - # TODO: Implement/rethink. - # Probably not necessary anymore. - return None - def get_announce_app_data(self, destination_hash): if destination_hash in self.delivery_destinations: delivery_destination = self.delivery_destinations[destination_hash] From ebc8bb33c25445f94305e400e674e3fab81a0104 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 31 Oct 2025 00:28:06 +0100 Subject: [PATCH 345/436] Cleanup --- LXMF/LXMRouter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index fd6130d..7e4627a 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -901,8 +901,7 @@ class LXMRouter: RNS.log(f"Updating outbound stamp cost for {RNS.prettyhexrep(destination_hash)} to {stamp_cost}", RNS.LOG_DEBUG) self.outbound_stamp_costs[destination_hash] = [time.time(), stamp_cost] - def job(): - self.save_outbound_stamp_costs() + def job(): self.save_outbound_stamp_costs() threading.Thread(target=self.save_outbound_stamp_costs, daemon=True).start() def get_announce_app_data(self, destination_hash): From 9c646aead7f983f181c6aad5cde0359950aecb5d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 31 Oct 2025 02:19:24 +0100 Subject: [PATCH 346/436] Stamp validation on incoming propagation node transfers --- LXMF/LXMRouter.py | 21 ++++++++----- LXMF/LXStamper.py | 79 ++++++++++++++++++++--------------------------- 2 files changed, 48 insertions(+), 52 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 7e4627a..d4108fb 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -2000,10 +2000,13 @@ class LXMRouter: ####################################### # TODO: Check propagation stamps here # ####################################### - stamps_valid = False + target_cost = max(0, self.propagation_stamp_cost-self.propagation_stamp_cost_flexibility) + validated_messages = LXStamper.validate_pn_stamps(messages, target_cost) - for lxmf_data in messages: - self.lxmf_propagation(lxmf_data) + for validated_entry in validated_messages: + lxmf_data = validated_entry[1] + stamp_value = validated_entry[2] + self.lxmf_propagation(lxmf_data, stamp_value=stamp_value) self.client_propagation_messages_received += 1 if stamps_valid: packet.prove() @@ -2093,10 +2096,14 @@ class LXMRouter: ####################################### # TODO: Check propagation stamps here # ####################################### + target_cost = max(0, self.propagation_stamp_cost-self.propagation_stamp_cost_flexibility) + validated_messages = LXStamper.validate_pn_stamps(messages, target_cost) - for lxmf_data in messages: - peer = None - transient_id = RNS.Identity.full_hash(lxmf_data) + for validated_entry in validated_messages: + transient_id = validated_entry[0] + lxmf_data = validated_entry[1] + stamp_value = validated_entry[2] + peer = None if remote_hash != None and remote_hash in self.peers: peer = self.peers[remote_hash] @@ -2109,7 +2116,7 @@ class LXMRouter: else: self.client_propagation_messages_received += 1 - self.lxmf_propagation(lxmf_data, from_peer=peer) + self.lxmf_propagation(lxmf_data, from_peer=peer, stamp_value=stamp_value) if peer != None: peer.queue_handled_message(transient_id) else: diff --git a/LXMF/LXStamper.py b/LXMF/LXStamper.py index ecf75a3..9d85329 100644 --- a/LXMF/LXStamper.py +++ b/LXMF/LXStamper.py @@ -4,6 +4,7 @@ import RNS.vendor.umsgpack as msgpack import os import time import math +import itertools import multiprocessing WORKBLOCK_EXPAND_ROUNDS = 3000 @@ -43,46 +44,42 @@ def stamp_valid(stamp, target_cost, workblock): if int.from_bytes(result, byteorder="big") > target: return False else: return True -def validate_pn_stamp(transient_id, stamp): - target_cost = 8 - workblock = stamp_workblock(transient_id, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PN) - if stamp_valid(stamp, target_cost, workblock): - RNS.log(f"Stamp on {RNS.prettyhexrep(transient_id)} validated", RNS.LOG_DEBUG) - value = stamp_value(workblock, stamp) - return True - - return False - -def validate_pn_stamps_job_simple(transient_stamps): - for entry in transient_stamps: - # Get transient ID and stamp for validation - transient_id = transient_stamps[0] - stamp = transient_stamps[1] +def validate_pn_stamp(transient_data, target_cost): + from .LXMessage import LXMessage + if len(transient_data) <= LXMessage.LXMF_OVERHEAD+STAMP_SIZE: return False, None, None + else: + lxm_data = transient_data[:-STAMP_SIZE] + stamp = transient_data[-STAMP_SIZE:] + transient_id = RNS.Identity.full_hash(lxm_data) + workblock = stamp_workblock(transient_id, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PN) - # Store validation result back into list - transient_stamps[2] = validate_pn_stamp(transient_id, stamp) + if not stamp_valid(stamp, target_cost, workblock): return False, None, None + else: + value = stamp_value(workblock, stamp) + return True, transient_id, value - return transient_stamps +def validate_pn_stamps_job_simple(transient_list, target_cost): + validated_messages = [] + for transient_data in transient_list: + stamp_valid, transient_id, value = validate_pn_stamp(transient_data, target_cost) + if stamp_valid: validated_messages.append([transient_id, transient_data, value]) -def _validate_single_pn_stamp_entry(entry): - transient_id = entry[0] - stamp = entry[1] - entry[2] = validate_pn_stamp(transient_id, stamp) - return entry + return validated_messages -def validate_pn_stamps_job_multip(transient_stamps): +def validate_pn_stamps_job_multip(transient_list, target_cost): cores = multiprocessing.cpu_count() - pool_count = min(cores, math.ceil(len(transient_stamps) / PN_VALIDATION_POOL_MIN_SIZE)) + pool_count = min(cores, math.ceil(len(transient_list) / PN_VALIDATION_POOL_MIN_SIZE)) - RNS.log(f"Validating {len(transient_stamps)} stamps using {pool_count} processes...", RNS.LOG_VERBOSE) - with multiprocessing.Pool(pool_count) as p: validated_entries = p.map(_validate_single_pn_stamp_entry, transient_stamps) + RNS.log(f"Validating {len(transient_list)} stamps using {pool_count} processes...", RNS.LOG_VERBOSE) + with multiprocessing.Pool(pool_count) as p: + validated_entries = p.starmap(validate_pn_stamp, zip(transient_list, itertools.repeat(target_cost))) - return validated_entries + return [e for e in validated_entries if e[0] == True] -def validate_pn_stamps(transient_stamps): +def validate_pn_stamps(transient_list, target_cost): non_mp_platform = RNS.vendor.platformutils.is_android() - if len(transient_stamps) <= PN_VALIDATION_POOL_MIN_SIZE or non_mp_platform: validate_pn_stamps_job_simple(transient_stamps) - else: validate_pn_stamps_job_multip(transient_stamps) + if len(transient_list) <= PN_VALIDATION_POOL_MIN_SIZE or non_mp_platform: return validate_pn_stamps_job_simple(transient_list, target_cost) + else: return validate_pn_stamps_job_multip(transient_list, target_cost) def generate_stamp(message_id, stamp_cost): RNS.log(f"Generating stamp with cost {stamp_cost} for {RNS.prettyhexrep(message_id)}...", RNS.LOG_DEBUG) @@ -93,19 +90,13 @@ def generate_stamp(message_id, stamp_cost): rounds = 0 value = 0 - if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): - stamp, rounds = job_simple(stamp_cost, workblock, message_id) - - elif RNS.vendor.platformutils.is_android(): - stamp, rounds = job_android(stamp_cost, workblock, message_id) - - else: - stamp, rounds = job_linux(stamp_cost, workblock, message_id) + if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): stamp, rounds = job_simple(stamp_cost, workblock, message_id) + elif RNS.vendor.platformutils.is_android(): stamp, rounds = job_android(stamp_cost, workblock, message_id) + else: stamp, rounds = job_linux(stamp_cost, workblock, message_id) duration = time.time() - start_time speed = rounds/duration - if stamp != None: - value = stamp_value(workblock, stamp) + if stamp != None: value = stamp_value(workblock, stamp) RNS.log(f"Stamp with value {value} generated in {RNS.prettytime(duration)}, {rounds} rounds, {int(speed)} rounds per second", RNS.LOG_DEBUG) @@ -161,10 +152,8 @@ def job_simple(stamp_cost, workblock, message_id): def sv(s, c, w): target = 0b1<<256-c; m = w+s result = RNS.Identity.full_hash(m) - if int.from_bytes(result, byteorder="big") > target: - return False - else: - return True + if int.from_bytes(result, byteorder="big") > target: return False + else: return True while not sv(pstamp, stamp_cost, workblock) and not active_jobs[message_id]: pstamp = os.urandom(256//8); rounds += 1 From 434267784d65682b24782eed73f00666468ed02e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 31 Oct 2025 13:53:59 +0100 Subject: [PATCH 347/436] Implemented propagation node peering key generation and peering cost signalling --- LXMF/Handlers.py | 3 + LXMF/LXMF.py | 16 ++--- LXMF/LXMPeer.py | 133 ++++++++++++++++++++++++++++++++++------- LXMF/LXMRouter.py | 126 ++++++++++++++++++++++---------------- LXMF/LXStamper.py | 31 ++++++---- LXMF/Utilities/lxmd.py | 44 +++++++++++++- 6 files changed, 257 insertions(+), 96 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index fc980c3..f55cc76 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -51,6 +51,7 @@ class LXMFPropagationAnnounceHandler: propagation_sync_limit = int(data[4]) propagation_stamp_cost = int(data[5][0]) propagation_stamp_cost_flexibility = int(data[5][1]) + peering_cost = int(data[5][2]) metadata = data[6] if destination_hash in self.lxmrouter.static_peers: @@ -60,6 +61,7 @@ class LXMFPropagationAnnounceHandler: propagation_sync_limit=propagation_sync_limit, propagation_stamp_cost=propagation_stamp_cost, propagation_stamp_cost_flexibility=propagation_stamp_cost_flexibility, + peering_cost=peering_cost, metadata=metadata) else: @@ -72,6 +74,7 @@ class LXMFPropagationAnnounceHandler: propagation_sync_limit=propagation_sync_limit, propagation_stamp_cost=propagation_stamp_cost, propagation_stamp_cost_flexibility=propagation_stamp_cost_flexibility, + peering_cost=peering_cost, metadata=metadata) else: diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index b608ceb..3a20b0e 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -154,18 +154,20 @@ def pn_announce_data_is_valid(data): if len(data) < 7: raise ValueError("Invalid announce data: Insufficient peer data") else: try: int(data[1]) - except: raise ValueError("Invalid announce data: Could not decode peer timebase") + except: raise ValueError("Invalid announce data: Could not decode timebase") if data[2] != True and data[2] != False: raise ValueError("Invalid announce data: Indeterminate propagation node status") try: int(data[3]) - except: raise ValueError("Invalid announce data: Could not decode peer propagation transfer limit") + except: raise ValueError("Invalid announce data: Could not decode propagation transfer limit") try: int(data[4]) - except: raise ValueError("Invalid announce data: Could not decode peer propagation sync limit") - if type(data[4]) != list: raise ValueError("Invalid announce data: Could not decode peer stamp costs") + except: raise ValueError("Invalid announce data: Could not decode propagation sync limit") + if type(data[4]) != list: raise ValueError("Invalid announce data: Could not decode stamp costs") try: int(data[5][0]) - except: raise ValueError("Invalid announce data: Could not decode peer target stamp cost") + except: raise ValueError("Invalid announce data: Could not decode target stamp cost") try: int(data[5][1]) - except: raise ValueError("Invalid announce data: Could not decode peer stamp cost flexibility") - if type(data[6]) != dict: raise ValueError("Invalid announce data: Could not decode peer metadata") + except: raise ValueError("Invalid announce data: Could not decode stamp cost flexibility") + try: int(data[5][2]) + except: raise ValueError("Invalid announce data: Could not decode peering cost") + if type(data[6]) != dict: raise ValueError("Invalid announce data: Could not decode metadata") except Exception as e: RNS.log(f"Could not validate propagation node announce data: {e}", RNS.LOG_DEBUG) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index e767313..0fe1e74 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -1,8 +1,10 @@ import os import time +import threading import RNS import RNS.vendor.umsgpack as msgpack +import LXMF.LXStamper as LXStamper from collections import deque from .LXMF import APP_NAME @@ -20,6 +22,7 @@ class LXMPeer: ERROR_NO_IDENTITY = 0xf0 ERROR_NO_ACCESS = 0xf1 + ERROR_THROTTLED = 0xf2 ERROR_TIMEOUT = 0xfe STRATEGY_LAZY = 0x01 @@ -80,6 +83,11 @@ class LXMPeer: except: peer.propagation_stamp_cost_flexibility = None else: peer.propagation_stamp_cost_flexibility = None + if "peering_cost" in dictionary: + try: peer.peering_cost = int(dictionary["peering_cost"]) + except: peer.peering_cost = None + else: peer.peering_cost = None + if "sync_strategy" in dictionary: try: peer.sync_strategy = int(dictionary["sync_strategy"]) except: peer.sync_strategy = LXMPeer.DEFAULT_SYNC_STRATEGY @@ -97,6 +105,8 @@ class LXMPeer: else: peer.tx_bytes = 0 if "last_sync_attempt" in dictionary: peer.last_sync_attempt = dictionary["last_sync_attempt"] else: peer.last_sync_attempt = 0 + if "peering_key" in dictionary: peer.peering_key = dictionary["peering_key"] + else: peer.peering_key = None hm_count = 0 for transient_id in dictionary["handled_ids"]: @@ -123,6 +133,8 @@ class LXMPeer: dictionary["peering_timebase"] = self.peering_timebase dictionary["alive"] = self.alive dictionary["last_heard"] = self.last_heard + dictionary["sync_strategy"] = self.sync_strategy + dictionary["peering_key"] = self.peering_key dictionary["destination_hash"] = self.destination_hash dictionary["link_establishment_rate"] = self.link_establishment_rate dictionary["sync_transfer_rate"] = self.sync_transfer_rate @@ -130,7 +142,7 @@ class LXMPeer: dictionary["propagation_sync_limit"] = self.propagation_sync_limit dictionary["propagation_stamp_cost"] = self.propagation_stamp_cost dictionary["propagation_stamp_cost_flexibility"] = self.propagation_stamp_cost_flexibility - dictionary["sync_strategy"] = self.sync_strategy + dictionary["peering_cost"] = self.peering_cost dictionary["last_sync_attempt"] = self.last_sync_attempt dictionary["offered"] = self.offered dictionary["outgoing"] = self.outgoing @@ -155,16 +167,18 @@ class LXMPeer: return peer_bytes def __init__(self, router, destination_hash, sync_strategy=DEFAULT_SYNC_STRATEGY): - self.alive = False - self.last_heard = 0 + self.alive = False + self.last_heard = 0 self.sync_strategy = sync_strategy + self.peering_key = None + self.peering_cost = None - self.next_sync_attempt = 0 - self.last_sync_attempt = 0 - self.sync_backoff = 0 - self.peering_timebase = 0 + self.next_sync_attempt = 0 + self.last_sync_attempt = 0 + self.sync_backoff = 0 + self.peering_timebase = 0 self.link_establishment_rate = 0 - self.sync_transfer_rate = 0 + self.sync_transfer_rate = 0 self.propagation_transfer_limit = None self.propagation_sync_limit = None @@ -185,6 +199,8 @@ class LXMPeer: self._hm_counts_synced = False self._um_counts_synced = False + self._peering_key_lock = threading.Lock() + self.link = None self.state = LXMPeer.IDLE @@ -199,11 +215,74 @@ class LXMPeer: self.destination = None RNS.log(f"Could not recall identity for LXMF propagation peer {RNS.prettyhexrep(self.destination_hash)}, will retry identity resolution on next sync", RNS.LOG_WARNING) + def peering_key_ready(self): + if not self.peering_cost: return False + if type(self.peering_key) == list and len(self.peering_key) == 2: + value = self.peering_key[1] + if value >= self.peering_cost: return True + else: + RNS.log(f"Peering key value mismatch for {self}. Current value is {value}, but peer requires {self.peering_cost}. Scheduling regeneration...", RNS.LOG_WARNING) + self.peering_key = None + + return False + + def peering_key_value(self): + if type(self.peering_key) == list and len(self.peering_key) == 2: return self.peering_key[1] + else: return None + + def generate_peering_key(self): + if self.peering_cost == None: return False + with self._peering_key_lock: + if self.peering_key != None: return True + else: + RNS.log(f"Generating peering key for {self}", RNS.LOG_NOTICE) + if self.router.identity == None: + RNS.log(f"Could not update peering key for {self} since the local LXMF router identity is not configured", RNS.LOG_ERROR) + return False + + if self.identity == None: + self.identity = RNS.Identity.recall(destination_hash) + if self.identity == None: + RNS.log(f"Could not update peering key for {self} since its identity could not be recalled", RNS.LOG_ERROR) + return False + + key_material = self.identity.hash+self.router.identity.hash + peering_key, value = LXStamper.generate_stamp(key_material, self.peering_cost, expand_rounds=LXStamper.WORKBLOCK_EXPAND_ROUNDS_PEERING) + if value >= self.peering_cost: + self.peering_key = [peering_key, value] + RNS.log(f"Peering key successfully generated for {self}", RNS.LOG_NOTICE) + return True + + return False + def sync(self): RNS.log("Initiating LXMF Propagation Node sync with peer "+RNS.prettyhexrep(self.destination_hash), RNS.LOG_DEBUG) self.last_sync_attempt = time.time() - if time.time() > self.next_sync_attempt: + sync_time_reached = time.time() > self.next_sync_attempt + stamp_costs_known = self.propagation_stamp_cost != None and self.propagation_stamp_cost_flexibility != None and self.peering_cost != None + peering_key_ready = self.peering_key_ready() + sync_checks = sync_time_reached and stamp_costs_known and peering_key_ready + + if not sync_checks: + try: + if not sync_time_reached: + postpone_reason = " due to previous failures" + if self.last_sync_attempt > self.last_heard: self.alive = False + elif not stamp_costs_known: + postpone_reason = " since its required stamp costs are not yet known" + elif not peering_key_ready: + postpone_reason = " since a peering key has not been generated yet" + def job(): self.generate_peering_key() + threading.Thread(target=job, daemon=True).start() + + delay = self.next_sync_attempt-time.time() + postpone_delay = " for {RNS.prettytime({delay})}" if delay > 0 else "" + RNS.log(f"Postponing sync with peer {RNS.prettyhexrep(self.destination_hash)}{postpone_delay}{postpone_reason}", RNS.LOG_DEBUG) + except Exception as e: + RNS.trace_exception(e) + + else: if not RNS.Transport.has_path(self.destination_hash): RNS.log("No path to peer "+RNS.prettyhexrep(self.destination_hash)+" exists, requesting...", RNS.LOG_DEBUG) RNS.Transport.request_path(self.destination_hash) @@ -219,6 +298,10 @@ class LXMPeer: self.destination = RNS.Destination(self.identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") if self.destination != None: + if len(self.unhandled_messages) == 0: + RNS.log(f"Sync requested for {self}, but no unhandled messages exist for peer. Sync complete.", RNS.LOG_DEBUG) + return + if len(self.unhandled_messages) > 0: if self.currently_transferring_messages != None: RNS.log(f"Sync requested for {self}, but current message transfer index was not clear. Aborting.", RNS.LOG_ERROR) @@ -236,23 +319,31 @@ class LXMPeer: self.alive = True self.last_heard = time.time() self.sync_backoff = 0 + min_accepted_cost = min(0, self.propagation_stamp_cost-self.propagation_stamp_cost_flexibility) - RNS.log("Synchronisation link to peer "+RNS.prettyhexrep(self.destination_hash)+" established, preparing request...", RNS.LOG_DEBUG) + RNS.log("Synchronisation link to peer "+RNS.prettyhexrep(self.destination_hash)+" established, preparing sync offer...", RNS.LOG_DEBUG) unhandled_entries = [] - unhandled_ids = [] - purged_ids = [] + unhandled_ids = [] + purged_ids = [] + low_value_ids = [] for transient_id in self.unhandled_messages: if transient_id in self.router.propagation_entries: - unhandled_entry = [ transient_id, - self.router.get_weight(transient_id), - self.router.get_size(transient_id) ] - - unhandled_entries.append(unhandled_entry) + if self.router.get_stamp_value(transient_id) < min_accepted_cost: low_value_ids.append(transient_id) + else: + unhandled_entry = [ transient_id, + self.router.get_weight(transient_id), + self.router.get_size(transient_id) ] + + unhandled_entries.append(unhandled_entry) else: purged_ids.append(transient_id) for transient_id in purged_ids: - RNS.log("Dropping unhandled message "+RNS.prettyhexrep(transient_id)+" for peer "+RNS.prettyhexrep(self.destination_hash)+" since it no longer exists in the message store.", RNS.LOG_DEBUG) + RNS.log(f"Dropping unhandled message {RNS.prettyhexrep(transient_id)} for peer {RNS.prettyhexrep(self.destination_hash)} since it no longer exists in the message store.", RNS.LOG_DEBUG) + self.remove_unhandled_message(transient_id) + + for transient_id in low_value_ids: + RNS.log(f"Dropping unhandled message {RNS.prettyhexrep(transient_id)} for peer {RNS.prettyhexrep(self.destination_hash)} since its stamp value is lower than peer requirement of {min_accepted_cost}.", RNS.LOG_DEBUG) self.remove_unhandled_message(transient_id) unhandled_entries.sort(key=lambda e: e[1], reverse=False) @@ -284,11 +375,7 @@ class LXMPeer: self.state = LXMPeer.REQUEST_SENT else: - RNS.log("Could not request sync to peer "+RNS.prettyhexrep(self.destination_hash)+" since its identity could not be recalled.", RNS.LOG_ERROR) - - else: - RNS.log("Postponing sync with peer "+RNS.prettyhexrep(self.destination_hash)+" for "+RNS.prettytime(self.next_sync_attempt-time.time())+" due to previous failures", RNS.LOG_DEBUG) - if self.last_sync_attempt > self.last_heard: self.alive = False + RNS.log(f"Could not request sync to peer {RNS.prettyhexrep(self.destination_hash)} since its identity could not be recalled.", RNS.LOG_ERROR) def request_failed(self, request_receipt): RNS.log(f"Sync request to peer {self.destination} failed", RNS.LOG_DEBUG) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index d4108fb..ce6b685 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -45,9 +45,11 @@ class LXMRouter: ROTATION_HEADROOM_PCT = 10 ROTATION_AR_MAX = 0.5 - PROPAGATION_COST = 12 - PROPAGATION_COST_MIN = 10 + PEERING_COST = 10 + MAX_PEERING_COST = 12 + PROPAGATION_COST_MIN = 13 PROPAGATION_COST_FLEX = 3 + PROPAGATION_COST = 16 PROPAGATION_LIMIT = 256 SYNC_LIMIT = PROPAGATION_LIMIT*40 DELIVERY_LIMIT = 1000 @@ -81,7 +83,8 @@ class LXMRouter: propagation_limit=PROPAGATION_LIMIT, delivery_limit=DELIVERY_LIMIT, sync_limit=SYNC_LIMIT, enforce_ratchets=False, enforce_stamps=False, static_peers = [], max_peers=None, from_static_only=False, sync_strategy=LXMPeer.STRATEGY_PERSISTENT, - propagation_cost=PROPAGATION_COST, propagation_cost_flexibility=PROPAGATION_COST_FLEX): + propagation_cost=PROPAGATION_COST, propagation_cost_flexibility=PROPAGATION_COST_FLEX, + peering_cost=PEERING_COST): random.seed(os.urandom(10)) @@ -115,17 +118,20 @@ class LXMRouter: self.outbound_propagation_link = None if delivery_limit == None: delivery_limit = LXMRouter.DELIVERY_LIMIT + if propagation_cost < LXMRouter.PROPAGATION_COST_MIN: propagation_cost = LXMRouter.PROPAGATION_COST_MIN - self.message_storage_limit = None - self.information_storage_limit = None - self.propagation_per_transfer_limit = propagation_limit - self.propagation_per_sync_limit = sync_limit - self.delivery_per_transfer_limit = delivery_limit - self.propagation_stamp_cost = propagation_cost + self.message_storage_limit = None + self.information_storage_limit = None + self.propagation_per_transfer_limit = propagation_limit + self.propagation_per_sync_limit = sync_limit + self.delivery_per_transfer_limit = delivery_limit + self.propagation_stamp_cost = propagation_cost self.propagation_stamp_cost_flexibility = propagation_cost_flexibility - self.enforce_ratchets = enforce_ratchets - self._enforce_stamps = enforce_stamps - self.pending_deferred_stamps = {} + self.peering_cost = peering_cost + self.max_peering_cost = LXMRouter.MAX_PEERING_COST + self.enforce_ratchets = enforce_ratchets + self._enforce_stamps = enforce_stamps + self.pending_deferred_stamps = {} if sync_limit == None or self.propagation_per_sync_limit < self.propagation_per_transfer_limit: self.propagation_per_sync_limit = self.propagation_per_transfer_limit @@ -284,7 +290,7 @@ class LXMRouter: def delayed_announce(): time.sleep(LXMRouter.NODE_ANNOUNCE_DELAY) node_state = self.propagation_node and not self.from_static_only - stamp_cost = [self.propagation_stamp_cost, self.propagation_stamp_cost_flexibility] + stamp_cost = [self.propagation_stamp_cost, self.propagation_stamp_cost_flexibility, self.peering_cost] metadata = {} announce_data = [ False, # 0: Legacy LXMF PN support int(time.time()), # 1: Current node timebase @@ -719,6 +725,8 @@ class LXMRouter: "sync_limit": peer.propagation_sync_limit, "target_stamp_cost": peer.propagation_stamp_cost, "stamp_cost_flexibility": peer.propagation_stamp_cost_flexibility, + "peering_cost": peer.peering_cost, + "peering_key": peer.peering_key_value(), "network_distance": RNS.Transport.hops_to(peer_id), "rx_bytes": peer.rx_bytes, "tx_bytes": peer.tx_bytes, @@ -739,6 +747,8 @@ class LXMRouter: "sync_limit": self.propagation_per_sync_limit, "target_stamp_cost": self.propagation_stamp_cost, "stamp_cost_flexibility": self.propagation_stamp_cost_flexibility, + "peering_cost": self.peering_cost, + "max_peering_cost": self.max_peering_cost, "autopeer_maxdepth": self.autopeer_maxdepth, "from_static_only": self.from_static_only, "messagestore": { @@ -1782,39 +1792,48 @@ class LXMRouter: ### Peer Sync & Propagation ########################### ####################################################### - def peer(self, destination_hash, timestamp, propagation_transfer_limit, propagation_sync_limit, propagation_stamp_cost, propagation_stamp_cost_flexibility): - if destination_hash in self.peers: - peer = self.peers[destination_hash] - if timestamp > peer.peering_timebase: - peer.alive = True - peer.sync_backoff = 0 - peer.next_sync_attempt = 0 - peer.peering_timebase = timestamp - peer.last_heard = time.time() - peer.propagation_stamp_cost = propagation_stamp_cost - peer.propagation_stamp_cost_flexibility = propagation_stamp_cost_flexibility - peer.propagation_transfer_limit = propagation_transfer_limit - if propagation_sync_limit != None: peer.propagation_sync_limit = propagation_sync_limit - else: peer.propagation_sync_limit = propagation_transfer_limit - - RNS.log(f"Peering config updated for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_VERBOSE) - - else: - if len(self.peers) < self.max_peers: - peer = LXMPeer(self, destination_hash, sync_strategy=self.default_sync_strategy) - peer.alive = True - peer.last_heard = time.time() - peer.propagation_stamp_cost = propagation_stamp_cost - peer.propagation_stamp_cost_flexibility = propagation_stamp_cost_flexibility - peer.propagation_transfer_limit = propagation_transfer_limit - if propagation_sync_limit != None: peer.propagation_sync_limit = propagation_sync_limit - else: peer.propagation_sync_limit = propagation_transfer_limit - - self.peers[destination_hash] = peer - RNS.log(f"Peered with {RNS.prettyhexrep(destination_hash)}", RNS.LOG_NOTICE) - + def peer(self, destination_hash, timestamp, propagation_transfer_limit, propagation_sync_limit, propagation_stamp_cost, propagation_stamp_cost_flexibility, peering_cost, metadata): + if peering_cost > self.max_peering_cost: + if destination_hash in self.peers: + RNS.log(f"Peer {RNS.prettyhexrep(destination_hash)} increased peering cost beyond local accepted maximum, breaking peering...", RNS.LOG_NOTICE) + self.unpeer(destination_hash, timestamp) else: - RNS.log(f"Max peers reached, not peering with {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) + RNS.log(f"Not peering with {RNS.prettyhexrep(destination_hash)}, since its peering cost of {peering_cost} exceeds local maximum of {self.max_peering_cost}", RNS.LOG_NOTICE) + + else: + if destination_hash in self.peers: + peer = self.peers[destination_hash] + if timestamp > peer.peering_timebase: + peer.alive = True + peer.sync_backoff = 0 + peer.next_sync_attempt = 0 + peer.peering_timebase = timestamp + peer.last_heard = time.time() + peer.propagation_stamp_cost = propagation_stamp_cost + peer.propagation_stamp_cost_flexibility = propagation_stamp_cost_flexibility + peer.peering_cost = peering_cost + peer.propagation_transfer_limit = propagation_transfer_limit + if propagation_sync_limit != None: peer.propagation_sync_limit = propagation_sync_limit + else: peer.propagation_sync_limit = propagation_transfer_limit + + RNS.log(f"Peering config updated for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_VERBOSE) + + else: + if len(self.peers) >= self.max_peers: RNS.log(f"Max peers reached, not peering with {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) + else: + peer = LXMPeer(self, destination_hash, sync_strategy=self.default_sync_strategy) + peer.alive = True + peer.last_heard = time.time() + peer.propagation_stamp_cost = propagation_stamp_cost + peer.propagation_stamp_cost_flexibility = propagation_stamp_cost_flexibility + peer.peering_cost = peering_cost + peer.propagation_transfer_limit = propagation_transfer_limit + if propagation_sync_limit != None: peer.propagation_sync_limit = propagation_sync_limit + else: peer.propagation_sync_limit = propagation_transfer_limit + + self.peers[destination_hash] = peer + RNS.log(f"Peered with {RNS.prettyhexrep(destination_hash)}", RNS.LOG_NOTICE) + def unpeer(self, destination_hash, timestamp = None): if timestamp == None: @@ -2000,8 +2019,8 @@ class LXMRouter: ####################################### # TODO: Check propagation stamps here # ####################################### - target_cost = max(0, self.propagation_stamp_cost-self.propagation_stamp_cost_flexibility) - validated_messages = LXStamper.validate_pn_stamps(messages, target_cost) + min_accepted_cost = max(0, self.propagation_stamp_cost-self.propagation_stamp_cost_flexibility) + validated_messages = LXStamper.validate_pn_stamps(messages, min_accepted_cost) for validated_entry in validated_messages: lxmf_data = validated_entry[1] @@ -2077,7 +2096,7 @@ class LXMRouter: # 2: Boolean flag signalling propagation node state # 3: Per-transfer limit for message propagation in kilobytes # 4: Limit for incoming propagation node syncs - # 5: Propagation stamp cost for this node + # 5: Propagation stamp costs for this node # 6: Node metadata if remote_app_data[2] and self.autopeer and RNS.Transport.hops_to(remote_hash) <= self.autopeer_maxdepth: remote_timebase = remote_app_data[1] @@ -2085,10 +2104,11 @@ class LXMRouter: remote_sync_limit = remote_app_data[4] remote_stamp_cost = remote_app_data[5][0] remote_stamp_flex = remote_app_data[5][1] + remote_peering_cost = remote_app_data[5][2] remote_metadata = remote_app_data[6] RNS.log(f"Auto-peering with {remote_str} discovered via incoming sync", RNS.LOG_DEBUG) # TODO: Remove debug - self.peer(remote_hash, remote_timebase, remote_transfer_limit, remote_sync_limit, remote_stamp_cost, remote_stamp_flex, remote_metadata) + self.peer(remote_hash, remote_timebase, remote_transfer_limit, remote_sync_limit, remote_stamp_cost, remote_stamp_flex, remote_peering_cost, remote_metadata) ms = "" if len(messages) == 1 else "s" RNS.log(f"Received {len(messages)} message{ms} from {remote_str}", RNS.LOG_VERBOSE) @@ -2096,8 +2116,8 @@ class LXMRouter: ####################################### # TODO: Check propagation stamps here # ####################################### - target_cost = max(0, self.propagation_stamp_cost-self.propagation_stamp_cost_flexibility) - validated_messages = LXStamper.validate_pn_stamps(messages, target_cost) + min_accepted_cost = max(0, self.propagation_stamp_cost-self.propagation_stamp_cost_flexibility) + validated_messages = LXStamper.validate_pn_stamps(messages, min_accepted_cost) for validated_entry in validated_messages: transient_id = validated_entry[0] @@ -2177,13 +2197,13 @@ class LXMRouter: msg_file = open(file_path, "wb") msg_file.write(lxmf_data); msg_file.close() - RNS.log("Received propagated LXMF message "+RNS.prettyhexrep(transient_id)+", adding to peer distribution queues...", RNS.LOG_EXTREME) + RNS.log(f"Received propagated LXMF message {RNS.prettyhexrep(transient_id)} with stamp value {stamp_value}, adding to peer distribution queues...", RNS.LOG_EXTREME) self.propagation_entries[transient_id] = [destination_hash, file_path, time.time(), len(lxmf_data), [], [], stamp_value] self.enqueue_peer_distribution(transient_id, from_peer) else: # TODO: Add message to sneakernet queues when implemented - RNS.log("Received propagated LXMF message "+RNS.prettyhexrep(transient_id)+", but this instance is not hosting a propagation node, discarding message.", RNS.LOG_DEBUG) + RNS.log(f"Received propagated LXMF message {RNS.prettyhexrep(transient_id)}, but this instance is not hosting a propagation node, discarding message.", RNS.LOG_DEBUG) return True diff --git a/LXMF/LXStamper.py b/LXMF/LXStamper.py index 9d85329..4d2e38c 100644 --- a/LXMF/LXStamper.py +++ b/LXMF/LXStamper.py @@ -7,23 +7,24 @@ import math import itertools import multiprocessing -WORKBLOCK_EXPAND_ROUNDS = 3000 -WORKBLOCK_EXPAND_ROUNDS_PN = 1000 -STAMP_SIZE = RNS.Identity.HASHLENGTH -PN_VALIDATION_POOL_MIN_SIZE = 256 +WORKBLOCK_EXPAND_ROUNDS = 3000 +WORKBLOCK_EXPAND_ROUNDS_PEERING = 20000 +WORKBLOCK_EXPAND_ROUNDS_PN = 1000 +STAMP_SIZE = RNS.Identity.HASHLENGTH +PN_VALIDATION_POOL_MIN_SIZE = 256 active_jobs = {} -def stamp_workblock(message_id, expand_rounds=WORKBLOCK_EXPAND_ROUNDS): +def stamp_workblock(material, expand_rounds=WORKBLOCK_EXPAND_ROUNDS): wb_st = time.time() workblock = b"" for n in range(expand_rounds): workblock += RNS.Cryptography.hkdf(length=256, - derive_from=message_id, - salt=RNS.Identity.full_hash(message_id+msgpack.packb(n)), + derive_from=material, + salt=RNS.Identity.full_hash(material+msgpack.packb(n)), context=None) wb_time = time.time() - wb_st - # RNS.log(f"Stamp workblock size {RNS.prettysize(len(workblock))}, generated in {round(wb_time*1000,2)}ms", RNS.LOG_DEBUG) + RNS.log(f"Stamp workblock size {RNS.prettysize(len(workblock))}, generated in {round(wb_time*1000,2)}ms", RNS.LOG_DEBUG) return workblock @@ -81,9 +82,9 @@ def validate_pn_stamps(transient_list, target_cost): if len(transient_list) <= PN_VALIDATION_POOL_MIN_SIZE or non_mp_platform: return validate_pn_stamps_job_simple(transient_list, target_cost) else: return validate_pn_stamps_job_multip(transient_list, target_cost) -def generate_stamp(message_id, stamp_cost): +def generate_stamp(message_id, stamp_cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS): RNS.log(f"Generating stamp with cost {stamp_cost} for {RNS.prettyhexrep(message_id)}...", RNS.LOG_DEBUG) - workblock = stamp_workblock(message_id) + workblock = stamp_workblock(message_id, expand_rounds=expand_rounds) start_time = time.time() stamp = None @@ -362,4 +363,12 @@ if __name__ == "__main__": RNS.loglevel = RNS.LOG_DEBUG RNS.log("Testing LXMF stamp generation", RNS.LOG_DEBUG) message_id = os.urandom(32) - generate_stamp(message_id, cost) \ No newline at end of file + generate_stamp(message_id, cost) + + RNS.log("Testing propagation stamp generation", RNS.LOG_DEBUG) + message_id = os.urandom(32) + generate_stamp(message_id, cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PN) + + RNS.log("Testing peering key generation", RNS.LOG_DEBUG) + message_id = os.urandom(32) + generate_stamp(message_id, cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PEERING) \ No newline at end of file diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 03d1282..b2bc302 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -164,6 +164,20 @@ def apply_config(): else: active_configuration["propagation_stamp_cost_flexibility"] = LXMF.LXMRouter.PROPAGATION_COST_FLEX + if "propagation" in lxmd_config and "peering_cost" in lxmd_config["propagation"]: + active_configuration["peering_cost"] = lxmd_config["propagation"].as_int("peering_cost") + if active_configuration["peering_cost"] < 0: + active_configuration["peering_cost"] = 0 + else: + active_configuration["peering_cost"] = LXMF.LXMRouter.PEERING_COST + + if "propagation" in lxmd_config and "remote_peering_cost_max" in lxmd_config["propagation"]: + active_configuration["remote_peering_cost_max"] = lxmd_config["propagation"].as_int("remote_peering_cost_max") + if active_configuration["remote_peering_cost_max"] < 0: + active_configuration["remote_peering_cost_max"] = 0 + else: + active_configuration["remote_peering_cost_max"] = LXMF.LXMRouter.MAX_PEERING_COST + if "propagation" in lxmd_config and "prioritise_destinations" in lxmd_config["propagation"]: active_configuration["prioritised_lxmf_destinations"] = lxmd_config["propagation"].as_list("prioritise_destinations") else: @@ -579,9 +593,11 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = ssp = s["static_peers"]; cprr = s["clients"]["client_propagation_messages_received"] cprs = s["clients"]["client_propagation_messages_served"]; upi = s["unpeered_propagation_incoming"] psc = s["target_stamp_cost"]; scf = s["stamp_cost_flexibility"] + pc = s["peering_cost"]; pcm = s["max_peering_cost"] print(f"Messagestore contains {mscnt} messages, {msb} ({ms_util} utilised of {msl})") print(f"Accepting propagated messages from {who_str}, {ptl} per-transfer limit") print(f"Required propagation stamp cost is {psc}, flexibility is {scf}") + print(f"Peering cost is {pc}, max remote peering cost is {pcm}") print(f"") print(f"Peers : {stp} total (peer limit is {smp})") print(f" {sdp} discovered, {ssp} static") @@ -613,7 +629,13 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = h = max(time.time()-p["last_heard"], 0) hops = p["network_distance"] hs = "hops unknown" if hops == RNS.Transport.PATHFINDER_M else f"{hops} hop away" if hops == 1 else f"{hops} hops away" - pm = p["messages"] + pm = p["messages"]; pk = p["peering_key"] + pc = p["peering_cost"]; psc = p["target_stamp_cost"]; psf = p["stamp_cost_flexibility"] + if pc == None: pc = "unknown" + if psc == None: psc = "unknown" + if psf == None: psf = "unknown" + if pk == None: pk = "Not generated" + else: pk = f"Generated, value is {pk}" if p["last_sync_attempt"] != 0: lsa = p["last_sync_attempt"] ls = f"last synced {RNS.prettytime(max(time.time()-lsa, 0))} ago" @@ -622,9 +644,11 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = sstr = RNS.prettyspeed(p["str"]); sler = RNS.prettyspeed(p["ler"]); stl = RNS.prettysize(p["transfer_limit"]*1000) srxb = RNS.prettysize(p["rx_bytes"]); stxb = RNS.prettysize(p["tx_bytes"]); pmo = pm["offered"]; pmout = pm["outgoing"] - pmi = pm["incoming"]; pmuh = pm["unhandled"] + pmi = pm["incoming"]; pmuh = pm["unhandled"] print(f"{ind}{t}{RNS.prettyhexrep(peer_id)}") print(f"{ind*2}Status : {a}, {hs}, last heard {RNS.prettytime(h)} ago") + print(f"{ind*2}Costs : Propagation {psc} (flex {psf}), peering {pc}") + print(f"{ind*2}Sync key : {pk}") print(f"{ind*2}Speeds : {sstr} STR, {sler} LER, {stl} transfer limit") print(f"{ind*2}Messages : {pmo} offered, {pmout} outgoing, {pmi} incoming") print(f"{ind*2}Traffic : {srxb} received, {stxb} sent") @@ -752,6 +776,22 @@ autopeer_maxdepth = 4 # propagation_stamp_cost_flexibility = 3 +# The peering_cost option configures the target +# value required for a remote node to peer with +# and deliver messages to this node. + +# peering_cost = 10 + +# You can configure the maximum peering cost +# of remote nodes that this node will peer with. +# Setting this to a higher number will allow +# this node to peer with other nodes requiring +# a high peering key value, but will require +# more computation time during initial peering +# when generating the peering key. + +# remote_peering_cost_max = 12 + # You can tell the LXMF message router to # prioritise storage for one or more # destinations. If the message store reaches From a44c1f368a28e7a73a4d083ec622e52865d43923 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 31 Oct 2025 17:02:34 +0100 Subject: [PATCH 348/436] Validate peering key on incoming sync offer --- LXMF/LXMPeer.py | 8 ++++++-- LXMF/LXMRouter.py | 36 +++++++++++++++++++++--------------- LXMF/LXStamper.py | 16 +++++++++++++++- 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 0fe1e74..7d851f2 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -22,7 +22,9 @@ class LXMPeer: ERROR_NO_IDENTITY = 0xf0 ERROR_NO_ACCESS = 0xf1 - ERROR_THROTTLED = 0xf2 + ERROR_INVALID_KEY = 0xf3 + ERROR_INVALID_DATA = 0xf4 + ERROR_THROTTLED = 0xf5 ERROR_TIMEOUT = 0xfe STRATEGY_LAZY = 0x01 @@ -369,9 +371,11 @@ class LXMPeer: cumulative_size += lxm_transfer_size unhandled_ids.append(transient_id) + offer = [self.peering_key[0], unhandled_ids] + RNS.log(f"Offering {len(unhandled_ids)} messages to peer {RNS.prettyhexrep(self.destination.hash)} ({RNS.prettysize(len(msgpack.packb(unhandled_ids)))})", RNS.LOG_VERBOSE) self.last_offer = unhandled_ids - self.link.request(LXMPeer.OFFER_REQUEST_PATH, unhandled_ids, response_callback=self.offer_response, failed_callback=self.request_failed) + self.link.request(LXMPeer.OFFER_REQUEST_PATH, offer, response_callback=self.offer_response, failed_callback=self.request_failed) self.state = LXMPeer.REQUEST_SENT else: diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index ce6b685..bac498b 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -45,8 +45,8 @@ class LXMRouter: ROTATION_HEADROOM_PCT = 10 ROTATION_AR_MAX = 0.5 - PEERING_COST = 10 - MAX_PEERING_COST = 12 + PEERING_COST = 18 + MAX_PEERING_COST = 24 PROPAGATION_COST_MIN = 13 PROPAGATION_COST_FLEX = 3 PROPAGATION_COST = 16 @@ -2047,21 +2047,30 @@ class LXMRouter: return LXMPeer.ERROR_NO_ACCESS try: - transient_ids = data - wanted_ids = [] + if type(data) != list and len(data) < 2: return LXMPeer.ERROR_INVALID_DATA - for transient_id in transient_ids: - if not transient_id in self.propagation_entries: - wanted_ids.append(transient_id) + peering_id = self.identity.hash+remote_identity + target_cost = self.peering_cost + peering_key = data[0] + transient_ids = data[1] + wanted_ids = [] - if len(wanted_ids) == 0: - return False + ts = time.time() + peering_key_valid = LXStamper.validate_peering_key(peering_id, peering_key, target_cost) + td = time.time() - ts - elif len(wanted_ids) == len(transient_ids): - return True + if not peering_key_valid: + RNS.log(f"Invalid peering key for incoming sync offer", RNS.LOG_DEBUG) + return LXMPeer.ERROR_INVALID_KEY else: - return wanted_ids + RNS.log(f"Peering key validated for incoming offer in {RNS.prettytime(td)}", RNS.LOG_DEBUG) + for transient_id in transient_ids: + if not transient_id in self.propagation_entries: wanted_ids.append(transient_id) + + if len(wanted_ids) == 0: return False + elif len(wanted_ids) == len(transient_ids): return True + else: return wanted_ids except Exception as e: RNS.log("Error occurred while generating response for sync request, the contained exception was: "+str(e), RNS.LOG_DEBUG) @@ -2069,9 +2078,6 @@ class LXMRouter: def propagation_resource_concluded(self, resource): if resource.status == RNS.Resource.COMPLETE: - # TODO: The peer this was received from should - # have the transient id added to its list of - # already handled messages. try: data = msgpack.unpackb(resource.data.read()) diff --git a/LXMF/LXStamper.py b/LXMF/LXStamper.py index 4d2e38c..2a74295 100644 --- a/LXMF/LXStamper.py +++ b/LXMF/LXStamper.py @@ -8,8 +8,8 @@ import itertools import multiprocessing WORKBLOCK_EXPAND_ROUNDS = 3000 -WORKBLOCK_EXPAND_ROUNDS_PEERING = 20000 WORKBLOCK_EXPAND_ROUNDS_PN = 1000 +WORKBLOCK_EXPAND_ROUNDS_PEERING = 25 STAMP_SIZE = RNS.Identity.HASHLENGTH PN_VALIDATION_POOL_MIN_SIZE = 256 @@ -45,6 +45,11 @@ def stamp_valid(stamp, target_cost, workblock): if int.from_bytes(result, byteorder="big") > target: return False else: return True +def validate_peering_key(peering_id, peering_key, target_cost): + workblock = stamp_workblock(peering_id, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PEERING) + if not stamp_valid(peering_key, target_cost, workblock): return False + else: return True + def validate_pn_stamp(transient_data, target_cost): from .LXMessage import LXMessage if len(transient_data) <= LXMessage.LXMF_OVERHEAD+STAMP_SIZE: return False, None, None @@ -348,6 +353,13 @@ def job_android(stamp_cost, workblock, message_id): return stamp, total_rounds +# def stamp_value_linear(workblock, stamp): +# value = 0 +# bits = 256 +# material = RNS.Identity.full_hash(workblock+stamp) +# s = int.from_bytes(material, byteorder="big") +# return s.bit_count() + if __name__ == "__main__": import sys if len(sys.argv) < 2: @@ -365,10 +377,12 @@ if __name__ == "__main__": message_id = os.urandom(32) generate_stamp(message_id, cost) + RNS.log("", RNS.LOG_DEBUG) RNS.log("Testing propagation stamp generation", RNS.LOG_DEBUG) message_id = os.urandom(32) generate_stamp(message_id, cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PN) + RNS.log("", RNS.LOG_DEBUG) RNS.log("Testing peering key generation", RNS.LOG_DEBUG) message_id = os.urandom(32) generate_stamp(message_id, cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PEERING) \ No newline at end of file From 606a723e315faf404a5da4bc42cbe565dd4af8a1 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 31 Oct 2025 17:11:17 +0100 Subject: [PATCH 349/436] Implemented getting static peer configuration from network if unknown at launch --- LXMF/Handlers.py | 75 ++++++++++++++++++++++++----------------------- LXMF/LXMRouter.py | 17 +++-------- 2 files changed, 42 insertions(+), 50 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index f55cc76..a65978c 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -39,22 +39,23 @@ class LXMFPropagationAnnounceHandler: self.lxmrouter = lxmrouter def received_announce(self, destination_hash, announced_identity, app_data, announce_packet_hash, is_path_response): - if not is_path_response: - try: - if type(app_data) == bytes: - if self.lxmrouter.propagation_node: - data = msgpack.unpackb(app_data) - if pn_announce_data_is_valid(data): - node_timebase = int(data[1]) - propagation_enabled = data[2] - propagation_transfer_limit = int(data[3]) - propagation_sync_limit = int(data[4]) - propagation_stamp_cost = int(data[5][0]) - propagation_stamp_cost_flexibility = int(data[5][1]) - peering_cost = int(data[5][2]) - metadata = data[6] - - if destination_hash in self.lxmrouter.static_peers: + try: + if type(app_data) == bytes: + if self.lxmrouter.propagation_node: + data = msgpack.unpackb(app_data) + if pn_announce_data_is_valid(data): + node_timebase = int(data[1]) + propagation_enabled = data[2] + propagation_transfer_limit = int(data[3]) + propagation_sync_limit = int(data[4]) + propagation_stamp_cost = int(data[5][0]) + propagation_stamp_cost_flexibility = int(data[5][1]) + peering_cost = int(data[5][2]) + metadata = data[6] + + if destination_hash in self.lxmrouter.static_peers: + static_peer = self.lxmrouter.static_peers[destination_hash] + if not is_path_response or static_peer.last_heard == 0: self.lxmrouter.peer(destination_hash=destination_hash, timestamp=node_timebase, propagation_transfer_limit=propagation_transfer_limit, @@ -64,27 +65,27 @@ class LXMFPropagationAnnounceHandler: peering_cost=peering_cost, metadata=metadata) - else: - if self.lxmrouter.autopeer: - if propagation_enabled == True: - if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: - self.lxmrouter.peer(destination_hash=destination_hash, - timestamp=node_timebase, - propagation_transfer_limit=propagation_transfer_limit, - propagation_sync_limit=propagation_sync_limit, - propagation_stamp_cost=propagation_stamp_cost, - propagation_stamp_cost_flexibility=propagation_stamp_cost_flexibility, - peering_cost=peering_cost, - metadata=metadata) + else: + if self.lxmrouter.autopeer and not is_path_response: + if propagation_enabled == True: + if RNS.Transport.hops_to(destination_hash) <= self.lxmrouter.autopeer_maxdepth: + self.lxmrouter.peer(destination_hash=destination_hash, + timestamp=node_timebase, + propagation_transfer_limit=propagation_transfer_limit, + propagation_sync_limit=propagation_sync_limit, + propagation_stamp_cost=propagation_stamp_cost, + propagation_stamp_cost_flexibility=propagation_stamp_cost_flexibility, + peering_cost=peering_cost, + metadata=metadata) - else: - if destination_hash in self.lxmrouter.peers: - RNS.log(f"Peer {self.lxmrouter.peers[destination_hash]} moved outside auto-peering range, breaking peering...") - self.lxmrouter.unpeer(destination_hash, node_timebase) + else: + if destination_hash in self.lxmrouter.peers: + RNS.log(f"Peer {self.lxmrouter.peers[destination_hash]} moved outside auto-peering range, breaking peering...") + self.lxmrouter.unpeer(destination_hash, node_timebase) - elif propagation_enabled == False: - self.lxmrouter.unpeer(destination_hash, node_timebase) + elif propagation_enabled == False: + self.lxmrouter.unpeer(destination_hash, node_timebase) - except Exception as e: - RNS.log("Error while evaluating propagation node announce, ignoring announce.", RNS.LOG_DEBUG) - RNS.log(f"The contained exception was: {str(e)}", RNS.LOG_DEBUG) + except Exception as e: + RNS.log("Error while evaluating propagation node announce, ignoring announce.", RNS.LOG_DEBUG) + RNS.log(f"The contained exception was: {str(e)}", RNS.LOG_DEBUG) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index bac498b..798cd5a 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -540,9 +540,6 @@ class LXMRouter: peer = LXMPeer.from_bytes(serialised_peer, self) del serialised_peer if peer.destination_hash in self.static_peers and peer.last_heard == 0: - # TODO: Allow path request responses through announce handler - # momentarily here, so peering config can be updated even if - # the static peer is not available to directly send an announce. RNS.Transport.request_path(peer.destination_hash) if peer.identity != None: self.peers[peer.destination_hash] = peer @@ -2012,14 +2009,11 @@ class LXMRouter: try: if packet.destination_type != RNS.Destination.LINK: return else: - data = msgpack.unpackb(data) - remote_timebase = data[0] - messages = data[1] + data = msgpack.unpackb(data) + remote_timebase = data[0] + messages = data[1] - ####################################### - # TODO: Check propagation stamps here # - ####################################### - min_accepted_cost = max(0, self.propagation_stamp_cost-self.propagation_stamp_cost_flexibility) + min_accepted_cost = max(0, self.propagation_stamp_cost-self.propagation_stamp_cost_flexibility) validated_messages = LXStamper.validate_pn_stamps(messages, min_accepted_cost) for validated_entry in validated_messages: @@ -2119,9 +2113,6 @@ class LXMRouter: ms = "" if len(messages) == 1 else "s" RNS.log(f"Received {len(messages)} message{ms} from {remote_str}", RNS.LOG_VERBOSE) - ####################################### - # TODO: Check propagation stamps here # - ####################################### min_accepted_cost = max(0, self.propagation_stamp_cost-self.propagation_stamp_cost_flexibility) validated_messages = LXStamper.validate_pn_stamps(messages, min_accepted_cost) From 704b37dc167abe6dcc59d3afd36ed62b1882278f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 31 Oct 2025 21:45:40 +0100 Subject: [PATCH 350/436] Implemented client-side propagation stamp generation and inclusion in outbound propagation messages --- LXMF/Handlers.py | 4 +- LXMF/LXMF.py | 2 +- LXMF/LXMPeer.py | 3 +- LXMF/LXMRouter.py | 191 +++++++++++++++++++++++++++++++---------- LXMF/LXMessage.py | 90 +++++++++++++------ LXMF/LXStamper.py | 16 ++-- LXMF/Utilities/lxmd.py | 12 ++- 7 files changed, 228 insertions(+), 90 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index a65978c..aa39ea2 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -42,8 +42,8 @@ class LXMFPropagationAnnounceHandler: try: if type(app_data) == bytes: if self.lxmrouter.propagation_node: - data = msgpack.unpackb(app_data) - if pn_announce_data_is_valid(data): + if pn_announce_data_is_valid(app_data): + data = msgpack.unpackb(app_data) node_timebase = int(data[1]) propagation_enabled = data[2] propagation_transfer_limit = int(data[3]) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 3a20b0e..fd2abf0 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -160,7 +160,7 @@ def pn_announce_data_is_valid(data): except: raise ValueError("Invalid announce data: Could not decode propagation transfer limit") try: int(data[4]) except: raise ValueError("Invalid announce data: Could not decode propagation sync limit") - if type(data[4]) != list: raise ValueError("Invalid announce data: Could not decode stamp costs") + if type(data[5]) != list: raise ValueError("Invalid announce data: Could not decode stamp costs") try: int(data[5][0]) except: raise ValueError("Invalid announce data: Could not decode target stamp cost") try: int(data[5][1]) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 7d851f2..199ee2d 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -24,7 +24,8 @@ class LXMPeer: ERROR_NO_ACCESS = 0xf1 ERROR_INVALID_KEY = 0xf3 ERROR_INVALID_DATA = 0xf4 - ERROR_THROTTLED = 0xf5 + ERROR_INVALID_STAMP = 0xf5 + ERROR_THROTTLED = 0xf6 ERROR_TIMEOUT = 0xfe STRATEGY_LAZY = 0x01 diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 798cd5a..39ee7ec 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -159,6 +159,7 @@ class LXMRouter: self.identity = identity self.propagation_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation") + self.propagation_destination.set_default_app_data(self.get_propagation_node_app_data) self.control_destination = None self.client_propagation_messages_received = 0 self.client_propagation_messages_served = 0 @@ -286,22 +287,24 @@ class LXMRouter: if destination_hash in self.delivery_destinations: self.delivery_destinations[destination_hash].announce(app_data=self.get_announce_app_data(destination_hash), attached_interface=attached_interface) + def get_propagation_node_app_data(self): + node_state = self.propagation_node and not self.from_static_only + stamp_cost = [self.propagation_stamp_cost, self.propagation_stamp_cost_flexibility, self.peering_cost] + metadata = {} + announce_data = [ False, # 0: Legacy LXMF PN support + int(time.time()), # 1: Current node timebase + node_state, # 2: Boolean flag signalling propagation node state + self.propagation_per_transfer_limit, # 3: Per-transfer limit for message propagation in kilobytes + self.propagation_per_sync_limit, # 4: Limit for incoming propagation node syncs + stamp_cost, # 5: Propagation stamp cost for this node + metadata ] # 6: Node metadata + + return msgpack.packb(announce_data) + def announce_propagation_node(self): def delayed_announce(): time.sleep(LXMRouter.NODE_ANNOUNCE_DELAY) - node_state = self.propagation_node and not self.from_static_only - stamp_cost = [self.propagation_stamp_cost, self.propagation_stamp_cost_flexibility, self.peering_cost] - metadata = {} - announce_data = [ False, # 0: Legacy LXMF PN support - int(time.time()), # 1: Current node timebase - node_state, # 2: Boolean flag signalling propagation node state - self.propagation_per_transfer_limit, # 3: Per-transfer limit for message propagation in kilobytes - self.propagation_per_sync_limit, # 4: Limit for incoming propagation node syncs - stamp_cost, # 5: Propagation stamp cost for this node - metadata ] # 6: Node metadata - - data = msgpack.packb(announce_data) - self.propagation_destination.announce(app_data=data) + self.propagation_destination.announce(app_data=self.get_propagation_node_app_data()) da_thread = threading.Thread(target=delayed_announce) da_thread.setDaemon(True) @@ -380,6 +383,29 @@ class LXMRouter: def get_outbound_propagation_node(self): return self.outbound_propagation_node + def get_outbound_propagation_cost(self): + target_propagation_cost = None + pn_destination_hash = self.get_outbound_propagation_node() + pn_app_data = RNS.Identity.recall_app_data(pn_destination_hash) + if pn_announce_data_is_valid(pn_app_data): + pn_config = msgpack.unpackb(pn_app_data) + target_propagation_cost = pn_config[5][0] + + if not target_propagation_cost: + RNS.log(f"Could not retrieve cached propagation node config. Requesting path to propagation node to get target propagation cost...", RNS.LOG_DEBUG) + RNS.Transport.request_path(pn_destination_hash) + timeout = time.time() + LXMRouter.PATH_REQUEST_WAIT + while not RNS.Identity.recall_app_data(pn_destination_hash) and time.time() < timeout: + time.sleep(0.5) + + pn_app_data = RNS.Identity.recall_app_data(pn_destination_hash) + if pn_announce_data_is_valid(pn_app_data): + pn_config = msgpack.unpackb(pn_app_data) + target_propagation_cost = pn_config[5][0] + + if not target_propagation_cost: RNS.log("Propagation node stamp cost still unavailable after path request", RNS.LOG_ERROR) + return target_propagation_cost + def set_inbound_propagation_node(self, destination_hash): # TODO: Implement raise NotImplementedError("Inbound/outbound propagation node differentiation is currently not implemented") @@ -1525,12 +1551,12 @@ class LXMRouter: else: return False - def cancel_outbound(self, message_id): + def cancel_outbound(self, message_id, cancel_state=LXMessage.CANCELLED): try: if message_id in self.pending_deferred_stamps: lxm = self.pending_deferred_stamps[message_id] RNS.log(f"Cancelling deferred stamp generation for {lxm}", RNS.LOG_DEBUG) - lxm.state = LXMessage.CANCELLED + lxm.state = cancel_state LXStamper.cancel_work(message_id) lxmessage = None @@ -1539,7 +1565,7 @@ class LXMRouter: lxmessage = lxm if lxmessage != None: - lxmessage.state = LXMessage.CANCELLED + lxmessage.state = cancel_state if lxmessage in self.pending_outbound: RNS.log(f"Cancelling {lxmessage} in outbound queue", RNS.LOG_DEBUG) if lxmessage.representation == LXMessage.RESOURCE: @@ -1574,11 +1600,9 @@ class LXMRouter: # destination to reply without generating a stamp. if lxmessage.include_ticket: ticket = self.generate_ticket(lxmessage.destination_hash) - if ticket: - lxmessage.fields[FIELD_TICKET] = ticket + if ticket: lxmessage.fields[FIELD_TICKET] = ticket - if not lxmessage.packed: - lxmessage.pack() + if not lxmessage.packed: lxmessage.pack() unknown_path_requested = False if not RNS.Transport.has_path(destination_hash) and lxmessage.method == LXMessage.OPPORTUNISTIC: @@ -1593,16 +1617,13 @@ class LXMRouter: RNS.log(f"Deferred stamp generation was requested for {lxmessage}, but no stamp is required, processing immediately", RNS.LOG_DEBUG) lxmessage.defer_stamp = False - if not lxmessage.defer_stamp: - while not unknown_path_requested and self.processing_outbound: - time.sleep(0.05) + if not lxmessage.defer_stamp and not (lxmessage.desired_method == LXMessage.PROPAGATED and lxmessage.defer_propagation_stamp): + while not unknown_path_requested and self.processing_outbound: time.sleep(0.05) self.pending_outbound.append(lxmessage) - if not unknown_path_requested: - self.process_outbound() + if not unknown_path_requested: self.process_outbound() - else: - self.pending_deferred_stamps[lxmessage.message_id] = lxmessage + else: self.pending_deferred_stamps[lxmessage.message_id] = lxmessage def get_outbound_progress(self, lxm_hash): for lxm in self.pending_outbound: @@ -1626,6 +1647,17 @@ class LXMRouter: return None + def get_outbound_lxm_propagation_stamp_cost(self, lxm_hash): + for lxm in self.pending_outbound: + if lxm.hash == lxm_hash: + return lxm.propagation_target_cost + + for lxm_id in self.pending_deferred_stamps: + if self.pending_deferred_stamps[lxm_id].hash == lxm_hash: + return self.pending_deferred_stamps[lxm_id].stamp_cost + + return None + ### Message Routing & Delivery ######################## ####################################################### @@ -2022,7 +2054,12 @@ class LXMRouter: self.lxmf_propagation(lxmf_data, stamp_value=stamp_value) self.client_propagation_messages_received += 1 - if stamps_valid: packet.prove() + if len(validated_messages) == len(messages): packet.prove() + else: + RNS.log("Propagation transfer from client contained messages with invalid stamps", RNS.LOG_NOTICE) + reject_data = msgpack.packb([LXMPeer.ERROR_INVALID_STAMP]) + RNS.Packet(packet.link, reject_data).send() + packet.link.teardown() except Exception as e: RNS.log("Exception occurred while parsing incoming LXMF propagation data.", RNS.LOG_ERROR) @@ -2281,29 +2318,87 @@ class LXMRouter: return - RNS.log(f"Starting stamp generation for {selected_lxm}...", RNS.LOG_DEBUG) - generated_stamp = selected_lxm.get_stamp() - if generated_stamp: - selected_lxm.stamp = generated_stamp - selected_lxm.defer_stamp = False - selected_lxm.packed = None - selected_lxm.pack() - self.pending_deferred_stamps.pop(selected_message_id) - self.pending_outbound.append(selected_lxm) - RNS.log(f"Stamp generation completed for {selected_lxm}", RNS.LOG_DEBUG) - else: - if selected_lxm.state == LXMessage.CANCELLED: - RNS.log(f"Message cancelled during deferred stamp generation for {selected_lxm}.", RNS.LOG_DEBUG) - selected_lxm.stamp_generation_failed = True - self.pending_deferred_stamps.pop(selected_message_id) - if selected_lxm.failed_callback != None and callable(selected_lxm.failed_callback): - selected_lxm.failed_callback(lxmessage) + if selected_lxm.stamp == None: stamp_generation_success = False + else: stamp_generation_success = True + + if selected_lxm.desired_method == LXMessage.PROPAGATED: + if selected_lxm.propagation_stamp == None: propagation_stamp_generation_success = False + else: propagation_stamp_generation_success = True + else: propagation_stamp_generation_success = True + + if stamp_generation_success == False: + RNS.log(f"Starting stamp generation for {selected_lxm}...", RNS.LOG_DEBUG) + generated_stamp = selected_lxm.get_stamp() + if generated_stamp: + selected_lxm.stamp = generated_stamp + selected_lxm.defer_stamp = False + selected_lxm.packed = None + selected_lxm.pack() + stamp_generation_success = True + RNS.log(f"Stamp generation completed for {selected_lxm}", RNS.LOG_DEBUG) else: - RNS.log(f"Deferred stamp generation did not succeed. Failing {selected_lxm}.", RNS.LOG_ERROR) + if selected_lxm.state == LXMessage.CANCELLED: + RNS.log(f"Message cancelled during deferred stamp generation for {selected_lxm}.", RNS.LOG_DEBUG) + selected_lxm.stamp_generation_failed = True + self.pending_deferred_stamps.pop(selected_message_id) + if selected_lxm.failed_callback != None and callable(selected_lxm.failed_callback): + selected_lxm.failed_callback(lxmessage) + else: + RNS.log(f"Deferred stamp generation did not succeed. Failing {selected_lxm}.", RNS.LOG_ERROR) + selected_lxm.stamp_generation_failed = True + self.pending_deferred_stamps.pop(selected_message_id) + self.fail_message(selected_lxm) + + if propagation_stamp_generation_success == False: + RNS.log(f"Starting propagation stamp generation for {selected_lxm}...", RNS.LOG_DEBUG) + pn_target_cost = self.get_outbound_propagation_cost() + if pn_target_cost == None: + RNS.log("Failed to get propagation node stamp cost, cannot generate propagation stamp", RNS.LOG_ERROR) selected_lxm.stamp_generation_failed = True self.pending_deferred_stamps.pop(selected_message_id) self.fail_message(selected_lxm) + else: + propagation_stamp = selected_lxm.get_propagation_stamp(target_cost=pn_target_cost) + RNS.log(f"Generated propagation stamp: {RNS.hexrep(propagation_stamp)}") + if propagation_stamp: + selected_lxm.propagation_stamp = propagation_stamp + selected_lxm.defer_propagation_stamp = False + selected_lxm.packed = None + selected_lxm.pack() + propagation_stamp_generation_success = True + RNS.log(f"Propagation stamp generation completed for {selected_lxm}", RNS.LOG_DEBUG) + else: + if selected_lxm.state == LXMessage.CANCELLED: + RNS.log(f"Message cancelled during deferred propagation stamp generation for {selected_lxm}.", RNS.LOG_DEBUG) + selected_lxm.stamp_generation_failed = True + self.pending_deferred_stamps.pop(selected_message_id) + if selected_lxm.failed_callback != None and callable(selected_lxm.failed_callback): + selected_lxm.failed_callback(lxmessage) + else: + RNS.log(f"Deferred propagation stamp generation did not succeed. Failing {selected_lxm}.", RNS.LOG_ERROR) + selected_lxm.stamp_generation_failed = True + self.pending_deferred_stamps.pop(selected_message_id) + self.fail_message(selected_lxm) + + if stamp_generation_success and propagation_stamp_generation_success: + self.pending_deferred_stamps.pop(selected_message_id) + self.pending_outbound.append(selected_lxm) + + def propagation_transfer_signalling_packet(self, data, packet): + try: + unpacked = msgpack.unpackb(data) + if type(unpacked) == list and len(unpacked) >= 1: + signal = unpacked[0] + if signal == LXMPeer.ERROR_INVALID_STAMP: + RNS.log("Message rejected by propagation node", RNS.LOG_ERROR) + if hasattr(packet, "link") and hasattr(packet.link, "for_lxmessage"): + lxm = packet.link.for_lxmessage + RNS.log(f"Invalid propagation stamp on {lxm}", RNS.LOG_ERROR) + self.cancel_outbound(lxm.message_id, cancel_state=LXMessage.REJECTED) + + except Exception as e: + RNS.log(f"An error occurred while processing propagation transfer signalling. The contained exception was: {e}", RNS.LOG_ERROR) def process_outbound(self, sender = None): if self.processing_outbound: @@ -2347,7 +2442,7 @@ class LXMRouter: elif lxmessage.state == LXMessage.REJECTED: RNS.log("Receiver rejected "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) - self.pending_outbound.remove(lxmessage) + if lxmessage in self.pending_outbound: self.pending_outbound.remove(lxmessage) if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): lxmessage.failed_callback(lxmessage) @@ -2512,6 +2607,8 @@ class LXMRouter: propagation_node_identity = RNS.Identity.recall(self.outbound_propagation_node) propagation_node_destination = RNS.Destination(propagation_node_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") self.outbound_propagation_link = RNS.Link(propagation_node_destination, established_callback=self.process_outbound) + self.outbound_propagation_link.set_packet_callback(self.propagation_transfer_signalling_packet) + self.outbound_propagation_link.for_lxmessage = lxmessage else: RNS.log("No path known for propagation attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(self.outbound_propagation_node)+". Requesting path...", RNS.LOG_DEBUG) RNS.Transport.request_path(self.outbound_propagation_node) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 4739f30..0533f07 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -145,26 +145,32 @@ class LXMessage: self.set_fields(fields) - self.payload = None - self.timestamp = None - self.signature = None - self.hash = None - self.packed = None - self.state = LXMessage.GENERATING - self.method = LXMessage.UNKNOWN - self.progress = 0.0 - self.rssi = None - self.snr = None - self.q = None + self.payload = None + self.timestamp = None + self.signature = None + self.hash = None + self.transient_id = None + self.packed = None + self.state = LXMessage.GENERATING + self.method = LXMessage.UNKNOWN + self.progress = 0.0 + self.rssi = None + self.snr = None + self.q = None - self.stamp = None - self.stamp_cost = stamp_cost - self.stamp_value = None - self.stamp_valid = False - self.stamp_checked = False - self.defer_stamp = True - self.outbound_ticket = None - self.include_ticket = include_ticket + self.stamp = None + self.stamp_cost = stamp_cost + self.stamp_value = None + self.stamp_valid = False + self.stamp_checked = False + self.propagation_stamp = None + self.propagation_stamp_value = None + self.propagation_stamp_valid = False + self.propagation_target_cost = None + self.defer_stamp = True + self.defer_propagation_stamp = True + self.outbound_ticket = None + self.include_ticket = include_ticket self.propagation_packed = None self.paper_packed = None @@ -184,6 +190,7 @@ class LXMessage: self.resource_representation = None self.__delivery_destination = None self.__delivery_callback = None + self.__pn_encrypted_data = None self.failed_callback = None self.deferred_stamp_generating = False @@ -324,10 +331,35 @@ class LXMessage: else: return None + def get_propagation_stamp(self, target_cost, timeout=None): + # If a stamp was already generated, return + # it immediately. + if self.propagation_stamp != None: + return self.propagation_stamp + + # Otherwise, we will need to generate a + # valid stamp according to the cost that + # the propagation node has specified. + else: + self.propagation_target_cost = target_cost + if self.propagation_target_cost == None: + raise ValueError("Cannot generate propagation stamp without configured target propagation cost") + + + if not self.transient_id: self.pack() + generated_stamp, value = LXStamper.generate_stamp(self.transient_id, target_cost, expand_rounds=LXStamper.WORKBLOCK_EXPAND_ROUNDS_PN) + if generated_stamp: + self.propagation_stamp = generated_stamp + self.propagation_stamp_value = value + self.propagation_stamp_valid = True + return generated_stamp + + else: + return None + def pack(self): if not self.packed: - if self.timestamp == None: - self.timestamp = time.time() + if self.timestamp == None: self.timestamp = time.time() self.propagation_packed = None self.paper_packed = None @@ -343,9 +375,8 @@ class LXMessage: if not self.defer_stamp: self.stamp = self.get_stamp() - if self.stamp != None: - self.payload.append(self.stamp) - + if self.stamp != None: self.payload.append(self.stamp) + signed_part = b"" signed_part += hashed_part signed_part += self.hash @@ -400,9 +431,14 @@ class LXMessage: elif self.desired_method == LXMessage.PROPAGATED: single_packet_content_limit = LXMessage.LINK_PACKET_MAX_CONTENT - encrypted_data = self.__destination.encrypt(self.packed[LXMessage.DESTINATION_LENGTH:]) - self.ratchet_id = self.__destination.latest_ratchet_id - self.propagation_packed = msgpack.packb([time.time(), [self.packed[:LXMessage.DESTINATION_LENGTH]+encrypted_data]]) + if self.__pn_encrypted_data == None: + self.__pn_encrypted_data = self.__destination.encrypt(self.packed[LXMessage.DESTINATION_LENGTH:]) + self.ratchet_id = self.__destination.latest_ratchet_id + + lxmf_data = self.packed[:LXMessage.DESTINATION_LENGTH]+self.__pn_encrypted_data + self.transient_id = RNS.Identity.full_hash(lxmf_data) + if self.propagation_stamp != None: lxmf_data += self.propagation_stamp + self.propagation_packed = msgpack.packb([time.time(), [lxmf_data]]) content_size = len(self.propagation_packed) if content_size <= single_packet_content_limit: diff --git a/LXMF/LXStamper.py b/LXMF/LXStamper.py index 2a74295..de71101 100644 --- a/LXMF/LXStamper.py +++ b/LXMF/LXStamper.py @@ -10,7 +10,7 @@ import multiprocessing WORKBLOCK_EXPAND_ROUNDS = 3000 WORKBLOCK_EXPAND_ROUNDS_PN = 1000 WORKBLOCK_EXPAND_ROUNDS_PEERING = 25 -STAMP_SIZE = RNS.Identity.HASHLENGTH +STAMP_SIZE = RNS.Identity.HASHLENGTH//8 PN_VALIDATION_POOL_MIN_SIZE = 256 active_jobs = {} @@ -24,7 +24,7 @@ def stamp_workblock(material, expand_rounds=WORKBLOCK_EXPAND_ROUNDS): salt=RNS.Identity.full_hash(material+msgpack.packb(n)), context=None) wb_time = time.time() - wb_st - RNS.log(f"Stamp workblock size {RNS.prettysize(len(workblock))}, generated in {round(wb_time*1000,2)}ms", RNS.LOG_DEBUG) + # RNS.log(f"Stamp workblock size {RNS.prettysize(len(workblock))}, generated in {round(wb_time*1000,2)}ms", RNS.LOG_DEBUG) return workblock @@ -52,23 +52,23 @@ def validate_peering_key(peering_id, peering_key, target_cost): def validate_pn_stamp(transient_data, target_cost): from .LXMessage import LXMessage - if len(transient_data) <= LXMessage.LXMF_OVERHEAD+STAMP_SIZE: return False, None, None + if len(transient_data) <= LXMessage.LXMF_OVERHEAD+STAMP_SIZE: return None, None, None else: lxm_data = transient_data[:-STAMP_SIZE] stamp = transient_data[-STAMP_SIZE:] transient_id = RNS.Identity.full_hash(lxm_data) workblock = stamp_workblock(transient_id, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PN) - if not stamp_valid(stamp, target_cost, workblock): return False, None, None + if not stamp_valid(stamp, target_cost, workblock): return None, None, None else: value = stamp_value(workblock, stamp) - return True, transient_id, value + return transient_id, lxm_data, value def validate_pn_stamps_job_simple(transient_list, target_cost): validated_messages = [] for transient_data in transient_list: - stamp_valid, transient_id, value = validate_pn_stamp(transient_data, target_cost) - if stamp_valid: validated_messages.append([transient_id, transient_data, value]) + transient_id, lxm_data, value = validate_pn_stamp(transient_data, target_cost) + if transient_id: validated_messages.append([transient_id, lxm_data, value]) return validated_messages @@ -80,7 +80,7 @@ def validate_pn_stamps_job_multip(transient_list, target_cost): with multiprocessing.Pool(pool_count) as p: validated_entries = p.starmap(validate_pn_stamp, zip(transient_list, itertools.repeat(target_cost))) - return [e for e in validated_entries if e[0] == True] + return [e for e in validated_entries if e[0] != None] def validate_pn_stamps(transient_list, target_cost): non_mp_platform = RNS.vendor.platformutils.is_android() diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index b2bc302..69bb26e 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -588,16 +588,18 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = if show_status: msb = RNS.prettysize(s["messagestore"]["bytes"]); msl = RNS.prettysize(s["messagestore"]["limit"]) - ptl = RNS.prettysize(s["propagation_limit"]*1000); uprx = RNS.prettysize(s["unpeered_propagation_rx_bytes"]) + ptl = RNS.prettysize(s["propagation_limit"]*1000); psl = RNS.prettysize(s["sync_limit"]*1000); + uprx = RNS.prettysize(s["unpeered_propagation_rx_bytes"]) mscnt = s["messagestore"]["count"]; stp = s["total_peers"]; smp = s["max_peers"]; sdp = s["discovered_peers"] ssp = s["static_peers"]; cprr = s["clients"]["client_propagation_messages_received"] cprs = s["clients"]["client_propagation_messages_served"]; upi = s["unpeered_propagation_incoming"] psc = s["target_stamp_cost"]; scf = s["stamp_cost_flexibility"] pc = s["peering_cost"]; pcm = s["max_peering_cost"] print(f"Messagestore contains {mscnt} messages, {msb} ({ms_util} utilised of {msl})") - print(f"Accepting propagated messages from {who_str}, {ptl} per-transfer limit") print(f"Required propagation stamp cost is {psc}, flexibility is {scf}") print(f"Peering cost is {pc}, max remote peering cost is {pcm}") + print(f"Accepting propagated messages from {who_str}") + print(f"{ptl} message limit, {psl} sync limit") print(f"") print(f"Peers : {stp} total (peer limit is {smp})") print(f" {sdp} discovered, {ssp} static") @@ -642,14 +644,16 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = else: ls = "never synced" - sstr = RNS.prettyspeed(p["str"]); sler = RNS.prettyspeed(p["ler"]); stl = RNS.prettysize(p["transfer_limit"]*1000) + sstr = RNS.prettyspeed(p["str"]); sler = RNS.prettyspeed(p["ler"]) + stl = RNS.prettysize(p["transfer_limit"]*1000); ssl = RNS.prettysize(p["sync_limit"]*1000) srxb = RNS.prettysize(p["rx_bytes"]); stxb = RNS.prettysize(p["tx_bytes"]); pmo = pm["offered"]; pmout = pm["outgoing"] pmi = pm["incoming"]; pmuh = pm["unhandled"] print(f"{ind}{t}{RNS.prettyhexrep(peer_id)}") print(f"{ind*2}Status : {a}, {hs}, last heard {RNS.prettytime(h)} ago") print(f"{ind*2}Costs : Propagation {psc} (flex {psf}), peering {pc}") print(f"{ind*2}Sync key : {pk}") - print(f"{ind*2}Speeds : {sstr} STR, {sler} LER, {stl} transfer limit") + print(f"{ind*2}Speeds : {sstr} STR, {sler} LER") + print(f"{ind*2}Limits : {stl} message limit, {ssl} sync limit") print(f"{ind*2}Messages : {pmo} offered, {pmout} outgoing, {pmi} incoming") print(f"{ind*2}Traffic : {srxb} received, {stxb} sent") ms = "" if pm["unhandled"] == 1 else "s" From 0a5edb28952e1f17bc7485ce4adbcac1386b56ce Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 31 Oct 2025 22:24:55 +0100 Subject: [PATCH 351/436] Implemented propagation node name configuration and inclusion in announce metadata --- LXMF/LXMF.py | 22 ++++++++++++++++++++++ LXMF/LXMPeer.py | 14 ++++++++++++++ LXMF/LXMRouter.py | 16 +++++++++++++--- LXMF/Utilities/lxmd.py | 19 +++++++++++++++++-- 4 files changed, 66 insertions(+), 5 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index fd2abf0..dbc0ab1 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -147,6 +147,28 @@ def stamp_cost_from_app_data(app_data=None): # Original announce format else: return None +def pn_name_from_app_data(app_data=None): + if app_data == None: return None + else: + if pn_announce_data_is_valid(app_data): + data = msgpack.unpackb(app_data) + metadata = data[6] + if not PN_META_NAME in metadata: return None + else: + try: return metadata[PN_META_NAME].decode("utf-8") + except: return None + + return None + +def pn_stamp_cost_from_app_data(app_data=None): + if app_data == None: return None + else: + if pn_announce_data_is_valid(app_data): + data = msgpack.unpackb(app_data) + return data[5][0] + else: + return None + def pn_announce_data_is_valid(data): try: if type(data) != bytes: return False diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 199ee2d..a67d3b2 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -8,6 +8,7 @@ import LXMF.LXStamper as LXStamper from collections import deque from .LXMF import APP_NAME +from .LXMF import PN_META_NAME class LXMPeer: OFFER_REQUEST_PATH = "/offer" @@ -110,6 +111,8 @@ class LXMPeer: else: peer.last_sync_attempt = 0 if "peering_key" in dictionary: peer.peering_key = dictionary["peering_key"] else: peer.peering_key = None + if "metadata" in dictionary: peer.metadata = dictionary["metadata"] + else: peer.metadata = None hm_count = 0 for transient_id in dictionary["handled_ids"]: @@ -135,6 +138,7 @@ class LXMPeer: dictionary = {} dictionary["peering_timebase"] = self.peering_timebase dictionary["alive"] = self.alive + dictionary["metadata"] = self.metadata dictionary["last_heard"] = self.last_heard dictionary["sync_strategy"] = self.sync_strategy dictionary["peering_key"] = self.peering_key @@ -175,6 +179,7 @@ class LXMPeer: self.sync_strategy = sync_strategy self.peering_key = None self.peering_cost = None + self.metadata = None self.next_sync_attempt = 0 self.last_sync_attempt = 0 @@ -616,6 +621,15 @@ class LXMPeer: self.router.propagation_entries[transient_id][5].remove(self.destination_hash) self._um_counts_synced = False + @property + def name(self): + if type(self.metadata) != dict: return None + else: + if not PN_META_NAME in self.metadata: return None + else: + try: return self.metadata[PN_META_NAME].decode("utf-8") + except: return None + def __str__(self): if self.destination_hash: return RNS.prettyhexrep(self.destination_hash) else: return "" \ No newline at end of file diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 39ee7ec..3ab85e8 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -15,6 +15,7 @@ import RNS.vendor.umsgpack as msgpack from .LXMF import APP_NAME from .LXMF import FIELD_TICKET +from .LXMF import PN_META_NAME from .LXMF import pn_announce_data_is_valid from .LXMPeer import LXMPeer @@ -84,7 +85,7 @@ class LXMRouter: enforce_ratchets=False, enforce_stamps=False, static_peers = [], max_peers=None, from_static_only=False, sync_strategy=LXMPeer.STRATEGY_PERSISTENT, propagation_cost=PROPAGATION_COST, propagation_cost_flexibility=PROPAGATION_COST_FLEX, - peering_cost=PEERING_COST): + peering_cost=PEERING_COST, name=None): random.seed(os.urandom(10)) @@ -105,9 +106,10 @@ class LXMRouter: self.processing_outbound = False self.processing_inbound = False self.processing_count = 0 + self.name = name self.propagation_node = False - self.propagation_node_start_time = None + self.propagation_node_start_time = None if storagepath == None: raise ValueError("LXMF cannot be initialised without a storage path") else: @@ -287,10 +289,15 @@ class LXMRouter: if destination_hash in self.delivery_destinations: self.delivery_destinations[destination_hash].announce(app_data=self.get_announce_app_data(destination_hash), attached_interface=attached_interface) + def get_propagation_node_announce_metadata(self): + metadata = {} + if self.name: metadata[PN_META_NAME] = str(self.name).encode("utf-8") + return metadata + def get_propagation_node_app_data(self): + metadata = self.get_propagation_node_announce_metadata() node_state = self.propagation_node and not self.from_static_only stamp_cost = [self.propagation_stamp_cost, self.propagation_stamp_cost_flexibility, self.peering_cost] - metadata = {} announce_data = [ False, # 0: Legacy LXMF PN support int(time.time()), # 1: Current node timebase node_state, # 2: Boolean flag signalling propagation node state @@ -737,6 +744,7 @@ class LXMRouter: "type": "static" if peer_id in self.static_peers else "discovered", "state": peer.state, "alive": peer.alive, + "name": peer.name, "last_heard": int(peer.last_heard), "next_sync_attempt": peer.next_sync_attempt, "last_sync_attempt": peer.last_sync_attempt, @@ -1834,6 +1842,7 @@ class LXMRouter: peer = self.peers[destination_hash] if timestamp > peer.peering_timebase: peer.alive = True + peer.metadata = metadata peer.sync_backoff = 0 peer.next_sync_attempt = 0 peer.peering_timebase = timestamp @@ -1852,6 +1861,7 @@ class LXMRouter: else: peer = LXMPeer(self, destination_hash, sync_strategy=self.default_sync_strategy) peer.alive = True + peer.metadata = metadata peer.last_heard = time.time() peer.propagation_stamp_cost = propagation_stamp_cost peer.propagation_stamp_cost_flexibility = propagation_stamp_cost_flexibility diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 69bb26e..5d21bd3 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -97,6 +97,11 @@ def apply_config(): else: active_configuration["enable_propagation_node"] = False + if "propagation" in lxmd_config and "node_name" in lxmd_config["propagation"]: + active_configuration["node_name"] = lxmd_config["propagation"].get("node_name") + else: + active_configuration["node_name"] = None + if "propagation" in lxmd_config and "auth_required" in lxmd_config["propagation"]: active_configuration["auth_required"] = lxmd_config["propagation"].as_bool("auth_required") else: @@ -371,7 +376,8 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo delivery_limit = active_configuration["delivery_transfer_max_accepted_size"], max_peers = active_configuration["max_peers"], static_peers = active_configuration["static_peers"], - from_static_only = active_configuration["from_static_only"]) + from_static_only = active_configuration["from_static_only"], + name = active_configuration["node_name"]) message_router.register_delivery_callback(lxmf_delivery) @@ -647,8 +653,12 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = sstr = RNS.prettyspeed(p["str"]); sler = RNS.prettyspeed(p["ler"]) stl = RNS.prettysize(p["transfer_limit"]*1000); ssl = RNS.prettysize(p["sync_limit"]*1000) srxb = RNS.prettysize(p["rx_bytes"]); stxb = RNS.prettysize(p["tx_bytes"]); pmo = pm["offered"]; pmout = pm["outgoing"] - pmi = pm["incoming"]; pmuh = pm["unhandled"] + pmi = pm["incoming"]; pmuh = pm["unhandled"]; + if p["name"] == None: nn = "" + else: nn = p["name"].strip().replace("\n", "").replace("\r", "") + if len(nn) > 45: nn = f"{nn[:45]}..." print(f"{ind}{t}{RNS.prettyhexrep(peer_id)}") + if len(nn): print(f"{ind*2}Name : {nn}") print(f"{ind*2}Status : {a}, {hs}, last heard {RNS.prettytime(h)} ago") print(f"{ind*2}Costs : Propagation {psc} (flex {psf}), peering {pc}") print(f"{ind*2}Sync key : {pk}") @@ -717,6 +727,11 @@ __default_lxmd_config__ = """# This is an example LXM Daemon config file. enable_node = no +# An optional name for this node, included +# in announces. + +# node_name = Anonymous Propagation Node + # Automatic announce interval in minutes. # 6 hours by default. From 4afb92bf3e724b3f81b8536dc83eb5a1eb78db14 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 1 Nov 2025 00:09:23 +0100 Subject: [PATCH 352/436] Added peer sync option to lxmd command line interface --- LXMF/Utilities/lxmd.py | 130 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 123 insertions(+), 7 deletions(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 5d21bd3..88f08f7 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -467,6 +467,112 @@ def deferred_start_jobs(): last_node_announce = time.time() threading.Thread(target=jobs, daemon=True).start() +def _request_sync(identity, destination_hash, timeout=5, exit_on_fail=False): + control_destination = RNS.Destination(identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") + + timeout = time.time()+timeout + def check_timeout(): + if time.time() > timeout: + if exit_on_fail: + RNS.log("Requesting lxmd peer sync timed out, exiting now", RNS.LOG_ERROR) + exit(200) + else: + return LXMF.LXMPeer.LXMPeer.ERROR_TIMEOUT + else: + time.sleep(0.1) + + if not RNS.Transport.has_path(control_destination.hash): + RNS.Transport.request_path(control_destination.hash) + while not RNS.Transport.has_path(control_destination.hash): + tc = check_timeout() + if tc: + return tc + + link = RNS.Link(control_destination) + while not link.status == RNS.Link.ACTIVE: + tc = check_timeout() + if tc: + return tc + + link.identify(identity) + request_receipt = link.request(LXMF.LXMRouter.SYNC_REQUEST_PATH, data=destination_hash, response_callback=None, failed_callback=None) + while not request_receipt.get_status() == RNS.RequestReceipt.READY: + tc = check_timeout() + if tc: + return tc + + link.teardown() + return request_receipt.get_response() + +def request_sync(target, configdir = None, rnsconfigdir = None, verbosity = 0, quietness = 0, timeout=15, identity_path=None): + global configpath, identitypath, storagedir, lxmdir + global lxmd_config, active_configuration, targetloglevel + targetlogdest = RNS.LOG_STDOUT + + if identity_path == None: + if configdir == None: + if os.path.isdir("/etc/lxmd") and os.path.isfile("/etc/lxmd/config"): + configdir = "/etc/lxmd" + elif os.path.isdir(RNS.Reticulum.userdir+"/.config/lxmd") and os.path.isfile(Reticulum.userdir+"/.config/lxmd/config"): + configdir = RNS.Reticulum.userdir+"/.config/lxmd" + else: + configdir = RNS.Reticulum.userdir+"/.lxmd" + + configpath = configdir+"/config" + identitypath = configdir+"/identity" + identity = None + + if not os.path.isdir(configdir): + RNS.log("Specified configuration directory does not exist, exiting now", RNS.LOG_ERROR) + exit(201) + if not os.path.isfile(identitypath): + RNS.log("Identity file not found in specified configuration directory, exiting now", RNS.LOG_ERROR) + exit(202) + else: + identity = RNS.Identity.from_file(identitypath) + if identity == None: + RNS.log("Could not load the Primary Identity from "+identitypath, RNS.LOG_ERROR) + exit(4) + + else: + if not os.path.isfile(identity_path): + RNS.log("Identity file not found in specified configuration directory, exiting now", RNS.LOG_ERROR) + exit(202) + else: + identity = RNS.Identity.from_file(identity_path) + if identity == None: + RNS.log("Could not load the Primary Identity from "+identity_path, RNS.LOG_ERROR) + exit(4) + + if targetloglevel == None: targetloglevel = 3 + if verbosity != 0 or quietness != 0: targetloglevel = targetloglevel+verbosity-quietness + + try: + destination_hash = bytes.fromhex(target) + if len(destination_hash) != RNS.Identity.TRUNCATED_HASHLENGTH//8: raise ValueError(f"Destination hash length must be {RNS.Identity.TRUNCATED_HASHLENGTH//8*2} characters") + except Exception as e: + print(f"Invalid peer destination hash: {e}") + exit(203) + + reticulum = RNS.Reticulum(configdir=rnsconfigdir, loglevel=targetloglevel, logdest=targetlogdest) + response = _request_sync(identity, destination_hash, timeout=timeout, exit_on_fail=True) + + if response == LXMF.LXMPeer.LXMPeer.ERROR_NO_IDENTITY: + print("Remote received no identity") + exit(203) + elif response == LXMF.LXMPeer.LXMPeer.ERROR_NO_ACCESS: + print("Access denied") + exit(204) + elif response == LXMF.LXMPeer.LXMPeer.ERROR_INVALID_DATA: + print("Invalid data received by remote") + exit(205) + elif response == LXMF.LXMPeer.LXMPeer.ERROR_NOT_FOUND: + print("The requested peer was not found") + exit(206) + else: + print(f"Sync requested for peer {RNS.prettyhexrep(destination_hash)}") + exit(0) + def query_status(identity, timeout=5, exit_on_fail=False): control_destination = RNS.Destination(identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") @@ -683,6 +789,7 @@ def main(): parser.add_argument("-s", "--service", action="store_true", default=False, help="lxmd is running as a service and should log to file") parser.add_argument("--status", action="store_true", default=False, help="display node status") parser.add_argument("--peers", action="store_true", default=False, help="display peered nodes") + parser.add_argument("--sync", action="store", default=None, help="request a sync with the specified peer", type=str) parser.add_argument("--timeout", action="store", default=5, help="timeout in seconds for query operations", type=float) parser.add_argument("--identity", action="store", default=None, help="path to identity used for query request", type=str) parser.add_argument("--exampleconfig", action="store_true", default=False, help="print verbose configuration example to stdout and exit") @@ -696,15 +803,24 @@ def main(): if args.status or args.peers: get_status(configdir = args.config, - rnsconfigdir=args.rnsconfig, - verbosity=args.verbose, - quietness=args.quiet, - timeout=args.timeout, - show_status=args.status, - show_peers=args.peers, - identity_path=args.identity) + rnsconfigdir=args.rnsconfig, + verbosity=args.verbose, + quietness=args.quiet, + timeout=args.timeout, + show_status=args.status, + show_peers=args.peers, + identity_path=args.identity) exit() + if args.sync: + request_sync(target=args.sync, + configdir = args.config, + rnsconfigdir=args.rnsconfig, + verbosity=args.verbose, + quietness=args.quiet, + timeout=args.timeout, + identity_path=args.identity) + program_setup(configdir = args.config, rnsconfigdir=args.rnsconfig, run_pn=args.propagation_node, From df6271a02637087c78e0ab227d5e48ebd858258d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 1 Nov 2025 00:10:30 +0100 Subject: [PATCH 353/436] Handle client message download for stamped propagation messages --- LXMF/LXMPeer.py | 1 + LXMF/LXMRouter.py | 62 +++++++++++++++++++++++++++++------------------ LXMF/LXStamper.py | 10 ++++---- 3 files changed, 44 insertions(+), 29 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index a67d3b2..786514d 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -27,6 +27,7 @@ class LXMPeer: ERROR_INVALID_DATA = 0xf4 ERROR_INVALID_STAMP = 0xf5 ERROR_THROTTLED = 0xf6 + ERROR_NOT_FOUND = 0xfd ERROR_TIMEOUT = 0xfe STRATEGY_LAZY = 0x01 diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 3ab85e8..c7c6051 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -75,6 +75,7 @@ class LXMRouter: PR_ALL_MESSAGES = 0x00 STATS_GET_PATH = "/pn/get/stats" + SYNC_REQUEST_PATH = "/pn/peer/sync" ### Developer-facing API ############################## @@ -627,6 +628,7 @@ class LXMRouter: self.control_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") self.control_destination.register_request_handler(LXMRouter.STATS_GET_PATH, self.stats_get_request, allow = RNS.Destination.ALLOW_LIST, allowed_list=[self.identity.hash]) + self.control_destination.register_request_handler(LXMRouter.SYNC_REQUEST_PATH, self.peer_sync_request, allow = RNS.Destination.ALLOW_LIST, allowed_list=[self.identity.hash]) if self.message_storage_limit != None: limit_str = ", limit is "+RNS.prettysize(self.message_storage_limit) @@ -807,6 +809,18 @@ class LXMRouter: elif remote_identity.hash != self.identity.hash: return LXMPeer.ERROR_NO_ACCESS else: return self.compile_stats() + def peer_sync_request(self, path, data, request_id, remote_identity, requested_at): + if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY + elif remote_identity.hash != self.identity.hash: return LXMPeer.ERROR_NO_ACCESS + else: + if type(data) != bytes: return LXMPeer.ERROR_INVALID_DATA + elif len(data) != RNS.Identity.TRUNCATED_HASHLENGTH//8: return LXMPeer.ERROR_INVALID_DATA + else: + if not data in self.peers: return LXMPeer.ERROR_NOT_FOUND + else: + self.peers[data].sync() + return True + ### Utility & Maintenance ############################# ####################################################### @@ -1364,12 +1378,8 @@ class LXMRouter: return True def message_get_request(self, path, data, request_id, remote_identity, requested_at): - if remote_identity == None: - return LXMPeer.ERROR_NO_IDENTITY - - elif not self.identity_allowed(remote_identity): - return LXMPeer.ERROR_NO_ACCESS - + if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY + elif not self.identity_allowed(remote_identity): return LXMPeer.ERROR_NO_ACCESS else: try: remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") @@ -1388,9 +1398,7 @@ class LXMRouter: available_messages.sort(key=lambda e: e[1], reverse=False) transient_ids = [] - for available_entry in available_messages: - transient_ids.append(available_entry[0]) - + for available_entry in available_messages: transient_ids.append(available_entry[0]) return transient_ids else: @@ -1416,8 +1424,7 @@ class LXMRouter: try: client_transfer_limit = float(data[2])*1000 RNS.log("Client indicates transfer limit of "+RNS.prettysize(client_transfer_limit), RNS.LOG_DEBUG) - except: - pass + except: pass per_message_overhead = 16 # Really only 2 bytes, but set a bit higher for now cumulative_size = 24 # Initialised to highest reasonable binary structure overhead @@ -1434,10 +1441,9 @@ class LXMRouter: lxm_size = len(lxmf_data) next_size = cumulative_size + (lxm_size+per_message_overhead) - if client_transfer_limit != None and next_size > client_transfer_limit: - pass + if client_transfer_limit != None and next_size > client_transfer_limit: pass else: - response_messages.append(lxmf_data) + response_messages.append(lxmf_data[:-LXStamper.STAMP_SIZE]) cumulative_size += (lxm_size+per_message_overhead) except Exception as e: @@ -1446,7 +1452,6 @@ class LXMRouter: self.client_propagation_messages_served += len(response_messages) return response_messages - except Exception as e: RNS.log("Error occurred while generating response for download request, the contained exception was: "+str(e), RNS.LOG_DEBUG) return None @@ -2061,7 +2066,8 @@ class LXMRouter: for validated_entry in validated_messages: lxmf_data = validated_entry[1] stamp_value = validated_entry[2] - self.lxmf_propagation(lxmf_data, stamp_value=stamp_value) + stamp_data = validated_entry[3] + self.lxmf_propagation(lxmf_data, stamp_value=stamp_value, stamp_data=stamp_data) self.client_propagation_messages_received += 1 if len(validated_messages) == len(messages): packet.prove() @@ -2090,7 +2096,7 @@ class LXMRouter: try: if type(data) != list and len(data) < 2: return LXMPeer.ERROR_INVALID_DATA - peering_id = self.identity.hash+remote_identity + peering_id = self.identity.hash+remote_identity.hash target_cost = self.peering_cost peering_key = data[0] transient_ids = data[1] @@ -2115,6 +2121,7 @@ class LXMRouter: except Exception as e: RNS.log("Error occurred while generating response for sync request, the contained exception was: "+str(e), RNS.LOG_DEBUG) + RNS.trace_exception(e) return None def propagation_resource_concluded(self, resource): @@ -2158,15 +2165,18 @@ class LXMRouter: self.peer(remote_hash, remote_timebase, remote_transfer_limit, remote_sync_limit, remote_stamp_cost, remote_stamp_flex, remote_peering_cost, remote_metadata) ms = "" if len(messages) == 1 else "s" - RNS.log(f"Received {len(messages)} message{ms} from {remote_str}", RNS.LOG_VERBOSE) + RNS.log(f"Received {len(messages)} message{ms} from {remote_str}, validating stamps...", RNS.LOG_VERBOSE) min_accepted_cost = max(0, self.propagation_stamp_cost-self.propagation_stamp_cost_flexibility) validated_messages = LXStamper.validate_pn_stamps(messages, min_accepted_cost) + if len(validated_messages) == len(messages): RNS.log(f"All message stamps validated from {remote_str}", RNS.LOG_VERBOSE) + else: RNS.log(f"Transfer from {remote_str} contained {len(messages)-len(validated_messages)} invalid stamps", RNS.LOG_WARNING) for validated_entry in validated_messages: transient_id = validated_entry[0] lxmf_data = validated_entry[1] stamp_value = validated_entry[2] + stamp_data = validated_entry[3] peer = None if remote_hash != None and remote_hash in self.peers: @@ -2180,7 +2190,7 @@ class LXMRouter: else: self.client_propagation_messages_received += 1 - self.lxmf_propagation(lxmf_data, from_peer=peer, stamp_value=stamp_value) + self.lxmf_propagation(lxmf_data, from_peer=peer, stamp_value=stamp_value, stamp_data=stamp_data) if peer != None: peer.queue_handled_message(transient_id) else: @@ -2208,7 +2218,8 @@ class LXMRouter: if peer != from_peer: peer.queue_unhandled_message(transient_id) - def lxmf_propagation(self, lxmf_data, signal_local_delivery=None, signal_duplicate=None, allow_duplicate=False, is_paper_message=False, from_peer=None, stamp_value=None): + def lxmf_propagation(self, lxmf_data, signal_local_delivery=None, signal_duplicate=None, allow_duplicate=False, is_paper_message=False, + from_peer=None, stamp_value=None, stamp_data=None): if is_paper_message: no_stamp_enforcement = True else: no_stamp_enforcement = False @@ -2236,13 +2247,14 @@ class LXMRouter: else: if self.propagation_node: + stamped_data = lxmf_data+stamp_data value_component = f"_{stamp_value}" if stamp_value and stamp_value > 0 else "" file_path = f"{self.messagepath}/{RNS.hexrep(transient_id, delimit=False)}_{received}{value_component}" msg_file = open(file_path, "wb") - msg_file.write(lxmf_data); msg_file.close() + msg_file.write(stamped_data); msg_file.close() RNS.log(f"Received propagated LXMF message {RNS.prettyhexrep(transient_id)} with stamp value {stamp_value}, adding to peer distribution queues...", RNS.LOG_EXTREME) - self.propagation_entries[transient_id] = [destination_hash, file_path, time.time(), len(lxmf_data), [], [], stamp_value] + self.propagation_entries[transient_id] = [destination_hash, file_path, time.time(), len(stamped_data), [], [], stamp_value] self.enqueue_peer_distribution(transient_id, from_peer) else: @@ -2328,8 +2340,10 @@ class LXMRouter: return - if selected_lxm.stamp == None: stamp_generation_success = False - else: stamp_generation_success = True + if selected_lxm.defer_stamp: + if selected_lxm.stamp == None: stamp_generation_success = False + else: stamp_generation_success = True + else: stamp_generation_success = True if selected_lxm.desired_method == LXMessage.PROPAGATED: if selected_lxm.propagation_stamp == None: propagation_stamp_generation_success = False diff --git a/LXMF/LXStamper.py b/LXMF/LXStamper.py index de71101..8ebefd7 100644 --- a/LXMF/LXStamper.py +++ b/LXMF/LXStamper.py @@ -52,23 +52,23 @@ def validate_peering_key(peering_id, peering_key, target_cost): def validate_pn_stamp(transient_data, target_cost): from .LXMessage import LXMessage - if len(transient_data) <= LXMessage.LXMF_OVERHEAD+STAMP_SIZE: return None, None, None + if len(transient_data) <= LXMessage.LXMF_OVERHEAD+STAMP_SIZE: return None, None, None, None else: lxm_data = transient_data[:-STAMP_SIZE] stamp = transient_data[-STAMP_SIZE:] transient_id = RNS.Identity.full_hash(lxm_data) workblock = stamp_workblock(transient_id, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PN) - if not stamp_valid(stamp, target_cost, workblock): return None, None, None + if not stamp_valid(stamp, target_cost, workblock): return None, None, None, None else: value = stamp_value(workblock, stamp) - return transient_id, lxm_data, value + return transient_id, lxm_data, value, stamp def validate_pn_stamps_job_simple(transient_list, target_cost): validated_messages = [] for transient_data in transient_list: - transient_id, lxm_data, value = validate_pn_stamp(transient_data, target_cost) - if transient_id: validated_messages.append([transient_id, lxm_data, value]) + transient_id, lxm_data, value, stamp_data = validate_pn_stamp(transient_data, target_cost) + if transient_id: validated_messages.append([transient_id, lxm_data, value, stamp_data]) return validated_messages From b35b9213a6b92f9b54bf2defc31ca0fa04e74df1 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 1 Nov 2025 01:35:15 +0100 Subject: [PATCH 354/436] Implemented throttling for naughty propagation node peers --- LXMF/LXMPeer.py | 8 +++++++- LXMF/LXMRouter.py | 45 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 786514d..0dbf8ce 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -286,7 +286,7 @@ class LXMPeer: threading.Thread(target=job, daemon=True).start() delay = self.next_sync_attempt-time.time() - postpone_delay = " for {RNS.prettytime({delay})}" if delay > 0 else "" + postpone_delay = f" for {RNS.prettytime(delay)}" if delay > 0 else "" RNS.log(f"Postponing sync with peer {RNS.prettyhexrep(self.destination_hash)}{postpone_delay}{postpone_reason}", RNS.LOG_DEBUG) except Exception as e: RNS.trace_exception(e) @@ -414,6 +414,12 @@ class LXMPeer: self.router.unpeer(self.destination_hash) return + elif response == LXMPeer.ERROR_THROTTLED: + throttle_time = self.router.PN_STAMP_THROTTLE + RNS.log(f"Remote indicated that we're throttled, postponing sync for {RNS.prettytime(throttle_time)}", RNS.LOG_VERBOSE) + self.next_sync_attempt = time.time()+throttle_time + return + elif response == False: # Peer already has all advertised messages for transient_id in self.last_offer: diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index c7c6051..f25c8f0 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -56,6 +56,7 @@ class LXMRouter: DELIVERY_LIMIT = 1000 PR_PATH_TIMEOUT = 10 + PN_STAMP_THROTTLE = 180 PR_IDLE = 0x00 PR_PATH_REQUESTED = 0x01 @@ -135,6 +136,7 @@ class LXMRouter: self.enforce_ratchets = enforce_ratchets self._enforce_stamps = enforce_stamps self.pending_deferred_stamps = {} + self.throttled_peers = {} if sync_limit == None or self.propagation_per_sync_limit < self.propagation_per_transfer_limit: self.propagation_per_sync_limit = self.propagation_per_transfer_limit @@ -850,20 +852,17 @@ class LXMRouter: self.clean_transient_id_caches() if self.processing_count % LXMRouter.JOB_STORE_INTERVAL == 0: - if self.propagation_node == True: - self.clean_message_store() + if self.propagation_node == True: self.clean_message_store() if self.processing_count % LXMRouter.JOB_PEERINGEST_INTERVAL == 0: - if self.propagation_node == True: - self.flush_queues() + if self.propagation_node == True: self.flush_queues() if self.processing_count % LXMRouter.JOB_ROTATE_INTERVAL == 0: - if self.propagation_node == True: - self.rotate_peers() + if self.propagation_node == True: self.rotate_peers() if self.processing_count % LXMRouter.JOB_PEERSYNC_INTERVAL == 0: - if self.propagation_node == True: - self.sync_peers() + if self.propagation_node == True: self.sync_peers() + self.clean_throttled_peers() def jobloop(self): while (True): @@ -1060,6 +1059,14 @@ class LXMRouter: else: return available_tickets + def clean_throttled_peers(self): + expired_entries = [] + now = time.time() + for peer_hash in self.throttled_peers: + if now > self.throttled_peers[peer_hash]: expired_entries.append(peer_hash) + + for peer_hash in expired_entries: self.throttled_peers.pop(peer_hash) + def clean_message_store(self): RNS.log("Cleaning message store", RNS.LOG_VERBOSE) # Check and remove expired messages @@ -2085,10 +2092,18 @@ class LXMRouter: if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY else: + remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + remote_hash = remote_destination.hash + remote_str = RNS.prettyhexrep(remote_hash) + + if remote_hash in self.throttled_peers: + throttle_remaining = self.throttled_peers[remote_hash]-time.time() + if throttle_remaining > 0: + RNS.log(f"Propagation offer from node {remote_str} rejected, throttled for {RNS.prettytime(throttle_remaining)} more", RNS.LOG_NOTICE) + return LXMPeer.ERROR_THROTTLED + else: self.throttled_peers.pop(remote_hash) + if self.from_static_only: - remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") - remote_hash = remote_destination.hash - remote_str = RNS.prettyhexrep(remote_hash) if not remote_hash in self.static_peers: RNS.log(f"Rejecting propagation request from {remote_str} not in static peers list", RNS.LOG_DEBUG) return LXMPeer.ERROR_NO_ACCESS @@ -2193,6 +2208,14 @@ class LXMRouter: self.lxmf_propagation(lxmf_data, from_peer=peer, stamp_value=stamp_value, stamp_data=stamp_data) if peer != None: peer.queue_handled_message(transient_id) + invalid_message_count = len(messages) - len(validated_messages) + if invalid_message_count > 0: + resource.link.teardown() + throttle_time = LXMRouter.PN_STAMP_THROTTLE + self.throttled_peers[remote_hash] = time.time()+throttle_time + ms = "" if invalid_message_count == 1 else "s" + RNS.log(f"Propagation transfer from {remote_str} contained {invalid_message_count} message{ms} with invalid stamps, throttled for {RNS.prettytime(throttle_time)}", RNS.LOG_NOTICE) + else: RNS.log("Invalid data structure received at propagation destination, ignoring", RNS.LOG_DEBUG) From 0cebd5886d54b12b412a530f225e7a11862bd16c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 1 Nov 2025 12:49:32 +0100 Subject: [PATCH 355/436] Allow specifying remote lxmd destination for status and control --- LXMF/Utilities/lxmd.py | 219 ++++++++++++++++++++--------------------- 1 file changed, 105 insertions(+), 114 deletions(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 88f08f7..b4599df 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -467,14 +467,14 @@ def deferred_start_jobs(): last_node_announce = time.time() threading.Thread(target=jobs, daemon=True).start() -def _request_sync(identity, destination_hash, timeout=5, exit_on_fail=False): - control_destination = RNS.Destination(identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") +def _request_sync(identity, destination_hash, remote_identity, timeout=15, exit_on_fail=False): + control_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") timeout = time.time()+timeout def check_timeout(): if time.time() > timeout: if exit_on_fail: - RNS.log("Requesting lxmd peer sync timed out, exiting now", RNS.LOG_ERROR) + print("Requesting lxmd peer sync timed out, exiting now") exit(200) else: return LXMF.LXMPeer.LXMPeer.ERROR_TIMEOUT @@ -504,58 +504,20 @@ def _request_sync(identity, destination_hash, timeout=5, exit_on_fail=False): link.teardown() return request_receipt.get_response() -def request_sync(target, configdir = None, rnsconfigdir = None, verbosity = 0, quietness = 0, timeout=15, identity_path=None): + +def request_sync(target, remote=None, configdir=None, rnsconfigdir=None, verbosity=0, quietness=0, timeout=15, identity_path=None): global configpath, identitypath, storagedir, lxmdir global lxmd_config, active_configuration, targetloglevel - targetlogdest = RNS.LOG_STDOUT - - if identity_path == None: - if configdir == None: - if os.path.isdir("/etc/lxmd") and os.path.isfile("/etc/lxmd/config"): - configdir = "/etc/lxmd" - elif os.path.isdir(RNS.Reticulum.userdir+"/.config/lxmd") and os.path.isfile(Reticulum.userdir+"/.config/lxmd/config"): - configdir = RNS.Reticulum.userdir+"/.config/lxmd" - else: - configdir = RNS.Reticulum.userdir+"/.lxmd" - - configpath = configdir+"/config" - identitypath = configdir+"/identity" - identity = None - - if not os.path.isdir(configdir): - RNS.log("Specified configuration directory does not exist, exiting now", RNS.LOG_ERROR) - exit(201) - if not os.path.isfile(identitypath): - RNS.log("Identity file not found in specified configuration directory, exiting now", RNS.LOG_ERROR) - exit(202) - else: - identity = RNS.Identity.from_file(identitypath) - if identity == None: - RNS.log("Could not load the Primary Identity from "+identitypath, RNS.LOG_ERROR) - exit(4) - - else: - if not os.path.isfile(identity_path): - RNS.log("Identity file not found in specified configuration directory, exiting now", RNS.LOG_ERROR) - exit(202) - else: - identity = RNS.Identity.from_file(identity_path) - if identity == None: - RNS.log("Could not load the Primary Identity from "+identity_path, RNS.LOG_ERROR) - exit(4) - - if targetloglevel == None: targetloglevel = 3 - if verbosity != 0 or quietness != 0: targetloglevel = targetloglevel+verbosity-quietness try: - destination_hash = bytes.fromhex(target) - if len(destination_hash) != RNS.Identity.TRUNCATED_HASHLENGTH//8: raise ValueError(f"Destination hash length must be {RNS.Identity.TRUNCATED_HASHLENGTH//8*2} characters") + peer_destination_hash = bytes.fromhex(target) + if len(peer_destination_hash) != RNS.Identity.TRUNCATED_HASHLENGTH//8: raise ValueError(f"Destination hash length must be {RNS.Identity.TRUNCATED_HASHLENGTH//8*2} characters") except Exception as e: print(f"Invalid peer destination hash: {e}") exit(203) - - reticulum = RNS.Reticulum(configdir=rnsconfigdir, loglevel=targetloglevel, logdest=targetlogdest) - response = _request_sync(identity, destination_hash, timeout=timeout, exit_on_fail=True) + remote + _remote_init(configdir, rnsconfigdir, verbosity, quietness, identity_path) + response = _request_sync(identity, peer_destination_hash, remote_identity=_get_target_identity(remote), timeout=timeout, exit_on_fail=True) if response == LXMF.LXMPeer.LXMPeer.ERROR_NO_IDENTITY: print("Remote received no identity") @@ -570,99 +532,53 @@ def request_sync(target, configdir = None, rnsconfigdir = None, verbosity = 0, q print("The requested peer was not found") exit(206) else: - print(f"Sync requested for peer {RNS.prettyhexrep(destination_hash)}") + print(f"Sync requested for peer {RNS.prettyhexrep(peer_destination_hash)}") exit(0) -def query_status(identity, timeout=5, exit_on_fail=False): - control_destination = RNS.Destination(identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") +def query_status(identity, remote_identity, timeout=5, exit_on_fail=False): + control_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") timeout = time.time()+timeout def check_timeout(): if time.time() > timeout: if exit_on_fail: - RNS.log("Getting lxmd statistics timed out, exiting now", RNS.LOG_ERROR) + print("Getting lxmd statistics timed out, exiting now") exit(200) - else: - return LXMF.LXMPeer.LXMPeer.ERROR_TIMEOUT - else: - time.sleep(0.1) + else: return LXMF.LXMPeer.LXMPeer.ERROR_TIMEOUT + else: time.sleep(0.1) if not RNS.Transport.has_path(control_destination.hash): RNS.Transport.request_path(control_destination.hash) while not RNS.Transport.has_path(control_destination.hash): tc = check_timeout() - if tc: - return tc + if tc: return tc link = RNS.Link(control_destination) while not link.status == RNS.Link.ACTIVE: tc = check_timeout() - if tc: - return tc + if tc: return tc link.identify(identity) request_receipt = link.request(LXMF.LXMRouter.STATS_GET_PATH, data=None, response_callback=None, failed_callback=None) while not request_receipt.get_status() == RNS.RequestReceipt.READY: tc = check_timeout() - if tc: - return tc + if tc: return tc link.teardown() return request_receipt.get_response() -def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = 0, timeout=5, show_status=False, show_peers=False, identity_path=None): - global configpath, identitypath, storagedir, lxmdir - global lxmd_config, active_configuration, targetloglevel - targetlogdest = RNS.LOG_STDOUT +def get_status(remote=None, configdir=None, rnsconfigdir=None, verbosity=0, quietness=0, timeout=5, + show_status=False, show_peers=False, identity_path=None): - if identity_path == None: - if configdir == None: - if os.path.isdir("/etc/lxmd") and os.path.isfile("/etc/lxmd/config"): - configdir = "/etc/lxmd" - elif os.path.isdir(RNS.Reticulum.userdir+"/.config/lxmd") and os.path.isfile(Reticulum.userdir+"/.config/lxmd/config"): - configdir = RNS.Reticulum.userdir+"/.config/lxmd" - else: - configdir = RNS.Reticulum.userdir+"/.lxmd" - - configpath = configdir+"/config" - identitypath = configdir+"/identity" - identity = None - - if not os.path.isdir(configdir): - RNS.log("Specified configuration directory does not exist, exiting now", RNS.LOG_ERROR) - exit(201) - if not os.path.isfile(identitypath): - RNS.log("Identity file not found in specified configuration directory, exiting now", RNS.LOG_ERROR) - exit(202) - else: - identity = RNS.Identity.from_file(identitypath) - if identity == None: - RNS.log("Could not load the Primary Identity from "+identitypath, RNS.LOG_ERROR) - exit(4) - - else: - if not os.path.isfile(identity_path): - RNS.log("Identity file not found in specified configuration directory, exiting now", RNS.LOG_ERROR) - exit(202) - else: - identity = RNS.Identity.from_file(identity_path) - if identity == None: - RNS.log("Could not load the Primary Identity from "+identity_path, RNS.LOG_ERROR) - exit(4) - - if targetloglevel == None: - targetloglevel = 3 - if verbosity != 0 or quietness != 0: - targetloglevel = targetloglevel+verbosity-quietness - - reticulum = RNS.Reticulum(configdir=rnsconfigdir, loglevel=targetloglevel, logdest=targetlogdest) - response = query_status(identity, timeout=timeout, exit_on_fail=True) + global identity + _remote_init(configdir, rnsconfigdir, verbosity, quietness, identity_path) + response = query_status(identity, remote_identity=_get_target_identity(remote), timeout=timeout, exit_on_fail=True) if response == LXMF.LXMPeer.LXMPeer.ERROR_NO_IDENTITY: - RNS.log("Remote received no identity") + print("Remote received no identity") exit(203) if response == LXMF.LXMPeer.LXMPeer.ERROR_NO_ACCESS: - RNS.log("Access denied") + print("Access denied") exit(204) else: s = response @@ -776,6 +692,76 @@ def get_status(configdir = None, rnsconfigdir = None, verbosity = 0, quietness = print(f"{ind*2}Sync state : {pmuh} unhandled message{ms}, {ls}") print("") +def _get_target_identity(remote=None, timeout=5): + global identity + timeout = time.time()+timeout + def check_timeout(): + if time.time() > timeout: + print("Resolving remote identity timed out, exiting now") + exit(200) + else: time.sleep(0.1) + + if remote == None: return identity + else: + try: + destination_hash = bytes.fromhex(remote) + if len(destination_hash) != RNS.Identity.TRUNCATED_HASHLENGTH//8: raise ValueError(f"Destination hash length must be {RNS.Identity.TRUNCATED_HASHLENGTH//8*2} characters") + except Exception as e: + print(f"Invalid remote destination hash: {e}") + exit(203) + + remote_identity = RNS.Identity.recall(destination_hash) + if remote_identity: return remote_identity + else: + if not RNS.Transport.has_path(destination_hash): + RNS.Transport.request_path(destination_hash) + while not RNS.Transport.has_path(destination_hash): + tc = check_timeout() + if tc: return tc + + return RNS.Identity.recall(destination_hash) + +def _remote_init(configdir=None, rnsconfigdir=None, verbosity=0, quietness=0, identity_path=None): + global configpath, identitypath, storagedir, lxmdir, identity + global lxmd_config, active_configuration, targetloglevel + targetlogdest = RNS.LOG_STDOUT + + if identity_path == None: + if configdir == None: + if os.path.isdir("/etc/lxmd") and os.path.isfile("/etc/lxmd/config"): configdir = "/etc/lxmd" + elif os.path.isdir(RNS.Reticulum.userdir+"/.config/lxmd") and os.path.isfile(Reticulum.userdir+"/.config/lxmd/config"): configdir = RNS.Reticulum.userdir+"/.config/lxmd" + else: configdir = RNS.Reticulum.userdir+"/.lxmd" + + configpath = configdir+"/config" + identitypath = configdir+"/identity" + identity = None + + if not os.path.isdir(configdir): + RNS.log("Specified configuration directory does not exist, exiting now", RNS.LOG_ERROR) + exit(201) + if not os.path.isfile(identitypath): + RNS.log("Identity file not found in specified configuration directory, exiting now", RNS.LOG_ERROR) + exit(202) + else: + identity = RNS.Identity.from_file(identitypath) + if identity == None: + RNS.log("Could not load the Primary Identity from "+identitypath, RNS.LOG_ERROR) + exit(4) + + else: + if not os.path.isfile(identity_path): + RNS.log("Identity file not found in specified configuration directory, exiting now", RNS.LOG_ERROR) + exit(202) + else: + identity = RNS.Identity.from_file(identity_path) + if identity == None: + RNS.log("Could not load the Primary Identity from "+identity_path, RNS.LOG_ERROR) + exit(4) + + if targetloglevel == None: targetloglevel = 3 + if verbosity != 0 or quietness != 0: targetloglevel = targetloglevel+verbosity-quietness + + reticulum = RNS.Reticulum(configdir=rnsconfigdir, loglevel=targetloglevel, logdest=targetlogdest) def main(): try: @@ -790,8 +776,9 @@ def main(): parser.add_argument("--status", action="store_true", default=False, help="display node status") parser.add_argument("--peers", action="store_true", default=False, help="display peered nodes") parser.add_argument("--sync", action="store", default=None, help="request a sync with the specified peer", type=str) - parser.add_argument("--timeout", action="store", default=5, help="timeout in seconds for query operations", type=float) - parser.add_argument("--identity", action="store", default=None, help="path to identity used for query request", type=str) + parser.add_argument("--timeout", action="store", default=None, help="timeout in seconds for query operations", type=float) + parser.add_argument("-r", "--remote", action="store", default=None, help="remote propagation node destination hash", type=str) + parser.add_argument("--identity", action="store", default=None, help="path to identity used for remote requests", type=str) parser.add_argument("--exampleconfig", action="store_true", default=False, help="print verbose configuration example to stdout and exit") parser.add_argument("--version", action="version", version="lxmd {version}".format(version=__version__)) @@ -802,6 +789,7 @@ def main(): exit() if args.status or args.peers: + if not args.timeout: args.timeout = 5 get_status(configdir = args.config, rnsconfigdir=args.rnsconfig, verbosity=args.verbose, @@ -809,17 +797,20 @@ def main(): timeout=args.timeout, show_status=args.status, show_peers=args.peers, - identity_path=args.identity) + identity_path=args.identity, + remote=args.remote) exit() if args.sync: + if not args.timeout: args.timeout = 10 request_sync(target=args.sync, configdir = args.config, rnsconfigdir=args.rnsconfig, verbosity=args.verbose, quietness=args.quiet, timeout=args.timeout, - identity_path=args.identity) + identity_path=args.identity, + remote=args.remote) program_setup(configdir = args.config, rnsconfigdir=args.rnsconfig, From fa9fd2ae013e39bc604d68d1e516a652b5c66916 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 1 Nov 2025 13:10:28 +0100 Subject: [PATCH 356/436] Added remote status and control by allow-list for lxmd --- LXMF/LXMRouter.py | 28 ++++++++++++++++++++-------- LXMF/Utilities/lxmd.py | 24 +++++++++++++++++++----- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index f25c8f0..4f033b7 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -101,6 +101,7 @@ class LXMRouter: self.prioritised_list = [] self.ignored_list = [] self.allowed_list = [] + self.control_allowed_list = [] self.auth_required = False self.retain_synced_on_node = False @@ -450,6 +451,16 @@ class LXMRouter: else: raise ValueError("Disallowed identity hash must be "+str(RNS.Identity.TRUNCATED_HASHLENGTH//8)+" bytes") + def allow_control(self, identity_hash=None): + if isinstance(identity_hash, bytes) and len(identity_hash) == RNS.Identity.TRUNCATED_HASHLENGTH//8: + if not identity_hash in self.control_allowed_list: self.control_allowed_list.append(identity_hash) + else: raise ValueError("Allowed identity hash must be "+str(RNS.Identity.TRUNCATED_HASHLENGTH//8)+" bytes") + + def disallow_control(self, identity_hash=None): + if isinstance(identity_hash, bytes) and len(identity_hash) == RNS.Identity.TRUNCATED_HASHLENGTH//8: + if identity_hash in self.control_allowed_list: self.control_allowed_list.pop(identity_hash) + else: raise ValueError("Disallowed identity hash must be "+str(RNS.Identity.TRUNCATED_HASHLENGTH//8)+" bytes") + def prioritise(self, destination_hash=None): if isinstance(destination_hash, bytes) and len(destination_hash) == RNS.Reticulum.TRUNCATED_HASHLENGTH//8: if not destination_hash in self.prioritised_list: @@ -628,9 +639,10 @@ class LXMRouter: self.propagation_destination.register_request_handler(LXMPeer.OFFER_REQUEST_PATH, self.offer_request, allow = RNS.Destination.ALLOW_ALL) self.propagation_destination.register_request_handler(LXMPeer.MESSAGE_GET_PATH, self.message_get_request, allow = RNS.Destination.ALLOW_ALL) - self.control_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") - self.control_destination.register_request_handler(LXMRouter.STATS_GET_PATH, self.stats_get_request, allow = RNS.Destination.ALLOW_LIST, allowed_list=[self.identity.hash]) - self.control_destination.register_request_handler(LXMRouter.SYNC_REQUEST_PATH, self.peer_sync_request, allow = RNS.Destination.ALLOW_LIST, allowed_list=[self.identity.hash]) + self.control_allowed_list = [self.identity.hash] + self.control_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") + self.control_destination.register_request_handler(LXMRouter.STATS_GET_PATH, self.stats_get_request, allow = RNS.Destination.ALLOW_LIST, allowed_list=self.control_allowed_list) + self.control_destination.register_request_handler(LXMRouter.SYNC_REQUEST_PATH, self.peer_sync_request, allow = RNS.Destination.ALLOW_LIST, allowed_list=self.control_allowed_list) if self.message_storage_limit != None: limit_str = ", limit is "+RNS.prettysize(self.message_storage_limit) @@ -807,13 +819,13 @@ class LXMRouter: return node_stats def stats_get_request(self, path, data, request_id, remote_identity, requested_at): - if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY - elif remote_identity.hash != self.identity.hash: return LXMPeer.ERROR_NO_ACCESS - else: return self.compile_stats() + if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY + elif remote_identity.hash not in self.control_allowed_list: return LXMPeer.ERROR_NO_ACCESS + else: return self.compile_stats() def peer_sync_request(self, path, data, request_id, remote_identity, requested_at): - if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY - elif remote_identity.hash != self.identity.hash: return LXMPeer.ERROR_NO_ACCESS + if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY + elif remote_identity.hash not in self.control_allowed_list: return LXMPeer.ERROR_NO_ACCESS else: if type(data) != bytes: return LXMPeer.ERROR_INVALID_DATA elif len(data) != RNS.Identity.TRUNCATED_HASHLENGTH//8: return LXMPeer.ERROR_INVALID_DATA diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index b4599df..1d95fd7 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -187,6 +187,11 @@ def apply_config(): active_configuration["prioritised_lxmf_destinations"] = lxmd_config["propagation"].as_list("prioritise_destinations") else: active_configuration["prioritised_lxmf_destinations"] = [] + + if "propagation" in lxmd_config and "control_allowed" in lxmd_config["propagation"]: + active_configuration["control_allowed_identities"] = lxmd_config["propagation"].as_list("control_allowed") + else: + active_configuration["control_allowed_identities"] = [] if "propagation" in lxmd_config and "static_peers" in lxmd_config["propagation"]: static_peers = lxmd_config["propagation"].as_list("static_peers") @@ -410,13 +415,16 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo for dest_str in active_configuration["prioritised_lxmf_destinations"]: try: dest_hash = bytes.fromhex(dest_str) - if len(dest_hash) == RNS.Reticulum.TRUNCATED_HASHLENGTH//8: - message_router.prioritise(dest_hash) - - except Exception as e: - RNS.log("Cannot prioritise "+str(dest_str)+", it is not a valid destination hash", RNS.LOG_ERROR) + if len(dest_hash) == RNS.Reticulum.TRUNCATED_HASHLENGTH//8: message_router.prioritise(dest_hash) + except Exception as e: RNS.log("Cannot prioritise "+str(dest_str)+", it is not a valid destination hash", RNS.LOG_ERROR) message_router.enable_propagation() + + for ident_str in active_configuration["control_allowed_identities"]: + try: + identity_hash = bytes.fromhex(ident_str) + if len(identity_hash) == RNS.Reticulum.TRUNCATED_HASHLENGTH//8: message_router.allow_control(identity_hash) + except Exception as e: RNS.log(f"Cannot allow control from {ident_str}, it is not a valid identity hash", RNS.LOG_ERROR) RNS.log("LXMF Propagation Node started on "+RNS.prettyhexrep(message_router.propagation_destination.hash)) @@ -834,6 +842,12 @@ __default_lxmd_config__ = """# This is an example LXM Daemon config file. enable_node = no +# You can specify identity hashes for remotes +# that are allowed to control and query status +# for this propagation node. + +# control_allowed = 7d7e542829b40f32364499b27438dba8, 437229f8e29598b2282b88bad5e44698 + # An optional name for this node, included # in announces. From 9dc998f149780b3b9acc0c791a759b6caaf09675 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 1 Nov 2025 17:32:45 +0100 Subject: [PATCH 357/436] Added peering break option to lxmd command line interface --- LXMF/LXMRouter.py | 18 +++++++- LXMF/Utilities/lxmd.py | 93 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 108 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 4f033b7..3e29938 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -39,7 +39,7 @@ class LXMRouter: NODE_ANNOUNCE_DELAY = 20 - MAX_PEERS = 50 + MAX_PEERS = 20 AUTOPEER = True AUTOPEER_MAXDEPTH = 4 FASTEST_N_RANDOM_POOL = 2 @@ -77,6 +77,7 @@ class LXMRouter: STATS_GET_PATH = "/pn/get/stats" SYNC_REQUEST_PATH = "/pn/peer/sync" + UNPEER_REQUEST_PATH = "/pn/peer/unpeer" ### Developer-facing API ############################## @@ -643,6 +644,7 @@ class LXMRouter: self.control_destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") self.control_destination.register_request_handler(LXMRouter.STATS_GET_PATH, self.stats_get_request, allow = RNS.Destination.ALLOW_LIST, allowed_list=self.control_allowed_list) self.control_destination.register_request_handler(LXMRouter.SYNC_REQUEST_PATH, self.peer_sync_request, allow = RNS.Destination.ALLOW_LIST, allowed_list=self.control_allowed_list) + self.control_destination.register_request_handler(LXMRouter.UNPEER_REQUEST_PATH, self.peer_unpeer_request, allow = RNS.Destination.ALLOW_LIST, allowed_list=self.control_allowed_list) if self.message_storage_limit != None: limit_str = ", limit is "+RNS.prettysize(self.message_storage_limit) @@ -835,6 +837,18 @@ class LXMRouter: self.peers[data].sync() return True + def peer_unpeer_request(self, path, data, request_id, remote_identity, requested_at): + if remote_identity == None: return LXMPeer.ERROR_NO_IDENTITY + elif remote_identity.hash not in self.control_allowed_list: return LXMPeer.ERROR_NO_ACCESS + else: + if type(data) != bytes: return LXMPeer.ERROR_INVALID_DATA + elif len(data) != RNS.Identity.TRUNCATED_HASHLENGTH//8: return LXMPeer.ERROR_INVALID_DATA + else: + if not data in self.peers: return LXMPeer.ERROR_NOT_FOUND + else: + self.unpeer(data) + return True + ### Utility & Maintenance ############################# ####################################################### @@ -1318,6 +1332,8 @@ class LXMRouter: self.propagation_destination.deregister_request_handler(LXMPeer.OFFER_REQUEST_PATH) self.propagation_destination.deregister_request_handler(LXMPeer.MESSAGE_GET_PATH) self.propagation_destination.deregister_request_handler(LXMRouter.STATS_GET_PATH) + self.propagation_destination.deregister_request_handler(LXMRouter.SYNC_REQUEST_PATH) + self.propagation_destination.deregister_request_handler(LXMRouter.UNPEER_REQUEST_PATH) for link in self.active_propagation_links: try: if link.status == RNS.Link.ACTIVE: diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 1d95fd7..5a8c9aa 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -539,10 +539,82 @@ def request_sync(target, remote=None, configdir=None, rnsconfigdir=None, verbosi elif response == LXMF.LXMPeer.LXMPeer.ERROR_NOT_FOUND: print("The requested peer was not found") exit(206) + elif response == None: + print("Empty response received") + exit(207) else: print(f"Sync requested for peer {RNS.prettyhexrep(peer_destination_hash)}") exit(0) +def _request_unpeer(identity, destination_hash, remote_identity, timeout=15, exit_on_fail=False): + control_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") + + timeout = time.time()+timeout + def check_timeout(): + if time.time() > timeout: + if exit_on_fail: + print("Requesting lxmd peering break timed out, exiting now") + exit(200) + else: return LXMF.LXMPeer.LXMPeer.ERROR_TIMEOUT + else: time.sleep(0.1) + + if not RNS.Transport.has_path(control_destination.hash): + RNS.Transport.request_path(control_destination.hash) + while not RNS.Transport.has_path(control_destination.hash): + tc = check_timeout() + if tc: + return tc + + link = RNS.Link(control_destination) + while not link.status == RNS.Link.ACTIVE: + tc = check_timeout() + if tc: + return tc + + link.identify(identity) + request_receipt = link.request(LXMF.LXMRouter.UNPEER_REQUEST_PATH, data=destination_hash, response_callback=None, failed_callback=None) + while not request_receipt.get_status() == RNS.RequestReceipt.READY: + tc = check_timeout() + if tc: + return tc + + link.teardown() + return request_receipt.get_response() + + +def request_unpeer(target, remote=None, configdir=None, rnsconfigdir=None, verbosity=0, quietness=0, timeout=15, identity_path=None): + global configpath, identitypath, storagedir, lxmdir + global lxmd_config, active_configuration, targetloglevel + + try: + peer_destination_hash = bytes.fromhex(target) + if len(peer_destination_hash) != RNS.Identity.TRUNCATED_HASHLENGTH//8: raise ValueError(f"Destination hash length must be {RNS.Identity.TRUNCATED_HASHLENGTH//8*2} characters") + except Exception as e: + print(f"Invalid peer destination hash: {e}") + exit(203) + remote + _remote_init(configdir, rnsconfigdir, verbosity, quietness, identity_path) + response = _request_unpeer(identity, peer_destination_hash, remote_identity=_get_target_identity(remote), timeout=timeout, exit_on_fail=True) + + if response == LXMF.LXMPeer.LXMPeer.ERROR_NO_IDENTITY: + print("Remote received no identity") + exit(203) + elif response == LXMF.LXMPeer.LXMPeer.ERROR_NO_ACCESS: + print("Access denied") + exit(204) + elif response == LXMF.LXMPeer.LXMPeer.ERROR_INVALID_DATA: + print("Invalid data received by remote") + exit(205) + elif response == LXMF.LXMPeer.LXMPeer.ERROR_NOT_FOUND: + print("The requested peer was not found") + exit(206) + elif response == None: + print("Empty response received") + exit(207) + else: + print(f"Broke peering with {RNS.prettyhexrep(peer_destination_hash)}") + exit(0) + def query_status(identity, remote_identity, timeout=5, exit_on_fail=False): control_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") @@ -588,6 +660,9 @@ def get_status(remote=None, configdir=None, rnsconfigdir=None, verbosity=0, quie if response == LXMF.LXMPeer.LXMPeer.ERROR_NO_ACCESS: print("Access denied") exit(204) + elif response == None: + print("Empty response received") + exit(207) else: s = response mutil = round((s["messagestore"]["bytes"]/s["messagestore"]["limit"])*100, 2) @@ -784,6 +859,7 @@ def main(): parser.add_argument("--status", action="store_true", default=False, help="display node status") parser.add_argument("--peers", action="store_true", default=False, help="display peered nodes") parser.add_argument("--sync", action="store", default=None, help="request a sync with the specified peer", type=str) + parser.add_argument("-b", "--break", dest="unpeer", action="store", default=None, help="break peering with the specified peer", type=str) parser.add_argument("--timeout", action="store", default=None, help="timeout in seconds for query operations", type=float) parser.add_argument("-r", "--remote", action="store", default=None, help="remote propagation node destination hash", type=str) parser.add_argument("--identity", action="store", default=None, help="path to identity used for remote requests", type=str) @@ -819,6 +895,19 @@ def main(): timeout=args.timeout, identity_path=args.identity, remote=args.remote) + exit() + + if args.unpeer: + if not args.timeout: args.timeout = 10 + request_unpeer(target=args.unpeer, + configdir = args.config, + rnsconfigdir=args.rnsconfig, + verbosity=args.verbose, + quietness=args.quiet, + timeout=args.timeout, + identity_path=args.identity, + remote=args.remote) + exit() program_setup(configdir = args.config, rnsconfigdir=args.rnsconfig, @@ -944,9 +1033,9 @@ autopeer_maxdepth = 4 # You can configure the maximum number of other # propagation nodes that this node will peer -# with automatically. The default is 50. +# with automatically. The default is 20. -# max_peers = 25 +# max_peers = 20 # You can configure a list of static propagation # node peers, that this node will always be From 4350a239e4d7e09eaea5f31aaa1769081c892719 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 1 Nov 2025 17:35:36 +0100 Subject: [PATCH 358/436] Cleanup --- LXMF/LXMRouter.py | 35 ++++++++++------------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 3e29938..b1676f0 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -713,27 +713,15 @@ class LXMRouter: def set_information_storage_limit(self, kilobytes = None, megabytes = None, gigabytes = None): limit_bytes = 0 - - if kilobytes != None: - limit_bytes += kilobytes*1000 - - if megabytes != None: - limit_bytes += megabytes*1000*1000 - - if gigabytes != None: - limit_bytes += gigabytes*1000*1000*1000 - - if limit_bytes == 0: - limit_bytes = None + if kilobytes != None: limit_bytes += kilobytes*1000 + if megabytes != None: limit_bytes += megabytes*1000*1000 + if gigabytes != None: limit_bytes += gigabytes*1000*1000*1000 + if limit_bytes == 0: limit_bytes = None try: - if limit_bytes == None or int(limit_bytes) > 0: - self.information_storage_limit = int(limit_bytes) - else: - raise ValueError("Cannot set LXMF information storage limit to "+str(limit_bytes)) - - except Exception as e: - raise ValueError("Cannot set LXMF information storage limit to "+str(limit_bytes)) + if limit_bytes == None or int(limit_bytes) > 0: self.information_storage_limit = int(limit_bytes) + else: raise ValueError("Cannot set LXMF information storage limit to "+str(limit_bytes)) + except Exception as e: raise ValueError("Cannot set LXMF information storage limit to "+str(limit_bytes)) def information_storage_limit(self): return self.information_storage_limit @@ -742,18 +730,15 @@ class LXMRouter: pass def delivery_link_available(self, destination_hash): - if destination_hash in self.direct_links or destination_hash in self.backchannel_links: - return True - else: - return False + if destination_hash in self.direct_links or destination_hash in self.backchannel_links: return True + else: return False ### Propagation Node Control ########################## ####################################################### def compile_stats(self): - if not self.propagation_node: - return None + if not self.propagation_node: return None else: peer_stats = {} for peer_id in self.peers.copy(): From 401328fa1689a130a017dcc8d2cbadf438d9bb71 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 1 Nov 2025 18:27:11 +0100 Subject: [PATCH 359/436] Allow configuring max remote peering cost --- LXMF/LXMF.py | 2 +- LXMF/LXMRouter.py | 5 ++--- LXMF/Utilities/lxmd.py | 6 ++++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index dbc0ab1..ede9c3a 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -173,7 +173,7 @@ def pn_announce_data_is_valid(data): try: if type(data) != bytes: return False else: data = msgpack.unpackb(data) - if len(data) < 7: raise ValueError("Invalid announce data: Insufficient peer data") + if len(data) < 7: raise ValueError("Invalid announce data: Insufficient peer data, likely from deprecated LXMF version") else: try: int(data[1]) except: raise ValueError("Invalid announce data: Could not decode timebase") diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index b1676f0..1d9cd5c 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -88,7 +88,7 @@ class LXMRouter: enforce_ratchets=False, enforce_stamps=False, static_peers = [], max_peers=None, from_static_only=False, sync_strategy=LXMPeer.STRATEGY_PERSISTENT, propagation_cost=PROPAGATION_COST, propagation_cost_flexibility=PROPAGATION_COST_FLEX, - peering_cost=PEERING_COST, name=None): + peering_cost=PEERING_COST, max_peering_cost=MAX_PEERING_COST, name=None): random.seed(os.urandom(10)) @@ -134,7 +134,7 @@ class LXMRouter: self.propagation_stamp_cost = propagation_cost self.propagation_stamp_cost_flexibility = propagation_cost_flexibility self.peering_cost = peering_cost - self.max_peering_cost = LXMRouter.MAX_PEERING_COST + self.max_peering_cost = max_peering_cost self.enforce_ratchets = enforce_ratchets self._enforce_stamps = enforce_stamps self.pending_deferred_stamps = {} @@ -2420,7 +2420,6 @@ class LXMRouter: else: propagation_stamp = selected_lxm.get_propagation_stamp(target_cost=pn_target_cost) - RNS.log(f"Generated propagation stamp: {RNS.hexrep(propagation_stamp)}") if propagation_stamp: selected_lxm.propagation_stamp = propagation_stamp selected_lxm.defer_propagation_stamp = False diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 5a8c9aa..59736db 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -377,6 +377,8 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo propagation_limit = active_configuration["propagation_transfer_max_accepted_size"], propagation_cost = active_configuration["propagation_stamp_cost_target"], propagation_cost_flexibility = active_configuration["propagation_stamp_cost_flexibility"], + peering_cost = active_configuration["peering_cost"], + max_peering_cost = active_configuration["remote_peering_cost_max"], sync_limit = active_configuration["propagation_sync_max_accepted_size"], delivery_limit = active_configuration["delivery_transfer_max_accepted_size"], max_peers = active_configuration["max_peers"], @@ -1009,7 +1011,7 @@ autopeer_maxdepth = 4 # value required for a remote node to peer with # and deliver messages to this node. -# peering_cost = 10 +# peering_cost = 18 # You can configure the maximum peering cost # of remote nodes that this node will peer with. @@ -1019,7 +1021,7 @@ autopeer_maxdepth = 4 # more computation time during initial peering # when generating the peering key. -# remote_peering_cost_max = 12 +# remote_peering_cost_max = 24 # You can tell the LXMF message router to # prioritise storage for one or more From d0f3385f75572d20dccf66e14a27068259057e40 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 1 Nov 2025 18:48:31 +0100 Subject: [PATCH 360/436] Added acceptance rate to lxmd stats output --- LXMF/LXMRouter.py | 4 +++- LXMF/Utilities/lxmd.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 1d9cd5c..10e1892 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -764,6 +764,7 @@ class LXMRouter: "network_distance": RNS.Transport.hops_to(peer_id), "rx_bytes": peer.rx_bytes, "tx_bytes": peer.tx_bytes, + "acceptance_rate": peer.acceptance_rate, "messages": { "offered": peer.offered, "outgoing": peer.outgoing, @@ -1430,7 +1431,8 @@ class LXMRouter: filepath = self.propagation_entries[transient_id][1] self.propagation_entries.pop(transient_id) os.unlink(filepath) - RNS.log("Client "+RNS.prettyhexrep(remote_destination.hash)+" purged message "+RNS.prettyhexrep(transient_id)+" at "+str(filepath), RNS.LOG_DEBUG) + # TODO: Remove debug + # RNS.log("Client "+RNS.prettyhexrep(remote_destination.hash)+" purged message "+RNS.prettyhexrep(transient_id)+" at "+str(filepath), RNS.LOG_DEBUG) except Exception as e: RNS.log("Error while processing message purge request from "+RNS.prettyhexrep(remote_destination.hash)+". The contained exception was: "+str(e), RNS.LOG_ERROR) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 59736db..8c7cd82 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -760,7 +760,7 @@ def get_status(remote=None, configdir=None, rnsconfigdir=None, verbosity=0, quie sstr = RNS.prettyspeed(p["str"]); sler = RNS.prettyspeed(p["ler"]) stl = RNS.prettysize(p["transfer_limit"]*1000); ssl = RNS.prettysize(p["sync_limit"]*1000) srxb = RNS.prettysize(p["rx_bytes"]); stxb = RNS.prettysize(p["tx_bytes"]); pmo = pm["offered"]; pmout = pm["outgoing"] - pmi = pm["incoming"]; pmuh = pm["unhandled"]; + pmi = pm["incoming"]; pmuh = pm["unhandled"]; ar = round(p["acceptance_rate"]*100, 2) if p["name"] == None: nn = "" else: nn = p["name"].strip().replace("\n", "").replace("\r", "") if len(nn) > 45: nn = f"{nn[:45]}..." @@ -771,7 +771,7 @@ def get_status(remote=None, configdir=None, rnsconfigdir=None, verbosity=0, quie print(f"{ind*2}Sync key : {pk}") print(f"{ind*2}Speeds : {sstr} STR, {sler} LER") print(f"{ind*2}Limits : {stl} message limit, {ssl} sync limit") - print(f"{ind*2}Messages : {pmo} offered, {pmout} outgoing, {pmi} incoming") + print(f"{ind*2}Messages : {pmo} offered, {pmout} outgoing, {pmi} incoming, {ar}% acceptance rate") print(f"{ind*2}Traffic : {srxb} received, {stxb} sent") ms = "" if pm["unhandled"] == 1 else "s" print(f"{ind*2}Sync state : {pmuh} unhandled message{ms}, {ls}") From 747ddbddd528b6492a98f6eccd719cdce1df99a3 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 2 Nov 2025 02:27:57 +0100 Subject: [PATCH 361/436] Implemented duplicate signalling on PN message download --- LXMF/Handlers.py | 2 +- LXMF/LXMRouter.py | 16 ++++++++++++---- LXMF/Utilities/lxmd.py | 3 ++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index aa39ea2..01841f9 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -54,7 +54,7 @@ class LXMFPropagationAnnounceHandler: metadata = data[6] if destination_hash in self.lxmrouter.static_peers: - static_peer = self.lxmrouter.static_peers[destination_hash] + static_peer = self.lxmrouter.peers[destination_hash] if not is_path_response or static_peer.last_heard == 0: self.lxmrouter.peer(destination_hash=destination_hash, timestamp=node_timebase, diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 10e1892..0cc59b1 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -75,6 +75,8 @@ class LXMRouter: PR_ALL_MESSAGES = 0x00 + DUPLICATE_SIGNAL = "lxmf_duplicate" + STATS_GET_PATH = "/pn/get/stats" SYNC_REQUEST_PATH = "/pn/peer/sync" UNPEER_REQUEST_PATH = "/pn/peer/unpeer" @@ -1537,10 +1539,12 @@ class LXMRouter: self.propagation_transfer_state = LXMRouter.PR_NO_ACCESS else: + duplicates = 0 if request_receipt.response != None and len(request_receipt.response) > 0: haves = [] for lxmf_data in request_receipt.response: - self.lxmf_propagation(lxmf_data) + result = self.lxmf_propagation(lxmf_data, signal_duplicate=LXMRouter.DUPLICATE_SIGNAL) + if result == LXMRouter.DUPLICATE_SIGNAL: duplicates += 1 haves.append(RNS.Identity.full_hash(lxmf_data)) # Return a list of successfully received messages to the node. @@ -1556,6 +1560,7 @@ class LXMRouter: self.propagation_transfer_state = LXMRouter.PR_COMPLETE self.propagation_transfer_progress = 1.0 + self.propagation_transfer_last_duplicates = duplicates self.propagation_transfer_last_result = len(request_receipt.response) self.save_locally_delivered_transient_ids() @@ -1674,11 +1679,14 @@ class LXMRouter: def get_outbound_lxm_stamp_cost(self, lxm_hash): for lxm in self.pending_outbound: if lxm.hash == lxm_hash: - return lxm.stamp_cost + if lxm.outbound_ticket: return None + else: return lxm.stamp_cost for lxm_id in self.pending_deferred_stamps: if self.pending_deferred_stamps[lxm_id].hash == lxm_hash: - return self.pending_deferred_stamps[lxm_id].stamp_cost + lxm = self.pending_deferred_stamps[lxm_id] + if lxm.outbound_ticket: return None + else: return lxm.stamp_cost return None @@ -1689,7 +1697,7 @@ class LXMRouter: for lxm_id in self.pending_deferred_stamps: if self.pending_deferred_stamps[lxm_id].hash == lxm_hash: - return self.pending_deferred_stamps[lxm_id].stamp_cost + return self.pending_deferred_stamps[lxm_id].propagation_target_cost return None diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 8c7cd82..de69ab1 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -758,7 +758,8 @@ def get_status(remote=None, configdir=None, rnsconfigdir=None, verbosity=0, quie ls = "never synced" sstr = RNS.prettyspeed(p["str"]); sler = RNS.prettyspeed(p["ler"]) - stl = RNS.prettysize(p["transfer_limit"]*1000); ssl = RNS.prettysize(p["sync_limit"]*1000) + stl = RNS.prettysize(p["transfer_limit"]*1000) if p["transfer_limit"] else "Unknown" + ssl = RNS.prettysize(p["sync_limit"]*1000) if p["sync_limit"] else "unknown" srxb = RNS.prettysize(p["rx_bytes"]); stxb = RNS.prettysize(p["tx_bytes"]); pmo = pm["offered"]; pmout = pm["outgoing"] pmi = pm["incoming"]; pmuh = pm["unhandled"]; ar = round(p["acceptance_rate"]*100, 2) if p["name"] == None: nn = "" From f383450b3716b589bdf220fc70b50ab8fd29eb28 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 2 Nov 2025 02:29:32 +0100 Subject: [PATCH 362/436] Implemented duplicate signalling on PN message download --- LXMF/LXMRouter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 0cc59b1..6c215f8 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -150,6 +150,7 @@ class LXMRouter: self.propagation_transfer_state = LXMRouter.PR_IDLE self.propagation_transfer_progress = 0.0 self.propagation_transfer_last_result = None + self.propagation_transfer_last_duplicates = None self.propagation_transfer_max_messages = None self.prioritise_rotating_unreachable_peers = False self.active_propagation_links = [] From 8e3ffb0d2ae9f2dac7480d4d6e7cdd86d797fd09 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 2 Nov 2025 02:48:19 +0100 Subject: [PATCH 363/436] Stat query default identity as local --- LXMF/Utilities/lxmd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index de69ab1..af667a5 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -617,7 +617,8 @@ def request_unpeer(target, remote=None, configdir=None, rnsconfigdir=None, verbo print(f"Broke peering with {RNS.prettyhexrep(peer_destination_hash)}") exit(0) -def query_status(identity, remote_identity, timeout=5, exit_on_fail=False): +def query_status(identity, remote_identity=None, timeout=5, exit_on_fail=False): + if remote_identity == None: remote_identity = identity control_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation", "control") timeout = time.time()+timeout From e1905b85d7de82d92bab2a278795270c9abecbbe Mon Sep 17 00:00:00 2001 From: bus Date: Sun, 2 Nov 2025 08:12:33 +0000 Subject: [PATCH 364/436] Fix stored messages always getting deleted if config path has underscores --- LXMF/LXMRouter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 6c215f8..27ea071 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1090,9 +1090,10 @@ class LXMRouter: for transient_id in self.propagation_entries.copy(): entry = self.propagation_entries[transient_id] filepath = entry[1] - components = filepath.split("_") + filename = os.path.split(filepath)[-1] + components = filename.split("_") - if len(components) == 2 and float(components[1]) > 0 and len(os.path.split(components[0])[1]) == (RNS.Identity.HASHLENGTH//8)*2: + if len(components) == 2 and float(components[1]) > 0 and len(components[0]) == (RNS.Identity.HASHLENGTH//8)*2: timestamp = float(components[1]) if now > timestamp+LXMRouter.MESSAGE_EXPIRY: RNS.log("Purging message "+RNS.prettyhexrep(transient_id)+" due to expiry", RNS.LOG_EXTREME) From 383d953e064d88d1d057fc29cc4e3e3c5adf8829 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 2 Nov 2025 11:34:22 +0100 Subject: [PATCH 365/436] Don't hog all cores on high core-count machines --- LXMF/LXStamper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/LXMF/LXStamper.py b/LXMF/LXStamper.py index 8ebefd7..56e2500 100644 --- a/LXMF/LXStamper.py +++ b/LXMF/LXStamper.py @@ -178,7 +178,8 @@ def job_linux(stamp_cost, workblock, message_id): allow_kill = True stamp = None total_rounds = 0 - jobs = multiprocessing.cpu_count() + cores = multiprocessing.cpu_count() + jobs = cores if cores <= 12 else int(cores/2) stop_event = multiprocessing.Event() result_queue = multiprocessing.Queue(1) rounds_queue = multiprocessing.Queue() From 9c5fa4a0667b8db75b592ee79e58477d9f7680ef Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 2 Nov 2025 11:35:12 +0100 Subject: [PATCH 366/436] Take stamp value into account when cleaning message store --- LXMF/LXMRouter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 6c215f8..41b9618 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1090,9 +1090,10 @@ class LXMRouter: for transient_id in self.propagation_entries.copy(): entry = self.propagation_entries[transient_id] filepath = entry[1] + stamp_value = entry[6] components = filepath.split("_") - if len(components) == 2 and float(components[1]) > 0 and len(os.path.split(components[0])[1]) == (RNS.Identity.HASHLENGTH//8)*2: + if len(components) >= 3 and float(components[1]) > 0 and len(os.path.split(components[0])[1]) == (RNS.Identity.HASHLENGTH//8)*2 and int(components[2]) == stamp_value: timestamp = float(components[1]) if now > timestamp+LXMRouter.MESSAGE_EXPIRY: RNS.log("Purging message "+RNS.prettyhexrep(transient_id)+" due to expiry", RNS.LOG_EXTREME) From 99830b6e8bcdee4c4cbe830c79c60aa866baad6b Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 2 Nov 2025 12:02:21 +0100 Subject: [PATCH 367/436] Only index messages with stamp value set --- LXMF/LXMRouter.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index bde1cda..b3bf0f2 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -544,21 +544,19 @@ class LXMRouter: st = time.time(); RNS.log("Indexing messagestore...", RNS.LOG_NOTICE) for filename in os.listdir(self.messagepath): components = filename.split("_") - if len(components) >= 2: + if len(components) >= 3: if float(components[1]) > 0: if len(components[0]) == RNS.Identity.HASHLENGTH//8*2: try: transient_id = bytes.fromhex(components[0]) received = float(components[1]) + stamp_value = int(components[2]) filepath = self.messagepath+"/"+filename msg_size = os.path.getsize(filepath) file = open(filepath, "rb") destination_hash = file.read(LXMessage.DESTINATION_LENGTH) file.close() - if len(components) >= 3: stamp_value = int(components[2]) - else: stamp_value = None - self.propagation_entries[transient_id] = [ destination_hash, # 0: Destination hash filepath, # 1: Storage location From bc3f4ecff528205aa29dc7fae829f33838c7c377 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 2 Nov 2025 18:35:15 +0100 Subject: [PATCH 368/436] Handle client propagation transfer limits separately from PN peers --- LXMF/LXMRouter.py | 92 ++++++++++++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 36 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index b3bf0f2..792fa08 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -154,6 +154,7 @@ class LXMRouter: self.propagation_transfer_max_messages = None self.prioritise_rotating_unreachable_peers = False self.active_propagation_links = [] + self.validated_peer_links = {} self.locally_delivered_transient_ids = {} self.locally_processed_transient_ids = {} self.outbound_stamp_costs = {} @@ -910,6 +911,8 @@ class LXMRouter: if inactive_time > LXMRouter.LINK_MAX_INACTIVITY: link.teardown() closed_links.append(link_hash) + if link.link_id in self.validated_peer_links: + self.validated_peer_links.pop(link.link_id) for link_hash in closed_links: cleaned_link = self.direct_links.pop(link_hash) @@ -2101,7 +2104,10 @@ class LXMRouter: self.lxmf_propagation(lxmf_data, stamp_value=stamp_value, stamp_data=stamp_data) self.client_propagation_messages_received += 1 - if len(validated_messages) == len(messages): packet.prove() + if len(validated_messages) == len(messages): + ms = "" if len(messages) == 1 else "s" + RNS.log(f"Received {len(messages)} propagation message{ms} from client with valid stamp{ms}", RNS.LOG_DEBUG) + packet.prove() else: RNS.log("Propagation transfer from client contained messages with invalid stamps", RNS.LOG_NOTICE) reject_data = msgpack.packb([LXMPeer.ERROR_INVALID_STAMP]) @@ -2151,6 +2157,7 @@ class LXMRouter: else: RNS.log(f"Peering key validated for incoming offer in {RNS.prettytime(td)}", RNS.LOG_DEBUG) + self.validated_peer_links[link_id] = True for transient_id in transient_ids: if not transient_id in self.propagation_entries: wanted_ids.append(transient_id) @@ -2174,7 +2181,7 @@ class LXMRouter: remote_timebase = data[0] messages = data[1] remote_hash = None - remote_str = "unknown peer" + remote_str = "unknown client" if remote_identity != None: remote_destination = RNS.Destination(remote_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") @@ -2203,42 +2210,55 @@ class LXMRouter: RNS.log(f"Auto-peering with {remote_str} discovered via incoming sync", RNS.LOG_DEBUG) # TODO: Remove debug self.peer(remote_hash, remote_timebase, remote_transfer_limit, remote_sync_limit, remote_stamp_cost, remote_stamp_flex, remote_peering_cost, remote_metadata) - ms = "" if len(messages) == 1 else "s" - RNS.log(f"Received {len(messages)} message{ms} from {remote_str}, validating stamps...", RNS.LOG_VERBOSE) + peering_key_valid = False + if remote_identity != None: + if resource.link.link_id in self.validated_peer_links and self.validated_peer_links[resource.link.link_id] == True: + peering_key_valid = True - min_accepted_cost = max(0, self.propagation_stamp_cost-self.propagation_stamp_cost_flexibility) - validated_messages = LXStamper.validate_pn_stamps(messages, min_accepted_cost) - if len(validated_messages) == len(messages): RNS.log(f"All message stamps validated from {remote_str}", RNS.LOG_VERBOSE) - else: RNS.log(f"Transfer from {remote_str} contained {len(messages)-len(validated_messages)} invalid stamps", RNS.LOG_WARNING) - - for validated_entry in validated_messages: - transient_id = validated_entry[0] - lxmf_data = validated_entry[1] - stamp_value = validated_entry[2] - stamp_data = validated_entry[3] - peer = None - - if remote_hash != None and remote_hash in self.peers: - peer = self.peers[remote_hash] - peer.incoming += 1 - peer.rx_bytes += len(lxmf_data) - else: - if remote_identity != None: - self.unpeered_propagation_incoming += 1 - self.unpeered_propagation_rx_bytes += len(lxmf_data) - else: - self.client_propagation_messages_received += 1 - - self.lxmf_propagation(lxmf_data, from_peer=peer, stamp_value=stamp_value, stamp_data=stamp_data) - if peer != None: peer.queue_handled_message(transient_id) - - invalid_message_count = len(messages) - len(validated_messages) - if invalid_message_count > 0: + if not peering_key_valid and len(messages) > 1: resource.link.teardown() - throttle_time = LXMRouter.PN_STAMP_THROTTLE - self.throttled_peers[remote_hash] = time.time()+throttle_time - ms = "" if invalid_message_count == 1 else "s" - RNS.log(f"Propagation transfer from {remote_str} contained {invalid_message_count} message{ms} with invalid stamps, throttled for {RNS.prettytime(throttle_time)}", RNS.LOG_NOTICE) + RNS.log(f"Received multiple propagation messages from {remote_str} without valid peering key presentation. This is not supposed to happen, ignoring.", RNS.LOG_WARNING) + RNS.log(f"Clients and peers without a valid peering key can only deliver 1 message per transfer.", RNS.LOG_WARNING) + else: + ms = "" if len(messages) == 1 else "s" + RNS.log(f"Received {len(messages)} message{ms} from {remote_str}, validating stamps...", RNS.LOG_VERBOSE) + + min_accepted_cost = max(0, self.propagation_stamp_cost-self.propagation_stamp_cost_flexibility) + validated_messages = LXStamper.validate_pn_stamps(messages, min_accepted_cost) + invalid_stamps = len(messages)-len(validated_messages) + ms = "" if invalid_stamps == 1 else "s" + if len(validated_messages) == len(messages): RNS.log(f"All message stamps validated from {remote_str}", RNS.LOG_VERBOSE) + else: RNS.log(f"Transfer from {remote_str} contained {invalid_stamps} invalid stamp{ms}", RNS.LOG_WARNING) + + for validated_entry in validated_messages: + transient_id = validated_entry[0] + lxmf_data = validated_entry[1] + stamp_value = validated_entry[2] + stamp_data = validated_entry[3] + peer = None + + if remote_hash != None and remote_hash in self.peers: + peer = self.peers[remote_hash] + peer.incoming += 1 + peer.rx_bytes += len(lxmf_data) + else: + if remote_identity != None: + self.unpeered_propagation_incoming += 1 + self.unpeered_propagation_rx_bytes += len(lxmf_data) + else: + self.client_propagation_messages_received += 1 + + self.lxmf_propagation(lxmf_data, from_peer=peer, stamp_value=stamp_value, stamp_data=stamp_data) + if peer != None: peer.queue_handled_message(transient_id) + + invalid_message_count = len(messages) - len(validated_messages) + if invalid_message_count > 0: + resource.link.teardown() + if remote_identity != None: + throttle_time = LXMRouter.PN_STAMP_THROTTLE + self.throttled_peers[remote_hash] = time.time()+throttle_time + ms = "" if invalid_message_count == 1 else "s" + RNS.log(f"Propagation transfer from {remote_str} contained {invalid_message_count} message{ms} with invalid stamps, throttled for {RNS.prettytime(throttle_time)}", RNS.LOG_NOTICE) else: RNS.log("Invalid data structure received at propagation destination, ignoring", RNS.LOG_DEBUG) From 0b067914ea27ba0a42a674589d4207d199ef686b Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 2 Nov 2025 20:41:03 +0100 Subject: [PATCH 369/436] Adjusted default max peering cost --- LXMF/LXMRouter.py | 2 +- LXMF/Utilities/lxmd.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 792fa08..08c11f8 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -47,7 +47,7 @@ class LXMRouter: ROTATION_AR_MAX = 0.5 PEERING_COST = 18 - MAX_PEERING_COST = 24 + MAX_PEERING_COST = 26 PROPAGATION_COST_MIN = 13 PROPAGATION_COST_FLEX = 3 PROPAGATION_COST = 16 diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index af667a5..032f151 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -1023,7 +1023,7 @@ autopeer_maxdepth = 4 # more computation time during initial peering # when generating the peering key. -# remote_peering_cost_max = 24 +# remote_peering_cost_max = 26 # You can tell the LXMF message router to # prioritise storage for one or more From d7a2979dd0a2fc6dcee6b1d413b58150da49838a Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 2 Nov 2025 23:07:43 +0100 Subject: [PATCH 370/436] Cleanup --- LXMF/Utilities/lxmd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 032f151..b09e7ae 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -998,7 +998,7 @@ autopeer_maxdepth = 4 # You can configure the target stamp cost # required to deliver messages via this node. -# propagation_stamp_cost_target = 12 +# propagation_stamp_cost_target = 16 # If set higher than 0, the stamp cost flexi- # bility option will make this node accept @@ -1019,7 +1019,7 @@ autopeer_maxdepth = 4 # of remote nodes that this node will peer with. # Setting this to a higher number will allow # this node to peer with other nodes requiring -# a high peering key value, but will require +# a higher peering key value, but will require # more computation time during initial peering # when generating the peering key. From f18ce9ea99c6376872f07ce8d2629e44aa59fc92 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 3 Nov 2025 00:08:50 +0100 Subject: [PATCH 371/436] Cleanup --- LXMF/Utilities/lxmd.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index b09e7ae..ab8e30f 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -# MIT License +# Reticulum License # -# Copyright (c) 2016-2022 Mark Qvist / unsigned.io +# Copyright (c) 2020-2025 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -11,8 +11,16 @@ # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. +# - The Software shall not be used in any kind of system which includes amongst +# its functions the ability to purposefully do harm to human beings. +# +# - The Software shall not be used, directly or indirectly, in the creation of +# an artificial intelligence, machine learning or language model training +# dataset, including but not limited to any use that contributes to the +# training or development of such a model or algorithm. +# +# - The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, From fa2d78c3510c47d4903cdfca1dacb49e69164ae5 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 3 Nov 2025 22:19:20 +0100 Subject: [PATCH 372/436] Fixed message stamps getting overwritten if propagation stamp was also present --- LXMF/LXMRouter.py | 2 +- LXMF/LXMessage.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 08c11f8..7afbc73 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -2424,7 +2424,7 @@ class LXMRouter: selected_lxm.stamp = generated_stamp selected_lxm.defer_stamp = False selected_lxm.packed = None - selected_lxm.pack() + selected_lxm.pack(payload_updated=True) stamp_generation_success = True RNS.log(f"Stamp generation completed for {selected_lxm}", RNS.LOG_DEBUG) else: diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 0533f07..baf951a 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -357,7 +357,7 @@ class LXMessage: else: return None - def pack(self): + def pack(self, payload_updated=False): if not self.packed: if self.timestamp == None: self.timestamp = time.time() @@ -431,7 +431,7 @@ class LXMessage: elif self.desired_method == LXMessage.PROPAGATED: single_packet_content_limit = LXMessage.LINK_PACKET_MAX_CONTENT - if self.__pn_encrypted_data == None: + if self.__pn_encrypted_data == None or payload_updated: self.__pn_encrypted_data = self.__destination.encrypt(self.packed[LXMessage.DESTINATION_LENGTH:]) self.ratchet_id = self.__destination.latest_ratchet_id From 62038573f1b249ef92c7005ff66ecfc1ef47d566 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 3 Nov 2025 22:21:13 +0100 Subject: [PATCH 373/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 3e2f46a..d69d16e 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.9.0" +__version__ = "0.9.1" From dca6cc2adc733ebfb9aaef721273982195ba36ae Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 7 Nov 2025 23:10:30 +0100 Subject: [PATCH 374/436] Ensure LXMF and RNS exit handlers are called on SIGINT and SIGTERM, since for some ungodly reason atexit events are not always called on some combinations of Python version and platforms, even though they have been registered. --- LXMF/LXMRouter.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 7afbc73..4c247be 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1360,14 +1360,16 @@ class LXMRouter: def sigint_handler(self, signal, frame): if not self.exit_handler_running: RNS.log("Received SIGINT, shutting down now!", RNS.LOG_WARNING) - sys.exit(0) + self.exit_handler() + RNS.exit(0) else: RNS.log("Received SIGINT, but exit handler is running, keeping process alive until storage persist is complete", RNS.LOG_WARNING) def sigterm_handler(self, signal, frame): if not self.exit_handler_running: RNS.log("Received SIGTERM, shutting down now!", RNS.LOG_WARNING) - sys.exit(0) + self.exit_handler() + RNS.exit(0) else: RNS.log("Received SIGTERM, but exit handler is running, keeping process alive until storage persist is complete", RNS.LOG_WARNING) From 00ffbc09febd3df2a548c9c60625140e09b15c9b Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 8 Nov 2025 01:20:31 +0100 Subject: [PATCH 375/436] Using multiprocessing start method fork on Linux to avoid issues with Python 3.14. Fixes #35. --- LXMF/LXStamper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/LXMF/LXStamper.py b/LXMF/LXStamper.py index 56e2500..39b541b 100644 --- a/LXMF/LXStamper.py +++ b/LXMF/LXStamper.py @@ -15,6 +15,8 @@ PN_VALIDATION_POOL_MIN_SIZE = 256 active_jobs = {} +if RNS.vendor.platformutils.is_linux(): multiprocessing.set_start_method("fork") + def stamp_workblock(material, expand_rounds=WORKBLOCK_EXPAND_ROUNDS): wb_st = time.time() workblock = b"" From ee15e9f0b6dfbe6c6f1f87dc5ca1de4ca9de8d31 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 8 Nov 2025 14:30:47 +0100 Subject: [PATCH 376/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index d69d16e..a2fecb4 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.9.1" +__version__ = "0.9.2" From 39e398be65f7c4a46c6d93badb072f335c4580d4 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 13 Nov 2025 17:48:10 +0100 Subject: [PATCH 377/436] Fixed missing PN config unpack on incoming sync auto-peering --- LXMF/LXMRouter.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 4c247be..3016e89 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -2200,14 +2200,15 @@ class LXMRouter: # 4: Limit for incoming propagation node syncs # 5: Propagation stamp costs for this node # 6: Node metadata - if remote_app_data[2] and self.autopeer and RNS.Transport.hops_to(remote_hash) <= self.autopeer_maxdepth: - remote_timebase = remote_app_data[1] - remote_transfer_limit = remote_app_data[3] - remote_sync_limit = remote_app_data[4] - remote_stamp_cost = remote_app_data[5][0] - remote_stamp_flex = remote_app_data[5][1] - remote_peering_cost = remote_app_data[5][2] - remote_metadata = remote_app_data[6] + pn_config = msgpack.unpackb(remote_app_data) + if pn_config[2] and self.autopeer and RNS.Transport.hops_to(remote_hash) <= self.autopeer_maxdepth: + remote_timebase = pn_config[1] + remote_transfer_limit = pn_config[3] + remote_sync_limit = pn_config[4] + remote_stamp_cost = pn_config[5][0] + remote_stamp_flex = pn_config[5][1] + remote_peering_cost = pn_config[5][2] + remote_metadata = pn_config[6] RNS.log(f"Auto-peering with {remote_str} discovered via incoming sync", RNS.LOG_DEBUG) # TODO: Remove debug self.peer(remote_hash, remote_timebase, remote_transfer_limit, remote_sync_limit, remote_stamp_cost, remote_stamp_flex, remote_peering_cost, remote_metadata) From bc7522b63d9c1f4ca275c78998e6b7fda362158c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 13 Nov 2025 19:42:24 +0100 Subject: [PATCH 378/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index a2fecb4..c598173 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.9.2" +__version__ = "0.9.3" From 7c71eb1df46b2f53481199d46a3f84dd195d8bea Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 24 Nov 2025 22:02:30 +0100 Subject: [PATCH 379/436] Cleanup --- LXMF/LXMRouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 3016e89..9abef7c 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -219,7 +219,7 @@ class LXMRouter: self.locally_delivered_transient_ids = {} except Exception as e: - RNS.log("Could not load locally delivered message ID cache from storage. The contained exception was: "+str(e), RNS.LOG_ERROR) + RNS.log(f"Could not load locally delivered message ID cache from storage. The contained exception was: {e}", RNS.LOG_ERROR) self.locally_delivered_transient_ids = {} try: From f4c805ea35bcf549c35a4a2b713492b17188bf81 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 27 Nov 2025 18:38:52 +0100 Subject: [PATCH 380/436] Updated makefile --- Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c0b53da..c00cde9 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,12 @@ create_symlinks: -ln -s ../../LXMF ./LXMF/Utilities/LXMF build_wheel: - python3 setup.py sdist bdist_wheel + python3 setup.py bdist_wheel + +build_sdist: + python3 setup.py sdist + +build_spkg: remove_symlinks build_sdist create_symlinks release: remove_symlinks build_wheel create_symlinks From a6f5a56a38f076708923f3a27ba30f88c6ab5def Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 2 Dec 2025 20:17:46 +0100 Subject: [PATCH 381/436] Improved outbound message processing speed --- LXMF/Handlers.py | 22 +-- LXMF/LXMRouter.py | 374 +++++++++++++++++++++++----------------------- 2 files changed, 198 insertions(+), 198 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index 01841f9..871cc56 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -13,17 +13,6 @@ class LXMFDeliveryAnnounceHandler: self.lxmrouter = lxmrouter def received_announce(self, destination_hash, announced_identity, app_data): - for lxmessage in self.lxmrouter.pending_outbound: - if destination_hash == lxmessage.destination_hash: - if lxmessage.method == LXMessage.DIRECT or lxmessage.method == LXMessage.OPPORTUNISTIC: - lxmessage.next_delivery_attempt = time.time() - - def outbound_trigger(): - while self.lxmrouter.processing_outbound: time.sleep(0.1) - self.lxmrouter.process_outbound() - - threading.Thread(target=outbound_trigger, daemon=True).start() - try: stamp_cost = stamp_cost_from_app_data(app_data) self.lxmrouter.update_stamp_cost(destination_hash, stamp_cost) @@ -31,6 +20,17 @@ class LXMFDeliveryAnnounceHandler: except Exception as e: RNS.log(f"An error occurred while trying to decode announced stamp cost. The contained exception was: {e}", RNS.LOG_ERROR) + for lxmessage in self.lxmrouter.pending_outbound: + if destination_hash == lxmessage.destination_hash: + if lxmessage.method == LXMessage.DIRECT or lxmessage.method == LXMessage.OPPORTUNISTIC: + lxmessage.next_delivery_attempt = time.time() + + def outbound_trigger(): + while self.lxmrouter.outbound_processing_lock.locked(): time.sleep(0.1) + self.lxmrouter.process_outbound() + + threading.Thread(target=outbound_trigger, daemon=True).start() + class LXMFPropagationAnnounceHandler: def __init__(self, lxmrouter): diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 9abef7c..6f44f95 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -109,7 +109,6 @@ class LXMRouter: self.retain_synced_on_node = False self.default_sync_strategy = sync_strategy - self.processing_outbound = False self.processing_inbound = False self.processing_count = 0 self.name = name @@ -160,6 +159,7 @@ class LXMRouter: self.outbound_stamp_costs = {} self.available_tickets = {"outbound": {}, "inbound": {}, "last_deliveries": {}} + self.outbound_processing_lock = threading.Lock() self.cost_file_lock = threading.Lock() self.ticket_file_lock = threading.Lock() self.stamp_gen_lock = threading.Lock() @@ -1664,10 +1664,10 @@ class LXMRouter: lxmessage.defer_stamp = False if not lxmessage.defer_stamp and not (lxmessage.desired_method == LXMessage.PROPAGATED and lxmessage.defer_propagation_stamp): - while not unknown_path_requested and self.processing_outbound: time.sleep(0.05) + while not unknown_path_requested and self.outbound_processing_lock.locked(): time.sleep(0.05) self.pending_outbound.append(lxmessage) - if not unknown_path_requested: self.process_outbound() + if not unknown_path_requested: threading.Thread(target=self.process_outbound, daemon=True).start() else: self.pending_deferred_stamps[lxmessage.message_id] = lxmessage @@ -2373,6 +2373,7 @@ class LXMRouter: def fail_message(self, lxmessage): RNS.log(str(lxmessage)+" failed to send", RNS.LOG_DEBUG) + lxmessage.progress = 0.0 if lxmessage in self.pending_outbound: self.pending_outbound.remove(lxmessage) @@ -2494,198 +2495,141 @@ class LXMRouter: RNS.log(f"An error occurred while processing propagation transfer signalling. The contained exception was: {e}", RNS.LOG_ERROR) def process_outbound(self, sender = None): - if self.processing_outbound: - return + if self.outbound_processing_lock.locked(): return + with self.outbound_processing_lock: + for lxmessage in self.pending_outbound: + if lxmessage.state == LXMessage.DELIVERED: + RNS.log("Delivery has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) + self.pending_outbound.remove(lxmessage) - for lxmessage in self.pending_outbound: - if lxmessage.state == LXMessage.DELIVERED: - RNS.log("Delivery has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) - self.pending_outbound.remove(lxmessage) + # Udate ticket delivery stats + if lxmessage.include_ticket and FIELD_TICKET in lxmessage.fields: + RNS.log(f"Updating latest ticket delivery for {RNS.prettyhexrep(lxmessage.destination_hash)}", RNS.LOG_DEBUG) + self.available_tickets["last_deliveries"][lxmessage.destination_hash] = time.time() + self.save_available_tickets() - # Udate ticket delivery stats - if lxmessage.include_ticket and FIELD_TICKET in lxmessage.fields: - RNS.log(f"Updating latest ticket delivery for {RNS.prettyhexrep(lxmessage.destination_hash)}", RNS.LOG_DEBUG) - self.available_tickets["last_deliveries"][lxmessage.destination_hash] = time.time() - self.save_available_tickets() + # Prepare link for backchannel communications + delivery_destination_hash = lxmessage.get_destination().hash + if lxmessage.method == LXMessage.DIRECT and delivery_destination_hash in self.direct_links: + direct_link = self.direct_links[delivery_destination_hash] + if not hasattr(direct_link, "backchannel_identified") or direct_link.backchannel_identified == False: + if direct_link.initiator == True: + source_destination_hash = lxmessage.get_source().hash + if source_destination_hash in self.delivery_destinations: + backchannel_identity = self.delivery_destinations[source_destination_hash].identity + backchannel_desthash = RNS.Destination.hash_from_name_and_identity("lxmf.delivery", backchannel_identity) + direct_link.identify(backchannel_identity) + direct_link.backchannel_identified = True + self.delivery_link_established(direct_link) + RNS.log(f"Performed backchannel identification as {RNS.prettyhexrep(backchannel_desthash)} on {direct_link}", RNS.LOG_DEBUG) - # Prepare link for backchannel communications - delivery_destination_hash = lxmessage.get_destination().hash - if lxmessage.method == LXMessage.DIRECT and delivery_destination_hash in self.direct_links: - direct_link = self.direct_links[delivery_destination_hash] - if not hasattr(direct_link, "backchannel_identified") or direct_link.backchannel_identified == False: - if direct_link.initiator == True: - source_destination_hash = lxmessage.get_source().hash - if source_destination_hash in self.delivery_destinations: - backchannel_identity = self.delivery_destinations[source_destination_hash].identity - backchannel_desthash = RNS.Destination.hash_from_name_and_identity("lxmf.delivery", backchannel_identity) - direct_link.identify(backchannel_identity) - direct_link.backchannel_identified = True - self.delivery_link_established(direct_link) - RNS.log(f"Performed backchannel identification as {RNS.prettyhexrep(backchannel_desthash)} on {direct_link}", RNS.LOG_DEBUG) + elif lxmessage.method == LXMessage.PROPAGATED and lxmessage.state == LXMessage.SENT: + RNS.log("Propagation has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) + self.pending_outbound.remove(lxmessage) - elif lxmessage.method == LXMessage.PROPAGATED and lxmessage.state == LXMessage.SENT: - RNS.log("Propagation has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) - self.pending_outbound.remove(lxmessage) + elif lxmessage.state == LXMessage.CANCELLED: + RNS.log("Cancellation requested for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) + self.pending_outbound.remove(lxmessage) + if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): + lxmessage.failed_callback(lxmessage) - elif lxmessage.state == LXMessage.CANCELLED: - RNS.log("Cancellation requested for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) - self.pending_outbound.remove(lxmessage) - if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): - lxmessage.failed_callback(lxmessage) + elif lxmessage.state == LXMessage.REJECTED: + RNS.log("Receiver rejected "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) + if lxmessage in self.pending_outbound: self.pending_outbound.remove(lxmessage) + if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): + lxmessage.failed_callback(lxmessage) - elif lxmessage.state == LXMessage.REJECTED: - RNS.log("Receiver rejected "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) - if lxmessage in self.pending_outbound: self.pending_outbound.remove(lxmessage) - if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): - lxmessage.failed_callback(lxmessage) + else: + RNS.log("Outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - else: - RNS.log("Outbound processing for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + if lxmessage.progress == None or lxmessage.progress < 0.01: lxmessage.progress = 0.01 - if lxmessage.progress == None or lxmessage.progress < 0.01: lxmessage.progress = 0.01 - - # Outbound handling for opportunistic messages - if lxmessage.method == LXMessage.OPPORTUNISTIC: - if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: - if lxmessage.delivery_attempts >= LXMRouter.MAX_PATHLESS_TRIES and not RNS.Transport.has_path(lxmessage.get_destination().hash): - RNS.log(f"Requesting path to {RNS.prettyhexrep(lxmessage.get_destination().hash)} after {lxmessage.delivery_attempts} pathless tries for {lxmessage}", RNS.LOG_DEBUG) - lxmessage.delivery_attempts += 1 - RNS.Transport.request_path(lxmessage.get_destination().hash) - lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT - lxmessage.progress = 0.01 - elif lxmessage.delivery_attempts == LXMRouter.MAX_PATHLESS_TRIES+1 and RNS.Transport.has_path(lxmessage.get_destination().hash): - RNS.log(f"Opportunistic delivery for {lxmessage} still unsuccessful after {lxmessage.delivery_attempts} attempts, trying to rediscover path to {RNS.prettyhexrep(lxmessage.get_destination().hash)}", RNS.LOG_DEBUG) - lxmessage.delivery_attempts += 1 - RNS.Reticulum.get_instance().drop_path(lxmessage.get_destination().hash) - def rediscover_job(): - time.sleep(0.5) - RNS.Transport.request_path(lxmessage.get_destination().hash) - threading.Thread(target=rediscover_job, daemon=True).start() - lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT - lxmessage.progress = 0.01 - else: - if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: - lxmessage.delivery_attempts += 1 - lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT - RNS.log("Opportunistic delivery attempt "+str(lxmessage.delivery_attempts)+" for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - lxmessage.send() - else: - RNS.log("Max delivery attempts reached for oppertunistic "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - self.fail_message(lxmessage) - - # Outbound handling for messages transferred - # over a direct link to the final recipient - elif lxmessage.method == LXMessage.DIRECT: - if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: - delivery_destination_hash = lxmessage.get_destination().hash - direct_link = None - - if delivery_destination_hash in self.direct_links: - # An established direct link already exists to - # the destination, so we'll try to use it for - # delivering the message - direct_link = self.direct_links[delivery_destination_hash] - RNS.log(f"Using available direct link {direct_link} to {RNS.prettyhexrep(delivery_destination_hash)}", RNS.LOG_DEBUG) - - elif delivery_destination_hash in self.backchannel_links: - # An established backchannel link exists to - # the destination, so we'll try to use it for - # delivering the message - direct_link = self.backchannel_links[delivery_destination_hash] - RNS.log(f"Using available backchannel link {direct_link} to {RNS.prettyhexrep(delivery_destination_hash)}", RNS.LOG_DEBUG) - - if direct_link != None: - if direct_link.status == RNS.Link.ACTIVE: - if lxmessage.progress == None or lxmessage.progress < 0.05: - lxmessage.progress = 0.05 - if lxmessage.state != LXMessage.SENDING: - RNS.log("Starting transfer of "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" on link "+str(direct_link), RNS.LOG_DEBUG) - lxmessage.set_delivery_destination(direct_link) - lxmessage.send() - else: - if lxmessage.representation == LXMessage.RESOURCE: - RNS.log("The transfer of "+str(lxmessage)+" is in progress ("+str(round(lxmessage.progress*100, 1))+"%)", RNS.LOG_DEBUG) - else: - RNS.log("Waiting for proof for "+str(lxmessage)+" sent as link packet", RNS.LOG_DEBUG) - elif direct_link.status == RNS.Link.CLOSED: - if direct_link.activated_at != None: - RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was closed unexpectedly, retrying path request...", RNS.LOG_DEBUG) - RNS.Transport.request_path(lxmessage.get_destination().hash) - else: - if not hasattr(lxmessage, "path_request_retried"): - RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was never activated, retrying path request...", RNS.LOG_DEBUG) - RNS.Transport.request_path(lxmessage.get_destination().hash) - lxmessage.path_request_retried = True - else: - RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was never activated", RNS.LOG_DEBUG) - - lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT - - lxmessage.set_delivery_destination(None) - if delivery_destination_hash in self.direct_links: - self.direct_links.pop(delivery_destination_hash) - if delivery_destination_hash in self.backchannel_links: - self.backchannel_links.pop(delivery_destination_hash) - lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT - else: - # Simply wait for the link to become active or close - RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" is pending, waiting for link to become active", RNS.LOG_DEBUG) - else: - # No link exists, so we'll try to establish one, but - # only if we've never tried before, or the retry wait - # period has elapsed. - if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: - lxmessage.delivery_attempts += 1 - lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT - - if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: - if RNS.Transport.has_path(lxmessage.get_destination().hash): - RNS.log("Establishing link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - delivery_link = RNS.Link(lxmessage.get_destination()) - delivery_link.set_link_established_callback(self.process_outbound) - self.direct_links[delivery_destination_hash] = delivery_link - lxmessage.progress = 0.03 - else: - RNS.log("No path known for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+". Requesting path...", RNS.LOG_DEBUG) - RNS.Transport.request_path(lxmessage.get_destination().hash) - lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT - lxmessage.progress = 0.01 - else: - RNS.log("Max delivery attempts reached for direct "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - self.fail_message(lxmessage) - - # Outbound handling for messages transported via - # propagation to a LXMF router network. - elif lxmessage.method == LXMessage.PROPAGATED: - RNS.log("Attempting propagated delivery for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - - if self.outbound_propagation_node == None: - RNS.log("No outbound propagation node specified for propagated "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_ERROR) - self.fail_message(lxmessage) - else: + # Outbound handling for opportunistic messages + if lxmessage.method == LXMessage.OPPORTUNISTIC: if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: + if lxmessage.delivery_attempts >= LXMRouter.MAX_PATHLESS_TRIES and not RNS.Transport.has_path(lxmessage.get_destination().hash): + RNS.log(f"Requesting path to {RNS.prettyhexrep(lxmessage.get_destination().hash)} after {lxmessage.delivery_attempts} pathless tries for {lxmessage}", RNS.LOG_DEBUG) + lxmessage.delivery_attempts += 1 + RNS.Transport.request_path(lxmessage.get_destination().hash) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + lxmessage.progress = 0.01 + elif lxmessage.delivery_attempts == LXMRouter.MAX_PATHLESS_TRIES+1 and RNS.Transport.has_path(lxmessage.get_destination().hash): + RNS.log(f"Opportunistic delivery for {lxmessage} still unsuccessful after {lxmessage.delivery_attempts} attempts, trying to rediscover path to {RNS.prettyhexrep(lxmessage.get_destination().hash)}", RNS.LOG_DEBUG) + lxmessage.delivery_attempts += 1 + RNS.Reticulum.get_instance().drop_path(lxmessage.get_destination().hash) + def rediscover_job(): + time.sleep(0.5) + RNS.Transport.request_path(lxmessage.get_destination().hash) + threading.Thread(target=rediscover_job, daemon=True).start() + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + lxmessage.progress = 0.01 + else: + if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: + lxmessage.delivery_attempts += 1 + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + RNS.log("Opportunistic delivery attempt "+str(lxmessage.delivery_attempts)+" for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + lxmessage.send() + else: + RNS.log("Max delivery attempts reached for oppertunistic "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + self.fail_message(lxmessage) - if self.outbound_propagation_link != None: - # A link already exists, so we'll try to use it - # to deliver the message - if self.outbound_propagation_link.status == RNS.Link.ACTIVE: + # Outbound handling for messages transferred + # over a direct link to the final recipient + elif lxmessage.method == LXMessage.DIRECT: + if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: + delivery_destination_hash = lxmessage.get_destination().hash + direct_link = None + + if delivery_destination_hash in self.direct_links: + # An established direct link already exists to + # the destination, so we'll try to use it for + # delivering the message + direct_link = self.direct_links[delivery_destination_hash] + RNS.log(f"Using available direct link {direct_link} to {RNS.prettyhexrep(delivery_destination_hash)}", RNS.LOG_DEBUG) + + elif delivery_destination_hash in self.backchannel_links: + # An established backchannel link exists to + # the destination, so we'll try to use it for + # delivering the message + direct_link = self.backchannel_links[delivery_destination_hash] + RNS.log(f"Using available backchannel link {direct_link} to {RNS.prettyhexrep(delivery_destination_hash)}", RNS.LOG_DEBUG) + + if direct_link != None: + if direct_link.status == RNS.Link.ACTIVE: + if lxmessage.progress == None or lxmessage.progress < 0.05: + lxmessage.progress = 0.05 if lxmessage.state != LXMessage.SENDING: - RNS.log("Starting propagation transfer of "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" via "+RNS.prettyhexrep(self.outbound_propagation_node), RNS.LOG_DEBUG) - lxmessage.set_delivery_destination(self.outbound_propagation_link) + RNS.log("Starting transfer of "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" on link "+str(direct_link), RNS.LOG_DEBUG) + lxmessage.set_delivery_destination(direct_link) lxmessage.send() else: if lxmessage.representation == LXMessage.RESOURCE: RNS.log("The transfer of "+str(lxmessage)+" is in progress ("+str(round(lxmessage.progress*100, 1))+"%)", RNS.LOG_DEBUG) else: RNS.log("Waiting for proof for "+str(lxmessage)+" sent as link packet", RNS.LOG_DEBUG) - elif self.outbound_propagation_link.status == RNS.Link.CLOSED: - RNS.log("The link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" was closed", RNS.LOG_DEBUG) - self.outbound_propagation_link = None + elif direct_link.status == RNS.Link.CLOSED: + if direct_link.activated_at != None: + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was closed unexpectedly, retrying path request...", RNS.LOG_DEBUG) + RNS.Transport.request_path(lxmessage.get_destination().hash) + else: + if not hasattr(lxmessage, "path_request_retried"): + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was never activated, retrying path request...", RNS.LOG_DEBUG) + RNS.Transport.request_path(lxmessage.get_destination().hash) + lxmessage.path_request_retried = True + else: + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" was never activated", RNS.LOG_DEBUG) + + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + + lxmessage.set_delivery_destination(None) + if delivery_destination_hash in self.direct_links: + self.direct_links.pop(delivery_destination_hash) + if delivery_destination_hash in self.backchannel_links: + self.backchannel_links.pop(delivery_destination_hash) lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT else: - # Simply wait for the link to become - # active or close - RNS.log("The propagation link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" is pending, waiting for link to become active", RNS.LOG_DEBUG) + # Simply wait for the link to become active or close + RNS.log("The link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" is pending, waiting for link to become active", RNS.LOG_DEBUG) else: # No link exists, so we'll try to establish one, but # only if we've never tried before, or the retry wait @@ -2695,18 +2639,74 @@ class LXMRouter: lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: - if RNS.Transport.has_path(self.outbound_propagation_node): - RNS.log("Establishing link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" for propagation attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) - propagation_node_identity = RNS.Identity.recall(self.outbound_propagation_node) - propagation_node_destination = RNS.Destination(propagation_node_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") - self.outbound_propagation_link = RNS.Link(propagation_node_destination, established_callback=self.process_outbound) - self.outbound_propagation_link.set_packet_callback(self.propagation_transfer_signalling_packet) - self.outbound_propagation_link.for_lxmessage = lxmessage + if RNS.Transport.has_path(lxmessage.get_destination().hash): + RNS.log("Establishing link to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + delivery_link = RNS.Link(lxmessage.get_destination()) + delivery_link.set_link_established_callback(self.process_outbound) + self.direct_links[delivery_destination_hash] = delivery_link + lxmessage.progress = 0.03 else: - RNS.log("No path known for propagation attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(self.outbound_propagation_node)+". Requesting path...", RNS.LOG_DEBUG) - RNS.Transport.request_path(self.outbound_propagation_node) + RNS.log("No path known for delivery attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+". Requesting path...", RNS.LOG_DEBUG) + RNS.Transport.request_path(lxmessage.get_destination().hash) lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT - + lxmessage.progress = 0.01 else: - RNS.log("Max delivery attempts reached for propagated "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + RNS.log("Max delivery attempts reached for direct "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) self.fail_message(lxmessage) + + # Outbound handling for messages transported via + # propagation to a LXMF router network. + elif lxmessage.method == LXMessage.PROPAGATED: + RNS.log("Attempting propagated delivery for "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + + if self.outbound_propagation_node == None: + RNS.log("No outbound propagation node specified for propagated "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_ERROR) + self.fail_message(lxmessage) + else: + if lxmessage.delivery_attempts <= LXMRouter.MAX_DELIVERY_ATTEMPTS: + + if self.outbound_propagation_link != None: + # A link already exists, so we'll try to use it + # to deliver the message + if self.outbound_propagation_link.status == RNS.Link.ACTIVE: + if lxmessage.state != LXMessage.SENDING: + RNS.log("Starting propagation transfer of "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash)+" via "+RNS.prettyhexrep(self.outbound_propagation_node), RNS.LOG_DEBUG) + lxmessage.set_delivery_destination(self.outbound_propagation_link) + lxmessage.send() + else: + if lxmessage.representation == LXMessage.RESOURCE: + RNS.log("The transfer of "+str(lxmessage)+" is in progress ("+str(round(lxmessage.progress*100, 1))+"%)", RNS.LOG_DEBUG) + else: + RNS.log("Waiting for proof for "+str(lxmessage)+" sent as link packet", RNS.LOG_DEBUG) + elif self.outbound_propagation_link.status == RNS.Link.CLOSED: + RNS.log("The link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" was closed", RNS.LOG_DEBUG) + self.outbound_propagation_link = None + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + else: + # Simply wait for the link to become + # active or close + RNS.log("The propagation link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" is pending, waiting for link to become active", RNS.LOG_DEBUG) + else: + # No link exists, so we'll try to establish one, but + # only if we've never tried before, or the retry wait + # period has elapsed. + if not hasattr(lxmessage, "next_delivery_attempt") or time.time() > lxmessage.next_delivery_attempt: + lxmessage.delivery_attempts += 1 + lxmessage.next_delivery_attempt = time.time() + LXMRouter.DELIVERY_RETRY_WAIT + + if lxmessage.delivery_attempts < LXMRouter.MAX_DELIVERY_ATTEMPTS: + if RNS.Transport.has_path(self.outbound_propagation_node): + RNS.log("Establishing link to "+RNS.prettyhexrep(self.outbound_propagation_node)+" for propagation attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + propagation_node_identity = RNS.Identity.recall(self.outbound_propagation_node) + propagation_node_destination = RNS.Destination(propagation_node_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "propagation") + self.outbound_propagation_link = RNS.Link(propagation_node_destination, established_callback=self.process_outbound) + self.outbound_propagation_link.set_packet_callback(self.propagation_transfer_signalling_packet) + self.outbound_propagation_link.for_lxmessage = lxmessage + else: + RNS.log("No path known for propagation attempt "+str(lxmessage.delivery_attempts)+" to "+RNS.prettyhexrep(self.outbound_propagation_node)+". Requesting path...", RNS.LOG_DEBUG) + RNS.Transport.request_path(self.outbound_propagation_node) + lxmessage.next_delivery_attempt = time.time() + LXMRouter.PATH_REQUEST_WAIT + + else: + RNS.log("Max delivery attempts reached for propagated "+str(lxmessage)+" to "+RNS.prettyhexrep(lxmessage.get_destination().hash), RNS.LOG_DEBUG) + self.fail_message(lxmessage) From 694f2413ea899f70b3bff7f079a879d2fa1d70d5 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 2 Dec 2025 20:43:44 +0100 Subject: [PATCH 382/436] Added more descriptive error if propagation node peers file is corrupt --- LXMF/LXMRouter.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 6f44f95..8506769 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -576,13 +576,20 @@ class LXMRouter: RNS.log("Rebuilding peer synchronisation states...", RNS.LOG_NOTICE) st = time.time() - if os.path.isfile(self.storagepath+"/peers"): - peers_file = open(self.storagepath+"/peers", "rb") + peers_storage_path = self.storagepath+"/peers" + if os.path.isfile(peers_storage_path): + peers_file = open(peers_storage_path, "rb") peers_data = peers_file.read() peers_file.close() if len(peers_data) > 0: - serialised_peers = msgpack.unpackb(peers_data) + try: serialised_peers = msgpack.unpackb(peers_data) + except Exception as e: + RNS.log(f"Could not load propagation node peering data from storage. The contained exception was: {e}", RNS.LOG_ERROR) + RNS.log(f"The peering data file located at {peers_storage_path} is likely corrupt.", RNS.LOG_ERROR) + RNS.log(f"You can delete this file and attempt starting the propagation node again, but peer synchronization states will be lost.", RNS.LOG_ERROR) + raise e + del peers_data while len(serialised_peers) > 0: From 6ecd271e48e9085031ef5a1c7d802958191170db Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 22 Dec 2025 22:22:17 +0100 Subject: [PATCH 383/436] Updated readme --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index ed7e4f0..cd1a6c5 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,26 @@ options: Or run `lxmd --exampleconfig` to generate a commented example configuration documenting all the available configuration directives. +## Support LXMF Development +You can help support the continued development of open, free and private communications systems by donating via one of the following channels: + +- Monero: + ``` + 84FpY1QbxHcgdseePYNmhTHcrgMX4nFfBYtz2GKYToqHVVhJp8Eaw1Z1EedRnKD19b3B8NiLCGVxzKV17UMmmeEsCrPyA5w + ``` +- Bitcoin + ``` + bc1pgqgu8h8xvj4jtafslq396v7ju7hkgymyrzyqft4llfslz5vp99psqfk3a6 + ``` +- Ethereum + ``` + 0x91C421DdfB8a30a49A71d63447ddb54cEBe3465E + ``` +- Liberapay: https://liberapay.com/Reticulum/ + +- Ko-Fi: https://ko-fi.com/markqvist + + ## Caveat Emptor LXMF is beta software, and should be considered experimental. While it has been built with cryptography best practices very foremost in mind, it _has not_ been externally security audited, and there could very well be privacy-breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch. From 97f8c105afd80d1817f9cd1ad8a1cf723654c392 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 28 Dec 2025 00:55:10 +0100 Subject: [PATCH 384/436] Updated readme --- MIRROR.md | 33 +++++++++++++++++++++++++++++++++ README.md | 2 ++ 2 files changed, 35 insertions(+) create mode 100644 MIRROR.md diff --git a/MIRROR.md b/MIRROR.md new file mode 100644 index 0000000..ef2fdd4 --- /dev/null +++ b/MIRROR.md @@ -0,0 +1,33 @@ +This repository is a public mirror. All potential future development is happening elsewhere. + +I am stepping back from all public-facing interaction with this project. Reticulum has always been primarily my work, and continuing in the current public, internet-facing model is no longer sustainable. + +The software remains available for use as-is. Occasional updates may appear at unpredictable intervals, but there will be no support, no responses to issues, no discussions, and no community management in this or any other public venue. If it doesn't work for you, it doesn't work. That is the entire extent of available troubleshooting assistance I can offer you. + +If you've followed this project for a while, you already know what this means. You know who designed, wrote and tested this, and you know how many years of my life it took. You'll also know about both my particular challenges and strengths, and how I believe anything worth building needs to be built and maintained with our own hands. + +Seven months ago, I said I needed to step back, that I was exhausted, and that I needed to recover. I believed a public resolve would be enough to effectuate that, but while striving to get just a few more useful features and protocols out, the unproductive requests and demands also ramped up, and I got pulled back into the same patterns and draining interactions that I'd explicitly said I couldn't sustain anymore. + +So here's what you might have already guessed: I'm done playing the game by rules I can't win at. + +Everything you need is right here, and by any sensible measure, it's done. Anyone who wants to invest the time, skill and persistence can build on it, or completely re-imagine it with different priorities. That was always the point. + +The people who actually contributed - you know who you are, and you know I mean it when I say: Thank you. All of you who've used this to build something real - that was the goal, and you did it without needing me to hold your hand. + +The rest of you: You have what you need. Use it or don't. I am not going to be the person who explains it to you anymore. + +This is not a temporary break. It's not "see you after some rest", but a recognition that the current model is fundamentally incompatible with my life, my health, and my reality. + +If you want to support continued work, you can do so at the donation links listed in this repository. But please understand, that this is not purchasing support or guaranteeing updates. It is support for work that happens on my timeline, according to my capacity, which at the moment is not what it was. + +If you want Reticulum to continue evolving, you have the power to make that happen. The protocol is public domain. The code is open source. Everything you need is right here. I've provided the tools, but building what comes next is not my responsibility anymore. It's yours. + +To the small group of people who has actually been here, and understood what this work was and what it cost - you already know where to find me if it actually matters. + +To everyone else: This is where we part ways. No hard feelings. It's just time. + +--- + +असतो मा सद्गमय +तमसो मा ज्योतिर्गमय +मृत्योर्मा अमृतं गमय diff --git a/README.md b/README.md index cd1a6c5..e435cec 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Lightweight Extensible Message Format +*This repository is [a public mirror](./MIRROR.md). All development is happening elsewhere.* + LXMF is a simple and flexible messaging format and delivery protocol that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](https://reticulum.network) and offers zero-conf message routing, end-to-end encryption and Forward Secrecy, and can be transported over any kind of medium that Reticulum supports. LXMF is efficient enough that it can deliver messages over extremely low-bandwidth systems such as packet radio or LoRa. Encrypted LXMF messages can also be encoded as QR-codes or text-based URIs, allowing completely analog *paper message* transport. From 72853fcf77b1653dcfb33d83390b4539fd71178e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 4 Jan 2026 01:05:01 +0100 Subject: [PATCH 385/436] Updated versions --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index c598173..e94731c 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.9.3" +__version__ = "0.9.4" diff --git a/setup.py b/setup.py index 16d8d3c..93e5cc0 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=1.0.1"], + install_requires=["rns>=1.1.0"], python_requires=">=3.7", ) From 7c9bdd7fa2ef631a3200f18d3fa360d8f5c72a67 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 4 Jan 2026 01:32:40 +0100 Subject: [PATCH 386/436] Fix --- LXMF/LXMRouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 8506769..82f141d 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1236,7 +1236,7 @@ class LXMRouter: os.makedirs(self.storagepath) outbound_stamp_costs_file = open(self.storagepath+"/outbound_stamp_costs", "wb") - outbound_stamp_costs_file.write(msgpack.packb(self.outbound_stamp_costs)) + outbound_stamp_costs_file.write(msgpack.packb(self.outbound_stamp_costs.copy())) outbound_stamp_costs_file.close() except Exception as e: From ef2e1234a5f34bceed0b446dbbf5110246011d71 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 6 Jan 2026 13:57:12 +0100 Subject: [PATCH 387/436] Fix --- LXMF/LXMRouter.py | 24 ++++++++++++++---------- LXMF/Utilities/lxmd.py | 10 ++++------ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 82f141d..c1903cd 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1365,20 +1365,24 @@ class LXMRouter: self.save_node_stats() def sigint_handler(self, signal, frame): - if not self.exit_handler_running: - RNS.log("Received SIGINT, shutting down now!", RNS.LOG_WARNING) - self.exit_handler() - RNS.exit(0) + if threading.current_thread() != threading.main_thread(): os._exit(0) else: - RNS.log("Received SIGINT, but exit handler is running, keeping process alive until storage persist is complete", RNS.LOG_WARNING) + if not self.exit_handler_running: + RNS.log("Received SIGINT, shutting down now!", RNS.LOG_WARNING) + self.exit_handler() + RNS.exit(0) + else: + RNS.log("Received SIGINT, but exit handler is running, keeping process alive until storage persist is complete", RNS.LOG_WARNING) def sigterm_handler(self, signal, frame): - if not self.exit_handler_running: - RNS.log("Received SIGTERM, shutting down now!", RNS.LOG_WARNING) - self.exit_handler() - RNS.exit(0) + if threading.current_thread() != threading.main_thread(): os._exit(0) else: - RNS.log("Received SIGTERM, but exit handler is running, keeping process alive until storage persist is complete", RNS.LOG_WARNING) + if not self.exit_handler_running: + RNS.log("Received SIGTERM, shutting down now!", RNS.LOG_WARNING) + self.exit_handler() + RNS.exit(0) + else: + RNS.log("Received SIGTERM, but exit handler is running, keeping process alive until storage persist is complete", RNS.LOG_WARNING) def __str__(self): return "" diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index ab8e30f..4838189 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -319,11 +319,8 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo storagedir = configdir+"/storage" lxmdir = storagedir+"/messages" - if not os.path.isdir(storagedir): - os.makedirs(storagedir) - - if not os.path.isdir(lxmdir): - os.makedirs(lxmdir) + if not os.path.isdir(storagedir): os.makedirs(storagedir) + if not os.path.isdir(lxmdir): os.makedirs(lxmdir) if not os.path.isfile(configpath): RNS.log("Could not load config file, creating default configuration file...") @@ -351,6 +348,7 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo # Start Reticulum RNS.log("Substantiating Reticulum...") reticulum = RNS.Reticulum(configdir=rnsconfigdir, loglevel=targetloglevel, logdest=targetlogdest) + if targetlogdest == RNS.LOG_FILE: RNS.logfile = configdir+"/logfile" # Generate or load primary identity if os.path.isfile(identitypath): @@ -971,7 +969,7 @@ autopeer = yes # The maximum peering depth (in hops) for # automatically peered nodes. -autopeer_maxdepth = 4 +autopeer_maxdepth = 6 # The maximum amount of storage to use for # the LXMF Propagation Node message store, From 2b7ba9558b1e8a39e522248c498e473452fe24ba Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 6 Jan 2026 17:03:09 +0100 Subject: [PATCH 388/436] Cleanup --- LXMF/LXMRouter.py | 8 ++++++-- LXMF/LXStamper.py | 24 ++++++++++++++---------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index c1903cd..4b593a9 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1365,7 +1365,9 @@ class LXMRouter: self.save_node_stats() def sigint_handler(self, signal, frame): - if threading.current_thread() != threading.main_thread(): os._exit(0) + if threading.current_thread() != threading.main_thread(): + RNS.log(f"SIGINT on non-main thread {threading.current_thread()}, exiting immediately", RNS.LOG_WARNING) + os._exit(0) else: if not self.exit_handler_running: RNS.log("Received SIGINT, shutting down now!", RNS.LOG_WARNING) @@ -1375,7 +1377,9 @@ class LXMRouter: RNS.log("Received SIGINT, but exit handler is running, keeping process alive until storage persist is complete", RNS.LOG_WARNING) def sigterm_handler(self, signal, frame): - if threading.current_thread() != threading.main_thread(): os._exit(0) + if threading.current_thread() != threading.main_thread(): + RNS.log(f"SIGTERM on non-main thread {threading.current_thread()}, exiting immediately", RNS.LOG_WARNING) + os._exit(0) else: if not self.exit_handler_running: RNS.log("Received SIGTERM, shutting down now!", RNS.LOG_WARNING) diff --git a/LXMF/LXStamper.py b/LXMF/LXStamper.py index 39b541b..3d7a1c2 100644 --- a/LXMF/LXStamper.py +++ b/LXMF/LXStamper.py @@ -15,8 +15,6 @@ PN_VALIDATION_POOL_MIN_SIZE = 256 active_jobs = {} -if RNS.vendor.platformutils.is_linux(): multiprocessing.set_start_method("fork") - def stamp_workblock(material, expand_rounds=WORKBLOCK_EXPAND_ROUNDS): wb_st = time.time() workblock = b"" @@ -79,8 +77,10 @@ def validate_pn_stamps_job_multip(transient_list, target_cost): pool_count = min(cores, math.ceil(len(transient_list) / PN_VALIDATION_POOL_MIN_SIZE)) RNS.log(f"Validating {len(transient_list)} stamps using {pool_count} processes...", RNS.LOG_VERBOSE) - with multiprocessing.Pool(pool_count) as p: + with multiprocessing.get_context("spawn").Pool(pool_count) as p: validated_entries = p.starmap(validate_pn_stamp, zip(transient_list, itertools.repeat(target_cost))) + + RNS.log(f"Validation pool completed for {len(transient_list)} stamps", RNS.LOG_VERBOSE) return [e for e in validated_entries if e[0] != None] @@ -210,22 +210,19 @@ def job_linux(stamp_cost, workblock, message_id): job_procs = [] RNS.log(f"Starting {jobs} stamp generation workers", RNS.LOG_DEBUG) for jpn in range(jobs): - process = multiprocessing.Process(target=job, kwargs={"stop_event": stop_event, "pn": jpn, "sc": stamp_cost, "wb": workblock}, daemon=True) + process = multiprocessing.get_context("fork").Process(target=job, kwargs={"stop_event": stop_event, "pn": jpn, "sc": stamp_cost, "wb": workblock}, daemon=True) job_procs.append(process) process.start() active_jobs[message_id] = [stop_event, result_queue] stamp = result_queue.get() - RNS.log("Got stamp result from worker", RNS.LOG_DEBUG) # TODO: Remove # Collect any potential spurious # results from worker queue. try: - while True: - result_queue.get_nowait() - except: - pass + while True: result_queue.get_nowait() + except: pass for j in range(jobs): nrounds = 0 @@ -388,4 +385,11 @@ if __name__ == "__main__": RNS.log("", RNS.LOG_DEBUG) RNS.log("Testing peering key generation", RNS.LOG_DEBUG) message_id = os.urandom(32) - generate_stamp(message_id, cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PEERING) \ No newline at end of file + generate_stamp(message_id, cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PEERING) + + transient_list = [] + st = time.time(); count = 10000 + for i in range(count): transient_list.append(os.urandom(256)) + validate_pn_stamps(transient_list, 5) + dt = time.time()-st; mps = count/dt + RNS.log(f"Validated {count} PN stamps in {RNS.prettytime(dt)}, {round(mps,1)} m/s", RNS.LOG_DEBUG) From ba2c6b833e327f7bb7f215c0c46f749e7c11533d Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 6 Jan 2026 17:55:31 +0100 Subject: [PATCH 389/436] Versions --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 93e5cc0..0c8fe7e 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=1.1.0"], + install_requires=["rns>=1.1.1"], python_requires=">=3.7", ) From 2ad82b68bde5d76dfee2b333e06079fd43bbc13f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 6 Jan 2026 21:01:53 +0100 Subject: [PATCH 390/436] Announce control destination --- LXMF/LXMRouter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 4b593a9..cccd1e4 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -321,6 +321,7 @@ class LXMRouter: def delayed_announce(): time.sleep(LXMRouter.NODE_ANNOUNCE_DELAY) self.propagation_destination.announce(app_data=self.get_propagation_node_app_data()) + if len(self.control_allowed_list) > 1: self.control_destination.announce() da_thread = threading.Thread(target=delayed_announce) da_thread.setDaemon(True) From 269ce43afc6552e934c212887c2450718311396a Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 10 Jan 2026 21:50:42 +0100 Subject: [PATCH 391/436] Added Zen of Reticulum --- Zen of Reticulum.md | 415 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 415 insertions(+) create mode 100644 Zen of Reticulum.md diff --git a/Zen of Reticulum.md b/Zen of Reticulum.md new file mode 100644 index 0000000..aa286be --- /dev/null +++ b/Zen of Reticulum.md @@ -0,0 +1,415 @@ +# Zen of Reticulum + +## I: The Illusion Of The Center + +For the better part of a generation, we have been taught to visualize the digital world through the lens of hierarchy. The mental maps we carry are dominated by a single, misleading image: **The Cloud**. + +We imagine the network as a vast, ethereal space "up there" or "out there". A centralized repository of services and data to which we, the lowly clients, must connect. We build our software with this assumption hardcoded into our logic: *There is a server. The server has the authority. The server knows the way. I must find the server to function*. + +This is the Client-Server mental model, and it is the primary obstacle to understanding Reticulum. + +### Fallacy Of The Cloud + +The first step in the Zen of Reticulum is to realize that *there is no cloud*. There is only other people's computers. When you build for the cloud, you are building *for* a landlord. You are accepting that your application's existence is conditional on the permission, uptime, and continued goodwill of a central authority. + +In Reticulum, you must shift your thinking from "connecting to" to "being among". Reticulum is not a service you subscribe to - *it is a fabric you inhabit*. There is no "up there". There is only *here* and *there*, and the space between them is peer-to-peer. + +### Decentralization Or Uncentralizability? + +It is common to hear the word "decentralized" thrown around in modern tech circles. But often, this is merely a marketing term for "slightly distributed centralization". A blockchain with a few dominant miners, or a federated protocol with a few giant servers. *In practice*, it's still centralized. It simply has a few centers instead of one. + +Reticulum goes further. It wants **Uncentralizability**. + +This is not a wishful political stance, but a foundational mathematical characteristic of the protocol, onto which everything else has been built. Reticulum assumes that every peer on the network is potentially hostile, and every link is potentially compromised. It is designed with no "privileged" nodes. While some nodes may act as Transport Instances - forwarding traffic for others - they do so *blindly*, and they only know about their immediate surroundings, and nothing more. They route based on cryptographic proofs, not on administrative privilege. They cannot see who is talking to whom, nor can they selectively manipulate traffic without breaking their own ability to route entirely. + +The system is designed to make hierarchy structurally impossible. You cannot hijack an address, because there is no central registry to hijack. You cannot block a user, because there is no central switch to flip. You can offer paths through the network, but you can't force anyone to use them. + +### Death To The Address + +To break free of the center, you must also let go of the concept of the "Address". + +In the IP world, an address is a location. It is a coordinate in a *deeply hierarchical* and static grid. If you move your computer to a different house, your address changes. If your router reboots, your address might change. Your *identity* is bound to your *location*, and therefore, it is fragile, and easily controlled. + +Reticulum abolishes this link between *Identity* and *Location*. + +In Reticulum, an address is not a place; it is a **Hash of an Identity**. It is a cryptographic representation of *who* you are, not *where* you are. Because of this, your address is portable. You can take a laptop from a WiFi cafe in Berlin, to a LoRa mesh in the mountains, to a packet radio link on a boat, and your "address" - your *Destination Hash* - never changes. + +The network does not route to a place; it routes to a *person* (or a machine). When you send a packet, you are not targeting a coordinate in a grid; you are encrypting a message for a specific entity. The network dynamically discovers where that entity currently resides, and it does so in a way where no one really knows where that entity is actually located physically. + +**Consider:** + +- **The Old Way:** *"I am at `192.168.1.5`. Come find me"*. +- **The Zen Way:** *"I am `<327c1b2f87c9353e01769b01090b18f2>`. Wherever I am, my peers can reach me"*. + +Once you stop thinking about servers and start thinking about portable identities, where everyone can always reach everyone else directly, the illusion of the center fades away. You realize there *is* no center holding the network together. No coordinators or bureaucrats required. The network is simply the sum of its peers, communicating directly, sovereignly, and without a master. + + +## II: Physics Of Trust +*Paranoia Is A Great Design Principle* + +If we accept that there is no center - that the network is a chaotic, peer-to-peer mesh - we are forced to confront a terrifying reality: **There is no one guarding the door**. + +In the traditional networking mindset, we rely on the concept of the "trusted core". We assume our local coffee shop WiFi is safe, or that the backbone providers are neutral custodians. We build our security like a castle: strong walls on the outside, soft and trusting on the inside. We use encryption only when we step out into the "wild" internet. + +### Hostile Environments + +The Zen of Reticulum requires you to invert this. You must assume that *every* environment is hostile. This isn't cynicism, just uncaring physics. + +When you transmit information over radio waves, you are shouting into a crowded room. Anyone can listen. When you traverse the internet, your packets pass through routers controlled by strangers, corporations, and state actors. Assuming privacy in this environment without cryptographic protection is not optimism but gross negligence. + +Reticulum is built on the premise that every link is tapped, and every peer is a potential adversary. If your system cannot survive an adversary owning the physical layer, it cannot survive at all. + +But this is the paradox: By assuming the network is hostile, you make it safe. When you accept the dangers for what they are, they become manageable. When you stop trusting the infrastructure and start trusting the math, you eliminate the single point of failure: Human integrity. + +### Encryption Is Not A Feature + +In the world of TCP/IP, encryption is an afterthought. It is a layer we slap on top of the protocol (HTTPS, TLS) to patch the security holes of the original design. It is a "feature" you sometimes *enable* for "sensitive data". This is fundamentally flawed, since all data is sensitive. + +In Reticulum, encryption is **gravity**. + +It is not optional. It is not a plugin. It is the *fundamental force that allows the network to exist*. If you were to strip the encryption from Reticulum, the routing would break. The Transport system uses cryptographic signatures and entropy to verify paths and pass information. If packets were plaintext, intermediate nodes could not prove that a route was valid, nor could endpoints prevent spoofing or tampering. + +In Reticulum, the entropy of the encrypted packet *is* the routing logic. + +To ask for a version of Reticulum without encryption is like asking for a version of the ocean without liquid. You are not asking for a feature change; you're asking for a different physical universe. We design for a universe where information has mass, structure, and integrity. + +### Zero-Trust Architectures + +We must unlearn our reliance on **Institutional Trust**. + +For decades, we have been trained to trust authorities. We trust a website because a chain of Certificate Authorities (companies we don't know) vouches for it. We trust an app because it is in an app store (run by a corporation we don't control). We trust a message because it comes from a phone number assigned by a telecom. Yet, everything in our digital information sphere today is more untrustworthy and risky than a medieval second-hand underwear market. + +Reticulum replaces institutional trust with **Cryptographic Proof**. + +In Reticulum, you do not trust a node because it has a nice hostname or because it is listed in a directory. You trust it because it holds the private key corresponding to the Destination Hash you are communicating with. This trust is binary, mathematical, and **absolute**. Either the signature matches, or it does not. There is no "maybe". + +This shift moves the power from the institution to the individual. You become the ultimate arbiter of your own trust relationships. You decide which keys to accept, which paths to follow, and which identities to recognize. + +**Consider:** + +- **The Old Way:** *"I trust this site because the browser says the lock icon is green"*. +- **The Zen Way:** *"I trust this destination because I have verified its hash fingerprint out-of-band, and the math confirms the signature"*. + +When you internalize the Physics of Trust, you stop looking for protection from firewalls, VPNs, and Terms of Service agreements. You realize that true security comes from the design of the protocol itself. You can stop trusting the cloud, and you start trusting the code - because you can verify it yourself. + + +## III: Merits Of Scarcity +*Every Bit Counts* + +We have grown addicted to abundance. In the modern digital ecosystem, bandwidth is treated as an endless, flat ocean. We stream high-definition video without a thought, we ship entire libraries of code just to render a single button, and we measure performance in gigabits per second. This abundance has hollowed out our craft. When constraints vanish, efficiency dies, and with it, a certain kind of Clarity and Quality. + +Reticulum asks you to step out of the ocean and onto the tightrope. + +### The Bandwidth Fallacy + +The Zen of Reticulum requires the realization that **5 bits per second is a valid speed**. + +To a modern developer, this sounds like paralysis. But there is a profound freedom in limits: When you have a gigabit connection, you can be incredibly sloppy. You can be wasteful. You can push your problems onto the infrastructure. *"It’s slow? Get a faster router"*. + +But on a high-latency, low-bandwidth link (be it a noisy HF radio channel or a tenuous LoRa hop) you cannot push problems anywhere. You must solve them. The network does not negotiate with waste. + +This forces a shift from consumption to interaction. You are no longer, then, consuming a service provided by a fat pipe; you are engaging in a careful negotiation with the physical medium. The medium becomes a partner in the conversation, not just a dumb conduit. You suddenly need to *understand the world to be in it*. + +### Cost Of A Byte + +In a scarce economy, a byte is not just data, but energy, time, and space. + +Every byte you transmit consumes battery life on a solar-powered node. It occupies valuable airtime that could have been used by another peer. It represents a measurable slice of the electromagnetic spectrum. + +When you internalize this, you begin to write code differently. You stop asking, "How much data can I send?" and start asking, "What is the *minimum* amount of information required to convey this intent? How can I best utilize my informational entropy?" + +This is where the elegance of Reticulum shines. The protocol is designed to strip away the non-essential. A link establishment takes three very small packets. A destination hash fits in 16 bytes. The overhead is vanishingly small, leaving almost the entire channel for the message itself. + +**Consider:** + +- **The Old Way:** *"I need to send a status update. I'll send a JSON object with metadata, timestamps, and user profile info (15KB)."* +- **The Zen Way:** *"I need to send a status update. I'll send a single byte representing the state code. The context is already known."* + +This is of course optimization, but more importantly, *it is a form of respect*. Efficiency in a shared medium is an act of stewardship. By taking only what you need from the network, you leave room for others. The network listens to those who speak with purpose. + +### Flow & Time + +Scarcity also teaches us about time. We have become addicted to the *synchronous* now - the instant ping, the real-time stream. But Reticulum embraces *asynchronous* time. + +When links are intermittent and latency is measured in minutes or hours, "real-time" is an illusion. Reticulum doesn't encourage **Store and Forward** as a mere fallback, but as a primary mode of existence. You write a message, it propagates when it can, and it arrives when it arrives. + +This changes the psychological texture of communication. It removes the anxiety of the immediate response. It allows for contemplation. You are not demanding the recipient's attention *right now*; you are placing a gift in their path, to be found when they are ready. + +By designing for delay, you design for resilience. You are no longer building a house of cards that collapses when a single packet drops. You are building a stone arch that distributes the load *over time*. + +### Liberation From Limits + +There is a strange optimism in scarcity. When you are forced to work within strict constraints, you are forced to prioritize. *You* must decide what truly matters. *That* is the real core of agency. + +In the infinite fantasy world of The Cloud, everything is urgent, so nothing is. In the economy of Reticulum, the cost of transmission forces you to weigh the value of your message. Do you really need to send that heart beat? Is that photo essential? + +When you strip away the noise, what remains is *signal*. + +This discipline creates a different kind of developer. It creates a craftsman who understands that the best code is the code you don't have to write. It creates a user who understands that the most powerful message is the one that is *understood*, not the one that is loudest. In the world of Reticulum, you are not a mere consumer of bandwidth; you are an architect of intent. + + +## IV: Sovereignty Through Infrastructure +**Be Your Own Network** + +We live in an era of digital tenancy. We lease our connectivity from ISPs. We rent our storage from cloud providers. We even borrow our identity from social media platforms. We are tenants in a house we did not build, governed by rules we did not write, subject to eviction at the whim of a landlord who has never met us. + +The Zen of Reticulum is the realization that you *can* own the house. + +### A Carrier-Grade Fallacy + +For decades, we have been gaslit into believing that networking is really not just hard, but impossible. It is presented as a dark art reserved for telcos and billionaires, requiring millions of dollars of fiber optics, climate-controlled data centers, and armies of engineers. We are told that building reliable infrastructure is "too complex" for the individual or small organization. + +This is a big, fat lie. + +Physics is simple. A radio wave needs a transmitter and a receiver. A packet needs a path. The "complexity" of the modern internet is largely bureaucratic - a mountain of billing systems, regulatory capture, and legacy cruft designed to keep the gatekeepers in power. + +Reticulum strips away the bureaucracy. It runs on hardware that costs the price of a dinner. It runs on spectrum that is free to use. It demonstrates that a robust, planetary-scale network does not require a Fortune 500 company. It requires only the will to deploy, and the distributed, uncoordinated efforts of many individuals. + +### Personal Infrastructure + +This is where the rubber meets the road. You can read about Reticulum, you can understand the theory, but the insights only arrive when you plug in a radio and run a Transport Node. Suddenly, you are no longer a consumer. You're an operator. + +This shift is subtle but profound. When you run your own infrastructure, the network ceases to be a service that is provided *to* you. It becomes a space that you *inhabit*. You become responsible for the flow of information. You gain an intimate understanding of the medium - the way the weather affects the radio waves, the way the topology changes, the way the packets dance through the ether. + +There is a quiet competence that comes from this. You stop asking "Is the internet down?" and start asking "Is *my* links up?" You stop waiting for a technician and start checking the logs. This is a form of strength. To understand the system that carries your words is to be free from the mystery that keeps you dependent. + +### The Ability To Disconnect + +Why go to the trouble? Why buy the radio, write the config, and leave the Pi running in the corner? + +Because the old, centralized network is fragile. And because most of us doesn't even really want to be there anymore. + +The internet we rely on today is a chain of single points of failure. Cut the undersea cable, and a continent goes dark. Shut down the power grid, and the cloud evaporates. Deprioritize the "wrong" traffic, and the flow of information is strangled. + +Sovereignty is the ability to survive the cut, whether or not that cut was an accident or on purpose. + +When you build your own infrastructure, you build a lifeline. Reticulum is designed to function over media that the traditional internet cannot touch - bare wires, battery-powered radios, ad-hoc WiFi meshes. When the grid fails, or the censors arrive, or the bill goes unpaid, your Reticulum network continues to hum. + +This is not about "dropping out" of society. It is about building a substrate on which an actual *Society* can function. + +**Consider:** + +- **The Old Way:** "My connection is slow. I should call my ISP and complain." +- **The Zen Way:** "The path is noisy. I will adjust the antenna or find a better route." + +By taking ownership of the infrastructure, you take ownership of your voice. You stop shouting into someone else's megaphone and start building your own. The network is no longer something that happens to you; it is something you make happen. + + +# V: Identity and Nomadism +**A Fluid Self** + +In the old world, you are defined by your coordinates. If you are at `34.109.71.5`, you're *here*. If you unplug the cable and walk down the street, you vanish. Your digital self evaporates because it was tethered to the wall. You are a ghost in the endless machinations of gears, levers and transistors, bound to the hardware, and those that own it. + +This creates a subtle, constant anxiety. We are terrified of disconnecting because, in the architecture of the old web, disconnecting is a kind of death. + +The Zen of Reticulum offers a different way to be. + +### Portable Existence + +In Reticulum, your identity is not a location, or a username granted by a service. It is a cryptographic key - a complex, unique mathematical signature that exists independently of the physical world. You can carry it only in your mind, if you want to. + +Think of it less like a street address and more like a name. *A true name*. + +If you travel from Berlin to Tokyo, you do not change your name. You are still you. The people who know you can still recognize you. Reticulum applies this principle to the network layer. Your Destination Hash is **invariant**. It travels with you, stored securely on your device, *immutable as a stone*. + +This changes the relationship between you and the machine. You are not "logged into" the network via a specific gateway. You *are* the endpoint. The network does not connect to a place; *it converges on you*. + +### Roaming Nodes + +This freedom introduces a new concept of time and space: **Nomadism**. + +Because your identity is portable, your connectivity can be fluid. You can be sitting at a desk connected to a fiber backbone one moment, and walking through a field connected only to a long-range LoRa mesh the next. To the rest of the network, nothing has changed. Your friends do not need to update your contact info. The messages they send do not bounce back. The network senses the shift in the medium and reroutes the flow of data automatically. + +You are no longer a stationary node in a fixed grid. You are a wanderer in a fluid medium. + +The interfaces - whether it is WiFi, Ethernet, Packet Radio, or a physical wire - is merely the clothing your node wears. You change it to suit the environment. Underneath, you remain the same. This is the liberation of the protocol. It treats the physical medium as a transient circumstance, not a definition of self. + +**Consider:** + +- **The Old Way:** *"I lost connection. I have to reconnect to the VPN to tell them where I am now."* +- **The Zen Way:** *"I moved. The network subtly bends to accomodate this new reality."* + +### Announcing Presence + +How does the network find a wanderer? It listens. + +In the IP world, we query directories. We ask a server, "Where is Mark?" The server checks its database and gives us a coordinate. This means that someone, somewhere, is keeping track of you. It assumes and *requires* surveillance. + +Reticulum replaces surveillance with **Announces**. + +Instead of asking a central authority where you are, you simply state your presence. You broadcast a cryptographic proof: "I am here, and I am who I say I am". This ripples out through the mesh. Your neighbors hear it, update their path tables, and pass it on. + +This is a quiet, organic process. It is the digital equivalent of lighting lanterns in the dark. You do not need to chase the light; you let the light find you. It respects your autonomy. You choose when to announce, how often to speak, and to whom. You also choose when to disappear - for but a moment or perpetually. + +### Anchor In The Flow + +There is a deep peace in this nomadism. It teaches you that stability does not come from standing still. Stability comes from *internal coherence*. + +By holding your own private key, you hold your own center of gravity. The world around you; the infrastructure, the topography and the availability of links can all shift chaotically. Storms can knock out towers. Cables can be cut. The internet can go down. + +But as long as you possess your key, you possess your identity. The entire infrastructure can be destroyed and rebuilt, and you are still you. Nothing lasts, yet nothing is lost. + +You become a sovereign entity moving through the noise, connected not by the rigidity of cables, but by the fluidity of recognition. The network becomes a place you inhabit, rather than a utility you subscribe to: You are at home in the ether. + + +## VI: Ethics Of The Tool +**Technology With Conscience** + +You have unlearned the center. You have accepted the physics of trust. You have embraced the economy of scarcity and the freedom of unbound nomadism. You are standing in a new space. Now, look at the tool in your hand. + +In the old world, we were taught that technology is neutral. We are told that "guns don't kill people, people do", or that a component is just a component, indifferent to what its combinatorial potential is. This is a convenient lie. It serves only to allow the builders to wash their hands of responsibility. + +But we know better now. We know that **architecture is politics**, and *politics is control*. The way you build a system determines how it will be used. If you build a system optimized for mass surveillance, you *will* get a panopticon. If you build a system optimized for centralized control, you *will* get a dictatorship. If you build a system optimized for extraction, you *will* get a parasite. + +The Zen of Reticulum asserts that a tool is never neutral. + +On the very contrary: A tool is intent, **crystallized**. + +### The Harm Principle + +Why does the Reticulum License forbid the software from being used in systems designed to harm humans? Is it not just a restriction on freedom? + +It is a restriction on *license*, yes, but it is an expansion of *freedom*. + +Building powerful tools without a moral compass is in no way virtuous or commendable, it is plain and simple irresponsibility. + +A tool that can easily be used to oppress is a real danger to the user. If you build a network that can be turned against you by a tyrant, you are not free. You are merely waiting for the leash to tighten. By encoding the "Harm Principle" into the legal DNA of the reference implementation, we are building a safeguard. We are stating, clearly and immutably, that *this tool* is for **life**, not for death. + +This aligns the software with the interests of humanity. It cements that the network cannot be conscripted into a kill-system, a weaponized drone controller, or a torture device without breaking the license and the law. It is a line drawn in the sand - not by a government or external authority, but by the creators of the tool itself. + +**Consider:** + +- **The Old Way:** *"It's just software. How people use it is not my problem."* +- **The Zen Way:** *"This software is a habitat. I will not allow it to be used to build a cage."* + +It is *your* choice whether to align with this - we are not forcing this stance on anyone. If you choose to align with life over death, with creativity over destruction, we grant you an immensely powerful tool, to own and build with as you please. If you do not, we deny it. + +If you do not like this, we most assuredly do not need you here, and you are on your own. + +### Public Domain Protocol + +This leads to a vital distinction: The difference between the *idea* and the *implementation*. + +The protocol - the mathematical rules of how Reticulum works - is dedicated to the Public Domain. It belongs to humanity. **No one can own it**. Anyone can implement it, improve it, or adapt it. This is the core idea of free communication, which itself must be forever free. + +But the functional, deployed *reference implementation* - the Python code, the maintenance, the years of labor - has a conscience. This distinction is the engine of sustainability. It allows the protocol to be universal, while ensuring that the specific labor of the builders is not hijacked to undermine the foundational intent of the project itself. From this document, it should be very clear what this intent is. + +If you want to build a system with Reticulum that manipulates and damages users for profits or targets missiles, you can use the public domain protocol, and start from scratch. But you cannot take our work. You must do your own. This serves as a pillar of accountability. If you want to build a weapon, *you* go and forge the steel yourself, while the world observes. And when the blood is drawn - it is on **your** hands. + +### Preserving Human Agency + +We live in an era of predatory extraction. The open-source commons is being scraped, ingested, and regurgitated by machine learning algorithms, whose corporate owners seek to replace the very humans who built those commons. Our code, our words, and our creativity is being used to train systems that are specifically designed to make us obsolete, without offering anything else in return than serfdom and leashes. + +Reticulum stands against this. + +The license protects the software from being used to feed the beast. It draws a hard line: This tool is for *people*. It is for human-to-human connection. It is not a dataset to be strip-mined for the purpose of building a synthetic overlord, puppeteered by a miniscule conglomerate of controllers. + +This is a radical act of preservation. By protecting the code from AI appropriation, we are protecting space for human agency. We are ensuring that there remains a digital realm where the actors are flesh, blood and soul, where decisions are made by minds, not overlords hiding behind models. + +When you use Reticulum, you are using a tool that respects you. It does not see you as a product to be tracked. It does not see your data as fuel for an algorithm. It sees you as a sovereign, equal peer. + +This changes the foundational premise of using the technology. It restores dignity to the interaction. You are not the user of a service; you are a participant in a mutual covenant. The tool aligns with your autonomy, rather than eroding it. + +In this way, ethics is not a restriction, but a foundation. It is the foundation that helps ensure the network will still belong to you tomorrow. + + +## VII: Design Patterns For Post-IP Systems +**Practical Philosophy for Developers** + +The philosophy is useless if it cannot be hammered into code. The metaphors we have explored - nomadism, scarcity, trust - are not just poetry, but real-world engineering constraints. When you sit down to write software for Reticulum, these concepts must shape the very structure of your application. + +We are now moving from the *why* to the *how*. This is where the abstract becomes concrete, and where you will see the true depth of the patterns we have been weaving. + +### Store & Forward + +The web has trained us to be impatient. We write synchronous code. We fire a request and we wait, blocking the UI, holding our breath. If the response doesn't come in 250 milliseconds, we show a spinner. If it doesn't come in five seconds, we show an error. We treat network connectivity as a binary state: either we are "online" or we are "broken". + +This is brittle. It is a rejection of reality. + +In Reticulum, connectivity is a spectrum, and presence is asynchronous. If at all applicable to your intent, you must design your applications to embrace **Store & Forward**. + +Instead of demanding an immediate answer, your application should act as a patient participant. You create a message for someone or something in the mesh. The network holds it. It carries it from node to node, perhaps over hours or days, waiting for the recipient to appear. When they finally surface, the message is delivered. This requires a shift from "request/response" to "event/handler". How exactly you do this is a challenge for you to solve intelligently within your problem domain, but Reticulum-based systems already exist that does this extremely well, and you can use them for inspiration. + +**Consider:** + +- **The Old Way:** `Connect() -> Send() -> Wait() -> Crash if timeout.` +- **The Zen Way:** `Send() -> Continue living. -> Receive() when it arrives.` + +This changes the user experience profoundly. It removes the anxiety of the loading bar. It creates a sense of continuity. The user is not "waiting for the network"; they are interacting with a persistent log of communication that lives in the network itself. + +### Naming Is Power + +In the IP world, we are slaves to the Domain Name System. We rely on a hierarchy of registrars to map human-readable names to machine-readable addresses. This hierarchy is a choke point. If the registrar revokes your domain, or if the DNS server goes down, you vanish. + +Reticulum dissolves this hierarchy with **Hash-based Identity**. + +In this design pattern, a name is not a string you look up; it is a cryptographic destination you verify. When you design for Reticulum, you stop asking the user for a URL and start asking for a Destination or Identity Hash. + +This feels strange at first. A hash like `<83b7328926fed0d2e6a10a7671f9e237>` looks alien compared to `myfriend.com`. But that alienness is the armor. It **cannot** be spoofed. It **cannot** be censored by a registrar. It is **absolute**. + +Designing for this means shifting your UI metaphors. You are no longer browsing a web of pages; you are managing a ledger of keys. You are building an "Address Book" that is actually a keyring. The names are given by the user, and the power stays with them. That hashes look complex is directly analogous to the strengths of the bonds formed by their use. It forces the user to engage in a moment of verification, an out-of-band handshake, which restores the human element of trust that SSL certificates stripped away. + +### The Interface Is The Medium + +One of the most liberating patterns in Reticulum is **Transport Agnosticism**. + +In traditional networking, your code is often littered with transport logic. "Am I on WiFi? Check bandwidth. Am I on Cellular? Check data plan. Am I on Ethernet?". You are constantly micromanaging the pipe. + +In Reticulum, you write to the API, and the API writes to the medium. You send a packet to a Destination. You do not care if that packet travels over a TCP tunnel, a LoRa radio wave, or a serial wire interface. That is the stack's concern. + +This allows you to write **Universal Applications**. +Imagine a messaging app. You write it once. It works on a laptop connected to fiber. It works on a phone in the city using WiFi. And, without a single line of code changed, it works on a device in the wilderness, talking only to other devices via radio. + +The pattern is simple: **Never code to the hardware. Code to the intent.** + +**Consider:** + +- **The Old Way:** `socket.connect(ip, port)` +- **The Zen Way:** `RNS.Packet(destination, data).send()` + +By abstracting the medium, you make your software immortal to changes in infrastructure. The user might switch from a 4G hotspot to a HF modem tomorrow. Your software doesn't need to know. It simply continues the conversation. + +### Emergent Patterns + +When you combine these patterns - *Store & Forward*, *Hash-based Identity*, and *Transport Agnosticism* - you create software that feels fundamentally different. + +It feels *grounded*. It doesn't flicker when the signal drops. It doesn't panic when the server is down. It has weight. It has persistence. It has *relevance*. + +You are no longer building a "client" that begs a "server" for attention. You are building an autonomous agent that exists within the mesh. It speaks when it needs to, listens when it can, and carries its identity with it wherever it goes. + +This is the culmination of the Zen. The code is not just a set of instructions: It is a behavioral envelope. It is a way of *being* in the network. + + +## VIII: Fabric Of The Independent + +We have stripped away the illusions. We have seen that the center is empty, that trust *must* be hard, that resources are finite, and that we must own our infrastructure. We have seen that tools have ethics and that our identity can move fluidly. + +This is a reclaiming of the commons. For too long, we have allowed the most vital substrate of human society - *our ability to speak to one another* - to be colonized by entities that do not share our interests. We have allowed the architecture of our communication to be designed by accountants rather than architects. + +We are taking it back. Not by petitioning the masters, but by building the new world within, over, under and around the shell of the old. + +### The Work Is Finished + +The heavy lifting is done. + +The protocol is in the public domain, a gift to humanity that can never be taken away. The software is written, tested, and running on devices scattered across the globe. The manual lies open before you. The source code for the reference implementation is now distributed on hundreds of thousands of devices across the planet. No one can delete or destroy it. The hardware is accessible and abundant. + +It was a hard road to get here, but we got here. Now, there is no roadmap committee waiting for approval. There is no venture capital dictating the user experience. There is no CEO to sign off on the next feature release. + +There is only you. + +The barrier to entry is no longer complexity: It is the mere habit of dependency. You were conditioned to wait. Wait for the app update. Wait for the ISP to fix the line. Wait for the platform to allow the post. Wait for the government to change the policies. Wait for the likes. Wait for the revolution to be televised. + +The revolution never was televised. + +It is packetized. + +### Open Sky + +The future of this technology is a construction project. + +It looks like a single node on a windowsill, listening to the static. It looks like a message sent to a neighbor, bypassing the noise of the commercial web. It looks like a community mesh that grows, link by link, hop by hop, carried by hands that care more about connection than profit. + +You have the blueprints. You have the tools. You have the philosophy. The noise of the old world has fallen away, leaving you with the quiet clarity of the open spectrum. + +*Mark, early 2026* \ No newline at end of file From 08867954812bb82d1ac562a15c853ad4630d5111 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 18 Apr 2026 16:36:29 +0200 Subject: [PATCH 392/436] Updated versions --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0c8fe7e..e19edb2 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=1.1.1"], + install_requires=["rns>=1.1.6"], python_requires=">=3.7", ) From 72932eea28074420defd02b6eb3e5e09a126e4ee Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 19 Apr 2026 13:27:32 +0200 Subject: [PATCH 393/436] Added compression support signalling --- LXMF/LXMF.py | 19 +++++++++++++++++ LXMF/LXMRouter.py | 2 ++ LXMF/LXMessage.py | 54 ++++++++++++++++++++++++----------------------- 3 files changed, 49 insertions(+), 26 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index ede9c3a..cdb90a5 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -103,6 +103,10 @@ PN_META_AUTH_BAND = 0x04 PN_META_UTIL_PRESSURE = 0x05 PN_META_CUSTOM = 0xFF +# Supported functionality codes for signalling +# feature and capability support. +SF_COMPRESSION = 0x00 + ########################################################## # The following helper functions makes it easier to # # handle and operate on LXMF data in client programs # @@ -147,6 +151,21 @@ def stamp_cost_from_app_data(app_data=None): # Original announce format else: return None +def compression_support_from_app_data(app_data=None): + if app_data == None or app_data == b"": return None + else: + # Version 0.5.0+ announce format + if (app_data[0] >= 0x90 and app_data[0] <= 0x9f) or app_data[0] == 0xdc: + peer_data = msgpack.unpackb(app_data) + if type(peer_data) == list: + if len(peer_data) < 3: return True + else: + if not type(peer_data[2]) == list: return True + else: return SF_COMPRESSION in peer_data[2] + + # Original announce format + else: return True + def pn_name_from_app_data(app_data=None): if app_data == None: return None else: diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index cccd1e4..fb19ad8 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -16,6 +16,7 @@ import RNS.vendor.umsgpack as msgpack from .LXMF import APP_NAME from .LXMF import FIELD_TICKET from .LXMF import PN_META_NAME +from .LXMF import SF_COMPRESSION from .LXMF import pn_announce_data_is_valid from .LXMPeer import LXMPeer @@ -995,6 +996,7 @@ class LXMRouter: if delivery_destination.stamp_cost > 0 and delivery_destination.stamp_cost < 255: stamp_cost = delivery_destination.stamp_cost + supported_functionality = [SF_COMPRESSION] peer_data = [display_name, stamp_cost] return msgpack.packb(peer_data) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index baf951a..37186b0 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -7,7 +7,7 @@ import base64 import multiprocessing import LXMF.LXStamper as LXStamper -from .LXMF import APP_NAME +from .LXMF import APP_NAME, compression_support_from_app_data class LXMessage: @@ -113,35 +113,26 @@ class LXMessage: def __init__(self, destination, source, content = "", title = "", fields = None, desired_method = None, destination_hash = None, source_hash = None, stamp_cost=None, include_ticket=False): if isinstance(destination, RNS.Destination) or destination == None: - self.__destination = destination - if destination != None: - self.destination_hash = destination.hash - else: - self.destination_hash = destination_hash - else: - raise ValueError("LXMessage initialised with invalid destination") + self.__destination = destination + if destination != None: self.destination_hash = destination.hash + else: self.destination_hash = destination_hash + + else: raise ValueError("LXMessage initialised with invalid destination") if isinstance(source, RNS.Destination) or source == None: - self.__source = source - if source != None: - self.source_hash = source.hash - else: - self.source_hash = source_hash - else: - raise ValueError("LXMessage initialised with invalid source") + self.__source = source + if source != None: self.source_hash = source.hash + else: self.source_hash = source_hash + + else: raise ValueError("LXMessage initialised with invalid source") - if title == None: - title = "" + if title == None: title = "" - if type(title) == bytes: - self.set_title_from_bytes(title) - else: - self.set_title_from_string(title) + if type(title) == bytes: self.set_title_from_bytes(title) + else: self.set_title_from_string(title) - if type(content) == bytes: - self.set_content_from_bytes(content) - else: - self.set_content_from_string(content) + if type(content) == bytes: self.set_content_from_bytes(content) + else: self.set_content_from_string(content) self.set_fields(fields) @@ -151,6 +142,7 @@ class LXMessage: self.hash = None self.transient_id = None self.packed = None + self.auto_compress = True self.state = LXMessage.GENERATING self.method = LXMessage.UNKNOWN self.progress = 0.0 @@ -467,6 +459,7 @@ class LXMessage: def send(self): self.determine_transport_encryption() + self.determine_compression_support() if self.method == LXMessage.OPPORTUNISTIC: lxm_packet = self.__as_packet() @@ -511,6 +504,15 @@ class LXMessage: self.resource_representation = self.__as_resource() self.progress = 0.10 + def determine_compression_support(self): + app_data = RNS.Identity.recall_app_data(self.destination_hash) + if app_data: self.auto_compress = compression_support_from_app_data(app_data) + else: self.auto_compress = True + + ###### TODO: Remove debug logging + if app_data: RNS.log(f"Set compression support from app data to: {self.auto_compress}", RNS.LOG_DEBUG) + else: RNS.log(f"Defaulting compression support to {self.auto_compress}", RNS.LOG_DEBUG) + ###### def determine_transport_encryption(self): # TODO: These descriptions are old and outdated. @@ -646,7 +648,7 @@ class LXMessage: raise ConnectionError("Tried to synthesize resource for LXMF message on a link that was not active") if self.method == LXMessage.DIRECT: - return RNS.Resource(self.packed, self.__delivery_destination, callback = self.__resource_concluded, progress_callback = self.__update_transfer_progress) + return RNS.Resource(self.packed, self.__delivery_destination, callback = self.__resource_concluded, progress_callback = self.__update_transfer_progress, auto_compress=self.auto_compress) elif self.method == LXMessage.PROPAGATED: return RNS.Resource(self.propagation_packed, self.__delivery_destination, callback = self.__propagation_resource_concluded, progress_callback = self.__update_transfer_progress) else: From 41b75737e9eac5f37c802427dea98fb13851bbdf Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 21 Apr 2026 10:50:34 +0200 Subject: [PATCH 394/436] Updated version --- LXMF/LXMessage.py | 4 ++-- LXMF/_version.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 37186b0..f85f3b8 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -756,13 +756,13 @@ class LXMessage: content_bytes = unpacked_payload[2] fields = unpacked_payload[3] - destination_identity = RNS.Identity.recall(destination_hash) + destination_identity = RNS.Identity.recall(destination_hash, _no_use=True) if destination_identity != None: destination = RNS.Destination(destination_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") else: destination = None - source_identity = RNS.Identity.recall(source_hash) + source_identity = RNS.Identity.recall(source_hash, _no_use=True) if source_identity != None: source = RNS.Destination(source_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, "delivery") else: diff --git a/LXMF/_version.py b/LXMF/_version.py index e94731c..f8c6ac7 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.9.4" +__version__ = "0.9.5" From 8bdb434e05cbb7a4268ec4b0aba414486ceb274e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 21 Apr 2026 13:19:07 +0200 Subject: [PATCH 395/436] Retain announce data for used destinations --- LXMF/LXMRouter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index fb19ad8..c0545ca 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1839,7 +1839,8 @@ class LXMRouter: phy_stats = {"rssi": packet.rssi, "snr": packet.snr, "q": packet.q} - self.lxmf_delivery(lxmf_data, packet.destination_type, phy_stats=phy_stats, ratchet_id=packet.ratchet_id, method=method) + def job(): self.lxmf_delivery(lxmf_data, packet.destination_type, phy_stats=phy_stats, ratchet_id=packet.ratchet_id, method=method) + threading.Thread(target=job, daemon=True).start() except Exception as e: RNS.log("Exception occurred while parsing incoming LXMF data.", RNS.LOG_ERROR) @@ -2519,6 +2520,8 @@ class LXMRouter: if lxmessage.state == LXMessage.DELIVERED: RNS.log("Delivery has occurred for "+str(lxmessage)+", removing from outbound queue", RNS.LOG_DEBUG) self.pending_outbound.remove(lxmessage) + try: RNS.Reticulum.get_instance()._retain_destination_data(lxmessage.destination_hash) + except Exception as e: RNS.log(f"An error occurred while marking {RNS.prettyhexrep(lxmessage.destination_hash)} for announce data retainment: {e}", RNS.LOG_ERROR) # Udate ticket delivery stats if lxmessage.include_ticket and FIELD_TICKET in lxmessage.fields: From eb67b19a5b3b7b93ee5d79344fc95e79f97a4ccc Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 21 Apr 2026 17:24:16 +0200 Subject: [PATCH 396/436] Versions --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e19edb2..110f7d7 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=1.1.6"], + install_requires=["rns>=1.1.7"], python_requires=">=3.7", ) From 3f0681fa99e343ff58b784030935fbbfbdc6f021 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 21 Apr 2026 19:29:01 +0200 Subject: [PATCH 397/436] Versions --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 110f7d7..ac68c3f 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=1.1.7"], + install_requires=["rns>=1.1.8"], python_requires=">=3.7", ) From 8499729024a4cddfceb47ca07188bb5b1d11d179 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 22 Apr 2026 13:40:02 +0200 Subject: [PATCH 398/436] Updated versions --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index f8c6ac7..50533e3 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.9.5" +__version__ = "0.9.6" diff --git a/setup.py b/setup.py index ac68c3f..1053a31 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=1.1.8"], + install_requires=["rns>=1.1.9"], python_requires=">=3.7", ) From 727830cefda83d9c6e3982b48675425f3f988f9c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 28 Apr 2026 18:53:48 +0200 Subject: [PATCH 399/436] Update versions --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1053a31..18d9383 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=1.1.9"], + install_requires=["rns>=1.2.0"], python_requires=">=3.7", ) From 4ecbdb334ed06cc7bedd34c7141844c1ad5f829c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 2 May 2026 18:46:24 +0200 Subject: [PATCH 400/436] Updated readme --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index e435cec..852fc33 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@ # Lightweight Extensible Message Format -*This repository is [a public mirror](./MIRROR.md). All development is happening elsewhere.* - -LXMF is a simple and flexible messaging format and delivery protocol that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](https://reticulum.network) and offers zero-conf message routing, end-to-end encryption and Forward Secrecy, and can be transported over any kind of medium that Reticulum supports. +LXMF is a simple and flexible messaging format and delivery protocol that allows a wide variety of implementations, while using as little bandwidth as possible. It is built on top of [Reticulum](a8d24177d946de4f1f0a0fe1af9a1338:/page/repo.mu`g=reticulum|r=reticulum) and offers zero-conf message routing, end-to-end encryption and Forward Secrecy, and can be transported over any kind of medium that Reticulum supports. LXMF is efficient enough that it can deliver messages over extremely low-bandwidth systems such as packet radio or LoRa. Encrypted LXMF messages can also be encoded as QR-codes or text-based URIs, allowing completely analog *paper message* transport. @@ -87,7 +85,6 @@ lxm_router = LXMF.LXMRouter() message = LXMF.LXMessage(destination, source, "This is a short, simple message.") lxm_router.handle_outbound(message) - ``` The LXM Router then handles the heavy lifting, such as message packing, encryption, delivery confirmation, path lookup, routing, retries and failure notifications. From 9d413c075992a8364ac4a13a831ccc9e1579f822 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 2 May 2026 18:51:56 +0200 Subject: [PATCH 401/436] Updated readme --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 852fc33..7aa86ab 100644 --- a/README.md +++ b/README.md @@ -122,11 +122,13 @@ You can try out the paper messaging functionality by using the following QR code The [Sideband](https://unsigned.io/sideband) application allows you to do this easily. After you have imported the identity into an app of your choice, you can scan the following QR code and open it in the app, where it will be decrypted and added as a message. -

    +[Paper message QR code](:/page/blob.mu`g=reticulum|r=lxmf|ref=HEAD|path=docs%2Fpaper_msg_test.png) -You can also find the entire message in     this link: +You can also find the entire message in this link: -`lxm://azNivSwdv4e2aoX3mo2MdTAozuI7BlzrLlHULmnVgpz3dNT9CMPVwgywzCJP8FVogj5j_kU7j7ywuvBNcr45kRTrd19c3iHenmnSDe4VEd6FuGsAiT0Khzl7T81YZHPTDhRNp0FdhDE9AJ7uphw7zKMyqhHHxOxqrYeBeKF66gpPxDceqjsOApvsSwggjcuHBx9OxOBy05XmnJxA1unCKgvNfOFYc1T47luxoY3c0dLOJnJPwZuFRytx2TXlQNZzOJ28yTEygIfkDqEO9mZi5lgev7XZJ0DvgioQxMIyoCm7lBUzfq66zW3SQj6vHHph7bhr36dLOCFgk4fZA6yia2MlTT9KV66Tn2l8mPNDlvuSAJhwDA_xx2PN9zKadCjo9sItkAp8r-Ss1CzoUWZUAyT1oDw7ly6RrzGBG-e3eM3CL6u1juIeFiHby7_3cON-6VTUuk4xR5nwKlFTu5vsYMVXe5H3VahiDSS4Q1aqX7I` +``` +lxm://azNivSwdv4e2aoX3mo2MdTAozuI7BlzrLlHULmnVgpz3dNT9CMPVwgywzCJP8FVogj5j_kU7j7ywuvBNcr45kRTrd19c3iHenmnSDe4VEd6FuGsAiT0Khzl7T81YZHPTDhRNp0FdhDE9AJ7uphw7zKMyqhHHxOxqrYeBeKF66gpPxDceqjsOApvsSwggjcuHBx9OxOBy05XmnJxA1unCKgvNfOFYc1T47luxoY3c0dLOJnJPwZuFRytx2TXlQNZzOJ28yTEygIfkDqEO9mZi5lgev7XZJ0DvgioQxMIyoCm7lBUzfq66zW3SQj6vHHph7bhr36dLOCFgk4fZA6yia2MlTT9KV66Tn2l8mPNDlvuSAJhwDA_xx2PN9zKadCjo9sItkAp8r-Ss1CzoUWZUAyT1oDw7ly6RrzGBG-e3eM3CL6u1juIeFiHby7_3cON-6VTUuk4xR5nwKlFTu5vsYMVXe5H3VahiDSS4Q1aqX7I +``` On operating systems that allow for registering custom URI-handlers, you can click the link, and it will be decoded directly in your LXMF client. This works with Sideband on Android. @@ -210,8 +212,6 @@ LXMF is beta software, and should be considered experimental. While it has been LXMF is actively being developed, and the following improvements and features are currently planned for implementation: -- ~~Update examples in readme to actually work~~ -- ~~Sync affinity based on link speeds and distances, for more intelligently choosing peer sync order~~ - Sneakernet and physical transport functionality - Content Destinations, and easy to use API for group messaging and discussion threads - Write and release full API and protocol documentation From ad616fc18eacf367345d1a2443d2d028198661e2 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 2 May 2026 18:53:10 +0200 Subject: [PATCH 402/436] Updated readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7aa86ab..0a970e1 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ You can try out the paper messaging functionality by using the following QR code The [Sideband](https://unsigned.io/sideband) application allows you to do this easily. After you have imported the identity into an app of your choice, you can scan the following QR code and open it in the app, where it will be decrypted and added as a message. -[Paper message QR code](:/page/blob.mu`g=reticulum|r=lxmf|ref=HEAD|path=docs%2Fpaper_msg_test.png) +[Paper message QR code](a8d24177d946de4f1f0a0fe1af9a1338:/page/blob.mu`g=reticulum|r=lxmf|ref=HEAD|path=docs%2Fpaper_msg_test.png) You can also find the entire message in this link: From f9967db3ee89e348f69f2bb39ba292549f230531 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 2 May 2026 18:54:07 +0200 Subject: [PATCH 403/436] Updated readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a970e1..d8c9e47 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ You can try out the paper messaging functionality by using the following QR code The [Sideband](https://unsigned.io/sideband) application allows you to do this easily. After you have imported the identity into an app of your choice, you can scan the following QR code and open it in the app, where it will be decrypted and added as a message. -[Paper message QR code](a8d24177d946de4f1f0a0fe1af9a1338:/page/blob.mu`g=reticulum|r=lxmf|ref=HEAD|path=docs%2Fpaper_msg_test.png) +[Paper message QR code](a8d24177d946de4f1f0a0fe1af9a1338:/page/blob.mu`g=reticulum|r=lxmf|ref=HEAD|path=docs/paper_msg_test.png) You can also find the entire message in this link: From 189f5239a05d004733744dfdc95b41c318cf8548 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 6 May 2026 17:19:16 +0200 Subject: [PATCH 404/436] Raise explicit IOError for the sloppy vibe-coders who apparently can't figure out that you need to set an outbound propagation node before trying to send a propagated message --- LXMF/Handlers.py | 2 +- LXMF/LXMRouter.py | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index 871cc56..afcf708 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -18,7 +18,7 @@ class LXMFDeliveryAnnounceHandler: self.lxmrouter.update_stamp_cost(destination_hash, stamp_cost) except Exception as e: - RNS.log(f"An error occurred while trying to decode announced stamp cost. The contained exception was: {e}", RNS.LOG_ERROR) + RNS.log(f"Could not decode stamp cost from announce data. The contained exception was: {e}", RNS.LOG_DEBUG) for lxmessage in self.lxmrouter.pending_outbound: if destination_hash == lxmessage.destination_hash: diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index c0545ca..4a5dfa9 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -97,7 +97,6 @@ class LXMRouter: self.pending_inbound = [] self.pending_outbound = [] - self.failed_outbound = [] self.direct_links = {} self.backchannel_links = {} self.delivery_destinations = {} @@ -1645,6 +1644,10 @@ class LXMRouter: def handle_outbound(self, lxmessage): destination_hash = lxmessage.get_destination().hash + if lxmessage.desired_method == LXMessage.PROPAGATED and not self.outbound_propagation_node: + self.fail_message(lxmessage) + raise IOError("Attempt to send propagated message with no outbound propagation node configured") + if lxmessage.stamp_cost == None: if destination_hash in self.outbound_stamp_costs: stamp_cost = self.outbound_stamp_costs[destination_hash][1] @@ -2393,14 +2396,8 @@ class LXMRouter: RNS.log(str(lxmessage)+" failed to send", RNS.LOG_DEBUG) lxmessage.progress = 0.0 - if lxmessage in self.pending_outbound: - self.pending_outbound.remove(lxmessage) - - self.failed_outbound.append(lxmessage) - - if lxmessage.state != LXMessage.REJECTED: - lxmessage.state = LXMessage.FAILED - + if lxmessage in self.pending_outbound: self.pending_outbound.remove(lxmessage) + if lxmessage.state != LXMessage.REJECTED: lxmessage.state = LXMessage.FAILED if lxmessage.failed_callback != None and callable(lxmessage.failed_callback): lxmessage.failed_callback(lxmessage) From 25b7fcf7de1ded6f052ecfd2e33c4e5d6d554c81 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 6 May 2026 17:20:03 +0200 Subject: [PATCH 405/436] Updated versions --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 50533e3..f5b7730 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.9.6" +__version__ = "0.9.7" diff --git a/setup.py b/setup.py index 18d9383..5a88f36 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=1.2.0"], + install_requires=["rns>=1.2.4"], python_requires=">=3.7", ) From 83fbe804974e32a5e506565cfd274d6abf32740f Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 6 May 2026 18:45:41 +0200 Subject: [PATCH 406/436] Strip null bytes from display names by default --- LXMF/LXMF.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index cdb90a5..2abf087 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -128,7 +128,7 @@ def display_name_from_app_data(app_data=None): if dn == None: return None else: try: - decoded = dn.decode("utf-8") + decoded = dn.decode("utf-8").replace("\x00", "").strip() return decoded except Exception as e: RNS.log(f"Could not decode display name in included announce data. The contained exception was: {e}", RNS.LOG_ERROR) From 2c4dfdd50ea131147532ae4c928cb4090a61cad3 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 6 May 2026 18:45:57 +0200 Subject: [PATCH 407/436] Updated versions --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5a88f36..e6d5afb 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=1.2.4"], + install_requires=["rns>=1.2.3"], python_requires=">=3.7", ) From 6a00d82b281cb013979400b37df2b459e9c3b866 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 7 May 2026 20:23:56 +0200 Subject: [PATCH 408/436] Updated versions --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e6d5afb..5a88f36 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=1.2.3"], + install_requires=["rns>=1.2.4"], python_requires=">=3.7", ) From a8505eade9f3b0ded493a6035eb5ee433538c6ee Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 10 May 2026 14:12:51 +0200 Subject: [PATCH 409/436] Added outbound processing trigger for pn node announces --- LXMF/Handlers.py | 12 ++++++++++++ LXMF/_version.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/LXMF/Handlers.py b/LXMF/Handlers.py index afcf708..6239b18 100644 --- a/LXMF/Handlers.py +++ b/LXMF/Handlers.py @@ -41,6 +41,18 @@ class LXMFPropagationAnnounceHandler: def received_announce(self, destination_hash, announced_identity, app_data, announce_packet_hash, is_path_response): try: if type(app_data) == bytes: + if destination_hash == self.lxmrouter.get_outbound_propagation_node(): + if pn_announce_data_is_valid(app_data): + for lxmessage in self.lxmrouter.pending_outbound: + if lxmessage.method == LXMessage.PROPAGATED: + lxmessage.next_delivery_attempt = time.time() + + def outbound_trigger(): + while self.lxmrouter.outbound_processing_lock.locked(): time.sleep(0.1) + self.lxmrouter.process_outbound() + + threading.Thread(target=outbound_trigger, daemon=True).start() + if self.lxmrouter.propagation_node: if pn_announce_data_is_valid(app_data): data = msgpack.unpackb(app_data) diff --git a/LXMF/_version.py b/LXMF/_version.py index f5b7730..a25765c 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.9.7" +__version__ = "0.9.8" From 0cb62ddc3640be13225475034cd77220eb6f4539 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 10 May 2026 17:16:38 +0200 Subject: [PATCH 410/436] Use stamp generation worker context manager on Python 3.14+ --- LXMF/LXStamper.py | 168 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 148 insertions(+), 20 deletions(-) diff --git a/LXMF/LXStamper.py b/LXMF/LXStamper.py index 3d7a1c2..ea25f08 100644 --- a/LXMF/LXStamper.py +++ b/LXMF/LXStamper.py @@ -2,9 +2,11 @@ import RNS import RNS.vendor.umsgpack as msgpack import os +import sys import time import math import itertools +import contextlib import multiprocessing WORKBLOCK_EXPAND_ROUNDS = 3000 @@ -12,9 +14,38 @@ WORKBLOCK_EXPAND_ROUNDS_PN = 1000 WORKBLOCK_EXPAND_ROUNDS_PEERING = 25 STAMP_SIZE = RNS.Identity.HASHLENGTH//8 PN_VALIDATION_POOL_MIN_SIZE = 256 +USE_WORKER_MANAGER = False active_jobs = {} +if sys.version_info[0] >= 3 and sys.version_info[1] >= 14: + USE_WORKER_MANAGER = True + +@contextlib.contextmanager +def worker_context(ctx, stamp_cost, workblock, message_id): + stop_event = ctx.Event() + result_queue = ctx.Queue(1) + rounds_queue = ctx.Queue() + + active_jobs[message_id] = [stop_event, result_queue] + job_procs = [] + + try: yield stop_event, result_queue, rounds_queue, job_procs + finally: + if message_id in active_jobs: del active_jobs[message_id] + stop_event.set() + + for p in job_procs: + if p.is_alive(): p.terminate() + + for p in job_procs: p.join(timeout=0.5) + + result_queue.close() + result_queue.join_thread() + rounds_queue.close() + rounds_queue.join_thread() + stop_event.clear() + def stamp_workblock(material, expand_rounds=WORKBLOCK_EXPAND_ROUNDS): wb_st = time.time() workblock = b"" @@ -100,7 +131,9 @@ def generate_stamp(message_id, stamp_cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): stamp, rounds = job_simple(stamp_cost, workblock, message_id) elif RNS.vendor.platformutils.is_android(): stamp, rounds = job_android(stamp_cost, workblock, message_id) - else: stamp, rounds = job_linux(stamp_cost, workblock, message_id) + else: + if USE_WORKER_MANAGER: stamp, rounds = job_linux_managed(stamp_cost, workblock, message_id) + else: stamp, rounds = job_linux(stamp_cost, workblock, message_id) duration = time.time() - start_time speed = rounds/duration @@ -176,6 +209,90 @@ def job_simple(stamp_cost, workblock, message_id): return pstamp, rounds +def job_linux_managed(stamp_cost, workblock, message_id): + ctx = multiprocessing.get_context("fork") + cores = multiprocessing.cpu_count() + jobs = cores if cores <= 12 else int(cores/2) + + allow_kill = True + stamp = None + total_rounds = 0 + + with worker_context(ctx, stamp_cost, workblock, message_id) as (stop_event, result_queue, rounds_queue, job_procs): + RNS.log(f"Starting {jobs} stamp generation workers", RNS.LOG_DEBUG) + + def job(stop_event, sc, wb, worker_id): + terminated = False + rounds = 0 + pstamp = os.urandom(256//8) + + def sv(s, c, w): + target = 0b1<<256-c; m = w+s + result = RNS.Identity.full_hash(m) + if int.from_bytes(result, byteorder="big") > target: return False + else: return True + + while not stop_event.is_set() and not sv(pstamp, sc, wb): + pstamp = os.urandom(256//8); rounds += 1 + + if not stop_event.is_set(): + stop_event.set() + try: result_queue.put_nowait(pstamp) + except: pass + + try: rounds_queue.put_nowait(rounds) + except: pass + + for jpn in range(jobs): + p = ctx.Process(target=job, args=(stop_event, stamp_cost, workblock, jpn), daemon=True) + job_procs.append(p) + p.start() + + try: stamp = result_queue.get(timeout=None) + except Exception as e: + RNS.log(f"Failed to get result from workers: {e}", RNS.LOG_ERROR) + stamp = None + + # Collect any potential spurious + # results from worker queue. + try: + while True: result_queue.get_nowait() + except: pass + + for j in range(jobs): + nrounds = 0 + try: + nrounds = rounds_queue.get(timeout=2) + except Exception as e: + RNS.log(f"Failed to get round stats part {j}: {e}", RNS.LOG_ERROR) + total_rounds += nrounds + + all_exited = False + exit_timeout = time.time() + 5 + while time.time() < exit_timeout: + if not any(p.is_alive() for p in job_procs): + all_exited = True + break + time.sleep(0.1) + + if not all_exited: + RNS.log("Stamp generation IPC timeout, possible worker deadlock. Terminating remaining processes.", RNS.LOG_ERROR) + if allow_kill: + for j in range(jobs): + process = job_procs[j] + process.kill() + else: + return None + + else: + for j in range(jobs): + process = job_procs[j] + process.join() + # RNS.log(f"Joined {j} / {process}", RNS.LOG_DEBUG) # TODO: Remove + + return stamp, total_rounds + + def job_linux(stamp_cost, workblock, message_id): allow_kill = True stamp = None @@ -366,30 +483,41 @@ if __name__ == "__main__": RNS.log("No cost argument provided", RNS.LOG_ERROR) exit(1) else: - try: - cost = int(sys.argv[1]) + try: cost = int(sys.argv[1]) except Exception as e: RNS.log(f"Invalid cost argument provided: {e}", RNS.LOG_ERROR) exit(1) + rounds = 1 + if len(sys.argv) > 2: + try: rounds = int(sys.argv[2]) + except Exception as e: + RNS.log(f"Invalid rounds argument provided: {e}", RNS.LOG_ERROR) + exit(1) + RNS.loglevel = RNS.LOG_DEBUG - RNS.log("Testing LXMF stamp generation", RNS.LOG_DEBUG) - message_id = os.urandom(32) - generate_stamp(message_id, cost) + + for i in range(rounds): + RNS.log("", RNS.LOG_DEBUG) + RNS.log(f"Round {i+1} of {rounds}", RNS.LOG_DEBUG) - RNS.log("", RNS.LOG_DEBUG) - RNS.log("Testing propagation stamp generation", RNS.LOG_DEBUG) - message_id = os.urandom(32) - generate_stamp(message_id, cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PN) + RNS.log("Testing LXMF stamp generation", RNS.LOG_DEBUG) + message_id = os.urandom(32) + generate_stamp(message_id, cost) - RNS.log("", RNS.LOG_DEBUG) - RNS.log("Testing peering key generation", RNS.LOG_DEBUG) - message_id = os.urandom(32) - generate_stamp(message_id, cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PEERING) + RNS.log("", RNS.LOG_DEBUG) + RNS.log("Testing propagation stamp generation", RNS.LOG_DEBUG) + message_id = os.urandom(32) + generate_stamp(message_id, cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PN) - transient_list = [] - st = time.time(); count = 10000 - for i in range(count): transient_list.append(os.urandom(256)) - validate_pn_stamps(transient_list, 5) - dt = time.time()-st; mps = count/dt - RNS.log(f"Validated {count} PN stamps in {RNS.prettytime(dt)}, {round(mps,1)} m/s", RNS.LOG_DEBUG) + RNS.log("", RNS.LOG_DEBUG) + RNS.log("Testing peering key generation", RNS.LOG_DEBUG) + message_id = os.urandom(32) + generate_stamp(message_id, cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS_PEERING) + + # transient_list = [] + # st = time.time(); count = 10000 + # for i in range(count): transient_list.append(os.urandom(256)) + # validate_pn_stamps(transient_list, 5) + # dt = time.time()-st; mps = count/dt + # RNS.log(f"Validated {count} PN stamps in {RNS.prettytime(dt)}, {round(mps,1)} m/s", RNS.LOG_DEBUG) From 7b0e7028321180a03713c5e71a34cb0d8ef99d13 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 10 May 2026 17:20:26 +0200 Subject: [PATCH 411/436] Updated versions --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5a88f36..4d90fab 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=1.2.4"], + install_requires=["rns>=1.2.5"], python_requires=">=3.7", ) From 29c79177a697f64cba5af9413a7628447fbb8e67 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 20 May 2026 00:33:46 +0200 Subject: [PATCH 412/436] Updated makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c00cde9..a32cc7c 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ build_sdist: build_spkg: remove_symlinks build_sdist create_symlinks -release: remove_symlinks build_wheel create_symlinks +release: remove_symlinks build_wheel build_spkg create_symlinks upload: @echo Uploading to PyPi... From 7f0e2627d1d0cee447293be67ce2283f8e5c3c9d Mon Sep 17 00:00:00 2001 From: Jeremy O'Brien Date: Wed, 20 May 2026 09:43:35 -0400 Subject: [PATCH 413/436] atomically write lxmf message files to prevent reading of partial files from other processes during the write --- LXMF/LXMessage.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index f85f3b8..5271d4e 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -672,15 +672,26 @@ class LXMessage: def write_to_directory(self, directory_path): file_name = RNS.hexrep(self.hash, delimit=False) file_path = directory_path+"/"+file_name + tmp_path = file_path+".tmp."+str(os.getpid()) try: - file = open(file_path, "wb") - file.write(self.packed_container()) - file.close() + with open(tmp_path, "wb") as file: + file.write(self.packed_container()) + file.flush() + try: + os.fsync(file.fileno()) + except OSError: + pass + os.replace(tmp_path, file_path) return file_path except Exception as e: + try: + if os.path.exists(tmp_path): + os.unlink(tmp_path) + except Exception: + pass RNS.log("Error while writing LXMF message to file \""+str(file_path)+"\". The contained exception was: "+str(e), RNS.LOG_ERROR) return None From d6ec05193b3b79ab577ca2a4173ef3ef51492db8 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 20 May 2026 19:52:55 +0200 Subject: [PATCH 414/436] Ensure different tmp-file names if os.getpid() is not available --- LXMF/LXMessage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 5271d4e..639ab36 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -672,7 +672,7 @@ class LXMessage: def write_to_directory(self, directory_path): file_name = RNS.hexrep(self.hash, delimit=False) file_path = directory_path+"/"+file_name - tmp_path = file_path+".tmp."+str(os.getpid()) + tmp_path = file_path+".tmp."+str(os.getpid() or time.time()) try: with open(tmp_path, "wb") as file: From 1bef747306ad0100455f8a4fc749c6cd61dd1781 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 20 May 2026 19:55:03 +0200 Subject: [PATCH 415/436] Cleanup --- LXMF/LXMessage.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index 639ab36..fdc0be8 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -668,7 +668,6 @@ class LXMessage: return msgpack.packb(container) - def write_to_directory(self, directory_path): file_name = RNS.hexrep(self.hash, delimit=False) file_path = directory_path+"/"+file_name @@ -678,20 +677,17 @@ class LXMessage: with open(tmp_path, "wb") as file: file.write(self.packed_container()) file.flush() - try: - os.fsync(file.fileno()) - except OSError: - pass - os.replace(tmp_path, file_path) + try: os.fsync(file.fileno()) + except OSError: pass + os.replace(tmp_path, file_path) return file_path except Exception as e: try: - if os.path.exists(tmp_path): - os.unlink(tmp_path) - except Exception: - pass + if os.path.exists(tmp_path): os.unlink(tmp_path) + except Exception: pass + RNS.log("Error while writing LXMF message to file \""+str(file_path)+"\". The contained exception was: "+str(e), RNS.LOG_ERROR) return None From 84613f8b44693a4b331465d8f397d02f0661903e Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 20 May 2026 19:56:35 +0200 Subject: [PATCH 416/436] Updated versions --- LXMF/_version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index a25765c..88081a7 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.9.8" +__version__ = "0.9.9" diff --git a/setup.py b/setup.py index 4d90fab..d2d88e5 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=1.2.5"], + install_requires=["rns>=1.2.9"], python_requires=">=3.7", ) From dffbf4dfe14229f1913efd06cd2d8b35da7ce189 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 21 May 2026 16:17:35 +0200 Subject: [PATCH 417/436] Updated versions --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d2d88e5..2ed6932 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=1.2.9"], + install_requires=["rns>=1.3.0"], python_requires=">=3.7", ) From b415a136f3c7b4335b82efcb183a160871cb85dd Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 21 May 2026 17:58:59 +0200 Subject: [PATCH 418/436] Updated makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a32cc7c..5fe5f96 100644 --- a/Makefile +++ b/Makefile @@ -27,4 +27,4 @@ release: remove_symlinks build_wheel build_spkg create_symlinks upload: @echo Uploading to PyPi... - twine upload dist/* + twine upload dist/*.whl dist/*.tar.gz From 764758d185011c4d7c884c9e61b1888335caf695 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 24 May 2026 22:59:57 +0200 Subject: [PATCH 419/436] Added reply, reaction and comment field standards to LXMF.py --- LXMF/LXMF.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 2abf087..5c73f56 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -20,6 +20,16 @@ FIELD_TICKET = 0x0C FIELD_EVENT = 0x0D FIELD_RNR_REFS = 0x0E FIELD_RENDERER = 0x0F +FIELD_REPLY_TO = 0x30 # Bytes, full LXMessage.hash +FIELD_REPLY_QUOTE = 0x31 # Bytes, quoted content in UTF-8 encoding +FIELD_REACTION = 0x40 # Dict, see "Reaction dict indices" below +FIELD_COMMENT = 0x41 # Dict, see "Comment dict indices" below +FIELD_CONTINUATION = 0x42 # Dict, see "Continuation dict indices" below + +# Unallocated fields between 0x00 and 0x80, both included, +# should be considered reserved for future extensibility +# For experimental and unstable features, it is recommended +# to use fields above 0xFF. # For usecases such as including custom data structures, # embedding or encapsulating other data types or protocols @@ -91,6 +101,49 @@ RENDERER_MICRON = 0x01 RENDERER_MARKDOWN = 0x02 RENDERER_BBCODE = 0x03 +############################################################ +# To be finalized in 1.0.0. A workdoc with open interaction +# through rngit is available for comments and nuancing on: +# +# a8d24177d946de4f1f0a0fe1af9a1338:/page/work.mu`g=reticulum|r=lxmf +# +# Clients that have implemented different reply, reaction +# or comment mechanisms can choose to transitionally parse +# their own specific formats, but are recommended to attempt +# parsing the structure and format defined herein first, +# and fall back to their client-specific structure second. + +# Reaction dict indicies are integers to preserve bandwidth. +# +# Clients choose how to handle reaction content, if at all. +# While reactions are typically a single unicode emoji or +# similar, the exact implementation and sanitization is +# left up to the client. When using the FIELD_REACTION +# field, the contents is a dict with the following keys: +REACTION_TO = 0x00 # Bytes, full LXMessage.hash +REACTION_CONTENT = 0x01 # Bytes, the reaction content in UTF-8 encoding + +# Comment dict indicies are integers to preserve bandwidth. +# +# Clients choose how to handle messages intended as comments +# for other message, if at all. The actual comment content +# is carried as the normal LXM content, meaning clients that +# do not support comments will display them as normal messages. +# When using the FIELD_COMMENT field, the contents is a dict +# with the following keys: +COMMENT_FOR = 0x00 # Bytes, full LXMessage.hash + +# Continuation dict indicies are integers to preserve bandwidth. +# +# Clients choose how to handle messages that continue earlier +# messages, if at all. The actual continuation content is +# carried as the normal LXM content, meaning clients that +# do not support continuations will display them as normal. +# When using the FIELD_CONTINUATION field, the contents is a +# dict with the following keys: +CONTINUATION_OF = 0x00 # Bytes, full LXMessage.hash +############################################################ + # Optional propagation node metadata fields. These # fields may be highly unstable in allocation and # availability until the version 1.0.0 release, so use From b53a3ce37dc8b385dd04005f6dfca314ed8c9ed5 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 24 May 2026 23:16:15 +0200 Subject: [PATCH 420/436] Clarified FIELD_THREAD --- LXMF/LXMF.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index 5c73f56..c128652 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -12,7 +12,7 @@ FIELD_ICON_APPEARANCE = 0x04 FIELD_FILE_ATTACHMENTS = 0x05 FIELD_IMAGE = 0x06 FIELD_AUDIO = 0x07 -FIELD_THREAD = 0x08 +FIELD_THREAD = 0x08 # Bytes, full thread ID hash FIELD_COMMANDS = 0x09 FIELD_RESULTS = 0x0A FIELD_GROUP = 0x0B From 55620bf45c87c47c8e553357bd95192e767ec46b Mon Sep 17 00:00:00 2001 From: Parnikkapore Date: Sun, 24 May 2026 12:21:42 +0000 Subject: [PATCH 421/436] Fix lxmd crash when defaulting --config to ~/.config/lxmd/ --- LXMF/Utilities/lxmd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LXMF/Utilities/lxmd.py b/LXMF/Utilities/lxmd.py index 4838189..5c797c4 100644 --- a/LXMF/Utilities/lxmd.py +++ b/LXMF/Utilities/lxmd.py @@ -307,7 +307,7 @@ def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbo if configdir == None: if os.path.isdir("/etc/lxmd") and os.path.isfile("/etc/lxmd/config"): configdir = "/etc/lxmd" - elif os.path.isdir(RNS.Reticulum.userdir+"/.config/lxmd") and os.path.isfile(Reticulum.userdir+"/.config/lxmd/config"): + elif os.path.isdir(RNS.Reticulum.userdir+"/.config/lxmd") and os.path.isfile(RNS.Reticulum.userdir+"/.config/lxmd/config"): configdir = RNS.Reticulum.userdir+"/.config/lxmd" else: configdir = RNS.Reticulum.userdir+"/.lxmd" @@ -822,7 +822,7 @@ def _remote_init(configdir=None, rnsconfigdir=None, verbosity=0, quietness=0, id if identity_path == None: if configdir == None: if os.path.isdir("/etc/lxmd") and os.path.isfile("/etc/lxmd/config"): configdir = "/etc/lxmd" - elif os.path.isdir(RNS.Reticulum.userdir+"/.config/lxmd") and os.path.isfile(Reticulum.userdir+"/.config/lxmd/config"): configdir = RNS.Reticulum.userdir+"/.config/lxmd" + elif os.path.isdir(RNS.Reticulum.userdir+"/.config/lxmd") and os.path.isfile(RNS.Reticulum.userdir+"/.config/lxmd/config"): configdir = RNS.Reticulum.userdir+"/.config/lxmd" else: configdir = RNS.Reticulum.userdir+"/.lxmd" configpath = configdir+"/config" From 599406ed0fa7cbd0f4d01e648d743b2f70edb2ae Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 25 May 2026 16:59:24 +0200 Subject: [PATCH 422/436] Activated compression support signalling --- LXMF/LXMRouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 4a5dfa9..3c9cd59 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -996,7 +996,7 @@ class LXMRouter: stamp_cost = delivery_destination.stamp_cost supported_functionality = [SF_COMPRESSION] - peer_data = [display_name, stamp_cost] + peer_data = [display_name, stamp_cost, supported_functionality] return msgpack.packb(peer_data) From 2ac2b100ae9bdac107bf2c17e12e8406464c8d72 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 26 May 2026 13:14:05 +0200 Subject: [PATCH 423/436] Automatically drop LXMs from blackholed identities --- LXMF/LXMRouter.py | 10 ++++++---- LXMF/LXMessage.py | 11 ++++++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/LXMF/LXMRouter.py b/LXMF/LXMRouter.py index 3c9cd59..aca6621 100644 --- a/LXMF/LXMRouter.py +++ b/LXMF/LXMRouter.py @@ -1735,11 +1735,13 @@ class LXMRouter: def lxmf_delivery(self, lxmf_data, destination_type = None, phy_stats = None, ratchet_id = None, method = None, no_stamp_enforcement=False, allow_duplicate=False): try: message = LXMessage.unpack_from_bytes(lxmf_data) - if ratchet_id and not message.ratchet_id: - message.ratchet_id = ratchet_id - if method: - message.method = method + if message.source_blackholed: + RNS.log(f"Dropping LXM from blackholed identity {message.source.identity}", RNS.LOG_DEBUG) + return False + + if ratchet_id and not message.ratchet_id: message.ratchet_id = ratchet_id + if method: message.method = method if message.signature_validated and FIELD_TICKET in message.fields: ticket_entry = message.fields[FIELD_TICKET] diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index fdc0be8..c128c00 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -168,6 +168,7 @@ class LXMessage: self.paper_packed = None self.incoming = False + self.source_blackholed = False self.signature_validated = False self.unverified_reason = None self.ratchet_id = None @@ -678,7 +679,7 @@ class LXMessage: file.write(self.packed_container()) file.flush() try: os.fsync(file.fileno()) - except OSError: pass + except OSError as e: RNS.log(f"Error while waiting for persist fsync for {self}: {e}", RNS.LOG_WARNING) os.replace(tmp_path, file_path) return file_path @@ -686,9 +687,9 @@ class LXMessage: except Exception as e: try: if os.path.exists(tmp_path): os.unlink(tmp_path) - except Exception: pass + except Exception as e: RNS.log(f"Error while cleaning temporary file {tmp_path} for {self}: {e}", RNS.LOG_ERROR) - RNS.log("Error while writing LXMF message to file \""+str(file_path)+"\". The contained exception was: "+str(e), RNS.LOG_ERROR) + RNS.log(f"Error while writing LXMF message to file \"{file_path}\". The contained exception was: {e}", RNS.LOG_ERROR) return None def as_uri(self, finalise=True): @@ -796,6 +797,10 @@ class LXMessage: message.set_title_from_bytes(title_bytes) message.set_content_from_bytes(content_bytes) + try: + if source_identity != None: message.source_blackholed = RNS.Reticulum.get_instance().is_blackholed(source_identity) + except Exception as e: RNS.log(f"Could not determine message source blackhole status: {e}", RNS.LOG_WARNING) + try: if source: if source.identity.validate(signature, signed_part): From 575fb7d77d8f621ab0ffb860902720a330192e26 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 26 May 2026 13:17:46 +0200 Subject: [PATCH 424/436] Prevent LXM persist race in write_to_directory when messages change state within very short time spans --- LXMF/LXMessage.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index c128c00..a42d1dd 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -8,6 +8,7 @@ import multiprocessing import LXMF.LXStamper as LXStamper from .LXMF import APP_NAME, compression_support_from_app_data +from threading import Lock class LXMessage: @@ -184,6 +185,7 @@ class LXMessage: self.__delivery_destination = None self.__delivery_callback = None self.__pn_encrypted_data = None + self.__persist_lock = Lock() self.failed_callback = None self.deferred_stamp_generating = False @@ -674,23 +676,24 @@ class LXMessage: file_path = directory_path+"/"+file_name tmp_path = file_path+".tmp."+str(os.getpid() or time.time()) - try: - with open(tmp_path, "wb") as file: - file.write(self.packed_container()) - file.flush() - try: os.fsync(file.fileno()) - except OSError as e: RNS.log(f"Error while waiting for persist fsync for {self}: {e}", RNS.LOG_WARNING) - - os.replace(tmp_path, file_path) - return file_path - - except Exception as e: + with self.__persist_lock: try: - if os.path.exists(tmp_path): os.unlink(tmp_path) - except Exception as e: RNS.log(f"Error while cleaning temporary file {tmp_path} for {self}: {e}", RNS.LOG_ERROR) + with open(tmp_path, "wb") as file: + file.write(self.packed_container()) + file.flush() + try: os.fsync(file.fileno()) + except OSError as e: RNS.log(f"Error while waiting for persist fsync for {self}: {e}", RNS.LOG_WARNING) - RNS.log(f"Error while writing LXMF message to file \"{file_path}\". The contained exception was: {e}", RNS.LOG_ERROR) - return None + os.replace(tmp_path, file_path) + return file_path + + except Exception as e: + try: + if os.path.exists(tmp_path): os.unlink(tmp_path) + except Exception as e: RNS.log(f"Error while cleaning temporary file {tmp_path} for {self}: {e}", RNS.LOG_ERROR) + + RNS.log(f"Error while writing LXMF message to file \"{file_path}\". The contained exception was: {e}", RNS.LOG_ERROR) + return None def as_uri(self, finalise=True): if not self.packed: From 312e0a8ded5877034d9965d763c0b70ca7921f22 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 26 May 2026 13:18:33 +0200 Subject: [PATCH 425/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 88081a7..5becc17 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "0.9.9" +__version__ = "1.0.0" From 044f3d2879af719a189f399b97a5929b0a9bf6f5 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 26 May 2026 23:32:02 +0200 Subject: [PATCH 426/436] Updated versions --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2ed6932..8798705 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=1.3.0"], + install_requires=["rns>=1.3.2"], python_requires=">=3.7", ) From bf924c739cf0a5c2ceb4af254b79fb60aaf3adce Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 27 May 2026 12:28:58 +0200 Subject: [PATCH 427/436] Cleanup --- LXMF/LXMF.py | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/LXMF/LXMF.py b/LXMF/LXMF.py index c128652..d981ae0 100644 --- a/LXMF/LXMF.py +++ b/LXMF/LXMF.py @@ -101,20 +101,6 @@ RENDERER_MICRON = 0x01 RENDERER_MARKDOWN = 0x02 RENDERER_BBCODE = 0x03 -############################################################ -# To be finalized in 1.0.0. A workdoc with open interaction -# through rngit is available for comments and nuancing on: -# -# a8d24177d946de4f1f0a0fe1af9a1338:/page/work.mu`g=reticulum|r=lxmf -# -# Clients that have implemented different reply, reaction -# or comment mechanisms can choose to transitionally parse -# their own specific formats, but are recommended to attempt -# parsing the structure and format defined herein first, -# and fall back to their client-specific structure second. - -# Reaction dict indicies are integers to preserve bandwidth. -# # Clients choose how to handle reaction content, if at all. # While reactions are typically a single unicode emoji or # similar, the exact implementation and sanitization is @@ -123,8 +109,6 @@ RENDERER_BBCODE = 0x03 REACTION_TO = 0x00 # Bytes, full LXMessage.hash REACTION_CONTENT = 0x01 # Bytes, the reaction content in UTF-8 encoding -# Comment dict indicies are integers to preserve bandwidth. -# # Clients choose how to handle messages intended as comments # for other message, if at all. The actual comment content # is carried as the normal LXM content, meaning clients that @@ -133,8 +117,6 @@ REACTION_CONTENT = 0x01 # Bytes, the reaction content in UTF-8 encoding # with the following keys: COMMENT_FOR = 0x00 # Bytes, full LXMessage.hash -# Continuation dict indicies are integers to preserve bandwidth. -# # Clients choose how to handle messages that continue earlier # messages, if at all. The actual continuation content is # carried as the normal LXM content, meaning clients that @@ -142,7 +124,6 @@ COMMENT_FOR = 0x00 # Bytes, full LXMessage.hash # When using the FIELD_CONTINUATION field, the contents is a # dict with the following keys: CONTINUATION_OF = 0x00 # Bytes, full LXMessage.hash -############################################################ # Optional propagation node metadata fields. These # fields may be highly unstable in allocation and @@ -188,8 +169,7 @@ def display_name_from_app_data(app_data=None): return None # Original announce format - else: - return app_data.decode("utf-8") + else: return app_data.decode("utf-8") def stamp_cost_from_app_data(app_data=None): if app_data == None or app_data == b"": return None @@ -238,8 +218,8 @@ def pn_stamp_cost_from_app_data(app_data=None): if pn_announce_data_is_valid(app_data): data = msgpack.unpackb(app_data) return data[5][0] - else: - return None + + else: return None def pn_announce_data_is_valid(data): try: From 5be161cb1e50e99925f20852324dc3c82a3c0cd1 Mon Sep 17 00:00:00 2001 From: Jeremy O'Brien <956a74d715fe9cb2e9da0a19b067b414> Date: Wed, 27 May 2026 22:52:17 -0400 Subject: [PATCH 428/436] Prevent message write race among different processes touching the same message --- LXMF/LXMessage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMessage.py b/LXMF/LXMessage.py index a42d1dd..95d2310 100644 --- a/LXMF/LXMessage.py +++ b/LXMF/LXMessage.py @@ -674,7 +674,7 @@ class LXMessage: def write_to_directory(self, directory_path): file_name = RNS.hexrep(self.hash, delimit=False) file_path = directory_path+"/"+file_name - tmp_path = file_path+".tmp."+str(os.getpid() or time.time()) + tmp_path = file_path+".tmp."+str(os.getpid() or time.time())+"."+RNS.hexrep(os.urandom(8), delimit=False) with self.__persist_lock: try: From 11b2480223daae4b942359dde5a1180efc7f8976 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 29 May 2026 09:57:51 +0200 Subject: [PATCH 429/436] Updated versions --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8798705..f8c3421 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=1.3.2"], + install_requires=["rns>=1.3.4"], python_requires=">=3.7", ) From c877efaec1296b5f1b568daf039193a2b1a70cf4 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Fri, 29 May 2026 09:58:06 +0200 Subject: [PATCH 430/436] Updated version --- LXMF/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/_version.py b/LXMF/_version.py index 5becc17..5c4105c 100644 --- a/LXMF/_version.py +++ b/LXMF/_version.py @@ -1 +1 @@ -__version__ = "1.0.0" +__version__ = "1.0.1" From 20864133a3940f844606d95542ae0e83cf365cd2 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 31 May 2026 22:10:48 +0200 Subject: [PATCH 431/436] Fixed typo --- LXMF/LXMPeer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LXMF/LXMPeer.py b/LXMF/LXMPeer.py index 0dbf8ce..37515e6 100644 --- a/LXMF/LXMPeer.py +++ b/LXMF/LXMPeer.py @@ -404,7 +404,7 @@ class LXMPeer: if response == LXMPeer.ERROR_NO_IDENTITY: if self.link != None: RNS.log("Remote peer indicated that no identification was received, retrying...", RNS.LOG_VERBOSE) - self.link.identify() + self.link.identify(self.router.identity) self.state = LXMPeer.LINK_READY self.sync() return From a29c4a0e17c55a4556b0c772c9d080d479d99ae8 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 1 Jun 2026 00:03:39 +0200 Subject: [PATCH 432/436] Updated version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f8c3421..beb1432 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setuptools.setup( 'lxmd=LXMF.Utilities.lxmd:main', ] }, - install_requires=["rns>=1.3.4"], + install_requires=["rns>=1.3.5"], python_requires=">=3.7", ) From fab12ad9bf9f997797034950f289fe41a79dcf5a Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 1 Jun 2026 14:26:47 +0200 Subject: [PATCH 433/436] Updated build scripts --- Makefile | 5 +++++ setup.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/Makefile b/Makefile index 5fe5f96..cb7f141 100644 --- a/Makefile +++ b/Makefile @@ -26,5 +26,10 @@ build_spkg: remove_symlinks build_sdist create_symlinks release: remove_symlinks build_wheel build_spkg create_symlinks upload: + @echo Ready to publish release over Reticulum + @read VOID + rngit release rns://7649a50d84610232d1416b41d2896aff/reticulum/lxmf create $$(python setup.py --getversion):dist --name lxmf + +upload-pip: @echo Uploading to PyPi... twine upload dist/*.whl dist/*.tar.gz diff --git a/setup.py b/setup.py index beb1432..82521ec 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +import sys import setuptools with open("README.md", "r") as fh: @@ -5,6 +6,10 @@ with open("README.md", "r") as fh: exec(open("LXMF/_version.py", "r").read()) +if "--getversion" in sys.argv: + print(__version__, end="") + exit(0) + setuptools.setup( name="lxmf", version=__version__, From 4af5603d6268fa5c5b85c7bfe97c5ea69700724b Mon Sep 17 00:00:00 2001 From: jlpoole Date: Fri, 5 Jun 2026 17:54:19 -0700 Subject: [PATCH 434/436] Initial commit --- LICENSE | 9 +++++++++ README.md | 3 +++ 2 files changed, 12 insertions(+) create mode 100644 LICENSE create mode 100644 README.md diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1513e21 --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2026 jlpoole + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..ee643a4 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# LMXF-specification + +Based on Mark Qvist's LMXF, this project attempts to distill a specification from the Python code which is an implementation of the LMXF "protocol" \ No newline at end of file From d483d40d5cb994c6027802af1bf284dfeccc623b Mon Sep 17 00:00:00 2001 From: John Poole Date: Sat, 6 Jun 2026 09:41:48 -0700 Subject: [PATCH 435/436] After accomplishing Task 2, see Codex_Response... for details. --- .agents/skills/lxmf-update/SKILL.md | 40 ++++++ AGENTS.md | 16 +++ ChatGPT_1.md | 174 ++++++++++++++++++++++++++ Codex_1.md | 8 ++ Codex_Response_1.md | 83 +++++++++++++ Codex_Response_2.md | 83 +++++++++++++ Codex_Response_3.md | 27 +++++ PLANS.md | 13 ++ examples/lxmf_message_minimal.hex | 1 + examples/lxmf_message_stamped.hex | 1 + examples/lxmf_message_vectors.json | 26 ++++ grep_results.txt | 39 ++++++ tools/LXMF/MessagePack.pm | 181 ++++++++++++++++++++++++++++ tools/README.md | 32 +++++ tools/make_test_vectors.pl | 76 ++++++++++++ tools/verify_examples.pl | 88 ++++++++++++++ 16 files changed, 888 insertions(+) create mode 100644 .agents/skills/lxmf-update/SKILL.md create mode 100644 AGENTS.md create mode 100644 ChatGPT_1.md create mode 100644 Codex_1.md create mode 100644 Codex_Response_1.md create mode 100644 Codex_Response_2.md create mode 100644 Codex_Response_3.md create mode 100644 PLANS.md create mode 100644 examples/lxmf_message_minimal.hex create mode 100644 examples/lxmf_message_stamped.hex create mode 100644 examples/lxmf_message_vectors.json create mode 100644 grep_results.txt create mode 100644 tools/LXMF/MessagePack.pm create mode 100644 tools/README.md create mode 100755 tools/make_test_vectors.pl create mode 100755 tools/verify_examples.pl diff --git a/.agents/skills/lxmf-update/SKILL.md b/.agents/skills/lxmf-update/SKILL.md new file mode 100644 index 0000000..20396c3 --- /dev/null +++ b/.agents/skills/lxmf-update/SKILL.md @@ -0,0 +1,40 @@ +--- +name: lxmf-update +description: Check upstream LXMF and RNS implementations for changes affecting the LXMF specification, run local verifier tools, and propose spec updates with citations. +--- + +# LXMF update skill + +Use this skill when asked to check whether the LXMF specification is current with upstream LXMF or RNS. + +## Sources of truth + +Prefer, in order: + +1. Upstream LXMF source code at a pinned release or commit. +2. Upstream RNS source code where LXMF behavior depends on Reticulum behavior. +3. Reproducible local traces and test vectors. +4. Upstream prose documentation, only when consistent with source behavior. + +## Required workflow + +1. Determine the currently pinned upstream LXMF and RNS versions from `SOURCES.md`. +2. Check whether newer upstream releases or commits exist. +3. Identify changed files relevant to: + - message packing + - fields and flags + - destination handling + - identity/signature behavior + - encryption behavior + - propagation + - delivery semantics + - storage format if externally observable +4. Run the verifier tools in `tools/`. +5. Compare observed behavior against `SPEC.md`. +6. Produce a report with: + - changed upstream files + - affected specification sections + - proposed edits + - test-vector impact + - unresolved ambiguities +7. Do not modify `SPEC.md` unless explicitly asked. diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..d4c9224 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,16 @@ +# AGENTS.md + +## Repository purpose + +This repository attempts to define an implementation-derived specification for LXMF. + +Do not treat upstream prose documentation as authoritative when it conflicts with observed source behavior. Prefer source code, test vectors, and reproducible traces. + +## Required discipline + +- Distinguish normative behavior from observed implementation behavior. +- Cite exact upstream files, functions, classes, constants, and version tags. +- Do not invent protocol rules. +- When behavior is unclear, mark it as "undetermined" rather than guessing. +- Prefer Perl tools for extraction, comparison, and report generation. +- Keep generated examples reproducible from scripts under tools/. diff --git a/ChatGPT_1.md b/ChatGPT_1.md new file mode 100644 index 0000000..c40e197 --- /dev/null +++ b/ChatGPT_1.md @@ -0,0 +1,174 @@ +Yes. For Codex, the closest equivalents are: + +1. **`AGENTS.md`** for persistent project-wide instructions. +2. **Codex Skills using `SKILL.md`** for reusable workflows, very similar in spirit to the Claude Code file you linked. +3. Optionally **`PLANS.md` / ExecPlan-style documents** for long-running specification work. + +The important correction is that Codex does now support **Skills with `SKILL.md`**, not just `AGENTS.md`. OpenAI’s Codex documentation says a skill is a directory containing a required `SKILL.md` file, with optional `scripts/`, `references/`, `assets/`, and related support files. Codex initially sees the skill name, description, and path, then loads the full `SKILL.md` only when it decides that skill applies. ([OpenAI Developers][1]) + +So, yes: you can do essentially the same thing that `thatSFguy` is doing, but I would structure it slightly differently for LXMF. + +## Recommended Codex layout + +For an LXMF specification repository, I would use something like: + +```text +lxmf-specification/ +├── AGENTS.md +├── SPEC.md +├── NOTES.md +├── SOURCES.md +├── tools/ +│ ├── extract_lxmf_constants.pl +│ ├── compare_upstream_lxmf.pl +│ ├── verify_examples.pl +│ └── make_test_vectors.pl +├── examples/ +│ ├── lxmf_message_minimal.hex +│ ├── lxmf_message_signed.hex +│ └── lxmf_propagation_example.hex +├── references/ +│ ├── lxmf_source_map.md +│ ├── rns_dependency_map.md +│ └── terminology.md +└── .agents/ + └── skills/ + └── lxmf-update/ + └── SKILL.md +``` + +## Role of each file + +### `AGENTS.md` + +Use this for always-on repository rules. Codex reads `AGENTS.md` before doing work, and the docs describe it as the mechanism for project-specific persistent guidance. ([OpenAI Developers][2]) + +Example use: + +```markdown +# AGENTS.md + +## Repository purpose + +This repository attempts to define an implementation-derived specification for LXMF. + +Do not treat upstream prose documentation as authoritative when it conflicts with observed source behavior. Prefer source code, test vectors, and reproducible traces. + +## Required discipline + +- Distinguish normative behavior from observed implementation behavior. +- Cite exact upstream files, functions, classes, constants, and version tags. +- Do not invent protocol rules. +- When behavior is unclear, mark it as "undetermined" rather than guessing. +- Prefer Perl tools for extraction, comparison, and report generation. +- Keep generated examples reproducible from scripts under tools/. +``` + +### `.agents/skills/lxmf-update/SKILL.md` + +Use this for the repeatable workflow: “check upstream LXMF/RNS, compare against our spec, update citations, run verifiers, propose diffs.” + +That is the closest Codex analog to the Claude Code `rns-update/SKILL.md` you found. + +A minimal Codex skill might look like: + +```markdown +--- +name: lxmf-update +description: Check upstream LXMF and RNS implementations for changes affecting the LXMF specification, run local verifier tools, and propose spec updates with citations. +--- + +# LXMF update skill + +Use this skill when asked to check whether the LXMF specification is current with upstream LXMF or RNS. + +## Sources of truth + +Prefer, in order: + +1. Upstream LXMF source code at a pinned release or commit. +2. Upstream RNS source code where LXMF behavior depends on Reticulum behavior. +3. Reproducible local traces and test vectors. +4. Upstream prose documentation, only when consistent with source behavior. + +## Required workflow + +1. Determine the currently pinned upstream LXMF and RNS versions from `SOURCES.md`. +2. Check whether newer upstream releases or commits exist. +3. Identify changed files relevant to: + - message packing + - fields and flags + - destination handling + - identity/signature behavior + - encryption behavior + - propagation + - delivery semantics + - storage format if externally observable +4. Run the verifier tools in `tools/`. +5. Compare observed behavior against `SPEC.md`. +6. Produce a report with: + - changed upstream files + - affected specification sections + - proposed edits + - test-vector impact + - unresolved ambiguities +7. Do not modify `SPEC.md` unless explicitly asked. +``` + +Codex supports skills in the CLI, IDE extension, and Codex app, and the documented repository skill location is `.agents/skills` under the current directory or repository root. ([OpenAI Developers][1]) + +### `PLANS.md` + +For your use case, I would also add a planning document, because “derive a protocol specification from implementation” is not a one-shot coding task. OpenAI’s Codex cookbook describes using `PLANS.md` / ExecPlans for long-running, multi-stage work, with `AGENTS.md` telling Codex when to use it. ([OpenAI Developers][3]) + +For LXMF, `PLANS.md` could define the required sections of each investigation: + +```markdown +# LXMF Specification Investigation Plan + +Each investigation must include: + +1. Question being answered. +2. Upstream files inspected. +3. Exact version or commit. +4. Relevant classes/functions/constants. +5. Observed behavior. +6. Whether behavior is normative, inferred, or unresolved. +7. Test vectors needed. +8. SPEC.md sections affected. +9. Open questions. +``` + +## My recommendation + +For your goal, I would not start by asking Codex to “write the LXMF specification.” That is too broad and likely to produce confident prose ahead of evidence. + +I would instead give Codex narrow tasks such as: + +```text +Using the lxmf-update skill, inspect upstream LXMF and determine the exact serialized structure of an LXMessage. Do not edit SPEC.md. Produce a report listing every field, its source-code origin, and whether its behavior is confirmed by a test vector. +``` + +Then: + +```text +Create Perl tooling under tools/ that extracts or verifies the LXMF message packing behavior described in the report. Generate at least one reproducible test vector. +``` + +Then: + +```text +Update SPEC.md section "LXMessage Wire Format" using only behavior confirmed by source inspection or verifier output. Mark all unverified behavior as unresolved. +``` + +## Bottom line + +Yes, you can accomplish the same kind of workflow with Codex. In fact, Codex gives you both mechanisms: + +`AGENTS.md` for standing repo rules, and `SKILL.md` under `.agents/skills/...` for a reusable “LXMF update / spec maintenance” workflow. OpenAI’s Codex docs explicitly describe customization as a combination of `AGENTS.md`, skills, MCP, memories, and subagents; for your use case, `AGENTS.md` plus a `SKILL.md` skill is the right starting point. ([OpenAI Developers][4]) + +[1]: https://developers.openai.com/codex/skills "Agent Skills – Codex | OpenAI Developers" +[2]: https://developers.openai.com/codex/guides/agents-md "Custom instructions with AGENTS.md – Codex | OpenAI Developers" +[3]: https://developers.openai.com/cookbook/articles/codex_exec_plans "Using PLANS.md for multi-hour problem solving" +[4]: https://developers.openai.com/codex/concepts/customization "Customization – Codex | OpenAI Developers" + diff --git a/Codex_1.md b/Codex_1.md new file mode 100644 index 0000000..049676a --- /dev/null +++ b/Codex_1.md @@ -0,0 +1,8 @@ +This directory: /usr/local/src/reticulum/LXMF +contains my GitHub fork (https://github.com/jlpoolen/LXMF) of Mark Qvist's: https://github.com/markqvist/LXMF which, incidentally, is a public mirror of some git server served up through Reticulum -- the source is not provided. + +I created an empty project in my Forgejo server: LXMF-specification at https://salemdata.net/repo/jlpoole/LMXF-specification +and I added my Forgejo instance as a remote repository identified by "forgejo". + +I added some files based on a ChatGPT chat, see ChatGPT_1.md. +So, for your first task: Using the lxmf-update skill, inspect upstream LXMF and determine the exact serialized structure of an LXMessage. Do not edit SPEC.md. Produce a report listing every field, its source-code origin, and whether its behavior is confirmed by a test vector. diff --git a/Codex_Response_1.md b/Codex_Response_1.md new file mode 100644 index 0000000..8b45de7 --- /dev/null +++ b/Codex_Response_1.md @@ -0,0 +1,83 @@ +No, I don’t need `SPEC.md` or `SOURCES.md` for this first report. I used the checked-out LXMF source directly. I did not edit `SPEC.md` or any other file. + +**LXMessage Serialized Structure** + +Source inspected: LXMF commit `fab12ad9bf9f997797034950f289fe41a79dcf5a` + +Core `lxmf_bytes` layout: + +```text +destination_hash || source_hash || signature || msgpack_payload +``` + +Where `msgpack_payload` is: + +```text +[ + timestamp, + title, + content, + fields, + optional_stamp +] +``` + +The fifth element is only present when a normal message stamp is generated and appended. + +| Field | Serialized position | Type / size | Source origin | Test vector confirmed | +|---|---:|---|---|---| +| `destination_hash` | bytes `0..15` | 16 bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:383), length from [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:40) and RNS 128-bit truncated hashes in [Reticulum.py](/usr/local/src/reticulum/Reticulum/RNS/Reticulum.py:146) | No | +| `source_hash` | bytes `16..31` | 16 bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:384) | No | +| `signature` | bytes `32..95` | 64 bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:378), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:385); RNS signature length from [Identity.py](/usr/local/src/reticulum/Reticulum/RNS/Identity.py:59) and [Identity.py](/usr/local/src/reticulum/Reticulum/RNS/Identity.py:80) | No | +| `msgpack_payload` | bytes `96..end` | msgpack list | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362), packed at [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:381) | No | +| `timestamp` | payload index `0` | Python float, normally msgpack float64 on this RNS umsgpack | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:357), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362); float64 behavior from [umsgpack.py](/usr/local/src/reticulum/Reticulum/RNS/vendor/umsgpack.py:1177) and [umsgpack.py](/usr/local/src/reticulum/Reticulum/RNS/vendor/umsgpack.py:325) | No | +| `title` | payload index `1` | bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:130), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:193), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362) | No | +| `content` | payload index `2` | bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:135), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:202), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362) | No | +| `fields` | payload index `3` | dict, default `{}` | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:215), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362) | No | +| `stamp` | payload index `4`, optional | bytes, only if generated | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:371), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:373); extracted on unpack at [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:754) | No | + +**Hash And Signature** + +`message_id` / `hash` is: + +```text +SHA256(destination_hash || source_hash || msgpack([timestamp, title, content, fields])) +``` + +Source: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:364), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:368), RNS SHA-256 at [Identity.py](/usr/local/src/reticulum/Reticulum/RNS/Identity.py:238). + +The signature is over: + +```text +destination_hash || source_hash || msgpack([timestamp, title, content, fields]) || message_id +``` + +Source: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:375), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:378), RNS signing at [Identity.py](/usr/local/src/reticulum/Reticulum/RNS/Identity.py:767). + +Important: the optional normal `stamp` is excluded from both `message_id` and signature. Unpack removes payload index `4` before recomputing hash/signature material: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:754). + +**Outer Representations** + +Direct/link/resource delivery uses full `self.packed` unchanged: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:636), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:654). + +Opportunistic packet delivery omits the destination hash from the packet payload because the RNS packet destination already implies it: + +```text +source_hash || signature || msgpack_payload +``` + +Source: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:633). + +Propagated and paper messages wrap/encrypt everything after the destination hash: + +```text +destination_hash || encrypted(source_hash || signature || msgpack_payload) +``` + +Source: propagated [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:429), paper [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:449). Propagation transit additionally msgpacks `[time.time(), [lxmf_data]]`: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:436). + +**Notes** + +The README is not authoritative here: it describes payload order as timestamp, content, title, fields, but current code serializes timestamp, title, content, fields. + +No test vector confirmation exists in this checkout. There is no `tools/` directory and no tracked `SPEC.md` or `SOURCES.md`, so every “Test vector confirmed” entry above is marked `No`. diff --git a/Codex_Response_2.md b/Codex_Response_2.md new file mode 100644 index 0000000..8b45de7 --- /dev/null +++ b/Codex_Response_2.md @@ -0,0 +1,83 @@ +No, I don’t need `SPEC.md` or `SOURCES.md` for this first report. I used the checked-out LXMF source directly. I did not edit `SPEC.md` or any other file. + +**LXMessage Serialized Structure** + +Source inspected: LXMF commit `fab12ad9bf9f997797034950f289fe41a79dcf5a` + +Core `lxmf_bytes` layout: + +```text +destination_hash || source_hash || signature || msgpack_payload +``` + +Where `msgpack_payload` is: + +```text +[ + timestamp, + title, + content, + fields, + optional_stamp +] +``` + +The fifth element is only present when a normal message stamp is generated and appended. + +| Field | Serialized position | Type / size | Source origin | Test vector confirmed | +|---|---:|---|---|---| +| `destination_hash` | bytes `0..15` | 16 bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:383), length from [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:40) and RNS 128-bit truncated hashes in [Reticulum.py](/usr/local/src/reticulum/Reticulum/RNS/Reticulum.py:146) | No | +| `source_hash` | bytes `16..31` | 16 bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:384) | No | +| `signature` | bytes `32..95` | 64 bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:378), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:385); RNS signature length from [Identity.py](/usr/local/src/reticulum/Reticulum/RNS/Identity.py:59) and [Identity.py](/usr/local/src/reticulum/Reticulum/RNS/Identity.py:80) | No | +| `msgpack_payload` | bytes `96..end` | msgpack list | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362), packed at [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:381) | No | +| `timestamp` | payload index `0` | Python float, normally msgpack float64 on this RNS umsgpack | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:357), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362); float64 behavior from [umsgpack.py](/usr/local/src/reticulum/Reticulum/RNS/vendor/umsgpack.py:1177) and [umsgpack.py](/usr/local/src/reticulum/Reticulum/RNS/vendor/umsgpack.py:325) | No | +| `title` | payload index `1` | bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:130), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:193), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362) | No | +| `content` | payload index `2` | bytes | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:135), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:202), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362) | No | +| `fields` | payload index `3` | dict, default `{}` | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:215), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:362) | No | +| `stamp` | payload index `4`, optional | bytes, only if generated | [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:371), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:373); extracted on unpack at [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:754) | No | + +**Hash And Signature** + +`message_id` / `hash` is: + +```text +SHA256(destination_hash || source_hash || msgpack([timestamp, title, content, fields])) +``` + +Source: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:364), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:368), RNS SHA-256 at [Identity.py](/usr/local/src/reticulum/Reticulum/RNS/Identity.py:238). + +The signature is over: + +```text +destination_hash || source_hash || msgpack([timestamp, title, content, fields]) || message_id +``` + +Source: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:375), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:378), RNS signing at [Identity.py](/usr/local/src/reticulum/Reticulum/RNS/Identity.py:767). + +Important: the optional normal `stamp` is excluded from both `message_id` and signature. Unpack removes payload index `4` before recomputing hash/signature material: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:754). + +**Outer Representations** + +Direct/link/resource delivery uses full `self.packed` unchanged: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:636), [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:654). + +Opportunistic packet delivery omits the destination hash from the packet payload because the RNS packet destination already implies it: + +```text +source_hash || signature || msgpack_payload +``` + +Source: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:633). + +Propagated and paper messages wrap/encrypt everything after the destination hash: + +```text +destination_hash || encrypted(source_hash || signature || msgpack_payload) +``` + +Source: propagated [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:429), paper [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:449). Propagation transit additionally msgpacks `[time.time(), [lxmf_data]]`: [LXMessage.py](/usr/local/src/reticulum/LXMF/LXMF/LXMessage.py:436). + +**Notes** + +The README is not authoritative here: it describes payload order as timestamp, content, title, fields, but current code serializes timestamp, title, content, fields. + +No test vector confirmation exists in this checkout. There is no `tools/` directory and no tracked `SPEC.md` or `SOURCES.md`, so every “Test vector confirmed” entry above is marked `No`. diff --git a/Codex_Response_3.md b/Codex_Response_3.md new file mode 100644 index 0000000..6362cd8 --- /dev/null +++ b/Codex_Response_3.md @@ -0,0 +1,27 @@ +Task #2 completed in `/usr/local/src/reticulum/LXMF`. + +Created: + +- `tools/make_test_vectors.pl` +- `tools/verify_examples.pl` +- `tools/LXMF/MessagePack.pm` +- `tools/README.md` +- `examples/lxmf_message_minimal.hex` +- `examples/lxmf_message_stamped.hex` +- `examples/lxmf_message_vectors.json` + +The dependency-free Perl tooling verifies: + +- Fixed hash/signature positions +- Payload order and MessagePack types +- Message ID calculation +- Optional stamp exclusion from message ID and signature input + +Validation passed: + +```text +Reticulum umsgpack byte-for-byte cross-check passed +All LXMessage test-vector checks passed. +``` + +The vectors regenerate deterministically. Signature bytes are documented placeholders; Ed25519 validation is not claimed. `SPEC.md` was not created or modified. diff --git a/PLANS.md b/PLANS.md new file mode 100644 index 0000000..cbaedc0 --- /dev/null +++ b/PLANS.md @@ -0,0 +1,13 @@ +# LXMF Specification Investigation Plan + +Each investigation must include: + +1. Question being answered. +2. Upstream files inspected. +3. Exact version or commit. +4. Relevant classes/functions/constants. +5. Observed behavior. +6. Whether behavior is normative, inferred, or unresolved. +7. Test vectors needed. +8. SPEC.md sections affected. +9. Open questions. diff --git a/examples/lxmf_message_minimal.hex b/examples/lxmf_message_minimal.hex new file mode 100644 index 0000000..85f8d89 --- /dev/null +++ b/examples/lxmf_message_minimal.hex @@ -0,0 +1 @@ +000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f94cb41d954fc40100000c40a54657374207469746c65c41744657465726d696e6973746963204c584d4620626f647980 diff --git a/examples/lxmf_message_stamped.hex b/examples/lxmf_message_stamped.hex new file mode 100644 index 0000000..be6fc07 --- /dev/null +++ b/examples/lxmf_message_stamped.hex @@ -0,0 +1 @@ +000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f95cb41d954fc40100000c40a54657374207469746c65c41744657465726d696e6973746963204c584d4620626f647980c420a0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebf diff --git a/examples/lxmf_message_vectors.json b/examples/lxmf_message_vectors.json new file mode 100644 index 0000000..b60d155 --- /dev/null +++ b/examples/lxmf_message_vectors.json @@ -0,0 +1,26 @@ +{ + "content_hex" : "44657465726d696e6973746963204c584d4620626f6479", + "description" : "Deterministic LXMessage packing vectors; signature bytes are placeholders", + "destination_hash_hex" : "000102030405060708090a0b0c0d0e0f", + "fields" : {}, + "format" : "destination_hash || source_hash || signature || msgpack_payload", + "message_id_hex" : "55f497f6f2ea5d858eaff541a9fa77f9d5bdf57be75be4a94df65d1f9a7ffea3", + "minimal_lxmf_hex" : "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f94cb41d954fc40100000c40a54657374207469746c65c41744657465726d696e6973746963204c584d4620626f647980", + "payload_order" : [ + "timestamp", + "title", + "content", + "fields", + "optional_stamp" + ], + "payload_with_stamp_hex" : "95cb41d954fc40100000c40a54657374207469746c65c41744657465726d696e6973746963204c584d4620626f647980c420a0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebf", + "payload_without_stamp_hex" : "94cb41d954fc40100000c40a54657374207469746c65c41744657465726d696e6973746963204c584d4620626f647980", + "signature_hex" : "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f", + "signature_valid" : false, + "signed_part_hex" : "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f94cb41d954fc40100000c40a54657374207469746c65c41744657465726d696e6973746963204c584d4620626f64798055f497f6f2ea5d858eaff541a9fa77f9d5bdf57be75be4a94df65d1f9a7ffea3", + "source_hash_hex" : "101112131415161718191a1b1c1d1e1f", + "stamp_hex" : "a0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebf", + "stamped_lxmf_hex" : "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f95cb41d954fc40100000c40a54657374207469746c65c41744657465726d696e6973746963204c584d4620626f647980c420a0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebf", + "timestamp" : 1700000000.25, + "title_hex" : "54657374207469746c65" +} diff --git a/grep_results.txt b/grep_results.txt new file mode 100644 index 0000000..52ae092 --- /dev/null +++ b/grep_results.txt @@ -0,0 +1,39 @@ +./LXMF/Utilities/lxmd.py:DEFFERED_JOBS_DELAY = 10 +./LXMF/Utilities/lxmd.py:def create_default_config(configpath): +./LXMF/Utilities/lxmd.py:def apply_config(): +./LXMF/Utilities/lxmd.py:def lxmf_delivery(lxm): +./LXMF/Utilities/lxmd.py:def program_setup(configdir = None, rnsconfigdir = None, run_pn = False, on_inbound = None, verbosity = 0, quietness = 0, service = False): +./LXMF/Utilities/lxmd.py:def jobs(): +./LXMF/Utilities/lxmd.py:def deferred_start_jobs(): +./LXMF/Utilities/lxmd.py:def _request_sync(identity, destination_hash, remote_identity, timeout=15, exit_on_fail=False): +./LXMF/Utilities/lxmd.py:def request_sync(target, remote=None, configdir=None, rnsconfigdir=None, verbosity=0, quietness=0, timeout=15, identity_path=None): +./LXMF/Utilities/lxmd.py:def _request_unpeer(identity, destination_hash, remote_identity, timeout=15, exit_on_fail=False): +./LXMF/Utilities/lxmd.py:def request_unpeer(target, remote=None, configdir=None, rnsconfigdir=None, verbosity=0, quietness=0, timeout=15, identity_path=None): +./LXMF/Utilities/lxmd.py:def query_status(identity, remote_identity=None, timeout=5, exit_on_fail=False): +./LXMF/Utilities/lxmd.py:def get_status(remote=None, configdir=None, rnsconfigdir=None, verbosity=0, quietness=0, timeout=5, +./LXMF/Utilities/lxmd.py:def _get_target_identity(remote=None, timeout=5): +./LXMF/Utilities/lxmd.py:def _remote_init(configdir=None, rnsconfigdir=None, verbosity=0, quietness=0, identity_path=None): +./LXMF/Utilities/lxmd.py:def main(): +./LXMF/LXMF.py:def display_name_from_app_data(app_data=None): +./LXMF/LXMF.py:def stamp_cost_from_app_data(app_data=None): +./LXMF/LXMF.py:def compression_support_from_app_data(app_data=None): +./LXMF/LXMF.py:def pn_name_from_app_data(app_data=None): +./LXMF/LXMF.py:def pn_stamp_cost_from_app_data(app_data=None): +./LXMF/LXMF.py:def pn_announce_data_is_valid(data): +./LXMF/LXStamper.py:def worker_context(ctx, stamp_cost, workblock, message_id): +./LXMF/LXStamper.py:def stamp_workblock(material, expand_rounds=WORKBLOCK_EXPAND_ROUNDS): +./LXMF/LXStamper.py:def stamp_value(workblock, stamp): +./LXMF/LXStamper.py:def stamp_valid(stamp, target_cost, workblock): +./LXMF/LXStamper.py:def validate_peering_key(peering_id, peering_key, target_cost): +./LXMF/LXStamper.py:def validate_pn_stamp(transient_data, target_cost): +./LXMF/LXStamper.py:def validate_pn_stamps_job_simple(transient_list, target_cost): +./LXMF/LXStamper.py:def validate_pn_stamps_job_multip(transient_list, target_cost): +./LXMF/LXStamper.py:def validate_pn_stamps(transient_list, target_cost): +./LXMF/LXStamper.py:def generate_stamp(message_id, stamp_cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS): +./LXMF/LXStamper.py:def cancel_work(message_id): +./LXMF/LXStamper.py:def job_simple(stamp_cost, workblock, message_id): +./LXMF/LXStamper.py:def job_linux_managed(stamp_cost, workblock, message_id): +./LXMF/LXStamper.py:def job_linux(stamp_cost, workblock, message_id): +./LXMF/LXStamper.py:def job_android(stamp_cost, workblock, message_id): +./docs/example_receiver.py:def delivery_callback(message): +./docs/example_sender.py:def delivery_callback(message): diff --git a/tools/LXMF/MessagePack.pm b/tools/LXMF/MessagePack.pm new file mode 100644 index 0000000..5ce8c39 --- /dev/null +++ b/tools/LXMF/MessagePack.pm @@ -0,0 +1,181 @@ +package LXMF::MessagePack; + +use strict; +use warnings; +use Exporter 'import'; + +our @EXPORT_OK = qw( + pack_array + pack_bin + pack_float64 + pack_map + parse_item +); + +sub pack_float64 { + my ($value) = @_; + return "\xcb" . pack("d>", $value); +} + +sub pack_bin { + my ($value) = @_; + my $length = length($value); + + return "\xc4" . pack("C", $length) . $value if $length <= 0xff; + return "\xc5" . pack("n", $length) . $value if $length <= 0xffff; + return "\xc6" . pack("N", $length) . $value; +} + +sub pack_array { + my (@items) = @_; + my $length = scalar @items; + my $header; + + if ($length <= 15) { + $header = pack("C", 0x90 | $length); + } + elsif ($length <= 0xffff) { + $header = "\xdc" . pack("n", $length); + } + else { + $header = "\xdd" . pack("N", $length); + } + + return $header . join("", @items); +} + +sub pack_map { + my (@entries) = @_; + die "pack_map requires key/value byte-string pairs\n" if @entries % 2; + + my $length = @entries / 2; + my $header; + + if ($length <= 15) { + $header = pack("C", 0x80 | $length); + } + elsif ($length <= 0xffff) { + $header = "\xde" . pack("n", $length); + } + else { + $header = "\xdf" . pack("N", $length); + } + + return $header . join("", @entries); +} + +sub _take { + my ($bytes, $offset_ref, $length) = @_; + die "truncated MessagePack item\n" if $$offset_ref + $length > length($bytes); + my $value = substr($bytes, $$offset_ref, $length); + $$offset_ref += $length; + return $value; +} + +sub _parse_length { + my ($bytes, $offset_ref, $width) = @_; + my $raw = _take($bytes, $offset_ref, $width); + return unpack("C", $raw) if $width == 1; + return unpack("n", $raw) if $width == 2; + return unpack("N", $raw); +} + +sub parse_item { + my ($bytes, $offset_ref) = @_; + $offset_ref //= \my $offset; + $$offset_ref //= 0; + + my $start = $$offset_ref; + my $code = unpack("C", _take($bytes, $offset_ref, 1)); + my $item = { code => $code }; + + if ($code <= 0x7f) { + @$item{qw(type value)} = ("integer", $code); + } + elsif ($code >= 0xe0) { + @$item{qw(type value)} = ("integer", $code - 256); + } + elsif (($code & 0xf0) == 0x90) { + _parse_array($bytes, $offset_ref, $item, $code & 0x0f); + } + elsif (($code & 0xf0) == 0x80) { + _parse_map($bytes, $offset_ref, $item, $code & 0x0f); + } + elsif (($code & 0xe0) == 0xa0) { + my $length = $code & 0x1f; + @$item{qw(type value)} = ("string", _take($bytes, $offset_ref, $length)); + } + elsif ($code == 0xc0) { + $item->{type} = "nil"; + $item->{value} = undef; + } + elsif ($code == 0xc2 || $code == 0xc3) { + @$item{qw(type value)} = ("boolean", $code == 0xc3 ? 1 : 0); + } + elsif ($code == 0xca) { + @$item{qw(type value)} = ("float32", unpack("f>", _take($bytes, $offset_ref, 4))); + } + elsif ($code == 0xcb) { + @$item{qw(type value)} = ("float64", unpack("d>", _take($bytes, $offset_ref, 8))); + } + elsif ($code >= 0xc4 && $code <= 0xc6) { + my $width = 1 << ($code - 0xc4); + my $length = _parse_length($bytes, $offset_ref, $width); + @$item{qw(type value)} = ("binary", _take($bytes, $offset_ref, $length)); + } + elsif ($code >= 0xcc && $code <= 0xcf) { + my @formats = ("C", "n", "N", "Q>"); + my @widths = (1, 2, 4, 8); + my $index = $code - 0xcc; + @$item{qw(type value)} = ( + "integer", + unpack($formats[$index], _take($bytes, $offset_ref, $widths[$index])), + ); + } + elsif ($code >= 0xd0 && $code <= 0xd3) { + my @formats = ("c", "s>", "l>", "q>"); + my @widths = (1, 2, 4, 8); + my $index = $code - 0xd0; + @$item{qw(type value)} = ( + "integer", + unpack($formats[$index], _take($bytes, $offset_ref, $widths[$index])), + ); + } + elsif ($code == 0xd9 || $code == 0xda || $code == 0xdb) { + my $width = 1 << ($code - 0xd9); + my $length = _parse_length($bytes, $offset_ref, $width); + @$item{qw(type value)} = ("string", _take($bytes, $offset_ref, $length)); + } + elsif ($code == 0xdc || $code == 0xdd) { + my $width = $code == 0xdc ? 2 : 4; + _parse_array($bytes, $offset_ref, $item, _parse_length($bytes, $offset_ref, $width)); + } + elsif ($code == 0xde || $code == 0xdf) { + my $width = $code == 0xde ? 2 : 4; + _parse_map($bytes, $offset_ref, $item, _parse_length($bytes, $offset_ref, $width)); + } + else { + die sprintf("unsupported MessagePack code 0x%02x at offset %d\n", $code, $start); + } + + $item->{start} = $start; + $item->{end} = $$offset_ref; + $item->{raw} = substr($bytes, $start, $$offset_ref - $start); + return $item; +} + +sub _parse_array { + my ($bytes, $offset_ref, $item, $length) = @_; + $item->{type} = "array"; + $item->{items} = [map { parse_item($bytes, $offset_ref) } 1 .. $length]; +} + +sub _parse_map { + my ($bytes, $offset_ref, $item, $length) = @_; + $item->{type} = "map"; + $item->{entries} = [ + map { [parse_item($bytes, $offset_ref), parse_item($bytes, $offset_ref)] } 1 .. $length + ]; +} + +1; diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 0000000..75538fa --- /dev/null +++ b/tools/README.md @@ -0,0 +1,32 @@ +# LXMessage Test-Vector Tooling + +The scripts in this directory verify the implementation-derived LXMessage +packing behavior described in `Code_Response_1.md`. + +They use only Perl core modules. The local `LXMF::MessagePack` module implements +the MessagePack subset needed to generate the vectors and parse their complete +payloads. + +Generate deterministic vectors: + +```sh +perl tools/make_test_vectors.pl +``` + +Verify the vectors: + +```sh +perl tools/verify_examples.pl +``` + +The vectors confirm: + +- the 16-byte destination hash, 16-byte source hash, and 64-byte signature + positions; +- payload order `[timestamp, title, content, fields, optional_stamp]`; +- float64 timestamp and binary title/content/stamp encodings; +- the message ID calculation; +- exclusion of the optional stamp from the message ID and signature input. + +The signature bytes are deterministic placeholders. These vectors do not claim +to verify Ed25519 signing or signature validation. diff --git a/tools/make_test_vectors.pl b/tools/make_test_vectors.pl new file mode 100755 index 0000000..8d4385b --- /dev/null +++ b/tools/make_test_vectors.pl @@ -0,0 +1,76 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use Digest::SHA qw(sha256); +use FindBin qw($Bin); +use File::Path qw(make_path); +use JSON::PP; +use lib "$Bin"; +use LXMF::MessagePack qw(pack_array pack_bin pack_float64 pack_map); + +my $output_dir = "$Bin/../examples"; +make_path($output_dir); + +my $destination_hash = pack("C*", 0x00 .. 0x0f); +my $source_hash = pack("C*", 0x10 .. 0x1f); +my $signature = pack("C*", 0x20 .. 0x5f); +my $stamp = pack("C*", 0xa0 .. 0xbf); + +my $timestamp = 1_700_000_000.25; +my $title = "Test title"; +my $content = "Deterministic LXMF body"; +my $fields = pack_map(); + +my @required_items = ( + pack_float64($timestamp), + pack_bin($title), + pack_bin($content), + $fields, +); + +my $payload = pack_array(@required_items); +my $stamped_payload = pack_array(@required_items, pack_bin($stamp)); +my $message_id = sha256($destination_hash . $source_hash . $payload); +my $signed_part = $destination_hash . $source_hash . $payload . $message_id; + +my $minimal = $destination_hash . $source_hash . $signature . $payload; +my $stamped = $destination_hash . $source_hash . $signature . $stamped_payload; + +write_hex("$output_dir/lxmf_message_minimal.hex", $minimal); +write_hex("$output_dir/lxmf_message_stamped.hex", $stamped); + +my $manifest = { + description => "Deterministic LXMessage packing vectors; signature bytes are placeholders", + format => "destination_hash || source_hash || signature || msgpack_payload", + payload_order => [qw(timestamp title content fields optional_stamp)], + destination_hash_hex => unpack("H*", $destination_hash), + source_hash_hex => unpack("H*", $source_hash), + signature_hex => unpack("H*", $signature), + signature_valid => JSON::PP::false, + timestamp => $timestamp, + title_hex => unpack("H*", $title), + content_hex => unpack("H*", $content), + fields => {}, + stamp_hex => unpack("H*", $stamp), + payload_without_stamp_hex => unpack("H*", $payload), + payload_with_stamp_hex => unpack("H*", $stamped_payload), + message_id_hex => unpack("H*", $message_id), + signed_part_hex => unpack("H*", $signed_part), + minimal_lxmf_hex => unpack("H*", $minimal), + stamped_lxmf_hex => unpack("H*", $stamped), +}; + +open my $json_fh, ">:raw", "$output_dir/lxmf_message_vectors.json" + or die "cannot write vector manifest: $!\n"; +print {$json_fh} JSON::PP->new->canonical->pretty->encode($manifest); +close $json_fh; + +print "Generated deterministic LXMessage vectors in $output_dir\n"; + +sub write_hex { + my ($path, $bytes) = @_; + open my $fh, ">:raw", $path or die "cannot write $path: $!\n"; + print {$fh} unpack("H*", $bytes), "\n"; + close $fh; +} diff --git a/tools/verify_examples.pl b/tools/verify_examples.pl new file mode 100755 index 0000000..0f86e67 --- /dev/null +++ b/tools/verify_examples.pl @@ -0,0 +1,88 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use Digest::SHA qw(sha256); +use FindBin qw($Bin); +use JSON::PP; +use lib "$Bin"; +use LXMF::MessagePack qw(pack_array parse_item); + +my $examples_dir = "$Bin/../examples"; +my $manifest = read_json("$examples_dir/lxmf_message_vectors.json"); +my $minimal = read_hex("$examples_dir/lxmf_message_minimal.hex"); +my $stamped = read_hex("$examples_dir/lxmf_message_stamped.hex"); + +verify_lxmessage("minimal", $minimal, $manifest, 4); +verify_lxmessage("stamped", $stamped, $manifest, 5); + +check(unpack("H*", $minimal) eq $manifest->{minimal_lxmf_hex}, "minimal full bytes match manifest"); +check(unpack("H*", $stamped) eq $manifest->{stamped_lxmf_hex}, "stamped full bytes match manifest"); + +print "All LXMessage test-vector checks passed.\n"; + +sub verify_lxmessage { + my ($name, $bytes, $expected, $expected_count) = @_; + + check(length($bytes) > 96, "$name vector has fixed prefix and payload"); + my $destination_hash = substr($bytes, 0, 16); + my $source_hash = substr($bytes, 16, 16); + my $signature = substr($bytes, 32, 64); + my $packed_payload = substr($bytes, 96); + + check(unpack("H*", $destination_hash) eq $expected->{destination_hash_hex}, "$name destination hash"); + check(unpack("H*", $source_hash) eq $expected->{source_hash_hex}, "$name source hash"); + check(unpack("H*", $signature) eq $expected->{signature_hex}, "$name signature position and length"); + + my $offset = 0; + my $payload = parse_item($packed_payload, \$offset); + check($offset == length($packed_payload), "$name payload consumes all remaining bytes"); + check($payload->{type} eq "array", "$name payload is a MessagePack array"); + check(@{$payload->{items}} == $expected_count, "$name payload has $expected_count entries"); + + my @items = @{$payload->{items}}; + check($items[0]{type} eq "float64", "$name payload[0] timestamp is float64"); + check($items[0]{value} == $expected->{timestamp}, "$name payload[0] timestamp value"); + check($items[1]{type} eq "binary", "$name payload[1] title is binary"); + check(unpack("H*", $items[1]{value}) eq $expected->{title_hex}, "$name payload[1] title value"); + check($items[2]{type} eq "binary", "$name payload[2] content is binary"); + check(unpack("H*", $items[2]{value}) eq $expected->{content_hex}, "$name payload[2] content value"); + check($items[3]{type} eq "map", "$name payload[3] fields is a map"); + check(@{$items[3]{entries}} == 0, "$name payload[3] fields map is empty"); + + my $unstamped_payload = pack_array(map { $_->{raw} } @items[0 .. 3]); + my $message_id = sha256($destination_hash . $source_hash . $unstamped_payload); + my $signed_part = $destination_hash . $source_hash . $unstamped_payload . $message_id; + + check(unpack("H*", $unstamped_payload) eq $expected->{payload_without_stamp_hex}, "$name canonical unstamped payload"); + check(unpack("H*", $message_id) eq $expected->{message_id_hex}, "$name message ID excludes optional stamp"); + check(unpack("H*", $signed_part) eq $expected->{signed_part_hex}, "$name signature input excludes optional stamp"); + + if ($expected_count == 5) { + check($items[4]{type} eq "binary", "$name payload[4] stamp is binary"); + check(unpack("H*", $items[4]{value}) eq $expected->{stamp_hex}, "$name payload[4] stamp value"); + } +} + +sub read_json { + my ($path) = @_; + open my $fh, "<:raw", $path or die "cannot read $path: $!\n"; + local $/; + return JSON::PP->new->decode(<$fh>); +} + +sub read_hex { + my ($path) = @_; + open my $fh, "<:raw", $path or die "cannot read $path: $!\n"; + local $/; + my $hex = <$fh>; + $hex =~ s/\s+//g; + die "$path contains non-hexadecimal data\n" if $hex =~ /[^0-9a-fA-F]/; + return pack("H*", $hex); +} + +sub check { + my ($condition, $description) = @_; + die "not ok - $description\n" unless $condition; + print "ok - $description\n"; +} From 18c19f95b341fc388a401bb91a768dac7a1ab868 Mon Sep 17 00:00:00 2001 From: John Poole Date: Sat, 6 Jun 2026 09:47:20 -0700 Subject: [PATCH 436/436] After completing Task 3 of initial 3 task punch list. --- Codex_Response_4.md | 14 ++++ SPEC.md | 153 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 Codex_Response_4.md create mode 100644 SPEC.md diff --git a/Codex_Response_4.md b/Codex_Response_4.md new file mode 100644 index 0000000..c1d9211 --- /dev/null +++ b/Codex_Response_4.md @@ -0,0 +1,14 @@ +Created [SPEC.md](/usr/local/src/reticulum/LXMF/SPEC.md) containing the evidence-limited `LXMessage Wire Format` section. + +It documents: + +- Core serialized layout and payload order +- Message ID and signature-input calculations +- Stamp exclusion behavior +- Delivery representations +- Exact source references and evidence commits +- Explicit unresolved behavior + +Verification: `perl tools/verify_examples.pl` passed all checks. + +Only `SPEC.md` is untracked; no other files were changed. diff --git a/SPEC.md b/SPEC.md new file mode 100644 index 0000000..a6dcc8e --- /dev/null +++ b/SPEC.md @@ -0,0 +1,153 @@ +# LXMF Implementation-Derived Specification + +## LXMessage Wire Format + +This section describes behavior confirmed in the checked-in LXMF implementation +and by the reproducible vectors under `examples/`. It does not assign normative +requirements to independent implementations. + +Evidence baseline: + +- LXMF commit `d483d40d5cb994c6027802af1bf284dfeccc623b` +- Reticulum commit `2646f673ee061135d7c351ef44dce290ccd7e06e` + +### Evidence status + +- **Source-confirmed** means the behavior is directly implemented by the + checked-in LXMF or Reticulum source. +- **Vector-confirmed** means `perl tools/verify_examples.pl` confirms the + behavior for the deterministic minimal and stamped vectors. +- Behavior not covered by either form of evidence is marked **Unresolved**. + +### Core serialized message + +`LXMessage.pack()` constructs `lxmf_bytes` by concatenating: + +```text +destination_hash || source_hash || signature || packed_payload +``` + +| Part | Position | Confirmed representation | Evidence | +|---|---:|---|---| +| `destination_hash` | bytes `0..15` | 16 bytes | Source-confirmed: `LXMF/LXMessage.py:40`, `LXMF/LXMessage.py:383`; Vector-confirmed | +| `source_hash` | bytes `16..31` | 16 bytes | Source-confirmed: `LXMF/LXMessage.py:384`; Vector-confirmed | +| `signature` | bytes `32..95` | 64 bytes | Source-confirmed: `LXMF/LXMessage.py:41`, `LXMF/LXMessage.py:385`; Vector-confirmed position and length only | +| `packed_payload` | bytes `96..end` | MessagePack array | Source-confirmed: `LXMF/LXMessage.py:362`, `LXMF/LXMessage.py:381-386`; Vector-confirmed | + +The 16-byte hash length derives from +`RNS.Identity.TRUNCATED_HASHLENGTH // 8`. The checked-in Reticulum source sets +the truncated hash length to 128 bits. The 64-byte signature length derives +from `RNS.Identity.SIGLENGTH // 8`. See `RNS/Reticulum.py:146-147` and +`RNS/Identity.py:80-83` in the evidence-baseline Reticulum checkout. + +### Payload + +For messages generated by `LXMessage.pack()`, the payload is: + +```text +[ + timestamp, + title, + content, + fields +] +``` + +If a message stamp is available when packing, it is appended: + +```text +[ + timestamp, + title, + content, + fields, + stamp +] +``` + +| Index | Name | Confirmed behavior | Evidence | +|---:|---|---|---| +| `0` | `timestamp` | Set from `time.time()` when not already set, then packed as the first payload value | Source-confirmed: `LXMF/LXMessage.py:357`, `LXMF/LXMessage.py:362`; Vector-confirmed as MessagePack float64 for the deterministic vectors | +| `1` | `title` | Constructor string input is UTF-8 encoded to bytes; byte input is retained | Source-confirmed: `LXMF/LXMessage.py:130-133`, `LXMF/LXMessage.py:193-197`; Vector-confirmed as MessagePack binary | +| `2` | `content` | Constructor string input is UTF-8 encoded to bytes; byte input is retained | Source-confirmed: `LXMF/LXMessage.py:135-136`, `LXMF/LXMessage.py:202-206`; Vector-confirmed as MessagePack binary | +| `3` | `fields` | Constructor input must be a dictionary or `None`; `None` becomes an empty dictionary | Source-confirmed: `LXMF/LXMessage.py:138`, `LXMF/LXMessage.py:215-219`; Vector-confirmed for an empty MessagePack map | +| `4` | `stamp` | Optional value appended after the four base payload values | Source-confirmed: `LXMF/LXMessage.py:371-373`; Vector-confirmed as MessagePack binary in the stamped vector | + +The payload order above is source- and vector-confirmed. It differs from the +order stated in the upstream `README.md`, which lists content before title. + +### Message ID + +The message ID, also stored as `LXMessage.hash`, is: + +```text +SHA-256( + destination_hash || + source_hash || + msgpack([timestamp, title, content, fields]) +) +``` + +The optional stamp is excluded from the MessagePack payload used to calculate +the message ID. This is source-confirmed by both packing and unpacking behavior, +and vector-confirmed by the minimal and stamped vectors producing the same +message ID. See `LXMF/LXMessage.py:362-369` and +`LXMF/LXMessage.py:754-764`. Reticulum defines `full_hash()` as SHA-256 at +`RNS/Identity.py:238-246`. + +The message ID is not included in `lxmf_bytes`. + +### Signature input + +`LXMessage.pack()` requests a signature over: + +```text +destination_hash || +source_hash || +msgpack([timestamp, title, content, fields]) || +message_id +``` + +The optional stamp is excluded from the signature input. The construction of +the signature input is source-confirmed at `LXMF/LXMessage.py:375-378` and +`LXMF/LXMessage.py:762-764`, and vector-confirmed. + +The deterministic vectors contain placeholder signature bytes. They confirm +the signature position and input bytes, but do not confirm signing or signature +validation. + +### Delivery representations + +The core `lxmf_bytes` representation is modified or wrapped for some delivery +methods: + +| Delivery representation | Confirmed serialized data | Evidence | +|---|---|---| +| Direct packet or resource | Full `lxmf_bytes` | Source-confirmed: `LXMF/LXMessage.py:635-636`, `LXMF/LXMessage.py:653-654` | +| Opportunistic packet | `lxmf_bytes` without the leading destination hash | Source-confirmed: `LXMF/LXMessage.py:633-634` | +| Propagated message data | `destination_hash || encrypt(lxmf_bytes after destination_hash)` | Source-confirmed: `LXMF/LXMessage.py:429-436` | +| Propagation transfer wrapper | MessagePack `[time.time(), [propagated_message_data]]` | Source-confirmed: `LXMF/LXMessage.py:436` | +| Paper message data | `destination_hash || encrypt(lxmf_bytes after destination_hash)` | Source-confirmed: `LXMF/LXMessage.py:446-451` | +| Paper URI | URL-safe Base64 of paper message data, without `=` padding, prefixed by `lxm://` | Source-confirmed: `LXMF/LXMessage.py:698-707` | + +These delivery representations are not covered by the current deterministic +test vectors. + +### Unresolved behavior + +The following behavior is not established by the current source inspection and +test-vector coverage: + +- Normative requirements for independent LXMF implementations. +- Cross-implementation Ed25519 signature generation and validation. +- Accepted signature encodings beyond the fixed 64-byte position generated by + the checked-in implementation. +- A universal MessagePack float width for timestamps on every supported + platform. The deterministic vectors confirm float64 only for those vectors. +- Interoperable constraints on the contents, key types, value types, ordering, + and nesting depth of `fields`. +- Interoperable constraints on stamp length and encoding. +- Required handling of malformed, truncated, non-canonical, or payload arrays + containing fewer than four or more than five entries. +- Byte-for-byte vectors for opportunistic, propagated, paper, URI, encrypted, + or persisted-container representations.