After various memory exercises, then back to GPS to get UBlox working. UBlox is working. TODO: start libraries, downplay SD Card as we can use memory for interim logging

exercises/17_Flash/README.md

@@ -0,0 +1,32 @@
+# Exercise 17_Flash
+
+This exercise demonstrates using Flash storage as a persistent directory-like file system on an ESP32-S3 board.
+
+Behavior:
+- Mounts SPIFFS at boot and reports total / used / free flash space.
+- Ensures a flash directory at `/flash_logs` exists.
+- Creates a new log file when the device boots, based on the current timestamp: `YYYYMMDD_HHMM.log`.
+- Writes a timestamped line into the new log file once per second.
+- Supports console commands to inspect the current file, read it, clear it, append or rewrite it, and list stored files.
+- Files persist across reboots and are stored in flash.
+
+Build and upload:
+
+```bash
+cd /usr/local/src/microreticulum/microReticulumTbeam/exercises/17_Flash
+pio run -e amy -t upload
+pio device monitor -b 115200
+```
+
+Commands:
+- `help` - show command menu
+- `stat` - show flash / current file status
+- `list` - list files under `/flash_logs`
+- `read` - read the current flash file contents
+- `clear` - clear the current flash file contents
+- `write <text>` - overwrite the current flash file with text
+- `append <text>` - append text to the current flash file
+
+Notes:
+- If the current timestamp file name already exists, the exercise will append a numeric suffix to keep the file unique.
+- On each reboot a new file is created so persistent flash logs accumulate.

exercises/17_Flash/platformio.ini

@@ -0,0 +1,50 @@
+; 20260403 ChatGPT
+; Exercise 17_Flash
+
+[platformio]
+default_envs = amy
+
+[env]
+platform = espressif32
+framework = arduino
+board = esp32-s3-devkitc-1
+monitor_speed = 115200
+extra_scripts = pre:scripts/set_build_epoch.py
+lib_deps =
+  Wire
+  olikraus/U8g2@^2.36.4
+  lewisxhe/XPowersLib@0.3.3
+
+build_flags =
+  -I ../../shared/boards
+  -I ../../external/microReticulum_Firmware
+  -D BOARD_MODEL=BOARD_TBEAM_S_V1
+  -D OLED_SDA=17
+  -D OLED_SCL=18
+  -D OLED_ADDR=0x3C
+  -D ARDUINO_USB_MODE=1
+  -D ARDUINO_USB_CDC_ON_BOOT=1
+
+[env:amy]
+extends = env
+build_flags =
+  ${env.build_flags}
+  -D NODE_LABEL=\"AMY\"
+
+[env:bob]
+extends = env
+build_flags =
+  ${env.build_flags}
+  -D NODE_LABEL=\"BOB\"
+
+[env:cy]
+extends = env
+build_flags =
+  ${env.build_flags}
+  -D NODE_LABEL=\"CY\"
+
+[env:dan]
+extends = env
+build_flags =
+  ${env.build_flags}
+  -D NODE_LABEL=\"DAN\"

exercises/17_Flash/read_partition_bin.py

@@ -0,0 +1,11 @@
+import struct
+with open('AMY_test_partitions_read.bin', 'rb') as f:
+    data = f.read()
+    seq0 = struct.unpack('<I', data[0:4])[0]
+    seq1 = struct.unpack('<I', data[32:36])[0]
+    print(f"OTA seq0: {seq0:08x}")
+    print(f"OTA seq1: {seq1:08x}")
+    if seq0 > seq1:
+        print("→ app0 is active, new uploads go to app1")
+    else:
+        print("→ app1 is active, new uploads go to app0")

exercises/17_Flash/scripts/set_build_epoch.py

@@ -0,0 +1,12 @@
+import time
+Import("env")
+
+epoch = int(time.time())
+utc_tag = time.strftime("%Y%m%d_%H%M%S_z", time.gmtime(epoch))
+
+env.Append(
+    CPPDEFINES=[
+        ("FW_BUILD_EPOCH", str(epoch)),
+        ("FW_BUILD_UTC", '\"%s\"' % utc_tag),
+    ]
+)

exercises/17_Flash/show_partition_table.py

@@ -0,0 +1,26 @@
+import struct
+
+with open('partitions_backup.bin', 'rb') as f:
+    data = f.read()
+
+print("Name          | Type | SubType | Offset      | Size        | Flags")
+print("-" * 75)
+
+for i in range(0, len(data), 32):
+    entry = data[i:i+32]
+    if len(entry) < 32:
+        break
+    
+    magic = struct.unpack('<H', entry[0:2])[0]
+    
+    if magic == 0x50aa:  # Valid partition magic
+        type_val = entry[2]
+        subtype = entry[3]
+        offset = struct.unpack('<I', entry[4:8])[0]
+        size = struct.unpack('<I', entry[8:12])[0]
+        flags = struct.unpack('<H', entry[12:14])[0]
+        name = entry[16:32].rstrip(b'\x00').decode('ascii', errors='ignore')
+        
+        print(f"{name:<13} | {type_val:02x}   | {subtype:02x}      | 0x{offset:08x}  | 0x{size:08x}  | {flags:04x}")
+    elif magic == 0xebeb:
+        break  # End marker

exercises/17_Flash/src/main.cpp

@@ -0,0 +1,608 @@
+// 20260403 ChatGPT
+// Exercise 17_Flash
+
+#include <Arduino.h>
+#include <Wire.h>
+#include <U8g2lib.h>
+#include <time.h>
+#include <SPIFFS.h>
+
+#include "tbeam_supreme_adapter.h"
+
+#ifndef NODE_LABEL
+#define NODE_LABEL "FLASH"
+#endif
+
+#ifndef RTC_I2C_ADDR
+#define RTC_I2C_ADDR 0x51
+#endif
+
+#ifndef OLED_SDA
+#define OLED_SDA 17
+#endif
+#ifndef OLED_SCL
+#define OLED_SCL 18
+#endif
+#ifndef OLED_ADDR
+#define OLED_ADDR 0x3C
+#endif
+
+static U8G2_SH1106_128X64_NONAME_F_HW_I2C g_oled(U8G2_R0, U8X8_PIN_NONE);
+static const char *kFlashDir = "/flash_logs";
+static char g_currentFilePath[64] = {0};
+static File g_flashFile;
+static unsigned g_flashLineNumber = 0;
+static XPowersLibInterface* g_pmu = nullptr;
+static bool g_hasRtc = false;
+static bool g_rtcLowVoltage = false;
+
+struct RtcDateTime {
+  uint16_t year;
+  uint8_t month;
+  uint8_t day;
+  uint8_t hour;
+  uint8_t minute;
+  uint8_t second;
+  uint8_t weekday;
+};
+
+static void oledShowLines(const char *l1,
+                          const char *l2 = nullptr,
+                          const char *l3 = nullptr,
+                          const char *l4 = nullptr,
+                          const char *l5 = nullptr)
+{
+  g_oled.clearBuffer();
+  g_oled.setFont(u8g2_font_5x8_tf);
+  if (l1) g_oled.drawUTF8(0, 12, l1);
+  if (l2) g_oled.drawUTF8(0, 24, l2);
+  if (l3) g_oled.drawUTF8(0, 36, l3);
+  if (l4) g_oled.drawUTF8(0, 48, l4);
+  if (l5) g_oled.drawUTF8(0, 60, l5);
+  g_oled.sendBuffer();
+}
+
+static size_t getFlashTotalBytes()
+{
+  return SPIFFS.totalBytes();
+}
+
+static size_t getFlashUsedBytes()
+{
+  return SPIFFS.usedBytes();
+}
+
+static size_t getFlashFreeBytes()
+{
+  const size_t total = getFlashTotalBytes();
+  const size_t used = getFlashUsedBytes();
+  return total > used ? total - used : 0;
+}
+
+static uint8_t toBcd(uint8_t v) {
+  return ((v / 10U) << 4U) | (v % 10U);
+}
+
+static uint8_t fromBcd(uint8_t b) {
+  return ((b >> 4U) * 10U) + (b & 0x0FU);
+}
+
+static bool isRtcDateTimeValid(const RtcDateTime& dt) {
+  if (dt.year < 2020 || dt.year > 2099) return false;
+  if (dt.month < 1 || dt.month > 12) return false;
+  if (dt.day < 1 || dt.day > 31) return false;
+  if (dt.hour > 23 || dt.minute > 59 || dt.second > 59) return false;
+  return true;
+}
+
+static bool rtcRead(RtcDateTime& out, bool& lowVoltageFlag) {
+  Wire1.beginTransmission(RTC_I2C_ADDR);
+  Wire1.write(0x02);
+  if (Wire1.endTransmission(false) != 0) {
+    return false;
+  }
+
+  const uint8_t need = 7;
+  uint8_t got = Wire1.requestFrom((int)RTC_I2C_ADDR, (int)need);
+  if (got != need) {
+    return false;
+  }
+
+  uint8_t sec = Wire1.read();
+  uint8_t min = Wire1.read();
+  uint8_t hour = Wire1.read();
+  uint8_t day = Wire1.read();
+  uint8_t weekday = Wire1.read();
+  uint8_t month = Wire1.read();
+  uint8_t year = Wire1.read();
+
+  lowVoltageFlag = (sec & 0x80U) != 0;
+  out.second = fromBcd(sec & 0x7FU);
+  out.minute = fromBcd(min & 0x7FU);
+  out.hour = fromBcd(hour & 0x3FU);
+  out.day = fromBcd(day & 0x3FU);
+  out.weekday = fromBcd(weekday & 0x07U);
+  out.month = fromBcd(month & 0x1FU);
+  uint8_t yy = fromBcd(year);
+  bool century = (month & 0x80U) != 0;
+  out.year = century ? (1900U + yy) : (2000U + yy);
+
+  return true;
+}
+
+static bool initRtc() {
+  if (!tbeam_supreme::initPmuForPeripherals(g_pmu, &Serial)) {
+    Serial.println("RTC init: PMU/i2c init failed");
+    return false;
+  }
+  RtcDateTime now{};
+  if (!rtcRead(now, g_rtcLowVoltage) || !isRtcDateTimeValid(now)) {
+    Serial.println("RTC init: no valid time available");
+    return false;
+  }
+  g_hasRtc = true;
+  Serial.printf("RTC init: %04u-%02u-%02u %02u:%02u:%02u%s\r\n",
+                (unsigned)now.year, (unsigned)now.month, (unsigned)now.day,
+                (unsigned)now.hour, (unsigned)now.minute, (unsigned)now.second,
+                g_rtcLowVoltage ? " [LOW_BATT]" : "");
+  return true;
+}
+
+static bool getRtcTimestamp(char *out, size_t outSize) {
+  if (!g_hasRtc) {
+    return false;
+  }
+  RtcDateTime now{};
+  bool low = false;
+  if (!rtcRead(now, low) || !isRtcDateTimeValid(now)) {
+    return false;
+  }
+  g_rtcLowVoltage = low;
+  snprintf(out, outSize, "%04u-%02u-%02u %02u:%02u:%02u",
+           now.year,
+           now.month,
+           now.day,
+           now.hour,
+           now.minute,
+           now.second);
+  return true;
+}
+
+static void getTimestamp(char *out, size_t outSize)
+{
+  if (getRtcTimestamp(out, outSize)) {
+    return;
+  }
+
+  const time_t now = time(nullptr);
+  if (now > 1700000000) {
+    struct tm tmNow;
+    localtime_r(&now, &tmNow);
+    snprintf(out, outSize, "%04d-%02d-%02d %02d:%02d:%02d",
+             tmNow.tm_year + 1900,
+             tmNow.tm_mon + 1,
+             tmNow.tm_mday,
+             tmNow.tm_hour,
+             tmNow.tm_min,
+             tmNow.tm_sec);
+    return;
+  }
+
+  const uint32_t sec = millis() / 1000;
+  const uint32_t hh = sec / 3600;
+  const uint32_t mm = (sec % 3600) / 60;
+  const uint32_t ss = sec % 60;
+  snprintf(out, outSize, "uptime %02u:%02u:%02u", (unsigned)hh, (unsigned)mm, (unsigned)ss);
+}
+
+static void getFilenameTimestamp(char *out, size_t outSize)
+{
+  if (g_hasRtc) {
+    RtcDateTime now{};
+    bool low = false;
+    if (rtcRead(now, low) && isRtcDateTimeValid(now)) {
+      snprintf(out, outSize, "%04u%02u%02u_%02u%02u",
+               now.year,
+               now.month,
+               now.day,
+               now.hour,
+               now.minute);
+      return;
+    }
+  }
+
+  const time_t now = time(nullptr);
+  if (now > 1700000000) {
+    struct tm tmNow;
+    localtime_r(&now, &tmNow);
+    snprintf(out, outSize, "%04d%02d%02d_%02d%02d",
+             tmNow.tm_year + 1900,
+             tmNow.tm_mon + 1,
+             tmNow.tm_mday,
+             tmNow.tm_hour,
+             tmNow.tm_min);
+    return;
+  }
+
+  const uint32_t sec = millis() / 1000;
+  const uint32_t hh = sec / 3600;
+  const uint32_t mm = (sec % 3600) / 60;
+  const uint32_t ss = sec % 60;
+  snprintf(out, outSize, "uptime_%02u%02u%02u", (unsigned)hh, (unsigned)mm, (unsigned)ss);
+}
+
+static String getNewFlashFilePath()
+{
+  char baseName[64];
+  getFilenameTimestamp(baseName, sizeof(baseName));
+
+  char candidate[96];
+  snprintf(candidate, sizeof(candidate), "%s/%s.log", kFlashDir, baseName);
+  if (!SPIFFS.exists(candidate)) {
+    return String(candidate);
+  }
+
+  int suffix = 1;
+  do {
+    snprintf(candidate, sizeof(candidate), "%s/%s-%d.log", kFlashDir, baseName, suffix);
+    suffix += 1;
+  } while (SPIFFS.exists(candidate));
+
+  return String(candidate);
+}
+
+static bool ensureFlashDirectory()
+{
+  if (SPIFFS.exists(kFlashDir)) {
+    return true;
+  }
+  if (!SPIFFS.mkdir(kFlashDir)) {
+    Serial.printf("Warning: failed to create %s\r\n", kFlashDir);
+    return false;
+  }
+  return true;
+}
+
+static bool openCurrentFlashFile(bool truncate = false)
+{
+  if (g_flashFile) {
+    g_flashFile.close();
+  }
+
+  if (truncate) {
+    g_flashFile = SPIFFS.open(g_currentFilePath, FILE_WRITE);
+  } else {
+    g_flashFile = SPIFFS.open(g_currentFilePath, FILE_APPEND);
+  }
+
+  if (!g_flashFile) {
+    Serial.printf("ERROR: cannot open %s\r\n", g_currentFilePath);
+    return false;
+  }
+  return true;
+}
+
+static bool createFlashLogFile()
+{
+  if (!ensureFlashDirectory()) {
+    return false;
+  }
+
+  String path = getNewFlashFilePath();
+  path.toCharArray(g_currentFilePath, sizeof(g_currentFilePath));
+
+  g_flashFile = SPIFFS.open(g_currentFilePath, FILE_WRITE);
+  if (!g_flashFile) {
+    Serial.printf("ERROR: could not create %s\r\n", g_currentFilePath);
+    return false;
+  }
+
+  const char *header = "FLASH log file created\r\n";
+  g_flashFile.print(header);
+  g_flashFile.flush();
+  g_flashLineNumber = 0;
+  return true;
+}
+
+static void appendFlashTimestampLine()
+{
+  if (!g_flashFile) {
+    return;
+  }
+
+  char timestamp[32];
+  getTimestamp(timestamp, sizeof(timestamp));
+
+  char line[96];
+  const int written = snprintf(line, sizeof(line), "%u, %s\r\n", g_flashLineNumber + 1, timestamp);
+  if (written <= 0) {
+    return;
+  }
+
+  const size_t lineLen = (size_t)written;
+  if (g_flashFile.write(reinterpret_cast<const uint8_t *>(line), lineLen) != lineLen) {
+    Serial.println("Warning: flash write failed");
+    return;
+  }
+  g_flashFile.flush();
+  g_flashLineNumber += 1;
+}
+
+static void printFlashStatus()
+{
+  const size_t total = getFlashTotalBytes();
+  const size_t used = getFlashUsedBytes();
+  const size_t freeBytes = getFlashFreeBytes();
+
+  Serial.printf("FLASH total=%u used=%u free=%u\r\n",
+                (unsigned)total, (unsigned)used, (unsigned)freeBytes);
+
+  char line1[32];
+  char line2[32];
+  char line3[32];
+  char line4[32];
+  char line5[32];
+
+  snprintf(line1, sizeof(line1), "Exercise 17 Flash");
+  snprintf(line2, sizeof(line2), "Node: %s", NODE_LABEL);
+  snprintf(line3, sizeof(line3), "Free: %u KB", (unsigned)(freeBytes / 1024U));
+  snprintf(line4, sizeof(line4), "Used: %u KB", (unsigned)(used / 1024U));
+  snprintf(line5, sizeof(line5), "Lines: %u", (unsigned)g_flashLineNumber);
+
+  oledShowLines(line1, line2, line3, line4, line5);
+}
+
+static void showHelp()
+{
+  Serial.println("Flash command list:");
+  Serial.println("  help           - show this menu");
+  Serial.println("  stat           - show flash/file state");
+  Serial.println("  rtc            - show RTC time status");
+  Serial.println("  list           - list files in /flash_logs");
+  Serial.println("  read           - read current flash file");
+  Serial.println("  clear          - clear current flash file");
+  Serial.println("  write <text>   - overwrite current flash file");
+  Serial.println("  append <text>  - append text to current flash file");
+}
+
+static void printFlashFileStat()
+{
+  Serial.printf("Current file: %s\r\n", g_currentFilePath);
+  if (!SPIFFS.exists(g_currentFilePath)) {
+    Serial.println("Current file missing");
+    return;
+  }
+
+  File file = SPIFFS.open(g_currentFilePath, FILE_READ);
+  if (!file) {
+    Serial.println("Unable to open current file for stats");
+    return;
+  }
+
+  Serial.printf("Size: %u bytes\r\n", (unsigned)file.size());
+  Serial.printf("Lines written: %u\r\n", (unsigned)g_flashLineNumber);
+  file.close();
+}
+
+static void printFlashFileContents()
+{
+  if (!SPIFFS.exists(g_currentFilePath)) {
+    Serial.println("Current flash file does not exist");
+    return;
+  }
+
+  File file = SPIFFS.open(g_currentFilePath, FILE_READ);
+  if (!file) {
+    Serial.println("Unable to open current flash file");
+    return;
+  }
+
+  if (file.size() == 0) {
+    Serial.println("Current flash file is empty");
+    file.close();
+    return;
+  }
+
+  Serial.print("Flash file contents: ");
+  while (file.available()) {
+    Serial.write(file.read());
+  }
+  if (file.size() > 0) {
+    Serial.println();
+  }
+  file.close();
+}
+
+static void clearFlashFileContents()
+{
+  if (!SPIFFS.exists(g_currentFilePath)) {
+    Serial.println("No current file to clear");
+    return;
+  }
+
+  if (!openCurrentFlashFile(true)) {
+    return;
+  }
+  g_flashFile.close();
+  g_flashLineNumber = 0;
+  openCurrentFlashFile(false);
+  Serial.println("Current flash file cleared");
+}
+
+static void setFlashFileContent(const char *text)
+{
+  if (!text) {
+    clearFlashFileContents();
+    return;
+  }
+
+  File file = SPIFFS.open(g_currentFilePath, FILE_WRITE);
+  if (!file) {
+    Serial.println("Unable to overwrite current flash file");
+    return;
+  }
+
+  file.print(text);
+  file.close();
+  openCurrentFlashFile(false);
+  g_flashLineNumber = 0;
+}
+
+static void appendFlashFileContent(const char *text)
+{
+  if (!text || text[0] == '\0') {
+    return;
+  }
+
+  if (!openCurrentFlashFile(false)) {
+    return;
+  }
+
+  g_flashFile.print(text);
+  g_flashFile.flush();
+}
+
+static void listFlashFiles()
+{
+  File dir = SPIFFS.open(kFlashDir);
+  if (!dir || !dir.isDirectory()) {
+    Serial.printf("Unable to list files in %s\r\n", kFlashDir);
+    return;
+  }
+
+  Serial.printf("Files in %s:\r\n", kFlashDir);
+  File file = dir.openNextFile();
+  while (file) {
+    Serial.printf("  %s (%u bytes)\r\n", file.name(), (unsigned)file.size());
+    file = dir.openNextFile();
+  }
+  dir.close();
+}
+
+static void processSerialCommand(const char *line)
+{
+  if (!line || line[0] == '\0') return;
+
+  char tmp[384];
+  strncpy(tmp, line, sizeof(tmp) - 1);
+  tmp[sizeof(tmp) - 1] = '\0';
+
+  char *cmd = strtok(tmp, " \t\r\n");
+  if (!cmd) return;
+
+  if (strcasecmp(cmd, "help") == 0) {
+    showHelp();
+    return;
+  }
+
+  if (strcasecmp(cmd, "stat") == 0) {
+    printFlashFileStat();
+    return;
+  }
+
+  if (strcasecmp(cmd, "rtc") == 0) {
+    if (g_hasRtc) {
+      char ts[32];
+      if (getRtcTimestamp(ts, sizeof(ts))) {
+        Serial.printf("RTC now: %s\r\n", ts);
+        if (g_rtcLowVoltage) {
+          Serial.println("RTC low-voltage flag is set");
+        }
+      } else {
+        Serial.println("RTC present but time read failed");
+      }
+    } else {
+      Serial.println("RTC unavailable");
+    }
+    return;
+  }
+
+  if (strcasecmp(cmd, "list") == 0) {
+    listFlashFiles();
+    return;
+  }
+
+  if (strcasecmp(cmd, "read") == 0) {
+    printFlashFileContents();
+    return;
+  }
+
+  if (strcasecmp(cmd, "clear") == 0) {
+    clearFlashFileContents();
+    return;
+  }
+
+  if (strcasecmp(cmd, "write") == 0 || strcasecmp(cmd, "append") == 0) {
+    const char *payload = line + strlen(cmd);
+    while (*payload == ' ' || *payload == '\t') payload++;
+    if (strcasecmp(cmd, "write") == 0)
+      setFlashFileContent(payload);
+    else
+      appendFlashFileContent(payload);
+
+    Serial.printf("%s: %s\r\n", cmd, payload);
+    return;
+  }
+
+  Serial.println("Unknown command (help for list)");
+}
+
+void setup()
+{
+  Serial.begin(115200);
+  delay(800);
+  Serial.println("Exercise 17_Flash boot");
+
+  initRtc();
+
+  if (!SPIFFS.begin(true)) {
+    Serial.println("ERROR: SPIFFS mount failed");
+    oledShowLines("Exercise 17_Flash", "Node: " NODE_LABEL, "SPIFFS mount FAILED");
+  } else {
+    Serial.println("SPIFFS mounted successfully");
+    if (createFlashLogFile()) {
+      Serial.printf("Current flash file: %s\r\n", g_currentFilePath);
+    }
+  }
+
+  Wire.begin(OLED_SDA, OLED_SCL);
+  g_oled.setI2CAddress(OLED_ADDR << 1);
+  g_oled.begin();
+  oledShowLines("Exercise 17_Flash", "Node: " NODE_LABEL, "Booting...");
+
+  delay(1000);
+}
+
+void loop()
+{
+  static uint32_t lastMs = 0;
+  const uint32_t now = millis();
+
+  static char rxLine[384];
+  static size_t rxLen = 0;
+
+  while (Serial.available()) {
+    int c = Serial.read();
+    if (c <= 0) continue;
+    if (c == '\r' || c == '\n') {
+      if (rxLen > 0) {
+        rxLine[rxLen] = '\0';
+        processSerialCommand(rxLine);
+        rxLen = 0;
+      }
+    } else if (rxLen + 1 < sizeof(rxLine)) {
+      rxLine[rxLen++] = (char)c;
+    }
+  }
+
+  if (now - lastMs < 1000) {
+    delay(10);
+    return;
+  }
+  lastMs = now;
+
+  if (g_flashFile) {
+    appendFlashTimestampLine();
+  }
+  printFlashStatus();
+}