Stabilized logging to SD Card, disciplined time, and web fetch & erase

This commit is contained in:
John Poole 2026-04-06 11:38:06 -07:00
commit e28ebe5b17
15 changed files with 1536 additions and 510 deletions

View file

@ -0,0 +1,241 @@
#include "ClockDiscipline.h"
#include "Config.h"
namespace field_qa {
ClockDiscipline::ClockDiscipline(TwoWire& wire) : m_wire(wire) {}
uint8_t ClockDiscipline::toBcd(uint8_t value) {
return (uint8_t)(((value / 10U) << 4U) | (value % 10U));
}
uint8_t ClockDiscipline::fromBcd(uint8_t value) {
return (uint8_t)(((value >> 4U) * 10U) + (value & 0x0FU));
}
bool ClockDiscipline::isLeapYear(uint16_t year) {
return ((year % 4U) == 0U && (year % 100U) != 0U) || ((year % 400U) == 0U);
}
uint8_t ClockDiscipline::daysInMonth(uint16_t year, uint8_t month) {
static const uint8_t kDays[12] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
if (month == 2U) {
return (uint8_t)(isLeapYear(year) ? 29U : 28U);
}
if (month >= 1U && month <= 12U) {
return kDays[month - 1U];
}
return 0;
}
bool ClockDiscipline::isValidDateTime(const ClockDateTime& dt) {
if (dt.year < 2000U || dt.year > 2099U) {
return false;
}
if (dt.month < 1U || dt.month > 12U) {
return false;
}
if (dt.day < 1U || dt.day > daysInMonth(dt.year, dt.month)) {
return false;
}
if (dt.hour > 23U || dt.minute > 59U || dt.second > 59U) {
return false;
}
return true;
}
int64_t ClockDiscipline::daysFromCivil(int year, unsigned month, unsigned day) {
year -= (month <= 2U);
const int era = (year >= 0 ? year : year - 399) / 400;
const unsigned yoe = (unsigned)(year - era * 400);
const unsigned doy = (153U * (month + (month > 2U ? (unsigned)-3 : 9U)) + 2U) / 5U + day - 1U;
const unsigned doe = yoe * 365U + yoe / 4U - yoe / 100U + doy;
return era * 146097 + (int)doe - 719468;
}
int64_t ClockDiscipline::toEpochSeconds(const ClockDateTime& dt) {
const int64_t days = daysFromCivil((int)dt.year, dt.month, dt.day);
return days * 86400LL + (int64_t)dt.hour * 3600LL + (int64_t)dt.minute * 60LL + (int64_t)dt.second;
}
bool ClockDiscipline::fromEpochSeconds(int64_t seconds, ClockDateTime& out) {
if (seconds < 0) {
return false;
}
int64_t days = seconds / 86400LL;
int64_t remainder = seconds % 86400LL;
if (remainder < 0) {
remainder += 86400LL;
days -= 1;
}
out.hour = (uint8_t)(remainder / 3600LL);
remainder %= 3600LL;
out.minute = (uint8_t)(remainder / 60LL);
out.second = (uint8_t)(remainder % 60LL);
days += 719468;
const int era = (days >= 0 ? days : days - 146096) / 146097;
const unsigned doe = (unsigned)(days - era * 146097);
const unsigned yoe = (doe - doe / 1460U + doe / 36524U - doe / 146096U) / 365U;
int year = (int)yoe + era * 400;
const unsigned doy = doe - (365U * yoe + yoe / 4U - yoe / 100U);
const unsigned mp = (5U * doy + 2U) / 153U;
const unsigned day = doy - (153U * mp + 2U) / 5U + 1U;
const unsigned month = mp + (mp < 10U ? 3U : (unsigned)-9);
year += (month <= 2U);
out.year = (uint16_t)year;
out.month = (uint8_t)month;
out.day = (uint8_t)day;
return isValidDateTime(out);
}
bool ClockDiscipline::readRtc(ClockDateTime& out, bool& lowVoltageFlag) const {
m_wire.beginTransmission(RTC_I2C_ADDR);
m_wire.write(0x02);
if (m_wire.endTransmission(false) != 0) {
return false;
}
const uint8_t need = 7;
const uint8_t got = m_wire.requestFrom((int)RTC_I2C_ADDR, (int)need);
if (got != need) {
return false;
}
const uint8_t sec = m_wire.read();
const uint8_t min = m_wire.read();
const uint8_t hour = m_wire.read();
const uint8_t day = m_wire.read();
(void)m_wire.read();
const uint8_t month = m_wire.read();
const uint8_t year = m_wire.read();
lowVoltageFlag = (sec & 0x80U) != 0;
out.second = fromBcd(sec & 0x7FU);
out.minute = fromBcd(min & 0x7FU);
out.hour = fromBcd(hour & 0x3FU);
out.day = fromBcd(day & 0x3FU);
out.month = fromBcd(month & 0x1FU);
const uint8_t yy = fromBcd(year);
out.year = (month & 0x80U) ? (1900U + yy) : (2000U + yy);
return true;
}
bool ClockDiscipline::readValidRtc(ClockDateTime& out, int64_t* epochOut) const {
bool lowVoltage = false;
if (!readRtc(out, lowVoltage) || lowVoltage || !isValidDateTime(out)) {
return false;
}
if (epochOut != nullptr) {
*epochOut = toEpochSeconds(out);
}
return true;
}
bool ClockDiscipline::writeRtc(const ClockDateTime& dt) const {
if (!isValidDateTime(dt)) {
return false;
}
m_wire.beginTransmission(RTC_I2C_ADDR);
m_wire.write(0x02);
m_wire.write(toBcd(dt.second & 0x7FU));
m_wire.write(toBcd(dt.minute));
m_wire.write(toBcd(dt.hour));
m_wire.write(toBcd(dt.day));
m_wire.write(0x00);
uint8_t monthReg = toBcd(dt.month);
if (dt.year < 2000U) {
monthReg |= 0x80U;
}
m_wire.write(monthReg);
m_wire.write(toBcd((uint8_t)(dt.year % 100U)));
return m_wire.endTransmission() == 0;
}
void ClockDiscipline::formatIsoUtc(const ClockDateTime& dt, char* out, size_t outSize) {
snprintf(out,
outSize,
"%04u-%02u-%02uT%02u:%02u:%02uZ",
(unsigned)dt.year,
(unsigned)dt.month,
(unsigned)dt.day,
(unsigned)dt.hour,
(unsigned)dt.minute,
(unsigned)dt.second);
}
void ClockDiscipline::formatCompactUtc(const ClockDateTime& dt, char* out, size_t outSize) {
snprintf(out,
outSize,
"%04u%02u%02u_%02u%02u%02u",
(unsigned)dt.year,
(unsigned)dt.month,
(unsigned)dt.day,
(unsigned)dt.hour,
(unsigned)dt.minute,
(unsigned)dt.second);
}
void ClockDiscipline::makeRunId(const ClockDateTime& dt, const char* boardId, char* out, size_t outSize) {
snprintf(out,
outSize,
"%04u%02u%02u_%02u%02u%02u_%s",
(unsigned)dt.year,
(unsigned)dt.month,
(unsigned)dt.day,
(unsigned)dt.hour,
(unsigned)dt.minute,
(unsigned)dt.second,
boardId ? boardId : "NODE");
}
bool ClockDiscipline::fromGnssSample(const GnssSample& sample, ClockDateTime& out) {
if (!sample.validTime) {
return false;
}
out.year = sample.year;
out.month = sample.month;
out.day = sample.day;
out.hour = sample.hour;
out.minute = sample.minute;
out.second = sample.second;
return isValidDateTime(out);
}
bool ClockDiscipline::disciplineFromGnss(const GnssSample& sample,
WaitForPpsCallback waitForPps,
void* context,
ClockDateTime& disciplinedUtc,
bool& hadPriorRtc,
int64_t& driftSeconds) const {
ClockDateTime gpsUtc{};
if (!fromGnssSample(sample, gpsUtc) || waitForPps == nullptr) {
return false;
}
ClockDateTime priorRtc{};
hadPriorRtc = readValidRtc(priorRtc, nullptr);
if (!waitForPps(context, kClockPpsWaitTimeoutMs)) {
return false;
}
const int64_t snappedEpoch = toEpochSeconds(gpsUtc);
if (!fromEpochSeconds(snappedEpoch + 1, disciplinedUtc)) {
return false;
}
if (!writeRtc(disciplinedUtc)) {
return false;
}
driftSeconds = hadPriorRtc ? (toEpochSeconds(priorRtc) - toEpochSeconds(disciplinedUtc)) : 0;
return true;
}
} // namespace field_qa

View file

@ -0,0 +1,53 @@
#pragma once
#include <Arduino.h>
#include <Wire.h>
#include "GnssTypes.h"
namespace field_qa {
struct ClockDateTime {
uint16_t year = 0;
uint8_t month = 0;
uint8_t day = 0;
uint8_t hour = 0;
uint8_t minute = 0;
uint8_t second = 0;
};
using WaitForPpsCallback = bool (*)(void* context, uint32_t timeoutMs);
class ClockDiscipline {
public:
explicit ClockDiscipline(TwoWire& wire = Wire1);
bool readRtc(ClockDateTime& out, bool& lowVoltageFlag) const;
bool readValidRtc(ClockDateTime& out, int64_t* epochOut = nullptr) const;
bool writeRtc(const ClockDateTime& dt) const;
bool disciplineFromGnss(const GnssSample& sample,
WaitForPpsCallback waitForPps,
void* context,
ClockDateTime& disciplinedUtc,
bool& hadPriorRtc,
int64_t& driftSeconds) const;
static bool isValidDateTime(const ClockDateTime& dt);
static int64_t toEpochSeconds(const ClockDateTime& dt);
static bool fromEpochSeconds(int64_t seconds, ClockDateTime& out);
static void formatIsoUtc(const ClockDateTime& dt, char* out, size_t outSize);
static void formatCompactUtc(const ClockDateTime& dt, char* out, size_t outSize);
static void makeRunId(const ClockDateTime& dt, const char* boardId, char* out, size_t outSize);
static bool fromGnssSample(const GnssSample& sample, ClockDateTime& out);
private:
static uint8_t toBcd(uint8_t value);
static uint8_t fromBcd(uint8_t value);
static bool isLeapYear(uint16_t year);
static uint8_t daysInMonth(uint16_t year, uint8_t month);
static int64_t daysFromCivil(int year, unsigned month, unsigned day);
TwoWire& m_wire;
};
} // namespace field_qa

View file

@ -51,8 +51,8 @@ static constexpr const char* kExerciseName = "18_GPS_Field_QA";
static constexpr const char* kFirmwareVersion = FIELD_QA_STR(FW_BUILD_UTC);
static constexpr const char* kBoardId = BOARD_ID;
static constexpr const char* kGnssChip = GNSS_CHIP_NAME;
static constexpr const char* kStorageName = "SPIFFS";
static constexpr const char* kLogDir = "/";
static constexpr const char* kStorageName = "SD";
static constexpr const char* kLogDir = "/logs";
static constexpr const char* kLogApPrefix = "GPSQA-";
static constexpr const char* kLogApPassword = "";
static constexpr uint8_t kLogApIpOctet = 23;
@ -64,7 +64,6 @@ static constexpr uint32_t kStatusPeriodMs = 1000;
static constexpr uint32_t kProbeWindowL76kMs = 20000;
static constexpr uint32_t kProbeWindowUbloxMs = 45000;
static constexpr uint32_t kFixFreshMs = 5000;
static constexpr size_t kMaxLogFilesBeforePause = 5;
static constexpr uint8_t kPoorMinSatsUsed = 4;
static constexpr uint8_t kGoodMinSatsUsed = 10;
static constexpr uint8_t kExcellentMinSatsUsed = 16;
@ -72,5 +71,10 @@ static constexpr float kMarginalHdop = 3.0f;
static constexpr float kExcellentHdop = 1.5f;
static constexpr size_t kBufferedSamples = 10;
static constexpr size_t kMaxSatellites = 64;
static constexpr size_t kStorageBufferBytes = 4096;
static constexpr uint32_t kClockDisciplineRetryMs = 5000;
static constexpr uint32_t kClockPpsWaitTimeoutMs = 1500;
static constexpr uint32_t kClockFreshSampleMs = 2000;
static constexpr uint32_t kMaxLogFilesBeforePause = 1000;
} // namespace field_qa

View file

@ -58,36 +58,12 @@ static const char* constellationForTalker(const char* talker) {
} // namespace
bool StorageManager::mount() {
m_ready = false;
m_lastError = "";
m_path = "";
m_buffer = "";
if (m_file) {
m_file.close();
}
if (!SPIFFS.begin(true)) {
m_lastError = "SPIFFS.begin failed";
return false;
}
if (!ensureDir()) {
return false;
}
return true;
}
bool StorageManager::begin(const char* runId) {
if (!mount()) {
return false;
}
return startLog(runId, "");
}
bool StorageManager::startLog(const char* runId, const char* bootTimestampUtc) {
close();
m_ready = false;
m_lastError = "";
m_path = makeFilePath(runId);
if (!openFile()) {
if (!ensureDir() || !openFile()) {
return false;
}
m_ready = true;
@ -96,8 +72,10 @@ bool StorageManager::startLog(const char* runId, const char* bootTimestampUtc) {
}
bool StorageManager::mounted() const {
File root = SPIFFS.open("/");
return root && root.isDirectory();
File root = SD.open("/", FILE_READ);
const bool ok = root && root.isDirectory();
root.close();
return ok;
}
bool StorageManager::ready() const {
@ -117,76 +95,76 @@ bool StorageManager::fileOpen() const {
}
size_t StorageManager::bufferedBytes() const {
return m_buffer.length();
return m_bufferLengths[0] + m_bufferLengths[1];
}
size_t StorageManager::logFileCount() const {
File dir = SPIFFS.open("/");
size_t StorageManager::countLogsRecursive(const char* path) const {
File dir = SD.open(path, FILE_READ);
if (!dir || !dir.isDirectory()) {
dir.close();
return 0;
}
size_t count = 0;
File file = dir.openNextFile();
while (file) {
String name = file.name();
if (isRecognizedLogName(name)) {
File entry = dir.openNextFile();
while (entry) {
String name = entry.name();
if (entry.isDirectory()) {
count += countLogsRecursive(name.c_str());
} else if (isRecognizedLogName(name)) {
++count;
}
file = dir.openNextFile();
entry.close();
entry = dir.openNextFile();
}
dir.close();
return count;
}
size_t StorageManager::logFileCount() const {
return countLogsRecursive(kLogDir);
}
bool StorageManager::ensureDir() {
if (strcmp(kLogDir, "/") == 0) {
return true;
String full(kLogDir);
if (!full.startsWith("/")) {
full = "/" + full;
}
if (SPIFFS.exists(kLogDir)) {
return true;
}
if (!SPIFFS.mkdir(kLogDir)) {
m_lastError = "SPIFFS.mkdir failed";
return false;
int start = 1;
while (start > 0 && start < (int)full.length()) {
const int slash = full.indexOf('/', start);
String partial = (slash < 0) ? full : full.substring(0, slash);
if (!SD.exists(partial.c_str()) && !SD.mkdir(partial.c_str())) {
m_lastError = "SD.mkdir failed";
return false;
}
if (slash < 0) {
break;
}
start = slash + 1;
}
return true;
}
String StorageManager::makeFilePath(const char* runId) const {
char path[96];
const char* rid = runId ? runId : "run";
char shortId[32];
if (strlen(rid) >= 19 && rid[8] == '_' && rid[15] == '_') {
snprintf(shortId,
sizeof(shortId),
"%.6s_%.6s_%s",
rid + 2,
rid + 9,
rid + 16);
} else {
snprintf(shortId, sizeof(shortId), "%s", rid);
}
if (strcmp(kLogDir, "/") == 0) {
snprintf(path, sizeof(path), "/%s.csv", shortId);
} else {
snprintf(path, sizeof(path), "%s/%s.csv", kLogDir, shortId);
}
const char* rid = (runId && runId[0] != '\0') ? runId : "run";
snprintf(path, sizeof(path), "%s/%s.csv", kLogDir, rid);
return String(path);
}
bool StorageManager::openFile() {
m_file = SPIFFS.open(m_path, FILE_WRITE);
m_file = SD.open(m_path.c_str(), FILE_WRITE);
if (!m_file) {
m_lastError = "SPIFFS.open write failed";
m_lastError = "SD.open write failed";
return false;
}
return true;
}
void StorageManager::writeHeader(const char* runId, const char* bootTimestampUtc) {
if (!m_file) {
return;
}
if (m_file.size() > 0) {
if (!m_file || m_file.size() > 0) {
return;
}
m_file.printf("# exercise: %s\n", kExerciseName);
@ -199,27 +177,99 @@ void StorageManager::writeHeader(const char* runId, const char* bootTimestampUtc
m_file.printf("# run_id: %s\n", runId ? runId : "");
m_file.printf("# boot_timestamp_utc: %s\n", bootTimestampUtc ? bootTimestampUtc : "");
m_file.printf("# created_by: ChatGPT/Codex handoff\n");
m_file.print("record_type,timestamp_utc,board_id,gnss_chip,firmware_exercise_name,firmware_version,boot_timestamp_utc,run_id,fix_type,fix_dimension,sats_in_view,sat_seen,sats_used,hdop,vdop,pdop,latitude,longitude,altitude_m,speed_mps,course_deg,pps_seen,quality_class,gps_count,galileo_count,glonass_count,beidou_count,navic_count,qzss_count,sbas_count,mean_cn0,max_cn0,age_of_fix_ms,ttff_ms,longest_no_fix_ms,sat_talker,sat_constellation,sat_prn,sat_elevation_deg,sat_azimuth_deg,sat_snr,sat_used_in_solution\n");
m_file.print("record_type,timestamp_utc,sample_seq,ms_since_run_start,board_id,gnss_chip,firmware_exercise_name,firmware_version,boot_timestamp_utc,run_id,fix_type,fix_dimension,sats_in_view,sat_seen,sats_used,hdop,vdop,pdop,latitude,longitude,altitude_m,speed_mps,course_deg,pps_seen,quality_class,gps_count,galileo_count,glonass_count,beidou_count,navic_count,qzss_count,sbas_count,mean_cn0,max_cn0,age_of_fix_ms,ttff_ms,longest_no_fix_ms,sat_talker,sat_constellation,sat_prn,sat_elevation_deg,sat_azimuth_deg,sat_snr,sat_used_in_solution\n");
m_file.flush();
}
void StorageManager::appendLine(const String& line) {
m_buffer += line;
if (!m_buffer.endsWith("\n")) {
m_buffer += "\n";
bool StorageManager::writePendingBuffer() {
if (!m_file) {
return false;
}
for (uint8_t i = 0; i < 2; ++i) {
if (!m_bufferPending[i] || m_bufferLengths[i] == 0) {
continue;
}
const size_t wrote = m_file.write((const uint8_t*)m_buffers[i], m_bufferLengths[i]);
if (wrote != m_bufferLengths[i]) {
m_lastError = "SD.write failed";
m_ready = false;
return false;
}
m_bufferLengths[i] = 0;
m_bufferPending[i] = false;
}
return true;
}
void StorageManager::appendSampleTsv(const GnssSample& sample, const char* runId, const char* bootTimestampUtc) {
bool StorageManager::appendBytes(const char* data, size_t len) {
if (!m_file || !data || len == 0) {
return false;
}
if (len > kStorageBufferBytes) {
if (!writePendingBuffer()) {
return false;
}
const size_t wrote = m_file.write((const uint8_t*)data, len);
if (wrote != len) {
m_lastError = "SD.write large block failed";
m_ready = false;
return false;
}
return true;
}
if ((m_bufferLengths[m_activeBuffer] + len) > kStorageBufferBytes) {
m_bufferPending[m_activeBuffer] = true;
m_activeBuffer ^= 1U;
if (m_bufferPending[m_activeBuffer]) {
if (!writePendingBuffer()) {
return false;
}
}
if (m_bufferLengths[m_activeBuffer] != 0) {
m_bufferPending[m_activeBuffer] = true;
if (!writePendingBuffer()) {
return false;
}
}
}
memcpy(m_buffers[m_activeBuffer] + m_bufferLengths[m_activeBuffer], data, len);
m_bufferLengths[m_activeBuffer] += len;
return true;
}
bool StorageManager::appendLine(const String& line) {
if (!appendBytes(line.c_str(), line.length())) {
return false;
}
if (!line.endsWith("\n")) {
static const char newline = '\n';
return appendBytes(&newline, 1);
}
return true;
}
void StorageManager::appendSampleCsv(const GnssSample& sample,
uint32_t sampleSeq,
uint32_t msSinceRunStart,
const char* runId,
const char* bootTimestampUtc) {
if (!m_file) {
return;
}
if (m_file.size() == 0) {
writeHeader(runId, bootTimestampUtc);
}
String line = "sample,";
line += sampleTimestamp(sample);
line += kLogFieldDelimiter;
line += String(sampleSeq);
line += kLogFieldDelimiter;
line += String(msSinceRunStart);
line += kLogFieldDelimiter;
line += kBoardId;
line += kLogFieldDelimiter;
line += kGnssChip;
@ -286,10 +336,12 @@ void StorageManager::appendSampleTsv(const GnssSample& sample, const char* runId
line += kLogFieldDelimiter;
line += String(sample.longestNoFixMs);
line += ",,,,,,,";
appendLine(line);
(void)appendLine(line);
}
void StorageManager::appendSatelliteTsv(const GnssSample& sample,
void StorageManager::appendSatelliteCsv(const GnssSample& sample,
uint32_t sampleSeq,
uint32_t msSinceRunStart,
const SatelliteInfo* satellites,
size_t satelliteCount,
const char* runId,
@ -309,6 +361,10 @@ void StorageManager::appendSatelliteTsv(const GnssSample& sample,
String line = "satellite,";
line += sampleTimestamp(sample);
line += kLogFieldDelimiter;
line += String(sampleSeq);
line += kLogFieldDelimiter;
line += String(msSinceRunStart);
line += kLogFieldDelimiter;
line += kBoardId;
line += kLogFieldDelimiter;
line += kGnssChip;
@ -388,17 +444,22 @@ void StorageManager::appendSatelliteTsv(const GnssSample& sample,
line += String(sat.snr);
line += kLogFieldDelimiter;
line += sat.usedInSolution ? "1" : "0";
appendLine(line);
(void)appendLine(line);
}
}
void StorageManager::flush() {
if (!m_file || m_buffer.isEmpty()) {
if (!m_file) {
return;
}
if (m_bufferLengths[m_activeBuffer] > 0) {
m_bufferPending[m_activeBuffer] = true;
}
if (!writePendingBuffer()) {
return;
}
m_file.print(m_buffer);
m_file.flush();
m_buffer = "";
}
void StorageManager::close() {
@ -406,6 +467,36 @@ void StorageManager::close() {
if (m_file) {
m_file.close();
}
m_ready = false;
}
bool StorageManager::normalizePath(const char* input, String& normalized) const {
normalized = "";
if (!input || input[0] == '\0') {
return false;
}
normalized = input[0] == '/' ? String(input) : (String("/") + input);
if (normalized.indexOf("..") >= 0) {
normalized = "";
return false;
}
return true;
}
void StorageManager::listFilesRecursive(File& dir, Stream& out) {
File entry = dir.openNextFile();
while (entry) {
String name = entry.name();
if (entry.isDirectory()) {
out.printf("%s/\n", name.c_str());
listFilesRecursive(entry, out);
} else if (isRecognizedLogName(name)) {
out.printf("%s\t%u\n", name.c_str(), (unsigned)entry.size());
}
entry.close();
entry = dir.openNextFile();
}
}
void StorageManager::listFiles(Stream& out) {
@ -413,23 +504,14 @@ void StorageManager::listFiles(Stream& out) {
out.println("storage not mounted");
return;
}
File dir = SPIFFS.open("/");
File dir = SD.open(kLogDir, FILE_READ);
if (!dir || !dir.isDirectory()) {
out.println("root directory unavailable");
out.println("log directory unavailable");
dir.close();
return;
}
File file = dir.openNextFile();
if (!file) {
out.println("(no files)");
return;
}
while (file) {
String name = file.name();
if (isRecognizedLogName(name)) {
out.printf("%s\t%u\n", file.name(), (unsigned)file.size());
}
file = dir.openNextFile();
}
listFilesRecursive(dir, out);
dir.close();
}
void StorageManager::catFile(Stream& out, const char* path) {
@ -437,12 +519,12 @@ void StorageManager::catFile(Stream& out, const char* path) {
out.println("storage not mounted");
return;
}
if (!path || path[0] == '\0') {
out.println("cat requires a filename");
String fullPath;
if (!normalizePath(path, fullPath)) {
out.println("cat requires a valid filename");
return;
}
String fullPath = path[0] == '/' ? String(path) : String("/") + path;
File file = SPIFFS.open(fullPath, FILE_READ);
File file = SD.open(fullPath.c_str(), FILE_READ);
if (!file) {
out.printf("unable to open %s\n", fullPath.c_str());
return;
@ -453,6 +535,26 @@ void StorageManager::catFile(Stream& out, const char* path) {
if (file.size() > 0) {
out.println();
}
file.close();
}
void StorageManager::eraseLogsRecursive(File& dir) {
File entry = dir.openNextFile();
while (entry) {
String path = entry.name();
const bool isDir = entry.isDirectory();
entry.close();
if (isDir) {
File subdir = SD.open(path.c_str(), FILE_READ);
if (subdir) {
eraseLogsRecursive(subdir);
subdir.close();
}
} else if (isRecognizedLogName(path)) {
SD.remove(path.c_str());
}
entry = dir.openNextFile();
}
}
void StorageManager::eraseLogs(Stream& out) {
@ -460,22 +562,35 @@ void StorageManager::eraseLogs(Stream& out) {
out.println("storage not mounted");
return;
}
File dir = SPIFFS.open("/");
File dir = SD.open(kLogDir, FILE_READ);
if (!dir || !dir.isDirectory()) {
out.println("root directory unavailable");
out.println("log directory unavailable");
dir.close();
return;
}
File file = dir.openNextFile();
while (file) {
String path = file.path();
bool isLog = isRecognizedLogName(path);
file.close();
if (isLog) {
SPIFFS.remove(path);
}
file = dir.openNextFile();
}
eraseLogsRecursive(dir);
dir.close();
out.println("logs erased");
}
bool StorageManager::eraseFile(const char* path) {
String fullPath;
if (!normalizePath(path, fullPath)) {
m_lastError = "invalid path";
return false;
}
if (m_path == fullPath && m_file) {
close();
}
if (!SD.exists(fullPath.c_str())) {
m_lastError = "path not found";
return false;
}
if (!SD.remove(fullPath.c_str())) {
m_lastError = "SD.remove failed";
return false;
}
return true;
}
} // namespace field_qa

View file

@ -1,15 +1,14 @@
#pragma once
#include <Arduino.h>
#include <SPIFFS.h>
#include <SD.h>
#include "Config.h"
#include "GnssTypes.h"
namespace field_qa {
class StorageManager {
public:
bool mount();
bool begin(const char* runId);
bool startLog(const char* runId, const char* bootTimestampUtc);
bool mounted() const;
bool ready() const;
@ -18,8 +17,14 @@ class StorageManager {
bool fileOpen() const;
size_t bufferedBytes() const;
size_t logFileCount() const;
void appendSampleTsv(const GnssSample& sample, const char* runId, const char* bootTimestampUtc);
void appendSatelliteTsv(const GnssSample& sample,
void appendSampleCsv(const GnssSample& sample,
uint32_t sampleSeq,
uint32_t msSinceRunStart,
const char* runId,
const char* bootTimestampUtc);
void appendSatelliteCsv(const GnssSample& sample,
uint32_t sampleSeq,
uint32_t msSinceRunStart,
const SatelliteInfo* satellites,
size_t satelliteCount,
const char* runId,
@ -29,19 +34,29 @@ class StorageManager {
void listFiles(Stream& out);
void catFile(Stream& out, const char* path);
void eraseLogs(Stream& out);
bool eraseFile(const char* path);
bool normalizePath(const char* input, String& normalized) const;
private:
bool ensureDir();
bool openFile();
void writeHeader(const char* runId, const char* bootTimestampUtc);
String makeFilePath(const char* runId) const;
void appendLine(const String& line);
bool appendLine(const String& line);
bool appendBytes(const char* data, size_t len);
bool writePendingBuffer();
size_t countLogsRecursive(const char* path) const;
void listFilesRecursive(File& dir, Stream& out);
void eraseLogsRecursive(File& dir);
bool m_ready = false;
String m_path;
String m_lastError;
File m_file;
String m_buffer;
char m_buffers[2][kStorageBufferBytes] = {};
size_t m_bufferLengths[2] = {0, 0};
bool m_bufferPending[2] = {false, false};
uint8_t m_activeBuffer = 0;
};
} // namespace field_qa

View file

@ -0,0 +1,360 @@
#include "StartupSdManager.h"
#include <stdarg.h>
#include "driver/gpio.h"
StartupSdManager::StartupSdManager(Print& serial) : serial_(serial) {}
bool StartupSdManager::begin(const SdWatcherConfig& cfg, SdStatusCallback callback) {
cfg_ = cfg;
callback_ = callback;
forceSpiDeselected();
dumpSdPins("very-early");
if (!initPmuForSdPower()) {
return false;
}
cycleSdRail();
delay(cfg_.startupWarmupMs);
bool warmMounted = false;
for (uint8_t i = 0; i < 3; ++i) {
if (mountPreferred(false)) {
warmMounted = true;
break;
}
delay(200);
}
// Some cards need a longer power/settle window after cold boot.
// Before declaring ABSENT, retry with extended settle and a full scan.
if (!warmMounted) {
logf("Watcher: startup preferred mount failed, retrying with extended settle");
cycleSdRail(400, 1200);
delay(cfg_.startupWarmupMs + 1500);
warmMounted = mountCardFullScan();
}
if (warmMounted) {
setStateMounted();
} else {
setStateAbsent();
}
return true;
}
void StartupSdManager::update() {
const uint32_t now = millis();
const uint32_t pollInterval =
(watchState_ == SdWatchState::MOUNTED) ? cfg_.pollIntervalMountedMs : cfg_.pollIntervalAbsentMs;
if ((uint32_t)(now - lastPollMs_) < pollInterval) {
return;
}
lastPollMs_ = now;
if (watchState_ == SdWatchState::MOUNTED) {
if (verifyMountedCard()) {
presentVotes_ = 0;
absentVotes_ = 0;
return;
}
if (mountPreferred(false) && verifyMountedCard()) {
presentVotes_ = 0;
absentVotes_ = 0;
return;
}
absentVotes_++;
presentVotes_ = 0;
if (absentVotes_ >= cfg_.votesToAbsent) {
setStateAbsent();
absentVotes_ = 0;
}
return;
}
bool mounted = mountPreferred(false);
if (!mounted && (uint32_t)(now - lastFullScanMs_) >= cfg_.fullScanIntervalMs) {
lastFullScanMs_ = now;
if (cfg_.recoveryRailCycleOnFullScan) {
logf("Watcher: recovery rail cycle before full scan");
cycleSdRail(cfg_.recoveryRailOffMs, cfg_.recoveryRailOnSettleMs);
delay(150);
}
logf("Watcher: preferred probe failed, running full scan");
mounted = mountCardFullScan();
}
if (mounted) {
presentVotes_++;
absentVotes_ = 0;
if (presentVotes_ >= cfg_.votesToPresent) {
setStateMounted();
presentVotes_ = 0;
}
} else {
absentVotes_++;
presentVotes_ = 0;
if (absentVotes_ >= cfg_.votesToAbsent) {
setStateAbsent();
absentVotes_ = 0;
}
}
}
bool StartupSdManager::consumeMountedEvent() {
bool out = mountedEventPending_;
mountedEventPending_ = false;
return out;
}
bool StartupSdManager::consumeRemovedEvent() {
bool out = removedEventPending_;
removedEventPending_ = false;
return out;
}
void StartupSdManager::logf(const char* fmt, ...) {
char msg[196];
va_list args;
va_start(args, fmt);
vsnprintf(msg, sizeof(msg), fmt, args);
va_end(args);
serial_.printf("[%10lu][%06lu] %s\r\n",
(unsigned long)millis(),
(unsigned long)logSeq_++,
msg);
}
void StartupSdManager::notify(SdEvent event, const char* message) {
if (callback_ != nullptr) {
callback_(event, message);
}
}
void StartupSdManager::forceSpiDeselected() {
pinMode(tbeam_supreme::sdCs(), OUTPUT);
digitalWrite(tbeam_supreme::sdCs(), HIGH);
pinMode(tbeam_supreme::imuCs(), OUTPUT);
digitalWrite(tbeam_supreme::imuCs(), HIGH);
}
void StartupSdManager::dumpSdPins(const char* tag) {
if (!cfg_.enablePinDumps) {
(void)tag;
return;
}
const gpio_num_t cs = (gpio_num_t)tbeam_supreme::sdCs();
const gpio_num_t sck = (gpio_num_t)tbeam_supreme::sdSck();
const gpio_num_t miso = (gpio_num_t)tbeam_supreme::sdMiso();
const gpio_num_t mosi = (gpio_num_t)tbeam_supreme::sdMosi();
logf("PINS(%s): CS=%d SCK=%d MISO=%d MOSI=%d",
tag, gpio_get_level(cs), gpio_get_level(sck), gpio_get_level(miso), gpio_get_level(mosi));
}
bool StartupSdManager::initPmuForSdPower() {
if (!tbeam_supreme::initPmuForPeripherals(pmu_, &serial_)) {
logf("ERROR: PMU init failed");
return false;
}
return true;
}
void StartupSdManager::cycleSdRail(uint32_t offMs, uint32_t onSettleMs) {
if (!cfg_.enableSdRailCycle) {
return;
}
if (!pmu_) {
logf("SD rail cycle skipped: pmu=null");
return;
}
forceSpiDeselected();
pmu_->disablePowerOutput(XPOWERS_BLDO1);
delay(offMs);
pmu_->setPowerChannelVoltage(XPOWERS_BLDO1, 3300);
pmu_->enablePowerOutput(XPOWERS_BLDO1);
delay(onSettleMs);
}
bool StartupSdManager::tryMountWithBus(SPIClass& bus, const char* busName, uint32_t hz, bool verbose) {
SD.end();
bus.end();
delay(10);
forceSpiDeselected();
bus.begin(tbeam_supreme::sdSck(), tbeam_supreme::sdMiso(), tbeam_supreme::sdMosi(), tbeam_supreme::sdCs());
digitalWrite(tbeam_supreme::sdCs(), HIGH);
delay(2);
for (int i = 0; i < 10; i++) {
bus.transfer(0xFF);
}
delay(2);
if (verbose) {
logf("SD: trying bus=%s freq=%lu Hz", busName, (unsigned long)hz);
}
if (!SD.begin(tbeam_supreme::sdCs(), bus, hz)) {
if (verbose) {
logf("SD: mount failed (possible non-FAT format, power, or bus issue)");
}
return false;
}
if (SD.cardType() == CARD_NONE) {
SD.end();
return false;
}
sdSpi_ = &bus;
sdBusName_ = busName;
sdFreq_ = hz;
return true;
}
bool StartupSdManager::mountPreferred(bool verbose) {
return tryMountWithBus(sdSpiH_, "HSPI", 400000, verbose);
}
bool StartupSdManager::mountCardFullScan() {
const uint32_t freqs[] = {400000, 1000000, 4000000, 10000000};
for (uint8_t i = 0; i < (sizeof(freqs) / sizeof(freqs[0])); ++i) {
if (tryMountWithBus(sdSpiH_, "HSPI", freqs[i], true)) {
logf("SD: card detected and mounted");
return true;
}
}
for (uint8_t i = 0; i < (sizeof(freqs) / sizeof(freqs[0])); ++i) {
if (tryMountWithBus(sdSpiF_, "FSPI", freqs[i], true)) {
logf("SD: card detected and mounted");
return true;
}
}
logf("SD: begin() failed on all bus/frequency attempts");
return false;
}
bool StartupSdManager::verifyMountedCard() {
File root = SD.open("/", FILE_READ);
if (!root) {
return false;
}
root.close();
return true;
}
const char* StartupSdManager::cardTypeToString(uint8_t type) {
switch (type) {
case CARD_MMC:
return "MMC";
case CARD_SD:
return "SDSC";
case CARD_SDHC:
return "SDHC/SDXC";
default:
return "UNKNOWN";
}
}
void StartupSdManager::printCardInfo() {
uint8_t cardType = SD.cardType();
uint64_t cardSizeMB = SD.cardSize() / (1024ULL * 1024ULL);
uint64_t totalMB = SD.totalBytes() / (1024ULL * 1024ULL);
uint64_t usedMB = SD.usedBytes() / (1024ULL * 1024ULL);
logf("SD type: %s", cardTypeToString(cardType));
logf("SD size: %llu MB", cardSizeMB);
logf("FS total: %llu MB", totalMB);
logf("FS used : %llu MB", usedMB);
logf("SPI bus: %s @ %lu Hz", sdBusName_, (unsigned long)sdFreq_);
}
bool StartupSdManager::ensureDirRecursive(const char* path) {
String full(path);
if (!full.startsWith("/")) {
full = "/" + full;
}
int start = 1;
while (start > 0 && start < (int)full.length()) {
int slash = full.indexOf('/', start);
String partial = (slash < 0) ? full : full.substring(0, slash);
if (!SD.exists(partial.c_str()) && !SD.mkdir(partial.c_str())) {
logf("ERROR: mkdir failed for %s", partial.c_str());
return false;
}
if (slash < 0) {
break;
}
start = slash + 1;
}
return true;
}
bool StartupSdManager::rewriteFile(const char* path, const char* payload) {
if (SD.exists(path) && !SD.remove(path)) {
logf("ERROR: failed to erase %s", path);
return false;
}
File f = SD.open(path, FILE_WRITE);
if (!f) {
logf("ERROR: failed to create %s", path);
return false;
}
size_t wrote = f.println(payload);
f.close();
if (wrote == 0) {
logf("ERROR: write failed for %s", path);
return false;
}
return true;
}
void StartupSdManager::permissionsDemo(const char* path) {
logf("Permissions demo: FAT has no Unix chmod/chown, use open mode only.");
File r = SD.open(path, FILE_READ);
if (!r) {
logf("Could not open %s as FILE_READ", path);
return;
}
size_t writeInReadMode = r.print("attempt write while opened read-only");
if (writeInReadMode == 0) {
logf("As expected, FILE_READ write was blocked.");
} else {
logf("NOTE: FILE_READ write returned %u (unexpected)", (unsigned)writeInReadMode);
}
r.close();
}
void StartupSdManager::setStateMounted() {
if (watchState_ != SdWatchState::MOUNTED) {
logf("EVENT: card inserted/mounted");
mountedEventPending_ = true;
notify(SdEvent::CARD_MOUNTED, "SD card mounted");
}
watchState_ = SdWatchState::MOUNTED;
}
void StartupSdManager::setStateAbsent() {
if (watchState_ == SdWatchState::MOUNTED) {
logf("EVENT: card removed/unavailable");
removedEventPending_ = true;
notify(SdEvent::CARD_REMOVED, "SD card removed");
} else if (watchState_ != SdWatchState::ABSENT) {
logf("EVENT: no card detected");
notify(SdEvent::NO_CARD, "Missing SD card or invalid FAT16/FAT32 format");
}
SD.end();
watchState_ = SdWatchState::ABSENT;
}

View file

@ -0,0 +1,90 @@
#pragma once
#include <Arduino.h>
#include <SD.h>
#include <SPI.h>
#include <Wire.h>
#include "tbeam_supreme_adapter.h"
enum class SdWatchState : uint8_t {
UNKNOWN = 0,
ABSENT,
MOUNTED
};
enum class SdEvent : uint8_t {
NO_CARD,
CARD_MOUNTED,
CARD_REMOVED
};
using SdStatusCallback = void (*)(SdEvent event, const char* message);
struct SdWatcherConfig {
bool enableSdRailCycle = true;
bool enablePinDumps = true;
bool recoveryRailCycleOnFullScan = true;
uint32_t recoveryRailOffMs = 250;
uint32_t recoveryRailOnSettleMs = 700;
uint32_t startupWarmupMs = 1500;
uint32_t pollIntervalAbsentMs = 1000;
uint32_t pollIntervalMountedMs = 2000;
uint32_t fullScanIntervalMs = 10000;
uint8_t votesToPresent = 2;
uint8_t votesToAbsent = 5;
};
class StartupSdManager {
public:
explicit StartupSdManager(Print& serial = Serial);
bool begin(const SdWatcherConfig& cfg, SdStatusCallback callback = nullptr);
void update();
bool isMounted() const { return watchState_ == SdWatchState::MOUNTED; }
SdWatchState state() const { return watchState_; }
bool consumeMountedEvent();
bool consumeRemovedEvent();
void printCardInfo();
bool ensureDirRecursive(const char* path);
bool rewriteFile(const char* path, const char* payload);
void permissionsDemo(const char* path);
private:
void logf(const char* fmt, ...);
void notify(SdEvent event, const char* message);
void forceSpiDeselected();
void dumpSdPins(const char* tag);
bool initPmuForSdPower();
void cycleSdRail(uint32_t offMs = 250, uint32_t onSettleMs = 600);
bool tryMountWithBus(SPIClass& bus, const char* busName, uint32_t hz, bool verbose);
bool mountPreferred(bool verbose);
bool mountCardFullScan();
bool verifyMountedCard();
const char* cardTypeToString(uint8_t type);
void setStateMounted();
void setStateAbsent();
Print& serial_;
SdWatcherConfig cfg_{};
SdStatusCallback callback_ = nullptr;
SPIClass sdSpiH_{HSPI};
SPIClass sdSpiF_{FSPI};
SPIClass* sdSpi_ = nullptr;
const char* sdBusName_ = "none";
uint32_t sdFreq_ = 0;
XPowersLibInterface* pmu_ = nullptr;
SdWatchState watchState_ = SdWatchState::UNKNOWN;
uint8_t presentVotes_ = 0;
uint8_t absentVotes_ = 0;
uint32_t lastPollMs_ = 0;
uint32_t lastFullScanMs_ = 0;
uint32_t logSeq_ = 0;
bool mountedEventPending_ = false;
bool removedEventPending_ = false;
};

View file

@ -0,0 +1,12 @@
{
"name": "startup_sd",
"version": "0.1.0",
"dependencies": [
{
"name": "XPowersLib"
},
{
"name": "Wire"
}
]
}