jlpoole/microReticulumTbeam
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Image for T-Beam is in good working shape, restructuring the Perl data importer to deal with the 44 columns using hashes rather than positions

Browse source
This commit is contained in:
John Poole 2026-04-06 16:42:35 -07:00
commit d3043533ce
9 changed files with 174 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

4
exercises/18_GPS_Field_QA/lib/field_qa/Config.h
View file

@ -38,6 +38,10 @@
#define GPS_TX_PIN 8
#endif
#ifndef BUTTON_PIN
#define BUTTON_PIN 0
#endif
#ifndef FW_BUILD_UTC
#define FW_BUILD_UTC unknown
#endif

5
exercises/18_GPS_Field_QA/lib/field_qa/DisplayManager.cpp
View file

@ -48,7 +48,7 @@ void DisplayManager::showError(const char* line1, const char* line2) {
drawLines(kExerciseName, "ERROR", line1, line2);
}
void DisplayManager::showSample(const GnssSample& sample, const RunStats& stats) {
void DisplayManager::showSample(const GnssSample& sample, const RunStats& stats, bool recording) {
char l1[24];
char l2[20];
char l3[20];
@ -56,7 +56,7 @@ void DisplayManager::showSample(const GnssSample& sample, const RunStats& stats)
char l5[20];
char l6[20];
snprintf(l1, sizeof(l1), "%s %.5s", __DATE__, __TIME__);
snprintf(l1, sizeof(l1), "%s", recording ? "*RECORDING" : "Halted");
snprintf(l2, sizeof(l2), "FIX: %s", fixTypeToString(sample.fixType));
snprintf(l3, sizeof(l3), "USED: %d/%d", sample.satsUsed < 0 ? 0 : sample.satsUsed, sample.satsInView < 0 ? 0 : sample.satsInView);
if (sample.validHdop) {
@ -70,4 +70,3 @@ void DisplayManager::showSample(const GnssSample& sample, const RunStats& stats)
}
} // namespace field_qa

3
exercises/18_GPS_Field_QA/lib/field_qa/DisplayManager.h
View file

@ -12,7 +12,7 @@ class DisplayManager {
void begin();
void showBoot(const char* line2, const char* line3 = nullptr);
void showError(const char* line1, const char* line2 = nullptr);
void showSample(const GnssSample& sample, const RunStats& stats);
void showSample(const GnssSample& sample, const RunStats& stats, bool recording);
private:
void drawLines(const char* l1,
@ -26,4 +26,3 @@ class DisplayManager {
};
} // namespace field_qa

37
exercises/18_GPS_Field_QA/lib/field_qa/StorageManager.cpp
View file

@ -63,6 +63,7 @@ bool StorageManager::startLog(const char* runId, const char* bootTimestampUtc) {
m_ready = false;
m_lastError = "";
m_path = makeFilePath(runId);
m_newFile = !SD.exists(m_path.c_str());
if (!ensureDir() || !openFile()) {
return false;
}
@ -148,10 +149,24 @@ bool StorageManager::ensureDir() {
}
String StorageManager::makeFilePath(const char* runId) const {
char path[96];
char basePath[96];
char candidatePath[112];
const char* rid = (runId && runId[0] != '\0') ? runId : "run";
snprintf(path, sizeof(path), "%s/%s.csv", kLogDir, rid);
return String(path);
snprintf(basePath, sizeof(basePath), "%s/%s", kLogDir, rid);
snprintf(candidatePath, sizeof(candidatePath), "%s.csv", basePath);
if (!SD.exists(candidatePath)) {
return String(candidatePath);
}
for (uint16_t suffix = 2; suffix < 1000; ++suffix) {
snprintf(candidatePath, sizeof(candidatePath), "%s_%02u.csv", basePath, (unsigned)suffix);
if (!SD.exists(candidatePath)) {
return String(candidatePath);
}
}
snprintf(candidatePath, sizeof(candidatePath), "%s_overflow.csv", basePath);
return String(candidatePath);
}
bool StorageManager::openFile() {
@ -164,7 +179,7 @@ bool StorageManager::openFile() {
}
void StorageManager::writeHeader(const char* runId, const char* bootTimestampUtc) {
if (!m_file || m_file.size() > 0) {
if (!m_file || !m_newFile) {
return;
}
m_file.printf("# exercise: %s\n", kExerciseName);
@ -179,6 +194,7 @@ void StorageManager::writeHeader(const char* runId, const char* bootTimestampUtc
m_file.printf("# created_by: ChatGPT/Codex handoff\n");
m_file.print("record_type,timestamp_utc,sample_seq,ms_since_run_start,board_id,gnss_chip,firmware_exercise_name,firmware_version,boot_timestamp_utc,run_id,fix_type,fix_dimension,sats_in_view,sat_seen,sats_used,hdop,vdop,pdop,latitude,longitude,altitude_m,speed_mps,course_deg,pps_seen,quality_class,gps_count,galileo_count,glonass_count,beidou_count,navic_count,qzss_count,sbas_count,mean_cn0,max_cn0,age_of_fix_ms,ttff_ms,longest_no_fix_ms,sat_talker,sat_constellation,sat_prn,sat_elevation_deg,sat_azimuth_deg,sat_snr,sat_used_in_solution\n");
m_file.flush();
m_newFile = false;
}
bool StorageManager::writePendingBuffer() {
@ -241,14 +257,12 @@ bool StorageManager::appendBytes(const char* data, size_t len) {
}
bool StorageManager::appendLine(const String& line) {
if (!appendBytes(line.c_str(), line.length())) {
return false;
if (line.endsWith("\n")) {
return appendBytes(line.c_str(), line.length());
}
if (!line.endsWith("\n")) {
static const char newline = '\n';
return appendBytes(&newline, 1);
}
return true;
String record = line;
record += '\n';
return appendBytes(record.c_str(), record.length());
}
void StorageManager::appendSampleCsv(const GnssSample& sample,
@ -468,6 +482,7 @@ void StorageManager::close() {
m_file.close();
}
m_ready = false;
m_newFile = false;
}
bool StorageManager::normalizePath(const char* input, String& normalized) const {

1
exercises/18_GPS_Field_QA/lib/field_qa/StorageManager.h
View file

@ -50,6 +50,7 @@ class StorageManager {
void eraseLogsRecursive(File& dir);
bool m_ready = false;
bool m_newFile = false;
String m_path;
String m_lastError;
File m_file;

19
exercises/18_GPS_Field_QA/lib/startup_sd/StartupSdManager.cpp
View file

@ -118,6 +118,25 @@ bool StartupSdManager::consumeRemovedEvent() {
return out;
}
bool StartupSdManager::forceRemount() {
logf("Watcher: manual rescan requested");
presentVotes_ = 0;
absentVotes_ = 0;
lastPollMs_ = 0;
lastFullScanMs_ = millis();
cycleSdRail(cfg_.recoveryRailOffMs, cfg_.recoveryRailOnSettleMs);
delay(cfg_.startupWarmupMs);
if (mountCardFullScan()) {
setStateMounted();
return true;
}
setStateAbsent();
return false;
}
void StartupSdManager::logf(const char* fmt, ...) {
char msg[196];
va_list args;

1
exercises/18_GPS_Field_QA/lib/startup_sd/StartupSdManager.h
View file

@ -46,6 +46,7 @@ class StartupSdManager {
bool consumeMountedEvent();
bool consumeRemovedEvent();
bool forceRemount();
void printCardInfo();
bool ensureDirRecursive(const char* path);

39
exercises/18_GPS_Field_QA/scripts/import_satellite_logs.pl
View file

@ -53,7 +53,7 @@ my $ENHANCED_HEADER = join ',', @ENHANCED_COLUMNS;
my %opt = (
dbname => 'satellite_data',
host => 'localhost',
host => 'ryzdesk',
port => 5432,
schema => 'public',
);
@ -99,13 +99,20 @@ exit 0;
sub import_file {
my ($dbh, $file, $opt) = @_;
#
# get a fixed-length hash (fingerprint) so we do not accidently
# load the same file twice.
#
my $sha256 = "";
my $blob;
{
open my $fh, '<:raw', $file or die "Cannot open $file: $!\n";
local $/;
my $blob = <$fh>;
$blob = <$fh>;
close $fh;
}
my $sha256 = sha256_hex($blob // '');
$sha256 = sha256_hex($blob // '');
my $file_size = -s $file;
open my $in, '<:encoding(UTF-8)', $file or die "Cannot open $file: $!\n";
@ -113,18 +120,19 @@ sub import_file {
my @header_lines;
my $csv_header_line;
my @data_lines;
my $line_count = 0;
while (my $line = <$in>) {
chomp $line;
$line =~ s/\r\z//;
$line =~ s/\r//;
next if $line =~ /^\s*$/ && !@data_lines && !defined $csv_header_line && !@header_lines;
$line_count++;
print "B Processing $line_count\n";
if ($line =~ /^#/) {
push @header_lines, $line;
next;
}
# record_type is the first entry in the column heading
if (!defined $csv_header_line && $line =~ /^record_type,/) {
$csv_header_line = $line;
next;
@ -172,8 +180,11 @@ sub import_file {
sat_azimuth_deg sat_snr sat_used_in_solution
);
my $col_count = 0;
for my $col (@columns) {
die "Unexpected column '$col' in $file\n" if !$allowed{$col};
$col_count++;
die "Unexpected column at column \# $col_count \"$col\" in $file\nHeader line: $csv_header_line\n"
if !$allowed{$col};
}
my $raw_header_text = join("\n", @header_lines);
@ -229,11 +240,11 @@ SQL
my ($first_ts, $last_ts, $board_id, $gnss_chip, $fw_name, $fw_ver, $boot_ts, $run_id);
$dbh->begin_work;
$line_count = 0; # reset
for my $i (0 .. $#data_lines) {
my $line = $data_lines[$i];
next if $line =~ /^\s*$/;
$line_count++;
$csv->parse($line) or die "CSV parse failed in $file line @{[$i+1]}: " . $csv->error_diag . "\n";
my @fields = $csv->fields;
@ -270,9 +281,9 @@ SQL
$line,
);
$sth->execute(@values);
$sth->execute(@values) or die "Line: $line_count ".$DBD::errstr;
}
die "halted before commit, but after all rows processed";
$dbh->commit;
my $update_sql = <<'SQL';
@ -313,9 +324,11 @@ SQL
sub parse_header_columns {
my ($line) = @_;
#print "DEBUG [".__LINE__."] header line = $line\n";
my $csv = Text::CSV_XS->new({ binary => 1, auto_diag => 1 });
$csv->parse($line) or die "Cannot parse header line: " . $csv->error_diag . "\n";
my @cols = $csv->fields;
#print "DEBUG [".__LINE__."] columns found: ".@cols."\n";
s/^\s+|\s+$//g for @cols;
return @cols;
}

98
exercises/18_GPS_Field_QA/src/main.cpp
View file

@ -47,6 +47,12 @@ bool g_webReady = false;
size_t g_logFileCount = 0;
uint32_t g_sampleSeq = 0;
uint32_t g_runStartMs = 0;
bool g_buttonPrevPressed = false;
bool g_buttonConfirmActive = false;
bool g_buttonHoldHandled = false;
uint32_t g_buttonPressedMs = 0;
uint32_t g_buttonConfirmDeadlineMs = 0;
uint32_t g_buttonStopMessageUntilMs = 0;
uint32_t g_lastSampleMs = 0;
uint32_t g_lastFlushMs = 0;
@ -55,6 +61,10 @@ uint32_t g_lastStatusMs = 0;
uint32_t g_lastDisciplineAttemptMs = 0;
volatile uint32_t g_ppsEdgeCount = 0;
static constexpr uint32_t kButtonHoldPromptMs = 1500;
static constexpr uint32_t kButtonConfirmWindowMs = 3000;
static constexpr uint32_t kButtonStopMessageMs = 4000;
void IRAM_ATTR onPpsEdge() {
++g_ppsEdgeCount;
}
@ -202,6 +212,66 @@ bool ensureStorageReady() {
return true;
}
void stopLoggingCleanly(const char* reason) {
if (!g_loggingEnabled && !g_storageReady) {
return;
}
g_storage.flush();
g_storage.close();
g_storageReady = false;
g_loggingEnabled = false;
if (reason && reason[0] != '\0') {
Serial.printf("logging stopped: %s\n", reason);
} else {
Serial.println("logging stopped");
}
g_buttonStopMessageUntilMs = millis() + kButtonStopMessageMs;
}
bool rescanSdCard() {
g_storage.flush();
g_storage.close();
g_storageReady = false;
g_loggingEnabled = false;
const bool mounted = g_sd.forceRemount();
g_storageMounted = g_sd.isMounted();
if (mounted) {
g_sd.printCardInfo();
(void)ensureStorageReady();
}
return mounted;
}
void pollStopButton() {
const uint32_t now = millis();
const bool pressed = (digitalRead(BUTTON_PIN) == LOW);
if (g_buttonConfirmActive && (int32_t)(now - g_buttonConfirmDeadlineMs) >= 0) {
g_buttonConfirmActive = false;
}
if (pressed && !g_buttonPrevPressed) {
g_buttonPressedMs = now;
g_buttonHoldHandled = false;
if (g_buttonConfirmActive && g_loggingEnabled) {
stopLoggingCleanly("button confirm");
g_buttonConfirmActive = false;
}
} else if (pressed && !g_buttonHoldHandled && !g_buttonConfirmActive && g_loggingEnabled &&
(uint32_t)(now - g_buttonPressedMs) >= kButtonHoldPromptMs) {
g_buttonHoldHandled = true;
g_buttonConfirmActive = true;
g_buttonConfirmDeadlineMs = now + kButtonConfirmWindowMs;
g_display.showBoot("Stop recording?", "Press again in 3s");
}
if (!pressed && g_buttonPrevPressed) {
g_buttonHoldHandled = false;
}
g_buttonPrevPressed = pressed;
}
void handleSdStateTransitions() {
g_sd.update();
if (g_sd.consumeMountedEvent()) {
@ -277,7 +347,13 @@ void sampleAndMaybeLog() {
if ((uint32_t)(millis() - g_lastDisplayMs) >= kDisplayPeriodMs) {
g_lastDisplayMs = millis();
if (g_clockDisciplined) {
g_display.showSample(sample, g_stats);
if (g_buttonConfirmActive) {
g_display.showBoot("Stop recording?", "Press again in 3s");
} else if ((uint32_t)(millis() - g_buttonStopMessageUntilMs) < kButtonStopMessageMs) {
g_display.showBoot("Halted", "Safe to power off");
} else {
g_display.showSample(sample, g_stats, g_loggingEnabled);
}
} else {
g_display.showBoot("Waiting for GPS UTC", sample.validTime ? "Awaiting PPS" : "No valid time yet");
}
@ -382,6 +458,7 @@ void handleWebIndex() {
html += "<a href='/cmd?flush=1'>flush</a> ";
html += "<a href='/cmd?start=1'>start</a> ";
html += "<a href='/cmd?stop=1'>stop</a> ";
html += "<a href='/cmd?sd_rescan=1'>sd_rescan</a> ";
html += "<a href='/cmd?erase_logs=1'>erase_logs</a></p>";
html += "<h2>SD Tree</h2><ul>";
@ -461,9 +538,10 @@ void handleWebCommand() {
g_storage.flush();
response = "buffer flushed";
} else if (g_server.hasArg("stop")) {
g_loggingEnabled = false;
g_storage.flush();
stopLoggingCleanly("web stop");
response = "logging stopped";
} else if (g_server.hasArg("sd_rescan")) {
response = rescanSdCard() ? "sd mounted" : "sd rescan failed";
} else if (g_server.hasArg("start")) {
if (!g_clockDisciplined) {
response = "clock not disciplined yet";
@ -486,8 +564,10 @@ void handleWebCommand() {
response += g_storageMounted ? "yes" : "no";
response += "\nstorage_ready=";
response += g_storageReady ? "yes" : "no";
response += "\nsd_state=";
response += g_storageMounted ? "mounted" : "absent";
} else {
response = "commands: status flush start stop erase=<path> erase_logs=1";
response = "commands: status flush start stop sd_rescan erase=<path> erase_logs=1";
}
g_server.send(200, "text/plain; charset=utf-8", response);
@ -546,9 +626,9 @@ void handleCommand(const char* line) {
}
}
} else if (strcasecmp(line, "stop") == 0) {
g_loggingEnabled = false;
g_storage.flush();
Serial.println("logging stopped");
stopLoggingCleanly("serial stop");
} else if (strcasecmp(line, "sd_rescan") == 0) {
Serial.println(rescanSdCard() ? "sd mounted" : "sd rescan failed");
} else if (strcasecmp(line, "flush") == 0) {
g_storage.flush();
Serial.println("log buffer flushed");
@ -579,7 +659,7 @@ void handleCommand(const char* line) {
g_lastDisciplineAttemptMs = 0;
Serial.println("clock discipline requested");
} else {
Serial.println("commands: status quiet verbose flush start stop summary ls cat <path> erase <path> erase_logs discipline");
Serial.println("commands: status quiet verbose flush start stop sd_rescan summary ls cat <path> erase <path> erase_logs discipline");
}
}
@ -621,6 +701,7 @@ void setup() {
}
g_display.begin();
pinMode(BUTTON_PIN, INPUT_PULLUP);
g_display.showBoot("Booting...", kBoardId);
g_stats.begin(millis());
g_gnss.begin();
@ -659,6 +740,7 @@ void setup() {
void loop() {
pollSerialConsole();
pollStopButton();
g_gnss.poll();
handleSdStateTransitions();
g_server.handleClient();