From d3043533cef07f53a057eb24034b6cf3363d5a2e Mon Sep 17 00:00:00 2001 From: John Poole Date: Mon, 6 Apr 2026 16:42:35 -0700 Subject: [PATCH] Image for T-Beam is in good working shape, restructuring the Perl data importer to deal with the 44 columns using hashes rather than positions --- .../18_GPS_Field_QA/lib/field_qa/Config.h | 4 + .../lib/field_qa/DisplayManager.cpp | 5 +- .../lib/field_qa/DisplayManager.h | 3 +- .../lib/field_qa/StorageManager.cpp | 37 ++++--- .../lib/field_qa/StorageManager.h | 1 + .../lib/startup_sd/StartupSdManager.cpp | 19 ++++ .../lib/startup_sd/StartupSdManager.h | 1 + .../scripts/import_satellite_logs.pl | 47 +++++---- exercises/18_GPS_Field_QA/src/main.cpp | 98 +++++++++++++++++-- 9 files changed, 174 insertions(+), 41 deletions(-) diff --git a/exercises/18_GPS_Field_QA/lib/field_qa/Config.h b/exercises/18_GPS_Field_QA/lib/field_qa/Config.h index 9eb7ff5..c15297a 100644 --- a/exercises/18_GPS_Field_QA/lib/field_qa/Config.h +++ b/exercises/18_GPS_Field_QA/lib/field_qa/Config.h @@ -38,6 +38,10 @@ #define GPS_TX_PIN 8 #endif +#ifndef BUTTON_PIN +#define BUTTON_PIN 0 +#endif + #ifndef FW_BUILD_UTC #define FW_BUILD_UTC unknown #endif diff --git a/exercises/18_GPS_Field_QA/lib/field_qa/DisplayManager.cpp b/exercises/18_GPS_Field_QA/lib/field_qa/DisplayManager.cpp index e4f6ec1..4a3cc42 100644 --- a/exercises/18_GPS_Field_QA/lib/field_qa/DisplayManager.cpp +++ b/exercises/18_GPS_Field_QA/lib/field_qa/DisplayManager.cpp @@ -48,7 +48,7 @@ void DisplayManager::showError(const char* line1, const char* line2) { drawLines(kExerciseName, "ERROR", line1, line2); } -void DisplayManager::showSample(const GnssSample& sample, const RunStats& stats) { +void DisplayManager::showSample(const GnssSample& sample, const RunStats& stats, bool recording) { char l1[24]; char l2[20]; char l3[20]; @@ -56,7 +56,7 @@ void DisplayManager::showSample(const GnssSample& sample, const RunStats& stats) char l5[20]; char l6[20]; - snprintf(l1, sizeof(l1), "%s %.5s", __DATE__, __TIME__); + snprintf(l1, sizeof(l1), "%s", recording ? "*RECORDING" : "Halted"); snprintf(l2, sizeof(l2), "FIX: %s", fixTypeToString(sample.fixType)); snprintf(l3, sizeof(l3), "USED: %d/%d", sample.satsUsed < 0 ? 0 : sample.satsUsed, sample.satsInView < 0 ? 0 : sample.satsInView); if (sample.validHdop) { @@ -70,4 +70,3 @@ void DisplayManager::showSample(const GnssSample& sample, const RunStats& stats) } } // namespace field_qa - diff --git a/exercises/18_GPS_Field_QA/lib/field_qa/DisplayManager.h b/exercises/18_GPS_Field_QA/lib/field_qa/DisplayManager.h index 93dec34..2859675 100644 --- a/exercises/18_GPS_Field_QA/lib/field_qa/DisplayManager.h +++ b/exercises/18_GPS_Field_QA/lib/field_qa/DisplayManager.h @@ -12,7 +12,7 @@ class DisplayManager { void begin(); void showBoot(const char* line2, const char* line3 = nullptr); void showError(const char* line1, const char* line2 = nullptr); - void showSample(const GnssSample& sample, const RunStats& stats); + void showSample(const GnssSample& sample, const RunStats& stats, bool recording); private: void drawLines(const char* l1, @@ -26,4 +26,3 @@ class DisplayManager { }; } // namespace field_qa - diff --git a/exercises/18_GPS_Field_QA/lib/field_qa/StorageManager.cpp b/exercises/18_GPS_Field_QA/lib/field_qa/StorageManager.cpp index de207bc..d368233 100644 --- a/exercises/18_GPS_Field_QA/lib/field_qa/StorageManager.cpp +++ b/exercises/18_GPS_Field_QA/lib/field_qa/StorageManager.cpp @@ -63,6 +63,7 @@ bool StorageManager::startLog(const char* runId, const char* bootTimestampUtc) { m_ready = false; m_lastError = ""; m_path = makeFilePath(runId); + m_newFile = !SD.exists(m_path.c_str()); if (!ensureDir() || !openFile()) { return false; } @@ -148,10 +149,24 @@ bool StorageManager::ensureDir() { } String StorageManager::makeFilePath(const char* runId) const { - char path[96]; + char basePath[96]; + char candidatePath[112]; const char* rid = (runId && runId[0] != '\0') ? runId : "run"; - snprintf(path, sizeof(path), "%s/%s.csv", kLogDir, rid); - return String(path); + snprintf(basePath, sizeof(basePath), "%s/%s", kLogDir, rid); + snprintf(candidatePath, sizeof(candidatePath), "%s.csv", basePath); + if (!SD.exists(candidatePath)) { + return String(candidatePath); + } + + for (uint16_t suffix = 2; suffix < 1000; ++suffix) { + snprintf(candidatePath, sizeof(candidatePath), "%s_%02u.csv", basePath, (unsigned)suffix); + if (!SD.exists(candidatePath)) { + return String(candidatePath); + } + } + + snprintf(candidatePath, sizeof(candidatePath), "%s_overflow.csv", basePath); + return String(candidatePath); } bool StorageManager::openFile() { @@ -164,7 +179,7 @@ bool StorageManager::openFile() { } void StorageManager::writeHeader(const char* runId, const char* bootTimestampUtc) { - if (!m_file || m_file.size() > 0) { + if (!m_file || !m_newFile) { return; } m_file.printf("# exercise: %s\n", kExerciseName); @@ -179,6 +194,7 @@ void StorageManager::writeHeader(const char* runId, const char* bootTimestampUtc m_file.printf("# created_by: ChatGPT/Codex handoff\n"); m_file.print("record_type,timestamp_utc,sample_seq,ms_since_run_start,board_id,gnss_chip,firmware_exercise_name,firmware_version,boot_timestamp_utc,run_id,fix_type,fix_dimension,sats_in_view,sat_seen,sats_used,hdop,vdop,pdop,latitude,longitude,altitude_m,speed_mps,course_deg,pps_seen,quality_class,gps_count,galileo_count,glonass_count,beidou_count,navic_count,qzss_count,sbas_count,mean_cn0,max_cn0,age_of_fix_ms,ttff_ms,longest_no_fix_ms,sat_talker,sat_constellation,sat_prn,sat_elevation_deg,sat_azimuth_deg,sat_snr,sat_used_in_solution\n"); m_file.flush(); + m_newFile = false; } bool StorageManager::writePendingBuffer() { @@ -241,14 +257,12 @@ bool StorageManager::appendBytes(const char* data, size_t len) { } bool StorageManager::appendLine(const String& line) { - if (!appendBytes(line.c_str(), line.length())) { - return false; + if (line.endsWith("\n")) { + return appendBytes(line.c_str(), line.length()); } - if (!line.endsWith("\n")) { - static const char newline = '\n'; - return appendBytes(&newline, 1); - } - return true; + String record = line; + record += '\n'; + return appendBytes(record.c_str(), record.length()); } void StorageManager::appendSampleCsv(const GnssSample& sample, @@ -468,6 +482,7 @@ void StorageManager::close() { m_file.close(); } m_ready = false; + m_newFile = false; } bool StorageManager::normalizePath(const char* input, String& normalized) const { diff --git a/exercises/18_GPS_Field_QA/lib/field_qa/StorageManager.h b/exercises/18_GPS_Field_QA/lib/field_qa/StorageManager.h index 650f66d..5acf8cf 100644 --- a/exercises/18_GPS_Field_QA/lib/field_qa/StorageManager.h +++ b/exercises/18_GPS_Field_QA/lib/field_qa/StorageManager.h @@ -50,6 +50,7 @@ class StorageManager { void eraseLogsRecursive(File& dir); bool m_ready = false; + bool m_newFile = false; String m_path; String m_lastError; File m_file; diff --git a/exercises/18_GPS_Field_QA/lib/startup_sd/StartupSdManager.cpp b/exercises/18_GPS_Field_QA/lib/startup_sd/StartupSdManager.cpp index 1e8791c..f768cb4 100644 --- a/exercises/18_GPS_Field_QA/lib/startup_sd/StartupSdManager.cpp +++ b/exercises/18_GPS_Field_QA/lib/startup_sd/StartupSdManager.cpp @@ -118,6 +118,25 @@ bool StartupSdManager::consumeRemovedEvent() { return out; } +bool StartupSdManager::forceRemount() { + logf("Watcher: manual rescan requested"); + presentVotes_ = 0; + absentVotes_ = 0; + lastPollMs_ = 0; + lastFullScanMs_ = millis(); + + cycleSdRail(cfg_.recoveryRailOffMs, cfg_.recoveryRailOnSettleMs); + delay(cfg_.startupWarmupMs); + + if (mountCardFullScan()) { + setStateMounted(); + return true; + } + + setStateAbsent(); + return false; +} + void StartupSdManager::logf(const char* fmt, ...) { char msg[196]; va_list args; diff --git a/exercises/18_GPS_Field_QA/lib/startup_sd/StartupSdManager.h b/exercises/18_GPS_Field_QA/lib/startup_sd/StartupSdManager.h index be9ef27..9a10cfd 100644 --- a/exercises/18_GPS_Field_QA/lib/startup_sd/StartupSdManager.h +++ b/exercises/18_GPS_Field_QA/lib/startup_sd/StartupSdManager.h @@ -46,6 +46,7 @@ class StartupSdManager { bool consumeMountedEvent(); bool consumeRemovedEvent(); + bool forceRemount(); void printCardInfo(); bool ensureDirRecursive(const char* path); diff --git a/exercises/18_GPS_Field_QA/scripts/import_satellite_logs.pl b/exercises/18_GPS_Field_QA/scripts/import_satellite_logs.pl index 4481210..aa7fc1e 100644 --- a/exercises/18_GPS_Field_QA/scripts/import_satellite_logs.pl +++ b/exercises/18_GPS_Field_QA/scripts/import_satellite_logs.pl @@ -53,7 +53,7 @@ my $ENHANCED_HEADER = join ',', @ENHANCED_COLUMNS; my %opt = ( dbname => 'satellite_data', - host => 'localhost', + host => 'ryzdesk', port => 5432, schema => 'public', ); @@ -99,32 +99,40 @@ exit 0; sub import_file { my ($dbh, $file, $opt) = @_; + # + # get a fixed-length hash (fingerprint) so we do not accidently + # load the same file twice. + # + my $sha256 = ""; + my $blob; + { + open my $fh, '<:raw', $file or die "Cannot open $file: $!\n"; + local $/; + $blob = <$fh>; + close $fh; + } - open my $fh, '<:raw', $file or die "Cannot open $file: $!\n"; - local $/; - my $blob = <$fh>; - close $fh; - - my $sha256 = sha256_hex($blob // ''); + $sha256 = sha256_hex($blob // ''); my $file_size = -s $file; open my $in, '<:encoding(UTF-8)', $file or die "Cannot open $file: $!\n"; - + my @header_lines; my $csv_header_line; my @data_lines; - + my $line_count = 0; while (my $line = <$in>) { chomp $line; - $line =~ s/\r\z//; + $line =~ s/\r//; next if $line =~ /^\s*$/ && !@data_lines && !defined $csv_header_line && !@header_lines; - + $line_count++; + print "B Processing $line_count\n"; if ($line =~ /^#/) { push @header_lines, $line; next; } - + # record_type is the first entry in the column heading if (!defined $csv_header_line && $line =~ /^record_type,/) { $csv_header_line = $line; next; @@ -172,8 +180,11 @@ sub import_file { sat_azimuth_deg sat_snr sat_used_in_solution ); + my $col_count = 0; for my $col (@columns) { - die "Unexpected column '$col' in $file\n" if !$allowed{$col}; + $col_count++; + die "Unexpected column at column \# $col_count \"$col\" in $file\nHeader line: $csv_header_line\n" + if !$allowed{$col}; } my $raw_header_text = join("\n", @header_lines); @@ -229,11 +240,11 @@ SQL my ($first_ts, $last_ts, $board_id, $gnss_chip, $fw_name, $fw_ver, $boot_ts, $run_id); $dbh->begin_work; - + $line_count = 0; # reset for my $i (0 .. $#data_lines) { my $line = $data_lines[$i]; next if $line =~ /^\s*$/; - + $line_count++; $csv->parse($line) or die "CSV parse failed in $file line @{[$i+1]}: " . $csv->error_diag . "\n"; my @fields = $csv->fields; @@ -270,9 +281,9 @@ SQL $line, ); - $sth->execute(@values); + $sth->execute(@values) or die "Line: $line_count ".$DBD::errstr; } - + die "halted before commit, but after all rows processed"; $dbh->commit; my $update_sql = <<'SQL'; @@ -313,9 +324,11 @@ SQL sub parse_header_columns { my ($line) = @_; + #print "DEBUG [".__LINE__."] header line = $line\n"; my $csv = Text::CSV_XS->new({ binary => 1, auto_diag => 1 }); $csv->parse($line) or die "Cannot parse header line: " . $csv->error_diag . "\n"; my @cols = $csv->fields; + #print "DEBUG [".__LINE__."] columns found: ".@cols."\n"; s/^\s+|\s+$//g for @cols; return @cols; } diff --git a/exercises/18_GPS_Field_QA/src/main.cpp b/exercises/18_GPS_Field_QA/src/main.cpp index cbf176a..3b515d8 100644 --- a/exercises/18_GPS_Field_QA/src/main.cpp +++ b/exercises/18_GPS_Field_QA/src/main.cpp @@ -47,6 +47,12 @@ bool g_webReady = false; size_t g_logFileCount = 0; uint32_t g_sampleSeq = 0; uint32_t g_runStartMs = 0; +bool g_buttonPrevPressed = false; +bool g_buttonConfirmActive = false; +bool g_buttonHoldHandled = false; +uint32_t g_buttonPressedMs = 0; +uint32_t g_buttonConfirmDeadlineMs = 0; +uint32_t g_buttonStopMessageUntilMs = 0; uint32_t g_lastSampleMs = 0; uint32_t g_lastFlushMs = 0; @@ -55,6 +61,10 @@ uint32_t g_lastStatusMs = 0; uint32_t g_lastDisciplineAttemptMs = 0; volatile uint32_t g_ppsEdgeCount = 0; +static constexpr uint32_t kButtonHoldPromptMs = 1500; +static constexpr uint32_t kButtonConfirmWindowMs = 3000; +static constexpr uint32_t kButtonStopMessageMs = 4000; + void IRAM_ATTR onPpsEdge() { ++g_ppsEdgeCount; } @@ -202,6 +212,66 @@ bool ensureStorageReady() { return true; } +void stopLoggingCleanly(const char* reason) { + if (!g_loggingEnabled && !g_storageReady) { + return; + } + g_storage.flush(); + g_storage.close(); + g_storageReady = false; + g_loggingEnabled = false; + if (reason && reason[0] != '\0') { + Serial.printf("logging stopped: %s\n", reason); + } else { + Serial.println("logging stopped"); + } + g_buttonStopMessageUntilMs = millis() + kButtonStopMessageMs; +} + +bool rescanSdCard() { + g_storage.flush(); + g_storage.close(); + g_storageReady = false; + g_loggingEnabled = false; + const bool mounted = g_sd.forceRemount(); + g_storageMounted = g_sd.isMounted(); + if (mounted) { + g_sd.printCardInfo(); + (void)ensureStorageReady(); + } + return mounted; +} + +void pollStopButton() { + const uint32_t now = millis(); + const bool pressed = (digitalRead(BUTTON_PIN) == LOW); + + if (g_buttonConfirmActive && (int32_t)(now - g_buttonConfirmDeadlineMs) >= 0) { + g_buttonConfirmActive = false; + } + + if (pressed && !g_buttonPrevPressed) { + g_buttonPressedMs = now; + g_buttonHoldHandled = false; + if (g_buttonConfirmActive && g_loggingEnabled) { + stopLoggingCleanly("button confirm"); + g_buttonConfirmActive = false; + } + } else if (pressed && !g_buttonHoldHandled && !g_buttonConfirmActive && g_loggingEnabled && + (uint32_t)(now - g_buttonPressedMs) >= kButtonHoldPromptMs) { + g_buttonHoldHandled = true; + g_buttonConfirmActive = true; + g_buttonConfirmDeadlineMs = now + kButtonConfirmWindowMs; + g_display.showBoot("Stop recording?", "Press again in 3s"); + } + + if (!pressed && g_buttonPrevPressed) { + g_buttonHoldHandled = false; + } + + g_buttonPrevPressed = pressed; +} + void handleSdStateTransitions() { g_sd.update(); if (g_sd.consumeMountedEvent()) { @@ -277,7 +347,13 @@ void sampleAndMaybeLog() { if ((uint32_t)(millis() - g_lastDisplayMs) >= kDisplayPeriodMs) { g_lastDisplayMs = millis(); if (g_clockDisciplined) { - g_display.showSample(sample, g_stats); + if (g_buttonConfirmActive) { + g_display.showBoot("Stop recording?", "Press again in 3s"); + } else if ((uint32_t)(millis() - g_buttonStopMessageUntilMs) < kButtonStopMessageMs) { + g_display.showBoot("Halted", "Safe to power off"); + } else { + g_display.showSample(sample, g_stats, g_loggingEnabled); + } } else { g_display.showBoot("Waiting for GPS UTC", sample.validTime ? "Awaiting PPS" : "No valid time yet"); } @@ -382,6 +458,7 @@ void handleWebIndex() { html += "flush "; html += "start "; html += "stop "; + html += "sd_rescan "; html += "erase_logs

"; html += "

SD Tree