Image for T-Beam is in good working shape, restructuring the Perl data importer to deal with the 44 columns using hashes rather than positions

exercises/18_GPS_Field_QA/lib/field_qa/Config.h

@@ -38,6 +38,10 @@
 #define GPS_TX_PIN 8
 #endif
 
+#ifndef BUTTON_PIN
+#define BUTTON_PIN 0
+#endif
+
 #ifndef FW_BUILD_UTC
 #define FW_BUILD_UTC unknown
 #endif

exercises/18_GPS_Field_QA/lib/field_qa/DisplayManager.cpp

@@ -48,7 +48,7 @@ void DisplayManager::showError(const char* line1, const char* line2) {
   drawLines(kExerciseName, "ERROR", line1, line2);
 }
 
-void DisplayManager::showSample(const GnssSample& sample, const RunStats& stats) {
+void DisplayManager::showSample(const GnssSample& sample, const RunStats& stats, bool recording) {
   char l1[24];
   char l2[20];
   char l3[20];
@@ -56,7 +56,7 @@ void DisplayManager::showSample(const GnssSample& sample, const RunStats& stats)
   char l5[20];
   char l6[20];
 
-  snprintf(l1, sizeof(l1), "%s %.5s", __DATE__, __TIME__);
+  snprintf(l1, sizeof(l1), "%s", recording ? "*RECORDING" : "Halted");
   snprintf(l2, sizeof(l2), "FIX: %s", fixTypeToString(sample.fixType));
   snprintf(l3, sizeof(l3), "USED: %d/%d", sample.satsUsed < 0 ? 0 : sample.satsUsed, sample.satsInView < 0 ? 0 : sample.satsInView);
   if (sample.validHdop) {
@@ -70,4 +70,3 @@ void DisplayManager::showSample(const GnssSample& sample, const RunStats& stats)
 }
 
 }  // namespace field_qa
-

exercises/18_GPS_Field_QA/lib/field_qa/DisplayManager.h

@@ -12,7 +12,7 @@ class DisplayManager {
   void begin();
   void showBoot(const char* line2, const char* line3 = nullptr);
   void showError(const char* line1, const char* line2 = nullptr);
-  void showSample(const GnssSample& sample, const RunStats& stats);
+  void showSample(const GnssSample& sample, const RunStats& stats, bool recording);
 
  private:
   void drawLines(const char* l1,
@@ -26,4 +26,3 @@ class DisplayManager {
 };
 
 }  // namespace field_qa
-

exercises/18_GPS_Field_QA/lib/field_qa/StorageManager.cpp

@@ -63,6 +63,7 @@ bool StorageManager::startLog(const char* runId, const char* bootTimestampUtc) {
   m_ready = false;
   m_lastError = "";
   m_path = makeFilePath(runId);
+  m_newFile = !SD.exists(m_path.c_str());
   if (!ensureDir() || !openFile()) {
     return false;
   }
@@ -148,10 +149,24 @@ bool StorageManager::ensureDir() {
 }
 
 String StorageManager::makeFilePath(const char* runId) const {
-  char path[96];
+  char basePath[96];
+  char candidatePath[112];
   const char* rid = (runId && runId[0] != '\0') ? runId : "run";
-  snprintf(path, sizeof(path), "%s/%s.csv", kLogDir, rid);
+  snprintf(basePath, sizeof(basePath), "%s/%s", kLogDir, rid);
-  return String(path);
+  snprintf(candidatePath, sizeof(candidatePath), "%s.csv", basePath);
+  if (!SD.exists(candidatePath)) {
+    return String(candidatePath);
+  }
+
+  for (uint16_t suffix = 2; suffix < 1000; ++suffix) {
+    snprintf(candidatePath, sizeof(candidatePath), "%s_%02u.csv", basePath, (unsigned)suffix);
+    if (!SD.exists(candidatePath)) {
+      return String(candidatePath);
+    }
+  }
+
+  snprintf(candidatePath, sizeof(candidatePath), "%s_overflow.csv", basePath);
+  return String(candidatePath);
 }
 
 bool StorageManager::openFile() {
@@ -164,7 +179,7 @@ bool StorageManager::openFile() {
 }
 
 void StorageManager::writeHeader(const char* runId, const char* bootTimestampUtc) {
-  if (!m_file || m_file.size() > 0) {
+  if (!m_file || !m_newFile) {
     return;
   }
   m_file.printf("# exercise: %s\n", kExerciseName);
@@ -179,6 +194,7 @@ void StorageManager::writeHeader(const char* runId, const char* bootTimestampUtc
   m_file.printf("# created_by: ChatGPT/Codex handoff\n");
   m_file.print("record_type,timestamp_utc,sample_seq,ms_since_run_start,board_id,gnss_chip,firmware_exercise_name,firmware_version,boot_timestamp_utc,run_id,fix_type,fix_dimension,sats_in_view,sat_seen,sats_used,hdop,vdop,pdop,latitude,longitude,altitude_m,speed_mps,course_deg,pps_seen,quality_class,gps_count,galileo_count,glonass_count,beidou_count,navic_count,qzss_count,sbas_count,mean_cn0,max_cn0,age_of_fix_ms,ttff_ms,longest_no_fix_ms,sat_talker,sat_constellation,sat_prn,sat_elevation_deg,sat_azimuth_deg,sat_snr,sat_used_in_solution\n");
   m_file.flush();
+  m_newFile = false;
 }
 
 bool StorageManager::writePendingBuffer() {
@@ -241,14 +257,12 @@ bool StorageManager::appendBytes(const char* data, size_t len) {
 }
 
 bool StorageManager::appendLine(const String& line) {
-  if (!appendBytes(line.c_str(), line.length())) {
+  if (line.endsWith("\n")) {
-    return false;
+    return appendBytes(line.c_str(), line.length());
   }
-  if (!line.endsWith("\n")) {
+  String record = line;
-    static const char newline = '\n';
+  record += '\n';
-    return appendBytes(&newline, 1);
+  return appendBytes(record.c_str(), record.length());
-  }
-  return true;
 }
 
 void StorageManager::appendSampleCsv(const GnssSample& sample,
@@ -468,6 +482,7 @@ void StorageManager::close() {
     m_file.close();
   }
   m_ready = false;
+  m_newFile = false;
 }
 
 bool StorageManager::normalizePath(const char* input, String& normalized) const {

exercises/18_GPS_Field_QA/lib/field_qa/StorageManager.h

@@ -50,6 +50,7 @@ class StorageManager {
   void eraseLogsRecursive(File& dir);
 
   bool m_ready = false;
+  bool m_newFile = false;
   String m_path;
   String m_lastError;
   File m_file;

exercises/18_GPS_Field_QA/lib/startup_sd/StartupSdManager.cpp

@@ -118,6 +118,25 @@ bool StartupSdManager::consumeRemovedEvent() {
   return out;
 }
 
+bool StartupSdManager::forceRemount() {
+  logf("Watcher: manual rescan requested");
+  presentVotes_ = 0;
+  absentVotes_ = 0;
+  lastPollMs_ = 0;
+  lastFullScanMs_ = millis();
+
+  cycleSdRail(cfg_.recoveryRailOffMs, cfg_.recoveryRailOnSettleMs);
+  delay(cfg_.startupWarmupMs);
+
+  if (mountCardFullScan()) {
+    setStateMounted();
+    return true;
+  }
+
+  setStateAbsent();
+  return false;
+}
+
 void StartupSdManager::logf(const char* fmt, ...) {
   char msg[196];
   va_list args;

exercises/18_GPS_Field_QA/lib/startup_sd/StartupSdManager.h

@@ -46,6 +46,7 @@ class StartupSdManager {
 
   bool consumeMountedEvent();
   bool consumeRemovedEvent();
+  bool forceRemount();
 
   void printCardInfo();
   bool ensureDirRecursive(const char* path);

exercises/18_GPS_Field_QA/scripts/import_satellite_logs.pl

@@ -53,7 +53,7 @@ my $ENHANCED_HEADER = join ',', @ENHANCED_COLUMNS;
 
 my %opt = (
     dbname => 'satellite_data',
-    host   => 'localhost',
+    host   => 'ryzdesk',
     port   => 5432,
     schema => 'public',
 );
@@ -99,13 +99,20 @@ exit 0;
 
 sub import_file {
     my ($dbh, $file, $opt) = @_;
+    #
+    # get a fixed-length hash (fingerprint) so we do not accidently
+    # load the same file twice.
+    #
+    my $sha256 = "";
+    my $blob;
+    {
+	open my $fh, '<:raw', $file or die "Cannot open $file: $!\n";
+	local $/;
+	$blob = <$fh>;
+	close $fh;
+    }
 
-    open my $fh, '<:raw', $file or die "Cannot open $file: $!\n";
+    $sha256    = sha256_hex($blob // '');
-    local $/;
-    my $blob = <$fh>;
-    close $fh;
-
-    my $sha256    = sha256_hex($blob // '');
     my $file_size = -s $file;
 
     open my $in, '<:encoding(UTF-8)', $file or die "Cannot open $file: $!\n";
@@ -113,18 +120,19 @@ sub import_file {
     my @header_lines;
     my $csv_header_line;
     my @data_lines;
-
+    my $line_count = 0;
     while (my $line = <$in>) {
         chomp $line;
-        $line =~ s/\r\z//;
+        $line =~ s/\r//;
 
         next if $line =~ /^\s*$/ && !@data_lines && !defined $csv_header_line && !@header_lines;
-
+	$line_count++;
+	print "B Processing $line_count\n";
         if ($line =~ /^#/) {
             push @header_lines, $line;
             next;
         }
-
+	# record_type is the first entry in the column heading
         if (!defined $csv_header_line && $line =~ /^record_type,/) {
             $csv_header_line = $line;
             next;
@@ -172,8 +180,11 @@ sub import_file {
         sat_azimuth_deg sat_snr sat_used_in_solution
     );
 
+    my $col_count = 0;
     for my $col (@columns) {
-        die "Unexpected column '$col' in $file\n" if !$allowed{$col};
+	$col_count++;
+        die "Unexpected column at column \# $col_count \"$col\" in $file\nHeader line: $csv_header_line\n"
+	    if !$allowed{$col};
     }
 
     my $raw_header_text = join("\n", @header_lines);
@@ -229,11 +240,11 @@ SQL
     my ($first_ts, $last_ts, $board_id, $gnss_chip, $fw_name, $fw_ver, $boot_ts, $run_id);
 
     $dbh->begin_work;
-
+    $line_count = 0;  # reset
     for my $i (0 .. $#data_lines) {
         my $line = $data_lines[$i];
         next if $line =~ /^\s*$/;
-
+	$line_count++;
         $csv->parse($line) or die "CSV parse failed in $file line @{[$i+1]}: " . $csv->error_diag . "\n";
         my @fields = $csv->fields;
 
@@ -270,9 +281,9 @@ SQL
             $line,
         );
 
-        $sth->execute(@values);
+        $sth->execute(@values) or die "Line: $line_count ".$DBD::errstr;
     }
-
+    die "halted before commit, but after all rows processed";
     $dbh->commit;
 
     my $update_sql = <<'SQL';
@@ -313,9 +324,11 @@ SQL
 
 sub parse_header_columns {
     my ($line) = @_;
+    #print "DEBUG [".__LINE__."] header line = $line\n";
     my $csv = Text::CSV_XS->new({ binary => 1, auto_diag => 1 });
     $csv->parse($line) or die "Cannot parse header line: " . $csv->error_diag . "\n";
     my @cols = $csv->fields;
+    #print "DEBUG [".__LINE__."] columns found: ".@cols."\n";
     s/^\s+|\s+$//g for @cols;
     return @cols;
 }

exercises/18_GPS_Field_QA/src/main.cpp

@@ -47,6 +47,12 @@ bool g_webReady = false;
 size_t g_logFileCount = 0;
 uint32_t g_sampleSeq = 0;
 uint32_t g_runStartMs = 0;
+bool g_buttonPrevPressed = false;
+bool g_buttonConfirmActive = false;
+bool g_buttonHoldHandled = false;
+uint32_t g_buttonPressedMs = 0;
+uint32_t g_buttonConfirmDeadlineMs = 0;
+uint32_t g_buttonStopMessageUntilMs = 0;
 
 uint32_t g_lastSampleMs = 0;
 uint32_t g_lastFlushMs = 0;
@@ -55,6 +61,10 @@ uint32_t g_lastStatusMs = 0;
 uint32_t g_lastDisciplineAttemptMs = 0;
 volatile uint32_t g_ppsEdgeCount = 0;
 
+static constexpr uint32_t kButtonHoldPromptMs = 1500;
+static constexpr uint32_t kButtonConfirmWindowMs = 3000;
+static constexpr uint32_t kButtonStopMessageMs = 4000;
+
 void IRAM_ATTR onPpsEdge() {
   ++g_ppsEdgeCount;
 }
@@ -202,6 +212,66 @@ bool ensureStorageReady() {
   return true;
 }
 
+void stopLoggingCleanly(const char* reason) {
+  if (!g_loggingEnabled && !g_storageReady) {
+    return;
+  }
+  g_storage.flush();
+  g_storage.close();
+  g_storageReady = false;
+  g_loggingEnabled = false;
+  if (reason && reason[0] != '\0') {
+    Serial.printf("logging stopped: %s\n", reason);
+  } else {
+    Serial.println("logging stopped");
+  }
+  g_buttonStopMessageUntilMs = millis() + kButtonStopMessageMs;
+}
+
+bool rescanSdCard() {
+  g_storage.flush();
+  g_storage.close();
+  g_storageReady = false;
+  g_loggingEnabled = false;
+  const bool mounted = g_sd.forceRemount();
+  g_storageMounted = g_sd.isMounted();
+  if (mounted) {
+    g_sd.printCardInfo();
+    (void)ensureStorageReady();
+  }
+  return mounted;
+}
+
+void pollStopButton() {
+  const uint32_t now = millis();
+  const bool pressed = (digitalRead(BUTTON_PIN) == LOW);
+
+  if (g_buttonConfirmActive && (int32_t)(now - g_buttonConfirmDeadlineMs) >= 0) {
+    g_buttonConfirmActive = false;
+  }
+
+  if (pressed && !g_buttonPrevPressed) {
+    g_buttonPressedMs = now;
+    g_buttonHoldHandled = false;
+    if (g_buttonConfirmActive && g_loggingEnabled) {
+      stopLoggingCleanly("button confirm");
+      g_buttonConfirmActive = false;
+    }
+  } else if (pressed && !g_buttonHoldHandled && !g_buttonConfirmActive && g_loggingEnabled &&
+             (uint32_t)(now - g_buttonPressedMs) >= kButtonHoldPromptMs) {
+    g_buttonHoldHandled = true;
+    g_buttonConfirmActive = true;
+    g_buttonConfirmDeadlineMs = now + kButtonConfirmWindowMs;
+    g_display.showBoot("Stop recording?", "Press again in 3s");
+  }
+
+  if (!pressed && g_buttonPrevPressed) {
+    g_buttonHoldHandled = false;
+  }
+
+  g_buttonPrevPressed = pressed;
+}
+
 void handleSdStateTransitions() {
   g_sd.update();
   if (g_sd.consumeMountedEvent()) {
@@ -277,7 +347,13 @@ void sampleAndMaybeLog() {
   if ((uint32_t)(millis() - g_lastDisplayMs) >= kDisplayPeriodMs) {
     g_lastDisplayMs = millis();
     if (g_clockDisciplined) {
-      g_display.showSample(sample, g_stats);
+      if (g_buttonConfirmActive) {
+        g_display.showBoot("Stop recording?", "Press again in 3s");
+      } else if ((uint32_t)(millis() - g_buttonStopMessageUntilMs) < kButtonStopMessageMs) {
+        g_display.showBoot("Halted", "Safe to power off");
+      } else {
+        g_display.showSample(sample, g_stats, g_loggingEnabled);
+      }
     } else {
       g_display.showBoot("Waiting for GPS UTC", sample.validTime ? "Awaiting PPS" : "No valid time yet");
     }
@@ -382,6 +458,7 @@ void handleWebIndex() {
   html += "<a href='/cmd?flush=1'>flush</a> ";
   html += "<a href='/cmd?start=1'>start</a> ";
   html += "<a href='/cmd?stop=1'>stop</a> ";
+  html += "<a href='/cmd?sd_rescan=1'>sd_rescan</a> ";
   html += "<a href='/cmd?erase_logs=1'>erase_logs</a></p>";
   html += "<h2>SD Tree</h2><ul>";
 
@@ -461,9 +538,10 @@ void handleWebCommand() {
     g_storage.flush();
     response = "buffer flushed";
   } else if (g_server.hasArg("stop")) {
-    g_loggingEnabled = false;
+    stopLoggingCleanly("web stop");
-    g_storage.flush();
     response = "logging stopped";
+  } else if (g_server.hasArg("sd_rescan")) {
+    response = rescanSdCard() ? "sd mounted" : "sd rescan failed";
   } else if (g_server.hasArg("start")) {
     if (!g_clockDisciplined) {
       response = "clock not disciplined yet";
@@ -486,8 +564,10 @@ void handleWebCommand() {
     response += g_storageMounted ? "yes" : "no";
     response += "\nstorage_ready=";
     response += g_storageReady ? "yes" : "no";
+    response += "\nsd_state=";
+    response += g_storageMounted ? "mounted" : "absent";
   } else {
-    response = "commands: status flush start stop erase=<path> erase_logs=1";
+    response = "commands: status flush start stop sd_rescan erase=<path> erase_logs=1";
   }
 
   g_server.send(200, "text/plain; charset=utf-8", response);
@@ -546,9 +626,9 @@ void handleCommand(const char* line) {
       }
     }
   } else if (strcasecmp(line, "stop") == 0) {
-    g_loggingEnabled = false;
+    stopLoggingCleanly("serial stop");
-    g_storage.flush();
+  } else if (strcasecmp(line, "sd_rescan") == 0) {
-    Serial.println("logging stopped");
+    Serial.println(rescanSdCard() ? "sd mounted" : "sd rescan failed");
   } else if (strcasecmp(line, "flush") == 0) {
     g_storage.flush();
     Serial.println("log buffer flushed");
@@ -579,7 +659,7 @@ void handleCommand(const char* line) {
     g_lastDisciplineAttemptMs = 0;
     Serial.println("clock discipline requested");
   } else {
-    Serial.println("commands: status quiet verbose flush start stop summary ls cat <path> erase <path> erase_logs discipline");
+    Serial.println("commands: status quiet verbose flush start stop sd_rescan summary ls cat <path> erase <path> erase_logs discipline");
   }
 }
 
@@ -621,6 +701,7 @@ void setup() {
   }
 
   g_display.begin();
+  pinMode(BUTTON_PIN, INPUT_PULLUP);
   g_display.showBoot("Booting...", kBoardId);
   g_stats.begin(millis());
   g_gnss.begin();
@@ -659,6 +740,7 @@ void setup() {
 
 void loop() {
   pollSerialConsole();
+  pollStopButton();
   g_gnss.poll();
   handleSdStateTransitions();
   g_server.handleClient();