latest schema for 18 column logs

exercises/26_Bluetooth_discover/scripts/create_exercise_26_ble_schema.sql

@@ -2,30 +2,34 @@
 -- $HeadURL$
 --
 -- Example:
---   sqlite3 ble_fieldtest_20260525_1945_ed_flo.sqlite < create_exercise_26_ble_schema.sql
+--   sqlite3 ble_fieldtest_YYYYMMDD_HHMM.sqlite < create_exercise_26_ble_schema.sql
 --
 -- SQLite schema for Exercise 26 BLE Discovery field-test logs.
 --
--- Design goals:
+-- This schema targets the current 18-column Exercise 26 log format only.
---   1. Preserve the raw log rows.
---   2. Preserve per-log manifest metadata.
---   3. Normalize parsed BLE observations.
---   4. Keep one portable SQLite database per field trial.
---   5. Support later export to GeoJSON / MapLibre / GIS tools.
 --
 -- Current Exercise 26 log format:
 --
 --   human_time,rx_epoch_ms,receiver,rx_lat,rx_lon,gps_fix_age_ms,
 --   clock_valid,gps_valid,discipline_age_ms,last_discipline_epoch_ms,
---   heard,rssi,avg_rssi,age_s,count,seq,tx_payload_epoch_ms,payload,vbat_mv
+--   heard,rssi,avg_rssi,age_s,count,seq,tx_payload_epoch_ms,payload
 --
 -- Current payload format:
 --
 --   B2|NODE|SEQ|TX_PAYLOAD_EPOCH_MS
 --
--- Legacy payload format:
+-- Timestamp semantics:
 --
---   TBMSND|1|NODE|SEQ|UPTIME
+--   rx_epoch_ms is the receiver's disciplined-clock time when the observation
+--   row is written.
+--
+--   tx_payload_epoch_ms is the sender's disciplined-clock time when the BLE
+--   advertising payload was generated/configured.  It is not a per-RF-packet
+--   transmit timestamp.
+--
+--   rx_tx_payload_delta_ms is diagnostic only.  It is useful for clock,
+--   scheduling, payload-age, and repeated-observation analysis.  It is not
+--   BLE propagation delay.
 --
 
 PRAGMA foreign_keys = ON;
@@ -39,7 +43,7 @@ BEGIN TRANSACTION;
 --
 -- A trial may contain multiple logs, usually one log per T-Beam receiver.
 -- Example:
---   trial_label = peck_cottage_ed_flo_20260525_1945
+--   trial_label = peck_cottage_four_unit_20260526_1430
 -- ---------------------------------------------------------------------------
 
 CREATE TABLE IF NOT EXISTS trial (
@@ -66,10 +70,6 @@ CREATE TABLE IF NOT EXISTS trial (
 -- unit
 --
 -- One row per named T-Beam unit encountered or described.
---
--- This table is intentionally sparse.  It allows later enrichment with
--- hardware-specific observations such as GNSS chip, SD card problems,
--- antenna notes, case color, battery notes, etc.
 -- ---------------------------------------------------------------------------
 
 CREATE TABLE IF NOT EXISTS unit (
@@ -90,15 +90,13 @@ CREATE TABLE IF NOT EXISTS unit (
 --
 -- One row per imported physical log file.
 --
--- The sha256 column allows repeatable provenance checks and prevents accidental
+-- sha256 prevents accidental duplicate ingestion.
--- duplicate ingestion of the same log.
 -- ---------------------------------------------------------------------------
 
 CREATE TABLE IF NOT EXISTS log_file (
     log_file_id           INTEGER PRIMARY KEY AUTOINCREMENT,
 
     trial_id              INTEGER NOT NULL REFERENCES trial(trial_id) ON DELETE CASCADE,
-
     receiver              TEXT NOT NULL REFERENCES unit(unit_name),
 
     path                  TEXT NOT NULL,
@@ -130,9 +128,6 @@ ON log_file(receiver);
 -- log_manifest_kv
 --
 -- Raw key/value manifest entries.
---
--- This preserves metadata exactly as supplied outside the CSV log.
--- The importer also copies selected keys into trial/log_file columns.
 -- ---------------------------------------------------------------------------
 
 CREATE TABLE IF NOT EXISTS log_manifest_kv (
@@ -149,10 +144,7 @@ CREATE TABLE IF NOT EXISTS log_manifest_kv (
 -- ---------------------------------------------------------------------------
 -- ble_observation_raw
 --
--- Faithful row-level import from the CSV log.
+-- Faithful row-level import from the current 18-column CSV log.
---
--- This table is meant to preserve the CSV fields as received, with minimal
--- interpretation.  Parsed/normalized fields live in ble_observation.
 -- ---------------------------------------------------------------------------
 
 CREATE TABLE IF NOT EXISTS ble_observation_raw (
@@ -169,6 +161,7 @@ CREATE TABLE IF NOT EXISTS ble_observation_raw (
     rx_lat                REAL,
     rx_lon                REAL,
     gps_fix_age_ms        INTEGER,
+
     clock_valid           INTEGER,
     gps_valid             INTEGER,
     discipline_age_ms     INTEGER,
@@ -182,7 +175,6 @@ CREATE TABLE IF NOT EXISTS ble_observation_raw (
     seq                   INTEGER,
     tx_payload_epoch_ms   INTEGER,
     payload               TEXT,
-    vbat_mv               INTEGER,
 
     UNIQUE(log_file_id, source_line_no)
 );
@@ -196,17 +188,16 @@ ON ble_observation_raw(receiver, heard, rx_epoch_ms);
 CREATE INDEX IF NOT EXISTS idx_raw_rx_position
 ON ble_observation_raw(rx_lat, rx_lon);
 
+CREATE INDEX IF NOT EXISTS idx_raw_validity
+ON ble_observation_raw(clock_valid, gps_valid, gps_fix_age_ms);
+
 -- ---------------------------------------------------------------------------
 -- ble_observation
 --
 -- Parsed and normalized BLE observation table.
 --
--- This table keeps the important analytical values in typed columns.
+-- rx_tx_payload_delta_ms is the receiver row timestamp minus the sender
--- rx_tx_delta_ms is the receiver timestamp minus the sender timestamp.
+-- payload-generation timestamp.  It is diagnostic only.
---
--- rx_tx_delta_ms is not a pure radio propagation delay.  It includes BLE
--- advertising/scanning timing, firmware scheduling, clock error, and logging
--- latency.  It is still useful for clock-alignment diagnostics.
 -- ---------------------------------------------------------------------------
 
 CREATE TABLE IF NOT EXISTS ble_observation (
@@ -225,7 +216,7 @@ CREATE TABLE IF NOT EXISTS ble_observation (
     tx_payload_epoch_ms   INTEGER,
     tx_payload_epoch_s    REAL,
 
-    rx_tx_delta_ms        INTEGER,
+    rx_tx_payload_delta_ms INTEGER,
 
     receiver              TEXT NOT NULL REFERENCES unit(unit_name),
     heard                 TEXT NOT NULL REFERENCES unit(unit_name),
@@ -233,11 +224,11 @@ CREATE TABLE IF NOT EXISTS ble_observation (
     rx_lat                REAL,
     rx_lon                REAL,
     gps_fix_age_ms        INTEGER,
+
     clock_valid           INTEGER,
     gps_valid             INTEGER,
     discipline_age_ms     INTEGER,
     last_discipline_epoch_ms INTEGER,
-    vbat_mv               INTEGER,
 
     rssi                  INTEGER,
     avg_rssi              INTEGER,
@@ -250,7 +241,7 @@ CREATE TABLE IF NOT EXISTS ble_observation (
     payload_kind          TEXT,
     payload_node          TEXT,
     payload_seq           INTEGER,
-    payload_tx_epoch_ms   INTEGER,
+    payload_tx_payload_epoch_ms INTEGER,
     payload_legacy_uptime INTEGER,
 
     parse_warning         TEXT
@@ -271,10 +262,14 @@ ON ble_observation(rx_lat, rx_lon);
 CREATE INDEX IF NOT EXISTS idx_obs_rssi
 ON ble_observation(receiver, heard, rssi);
 
+CREATE INDEX IF NOT EXISTS idx_obs_validity
+ON ble_observation(clock_valid, gps_valid, gps_fix_age_ms);
+
+CREATE INDEX IF NOT EXISTS idx_obs_payload_time
+ON ble_observation(tx_payload_epoch_ms, rx_tx_payload_delta_ms);
+
 -- ---------------------------------------------------------------------------
 -- v_log_file_summary
---
--- Per-log summary.
 -- ---------------------------------------------------------------------------
 
 CREATE VIEW IF NOT EXISTS v_log_file_summary AS
@@ -296,10 +291,8 @@ JOIN trial t ON t.trial_id = lf.trial_id;
 -- ---------------------------------------------------------------------------
 -- v_receiver_gps_summary
 --
--- Per receiver GPS spread summary in degrees.
+-- Per receiver GPS spread and validity summary in degrees.
---
+-- Distance conversion remains outside SQLite.
--- Distance conversion is intentionally left outside SQLite because plain SQLite
--- may not have trigonometric math functions compiled in.
 -- ---------------------------------------------------------------------------
 
 CREATE VIEW IF NOT EXISTS v_receiver_gps_summary AS
@@ -327,7 +320,19 @@ SELECT
 
     MIN(gps_fix_age_ms) AS min_gps_fix_age_ms,
     AVG(gps_fix_age_ms) AS avg_gps_fix_age_ms,
-    MAX(gps_fix_age_ms) AS max_gps_fix_age_ms
+    MAX(gps_fix_age_ms) AS max_gps_fix_age_ms,
+
+    MIN(clock_valid) AS min_clock_valid,
+    MAX(clock_valid) AS max_clock_valid,
+    MIN(gps_valid) AS min_gps_valid,
+    MAX(gps_valid) AS max_gps_valid,
+
+    MIN(discipline_age_ms) AS min_discipline_age_ms,
+    AVG(discipline_age_ms) AS avg_discipline_age_ms,
+    MAX(discipline_age_ms) AS max_discipline_age_ms,
+
+    MIN(last_discipline_epoch_ms) AS min_last_discipline_epoch_ms,
+    MAX(last_discipline_epoch_ms) AS max_last_discipline_epoch_ms
 FROM ble_observation
 GROUP BY trial_id, log_file_id, receiver;
 
@@ -358,20 +363,24 @@ SELECT
     MAX(avg_rssi) AS max_rolling_avg_rssi,
 
     MIN(payload_seq) AS min_payload_seq,
-    MAX(payload_seq) AS max_payload_seq
+    MAX(payload_seq) AS max_payload_seq,
+
+    MIN(clock_valid) AS min_clock_valid,
+    MIN(gps_valid) AS min_gps_valid
 FROM ble_observation
 GROUP BY trial_id, receiver, heard;
 
 -- ---------------------------------------------------------------------------
--- v_rx_tx_timing_summary
+-- v_rx_tx_payload_timing_summary
 --
 -- Per receiver/heard timing-delta summary.
 --
--- rx_tx_delta_ms is useful for detecting clock disagreement, restarts, and
+-- rx_tx_payload_delta_ms is useful for detecting clock disagreement,
--- logging/scan oddities.  It should not be interpreted as RF travel time.
+-- scheduling, repeated payload observations, and payload-age behavior.
+-- It is not RF propagation delay.
 -- ---------------------------------------------------------------------------
 
-CREATE VIEW IF NOT EXISTS v_rx_tx_timing_summary AS
+CREATE VIEW IF NOT EXISTS v_rx_tx_payload_timing_summary AS
 SELECT
     trial_id,
     receiver,
@@ -379,28 +388,28 @@ SELECT
 
     COUNT(*) AS observation_count,
 
-    MIN(rx_tx_delta_ms) AS min_rx_tx_delta_ms,
+    MIN(rx_tx_payload_delta_ms) AS min_rx_tx_payload_delta_ms,
-    AVG(rx_tx_delta_ms) AS avg_rx_tx_delta_ms,
+    AVG(rx_tx_payload_delta_ms) AS avg_rx_tx_payload_delta_ms,
-    MAX(rx_tx_delta_ms) AS max_rx_tx_delta_ms
+    MAX(rx_tx_payload_delta_ms) AS max_rx_tx_payload_delta_ms
 FROM ble_observation
 WHERE tx_payload_epoch_ms IS NOT NULL
   AND tx_payload_epoch_ms > 0
 GROUP BY trial_id, receiver, heard;
 
---
+-- ---------------------------------------------------------------------------
--- Additiona views created after initial import of 1st test set
+-- v_rx_tx_payload_delta_by_seq
--- and anomolies seen
+-- ---------------------------------------------------------------------------
---
+
-CREATE VIEW IF NOT EXISTS v_rx_tx_delta_by_seq AS
+CREATE VIEW IF NOT EXISTS v_rx_tx_payload_delta_by_seq AS
 SELECT
     trial_id,
     receiver,
     heard,
     payload_seq,
     COUNT(*) AS observations,
-    MIN(rx_tx_delta_ms) AS min_delta_ms,
+    MIN(rx_tx_payload_delta_ms) AS min_delta_ms,
-    AVG(rx_tx_delta_ms) AS avg_delta_ms,
+    AVG(rx_tx_payload_delta_ms) AS avg_delta_ms,
-    MAX(rx_tx_delta_ms) AS max_delta_ms,
+    MAX(rx_tx_payload_delta_ms) AS max_delta_ms,
     MIN(rx_epoch_ms) AS first_rx_epoch_ms,
     MAX(rx_epoch_ms) AS last_rx_epoch_ms
 FROM ble_observation
@@ -408,6 +417,10 @@ WHERE tx_payload_epoch_ms IS NOT NULL
   AND tx_payload_epoch_ms > 0
 GROUP BY trial_id, receiver, heard, payload_seq;
 
+-- ---------------------------------------------------------------------------
+-- v_payload_seq_summary
+-- ---------------------------------------------------------------------------
+
 CREATE VIEW IF NOT EXISTS v_payload_seq_summary AS
 SELECT
     trial_id,
@@ -421,13 +434,78 @@ SELECT
 FROM ble_observation
 GROUP BY trial_id, receiver, heard;
 
+-- ---------------------------------------------------------------------------
+-- v_payload_seq_receive_spread
 --
--- for mapping
+-- Shows how long the same advertised payload sequence was observed by a
+-- receiver.  Useful for confirming repeated observation of one payload.
+-- ---------------------------------------------------------------------------
+
+CREATE VIEW IF NOT EXISTS v_payload_seq_receive_spread AS
+SELECT
+    trial_id,
+    receiver,
+    heard,
+    payload_seq,
+    COUNT(*) AS observations,
+    MIN(rx_epoch_ms) AS first_rx_epoch_ms,
+    MAX(rx_epoch_ms) AS last_rx_epoch_ms,
+    MAX(rx_epoch_ms) - MIN(rx_epoch_ms) AS rx_spread_ms,
+    MIN(rx_tx_payload_delta_ms) AS min_delta_ms,
+    AVG(rx_tx_payload_delta_ms) AS avg_delta_ms,
+    MAX(rx_tx_payload_delta_ms) AS max_delta_ms
+FROM ble_observation
+WHERE tx_payload_epoch_ms IS NOT NULL
+  AND tx_payload_epoch_ms > 0
+GROUP BY trial_id, receiver, heard, payload_seq;
+
+-- ---------------------------------------------------------------------------
+-- v_payload_tx_epoch_step
 --
+-- Shows elapsed sender payload-generation time between successive payload
+-- sequence values as observed by each receiver/heard pair.
+-- ---------------------------------------------------------------------------
+
+CREATE VIEW IF NOT EXISTS v_payload_tx_epoch_step AS
+WITH x AS (
+    SELECT
+        trial_id,
+        receiver,
+        heard,
+        payload_seq,
+        MIN(payload_tx_payload_epoch_ms) AS payload_tx_payload_epoch_ms
+    FROM ble_observation
+    WHERE payload_tx_payload_epoch_ms IS NOT NULL
+      AND payload_tx_payload_epoch_ms > 0
+    GROUP BY trial_id, receiver, heard, payload_seq
+),
+y AS (
+    SELECT
+        x.*,
+        LAG(payload_tx_payload_epoch_ms) OVER (
+            PARTITION BY trial_id, receiver, heard
+            ORDER BY payload_seq
+        ) AS prev_payload_tx_payload_epoch_ms
+    FROM x
+)
+SELECT
+    *,
+    payload_tx_payload_epoch_ms - prev_payload_tx_payload_epoch_ms AS tx_payload_epoch_step_ms
+FROM y
+WHERE prev_payload_tx_payload_epoch_ms IS NOT NULL;
+
+-- ---------------------------------------------------------------------------
+-- v_map_observation_points
+--
+-- Mapping-oriented observation view.
+-- ---------------------------------------------------------------------------
+
 CREATE VIEW IF NOT EXISTS v_map_observation_points AS
 SELECT
     obs_id,
     trial_id,
+    log_file_id,
+    source_line_no,
     receiver,
     heard,
     rx_epoch_ms,
@@ -435,11 +513,15 @@ SELECT
     rx_lat,
     rx_lon,
     gps_fix_age_ms,
+    clock_valid,
+    gps_valid,
+    discipline_age_ms,
+    last_discipline_epoch_ms,
     rssi,
     avg_rssi,
     payload_seq,
     tx_payload_epoch_ms,
-    rx_tx_delta_ms
+    rx_tx_payload_delta_ms
 FROM ble_observation
 WHERE rx_lat IS NOT NULL
   AND rx_lon IS NOT NULL;
@@ -447,29 +529,8 @@ WHERE rx_lat IS NOT NULL
 CREATE VIEW IF NOT EXISTS v_map_good_gps_points AS
 SELECT *
 FROM v_map_observation_points
-WHERE gps_fix_age_ms <= 5000;
+WHERE clock_valid = 1
-
+  AND gps_valid = 1
-
+  AND gps_fix_age_ms <= 5000;
--- ---------------------------------------------------------------------------
--- v_battery_summary
---
--- Per receiver battery voltage summary for diagnosing power-related logging
--- interruptions.  vbat_mv = -1 means the PMU value was unavailable.
--- ---------------------------------------------------------------------------
-
-CREATE VIEW IF NOT EXISTS v_battery_summary AS
-SELECT
-    trial_id,
-    log_file_id,
-    receiver,
-
-    COUNT(*) AS row_count,
-    SUM(CASE WHEN vbat_mv IS NOT NULL AND vbat_mv >= 0 THEN 1 ELSE 0 END) AS vbat_sample_count,
-
-    MIN(CASE WHEN vbat_mv >= 0 THEN vbat_mv END) AS min_vbat_mv,
-    AVG(CASE WHEN vbat_mv >= 0 THEN vbat_mv END) AS avg_vbat_mv,
-    MAX(CASE WHEN vbat_mv >= 0 THEN vbat_mv END) AS max_vbat_mv
-FROM ble_observation
-GROUP BY trial_id, log_file_id, receiver;
 
 COMMIT;