From fe46db2b3ccecc7bb5f3dc11ac544536d8bfce77 Mon Sep 17 00:00:00 2001 From: John Poole Date: Wed, 27 May 2026 11:10:14 -0700 Subject: [PATCH] latest schema for 18 column logs --- .../scripts/create_exercise_26_ble_schema.sql | 229 +++++++++++------- 1 file changed, 145 insertions(+), 84 deletions(-) diff --git a/exercises/26_Bluetooth_discover/scripts/create_exercise_26_ble_schema.sql b/exercises/26_Bluetooth_discover/scripts/create_exercise_26_ble_schema.sql index b8cb351..7dfa192 100644 --- a/exercises/26_Bluetooth_discover/scripts/create_exercise_26_ble_schema.sql +++ b/exercises/26_Bluetooth_discover/scripts/create_exercise_26_ble_schema.sql @@ -2,30 +2,34 @@ -- $HeadURL$ -- -- Example: --- sqlite3 ble_fieldtest_20260525_1945_ed_flo.sqlite < create_exercise_26_ble_schema.sql +-- sqlite3 ble_fieldtest_YYYYMMDD_HHMM.sqlite < create_exercise_26_ble_schema.sql -- -- SQLite schema for Exercise 26 BLE Discovery field-test logs. -- --- Design goals: --- 1. Preserve the raw log rows. --- 2. Preserve per-log manifest metadata. --- 3. Normalize parsed BLE observations. --- 4. Keep one portable SQLite database per field trial. --- 5. Support later export to GeoJSON / MapLibre / GIS tools. +-- This schema targets the current 18-column Exercise 26 log format only. -- -- Current Exercise 26 log format: -- -- human_time,rx_epoch_ms,receiver,rx_lat,rx_lon,gps_fix_age_ms, -- clock_valid,gps_valid,discipline_age_ms,last_discipline_epoch_ms, --- heard,rssi,avg_rssi,age_s,count,seq,tx_payload_epoch_ms,payload,vbat_mv +-- heard,rssi,avg_rssi,age_s,count,seq,tx_payload_epoch_ms,payload -- -- Current payload format: -- -- B2|NODE|SEQ|TX_PAYLOAD_EPOCH_MS -- --- Legacy payload format: +-- Timestamp semantics: -- --- TBMSND|1|NODE|SEQ|UPTIME +-- rx_epoch_ms is the receiver's disciplined-clock time when the observation +-- row is written. +-- +-- tx_payload_epoch_ms is the sender's disciplined-clock time when the BLE +-- advertising payload was generated/configured. It is not a per-RF-packet +-- transmit timestamp. +-- +-- rx_tx_payload_delta_ms is diagnostic only. It is useful for clock, +-- scheduling, payload-age, and repeated-observation analysis. It is not +-- BLE propagation delay. -- PRAGMA foreign_keys = ON; @@ -39,7 +43,7 @@ BEGIN TRANSACTION; -- -- A trial may contain multiple logs, usually one log per T-Beam receiver. -- Example: --- trial_label = peck_cottage_ed_flo_20260525_1945 +-- trial_label = peck_cottage_four_unit_20260526_1430 -- --------------------------------------------------------------------------- CREATE TABLE IF NOT EXISTS trial ( @@ -66,10 +70,6 @@ CREATE TABLE IF NOT EXISTS trial ( -- unit -- -- One row per named T-Beam unit encountered or described. --- --- This table is intentionally sparse. It allows later enrichment with --- hardware-specific observations such as GNSS chip, SD card problems, --- antenna notes, case color, battery notes, etc. -- --------------------------------------------------------------------------- CREATE TABLE IF NOT EXISTS unit ( @@ -90,15 +90,13 @@ CREATE TABLE IF NOT EXISTS unit ( -- -- One row per imported physical log file. -- --- The sha256 column allows repeatable provenance checks and prevents accidental --- duplicate ingestion of the same log. +-- sha256 prevents accidental duplicate ingestion. -- --------------------------------------------------------------------------- CREATE TABLE IF NOT EXISTS log_file ( log_file_id INTEGER PRIMARY KEY AUTOINCREMENT, trial_id INTEGER NOT NULL REFERENCES trial(trial_id) ON DELETE CASCADE, - receiver TEXT NOT NULL REFERENCES unit(unit_name), path TEXT NOT NULL, @@ -130,9 +128,6 @@ ON log_file(receiver); -- log_manifest_kv -- -- Raw key/value manifest entries. --- --- This preserves metadata exactly as supplied outside the CSV log. --- The importer also copies selected keys into trial/log_file columns. -- --------------------------------------------------------------------------- CREATE TABLE IF NOT EXISTS log_manifest_kv ( @@ -149,10 +144,7 @@ CREATE TABLE IF NOT EXISTS log_manifest_kv ( -- --------------------------------------------------------------------------- -- ble_observation_raw -- --- Faithful row-level import from the CSV log. --- --- This table is meant to preserve the CSV fields as received, with minimal --- interpretation. Parsed/normalized fields live in ble_observation. +-- Faithful row-level import from the current 18-column CSV log. -- --------------------------------------------------------------------------- CREATE TABLE IF NOT EXISTS ble_observation_raw ( @@ -169,6 +161,7 @@ CREATE TABLE IF NOT EXISTS ble_observation_raw ( rx_lat REAL, rx_lon REAL, gps_fix_age_ms INTEGER, + clock_valid INTEGER, gps_valid INTEGER, discipline_age_ms INTEGER, @@ -182,7 +175,6 @@ CREATE TABLE IF NOT EXISTS ble_observation_raw ( seq INTEGER, tx_payload_epoch_ms INTEGER, payload TEXT, - vbat_mv INTEGER, UNIQUE(log_file_id, source_line_no) ); @@ -196,17 +188,16 @@ ON ble_observation_raw(receiver, heard, rx_epoch_ms); CREATE INDEX IF NOT EXISTS idx_raw_rx_position ON ble_observation_raw(rx_lat, rx_lon); +CREATE INDEX IF NOT EXISTS idx_raw_validity +ON ble_observation_raw(clock_valid, gps_valid, gps_fix_age_ms); + -- --------------------------------------------------------------------------- -- ble_observation -- -- Parsed and normalized BLE observation table. -- --- This table keeps the important analytical values in typed columns. --- rx_tx_delta_ms is the receiver timestamp minus the sender timestamp. --- --- rx_tx_delta_ms is not a pure radio propagation delay. It includes BLE --- advertising/scanning timing, firmware scheduling, clock error, and logging --- latency. It is still useful for clock-alignment diagnostics. +-- rx_tx_payload_delta_ms is the receiver row timestamp minus the sender +-- payload-generation timestamp. It is diagnostic only. -- --------------------------------------------------------------------------- CREATE TABLE IF NOT EXISTS ble_observation ( @@ -225,7 +216,7 @@ CREATE TABLE IF NOT EXISTS ble_observation ( tx_payload_epoch_ms INTEGER, tx_payload_epoch_s REAL, - rx_tx_delta_ms INTEGER, + rx_tx_payload_delta_ms INTEGER, receiver TEXT NOT NULL REFERENCES unit(unit_name), heard TEXT NOT NULL REFERENCES unit(unit_name), @@ -233,11 +224,11 @@ CREATE TABLE IF NOT EXISTS ble_observation ( rx_lat REAL, rx_lon REAL, gps_fix_age_ms INTEGER, + clock_valid INTEGER, gps_valid INTEGER, discipline_age_ms INTEGER, last_discipline_epoch_ms INTEGER, - vbat_mv INTEGER, rssi INTEGER, avg_rssi INTEGER, @@ -250,7 +241,7 @@ CREATE TABLE IF NOT EXISTS ble_observation ( payload_kind TEXT, payload_node TEXT, payload_seq INTEGER, - payload_tx_epoch_ms INTEGER, + payload_tx_payload_epoch_ms INTEGER, payload_legacy_uptime INTEGER, parse_warning TEXT @@ -271,10 +262,14 @@ ON ble_observation(rx_lat, rx_lon); CREATE INDEX IF NOT EXISTS idx_obs_rssi ON ble_observation(receiver, heard, rssi); +CREATE INDEX IF NOT EXISTS idx_obs_validity +ON ble_observation(clock_valid, gps_valid, gps_fix_age_ms); + +CREATE INDEX IF NOT EXISTS idx_obs_payload_time +ON ble_observation(tx_payload_epoch_ms, rx_tx_payload_delta_ms); + -- --------------------------------------------------------------------------- -- v_log_file_summary --- --- Per-log summary. -- --------------------------------------------------------------------------- CREATE VIEW IF NOT EXISTS v_log_file_summary AS @@ -296,10 +291,8 @@ JOIN trial t ON t.trial_id = lf.trial_id; -- --------------------------------------------------------------------------- -- v_receiver_gps_summary -- --- Per receiver GPS spread summary in degrees. --- --- Distance conversion is intentionally left outside SQLite because plain SQLite --- may not have trigonometric math functions compiled in. +-- Per receiver GPS spread and validity summary in degrees. +-- Distance conversion remains outside SQLite. -- --------------------------------------------------------------------------- CREATE VIEW IF NOT EXISTS v_receiver_gps_summary AS @@ -327,7 +320,19 @@ SELECT MIN(gps_fix_age_ms) AS min_gps_fix_age_ms, AVG(gps_fix_age_ms) AS avg_gps_fix_age_ms, - MAX(gps_fix_age_ms) AS max_gps_fix_age_ms + MAX(gps_fix_age_ms) AS max_gps_fix_age_ms, + + MIN(clock_valid) AS min_clock_valid, + MAX(clock_valid) AS max_clock_valid, + MIN(gps_valid) AS min_gps_valid, + MAX(gps_valid) AS max_gps_valid, + + MIN(discipline_age_ms) AS min_discipline_age_ms, + AVG(discipline_age_ms) AS avg_discipline_age_ms, + MAX(discipline_age_ms) AS max_discipline_age_ms, + + MIN(last_discipline_epoch_ms) AS min_last_discipline_epoch_ms, + MAX(last_discipline_epoch_ms) AS max_last_discipline_epoch_ms FROM ble_observation GROUP BY trial_id, log_file_id, receiver; @@ -358,20 +363,24 @@ SELECT MAX(avg_rssi) AS max_rolling_avg_rssi, MIN(payload_seq) AS min_payload_seq, - MAX(payload_seq) AS max_payload_seq + MAX(payload_seq) AS max_payload_seq, + + MIN(clock_valid) AS min_clock_valid, + MIN(gps_valid) AS min_gps_valid FROM ble_observation GROUP BY trial_id, receiver, heard; -- --------------------------------------------------------------------------- --- v_rx_tx_timing_summary +-- v_rx_tx_payload_timing_summary -- -- Per receiver/heard timing-delta summary. -- --- rx_tx_delta_ms is useful for detecting clock disagreement, restarts, and --- logging/scan oddities. It should not be interpreted as RF travel time. +-- rx_tx_payload_delta_ms is useful for detecting clock disagreement, +-- scheduling, repeated payload observations, and payload-age behavior. +-- It is not RF propagation delay. -- --------------------------------------------------------------------------- -CREATE VIEW IF NOT EXISTS v_rx_tx_timing_summary AS +CREATE VIEW IF NOT EXISTS v_rx_tx_payload_timing_summary AS SELECT trial_id, receiver, @@ -379,28 +388,28 @@ SELECT COUNT(*) AS observation_count, - MIN(rx_tx_delta_ms) AS min_rx_tx_delta_ms, - AVG(rx_tx_delta_ms) AS avg_rx_tx_delta_ms, - MAX(rx_tx_delta_ms) AS max_rx_tx_delta_ms + MIN(rx_tx_payload_delta_ms) AS min_rx_tx_payload_delta_ms, + AVG(rx_tx_payload_delta_ms) AS avg_rx_tx_payload_delta_ms, + MAX(rx_tx_payload_delta_ms) AS max_rx_tx_payload_delta_ms FROM ble_observation WHERE tx_payload_epoch_ms IS NOT NULL AND tx_payload_epoch_ms > 0 GROUP BY trial_id, receiver, heard; --- --- Additiona views created after initial import of 1st test set --- and anomolies seen --- -CREATE VIEW IF NOT EXISTS v_rx_tx_delta_by_seq AS +-- --------------------------------------------------------------------------- +-- v_rx_tx_payload_delta_by_seq +-- --------------------------------------------------------------------------- + +CREATE VIEW IF NOT EXISTS v_rx_tx_payload_delta_by_seq AS SELECT trial_id, receiver, heard, payload_seq, COUNT(*) AS observations, - MIN(rx_tx_delta_ms) AS min_delta_ms, - AVG(rx_tx_delta_ms) AS avg_delta_ms, - MAX(rx_tx_delta_ms) AS max_delta_ms, + MIN(rx_tx_payload_delta_ms) AS min_delta_ms, + AVG(rx_tx_payload_delta_ms) AS avg_delta_ms, + MAX(rx_tx_payload_delta_ms) AS max_delta_ms, MIN(rx_epoch_ms) AS first_rx_epoch_ms, MAX(rx_epoch_ms) AS last_rx_epoch_ms FROM ble_observation @@ -408,6 +417,10 @@ WHERE tx_payload_epoch_ms IS NOT NULL AND tx_payload_epoch_ms > 0 GROUP BY trial_id, receiver, heard, payload_seq; +-- --------------------------------------------------------------------------- +-- v_payload_seq_summary +-- --------------------------------------------------------------------------- + CREATE VIEW IF NOT EXISTS v_payload_seq_summary AS SELECT trial_id, @@ -421,13 +434,78 @@ SELECT FROM ble_observation GROUP BY trial_id, receiver, heard; +-- --------------------------------------------------------------------------- +-- v_payload_seq_receive_spread -- --- for mapping +-- Shows how long the same advertised payload sequence was observed by a +-- receiver. Useful for confirming repeated observation of one payload. +-- --------------------------------------------------------------------------- + +CREATE VIEW IF NOT EXISTS v_payload_seq_receive_spread AS +SELECT + trial_id, + receiver, + heard, + payload_seq, + COUNT(*) AS observations, + MIN(rx_epoch_ms) AS first_rx_epoch_ms, + MAX(rx_epoch_ms) AS last_rx_epoch_ms, + MAX(rx_epoch_ms) - MIN(rx_epoch_ms) AS rx_spread_ms, + MIN(rx_tx_payload_delta_ms) AS min_delta_ms, + AVG(rx_tx_payload_delta_ms) AS avg_delta_ms, + MAX(rx_tx_payload_delta_ms) AS max_delta_ms +FROM ble_observation +WHERE tx_payload_epoch_ms IS NOT NULL + AND tx_payload_epoch_ms > 0 +GROUP BY trial_id, receiver, heard, payload_seq; + +-- --------------------------------------------------------------------------- +-- v_payload_tx_epoch_step -- +-- Shows elapsed sender payload-generation time between successive payload +-- sequence values as observed by each receiver/heard pair. +-- --------------------------------------------------------------------------- + +CREATE VIEW IF NOT EXISTS v_payload_tx_epoch_step AS +WITH x AS ( + SELECT + trial_id, + receiver, + heard, + payload_seq, + MIN(payload_tx_payload_epoch_ms) AS payload_tx_payload_epoch_ms + FROM ble_observation + WHERE payload_tx_payload_epoch_ms IS NOT NULL + AND payload_tx_payload_epoch_ms > 0 + GROUP BY trial_id, receiver, heard, payload_seq +), +y AS ( + SELECT + x.*, + LAG(payload_tx_payload_epoch_ms) OVER ( + PARTITION BY trial_id, receiver, heard + ORDER BY payload_seq + ) AS prev_payload_tx_payload_epoch_ms + FROM x +) +SELECT + *, + payload_tx_payload_epoch_ms - prev_payload_tx_payload_epoch_ms AS tx_payload_epoch_step_ms +FROM y +WHERE prev_payload_tx_payload_epoch_ms IS NOT NULL; + +-- --------------------------------------------------------------------------- +-- v_map_observation_points +-- +-- Mapping-oriented observation view. +-- --------------------------------------------------------------------------- + CREATE VIEW IF NOT EXISTS v_map_observation_points AS SELECT obs_id, trial_id, + log_file_id, + source_line_no, receiver, heard, rx_epoch_ms, @@ -435,11 +513,15 @@ SELECT rx_lat, rx_lon, gps_fix_age_ms, + clock_valid, + gps_valid, + discipline_age_ms, + last_discipline_epoch_ms, rssi, avg_rssi, payload_seq, tx_payload_epoch_ms, - rx_tx_delta_ms + rx_tx_payload_delta_ms FROM ble_observation WHERE rx_lat IS NOT NULL AND rx_lon IS NOT NULL; @@ -447,29 +529,8 @@ WHERE rx_lat IS NOT NULL CREATE VIEW IF NOT EXISTS v_map_good_gps_points AS SELECT * FROM v_map_observation_points -WHERE gps_fix_age_ms <= 5000; - - --- --------------------------------------------------------------------------- --- v_battery_summary --- --- Per receiver battery voltage summary for diagnosing power-related logging --- interruptions. vbat_mv = -1 means the PMU value was unavailable. --- --------------------------------------------------------------------------- - -CREATE VIEW IF NOT EXISTS v_battery_summary AS -SELECT - trial_id, - log_file_id, - receiver, - - COUNT(*) AS row_count, - SUM(CASE WHEN vbat_mv IS NOT NULL AND vbat_mv >= 0 THEN 1 ELSE 0 END) AS vbat_sample_count, - - MIN(CASE WHEN vbat_mv >= 0 THEN vbat_mv END) AS min_vbat_mv, - AVG(CASE WHEN vbat_mv >= 0 THEN vbat_mv END) AS avg_vbat_mv, - MAX(CASE WHEN vbat_mv >= 0 THEN vbat_mv END) AS max_vbat_mv -FROM ble_observation -GROUP BY trial_id, log_file_id, receiver; +WHERE clock_valid = 1 + AND gps_valid = 1 + AND gps_fix_age_ms <= 5000; COMMIT;