latest schema for 18 column logs
This commit is contained in:
parent
a8fc2cf9d5
commit
fe46db2b3c
1 changed files with 145 additions and 84 deletions
|
|
@ -2,30 +2,34 @@
|
|||
-- $HeadURL$
|
||||
--
|
||||
-- Example:
|
||||
-- sqlite3 ble_fieldtest_20260525_1945_ed_flo.sqlite < create_exercise_26_ble_schema.sql
|
||||
-- sqlite3 ble_fieldtest_YYYYMMDD_HHMM.sqlite < create_exercise_26_ble_schema.sql
|
||||
--
|
||||
-- SQLite schema for Exercise 26 BLE Discovery field-test logs.
|
||||
--
|
||||
-- Design goals:
|
||||
-- 1. Preserve the raw log rows.
|
||||
-- 2. Preserve per-log manifest metadata.
|
||||
-- 3. Normalize parsed BLE observations.
|
||||
-- 4. Keep one portable SQLite database per field trial.
|
||||
-- 5. Support later export to GeoJSON / MapLibre / GIS tools.
|
||||
-- This schema targets the current 18-column Exercise 26 log format only.
|
||||
--
|
||||
-- Current Exercise 26 log format:
|
||||
--
|
||||
-- human_time,rx_epoch_ms,receiver,rx_lat,rx_lon,gps_fix_age_ms,
|
||||
-- clock_valid,gps_valid,discipline_age_ms,last_discipline_epoch_ms,
|
||||
-- heard,rssi,avg_rssi,age_s,count,seq,tx_payload_epoch_ms,payload,vbat_mv
|
||||
-- heard,rssi,avg_rssi,age_s,count,seq,tx_payload_epoch_ms,payload
|
||||
--
|
||||
-- Current payload format:
|
||||
--
|
||||
-- B2|NODE|SEQ|TX_PAYLOAD_EPOCH_MS
|
||||
--
|
||||
-- Legacy payload format:
|
||||
-- Timestamp semantics:
|
||||
--
|
||||
-- TBMSND|1|NODE|SEQ|UPTIME
|
||||
-- rx_epoch_ms is the receiver's disciplined-clock time when the observation
|
||||
-- row is written.
|
||||
--
|
||||
-- tx_payload_epoch_ms is the sender's disciplined-clock time when the BLE
|
||||
-- advertising payload was generated/configured. It is not a per-RF-packet
|
||||
-- transmit timestamp.
|
||||
--
|
||||
-- rx_tx_payload_delta_ms is diagnostic only. It is useful for clock,
|
||||
-- scheduling, payload-age, and repeated-observation analysis. It is not
|
||||
-- BLE propagation delay.
|
||||
--
|
||||
|
||||
PRAGMA foreign_keys = ON;
|
||||
|
|
@ -39,7 +43,7 @@ BEGIN TRANSACTION;
|
|||
--
|
||||
-- A trial may contain multiple logs, usually one log per T-Beam receiver.
|
||||
-- Example:
|
||||
-- trial_label = peck_cottage_ed_flo_20260525_1945
|
||||
-- trial_label = peck_cottage_four_unit_20260526_1430
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE IF NOT EXISTS trial (
|
||||
|
|
@ -66,10 +70,6 @@ CREATE TABLE IF NOT EXISTS trial (
|
|||
-- unit
|
||||
--
|
||||
-- One row per named T-Beam unit encountered or described.
|
||||
--
|
||||
-- This table is intentionally sparse. It allows later enrichment with
|
||||
-- hardware-specific observations such as GNSS chip, SD card problems,
|
||||
-- antenna notes, case color, battery notes, etc.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE IF NOT EXISTS unit (
|
||||
|
|
@ -90,15 +90,13 @@ CREATE TABLE IF NOT EXISTS unit (
|
|||
--
|
||||
-- One row per imported physical log file.
|
||||
--
|
||||
-- The sha256 column allows repeatable provenance checks and prevents accidental
|
||||
-- duplicate ingestion of the same log.
|
||||
-- sha256 prevents accidental duplicate ingestion.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE IF NOT EXISTS log_file (
|
||||
log_file_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
|
||||
trial_id INTEGER NOT NULL REFERENCES trial(trial_id) ON DELETE CASCADE,
|
||||
|
||||
receiver TEXT NOT NULL REFERENCES unit(unit_name),
|
||||
|
||||
path TEXT NOT NULL,
|
||||
|
|
@ -130,9 +128,6 @@ ON log_file(receiver);
|
|||
-- log_manifest_kv
|
||||
--
|
||||
-- Raw key/value manifest entries.
|
||||
--
|
||||
-- This preserves metadata exactly as supplied outside the CSV log.
|
||||
-- The importer also copies selected keys into trial/log_file columns.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE IF NOT EXISTS log_manifest_kv (
|
||||
|
|
@ -149,10 +144,7 @@ CREATE TABLE IF NOT EXISTS log_manifest_kv (
|
|||
-- ---------------------------------------------------------------------------
|
||||
-- ble_observation_raw
|
||||
--
|
||||
-- Faithful row-level import from the CSV log.
|
||||
--
|
||||
-- This table is meant to preserve the CSV fields as received, with minimal
|
||||
-- interpretation. Parsed/normalized fields live in ble_observation.
|
||||
-- Faithful row-level import from the current 18-column CSV log.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ble_observation_raw (
|
||||
|
|
@ -169,6 +161,7 @@ CREATE TABLE IF NOT EXISTS ble_observation_raw (
|
|||
rx_lat REAL,
|
||||
rx_lon REAL,
|
||||
gps_fix_age_ms INTEGER,
|
||||
|
||||
clock_valid INTEGER,
|
||||
gps_valid INTEGER,
|
||||
discipline_age_ms INTEGER,
|
||||
|
|
@ -182,7 +175,6 @@ CREATE TABLE IF NOT EXISTS ble_observation_raw (
|
|||
seq INTEGER,
|
||||
tx_payload_epoch_ms INTEGER,
|
||||
payload TEXT,
|
||||
vbat_mv INTEGER,
|
||||
|
||||
UNIQUE(log_file_id, source_line_no)
|
||||
);
|
||||
|
|
@ -196,17 +188,16 @@ ON ble_observation_raw(receiver, heard, rx_epoch_ms);
|
|||
CREATE INDEX IF NOT EXISTS idx_raw_rx_position
|
||||
ON ble_observation_raw(rx_lat, rx_lon);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_validity
|
||||
ON ble_observation_raw(clock_valid, gps_valid, gps_fix_age_ms);
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- ble_observation
|
||||
--
|
||||
-- Parsed and normalized BLE observation table.
|
||||
--
|
||||
-- This table keeps the important analytical values in typed columns.
|
||||
-- rx_tx_delta_ms is the receiver timestamp minus the sender timestamp.
|
||||
--
|
||||
-- rx_tx_delta_ms is not a pure radio propagation delay. It includes BLE
|
||||
-- advertising/scanning timing, firmware scheduling, clock error, and logging
|
||||
-- latency. It is still useful for clock-alignment diagnostics.
|
||||
-- rx_tx_payload_delta_ms is the receiver row timestamp minus the sender
|
||||
-- payload-generation timestamp. It is diagnostic only.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ble_observation (
|
||||
|
|
@ -225,7 +216,7 @@ CREATE TABLE IF NOT EXISTS ble_observation (
|
|||
tx_payload_epoch_ms INTEGER,
|
||||
tx_payload_epoch_s REAL,
|
||||
|
||||
rx_tx_delta_ms INTEGER,
|
||||
rx_tx_payload_delta_ms INTEGER,
|
||||
|
||||
receiver TEXT NOT NULL REFERENCES unit(unit_name),
|
||||
heard TEXT NOT NULL REFERENCES unit(unit_name),
|
||||
|
|
@ -233,11 +224,11 @@ CREATE TABLE IF NOT EXISTS ble_observation (
|
|||
rx_lat REAL,
|
||||
rx_lon REAL,
|
||||
gps_fix_age_ms INTEGER,
|
||||
|
||||
clock_valid INTEGER,
|
||||
gps_valid INTEGER,
|
||||
discipline_age_ms INTEGER,
|
||||
last_discipline_epoch_ms INTEGER,
|
||||
vbat_mv INTEGER,
|
||||
|
||||
rssi INTEGER,
|
||||
avg_rssi INTEGER,
|
||||
|
|
@ -250,7 +241,7 @@ CREATE TABLE IF NOT EXISTS ble_observation (
|
|||
payload_kind TEXT,
|
||||
payload_node TEXT,
|
||||
payload_seq INTEGER,
|
||||
payload_tx_epoch_ms INTEGER,
|
||||
payload_tx_payload_epoch_ms INTEGER,
|
||||
payload_legacy_uptime INTEGER,
|
||||
|
||||
parse_warning TEXT
|
||||
|
|
@ -271,10 +262,14 @@ ON ble_observation(rx_lat, rx_lon);
|
|||
CREATE INDEX IF NOT EXISTS idx_obs_rssi
|
||||
ON ble_observation(receiver, heard, rssi);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_obs_validity
|
||||
ON ble_observation(clock_valid, gps_valid, gps_fix_age_ms);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_obs_payload_time
|
||||
ON ble_observation(tx_payload_epoch_ms, rx_tx_payload_delta_ms);
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_log_file_summary
|
||||
--
|
||||
-- Per-log summary.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE VIEW IF NOT EXISTS v_log_file_summary AS
|
||||
|
|
@ -296,10 +291,8 @@ JOIN trial t ON t.trial_id = lf.trial_id;
|
|||
-- ---------------------------------------------------------------------------
|
||||
-- v_receiver_gps_summary
|
||||
--
|
||||
-- Per receiver GPS spread summary in degrees.
|
||||
--
|
||||
-- Distance conversion is intentionally left outside SQLite because plain SQLite
|
||||
-- may not have trigonometric math functions compiled in.
|
||||
-- Per receiver GPS spread and validity summary in degrees.
|
||||
-- Distance conversion remains outside SQLite.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE VIEW IF NOT EXISTS v_receiver_gps_summary AS
|
||||
|
|
@ -327,7 +320,19 @@ SELECT
|
|||
|
||||
MIN(gps_fix_age_ms) AS min_gps_fix_age_ms,
|
||||
AVG(gps_fix_age_ms) AS avg_gps_fix_age_ms,
|
||||
MAX(gps_fix_age_ms) AS max_gps_fix_age_ms
|
||||
MAX(gps_fix_age_ms) AS max_gps_fix_age_ms,
|
||||
|
||||
MIN(clock_valid) AS min_clock_valid,
|
||||
MAX(clock_valid) AS max_clock_valid,
|
||||
MIN(gps_valid) AS min_gps_valid,
|
||||
MAX(gps_valid) AS max_gps_valid,
|
||||
|
||||
MIN(discipline_age_ms) AS min_discipline_age_ms,
|
||||
AVG(discipline_age_ms) AS avg_discipline_age_ms,
|
||||
MAX(discipline_age_ms) AS max_discipline_age_ms,
|
||||
|
||||
MIN(last_discipline_epoch_ms) AS min_last_discipline_epoch_ms,
|
||||
MAX(last_discipline_epoch_ms) AS max_last_discipline_epoch_ms
|
||||
FROM ble_observation
|
||||
GROUP BY trial_id, log_file_id, receiver;
|
||||
|
||||
|
|
@ -358,20 +363,24 @@ SELECT
|
|||
MAX(avg_rssi) AS max_rolling_avg_rssi,
|
||||
|
||||
MIN(payload_seq) AS min_payload_seq,
|
||||
MAX(payload_seq) AS max_payload_seq
|
||||
MAX(payload_seq) AS max_payload_seq,
|
||||
|
||||
MIN(clock_valid) AS min_clock_valid,
|
||||
MIN(gps_valid) AS min_gps_valid
|
||||
FROM ble_observation
|
||||
GROUP BY trial_id, receiver, heard;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_rx_tx_timing_summary
|
||||
-- v_rx_tx_payload_timing_summary
|
||||
--
|
||||
-- Per receiver/heard timing-delta summary.
|
||||
--
|
||||
-- rx_tx_delta_ms is useful for detecting clock disagreement, restarts, and
|
||||
-- logging/scan oddities. It should not be interpreted as RF travel time.
|
||||
-- rx_tx_payload_delta_ms is useful for detecting clock disagreement,
|
||||
-- scheduling, repeated payload observations, and payload-age behavior.
|
||||
-- It is not RF propagation delay.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE VIEW IF NOT EXISTS v_rx_tx_timing_summary AS
|
||||
CREATE VIEW IF NOT EXISTS v_rx_tx_payload_timing_summary AS
|
||||
SELECT
|
||||
trial_id,
|
||||
receiver,
|
||||
|
|
@ -379,28 +388,28 @@ SELECT
|
|||
|
||||
COUNT(*) AS observation_count,
|
||||
|
||||
MIN(rx_tx_delta_ms) AS min_rx_tx_delta_ms,
|
||||
AVG(rx_tx_delta_ms) AS avg_rx_tx_delta_ms,
|
||||
MAX(rx_tx_delta_ms) AS max_rx_tx_delta_ms
|
||||
MIN(rx_tx_payload_delta_ms) AS min_rx_tx_payload_delta_ms,
|
||||
AVG(rx_tx_payload_delta_ms) AS avg_rx_tx_payload_delta_ms,
|
||||
MAX(rx_tx_payload_delta_ms) AS max_rx_tx_payload_delta_ms
|
||||
FROM ble_observation
|
||||
WHERE tx_payload_epoch_ms IS NOT NULL
|
||||
AND tx_payload_epoch_ms > 0
|
||||
GROUP BY trial_id, receiver, heard;
|
||||
|
||||
--
|
||||
-- Additiona views created after initial import of 1st test set
|
||||
-- and anomolies seen
|
||||
--
|
||||
CREATE VIEW IF NOT EXISTS v_rx_tx_delta_by_seq AS
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_rx_tx_payload_delta_by_seq
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE VIEW IF NOT EXISTS v_rx_tx_payload_delta_by_seq AS
|
||||
SELECT
|
||||
trial_id,
|
||||
receiver,
|
||||
heard,
|
||||
payload_seq,
|
||||
COUNT(*) AS observations,
|
||||
MIN(rx_tx_delta_ms) AS min_delta_ms,
|
||||
AVG(rx_tx_delta_ms) AS avg_delta_ms,
|
||||
MAX(rx_tx_delta_ms) AS max_delta_ms,
|
||||
MIN(rx_tx_payload_delta_ms) AS min_delta_ms,
|
||||
AVG(rx_tx_payload_delta_ms) AS avg_delta_ms,
|
||||
MAX(rx_tx_payload_delta_ms) AS max_delta_ms,
|
||||
MIN(rx_epoch_ms) AS first_rx_epoch_ms,
|
||||
MAX(rx_epoch_ms) AS last_rx_epoch_ms
|
||||
FROM ble_observation
|
||||
|
|
@ -408,6 +417,10 @@ WHERE tx_payload_epoch_ms IS NOT NULL
|
|||
AND tx_payload_epoch_ms > 0
|
||||
GROUP BY trial_id, receiver, heard, payload_seq;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_payload_seq_summary
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE VIEW IF NOT EXISTS v_payload_seq_summary AS
|
||||
SELECT
|
||||
trial_id,
|
||||
|
|
@ -421,13 +434,78 @@ SELECT
|
|||
FROM ble_observation
|
||||
GROUP BY trial_id, receiver, heard;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_payload_seq_receive_spread
|
||||
--
|
||||
-- for mapping
|
||||
-- Shows how long the same advertised payload sequence was observed by a
|
||||
-- receiver. Useful for confirming repeated observation of one payload.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE VIEW IF NOT EXISTS v_payload_seq_receive_spread AS
|
||||
SELECT
|
||||
trial_id,
|
||||
receiver,
|
||||
heard,
|
||||
payload_seq,
|
||||
COUNT(*) AS observations,
|
||||
MIN(rx_epoch_ms) AS first_rx_epoch_ms,
|
||||
MAX(rx_epoch_ms) AS last_rx_epoch_ms,
|
||||
MAX(rx_epoch_ms) - MIN(rx_epoch_ms) AS rx_spread_ms,
|
||||
MIN(rx_tx_payload_delta_ms) AS min_delta_ms,
|
||||
AVG(rx_tx_payload_delta_ms) AS avg_delta_ms,
|
||||
MAX(rx_tx_payload_delta_ms) AS max_delta_ms
|
||||
FROM ble_observation
|
||||
WHERE tx_payload_epoch_ms IS NOT NULL
|
||||
AND tx_payload_epoch_ms > 0
|
||||
GROUP BY trial_id, receiver, heard, payload_seq;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_payload_tx_epoch_step
|
||||
--
|
||||
-- Shows elapsed sender payload-generation time between successive payload
|
||||
-- sequence values as observed by each receiver/heard pair.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE VIEW IF NOT EXISTS v_payload_tx_epoch_step AS
|
||||
WITH x AS (
|
||||
SELECT
|
||||
trial_id,
|
||||
receiver,
|
||||
heard,
|
||||
payload_seq,
|
||||
MIN(payload_tx_payload_epoch_ms) AS payload_tx_payload_epoch_ms
|
||||
FROM ble_observation
|
||||
WHERE payload_tx_payload_epoch_ms IS NOT NULL
|
||||
AND payload_tx_payload_epoch_ms > 0
|
||||
GROUP BY trial_id, receiver, heard, payload_seq
|
||||
),
|
||||
y AS (
|
||||
SELECT
|
||||
x.*,
|
||||
LAG(payload_tx_payload_epoch_ms) OVER (
|
||||
PARTITION BY trial_id, receiver, heard
|
||||
ORDER BY payload_seq
|
||||
) AS prev_payload_tx_payload_epoch_ms
|
||||
FROM x
|
||||
)
|
||||
SELECT
|
||||
*,
|
||||
payload_tx_payload_epoch_ms - prev_payload_tx_payload_epoch_ms AS tx_payload_epoch_step_ms
|
||||
FROM y
|
||||
WHERE prev_payload_tx_payload_epoch_ms IS NOT NULL;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_map_observation_points
|
||||
--
|
||||
-- Mapping-oriented observation view.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE VIEW IF NOT EXISTS v_map_observation_points AS
|
||||
SELECT
|
||||
obs_id,
|
||||
trial_id,
|
||||
log_file_id,
|
||||
source_line_no,
|
||||
receiver,
|
||||
heard,
|
||||
rx_epoch_ms,
|
||||
|
|
@ -435,11 +513,15 @@ SELECT
|
|||
rx_lat,
|
||||
rx_lon,
|
||||
gps_fix_age_ms,
|
||||
clock_valid,
|
||||
gps_valid,
|
||||
discipline_age_ms,
|
||||
last_discipline_epoch_ms,
|
||||
rssi,
|
||||
avg_rssi,
|
||||
payload_seq,
|
||||
tx_payload_epoch_ms,
|
||||
rx_tx_delta_ms
|
||||
rx_tx_payload_delta_ms
|
||||
FROM ble_observation
|
||||
WHERE rx_lat IS NOT NULL
|
||||
AND rx_lon IS NOT NULL;
|
||||
|
|
@ -447,29 +529,8 @@ WHERE rx_lat IS NOT NULL
|
|||
CREATE VIEW IF NOT EXISTS v_map_good_gps_points AS
|
||||
SELECT *
|
||||
FROM v_map_observation_points
|
||||
WHERE gps_fix_age_ms <= 5000;
|
||||
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_battery_summary
|
||||
--
|
||||
-- Per receiver battery voltage summary for diagnosing power-related logging
|
||||
-- interruptions. vbat_mv = -1 means the PMU value was unavailable.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE VIEW IF NOT EXISTS v_battery_summary AS
|
||||
SELECT
|
||||
trial_id,
|
||||
log_file_id,
|
||||
receiver,
|
||||
|
||||
COUNT(*) AS row_count,
|
||||
SUM(CASE WHEN vbat_mv IS NOT NULL AND vbat_mv >= 0 THEN 1 ELSE 0 END) AS vbat_sample_count,
|
||||
|
||||
MIN(CASE WHEN vbat_mv >= 0 THEN vbat_mv END) AS min_vbat_mv,
|
||||
AVG(CASE WHEN vbat_mv >= 0 THEN vbat_mv END) AS avg_vbat_mv,
|
||||
MAX(CASE WHEN vbat_mv >= 0 THEN vbat_mv END) AS max_vbat_mv
|
||||
FROM ble_observation
|
||||
GROUP BY trial_id, log_file_id, receiver;
|
||||
WHERE clock_valid = 1
|
||||
AND gps_valid = 1
|
||||
AND gps_fix_age_ms <= 5000;
|
||||
|
||||
COMMIT;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue