Database support, initial
This commit is contained in:
parent
02721701a0
commit
32ad481fcf
2 changed files with 655 additions and 0 deletions
362
exercises/18_GPS_Field_QA/scripts/import_satellite_logs.pl
Normal file
362
exercises/18_GPS_Field_QA/scripts/import_satellite_logs.pl
Normal file
|
|
@ -0,0 +1,362 @@
|
|||
#!/usr/bin/env perl
|
||||
# 20260406 ChatGPT
|
||||
# $Header$
|
||||
#
|
||||
# Example:
|
||||
# perl import_satellite_logs.pl \
|
||||
# --dbname satellite_data \
|
||||
# --host localhost \
|
||||
# --user jlpoole \
|
||||
# --schema public \
|
||||
# /path/to/20260406_175441_GUY.csv
|
||||
#
|
||||
# Notes:
|
||||
# * Imports one or more CSV files into tables logs and log_data.
|
||||
# * Preserves all leading hash-prefixed header lines in logs.raw_header_text.
|
||||
# * Uses the file's own CSV header row when present; otherwise falls back to
|
||||
# the expected project header defined in this script.
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use utf8;
|
||||
|
||||
use DBI;
|
||||
use Digest::SHA qw(sha256_hex);
|
||||
use File::Basename qw(basename);
|
||||
use Getopt::Long qw(GetOptions);
|
||||
use Text::CSV_XS;
|
||||
|
||||
my $DEFAULT_HEADER = join ',', qw(
|
||||
record_type timestamp_utc board_id gnss_chip firmware_exercise_name firmware_version
|
||||
boot_timestamp_utc run_id fix_type fix_dimension sats_in_view sat_seen sats_used
|
||||
hdop vdop pdop latitude longitude altitude_m speed_mps course_deg pps_seen
|
||||
quality_class gps_count galileo_count glonass_count beidou_count navic_count
|
||||
qzss_count sbas_count mean_cn0 max_cn0 age_of_fix_ms ttff_ms longest_no_fix_ms
|
||||
sat_talker sat_constellation sat_prn sat_elevation_deg sat_azimuth_deg sat_snr
|
||||
sat_used_in_solution
|
||||
);
|
||||
|
||||
my %opt = (
|
||||
dbname => 'satellite_data',
|
||||
host => 'localhost',
|
||||
port => 5432,
|
||||
schema => 'public',
|
||||
);
|
||||
|
||||
GetOptions(
|
||||
'dbname=s' => \$opt{dbname},
|
||||
'host=s' => \$opt{host},
|
||||
'port=i' => \$opt{port},
|
||||
'user=s' => \$opt{user},
|
||||
'password=s' => \$opt{password},
|
||||
'schema=s' => \$opt{schema},
|
||||
'header-line=s' => \$opt{header_line},
|
||||
'notes=s' => \$opt{import_notes},
|
||||
'help' => \$opt{help},
|
||||
) or die usage();
|
||||
|
||||
if ($opt{help} || !@ARGV) {
|
||||
print usage();
|
||||
exit 0;
|
||||
}
|
||||
|
||||
my $dsn = sprintf 'dbi:Pg:dbname=%s;host=%s;port=%d',
|
||||
$opt{dbname}, $opt{host}, $opt{port};
|
||||
|
||||
my %dbh_attr = (
|
||||
RaiseError => 1,
|
||||
AutoCommit => 1,
|
||||
PrintError => 0,
|
||||
pg_enable_utf8 => 1,
|
||||
);
|
||||
|
||||
my $dbh = DBI->connect($dsn, $opt{user}, $opt{password}, \%dbh_attr)
|
||||
or die "Unable to connect to PostgreSQL\n";
|
||||
|
||||
$dbh->do(sprintf 'SET search_path TO %s', $dbh->quote_identifier($opt{schema}));
|
||||
|
||||
for my $file (@ARGV) {
|
||||
import_file($dbh, $file, \%opt);
|
||||
}
|
||||
|
||||
$dbh->disconnect;
|
||||
exit 0;
|
||||
|
||||
sub usage {
|
||||
return <<'USAGE';
|
||||
Usage:
|
||||
perl import_satellite_logs.pl [options] file1.csv [file2.csv ...]
|
||||
|
||||
Options:
|
||||
--dbname NAME PostgreSQL database name. Default: satellite_data
|
||||
--host HOST PostgreSQL host. Default: localhost
|
||||
--port PORT PostgreSQL port. Default: 5432
|
||||
--user USER PostgreSQL user name
|
||||
--password PASS PostgreSQL password
|
||||
--schema NAME Target schema. Default: public
|
||||
--header-line TEXT Override the expected CSV header line when file lacks one
|
||||
--notes TEXT Import notes stored in logs.import_notes
|
||||
--help Show this help text
|
||||
USAGE
|
||||
}
|
||||
|
||||
sub import_file {
|
||||
my ($dbh, $file, $opt) = @_;
|
||||
|
||||
open my $fh, '<:encoding(UTF-8)', $file
|
||||
or die "Unable to open $file: $!\n";
|
||||
|
||||
my $file_text = do { local $/; <$fh> };
|
||||
close $fh;
|
||||
|
||||
my $sha256 = sha256_hex($file_text);
|
||||
my $file_size_bytes = length $file_text;
|
||||
|
||||
my @lines = split /\n/, $file_text, -1;
|
||||
my @comment_lines;
|
||||
my $header_line;
|
||||
my @data_lines;
|
||||
my $saw_header = 0;
|
||||
|
||||
while (@lines) {
|
||||
my $line = shift @lines;
|
||||
next if !defined $line;
|
||||
|
||||
if ($line =~ /^#/) {
|
||||
push @comment_lines, $line;
|
||||
next;
|
||||
}
|
||||
|
||||
if ($line =~ /^\s*$/ && !@data_lines && !$saw_header) {
|
||||
next;
|
||||
}
|
||||
|
||||
if (!$saw_header && $line =~ /^record_type,/) {
|
||||
$header_line = $line;
|
||||
$saw_header = 1;
|
||||
next;
|
||||
}
|
||||
|
||||
push @data_lines, $line;
|
||||
push @data_lines, @lines;
|
||||
last;
|
||||
}
|
||||
|
||||
@data_lines = grep { defined $_ && $_ !~ /^\s*$/ } @data_lines;
|
||||
|
||||
$header_line ||= $opt->{header_line} || $DEFAULT_HEADER;
|
||||
|
||||
my $raw_header_text = @comment_lines ? join("\n", @comment_lines) . "\n" : undef;
|
||||
|
||||
my $csv = Text::CSV_XS->new({
|
||||
binary => 1,
|
||||
auto_diag => 1,
|
||||
allow_loose_quotes => 1,
|
||||
allow_loose_escapes => 1,
|
||||
});
|
||||
|
||||
$csv->parse($header_line);
|
||||
my @header = $csv->fields;
|
||||
|
||||
my %idx;
|
||||
for my $i (0 .. $#header) {
|
||||
$idx{$header[$i]} = $i;
|
||||
}
|
||||
|
||||
my @required = qw(record_type timestamp_utc board_id gnss_chip run_id);
|
||||
for my $name (@required) {
|
||||
die "Header is missing required column: $name\n" if !exists $idx{$name};
|
||||
}
|
||||
|
||||
$dbh->begin_work;
|
||||
|
||||
my $log_insert_sql = <<'SQL';
|
||||
INSERT INTO logs (
|
||||
source_filename,
|
||||
source_path,
|
||||
file_sha256,
|
||||
file_size_bytes,
|
||||
raw_header_text,
|
||||
csv_header_line,
|
||||
import_notes
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
RETURNING log_id
|
||||
SQL
|
||||
|
||||
my $log_sth = $dbh->prepare($log_insert_sql);
|
||||
$log_sth->execute(
|
||||
basename($file),
|
||||
$file,
|
||||
$sha256,
|
||||
$file_size_bytes,
|
||||
$raw_header_text,
|
||||
$header_line,
|
||||
$opt->{import_notes},
|
||||
);
|
||||
my ($log_id) = $log_sth->fetchrow_array;
|
||||
|
||||
my $data_insert_sql = <<'SQL';
|
||||
INSERT INTO log_data (
|
||||
log_id, row_num, record_type, timestamp_utc, board_id, gnss_chip,
|
||||
firmware_exercise_name, firmware_version, boot_timestamp_utc, run_id,
|
||||
fix_type, fix_dimension, sats_in_view, sat_seen, sats_used,
|
||||
hdop, vdop, pdop, latitude, longitude, altitude_m, speed_mps, course_deg,
|
||||
pps_seen, quality_class, gps_count, galileo_count, glonass_count,
|
||||
beidou_count, navic_count, qzss_count, sbas_count, mean_cn0, max_cn0,
|
||||
age_of_fix_ms, ttff_ms, longest_no_fix_ms, sat_talker, sat_constellation,
|
||||
sat_prn, sat_elevation_deg, sat_azimuth_deg, sat_snr, sat_used_in_solution
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
||||
)
|
||||
SQL
|
||||
|
||||
my $data_sth = $dbh->prepare($data_insert_sql);
|
||||
|
||||
my ($row_count, $sample_count, $satellite_count) = (0, 0, 0);
|
||||
my ($first_ts, $last_ts);
|
||||
my ($board_id, $gnss_chip, $firmware_exercise_name, $firmware_version, $boot_ts, $run_id);
|
||||
|
||||
ROW:
|
||||
for my $line (@data_lines) {
|
||||
next ROW if $line =~ /^\s*$/;
|
||||
|
||||
$csv->parse($line);
|
||||
my @f = $csv->fields;
|
||||
|
||||
my %row;
|
||||
for my $name (@header) {
|
||||
my $value = $f[$idx{$name}];
|
||||
$row{$name} = normalize_value($value);
|
||||
}
|
||||
|
||||
++$row_count;
|
||||
++$sample_count if defined $row{record_type} && $row{record_type} eq 'sample';
|
||||
++$satellite_count if defined $row{record_type} && $row{record_type} eq 'satellite';
|
||||
|
||||
$first_ts //= $row{timestamp_utc};
|
||||
$last_ts = $row{timestamp_utc} if defined $row{timestamp_utc};
|
||||
|
||||
$board_id //= $row{board_id};
|
||||
$gnss_chip //= $row{gnss_chip};
|
||||
$firmware_exercise_name //= $row{firmware_exercise_name};
|
||||
$firmware_version //= $row{firmware_version};
|
||||
$boot_ts //= $row{boot_timestamp_utc};
|
||||
$run_id //= $row{run_id};
|
||||
|
||||
$data_sth->execute(
|
||||
$log_id,
|
||||
$row_count,
|
||||
$row{record_type},
|
||||
$row{timestamp_utc},
|
||||
$row{board_id},
|
||||
$row{gnss_chip},
|
||||
$row{firmware_exercise_name},
|
||||
$row{firmware_version},
|
||||
$row{boot_timestamp_utc},
|
||||
$row{run_id},
|
||||
$row{fix_type},
|
||||
to_int($row{fix_dimension}),
|
||||
to_int($row{sats_in_view}),
|
||||
to_int($row{sat_seen}),
|
||||
to_int($row{sats_used}),
|
||||
to_num($row{hdop}),
|
||||
to_num($row{vdop}),
|
||||
to_num($row{pdop}),
|
||||
to_num($row{latitude}),
|
||||
to_num($row{longitude}),
|
||||
to_num($row{altitude_m}),
|
||||
to_num($row{speed_mps}),
|
||||
to_num($row{course_deg}),
|
||||
to_bool($row{pps_seen}),
|
||||
$row{quality_class},
|
||||
to_int($row{gps_count}),
|
||||
to_int($row{galileo_count}),
|
||||
to_int($row{glonass_count}),
|
||||
to_int($row{beidou_count}),
|
||||
to_int($row{navic_count}),
|
||||
to_int($row{qzss_count}),
|
||||
to_int($row{sbas_count}),
|
||||
to_num($row{mean_cn0}),
|
||||
to_num($row{max_cn0}),
|
||||
to_int($row{age_of_fix_ms}),
|
||||
to_int($row{ttff_ms}),
|
||||
to_int($row{longest_no_fix_ms}),
|
||||
$row{sat_talker},
|
||||
$row{sat_constellation},
|
||||
to_int($row{sat_prn}),
|
||||
to_int($row{sat_elevation_deg}),
|
||||
to_int($row{sat_azimuth_deg}),
|
||||
to_num($row{sat_snr}),
|
||||
to_bool($row{sat_used_in_solution}),
|
||||
);
|
||||
}
|
||||
|
||||
my $update_sql = <<'SQL';
|
||||
UPDATE logs
|
||||
SET board_id = ?,
|
||||
gnss_chip = ?,
|
||||
firmware_exercise_name = ?,
|
||||
firmware_version = ?,
|
||||
boot_timestamp_utc = ?,
|
||||
run_id = ?,
|
||||
first_timestamp_utc = ?,
|
||||
last_timestamp_utc = ?,
|
||||
row_count = ?,
|
||||
sample_count = ?,
|
||||
satellite_count = ?
|
||||
WHERE log_id = ?
|
||||
SQL
|
||||
|
||||
my $update_sth = $dbh->prepare($update_sql);
|
||||
$update_sth->execute(
|
||||
$board_id,
|
||||
$gnss_chip,
|
||||
$firmware_exercise_name,
|
||||
$firmware_version,
|
||||
$boot_ts,
|
||||
$run_id,
|
||||
$first_ts,
|
||||
$last_ts,
|
||||
$row_count,
|
||||
$sample_count,
|
||||
$satellite_count,
|
||||
$log_id,
|
||||
);
|
||||
|
||||
$dbh->commit;
|
||||
|
||||
print STDERR sprintf(
|
||||
"Imported %s => log_id=%d rows=%d samples=%d satellites=%d\n",
|
||||
$file, $log_id, $row_count, $sample_count, $satellite_count,
|
||||
);
|
||||
}
|
||||
|
||||
sub normalize_value {
|
||||
my ($value) = @_;
|
||||
return undef if !defined $value;
|
||||
$value =~ s/^\s+//;
|
||||
$value =~ s/\s+$//;
|
||||
return undef if $value eq '';
|
||||
return $value;
|
||||
}
|
||||
|
||||
sub to_int {
|
||||
my ($value) = @_;
|
||||
return undef if !defined $value;
|
||||
return int($value);
|
||||
}
|
||||
|
||||
sub to_num {
|
||||
my ($value) = @_;
|
||||
return undef if !defined $value;
|
||||
return $value + 0;
|
||||
}
|
||||
|
||||
sub to_bool {
|
||||
my ($value) = @_;
|
||||
return undef if !defined $value;
|
||||
return 1 if $value =~ /^(?:1|true|t|yes|y)$/i;
|
||||
return 0 if $value =~ /^(?:0|false|f|no|n)$/i;
|
||||
return undef;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue