[scripts] Add AggSetCoord

Add the script owlps-aggsetcoord.pl, which allows to filter an
aggregation file and interpolate the coordinates already present in it.
This commit is contained in:
Matteo Cypriani 2013-07-29 14:14:24 -04:00
parent 9073013997
commit 0addbc2a0f
1 changed files with 480 additions and 0 deletions

480
scripts/owlps-aggsetcoord.pl Executable file
View File

@ -0,0 +1,480 @@
#!/usr/bin/perl -w
=head1 NAME
owlps-aggsetcoord - set the coordinates in an aggregation CSV file
=head1 SYNOPSIS
B<owlps-aggsetcoord> [ B<-h> | B<-V> ] [ B<-v> ] [ <I<aggregation_file>> ]
=head1 DESCRIPTION
B<owlps-aggsetcoord> automatically filters an aggregation file and interpolates
the blank coordinates according to the previous and next known coordinates and
their timestamps, assuming the mobile moves at a constant speed between two
timestamps. The interpolation is done in 3 dimensions, each dimension being
interpolated independently.
Note that the default coordinate in OwlPS is registered in aggregation files as
"0.00", therefore this program considers the string "0.00" as an undefined
coordinate. Zero coordinates are not prevented, but you have to write them in
another way in your input file, such as "0", "00.00", "0.0", "0.000", etc.
In theory, you can provide more than one I<aggregation_file> at once, but don't
do that unless you really know what you're doing. The (list of) aggregation file
name(s) I<must> be placed after the options. If no file is provided, the
standard input is read.
=head1 LIMITATIONS
=over 3
=item *
The mobile terminal's MAC address is not taken into account, so if the input
file includes several mobiles the results won't make any sense.
=item *
It is not yet possible to choose an interval of lines to work on: the whole file
is treated.
=back
=head1 OPTIONS
=over 7
=item B<-h>, B<--help>
Print version and help message and exit.
=item B<-V>, B<--version>
Print version message and exit.
=item B<-v>
Turn on verbose mode (displays a trace on the standard error).
=back
=head1 COPYING
This script and its documentation are part of the Owl Positioning System (OwlPS)
project. They are subject to the copyright notice and license terms in the
COPYRIGHT.t2t file found in the top-level directory of the OwlPS distribution
and at http://code.lm7.fr/p/owlps/source/tree/master/COPYRIGHT.t2t
=head1 SEE ALSO
owlps(7), owlps-aggregatord(1)
=cut
use strict;
use Getopt::Std;
use Pod::Usage;
use OwlPS::TimeInterpolation;
use OwlPS::CSV;
## Constants ##
# CSV format version handled by this program
use constant FORMAT_VERSION => 1;
# Indexes of the fields in the CSV lines (first field is #0)
use constant TYPE_FIELD_INDEX => 2;
use constant TIMESTAMP_FIELD_INDEX => 4;
use constant X_FIELD_INDEX => 5;
use constant Y_FIELD_INDEX => 6;
use constant Z_FIELD_INDEX => 7;
# Number corresponding to a normal positioning request
use constant REQUEST_TYPE_NORMAL => 0;
# String corresponding to an "empty" coordinate in the input file
use constant BASE_COORD => "0.00";
## Global variables ##
# Verbose mode
my $verbose;
# Current line
my $cur_line;
# Current line number
my $line_nb = 0;
# Last "real" values of X, Y and Z to interpolate with
my ($last_x, $last_y, $last_z);
# Timestamps corresponding to each of these last values
my ($last_x_t, $last_y_t, $last_z_t);
# Number of waiting Xs, Ys and Zs that we will have to interpolate
my ($xs, $ys, $zs) = (0, 0, 0);
# Array of the previous lines to interpolate (we store them while we are waiting
# for the next "complete" line)
my @prev_lines;
## Functions ##
sub VERSION_MESSAGE {
my $handle = $_[0] || *STDOUT;
print $handle
"This is OwlPS AggSetCoord, part of the Owl Positioning System project.\n"
. "CSV aggregation format version handled: "
. FORMAT_VERSION . "\n";
}
sub HELP_MESSAGE {
my $handle = $_[0] || *STDOUT;
pod2usage(-output => $handle,
-exitval => "NOEXIT");
}
# Prints a trace message if verbose mode is active (no new line is printed).
# Parameters: @message
sub trace(@) {
print STDERR "Line #$line_nb: @_" if ($verbose);
}
# Prints a warning message terminated by a new line.
# Parameters: @message
sub print_warning(@) {
print STDERR "Warning on line #$line_nb: @_\n";
}
# Returns true if all $x, $y and $z are ready to be printed.
# Arguments: $x, $y, $z
sub is_complete($$$) {
my ($x, $y, $z) = @_;
return ($x ne BASE_COORD && $y ne BASE_COORD && $z ne BASE_COORD);
}
# Returns true if all $x, $y and $z are untouched (fully incomplete line) or
# undefined.
# Arguments: $x, $y, $z
sub is_blank($$$) {
my ($x, $y, $z) = @_;
return (!defined($x) && !defined($y) && !defined($z))
|| ($x eq BASE_COORD && $y eq BASE_COORD && $z eq BASE_COORD);
}
# Checks the stored previous line and prints/delete what the complete lines,
# starting with the oldest line stored and stopping with the first incomplete
# line.
sub check_oldest_lines() {
while ($prev_lines[0]) {
# If the oldest stored line is empty, just print it and delete it
if ($prev_lines[0] eq "\n") {
print shift @prev_lines;
next;
}
# Check the oldest (non-empty) line
my (
$csv_format,
$mac,
$type,
$nb_pkt,
$timestamp,
$x,
$y,
$z,
$end_line
) = split(';', $prev_lines[0], 9);
# Display and delete the line if it is not a positioning request or if
# it is complete
if ($type != REQUEST_TYPE_NORMAL
|| is_complete($x, $y, $z))
{
print shift @prev_lines;
next; # Jump to the next oldest line
}
# If this line is not complete, it will have to be interpolated and the
# potential newer stored lines as well, so we end the check here
last;
}
}
# Modifies the field number $idx of $nb_lines previous lines stored in
# @prev_lines with values interpolated from real coordinates $start and $stop
# registered at $start_time and $stop_time. Lines containing non-positioning
# requests are skipped and do not count in $nb_lines. Parameters: $nb_lines,
# $idx, $start, $start_time, $stop, $stop_time
sub apply_interpolation($$$$$$) {
my ($nb_lines, $idx, $start, $start_time, $stop, $stop_time) = @_;
for (my $i = 1 ; $i <= $nb_lines ; $i++) {
# Skip blank lines
if ($prev_lines[-$i] eq "\n") {
# Incrementing the number of lines to treat has no incidence but to
# allow the loop to go one element further to take into account the
# blank line
$nb_lines++;
next;
}
# Skip non-positioning requests
my $type = get_field($prev_lines[-$i], TYPE_FIELD_INDEX);
if ($type != REQUEST_TYPE_NORMAL) {
$nb_lines++;
next;
}
my $time = get_field($prev_lines[-$i], TIMESTAMP_FIELD_INDEX);
my $newvalue =
interpolate_1d($start, $start_time, $stop, $stop_time, $time);
# Round the interpolated coordinate to two decimals
$newvalue = round2($newvalue);
# But if the new value is "0.00", it will be considered as "incomplete",
# so we add a leading 0 to avoid being confused when we later check the
# line
$newvalue = "0" . $newvalue if ($newvalue == BASE_COORD);
$prev_lines[-$i] = update_field($prev_lines[-$i], $idx, $newvalue);
}
}
## Option parsing ##
$Getopt::Std::STANDARD_HELP_VERSION = 1;
use constant OPTIONS => 'hvV';
my %options;
if (!getopts(OPTIONS, \%options)) {
HELP_MESSAGE(*STDERR);
exit 1;
}
if ($options{'h'}) {
VERSION_MESSAGE();
HELP_MESSAGE();
exit 0;
}
if ($options{'V'}) {
VERSION_MESSAGE();
exit 0;
}
$verbose = $options{'v'};
## Main loop: read lines ##
while ($cur_line = <>) {
$line_nb++;
trace($cur_line);
# The current line is empty
if ($cur_line eq "\n") {
# If we don't have previous lines, we print it
if (@prev_lines == 0) { print "\n" }
# If we have previous lines, we store it
else { push @prev_lines, "\n" }
next;
}
my (
$csv_format,
$mac,
$type,
$nb_pkt,
$timestamp,
$x,
$y,
$z,
$end_line
) = split(';', $cur_line, 9);
# Check the CSV format version
if ($csv_format != FORMAT_VERSION) {
die "CSV format \"$csv_format\" not supported!";
}
# Check the stored previous line and print what we can
check_oldest_lines();
# Check the request type: we work only on positioning requests but must take
# into account other request types
if ($type != REQUEST_TYPE_NORMAL) {
# If we have no lines in memory, we can just print the line and jump to
# the next one
if (@prev_lines == 0) {
print $cur_line;
next;
}
# Otherwise, we have to store the line
trace("Non-positioning request line stored.\n");
push @prev_lines, $cur_line;
next;
}
# No previous line in memory: we might have got a chance to get rid of the
# current line right now
if (@prev_lines == 0) {
# The current line is "complete": print it and store the coordinates for
# later interpolation
if (is_complete($x, $y, $z)) {
# Print current line
print $cur_line;
# Store current values
$last_x = $x;
$last_y = $y;
$last_z = $z;
# Store current timestamps
$last_x_t = $last_y_t = $last_z_t = $timestamp;
# Jump to next line
next;
}
# The current line is fully incomplete (all the coordinates at 0) and we
# don't have previous coordinates to interpolate: this should not happen
# if the input file is correct, so we print a warning but don't store
# the line since we won't be able to interpolate anything anyway
if (is_blank($x, $y, $z) && is_blank($last_x, $last_y, $last_z)) {
print_warning("No previous X, Y and Z values to interpolate!");
print $cur_line;
next;
}
}
# OK, the serious work starts here. The current line is either complete or
# incomplete. If it is incomplete, it can still contain non-zero values, in
# which case we will have to interpolate the previously stored values with
# this value. For instance, if the current line have X = 0, Y = 42, Z = 0,
# it is incomplete because X and Z are zero, but we will use the Y value
# (42) as the end bound to interpolate Y values of the previous lines we
# stored (assuming we have a begin bound value for Z). In this case, the
# interpolation is done by modifying the lines in memory (which can still
# have missing values even after the interpolation, but that will be checked
# by the code above, at the next loop).
#
# In any case, if the line is incomplete, we will store it for later
# interpolation (except if there are no real coordinates before this line,
# but this should not happen if your input file is correct).
#
# If it is complete, we will also store it after having interpolated the
# previous lines in memory, and all the lines will be printed at the next
# loop in the check above (or after the end of the loop if it's the last
# line).
# Check if we have to compute X's value
if ($x eq BASE_COORD) {
if (!defined($last_x)) {
# We've got a problem if we don't have a previous value to
# interpolate
print_warning("No previous X value to interpolate!");
}
else {
$xs++;
}
}
# Check if we have to compute Y's value
if ($y eq BASE_COORD) {
if (!defined($last_y)) {
print_warning("No previous Y value to interpolate!");
}
else {
$ys++;
}
}
# Check if we have to compute Z's value
if ($z eq BASE_COORD) {
if (!defined($last_z)) {
print_warning("No previous Z value to interpolate!");
}
else {
$zs++;
}
}
# Interpolate what we can if needed
if (@prev_lines > 0) {
# X interpolation
if ($xs > 0 && $x ne BASE_COORD) {
apply_interpolation($xs, X_FIELD_INDEX, $last_x, $last_x_t, $x,
$timestamp);
$xs = 0; # There are no more Xs waiting
}
# Y interpolation
if ($ys > 0 && $y ne BASE_COORD) {
apply_interpolation($ys, Y_FIELD_INDEX, $last_y, $last_y_t, $y,
$timestamp);
$ys = 0; # There are no more Ys waiting
}
# Z interpolation
if ($zs > 0 && $z ne BASE_COORD) {
apply_interpolation($zs, Z_FIELD_INDEX, $last_z, $last_z_t, $z,
$timestamp);
$zs = 0; # There are no more Zs waiting
}
}
# Store the current line; if it is complete, previous lines should be
# complete as well now, since we just interpolated what needed to do, so all
# the lines in @prev_lines will be printed at the check at the beginning of
# the next loop
trace("Line stored.\n");
push @prev_lines, $cur_line;
# Store the current coordinates if they are non-zero
if ($x ne BASE_COORD) {
$last_x = $x;
$last_x_t = $timestamp;
}
if ($y ne BASE_COORD) {
$last_y = $y;
$last_y_t = $timestamp;
}
if ($z ne BASE_COORD) {
$last_z = $z;
$last_z_t = $timestamp;
}
}
## Display the lines remaining in memory if any ##
if (@prev_lines) {
trace("Printing " . scalar(@prev_lines) . " lines left in memory...\n");
print foreach (@prev_lines);
}
trace("The end.\n");
# vim: tabstop=4:shiftwidth=4:expandtab:textwidth=80