From 0addbc2a0f2a9c589ffd72e929cdfe22d6a07903 Mon Sep 17 00:00:00 2001 From: Matteo Cypriani Date: Mon, 29 Jul 2013 14:14:24 -0400 Subject: [PATCH] [scripts] Add AggSetCoord Add the script owlps-aggsetcoord.pl, which allows to filter an aggregation file and interpolate the coordinates already present in it. --- scripts/owlps-aggsetcoord.pl | 480 +++++++++++++++++++++++++++++++++++ 1 file changed, 480 insertions(+) create mode 100755 scripts/owlps-aggsetcoord.pl diff --git a/scripts/owlps-aggsetcoord.pl b/scripts/owlps-aggsetcoord.pl new file mode 100755 index 0000000..860fe26 --- /dev/null +++ b/scripts/owlps-aggsetcoord.pl @@ -0,0 +1,480 @@ +#!/usr/bin/perl -w + +=head1 NAME + +owlps-aggsetcoord - set the coordinates in an aggregation CSV file + + +=head1 SYNOPSIS + +B [ B<-h> | B<-V> ] [ B<-v> ] [ > ] + + +=head1 DESCRIPTION + +B automatically filters an aggregation file and interpolates +the blank coordinates according to the previous and next known coordinates and +their timestamps, assuming the mobile moves at a constant speed between two +timestamps. The interpolation is done in 3 dimensions, each dimension being +interpolated independently. + +Note that the default coordinate in OwlPS is registered in aggregation files as +"0.00", therefore this program considers the string "0.00" as an undefined +coordinate. Zero coordinates are not prevented, but you have to write them in +another way in your input file, such as "0", "00.00", "0.0", "0.000", etc. + +In theory, you can provide more than one I at once, but don't +do that unless you really know what you're doing. The (list of) aggregation file +name(s) I be placed after the options. If no file is provided, the +standard input is read. + + +=head1 LIMITATIONS + +=over 3 + +=item * +The mobile terminal's MAC address is not taken into account, so if the input +file includes several mobiles the results won't make any sense. + +=item * +It is not yet possible to choose an interval of lines to work on: the whole file +is treated. + +=back + + +=head1 OPTIONS + +=over 7 + +=item B<-h>, B<--help> + +Print version and help message and exit. + +=item B<-V>, B<--version> + +Print version message and exit. + +=item B<-v> + +Turn on verbose mode (displays a trace on the standard error). + +=back + + +=head1 COPYING + +This script and its documentation are part of the Owl Positioning System (OwlPS) +project. They are subject to the copyright notice and license terms in the +COPYRIGHT.t2t file found in the top-level directory of the OwlPS distribution +and at http://code.lm7.fr/p/owlps/source/tree/master/COPYRIGHT.t2t + + +=head1 SEE ALSO + +owlps(7), owlps-aggregatord(1) + +=cut + + +use strict; +use Getopt::Std; +use Pod::Usage; +use OwlPS::TimeInterpolation; +use OwlPS::CSV; + + +## Constants ## + +# CSV format version handled by this program +use constant FORMAT_VERSION => 1; + +# Indexes of the fields in the CSV lines (first field is #0) +use constant TYPE_FIELD_INDEX => 2; +use constant TIMESTAMP_FIELD_INDEX => 4; +use constant X_FIELD_INDEX => 5; +use constant Y_FIELD_INDEX => 6; +use constant Z_FIELD_INDEX => 7; + +# Number corresponding to a normal positioning request +use constant REQUEST_TYPE_NORMAL => 0; + +# String corresponding to an "empty" coordinate in the input file +use constant BASE_COORD => "0.00"; + + +## Global variables ## + +# Verbose mode +my $verbose; + +# Current line +my $cur_line; + +# Current line number +my $line_nb = 0; + +# Last "real" values of X, Y and Z to interpolate with +my ($last_x, $last_y, $last_z); + +# Timestamps corresponding to each of these last values +my ($last_x_t, $last_y_t, $last_z_t); + +# Number of waiting Xs, Ys and Zs that we will have to interpolate +my ($xs, $ys, $zs) = (0, 0, 0); + +# Array of the previous lines to interpolate (we store them while we are waiting +# for the next "complete" line) +my @prev_lines; + + +## Functions ## + +sub VERSION_MESSAGE { + my $handle = $_[0] || *STDOUT; + print $handle + "This is OwlPS AggSetCoord, part of the Owl Positioning System project.\n" + . "CSV aggregation format version handled: " + . FORMAT_VERSION . "\n"; +} + + +sub HELP_MESSAGE { + my $handle = $_[0] || *STDOUT; + pod2usage(-output => $handle, + -exitval => "NOEXIT"); +} + + +# Prints a trace message if verbose mode is active (no new line is printed). +# Parameters: @message +sub trace(@) { + print STDERR "Line #$line_nb: @_" if ($verbose); +} + + +# Prints a warning message terminated by a new line. +# Parameters: @message +sub print_warning(@) { + print STDERR "Warning on line #$line_nb: @_\n"; +} + + +# Returns true if all $x, $y and $z are ready to be printed. +# Arguments: $x, $y, $z +sub is_complete($$$) { + my ($x, $y, $z) = @_; + return ($x ne BASE_COORD && $y ne BASE_COORD && $z ne BASE_COORD); +} + + +# Returns true if all $x, $y and $z are untouched (fully incomplete line) or +# undefined. +# Arguments: $x, $y, $z +sub is_blank($$$) { + my ($x, $y, $z) = @_; + return (!defined($x) && !defined($y) && !defined($z)) + || ($x eq BASE_COORD && $y eq BASE_COORD && $z eq BASE_COORD); +} + + +# Checks the stored previous line and prints/delete what the complete lines, +# starting with the oldest line stored and stopping with the first incomplete +# line. +sub check_oldest_lines() { + while ($prev_lines[0]) { + # If the oldest stored line is empty, just print it and delete it + if ($prev_lines[0] eq "\n") { + print shift @prev_lines; + next; + } + + # Check the oldest (non-empty) line + my ( + $csv_format, + $mac, + $type, + $nb_pkt, + $timestamp, + $x, + $y, + $z, + $end_line + ) = split(';', $prev_lines[0], 9); + + # Display and delete the line if it is not a positioning request or if + # it is complete + if ($type != REQUEST_TYPE_NORMAL + || is_complete($x, $y, $z)) + { + print shift @prev_lines; + next; # Jump to the next oldest line + } + + # If this line is not complete, it will have to be interpolated and the + # potential newer stored lines as well, so we end the check here + last; + } +} + + +# Modifies the field number $idx of $nb_lines previous lines stored in +# @prev_lines with values interpolated from real coordinates $start and $stop +# registered at $start_time and $stop_time. Lines containing non-positioning +# requests are skipped and do not count in $nb_lines. Parameters: $nb_lines, +# $idx, $start, $start_time, $stop, $stop_time +sub apply_interpolation($$$$$$) { + my ($nb_lines, $idx, $start, $start_time, $stop, $stop_time) = @_; + + for (my $i = 1 ; $i <= $nb_lines ; $i++) { + # Skip blank lines + if ($prev_lines[-$i] eq "\n") { + # Incrementing the number of lines to treat has no incidence but to + # allow the loop to go one element further to take into account the + # blank line + $nb_lines++; + next; + } + + # Skip non-positioning requests + my $type = get_field($prev_lines[-$i], TYPE_FIELD_INDEX); + if ($type != REQUEST_TYPE_NORMAL) { + $nb_lines++; + next; + } + + my $time = get_field($prev_lines[-$i], TIMESTAMP_FIELD_INDEX); + my $newvalue = + interpolate_1d($start, $start_time, $stop, $stop_time, $time); + # Round the interpolated coordinate to two decimals + $newvalue = round2($newvalue); + # But if the new value is "0.00", it will be considered as "incomplete", + # so we add a leading 0 to avoid being confused when we later check the + # line + $newvalue = "0" . $newvalue if ($newvalue == BASE_COORD); + $prev_lines[-$i] = update_field($prev_lines[-$i], $idx, $newvalue); + } +} + + +## Option parsing ## + +$Getopt::Std::STANDARD_HELP_VERSION = 1; +use constant OPTIONS => 'hvV'; +my %options; +if (!getopts(OPTIONS, \%options)) { + HELP_MESSAGE(*STDERR); + exit 1; +} + +if ($options{'h'}) { + VERSION_MESSAGE(); + HELP_MESSAGE(); + exit 0; +} + +if ($options{'V'}) { + VERSION_MESSAGE(); + exit 0; +} + +$verbose = $options{'v'}; + + +## Main loop: read lines ## + +while ($cur_line = <>) { + $line_nb++; + trace($cur_line); + + + # The current line is empty + if ($cur_line eq "\n") { + # If we don't have previous lines, we print it + if (@prev_lines == 0) { print "\n" } + # If we have previous lines, we store it + else { push @prev_lines, "\n" } + next; + } + + + my ( + $csv_format, + $mac, + $type, + $nb_pkt, + $timestamp, + $x, + $y, + $z, + $end_line + ) = split(';', $cur_line, 9); + + # Check the CSV format version + if ($csv_format != FORMAT_VERSION) { + die "CSV format \"$csv_format\" not supported!"; + } + + + # Check the stored previous line and print what we can + check_oldest_lines(); + + + # Check the request type: we work only on positioning requests but must take + # into account other request types + if ($type != REQUEST_TYPE_NORMAL) { + # If we have no lines in memory, we can just print the line and jump to + # the next one + if (@prev_lines == 0) { + print $cur_line; + next; + } + + # Otherwise, we have to store the line + trace("Non-positioning request line stored.\n"); + push @prev_lines, $cur_line; + next; + } + + + # No previous line in memory: we might have got a chance to get rid of the + # current line right now + if (@prev_lines == 0) { + # The current line is "complete": print it and store the coordinates for + # later interpolation + if (is_complete($x, $y, $z)) { + # Print current line + print $cur_line; + # Store current values + $last_x = $x; + $last_y = $y; + $last_z = $z; + # Store current timestamps + $last_x_t = $last_y_t = $last_z_t = $timestamp; + # Jump to next line + next; + } + + # The current line is fully incomplete (all the coordinates at 0) and we + # don't have previous coordinates to interpolate: this should not happen + # if the input file is correct, so we print a warning but don't store + # the line since we won't be able to interpolate anything anyway + if (is_blank($x, $y, $z) && is_blank($last_x, $last_y, $last_z)) { + print_warning("No previous X, Y and Z values to interpolate!"); + print $cur_line; + next; + } + } + + + # OK, the serious work starts here. The current line is either complete or + # incomplete. If it is incomplete, it can still contain non-zero values, in + # which case we will have to interpolate the previously stored values with + # this value. For instance, if the current line have X = 0, Y = 42, Z = 0, + # it is incomplete because X and Z are zero, but we will use the Y value + # (42) as the end bound to interpolate Y values of the previous lines we + # stored (assuming we have a begin bound value for Z). In this case, the + # interpolation is done by modifying the lines in memory (which can still + # have missing values even after the interpolation, but that will be checked + # by the code above, at the next loop). + # + # In any case, if the line is incomplete, we will store it for later + # interpolation (except if there are no real coordinates before this line, + # but this should not happen if your input file is correct). + # + # If it is complete, we will also store it after having interpolated the + # previous lines in memory, and all the lines will be printed at the next + # loop in the check above (or after the end of the loop if it's the last + # line). + + + # Check if we have to compute X's value + if ($x eq BASE_COORD) { + if (!defined($last_x)) { + # We've got a problem if we don't have a previous value to + # interpolate + print_warning("No previous X value to interpolate!"); + } + else { + $xs++; + } + } + + # Check if we have to compute Y's value + if ($y eq BASE_COORD) { + if (!defined($last_y)) { + print_warning("No previous Y value to interpolate!"); + } + else { + $ys++; + } + } + + # Check if we have to compute Z's value + if ($z eq BASE_COORD) { + if (!defined($last_z)) { + print_warning("No previous Z value to interpolate!"); + } + else { + $zs++; + } + } + + # Interpolate what we can if needed + if (@prev_lines > 0) { + # X interpolation + if ($xs > 0 && $x ne BASE_COORD) { + apply_interpolation($xs, X_FIELD_INDEX, $last_x, $last_x_t, $x, + $timestamp); + $xs = 0; # There are no more Xs waiting + } + + # Y interpolation + if ($ys > 0 && $y ne BASE_COORD) { + apply_interpolation($ys, Y_FIELD_INDEX, $last_y, $last_y_t, $y, + $timestamp); + $ys = 0; # There are no more Ys waiting + } + + # Z interpolation + if ($zs > 0 && $z ne BASE_COORD) { + apply_interpolation($zs, Z_FIELD_INDEX, $last_z, $last_z_t, $z, + $timestamp); + $zs = 0; # There are no more Zs waiting + } + } + + # Store the current line; if it is complete, previous lines should be + # complete as well now, since we just interpolated what needed to do, so all + # the lines in @prev_lines will be printed at the check at the beginning of + # the next loop + trace("Line stored.\n"); + push @prev_lines, $cur_line; + + # Store the current coordinates if they are non-zero + if ($x ne BASE_COORD) { + $last_x = $x; + $last_x_t = $timestamp; + } + if ($y ne BASE_COORD) { + $last_y = $y; + $last_y_t = $timestamp; + } + if ($z ne BASE_COORD) { + $last_z = $z; + $last_z_t = $timestamp; + } +} + + +## Display the lines remaining in memory if any ## + +if (@prev_lines) { + trace("Printing " . scalar(@prev_lines) . " lines left in memory...\n"); + print foreach (@prev_lines); +} +trace("The end.\n"); + + +# vim: tabstop=4:shiftwidth=4:expandtab:textwidth=80