From 8e6883cfc9906be07806ec25ebce51d2bd8cdb43 Mon Sep 17 00:00:00 2001 From: Matteo Cypriani Date: Thu, 1 Apr 2010 11:15:48 +0200 Subject: [PATCH] [Positioning] Add class CSVFileReader CSVFileReader is a specialisation of TextFileReader, that allow reading a CSV line field by field. TextFileReader: - Make private attributes protected to allow derivation. - Add attribute current_line_nb. - Add function read_nonblank_line(). - Delete useless accessor get_name(). Update InputCSV to use CSVFileReader. Adapt unit tests for InputCSV and InputLogCSV: EOF is not reached right after reading the last line, but after trying (and failing) to read one more line. --- owlps-positioning/Makefile | 5 +- owlps-positioning/src/csvfilereader.cc | 58 +++++++ owlps-positioning/src/csvfilereader.hh | 69 ++++++++ owlps-positioning/src/inputcsv.cc | 150 +++------------- owlps-positioning/src/inputcsv.hh | 11 +- owlps-positioning/src/textfilereader.cc | 27 ++- owlps-positioning/src/textfilereader.hh | 23 +-- owlps-positioning/tests/csvfilereader_test.hh | 162 ++++++++++++++++++ owlps-positioning/tests/inputcsv_test.hh | 5 +- owlps-positioning/tests/inputlogcsv_test.hh | 6 + 10 files changed, 357 insertions(+), 159 deletions(-) create mode 100644 owlps-positioning/src/csvfilereader.cc create mode 100644 owlps-positioning/src/csvfilereader.hh create mode 100644 owlps-positioning/tests/csvfilereader_test.hh diff --git a/owlps-positioning/Makefile b/owlps-positioning/Makefile index b77afd6..ca02073 100644 --- a/owlps-positioning/Makefile +++ b/owlps-positioning/Makefile @@ -58,6 +58,7 @@ OBJ_LIST = \ configuration.o \ userinterface.o \ textfilereader.o \ + csvfilereader.o \ textfilewriter.o \ output.o \ outputterminal.o \ @@ -136,11 +137,13 @@ $(OBJ_DIR)/calibrationrequest.o: \ $(OBJ_DIR)/direction.o $(OBJ_DIR)/textfilereader.o: \ $(OBJ_DIR)/posexcept.o +$(OBJ_DIR)/csvfilereader.o: \ + $(OBJ_DIR)/textfilereader.o $(OBJ_DIR)/textfilewriter.o: \ $(OBJ_DIR)/posexcept.o $(OBJ_DIR)/inputcsv.o: \ $(OBJ_DIR)/inputmedium.o \ - $(OBJ_DIR)/textfilereader.o \ + $(OBJ_DIR)/csvfilereader.o \ $(OBJ_DIR)/request.o \ $(OBJ_DIR)/calibrationrequest.o \ $(OBJ_DIR)/stock.o diff --git a/owlps-positioning/src/csvfilereader.cc b/owlps-positioning/src/csvfilereader.cc new file mode 100644 index 0000000..8855ded --- /dev/null +++ b/owlps-positioning/src/csvfilereader.cc @@ -0,0 +1,58 @@ +#include "csvfilereader.hh" + +#include + +using namespace std ; +using namespace boost ; + + + +/* *** Constructors *** */ + + +CSVFileReader::~CSVFileReader() +{ + delete current_token ; +} + + + +/* *** Operations *** */ + + +/** + * Note that this function is not called by the constructor, so you + * must manually call it prior to use read_field(). + * @return false in case of error (EOF, etc.). + */ +bool CSVFileReader::next_line() +{ + if (current_token != NULL) + { + delete current_token ; + current_token = NULL ; + } + + if (! read_nonblank_line(current_line)) + return false ; + + // Split read string into fields (semicolon-separated) + current_token = new tokenizer >( + current_line, escaped_list_separator('\\', separator, '\"')) ; + + token_iterator = current_token->begin() ; + current_field_nb = 0 ; + + return true ; +} + + +void CSVFileReader::print_error_cast() +{ + cerr + << "Bad value « " + << *token_iterator << " » at line " + << current_line_nb << ", field #" + << current_field_nb << ", of input file « " + << file_name << " »!" << endl ; +} diff --git a/owlps-positioning/src/csvfilereader.hh b/owlps-positioning/src/csvfilereader.hh new file mode 100644 index 0000000..f418740 --- /dev/null +++ b/owlps-positioning/src/csvfilereader.hh @@ -0,0 +1,69 @@ +#ifndef _OWLPS_POSITIONING_CSVFILEREADER_HH_ +#define _OWLPS_POSITIONING_CSVFILEREADER_HH_ + +#include "textfilereader.hh" + +#include +#include + + +/// Reads a CSV file, line by line, field by field +class CSVFileReader: public TextFileReader +{ +protected: + const char separator ; + + std::string current_line ; + boost::tokenizer > *current_token ; + boost::tokenizer >::const_iterator + token_iterator ; + unsigned int current_field_nb ; + + void print_error_cast(void) ; + +public: + CSVFileReader(const std::string &filename, + const char _separator = ';'): + TextFileReader(filename), separator(_separator), + current_token(NULL) {} + + virtual ~CSVFileReader(void) ; + + /** @name Operations */ + //@{ + /// Load the first next non-blank line + bool next_line(void) ; + /// Read the next field of the current line + template bool read_field(T &field) ; + //@} +} ; + + + +/* *** Operations *** */ + + +template +inline bool CSVFileReader::read_field(T &field) +{ + if (token_iterator == current_token->end()) + return false ; + ++current_field_nb ; + + try + { + field = boost::lexical_cast(*token_iterator) ; + } + catch (boost::bad_lexical_cast &e) + { + print_error_cast() ; + return false ; + } + + ++token_iterator ; + return true ; +} + + + +#endif // _OWLPS_POSITIONING_CSVFILEREADER_HH_ diff --git a/owlps-positioning/src/inputcsv.cc b/owlps-positioning/src/inputcsv.cc index 355b993..474476b 100644 --- a/owlps-positioning/src/inputcsv.cc +++ b/owlps-positioning/src/inputcsv.cc @@ -10,57 +10,11 @@ using namespace std ; using std::tr1::unordered_map ; -#include -#include - -using namespace boost ; - - - -/* *** Constructors *** */ - - -/** - * Prepares the InputCSV to read a CSV file. - * @param filename The name of the file to open. - * @throw error_opening_input_file if the file cannot be opened. - */ -InputCSV::InputCSV(const string &filename): - file(filename) -{ - read_next_line() ; -} - /* *** Operations *** */ -/** - * Read the next non-blank line. Tabs and spaces at the begining of a - * line are skipped. - * #current_line is set to empty string in case of error. - */ -void InputCSV::read_next_line() -{ - string::size_type first_non_blank = 0 ; - - do - { - if (! file.read_line(current_line)) - { - current_line.clear() ; - return ; - } - ++current_line_nb ; - } - while ((first_non_blank = current_line.find_first_not_of(" \t")) - == string::npos) ; - - current_line.erase(0, first_non_blank) ; -} - - /** * This function reads the next Request in the CSV input file. Blank * lines and lines containing only spaces are skipped. @@ -78,132 +32,70 @@ const Request& InputCSV::get_next_request() { clear_current_request() ; - if (current_line.empty()) // End of file or error + if (! file.next_line()) // End of file or error return *current_request ; - - // Split read string into fields (semicolon-separated) - tokenizer > tok( - current_line, escaped_list_separator('\\', ';', '\"')) ; - - tokenizer >::const_iterator ti(tok.begin()) ; + ++current_line_nb ; // Read Mobile MAC field - if (ti == tok.end()) // Wrong number of fields + string mac_mobile ; + if (! file.read_field(mac_mobile)) // Wrong number of fields return *current_request ; - const Mobile &mobile = Stock::find_create_mobile(*ti) ; + const Mobile &mobile = Stock::find_create_mobile(mac_mobile) ; current_request->set_mobile(&mobile) ; // Read Timestamp field - if (++ti == tok.end()) + uint64_t timestamp_ms ; + if (! file.read_field(timestamp_ms)) { // Wrong number of fields: blank current request current_request->clear() ; return *current_request ; } - try - { - current_request->set_time_sent(Timestamp - (lexical_cast(*ti))) ; - } - catch (bad_lexical_cast &e) - { - cerr - << "InputCSV::get_next_request(): Bad value « " - << *ti << " » at line " - << current_line_nb << ", field « Timestamp », of input file « " - << file.get_name() << " »!" << endl ; - current_request->clear() ; - return *current_request ; - } + current_request->set_time_sent(Timestamp(timestamp_ms)) ; // Read position fields bool is_calibration_request = false ; float pos[3] ; for (int i = 0 ; i < 3 ; ++i) { - if (++ti == tok.end()) + if (! file.read_field(pos[i])) { // Wrong number of fields: blank current request current_request->clear() ; return *current_request ; } - try - { - pos[i] = lexical_cast(*ti) ; - if (pos[i] != 0) - is_calibration_request = true ; - } - catch (bad_lexical_cast &e) - { - cerr - << "InputCSV::get_next_request(): Bad value « " - << *ti << " » at line " - << current_line_nb << ", position field #" << i - << ", of input file « " << file.get_name() << " »!" - << endl ; - current_request->clear() ; - return *current_request ; - } + if (pos[i] != 0) + is_calibration_request = true ; } // Read direction field Direction direction ; - if (++ti == tok.end()) + int direction_int ; + if (! file.read_field(direction_int)) { // Wrong number of fields: blank current request current_request->clear() ; return *current_request ; } - try + if (direction_int != 0) { - int d = lexical_cast(*ti) ; - if (d != 0) - { - is_calibration_request = true ; - direction = d ; - } - } - catch (bad_lexical_cast &e) - { - cerr - << "InputCSV::get_next_request(): Bad value « " - << *ti << " » at line " - << current_line_nb << ", field « Direction », of input file « " - << file.get_name() << " »!" << endl ; - current_request->clear() ; - return *current_request ; + is_calibration_request = true ; + direction = direction_int ; } // Reading {MAC_AP;SS} couples unordered_map measurements ; - for (++ti ; ti != tok.end() ; ++ti) + string mac_ap ; + while (file.read_field(mac_ap)) { - string mac_ap(*ti) ; - - if (++ti == tok.end()) + int ss ; + if (! file.read_field(ss)) { // Wrong number of fields: blank current request current_request->clear() ; return *current_request ; } - - int ss ; - try - { - ss = lexical_cast(*ti) ; - } - catch (bad_lexical_cast &e) - { - cerr - << "InputCSV::get_next_request(): Bad value « " - << *ti << " » at line " - << current_line_nb - << " of input file « " << file.get_name() << " »!" - << endl ; - current_request->clear() ; // Blank current request - return *current_request ; - } const AccessPoint &ap = Stock::find_create_ap(mac_ap) ; measurements[mac_ap].set_ap(&ap) ; measurements[mac_ap].add_ss(ss) ; @@ -223,7 +115,5 @@ const Request& InputCSV::get_next_request() request->set_reference_point(&reference_point) ; } - read_next_line() ; - return *current_request ; } diff --git a/owlps-positioning/src/inputcsv.hh b/owlps-positioning/src/inputcsv.hh index 84d40b9..9310f51 100644 --- a/owlps-positioning/src/inputcsv.hh +++ b/owlps-positioning/src/inputcsv.hh @@ -2,7 +2,7 @@ #define _OWLPS_POSITIONING_INPUTCSV_HH_ #include "inputmedium.hh" -#include "textfilereader.hh" +#include "csvfilereader.hh" #include @@ -14,14 +14,11 @@ class InputCSV: public InputMedium { protected: - TextFileReader file ; - /// Current line contents - std::string current_line ; - - void read_next_line(void) ; + CSVFileReader file ; public: - InputCSV(const std::string &filename) ; + InputCSV(const std::string &filename): + file(filename) {} ~InputCSV(void) {} diff --git a/owlps-positioning/src/textfilereader.cc b/owlps-positioning/src/textfilereader.cc index 7436324..1585574 100644 --- a/owlps-positioning/src/textfilereader.cc +++ b/owlps-positioning/src/textfilereader.cc @@ -13,7 +13,7 @@ using namespace std ; * @throw error_opening_input_file if the file cannot be opened. */ TextFileReader::TextFileReader(const string &_file_name): - file_name(_file_name) + file_name(_file_name), current_line_nb(0) { file.open(file_name.c_str()) ; if (! file) @@ -31,6 +31,30 @@ TextFileReader::~TextFileReader() /* *** Operations *** */ +/** + * Tabs and spaces at the begining of a + * line are deleted. + * \em text is unchanged in case of error. + * @return false in case of error, true else. + */ +bool TextFileReader::read_nonblank_line(string &text) +{ + string line ; + string::size_type first_non_blank = 0 ; + + do + if (! read_line(line)) + return false ; + while ((first_non_blank = line.find_first_not_of(" \t")) + == string::npos) ; + + line.erase(0, first_non_blank) ; + text = line ; + + return true ; +} + + /** * @return The string read, or an empty string in case of error. */ @@ -40,6 +64,7 @@ bool TextFileReader::read_line(string &text) return false ; getline(file, text) ; + ++current_line_nb ; return true ; } diff --git a/owlps-positioning/src/textfilereader.hh b/owlps-positioning/src/textfilereader.hh index a199f8e..9a55f53 100644 --- a/owlps-positioning/src/textfilereader.hh +++ b/owlps-positioning/src/textfilereader.hh @@ -4,12 +4,13 @@ #include #include -/// Read text from a file +/// Read text from a file, line by line class TextFileReader { -private: +protected: std::string file_name ; std::ifstream file ; + unsigned int current_line_nb ; /// Checks if the file is readable and closes it if not bool eof_close(void) ; @@ -20,14 +21,12 @@ public: /** @name Operations */ //@{ + /// Read the next non-blank line + bool read_nonblank_line(std::string &text) ; + /// Read the next line bool read_line(std::string &text) ; bool eof(void) const ; //@} - - /** @name Accessors */ - //@{ - const std::string& get_name(void) const ; - //@} } ; @@ -46,14 +45,4 @@ inline bool TextFileReader::eof() const -/* *** Accessors *** */ - - -inline const std::string& TextFileReader::get_name() const -{ - return file_name ; -} - - - #endif // _OWLPS_POSITIONING_TEXTFILEREADER_HH_ diff --git a/owlps-positioning/tests/csvfilereader_test.hh b/owlps-positioning/tests/csvfilereader_test.hh new file mode 100644 index 0000000..64256f2 --- /dev/null +++ b/owlps-positioning/tests/csvfilereader_test.hh @@ -0,0 +1,162 @@ +#include + +#include "csvfilereader.hh" + +class CSVFileReader_test: public CxxTest::TestSuite +{ +private: + std::string test_file_name ; + +public: + + CSVFileReader_test(void) + { + test_file_name = "/tmp/CSVFileReader_test_file.csv" ; + } + + + ~CSVFileReader_test(void) + { + TestUtil::remove_file(test_file_name) ; + } + + + static CSVFileReader_test* createSuite(void) + { + return new CSVFileReader_test() ; + } + + + static void destroySuite(CSVFileReader_test *suite) + { + delete suite ; + } + + + void test_csvfilereader(void) + { + const unsigned int nlines = 6 ; + + std::ofstream output_file(test_file_name.c_str()) ; + assert(output_file) ; + for (unsigned int i = 0 ; i < nlines ; ++i) + output_file << " \n c,string,42,3.1416\n \n" ; + output_file.close() ; + + CSVFileReader *csvfilereader ; + TS_ASSERT_THROWS_NOTHING( + csvfilereader = new CSVFileReader(test_file_name, ',')) ; + + for (unsigned int i = 0 ; i < nlines ; ++i) + { + TS_ASSERT(csvfilereader->next_line()) ; + + char c ; + TS_ASSERT(csvfilereader->read_field(c)) ; + TS_ASSERT_EQUALS(c, 'c') ; + + std::string s ; + TS_ASSERT(csvfilereader->read_field(s)) ; + TS_ASSERT_EQUALS(s, "string") ; + + uint64_t i ; + TS_ASSERT(csvfilereader->read_field(i)) ; + TS_ASSERT_EQUALS(i, 42u) ; + + double f ; + TS_ASSERT(csvfilereader->read_field(f)) ; + TS_ASSERT_EQUALS(f, 3.1416) ; + + TS_ASSERT(! csvfilereader->read_field(f)) ; + TS_ASSERT(! csvfilereader->eof()) ; + } + + TS_ASSERT(! csvfilereader->next_line()) ; + TS_ASSERT(csvfilereader->eof()) ; + + delete csvfilereader ; + } + + + void test_request(void) + { + const unsigned int nlines = 10 ; + + std::ofstream output_file(test_file_name.c_str()) ; + assert(output_file) ; + for (unsigned int i = 0 ; i < nlines ; ++i) + output_file << "aa:bb:cc:dd:ee:ff;1265120910725;0;0;0;0;11:22:33:44:55:01;-58;11:22:33:44:55:03;-50;11:22:33:44:55:02;-42;11:22:33:44:55:01;-55;11:22:33:44:55:02;-37\n" ; + output_file.close() ; + + CSVFileReader *csvfilereader ; + TS_ASSERT_THROWS_NOTHING( + csvfilereader = new CSVFileReader(test_file_name)) ; + + for (unsigned int i = 0 ; i < nlines ; ++i) + { + TS_ASSERT(csvfilereader->next_line()) ; + + std::string mac ; + TS_ASSERT(csvfilereader->read_field(mac)) ; + TS_ASSERT_EQUALS(mac, "aa:bb:cc:dd:ee:ff") ; + + uint64_t timestamp ; + TS_ASSERT(csvfilereader->read_field(timestamp)) ; + TS_ASSERT_EQUALS(timestamp, 1265120910725ull) ; + + float position ; + TS_ASSERT(csvfilereader->read_field(position)) ; + TS_ASSERT_EQUALS(position, 0) ; + + TS_ASSERT(csvfilereader->read_field(position)) ; + TS_ASSERT_EQUALS(position, 0) ; + + TS_ASSERT(csvfilereader->read_field(position)) ; + TS_ASSERT_EQUALS(position, 0) ; + + int direction ; + TS_ASSERT(csvfilereader->read_field(direction)) ; + TS_ASSERT_EQUALS(direction, 0) ; + + TS_ASSERT(csvfilereader->read_field(mac)) ; + TS_ASSERT_EQUALS(mac, "11:22:33:44:55:01") ; + + int ss ; + TS_ASSERT(csvfilereader->read_field(ss)) ; + TS_ASSERT_EQUALS(ss, -58) ; + + TS_ASSERT(csvfilereader->read_field(mac)) ; + TS_ASSERT_EQUALS(mac, "11:22:33:44:55:03") ; + + TS_ASSERT(csvfilereader->read_field(ss)) ; + TS_ASSERT_EQUALS(ss, -50) ; + + TS_ASSERT(csvfilereader->read_field(mac)) ; + TS_ASSERT_EQUALS(mac, "11:22:33:44:55:02") ; + + TS_ASSERT(csvfilereader->read_field(ss)) ; + TS_ASSERT_EQUALS(ss, -42) ; + + TS_ASSERT(csvfilereader->read_field(mac)) ; + TS_ASSERT_EQUALS(mac, "11:22:33:44:55:01") ; + + TS_ASSERT(csvfilereader->read_field(ss)) ; + TS_ASSERT_EQUALS(ss, -55) ; + + TS_ASSERT(csvfilereader->read_field(mac)) ; + TS_ASSERT_EQUALS(mac, "11:22:33:44:55:02") ; + + TS_ASSERT(csvfilereader->read_field(ss)) ; + TS_ASSERT_EQUALS(ss, -37) ; + + TS_ASSERT(! csvfilereader->read_field(mac)) ; + TS_ASSERT(! csvfilereader->eof()) ; + } + + TS_ASSERT(! csvfilereader->next_line()) ; + TS_ASSERT(csvfilereader->eof()) ; + + delete csvfilereader ; + } + +} ; diff --git a/owlps-positioning/tests/inputcsv_test.hh b/owlps-positioning/tests/inputcsv_test.hh index cb61a1c..d92c7d1 100644 --- a/owlps-positioning/tests/inputcsv_test.hh +++ b/owlps-positioning/tests/inputcsv_test.hh @@ -136,12 +136,11 @@ public: // End of file - TS_ASSERT(! inputcsv1) ; - TS_ASSERT(inputcsv1.eof()) ; + TS_ASSERT(inputcsv1) ; + TS_ASSERT(! inputcsv1.eof()) ; request1 = inputcsv1.get_next_request() ; TS_ASSERT(! request1) ; - TS_ASSERT(! inputcsv1) ; TS_ASSERT(inputcsv1.eof()) ; } diff --git a/owlps-positioning/tests/inputlogcsv_test.hh b/owlps-positioning/tests/inputlogcsv_test.hh index e85a361..938e449 100644 --- a/owlps-positioning/tests/inputlogcsv_test.hh +++ b/owlps-positioning/tests/inputlogcsv_test.hh @@ -83,6 +83,12 @@ public: TS_ASSERT(TestUtil::request_equals(**i, request_log)) ; } + TS_ASSERT(! inputcsv_csv.eof()) ; + TS_ASSERT(! inputcsv_log.eof()) ; + + TS_ASSERT(! inputcsv_csv.get_next_request()) ; + TS_ASSERT(! inputcsv_log.get_next_request()) ; + TS_ASSERT(inputcsv_csv.eof()) ; TS_ASSERT(inputcsv_log.eof()) ; }