wip

include/updater/csv_source.hpp

@@ -2,6 +2,7 @@
 #define OSRM_UPDATER_CSV_SOURCE_HPP
 
 #include "updater/source.hpp"
+#include "updater/updater_config.hpp"
 
 namespace osrm
 {
@@ -9,8 +10,8 @@ namespace updater
 {
 namespace csv
 {
-SegmentLookupTable readSegmentValues(const std::vector<std::string> &paths);
+SegmentLookupTable readSegmentValues(const std::vector<std::string> &paths, UpdaterConfig::SpeedAndTurnPenaltyFormat format);
-TurnLookupTable readTurnValues(const std::vector<std::string> &paths);
+TurnLookupTable readTurnValues(const std::vector<std::string> &paths, UpdaterConfig::SpeedAndTurnPenaltyFormat format);
 } // namespace csv
 } // namespace updater
 } // namespace osrm

include/updater/updater_config.hpp

@@ -69,6 +69,12 @@ struct UpdaterConfig final : storage::IOConfig
     double log_edge_updates_factor = 0.0;
     std::time_t valid_now;
 
+
+    enum class SpeedAndTurnPenaltyFormat {
+        CSV,
+        PARQUET
+    } speed_and_turn_penalty_format = SpeedAndTurnPenaltyFormat::CSV;
+
     std::vector<std::string> segment_speed_lookup_paths;
     std::vector<std::string> turn_penalty_lookup_paths;
     std::string tz_file_path;

src/updater/csv_source.cpp

@@ -2,6 +2,7 @@
 
 #include "updater/csv_file_parser.hpp"
 #include "updater/parquet_file_parser.hpp"
+#include "updater/source.hpp"
 
 #include <boost/fusion/adapted/std_pair.hpp>
 #include <boost/fusion/include/adapt_adt.hpp>
@@ -32,14 +33,43 @@ namespace updater
 {
 namespace csv
 {
-SegmentLookupTable readSegmentValues(const std::vector<std::string> &paths)
+
+namespace {
+std::unique_ptr<FilesParser<Segment, SpeedSource>> makeSegmentParser(UpdaterConfig::SpeedAndTurnPenaltyFormat format) {
+    switch (format) {
+        case UpdaterConfig::SpeedAndTurnPenaltyFormat::CSV:
+        {
+            static const auto value_if_blank = std::numeric_limits<double>::quiet_NaN();
+            const qi::real_parser<double, qi::ureal_policies<double>> unsigned_double;
+            return std::make_unique<CSVFilesParser<Segment, SpeedSource>>(qi::ulong_long >> ',' >> qi::ulong_long,
+        unsigned_double >> -(',' >> (qi::double_ | qi::attr(value_if_blank))));
+        }
+        case UpdaterConfig::SpeedAndTurnPenaltyFormat::PARQUET:
+            return std::make_unique<ParquetFilesParser<Segment, SpeedSource>>();
+    }
+} 
+
+std::unique_ptr<FilesParser<Turn, PenaltySource>> makeTurnParser(UpdaterConfig::SpeedAndTurnPenaltyFormat format) {
+    switch (format) {
+        case UpdaterConfig::SpeedAndTurnPenaltyFormat::CSV:
+        {
+            return std::make_unique<CSVFilesParser<Turn, PenaltySource>>(qi::ulong_long >> ',' >> qi::ulong_long >> ',' >>
+                                                   qi::ulong_long,
+                                               qi::double_ >> -(',' >> qi::double_));
+        }
+        case UpdaterConfig::SpeedAndTurnPenaltyFormat::PARQUET:
+            return std::make_unique<ParquetFilesParser<Turn, PenaltySource>>();
+    }
+} 
+
+} // namespace
+
+SegmentLookupTable readSegmentValues(const std::vector<std::string> &paths, UpdaterConfig::SpeedAndTurnPenaltyFormat format)
 {
-    //static const auto value_if_blank = std::numeric_limits<double>::quiet_NaN();
+    auto parser = makeSegmentParser(format);
-    //const qi::real_parser<double, qi::ureal_policies<double>> unsigned_double;
-    ParquetFilesParser<Segment, SpeedSource> parser;
 
     // Check consistency of keys in the result lookup table
-    auto result = parser(paths);
+    auto result = (*parser)(paths);
     const auto found_inconsistency =
         std::find_if(std::begin(result.lookup), std::end(result.lookup), [](const auto &entry) {
             return entry.first.from == entry.first.to;
@@ -53,10 +83,10 @@ SegmentLookupTable readSegmentValues(const std::vector<std::string> &paths)
     return result;
 }
 
-TurnLookupTable readTurnValues(const std::vector<std::string> &paths)
+TurnLookupTable readTurnValues(const std::vector<std::string> &paths, UpdaterConfig::SpeedAndTurnPenaltyFormat format)
 {
-    ParquetFilesParser<Turn, PenaltySource> parser;
+    auto parser = makeTurnParser(format);
-    return parser(paths);
+    return (*parser)(paths);
 }
 } // namespace csv
 } // namespace updater

src/updater/updater.cpp

@@ -618,7 +618,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
     tbb::concurrent_vector<GeometryID> updated_segments;
     if (update_edge_weights)
     {
-        auto segment_speed_lookup = csv::readSegmentValues(config.segment_speed_lookup_paths);
+        auto segment_speed_lookup = csv::readSegmentValues(config.segment_speed_lookup_paths, config.speed_and_turn_penalty_format);
 
         TIMER_START(segment);
         updated_segments = updateSegmentData(config,
@@ -633,7 +633,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
         util::Log() << "Updating segment data took " << TIMER_MSEC(segment) << "ms.";
     }
 
-    auto turn_penalty_lookup = csv::readTurnValues(config.turn_penalty_lookup_paths);
+    auto turn_penalty_lookup = csv::readTurnValues(config.turn_penalty_lookup_paths, config.speed_and_turn_penalty_format);
     if (update_turn_penalties)
     {
         auto updated_turn_penalties = updateTurnPenalties(config,