implement parsing thru libosmium

Extractor/Extractor.cpp

@@ -27,23 +27,29 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "Extractor.h"
 
-#include "ExtractorCallbacks.h"
 #include "ExtractionContainers.h"
-#include "PBFParser.h"
+#include "ExtractionNode.h"
+#include "ExtractionWay.h"
+#include "ExtractorCallbacks.h"
+#include "RestrictionParser.h"
 #include "ScriptingEnvironment.h"
-#include "XMLParser.h"
 
 #include "../Util/GitDescription.h"
 #include "../Util/IniFileUtil.h"
 #include "../Util/OSRMException.h"
 #include "../Util/simple_logger.hpp"
 #include "../Util/TimingUtil.h"
+#include "../Util/make_unique.hpp"
 #include "../typedefs.h"
 
 #include <boost/program_options.hpp>
 
+#include <luabind/luabind.hpp>
+
 #include <tbb/task_scheduler_init.h>
 
+#include <osmium/io/any_input.hpp>
+
 #include <cstdlib>
 
 #include <chrono>
@@ -56,7 +62,14 @@ Extractor::Extractor() : requested_num_threads(0), file_has_pbf_format(false)
 {
 }
 
-Extractor::~Extractor() {}
+int lua_error_callback(lua_State *L) // This is so I can use my own function as an
+// exception handler, pcall_log()
+{
+    luabind::object error_msg(luabind::from_stack(L, -1));
+    std::ostringstream error_stream;
+    error_stream << error_msg;
+    throw OSRMException("ERROR occured in profile script:\n" + error_stream.str());
+}
 
 bool Extractor::ParseArguments(int argc, char *argv[])
 {
@@ -176,17 +189,20 @@ void Extractor::GenerateOutputFilesNames()
         {
             output_file_name.append(".osrm");
             restriction_file_name.append(".osrm.restrictions");
+            timestamp_file_name.append(".osrm.timestamp");
         }
         else
         {
             output_file_name.replace(pos, 5, ".osrm");
             restriction_file_name.replace(pos, 5, ".osrm.restrictions");
+            timestamp_file_name.replace(pos, 5, ".osrm.timestamp");
         }
     }
     else
     {
         output_file_name.replace(pos, 8, ".osrm");
         restriction_file_name.replace(pos, 8, ".osrm.restrictions");
+        timestamp_file_name.replace(pos, 8, ".osrm.timestamp");
     }
 }
 
@@ -244,36 +260,88 @@ int Extractor::Run(int argc, char *argv[])
         ExtractionContainers extraction_containers;
 
         string_map[""] = 0;
-        auto extractor_callbacks = new ExtractorCallbacks(extraction_containers, string_map);
+        auto extractor_callbacks = osrm::make_unique<ExtractorCallbacks>(extraction_containers, string_map);
-        BaseParser *parser;
-        if (file_has_pbf_format)
-        {
-            parser = new PBFParser(input_path.string().c_str(),
-                                   extractor_callbacks,
-                                   scripting_environment,
-                                   requested_num_threads);
-        }
-        else
-        {
-            parser = new XMLParser(input_path.string().c_str(),
-                                   extractor_callbacks,
-                                   scripting_environment);
-        }
 
-        if (!parser->ReadHeader())
+        osmium::io::File infile(input_path.string());
-        {
+        osmium::io::Reader reader(infile);
-            throw OSRMException("Parser not initialized!");
+        osmium::io::Header header = reader.header();
-        }
+
+        unsigned number_of_nodes = 0;
+        unsigned number_of_ways = 0;
+        unsigned number_of_relations = 0;
+        unsigned number_of_others = 0;
 
         SimpleLogger().Write() << "Parsing in progress..";
         TIMER_START(parsing);
 
-        parser->Parse();
+        std::string generator = header.get("generator");
-        delete parser;
+        if (generator.empty())
-        delete extractor_callbacks;
+        {
+            generator = "unknown tool";
+        }
+        SimpleLogger().Write() << "input file generated by " << generator;
 
+        // TODO: write timestamp if non-empty
+        std::string timestamp = header.get("osmosis_replication_timestamp");
+        if (timestamp.empty())
+        {
+            timestamp = "n/a";
+        }
+        SimpleLogger().Write() << "timestamp: " << timestamp;
+
+        boost::filesystem::ofstream timestamp_out(timestamp_file_name);
+        timestamp_out.write(timestamp.c_str(), timestamp.length());
+        timestamp_out.close();
+
+        lua_State *lua_state = scripting_environment.getLuaState();
+        luabind::set_pcall_callback(&lua_error_callback);
+
+        RestrictionParser restriction_parser(scripting_environment);
+
+        ExtractionNode result_node;
+        ExtractionWay result_way;
+
+        while (osmium::memory::Buffer buffer = reader.read()) {
+            for (osmium::OSMEntity &entity : buffer)
+            {
+                switch (entity.type())
+                {
+                case osmium::item_type::node:
+                    ++number_of_nodes;
+                    result_node.Clear();
+                    luabind::call_function<void>(
+                        lua_state,
+                        "node_function",
+                        boost::cref(static_cast<osmium::Node &>(entity)),
+                        boost::ref(result_node));
+                    extractor_callbacks->ProcessNode(static_cast<osmium::Node &>(entity),
+                                                     result_node);
+                    break;
+                case osmium::item_type::way:
+                    ++number_of_ways;
+                    result_way.Clear();
+                    luabind::call_function<void>(
+                        lua_state,
+                        "way_function",
+                        boost::cref(static_cast<osmium::Way &>(entity)),
+                        boost::ref(result_way));
+                    extractor_callbacks->ProcessWay(static_cast<osmium::Way &>(entity),
+                                                    result_way);
+                    break;
+                case osmium::item_type::relation:
+                    ++number_of_relations;
+                    extractor_callbacks->ProcessRestriction(
+                        restriction_parser.TryParse(static_cast<osmium::Relation &>(entity)));
+                    break;
+                default:
+                    ++number_of_others;
+                    break;
+                }
+            }
+        }
         TIMER_STOP(parsing);
         SimpleLogger().Write() << "Parsing finished after " << TIMER_SEC(parsing) << " seconds";
+        extractor_callbacks.reset();
 
         if (extraction_containers.all_edges_list.empty())
         {

Extractor/Extractor.h

@@ -18,6 +18,7 @@ class Extractor
 
     std::string output_file_name;
     std::string restriction_file_name;
+    std::string timestamp_file_name;
     bool file_has_pbf_format;
 
     /** \brief Parses "extractor's" command line arguments */
@@ -29,7 +30,7 @@ class Extractor
   public:
     explicit Extractor();
     Extractor(const Extractor &) = delete;
-    virtual ~Extractor();
+    virtual ~Extractor() = default;
 
     int Run(int argc, char *argv[]);
 };

Extractor/ExtractorCallbacks.cpp

@@ -27,10 +27,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "ExtractorCallbacks.h"
 #include "ExtractionContainers.h"
+#include "ExtractionNode.h"
 #include "ExtractionWay.h"
 
 #include "../DataStructures/Restriction.h"
 #include "../DataStructures/ImportNode.h"
+#include "../Util/container.hpp"
 #include "../Util/simple_logger.hpp"
 
 #include <osrm/Coordinate.h>
@@ -46,22 +48,28 @@ ExtractorCallbacks::ExtractorCallbacks(ExtractionContainers &extraction_containe
 }
 
 /** warning: caller needs to take care of synchronization! */
-void ExtractorCallbacks::ProcessNode(const ExternalMemoryNode &n)
+void ExtractorCallbacks::ProcessNode(const osmium::Node &osm_input_node, const ExtractionNode &result_node)
 {
-    if (n.lat <= 85 * COORDINATE_PRECISION && n.lat >= -85 * COORDINATE_PRECISION)
+    //TODO: sort out datatype issues
-    {
+    ExternalMemoryNode node;
-        external_memory.all_nodes_list.push_back(n);
+    node.bollard = result_node.barrier;
-    }
+    node.trafficLight = result_node.traffic_lights;
+    node.lat = osm_input_node.location().lat() * COORDINATE_PRECISION;
+    node.lon = osm_input_node.location().lon() * COORDINATE_PRECISION;
+    node.node_id = osm_input_node.id();
+    external_memory.all_nodes_list.push_back(node);
 }
 
-bool ExtractorCallbacks::ProcessRestriction(const InputRestrictionContainer &restriction)
+void ExtractorCallbacks::ProcessRestriction(const boost::optional<InputRestrictionContainer> &restriction)
 {
-    external_memory.restrictions_list.push_back(restriction);
+    if (!restriction.is_initialized())
-    return true;
+    {
+      return;
+    }
+    external_memory.restrictions_list.push_back(restriction.get());
 }
-
 /** warning: caller needs to take care of synchronization! */
-void ExtractorCallbacks::ProcessWay(ExtractionWay &parsed_way)
+void ExtractorCallbacks::ProcessWay(const osmium::Way &current_way, ExtractionWay &parsed_way)
 {
     if (((0 >= parsed_way.forward_speed) ||
             (TRAVEL_MODE_INACCESSIBLE == parsed_way.forward_travel_mode)) &&
@@ -72,15 +80,15 @@ void ExtractorCallbacks::ProcessWay(ExtractionWay &parsed_way)
         return;
     }
 
-    if (parsed_way.path.size() <= 1)
+    if (current_way.nodes().size() <= 1)
     { // safe-guard against broken data
         return;
     }
 
-    if (std::numeric_limits<unsigned>::max() == parsed_way.id)
+    if (std::numeric_limits<unsigned>::max() == current_way.id())
     {
-        SimpleLogger().Write(logDEBUG) << "found bogus way with id: " << parsed_way.id
+        SimpleLogger().Write(logDEBUG) << "found bogus way with id: " << current_way.id()
-                                       << " of size " << parsed_way.path.size();
+                                       << " of size " << current_way.nodes().size();
         return;
     }
 
@@ -88,34 +96,27 @@ void ExtractorCallbacks::ProcessWay(ExtractionWay &parsed_way)
     {
         // TODO: iterate all way segments and set duration corresponding to the length of each
         // segment
-        parsed_way.forward_speed = parsed_way.duration / (parsed_way.path.size() - 1);
+        parsed_way.forward_speed = parsed_way.duration / (current_way.nodes().size() - 1);
-        parsed_way.backward_speed = parsed_way.duration / (parsed_way.path.size() - 1);
+        parsed_way.backward_speed = parsed_way.duration / (current_way.nodes().size() - 1);
     }
 
     if (std::numeric_limits<double>::epsilon() >= std::abs(-1. - parsed_way.forward_speed))
     {
-        SimpleLogger().Write(logDEBUG) << "found way with bogus speed, id: " << parsed_way.id;
+        SimpleLogger().Write(logDEBUG) << "found way with bogus speed, id: " << current_way.id();
         return;
     }
 
     // Get the unique identifier for the street name
     const auto &string_map_iterator = string_map.find(parsed_way.name);
+    unsigned name_id = external_memory.name_list.size();
     if (string_map.end() == string_map_iterator)
     {
-        parsed_way.nameID = external_memory.name_list.size();
         external_memory.name_list.push_back(parsed_way.name);
-        string_map.insert(std::make_pair(parsed_way.name, parsed_way.nameID));
+        string_map.insert(std::make_pair(parsed_way.name, name_id));
     }
     else
     {
-        parsed_way.nameID = string_map_iterator->second;
+        name_id = string_map_iterator->second;
-    }
-
-    if (TRAVEL_MODE_INACCESSIBLE == parsed_way.forward_travel_mode)
-    {
-        std::reverse(parsed_way.path.begin(), parsed_way.path.end());
-        parsed_way.forward_travel_mode = parsed_way.backward_travel_mode;
-        parsed_way.backward_travel_mode = TRAVEL_MODE_INACCESSIBLE;
     }
 
     const bool split_edge =
@@ -124,59 +125,89 @@ void ExtractorCallbacks::ProcessWay(ExtractionWay &parsed_way)
       ((parsed_way.forward_speed != parsed_way.backward_speed) ||
       (parsed_way.forward_travel_mode != parsed_way.backward_travel_mode));
 
-    BOOST_ASSERT(parsed_way.forward_travel_mode>0);
+    auto pair_wise_segment_split = [&](const osmium::NodeRef &first_node,
-    for (unsigned n = 0; n < (parsed_way.path.size() - 1); ++n)
+                                       const osmium::NodeRef &last_node) {
-    {
+        // SimpleLogger().Write() << "adding edge (" << first_node.ref() << "," <<
-        external_memory.all_edges_list.push_back(InternalExtractorEdge(
+        // last_node.ref() << "), speed: " << parsed_way.speed;
-            parsed_way.path[n],
+        external_memory.all_edges_list.push_back(
-            parsed_way.path[n + 1],
+            InternalExtractorEdge(first_node.ref(),
+                                  last_node.ref(),
             ((split_edge || TRAVEL_MODE_INACCESSIBLE == parsed_way.backward_travel_mode) ? ExtractionWay::oneway
                                                                                  : ExtractionWay::bidirectional),
             parsed_way.forward_speed,
-            parsed_way.nameID,
+            name_id,
             parsed_way.roundabout,
-            parsed_way.ignoreInGrid,
+            parsed_way.ignore_in_grid,
             (0 < parsed_way.duration),
-            parsed_way.isAccessRestricted,
+            parsed_way.is_access_restricted,
             parsed_way.forward_travel_mode,
             split_edge));
-        external_memory.used_node_id_list.push_back(parsed_way.path[n]);
+        external_memory.used_node_id_list.push_back(first_node.ref());
+    };
+
+    const bool is_opposite_way = TRAVEL_MODE_INACCESSIBLE == parsed_way.forward_travel_mode;
+    if (is_opposite_way)
+    {
+        parsed_way.forward_travel_mode = parsed_way.backward_travel_mode;
+        parsed_way.backward_travel_mode = TRAVEL_MODE_INACCESSIBLE;
+        osrm::for_each_pair(current_way.nodes().crbegin(), current_way.nodes().crend(), pair_wise_segment_split);
+        external_memory.used_node_id_list.push_back(current_way.nodes().front().ref());
+    }
+    else
+    {
+        osrm::for_each_pair(current_way.nodes().cbegin(), current_way.nodes().cend(), pair_wise_segment_split);
+        external_memory.used_node_id_list.push_back(current_way.nodes().back().ref());
     }
-    external_memory.used_node_id_list.push_back(parsed_way.path.back());
 
+    // The following information is needed to identify start and end segments of restrictions
      // The following information is needed to identify start and end segments of restrictions
     external_memory.way_start_end_id_list.push_back(
-        WayIDStartAndEndEdge(parsed_way.id,
+        {(EdgeID)current_way.id(),
-                             parsed_way.path[0],
+         (NodeID)current_way.nodes()[0].ref(),
-                             parsed_way.path[1],
+         (NodeID)current_way.nodes()[1].ref(),
-                             parsed_way.path[parsed_way.path.size() - 2],
+         (NodeID)current_way.nodes()[current_way.nodes().size() - 2].ref(),
-                             parsed_way.path.back()));
+         (NodeID)current_way.nodes().back().ref()});
 
     if (split_edge)
     { // Only true if the way should be split
         BOOST_ASSERT(parsed_way.backward_travel_mode>0);
-        std::reverse(parsed_way.path.begin(), parsed_way.path.end());
+        auto pair_wise_segment_split_2 = [&](const osmium::NodeRef &first_node,
-
+                                             const osmium::NodeRef &last_node)
-        for (std::vector<NodeID>::size_type n = 0; n < parsed_way.path.size() - 1; ++n)
         {
+            // SimpleLogger().Write() << "adding edge (" << last_node.ref() << "," <<
+            // first_node.ref() << "), speed: " << parsed_way.backward_speed;
             external_memory.all_edges_list.push_back(
-                InternalExtractorEdge(parsed_way.path[n],
+                InternalExtractorEdge(last_node.ref(),
-                                      parsed_way.path[n + 1],
+                                      first_node.ref(),
                                       ExtractionWay::oneway,
                                       parsed_way.backward_speed,
-                                      parsed_way.nameID,
+                                      name_id,
                                       parsed_way.roundabout,
-                                      parsed_way.ignoreInGrid,
+                                      parsed_way.ignore_in_grid,
                                       (0 < parsed_way.duration),
-                                      parsed_way.isAccessRestricted,
+                                      parsed_way.is_access_restricted,
                                       parsed_way.backward_travel_mode,
                                       split_edge));
+        };
+
+        if (is_opposite_way)
+        {
+            // SimpleLogger().Write() << "opposite2";
+            osrm::for_each_pair(current_way.nodes().crbegin(), current_way.nodes().crend(), pair_wise_segment_split_2);
+            external_memory.used_node_id_list.push_back(current_way.nodes().front().ref());
         }
+        else
+        {
+            osrm::for_each_pair(current_way.nodes().cbegin(), current_way.nodes().cend(), pair_wise_segment_split_2);
+            external_memory.used_node_id_list.push_back(current_way.nodes().back().ref());
+        }
+
+
         external_memory.way_start_end_id_list.push_back(
-            WayIDStartAndEndEdge(parsed_way.id,
+            {(EdgeID)current_way.id(),
-                                 parsed_way.path[0],
+             (NodeID)current_way.nodes()[1].ref(),
-                                 parsed_way.path[1],
+             (NodeID)current_way.nodes()[0].ref(),
-                                 parsed_way.path[parsed_way.path.size() - 2],
+             (NodeID)current_way.nodes().back().ref(),
-                                 parsed_way.path.back()));
+             (NodeID)current_way.nodes()[current_way.nodes().size() - 2].ref()});
     }
 }

Extractor/ExtractorCallbacks.h

@@ -28,15 +28,20 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifndef EXTRACTOR_CALLBACKS_H
 #define EXTRACTOR_CALLBACKS_H
 
+#include "ExtractionWay.h"
 #include "../typedefs.h"
 
-#include <unordered_map>
+#include <boost/optional.hpp>
+
+#include <osmium/osm.hpp>
+
 #include <string>
+#include <unordered_map>
 
 struct ExternalMemoryNode;
 class ExtractionContainers;
-struct ExtractionWay;
 struct InputRestrictionContainer;
+struct ExtractionNode;
 
 class ExtractorCallbacks
 {
@@ -51,13 +56,13 @@ class ExtractorCallbacks
                                 std::unordered_map<std::string, NodeID> &string_map);
 
     // warning: caller needs to take care of synchronization!
-    void ProcessNode(const ExternalMemoryNode &node);
+    void ProcessNode(const osmium::Node &current_node, const ExtractionNode &result_node);
 
     // warning: caller needs to take care of synchronization!
-    bool ProcessRestriction(const InputRestrictionContainer &restriction);
+    void ProcessRestriction(const boost::optional<InputRestrictionContainer> &restriction);
 
     // warning: caller needs to take care of synchronization!
-    void ProcessWay(ExtractionWay &way);
+    void ProcessWay(const osmium::Way &current_way, ExtractionWay &result_way);
 };
 
 #endif /* EXTRACTOR_CALLBACKS_H */