Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

implement parsing thru libosmium

Browse Source
This commit is contained in:
Dennis Luxen 2014-08-26 17:50:34 +02:00
parent 004c237085
commit b6e469abd1
4 changed files with 194 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
Extractor/Extractor.cpp
View File

@ -27,23 +27,29 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "Extractor.h"
#include "ExtractorCallbacks.h"
#include "ExtractionContainers.h"
#include "PBFParser.h"
#include "ExtractionNode.h"
#include "ExtractionWay.h"
#include "ExtractorCallbacks.h"
#include "RestrictionParser.h"
#include "ScriptingEnvironment.h"
#include "XMLParser.h"
#include "../Util/GitDescription.h"
#include "../Util/IniFileUtil.h"
#include "../Util/OSRMException.h"
#include "../Util/simple_logger.hpp"
#include "../Util/TimingUtil.h"
#include "../Util/make_unique.hpp"
#include "../typedefs.h"
#include <boost/program_options.hpp>
#include <luabind/luabind.hpp>
#include <tbb/task_scheduler_init.h>
#include <osmium/io/any_input.hpp>
#include <cstdlib>
#include <chrono>
@ -56,7 +62,14 @@ Extractor::Extractor() : requested_num_threads(0), file_has_pbf_format(false)
{
}
Extractor::~Extractor() {}
int lua_error_callback(lua_State *L) // This is so I can use my own function as an
// exception handler, pcall_log()
{
luabind::object error_msg(luabind::from_stack(L, -1));
std::ostringstream error_stream;
error_stream << error_msg;
throw OSRMException("ERROR occured in profile script:\n" + error_stream.str());
}
bool Extractor::ParseArguments(int argc, char *argv[])
{
@ -176,17 +189,20 @@ void Extractor::GenerateOutputFilesNames()
{
output_file_name.append(".osrm");
restriction_file_name.append(".osrm.restrictions");
timestamp_file_name.append(".osrm.timestamp");
}
else
{
output_file_name.replace(pos, 5, ".osrm");
restriction_file_name.replace(pos, 5, ".osrm.restrictions");
timestamp_file_name.replace(pos, 5, ".osrm.timestamp");
}
}
else
{
output_file_name.replace(pos, 8, ".osrm");
restriction_file_name.replace(pos, 8, ".osrm.restrictions");
timestamp_file_name.replace(pos, 8, ".osrm.timestamp");
}
}
@ -244,36 +260,88 @@ int Extractor::Run(int argc, char *argv[])
ExtractionContainers extraction_containers;
string_map[""] = 0;
auto extractor_callbacks = new ExtractorCallbacks(extraction_containers, string_map);
BaseParser *parser;
if (file_has_pbf_format)
{
parser = new PBFParser(input_path.string().c_str(),
extractor_callbacks,
scripting_environment,
requested_num_threads);
}
else
{
parser = new XMLParser(input_path.string().c_str(),
extractor_callbacks,
scripting_environment);
}
auto extractor_callbacks = osrm::make_unique<ExtractorCallbacks>(extraction_containers, string_map);
if (!parser->ReadHeader())
{
throw OSRMException("Parser not initialized!");
}
osmium::io::File infile(input_path.string());
osmium::io::Reader reader(infile);
osmium::io::Header header = reader.header();
unsigned number_of_nodes = 0;
unsigned number_of_ways = 0;
unsigned number_of_relations = 0;
unsigned number_of_others = 0;
SimpleLogger().Write() << "Parsing in progress..";
TIMER_START(parsing);
parser->Parse();
delete parser;
delete extractor_callbacks;
std::string generator = header.get("generator");
if (generator.empty())
{
generator = "unknown tool";
}
SimpleLogger().Write() << "input file generated by " << generator;
// TODO: write timestamp if non-empty
std::string timestamp = header.get("osmosis_replication_timestamp");
if (timestamp.empty())
{
timestamp = "n/a";
}
SimpleLogger().Write() << "timestamp: " << timestamp;
boost::filesystem::ofstream timestamp_out(timestamp_file_name);
timestamp_out.write(timestamp.c_str(), timestamp.length());
timestamp_out.close();
lua_State *lua_state = scripting_environment.getLuaState();
luabind::set_pcall_callback(&lua_error_callback);
RestrictionParser restriction_parser(scripting_environment);
ExtractionNode result_node;
ExtractionWay result_way;
while (osmium::memory::Buffer buffer = reader.read()) {
for (osmium::OSMEntity &entity : buffer)
{
switch (entity.type())
{
case osmium::item_type::node:
++number_of_nodes;
result_node.Clear();
luabind::call_function<void>(
lua_state,
"node_function",
boost::cref(static_cast<osmium::Node &>(entity)),
boost::ref(result_node));
extractor_callbacks->ProcessNode(static_cast<osmium::Node &>(entity),
result_node);
break;
case osmium::item_type::way:
++number_of_ways;
result_way.Clear();
luabind::call_function<void>(
lua_state,
"way_function",
boost::cref(static_cast<osmium::Way &>(entity)),
boost::ref(result_way));
extractor_callbacks->ProcessWay(static_cast<osmium::Way &>(entity),
result_way);
break;
case osmium::item_type::relation:
++number_of_relations;
extractor_callbacks->ProcessRestriction(
restriction_parser.TryParse(static_cast<osmium::Relation &>(entity)));
break;
default:
++number_of_others;
break;
}
}
}
TIMER_STOP(parsing);
SimpleLogger().Write() << "Parsing finished after " << TIMER_SEC(parsing) << " seconds";
extractor_callbacks.reset();
if (extraction_containers.all_edges_list.empty())
{

3
Extractor/Extractor.h
View File

@ -18,6 +18,7 @@ class Extractor
std::string output_file_name;
std::string restriction_file_name;
std::string timestamp_file_name;
bool file_has_pbf_format;
/** \brief Parses "extractor's" command line arguments */
@ -29,7 +30,7 @@ class Extractor
public:
explicit Extractor();
Extractor(const Extractor &) = delete;
virtual ~Extractor();
virtual ~Extractor() = default;
int Run(int argc, char *argv[]);
};

145
Extractor/ExtractorCallbacks.cpp
View File

@ -27,10 +27,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "ExtractorCallbacks.h"
#include "ExtractionContainers.h"
#include "ExtractionNode.h"
#include "ExtractionWay.h"
#include "../DataStructures/Restriction.h"
#include "../DataStructures/ImportNode.h"
#include "../Util/container.hpp"
#include "../Util/simple_logger.hpp"
#include <osrm/Coordinate.h>
@ -46,22 +48,28 @@ ExtractorCallbacks::ExtractorCallbacks(ExtractionContainers &extraction_containe
}
/** warning: caller needs to take care of synchronization! */
void ExtractorCallbacks::ProcessNode(const ExternalMemoryNode &n)
void ExtractorCallbacks::ProcessNode(const osmium::Node &osm_input_node, const ExtractionNode &result_node)
{
if (n.lat <= 85 * COORDINATE_PRECISION && n.lat >= -85 * COORDINATE_PRECISION)
//TODO: sort out datatype issues
ExternalMemoryNode node;
node.bollard = result_node.barrier;
node.trafficLight = result_node.traffic_lights;
node.lat = osm_input_node.location().lat() * COORDINATE_PRECISION;
node.lon = osm_input_node.location().lon() * COORDINATE_PRECISION;
node.node_id = osm_input_node.id();
external_memory.all_nodes_list.push_back(node);
}
void ExtractorCallbacks::ProcessRestriction(const boost::optional<InputRestrictionContainer> &restriction)
{
if (!restriction.is_initialized())
{
external_memory.all_nodes_list.push_back(n);
return;
}
external_memory.restrictions_list.push_back(restriction.get());
}
bool ExtractorCallbacks::ProcessRestriction(const InputRestrictionContainer &restriction)
{
external_memory.restrictions_list.push_back(restriction);
return true;
}
/** warning: caller needs to take care of synchronization! */
void ExtractorCallbacks::ProcessWay(ExtractionWay &parsed_way)
void ExtractorCallbacks::ProcessWay(const osmium::Way &current_way, ExtractionWay &parsed_way)
{
if (((0 >= parsed_way.forward_speed) ||
(TRAVEL_MODE_INACCESSIBLE == parsed_way.forward_travel_mode)) &&
@ -72,15 +80,15 @@ void ExtractorCallbacks::ProcessWay(ExtractionWay &parsed_way)
return;
}
if (parsed_way.path.size() <= 1)
if (current_way.nodes().size() <= 1)
{ // safe-guard against broken data
return;
}
if (std::numeric_limits<unsigned>::max() == parsed_way.id)
if (std::numeric_limits<unsigned>::max() == current_way.id())
{
SimpleLogger().Write(logDEBUG) << "found bogus way with id: " << parsed_way.id
<< " of size " << parsed_way.path.size();
SimpleLogger().Write(logDEBUG) << "found bogus way with id: " << current_way.id()
<< " of size " << current_way.nodes().size();
return;
}
@ -88,34 +96,27 @@ void ExtractorCallbacks::ProcessWay(ExtractionWay &parsed_way)
{
// TODO: iterate all way segments and set duration corresponding to the length of each
// segment
parsed_way.forward_speed = parsed_way.duration / (parsed_way.path.size() - 1);
parsed_way.backward_speed = parsed_way.duration / (parsed_way.path.size() - 1);
parsed_way.forward_speed = parsed_way.duration / (current_way.nodes().size() - 1);
parsed_way.backward_speed = parsed_way.duration / (current_way.nodes().size() - 1);
}
if (std::numeric_limits<double>::epsilon() >= std::abs(-1. - parsed_way.forward_speed))
{
SimpleLogger().Write(logDEBUG) << "found way with bogus speed, id: " << parsed_way.id;
SimpleLogger().Write(logDEBUG) << "found way with bogus speed, id: " << current_way.id();
return;
}
// Get the unique identifier for the street name
const auto &string_map_iterator = string_map.find(parsed_way.name);
unsigned name_id = external_memory.name_list.size();
if (string_map.end() == string_map_iterator)
{
parsed_way.nameID = external_memory.name_list.size();
external_memory.name_list.push_back(parsed_way.name);
string_map.insert(std::make_pair(parsed_way.name, parsed_way.nameID));
string_map.insert(std::make_pair(parsed_way.name, name_id));
}
else
{
parsed_way.nameID = string_map_iterator->second;
}
if (TRAVEL_MODE_INACCESSIBLE == parsed_way.forward_travel_mode)
{
std::reverse(parsed_way.path.begin(), parsed_way.path.end());
parsed_way.forward_travel_mode = parsed_way.backward_travel_mode;
parsed_way.backward_travel_mode = TRAVEL_MODE_INACCESSIBLE;
name_id = string_map_iterator->second;
}
const bool split_edge =
@ -124,59 +125,89 @@ void ExtractorCallbacks::ProcessWay(ExtractionWay &parsed_way)
((parsed_way.forward_speed != parsed_way.backward_speed) ||
(parsed_way.forward_travel_mode != parsed_way.backward_travel_mode));
BOOST_ASSERT(parsed_way.forward_travel_mode>0);
for (unsigned n = 0; n < (parsed_way.path.size() - 1); ++n)
{
external_memory.all_edges_list.push_back(InternalExtractorEdge(
parsed_way.path[n],
parsed_way.path[n + 1],
auto pair_wise_segment_split = [&](const osmium::NodeRef &first_node,
const osmium::NodeRef &last_node) {
// SimpleLogger().Write() << "adding edge (" << first_node.ref() << "," <<
// last_node.ref() << "), speed: " << parsed_way.speed;
external_memory.all_edges_list.push_back(
InternalExtractorEdge(first_node.ref(),
last_node.ref(),
((split_edge || TRAVEL_MODE_INACCESSIBLE == parsed_way.backward_travel_mode) ? ExtractionWay::oneway
: ExtractionWay::bidirectional),
parsed_way.forward_speed,
parsed_way.nameID,
name_id,
parsed_way.roundabout,
parsed_way.ignoreInGrid,
parsed_way.ignore_in_grid,
(0 < parsed_way.duration),
parsed_way.isAccessRestricted,
parsed_way.is_access_restricted,
parsed_way.forward_travel_mode,
split_edge));
external_memory.used_node_id_list.push_back(parsed_way.path[n]);
external_memory.used_node_id_list.push_back(first_node.ref());
};
const bool is_opposite_way = TRAVEL_MODE_INACCESSIBLE == parsed_way.forward_travel_mode;
if (is_opposite_way)
{
parsed_way.forward_travel_mode = parsed_way.backward_travel_mode;
parsed_way.backward_travel_mode = TRAVEL_MODE_INACCESSIBLE;
osrm::for_each_pair(current_way.nodes().crbegin(), current_way.nodes().crend(), pair_wise_segment_split);
external_memory.used_node_id_list.push_back(current_way.nodes().front().ref());
}
else
{
osrm::for_each_pair(current_way.nodes().cbegin(), current_way.nodes().cend(), pair_wise_segment_split);
external_memory.used_node_id_list.push_back(current_way.nodes().back().ref());
}
external_memory.used_node_id_list.push_back(parsed_way.path.back());
// The following information is needed to identify start and end segments of restrictions
// The following information is needed to identify start and end segments of restrictions
external_memory.way_start_end_id_list.push_back(
WayIDStartAndEndEdge(parsed_way.id,
parsed_way.path[0],
parsed_way.path[1],
parsed_way.path[parsed_way.path.size() - 2],
parsed_way.path.back()));
{(EdgeID)current_way.id(),
(NodeID)current_way.nodes()[0].ref(),
(NodeID)current_way.nodes()[1].ref(),
(NodeID)current_way.nodes()[current_way.nodes().size() - 2].ref(),
(NodeID)current_way.nodes().back().ref()});
if (split_edge)
{ // Only true if the way should be split
BOOST_ASSERT(parsed_way.backward_travel_mode>0);
std::reverse(parsed_way.path.begin(), parsed_way.path.end());
for (std::vector<NodeID>::size_type n = 0; n < parsed_way.path.size() - 1; ++n)
auto pair_wise_segment_split_2 = [&](const osmium::NodeRef &first_node,
const osmium::NodeRef &last_node)
{
// SimpleLogger().Write() << "adding edge (" << last_node.ref() << "," <<
// first_node.ref() << "), speed: " << parsed_way.backward_speed;
external_memory.all_edges_list.push_back(
InternalExtractorEdge(parsed_way.path[n],
parsed_way.path[n + 1],
InternalExtractorEdge(last_node.ref(),
first_node.ref(),
ExtractionWay::oneway,
parsed_way.backward_speed,
parsed_way.nameID,
name_id,
parsed_way.roundabout,
parsed_way.ignoreInGrid,
parsed_way.ignore_in_grid,
(0 < parsed_way.duration),
parsed_way.isAccessRestricted,
parsed_way.is_access_restricted,
parsed_way.backward_travel_mode,
split_edge));
};
if (is_opposite_way)
{
// SimpleLogger().Write() << "opposite2";
osrm::for_each_pair(current_way.nodes().crbegin(), current_way.nodes().crend(), pair_wise_segment_split_2);
external_memory.used_node_id_list.push_back(current_way.nodes().front().ref());
}
else
{
osrm::for_each_pair(current_way.nodes().cbegin(), current_way.nodes().cend(), pair_wise_segment_split_2);
external_memory.used_node_id_list.push_back(current_way.nodes().back().ref());
}
external_memory.way_start_end_id_list.push_back(
WayIDStartAndEndEdge(parsed_way.id,
parsed_way.path[0],
parsed_way.path[1],
parsed_way.path[parsed_way.path.size() - 2],
parsed_way.path.back()));
{(EdgeID)current_way.id(),
(NodeID)current_way.nodes()[1].ref(),
(NodeID)current_way.nodes()[0].ref(),
(NodeID)current_way.nodes().back().ref(),
(NodeID)current_way.nodes()[current_way.nodes().size() - 2].ref()});
}
}

15
Extractor/ExtractorCallbacks.h
View File

@ -28,15 +28,20 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef EXTRACTOR_CALLBACKS_H
#define EXTRACTOR_CALLBACKS_H
#include "ExtractionWay.h"
#include "../typedefs.h"
#include <unordered_map>
#include <boost/optional.hpp>
#include <osmium/osm.hpp>
#include <string>
#include <unordered_map>
struct ExternalMemoryNode;
class ExtractionContainers;
struct ExtractionWay;
struct InputRestrictionContainer;
struct ExtractionNode;
class ExtractorCallbacks
{
@ -51,13 +56,13 @@ class ExtractorCallbacks
std::unordered_map<std::string, NodeID> &string_map);
// warning: caller needs to take care of synchronization!
void ProcessNode(const ExternalMemoryNode &node);
void ProcessNode(const osmium::Node &current_node, const ExtractionNode &result_node);
// warning: caller needs to take care of synchronization!
bool ProcessRestriction(const InputRestrictionContainer &restriction);
void ProcessRestriction(const boost::optional<InputRestrictionContainer> &restriction);
// warning: caller needs to take care of synchronization!
void ProcessWay(ExtractionWay &way);
void ProcessWay(const osmium::Way &current_way, ExtractionWay &result_way);
};
#endif /* EXTRACTOR_CALLBACKS_H */