Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

refactor Extractor/

Browse Source
This commit is contained in:
Dennis Luxen 2014-05-09 16:17:31 +02:00
parent 7e639d6bc1
commit 0fccd0f0d2
16 changed files with 1171 additions and 1193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
Extractor/BaseParser.cpp
View File

@ -38,84 +38,86 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <boost/algorithm/string/regex.hpp> #include <boost/algorithm/string/regex.hpp>
#include <boost/regex.hpp> #include <boost/regex.hpp>
BaseParser::BaseParser( BaseParser::BaseParser(ExtractorCallbacks *extractor_callbacks,
ExtractorCallbacks * extractor_callbacks, ScriptingEnvironment &scripting_environment)
ScriptingEnvironment & scripting_environment : extractor_callbacks(extractor_callbacks),
) : extractor_callbacks(extractor_callbacks),
lua_state(scripting_environment.getLuaStateForThreadID(0)), lua_state(scripting_environment.getLuaStateForThreadID(0)),
scripting_environment(scripting_environment), scripting_environment(scripting_environment), use_turn_restrictions(true)
use_turn_restrictions(true)
{ {
ReadUseRestrictionsSetting(); ReadUseRestrictionsSetting();
ReadRestrictionExceptions(); ReadRestrictionExceptions();
} }
void BaseParser::ReadUseRestrictionsSetting() { void BaseParser::ReadUseRestrictionsSetting()
if( 0 != luaL_dostring( lua_state, "return use_turn_restrictions\n") ) { {
if (0 != luaL_dostring(lua_state, "return use_turn_restrictions\n"))
{
throw OSRMException("ERROR occured in scripting block"); throw OSRMException("ERROR occured in scripting block");
} }
if( lua_isboolean( lua_state, -1) ) { if (lua_isboolean(lua_state, -1))
{
use_turn_restrictions = lua_toboolean(lua_state, -1); use_turn_restrictions = lua_toboolean(lua_state, -1);
} }
if( use_turn_restrictions ) { if (use_turn_restrictions)
{
SimpleLogger().Write() << "Using turn restrictions"; SimpleLogger().Write() << "Using turn restrictions";
} else { }
else
{
SimpleLogger().Write() << "Ignoring turn restrictions"; SimpleLogger().Write() << "Ignoring turn restrictions";
} }
} }
void BaseParser::ReadRestrictionExceptions() { void BaseParser::ReadRestrictionExceptions()
if(lua_function_exists(lua_state, "get_exceptions" )) { {
if (lua_function_exists(lua_state, "get_exceptions"))
{
// get list of turn restriction exceptions // get list of turn restriction exceptions
luabind::call_function<void>( luabind::call_function<void>(
lua_state, lua_state, "get_exceptions", boost::ref(restriction_exceptions));
"get_exceptions",
boost::ref(restriction_exceptions)
);
const unsigned exception_count = restriction_exceptions.size(); const unsigned exception_count = restriction_exceptions.size();
SimpleLogger().Write() << SimpleLogger().Write() << "Found " << exception_count
"Found " << exception_count << " exceptions to turn restrictions:"; << " exceptions to turn restrictions:";
for(const std::string & str : restriction_exceptions) { for (const std::string &str : restriction_exceptions)
{
SimpleLogger().Write() << " " << str; SimpleLogger().Write() << " " << str;
} }
} else { }
else
{
SimpleLogger().Write() << "Found no exceptions to turn restrictions"; SimpleLogger().Write() << "Found no exceptions to turn restrictions";
} }
} }
void BaseParser::report_errors(lua_State *L, const int status) const { void BaseParser::report_errors(lua_State *lua_state, const int status) const
if( 0!=status ) { {
std::cerr << "-- " << lua_tostring(L, -1) << std::endl; if (0 != status)
lua_pop(L, 1); // remove error message {
std::cerr << "-- " << lua_tostring(lua_state, -1) << std::endl;
lua_pop(lua_state, 1); // remove error message
} }
} }
void BaseParser::ParseNodeInLua(ImportNode& n, lua_State* local_lua_state) { void BaseParser::ParseNodeInLua(ImportNode &node, lua_State *local_lua_state)
luabind::call_function<void>( {
local_lua_state, luabind::call_function<void>(local_lua_state, "node_function", boost::ref(node));
"node_function",
boost::ref(n)
);
} }
void BaseParser::ParseWayInLua(ExtractionWay& w, lua_State* local_lua_state) { void BaseParser::ParseWayInLua(ExtractionWay &way, lua_State *local_lua_state)
luabind::call_function<void>( {
local_lua_state, luabind::call_function<void>(local_lua_state, "way_function", boost::ref(way));
"way_function",
boost::ref(w)
);
} }
bool BaseParser::ShouldIgnoreRestriction( bool BaseParser::ShouldIgnoreRestriction(const std::string &except_tag_string) const
const std::string & except_tag_string {
) const {
// should this restriction be ignored? yes if there's an overlap between: // should this restriction be ignored? yes if there's an overlap between:
// a) the list of modes in the except tag of the restriction // a) the list of modes in the except tag of the restriction
// (except_tag_string), eg: except=bus;bicycle // (except_tag_string), eg: except=bus;bicycle
// b) the lua profile defines a hierachy of modes, // b) the lua profile defines a hierachy of modes,
// eg: [access, vehicle, bicycle] // eg: [access, vehicle, bicycle]
if( except_tag_string.empty() ) { if (except_tag_string.empty())
{
return false; return false;
} }
@ -123,14 +125,12 @@ bool BaseParser::ShouldIgnoreRestriction(
// only a few exceptions are actually defined. // only a few exceptions are actually defined.
std::vector<std::string> exceptions; std::vector<std::string> exceptions;
boost::algorithm::split_regex(exceptions, except_tag_string, boost::regex("[;][ ]*")); boost::algorithm::split_regex(exceptions, except_tag_string, boost::regex("[;][ ]*"));
for (std::string& current_string : exceptions) { for (std::string &current_string : exceptions)
std::vector<std::string>::const_iterator string_iterator; {
string_iterator = std::find( const auto string_iterator =
restriction_exceptions.begin(), std::find(restriction_exceptions.begin(), restriction_exceptions.end(), current_string);
restriction_exceptions.end(), if (restriction_exceptions.end() != string_iterator)
current_string {
);
if( restriction_exceptions.end() != string_iterator ) {
return true; return true;
} }
} }

17
Extractor/BaseParser.h
View File

@ -37,28 +37,25 @@ class ScriptingEnvironment;
struct ExtractionWay; struct ExtractionWay;
struct ImportNode; struct ImportNode;
class BaseParser { class BaseParser
{
public: public:
BaseParser() = delete; BaseParser() = delete;
BaseParser(const BaseParser &) = delete; BaseParser(const BaseParser &) = delete;
BaseParser( BaseParser(ExtractorCallbacks *extractor_callbacks,
ExtractorCallbacks * extractor_callbacks, ScriptingEnvironment &scripting_environment);
ScriptingEnvironment & scripting_environment
);
virtual ~BaseParser() {} virtual ~BaseParser() {}
virtual bool ReadHeader() = 0; virtual bool ReadHeader() = 0;
virtual bool Parse() = 0; virtual bool Parse() = 0;
virtual void ParseNodeInLua(ImportNode & n, lua_State* thread_lua_state); virtual void ParseNodeInLua(ImportNode &node, lua_State *lua_state);
virtual void ParseWayInLua(ExtractionWay & n, lua_State* thread_lua_state); virtual void ParseWayInLua(ExtractionWay &way, lua_State *lua_state);
virtual void report_errors(lua_State *lua_state, const int status) const; virtual void report_errors(lua_State *lua_state, const int status) const;
protected: protected:
virtual void ReadUseRestrictionsSetting(); virtual void ReadUseRestrictionsSetting();
virtual void ReadRestrictionExceptions(); virtual void ReadRestrictionExceptions();
virtual bool ShouldIgnoreRestriction( virtual bool ShouldIgnoreRestriction(const std::string &except_tag_string) const;
const std::string & except_tag_string
) const;
ExtractorCallbacks *extractor_callbacks; ExtractorCallbacks *extractor_callbacks;
lua_State *lua_state; lua_State *lua_state;

346
Extractor/ExtractionContainers.cpp
View File

@ -38,13 +38,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <chrono> #include <chrono>
ExtractionContainers::ExtractionContainers() { ExtractionContainers::ExtractionContainers()
{
// Check if stxxl can be instantiated // Check if stxxl can be instantiated
stxxl::vector<unsigned> dummy_vector; stxxl::vector<unsigned> dummy_vector;
name_list.push_back(""); name_list.push_back("");
} }
ExtractionContainers::~ExtractionContainers() { ExtractionContainers::~ExtractionContainers()
{
used_node_id_list.clear(); used_node_id_list.clear();
all_nodes_list.clear(); all_nodes_list.clear();
all_edges_list.clear(); all_edges_list.clear();
@ -53,30 +55,25 @@ ExtractionContainers::~ExtractionContainers() {
way_start_end_id_list.clear(); way_start_end_id_list.clear();
} }
void ExtractionContainers::PrepareData( void ExtractionContainers::PrepareData(const std::string &output_file_name,
const std::string & output_file_name, const std::string &restrictions_file_name)
const std::string & restrictions_file_name {
) { try
try { {
unsigned number_of_used_nodes = 0; unsigned number_of_used_nodes = 0;
unsigned number_of_used_edges = 0; unsigned number_of_used_edges = 0;
std::chrono::time_point<std::chrono::steady_clock> time1 = std::chrono::steady_clock::now(); std::chrono::time_point<std::chrono::steady_clock> time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting used nodes ... " << std::flush; std::cout << "[extractor] Sorting used nodes ... " << std::flush;
stxxl::sort( stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), 4294967296);
used_node_id_list.begin(),
used_node_id_list.end(),
Cmp(),
4294967296
);
std::chrono::time_point<std::chrono::steady_clock> time2 = std::chrono::steady_clock::now(); std::chrono::time_point<std::chrono::steady_clock> time2 = std::chrono::steady_clock::now();
std::chrono::duration<double> elapsed_seconds = time2 - time1; std::chrono::duration<double> elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now(); time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush; std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush;
stxxl::vector<NodeID>::iterator NewEnd = std::unique ( used_node_id_list.begin(),used_node_id_list.end() ) ; auto new_end = std::unique(used_node_id_list.begin(), used_node_id_list.end());
used_node_id_list.resize ( NewEnd - used_node_id_list.begin() ); used_node_id_list.resize(new_end - used_node_id_list.begin());
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
@ -84,12 +81,7 @@ void ExtractionContainers::PrepareData(
time1 = std::chrono::steady_clock::now(); time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting all nodes ... " << std::flush; std::cout << "[extractor] Sorting all nodes ... " << std::flush;
stxxl::sort( stxxl::sort(all_nodes_list.begin(), all_nodes_list.end(), CmpNodeByID(), 4294967296);
all_nodes_list.begin(),
all_nodes_list.end(),
CmpNodeByID(),
4294967296
);
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
@ -98,57 +90,59 @@ void ExtractionContainers::PrepareData(
std::cout << "[extractor] Sorting used ways ... " << std::flush; std::cout << "[extractor] Sorting used ways ... " << std::flush;
stxxl::sort( stxxl::sort(
way_start_end_id_list.begin(), way_start_end_id_list.begin(), way_start_end_id_list.end(), CmpWayByID(), 4294967296);
way_start_end_id_list.end(),
CmpWayByID(),
4294967296
);
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] Sorting restrictions. by from... " << std::flush; std::cout << "[extractor] Sorting restrictions. by from... " << std::flush;
stxxl::sort( stxxl::sort(restrictions_list.begin(),
restrictions_list.begin(),
restrictions_list.end(), restrictions_list.end(),
CmpRestrictionContainerByFrom(), CmpRestrictionContainerByFrom(),
4294967296 4294967296);
);
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] Fixing restriction starts ... " << std::flush; std::cout << "[extractor] Fixing restriction starts ... " << std::flush;
STXXLRestrictionsVector::iterator restrictions_iterator = restrictions_list.begin(); auto restrictions_iterator = restrictions_list.begin();
STXXLWayIDStartEndVector::iterator way_start_and_end_iterator = way_start_end_id_list.begin(); auto way_start_and_end_iterator = way_start_end_id_list.begin();
while( while (way_start_and_end_iterator != way_start_end_id_list.end() &&
way_start_and_end_iterator != way_start_end_id_list.end() && restrictions_iterator != restrictions_list.end())
restrictions_iterator != restrictions_list.end() {
) { if (way_start_and_end_iterator->wayID < restrictions_iterator->fromWay)
{
if(way_start_and_end_iterator->wayID < restrictions_iterator->fromWay){
++way_start_and_end_iterator; ++way_start_and_end_iterator;
continue; continue;
} }
if(way_start_and_end_iterator->wayID > restrictions_iterator->fromWay) { if (way_start_and_end_iterator->wayID > restrictions_iterator->fromWay)
{
++restrictions_iterator; ++restrictions_iterator;
continue; continue;
} }
BOOST_ASSERT(way_start_and_end_iterator->wayID == restrictions_iterator->fromWay); BOOST_ASSERT(way_start_and_end_iterator->wayID == restrictions_iterator->fromWay);
NodeID via_node_id = restrictions_iterator->restriction.viaNode; const NodeID via_node_id = restrictions_iterator->restriction.viaNode;
if(way_start_and_end_iterator->firstStart == via_node_id) { if (way_start_and_end_iterator->firstStart == via_node_id)
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->firstTarget; {
} else if(way_start_and_end_iterator->firstTarget == via_node_id) { restrictions_iterator->restriction.fromNode =
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->firstStart; way_start_and_end_iterator->firstTarget;
} else if(way_start_and_end_iterator->lastStart == via_node_id) { }
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->lastTarget; else if (way_start_and_end_iterator->firstTarget == via_node_id)
} else if(way_start_and_end_iterator->lastTarget == via_node_id) { {
restrictions_iterator->restriction.fromNode =
way_start_and_end_iterator->firstStart;
}
else if (way_start_and_end_iterator->lastStart == via_node_id)
{
restrictions_iterator->restriction.fromNode =
way_start_and_end_iterator->lastTarget;
}
else if (way_start_and_end_iterator->lastTarget == via_node_id)
{
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->lastStart; restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->lastStart;
} }
++restrictions_iterator; ++restrictions_iterator;
@ -161,50 +155,54 @@ void ExtractionContainers::PrepareData(
time1 = std::chrono::steady_clock::now(); time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting restrictions. by to ... " << std::flush; std::cout << "[extractor] Sorting restrictions. by to ... " << std::flush;
stxxl::sort( stxxl::sort(restrictions_list.begin(),
restrictions_list.begin(),
restrictions_list.end(), restrictions_list.end(),
CmpRestrictionContainerByTo(), CmpRestrictionContainerByTo(),
4294967296 4294967296);
);
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now(); time1 = std::chrono::steady_clock::now();
unsigned usableRestrictionsCounter(0); unsigned number_of_useable_restrictions = 0;
std::cout << "[extractor] Fixing restriction ends ... " << std::flush; std::cout << "[extractor] Fixing restriction ends ... " << std::flush;
restrictions_iterator = restrictions_list.begin(); restrictions_iterator = restrictions_list.begin();
way_start_and_end_iterator = way_start_end_id_list.begin(); way_start_and_end_iterator = way_start_end_id_list.begin();
while( while (way_start_and_end_iterator != way_start_end_id_list.end() &&
way_start_and_end_iterator != way_start_end_id_list.end() && restrictions_iterator != restrictions_list.end())
restrictions_iterator != restrictions_list.end() {
) { if (way_start_and_end_iterator->wayID < restrictions_iterator->toWay)
if(way_start_and_end_iterator->wayID < restrictions_iterator->toWay){ {
++way_start_and_end_iterator; ++way_start_and_end_iterator;
continue; continue;
} }
if(way_start_and_end_iterator->wayID > restrictions_iterator->toWay) { if (way_start_and_end_iterator->wayID > restrictions_iterator->toWay)
{
++restrictions_iterator; ++restrictions_iterator;
continue; continue;
} }
NodeID via_node_id = restrictions_iterator->restriction.viaNode; NodeID via_node_id = restrictions_iterator->restriction.viaNode;
if(way_start_and_end_iterator->lastStart == via_node_id) { if (way_start_and_end_iterator->lastStart == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastTarget; restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastTarget;
} else if(way_start_and_end_iterator->lastTarget == via_node_id) { }
else if (way_start_and_end_iterator->lastTarget == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastStart; restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastStart;
} else if(way_start_and_end_iterator->firstStart == via_node_id) { }
else if (way_start_and_end_iterator->firstStart == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstTarget; restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstTarget;
} else if(way_start_and_end_iterator->firstTarget == via_node_id) { }
else if (way_start_and_end_iterator->firstTarget == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstStart; restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstStart;
} }
if( if (UINT_MAX != restrictions_iterator->restriction.fromNode &&
UINT_MAX != restrictions_iterator->restriction.fromNode && UINT_MAX != restrictions_iterator->restriction.toNode)
UINT_MAX != restrictions_iterator->restriction.toNode {
) { ++number_of_useable_restrictions;
++usableRestrictionsCounter;
} }
++restrictions_iterator; ++restrictions_iterator;
} }
@ -212,28 +210,21 @@ void ExtractionContainers::PrepareData(
elapsed_seconds = time2 - time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
SimpleLogger().Write() << "usable restrictions: " << usableRestrictionsCounter; SimpleLogger().Write() << "usable restrictions: " << number_of_useable_restrictions;
// serialize restrictions // serialize restrictions
std::ofstream restrictions_out_stream; std::ofstream restrictions_out_stream;
restrictions_out_stream.open(restrictions_file_name.c_str(), std::ios::binary); restrictions_out_stream.open(restrictions_file_name.c_str(), std::ios::binary);
restrictions_out_stream.write((char *)&uuid, sizeof(UUID)); restrictions_out_stream.write((char *)&uuid, sizeof(UUID));
restrictions_out_stream.write( restrictions_out_stream.write((char *)&number_of_useable_restrictions, sizeof(unsigned));
(char*)&usableRestrictionsCounter, for (restrictions_iterator = restrictions_list.begin();
sizeof(unsigned)
);
for(
restrictions_iterator = restrictions_list.begin();
restrictions_iterator != restrictions_list.end(); restrictions_iterator != restrictions_list.end();
++restrictions_iterator ++restrictions_iterator)
) { {
if( if (UINT_MAX != restrictions_iterator->restriction.fromNode &&
UINT_MAX != restrictions_iterator->restriction.fromNode && UINT_MAX != restrictions_iterator->restriction.toNode)
UINT_MAX != restrictions_iterator->restriction.toNode {
) { restrictions_out_stream.write((char *)&(restrictions_iterator->restriction),
restrictions_out_stream.write( sizeof(TurnRestriction));
(char *)&(restrictions_iterator->restriction),
sizeof(TurnRestriction)
);
} }
} }
restrictions_out_stream.close(); restrictions_out_stream.close();
@ -246,26 +237,23 @@ void ExtractionContainers::PrepareData(
std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush; std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush;
// identify all used nodes by a merging step of two sorted lists // identify all used nodes by a merging step of two sorted lists
STXXLNodeVector::iterator node_iterator = all_nodes_list.begin(); auto node_iterator = all_nodes_list.begin();
STXXLNodeIDVector::iterator node_id_iterator = used_node_id_list.begin(); auto node_id_iterator = used_node_id_list.begin();
while( while (node_id_iterator != used_node_id_list.end() && node_iterator != all_nodes_list.end())
node_id_iterator != used_node_id_list.end() && {
node_iterator != all_nodes_list.end() if (*node_id_iterator < node_iterator->id)
) { {
if(*node_id_iterator < node_iterator->id){
++node_id_iterator; ++node_id_iterator;
continue; continue;
} }
if(*node_id_iterator > node_iterator->id) { if (*node_id_iterator > node_iterator->id)
{
++node_iterator; ++node_iterator;
continue; continue;
} }
BOOST_ASSERT(*node_id_iterator == node_iterator->id); BOOST_ASSERT(*node_id_iterator == node_iterator->id);
file_out_stream.write( file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode));
(char*)&(*node_iterator),
sizeof(ExternalMemoryNode)
);
++number_of_used_nodes; ++number_of_used_nodes;
++node_id_iterator; ++node_id_iterator;
@ -276,7 +264,6 @@ void ExtractionContainers::PrepareData(
elapsed_seconds = time2 - time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] setting number of nodes ... " << std::flush; std::cout << "[extractor] setting number of nodes ... " << std::flush;
std::ios::pos_type previous_file_position = file_out_stream.tellp(); std::ios::pos_type previous_file_position = file_out_stream.tellp();
file_out_stream.seekp(std::ios::beg + sizeof(UUID)); file_out_stream.seekp(std::ios::beg + sizeof(UUID));
@ -288,12 +275,7 @@ void ExtractionContainers::PrepareData(
// Sort edges by start. // Sort edges by start.
std::cout << "[extractor] Sorting edges by start ... " << std::flush; std::cout << "[extractor] Sorting edges by start ... " << std::flush;
stxxl::sort( stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartID(), 4294967296);
all_edges_list.begin(),
all_edges_list.end(),
CmpEdgeByStartID(),
4294967296
);
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
@ -304,23 +286,23 @@ void ExtractionContainers::PrepareData(
file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned)); file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
// Traverse list of edges and nodes in parallel and set start coord // Traverse list of edges and nodes in parallel and set start coord
node_iterator = all_nodes_list.begin(); node_iterator = all_nodes_list.begin();
STXXLEdgeVector::iterator edge_iterator = all_edges_list.begin(); auto edge_iterator = all_edges_list.begin();
while( while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
edge_iterator != all_edges_list.end() && {
node_iterator != all_nodes_list.end() if (edge_iterator->start < node_iterator->id)
) { {
if(edge_iterator->start < node_iterator->id){
++edge_iterator; ++edge_iterator;
continue; continue;
} }
if(edge_iterator->start > node_iterator->id) { if (edge_iterator->start > node_iterator->id)
{
node_iterator++; node_iterator++;
continue; continue;
} }
BOOST_ASSERT(edge_iterator->start == node_iterator->id); BOOST_ASSERT(edge_iterator->start == node_iterator->id);
edge_iterator->startCoord.lat = node_iterator->lat; edge_iterator->source_coordinate.lat = node_iterator->lat;
edge_iterator->startCoord.lon = node_iterator->lon; edge_iterator->source_coordinate.lon = node_iterator->lon;
++edge_iterator; ++edge_iterator;
} }
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
@ -328,53 +310,51 @@ void ExtractionContainers::PrepareData(
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now(); time1 = std::chrono::steady_clock::now();
// Sort Edges by target // Sort Edges by target
std::cout << "[extractor] Sorting edges by target ... " << std::flush; std::cout << "[extractor] Sorting edges by target ... " << std::flush;
stxxl::sort( stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByTargetID(), 4294967296);
all_edges_list.begin(),
all_edges_list.end(),
CmpEdgeByTargetID(),
4294967296
);
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now(); time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Setting target coords ... " << std::flush; std::cout << "[extractor] Setting target coords ... " << std::flush;
// Traverse list of edges and nodes in parallel and set target coord // Traverse list of edges and nodes in parallel and set target coord
node_iterator = all_nodes_list.begin(); node_iterator = all_nodes_list.begin();
edge_iterator = all_edges_list.begin(); edge_iterator = all_edges_list.begin();
while( while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
edge_iterator != all_edges_list.end() && {
node_iterator != all_nodes_list.end() if (edge_iterator->target < node_iterator->id)
) { {
if(edge_iterator->target < node_iterator->id){
++edge_iterator; ++edge_iterator;
continue; continue;
} }
if(edge_iterator->target > node_iterator->id) { if (edge_iterator->target > node_iterator->id)
{
++node_iterator; ++node_iterator;
continue; continue;
} }
BOOST_ASSERT(edge_iterator->target == node_iterator->id); BOOST_ASSERT(edge_iterator->target == node_iterator->id);
if(edge_iterator->startCoord.lat != INT_MIN && edge_iterator->startCoord.lon != INT_MIN) { if (edge_iterator->source_coordinate.lat != INT_MIN &&
edge_iterator->targetCoord.lat = node_iterator->lat; edge_iterator->source_coordinate.lon != INT_MIN)
edge_iterator->targetCoord.lon = node_iterator->lon; {
const double distance = FixedPointCoordinate::ApproximateDistance(
edge_iterator->startCoord.lat,
edge_iterator->startCoord.lon,
node_iterator->lat,
node_iterator->lon
);
BOOST_ASSERT(edge_iterator->speed != -1); BOOST_ASSERT(edge_iterator->speed != -1);
BOOST_ASSERT(edge_iterator->type >= 0);
edge_iterator->target_coordinate.lat = node_iterator->lat;
edge_iterator->target_coordinate.lon = node_iterator->lon;
const double distance = FixedPointCoordinate::ApproximateEuclideanDistance(
edge_iterator->source_coordinate.lat,
edge_iterator->source_coordinate.lon,
node_iterator->lat,
node_iterator->lon);
const double weight = (distance * 10.) / (edge_iterator->speed / 3.6); const double weight = (distance * 10.) / (edge_iterator->speed / 3.6);
int integer_weight = std::max( 1, (int)std::floor((edge_iterator->isDurationSet ? edge_iterator->speed : weight)+.5) ); int integer_weight = std::max(
1,
(int)std::floor(
(edge_iterator->is_duration_set ? edge_iterator->speed : weight) + .5));
int integer_distance = std::max(1, (int)distance); int integer_distance = std::max(1, (int)distance);
short zero = 0; short zero = 0;
short one = 1; short one = 1;
@ -382,7 +362,8 @@ void ExtractionContainers::PrepareData(
file_out_stream.write((char *)&edge_iterator->start, sizeof(unsigned)); file_out_stream.write((char *)&edge_iterator->start, sizeof(unsigned));
file_out_stream.write((char *)&edge_iterator->target, sizeof(unsigned)); file_out_stream.write((char *)&edge_iterator->target, sizeof(unsigned));
file_out_stream.write((char *)&integer_distance, sizeof(int)); file_out_stream.write((char *)&integer_distance, sizeof(int));
switch(edge_iterator->direction) { switch (edge_iterator->direction)
{
case ExtractionWay::notSure: case ExtractionWay::notSure:
file_out_stream.write((char *)&zero, sizeof(short)); file_out_stream.write((char *)&zero, sizeof(short));
break; break;
@ -391,7 +372,6 @@ void ExtractionContainers::PrepareData(
break; break;
case ExtractionWay::bidirectional: case ExtractionWay::bidirectional:
file_out_stream.write((char *)&zero, sizeof(short)); file_out_stream.write((char *)&zero, sizeof(short));
break; break;
case ExtractionWay::opposite: case ExtractionWay::opposite:
file_out_stream.write((char *)&one, sizeof(short)); file_out_stream.write((char *)&one, sizeof(short));
@ -399,38 +379,15 @@ void ExtractionContainers::PrepareData(
default: default:
throw OSRMException("edge has broken direction"); throw OSRMException("edge has broken direction");
} }
file_out_stream.write(
(char*)&integer_weight, sizeof(int) file_out_stream.write((char *)&integer_weight, sizeof(int));
); file_out_stream.write((char *)&edge_iterator->type, sizeof(short));
BOOST_ASSERT(edge_iterator->type >= 0); file_out_stream.write((char *)&edge_iterator->name_id, sizeof(unsigned));
file_out_stream.write( file_out_stream.write((char *)&edge_iterator->is_roundabout, sizeof(bool));
(char*)&edge_iterator->type, file_out_stream.write((char *)&edge_iterator->is_in_tiny_cc, sizeof(bool));
sizeof(short) file_out_stream.write((char *)&edge_iterator->is_access_restricted, sizeof(bool));
); file_out_stream.write((char *)&edge_iterator->is_contra_flow, sizeof(bool));
file_out_stream.write( file_out_stream.write((char *)&edge_iterator->is_split, sizeof(bool));
(char *) &edge_iterator->nameID,
sizeof(unsigned)
);
file_out_stream.write(
(char *) &edge_iterator->isRoundabout,
sizeof(bool)
);
file_out_stream.write(
(char *) &edge_iterator->ignoreInGrid,
sizeof(bool)
);
file_out_stream.write(
(char *) &edge_iterator->isAccessRestricted,
sizeof(bool)
);
file_out_stream.write(
(char *) &edge_iterator->isContraFlow,
sizeof(bool)
);
file_out_stream.write(
(char *) &edge_iterator->is_split,
sizeof(bool)
);
++number_of_used_edges; ++number_of_used_edges;
} }
++edge_iterator; ++edge_iterator;
@ -449,10 +406,7 @@ void ExtractionContainers::PrepareData(
std::cout << "[extractor] writing street name index ... " << std::flush; std::cout << "[extractor] writing street name index ... " << std::flush;
std::string name_file_streamName = (output_file_name + ".names"); std::string name_file_streamName = (output_file_name + ".names");
boost::filesystem::ofstream name_file_stream( boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary);
name_file_streamName,
std::ios::binary
);
// write number of names // write number of names
const unsigned number_of_names = name_list.size() + 1; const unsigned number_of_names = name_list.size() + 1;
@ -460,31 +414,25 @@ void ExtractionContainers::PrepareData(
// compute total number of chars // compute total number of chars
unsigned total_number_of_chars = 0; unsigned total_number_of_chars = 0;
for (const std::string & temp_string : name_list) { for (const std::string &temp_string : name_list)
{
total_number_of_chars += temp_string.length(); total_number_of_chars += temp_string.length();
} }
// write total number of chars // write total number of chars
name_file_stream.write( name_file_stream.write((char *)&(total_number_of_chars), sizeof(unsigned));
(char *)&(total_number_of_chars),
sizeof(unsigned)
);
// write prefixe sums // write prefixe sums
unsigned name_lengths_prefix_sum = 0; unsigned name_lengths_prefix_sum = 0;
for (const std::string & temp_string : name_list) { for (const std::string &temp_string : name_list)
name_file_stream.write( {
(char *)&(name_lengths_prefix_sum), name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
sizeof(unsigned)
);
name_lengths_prefix_sum += temp_string.length(); name_lengths_prefix_sum += temp_string.length();
} }
// duplicate on purpose! // duplicate on purpose!
name_file_stream.write( name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
(char *)&(name_lengths_prefix_sum),
sizeof(unsigned)
);
// write all chars consecutively // write all chars consecutively
for (const std::string & temp_string : name_list) { for (const std::string &temp_string : name_list)
{
const unsigned string_length = temp_string.length(); const unsigned string_length = temp_string.length();
name_file_stream.write(temp_string.c_str(), string_length); name_file_stream.write(temp_string.c_str(), string_length);
} }
@ -494,12 +442,8 @@ void ExtractionContainers::PrepareData(
elapsed_seconds = time2 - time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
SimpleLogger().Write() << "Processed " << SimpleLogger().Write() << "Processed " << number_of_used_nodes << " nodes and "
number_of_used_nodes << " nodes and " << << number_of_used_edges << " edges";
number_of_used_edges << " edges";
} catch ( const std::exception& e ) {
std::cerr << "Caught Execption:" << e.what() << std::endl;
} }
catch (const std::exception &e) { std::cerr << "Caught Execption:" << e.what() << std::endl; }
} }

11
Extractor/ExtractionContainers.h
View File

@ -35,14 +35,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <stxxl/vector> #include <stxxl/vector>
class ExtractionContainers { class ExtractionContainers
{
public: public:
typedef stxxl::vector<NodeID> STXXLNodeIDVector; typedef stxxl::vector<NodeID> STXXLNodeIDVector;
typedef stxxl::vector<ExternalMemoryNode> STXXLNodeVector; typedef stxxl::vector<ExternalMemoryNode> STXXLNodeVector;
typedef stxxl::vector<InternalExtractorEdge> STXXLEdgeVector; typedef stxxl::vector<InternalExtractorEdge> STXXLEdgeVector;
typedef stxxl::vector<std::string> STXXLStringVector; typedef stxxl::vector<std::string> STXXLStringVector;
typedef stxxl::vector<InputRestrictionContainer> STXXLRestrictionsVector; typedef stxxl::vector<InputRestrictionContainer> STXXLRestrictionsVector;
typedef stxxl::vector<_WayIDStartAndEndEdge> STXXLWayIDStartEndVector; typedef stxxl::vector<WayIDStartAndEndEdge> STXXLWayIDStartEndVector;
STXXLNodeIDVector used_node_id_list; STXXLNodeIDVector used_node_id_list;
STXXLNodeVector all_nodes_list; STXXLNodeVector all_nodes_list;
@ -56,10 +57,8 @@ public:
virtual ~ExtractionContainers(); virtual ~ExtractionContainers();
void PrepareData( void PrepareData(const std::string &output_file_name,
const std::string & output_file_name, const std::string &restrictions_file_name);
const std::string & restrictions_file_name
);
}; };
#endif /* EXTRACTIONCONTAINERS_H_ */ #endif /* EXTRACTIONCONTAINERS_H_ */

46
Extractor/ExtractionHelperFunctions.h
View File

@ -25,22 +25,26 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
#ifndef EXTRACTIONHELPERFUNCTIONS_H_ #ifndef EXTRACTION_HELPER_FUNCTIONS_H
#define EXTRACTIONHELPERFUNCTIONS_H_ #define EXTRACTION_HELPER_FUNCTIONS_H
#include "../Util/StringUtil.h" #include "../Util/StringUtil.h"
#include <boost/algorithm/string.hpp> #include <boost/algorithm/string.hpp>
#include <boost/algorithm/string_regex.hpp> #include <boost/algorithm/string_regex.hpp>
#include <boost/regex.hpp> #include <boost/regex.hpp>
#include <climits>
#include <limits>
namespace qi = boost::spirit::qi; namespace qi = boost::spirit::qi;
// TODO: Move into LUA // TODO: Move into LUA
inline bool durationIsValid(const std::string &s) { inline bool durationIsValid(const std::string &s)
boost::regex e ("((\\d|\\d\\d):(\\d|\\d\\d):(\\d|\\d\\d))|((\\d|\\d\\d):(\\d|\\d\\d))|(\\d|\\d\\d)",boost::regex_constants::icase|boost::regex_constants::perl); {
boost::regex e(
"((\\d|\\d\\d):(\\d|\\d\\d):(\\d|\\d\\d))|((\\d|\\d\\d):(\\d|\\d\\d))|(\\d|\\d\\d)",
boost::regex_constants::icase | boost::regex_constants::perl);
std::vector<std::string> result; std::vector<std::string> result;
boost::algorithm::split_regex(result, s, boost::regex(":")); boost::algorithm::split_regex(result, s, boost::regex(":"));
@ -48,40 +52,38 @@ inline bool durationIsValid(const std::string &s) {
return matched; return matched;
} }
inline unsigned parseDuration(const std::string &s) { inline unsigned parseDuration(const std::string &s)
{
unsigned hours = 0; unsigned hours = 0;
unsigned minutes = 0; unsigned minutes = 0;
unsigned seconds = 0; unsigned seconds = 0;
boost::regex e ("((\\d|\\d\\d):(\\d|\\d\\d):(\\d|\\d\\d))|((\\d|\\d\\d):(\\d|\\d\\d))|(\\d|\\d\\d)",boost::regex_constants::icase|boost::regex_constants::perl); boost::regex e(
"((\\d|\\d\\d):(\\d|\\d\\d):(\\d|\\d\\d))|((\\d|\\d\\d):(\\d|\\d\\d))|(\\d|\\d\\d)",
boost::regex_constants::icase | boost::regex_constants::perl);
std::vector<std::string> result; std::vector<std::string> result;
boost::algorithm::split_regex(result, s, boost::regex(":")); boost::algorithm::split_regex(result, s, boost::regex(":"));
bool matched = regex_match(s, e); bool matched = regex_match(s, e);
if(matched) { if (matched)
if(1 == result.size()) { {
if (1 == result.size())
{
minutes = stringToInt(result[0]); minutes = stringToInt(result[0]);
} }
if(2 == result.size()) { if (2 == result.size())
{
minutes = stringToInt(result[1]); minutes = stringToInt(result[1]);
hours = stringToInt(result[0]); hours = stringToInt(result[0]);
} }
if(3 == result.size()) { if (3 == result.size())
{
seconds = stringToInt(result[2]); seconds = stringToInt(result[2]);
minutes = stringToInt(result[1]); minutes = stringToInt(result[1]);
hours = stringToInt(result[0]); hours = stringToInt(result[0]);
} }
return 10 * (3600 * hours + 60 * minutes + seconds); return 10 * (3600 * hours + 60 * minutes + seconds);
} }
return UINT_MAX; return std::numeric_limits<unsigned>::max();
} }
// inline int parseMaxspeed(std::string input) { //call-by-value on purpose. #endif // EXTRACTION_HELPER_FUNCTIONS_H_
// boost::algorithm::to_lower(input);
// int n = stringToInt(input);
// if (input.find("mph") != std::string::npos || input.find("mp/h") != std::string::npos) {
// n = (n*1609)/1000;
// }
// return n;
// }
#endif /* EXTRACTIONHELPERFUNCTIONS_H_ */

23
Extractor/ExtractionWay.h
View File

@ -34,14 +34,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <string> #include <string>
#include <vector> #include <vector>
struct ExtractionWay { struct ExtractionWay
ExtractionWay() { {
Clear(); ExtractionWay() { Clear(); }
}
inline void Clear(){ inline void Clear()
id = UINT_MAX; {
nameID = UINT_MAX; id = SPECIAL_NODEID;
nameID = INVALID_NAMEID;
path.clear(); path.clear();
keyVals.clear(); keyVals.clear();
direction = ExtractionWay::notSure; direction = ExtractionWay::notSure;
@ -55,9 +55,11 @@ struct ExtractionWay {
ignoreInGrid = false; ignoreInGrid = false;
} }
enum Directions { enum Directions
notSure = 0, oneway, bidirectional, opposite { notSure = 0,
}; oneway,
bidirectional,
opposite };
unsigned id; unsigned id;
unsigned nameID; unsigned nameID;
double speed; double speed;
@ -74,5 +76,4 @@ struct ExtractionWay {
HashTable<std::string, std::string> keyVals; HashTable<std::string, std::string> keyVals;
}; };
#endif // EXTRACTION_WAY_H #endif // EXTRACTION_WAY_H

123
Extractor/ExtractorCallbacks.cpp
View File

@ -38,75 +38,77 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <string> #include <string>
#include <vector> #include <vector>
ExtractorCallbacks::ExtractorCallbacks() ExtractorCallbacks::ExtractorCallbacks(ExtractionContainers &extraction_containers,
: boost::unordered_map<std::string, NodeID> &string_map)
string_map(NULL), : string_map(string_map), external_memory(extraction_containers)
externalMemory(NULL) {
{ } }
ExtractorCallbacks::ExtractorCallbacks(
ExtractionContainers * ext,
boost::unordered_map<std::string, NodeID> * string_map
) :
string_map(string_map),
externalMemory(ext)
{ }
ExtractorCallbacks::~ExtractorCallbacks() { }
/** warning: caller needs to take care of synchronization! */ /** warning: caller needs to take care of synchronization! */
void ExtractorCallbacks::nodeFunction(const ExternalMemoryNode &n) { void ExtractorCallbacks::ProcessNode(const ExternalMemoryNode &n)
if(n.lat <= 85*COORDINATE_PRECISION && n.lat >= -85*COORDINATE_PRECISION) { {
externalMemory->all_nodes_list.push_back(n); if (n.lat <= 85 * COORDINATE_PRECISION && n.lat >= -85 * COORDINATE_PRECISION)
{
external_memory.all_nodes_list.push_back(n);
} }
} }
bool ExtractorCallbacks::restrictionFunction(const InputRestrictionContainer &r) { bool ExtractorCallbacks::ProcessRestriction(const InputRestrictionContainer &restriction)
externalMemory->restrictions_list.push_back(r); {
external_memory.restrictions_list.push_back(restriction);
return true; return true;
} }
/** warning: caller needs to take care of synchronization! */ /** warning: caller needs to take care of synchronization! */
void ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) { void ExtractorCallbacks::ProcessWay(ExtractionWay &parsed_way)
if((0 < parsed_way.speed) || (0 < parsed_way.duration)) { //Only true if the way is specified by the speed profile {
if(UINT_MAX == parsed_way.id){ if ((0 < parsed_way.speed) || (0 < parsed_way.duration))
SimpleLogger().Write(logDEBUG) << { // Only true if the way is specified by the speed profile
"found bogus way with id: " << parsed_way.id << if (UINT_MAX == parsed_way.id)
" of size " << parsed_way.path.size(); {
SimpleLogger().Write(logDEBUG) << "found bogus way with id: " << parsed_way.id
<< " of size " << parsed_way.path.size();
return; return;
} }
if(0 < parsed_way.duration) { if (0 < parsed_way.duration)
//TODO: iterate all way segments and set duration corresponding to the length of each segment {
// TODO: iterate all way segments and set duration corresponding to the length of each
// segment
parsed_way.speed = parsed_way.duration / (parsed_way.path.size() - 1); parsed_way.speed = parsed_way.duration / (parsed_way.path.size() - 1);
} }
if(std::numeric_limits<double>::epsilon() >= std::abs(-1. - parsed_way.speed)){ if (std::numeric_limits<double>::epsilon() >= std::abs(-1. - parsed_way.speed))
SimpleLogger().Write(logDEBUG) << {
"found way with bogus speed, id: " << parsed_way.id; SimpleLogger().Write(logDEBUG) << "found way with bogus speed, id: " << parsed_way.id;
return; return;
} }
// Get the unique identifier for the street name // Get the unique identifier for the street name
const boost::unordered_map<std::string, NodeID>::const_iterator & string_map_iterator = string_map->find(parsed_way.name); const auto &string_map_iterator = string_map.find(parsed_way.name);
if(string_map->end() == string_map_iterator) { if (string_map.end() == string_map_iterator)
parsed_way.nameID = externalMemory->name_list.size(); {
externalMemory->name_list.push_back(parsed_way.name); parsed_way.nameID = external_memory.name_list.size();
string_map->insert(std::make_pair(parsed_way.name, parsed_way.nameID)); external_memory.name_list.push_back(parsed_way.name);
} else { string_map.insert(std::make_pair(parsed_way.name, parsed_way.nameID));
}
else
{
parsed_way.nameID = string_map_iterator->second; parsed_way.nameID = string_map_iterator->second;
} }
if(ExtractionWay::opposite == parsed_way.direction) { if (ExtractionWay::opposite == parsed_way.direction)
{
std::reverse(parsed_way.path.begin(), parsed_way.path.end()); std::reverse(parsed_way.path.begin(), parsed_way.path.end());
parsed_way.direction = ExtractionWay::oneway; parsed_way.direction = ExtractionWay::oneway;
} }
const bool split_bidirectional_edge = (parsed_way.backward_speed > 0) && (parsed_way.speed != parsed_way.backward_speed); const bool split_bidirectional_edge =
(parsed_way.backward_speed > 0) && (parsed_way.speed != parsed_way.backward_speed);
for(std::vector< NodeID >::size_type n = 0; n < parsed_way.path.size()-1; ++n) { for (unsigned n = 0; n < parsed_way.path.size() - 1; ++n)
externalMemory->all_edges_list.push_back( {
InternalExtractorEdge( external_memory.all_edges_list.push_back(InternalExtractorEdge(
parsed_way.path[n], parsed_way.path[n],
parsed_way.path[n + 1], parsed_way.path[n + 1],
parsed_way.type, parsed_way.type,
@ -118,22 +120,26 @@ void ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) {
(0 < parsed_way.duration), (0 < parsed_way.duration),
parsed_way.isAccessRestricted, parsed_way.isAccessRestricted,
false, false,
split_bidirectional_edge split_bidirectional_edge));
) external_memory.used_node_id_list.push_back(parsed_way.path[n]);
);
externalMemory->used_node_id_list.push_back(parsed_way.path[n]);
} }
externalMemory->used_node_id_list.push_back(parsed_way.path.back()); external_memory.used_node_id_list.push_back(parsed_way.path.back());
// The following information is needed to identify start and end segments of restrictions // The following information is needed to identify start and end segments of restrictions
externalMemory->way_start_end_id_list.push_back(_WayIDStartAndEndEdge(parsed_way.id, parsed_way.path[0], parsed_way.path[1], parsed_way.path[parsed_way.path.size()-2], parsed_way.path.back())); external_memory.way_start_end_id_list.push_back(
WayIDStartAndEndEdge(parsed_way.id,
parsed_way.path[0],
parsed_way.path[1],
parsed_way.path[parsed_way.path.size() - 2],
parsed_way.path.back()));
if(split_bidirectional_edge) { //Only true if the way should be split if (split_bidirectional_edge)
{ // Only true if the way should be split
std::reverse(parsed_way.path.begin(), parsed_way.path.end()); std::reverse(parsed_way.path.begin(), parsed_way.path.end());
for(std::vector< NodeID >::size_type n = 0; n < parsed_way.path.size()-1; ++n) { for (std::vector<NodeID>::size_type n = 0; n < parsed_way.path.size() - 1; ++n)
externalMemory->all_edges_list.push_back( {
InternalExtractorEdge( external_memory.all_edges_list.push_back(
parsed_way.path[n], InternalExtractorEdge(parsed_way.path[n],
parsed_way.path[n + 1], parsed_way.path[n + 1],
parsed_way.type, parsed_way.type,
ExtractionWay::oneway, ExtractionWay::oneway,
@ -144,11 +150,14 @@ void ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) {
(0 < parsed_way.duration), (0 < parsed_way.duration),
parsed_way.isAccessRestricted, parsed_way.isAccessRestricted,
(ExtractionWay::oneway == parsed_way.direction), (ExtractionWay::oneway == parsed_way.direction),
split_bidirectional_edge split_bidirectional_edge));
)
);
} }
externalMemory->way_start_end_id_list.push_back(_WayIDStartAndEndEdge(parsed_way.id, parsed_way.path[0], parsed_way.path[1], parsed_way.path[parsed_way.path.size()-2], parsed_way.path.back())); external_memory.way_start_end_id_list.push_back(
WayIDStartAndEndEdge(parsed_way.id,
parsed_way.path[0],
parsed_way.path[1],
parsed_way.path[parsed_way.path.size() - 2],
parsed_way.path.back()));
} }
} }
} }

37
Extractor/ExtractorCallbacks.h
View File

@ -25,8 +25,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
#ifndef EXTRACTORCALLBACKS_H_ #ifndef EXTRACTOR_CALLBACKS_H
#define EXTRACTORCALLBACKS_H_ #define EXTRACTOR_CALLBACKS_H
#include "../typedefs.h" #include "../typedefs.h"
@ -38,29 +38,26 @@ class ExtractionContainers;
struct ExtractionWay; struct ExtractionWay;
struct InputRestrictionContainer; struct InputRestrictionContainer;
class ExtractorCallbacks{ class ExtractorCallbacks
{
private: private:
boost::unordered_map<std::string, NodeID> &string_map;
ExtractionContainers &external_memory;
boost::unordered_map<std::string, NodeID> * string_map;
ExtractionContainers * externalMemory;
ExtractorCallbacks();
public: public:
explicit ExtractorCallbacks( ExtractorCallbacks() = delete;
ExtractionContainers * ext, ExtractorCallbacks(const ExtractorCallbacks &) = delete;
boost::unordered_map<std::string, NodeID> * string_map explicit ExtractorCallbacks(ExtractionContainers &extraction_containers,
); boost::unordered_map<std::string, NodeID> &string_map);
~ExtractorCallbacks(); // warning: caller needs to take care of synchronization!
void ProcessNode(const ExternalMemoryNode &node);
/** warning: caller needs to take care of synchronization! */ // warning: caller needs to take care of synchronization!
void nodeFunction(const ExternalMemoryNode &n); bool ProcessRestriction(const InputRestrictionContainer &restriction);
bool restrictionFunction(const InputRestrictionContainer &r);
/** warning: caller needs to take care of synchronization! */
void wayFunction(ExtractionWay &w);
// warning: caller needs to take care of synchronization!
void ProcessWay(ExtractionWay &way);
}; };
#endif /* EXTRACTORCALLBACKS_H_ */ #endif /* EXTRACTOR_CALLBACKS_H */

117
Extractor/ExtractorStructs.h
View File

@ -34,97 +34,80 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <string> #include <string>
struct ExtractorRelation { struct ExtractorRelation
{
ExtractorRelation() : type(unknown) {} ExtractorRelation() : type(unknown) {}
enum { enum
unknown = 0, ferry, turnRestriction { unknown = 0,
} type; ferry,
turnRestriction } type;
HashTable<std::string, std::string> keyVals; HashTable<std::string, std::string> keyVals;
}; };
struct _WayIDStartAndEndEdge { struct WayIDStartAndEndEdge
{
unsigned wayID; unsigned wayID;
NodeID firstStart; NodeID firstStart;
NodeID firstTarget; NodeID firstTarget;
NodeID lastStart; NodeID lastStart;
NodeID lastTarget; NodeID lastTarget;
_WayIDStartAndEndEdge() WayIDStartAndEndEdge()
: : wayID(UINT_MAX), firstStart(UINT_MAX), firstTarget(UINT_MAX), lastStart(UINT_MAX),
wayID(UINT_MAX),
firstStart(UINT_MAX),
firstTarget(UINT_MAX),
lastStart(UINT_MAX),
lastTarget(UINT_MAX) lastTarget(UINT_MAX)
{ } {
explicit _WayIDStartAndEndEdge(
unsigned w,
NodeID fs,
NodeID ft,
NodeID ls,
NodeID lt
) :
wayID(w),
firstStart(fs),
firstTarget(ft),
lastStart(ls),
lastTarget(lt)
{ }
static _WayIDStartAndEndEdge min_value() {
return _WayIDStartAndEndEdge((std::numeric_limits<unsigned>::min)(), (std::numeric_limits<unsigned>::min)(), (std::numeric_limits<unsigned>::min)(), (std::numeric_limits<unsigned>::min)(), (std::numeric_limits<unsigned>::min)());
} }
static _WayIDStartAndEndEdge max_value() {
return _WayIDStartAndEndEdge((std::numeric_limits<unsigned>::max)(), (std::numeric_limits<unsigned>::max)(), (std::numeric_limits<unsigned>::max)(), (std::numeric_limits<unsigned>::max)(), (std::numeric_limits<unsigned>::max)()); explicit WayIDStartAndEndEdge(unsigned w, NodeID fs, NodeID ft, NodeID ls, NodeID lt)
: wayID(w), firstStart(fs), firstTarget(ft), lastStart(ls), lastTarget(lt)
{
}
static WayIDStartAndEndEdge min_value()
{
return WayIDStartAndEndEdge((std::numeric_limits<unsigned>::min)(),
(std::numeric_limits<unsigned>::min)(),
(std::numeric_limits<unsigned>::min)(),
(std::numeric_limits<unsigned>::min)(),
(std::numeric_limits<unsigned>::min)());
}
static WayIDStartAndEndEdge max_value()
{
return WayIDStartAndEndEdge((std::numeric_limits<unsigned>::max)(),
(std::numeric_limits<unsigned>::max)(),
(std::numeric_limits<unsigned>::max)(),
(std::numeric_limits<unsigned>::max)(),
(std::numeric_limits<unsigned>::max)());
} }
}; };
struct CmpWayByID { struct CmpWayByID
typedef _WayIDStartAndEndEdge value_type; {
bool operator ()( typedef WayIDStartAndEndEdge value_type;
const _WayIDStartAndEndEdge & a, bool operator()(const WayIDStartAndEndEdge &a, const WayIDStartAndEndEdge &b) const
const _WayIDStartAndEndEdge & b {
) const {
return a.wayID < b.wayID; return a.wayID < b.wayID;
} }
value_type max_value() { value_type max_value() { return WayIDStartAndEndEdge::max_value(); }
return _WayIDStartAndEndEdge::max_value(); value_type min_value() { return WayIDStartAndEndEdge::min_value(); }
}
value_type min_value() {
return _WayIDStartAndEndEdge::min_value();
}
}; };
struct Cmp { struct Cmp
{
typedef NodeID value_type; typedef NodeID value_type;
bool operator ()( bool operator()(const NodeID a, const NodeID b) const { return a < b; }
const NodeID a, value_type max_value() { return 0xffffffff; }
const NodeID b value_type min_value() { return 0x0; }
) const {
return a < b;
}
value_type max_value() {
return 0xffffffff;
}
value_type min_value() {
return 0x0;
}
}; };
struct CmpNodeByID { struct CmpNodeByID
{
typedef ExternalMemoryNode value_type; typedef ExternalMemoryNode value_type;
bool operator () ( bool operator()(const ExternalMemoryNode &a, const ExternalMemoryNode &b) const
const ExternalMemoryNode & a, {
const ExternalMemoryNode & b
) const {
return a.id < b.id; return a.id < b.id;
} }
value_type max_value() { value_type max_value() { return ExternalMemoryNode::max_value(); }
return ExternalMemoryNode::max_value(); value_type min_value() { return ExternalMemoryNode::min_value(); }
}
value_type min_value() {
return ExternalMemoryNode::min_value();
}
}; };
#endif /* EXTRACTORSTRUCTS_H_ */ #endif /* EXTRACTORSTRUCTS_H_ */

143
Extractor/InternalExtractorEdge.h
View File

@ -28,92 +28,49 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef INTERNAL_EXTRACTOR_EDGE_H #ifndef INTERNAL_EXTRACTOR_EDGE_H
#define INTERNAL_EXTRACTOR_EDGE_H #define INTERNAL_EXTRACTOR_EDGE_H
#include "../typedefs.h" #include "../typedefs.h"
#include <osrm/Coordinate.h> #include <osrm/Coordinate.h>
#include <boost/assert.hpp> #include <boost/assert.hpp>
struct InternalExtractorEdge { struct InternalExtractorEdge
{
InternalExtractorEdge() InternalExtractorEdge()
: : start(0), target(0), type(0), direction(0), speed(0), name_id(0), is_roundabout(false),
start(0), is_in_tiny_cc(false), is_duration_set(false), is_access_restricted(false),
target(0), is_contra_flow(false), is_split(false)
type(0), {
direction(0), }
speed(0),
nameID(0),
isRoundabout(false),
ignoreInGrid(false),
isDurationSet(false),
isAccessRestricted(false),
isContraFlow(false),
is_split(false)
{ }
explicit InternalExtractorEdge(NodeID start,
explicit InternalExtractorEdge(
NodeID start,
NodeID target, NodeID target,
short type, short type,
short direction, short direction,
double speed, double speed,
unsigned nameID, unsigned name_id,
bool isRoundabout, bool is_roundabout,
bool ignoreInGrid, bool is_in_tiny_cc,
bool isDurationSet, bool is_duration_set,
bool isAccressRestricted, bool is_access_restricted,
bool isContraFlow, bool is_contra_flow,
bool is_split bool is_split)
) : : start(start), target(target), type(type), direction(direction), speed(speed),
start(start), name_id(name_id), is_roundabout(is_roundabout), is_in_tiny_cc(is_in_tiny_cc),
target(target), is_duration_set(is_duration_set), is_access_restricted(is_access_restricted),
type(type), is_contra_flow(is_contra_flow), is_split(is_split)
direction(direction),
speed(speed),
nameID(nameID),
isRoundabout(isRoundabout),
ignoreInGrid(ignoreInGrid),
isDurationSet(isDurationSet),
isAccessRestricted(isAccressRestricted),
isContraFlow(isContraFlow),
is_split(is_split)
{ {
BOOST_ASSERT(0 <= type); BOOST_ASSERT(0 <= type);
} }
// necessary static util functions for stxxl's sorting // necessary static util functions for stxxl's sorting
static InternalExtractorEdge min_value() { static InternalExtractorEdge min_value()
return InternalExtractorEdge( {
0, return InternalExtractorEdge(0, 0, 0, 0, 0, 0, false, false, false, false, false, false);
0,
0,
0,
0,
0,
false,
false,
false,
false,
false,
false
);
} }
static InternalExtractorEdge max_value() { static InternalExtractorEdge max_value()
{
return InternalExtractorEdge( return InternalExtractorEdge(
std::numeric_limits<unsigned>::max(), SPECIAL_NODEID, SPECIAL_NODEID, 0, 0, 0, 0, false, false, false, false, false, false);
std::numeric_limits<unsigned>::max(),
0,
0,
0,
0,
false,
false,
false,
false,
false,
false
);
} }
NodeID start; NodeID start;
@ -121,53 +78,43 @@ struct InternalExtractorEdge {
short type; short type;
short direction; short direction;
double speed; double speed;
unsigned nameID; unsigned name_id;
bool isRoundabout; bool is_roundabout;
bool ignoreInGrid; bool is_in_tiny_cc;
bool isDurationSet; bool is_duration_set;
bool isAccessRestricted; bool is_access_restricted;
bool isContraFlow; bool is_contra_flow;
bool is_split; bool is_split;
FixedPointCoordinate startCoord; FixedPointCoordinate source_coordinate;
FixedPointCoordinate targetCoord; FixedPointCoordinate target_coordinate;
}; };
struct CmpEdgeByStartID { struct CmpEdgeByStartID
{
typedef InternalExtractorEdge value_type; typedef InternalExtractorEdge value_type;
bool operator ()( bool operator()(const InternalExtractorEdge &a, const InternalExtractorEdge &b) const
const InternalExtractorEdge & a, {
const InternalExtractorEdge & b
) const {
return a.start < b.start; return a.start < b.start;
} }
value_type max_value() { value_type max_value() { return InternalExtractorEdge::max_value(); }
return InternalExtractorEdge::max_value();
}
value_type min_value() { value_type min_value() { return InternalExtractorEdge::min_value(); }
return InternalExtractorEdge::min_value();
}
}; };
struct CmpEdgeByTargetID { struct CmpEdgeByTargetID
{
typedef InternalExtractorEdge value_type; typedef InternalExtractorEdge value_type;
bool operator ()( bool operator()(const InternalExtractorEdge &a, const InternalExtractorEdge &b) const
const InternalExtractorEdge & a, {
const InternalExtractorEdge & b
) const {
return a.target < b.target; return a.target < b.target;
} }
value_type max_value() { value_type max_value() { return InternalExtractorEdge::max_value(); }
return InternalExtractorEdge::max_value();
}
value_type min_value() { value_type min_value() { return InternalExtractorEdge::min_value(); }
return InternalExtractorEdge::min_value();
}
}; };
#endif // INTERNAL_EXTRACTOR_EDGE_H #endif // INTERNAL_EXTRACTOR_EDGE_H

447
Extractor/PBFParser.cpp
View File

@ -42,151 +42,184 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <osrm/Coordinate.h> #include <osrm/Coordinate.h>
#include <boost/make_shared.hpp>
#include <boost/ref.hpp> #include <boost/ref.hpp>
#include <boost/thread.hpp>
#include <zlib.h> #include <zlib.h>
PBFParser::PBFParser( #include <functional>
const char * fileName, #include <thread>
PBFParser::PBFParser(const char *fileName,
ExtractorCallbacks *extractor_callbacks, ExtractorCallbacks *extractor_callbacks,
ScriptingEnvironment& scripting_environment ScriptingEnvironment &scripting_environment)
) : BaseParser( extractor_callbacks, scripting_environment ) { : BaseParser(extractor_callbacks, scripting_environment)
{
GOOGLE_PROTOBUF_VERIFY_VERSION; GOOGLE_PROTOBUF_VERIFY_VERSION;
// TODO: What is the bottleneck here? Filling the queue or reading the stuff from disk? // TODO: What is the bottleneck here? Filling the queue or reading the stuff from disk?
// NOTE: With Lua scripting, it is parsing the stuff. I/O is virtually for free. // NOTE: With Lua scripting, it is parsing the stuff. I/O is virtually for free.
// Max 2500 items in queue, hardcoded. // Max 2500 items in queue, hardcoded.
threadDataQueue = boost::make_shared<ConcurrentQueue<_ThreadData*> >( 2500 ); thread_data_queue = std::make_shared<ConcurrentQueue<ParserThreadData *>>(2500);
input.open(fileName, std::ios::in | std::ios::binary); input.open(fileName, std::ios::in | std::ios::binary);
if (!input) { if (!input)
{
throw OSRMException("pbf file not found."); throw OSRMException("pbf file not found.");
} }
blockCount = 0; block_count = 0;
groupCount = 0; group_count = 0;
} }
PBFParser::~PBFParser() { PBFParser::~PBFParser()
if(input.is_open()) { {
if (input.is_open())
{
input.close(); input.close();
} }
// Clean up any leftover ThreadData objects in the queue // Clean up any leftover ThreadData objects in the queue
_ThreadData* thread_data; ParserThreadData *thread_data;
while (threadDataQueue->try_pop(thread_data)) while (thread_data_queue->try_pop(thread_data))
{ {
delete thread_data; delete thread_data;
} }
google::protobuf::ShutdownProtobufLibrary(); google::protobuf::ShutdownProtobufLibrary();
SimpleLogger().Write(logDEBUG) << SimpleLogger().Write(logDEBUG) << "parsed " << block_count << " blocks from pbf with "
"parsed " << blockCount << << group_count << " groups";
" blocks from pbf with " << groupCount <<
" groups";
} }
inline bool PBFParser::ReadHeader() { inline bool PBFParser::ReadHeader()
_ThreadData initData; {
ParserThreadData init_data;
/** read Header */ /** read Header */
if(!readPBFBlobHeader(input, &initData)) { if (!readPBFBlobHeader(input, &init_data))
{
return false; return false;
} }
if(readBlob(input, &initData)) { if (readBlob(input, &init_data))
if(!initData.PBFHeaderBlock.ParseFromArray(&(initData.charBuffer[0]), initData.charBuffer.size() ) ) { {
if (!init_data.PBFHeaderBlock.ParseFromArray(&(init_data.charBuffer[0]),
init_data.charBuffer.size()))
{
std::cerr << "[error] Header not parseable!" << std::endl; std::cerr << "[error] Header not parseable!" << std::endl;
return false; return false;
} }
for(int i = 0, featureSize = initData.PBFHeaderBlock.required_features_size(); i < featureSize; ++i) { const auto feature_size = init_data.PBFHeaderBlock.required_features_size();
const std::string& feature = initData.PBFHeaderBlock.required_features( i ); for (int i = 0; i < feature_size; ++i)
{
const std::string &feature = init_data.PBFHeaderBlock.required_features(i);
bool supported = false; bool supported = false;
if ( "OsmSchema-V0.6" == feature ) { if ("OsmSchema-V0.6" == feature)
{
supported = true; supported = true;
} }
else if ( "DenseNodes" == feature ) { else if ("DenseNodes" == feature)
{
supported = true; supported = true;
} }
if ( !supported ) { if (!supported)
std::cerr << "[error] required feature not supported: " << feature.data() << std::endl; {
std::cerr << "[error] required feature not supported: " << feature.data()
<< std::endl;
return false; return false;
} }
} }
} else { }
else
{
std::cerr << "[error] blob not loaded!" << std::endl; std::cerr << "[error] blob not loaded!" << std::endl;
} }
return true; return true;
} }
inline void PBFParser::ReadData() { inline void PBFParser::ReadData()
bool keepRunning = true; {
do { bool keep_running = true;
_ThreadData *threadData = new _ThreadData(); do
keepRunning = readNextBlock(input, threadData); {
ParserThreadData *thread_data = new ParserThreadData();
keep_running = readNextBlock(input, thread_data);
if (keepRunning) { if (keep_running)
threadDataQueue->push(threadData); {
} else { thread_data_queue->push(thread_data);
threadDataQueue->push(nullptr); // No more data to read, parse stops when nullptr encountered
delete threadData;
} }
} while(keepRunning); else
{
// No more data to read, parse stops when nullptr encountered
thread_data_queue->push(nullptr);
delete thread_data;
}
} while (keep_running);
} }
inline void PBFParser::ParseData() { inline void PBFParser::ParseData()
while (true) { {
_ThreadData *threadData; while (true)
threadDataQueue->wait_and_pop(threadData); {
if( nullptr==threadData ) { ParserThreadData *thread_data;
threadDataQueue->push(nullptr); // Signal end of data for other threads thread_data_queue->wait_and_pop(thread_data);
if (nullptr == thread_data)
{
thread_data_queue->push(nullptr); // Signal end of data for other threads
break; break;
} }
loadBlock(threadData); loadBlock(thread_data);
for(int i = 0, groupSize = threadData->PBFprimitiveBlock.primitivegroup_size(); i < groupSize; ++i) { int group_size = group_size = thread_data->PBFprimitiveBlock.primitivegroup_size();
threadData->currentGroupID = i; for (int i = 0; i < group_size; ++i)
loadGroup(threadData); {
thread_data->currentGroupID = i;
loadGroup(thread_data);
if(threadData->entityTypeIndicator == TypeNode) { if (thread_data->entityTypeIndicator == TypeNode)
parseNode(threadData); {
parseNode(thread_data);
} }
if(threadData->entityTypeIndicator == TypeWay) { if (thread_data->entityTypeIndicator == TypeWay)
parseWay(threadData); {
parseWay(thread_data);
} }
if(threadData->entityTypeIndicator == TypeRelation) { if (thread_data->entityTypeIndicator == TypeRelation)
parseRelation(threadData); {
parseRelation(thread_data);
} }
if(threadData->entityTypeIndicator == TypeDenseNode) { if (thread_data->entityTypeIndicator == TypeDenseNode)
parseDenseNode(threadData); {
parseDenseNode(thread_data);
} }
} }
delete threadData; delete thread_data;
threadData = NULL; thread_data = nullptr;
} }
} }
inline bool PBFParser::Parse() { inline bool PBFParser::Parse()
{
// Start the read and parse threads // Start the read and parse threads
boost::thread readThread(boost::bind(&PBFParser::ReadData, this)); std::thread read_thread(std::bind(&PBFParser::ReadData, this));
// Open several parse threads that are synchronized before call to // Open several parse threads that are synchronized before call to
boost::thread parseThread(boost::bind(&PBFParser::ParseData, this)); std::thread parse_thread(std::bind(&PBFParser::ParseData, this));
// Wait for the threads to finish // Wait for the threads to finish
readThread.join(); read_thread.join();
parseThread.join(); parse_thread.join();
return true; return true;
} }
inline void PBFParser::parseDenseNode(_ThreadData * threadData) { inline void PBFParser::parseDenseNode(ParserThreadData *thread_data)
const OSMPBF::DenseNodes& dense = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).dense(); {
const OSMPBF::DenseNodes &dense =
thread_data->PBFprimitiveBlock.primitivegroup(thread_data->currentGroupID).dense();
int denseTagIndex = 0; int denseTagIndex = 0;
int64_t m_lastDenseID = 0; int64_t m_lastDenseID = 0;
int64_t m_lastDenseLatitude = 0; int64_t m_lastDenseLatitude = 0;
@ -194,22 +227,33 @@ inline void PBFParser::parseDenseNode(_ThreadData * threadData) {
const int number_of_nodes = dense.id_size(); const int number_of_nodes = dense.id_size();
std::vector<ImportNode> extracted_nodes_vector(number_of_nodes); std::vector<ImportNode> extracted_nodes_vector(number_of_nodes);
for(int i = 0; i < number_of_nodes; ++i) { for (int i = 0; i < number_of_nodes; ++i)
{
m_lastDenseID += dense.id(i); m_lastDenseID += dense.id(i);
m_lastDenseLatitude += dense.lat(i); m_lastDenseLatitude += dense.lat(i);
m_lastDenseLongitude += dense.lon(i); m_lastDenseLongitude += dense.lon(i);
extracted_nodes_vector[i].id = m_lastDenseID; extracted_nodes_vector[i].id = m_lastDenseID;
extracted_nodes_vector[i].lat = COORDINATE_PRECISION*( ( double ) m_lastDenseLatitude * threadData->PBFprimitiveBlock.granularity() + threadData->PBFprimitiveBlock.lat_offset() ) / NANO; extracted_nodes_vector[i].lat =
extracted_nodes_vector[i].lon = COORDINATE_PRECISION*( ( double ) m_lastDenseLongitude * threadData->PBFprimitiveBlock.granularity() + threadData->PBFprimitiveBlock.lon_offset() ) / NANO; COORDINATE_PRECISION *
while (denseTagIndex < dense.keys_vals_size()) { ((double)m_lastDenseLatitude * thread_data->PBFprimitiveBlock.granularity() +
thread_data->PBFprimitiveBlock.lat_offset()) /
NANO;
extracted_nodes_vector[i].lon =
COORDINATE_PRECISION *
((double)m_lastDenseLongitude * thread_data->PBFprimitiveBlock.granularity() +
thread_data->PBFprimitiveBlock.lon_offset()) /
NANO;
while (denseTagIndex < dense.keys_vals_size())
{
const int tagValue = dense.keys_vals(denseTagIndex); const int tagValue = dense.keys_vals(denseTagIndex);
if( 0 == tagValue ) { if (0 == tagValue)
{
++denseTagIndex; ++denseTagIndex;
break; break;
} }
const int keyValue = dense.keys_vals(denseTagIndex + 1); const int keyValue = dense.keys_vals(denseTagIndex + 1);
const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(tagValue); const std::string &key = thread_data->PBFprimitiveBlock.stringtable().s(tagValue);
const std::string & value = threadData->PBFprimitiveBlock.stringtable().s(keyValue); const std::string &value = thread_data->PBFprimitiveBlock.stringtable().s(keyValue);
extracted_nodes_vector[i].keyVals.emplace(key, value); extracted_nodes_vector[i].keyVals.emplace(key, value);
denseTagIndex += 2; denseTagIndex += 2;
} }
@ -227,35 +271,47 @@ inline void PBFParser::parseDenseNode(_ThreadData * threadData) {
for (const ImportNode &import_node : extracted_nodes_vector) for (const ImportNode &import_node : extracted_nodes_vector)
{ {
extractor_callbacks->nodeFunction(import_node); extractor_callbacks->ProcessNode(import_node);
} }
} }
inline void PBFParser::parseNode(_ThreadData * ) inline void PBFParser::parseNode(ParserThreadData *)
{ {
throw OSRMException("Parsing of simple nodes not supported. PBF should use dense nodes"); throw OSRMException("Parsing of simple nodes not supported. PBF should use dense nodes");
} }
inline void PBFParser::parseRelation(_ThreadData * threadData) { inline void PBFParser::parseRelation(ParserThreadData *thread_data)
{
// TODO: leave early, if relation is not a restriction // TODO: leave early, if relation is not a restriction
// TODO: reuse rawRestriction container // TODO: reuse rawRestriction container
if( !use_turn_restrictions ) { if (!use_turn_restrictions)
{
return; return;
} }
const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ); const OSMPBF::PrimitiveGroup &group =
thread_data->PBFprimitiveBlock.primitivegroup(thread_data->currentGroupID);
for(int i = 0, relation_size = group.relations_size(); i < relation_size; ++i ) { for (int i = 0, relation_size = group.relations_size(); i < relation_size; ++i)
{
std::string except_tag_string; std::string except_tag_string;
const OSMPBF::Relation& inputRelation = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).relations(i); const OSMPBF::Relation &inputRelation =
thread_data->PBFprimitiveBlock.primitivegroup(thread_data->currentGroupID).relations(i);
bool isRestriction = false; bool isRestriction = false;
bool isOnlyRestriction = false; bool isOnlyRestriction = false;
for(int k = 0, endOfKeys = inputRelation.keys_size(); k < endOfKeys; ++k) { for (int k = 0, endOfKeys = inputRelation.keys_size(); k < endOfKeys; ++k)
const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(inputRelation.keys(k)); {
const std::string & val = threadData->PBFprimitiveBlock.stringtable().s(inputRelation.vals(k)); const std::string &key =
if ("type" == key) { thread_data->PBFprimitiveBlock.stringtable().s(inputRelation.keys(k));
if( "restriction" == val) { const std::string &val =
thread_data->PBFprimitiveBlock.stringtable().s(inputRelation.vals(k));
if ("type" == key)
{
if ("restriction" == val)
{
isRestriction = true; isRestriction = true;
} else { }
else
{
break; break;
} }
} }
@ -269,32 +325,38 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
} }
} }
if( isRestriction && ShouldIgnoreRestriction(except_tag_string) ) { if (isRestriction && ShouldIgnoreRestriction(except_tag_string))
{
continue; continue;
} }
if(isRestriction) { if (isRestriction)
{
int64_t lastRef = 0; int64_t lastRef = 0;
InputRestrictionContainer currentRestrictionContainer(isOnlyRestriction); InputRestrictionContainer currentRestrictionContainer(isOnlyRestriction);
for( for (int rolesIndex = 0, last_role = inputRelation.roles_sid_size();
int rolesIndex = 0, last_role = inputRelation.roles_sid_size();
rolesIndex < last_role; rolesIndex < last_role;
++rolesIndex ++rolesIndex)
) { {
const std::string & role = threadData->PBFprimitiveBlock.stringtable().s( inputRelation.roles_sid( rolesIndex ) ); const std::string &role = thread_data->PBFprimitiveBlock.stringtable().s(
inputRelation.roles_sid(rolesIndex));
lastRef += inputRelation.memids(rolesIndex); lastRef += inputRelation.memids(rolesIndex);
if(!("from" == role || "to" == role || "via" == role)) { if (!("from" == role || "to" == role || "via" == role))
{
continue; continue;
} }
switch(inputRelation.types(rolesIndex)) { switch (inputRelation.types(rolesIndex))
{
case 0: // node case 0: // node
if("from" == role || "to" == role) { //Only via should be a node if ("from" == role || "to" == role)
{ // Only via should be a node
continue; continue;
} }
assert("via" == role); assert("via" == role);
if(UINT_MAX != currentRestrictionContainer.viaNode) { if (UINT_MAX != currentRestrictionContainer.viaNode)
{
currentRestrictionContainer.viaNode = UINT_MAX; currentRestrictionContainer.viaNode = UINT_MAX;
} }
assert(UINT_MAX == currentRestrictionContainer.viaNode); assert(UINT_MAX == currentRestrictionContainer.viaNode);
@ -302,13 +364,16 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
break; break;
case 1: // way case 1: // way
assert("from" == role || "to" == role || "via" == role); assert("from" == role || "to" == role || "via" == role);
if("from" == role) { if ("from" == role)
{
currentRestrictionContainer.fromWay = lastRef; currentRestrictionContainer.fromWay = lastRef;
} }
if ("to" == role) { if ("to" == role)
{
currentRestrictionContainer.toWay = lastRef; currentRestrictionContainer.toWay = lastRef;
} }
if ("via" == role) { if ("via" == role)
{
assert(currentRestrictionContainer.restriction.toNode == UINT_MAX); assert(currentRestrictionContainer.restriction.toNode == UINT_MAX);
currentRestrictionContainer.viaNode = lastRef; currentRestrictionContainer.viaNode = lastRef;
} }
@ -323,30 +388,39 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
break; break;
} }
} }
if(!extractor_callbacks->restrictionFunction(currentRestrictionContainer)) { if (!extractor_callbacks->ProcessRestriction(currentRestrictionContainer))
{
std::cerr << "[PBFParser] relation not parsed" << std::endl; std::cerr << "[PBFParser] relation not parsed" << std::endl;
} }
} }
} }
} }
inline void PBFParser::parseWay(_ThreadData * threadData) { inline void PBFParser::parseWay(ParserThreadData *thread_data)
const int number_of_ways = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).ways_size(); {
const int number_of_ways =
thread_data->PBFprimitiveBlock.primitivegroup(thread_data->currentGroupID).ways_size();
std::vector<ExtractionWay> parsed_way_vector(number_of_ways); std::vector<ExtractionWay> parsed_way_vector(number_of_ways);
for(int i = 0; i < number_of_ways; ++i) { for (int i = 0; i < number_of_ways; ++i)
const OSMPBF::Way& inputWay = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).ways( i ); {
const OSMPBF::Way &inputWay =
thread_data->PBFprimitiveBlock.primitivegroup(thread_data->currentGroupID).ways(i);
parsed_way_vector[i].id = inputWay.id(); parsed_way_vector[i].id = inputWay.id();
unsigned pathNode(0); unsigned pathNode(0);
const int number_of_referenced_nodes = inputWay.refs_size(); const int number_of_referenced_nodes = inputWay.refs_size();
for(int j = 0; j < number_of_referenced_nodes; ++j) { for (int j = 0; j < number_of_referenced_nodes; ++j)
{
pathNode += inputWay.refs(j); pathNode += inputWay.refs(j);
parsed_way_vector[i].path.push_back(pathNode); parsed_way_vector[i].path.push_back(pathNode);
} }
assert(inputWay.keys_size() == inputWay.vals_size()); assert(inputWay.keys_size() == inputWay.vals_size());
const int number_of_keys = inputWay.keys_size(); const int number_of_keys = inputWay.keys_size();
for(int j = 0; j < number_of_keys; ++j) { for (int j = 0; j < number_of_keys; ++j)
const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(inputWay.keys(j)); {
const std::string & val = threadData->PBFprimitiveBlock.stringtable().s(inputWay.vals(j)); const std::string &key =
thread_data->PBFprimitiveBlock.stringtable().s(inputWay.keys(j));
const std::string &val =
thread_data->PBFprimitiveBlock.stringtable().s(inputWay.vals(j));
parsed_way_vector[i].keyVals.emplace(key, val); parsed_way_vector[i].keyVals.emplace(key, val);
} }
} }
@ -357,10 +431,8 @@ inline void PBFParser::parseWay(_ThreadData * threadData) {
ExtractionWay &extraction_way = parsed_way_vector[i]; ExtractionWay &extraction_way = parsed_way_vector[i];
if (2 <= extraction_way.path.size()) if (2 <= extraction_way.path.size())
{ {
ParseWayInLua( ParseWayInLua(extraction_way,
extraction_way, scripting_environment.getLuaStateForThreadID(omp_get_thread_num()));
scripting_environment.getLuaStateForThreadID(omp_get_thread_num())
);
} }
} }
@ -368,78 +440,91 @@ inline void PBFParser::parseWay(_ThreadData * threadData) {
{ {
if (2 <= extraction_way.path.size()) if (2 <= extraction_way.path.size())
{ {
extractor_callbacks->wayFunction(extraction_way); extractor_callbacks->ProcessWay(extraction_way);
} }
} }
} }
inline void PBFParser::loadGroup(_ThreadData * threadData) { inline void PBFParser::loadGroup(ParserThreadData *thread_data)
{
#ifndef NDEBUG #ifndef NDEBUG
++groupCount; ++group_count;
#endif #endif
const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ); const OSMPBF::PrimitiveGroup &group =
threadData->entityTypeIndicator = TypeDummy; thread_data->PBFprimitiveBlock.primitivegroup(thread_data->currentGroupID);
if ( 0 != group.nodes_size() ) { thread_data->entityTypeIndicator = TypeDummy;
threadData->entityTypeIndicator = TypeNode; if (0 != group.nodes_size())
{
thread_data->entityTypeIndicator = TypeNode;
} }
if ( 0 != group.ways_size() ) { if (0 != group.ways_size())
threadData->entityTypeIndicator = TypeWay; {
thread_data->entityTypeIndicator = TypeWay;
} }
if ( 0 != group.relations_size() ) { if (0 != group.relations_size())
threadData->entityTypeIndicator = TypeRelation; {
thread_data->entityTypeIndicator = TypeRelation;
} }
if ( group.has_dense() ) { if (group.has_dense())
threadData->entityTypeIndicator = TypeDenseNode; {
thread_data->entityTypeIndicator = TypeDenseNode;
assert(0 != group.dense().id_size()); assert(0 != group.dense().id_size());
} }
assert( threadData->entityTypeIndicator != TypeDummy ); assert(thread_data->entityTypeIndicator != TypeDummy);
} }
inline void PBFParser::loadBlock(_ThreadData * threadData) { inline void PBFParser::loadBlock(ParserThreadData *thread_data)
++blockCount; {
threadData->currentGroupID = 0; ++block_count;
threadData->currentEntityID = 0; thread_data->currentGroupID = 0;
thread_data->currentEntityID = 0;
} }
inline bool PBFParser::readPBFBlobHeader(std::fstream& stream, _ThreadData * threadData) { inline bool PBFParser::readPBFBlobHeader(std::fstream &stream, ParserThreadData *thread_data)
{
int size(0); int size(0);
stream.read((char *)&size, sizeof(int)); stream.read((char *)&size, sizeof(int));
size = SwapEndian(size); size = SwapEndian(size);
if(stream.eof()) { if (stream.eof())
{
return false; return false;
} }
if ( size > MAX_BLOB_HEADER_SIZE || size < 0 ) { if (size > MAX_BLOB_HEADER_SIZE || size < 0)
{
return false; return false;
} }
char *data = new char[size]; char *data = new char[size];
stream.read(data, size * sizeof(data[0])); stream.read(data, size * sizeof(data[0]));
bool dataSuccessfullyParsed = (threadData->PBFBlobHeader).ParseFromArray( data, size); bool dataSuccessfullyParsed = (thread_data->PBFBlobHeader).ParseFromArray(data, size);
delete[] data; delete[] data;
return dataSuccessfullyParsed; return dataSuccessfullyParsed;
} }
inline bool PBFParser::unpackZLIB(std::fstream &, _ThreadData * threadData) { inline bool PBFParser::unpackZLIB(std::fstream &, ParserThreadData *thread_data)
unsigned rawSize = threadData->PBFBlob.raw_size(); {
unsigned rawSize = thread_data->PBFBlob.raw_size();
char *unpackedDataArray = new char[rawSize]; char *unpackedDataArray = new char[rawSize];
z_stream compressedDataStream; z_stream compressedDataStream;
compressedDataStream.next_in = ( unsigned char* ) threadData->PBFBlob.zlib_data().data(); compressedDataStream.next_in = (unsigned char *)thread_data->PBFBlob.zlib_data().data();
compressedDataStream.avail_in = threadData->PBFBlob.zlib_data().size(); compressedDataStream.avail_in = thread_data->PBFBlob.zlib_data().size();
compressedDataStream.next_out = (unsigned char *)unpackedDataArray; compressedDataStream.next_out = (unsigned char *)unpackedDataArray;
compressedDataStream.avail_out = rawSize; compressedDataStream.avail_out = rawSize;
compressedDataStream.zalloc = Z_NULL; compressedDataStream.zalloc = Z_NULL;
compressedDataStream.zfree = Z_NULL; compressedDataStream.zfree = Z_NULL;
compressedDataStream.opaque = Z_NULL; compressedDataStream.opaque = Z_NULL;
int ret = inflateInit(&compressedDataStream); int ret = inflateInit(&compressedDataStream);
if ( ret != Z_OK ) { if (ret != Z_OK)
{
std::cerr << "[error] failed to init zlib stream" << std::endl; std::cerr << "[error] failed to init zlib stream" << std::endl;
delete[] unpackedDataArray; delete[] unpackedDataArray;
return false; return false;
} }
ret = inflate(&compressedDataStream, Z_FINISH); ret = inflate(&compressedDataStream, Z_FINISH);
if ( ret != Z_STREAM_END ) { if (ret != Z_STREAM_END)
{
std::cerr << "[error] failed to inflate zlib stream" << std::endl; std::cerr << "[error] failed to inflate zlib stream" << std::endl;
std::cerr << "[error] Error type: " << ret << std::endl; std::cerr << "[error] Error type: " << ret << std::endl;
delete[] unpackedDataArray; delete[] unpackedDataArray;
@ -447,29 +532,32 @@ inline bool PBFParser::unpackZLIB(std::fstream &, _ThreadData * threadData) {
} }
ret = inflateEnd(&compressedDataStream); ret = inflateEnd(&compressedDataStream);
if ( ret != Z_OK ) { if (ret != Z_OK)
{
std::cerr << "[error] failed to deinit zlib stream" << std::endl; std::cerr << "[error] failed to deinit zlib stream" << std::endl;
delete[] unpackedDataArray; delete[] unpackedDataArray;
return false; return false;
} }
threadData->charBuffer.clear(); threadData->charBuffer.resize(rawSize); thread_data->charBuffer.clear();
std::copy(unpackedDataArray, unpackedDataArray + rawSize, threadData->charBuffer.begin()); thread_data->charBuffer.resize(rawSize);
std::copy(unpackedDataArray, unpackedDataArray + rawSize, thread_data->charBuffer.begin());
delete[] unpackedDataArray; delete[] unpackedDataArray;
return true; return true;
} }
inline bool PBFParser::unpackLZMA(std::fstream &, _ThreadData * ) { inline bool PBFParser::unpackLZMA(std::fstream &, ParserThreadData *) { return false; }
inline bool PBFParser::readBlob(std::fstream &stream, ParserThreadData *thread_data)
{
if (stream.eof())
{
return false; return false;
} }
inline bool PBFParser::readBlob(std::fstream& stream, _ThreadData * threadData) { const int size = thread_data->PBFBlobHeader.datasize();
if(stream.eof()) { if (size < 0 || size > MAX_BLOB_SIZE)
return false; {
}
const int size = threadData->PBFBlobHeader.datasize();
if ( size < 0 || size > MAX_BLOB_SIZE ) {
std::cerr << "[error] invalid Blob size:" << size << std::endl; std::cerr << "[error] invalid Blob size:" << size << std::endl;
return false; return false;
} }
@ -477,30 +565,40 @@ inline bool PBFParser::readBlob(std::fstream& stream, _ThreadData * threadData)
char *data = new char[size]; char *data = new char[size];
stream.read(data, sizeof(data[0]) * size); stream.read(data, sizeof(data[0]) * size);
if ( !threadData->PBFBlob.ParseFromArray( data, size ) ) { if (!thread_data->PBFBlob.ParseFromArray(data, size))
{
std::cerr << "[error] failed to parse blob" << std::endl; std::cerr << "[error] failed to parse blob" << std::endl;
delete[] data; delete[] data;
return false; return false;
} }
if ( threadData->PBFBlob.has_raw() ) { if (thread_data->PBFBlob.has_raw())
const std::string& data = threadData->PBFBlob.raw(); {
threadData->charBuffer.clear(); const std::string &data = thread_data->PBFBlob.raw();
threadData->charBuffer.resize( data.size() ); thread_data->charBuffer.clear();
std::copy(data.begin(), data.end(), threadData->charBuffer.begin()); thread_data->charBuffer.resize(data.size());
} else if ( threadData->PBFBlob.has_zlib_data() ) { std::copy(data.begin(), data.end(), thread_data->charBuffer.begin());
if ( !unpackZLIB(stream, threadData) ) { }
else if (thread_data->PBFBlob.has_zlib_data())
{
if (!unpackZLIB(stream, thread_data))
{
std::cerr << "[error] zlib data encountered that could not be unpacked" << std::endl; std::cerr << "[error] zlib data encountered that could not be unpacked" << std::endl;
delete[] data; delete[] data;
return false; return false;
} }
} else if ( threadData->PBFBlob.has_lzma_data() ) { }
if ( !unpackLZMA(stream, threadData) ) { else if (thread_data->PBFBlob.has_lzma_data())
{
if (!unpackLZMA(stream, thread_data))
{
std::cerr << "[error] lzma data encountered that could not be unpacked" << std::endl; std::cerr << "[error] lzma data encountered that could not be unpacked" << std::endl;
} }
delete[] data; delete[] data;
return false; return false;
} else { }
else
{
std::cerr << "[error] Blob contains no data" << std::endl; std::cerr << "[error] Blob contains no data" << std::endl;
delete[] data; delete[] data;
return false; return false;
@ -509,24 +607,31 @@ inline bool PBFParser::readBlob(std::fstream& stream, _ThreadData * threadData)
return true; return true;
} }
bool PBFParser::readNextBlock(std::fstream& stream, _ThreadData * threadData) { bool PBFParser::readNextBlock(std::fstream &stream, ParserThreadData *thread_data)
if(stream.eof()) { {
if (stream.eof())
{
return false; return false;
} }
if ( !readPBFBlobHeader(stream, threadData) ){ if (!readPBFBlobHeader(stream, thread_data))
{
return false; return false;
} }
if ( threadData->PBFBlobHeader.type() != "OSMData" ) { if (thread_data->PBFBlobHeader.type() != "OSMData")
{
return false; return false;
} }
if ( !readBlob(stream, threadData) ) { if (!readBlob(stream, thread_data))
{
return false; return false;
} }
if ( !threadData->PBFprimitiveBlock.ParseFromArray( &(threadData->charBuffer[0]), threadData-> charBuffer.size() ) ) { if (!thread_data->PBFprimitiveBlock.ParseFromArray(&(thread_data->charBuffer[0]),
thread_data->charBuffer.size()))
{
std::cerr << "failed to parse PrimitiveBlock" << std::endl; std::cerr << "failed to parse PrimitiveBlock" << std::endl;
return false; return false;
} }

50
Extractor/PBFParser.h
View File

@ -31,24 +31,24 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "BaseParser.h" #include "BaseParser.h"
#include "../DataStructures/ConcurrentQueue.h" #include "../DataStructures/ConcurrentQueue.h"
#include <boost/shared_ptr.hpp>
#include <osmpbf/fileformat.pb.h> #include <osmpbf/fileformat.pb.h>
#include <osmpbf/osmformat.pb.h> #include <osmpbf/osmformat.pb.h>
#include <fstream> #include <fstream>
#include <memory>
class PBFParser : public BaseParser { class PBFParser : public BaseParser
{
enum EntityType { enum EntityType
TypeDummy = 0, { TypeDummy = 0,
TypeNode = 1, TypeNode = 1,
TypeWay = 2, TypeWay = 2,
TypeRelation = 4, TypeRelation = 4,
TypeDenseNode = 8 TypeDenseNode = 8 };
};
struct _ThreadData { struct ParserThreadData
{
int currentGroupID; int currentGroupID;
int currentEntityID; int currentEntityID;
EntityType entityTypeIndicator; EntityType entityTypeIndicator;
@ -63,11 +63,7 @@ class PBFParser : public BaseParser {
}; };
public: public:
PBFParser( PBFParser(const char *file_name, ExtractorCallbacks *extractor_callbacks, ScriptingEnvironment &scripting_environment);
const char * fileName,
ExtractorCallbacks* ec,
ScriptingEnvironment& se
);
virtual ~PBFParser(); virtual ~PBFParser();
inline bool ReadHeader(); inline bool ReadHeader();
@ -76,28 +72,28 @@ public:
private: private:
inline void ReadData(); inline void ReadData();
inline void ParseData(); inline void ParseData();
inline void parseDenseNode (_ThreadData * threadData); inline void parseDenseNode(ParserThreadData *thread_data);
inline void parseNode (_ThreadData * threadData); inline void parseNode(ParserThreadData *thread_data);
inline void parseRelation (_ThreadData * threadData); inline void parseRelation(ParserThreadData *thread_data);
inline void parseWay (_ThreadData * threadData); inline void parseWay(ParserThreadData *thread_data);
inline void loadGroup (_ThreadData * threadData); inline void loadGroup(ParserThreadData *thread_data);
inline void loadBlock (_ThreadData * threadData); inline void loadBlock(ParserThreadData *thread_data);
inline bool readPBFBlobHeader(std::fstream & stream, _ThreadData * threadData); inline bool readPBFBlobHeader(std::fstream &stream, ParserThreadData *thread_data);
inline bool unpackZLIB (std::fstream & stream, _ThreadData * threadData); inline bool unpackZLIB(std::fstream &stream, ParserThreadData *thread_data);
inline bool unpackLZMA (std::fstream & stream, _ThreadData * threadData); inline bool unpackLZMA(std::fstream &stream, ParserThreadData *thread_data);
inline bool readBlob (std::fstream & stream, _ThreadData * threadData); inline bool readBlob(std::fstream &stream, ParserThreadData *thread_data);
inline bool readNextBlock (std::fstream & stream, _ThreadData * threadData); inline bool readNextBlock(std::fstream &stream, ParserThreadData *thread_data);
static const int NANO = 1000 * 1000 * 1000; static const int NANO = 1000 * 1000 * 1000;
static const int MAX_BLOB_HEADER_SIZE = 64 * 1024; static const int MAX_BLOB_HEADER_SIZE = 64 * 1024;
static const int MAX_BLOB_SIZE = 32 * 1024 * 1024; static const int MAX_BLOB_SIZE = 32 * 1024 * 1024;
unsigned groupCount; unsigned group_count;
unsigned blockCount; unsigned block_count;
std::fstream input; // the input stream to parse std::fstream input; // the input stream to parse
boost::shared_ptr<ConcurrentQueue < _ThreadData* > > threadDataQueue; std::shared_ptr<ConcurrentQueue<ParserThreadData *>> thread_data_queue;
}; };
#endif /* PBFPARSER_H_ */ #endif /* PBFPARSER_H_ */

62
Extractor/ScriptingEnvironment.cpp
View File

@ -37,51 +37,49 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "../typedefs.h" #include "../typedefs.h"
ScriptingEnvironment::ScriptingEnvironment() {} ScriptingEnvironment::ScriptingEnvironment() {}
ScriptingEnvironment::ScriptingEnvironment(const char * fileName) { ScriptingEnvironment::ScriptingEnvironment(const char *file_name)
SimpleLogger().Write() << "Using script " << fileName; {
SimpleLogger().Write() << "Using script " << file_name;
// Create a new lua state // Create a new lua state
for(int i = 0; i < omp_get_max_threads(); ++i) { for (int i = 0; i < omp_get_max_threads(); ++i)
luaStateVector.push_back(luaL_newstate()); {
lua_state_vector.push_back(luaL_newstate());
} }
// Connect LuaBind to this lua state for all threads // Connect LuaBind to this lua state for all threads
#pragma omp parallel #pragma omp parallel
{ {
lua_State * myLuaState = getLuaStateForThreadID(omp_get_thread_num()); lua_State *lua_state = getLuaStateForThreadID(omp_get_thread_num());
luabind::open(myLuaState); luabind::open(lua_state);
// open utility libraries string library; // open utility libraries string library;
luaL_openlibs(myLuaState); luaL_openlibs(lua_state);
luaAddScriptFolderToLoadPath( myLuaState, fileName ); luaAddScriptFolderToLoadPath(lua_state, file_name);
// Add our function to the state's global scope // Add our function to the state's global scope
luabind::module(myLuaState) [ luabind::module(lua_state)[
luabind::def("print", LUA_print<std::string>), luabind::def("print", LUA_print<std::string>),
luabind::def("durationIsValid", durationIsValid), luabind::def("durationIsValid", durationIsValid),
luabind::def("parseDuration", parseDuration) luabind::def("parseDuration", parseDuration)
]; ];
luabind::module(myLuaState) [ luabind::module(lua_state)[luabind::class_<HashTable<std::string, std::string>>("keyVals")
luabind::class_<HashTable<std::string, std::string> >("keyVals")
.def("Add", &HashTable<std::string, std::string>::Add) .def("Add", &HashTable<std::string, std::string>::Add)
.def("Find", &HashTable<std::string, std::string>::Find) .def("Find", &HashTable<std::string, std::string>::Find)
.def("Holds", &HashTable<std::string, std::string>::Holds) .def("Holds", &HashTable<std::string, std::string>::Holds)];
];
luabind::module(myLuaState) [ luabind::module(lua_state)[luabind::class_<ImportNode>("Node")
luabind::class_<ImportNode>("Node")
.def(luabind::constructor<>()) .def(luabind::constructor<>())
.def_readwrite("lat", &ImportNode::lat) .def_readwrite("lat", &ImportNode::lat)
.def_readwrite("lon", &ImportNode::lon) .def_readwrite("lon", &ImportNode::lon)
.def_readonly("id", &ImportNode::id) .def_readonly("id", &ImportNode::id)
.def_readwrite("bollard", &ImportNode::bollard) .def_readwrite("bollard", &ImportNode::bollard)
.def_readwrite("traffic_light", &ImportNode::trafficLight) .def_readwrite("traffic_light", &ImportNode::trafficLight)
.def_readwrite("tags", &ImportNode::keyVals) .def_readwrite("tags", &ImportNode::keyVals)];
];
luabind::module(myLuaState) [ luabind::module(lua_state)
luabind::class_<ExtractionWay>("Way") [luabind::class_<ExtractionWay>("Way")
.def(luabind::constructor<>()) .def(luabind::constructor<>())
.def_readonly("id", &ExtractionWay::id) .def_readonly("id", &ExtractionWay::id)
.def_readwrite("name", &ExtractionWay::name) .def_readwrite("name", &ExtractionWay::name)
@ -100,27 +98,27 @@ ScriptingEnvironment::ScriptingEnvironment(const char * fileName) {
luabind::value("oneway", 1), luabind::value("oneway", 1),
luabind::value("bidirectional", 2), luabind::value("bidirectional", 2),
luabind::value("opposite", 3) luabind::value("opposite", 3)
] ]];
];
// fails on c++11/OS X 10.9 // fails on c++11/OS X 10.9
luabind::module(myLuaState) [ luabind::module(lua_state)[luabind::class_<std::vector<std::string>>("vector").def(
luabind::class_<std::vector<std::string> >("vector") "Add",
.def("Add", static_cast<void (std::vector<std::string>::*)(const std::string&)>(&std::vector<std::string>::push_back)) static_cast<void (std::vector<std::string>::*)(const std::string &)>(
]; &std::vector<std::string>::push_back))];
if(0 != luaL_dofile(myLuaState, fileName) ) { if (0 != luaL_dofile(lua_state, file_name))
{
throw OSRMException("ERROR occured in scripting block"); throw OSRMException("ERROR occured in scripting block");
} }
} }
} }
ScriptingEnvironment::~ScriptingEnvironment() { ScriptingEnvironment::~ScriptingEnvironment()
for(unsigned i = 0; i < luaStateVector.size(); ++i) { {
// luaStateVector[i]; for (unsigned i = 0; i < lua_state_vector.size(); ++i)
{
// lua_state_vector[i];
} }
} }
lua_State * ScriptingEnvironment::getLuaStateForThreadID(const int id) { lua_State *ScriptingEnvironment::getLuaStateForThreadID(const int id) { return lua_state_vector[id]; }
return luaStateVector[id];
}

7
Extractor/ScriptingEnvironment.h
View File

@ -32,15 +32,16 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
struct lua_State; struct lua_State;
class ScriptingEnvironment { class ScriptingEnvironment
{
public: public:
ScriptingEnvironment(); ScriptingEnvironment();
explicit ScriptingEnvironment(const char * fileName); explicit ScriptingEnvironment(const char *file_name);
virtual ~ScriptingEnvironment(); virtual ~ScriptingEnvironment();
lua_State *getLuaStateForThreadID(const int); lua_State *getLuaStateForThreadID(const int);
std::vector<lua_State *> luaStateVector; std::vector<lua_State *> lua_state_vector;
}; };
#endif /* SCRIPTINGENVIRONMENT_H_ */ #endif /* SCRIPTINGENVIRONMENT_H_ */

9
Extractor/XMLParser.cpp
View File

@ -71,19 +71,19 @@ bool XMLParser::Parse()
{ {
ImportNode n = ReadXMLNode(); ImportNode n = ReadXMLNode();
ParseNodeInLua(n, lua_state); ParseNodeInLua(n, lua_state);
extractor_callbacks->nodeFunction(n); extractor_callbacks->ProcessNode(n);
} }
if (xmlStrEqual(currentName, (const xmlChar *)"way") == 1) if (xmlStrEqual(currentName, (const xmlChar *)"way") == 1)
{ {
ExtractionWay way = ReadXMLWay(); ExtractionWay way = ReadXMLWay();
ParseWayInLua(way, lua_state); ParseWayInLua(way, lua_state);
extractor_callbacks->wayFunction(way); extractor_callbacks->ProcessWay(way);
} }
if (use_turn_restrictions && xmlStrEqual(currentName, (const xmlChar *)"relation") == 1) if (use_turn_restrictions && xmlStrEqual(currentName, (const xmlChar *)"relation") == 1)
{ {
InputRestrictionContainer r = ReadXMLRestriction(); InputRestrictionContainer r = ReadXMLRestriction();
if ((UINT_MAX != r.fromWay) && !extractor_callbacks->restrictionFunction(r)) if ((UINT_MAX != r.fromWay) && !extractor_callbacks->ProcessRestriction(r))
{ {
std::cerr << "[XMLParser] restriction not parsed" << std::endl; std::cerr << "[XMLParser] restriction not parsed" << std::endl;
} }
@ -328,8 +328,7 @@ ImportNode XMLParser::ReadXMLNode()
xmlChar *value = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"v"); xmlChar *value = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"v");
if (k != NULL && value != NULL) if (k != NULL && value != NULL)
{ {
node.keyVals.emplace(std::string((char *)(k)), node.keyVals.emplace(std::string((char *)(k)), std::string((char *)(value)));
std::string((char *)(value)));
} }
if (k != NULL) if (k != NULL)
{ {

8
extractor.cpp
View File

@ -221,7 +221,7 @@ int main(int argc, char *argv[])
ExtractionContainers extraction_containers; ExtractionContainers extraction_containers;
string_map[""] = 0; string_map[""] = 0;
extractor_callbacks = new ExtractorCallbacks(&extraction_containers, &string_map); extractor_callbacks = new ExtractorCallbacks(extraction_containers, string_map);
BaseParser *parser; BaseParser *parser;
if (file_has_pbf_format) if (file_has_pbf_format)
{ {
@ -241,6 +241,9 @@ int main(int argc, char *argv[])
std::chrono::steady_clock::now(); std::chrono::steady_clock::now();
parser->Parse(); parser->Parse();
delete parser;
delete extractor_callbacks;
std::chrono::duration<double> parsing_duration = std::chrono::duration<double> parsing_duration =
std::chrono::steady_clock::now() - parsing_start_time; std::chrono::steady_clock::now() - parsing_start_time;
SimpleLogger().Write() << "Parsing finished after " << parsing_duration.count() SimpleLogger().Write() << "Parsing finished after " << parsing_duration.count()
@ -254,9 +257,6 @@ int main(int argc, char *argv[])
extraction_containers.PrepareData(output_file_name, restriction_fileName); extraction_containers.PrepareData(output_file_name, restriction_fileName);
delete parser;
delete extractor_callbacks;
std::chrono::duration<double> extraction_duration = std::chrono::duration<double> extraction_duration =
std::chrono::steady_clock::now() - startup_time; std::chrono::steady_clock::now() - startup_time;
SimpleLogger().Write() << "extraction finished after " << extraction_duration.count() SimpleLogger().Write() << "extraction finished after " << extraction_duration.count()