Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

refactor Extractor/

Browse Source
This commit is contained in:
Dennis Luxen 2014-05-09 16:17:31 +02:00
parent 7e639d6bc1
commit 0fccd0f0d2
16 changed files with 1171 additions and 1193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
Extractor/BaseParser.cpp
View File

@ -38,99 +38,99 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <boost/algorithm/string/regex.hpp> #include <boost/algorithm/string/regex.hpp>
#include <boost/regex.hpp> #include <boost/regex.hpp>
BaseParser::BaseParser( BaseParser::BaseParser(ExtractorCallbacks *extractor_callbacks,
ExtractorCallbacks * extractor_callbacks, ScriptingEnvironment &scripting_environment)
ScriptingEnvironment & scripting_environment : extractor_callbacks(extractor_callbacks),
) : extractor_callbacks(extractor_callbacks), lua_state(scripting_environment.getLuaStateForThreadID(0)),
lua_state(scripting_environment.getLuaStateForThreadID(0)), scripting_environment(scripting_environment), use_turn_restrictions(true)
scripting_environment(scripting_environment),
use_turn_restrictions(true)
{ {
ReadUseRestrictionsSetting(); ReadUseRestrictionsSetting();
ReadRestrictionExceptions(); ReadRestrictionExceptions();
} }
void BaseParser::ReadUseRestrictionsSetting() { void BaseParser::ReadUseRestrictionsSetting()
if( 0 != luaL_dostring( lua_state, "return use_turn_restrictions\n") ) { {
if (0 != luaL_dostring(lua_state, "return use_turn_restrictions\n"))
{
throw OSRMException("ERROR occured in scripting block"); throw OSRMException("ERROR occured in scripting block");
} }
if( lua_isboolean( lua_state, -1) ) { if (lua_isboolean(lua_state, -1))
{
use_turn_restrictions = lua_toboolean(lua_state, -1); use_turn_restrictions = lua_toboolean(lua_state, -1);
} }
if( use_turn_restrictions ) { if (use_turn_restrictions)
{
SimpleLogger().Write() << "Using turn restrictions"; SimpleLogger().Write() << "Using turn restrictions";
} else { }
else
{
SimpleLogger().Write() << "Ignoring turn restrictions"; SimpleLogger().Write() << "Ignoring turn restrictions";
} }
} }
void BaseParser::ReadRestrictionExceptions() { void BaseParser::ReadRestrictionExceptions()
if(lua_function_exists(lua_state, "get_exceptions" )) { {
//get list of turn restriction exceptions if (lua_function_exists(lua_state, "get_exceptions"))
{
// get list of turn restriction exceptions
luabind::call_function<void>( luabind::call_function<void>(
lua_state, lua_state, "get_exceptions", boost::ref(restriction_exceptions));
"get_exceptions",
boost::ref(restriction_exceptions)
);
const unsigned exception_count = restriction_exceptions.size(); const unsigned exception_count = restriction_exceptions.size();
SimpleLogger().Write() << SimpleLogger().Write() << "Found " << exception_count
"Found " << exception_count << " exceptions to turn restrictions:"; << " exceptions to turn restrictions:";
for(const std::string & str : restriction_exceptions) { for (const std::string &str : restriction_exceptions)
{
SimpleLogger().Write() << " " << str; SimpleLogger().Write() << " " << str;
} }
} else { }
else
{
SimpleLogger().Write() << "Found no exceptions to turn restrictions"; SimpleLogger().Write() << "Found no exceptions to turn restrictions";
} }
} }
void BaseParser::report_errors(lua_State *L, const int status) const { void BaseParser::report_errors(lua_State *lua_state, const int status) const
if( 0!=status ) { {
std::cerr << "-- " << lua_tostring(L, -1) << std::endl; if (0 != status)
lua_pop(L, 1); // remove error message {
std::cerr << "-- " << lua_tostring(lua_state, -1) << std::endl;
lua_pop(lua_state, 1); // remove error message
} }
} }
void BaseParser::ParseNodeInLua(ImportNode& n, lua_State* local_lua_state) { void BaseParser::ParseNodeInLua(ImportNode &node, lua_State *local_lua_state)
luabind::call_function<void>( {
local_lua_state, luabind::call_function<void>(local_lua_state, "node_function", boost::ref(node));
"node_function",
boost::ref(n)
);
} }
void BaseParser::ParseWayInLua(ExtractionWay& w, lua_State* local_lua_state) { void BaseParser::ParseWayInLua(ExtractionWay &way, lua_State *local_lua_state)
luabind::call_function<void>( {
local_lua_state, luabind::call_function<void>(local_lua_state, "way_function", boost::ref(way));
"way_function",
boost::ref(w)
);
} }
bool BaseParser::ShouldIgnoreRestriction( bool BaseParser::ShouldIgnoreRestriction(const std::string &except_tag_string) const
const std::string & except_tag_string {
) const { // should this restriction be ignored? yes if there's an overlap between:
//should this restriction be ignored? yes if there's an overlap between:
// a) the list of modes in the except tag of the restriction // a) the list of modes in the except tag of the restriction
// (except_tag_string), eg: except=bus;bicycle // (except_tag_string), eg: except=bus;bicycle
// b) the lua profile defines a hierachy of modes, // b) the lua profile defines a hierachy of modes,
// eg: [access, vehicle, bicycle] // eg: [access, vehicle, bicycle]
if( except_tag_string.empty() ) { if (except_tag_string.empty())
{
return false; return false;
} }
//Be warned, this is quadratic work here, but we assume that // Be warned, this is quadratic work here, but we assume that
//only a few exceptions are actually defined. // only a few exceptions are actually defined.
std::vector<std::string> exceptions; std::vector<std::string> exceptions;
boost::algorithm::split_regex(exceptions, except_tag_string, boost::regex("[;][ ]*")); boost::algorithm::split_regex(exceptions, except_tag_string, boost::regex("[;][ ]*"));
for (std::string& current_string : exceptions) { for (std::string &current_string : exceptions)
std::vector<std::string>::const_iterator string_iterator; {
string_iterator = std::find( const auto string_iterator =
restriction_exceptions.begin(), std::find(restriction_exceptions.begin(), restriction_exceptions.end(), current_string);
restriction_exceptions.end(), if (restriction_exceptions.end() != string_iterator)
current_string {
);
if( restriction_exceptions.end() != string_iterator ) {
return true; return true;
} }
} }

31
Extractor/BaseParser.h
View File

@ -37,32 +37,29 @@ class ScriptingEnvironment;
struct ExtractionWay; struct ExtractionWay;
struct ImportNode; struct ImportNode;
class BaseParser { class BaseParser
public: {
public:
BaseParser() = delete; BaseParser() = delete;
BaseParser(const BaseParser&) = delete; BaseParser(const BaseParser &) = delete;
BaseParser( BaseParser(ExtractorCallbacks *extractor_callbacks,
ExtractorCallbacks * extractor_callbacks, ScriptingEnvironment &scripting_environment);
ScriptingEnvironment & scripting_environment
);
virtual ~BaseParser() {} virtual ~BaseParser() {}
virtual bool ReadHeader() = 0; virtual bool ReadHeader() = 0;
virtual bool Parse() = 0; virtual bool Parse() = 0;
virtual void ParseNodeInLua(ImportNode & n, lua_State* thread_lua_state); virtual void ParseNodeInLua(ImportNode &node, lua_State *lua_state);
virtual void ParseWayInLua(ExtractionWay & n, lua_State* thread_lua_state); virtual void ParseWayInLua(ExtractionWay &way, lua_State *lua_state);
virtual void report_errors(lua_State * lua_state, const int status) const; virtual void report_errors(lua_State *lua_state, const int status) const;
protected: protected:
virtual void ReadUseRestrictionsSetting(); virtual void ReadUseRestrictionsSetting();
virtual void ReadRestrictionExceptions(); virtual void ReadRestrictionExceptions();
virtual bool ShouldIgnoreRestriction( virtual bool ShouldIgnoreRestriction(const std::string &except_tag_string) const;
const std::string & except_tag_string
) const;
ExtractorCallbacks * extractor_callbacks; ExtractorCallbacks *extractor_callbacks;
lua_State * lua_state; lua_State *lua_state;
ScriptingEnvironment & scripting_environment; ScriptingEnvironment &scripting_environment;
std::vector<std::string> restriction_exceptions; std::vector<std::string> restriction_exceptions;
bool use_turn_restrictions; bool use_turn_restrictions;
}; };

438
Extractor/ExtractionContainers.cpp
View File

@ -38,13 +38,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <chrono> #include <chrono>
ExtractionContainers::ExtractionContainers() { ExtractionContainers::ExtractionContainers()
//Check if stxxl can be instantiated {
// Check if stxxl can be instantiated
stxxl::vector<unsigned> dummy_vector; stxxl::vector<unsigned> dummy_vector;
name_list.push_back(""); name_list.push_back("");
} }
ExtractionContainers::~ExtractionContainers() { ExtractionContainers::~ExtractionContainers()
{
used_node_id_list.clear(); used_node_id_list.clear();
all_nodes_list.clear(); all_nodes_list.clear();
all_edges_list.clear(); all_edges_list.clear();
@ -53,219 +55,205 @@ ExtractionContainers::~ExtractionContainers() {
way_start_end_id_list.clear(); way_start_end_id_list.clear();
} }
void ExtractionContainers::PrepareData( void ExtractionContainers::PrepareData(const std::string &output_file_name,
const std::string & output_file_name, const std::string &restrictions_file_name)
const std::string & restrictions_file_name {
) { try
try { {
unsigned number_of_used_nodes = 0; unsigned number_of_used_nodes = 0;
unsigned number_of_used_edges = 0; unsigned number_of_used_edges = 0;
std::chrono::time_point<std::chrono::steady_clock> time1 = std::chrono::steady_clock::now(); std::chrono::time_point<std::chrono::steady_clock> time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting used nodes ... " << std::flush; std::cout << "[extractor] Sorting used nodes ... " << std::flush;
stxxl::sort( stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), 4294967296);
used_node_id_list.begin(),
used_node_id_list.end(),
Cmp(),
4294967296
);
std::chrono::time_point<std::chrono::steady_clock> time2 = std::chrono::steady_clock::now(); std::chrono::time_point<std::chrono::steady_clock> time2 = std::chrono::steady_clock::now();
std::chrono::duration<double> elapsed_seconds = time2-time1; std::chrono::duration<double> elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now(); time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush; std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush;
stxxl::vector<NodeID>::iterator NewEnd = std::unique ( used_node_id_list.begin(),used_node_id_list.end() ) ; auto new_end = std::unique(used_node_id_list.begin(), used_node_id_list.end());
used_node_id_list.resize ( NewEnd - used_node_id_list.begin() ); used_node_id_list.resize(new_end - used_node_id_list.begin());
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now(); time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting all nodes ... " << std::flush; std::cout << "[extractor] Sorting all nodes ... " << std::flush;
stxxl::sort( stxxl::sort(all_nodes_list.begin(), all_nodes_list.end(), CmpNodeByID(), 4294967296);
all_nodes_list.begin(),
all_nodes_list.end(),
CmpNodeByID(),
4294967296
);
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now(); time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting used ways ... " << std::flush; std::cout << "[extractor] Sorting used ways ... " << std::flush;
stxxl::sort( stxxl::sort(
way_start_end_id_list.begin(), way_start_end_id_list.begin(), way_start_end_id_list.end(), CmpWayByID(), 4294967296);
way_start_end_id_list.end(),
CmpWayByID(),
4294967296
);
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] Sorting restrictions. by from... " << std::flush; std::cout << "[extractor] Sorting restrictions. by from... " << std::flush;
stxxl::sort( stxxl::sort(restrictions_list.begin(),
restrictions_list.begin(), restrictions_list.end(),
restrictions_list.end(), CmpRestrictionContainerByFrom(),
CmpRestrictionContainerByFrom(), 4294967296);
4294967296
);
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] Fixing restriction starts ... " << std::flush; std::cout << "[extractor] Fixing restriction starts ... " << std::flush;
STXXLRestrictionsVector::iterator restrictions_iterator = restrictions_list.begin(); auto restrictions_iterator = restrictions_list.begin();
STXXLWayIDStartEndVector::iterator way_start_and_end_iterator = way_start_end_id_list.begin(); auto way_start_and_end_iterator = way_start_end_id_list.begin();
while( while (way_start_and_end_iterator != way_start_end_id_list.end() &&
way_start_and_end_iterator != way_start_end_id_list.end() && restrictions_iterator != restrictions_list.end())
restrictions_iterator != restrictions_list.end() {
) { if (way_start_and_end_iterator->wayID < restrictions_iterator->fromWay)
{
if(way_start_and_end_iterator->wayID < restrictions_iterator->fromWay){
++way_start_and_end_iterator; ++way_start_and_end_iterator;
continue; continue;
} }
if(way_start_and_end_iterator->wayID > restrictions_iterator->fromWay) { if (way_start_and_end_iterator->wayID > restrictions_iterator->fromWay)
{
++restrictions_iterator; ++restrictions_iterator;
continue; continue;
} }
BOOST_ASSERT(way_start_and_end_iterator->wayID == restrictions_iterator->fromWay); BOOST_ASSERT(way_start_and_end_iterator->wayID == restrictions_iterator->fromWay);
NodeID via_node_id = restrictions_iterator->restriction.viaNode; const NodeID via_node_id = restrictions_iterator->restriction.viaNode;
if(way_start_and_end_iterator->firstStart == via_node_id) { if (way_start_and_end_iterator->firstStart == via_node_id)
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->firstTarget; {
} else if(way_start_and_end_iterator->firstTarget == via_node_id) { restrictions_iterator->restriction.fromNode =
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->firstStart; way_start_and_end_iterator->firstTarget;
} else if(way_start_and_end_iterator->lastStart == via_node_id) { }
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->lastTarget; else if (way_start_and_end_iterator->firstTarget == via_node_id)
} else if(way_start_and_end_iterator->lastTarget == via_node_id) { {
restrictions_iterator->restriction.fromNode =
way_start_and_end_iterator->firstStart;
}
else if (way_start_and_end_iterator->lastStart == via_node_id)
{
restrictions_iterator->restriction.fromNode =
way_start_and_end_iterator->lastTarget;
}
else if (way_start_and_end_iterator->lastTarget == via_node_id)
{
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->lastStart; restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->lastStart;
} }
++restrictions_iterator; ++restrictions_iterator;
} }
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now(); time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting restrictions. by to ... " << std::flush; std::cout << "[extractor] Sorting restrictions. by to ... " << std::flush;
stxxl::sort( stxxl::sort(restrictions_list.begin(),
restrictions_list.begin(), restrictions_list.end(),
restrictions_list.end(), CmpRestrictionContainerByTo(),
CmpRestrictionContainerByTo(), 4294967296);
4294967296
);
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now(); time1 = std::chrono::steady_clock::now();
unsigned usableRestrictionsCounter(0); unsigned number_of_useable_restrictions = 0;
std::cout << "[extractor] Fixing restriction ends ... " << std::flush; std::cout << "[extractor] Fixing restriction ends ... " << std::flush;
restrictions_iterator = restrictions_list.begin(); restrictions_iterator = restrictions_list.begin();
way_start_and_end_iterator = way_start_end_id_list.begin(); way_start_and_end_iterator = way_start_end_id_list.begin();
while( while (way_start_and_end_iterator != way_start_end_id_list.end() &&
way_start_and_end_iterator != way_start_end_id_list.end() && restrictions_iterator != restrictions_list.end())
restrictions_iterator != restrictions_list.end() {
) { if (way_start_and_end_iterator->wayID < restrictions_iterator->toWay)
if(way_start_and_end_iterator->wayID < restrictions_iterator->toWay){ {
++way_start_and_end_iterator; ++way_start_and_end_iterator;
continue; continue;
} }
if(way_start_and_end_iterator->wayID > restrictions_iterator->toWay) { if (way_start_and_end_iterator->wayID > restrictions_iterator->toWay)
{
++restrictions_iterator; ++restrictions_iterator;
continue; continue;
} }
NodeID via_node_id = restrictions_iterator->restriction.viaNode; NodeID via_node_id = restrictions_iterator->restriction.viaNode;
if(way_start_and_end_iterator->lastStart == via_node_id) { if (way_start_and_end_iterator->lastStart == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastTarget; restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastTarget;
} else if(way_start_and_end_iterator->lastTarget == via_node_id) { }
else if (way_start_and_end_iterator->lastTarget == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastStart; restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastStart;
} else if(way_start_and_end_iterator->firstStart == via_node_id) { }
else if (way_start_and_end_iterator->firstStart == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstTarget; restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstTarget;
} else if(way_start_and_end_iterator->firstTarget == via_node_id) { }
else if (way_start_and_end_iterator->firstTarget == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstStart; restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstStart;
} }
if( if (UINT_MAX != restrictions_iterator->restriction.fromNode &&
UINT_MAX != restrictions_iterator->restriction.fromNode && UINT_MAX != restrictions_iterator->restriction.toNode)
UINT_MAX != restrictions_iterator->restriction.toNode {
) { ++number_of_useable_restrictions;
++usableRestrictionsCounter;
} }
++restrictions_iterator; ++restrictions_iterator;
} }
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
SimpleLogger().Write() << "usable restrictions: " << usableRestrictionsCounter; SimpleLogger().Write() << "usable restrictions: " << number_of_useable_restrictions;
//serialize restrictions // serialize restrictions
std::ofstream restrictions_out_stream; std::ofstream restrictions_out_stream;
restrictions_out_stream.open(restrictions_file_name.c_str(), std::ios::binary); restrictions_out_stream.open(restrictions_file_name.c_str(), std::ios::binary);
restrictions_out_stream.write((char*)&uuid, sizeof(UUID)); restrictions_out_stream.write((char *)&uuid, sizeof(UUID));
restrictions_out_stream.write( restrictions_out_stream.write((char *)&number_of_useable_restrictions, sizeof(unsigned));
(char*)&usableRestrictionsCounter, for (restrictions_iterator = restrictions_list.begin();
sizeof(unsigned) restrictions_iterator != restrictions_list.end();
); ++restrictions_iterator)
for( {
restrictions_iterator = restrictions_list.begin(); if (UINT_MAX != restrictions_iterator->restriction.fromNode &&
restrictions_iterator != restrictions_list.end(); UINT_MAX != restrictions_iterator->restriction.toNode)
++restrictions_iterator {
) { restrictions_out_stream.write((char *)&(restrictions_iterator->restriction),
if( sizeof(TurnRestriction));
UINT_MAX != restrictions_iterator->restriction.fromNode &&
UINT_MAX != restrictions_iterator->restriction.toNode
) {
restrictions_out_stream.write(
(char *)&(restrictions_iterator->restriction),
sizeof(TurnRestriction)
);
} }
} }
restrictions_out_stream.close(); restrictions_out_stream.close();
std::ofstream file_out_stream; std::ofstream file_out_stream;
file_out_stream.open(output_file_name.c_str(), std::ios::binary); file_out_stream.open(output_file_name.c_str(), std::ios::binary);
file_out_stream.write((char*)&uuid, sizeof(UUID)); file_out_stream.write((char *)&uuid, sizeof(UUID));
file_out_stream.write((char*)&number_of_used_nodes, sizeof(unsigned)); file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
time1 = std::chrono::steady_clock::now(); time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush; std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush;
//identify all used nodes by a merging step of two sorted lists // identify all used nodes by a merging step of two sorted lists
STXXLNodeVector::iterator node_iterator = all_nodes_list.begin(); auto node_iterator = all_nodes_list.begin();
STXXLNodeIDVector::iterator node_id_iterator = used_node_id_list.begin(); auto node_id_iterator = used_node_id_list.begin();
while( while (node_id_iterator != used_node_id_list.end() && node_iterator != all_nodes_list.end())
node_id_iterator != used_node_id_list.end() && {
node_iterator != all_nodes_list.end() if (*node_id_iterator < node_iterator->id)
) { {
if(*node_id_iterator < node_iterator->id){
++node_id_iterator; ++node_id_iterator;
continue; continue;
} }
if(*node_id_iterator > node_iterator->id) { if (*node_id_iterator > node_iterator->id)
{
++node_iterator; ++node_iterator;
continue; continue;
} }
BOOST_ASSERT( *node_id_iterator == node_iterator->id); BOOST_ASSERT(*node_id_iterator == node_iterator->id);
file_out_stream.write( file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode));
(char*)&(*node_iterator),
sizeof(ExternalMemoryNode)
);
++number_of_used_nodes; ++number_of_used_nodes;
++node_id_iterator; ++node_id_iterator;
@ -273,14 +261,13 @@ void ExtractionContainers::PrepareData(
} }
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] setting number of nodes ... " << std::flush; std::cout << "[extractor] setting number of nodes ... " << std::flush;
std::ios::pos_type previous_file_position = file_out_stream.tellp(); std::ios::pos_type previous_file_position = file_out_stream.tellp();
file_out_stream.seekp(std::ios::beg+sizeof(UUID)); file_out_stream.seekp(std::ios::beg + sizeof(UUID));
file_out_stream.write((char*)&number_of_used_nodes, sizeof(unsigned)); file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
file_out_stream.seekp(previous_file_position); file_out_stream.seekp(previous_file_position);
std::cout << "ok" << std::endl; std::cout << "ok" << std::endl;
@ -288,218 +275,175 @@ void ExtractionContainers::PrepareData(
// Sort edges by start. // Sort edges by start.
std::cout << "[extractor] Sorting edges by start ... " << std::flush; std::cout << "[extractor] Sorting edges by start ... " << std::flush;
stxxl::sort( stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartID(), 4294967296);
all_edges_list.begin(),
all_edges_list.end(),
CmpEdgeByStartID(),
4294967296
);
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now(); time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Setting start coords ... " << std::flush; std::cout << "[extractor] Setting start coords ... " << std::flush;
file_out_stream.write((char*)&number_of_used_edges, sizeof(unsigned)); file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
// Traverse list of edges and nodes in parallel and set start coord // Traverse list of edges and nodes in parallel and set start coord
node_iterator = all_nodes_list.begin(); node_iterator = all_nodes_list.begin();
STXXLEdgeVector::iterator edge_iterator = all_edges_list.begin(); auto edge_iterator = all_edges_list.begin();
while( while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
edge_iterator != all_edges_list.end() && {
node_iterator != all_nodes_list.end() if (edge_iterator->start < node_iterator->id)
) { {
if(edge_iterator->start < node_iterator->id){
++edge_iterator; ++edge_iterator;
continue; continue;
} }
if(edge_iterator->start > node_iterator->id) { if (edge_iterator->start > node_iterator->id)
{
node_iterator++; node_iterator++;
continue; continue;
} }
BOOST_ASSERT(edge_iterator->start == node_iterator->id); BOOST_ASSERT(edge_iterator->start == node_iterator->id);
edge_iterator->startCoord.lat = node_iterator->lat; edge_iterator->source_coordinate.lat = node_iterator->lat;
edge_iterator->startCoord.lon = node_iterator->lon; edge_iterator->source_coordinate.lon = node_iterator->lon;
++edge_iterator; ++edge_iterator;
} }
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now(); time1 = std::chrono::steady_clock::now();
// Sort Edges by target // Sort Edges by target
std::cout << "[extractor] Sorting edges by target ... " << std::flush; std::cout << "[extractor] Sorting edges by target ... " << std::flush;
stxxl::sort( stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByTargetID(), 4294967296);
all_edges_list.begin(),
all_edges_list.end(),
CmpEdgeByTargetID(),
4294967296
);
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now(); time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Setting target coords ... " << std::flush; std::cout << "[extractor] Setting target coords ... " << std::flush;
// Traverse list of edges and nodes in parallel and set target coord // Traverse list of edges and nodes in parallel and set target coord
node_iterator = all_nodes_list.begin(); node_iterator = all_nodes_list.begin();
edge_iterator = all_edges_list.begin(); edge_iterator = all_edges_list.begin();
while( while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
edge_iterator != all_edges_list.end() && {
node_iterator != all_nodes_list.end() if (edge_iterator->target < node_iterator->id)
) { {
if(edge_iterator->target < node_iterator->id){
++edge_iterator; ++edge_iterator;
continue; continue;
} }
if(edge_iterator->target > node_iterator->id) { if (edge_iterator->target > node_iterator->id)
{
++node_iterator; ++node_iterator;
continue; continue;
} }
BOOST_ASSERT(edge_iterator->target == node_iterator->id); BOOST_ASSERT(edge_iterator->target == node_iterator->id);
if(edge_iterator->startCoord.lat != INT_MIN && edge_iterator->startCoord.lon != INT_MIN) { if (edge_iterator->source_coordinate.lat != INT_MIN &&
edge_iterator->targetCoord.lat = node_iterator->lat; edge_iterator->source_coordinate.lon != INT_MIN)
edge_iterator->targetCoord.lon = node_iterator->lon; {
const double distance = FixedPointCoordinate::ApproximateDistance(
edge_iterator->startCoord.lat,
edge_iterator->startCoord.lon,
node_iterator->lat,
node_iterator->lon
);
BOOST_ASSERT(edge_iterator->speed != -1); BOOST_ASSERT(edge_iterator->speed != -1);
const double weight = ( distance * 10. ) / (edge_iterator->speed / 3.6); BOOST_ASSERT(edge_iterator->type >= 0);
int integer_weight = std::max( 1, (int)std::floor((edge_iterator->isDurationSet ? edge_iterator->speed : weight)+.5) ); edge_iterator->target_coordinate.lat = node_iterator->lat;
int integer_distance = std::max( 1, (int)distance ); edge_iterator->target_coordinate.lon = node_iterator->lon;
const double distance = FixedPointCoordinate::ApproximateEuclideanDistance(
edge_iterator->source_coordinate.lat,
edge_iterator->source_coordinate.lon,
node_iterator->lat,
node_iterator->lon);
const double weight = (distance * 10.) / (edge_iterator->speed / 3.6);
int integer_weight = std::max(
1,
(int)std::floor(
(edge_iterator->is_duration_set ? edge_iterator->speed : weight) + .5));
int integer_distance = std::max(1, (int)distance);
short zero = 0; short zero = 0;
short one = 1; short one = 1;
file_out_stream.write((char*)&edge_iterator->start, sizeof(unsigned)); file_out_stream.write((char *)&edge_iterator->start, sizeof(unsigned));
file_out_stream.write((char*)&edge_iterator->target, sizeof(unsigned)); file_out_stream.write((char *)&edge_iterator->target, sizeof(unsigned));
file_out_stream.write((char*)&integer_distance, sizeof(int)); file_out_stream.write((char *)&integer_distance, sizeof(int));
switch(edge_iterator->direction) { switch (edge_iterator->direction)
{
case ExtractionWay::notSure: case ExtractionWay::notSure:
file_out_stream.write((char*)&zero, sizeof(short)); file_out_stream.write((char *)&zero, sizeof(short));
break; break;
case ExtractionWay::oneway: case ExtractionWay::oneway:
file_out_stream.write((char*)&one, sizeof(short)); file_out_stream.write((char *)&one, sizeof(short));
break; break;
case ExtractionWay::bidirectional: case ExtractionWay::bidirectional:
file_out_stream.write((char*)&zero, sizeof(short)); file_out_stream.write((char *)&zero, sizeof(short));
break; break;
case ExtractionWay::opposite: case ExtractionWay::opposite:
file_out_stream.write((char*)&one, sizeof(short)); file_out_stream.write((char *)&one, sizeof(short));
break; break;
default: default:
throw OSRMException("edge has broken direction"); throw OSRMException("edge has broken direction");
} }
file_out_stream.write(
(char*)&integer_weight, sizeof(int) file_out_stream.write((char *)&integer_weight, sizeof(int));
); file_out_stream.write((char *)&edge_iterator->type, sizeof(short));
BOOST_ASSERT(edge_iterator->type >= 0); file_out_stream.write((char *)&edge_iterator->name_id, sizeof(unsigned));
file_out_stream.write( file_out_stream.write((char *)&edge_iterator->is_roundabout, sizeof(bool));
(char*)&edge_iterator->type, file_out_stream.write((char *)&edge_iterator->is_in_tiny_cc, sizeof(bool));
sizeof(short) file_out_stream.write((char *)&edge_iterator->is_access_restricted, sizeof(bool));
); file_out_stream.write((char *)&edge_iterator->is_contra_flow, sizeof(bool));
file_out_stream.write( file_out_stream.write((char *)&edge_iterator->is_split, sizeof(bool));
(char *) &edge_iterator->nameID,
sizeof(unsigned)
);
file_out_stream.write(
(char *) &edge_iterator->isRoundabout,
sizeof(bool)
);
file_out_stream.write(
(char *) &edge_iterator->ignoreInGrid,
sizeof(bool)
);
file_out_stream.write(
(char *) &edge_iterator->isAccessRestricted,
sizeof(bool)
);
file_out_stream.write(
(char *) &edge_iterator->isContraFlow,
sizeof(bool)
);
file_out_stream.write(
(char *) &edge_iterator->is_split,
sizeof(bool)
);
++number_of_used_edges; ++number_of_used_edges;
} }
++edge_iterator; ++edge_iterator;
} }
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] setting number of edges ... " << std::flush; std::cout << "[extractor] setting number of edges ... " << std::flush;
file_out_stream.seekp(previous_file_position); file_out_stream.seekp(previous_file_position);
file_out_stream.write((char*)&number_of_used_edges, sizeof(unsigned)); file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
file_out_stream.close(); file_out_stream.close();
std::cout << "ok" << std::endl; std::cout << "ok" << std::endl;
time1 = std::chrono::steady_clock::now(); time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] writing street name index ... " << std::flush; std::cout << "[extractor] writing street name index ... " << std::flush;
std::string name_file_streamName = (output_file_name + ".names"); std::string name_file_streamName = (output_file_name + ".names");
boost::filesystem::ofstream name_file_stream( boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary);
name_file_streamName,
std::ios::binary
);
//write number of names // write number of names
const unsigned number_of_names = name_list.size()+1; const unsigned number_of_names = name_list.size() + 1;
name_file_stream.write((char *)&(number_of_names), sizeof(unsigned)); name_file_stream.write((char *)&(number_of_names), sizeof(unsigned));
//compute total number of chars // compute total number of chars
unsigned total_number_of_chars = 0; unsigned total_number_of_chars = 0;
for (const std::string & temp_string : name_list) { for (const std::string &temp_string : name_list)
{
total_number_of_chars += temp_string.length(); total_number_of_chars += temp_string.length();
} }
//write total number of chars // write total number of chars
name_file_stream.write( name_file_stream.write((char *)&(total_number_of_chars), sizeof(unsigned));
(char *)&(total_number_of_chars), // write prefixe sums
sizeof(unsigned)
);
//write prefixe sums
unsigned name_lengths_prefix_sum = 0; unsigned name_lengths_prefix_sum = 0;
for (const std::string & temp_string : name_list) { for (const std::string &temp_string : name_list)
name_file_stream.write( {
(char *)&(name_lengths_prefix_sum), name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
sizeof(unsigned)
);
name_lengths_prefix_sum += temp_string.length(); name_lengths_prefix_sum += temp_string.length();
} }
//duplicate on purpose! // duplicate on purpose!
name_file_stream.write( name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
(char *)&(name_lengths_prefix_sum),
sizeof(unsigned)
);
//write all chars consecutively // write all chars consecutively
for (const std::string & temp_string : name_list) { for (const std::string &temp_string : name_list)
{
const unsigned string_length = temp_string.length(); const unsigned string_length = temp_string.length();
name_file_stream.write(temp_string.c_str(), string_length); name_file_stream.write(temp_string.c_str(), string_length);
} }
name_file_stream.close(); name_file_stream.close();
time2 = std::chrono::steady_clock::now(); time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1; elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
SimpleLogger().Write() << "Processed " << SimpleLogger().Write() << "Processed " << number_of_used_nodes << " nodes and "
number_of_used_nodes << " nodes and " << << number_of_used_edges << " edges";
number_of_used_edges << " edges";
} catch ( const std::exception& e ) {
std::cerr << "Caught Execption:" << e.what() << std::endl;
} }
catch (const std::exception &e) { std::cerr << "Caught Execption:" << e.what() << std::endl; }
} }

33
Extractor/ExtractionContainers.h
View File

@ -35,31 +35,30 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <stxxl/vector> #include <stxxl/vector>
class ExtractionContainers { class ExtractionContainers
public: {
typedef stxxl::vector<NodeID> STXXLNodeIDVector; public:
typedef stxxl::vector<ExternalMemoryNode> STXXLNodeVector; typedef stxxl::vector<NodeID> STXXLNodeIDVector;
typedef stxxl::vector<InternalExtractorEdge> STXXLEdgeVector; typedef stxxl::vector<ExternalMemoryNode> STXXLNodeVector;
typedef stxxl::vector<std::string> STXXLStringVector; typedef stxxl::vector<InternalExtractorEdge> STXXLEdgeVector;
typedef stxxl::vector<std::string> STXXLStringVector;
typedef stxxl::vector<InputRestrictionContainer> STXXLRestrictionsVector; typedef stxxl::vector<InputRestrictionContainer> STXXLRestrictionsVector;
typedef stxxl::vector<_WayIDStartAndEndEdge> STXXLWayIDStartEndVector; typedef stxxl::vector<WayIDStartAndEndEdge> STXXLWayIDStartEndVector;
STXXLNodeIDVector used_node_id_list; STXXLNodeIDVector used_node_id_list;
STXXLNodeVector all_nodes_list; STXXLNodeVector all_nodes_list;
STXXLEdgeVector all_edges_list; STXXLEdgeVector all_edges_list;
STXXLStringVector name_list; STXXLStringVector name_list;
STXXLRestrictionsVector restrictions_list; STXXLRestrictionsVector restrictions_list;
STXXLWayIDStartEndVector way_start_end_id_list; STXXLWayIDStartEndVector way_start_end_id_list;
const UUID uuid; const UUID uuid;
ExtractionContainers(); ExtractionContainers();
virtual ~ExtractionContainers(); virtual ~ExtractionContainers();
void PrepareData( void PrepareData(const std::string &output_file_name,
const std::string & output_file_name, const std::string &restrictions_file_name);
const std::string & restrictions_file_name
);
}; };
#endif /* EXTRACTIONCONTAINERS_H_ */ #endif /* EXTRACTIONCONTAINERS_H_ */

74
Extractor/ExtractionHelperFunctions.h
View File

@ -25,63 +25,65 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
#ifndef EXTRACTIONHELPERFUNCTIONS_H_ #ifndef EXTRACTION_HELPER_FUNCTIONS_H
#define EXTRACTIONHELPERFUNCTIONS_H_ #define EXTRACTION_HELPER_FUNCTIONS_H
#include "../Util/StringUtil.h" #include "../Util/StringUtil.h"
#include <boost/algorithm/string.hpp> #include <boost/algorithm/string.hpp>
#include <boost/algorithm/string_regex.hpp> #include <boost/algorithm/string_regex.hpp>
#include <boost/regex.hpp> #include <boost/regex.hpp>
#include <climits>
#include <limits>
namespace qi = boost::spirit::qi; namespace qi = boost::spirit::qi;
//TODO: Move into LUA // TODO: Move into LUA
inline bool durationIsValid(const std::string &s) { inline bool durationIsValid(const std::string &s)
boost::regex e ("((\\d|\\d\\d):(\\d|\\d\\d):(\\d|\\d\\d))|((\\d|\\d\\d):(\\d|\\d\\d))|(\\d|\\d\\d)",boost::regex_constants::icase|boost::regex_constants::perl); {
boost::regex e(
"((\\d|\\d\\d):(\\d|\\d\\d):(\\d|\\d\\d))|((\\d|\\d\\d):(\\d|\\d\\d))|(\\d|\\d\\d)",
boost::regex_constants::icase | boost::regex_constants::perl);
std::vector< std::string > result; std::vector<std::string> result;
boost::algorithm::split_regex( result, s, boost::regex( ":" ) ) ; boost::algorithm::split_regex(result, s, boost::regex(":"));
bool matched = regex_match(s, e); bool matched = regex_match(s, e);
return matched; return matched;
} }
inline unsigned parseDuration(const std::string &s) { inline unsigned parseDuration(const std::string &s)
unsigned hours = 0; {
unsigned hours = 0;
unsigned minutes = 0; unsigned minutes = 0;
unsigned seconds = 0; unsigned seconds = 0;
boost::regex e ("((\\d|\\d\\d):(\\d|\\d\\d):(\\d|\\d\\d))|((\\d|\\d\\d):(\\d|\\d\\d))|(\\d|\\d\\d)",boost::regex_constants::icase|boost::regex_constants::perl); boost::regex e(
"((\\d|\\d\\d):(\\d|\\d\\d):(\\d|\\d\\d))|((\\d|\\d\\d):(\\d|\\d\\d))|(\\d|\\d\\d)",
boost::regex_constants::icase | boost::regex_constants::perl);
std::vector< std::string > result; std::vector<std::string> result;
boost::algorithm::split_regex( result, s, boost::regex( ":" ) ) ; boost::algorithm::split_regex(result, s, boost::regex(":"));
bool matched = regex_match(s, e); bool matched = regex_match(s, e);
if(matched) { if (matched)
if(1 == result.size()) { {
minutes = stringToInt(result[0]); if (1 == result.size())
} {
if(2 == result.size()) { minutes = stringToInt(result[0]);
minutes = stringToInt(result[1]); }
hours = stringToInt(result[0]); if (2 == result.size())
} {
if(3 == result.size()) { minutes = stringToInt(result[1]);
hours = stringToInt(result[0]);
}
if (3 == result.size())
{
seconds = stringToInt(result[2]); seconds = stringToInt(result[2]);
minutes = stringToInt(result[1]); minutes = stringToInt(result[1]);
hours = stringToInt(result[0]); hours = stringToInt(result[0]);
} }
return 10*(3600*hours+60*minutes+seconds); return 10 * (3600 * hours + 60 * minutes + seconds);
} }
return UINT_MAX; return std::numeric_limits<unsigned>::max();
} }
// inline int parseMaxspeed(std::string input) { //call-by-value on purpose. #endif // EXTRACTION_HELPER_FUNCTIONS_H_
// boost::algorithm::to_lower(input);
// int n = stringToInt(input);
// if (input.find("mph") != std::string::npos || input.find("mp/h") != std::string::npos) {
// n = (n*1609)/1000;
// }
// return n;
// }
#endif /* EXTRACTIONHELPERFUNCTIONS_H_ */

27
Extractor/ExtractionWay.h
View File

@ -34,14 +34,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <string> #include <string>
#include <vector> #include <vector>
struct ExtractionWay { struct ExtractionWay
ExtractionWay() { {
Clear(); ExtractionWay() { Clear(); }
}
inline void Clear(){ inline void Clear()
id = UINT_MAX; {
nameID = UINT_MAX; id = SPECIAL_NODEID;
nameID = INVALID_NAMEID;
path.clear(); path.clear();
keyVals.clear(); keyVals.clear();
direction = ExtractionWay::notSure; direction = ExtractionWay::notSure;
@ -55,9 +55,11 @@ struct ExtractionWay {
ignoreInGrid = false; ignoreInGrid = false;
} }
enum Directions { enum Directions
notSure = 0, oneway, bidirectional, opposite { notSure = 0,
}; oneway,
bidirectional,
opposite };
unsigned id; unsigned id;
unsigned nameID; unsigned nameID;
double speed; double speed;
@ -70,9 +72,8 @@ struct ExtractionWay {
bool roundabout; bool roundabout;
bool isAccessRestricted; bool isAccessRestricted;
bool ignoreInGrid; bool ignoreInGrid;
std::vector< NodeID > path; std::vector<NodeID> path;
HashTable<std::string, std::string> keyVals; HashTable<std::string, std::string> keyVals;
}; };
#endif // EXTRACTION_WAY_H
#endif //EXTRACTION_WAY_H

175
Extractor/ExtractorCallbacks.cpp
View File

@ -38,117 +38,126 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <string> #include <string>
#include <vector> #include <vector>
ExtractorCallbacks::ExtractorCallbacks() ExtractorCallbacks::ExtractorCallbacks(ExtractionContainers &extraction_containers,
: boost::unordered_map<std::string, NodeID> &string_map)
string_map(NULL), : string_map(string_map), external_memory(extraction_containers)
externalMemory(NULL) {
{ } }
ExtractorCallbacks::ExtractorCallbacks(
ExtractionContainers * ext,
boost::unordered_map<std::string, NodeID> * string_map
) :
string_map(string_map),
externalMemory(ext)
{ }
ExtractorCallbacks::~ExtractorCallbacks() { }
/** warning: caller needs to take care of synchronization! */ /** warning: caller needs to take care of synchronization! */
void ExtractorCallbacks::nodeFunction(const ExternalMemoryNode &n) { void ExtractorCallbacks::ProcessNode(const ExternalMemoryNode &n)
if(n.lat <= 85*COORDINATE_PRECISION && n.lat >= -85*COORDINATE_PRECISION) { {
externalMemory->all_nodes_list.push_back(n); if (n.lat <= 85 * COORDINATE_PRECISION && n.lat >= -85 * COORDINATE_PRECISION)
{
external_memory.all_nodes_list.push_back(n);
} }
} }
bool ExtractorCallbacks::restrictionFunction(const InputRestrictionContainer &r) { bool ExtractorCallbacks::ProcessRestriction(const InputRestrictionContainer &restriction)
externalMemory->restrictions_list.push_back(r); {
external_memory.restrictions_list.push_back(restriction);
return true; return true;
} }
/** warning: caller needs to take care of synchronization! */ /** warning: caller needs to take care of synchronization! */
void ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) { void ExtractorCallbacks::ProcessWay(ExtractionWay &parsed_way)
if((0 < parsed_way.speed) || (0 < parsed_way.duration)) { //Only true if the way is specified by the speed profile {
if(UINT_MAX == parsed_way.id){ if ((0 < parsed_way.speed) || (0 < parsed_way.duration))
SimpleLogger().Write(logDEBUG) << { // Only true if the way is specified by the speed profile
"found bogus way with id: " << parsed_way.id << if (UINT_MAX == parsed_way.id)
" of size " << parsed_way.path.size(); {
SimpleLogger().Write(logDEBUG) << "found bogus way with id: " << parsed_way.id
<< " of size " << parsed_way.path.size();
return; return;
} }
if(0 < parsed_way.duration) { if (0 < parsed_way.duration)
//TODO: iterate all way segments and set duration corresponding to the length of each segment {
parsed_way.speed = parsed_way.duration/(parsed_way.path.size()-1); // TODO: iterate all way segments and set duration corresponding to the length of each
// segment
parsed_way.speed = parsed_way.duration / (parsed_way.path.size() - 1);
} }
if(std::numeric_limits<double>::epsilon() >= std::abs(-1. - parsed_way.speed)){ if (std::numeric_limits<double>::epsilon() >= std::abs(-1. - parsed_way.speed))
SimpleLogger().Write(logDEBUG) << {
"found way with bogus speed, id: " << parsed_way.id; SimpleLogger().Write(logDEBUG) << "found way with bogus speed, id: " << parsed_way.id;
return; return;
} }
//Get the unique identifier for the street name // Get the unique identifier for the street name
const boost::unordered_map<std::string, NodeID>::const_iterator & string_map_iterator = string_map->find(parsed_way.name); const auto &string_map_iterator = string_map.find(parsed_way.name);
if(string_map->end() == string_map_iterator) { if (string_map.end() == string_map_iterator)
parsed_way.nameID = externalMemory->name_list.size(); {
externalMemory->name_list.push_back(parsed_way.name); parsed_way.nameID = external_memory.name_list.size();
string_map->insert(std::make_pair(parsed_way.name, parsed_way.nameID)); external_memory.name_list.push_back(parsed_way.name);
} else { string_map.insert(std::make_pair(parsed_way.name, parsed_way.nameID));
}
else
{
parsed_way.nameID = string_map_iterator->second; parsed_way.nameID = string_map_iterator->second;
} }
if(ExtractionWay::opposite == parsed_way.direction) { if (ExtractionWay::opposite == parsed_way.direction)
std::reverse( parsed_way.path.begin(), parsed_way.path.end() ); {
std::reverse(parsed_way.path.begin(), parsed_way.path.end());
parsed_way.direction = ExtractionWay::oneway; parsed_way.direction = ExtractionWay::oneway;
} }
const bool split_bidirectional_edge = (parsed_way.backward_speed > 0) && (parsed_way.speed != parsed_way.backward_speed); const bool split_bidirectional_edge =
(parsed_way.backward_speed > 0) && (parsed_way.speed != parsed_way.backward_speed);
for(std::vector< NodeID >::size_type n = 0; n < parsed_way.path.size()-1; ++n) { for (unsigned n = 0; n < parsed_way.path.size() - 1; ++n)
externalMemory->all_edges_list.push_back( {
InternalExtractorEdge( external_memory.all_edges_list.push_back(InternalExtractorEdge(
parsed_way.path[n], parsed_way.path[n],
parsed_way.path[n+1], parsed_way.path[n + 1],
parsed_way.type, parsed_way.type,
(split_bidirectional_edge ? ExtractionWay::oneway : parsed_way.direction), (split_bidirectional_edge ? ExtractionWay::oneway : parsed_way.direction),
parsed_way.speed, parsed_way.speed,
parsed_way.nameID, parsed_way.nameID,
parsed_way.roundabout, parsed_way.roundabout,
parsed_way.ignoreInGrid, parsed_way.ignoreInGrid,
(0 < parsed_way.duration), (0 < parsed_way.duration),
parsed_way.isAccessRestricted, parsed_way.isAccessRestricted,
false, false,
split_bidirectional_edge split_bidirectional_edge));
) external_memory.used_node_id_list.push_back(parsed_way.path[n]);
);
externalMemory->used_node_id_list.push_back(parsed_way.path[n]);
} }
externalMemory->used_node_id_list.push_back(parsed_way.path.back()); external_memory.used_node_id_list.push_back(parsed_way.path.back());
//The following information is needed to identify start and end segments of restrictions // The following information is needed to identify start and end segments of restrictions
externalMemory->way_start_end_id_list.push_back(_WayIDStartAndEndEdge(parsed_way.id, parsed_way.path[0], parsed_way.path[1], parsed_way.path[parsed_way.path.size()-2], parsed_way.path.back())); external_memory.way_start_end_id_list.push_back(
WayIDStartAndEndEdge(parsed_way.id,
parsed_way.path[0],
parsed_way.path[1],
parsed_way.path[parsed_way.path.size() - 2],
parsed_way.path.back()));
if(split_bidirectional_edge) { //Only true if the way should be split if (split_bidirectional_edge)
std::reverse( parsed_way.path.begin(), parsed_way.path.end() ); { // Only true if the way should be split
for(std::vector< NodeID >::size_type n = 0; n < parsed_way.path.size()-1; ++n) { std::reverse(parsed_way.path.begin(), parsed_way.path.end());
externalMemory->all_edges_list.push_back( for (std::vector<NodeID>::size_type n = 0; n < parsed_way.path.size() - 1; ++n)
InternalExtractorEdge( {
parsed_way.path[n], external_memory.all_edges_list.push_back(
parsed_way.path[n+1], InternalExtractorEdge(parsed_way.path[n],
parsed_way.type, parsed_way.path[n + 1],
ExtractionWay::oneway, parsed_way.type,
parsed_way.backward_speed, ExtractionWay::oneway,
parsed_way.nameID, parsed_way.backward_speed,
parsed_way.roundabout, parsed_way.nameID,
parsed_way.ignoreInGrid, parsed_way.roundabout,
(0 < parsed_way.duration), parsed_way.ignoreInGrid,
parsed_way.isAccessRestricted, (0 < parsed_way.duration),
(ExtractionWay::oneway == parsed_way.direction), parsed_way.isAccessRestricted,
split_bidirectional_edge (ExtractionWay::oneway == parsed_way.direction),
) split_bidirectional_edge));
);
} }
externalMemory->way_start_end_id_list.push_back(_WayIDStartAndEndEdge(parsed_way.id, parsed_way.path[0], parsed_way.path[1], parsed_way.path[parsed_way.path.size()-2], parsed_way.path.back())); external_memory.way_start_end_id_list.push_back(
WayIDStartAndEndEdge(parsed_way.id,
parsed_way.path[0],
parsed_way.path[1],
parsed_way.path[parsed_way.path.size() - 2],
parsed_way.path.back()));
} }
} }
} }

41
Extractor/ExtractorCallbacks.h
View File

@ -25,8 +25,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
#ifndef EXTRACTORCALLBACKS_H_ #ifndef EXTRACTOR_CALLBACKS_H
#define EXTRACTORCALLBACKS_H_ #define EXTRACTOR_CALLBACKS_H
#include "../typedefs.h" #include "../typedefs.h"
@ -38,29 +38,26 @@ class ExtractionContainers;
struct ExtractionWay; struct ExtractionWay;
struct InputRestrictionContainer; struct InputRestrictionContainer;
class ExtractorCallbacks{ class ExtractorCallbacks
private: {
private:
boost::unordered_map<std::string, NodeID> &string_map;
ExtractionContainers &external_memory;
boost::unordered_map<std::string, NodeID> * string_map; public:
ExtractionContainers * externalMemory; ExtractorCallbacks() = delete;
ExtractorCallbacks(const ExtractorCallbacks &) = delete;
explicit ExtractorCallbacks(ExtractionContainers &extraction_containers,
boost::unordered_map<std::string, NodeID> &string_map);
ExtractorCallbacks(); // warning: caller needs to take care of synchronization!
public: void ProcessNode(const ExternalMemoryNode &node);
explicit ExtractorCallbacks(
ExtractionContainers * ext,
boost::unordered_map<std::string, NodeID> * string_map
);
~ExtractorCallbacks(); // warning: caller needs to take care of synchronization!
bool ProcessRestriction(const InputRestrictionContainer &restriction);
/** warning: caller needs to take care of synchronization! */
void nodeFunction(const ExternalMemoryNode &n);
bool restrictionFunction(const InputRestrictionContainer &r);
/** warning: caller needs to take care of synchronization! */
void wayFunction(ExtractionWay &w);
// warning: caller needs to take care of synchronization!
void ProcessWay(ExtractionWay &way);
}; };
#endif /* EXTRACTORCALLBACKS_H_ */ #endif /* EXTRACTOR_CALLBACKS_H */

121
Extractor/ExtractorStructs.h
View File

@ -34,97 +34,80 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <string> #include <string>
struct ExtractorRelation { struct ExtractorRelation
ExtractorRelation() : type(unknown){} {
enum { ExtractorRelation() : type(unknown) {}
unknown = 0, ferry, turnRestriction enum
} type; { unknown = 0,
ferry,
turnRestriction } type;
HashTable<std::string, std::string> keyVals; HashTable<std::string, std::string> keyVals;
}; };
struct _WayIDStartAndEndEdge { struct WayIDStartAndEndEdge
{
unsigned wayID; unsigned wayID;
NodeID firstStart; NodeID firstStart;
NodeID firstTarget; NodeID firstTarget;
NodeID lastStart; NodeID lastStart;
NodeID lastTarget; NodeID lastTarget;
_WayIDStartAndEndEdge() WayIDStartAndEndEdge()
: : wayID(UINT_MAX), firstStart(UINT_MAX), firstTarget(UINT_MAX), lastStart(UINT_MAX),
wayID(UINT_MAX), lastTarget(UINT_MAX)
firstStart(UINT_MAX), {
firstTarget(UINT_MAX),
lastStart(UINT_MAX),
lastTarget(UINT_MAX)
{ }
explicit _WayIDStartAndEndEdge(
unsigned w,
NodeID fs,
NodeID ft,
NodeID ls,
NodeID lt
) :
wayID(w),
firstStart(fs),
firstTarget(ft),
lastStart(ls),
lastTarget(lt)
{ }
static _WayIDStartAndEndEdge min_value() {
return _WayIDStartAndEndEdge((std::numeric_limits<unsigned>::min)(), (std::numeric_limits<unsigned>::min)(), (std::numeric_limits<unsigned>::min)(), (std::numeric_limits<unsigned>::min)(), (std::numeric_limits<unsigned>::min)());
} }
static _WayIDStartAndEndEdge max_value() {
return _WayIDStartAndEndEdge((std::numeric_limits<unsigned>::max)(), (std::numeric_limits<unsigned>::max)(), (std::numeric_limits<unsigned>::max)(), (std::numeric_limits<unsigned>::max)(), (std::numeric_limits<unsigned>::max)()); explicit WayIDStartAndEndEdge(unsigned w, NodeID fs, NodeID ft, NodeID ls, NodeID lt)
: wayID(w), firstStart(fs), firstTarget(ft), lastStart(ls), lastTarget(lt)
{
}
static WayIDStartAndEndEdge min_value()
{
return WayIDStartAndEndEdge((std::numeric_limits<unsigned>::min)(),
(std::numeric_limits<unsigned>::min)(),
(std::numeric_limits<unsigned>::min)(),
(std::numeric_limits<unsigned>::min)(),
(std::numeric_limits<unsigned>::min)());
}
static WayIDStartAndEndEdge max_value()
{
return WayIDStartAndEndEdge((std::numeric_limits<unsigned>::max)(),
(std::numeric_limits<unsigned>::max)(),
(std::numeric_limits<unsigned>::max)(),
(std::numeric_limits<unsigned>::max)(),
(std::numeric_limits<unsigned>::max)());
} }
}; };
struct CmpWayByID { struct CmpWayByID
typedef _WayIDStartAndEndEdge value_type; {
bool operator ()( typedef WayIDStartAndEndEdge value_type;
const _WayIDStartAndEndEdge & a, bool operator()(const WayIDStartAndEndEdge &a, const WayIDStartAndEndEdge &b) const
const _WayIDStartAndEndEdge & b {
) const {
return a.wayID < b.wayID; return a.wayID < b.wayID;
} }
value_type max_value() { value_type max_value() { return WayIDStartAndEndEdge::max_value(); }
return _WayIDStartAndEndEdge::max_value(); value_type min_value() { return WayIDStartAndEndEdge::min_value(); }
}
value_type min_value() {
return _WayIDStartAndEndEdge::min_value();
}
}; };
struct Cmp { struct Cmp
{
typedef NodeID value_type; typedef NodeID value_type;
bool operator ()( bool operator()(const NodeID a, const NodeID b) const { return a < b; }
const NodeID a, value_type max_value() { return 0xffffffff; }
const NodeID b value_type min_value() { return 0x0; }
) const {
return a < b;
}
value_type max_value() {
return 0xffffffff;
}
value_type min_value() {
return 0x0;
}
}; };
struct CmpNodeByID { struct CmpNodeByID
{
typedef ExternalMemoryNode value_type; typedef ExternalMemoryNode value_type;
bool operator () ( bool operator()(const ExternalMemoryNode &a, const ExternalMemoryNode &b) const
const ExternalMemoryNode & a, {
const ExternalMemoryNode & b
) const {
return a.id < b.id; return a.id < b.id;
} }
value_type max_value() { value_type max_value() { return ExternalMemoryNode::max_value(); }
return ExternalMemoryNode::max_value(); value_type min_value() { return ExternalMemoryNode::min_value(); }
}
value_type min_value() {
return ExternalMemoryNode::min_value();
}
}; };
#endif /* EXTRACTORSTRUCTS_H_ */ #endif /* EXTRACTORSTRUCTS_H_ */

153
Extractor/InternalExtractorEdge.h
View File

@ -28,92 +28,49 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef INTERNAL_EXTRACTOR_EDGE_H #ifndef INTERNAL_EXTRACTOR_EDGE_H
#define INTERNAL_EXTRACTOR_EDGE_H #define INTERNAL_EXTRACTOR_EDGE_H
#include "../typedefs.h" #include "../typedefs.h"
#include <osrm/Coordinate.h> #include <osrm/Coordinate.h>
#include <boost/assert.hpp> #include <boost/assert.hpp>
struct InternalExtractorEdge { struct InternalExtractorEdge
{
InternalExtractorEdge() InternalExtractorEdge()
: : start(0), target(0), type(0), direction(0), speed(0), name_id(0), is_roundabout(false),
start(0), is_in_tiny_cc(false), is_duration_set(false), is_access_restricted(false),
target(0), is_contra_flow(false), is_split(false)
type(0), {
direction(0), }
speed(0),
nameID(0),
isRoundabout(false),
ignoreInGrid(false),
isDurationSet(false),
isAccessRestricted(false),
isContraFlow(false),
is_split(false)
{ }
explicit InternalExtractorEdge(NodeID start,
explicit InternalExtractorEdge( NodeID target,
NodeID start, short type,
NodeID target, short direction,
short type, double speed,
short direction, unsigned name_id,
double speed, bool is_roundabout,
unsigned nameID, bool is_in_tiny_cc,
bool isRoundabout, bool is_duration_set,
bool ignoreInGrid, bool is_access_restricted,
bool isDurationSet, bool is_contra_flow,
bool isAccressRestricted, bool is_split)
bool isContraFlow, : start(start), target(target), type(type), direction(direction), speed(speed),
bool is_split name_id(name_id), is_roundabout(is_roundabout), is_in_tiny_cc(is_in_tiny_cc),
) : is_duration_set(is_duration_set), is_access_restricted(is_access_restricted),
start(start), is_contra_flow(is_contra_flow), is_split(is_split)
target(target),
type(type),
direction(direction),
speed(speed),
nameID(nameID),
isRoundabout(isRoundabout),
ignoreInGrid(ignoreInGrid),
isDurationSet(isDurationSet),
isAccessRestricted(isAccressRestricted),
isContraFlow(isContraFlow),
is_split(is_split)
{ {
BOOST_ASSERT(0 <= type); BOOST_ASSERT(0 <= type);
} }
// necessary static util functions for stxxl's sorting // necessary static util functions for stxxl's sorting
static InternalExtractorEdge min_value() { static InternalExtractorEdge min_value()
return InternalExtractorEdge( {
0, return InternalExtractorEdge(0, 0, 0, 0, 0, 0, false, false, false, false, false, false);
0,
0,
0,
0,
0,
false,
false,
false,
false,
false,
false
);
} }
static InternalExtractorEdge max_value() { static InternalExtractorEdge max_value()
{
return InternalExtractorEdge( return InternalExtractorEdge(
std::numeric_limits<unsigned>::max(), SPECIAL_NODEID, SPECIAL_NODEID, 0, 0, 0, 0, false, false, false, false, false, false);
std::numeric_limits<unsigned>::max(),
0,
0,
0,
0,
false,
false,
false,
false,
false,
false
);
} }
NodeID start; NodeID start;
@ -121,53 +78,43 @@ struct InternalExtractorEdge {
short type; short type;
short direction; short direction;
double speed; double speed;
unsigned nameID; unsigned name_id;
bool isRoundabout; bool is_roundabout;
bool ignoreInGrid; bool is_in_tiny_cc;
bool isDurationSet; bool is_duration_set;
bool isAccessRestricted; bool is_access_restricted;
bool isContraFlow; bool is_contra_flow;
bool is_split; bool is_split;
FixedPointCoordinate startCoord; FixedPointCoordinate source_coordinate;
FixedPointCoordinate targetCoord; FixedPointCoordinate target_coordinate;
}; };
struct CmpEdgeByStartID { struct CmpEdgeByStartID
{
typedef InternalExtractorEdge value_type; typedef InternalExtractorEdge value_type;
bool operator ()( bool operator()(const InternalExtractorEdge &a, const InternalExtractorEdge &b) const
const InternalExtractorEdge & a, {
const InternalExtractorEdge & b
) const {
return a.start < b.start; return a.start < b.start;
} }
value_type max_value() { value_type max_value() { return InternalExtractorEdge::max_value(); }
return InternalExtractorEdge::max_value();
}
value_type min_value() { value_type min_value() { return InternalExtractorEdge::min_value(); }
return InternalExtractorEdge::min_value();
}
}; };
struct CmpEdgeByTargetID { struct CmpEdgeByTargetID
{
typedef InternalExtractorEdge value_type; typedef InternalExtractorEdge value_type;
bool operator ()( bool operator()(const InternalExtractorEdge &a, const InternalExtractorEdge &b) const
const InternalExtractorEdge & a, {
const InternalExtractorEdge & b
) const {
return a.target < b.target; return a.target < b.target;
} }
value_type max_value() { value_type max_value() { return InternalExtractorEdge::max_value(); }
return InternalExtractorEdge::max_value();
}
value_type min_value() { value_type min_value() { return InternalExtractorEdge::min_value(); }
return InternalExtractorEdge::min_value();
}
}; };
#endif //INTERNAL_EXTRACTOR_EDGE_H #endif // INTERNAL_EXTRACTOR_EDGE_H

953
Extractor/PBFParser.cpp
View File

File diff suppressed because it is too large Load Diff

64
Extractor/PBFParser.h
View File

@ -31,24 +31,24 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "BaseParser.h" #include "BaseParser.h"
#include "../DataStructures/ConcurrentQueue.h" #include "../DataStructures/ConcurrentQueue.h"
#include <boost/shared_ptr.hpp>
#include <osmpbf/fileformat.pb.h> #include <osmpbf/fileformat.pb.h>
#include <osmpbf/osmformat.pb.h> #include <osmpbf/osmformat.pb.h>
#include <fstream> #include <fstream>
#include <memory>
class PBFParser : public BaseParser { class PBFParser : public BaseParser
{
enum EntityType { enum EntityType
TypeDummy = 0, { TypeDummy = 0,
TypeNode = 1, TypeNode = 1,
TypeWay = 2, TypeWay = 2,
TypeRelation = 4, TypeRelation = 4,
TypeDenseNode = 8 TypeDenseNode = 8 };
};
struct _ThreadData { struct ParserThreadData
{
int currentGroupID; int currentGroupID;
int currentEntityID; int currentEntityID;
EntityType entityTypeIndicator; EntityType entityTypeIndicator;
@ -62,42 +62,38 @@ class PBFParser : public BaseParser {
std::vector<char> charBuffer; std::vector<char> charBuffer;
}; };
public: public:
PBFParser( PBFParser(const char *file_name, ExtractorCallbacks *extractor_callbacks, ScriptingEnvironment &scripting_environment);
const char * fileName,
ExtractorCallbacks* ec,
ScriptingEnvironment& se
);
virtual ~PBFParser(); virtual ~PBFParser();
inline bool ReadHeader(); inline bool ReadHeader();
inline bool Parse(); inline bool Parse();
private: private:
inline void ReadData(); inline void ReadData();
inline void ParseData(); inline void ParseData();
inline void parseDenseNode (_ThreadData * threadData); inline void parseDenseNode(ParserThreadData *thread_data);
inline void parseNode (_ThreadData * threadData); inline void parseNode(ParserThreadData *thread_data);
inline void parseRelation (_ThreadData * threadData); inline void parseRelation(ParserThreadData *thread_data);
inline void parseWay (_ThreadData * threadData); inline void parseWay(ParserThreadData *thread_data);
inline void loadGroup (_ThreadData * threadData); inline void loadGroup(ParserThreadData *thread_data);
inline void loadBlock (_ThreadData * threadData); inline void loadBlock(ParserThreadData *thread_data);
inline bool readPBFBlobHeader(std::fstream & stream, _ThreadData * threadData); inline bool readPBFBlobHeader(std::fstream &stream, ParserThreadData *thread_data);
inline bool unpackZLIB (std::fstream & stream, _ThreadData * threadData); inline bool unpackZLIB(std::fstream &stream, ParserThreadData *thread_data);
inline bool unpackLZMA (std::fstream & stream, _ThreadData * threadData); inline bool unpackLZMA(std::fstream &stream, ParserThreadData *thread_data);
inline bool readBlob (std::fstream & stream, _ThreadData * threadData); inline bool readBlob(std::fstream &stream, ParserThreadData *thread_data);
inline bool readNextBlock (std::fstream & stream, _ThreadData * threadData); inline bool readNextBlock(std::fstream &stream, ParserThreadData *thread_data);
static const int NANO = 1000 * 1000 * 1000; static const int NANO = 1000 * 1000 * 1000;
static const int MAX_BLOB_HEADER_SIZE = 64 * 1024; static const int MAX_BLOB_HEADER_SIZE = 64 * 1024;
static const int MAX_BLOB_SIZE = 32 * 1024 * 1024; static const int MAX_BLOB_SIZE = 32 * 1024 * 1024;
unsigned groupCount; unsigned group_count;
unsigned blockCount; unsigned block_count;
std::fstream input; // the input stream to parse std::fstream input; // the input stream to parse
boost::shared_ptr<ConcurrentQueue < _ThreadData* > > threadDataQueue; std::shared_ptr<ConcurrentQueue<ParserThreadData *>> thread_data_queue;
}; };
#endif /* PBFPARSER_H_ */ #endif /* PBFPARSER_H_ */

118
Extractor/ScriptingEnvironment.cpp
View File

@ -37,90 +37,88 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "../typedefs.h" #include "../typedefs.h"
ScriptingEnvironment::ScriptingEnvironment() {} ScriptingEnvironment::ScriptingEnvironment() {}
ScriptingEnvironment::ScriptingEnvironment(const char * fileName) { ScriptingEnvironment::ScriptingEnvironment(const char *file_name)
SimpleLogger().Write() << "Using script " << fileName; {
SimpleLogger().Write() << "Using script " << file_name;
// Create a new lua state // Create a new lua state
for(int i = 0; i < omp_get_max_threads(); ++i) { for (int i = 0; i < omp_get_max_threads(); ++i)
luaStateVector.push_back(luaL_newstate()); {
lua_state_vector.push_back(luaL_newstate());
} }
// Connect LuaBind to this lua state for all threads // Connect LuaBind to this lua state for all threads
#pragma omp parallel #pragma omp parallel
{ {
lua_State * myLuaState = getLuaStateForThreadID(omp_get_thread_num()); lua_State *lua_state = getLuaStateForThreadID(omp_get_thread_num());
luabind::open(myLuaState); luabind::open(lua_state);
//open utility libraries string library; // open utility libraries string library;
luaL_openlibs(myLuaState); luaL_openlibs(lua_state);
luaAddScriptFolderToLoadPath( myLuaState, fileName ); luaAddScriptFolderToLoadPath(lua_state, file_name);
// Add our function to the state's global scope // Add our function to the state's global scope
luabind::module(myLuaState) [ luabind::module(lua_state)[
luabind::def("print", LUA_print<std::string>), luabind::def("print", LUA_print<std::string>),
luabind::def("durationIsValid", durationIsValid), luabind::def("durationIsValid", durationIsValid),
luabind::def("parseDuration", parseDuration) luabind::def("parseDuration", parseDuration)
]; ];
luabind::module(myLuaState) [ luabind::module(lua_state)[luabind::class_<HashTable<std::string, std::string>>("keyVals")
luabind::class_<HashTable<std::string, std::string> >("keyVals") .def("Add", &HashTable<std::string, std::string>::Add)
.def("Add", &HashTable<std::string, std::string>::Add) .def("Find", &HashTable<std::string, std::string>::Find)
.def("Find", &HashTable<std::string, std::string>::Find) .def("Holds", &HashTable<std::string, std::string>::Holds)];
.def("Holds", &HashTable<std::string, std::string>::Holds)
];
luabind::module(myLuaState) [ luabind::module(lua_state)[luabind::class_<ImportNode>("Node")
luabind::class_<ImportNode>("Node") .def(luabind::constructor<>())
.def(luabind::constructor<>()) .def_readwrite("lat", &ImportNode::lat)
.def_readwrite("lat", &ImportNode::lat) .def_readwrite("lon", &ImportNode::lon)
.def_readwrite("lon", &ImportNode::lon) .def_readonly("id", &ImportNode::id)
.def_readonly("id", &ImportNode::id) .def_readwrite("bollard", &ImportNode::bollard)
.def_readwrite("bollard", &ImportNode::bollard) .def_readwrite("traffic_light", &ImportNode::trafficLight)
.def_readwrite("traffic_light", &ImportNode::trafficLight) .def_readwrite("tags", &ImportNode::keyVals)];
.def_readwrite("tags", &ImportNode::keyVals)
];
luabind::module(myLuaState) [ luabind::module(lua_state)
luabind::class_<ExtractionWay>("Way") [luabind::class_<ExtractionWay>("Way")
.def(luabind::constructor<>()) .def(luabind::constructor<>())
.def_readonly("id", &ExtractionWay::id) .def_readonly("id", &ExtractionWay::id)
.def_readwrite("name", &ExtractionWay::name) .def_readwrite("name", &ExtractionWay::name)
.def_readwrite("speed", &ExtractionWay::speed) .def_readwrite("speed", &ExtractionWay::speed)
.def_readwrite("backward_speed", &ExtractionWay::backward_speed) .def_readwrite("backward_speed", &ExtractionWay::backward_speed)
.def_readwrite("duration", &ExtractionWay::duration) .def_readwrite("duration", &ExtractionWay::duration)
.def_readwrite("type", &ExtractionWay::type) .def_readwrite("type", &ExtractionWay::type)
.def_readwrite("access", &ExtractionWay::access) .def_readwrite("access", &ExtractionWay::access)
.def_readwrite("roundabout", &ExtractionWay::roundabout) .def_readwrite("roundabout", &ExtractionWay::roundabout)
.def_readwrite("is_access_restricted", &ExtractionWay::isAccessRestricted) .def_readwrite("is_access_restricted", &ExtractionWay::isAccessRestricted)
.def_readwrite("ignore_in_grid", &ExtractionWay::ignoreInGrid) .def_readwrite("ignore_in_grid", &ExtractionWay::ignoreInGrid)
.def_readwrite("tags", &ExtractionWay::keyVals) .def_readwrite("tags", &ExtractionWay::keyVals)
.def_readwrite("direction", &ExtractionWay::direction) .def_readwrite("direction", &ExtractionWay::direction)
.enum_("constants") [ .enum_("constants")[
luabind::value("notSure", 0), luabind::value("notSure", 0),
luabind::value("oneway", 1), luabind::value("oneway", 1),
luabind::value("bidirectional", 2), luabind::value("bidirectional", 2),
luabind::value("opposite", 3) luabind::value("opposite", 3)
] ]];
];
// fails on c++11/OS X 10.9 // fails on c++11/OS X 10.9
luabind::module(myLuaState) [ luabind::module(lua_state)[luabind::class_<std::vector<std::string>>("vector").def(
luabind::class_<std::vector<std::string> >("vector") "Add",
.def("Add", static_cast<void (std::vector<std::string>::*)(const std::string&)>(&std::vector<std::string>::push_back)) static_cast<void (std::vector<std::string>::*)(const std::string &)>(
]; &std::vector<std::string>::push_back))];
if(0 != luaL_dofile(myLuaState, fileName) ) { if (0 != luaL_dofile(lua_state, file_name))
{
throw OSRMException("ERROR occured in scripting block"); throw OSRMException("ERROR occured in scripting block");
} }
} }
} }
ScriptingEnvironment::~ScriptingEnvironment() { ScriptingEnvironment::~ScriptingEnvironment()
for(unsigned i = 0; i < luaStateVector.size(); ++i) { {
// luaStateVector[i]; for (unsigned i = 0; i < lua_state_vector.size(); ++i)
{
// lua_state_vector[i];
} }
} }
lua_State * ScriptingEnvironment::getLuaStateForThreadID(const int id) { lua_State *ScriptingEnvironment::getLuaStateForThreadID(const int id) { return lua_state_vector[id]; }
return luaStateVector[id];
}

11
Extractor/ScriptingEnvironment.h
View File

@ -32,15 +32,16 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
struct lua_State; struct lua_State;
class ScriptingEnvironment { class ScriptingEnvironment
public: {
public:
ScriptingEnvironment(); ScriptingEnvironment();
explicit ScriptingEnvironment(const char * fileName); explicit ScriptingEnvironment(const char *file_name);
virtual ~ScriptingEnvironment(); virtual ~ScriptingEnvironment();
lua_State * getLuaStateForThreadID(const int); lua_State *getLuaStateForThreadID(const int);
std::vector<lua_State *> luaStateVector; std::vector<lua_State *> lua_state_vector;
}; };
#endif /* SCRIPTINGENVIRONMENT_H_ */ #endif /* SCRIPTINGENVIRONMENT_H_ */

9
Extractor/XMLParser.cpp
View File

@ -71,19 +71,19 @@ bool XMLParser::Parse()
{ {
ImportNode n = ReadXMLNode(); ImportNode n = ReadXMLNode();
ParseNodeInLua(n, lua_state); ParseNodeInLua(n, lua_state);
extractor_callbacks->nodeFunction(n); extractor_callbacks->ProcessNode(n);
} }
if (xmlStrEqual(currentName, (const xmlChar *)"way") == 1) if (xmlStrEqual(currentName, (const xmlChar *)"way") == 1)
{ {
ExtractionWay way = ReadXMLWay(); ExtractionWay way = ReadXMLWay();
ParseWayInLua(way, lua_state); ParseWayInLua(way, lua_state);
extractor_callbacks->wayFunction(way); extractor_callbacks->ProcessWay(way);
} }
if (use_turn_restrictions && xmlStrEqual(currentName, (const xmlChar *)"relation") == 1) if (use_turn_restrictions && xmlStrEqual(currentName, (const xmlChar *)"relation") == 1)
{ {
InputRestrictionContainer r = ReadXMLRestriction(); InputRestrictionContainer r = ReadXMLRestriction();
if ((UINT_MAX != r.fromWay) && !extractor_callbacks->restrictionFunction(r)) if ((UINT_MAX != r.fromWay) && !extractor_callbacks->ProcessRestriction(r))
{ {
std::cerr << "[XMLParser] restriction not parsed" << std::endl; std::cerr << "[XMLParser] restriction not parsed" << std::endl;
} }
@ -328,8 +328,7 @@ ImportNode XMLParser::ReadXMLNode()
xmlChar *value = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"v"); xmlChar *value = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"v");
if (k != NULL && value != NULL) if (k != NULL && value != NULL)
{ {
node.keyVals.emplace(std::string((char *)(k)), node.keyVals.emplace(std::string((char *)(k)), std::string((char *)(value)));
std::string((char *)(value)));
} }
if (k != NULL) if (k != NULL)
{ {

8
extractor.cpp
View File

@ -221,7 +221,7 @@ int main(int argc, char *argv[])
ExtractionContainers extraction_containers; ExtractionContainers extraction_containers;
string_map[""] = 0; string_map[""] = 0;
extractor_callbacks = new ExtractorCallbacks(&extraction_containers, &string_map); extractor_callbacks = new ExtractorCallbacks(extraction_containers, string_map);
BaseParser *parser; BaseParser *parser;
if (file_has_pbf_format) if (file_has_pbf_format)
{ {
@ -241,6 +241,9 @@ int main(int argc, char *argv[])
std::chrono::steady_clock::now(); std::chrono::steady_clock::now();
parser->Parse(); parser->Parse();
delete parser;
delete extractor_callbacks;
std::chrono::duration<double> parsing_duration = std::chrono::duration<double> parsing_duration =
std::chrono::steady_clock::now() - parsing_start_time; std::chrono::steady_clock::now() - parsing_start_time;
SimpleLogger().Write() << "Parsing finished after " << parsing_duration.count() SimpleLogger().Write() << "Parsing finished after " << parsing_duration.count()
@ -254,9 +257,6 @@ int main(int argc, char *argv[])
extraction_containers.PrepareData(output_file_name, restriction_fileName); extraction_containers.PrepareData(output_file_name, restriction_fileName);
delete parser;
delete extractor_callbacks;
std::chrono::duration<double> extraction_duration = std::chrono::duration<double> extraction_duration =
std::chrono::steady_clock::now() - startup_time; std::chrono::steady_clock::now() - startup_time;
SimpleLogger().Write() << "extraction finished after " << extraction_duration.count() SimpleLogger().Write() << "extraction finished after " << extraction_duration.count()