Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

refactor Extractor/

Browse Source
This commit is contained in:
Dennis Luxen 2014-05-09 16:17:31 +02:00
parent 7e639d6bc1
commit 0fccd0f0d2
16 changed files with 1171 additions and 1193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
Extractor/BaseParser.cpp
View File

@ -38,99 +38,99 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <boost/algorithm/string/regex.hpp>
#include <boost/regex.hpp>
BaseParser::BaseParser(
ExtractorCallbacks * extractor_callbacks,
ScriptingEnvironment & scripting_environment
) : extractor_callbacks(extractor_callbacks),
BaseParser::BaseParser(ExtractorCallbacks *extractor_callbacks,
ScriptingEnvironment &scripting_environment)
: extractor_callbacks(extractor_callbacks),
lua_state(scripting_environment.getLuaStateForThreadID(0)),
scripting_environment(scripting_environment),
use_turn_restrictions(true)
scripting_environment(scripting_environment), use_turn_restrictions(true)
{
ReadUseRestrictionsSetting();
ReadRestrictionExceptions();
}
void BaseParser::ReadUseRestrictionsSetting() {
if( 0 != luaL_dostring( lua_state, "return use_turn_restrictions\n") ) {
void BaseParser::ReadUseRestrictionsSetting()
{
if (0 != luaL_dostring(lua_state, "return use_turn_restrictions\n"))
{
throw OSRMException("ERROR occured in scripting block");
}
if( lua_isboolean( lua_state, -1) ) {
if (lua_isboolean(lua_state, -1))
{
use_turn_restrictions = lua_toboolean(lua_state, -1);
}
if( use_turn_restrictions ) {
if (use_turn_restrictions)
{
SimpleLogger().Write() << "Using turn restrictions";
} else {
}
else
{
SimpleLogger().Write() << "Ignoring turn restrictions";
}
}
void BaseParser::ReadRestrictionExceptions() {
if(lua_function_exists(lua_state, "get_exceptions" )) {
//get list of turn restriction exceptions
void BaseParser::ReadRestrictionExceptions()
{
if (lua_function_exists(lua_state, "get_exceptions"))
{
// get list of turn restriction exceptions
luabind::call_function<void>(
lua_state,
"get_exceptions",
boost::ref(restriction_exceptions)
);
lua_state, "get_exceptions", boost::ref(restriction_exceptions));
const unsigned exception_count = restriction_exceptions.size();
SimpleLogger().Write() <<
"Found " << exception_count << " exceptions to turn restrictions:";
for(const std::string & str : restriction_exceptions) {
SimpleLogger().Write() << "Found " << exception_count
<< " exceptions to turn restrictions:";
for (const std::string &str : restriction_exceptions)
{
SimpleLogger().Write() << " " << str;
}
} else {
}
else
{
SimpleLogger().Write() << "Found no exceptions to turn restrictions";
}
}
void BaseParser::report_errors(lua_State *L, const int status) const {
if( 0!=status ) {
std::cerr << "-- " << lua_tostring(L, -1) << std::endl;
lua_pop(L, 1); // remove error message
void BaseParser::report_errors(lua_State *lua_state, const int status) const
{
if (0 != status)
{
std::cerr << "-- " << lua_tostring(lua_state, -1) << std::endl;
lua_pop(lua_state, 1); // remove error message
}
}
void BaseParser::ParseNodeInLua(ImportNode& n, lua_State* local_lua_state) {
luabind::call_function<void>(
local_lua_state,
"node_function",
boost::ref(n)
);
void BaseParser::ParseNodeInLua(ImportNode &node, lua_State *local_lua_state)
{
luabind::call_function<void>(local_lua_state, "node_function", boost::ref(node));
}
void BaseParser::ParseWayInLua(ExtractionWay& w, lua_State* local_lua_state) {
luabind::call_function<void>(
local_lua_state,
"way_function",
boost::ref(w)
);
void BaseParser::ParseWayInLua(ExtractionWay &way, lua_State *local_lua_state)
{
luabind::call_function<void>(local_lua_state, "way_function", boost::ref(way));
}
bool BaseParser::ShouldIgnoreRestriction(
const std::string & except_tag_string
) const {
//should this restriction be ignored? yes if there's an overlap between:
bool BaseParser::ShouldIgnoreRestriction(const std::string &except_tag_string) const
{
// should this restriction be ignored? yes if there's an overlap between:
// a) the list of modes in the except tag of the restriction
// (except_tag_string), eg: except=bus;bicycle
// b) the lua profile defines a hierachy of modes,
// eg: [access, vehicle, bicycle]
if( except_tag_string.empty() ) {
if (except_tag_string.empty())
{
return false;
}
//Be warned, this is quadratic work here, but we assume that
//only a few exceptions are actually defined.
// Be warned, this is quadratic work here, but we assume that
// only a few exceptions are actually defined.
std::vector<std::string> exceptions;
boost::algorithm::split_regex(exceptions, except_tag_string, boost::regex("[;][ ]*"));
for (std::string& current_string : exceptions) {
std::vector<std::string>::const_iterator string_iterator;
string_iterator = std::find(
restriction_exceptions.begin(),
restriction_exceptions.end(),
current_string
);
if( restriction_exceptions.end() != string_iterator ) {
for (std::string &current_string : exceptions)
{
const auto string_iterator =
std::find(restriction_exceptions.begin(), restriction_exceptions.end(), current_string);
if (restriction_exceptions.end() != string_iterator)
{
return true;
}
}

31
Extractor/BaseParser.h
View File

@ -37,32 +37,29 @@ class ScriptingEnvironment;
struct ExtractionWay;
struct ImportNode;
class BaseParser {
public:
class BaseParser
{
public:
BaseParser() = delete;
BaseParser(const BaseParser&) = delete;
BaseParser(
ExtractorCallbacks * extractor_callbacks,
ScriptingEnvironment & scripting_environment
);
BaseParser(const BaseParser &) = delete;
BaseParser(ExtractorCallbacks *extractor_callbacks,
ScriptingEnvironment &scripting_environment);
virtual ~BaseParser() {}
virtual bool ReadHeader() = 0;
virtual bool Parse() = 0;
virtual void ParseNodeInLua(ImportNode & n, lua_State* thread_lua_state);
virtual void ParseWayInLua(ExtractionWay & n, lua_State* thread_lua_state);
virtual void report_errors(lua_State * lua_state, const int status) const;
virtual void ParseNodeInLua(ImportNode &node, lua_State *lua_state);
virtual void ParseWayInLua(ExtractionWay &way, lua_State *lua_state);
virtual void report_errors(lua_State *lua_state, const int status) const;
protected:
protected:
virtual void ReadUseRestrictionsSetting();
virtual void ReadRestrictionExceptions();
virtual bool ShouldIgnoreRestriction(
const std::string & except_tag_string
) const;
virtual bool ShouldIgnoreRestriction(const std::string &except_tag_string) const;
ExtractorCallbacks * extractor_callbacks;
lua_State * lua_state;
ScriptingEnvironment & scripting_environment;
ExtractorCallbacks *extractor_callbacks;
lua_State *lua_state;
ScriptingEnvironment &scripting_environment;
std::vector<std::string> restriction_exceptions;
bool use_turn_restrictions;
};

428
Extractor/ExtractionContainers.cpp
View File

@ -38,13 +38,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <chrono>
ExtractionContainers::ExtractionContainers() {
//Check if stxxl can be instantiated
ExtractionContainers::ExtractionContainers()
{
// Check if stxxl can be instantiated
stxxl::vector<unsigned> dummy_vector;
name_list.push_back("");
}
ExtractionContainers::~ExtractionContainers() {
ExtractionContainers::~ExtractionContainers()
{
used_node_id_list.clear();
all_nodes_list.clear();
all_edges_list.clear();
@ -53,219 +55,205 @@ ExtractionContainers::~ExtractionContainers() {
way_start_end_id_list.clear();
}
void ExtractionContainers::PrepareData(
const std::string & output_file_name,
const std::string & restrictions_file_name
) {
try {
void ExtractionContainers::PrepareData(const std::string &output_file_name,
const std::string &restrictions_file_name)
{
try
{
unsigned number_of_used_nodes = 0;
unsigned number_of_used_edges = 0;
std::chrono::time_point<std::chrono::steady_clock> time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting used nodes ... " << std::flush;
stxxl::sort(
used_node_id_list.begin(),
used_node_id_list.end(),
Cmp(),
4294967296
);
stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), 4294967296);
std::chrono::time_point<std::chrono::steady_clock> time2 = std::chrono::steady_clock::now();
std::chrono::duration<double> elapsed_seconds = time2-time1;
std::chrono::duration<double> elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush;
stxxl::vector<NodeID>::iterator NewEnd = std::unique ( used_node_id_list.begin(),used_node_id_list.end() ) ;
used_node_id_list.resize ( NewEnd - used_node_id_list.begin() );
auto new_end = std::unique(used_node_id_list.begin(), used_node_id_list.end());
used_node_id_list.resize(new_end - used_node_id_list.begin());
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting all nodes ... " << std::flush;
stxxl::sort(
all_nodes_list.begin(),
all_nodes_list.end(),
CmpNodeByID(),
4294967296
);
stxxl::sort(all_nodes_list.begin(), all_nodes_list.end(), CmpNodeByID(), 4294967296);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting used ways ... " << std::flush;
stxxl::sort(
way_start_end_id_list.begin(),
way_start_end_id_list.end(),
CmpWayByID(),
4294967296
);
way_start_end_id_list.begin(), way_start_end_id_list.end(), CmpWayByID(), 4294967296);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] Sorting restrictions. by from... " << std::flush;
stxxl::sort(
restrictions_list.begin(),
stxxl::sort(restrictions_list.begin(),
restrictions_list.end(),
CmpRestrictionContainerByFrom(),
4294967296
);
4294967296);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] Fixing restriction starts ... " << std::flush;
STXXLRestrictionsVector::iterator restrictions_iterator = restrictions_list.begin();
STXXLWayIDStartEndVector::iterator way_start_and_end_iterator = way_start_end_id_list.begin();
auto restrictions_iterator = restrictions_list.begin();
auto way_start_and_end_iterator = way_start_end_id_list.begin();
while(
way_start_and_end_iterator != way_start_end_id_list.end() &&
restrictions_iterator != restrictions_list.end()
) {
if(way_start_and_end_iterator->wayID < restrictions_iterator->fromWay){
while (way_start_and_end_iterator != way_start_end_id_list.end() &&
restrictions_iterator != restrictions_list.end())
{
if (way_start_and_end_iterator->wayID < restrictions_iterator->fromWay)
{
++way_start_and_end_iterator;
continue;
}
if(way_start_and_end_iterator->wayID > restrictions_iterator->fromWay) {
if (way_start_and_end_iterator->wayID > restrictions_iterator->fromWay)
{
++restrictions_iterator;
continue;
}
BOOST_ASSERT(way_start_and_end_iterator->wayID == restrictions_iterator->fromWay);
NodeID via_node_id = restrictions_iterator->restriction.viaNode;
const NodeID via_node_id = restrictions_iterator->restriction.viaNode;
if(way_start_and_end_iterator->firstStart == via_node_id) {
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->firstTarget;
} else if(way_start_and_end_iterator->firstTarget == via_node_id) {
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->firstStart;
} else if(way_start_and_end_iterator->lastStart == via_node_id) {
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->lastTarget;
} else if(way_start_and_end_iterator->lastTarget == via_node_id) {
if (way_start_and_end_iterator->firstStart == via_node_id)
{
restrictions_iterator->restriction.fromNode =
way_start_and_end_iterator->firstTarget;
}
else if (way_start_and_end_iterator->firstTarget == via_node_id)
{
restrictions_iterator->restriction.fromNode =
way_start_and_end_iterator->firstStart;
}
else if (way_start_and_end_iterator->lastStart == via_node_id)
{
restrictions_iterator->restriction.fromNode =
way_start_and_end_iterator->lastTarget;
}
else if (way_start_and_end_iterator->lastTarget == via_node_id)
{
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->lastStart;
}
++restrictions_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting restrictions. by to ... " << std::flush;
stxxl::sort(
restrictions_list.begin(),
stxxl::sort(restrictions_list.begin(),
restrictions_list.end(),
CmpRestrictionContainerByTo(),
4294967296
);
4294967296);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
unsigned usableRestrictionsCounter(0);
unsigned number_of_useable_restrictions = 0;
std::cout << "[extractor] Fixing restriction ends ... " << std::flush;
restrictions_iterator = restrictions_list.begin();
way_start_and_end_iterator = way_start_end_id_list.begin();
while(
way_start_and_end_iterator != way_start_end_id_list.end() &&
restrictions_iterator != restrictions_list.end()
) {
if(way_start_and_end_iterator->wayID < restrictions_iterator->toWay){
while (way_start_and_end_iterator != way_start_end_id_list.end() &&
restrictions_iterator != restrictions_list.end())
{
if (way_start_and_end_iterator->wayID < restrictions_iterator->toWay)
{
++way_start_and_end_iterator;
continue;
}
if(way_start_and_end_iterator->wayID > restrictions_iterator->toWay) {
if (way_start_and_end_iterator->wayID > restrictions_iterator->toWay)
{
++restrictions_iterator;
continue;
}
NodeID via_node_id = restrictions_iterator->restriction.viaNode;
if(way_start_and_end_iterator->lastStart == via_node_id) {
if (way_start_and_end_iterator->lastStart == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastTarget;
} else if(way_start_and_end_iterator->lastTarget == via_node_id) {
}
else if (way_start_and_end_iterator->lastTarget == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastStart;
} else if(way_start_and_end_iterator->firstStart == via_node_id) {
}
else if (way_start_and_end_iterator->firstStart == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstTarget;
} else if(way_start_and_end_iterator->firstTarget == via_node_id) {
}
else if (way_start_and_end_iterator->firstTarget == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstStart;
}
if(
UINT_MAX != restrictions_iterator->restriction.fromNode &&
UINT_MAX != restrictions_iterator->restriction.toNode
) {
++usableRestrictionsCounter;
if (UINT_MAX != restrictions_iterator->restriction.fromNode &&
UINT_MAX != restrictions_iterator->restriction.toNode)
{
++number_of_useable_restrictions;
}
++restrictions_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
SimpleLogger().Write() << "usable restrictions: " << usableRestrictionsCounter;
//serialize restrictions
SimpleLogger().Write() << "usable restrictions: " << number_of_useable_restrictions;
// serialize restrictions
std::ofstream restrictions_out_stream;
restrictions_out_stream.open(restrictions_file_name.c_str(), std::ios::binary);
restrictions_out_stream.write((char*)&uuid, sizeof(UUID));
restrictions_out_stream.write(
(char*)&usableRestrictionsCounter,
sizeof(unsigned)
);
for(
restrictions_iterator = restrictions_list.begin();
restrictions_out_stream.write((char *)&uuid, sizeof(UUID));
restrictions_out_stream.write((char *)&number_of_useable_restrictions, sizeof(unsigned));
for (restrictions_iterator = restrictions_list.begin();
restrictions_iterator != restrictions_list.end();
++restrictions_iterator
) {
if(
UINT_MAX != restrictions_iterator->restriction.fromNode &&
UINT_MAX != restrictions_iterator->restriction.toNode
) {
restrictions_out_stream.write(
(char *)&(restrictions_iterator->restriction),
sizeof(TurnRestriction)
);
++restrictions_iterator)
{
if (UINT_MAX != restrictions_iterator->restriction.fromNode &&
UINT_MAX != restrictions_iterator->restriction.toNode)
{
restrictions_out_stream.write((char *)&(restrictions_iterator->restriction),
sizeof(TurnRestriction));
}
}
restrictions_out_stream.close();
std::ofstream file_out_stream;
file_out_stream.open(output_file_name.c_str(), std::ios::binary);
file_out_stream.write((char*)&uuid, sizeof(UUID));
file_out_stream.write((char*)&number_of_used_nodes, sizeof(unsigned));
file_out_stream.write((char *)&uuid, sizeof(UUID));
file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush;
//identify all used nodes by a merging step of two sorted lists
STXXLNodeVector::iterator node_iterator = all_nodes_list.begin();
STXXLNodeIDVector::iterator node_id_iterator = used_node_id_list.begin();
while(
node_id_iterator != used_node_id_list.end() &&
node_iterator != all_nodes_list.end()
) {
if(*node_id_iterator < node_iterator->id){
// identify all used nodes by a merging step of two sorted lists
auto node_iterator = all_nodes_list.begin();
auto node_id_iterator = used_node_id_list.begin();
while (node_id_iterator != used_node_id_list.end() && node_iterator != all_nodes_list.end())
{
if (*node_id_iterator < node_iterator->id)
{
++node_id_iterator;
continue;
}
if(*node_id_iterator > node_iterator->id) {
if (*node_id_iterator > node_iterator->id)
{
++node_iterator;
continue;
}
BOOST_ASSERT( *node_id_iterator == node_iterator->id);
BOOST_ASSERT(*node_id_iterator == node_iterator->id);
file_out_stream.write(
(char*)&(*node_iterator),
sizeof(ExternalMemoryNode)
);
file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode));
++number_of_used_nodes;
++node_id_iterator;
@ -273,14 +261,13 @@ void ExtractionContainers::PrepareData(
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] setting number of nodes ... " << std::flush;
std::ios::pos_type previous_file_position = file_out_stream.tellp();
file_out_stream.seekp(std::ios::beg+sizeof(UUID));
file_out_stream.write((char*)&number_of_used_nodes, sizeof(unsigned));
file_out_stream.seekp(std::ios::beg + sizeof(UUID));
file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
file_out_stream.seekp(previous_file_position);
std::cout << "ok" << std::endl;
@ -288,218 +275,175 @@ void ExtractionContainers::PrepareData(
// Sort edges by start.
std::cout << "[extractor] Sorting edges by start ... " << std::flush;
stxxl::sort(
all_edges_list.begin(),
all_edges_list.end(),
CmpEdgeByStartID(),
4294967296
);
stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartID(), 4294967296);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Setting start coords ... " << std::flush;
file_out_stream.write((char*)&number_of_used_edges, sizeof(unsigned));
file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
// Traverse list of edges and nodes in parallel and set start coord
node_iterator = all_nodes_list.begin();
STXXLEdgeVector::iterator edge_iterator = all_edges_list.begin();
while(
edge_iterator != all_edges_list.end() &&
node_iterator != all_nodes_list.end()
) {
if(edge_iterator->start < node_iterator->id){
auto edge_iterator = all_edges_list.begin();
while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
{
if (edge_iterator->start < node_iterator->id)
{
++edge_iterator;
continue;
}
if(edge_iterator->start > node_iterator->id) {
if (edge_iterator->start > node_iterator->id)
{
node_iterator++;
continue;
}
BOOST_ASSERT(edge_iterator->start == node_iterator->id);
edge_iterator->startCoord.lat = node_iterator->lat;
edge_iterator->startCoord.lon = node_iterator->lon;
edge_iterator->source_coordinate.lat = node_iterator->lat;
edge_iterator->source_coordinate.lon = node_iterator->lon;
++edge_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
// Sort Edges by target
std::cout << "[extractor] Sorting edges by target ... " << std::flush;
stxxl::sort(
all_edges_list.begin(),
all_edges_list.end(),
CmpEdgeByTargetID(),
4294967296
);
stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByTargetID(), 4294967296);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Setting target coords ... " << std::flush;
// Traverse list of edges and nodes in parallel and set target coord
node_iterator = all_nodes_list.begin();
edge_iterator = all_edges_list.begin();
while(
edge_iterator != all_edges_list.end() &&
node_iterator != all_nodes_list.end()
) {
if(edge_iterator->target < node_iterator->id){
while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
{
if (edge_iterator->target < node_iterator->id)
{
++edge_iterator;
continue;
}
if(edge_iterator->target > node_iterator->id) {
if (edge_iterator->target > node_iterator->id)
{
++node_iterator;
continue;
}
BOOST_ASSERT(edge_iterator->target == node_iterator->id);
if(edge_iterator->startCoord.lat != INT_MIN && edge_iterator->startCoord.lon != INT_MIN) {
edge_iterator->targetCoord.lat = node_iterator->lat;
edge_iterator->targetCoord.lon = node_iterator->lon;
const double distance = FixedPointCoordinate::ApproximateDistance(
edge_iterator->startCoord.lat,
edge_iterator->startCoord.lon,
node_iterator->lat,
node_iterator->lon
);
if (edge_iterator->source_coordinate.lat != INT_MIN &&
edge_iterator->source_coordinate.lon != INT_MIN)
{
BOOST_ASSERT(edge_iterator->speed != -1);
const double weight = ( distance * 10. ) / (edge_iterator->speed / 3.6);
int integer_weight = std::max( 1, (int)std::floor((edge_iterator->isDurationSet ? edge_iterator->speed : weight)+.5) );
int integer_distance = std::max( 1, (int)distance );
BOOST_ASSERT(edge_iterator->type >= 0);
edge_iterator->target_coordinate.lat = node_iterator->lat;
edge_iterator->target_coordinate.lon = node_iterator->lon;
const double distance = FixedPointCoordinate::ApproximateEuclideanDistance(
edge_iterator->source_coordinate.lat,
edge_iterator->source_coordinate.lon,
node_iterator->lat,
node_iterator->lon);
const double weight = (distance * 10.) / (edge_iterator->speed / 3.6);
int integer_weight = std::max(
1,
(int)std::floor(
(edge_iterator->is_duration_set ? edge_iterator->speed : weight) + .5));
int integer_distance = std::max(1, (int)distance);
short zero = 0;
short one = 1;
file_out_stream.write((char*)&edge_iterator->start, sizeof(unsigned));
file_out_stream.write((char*)&edge_iterator->target, sizeof(unsigned));
file_out_stream.write((char*)&integer_distance, sizeof(int));
switch(edge_iterator->direction) {
file_out_stream.write((char *)&edge_iterator->start, sizeof(unsigned));
file_out_stream.write((char *)&edge_iterator->target, sizeof(unsigned));
file_out_stream.write((char *)&integer_distance, sizeof(int));
switch (edge_iterator->direction)
{
case ExtractionWay::notSure:
file_out_stream.write((char*)&zero, sizeof(short));
file_out_stream.write((char *)&zero, sizeof(short));
break;
case ExtractionWay::oneway:
file_out_stream.write((char*)&one, sizeof(short));
file_out_stream.write((char *)&one, sizeof(short));
break;
case ExtractionWay::bidirectional:
file_out_stream.write((char*)&zero, sizeof(short));
file_out_stream.write((char *)&zero, sizeof(short));
break;
case ExtractionWay::opposite:
file_out_stream.write((char*)&one, sizeof(short));
file_out_stream.write((char *)&one, sizeof(short));
break;
default:
throw OSRMException("edge has broken direction");
}
file_out_stream.write(
(char*)&integer_weight, sizeof(int)
);
BOOST_ASSERT(edge_iterator->type >= 0);
file_out_stream.write(
(char*)&edge_iterator->type,
sizeof(short)
);
file_out_stream.write(
(char *) &edge_iterator->nameID,
sizeof(unsigned)
);
file_out_stream.write(
(char *) &edge_iterator->isRoundabout,
sizeof(bool)
);
file_out_stream.write(
(char *) &edge_iterator->ignoreInGrid,
sizeof(bool)
);
file_out_stream.write(
(char *) &edge_iterator->isAccessRestricted,
sizeof(bool)
);
file_out_stream.write(
(char *) &edge_iterator->isContraFlow,
sizeof(bool)
);
file_out_stream.write(
(char *) &edge_iterator->is_split,
sizeof(bool)
);
file_out_stream.write((char *)&integer_weight, sizeof(int));
file_out_stream.write((char *)&edge_iterator->type, sizeof(short));
file_out_stream.write((char *)&edge_iterator->name_id, sizeof(unsigned));
file_out_stream.write((char *)&edge_iterator->is_roundabout, sizeof(bool));
file_out_stream.write((char *)&edge_iterator->is_in_tiny_cc, sizeof(bool));
file_out_stream.write((char *)&edge_iterator->is_access_restricted, sizeof(bool));
file_out_stream.write((char *)&edge_iterator->is_contra_flow, sizeof(bool));
file_out_stream.write((char *)&edge_iterator->is_split, sizeof(bool));
++number_of_used_edges;
}
++edge_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] setting number of edges ... " << std::flush;
file_out_stream.seekp(previous_file_position);
file_out_stream.write((char*)&number_of_used_edges, sizeof(unsigned));
file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
file_out_stream.close();
std::cout << "ok" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] writing street name index ... " << std::flush;
std::string name_file_streamName = (output_file_name + ".names");
boost::filesystem::ofstream name_file_stream(
name_file_streamName,
std::ios::binary
);
boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary);
//write number of names
const unsigned number_of_names = name_list.size()+1;
// write number of names
const unsigned number_of_names = name_list.size() + 1;
name_file_stream.write((char *)&(number_of_names), sizeof(unsigned));
//compute total number of chars
// compute total number of chars
unsigned total_number_of_chars = 0;
for (const std::string & temp_string : name_list) {
for (const std::string &temp_string : name_list)
{
total_number_of_chars += temp_string.length();
}
//write total number of chars
name_file_stream.write(
(char *)&(total_number_of_chars),
sizeof(unsigned)
);
//write prefixe sums
// write total number of chars
name_file_stream.write((char *)&(total_number_of_chars), sizeof(unsigned));
// write prefixe sums
unsigned name_lengths_prefix_sum = 0;
for (const std::string & temp_string : name_list) {
name_file_stream.write(
(char *)&(name_lengths_prefix_sum),
sizeof(unsigned)
);
for (const std::string &temp_string : name_list)
{
name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
name_lengths_prefix_sum += temp_string.length();
}
//duplicate on purpose!
name_file_stream.write(
(char *)&(name_lengths_prefix_sum),
sizeof(unsigned)
);
// duplicate on purpose!
name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
//write all chars consecutively
for (const std::string & temp_string : name_list) {
// write all chars consecutively
for (const std::string &temp_string : name_list)
{
const unsigned string_length = temp_string.length();
name_file_stream.write(temp_string.c_str(), string_length);
}
name_file_stream.close();
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
SimpleLogger().Write() << "Processed " <<
number_of_used_nodes << " nodes and " <<
number_of_used_edges << " edges";
} catch ( const std::exception& e ) {
std::cerr << "Caught Execption:" << e.what() << std::endl;
SimpleLogger().Write() << "Processed " << number_of_used_nodes << " nodes and "
<< number_of_used_edges << " edges";
}
catch (const std::exception &e) { std::cerr << "Caught Execption:" << e.what() << std::endl; }
}

13
Extractor/ExtractionContainers.h
View File

@ -35,14 +35,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <stxxl/vector>
class ExtractionContainers {
public:
class ExtractionContainers
{
public:
typedef stxxl::vector<NodeID> STXXLNodeIDVector;
typedef stxxl::vector<ExternalMemoryNode> STXXLNodeVector;
typedef stxxl::vector<InternalExtractorEdge> STXXLEdgeVector;
typedef stxxl::vector<std::string> STXXLStringVector;
typedef stxxl::vector<InputRestrictionContainer> STXXLRestrictionsVector;
typedef stxxl::vector<_WayIDStartAndEndEdge> STXXLWayIDStartEndVector;
typedef stxxl::vector<WayIDStartAndEndEdge> STXXLWayIDStartEndVector;
STXXLNodeIDVector used_node_id_list;
STXXLNodeVector all_nodes_list;
@ -56,10 +57,8 @@ public:
virtual ~ExtractionContainers();
void PrepareData(
const std::string & output_file_name,
const std::string & restrictions_file_name
);
void PrepareData(const std::string &output_file_name,
const std::string &restrictions_file_name);
};
#endif /* EXTRACTIONCONTAINERS_H_ */

58
Extractor/ExtractionHelperFunctions.h
View File

@ -25,63 +25,65 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef EXTRACTIONHELPERFUNCTIONS_H_
#define EXTRACTIONHELPERFUNCTIONS_H_
#ifndef EXTRACTION_HELPER_FUNCTIONS_H
#define EXTRACTION_HELPER_FUNCTIONS_H
#include "../Util/StringUtil.h"
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string_regex.hpp>
#include <boost/regex.hpp>
#include <climits>
#include <limits>
namespace qi = boost::spirit::qi;
//TODO: Move into LUA
// TODO: Move into LUA
inline bool durationIsValid(const std::string &s) {
boost::regex e ("((\\d|\\d\\d):(\\d|\\d\\d):(\\d|\\d\\d))|((\\d|\\d\\d):(\\d|\\d\\d))|(\\d|\\d\\d)",boost::regex_constants::icase|boost::regex_constants::perl);
inline bool durationIsValid(const std::string &s)
{
boost::regex e(
"((\\d|\\d\\d):(\\d|\\d\\d):(\\d|\\d\\d))|((\\d|\\d\\d):(\\d|\\d\\d))|(\\d|\\d\\d)",
boost::regex_constants::icase | boost::regex_constants::perl);
std::vector< std::string > result;
boost::algorithm::split_regex( result, s, boost::regex( ":" ) ) ;
std::vector<std::string> result;
boost::algorithm::split_regex(result, s, boost::regex(":"));
bool matched = regex_match(s, e);
return matched;
}
inline unsigned parseDuration(const std::string &s) {
inline unsigned parseDuration(const std::string &s)
{
unsigned hours = 0;
unsigned minutes = 0;
unsigned seconds = 0;
boost::regex e ("((\\d|\\d\\d):(\\d|\\d\\d):(\\d|\\d\\d))|((\\d|\\d\\d):(\\d|\\d\\d))|(\\d|\\d\\d)",boost::regex_constants::icase|boost::regex_constants::perl);
boost::regex e(
"((\\d|\\d\\d):(\\d|\\d\\d):(\\d|\\d\\d))|((\\d|\\d\\d):(\\d|\\d\\d))|(\\d|\\d\\d)",
boost::regex_constants::icase | boost::regex_constants::perl);
std::vector< std::string > result;
boost::algorithm::split_regex( result, s, boost::regex( ":" ) ) ;
std::vector<std::string> result;
boost::algorithm::split_regex(result, s, boost::regex(":"));
bool matched = regex_match(s, e);
if(matched) {
if(1 == result.size()) {
if (matched)
{
if (1 == result.size())
{
minutes = stringToInt(result[0]);
}
if(2 == result.size()) {
if (2 == result.size())
{
minutes = stringToInt(result[1]);
hours = stringToInt(result[0]);
}
if(3 == result.size()) {
if (3 == result.size())
{
seconds = stringToInt(result[2]);
minutes = stringToInt(result[1]);
hours = stringToInt(result[0]);
}
return 10*(3600*hours+60*minutes+seconds);
return 10 * (3600 * hours + 60 * minutes + seconds);
}
return UINT_MAX;
return std::numeric_limits<unsigned>::max();
}
// inline int parseMaxspeed(std::string input) { //call-by-value on purpose.
// boost::algorithm::to_lower(input);
// int n = stringToInt(input);
// if (input.find("mph") != std::string::npos || input.find("mp/h") != std::string::npos) {
// n = (n*1609)/1000;
// }
// return n;
// }
#endif /* EXTRACTIONHELPERFUNCTIONS_H_ */
#endif // EXTRACTION_HELPER_FUNCTIONS_H_

27
Extractor/ExtractionWay.h
View File

@ -34,14 +34,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <string>
#include <vector>
struct ExtractionWay {
ExtractionWay() {
Clear();
}
struct ExtractionWay
{
ExtractionWay() { Clear(); }
inline void Clear(){
id = UINT_MAX;
nameID = UINT_MAX;
inline void Clear()
{
id = SPECIAL_NODEID;
nameID = INVALID_NAMEID;
path.clear();
keyVals.clear();
direction = ExtractionWay::notSure;
@ -55,9 +55,11 @@ struct ExtractionWay {
ignoreInGrid = false;
}
enum Directions {
notSure = 0, oneway, bidirectional, opposite
};
enum Directions
{ notSure = 0,
oneway,
bidirectional,
opposite };
unsigned id;
unsigned nameID;
double speed;
@ -70,9 +72,8 @@ struct ExtractionWay {
bool roundabout;
bool isAccessRestricted;
bool ignoreInGrid;
std::vector< NodeID > path;
std::vector<NodeID> path;
HashTable<std::string, std::string> keyVals;
};
#endif //EXTRACTION_WAY_H
#endif // EXTRACTION_WAY_H

137
Extractor/ExtractorCallbacks.cpp
View File

@ -38,77 +38,79 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <string>
#include <vector>
ExtractorCallbacks::ExtractorCallbacks()
:
string_map(NULL),
externalMemory(NULL)
{ }
ExtractorCallbacks::ExtractorCallbacks(
ExtractionContainers * ext,
boost::unordered_map<std::string, NodeID> * string_map
) :
string_map(string_map),
externalMemory(ext)
{ }
ExtractorCallbacks::~ExtractorCallbacks() { }
ExtractorCallbacks::ExtractorCallbacks(ExtractionContainers &extraction_containers,
boost::unordered_map<std::string, NodeID> &string_map)
: string_map(string_map), external_memory(extraction_containers)
{
}
/** warning: caller needs to take care of synchronization! */
void ExtractorCallbacks::nodeFunction(const ExternalMemoryNode &n) {
if(n.lat <= 85*COORDINATE_PRECISION && n.lat >= -85*COORDINATE_PRECISION) {
externalMemory->all_nodes_list.push_back(n);
void ExtractorCallbacks::ProcessNode(const ExternalMemoryNode &n)
{
if (n.lat <= 85 * COORDINATE_PRECISION && n.lat >= -85 * COORDINATE_PRECISION)
{
external_memory.all_nodes_list.push_back(n);
}
}
bool ExtractorCallbacks::restrictionFunction(const InputRestrictionContainer &r) {
externalMemory->restrictions_list.push_back(r);
bool ExtractorCallbacks::ProcessRestriction(const InputRestrictionContainer &restriction)
{
external_memory.restrictions_list.push_back(restriction);
return true;
}
/** warning: caller needs to take care of synchronization! */
void ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) {
if((0 < parsed_way.speed) || (0 < parsed_way.duration)) { //Only true if the way is specified by the speed profile
if(UINT_MAX == parsed_way.id){
SimpleLogger().Write(logDEBUG) <<
"found bogus way with id: " << parsed_way.id <<
" of size " << parsed_way.path.size();
void ExtractorCallbacks::ProcessWay(ExtractionWay &parsed_way)
{
if ((0 < parsed_way.speed) || (0 < parsed_way.duration))
{ // Only true if the way is specified by the speed profile
if (UINT_MAX == parsed_way.id)
{
SimpleLogger().Write(logDEBUG) << "found bogus way with id: " << parsed_way.id
<< " of size " << parsed_way.path.size();
return;
}
if(0 < parsed_way.duration) {
//TODO: iterate all way segments and set duration corresponding to the length of each segment
parsed_way.speed = parsed_way.duration/(parsed_way.path.size()-1);
if (0 < parsed_way.duration)
{
// TODO: iterate all way segments and set duration corresponding to the length of each
// segment
parsed_way.speed = parsed_way.duration / (parsed_way.path.size() - 1);
}
if(std::numeric_limits<double>::epsilon() >= std::abs(-1. - parsed_way.speed)){
SimpleLogger().Write(logDEBUG) <<
"found way with bogus speed, id: " << parsed_way.id;
if (std::numeric_limits<double>::epsilon() >= std::abs(-1. - parsed_way.speed))
{
SimpleLogger().Write(logDEBUG) << "found way with bogus speed, id: " << parsed_way.id;
return;
}
//Get the unique identifier for the street name
const boost::unordered_map<std::string, NodeID>::const_iterator & string_map_iterator = string_map->find(parsed_way.name);
if(string_map->end() == string_map_iterator) {
parsed_way.nameID = externalMemory->name_list.size();
externalMemory->name_list.push_back(parsed_way.name);
string_map->insert(std::make_pair(parsed_way.name, parsed_way.nameID));
} else {
// Get the unique identifier for the street name
const auto &string_map_iterator = string_map.find(parsed_way.name);
if (string_map.end() == string_map_iterator)
{
parsed_way.nameID = external_memory.name_list.size();
external_memory.name_list.push_back(parsed_way.name);
string_map.insert(std::make_pair(parsed_way.name, parsed_way.nameID));
}
else
{
parsed_way.nameID = string_map_iterator->second;
}
if(ExtractionWay::opposite == parsed_way.direction) {
std::reverse( parsed_way.path.begin(), parsed_way.path.end() );
if (ExtractionWay::opposite == parsed_way.direction)
{
std::reverse(parsed_way.path.begin(), parsed_way.path.end());
parsed_way.direction = ExtractionWay::oneway;
}
const bool split_bidirectional_edge = (parsed_way.backward_speed > 0) && (parsed_way.speed != parsed_way.backward_speed);
const bool split_bidirectional_edge =
(parsed_way.backward_speed > 0) && (parsed_way.speed != parsed_way.backward_speed);
for(std::vector< NodeID >::size_type n = 0; n < parsed_way.path.size()-1; ++n) {
externalMemory->all_edges_list.push_back(
InternalExtractorEdge(
for (unsigned n = 0; n < parsed_way.path.size() - 1; ++n)
{
external_memory.all_edges_list.push_back(InternalExtractorEdge(
parsed_way.path[n],
parsed_way.path[n+1],
parsed_way.path[n + 1],
parsed_way.type,
(split_bidirectional_edge ? ExtractionWay::oneway : parsed_way.direction),
parsed_way.speed,
@ -118,23 +120,27 @@ void ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) {
(0 < parsed_way.duration),
parsed_way.isAccessRestricted,
false,
split_bidirectional_edge
)
);
externalMemory->used_node_id_list.push_back(parsed_way.path[n]);
split_bidirectional_edge));
external_memory.used_node_id_list.push_back(parsed_way.path[n]);
}
externalMemory->used_node_id_list.push_back(parsed_way.path.back());
external_memory.used_node_id_list.push_back(parsed_way.path.back());
//The following information is needed to identify start and end segments of restrictions
externalMemory->way_start_end_id_list.push_back(_WayIDStartAndEndEdge(parsed_way.id, parsed_way.path[0], parsed_way.path[1], parsed_way.path[parsed_way.path.size()-2], parsed_way.path.back()));
// The following information is needed to identify start and end segments of restrictions
external_memory.way_start_end_id_list.push_back(
WayIDStartAndEndEdge(parsed_way.id,
parsed_way.path[0],
parsed_way.path[1],
parsed_way.path[parsed_way.path.size() - 2],
parsed_way.path.back()));
if(split_bidirectional_edge) { //Only true if the way should be split
std::reverse( parsed_way.path.begin(), parsed_way.path.end() );
for(std::vector< NodeID >::size_type n = 0; n < parsed_way.path.size()-1; ++n) {
externalMemory->all_edges_list.push_back(
InternalExtractorEdge(
parsed_way.path[n],
parsed_way.path[n+1],
if (split_bidirectional_edge)
{ // Only true if the way should be split
std::reverse(parsed_way.path.begin(), parsed_way.path.end());
for (std::vector<NodeID>::size_type n = 0; n < parsed_way.path.size() - 1; ++n)
{
external_memory.all_edges_list.push_back(
InternalExtractorEdge(parsed_way.path[n],
parsed_way.path[n + 1],
parsed_way.type,
ExtractionWay::oneway,
parsed_way.backward_speed,
@ -144,11 +150,14 @@ void ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) {
(0 < parsed_way.duration),
parsed_way.isAccessRestricted,
(ExtractionWay::oneway == parsed_way.direction),
split_bidirectional_edge
)
);
split_bidirectional_edge));
}
externalMemory->way_start_end_id_list.push_back(_WayIDStartAndEndEdge(parsed_way.id, parsed_way.path[0], parsed_way.path[1], parsed_way.path[parsed_way.path.size()-2], parsed_way.path.back()));
external_memory.way_start_end_id_list.push_back(
WayIDStartAndEndEdge(parsed_way.id,
parsed_way.path[0],
parsed_way.path[1],
parsed_way.path[parsed_way.path.size() - 2],
parsed_way.path.back()));
}
}
}

41
Extractor/ExtractorCallbacks.h
View File

@ -25,8 +25,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef EXTRACTORCALLBACKS_H_
#define EXTRACTORCALLBACKS_H_
#ifndef EXTRACTOR_CALLBACKS_H
#define EXTRACTOR_CALLBACKS_H
#include "../typedefs.h"
@ -38,29 +38,26 @@ class ExtractionContainers;
struct ExtractionWay;
struct InputRestrictionContainer;
class ExtractorCallbacks{
private:
class ExtractorCallbacks
{
private:
boost::unordered_map<std::string, NodeID> &string_map;
ExtractionContainers &external_memory;
boost::unordered_map<std::string, NodeID> * string_map;
ExtractionContainers * externalMemory;
public:
ExtractorCallbacks() = delete;
ExtractorCallbacks(const ExtractorCallbacks &) = delete;
explicit ExtractorCallbacks(ExtractionContainers &extraction_containers,
boost::unordered_map<std::string, NodeID> &string_map);
ExtractorCallbacks();
public:
explicit ExtractorCallbacks(
ExtractionContainers * ext,
boost::unordered_map<std::string, NodeID> * string_map
);
// warning: caller needs to take care of synchronization!
void ProcessNode(const ExternalMemoryNode &node);
~ExtractorCallbacks();
/** warning: caller needs to take care of synchronization! */
void nodeFunction(const ExternalMemoryNode &n);
bool restrictionFunction(const InputRestrictionContainer &r);
/** warning: caller needs to take care of synchronization! */
void wayFunction(ExtractionWay &w);
// warning: caller needs to take care of synchronization!
bool ProcessRestriction(const InputRestrictionContainer &restriction);
// warning: caller needs to take care of synchronization!
void ProcessWay(ExtractionWay &way);
};
#endif /* EXTRACTORCALLBACKS_H_ */
#endif /* EXTRACTOR_CALLBACKS_H */

119
Extractor/ExtractorStructs.h
View File

@ -34,97 +34,80 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <string>
struct ExtractorRelation {
ExtractorRelation() : type(unknown){}
enum {
unknown = 0, ferry, turnRestriction
} type;
struct ExtractorRelation
{
ExtractorRelation() : type(unknown) {}
enum
{ unknown = 0,
ferry,
turnRestriction } type;
HashTable<std::string, std::string> keyVals;
};
struct _WayIDStartAndEndEdge {
struct WayIDStartAndEndEdge
{
unsigned wayID;
NodeID firstStart;
NodeID firstTarget;
NodeID lastStart;
NodeID lastTarget;
_WayIDStartAndEndEdge()
:
wayID(UINT_MAX),
firstStart(UINT_MAX),
firstTarget(UINT_MAX),
lastStart(UINT_MAX),
WayIDStartAndEndEdge()
: wayID(UINT_MAX), firstStart(UINT_MAX), firstTarget(UINT_MAX), lastStart(UINT_MAX),
lastTarget(UINT_MAX)
{ }
explicit _WayIDStartAndEndEdge(
unsigned w,
NodeID fs,
NodeID ft,
NodeID ls,
NodeID lt
) :
wayID(w),
firstStart(fs),
firstTarget(ft),
lastStart(ls),
lastTarget(lt)
{ }
static _WayIDStartAndEndEdge min_value() {
return _WayIDStartAndEndEdge((std::numeric_limits<unsigned>::min)(), (std::numeric_limits<unsigned>::min)(), (std::numeric_limits<unsigned>::min)(), (std::numeric_limits<unsigned>::min)(), (std::numeric_limits<unsigned>::min)());
{
}
static _WayIDStartAndEndEdge max_value() {
return _WayIDStartAndEndEdge((std::numeric_limits<unsigned>::max)(), (std::numeric_limits<unsigned>::max)(), (std::numeric_limits<unsigned>::max)(), (std::numeric_limits<unsigned>::max)(), (std::numeric_limits<unsigned>::max)());
explicit WayIDStartAndEndEdge(unsigned w, NodeID fs, NodeID ft, NodeID ls, NodeID lt)
: wayID(w), firstStart(fs), firstTarget(ft), lastStart(ls), lastTarget(lt)
{
}
static WayIDStartAndEndEdge min_value()
{
return WayIDStartAndEndEdge((std::numeric_limits<unsigned>::min)(),
(std::numeric_limits<unsigned>::min)(),
(std::numeric_limits<unsigned>::min)(),
(std::numeric_limits<unsigned>::min)(),
(std::numeric_limits<unsigned>::min)());
}
static WayIDStartAndEndEdge max_value()
{
return WayIDStartAndEndEdge((std::numeric_limits<unsigned>::max)(),
(std::numeric_limits<unsigned>::max)(),
(std::numeric_limits<unsigned>::max)(),
(std::numeric_limits<unsigned>::max)(),
(std::numeric_limits<unsigned>::max)());
}
};
struct CmpWayByID {
typedef _WayIDStartAndEndEdge value_type;
bool operator ()(
const _WayIDStartAndEndEdge & a,
const _WayIDStartAndEndEdge & b
) const {
struct CmpWayByID
{
typedef WayIDStartAndEndEdge value_type;
bool operator()(const WayIDStartAndEndEdge &a, const WayIDStartAndEndEdge &b) const
{
return a.wayID < b.wayID;
}
value_type max_value() {
return _WayIDStartAndEndEdge::max_value();
}
value_type min_value() {
return _WayIDStartAndEndEdge::min_value();
}
value_type max_value() { return WayIDStartAndEndEdge::max_value(); }
value_type min_value() { return WayIDStartAndEndEdge::min_value(); }
};
struct Cmp {
struct Cmp
{
typedef NodeID value_type;
bool operator ()(
const NodeID a,
const NodeID b
) const {
return a < b;
}
value_type max_value() {
return 0xffffffff;
}
value_type min_value() {
return 0x0;
}
bool operator()(const NodeID a, const NodeID b) const { return a < b; }
value_type max_value() { return 0xffffffff; }
value_type min_value() { return 0x0; }
};
struct CmpNodeByID {
struct CmpNodeByID
{
typedef ExternalMemoryNode value_type;
bool operator () (
const ExternalMemoryNode & a,
const ExternalMemoryNode & b
) const {
bool operator()(const ExternalMemoryNode &a, const ExternalMemoryNode &b) const
{
return a.id < b.id;
}
value_type max_value() {
return ExternalMemoryNode::max_value();
}
value_type min_value() {
return ExternalMemoryNode::min_value();
}
value_type max_value() { return ExternalMemoryNode::max_value(); }
value_type min_value() { return ExternalMemoryNode::min_value(); }
};
#endif /* EXTRACTORSTRUCTS_H_ */

145
Extractor/InternalExtractorEdge.h
View File

@ -28,92 +28,49 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef INTERNAL_EXTRACTOR_EDGE_H
#define INTERNAL_EXTRACTOR_EDGE_H
#include "../typedefs.h"
#include <osrm/Coordinate.h>
#include <boost/assert.hpp>
struct InternalExtractorEdge {
struct InternalExtractorEdge
{
InternalExtractorEdge()
:
start(0),
target(0),
type(0),
direction(0),
speed(0),
nameID(0),
isRoundabout(false),
ignoreInGrid(false),
isDurationSet(false),
isAccessRestricted(false),
isContraFlow(false),
is_split(false)
{ }
: start(0), target(0), type(0), direction(0), speed(0), name_id(0), is_roundabout(false),
is_in_tiny_cc(false), is_duration_set(false), is_access_restricted(false),
is_contra_flow(false), is_split(false)
{
}
explicit InternalExtractorEdge(
NodeID start,
explicit InternalExtractorEdge(NodeID start,
NodeID target,
short type,
short direction,
double speed,
unsigned nameID,
bool isRoundabout,
bool ignoreInGrid,
bool isDurationSet,
bool isAccressRestricted,
bool isContraFlow,
bool is_split
) :
start(start),
target(target),
type(type),
direction(direction),
speed(speed),
nameID(nameID),
isRoundabout(isRoundabout),
ignoreInGrid(ignoreInGrid),
isDurationSet(isDurationSet),
isAccessRestricted(isAccressRestricted),
isContraFlow(isContraFlow),
is_split(is_split)
unsigned name_id,
bool is_roundabout,
bool is_in_tiny_cc,
bool is_duration_set,
bool is_access_restricted,
bool is_contra_flow,
bool is_split)
: start(start), target(target), type(type), direction(direction), speed(speed),
name_id(name_id), is_roundabout(is_roundabout), is_in_tiny_cc(is_in_tiny_cc),
is_duration_set(is_duration_set), is_access_restricted(is_access_restricted),
is_contra_flow(is_contra_flow), is_split(is_split)
{
BOOST_ASSERT(0 <= type);
}
// necessary static util functions for stxxl's sorting
static InternalExtractorEdge min_value() {
return InternalExtractorEdge(
0,
0,
0,
0,
0,
0,
false,
false,
false,
false,
false,
false
);
static InternalExtractorEdge min_value()
{
return InternalExtractorEdge(0, 0, 0, 0, 0, 0, false, false, false, false, false, false);
}
static InternalExtractorEdge max_value() {
static InternalExtractorEdge max_value()
{
return InternalExtractorEdge(
std::numeric_limits<unsigned>::max(),
std::numeric_limits<unsigned>::max(),
0,
0,
0,
0,
false,
false,
false,
false,
false,
false
);
SPECIAL_NODEID, SPECIAL_NODEID, 0, 0, 0, 0, false, false, false, false, false, false);
}
NodeID start;
@ -121,53 +78,43 @@ struct InternalExtractorEdge {
short type;
short direction;
double speed;
unsigned nameID;
bool isRoundabout;
bool ignoreInGrid;
bool isDurationSet;
bool isAccessRestricted;
bool isContraFlow;
unsigned name_id;
bool is_roundabout;
bool is_in_tiny_cc;
bool is_duration_set;
bool is_access_restricted;
bool is_contra_flow;
bool is_split;
FixedPointCoordinate startCoord;
FixedPointCoordinate targetCoord;
FixedPointCoordinate source_coordinate;
FixedPointCoordinate target_coordinate;
};
struct CmpEdgeByStartID {
struct CmpEdgeByStartID
{
typedef InternalExtractorEdge value_type;
bool operator ()(
const InternalExtractorEdge & a,
const InternalExtractorEdge & b
) const {
bool operator()(const InternalExtractorEdge &a, const InternalExtractorEdge &b) const
{
return a.start < b.start;
}
value_type max_value() {
return InternalExtractorEdge::max_value();
}
value_type max_value() { return InternalExtractorEdge::max_value(); }
value_type min_value() {
return InternalExtractorEdge::min_value();
}
value_type min_value() { return InternalExtractorEdge::min_value(); }
};
struct CmpEdgeByTargetID {
struct CmpEdgeByTargetID
{
typedef InternalExtractorEdge value_type;
bool operator ()(
const InternalExtractorEdge & a,
const InternalExtractorEdge & b
) const {
bool operator()(const InternalExtractorEdge &a, const InternalExtractorEdge &b) const
{
return a.target < b.target;
}
value_type max_value() {
return InternalExtractorEdge::max_value();
}
value_type max_value() { return InternalExtractorEdge::max_value(); }
value_type min_value() {
return InternalExtractorEdge::min_value();
}
value_type min_value() { return InternalExtractorEdge::min_value(); }
};
#endif //INTERNAL_EXTRACTOR_EDGE_H
#endif // INTERNAL_EXTRACTOR_EDGE_H

517
Extractor/PBFParser.cpp
View File

@ -42,151 +42,184 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <osrm/Coordinate.h>
#include <boost/make_shared.hpp>
#include <boost/ref.hpp>
#include <boost/thread.hpp>
#include <zlib.h>
PBFParser::PBFParser(
const char * fileName,
ExtractorCallbacks * extractor_callbacks,
ScriptingEnvironment& scripting_environment
) : BaseParser( extractor_callbacks, scripting_environment ) {
#include <functional>
#include <thread>
PBFParser::PBFParser(const char *fileName,
ExtractorCallbacks *extractor_callbacks,
ScriptingEnvironment &scripting_environment)
: BaseParser(extractor_callbacks, scripting_environment)
{
GOOGLE_PROTOBUF_VERIFY_VERSION;
//TODO: What is the bottleneck here? Filling the queue or reading the stuff from disk?
//NOTE: With Lua scripting, it is parsing the stuff. I/O is virtually for free.
// TODO: What is the bottleneck here? Filling the queue or reading the stuff from disk?
// NOTE: With Lua scripting, it is parsing the stuff. I/O is virtually for free.
// Max 2500 items in queue, hardcoded.
threadDataQueue = boost::make_shared<ConcurrentQueue<_ThreadData*> >( 2500 );
thread_data_queue = std::make_shared<ConcurrentQueue<ParserThreadData *>>(2500);
input.open(fileName, std::ios::in | std::ios::binary);
if (!input) {
if (!input)
{
throw OSRMException("pbf file not found.");
}
blockCount = 0;
groupCount = 0;
block_count = 0;
group_count = 0;
}
PBFParser::~PBFParser() {
if(input.is_open()) {
PBFParser::~PBFParser()
{
if (input.is_open())
{
input.close();
}
// Clean up any leftover ThreadData objects in the queue
_ThreadData* thread_data;
while (threadDataQueue->try_pop(thread_data))
ParserThreadData *thread_data;
while (thread_data_queue->try_pop(thread_data))
{
delete thread_data;
}
google::protobuf::ShutdownProtobufLibrary();
SimpleLogger().Write(logDEBUG) <<
"parsed " << blockCount <<
" blocks from pbf with " << groupCount <<
" groups";
SimpleLogger().Write(logDEBUG) << "parsed " << block_count << " blocks from pbf with "
<< group_count << " groups";
}
inline bool PBFParser::ReadHeader() {
_ThreadData initData;
inline bool PBFParser::ReadHeader()
{
ParserThreadData init_data;
/** read Header */
if(!readPBFBlobHeader(input, &initData)) {
if (!readPBFBlobHeader(input, &init_data))
{
return false;
}
if(readBlob(input, &initData)) {
if(!initData.PBFHeaderBlock.ParseFromArray(&(initData.charBuffer[0]), initData.charBuffer.size() ) ) {
if (readBlob(input, &init_data))
{
if (!init_data.PBFHeaderBlock.ParseFromArray(&(init_data.charBuffer[0]),
init_data.charBuffer.size()))
{
std::cerr << "[error] Header not parseable!" << std::endl;
return false;
}
for(int i = 0, featureSize = initData.PBFHeaderBlock.required_features_size(); i < featureSize; ++i) {
const std::string& feature = initData.PBFHeaderBlock.required_features( i );
const auto feature_size = init_data.PBFHeaderBlock.required_features_size();
for (int i = 0; i < feature_size; ++i)
{
const std::string &feature = init_data.PBFHeaderBlock.required_features(i);
bool supported = false;
if ( "OsmSchema-V0.6" == feature ) {
if ("OsmSchema-V0.6" == feature)
{
supported = true;
}
else if ( "DenseNodes" == feature ) {
else if ("DenseNodes" == feature)
{
supported = true;
}
if ( !supported ) {
std::cerr << "[error] required feature not supported: " << feature.data() << std::endl;
if (!supported)
{
std::cerr << "[error] required feature not supported: " << feature.data()
<< std::endl;
return false;
}
}
} else {
}
else
{
std::cerr << "[error] blob not loaded!" << std::endl;
}
return true;
}
inline void PBFParser::ReadData() {
bool keepRunning = true;
do {
_ThreadData *threadData = new _ThreadData();
keepRunning = readNextBlock(input, threadData);
inline void PBFParser::ReadData()
{
bool keep_running = true;
do
{
ParserThreadData *thread_data = new ParserThreadData();
keep_running = readNextBlock(input, thread_data);
if (keepRunning) {
threadDataQueue->push(threadData);
} else {
threadDataQueue->push(nullptr); // No more data to read, parse stops when nullptr encountered
delete threadData;
if (keep_running)
{
thread_data_queue->push(thread_data);
}
} while(keepRunning);
else
{
// No more data to read, parse stops when nullptr encountered
thread_data_queue->push(nullptr);
delete thread_data;
}
} while (keep_running);
}
inline void PBFParser::ParseData() {
while (true) {
_ThreadData *threadData;
threadDataQueue->wait_and_pop(threadData);
if( nullptr==threadData ) {
threadDataQueue->push(nullptr); // Signal end of data for other threads
inline void PBFParser::ParseData()
{
while (true)
{
ParserThreadData *thread_data;
thread_data_queue->wait_and_pop(thread_data);
if (nullptr == thread_data)
{
thread_data_queue->push(nullptr); // Signal end of data for other threads
break;
}
loadBlock(threadData);
loadBlock(thread_data);
for(int i = 0, groupSize = threadData->PBFprimitiveBlock.primitivegroup_size(); i < groupSize; ++i) {
threadData->currentGroupID = i;
loadGroup(threadData);
int group_size = group_size = thread_data->PBFprimitiveBlock.primitivegroup_size();
for (int i = 0; i < group_size; ++i)
{
thread_data->currentGroupID = i;
loadGroup(thread_data);
if(threadData->entityTypeIndicator == TypeNode) {
parseNode(threadData);
if (thread_data->entityTypeIndicator == TypeNode)
{
parseNode(thread_data);
}
if(threadData->entityTypeIndicator == TypeWay) {
parseWay(threadData);
if (thread_data->entityTypeIndicator == TypeWay)
{
parseWay(thread_data);
}
if(threadData->entityTypeIndicator == TypeRelation) {
parseRelation(threadData);
if (thread_data->entityTypeIndicator == TypeRelation)
{
parseRelation(thread_data);
}
if(threadData->entityTypeIndicator == TypeDenseNode) {
parseDenseNode(threadData);
if (thread_data->entityTypeIndicator == TypeDenseNode)
{
parseDenseNode(thread_data);
}
}
delete threadData;
threadData = NULL;
delete thread_data;
thread_data = nullptr;
}
}
inline bool PBFParser::Parse() {
inline bool PBFParser::Parse()
{
// Start the read and parse threads
boost::thread readThread(boost::bind(&PBFParser::ReadData, this));
std::thread read_thread(std::bind(&PBFParser::ReadData, this));
//Open several parse threads that are synchronized before call to
boost::thread parseThread(boost::bind(&PBFParser::ParseData, this));
// Open several parse threads that are synchronized before call to
std::thread parse_thread(std::bind(&PBFParser::ParseData, this));
// Wait for the threads to finish
readThread.join();
parseThread.join();
read_thread.join();
parse_thread.join();
return true;
}
inline void PBFParser::parseDenseNode(_ThreadData * threadData) {
const OSMPBF::DenseNodes& dense = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).dense();
inline void PBFParser::parseDenseNode(ParserThreadData *thread_data)
{
const OSMPBF::DenseNodes &dense =
thread_data->PBFprimitiveBlock.primitivegroup(thread_data->currentGroupID).dense();
int denseTagIndex = 0;
int64_t m_lastDenseID = 0;
int64_t m_lastDenseLatitude = 0;
@ -194,72 +227,95 @@ inline void PBFParser::parseDenseNode(_ThreadData * threadData) {
const int number_of_nodes = dense.id_size();
std::vector<ImportNode> extracted_nodes_vector(number_of_nodes);
for(int i = 0; i < number_of_nodes; ++i) {
m_lastDenseID += dense.id( i );
m_lastDenseLatitude += dense.lat( i );
m_lastDenseLongitude += dense.lon( i );
for (int i = 0; i < number_of_nodes; ++i)
{
m_lastDenseID += dense.id(i);
m_lastDenseLatitude += dense.lat(i);
m_lastDenseLongitude += dense.lon(i);
extracted_nodes_vector[i].id = m_lastDenseID;
extracted_nodes_vector[i].lat = COORDINATE_PRECISION*( ( double ) m_lastDenseLatitude * threadData->PBFprimitiveBlock.granularity() + threadData->PBFprimitiveBlock.lat_offset() ) / NANO;
extracted_nodes_vector[i].lon = COORDINATE_PRECISION*( ( double ) m_lastDenseLongitude * threadData->PBFprimitiveBlock.granularity() + threadData->PBFprimitiveBlock.lon_offset() ) / NANO;
while (denseTagIndex < dense.keys_vals_size()) {
const int tagValue = dense.keys_vals( denseTagIndex );
if( 0 == tagValue ) {
extracted_nodes_vector[i].lat =
COORDINATE_PRECISION *
((double)m_lastDenseLatitude * thread_data->PBFprimitiveBlock.granularity() +
thread_data->PBFprimitiveBlock.lat_offset()) /
NANO;
extracted_nodes_vector[i].lon =
COORDINATE_PRECISION *
((double)m_lastDenseLongitude * thread_data->PBFprimitiveBlock.granularity() +
thread_data->PBFprimitiveBlock.lon_offset()) /
NANO;
while (denseTagIndex < dense.keys_vals_size())
{
const int tagValue = dense.keys_vals(denseTagIndex);
if (0 == tagValue)
{
++denseTagIndex;
break;
}
const int keyValue = dense.keys_vals ( denseTagIndex+1 );
const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(tagValue);
const std::string & value = threadData->PBFprimitiveBlock.stringtable().s(keyValue);
const int keyValue = dense.keys_vals(denseTagIndex + 1);
const std::string &key = thread_data->PBFprimitiveBlock.stringtable().s(tagValue);
const std::string &value = thread_data->PBFprimitiveBlock.stringtable().s(keyValue);
extracted_nodes_vector[i].keyVals.emplace(key, value);
denseTagIndex += 2;
}
}
#pragma omp parallel
{
const int thread_num = omp_get_thread_num();
#pragma omp parallel for schedule ( guided )
for(int i = 0; i < number_of_nodes; ++i)
{
ImportNode & import_node = extracted_nodes_vector[i];
const int thread_num = omp_get_thread_num();
#pragma omp parallel for schedule(guided)
for (int i = 0; i < number_of_nodes; ++i)
{
ImportNode &import_node = extracted_nodes_vector[i];
ParseNodeInLua(import_node, scripting_environment.getLuaStateForThreadID(thread_num));
}
}
}
for(const ImportNode &import_node : extracted_nodes_vector)
for (const ImportNode &import_node : extracted_nodes_vector)
{
extractor_callbacks->nodeFunction(import_node);
extractor_callbacks->ProcessNode(import_node);
}
}
inline void PBFParser::parseNode(_ThreadData * )
inline void PBFParser::parseNode(ParserThreadData *)
{
throw OSRMException("Parsing of simple nodes not supported. PBF should use dense nodes");
}
inline void PBFParser::parseRelation(_ThreadData * threadData) {
//TODO: leave early, if relation is not a restriction
//TODO: reuse rawRestriction container
if( !use_turn_restrictions ) {
inline void PBFParser::parseRelation(ParserThreadData *thread_data)
{
// TODO: leave early, if relation is not a restriction
// TODO: reuse rawRestriction container
if (!use_turn_restrictions)
{
return;
}
const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID );
const OSMPBF::PrimitiveGroup &group =
thread_data->PBFprimitiveBlock.primitivegroup(thread_data->currentGroupID);
for(int i = 0, relation_size = group.relations_size(); i < relation_size; ++i ) {
for (int i = 0, relation_size = group.relations_size(); i < relation_size; ++i)
{
std::string except_tag_string;
const OSMPBF::Relation& inputRelation = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).relations(i);
const OSMPBF::Relation &inputRelation =
thread_data->PBFprimitiveBlock.primitivegroup(thread_data->currentGroupID).relations(i);
bool isRestriction = false;
bool isOnlyRestriction = false;
for(int k = 0, endOfKeys = inputRelation.keys_size(); k < endOfKeys; ++k) {
const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(inputRelation.keys(k));
const std::string & val = threadData->PBFprimitiveBlock.stringtable().s(inputRelation.vals(k));
if ("type" == key) {
if( "restriction" == val) {
for (int k = 0, endOfKeys = inputRelation.keys_size(); k < endOfKeys; ++k)
{
const std::string &key =
thread_data->PBFprimitiveBlock.stringtable().s(inputRelation.keys(k));
const std::string &val =
thread_data->PBFprimitiveBlock.stringtable().s(inputRelation.vals(k));
if ("type" == key)
{
if ("restriction" == val)
{
isRestriction = true;
} else {
}
else
{
break;
}
}
if ( ("restriction" == key) && (val.find("only_") == 0) )
if (("restriction" == key) && (val.find("only_") == 0))
{
isOnlyRestriction = true;
}
@ -269,238 +325,280 @@ inline void PBFParser::parseRelation(_ThreadData * threadData) {
}
}
if( isRestriction && ShouldIgnoreRestriction(except_tag_string) ) {
if (isRestriction && ShouldIgnoreRestriction(except_tag_string))
{
continue;
}
if(isRestriction) {
if (isRestriction)
{
int64_t lastRef = 0;
InputRestrictionContainer currentRestrictionContainer(isOnlyRestriction);
for(
int rolesIndex = 0, last_role = inputRelation.roles_sid_size();
for (int rolesIndex = 0, last_role = inputRelation.roles_sid_size();
rolesIndex < last_role;
++rolesIndex
) {
const std::string & role = threadData->PBFprimitiveBlock.stringtable().s( inputRelation.roles_sid( rolesIndex ) );
++rolesIndex)
{
const std::string &role = thread_data->PBFprimitiveBlock.stringtable().s(
inputRelation.roles_sid(rolesIndex));
lastRef += inputRelation.memids(rolesIndex);
if(!("from" == role || "to" == role || "via" == role)) {
if (!("from" == role || "to" == role || "via" == role))
{
continue;
}
switch(inputRelation.types(rolesIndex)) {
case 0: //node
if("from" == role || "to" == role) { //Only via should be a node
switch (inputRelation.types(rolesIndex))
{
case 0: // node
if ("from" == role || "to" == role)
{ // Only via should be a node
continue;
}
assert("via" == role);
if(UINT_MAX != currentRestrictionContainer.viaNode) {
if (UINT_MAX != currentRestrictionContainer.viaNode)
{
currentRestrictionContainer.viaNode = UINT_MAX;
}
assert(UINT_MAX == currentRestrictionContainer.viaNode);
currentRestrictionContainer.restriction.viaNode = lastRef;
break;
case 1: //way
case 1: // way
assert("from" == role || "to" == role || "via" == role);
if("from" == role) {
if ("from" == role)
{
currentRestrictionContainer.fromWay = lastRef;
}
if ("to" == role) {
if ("to" == role)
{
currentRestrictionContainer.toWay = lastRef;
}
if ("via" == role) {
if ("via" == role)
{
assert(currentRestrictionContainer.restriction.toNode == UINT_MAX);
currentRestrictionContainer.viaNode = lastRef;
}
break;
case 2: //relation, not used. relations relating to relations are evil.
case 2: // relation, not used. relations relating to relations are evil.
continue;
assert(false);
break;
default: //should not happen
default: // should not happen
assert(false);
break;
}
}
if(!extractor_callbacks->restrictionFunction(currentRestrictionContainer)) {
if (!extractor_callbacks->ProcessRestriction(currentRestrictionContainer))
{
std::cerr << "[PBFParser] relation not parsed" << std::endl;
}
}
}
}
inline void PBFParser::parseWay(_ThreadData * threadData) {
const int number_of_ways = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).ways_size();
inline void PBFParser::parseWay(ParserThreadData *thread_data)
{
const int number_of_ways =
thread_data->PBFprimitiveBlock.primitivegroup(thread_data->currentGroupID).ways_size();
std::vector<ExtractionWay> parsed_way_vector(number_of_ways);
for(int i = 0; i < number_of_ways; ++i) {
const OSMPBF::Way& inputWay = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID ).ways( i );
for (int i = 0; i < number_of_ways; ++i)
{
const OSMPBF::Way &inputWay =
thread_data->PBFprimitiveBlock.primitivegroup(thread_data->currentGroupID).ways(i);
parsed_way_vector[i].id = inputWay.id();
unsigned pathNode(0);
const int number_of_referenced_nodes = inputWay.refs_size();
for(int j = 0; j < number_of_referenced_nodes; ++j) {
for (int j = 0; j < number_of_referenced_nodes; ++j)
{
pathNode += inputWay.refs(j);
parsed_way_vector[i].path.push_back(pathNode);
}
assert(inputWay.keys_size() == inputWay.vals_size());
const int number_of_keys = inputWay.keys_size();
for(int j = 0; j < number_of_keys; ++j) {
const std::string & key = threadData->PBFprimitiveBlock.stringtable().s(inputWay.keys(j));
const std::string & val = threadData->PBFprimitiveBlock.stringtable().s(inputWay.vals(j));
for (int j = 0; j < number_of_keys; ++j)
{
const std::string &key =
thread_data->PBFprimitiveBlock.stringtable().s(inputWay.keys(j));
const std::string &val =
thread_data->PBFprimitiveBlock.stringtable().s(inputWay.vals(j));
parsed_way_vector[i].keyVals.emplace(key, val);
}
}
#pragma omp parallel for schedule ( guided )
for(int i = 0; i < number_of_ways; ++i)
#pragma omp parallel for schedule(guided)
for (int i = 0; i < number_of_ways; ++i)
{
ExtractionWay & extraction_way = parsed_way_vector[i];
ExtractionWay &extraction_way = parsed_way_vector[i];
if (2 <= extraction_way.path.size())
{
ParseWayInLua(
extraction_way,
scripting_environment.getLuaStateForThreadID(omp_get_thread_num())
);
ParseWayInLua(extraction_way,
scripting_environment.getLuaStateForThreadID(omp_get_thread_num()));
}
}
for(ExtractionWay & extraction_way : parsed_way_vector)
for (ExtractionWay &extraction_way : parsed_way_vector)
{
if (2 <= extraction_way.path.size())
{
extractor_callbacks->wayFunction(extraction_way);
extractor_callbacks->ProcessWay(extraction_way);
}
}
}
inline void PBFParser::loadGroup(_ThreadData * threadData) {
inline void PBFParser::loadGroup(ParserThreadData *thread_data)
{
#ifndef NDEBUG
++groupCount;
++group_count;
#endif
const OSMPBF::PrimitiveGroup& group = threadData->PBFprimitiveBlock.primitivegroup( threadData->currentGroupID );
threadData->entityTypeIndicator = TypeDummy;
if ( 0 != group.nodes_size() ) {
threadData->entityTypeIndicator = TypeNode;
const OSMPBF::PrimitiveGroup &group =
thread_data->PBFprimitiveBlock.primitivegroup(thread_data->currentGroupID);
thread_data->entityTypeIndicator = TypeDummy;
if (0 != group.nodes_size())
{
thread_data->entityTypeIndicator = TypeNode;
}
if ( 0 != group.ways_size() ) {
threadData->entityTypeIndicator = TypeWay;
if (0 != group.ways_size())
{
thread_data->entityTypeIndicator = TypeWay;
}
if ( 0 != group.relations_size() ) {
threadData->entityTypeIndicator = TypeRelation;
if (0 != group.relations_size())
{
thread_data->entityTypeIndicator = TypeRelation;
}
if ( group.has_dense() ) {
threadData->entityTypeIndicator = TypeDenseNode;
assert( 0 != group.dense().id_size() );
if (group.has_dense())
{
thread_data->entityTypeIndicator = TypeDenseNode;
assert(0 != group.dense().id_size());
}
assert( threadData->entityTypeIndicator != TypeDummy );
assert(thread_data->entityTypeIndicator != TypeDummy);
}
inline void PBFParser::loadBlock(_ThreadData * threadData) {
++blockCount;
threadData->currentGroupID = 0;
threadData->currentEntityID = 0;
inline void PBFParser::loadBlock(ParserThreadData *thread_data)
{
++block_count;
thread_data->currentGroupID = 0;
thread_data->currentEntityID = 0;
}
inline bool PBFParser::readPBFBlobHeader(std::fstream& stream, _ThreadData * threadData) {
inline bool PBFParser::readPBFBlobHeader(std::fstream &stream, ParserThreadData *thread_data)
{
int size(0);
stream.read((char *)&size, sizeof(int));
size = SwapEndian(size);
if(stream.eof()) {
if (stream.eof())
{
return false;
}
if ( size > MAX_BLOB_HEADER_SIZE || size < 0 ) {
if (size > MAX_BLOB_HEADER_SIZE || size < 0)
{
return false;
}
char *data = new char[size];
stream.read(data, size*sizeof(data[0]));
stream.read(data, size * sizeof(data[0]));
bool dataSuccessfullyParsed = (threadData->PBFBlobHeader).ParseFromArray( data, size);
bool dataSuccessfullyParsed = (thread_data->PBFBlobHeader).ParseFromArray(data, size);
delete[] data;
return dataSuccessfullyParsed;
}
inline bool PBFParser::unpackZLIB(std::fstream &, _ThreadData * threadData) {
unsigned rawSize = threadData->PBFBlob.raw_size();
char* unpackedDataArray = new char[rawSize];
inline bool PBFParser::unpackZLIB(std::fstream &, ParserThreadData *thread_data)
{
unsigned rawSize = thread_data->PBFBlob.raw_size();
char *unpackedDataArray = new char[rawSize];
z_stream compressedDataStream;
compressedDataStream.next_in = ( unsigned char* ) threadData->PBFBlob.zlib_data().data();
compressedDataStream.avail_in = threadData->PBFBlob.zlib_data().size();
compressedDataStream.next_out = ( unsigned char* ) unpackedDataArray;
compressedDataStream.next_in = (unsigned char *)thread_data->PBFBlob.zlib_data().data();
compressedDataStream.avail_in = thread_data->PBFBlob.zlib_data().size();
compressedDataStream.next_out = (unsigned char *)unpackedDataArray;
compressedDataStream.avail_out = rawSize;
compressedDataStream.zalloc = Z_NULL;
compressedDataStream.zfree = Z_NULL;
compressedDataStream.opaque = Z_NULL;
int ret = inflateInit( &compressedDataStream );
if ( ret != Z_OK ) {
int ret = inflateInit(&compressedDataStream);
if (ret != Z_OK)
{
std::cerr << "[error] failed to init zlib stream" << std::endl;
delete[] unpackedDataArray;
return false;
}
ret = inflate( &compressedDataStream, Z_FINISH );
if ( ret != Z_STREAM_END ) {
ret = inflate(&compressedDataStream, Z_FINISH);
if (ret != Z_STREAM_END)
{
std::cerr << "[error] failed to inflate zlib stream" << std::endl;
std::cerr << "[error] Error type: " << ret << std::endl;
delete[] unpackedDataArray;
return false;
}
ret = inflateEnd( &compressedDataStream );
if ( ret != Z_OK ) {
ret = inflateEnd(&compressedDataStream);
if (ret != Z_OK)
{
std::cerr << "[error] failed to deinit zlib stream" << std::endl;
delete[] unpackedDataArray;
return false;
}
threadData->charBuffer.clear(); threadData->charBuffer.resize(rawSize);
std::copy(unpackedDataArray, unpackedDataArray + rawSize, threadData->charBuffer.begin());
thread_data->charBuffer.clear();
thread_data->charBuffer.resize(rawSize);
std::copy(unpackedDataArray, unpackedDataArray + rawSize, thread_data->charBuffer.begin());
delete[] unpackedDataArray;
return true;
}
inline bool PBFParser::unpackLZMA(std::fstream &, _ThreadData * ) {
return false;
}
inline bool PBFParser::unpackLZMA(std::fstream &, ParserThreadData *) { return false; }
inline bool PBFParser::readBlob(std::fstream& stream, _ThreadData * threadData) {
if(stream.eof()) {
inline bool PBFParser::readBlob(std::fstream &stream, ParserThreadData *thread_data)
{
if (stream.eof())
{
return false;
}
const int size = threadData->PBFBlobHeader.datasize();
if ( size < 0 || size > MAX_BLOB_SIZE ) {
const int size = thread_data->PBFBlobHeader.datasize();
if (size < 0 || size > MAX_BLOB_SIZE)
{
std::cerr << "[error] invalid Blob size:" << size << std::endl;
return false;
}
char* data = new char[size];
stream.read(data, sizeof(data[0])*size);
char *data = new char[size];
stream.read(data, sizeof(data[0]) * size);
if ( !threadData->PBFBlob.ParseFromArray( data, size ) ) {
if (!thread_data->PBFBlob.ParseFromArray(data, size))
{
std::cerr << "[error] failed to parse blob" << std::endl;
delete[] data;
return false;
}
if ( threadData->PBFBlob.has_raw() ) {
const std::string& data = threadData->PBFBlob.raw();
threadData->charBuffer.clear();
threadData->charBuffer.resize( data.size() );
std::copy(data.begin(), data.end(), threadData->charBuffer.begin());
} else if ( threadData->PBFBlob.has_zlib_data() ) {
if ( !unpackZLIB(stream, threadData) ) {
if (thread_data->PBFBlob.has_raw())
{
const std::string &data = thread_data->PBFBlob.raw();
thread_data->charBuffer.clear();
thread_data->charBuffer.resize(data.size());
std::copy(data.begin(), data.end(), thread_data->charBuffer.begin());
}
else if (thread_data->PBFBlob.has_zlib_data())
{
if (!unpackZLIB(stream, thread_data))
{
std::cerr << "[error] zlib data encountered that could not be unpacked" << std::endl;
delete[] data;
return false;
}
} else if ( threadData->PBFBlob.has_lzma_data() ) {
if ( !unpackLZMA(stream, threadData) ) {
}
else if (thread_data->PBFBlob.has_lzma_data())
{
if (!unpackLZMA(stream, thread_data))
{
std::cerr << "[error] lzma data encountered that could not be unpacked" << std::endl;
}
delete[] data;
return false;
} else {
}
else
{
std::cerr << "[error] Blob contains no data" << std::endl;
delete[] data;
return false;
@ -509,24 +607,31 @@ inline bool PBFParser::readBlob(std::fstream& stream, _ThreadData * threadData)
return true;
}
bool PBFParser::readNextBlock(std::fstream& stream, _ThreadData * threadData) {
if(stream.eof()) {
bool PBFParser::readNextBlock(std::fstream &stream, ParserThreadData *thread_data)
{
if (stream.eof())
{
return false;
}
if ( !readPBFBlobHeader(stream, threadData) ){
if (!readPBFBlobHeader(stream, thread_data))
{
return false;
}
if ( threadData->PBFBlobHeader.type() != "OSMData" ) {
if (thread_data->PBFBlobHeader.type() != "OSMData")
{
return false;
}
if ( !readBlob(stream, threadData) ) {
if (!readBlob(stream, thread_data))
{
return false;
}
if ( !threadData->PBFprimitiveBlock.ParseFromArray( &(threadData->charBuffer[0]), threadData-> charBuffer.size() ) ) {
if (!thread_data->PBFprimitiveBlock.ParseFromArray(&(thread_data->charBuffer[0]),
thread_data->charBuffer.size()))
{
std::cerr << "failed to parse PrimitiveBlock" << std::endl;
return false;
}

54
Extractor/PBFParser.h
View File

@ -31,24 +31,24 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "BaseParser.h"
#include "../DataStructures/ConcurrentQueue.h"
#include <boost/shared_ptr.hpp>
#include <osmpbf/fileformat.pb.h>
#include <osmpbf/osmformat.pb.h>
#include <fstream>
#include <memory>
class PBFParser : public BaseParser {
class PBFParser : public BaseParser
{
enum EntityType {
TypeDummy = 0,
enum EntityType
{ TypeDummy = 0,
TypeNode = 1,
TypeWay = 2,
TypeRelation = 4,
TypeDenseNode = 8
};
TypeDenseNode = 8 };
struct _ThreadData {
struct ParserThreadData
{
int currentGroupID;
int currentEntityID;
EntityType entityTypeIndicator;
@ -62,42 +62,38 @@ class PBFParser : public BaseParser {
std::vector<char> charBuffer;
};
public:
PBFParser(
const char * fileName,
ExtractorCallbacks* ec,
ScriptingEnvironment& se
);
public:
PBFParser(const char *file_name, ExtractorCallbacks *extractor_callbacks, ScriptingEnvironment &scripting_environment);
virtual ~PBFParser();
inline bool ReadHeader();
inline bool Parse();
private:
private:
inline void ReadData();
inline void ParseData();
inline void parseDenseNode (_ThreadData * threadData);
inline void parseNode (_ThreadData * threadData);
inline void parseRelation (_ThreadData * threadData);
inline void parseWay (_ThreadData * threadData);
inline void parseDenseNode(ParserThreadData *thread_data);
inline void parseNode(ParserThreadData *thread_data);
inline void parseRelation(ParserThreadData *thread_data);
inline void parseWay(ParserThreadData *thread_data);
inline void loadGroup (_ThreadData * threadData);
inline void loadBlock (_ThreadData * threadData);
inline bool readPBFBlobHeader(std::fstream & stream, _ThreadData * threadData);
inline bool unpackZLIB (std::fstream & stream, _ThreadData * threadData);
inline bool unpackLZMA (std::fstream & stream, _ThreadData * threadData);
inline bool readBlob (std::fstream & stream, _ThreadData * threadData);
inline bool readNextBlock (std::fstream & stream, _ThreadData * threadData);
inline void loadGroup(ParserThreadData *thread_data);
inline void loadBlock(ParserThreadData *thread_data);
inline bool readPBFBlobHeader(std::fstream &stream, ParserThreadData *thread_data);
inline bool unpackZLIB(std::fstream &stream, ParserThreadData *thread_data);
inline bool unpackLZMA(std::fstream &stream, ParserThreadData *thread_data);
inline bool readBlob(std::fstream &stream, ParserThreadData *thread_data);
inline bool readNextBlock(std::fstream &stream, ParserThreadData *thread_data);
static const int NANO = 1000 * 1000 * 1000;
static const int MAX_BLOB_HEADER_SIZE = 64 * 1024;
static const int MAX_BLOB_SIZE = 32 * 1024 * 1024;
unsigned groupCount;
unsigned blockCount;
unsigned group_count;
unsigned block_count;
std::fstream input; // the input stream to parse
boost::shared_ptr<ConcurrentQueue < _ThreadData* > > threadDataQueue;
std::shared_ptr<ConcurrentQueue<ParserThreadData *>> thread_data_queue;
};
#endif /* PBFPARSER_H_ */

68
Extractor/ScriptingEnvironment.cpp
View File

@ -37,51 +37,49 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "../typedefs.h"
ScriptingEnvironment::ScriptingEnvironment() {}
ScriptingEnvironment::ScriptingEnvironment(const char * fileName) {
SimpleLogger().Write() << "Using script " << fileName;
ScriptingEnvironment::ScriptingEnvironment(const char *file_name)
{
SimpleLogger().Write() << "Using script " << file_name;
// Create a new lua state
for(int i = 0; i < omp_get_max_threads(); ++i) {
luaStateVector.push_back(luaL_newstate());
for (int i = 0; i < omp_get_max_threads(); ++i)
{
lua_state_vector.push_back(luaL_newstate());
}
// Connect LuaBind to this lua state for all threads
// Connect LuaBind to this lua state for all threads
#pragma omp parallel
{
lua_State * myLuaState = getLuaStateForThreadID(omp_get_thread_num());
luabind::open(myLuaState);
//open utility libraries string library;
luaL_openlibs(myLuaState);
lua_State *lua_state = getLuaStateForThreadID(omp_get_thread_num());
luabind::open(lua_state);
// open utility libraries string library;
luaL_openlibs(lua_state);
luaAddScriptFolderToLoadPath( myLuaState, fileName );
luaAddScriptFolderToLoadPath(lua_state, file_name);
// Add our function to the state's global scope
luabind::module(myLuaState) [
luabind::module(lua_state)[
luabind::def("print", LUA_print<std::string>),
luabind::def("durationIsValid", durationIsValid),
luabind::def("parseDuration", parseDuration)
];
luabind::module(myLuaState) [
luabind::class_<HashTable<std::string, std::string> >("keyVals")
luabind::module(lua_state)[luabind::class_<HashTable<std::string, std::string>>("keyVals")
.def("Add", &HashTable<std::string, std::string>::Add)
.def("Find", &HashTable<std::string, std::string>::Find)
.def("Holds", &HashTable<std::string, std::string>::Holds)
];
.def("Holds", &HashTable<std::string, std::string>::Holds)];
luabind::module(myLuaState) [
luabind::class_<ImportNode>("Node")
luabind::module(lua_state)[luabind::class_<ImportNode>("Node")
.def(luabind::constructor<>())
.def_readwrite("lat", &ImportNode::lat)
.def_readwrite("lon", &ImportNode::lon)
.def_readonly("id", &ImportNode::id)
.def_readwrite("bollard", &ImportNode::bollard)
.def_readwrite("traffic_light", &ImportNode::trafficLight)
.def_readwrite("tags", &ImportNode::keyVals)
];
.def_readwrite("tags", &ImportNode::keyVals)];
luabind::module(myLuaState) [
luabind::class_<ExtractionWay>("Way")
luabind::module(lua_state)
[luabind::class_<ExtractionWay>("Way")
.def(luabind::constructor<>())
.def_readonly("id", &ExtractionWay::id)
.def_readwrite("name", &ExtractionWay::name)
@ -95,32 +93,32 @@ ScriptingEnvironment::ScriptingEnvironment(const char * fileName) {
.def_readwrite("ignore_in_grid", &ExtractionWay::ignoreInGrid)
.def_readwrite("tags", &ExtractionWay::keyVals)
.def_readwrite("direction", &ExtractionWay::direction)
.enum_("constants") [
.enum_("constants")[
luabind::value("notSure", 0),
luabind::value("oneway", 1),
luabind::value("bidirectional", 2),
luabind::value("opposite", 3)
]
];
]];
// fails on c++11/OS X 10.9
luabind::module(myLuaState) [
luabind::class_<std::vector<std::string> >("vector")
.def("Add", static_cast<void (std::vector<std::string>::*)(const std::string&)>(&std::vector<std::string>::push_back))
];
luabind::module(lua_state)[luabind::class_<std::vector<std::string>>("vector").def(
"Add",
static_cast<void (std::vector<std::string>::*)(const std::string &)>(
&std::vector<std::string>::push_back))];
if(0 != luaL_dofile(myLuaState, fileName) ) {
if (0 != luaL_dofile(lua_state, file_name))
{
throw OSRMException("ERROR occured in scripting block");
}
}
}
ScriptingEnvironment::~ScriptingEnvironment() {
for(unsigned i = 0; i < luaStateVector.size(); ++i) {
// luaStateVector[i];
ScriptingEnvironment::~ScriptingEnvironment()
{
for (unsigned i = 0; i < lua_state_vector.size(); ++i)
{
// lua_state_vector[i];
}
}
lua_State * ScriptingEnvironment::getLuaStateForThreadID(const int id) {
return luaStateVector[id];
}
lua_State *ScriptingEnvironment::getLuaStateForThreadID(const int id) { return lua_state_vector[id]; }

11
Extractor/ScriptingEnvironment.h
View File

@ -32,15 +32,16 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
struct lua_State;
class ScriptingEnvironment {
public:
class ScriptingEnvironment
{
public:
ScriptingEnvironment();
explicit ScriptingEnvironment(const char * fileName);
explicit ScriptingEnvironment(const char *file_name);
virtual ~ScriptingEnvironment();
lua_State * getLuaStateForThreadID(const int);
lua_State *getLuaStateForThreadID(const int);
std::vector<lua_State *> luaStateVector;
std::vector<lua_State *> lua_state_vector;
};
#endif /* SCRIPTINGENVIRONMENT_H_ */

9
Extractor/XMLParser.cpp
View File

@ -71,19 +71,19 @@ bool XMLParser::Parse()
{
ImportNode n = ReadXMLNode();
ParseNodeInLua(n, lua_state);
extractor_callbacks->nodeFunction(n);
extractor_callbacks->ProcessNode(n);
}
if (xmlStrEqual(currentName, (const xmlChar *)"way") == 1)
{
ExtractionWay way = ReadXMLWay();
ParseWayInLua(way, lua_state);
extractor_callbacks->wayFunction(way);
extractor_callbacks->ProcessWay(way);
}
if (use_turn_restrictions && xmlStrEqual(currentName, (const xmlChar *)"relation") == 1)
{
InputRestrictionContainer r = ReadXMLRestriction();
if ((UINT_MAX != r.fromWay) && !extractor_callbacks->restrictionFunction(r))
if ((UINT_MAX != r.fromWay) && !extractor_callbacks->ProcessRestriction(r))
{
std::cerr << "[XMLParser] restriction not parsed" << std::endl;
}
@ -328,8 +328,7 @@ ImportNode XMLParser::ReadXMLNode()
xmlChar *value = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"v");
if (k != NULL && value != NULL)
{
node.keyVals.emplace(std::string((char *)(k)),
std::string((char *)(value)));
node.keyVals.emplace(std::string((char *)(k)), std::string((char *)(value)));
}
if (k != NULL)
{

8
extractor.cpp
View File

@ -221,7 +221,7 @@ int main(int argc, char *argv[])
ExtractionContainers extraction_containers;
string_map[""] = 0;
extractor_callbacks = new ExtractorCallbacks(&extraction_containers, &string_map);
extractor_callbacks = new ExtractorCallbacks(extraction_containers, string_map);
BaseParser *parser;
if (file_has_pbf_format)
{
@ -241,6 +241,9 @@ int main(int argc, char *argv[])
std::chrono::steady_clock::now();
parser->Parse();
delete parser;
delete extractor_callbacks;
std::chrono::duration<double> parsing_duration =
std::chrono::steady_clock::now() - parsing_start_time;
SimpleLogger().Write() << "Parsing finished after " << parsing_duration.count()
@ -254,9 +257,6 @@ int main(int argc, char *argv[])
extraction_containers.PrepareData(output_file_name, restriction_fileName);
delete parser;
delete extractor_callbacks;
std::chrono::duration<double> extraction_duration =
std::chrono::steady_clock::now() - startup_time;
SimpleLogger().Write() << "extraction finished after " << extraction_duration.count()