Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

refactor Extractor/

Browse Source
This commit is contained in:
Dennis Luxen 2014-05-09 16:17:31 +02:00
parent 7e639d6bc1
commit 0fccd0f0d2
16 changed files with 1171 additions and 1193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
Extractor/BaseParser.cpp
View File

@ -38,99 +38,99 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <boost/algorithm/string/regex.hpp>
#include <boost/regex.hpp>
BaseParser::BaseParser(
ExtractorCallbacks * extractor_callbacks,
ScriptingEnvironment & scripting_environment
) : extractor_callbacks(extractor_callbacks),
lua_state(scripting_environment.getLuaStateForThreadID(0)),
scripting_environment(scripting_environment),
use_turn_restrictions(true)
BaseParser::BaseParser(ExtractorCallbacks *extractor_callbacks,
ScriptingEnvironment &scripting_environment)
: extractor_callbacks(extractor_callbacks),
lua_state(scripting_environment.getLuaStateForThreadID(0)),
scripting_environment(scripting_environment), use_turn_restrictions(true)
{
ReadUseRestrictionsSetting();
ReadRestrictionExceptions();
}
void BaseParser::ReadUseRestrictionsSetting() {
if( 0 != luaL_dostring( lua_state, "return use_turn_restrictions\n") ) {
void BaseParser::ReadUseRestrictionsSetting()
{
if (0 != luaL_dostring(lua_state, "return use_turn_restrictions\n"))
{
throw OSRMException("ERROR occured in scripting block");
}
if( lua_isboolean( lua_state, -1) ) {
if (lua_isboolean(lua_state, -1))
{
use_turn_restrictions = lua_toboolean(lua_state, -1);
}
if( use_turn_restrictions ) {
if (use_turn_restrictions)
{
SimpleLogger().Write() << "Using turn restrictions";
} else {
}
else
{
SimpleLogger().Write() << "Ignoring turn restrictions";
}
}
void BaseParser::ReadRestrictionExceptions() {
if(lua_function_exists(lua_state, "get_exceptions" )) {
//get list of turn restriction exceptions
void BaseParser::ReadRestrictionExceptions()
{
if (lua_function_exists(lua_state, "get_exceptions"))
{
// get list of turn restriction exceptions
luabind::call_function<void>(
lua_state,
"get_exceptions",
boost::ref(restriction_exceptions)
);
lua_state, "get_exceptions", boost::ref(restriction_exceptions));
const unsigned exception_count = restriction_exceptions.size();
SimpleLogger().Write() <<
"Found " << exception_count << " exceptions to turn restrictions:";
for(const std::string & str : restriction_exceptions) {
SimpleLogger().Write() << "Found " << exception_count
<< " exceptions to turn restrictions:";
for (const std::string &str : restriction_exceptions)
{
SimpleLogger().Write() << " " << str;
}
} else {
}
else
{
SimpleLogger().Write() << "Found no exceptions to turn restrictions";
}
}
void BaseParser::report_errors(lua_State *L, const int status) const {
if( 0!=status ) {
std::cerr << "-- " << lua_tostring(L, -1) << std::endl;
lua_pop(L, 1); // remove error message
void BaseParser::report_errors(lua_State *lua_state, const int status) const
{
if (0 != status)
{
std::cerr << "-- " << lua_tostring(lua_state, -1) << std::endl;
lua_pop(lua_state, 1); // remove error message
}
}
void BaseParser::ParseNodeInLua(ImportNode& n, lua_State* local_lua_state) {
luabind::call_function<void>(
local_lua_state,
"node_function",
boost::ref(n)
);
void BaseParser::ParseNodeInLua(ImportNode &node, lua_State *local_lua_state)
{
luabind::call_function<void>(local_lua_state, "node_function", boost::ref(node));
}
void BaseParser::ParseWayInLua(ExtractionWay& w, lua_State* local_lua_state) {
luabind::call_function<void>(
local_lua_state,
"way_function",
boost::ref(w)
);
void BaseParser::ParseWayInLua(ExtractionWay &way, lua_State *local_lua_state)
{
luabind::call_function<void>(local_lua_state, "way_function", boost::ref(way));
}
bool BaseParser::ShouldIgnoreRestriction(
const std::string & except_tag_string
) const {
//should this restriction be ignored? yes if there's an overlap between:
bool BaseParser::ShouldIgnoreRestriction(const std::string &except_tag_string) const
{
// should this restriction be ignored? yes if there's an overlap between:
// a) the list of modes in the except tag of the restriction
// (except_tag_string), eg: except=bus;bicycle
// b) the lua profile defines a hierachy of modes,
// eg: [access, vehicle, bicycle]
if( except_tag_string.empty() ) {
if (except_tag_string.empty())
{
return false;
}
//Be warned, this is quadratic work here, but we assume that
//only a few exceptions are actually defined.
// Be warned, this is quadratic work here, but we assume that
// only a few exceptions are actually defined.
std::vector<std::string> exceptions;
boost::algorithm::split_regex(exceptions, except_tag_string, boost::regex("[;][ ]*"));
for (std::string& current_string : exceptions) {
std::vector<std::string>::const_iterator string_iterator;
string_iterator = std::find(
restriction_exceptions.begin(),
restriction_exceptions.end(),
current_string
);
if( restriction_exceptions.end() != string_iterator ) {
for (std::string &current_string : exceptions)
{
const auto string_iterator =
std::find(restriction_exceptions.begin(), restriction_exceptions.end(), current_string);
if (restriction_exceptions.end() != string_iterator)
{
return true;
}
}

31
Extractor/BaseParser.h
View File

@ -37,32 +37,29 @@ class ScriptingEnvironment;
struct ExtractionWay;
struct ImportNode;
class BaseParser {
public:
class BaseParser
{
public:
BaseParser() = delete;
BaseParser(const BaseParser&) = delete;
BaseParser(
ExtractorCallbacks * extractor_callbacks,
ScriptingEnvironment & scripting_environment
);
BaseParser(const BaseParser &) = delete;
BaseParser(ExtractorCallbacks *extractor_callbacks,
ScriptingEnvironment &scripting_environment);
virtual ~BaseParser() {}
virtual bool ReadHeader() = 0;
virtual bool Parse() = 0;
virtual void ParseNodeInLua(ImportNode & n, lua_State* thread_lua_state);
virtual void ParseWayInLua(ExtractionWay & n, lua_State* thread_lua_state);
virtual void report_errors(lua_State * lua_state, const int status) const;
virtual void ParseNodeInLua(ImportNode &node, lua_State *lua_state);
virtual void ParseWayInLua(ExtractionWay &way, lua_State *lua_state);
virtual void report_errors(lua_State *lua_state, const int status) const;
protected:
protected:
virtual void ReadUseRestrictionsSetting();
virtual void ReadRestrictionExceptions();
virtual bool ShouldIgnoreRestriction(
const std::string & except_tag_string
) const;
virtual bool ShouldIgnoreRestriction(const std::string &except_tag_string) const;
ExtractorCallbacks * extractor_callbacks;
lua_State * lua_state;
ScriptingEnvironment & scripting_environment;
ExtractorCallbacks *extractor_callbacks;
lua_State *lua_state;
ScriptingEnvironment &scripting_environment;
std::vector<std::string> restriction_exceptions;
bool use_turn_restrictions;
};

438
Extractor/ExtractionContainers.cpp
View File

@ -38,13 +38,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <chrono>
ExtractionContainers::ExtractionContainers() {
//Check if stxxl can be instantiated
ExtractionContainers::ExtractionContainers()
{
// Check if stxxl can be instantiated
stxxl::vector<unsigned> dummy_vector;
name_list.push_back("");
}
ExtractionContainers::~ExtractionContainers() {
ExtractionContainers::~ExtractionContainers()
{
used_node_id_list.clear();
all_nodes_list.clear();
all_edges_list.clear();
@ -53,219 +55,205 @@ ExtractionContainers::~ExtractionContainers() {
way_start_end_id_list.clear();
}
void ExtractionContainers::PrepareData(
const std::string & output_file_name,
const std::string & restrictions_file_name
) {
try {
void ExtractionContainers::PrepareData(const std::string &output_file_name,
const std::string &restrictions_file_name)
{
try
{
unsigned number_of_used_nodes = 0;
unsigned number_of_used_edges = 0;
std::chrono::time_point<std::chrono::steady_clock> time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting used nodes ... " << std::flush;
stxxl::sort(
used_node_id_list.begin(),
used_node_id_list.end(),
Cmp(),
4294967296
);
stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), 4294967296);
std::chrono::time_point<std::chrono::steady_clock> time2 = std::chrono::steady_clock::now();
std::chrono::duration<double> elapsed_seconds = time2-time1;
std::chrono::duration<double> elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush;
stxxl::vector<NodeID>::iterator NewEnd = std::unique ( used_node_id_list.begin(),used_node_id_list.end() ) ;
used_node_id_list.resize ( NewEnd - used_node_id_list.begin() );
auto new_end = std::unique(used_node_id_list.begin(), used_node_id_list.end());
used_node_id_list.resize(new_end - used_node_id_list.begin());
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting all nodes ... " << std::flush;
stxxl::sort(
all_nodes_list.begin(),
all_nodes_list.end(),
CmpNodeByID(),
4294967296
);
stxxl::sort(all_nodes_list.begin(), all_nodes_list.end(), CmpNodeByID(), 4294967296);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting used ways ... " << std::flush;
stxxl::sort(
way_start_end_id_list.begin(),
way_start_end_id_list.end(),
CmpWayByID(),
4294967296
);
way_start_end_id_list.begin(), way_start_end_id_list.end(), CmpWayByID(), 4294967296);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] Sorting restrictions. by from... " << std::flush;
stxxl::sort(
restrictions_list.begin(),
restrictions_list.end(),
CmpRestrictionContainerByFrom(),
4294967296
);
stxxl::sort(restrictions_list.begin(),
restrictions_list.end(),
CmpRestrictionContainerByFrom(),
4294967296);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] Fixing restriction starts ... " << std::flush;
STXXLRestrictionsVector::iterator restrictions_iterator = restrictions_list.begin();
STXXLWayIDStartEndVector::iterator way_start_and_end_iterator = way_start_end_id_list.begin();
auto restrictions_iterator = restrictions_list.begin();
auto way_start_and_end_iterator = way_start_end_id_list.begin();
while(
way_start_and_end_iterator != way_start_end_id_list.end() &&
restrictions_iterator != restrictions_list.end()
) {
if(way_start_and_end_iterator->wayID < restrictions_iterator->fromWay){
while (way_start_and_end_iterator != way_start_end_id_list.end() &&
restrictions_iterator != restrictions_list.end())
{
if (way_start_and_end_iterator->wayID < restrictions_iterator->fromWay)
{
++way_start_and_end_iterator;
continue;
}
if(way_start_and_end_iterator->wayID > restrictions_iterator->fromWay) {
if (way_start_and_end_iterator->wayID > restrictions_iterator->fromWay)
{
++restrictions_iterator;
continue;
}
BOOST_ASSERT(way_start_and_end_iterator->wayID == restrictions_iterator->fromWay);
NodeID via_node_id = restrictions_iterator->restriction.viaNode;
const NodeID via_node_id = restrictions_iterator->restriction.viaNode;
if(way_start_and_end_iterator->firstStart == via_node_id) {
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->firstTarget;
} else if(way_start_and_end_iterator->firstTarget == via_node_id) {
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->firstStart;
} else if(way_start_and_end_iterator->lastStart == via_node_id) {
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->lastTarget;
} else if(way_start_and_end_iterator->lastTarget == via_node_id) {
if (way_start_and_end_iterator->firstStart == via_node_id)
{
restrictions_iterator->restriction.fromNode =
way_start_and_end_iterator->firstTarget;
}
else if (way_start_and_end_iterator->firstTarget == via_node_id)
{
restrictions_iterator->restriction.fromNode =
way_start_and_end_iterator->firstStart;
}
else if (way_start_and_end_iterator->lastStart == via_node_id)
{
restrictions_iterator->restriction.fromNode =
way_start_and_end_iterator->lastTarget;
}
else if (way_start_and_end_iterator->lastTarget == via_node_id)
{
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->lastStart;
}
++restrictions_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting restrictions. by to ... " << std::flush;
stxxl::sort(
restrictions_list.begin(),
restrictions_list.end(),
CmpRestrictionContainerByTo(),
4294967296
);
stxxl::sort(restrictions_list.begin(),
restrictions_list.end(),
CmpRestrictionContainerByTo(),
4294967296);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
unsigned usableRestrictionsCounter(0);
unsigned number_of_useable_restrictions = 0;
std::cout << "[extractor] Fixing restriction ends ... " << std::flush;
restrictions_iterator = restrictions_list.begin();
way_start_and_end_iterator = way_start_end_id_list.begin();
while(
way_start_and_end_iterator != way_start_end_id_list.end() &&
restrictions_iterator != restrictions_list.end()
) {
if(way_start_and_end_iterator->wayID < restrictions_iterator->toWay){
while (way_start_and_end_iterator != way_start_end_id_list.end() &&
restrictions_iterator != restrictions_list.end())
{
if (way_start_and_end_iterator->wayID < restrictions_iterator->toWay)
{
++way_start_and_end_iterator;
continue;
}
if(way_start_and_end_iterator->wayID > restrictions_iterator->toWay) {
if (way_start_and_end_iterator->wayID > restrictions_iterator->toWay)
{
++restrictions_iterator;
continue;
}
NodeID via_node_id = restrictions_iterator->restriction.viaNode;
if(way_start_and_end_iterator->lastStart == via_node_id) {
if (way_start_and_end_iterator->lastStart == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastTarget;
} else if(way_start_and_end_iterator->lastTarget == via_node_id) {
}
else if (way_start_and_end_iterator->lastTarget == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastStart;
} else if(way_start_and_end_iterator->firstStart == via_node_id) {
}
else if (way_start_and_end_iterator->firstStart == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstTarget;
} else if(way_start_and_end_iterator->firstTarget == via_node_id) {
}
else if (way_start_and_end_iterator->firstTarget == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstStart;
}
if(
UINT_MAX != restrictions_iterator->restriction.fromNode &&
UINT_MAX != restrictions_iterator->restriction.toNode
) {
++usableRestrictionsCounter;
if (UINT_MAX != restrictions_iterator->restriction.fromNode &&
UINT_MAX != restrictions_iterator->restriction.toNode)
{
++number_of_useable_restrictions;
}
++restrictions_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
SimpleLogger().Write() << "usable restrictions: " << usableRestrictionsCounter;
//serialize restrictions
SimpleLogger().Write() << "usable restrictions: " << number_of_useable_restrictions;
// serialize restrictions
std::ofstream restrictions_out_stream;
restrictions_out_stream.open(restrictions_file_name.c_str(), std::ios::binary);
restrictions_out_stream.write((char*)&uuid, sizeof(UUID));
restrictions_out_stream.write(
(char*)&usableRestrictionsCounter,
sizeof(unsigned)
);
for(
restrictions_iterator = restrictions_list.begin();
restrictions_iterator != restrictions_list.end();
++restrictions_iterator
) {
if(
UINT_MAX != restrictions_iterator->restriction.fromNode &&
UINT_MAX != restrictions_iterator->restriction.toNode
) {
restrictions_out_stream.write(
(char *)&(restrictions_iterator->restriction),
sizeof(TurnRestriction)
);
restrictions_out_stream.write((char *)&uuid, sizeof(UUID));
restrictions_out_stream.write((char *)&number_of_useable_restrictions, sizeof(unsigned));
for (restrictions_iterator = restrictions_list.begin();
restrictions_iterator != restrictions_list.end();
++restrictions_iterator)
{
if (UINT_MAX != restrictions_iterator->restriction.fromNode &&
UINT_MAX != restrictions_iterator->restriction.toNode)
{
restrictions_out_stream.write((char *)&(restrictions_iterator->restriction),
sizeof(TurnRestriction));
}
}
restrictions_out_stream.close();
std::ofstream file_out_stream;
file_out_stream.open(output_file_name.c_str(), std::ios::binary);
file_out_stream.write((char*)&uuid, sizeof(UUID));
file_out_stream.write((char*)&number_of_used_nodes, sizeof(unsigned));
file_out_stream.write((char *)&uuid, sizeof(UUID));
file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush;
//identify all used nodes by a merging step of two sorted lists
STXXLNodeVector::iterator node_iterator = all_nodes_list.begin();
STXXLNodeIDVector::iterator node_id_iterator = used_node_id_list.begin();
while(
node_id_iterator != used_node_id_list.end() &&
node_iterator != all_nodes_list.end()
) {
if(*node_id_iterator < node_iterator->id){
// identify all used nodes by a merging step of two sorted lists
auto node_iterator = all_nodes_list.begin();
auto node_id_iterator = used_node_id_list.begin();
while (node_id_iterator != used_node_id_list.end() && node_iterator != all_nodes_list.end())
{
if (*node_id_iterator < node_iterator->id)
{
++node_id_iterator;
continue;
}
if(*node_id_iterator > node_iterator->id) {
if (*node_id_iterator > node_iterator->id)
{
++node_iterator;
continue;
}
BOOST_ASSERT( *node_id_iterator == node_iterator->id);
BOOST_ASSERT(*node_id_iterator == node_iterator->id);
file_out_stream.write(
(char*)&(*node_iterator),
sizeof(ExternalMemoryNode)
);
file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode));
++number_of_used_nodes;
++node_id_iterator;
@ -273,14 +261,13 @@ void ExtractionContainers::PrepareData(
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] setting number of nodes ... " << std::flush;
std::ios::pos_type previous_file_position = file_out_stream.tellp();
file_out_stream.seekp(std::ios::beg+sizeof(UUID));
file_out_stream.write((char*)&number_of_used_nodes, sizeof(unsigned));
file_out_stream.seekp(std::ios::beg + sizeof(UUID));
file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
file_out_stream.seekp(previous_file_position);
std::cout << "ok" << std::endl;
@ -288,218 +275,175 @@ void ExtractionContainers::PrepareData(
// Sort edges by start.
std::cout << "[extractor] Sorting edges by start ... " << std::flush;
stxxl::sort(
all_edges_list.begin(),
all_edges_list.end(),
CmpEdgeByStartID(),
4294967296
);
stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartID(), 4294967296);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Setting start coords ... " << std::flush;
file_out_stream.write((char*)&number_of_used_edges, sizeof(unsigned));
file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
// Traverse list of edges and nodes in parallel and set start coord
node_iterator = all_nodes_list.begin();
STXXLEdgeVector::iterator edge_iterator = all_edges_list.begin();
while(
edge_iterator != all_edges_list.end() &&
node_iterator != all_nodes_list.end()
) {
if(edge_iterator->start < node_iterator->id){
auto edge_iterator = all_edges_list.begin();
while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
{
if (edge_iterator->start < node_iterator->id)
{
++edge_iterator;
continue;
}
if(edge_iterator->start > node_iterator->id) {
if (edge_iterator->start > node_iterator->id)
{
node_iterator++;
continue;
}
BOOST_ASSERT(edge_iterator->start == node_iterator->id);
edge_iterator->startCoord.lat = node_iterator->lat;
edge_iterator->startCoord.lon = node_iterator->lon;
edge_iterator->source_coordinate.lat = node_iterator->lat;
edge_iterator->source_coordinate.lon = node_iterator->lon;
++edge_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
// Sort Edges by target
std::cout << "[extractor] Sorting edges by target ... " << std::flush;
stxxl::sort(
all_edges_list.begin(),
all_edges_list.end(),
CmpEdgeByTargetID(),
4294967296
);
stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByTargetID(), 4294967296);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Setting target coords ... " << std::flush;
// Traverse list of edges and nodes in parallel and set target coord
node_iterator = all_nodes_list.begin();
edge_iterator = all_edges_list.begin();
while(
edge_iterator != all_edges_list.end() &&
node_iterator != all_nodes_list.end()
) {
if(edge_iterator->target < node_iterator->id){
while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
{
if (edge_iterator->target < node_iterator->id)
{
++edge_iterator;
continue;
}
if(edge_iterator->target > node_iterator->id) {
if (edge_iterator->target > node_iterator->id)
{
++node_iterator;
continue;
}
BOOST_ASSERT(edge_iterator->target == node_iterator->id);
if(edge_iterator->startCoord.lat != INT_MIN && edge_iterator->startCoord.lon != INT_MIN) {
edge_iterator->targetCoord.lat = node_iterator->lat;
edge_iterator->targetCoord.lon = node_iterator->lon;
const double distance = FixedPointCoordinate::ApproximateDistance(
edge_iterator->startCoord.lat,
edge_iterator->startCoord.lon,
node_iterator->lat,
node_iterator->lon
);
if (edge_iterator->source_coordinate.lat != INT_MIN &&
edge_iterator->source_coordinate.lon != INT_MIN)
{
BOOST_ASSERT(edge_iterator->speed != -1);
const double weight = ( distance * 10. ) / (edge_iterator->speed / 3.6);
int integer_weight = std::max( 1, (int)std::floor((edge_iterator->isDurationSet ? edge_iterator->speed : weight)+.5) );
int integer_distance = std::max( 1, (int)distance );
BOOST_ASSERT(edge_iterator->type >= 0);
edge_iterator->target_coordinate.lat = node_iterator->lat;
edge_iterator->target_coordinate.lon = node_iterator->lon;
const double distance = FixedPointCoordinate::ApproximateEuclideanDistance(
edge_iterator->source_coordinate.lat,
edge_iterator->source_coordinate.lon,
node_iterator->lat,
node_iterator->lon);
const double weight = (distance * 10.) / (edge_iterator->speed / 3.6);
int integer_weight = std::max(
1,
(int)std::floor(
(edge_iterator->is_duration_set ? edge_iterator->speed : weight) + .5));
int integer_distance = std::max(1, (int)distance);
short zero = 0;
short one = 1;
file_out_stream.write((char*)&edge_iterator->start, sizeof(unsigned));
file_out_stream.write((char*)&edge_iterator->target, sizeof(unsigned));
file_out_stream.write((char*)&integer_distance, sizeof(int));
switch(edge_iterator->direction) {
file_out_stream.write((char *)&edge_iterator->start, sizeof(unsigned));
file_out_stream.write((char *)&edge_iterator->target, sizeof(unsigned));
file_out_stream.write((char *)&integer_distance, sizeof(int));
switch (edge_iterator->direction)
{
case ExtractionWay::notSure:
file_out_stream.write((char*)&zero, sizeof(short));
file_out_stream.write((char *)&zero, sizeof(short));
break;
case ExtractionWay::oneway:
file_out_stream.write((char*)&one, sizeof(short));
file_out_stream.write((char *)&one, sizeof(short));
break;
case ExtractionWay::bidirectional:
file_out_stream.write((char*)&zero, sizeof(short));
file_out_stream.write((char *)&zero, sizeof(short));
break;
case ExtractionWay::opposite:
file_out_stream.write((char*)&one, sizeof(short));
file_out_stream.write((char *)&one, sizeof(short));
break;
default:
throw OSRMException("edge has broken direction");
}
file_out_stream.write(
(char*)&integer_weight, sizeof(int)
);
BOOST_ASSERT(edge_iterator->type >= 0);
file_out_stream.write(
(char*)&edge_iterator->type,
sizeof(short)
);
file_out_stream.write(
(char *) &edge_iterator->nameID,
sizeof(unsigned)
);
file_out_stream.write(
(char *) &edge_iterator->isRoundabout,
sizeof(bool)
);
file_out_stream.write(
(char *) &edge_iterator->ignoreInGrid,
sizeof(bool)
);
file_out_stream.write(
(char *) &edge_iterator->isAccessRestricted,
sizeof(bool)
);
file_out_stream.write(
(char *) &edge_iterator->isContraFlow,
sizeof(bool)
);
file_out_stream.write(
(char *) &edge_iterator->is_split,
sizeof(bool)
);
file_out_stream.write((char *)&integer_weight, sizeof(int));
file_out_stream.write((char *)&edge_iterator->type, sizeof(short));
file_out_stream.write((char *)&edge_iterator->name_id, sizeof(unsigned));
file_out_stream.write((char *)&edge_iterator->is_roundabout, sizeof(bool));
file_out_stream.write((char *)&edge_iterator->is_in_tiny_cc, sizeof(bool));
file_out_stream.write((char *)&edge_iterator->is_access_restricted, sizeof(bool));
file_out_stream.write((char *)&edge_iterator->is_contra_flow, sizeof(bool));
file_out_stream.write((char *)&edge_iterator->is_split, sizeof(bool));
++number_of_used_edges;
}
++edge_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] setting number of edges ... " << std::flush;
file_out_stream.seekp(previous_file_position);
file_out_stream.write((char*)&number_of_used_edges, sizeof(unsigned));
file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
file_out_stream.close();
std::cout << "ok" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] writing street name index ... " << std::flush;
std::string name_file_streamName = (output_file_name + ".names");
boost::filesystem::ofstream name_file_stream(
name_file_streamName,
std::ios::binary
);
boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary);
//write number of names
const unsigned number_of_names = name_list.size()+1;
// write number of names
const unsigned number_of_names = name_list.size() + 1;
name_file_stream.write((char *)&(number_of_names), sizeof(unsigned));
//compute total number of chars
// compute total number of chars
unsigned total_number_of_chars = 0;
for (const std::string & temp_string : name_list) {
for (const std::string &temp_string : name_list)
{
total_number_of_chars += temp_string.length();
}
//write total number of chars
name_file_stream.write(
(char *)&(total_number_of_chars),
sizeof(unsigned)
);
//write prefixe sums
// write total number of chars
name_file_stream.write((char *)&(total_number_of_chars), sizeof(unsigned));
// write prefixe sums
unsigned name_lengths_prefix_sum = 0;
for (const std::string & temp_string : name_list) {
name_file_stream.write(
(char *)&(name_lengths_prefix_sum),
sizeof(unsigned)
);
for (const std::string &temp_string : name_list)
{
name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
name_lengths_prefix_sum += temp_string.length();
}
//duplicate on purpose!
name_file_stream.write(
(char *)&(name_lengths_prefix_sum),
sizeof(unsigned)
);
// duplicate on purpose!
name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
//write all chars consecutively
for (const std::string & temp_string : name_list) {
// write all chars consecutively
for (const std::string &temp_string : name_list)
{
const unsigned string_length = temp_string.length();
name_file_stream.write(temp_string.c_str(), string_length);
}
name_file_stream.close();
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2-time1;
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
SimpleLogger().Write() << "Processed " <<
number_of_used_nodes << " nodes and " <<
number_of_used_edges << " edges";
} catch ( const std::exception& e ) {
std::cerr << "Caught Execption:" << e.what() << std::endl;
SimpleLogger().Write() << "Processed " << number_of_used_nodes << " nodes and "
<< number_of_used_edges << " edges";
}
catch (const std::exception &e) { std::cerr << "Caught Execption:" << e.what() << std::endl; }
}

33
Extractor/ExtractionContainers.h
View File

@ -35,31 +35,30 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <stxxl/vector>
class ExtractionContainers {
public:
typedef stxxl::vector<NodeID> STXXLNodeIDVector;
typedef stxxl::vector<ExternalMemoryNode> STXXLNodeVector;
typedef stxxl::vector<InternalExtractorEdge> STXXLEdgeVector;
typedef stxxl::vector<std::string> STXXLStringVector;
class ExtractionContainers
{
public:
typedef stxxl::vector<NodeID> STXXLNodeIDVector;
typedef stxxl::vector<ExternalMemoryNode> STXXLNodeVector;
typedef stxxl::vector<InternalExtractorEdge> STXXLEdgeVector;
typedef stxxl::vector<std::string> STXXLStringVector;
typedef stxxl::vector<InputRestrictionContainer> STXXLRestrictionsVector;
typedef stxxl::vector<_WayIDStartAndEndEdge> STXXLWayIDStartEndVector;
typedef stxxl::vector<WayIDStartAndEndEdge> STXXLWayIDStartEndVector;
STXXLNodeIDVector used_node_id_list;
STXXLNodeVector all_nodes_list;
STXXLEdgeVector all_edges_list;
STXXLStringVector name_list;
STXXLRestrictionsVector restrictions_list;
STXXLWayIDStartEndVector way_start_end_id_list;
STXXLNodeIDVector used_node_id_list;
STXXLNodeVector all_nodes_list;
STXXLEdgeVector all_edges_list;
STXXLStringVector name_list;
STXXLRestrictionsVector restrictions_list;
STXXLWayIDStartEndVector way_start_end_id_list;
const UUID uuid;
ExtractionContainers();
virtual ~ExtractionContainers();
void PrepareData(
const std::string & output_file_name,
const std::string & restrictions_file_name
);
void PrepareData(const std::string &output_file_name,
const std::string &restrictions_file_name);
};
#endif /* EXTRACTIONCONTAINERS_H_ */

74
Extractor/ExtractionHelperFunctions.h
View File

@ -25,63 +25,65 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef EXTRACTIONHELPERFUNCTIONS_H_
#define EXTRACTIONHELPERFUNCTIONS_H_
#ifndef EXTRACTION_HELPER_FUNCTIONS_H
#define EXTRACTION_HELPER_FUNCTIONS_H
#include "../Util/StringUtil.h"
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string_regex.hpp>
#include <boost/regex.hpp>
#include <climits>
#include <limits>
namespace qi = boost::spirit::qi;
//TODO: Move into LUA
// TODO: Move into LUA
inline bool durationIsValid(const std::string &s) {
boost::regex e ("((\\d|\\d\\d):(\\d|\\d\\d):(\\d|\\d\\d))|((\\d|\\d\\d):(\\d|\\d\\d))|(\\d|\\d\\d)",boost::regex_constants::icase|boost::regex_constants::perl);
inline bool durationIsValid(const std::string &s)
{
boost::regex e(
"((\\d|\\d\\d):(\\d|\\d\\d):(\\d|\\d\\d))|((\\d|\\d\\d):(\\d|\\d\\d))|(\\d|\\d\\d)",
boost::regex_constants::icase | boost::regex_constants::perl);
std::vector< std::string > result;
boost::algorithm::split_regex( result, s, boost::regex( ":" ) ) ;
std::vector<std::string> result;
boost::algorithm::split_regex(result, s, boost::regex(":"));
bool matched = regex_match(s, e);
return matched;
}
inline unsigned parseDuration(const std::string &s) {
unsigned hours = 0;
inline unsigned parseDuration(const std::string &s)
{
unsigned hours = 0;
unsigned minutes = 0;
unsigned seconds = 0;
boost::regex e ("((\\d|\\d\\d):(\\d|\\d\\d):(\\d|\\d\\d))|((\\d|\\d\\d):(\\d|\\d\\d))|(\\d|\\d\\d)",boost::regex_constants::icase|boost::regex_constants::perl);
boost::regex e(
"((\\d|\\d\\d):(\\d|\\d\\d):(\\d|\\d\\d))|((\\d|\\d\\d):(\\d|\\d\\d))|(\\d|\\d\\d)",
boost::regex_constants::icase | boost::regex_constants::perl);
std::vector< std::string > result;
boost::algorithm::split_regex( result, s, boost::regex( ":" ) ) ;
std::vector<std::string> result;
boost::algorithm::split_regex(result, s, boost::regex(":"));
bool matched = regex_match(s, e);
if(matched) {
if(1 == result.size()) {
minutes = stringToInt(result[0]);
}
if(2 == result.size()) {
minutes = stringToInt(result[1]);
hours = stringToInt(result[0]);
}
if(3 == result.size()) {
if (matched)
{
if (1 == result.size())
{
minutes = stringToInt(result[0]);
}
if (2 == result.size())
{
minutes = stringToInt(result[1]);
hours = stringToInt(result[0]);
}
if (3 == result.size())
{
seconds = stringToInt(result[2]);
minutes = stringToInt(result[1]);
hours = stringToInt(result[0]);
}
return 10*(3600*hours+60*minutes+seconds);
hours = stringToInt(result[0]);
}
return 10 * (3600 * hours + 60 * minutes + seconds);
}
return UINT_MAX;
return std::numeric_limits<unsigned>::max();
}
// inline int parseMaxspeed(std::string input) { //call-by-value on purpose.
// boost::algorithm::to_lower(input);
// int n = stringToInt(input);
// if (input.find("mph") != std::string::npos || input.find("mp/h") != std::string::npos) {
// n = (n*1609)/1000;
// }
// return n;
// }
#endif /* EXTRACTIONHELPERFUNCTIONS_H_ */
#endif // EXTRACTION_HELPER_FUNCTIONS_H_

27
Extractor/ExtractionWay.h
View File

@ -34,14 +34,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <string>
#include <vector>
struct ExtractionWay {
ExtractionWay() {
Clear();
}
struct ExtractionWay
{
ExtractionWay() { Clear(); }
inline void Clear(){
id = UINT_MAX;
nameID = UINT_MAX;
inline void Clear()
{
id = SPECIAL_NODEID;
nameID = INVALID_NAMEID;
path.clear();
keyVals.clear();
direction = ExtractionWay::notSure;
@ -55,9 +55,11 @@ struct ExtractionWay {
ignoreInGrid = false;
}
enum Directions {
notSure = 0, oneway, bidirectional, opposite
};
enum Directions
{ notSure = 0,
oneway,
bidirectional,
opposite };
unsigned id;
unsigned nameID;
double speed;
@ -70,9 +72,8 @@ struct ExtractionWay {
bool roundabout;
bool isAccessRestricted;
bool ignoreInGrid;
std::vector< NodeID > path;
std::vector<NodeID> path;
HashTable<std::string, std::string> keyVals;
};
#endif //EXTRACTION_WAY_H
#endif // EXTRACTION_WAY_H

175
Extractor/ExtractorCallbacks.cpp
View File

@ -38,117 +38,126 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <string>
#include <vector>
ExtractorCallbacks::ExtractorCallbacks()
:
string_map(NULL),
externalMemory(NULL)
{ }
ExtractorCallbacks::ExtractorCallbacks(
ExtractionContainers * ext,
boost::unordered_map<std::string, NodeID> * string_map
) :
string_map(string_map),
externalMemory(ext)
{ }
ExtractorCallbacks::~ExtractorCallbacks() { }
ExtractorCallbacks::ExtractorCallbacks(ExtractionContainers &extraction_containers,
boost::unordered_map<std::string, NodeID> &string_map)
: string_map(string_map), external_memory(extraction_containers)
{
}
/** warning: caller needs to take care of synchronization! */
void ExtractorCallbacks::nodeFunction(const ExternalMemoryNode &n) {
if(n.lat <= 85*COORDINATE_PRECISION && n.lat >= -85*COORDINATE_PRECISION) {
externalMemory->all_nodes_list.push_back(n);
void ExtractorCallbacks::ProcessNode(const ExternalMemoryNode &n)
{
if (n.lat <= 85 * COORDINATE_PRECISION && n.lat >= -85 * COORDINATE_PRECISION)
{
external_memory.all_nodes_list.push_back(n);
}
}
bool ExtractorCallbacks::restrictionFunction(const InputRestrictionContainer &r) {
externalMemory->restrictions_list.push_back(r);
bool ExtractorCallbacks::ProcessRestriction(const InputRestrictionContainer &restriction)
{
external_memory.restrictions_list.push_back(restriction);
return true;
}
/** warning: caller needs to take care of synchronization! */
void ExtractorCallbacks::wayFunction(ExtractionWay &parsed_way) {
if((0 < parsed_way.speed) || (0 < parsed_way.duration)) { //Only true if the way is specified by the speed profile
if(UINT_MAX == parsed_way.id){
SimpleLogger().Write(logDEBUG) <<
"found bogus way with id: " << parsed_way.id <<
" of size " << parsed_way.path.size();
void ExtractorCallbacks::ProcessWay(ExtractionWay &parsed_way)
{
if ((0 < parsed_way.speed) || (0 < parsed_way.duration))
{ // Only true if the way is specified by the speed profile
if (UINT_MAX == parsed_way.id)
{
SimpleLogger().Write(logDEBUG) << "found bogus way with id: " << parsed_way.id
<< " of size " << parsed_way.path.size();
return;
}
if(0 < parsed_way.duration) {
//TODO: iterate all way segments and set duration corresponding to the length of each segment
parsed_way.speed = parsed_way.duration/(parsed_way.path.size()-1);
if (0 < parsed_way.duration)
{
// TODO: iterate all way segments and set duration corresponding to the length of each
// segment
parsed_way.speed = parsed_way.duration / (parsed_way.path.size() - 1);
}
if(std::numeric_limits<double>::epsilon() >= std::abs(-1. - parsed_way.speed)){
SimpleLogger().Write(logDEBUG) <<
"found way with bogus speed, id: " << parsed_way.id;
if (std::numeric_limits<double>::epsilon() >= std::abs(-1. - parsed_way.speed))
{
SimpleLogger().Write(logDEBUG) << "found way with bogus speed, id: " << parsed_way.id;
return;
}
//Get the unique identifier for the street name
const boost::unordered_map<std::string, NodeID>::const_iterator & string_map_iterator = string_map->find(parsed_way.name);
if(string_map->end() == string_map_iterator) {
parsed_way.nameID = externalMemory->name_list.size();
externalMemory->name_list.push_back(parsed_way.name);
string_map->insert(std::make_pair(parsed_way.name, parsed_way.nameID));
} else {
// Get the unique identifier for the street name
const auto &string_map_iterator = string_map.find(parsed_way.name);
if (string_map.end() == string_map_iterator)
{
parsed_way.nameID = external_memory.name_list.size();
external_memory.name_list.push_back(parsed_way.name);
string_map.insert(std::make_pair(parsed_way.name, parsed_way.nameID));
}
else
{
parsed_way.nameID = string_map_iterator->second;
}
if(ExtractionWay::opposite == parsed_way.direction) {
std::reverse( parsed_way.path.begin(), parsed_way.path.end() );
if (ExtractionWay::opposite == parsed_way.direction)
{
std::reverse(parsed_way.path.begin(), parsed_way.path.end());
parsed_way.direction = ExtractionWay::oneway;
}
const bool split_bidirectional_edge = (parsed_way.backward_speed > 0) && (parsed_way.speed != parsed_way.backward_speed);
const bool split_bidirectional_edge =
(parsed_way.backward_speed > 0) && (parsed_way.speed != parsed_way.backward_speed);
for(std::vector< NodeID >::size_type n = 0; n < parsed_way.path.size()-1; ++n) {
externalMemory->all_edges_list.push_back(
InternalExtractorEdge(
parsed_way.path[n],
parsed_way.path[n+1],
parsed_way.type,
(split_bidirectional_edge ? ExtractionWay::oneway : parsed_way.direction),
parsed_way.speed,
parsed_way.nameID,
parsed_way.roundabout,
parsed_way.ignoreInGrid,
(0 < parsed_way.duration),
parsed_way.isAccessRestricted,
false,
split_bidirectional_edge
)
);
externalMemory->used_node_id_list.push_back(parsed_way.path[n]);
for (unsigned n = 0; n < parsed_way.path.size() - 1; ++n)
{
external_memory.all_edges_list.push_back(InternalExtractorEdge(
parsed_way.path[n],
parsed_way.path[n + 1],
parsed_way.type,
(split_bidirectional_edge ? ExtractionWay::oneway : parsed_way.direction),
parsed_way.speed,
parsed_way.nameID,
parsed_way.roundabout,
parsed_way.ignoreInGrid,
(0 < parsed_way.duration),
parsed_way.isAccessRestricted,
false,
split_bidirectional_edge));
external_memory.used_node_id_list.push_back(parsed_way.path[n]);
}
externalMemory->used_node_id_list.push_back(parsed_way.path.back());
external_memory.used_node_id_list.push_back(parsed_way.path.back());
//The following information is needed to identify start and end segments of restrictions
externalMemory->way_start_end_id_list.push_back(_WayIDStartAndEndEdge(parsed_way.id, parsed_way.path[0], parsed_way.path[1], parsed_way.path[parsed_way.path.size()-2], parsed_way.path.back()));
// The following information is needed to identify start and end segments of restrictions
external_memory.way_start_end_id_list.push_back(
WayIDStartAndEndEdge(parsed_way.id,
parsed_way.path[0],
parsed_way.path[1],
parsed_way.path[parsed_way.path.size() - 2],
parsed_way.path.back()));
if(split_bidirectional_edge) { //Only true if the way should be split
std::reverse( parsed_way.path.begin(), parsed_way.path.end() );
for(std::vector< NodeID >::size_type n = 0; n < parsed_way.path.size()-1; ++n) {
externalMemory->all_edges_list.push_back(
InternalExtractorEdge(
parsed_way.path[n],
parsed_way.path[n+1],
parsed_way.type,
ExtractionWay::oneway,
parsed_way.backward_speed,
parsed_way.nameID,
parsed_way.roundabout,
parsed_way.ignoreInGrid,
(0 < parsed_way.duration),
parsed_way.isAccessRestricted,
(ExtractionWay::oneway == parsed_way.direction),
split_bidirectional_edge
)
);
if (split_bidirectional_edge)
{ // Only true if the way should be split
std::reverse(parsed_way.path.begin(), parsed_way.path.end());
for (std::vector<NodeID>::size_type n = 0; n < parsed_way.path.size() - 1; ++n)
{
external_memory.all_edges_list.push_back(
InternalExtractorEdge(parsed_way.path[n],
parsed_way.path[n + 1],
parsed_way.type,
ExtractionWay::oneway,
parsed_way.backward_speed,
parsed_way.nameID,
parsed_way.roundabout,
parsed_way.ignoreInGrid,
(0 < parsed_way.duration),
parsed_way.isAccessRestricted,
(ExtractionWay::oneway == parsed_way.direction),
split_bidirectional_edge));
}
externalMemory->way_start_end_id_list.push_back(_WayIDStartAndEndEdge(parsed_way.id, parsed_way.path[0], parsed_way.path[1], parsed_way.path[parsed_way.path.size()-2], parsed_way.path.back()));
external_memory.way_start_end_id_list.push_back(
WayIDStartAndEndEdge(parsed_way.id,
parsed_way.path[0],
parsed_way.path[1],
parsed_way.path[parsed_way.path.size() - 2],
parsed_way.path.back()));
}
}
}

41
Extractor/ExtractorCallbacks.h
View File

@ -25,8 +25,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef EXTRACTORCALLBACKS_H_
#define EXTRACTORCALLBACKS_H_
#ifndef EXTRACTOR_CALLBACKS_H
#define EXTRACTOR_CALLBACKS_H
#include "../typedefs.h"
@ -38,29 +38,26 @@ class ExtractionContainers;
struct ExtractionWay;
struct InputRestrictionContainer;
class ExtractorCallbacks{
private:
class ExtractorCallbacks
{
private:
boost::unordered_map<std::string, NodeID> &string_map;
ExtractionContainers &external_memory;
boost::unordered_map<std::string, NodeID> * string_map;
ExtractionContainers * externalMemory;
public:
ExtractorCallbacks() = delete;
ExtractorCallbacks(const ExtractorCallbacks &) = delete;
explicit ExtractorCallbacks(ExtractionContainers &extraction_containers,
boost::unordered_map<std::string, NodeID> &string_map);
ExtractorCallbacks();
public:
explicit ExtractorCallbacks(
ExtractionContainers * ext,
boost::unordered_map<std::string, NodeID> * string_map
);
// warning: caller needs to take care of synchronization!
void ProcessNode(const ExternalMemoryNode &node);
~ExtractorCallbacks();
/** warning: caller needs to take care of synchronization! */
void nodeFunction(const ExternalMemoryNode &n);
bool restrictionFunction(const InputRestrictionContainer &r);
/** warning: caller needs to take care of synchronization! */
void wayFunction(ExtractionWay &w);
// warning: caller needs to take care of synchronization!
bool ProcessRestriction(const InputRestrictionContainer &restriction);
// warning: caller needs to take care of synchronization!
void ProcessWay(ExtractionWay &way);
};
#endif /* EXTRACTORCALLBACKS_H_ */
#endif /* EXTRACTOR_CALLBACKS_H */

121
Extractor/ExtractorStructs.h
View File

@ -34,97 +34,80 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <string>
struct ExtractorRelation {
ExtractorRelation() : type(unknown){}
enum {
unknown = 0, ferry, turnRestriction
} type;
struct ExtractorRelation
{
ExtractorRelation() : type(unknown) {}
enum
{ unknown = 0,
ferry,
turnRestriction } type;
HashTable<std::string, std::string> keyVals;
};
struct _WayIDStartAndEndEdge {
struct WayIDStartAndEndEdge
{
unsigned wayID;
NodeID firstStart;
NodeID firstTarget;
NodeID lastStart;
NodeID lastTarget;
_WayIDStartAndEndEdge()
:
wayID(UINT_MAX),
firstStart(UINT_MAX),
firstTarget(UINT_MAX),
lastStart(UINT_MAX),
lastTarget(UINT_MAX)
{ }
explicit _WayIDStartAndEndEdge(
unsigned w,
NodeID fs,
NodeID ft,
NodeID ls,
NodeID lt
) :
wayID(w),
firstStart(fs),
firstTarget(ft),
lastStart(ls),
lastTarget(lt)
{ }
static _WayIDStartAndEndEdge min_value() {
return _WayIDStartAndEndEdge((std::numeric_limits<unsigned>::min)(), (std::numeric_limits<unsigned>::min)(), (std::numeric_limits<unsigned>::min)(), (std::numeric_limits<unsigned>::min)(), (std::numeric_limits<unsigned>::min)());
WayIDStartAndEndEdge()
: wayID(UINT_MAX), firstStart(UINT_MAX), firstTarget(UINT_MAX), lastStart(UINT_MAX),
lastTarget(UINT_MAX)
{
}
static _WayIDStartAndEndEdge max_value() {
return _WayIDStartAndEndEdge((std::numeric_limits<unsigned>::max)(), (std::numeric_limits<unsigned>::max)(), (std::numeric_limits<unsigned>::max)(), (std::numeric_limits<unsigned>::max)(), (std::numeric_limits<unsigned>::max)());
explicit WayIDStartAndEndEdge(unsigned w, NodeID fs, NodeID ft, NodeID ls, NodeID lt)
: wayID(w), firstStart(fs), firstTarget(ft), lastStart(ls), lastTarget(lt)
{
}
static WayIDStartAndEndEdge min_value()
{
return WayIDStartAndEndEdge((std::numeric_limits<unsigned>::min)(),
(std::numeric_limits<unsigned>::min)(),
(std::numeric_limits<unsigned>::min)(),
(std::numeric_limits<unsigned>::min)(),
(std::numeric_limits<unsigned>::min)());
}
static WayIDStartAndEndEdge max_value()
{
return WayIDStartAndEndEdge((std::numeric_limits<unsigned>::max)(),
(std::numeric_limits<unsigned>::max)(),
(std::numeric_limits<unsigned>::max)(),
(std::numeric_limits<unsigned>::max)(),
(std::numeric_limits<unsigned>::max)());
}
};
struct CmpWayByID {
typedef _WayIDStartAndEndEdge value_type;
bool operator ()(
const _WayIDStartAndEndEdge & a,
const _WayIDStartAndEndEdge & b
) const {
struct CmpWayByID
{
typedef WayIDStartAndEndEdge value_type;
bool operator()(const WayIDStartAndEndEdge &a, const WayIDStartAndEndEdge &b) const
{
return a.wayID < b.wayID;
}
value_type max_value() {
return _WayIDStartAndEndEdge::max_value();
}
value_type min_value() {
return _WayIDStartAndEndEdge::min_value();
}
value_type max_value() { return WayIDStartAndEndEdge::max_value(); }
value_type min_value() { return WayIDStartAndEndEdge::min_value(); }
};
struct Cmp {
struct Cmp
{
typedef NodeID value_type;
bool operator ()(
const NodeID a,
const NodeID b
) const {
return a < b;
}
value_type max_value() {
return 0xffffffff;
}
value_type min_value() {
return 0x0;
}
bool operator()(const NodeID a, const NodeID b) const { return a < b; }
value_type max_value() { return 0xffffffff; }
value_type min_value() { return 0x0; }
};
struct CmpNodeByID {
struct CmpNodeByID
{
typedef ExternalMemoryNode value_type;
bool operator () (
const ExternalMemoryNode & a,
const ExternalMemoryNode & b
) const {
bool operator()(const ExternalMemoryNode &a, const ExternalMemoryNode &b) const
{
return a.id < b.id;
}
value_type max_value() {
return ExternalMemoryNode::max_value();
}
value_type min_value() {
return ExternalMemoryNode::min_value();
}
value_type max_value() { return ExternalMemoryNode::max_value(); }
value_type min_value() { return ExternalMemoryNode::min_value(); }
};
#endif /* EXTRACTORSTRUCTS_H_ */

153
Extractor/InternalExtractorEdge.h
View File

@ -28,92 +28,49 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef INTERNAL_EXTRACTOR_EDGE_H
#define INTERNAL_EXTRACTOR_EDGE_H
#include "../typedefs.h"
#include <osrm/Coordinate.h>
#include <boost/assert.hpp>
struct InternalExtractorEdge {
struct InternalExtractorEdge
{
InternalExtractorEdge()
:
start(0),
target(0),
type(0),
direction(0),
speed(0),
nameID(0),
isRoundabout(false),
ignoreInGrid(false),
isDurationSet(false),
isAccessRestricted(false),
isContraFlow(false),
is_split(false)
{ }
: start(0), target(0), type(0), direction(0), speed(0), name_id(0), is_roundabout(false),
is_in_tiny_cc(false), is_duration_set(false), is_access_restricted(false),
is_contra_flow(false), is_split(false)
{
}
explicit InternalExtractorEdge(
NodeID start,
NodeID target,
short type,
short direction,
double speed,
unsigned nameID,
bool isRoundabout,
bool ignoreInGrid,
bool isDurationSet,
bool isAccressRestricted,
bool isContraFlow,
bool is_split
) :
start(start),
target(target),
type(type),
direction(direction),
speed(speed),
nameID(nameID),
isRoundabout(isRoundabout),
ignoreInGrid(ignoreInGrid),
isDurationSet(isDurationSet),
isAccessRestricted(isAccressRestricted),
isContraFlow(isContraFlow),
is_split(is_split)
explicit InternalExtractorEdge(NodeID start,
NodeID target,
short type,
short direction,
double speed,
unsigned name_id,
bool is_roundabout,
bool is_in_tiny_cc,
bool is_duration_set,
bool is_access_restricted,
bool is_contra_flow,
bool is_split)
: start(start), target(target), type(type), direction(direction), speed(speed),
name_id(name_id), is_roundabout(is_roundabout), is_in_tiny_cc(is_in_tiny_cc),
is_duration_set(is_duration_set), is_access_restricted(is_access_restricted),
is_contra_flow(is_contra_flow), is_split(is_split)
{
BOOST_ASSERT(0 <= type);
}
// necessary static util functions for stxxl's sorting
static InternalExtractorEdge min_value() {
return InternalExtractorEdge(
0,
0,
0,
0,
0,
0,
false,
false,
false,
false,
false,
false
);
static InternalExtractorEdge min_value()
{
return InternalExtractorEdge(0, 0, 0, 0, 0, 0, false, false, false, false, false, false);
}
static InternalExtractorEdge max_value() {
static InternalExtractorEdge max_value()
{
return InternalExtractorEdge(
std::numeric_limits<unsigned>::max(),
std::numeric_limits<unsigned>::max(),
0,
0,
0,
0,
false,
false,
false,
false,
false,
false
);
SPECIAL_NODEID, SPECIAL_NODEID, 0, 0, 0, 0, false, false, false, false, false, false);
}
NodeID start;
@ -121,53 +78,43 @@ struct InternalExtractorEdge {
short type;
short direction;
double speed;
unsigned nameID;
bool isRoundabout;
bool ignoreInGrid;
bool isDurationSet;
bool isAccessRestricted;
bool isContraFlow;
unsigned name_id;
bool is_roundabout;
bool is_in_tiny_cc;
bool is_duration_set;
bool is_access_restricted;
bool is_contra_flow;
bool is_split;
FixedPointCoordinate startCoord;
FixedPointCoordinate targetCoord;
FixedPointCoordinate source_coordinate;
FixedPointCoordinate target_coordinate;
};
struct CmpEdgeByStartID {
struct CmpEdgeByStartID
{
typedef InternalExtractorEdge value_type;
bool operator ()(
const InternalExtractorEdge & a,
const InternalExtractorEdge & b
) const {
bool operator()(const InternalExtractorEdge &a, const InternalExtractorEdge &b) const
{
return a.start < b.start;
}
value_type max_value() {
return InternalExtractorEdge::max_value();
}
value_type max_value() { return InternalExtractorEdge::max_value(); }
value_type min_value() {
return InternalExtractorEdge::min_value();
}
value_type min_value() { return InternalExtractorEdge::min_value(); }
};
struct CmpEdgeByTargetID {
struct CmpEdgeByTargetID
{
typedef InternalExtractorEdge value_type;
bool operator ()(
const InternalExtractorEdge & a,
const InternalExtractorEdge & b
) const {
bool operator()(const InternalExtractorEdge &a, const InternalExtractorEdge &b) const
{
return a.target < b.target;
}
value_type max_value() {
return InternalExtractorEdge::max_value();
}
value_type max_value() { return InternalExtractorEdge::max_value(); }
value_type min_value() {
return InternalExtractorEdge::min_value();
}
value_type min_value() { return InternalExtractorEdge::min_value(); }
};
#endif //INTERNAL_EXTRACTOR_EDGE_H
#endif // INTERNAL_EXTRACTOR_EDGE_H

953
Extractor/PBFParser.cpp
View File

File diff suppressed because it is too large Load Diff

64
Extractor/PBFParser.h
View File

@ -31,24 +31,24 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "BaseParser.h"
#include "../DataStructures/ConcurrentQueue.h"
#include <boost/shared_ptr.hpp>
#include <osmpbf/fileformat.pb.h>
#include <osmpbf/osmformat.pb.h>
#include <fstream>
#include <memory>
class PBFParser : public BaseParser {
class PBFParser : public BaseParser
{
enum EntityType {
TypeDummy = 0,
TypeNode = 1,
TypeWay = 2,
TypeRelation = 4,
TypeDenseNode = 8
};
enum EntityType
{ TypeDummy = 0,
TypeNode = 1,
TypeWay = 2,
TypeRelation = 4,
TypeDenseNode = 8 };
struct _ThreadData {
struct ParserThreadData
{
int currentGroupID;
int currentEntityID;
EntityType entityTypeIndicator;
@ -62,42 +62,38 @@ class PBFParser : public BaseParser {
std::vector<char> charBuffer;
};
public:
PBFParser(
const char * fileName,
ExtractorCallbacks* ec,
ScriptingEnvironment& se
);
public:
PBFParser(const char *file_name, ExtractorCallbacks *extractor_callbacks, ScriptingEnvironment &scripting_environment);
virtual ~PBFParser();
inline bool ReadHeader();
inline bool Parse();
inline bool Parse();
private:
private:
inline void ReadData();
inline void ParseData();
inline void parseDenseNode (_ThreadData * threadData);
inline void parseNode (_ThreadData * threadData);
inline void parseRelation (_ThreadData * threadData);
inline void parseWay (_ThreadData * threadData);
inline void parseDenseNode(ParserThreadData *thread_data);
inline void parseNode(ParserThreadData *thread_data);
inline void parseRelation(ParserThreadData *thread_data);
inline void parseWay(ParserThreadData *thread_data);
inline void loadGroup (_ThreadData * threadData);
inline void loadBlock (_ThreadData * threadData);
inline bool readPBFBlobHeader(std::fstream & stream, _ThreadData * threadData);
inline bool unpackZLIB (std::fstream & stream, _ThreadData * threadData);
inline bool unpackLZMA (std::fstream & stream, _ThreadData * threadData);
inline bool readBlob (std::fstream & stream, _ThreadData * threadData);
inline bool readNextBlock (std::fstream & stream, _ThreadData * threadData);
inline void loadGroup(ParserThreadData *thread_data);
inline void loadBlock(ParserThreadData *thread_data);
inline bool readPBFBlobHeader(std::fstream &stream, ParserThreadData *thread_data);
inline bool unpackZLIB(std::fstream &stream, ParserThreadData *thread_data);
inline bool unpackLZMA(std::fstream &stream, ParserThreadData *thread_data);
inline bool readBlob(std::fstream &stream, ParserThreadData *thread_data);
inline bool readNextBlock(std::fstream &stream, ParserThreadData *thread_data);
static const int NANO = 1000 * 1000 * 1000;
static const int MAX_BLOB_HEADER_SIZE = 64 * 1024;
static const int MAX_BLOB_SIZE = 32 * 1024 * 1024;
unsigned groupCount;
unsigned blockCount;
unsigned group_count;
unsigned block_count;
std::fstream input; // the input stream to parse
boost::shared_ptr<ConcurrentQueue < _ThreadData* > > threadDataQueue;
std::fstream input; // the input stream to parse
std::shared_ptr<ConcurrentQueue<ParserThreadData *>> thread_data_queue;
};
#endif /* PBFPARSER_H_ */

118
Extractor/ScriptingEnvironment.cpp
View File

@ -37,90 +37,88 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "../typedefs.h"
ScriptingEnvironment::ScriptingEnvironment() {}
ScriptingEnvironment::ScriptingEnvironment(const char * fileName) {
SimpleLogger().Write() << "Using script " << fileName;
ScriptingEnvironment::ScriptingEnvironment(const char *file_name)
{
SimpleLogger().Write() << "Using script " << file_name;
// Create a new lua state
for(int i = 0; i < omp_get_max_threads(); ++i) {
luaStateVector.push_back(luaL_newstate());
for (int i = 0; i < omp_get_max_threads(); ++i)
{
lua_state_vector.push_back(luaL_newstate());
}
// Connect LuaBind to this lua state for all threads
// Connect LuaBind to this lua state for all threads
#pragma omp parallel
{
lua_State * myLuaState = getLuaStateForThreadID(omp_get_thread_num());
luabind::open(myLuaState);
//open utility libraries string library;
luaL_openlibs(myLuaState);
lua_State *lua_state = getLuaStateForThreadID(omp_get_thread_num());
luabind::open(lua_state);
// open utility libraries string library;
luaL_openlibs(lua_state);
luaAddScriptFolderToLoadPath( myLuaState, fileName );
luaAddScriptFolderToLoadPath(lua_state, file_name);
// Add our function to the state's global scope
luabind::module(myLuaState) [
luabind::module(lua_state)[
luabind::def("print", LUA_print<std::string>),
luabind::def("durationIsValid", durationIsValid),
luabind::def("parseDuration", parseDuration)
];
luabind::module(myLuaState) [
luabind::class_<HashTable<std::string, std::string> >("keyVals")
.def("Add", &HashTable<std::string, std::string>::Add)
.def("Find", &HashTable<std::string, std::string>::Find)
.def("Holds", &HashTable<std::string, std::string>::Holds)
];
luabind::module(lua_state)[luabind::class_<HashTable<std::string, std::string>>("keyVals")
.def("Add", &HashTable<std::string, std::string>::Add)
.def("Find", &HashTable<std::string, std::string>::Find)
.def("Holds", &HashTable<std::string, std::string>::Holds)];
luabind::module(myLuaState) [
luabind::class_<ImportNode>("Node")
.def(luabind::constructor<>())
.def_readwrite("lat", &ImportNode::lat)
.def_readwrite("lon", &ImportNode::lon)
.def_readonly("id", &ImportNode::id)
.def_readwrite("bollard", &ImportNode::bollard)
.def_readwrite("traffic_light", &ImportNode::trafficLight)
.def_readwrite("tags", &ImportNode::keyVals)
];
luabind::module(lua_state)[luabind::class_<ImportNode>("Node")
.def(luabind::constructor<>())
.def_readwrite("lat", &ImportNode::lat)
.def_readwrite("lon", &ImportNode::lon)
.def_readonly("id", &ImportNode::id)
.def_readwrite("bollard", &ImportNode::bollard)
.def_readwrite("traffic_light", &ImportNode::trafficLight)
.def_readwrite("tags", &ImportNode::keyVals)];
luabind::module(myLuaState) [
luabind::class_<ExtractionWay>("Way")
.def(luabind::constructor<>())
.def_readonly("id", &ExtractionWay::id)
.def_readwrite("name", &ExtractionWay::name)
.def_readwrite("speed", &ExtractionWay::speed)
.def_readwrite("backward_speed", &ExtractionWay::backward_speed)
.def_readwrite("duration", &ExtractionWay::duration)
.def_readwrite("type", &ExtractionWay::type)
.def_readwrite("access", &ExtractionWay::access)
.def_readwrite("roundabout", &ExtractionWay::roundabout)
.def_readwrite("is_access_restricted", &ExtractionWay::isAccessRestricted)
.def_readwrite("ignore_in_grid", &ExtractionWay::ignoreInGrid)
.def_readwrite("tags", &ExtractionWay::keyVals)
.def_readwrite("direction", &ExtractionWay::direction)
.enum_("constants") [
luabind::value("notSure", 0),
luabind::value("oneway", 1),
luabind::value("bidirectional", 2),
luabind::value("opposite", 3)
]
];
luabind::module(lua_state)
[luabind::class_<ExtractionWay>("Way")
.def(luabind::constructor<>())
.def_readonly("id", &ExtractionWay::id)
.def_readwrite("name", &ExtractionWay::name)
.def_readwrite("speed", &ExtractionWay::speed)
.def_readwrite("backward_speed", &ExtractionWay::backward_speed)
.def_readwrite("duration", &ExtractionWay::duration)
.def_readwrite("type", &ExtractionWay::type)
.def_readwrite("access", &ExtractionWay::access)
.def_readwrite("roundabout", &ExtractionWay::roundabout)
.def_readwrite("is_access_restricted", &ExtractionWay::isAccessRestricted)
.def_readwrite("ignore_in_grid", &ExtractionWay::ignoreInGrid)
.def_readwrite("tags", &ExtractionWay::keyVals)
.def_readwrite("direction", &ExtractionWay::direction)
.enum_("constants")[
luabind::value("notSure", 0),
luabind::value("oneway", 1),
luabind::value("bidirectional", 2),
luabind::value("opposite", 3)
]];
// fails on c++11/OS X 10.9
luabind::module(myLuaState) [
luabind::class_<std::vector<std::string> >("vector")
.def("Add", static_cast<void (std::vector<std::string>::*)(const std::string&)>(&std::vector<std::string>::push_back))
];
luabind::module(lua_state)[luabind::class_<std::vector<std::string>>("vector").def(
"Add",
static_cast<void (std::vector<std::string>::*)(const std::string &)>(
&std::vector<std::string>::push_back))];
if(0 != luaL_dofile(myLuaState, fileName) ) {
if (0 != luaL_dofile(lua_state, file_name))
{
throw OSRMException("ERROR occured in scripting block");
}
}
}
ScriptingEnvironment::~ScriptingEnvironment() {
for(unsigned i = 0; i < luaStateVector.size(); ++i) {
// luaStateVector[i];
ScriptingEnvironment::~ScriptingEnvironment()
{
for (unsigned i = 0; i < lua_state_vector.size(); ++i)
{
// lua_state_vector[i];
}
}
lua_State * ScriptingEnvironment::getLuaStateForThreadID(const int id) {
return luaStateVector[id];
}
lua_State *ScriptingEnvironment::getLuaStateForThreadID(const int id) { return lua_state_vector[id]; }

11
Extractor/ScriptingEnvironment.h
View File

@ -32,15 +32,16 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
struct lua_State;
class ScriptingEnvironment {
public:
class ScriptingEnvironment
{
public:
ScriptingEnvironment();
explicit ScriptingEnvironment(const char * fileName);
explicit ScriptingEnvironment(const char *file_name);
virtual ~ScriptingEnvironment();
lua_State * getLuaStateForThreadID(const int);
lua_State *getLuaStateForThreadID(const int);
std::vector<lua_State *> luaStateVector;
std::vector<lua_State *> lua_state_vector;
};
#endif /* SCRIPTINGENVIRONMENT_H_ */

9
Extractor/XMLParser.cpp
View File

@ -71,19 +71,19 @@ bool XMLParser::Parse()
{
ImportNode n = ReadXMLNode();
ParseNodeInLua(n, lua_state);
extractor_callbacks->nodeFunction(n);
extractor_callbacks->ProcessNode(n);
}
if (xmlStrEqual(currentName, (const xmlChar *)"way") == 1)
{
ExtractionWay way = ReadXMLWay();
ParseWayInLua(way, lua_state);
extractor_callbacks->wayFunction(way);
extractor_callbacks->ProcessWay(way);
}
if (use_turn_restrictions && xmlStrEqual(currentName, (const xmlChar *)"relation") == 1)
{
InputRestrictionContainer r = ReadXMLRestriction();
if ((UINT_MAX != r.fromWay) && !extractor_callbacks->restrictionFunction(r))
if ((UINT_MAX != r.fromWay) && !extractor_callbacks->ProcessRestriction(r))
{
std::cerr << "[XMLParser] restriction not parsed" << std::endl;
}
@ -328,8 +328,7 @@ ImportNode XMLParser::ReadXMLNode()
xmlChar *value = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"v");
if (k != NULL && value != NULL)
{
node.keyVals.emplace(std::string((char *)(k)),
std::string((char *)(value)));
node.keyVals.emplace(std::string((char *)(k)), std::string((char *)(value)));
}
if (k != NULL)
{

8
extractor.cpp
View File

@ -221,7 +221,7 @@ int main(int argc, char *argv[])
ExtractionContainers extraction_containers;
string_map[""] = 0;
extractor_callbacks = new ExtractorCallbacks(&extraction_containers, &string_map);
extractor_callbacks = new ExtractorCallbacks(extraction_containers, string_map);
BaseParser *parser;
if (file_has_pbf_format)
{
@ -241,6 +241,9 @@ int main(int argc, char *argv[])
std::chrono::steady_clock::now();
parser->Parse();
delete parser;
delete extractor_callbacks;
std::chrono::duration<double> parsing_duration =
std::chrono::steady_clock::now() - parsing_start_time;
SimpleLogger().Write() << "Parsing finished after " << parsing_duration.count()
@ -254,9 +257,6 @@ int main(int argc, char *argv[])
extraction_containers.PrepareData(output_file_name, restriction_fileName);
delete parser;
delete extractor_callbacks;
std::chrono::duration<double> extraction_duration =
std::chrono::steady_clock::now() - startup_time;
SimpleLogger().Write() << "extraction finished after " << extraction_duration.count()