Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

streamline PBF parsing code

Browse Source
This commit is contained in:
Dennis Luxen 2014-06-24 12:25:19 +02:00
parent efbda436f3
commit dd7d6df4c6
2 changed files with 82 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

153
Extractor/PBFParser.cpp
View File

@ -39,6 +39,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "../Util/SimpleLogger.h" #include "../Util/SimpleLogger.h"
#include "../typedefs.h" #include "../typedefs.h"
#include <boost/assert.hpp>
#include <tbb/parallel_for.h> #include <tbb/parallel_for.h>
#include <tbb/task_scheduler_init.h> #include <tbb/task_scheduler_init.h>
@ -113,7 +115,7 @@ inline bool PBFParser::ReadHeader()
if (readBlob(input, &init_data)) if (readBlob(input, &init_data))
{ {
if (!init_data.PBFHeaderBlock.ParseFromArray(&(init_data.charBuffer[0]), if (!init_data.PBFHeaderBlock.ParseFromArray(&(init_data.charBuffer[0]),
init_data.charBuffer.size())) static_cast<int>(init_data.charBuffer.size())))
{ {
std::cerr << "[error] Header not parseable!" << std::endl; std::cerr << "[error] Header not parseable!" << std::endl;
return false; return false;
@ -245,17 +247,17 @@ inline void PBFParser::parseDenseNode(ParserThreadData *thread_data)
m_lastDenseID += dense.id(i); m_lastDenseID += dense.id(i);
m_lastDenseLatitude += dense.lat(i); m_lastDenseLatitude += dense.lat(i);
m_lastDenseLongitude += dense.lon(i); m_lastDenseLongitude += dense.lon(i);
extracted_nodes_vector[i].node_id = m_lastDenseID; extracted_nodes_vector[i].node_id = static_cast<NodeID>(m_lastDenseID);
extracted_nodes_vector[i].lat = extracted_nodes_vector[i].lat = static_cast<int>(
COORDINATE_PRECISION * COORDINATE_PRECISION *
((double)m_lastDenseLatitude * thread_data->PBFprimitiveBlock.granularity() + ((double)m_lastDenseLatitude * thread_data->PBFprimitiveBlock.granularity() +
thread_data->PBFprimitiveBlock.lat_offset()) / thread_data->PBFprimitiveBlock.lat_offset()) /
NANO; NANO);
extracted_nodes_vector[i].lon = extracted_nodes_vector[i].lon = static_cast<int>(
COORDINATE_PRECISION * COORDINATE_PRECISION *
((double)m_lastDenseLongitude * thread_data->PBFprimitiveBlock.granularity() + ((double)m_lastDenseLongitude * thread_data->PBFprimitiveBlock.granularity() +
thread_data->PBFprimitiveBlock.lon_offset()) / thread_data->PBFprimitiveBlock.lon_offset()) /
NANO; NANO);
while (denseTagIndex < dense.keys_vals_size()) while (denseTagIndex < dense.keys_vals_size())
{ {
const int tagValue = dense.keys_vals(denseTagIndex); const int tagValue = dense.keys_vals(denseTagIndex);
@ -273,16 +275,15 @@ inline void PBFParser::parseDenseNode(ParserThreadData *thread_data)
} }
tbb::parallel_for(tbb::blocked_range<size_t>(0, extracted_nodes_vector.size()), tbb::parallel_for(tbb::blocked_range<size_t>(0, extracted_nodes_vector.size()),
[this, &extracted_nodes_vector](const tbb::blocked_range<size_t>& range) [this, &extracted_nodes_vector](const tbb::blocked_range<size_t> &range)
{
lua_State *lua_state = this->scripting_environment.getLuaState();
for (size_t i = range.begin(); i != range.end(); ++i)
{ {
lua_State* lua_state = this->scripting_environment.getLuaState(); ImportNode &import_node = extracted_nodes_vector[i];
for (size_t i = range.begin(); i != range.end(); ++i) ParseNodeInLua(import_node, lua_state);
{
ImportNode &import_node = extracted_nodes_vector[i];
ParseNodeInLua(import_node, lua_state);
}
} }
); });
for (const ImportNode &import_node : extracted_nodes_vector) for (const ImportNode &import_node : extracted_nodes_vector)
{ {
@ -347,7 +348,7 @@ inline void PBFParser::parseRelation(ParserThreadData *thread_data)
if (is_restriction) if (is_restriction)
{ {
int64_t lastRef = 0; int64_t last_ref = 0;
InputRestrictionContainer current_restriction_container(is_only_restriction); InputRestrictionContainer current_restriction_container(is_only_restriction);
for (int rolesIndex = 0, last_role = inputRelation.roles_sid_size(); for (int rolesIndex = 0, last_role = inputRelation.roles_sid_size();
rolesIndex < last_role; rolesIndex < last_role;
@ -355,7 +356,7 @@ inline void PBFParser::parseRelation(ParserThreadData *thread_data)
{ {
const std::string &role = thread_data->PBFprimitiveBlock.stringtable().s( const std::string &role = thread_data->PBFprimitiveBlock.stringtable().s(
inputRelation.roles_sid(rolesIndex)); inputRelation.roles_sid(rolesIndex));
lastRef += inputRelation.memids(rolesIndex); last_ref += inputRelation.memids(rolesIndex);
if (!("from" == role || "to" == role || "via" == role)) if (!("from" == role || "to" == role || "via" == role))
{ {
@ -369,37 +370,42 @@ inline void PBFParser::parseRelation(ParserThreadData *thread_data)
{ // Only via should be a node { // Only via should be a node
continue; continue;
} }
assert("via" == role); BOOST_ASSERT("via" == role);
if (std::numeric_limits<unsigned>::max() != current_restriction_container.viaNode) if (std::numeric_limits<unsigned>::max() !=
current_restriction_container.viaNode)
{ {
current_restriction_container.viaNode = std::numeric_limits<unsigned>::max(); current_restriction_container.viaNode =
std::numeric_limits<unsigned>::max();
} }
assert(std::numeric_limits<unsigned>::max() == current_restriction_container.viaNode); BOOST_ASSERT(std::numeric_limits<unsigned>::max() ==
current_restriction_container.restriction.viaNode = lastRef; current_restriction_container.viaNode);
current_restriction_container.restriction.viaNode =
static_cast<NodeID>(last_ref);
break; break;
case 1: // way case 1: // way
assert("from" == role || "to" == role || "via" == role); BOOST_ASSERT("from" == role || "to" == role || "via" == role);
if ("from" == role) if ("from" == role)
{ {
current_restriction_container.fromWay = lastRef; current_restriction_container.fromWay = static_cast<EdgeID>(last_ref);
} }
if ("to" == role) if ("to" == role)
{ {
current_restriction_container.toWay = lastRef; current_restriction_container.toWay = static_cast<EdgeID>(last_ref);
} }
if ("via" == role) if ("via" == role)
{ {
assert(current_restriction_container.restriction.toNode == std::numeric_limits<unsigned>::max()); BOOST_ASSERT(current_restriction_container.restriction.toNode ==
current_restriction_container.viaNode = lastRef; std::numeric_limits<unsigned>::max());
current_restriction_container.viaNode = static_cast<NodeID>(last_ref);
} }
break; break;
case 2: // relation, not used. relations relating to relations are evil. case 2: // relation, not used. relations relating to relations are evil.
continue; continue;
assert(false); BOOST_ASSERT(false);
break; break;
default: // should not happen default: // should not happen
assert(false); BOOST_ASSERT(false);
break; break;
} }
} }
@ -420,17 +426,17 @@ inline void PBFParser::parseWay(ParserThreadData *thread_data)
{ {
const OSMPBF::Way &input_way = const OSMPBF::Way &input_way =
thread_data->PBFprimitiveBlock.primitivegroup(thread_data->currentGroupID).ways(i); thread_data->PBFprimitiveBlock.primitivegroup(thread_data->currentGroupID).ways(i);
parsed_way_vector[i].id = input_way.id(); parsed_way_vector[i].id = static_cast<EdgeID>(input_way.id());
unsigned node_id_in_path = 0; unsigned node_id_in_path = 0;
const int number_of_referenced_nodes = input_way.refs_size(); const auto number_of_referenced_nodes = input_way.refs_size();
for (int j = 0; j < number_of_referenced_nodes; ++j) for (auto j = 0; j < number_of_referenced_nodes; ++j)
{ {
node_id_in_path += input_way.refs(j); node_id_in_path += static_cast<NodeID>(input_way.refs(j));
parsed_way_vector[i].path.push_back(node_id_in_path); parsed_way_vector[i].path.push_back(node_id_in_path);
} }
assert(input_way.keys_size() == input_way.vals_size()); BOOST_ASSERT(input_way.keys_size() == input_way.vals_size());
const int number_of_keys = input_way.keys_size(); const auto number_of_keys = input_way.keys_size();
for (int j = 0; j < number_of_keys; ++j) for (auto j = 0; j < number_of_keys; ++j)
{ {
const std::string &key = const std::string &key =
thread_data->PBFprimitiveBlock.stringtable().s(input_way.keys(j)); thread_data->PBFprimitiveBlock.stringtable().s(input_way.keys(j));
@ -442,19 +448,18 @@ inline void PBFParser::parseWay(ParserThreadData *thread_data)
// TODO: investigate if schedule guided will be handled by tbb automatically // TODO: investigate if schedule guided will be handled by tbb automatically
tbb::parallel_for(tbb::blocked_range<size_t>(0, parsed_way_vector.size()), tbb::parallel_for(tbb::blocked_range<size_t>(0, parsed_way_vector.size()),
[this, &parsed_way_vector](const tbb::blocked_range<size_t>& range) [this, &parsed_way_vector](const tbb::blocked_range<size_t> &range)
{
lua_State *lua_state = this->scripting_environment.getLuaState();
for (size_t i = range.begin(); i != range.end(); i++)
{ {
lua_State* lua_state = this->scripting_environment.getLuaState(); ExtractionWay &extraction_way = parsed_way_vector[i];
for (size_t i = range.begin(); i != range.end(); i++) if (2 <= extraction_way.path.size())
{ {
ExtractionWay &extraction_way = parsed_way_vector[i]; ParseWayInLua(extraction_way, lua_state);
if (2 <= extraction_way.path.size())
{
ParseWayInLua(extraction_way, lua_state);
}
} }
} }
); });
for (ExtractionWay &extraction_way : parsed_way_vector) for (ExtractionWay &extraction_way : parsed_way_vector)
{ {
@ -489,9 +494,9 @@ inline void PBFParser::loadGroup(ParserThreadData *thread_data)
if (group.has_dense()) if (group.has_dense())
{ {
thread_data->entityTypeIndicator = TypeDenseNode; thread_data->entityTypeIndicator = TypeDenseNode;
assert(0 != group.dense().id_size()); BOOST_ASSERT(0 != group.dense().id_size());
} }
assert(thread_data->entityTypeIndicator != TypeDummy); BOOST_ASSERT(thread_data->entityTypeIndicator != TypeDummy);
} }
inline void PBFParser::loadBlock(ParserThreadData *thread_data) inline void PBFParser::loadBlock(ParserThreadData *thread_data)
@ -522,51 +527,51 @@ inline bool PBFParser::readPBFBlobHeader(std::fstream &stream, ParserThreadData
return dataSuccessfullyParsed; return dataSuccessfullyParsed;
} }
inline bool PBFParser::unpackZLIB(std::fstream &, ParserThreadData *thread_data) inline bool PBFParser::unpackZLIB(ParserThreadData *thread_data)
{ {
unsigned rawSize = thread_data->PBFBlob.raw_size(); auto raw_size = thread_data->PBFBlob.raw_size();
char *unpackedDataArray = new char[rawSize]; char *unpacked_data_array = new char[raw_size];
z_stream compressedDataStream; z_stream compressed_data_stream;
compressedDataStream.next_in = (unsigned char *)thread_data->PBFBlob.zlib_data().data(); compressed_data_stream.next_in = (unsigned char *)thread_data->PBFBlob.zlib_data().data();
compressedDataStream.avail_in = thread_data->PBFBlob.zlib_data().size(); compressed_data_stream.avail_in = thread_data->PBFBlob.zlib_data().size();
compressedDataStream.next_out = (unsigned char *)unpackedDataArray; compressed_data_stream.next_out = (unsigned char *)unpacked_data_array;
compressedDataStream.avail_out = rawSize; compressed_data_stream.avail_out = raw_size;
compressedDataStream.zalloc = Z_NULL; compressed_data_stream.zalloc = Z_NULL;
compressedDataStream.zfree = Z_NULL; compressed_data_stream.zfree = Z_NULL;
compressedDataStream.opaque = Z_NULL; compressed_data_stream.opaque = Z_NULL;
int ret = inflateInit(&compressedDataStream); int return_code = inflateInit(&compressed_data_stream);
if (ret != Z_OK) if (return_code != Z_OK)
{ {
std::cerr << "[error] failed to init zlib stream" << std::endl; std::cerr << "[error] failed to init zlib stream" << std::endl;
delete[] unpackedDataArray; delete[] unpacked_data_array;
return false; return false;
} }
ret = inflate(&compressedDataStream, Z_FINISH); return_code = inflate(&compressed_data_stream, Z_FINISH);
if (ret != Z_STREAM_END) if (return_code != Z_STREAM_END)
{ {
std::cerr << "[error] failed to inflate zlib stream" << std::endl; std::cerr << "[error] failed to inflate zlib stream" << std::endl;
std::cerr << "[error] Error type: " << ret << std::endl; std::cerr << "[error] Error type: " << return_code << std::endl;
delete[] unpackedDataArray; delete[] unpacked_data_array;
return false; return false;
} }
ret = inflateEnd(&compressedDataStream); return_code = inflateEnd(&compressed_data_stream);
if (ret != Z_OK) if (return_code != Z_OK)
{ {
std::cerr << "[error] failed to deinit zlib stream" << std::endl; std::cerr << "[error] failed to deinit zlib stream" << std::endl;
delete[] unpackedDataArray; delete[] unpacked_data_array;
return false; return false;
} }
thread_data->charBuffer.clear(); thread_data->charBuffer.clear();
thread_data->charBuffer.resize(rawSize); thread_data->charBuffer.resize(raw_size);
std::copy(unpackedDataArray, unpackedDataArray + rawSize, thread_data->charBuffer.begin()); std::copy(unpacked_data_array, unpacked_data_array + raw_size, thread_data->charBuffer.begin());
delete[] unpackedDataArray; delete[] unpacked_data_array;
return true; return true;
} }
inline bool PBFParser::unpackLZMA(std::fstream &, ParserThreadData *) { return false; } inline bool PBFParser::unpackLZMA(ParserThreadData *) { return false; }
inline bool PBFParser::readBlob(std::fstream &stream, ParserThreadData *thread_data) inline bool PBFParser::readBlob(std::fstream &stream, ParserThreadData *thread_data)
{ {
@ -601,7 +606,7 @@ inline bool PBFParser::readBlob(std::fstream &stream, ParserThreadData *thread_d
} }
else if (thread_data->PBFBlob.has_zlib_data()) else if (thread_data->PBFBlob.has_zlib_data())
{ {
if (!unpackZLIB(stream, thread_data)) if (!unpackZLIB(thread_data))
{ {
std::cerr << "[error] zlib data encountered that could not be unpacked" << std::endl; std::cerr << "[error] zlib data encountered that could not be unpacked" << std::endl;
delete[] data; delete[] data;
@ -610,7 +615,7 @@ inline bool PBFParser::readBlob(std::fstream &stream, ParserThreadData *thread_d
} }
else if (thread_data->PBFBlob.has_lzma_data()) else if (thread_data->PBFBlob.has_lzma_data())
{ {
if (!unpackLZMA(stream, thread_data)) if (!unpackLZMA(thread_data))
{ {
std::cerr << "[error] lzma data encountered that could not be unpacked" << std::endl; std::cerr << "[error] lzma data encountered that could not be unpacked" << std::endl;
} }
@ -650,7 +655,7 @@ bool PBFParser::readNextBlock(std::fstream &stream, ParserThreadData *thread_dat
} }
if (!thread_data->PBFprimitiveBlock.ParseFromArray(&(thread_data->charBuffer[0]), if (!thread_data->PBFprimitiveBlock.ParseFromArray(&(thread_data->charBuffer[0]),
thread_data->charBuffer.size())) thread_data->charBuffer.size()))
{ {
std::cerr << "failed to parse PrimitiveBlock" << std::endl; std::cerr << "failed to parse PrimitiveBlock" << std::endl;
return false; return false;

6
Extractor/PBFParser.h
View File

@ -66,7 +66,7 @@ class PBFParser : public BaseParser
PBFParser(const char *file_name, PBFParser(const char *file_name,
ExtractorCallbacks *extractor_callbacks, ExtractorCallbacks *extractor_callbacks,
ScriptingEnvironment &scripting_environment, ScriptingEnvironment &scripting_environment,
unsigned num_parser_threads=0); unsigned num_parser_threads = 0);
virtual ~PBFParser(); virtual ~PBFParser();
inline bool ReadHeader(); inline bool ReadHeader();
@ -83,8 +83,8 @@ class PBFParser : public BaseParser
inline void loadGroup(ParserThreadData *thread_data); inline void loadGroup(ParserThreadData *thread_data);
inline void loadBlock(ParserThreadData *thread_data); inline void loadBlock(ParserThreadData *thread_data);
inline bool readPBFBlobHeader(std::fstream &stream, ParserThreadData *thread_data); inline bool readPBFBlobHeader(std::fstream &stream, ParserThreadData *thread_data);
inline bool unpackZLIB(std::fstream &stream, ParserThreadData *thread_data); inline bool unpackZLIB(ParserThreadData *thread_data);
inline bool unpackLZMA(std::fstream &stream, ParserThreadData *thread_data); inline bool unpackLZMA(ParserThreadData *thread_data);
inline bool readBlob(std::fstream &stream, ParserThreadData *thread_data); inline bool readBlob(std::fstream &stream, ParserThreadData *thread_data);
inline bool readNextBlock(std::fstream &stream, ParserThreadData *thread_data); inline bool readNextBlock(std::fstream &stream, ParserThreadData *thread_data);