Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Pass relation data to way and node functions

Browse Source
This commit is contained in:
Denis Koronchik 2017-08-30 19:35:38 +03:00 committed by Michael Krasnyk
parent f2b63ba0aa
commit f79bcc6b8d
7 changed files with 94 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
include/extractor/extraction_relation.hpp
View File

@ -5,6 +5,7 @@
#include <string>
#include <unordered_map>
#include <vector>
namespace osrm
{
@ -45,15 +46,39 @@ struct ExtractionRelation
return values[id.Hash()];
}
// AttributesMap & operator[] (util::OsmIDTyped id)
// {
// return values[id];
// }
bool is_restriction;
std::unordered_map<util::OsmIDTyped::HashType, AttributesMap> values;
};
// It contains data of all parsed relations for each node/way element
class ExtractionRelationContainer
{
public:
using AttributesMap = ExtractionRelation::AttributesMap;
using RelationList = std::vector<AttributesMap>;
void AddRelation(const ExtractionRelation & rel)
{
BOOST_ASSERT(!rel.is_restriction);
for (auto it : rel.values)
data[it.first].push_back(it.second);
}
const RelationList & Get(const util::OsmIDTyped & id) const
{
const auto it = data.find(id.Hash());
if (it != data.end())
return it->second;
static RelationList empty;
return empty;
}
private:
// TODO: need to store more common data
std::unordered_map<util::OsmIDTyped::HashType, RelationList> data;
};
} // namespace extractor
} // namespace osrm

3
include/extractor/extractor_callbacks.hpp
View File

@ -15,6 +15,7 @@ namespace osmium
{
class Node;
class Way;
class Relation;
}
namespace std
@ -44,6 +45,7 @@ namespace extractor
class ExtractionContainers;
struct ExtractionNode;
struct ExtractionWay;
struct ExtractionRelation;
struct ProfileProperties;
struct InputConditionalTurnRestriction;
@ -88,6 +90,7 @@ class ExtractorCallbacks
// warning: caller needs to take care of synchronization!
void ProcessWay(const osmium::Way &current_way, const ExtractionWay &result_way);
};
}
}

2
include/extractor/scripting_environment.hpp
View File

@ -34,6 +34,7 @@ namespace extractor
{
class RestrictionParser;
class ExtractionRelationContainer;
struct ExtractionNode;
struct ExtractionWay;
struct ExtractionRelation;
@ -64,6 +65,7 @@ class ScriptingEnvironment
virtual void
ProcessElements(const osmium::memory::Buffer &buffer,
const RestrictionParser &restriction_parser,
const ExtractionRelationContainer &relations,
std::vector<std::pair<const osmium::Node &, ExtractionNode>> &resulting_nodes,
std::vector<std::pair<const osmium::Way &, ExtractionWay>> &resulting_ways,
std::vector<std::pair<const osmium::Relation &, ExtractionRelation>> &resulting_relations,

6
include/extractor/scripting_environment_lua.hpp
View File

@ -3,6 +3,7 @@
#include "extractor/raster_source.hpp"
#include "extractor/scripting_environment.hpp"
#include "extractor/extraction_relation.hpp"
#include <tbb/enumerable_thread_specific.h>
@ -19,8 +20,8 @@ namespace extractor
struct LuaScriptingContext final
{
void ProcessNode(const osmium::Node &, ExtractionNode &result);
void ProcessWay(const osmium::Way &, ExtractionWay &result);
void ProcessNode(const osmium::Node &, ExtractionNode &result, const ExtractionRelationContainer::RelationList &relations);
void ProcessWay(const osmium::Way &, ExtractionWay &result, const ExtractionRelationContainer::RelationList &relations);
void ProcessRelation(const osmium::Relation &, ExtractionRelation &result);
ProfileProperties properties;
@ -71,6 +72,7 @@ class Sol2ScriptingEnvironment final : public ScriptingEnvironment
void
ProcessElements(const osmium::memory::Buffer &buffer,
const RestrictionParser &restriction_parser,
const ExtractionRelationContainer &relations,
std::vector<std::pair<const osmium::Node &, ExtractionNode>> &resulting_nodes,
std::vector<std::pair<const osmium::Way &, ExtractionWay>> &resulting_ways,
std::vector<std::pair<const osmium::Relation &, ExtractionRelation>> &resulting_relations,

49
src/extractor/extractor.cpp
View File

@ -289,12 +289,12 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
const osmium::io::File input_file(config.input_path.string());
osmium::thread::Pool pool(number_of_threads);
osmium::io::Reader reader(
input_file,
pool,
(config.use_metadata ? osmium::io::read_meta::yes : osmium::io::read_meta::no));
const osmium::io::Header header = reader.header();
std::unique_ptr<osmium::io::Reader> reader(
new osmium::io::Reader(input_file, osmium::osm_entity_bits::relation,
(config.use_metadata ? osmium::io::read_meta::yes : osmium::io::read_meta::no)));
osmium::io::Header header = reader->header();
unsigned number_of_nodes = 0;
unsigned number_of_ways = 0;
@ -332,6 +332,7 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
timestamp_file.WriteFrom(timestamp.c_str(), timestamp.length());
ExtractionRelationContainer relations;
std::vector<std::string> restrictions = scripting_environment.GetRestrictions();
// setup restriction parser
const RestrictionParser restriction_parser(
@ -339,8 +340,6 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
config.parse_conditionals,
restrictions);
std::mutex process_mutex;
using SharedBuffer = std::shared_ptr<const osmium::memory::Buffer>;
struct ParsedBuffer
{
@ -353,7 +352,7 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
tbb::filter_t<void, SharedBuffer> buffer_reader(
tbb::filter::serial_in_order, [&](tbb::flow_control &fc) {
if (auto buffer = reader.read())
if (auto buffer = reader->read())
{
return std::make_shared<const osmium::memory::Buffer>(std::move(buffer));
}
@ -372,6 +371,7 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
parsed_buffer->buffer = buffer;
scripting_environment.ProcessElements(*buffer,
restriction_parser,
relations,
parsed_buffer->resulting_nodes,
parsed_buffer->resulting_ways,
parsed_buffer->resulting_relations,
@ -394,13 +394,44 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
{
extractor_callbacks->ProcessWay(result.first, result.second);
}
number_of_relations += parsed_buffer->resulting_restrictions.size();
});
tbb::filter_t<std::shared_ptr<ParsedBuffer>, void> buffer_storage_relation(
tbb::filter::serial_in_order, [&](const std::shared_ptr<ParsedBuffer> parsed_buffer) {
if (!parsed_buffer)
return;
number_of_relations += parsed_buffer->resulting_relations.size();
for (const auto &result : parsed_buffer->resulting_relations)
{
/// TODO: add restriction processing
if (result.second.is_restriction)
continue;
relations.AddRelation(result.second);
}
for (const auto &result : parsed_buffer->resulting_restrictions)
{
extractor_callbacks->ProcessRestriction(result);
}
});
/* Main trick that we can use the same pipeline. It just receive relation objects
* from osmium. So other containers would be empty and doesn't process anything
*/
util::Log() << "Parse relations ...";
tbb::parallel_pipeline(tbb::task_scheduler_init::default_num_threads() * 1.5,
buffer_reader & buffer_transform & buffer_storage_relation);
reader->close();
/* At this step we just filter ways and nodes from osmium, so any relation wouldn't be
* processed there.
*/
util::Log() << "Parse ways and nodes ...";
reader.reset(new osmium::io::Reader(input_file, osmium::osm_entity_bits::node | osmium::osm_entity_bits::way,
(config.use_metadata ? osmium::io::read_meta::yes : osmium::io::read_meta::no)));
// Number of pipeline tokens that yielded the best speedup was about 1.5 * num_cores
tbb::parallel_pipeline(tbb::task_scheduler_init::default_num_threads() * 1.5,
buffer_reader & buffer_transform & buffer_storage);

21
src/extractor/scripting_environment_lua.cpp
View File

@ -737,6 +737,7 @@ LuaScriptingContext &Sol2ScriptingEnvironment::GetSol2Context()
void Sol2ScriptingEnvironment::ProcessElements(
const osmium::memory::Buffer &buffer,
const RestrictionParser &restriction_parser,
const ExtractionRelationContainer &relations,
std::vector<std::pair<const osmium::Node &, ExtractionNode>> &resulting_nodes,
std::vector<std::pair<const osmium::Way &, ExtractionWay>> &resulting_ways,
std::vector<std::pair<const osmium::Relation &, ExtractionRelation>> &resulting_relations,
@ -757,7 +758,9 @@ void Sol2ScriptingEnvironment::ProcessElements(
(!static_cast<const osmium::Node &>(*entity).tags().empty() ||
local_context.properties.call_tagless_node_function))
{
local_context.ProcessNode(static_cast<const osmium::Node &>(*entity), result_node);
const osmium::Node & node = static_cast<const osmium::Node &>(*entity);
const util::OsmIDTyped id(node.id(), std::uint8_t(node.type()));
local_context.ProcessNode(node, result_node, relations.Get(id));
}
resulting_nodes.push_back(std::pair<const osmium::Node &, ExtractionNode>(
static_cast<const osmium::Node &>(*entity), std::move(result_node)));
@ -766,7 +769,9 @@ void Sol2ScriptingEnvironment::ProcessElements(
result_way.clear();
if (local_context.has_way_function)
{
local_context.ProcessWay(static_cast<const osmium::Way &>(*entity), result_way);
const osmium::Way & way = static_cast<const osmium::Way &>(*entity);
const util::OsmIDTyped id(way.id(), std::uint8_t(way.type()));
local_context.ProcessWay(way, result_way, relations.Get(id));
}
resulting_ways.push_back(std::pair<const osmium::Way &, ExtractionWay>(
static_cast<const osmium::Way &>(*entity), std::move(result_way)));
@ -1007,15 +1012,15 @@ void Sol2ScriptingEnvironment::ProcessSegment(ExtractionSegment &segment)
}
}
void LuaScriptingContext::ProcessNode(const osmium::Node &node, ExtractionNode &result)
void LuaScriptingContext::ProcessNode(const osmium::Node &node, ExtractionNode &result, const ExtractionRelationContainer::RelationList &relations)
{
BOOST_ASSERT(state.lua_state() != nullptr);
switch (api_version)
{
case 3:
// BOOST_ASSERT(false); // TODO: implement me
// break;
node_function(profile_table, node, result, relations);
break;
case 2:
node_function(profile_table, node, result);
break;
@ -1026,15 +1031,15 @@ void LuaScriptingContext::ProcessNode(const osmium::Node &node, ExtractionNode &
}
}
void LuaScriptingContext::ProcessWay(const osmium::Way &way, ExtractionWay &result)
void LuaScriptingContext::ProcessWay(const osmium::Way &way, ExtractionWay &result, const ExtractionRelationContainer::RelationList &relations)
{
BOOST_ASSERT(state.lua_state() != nullptr);
switch (api_version)
{
case 3:
// BOOST_ASSERT(false); // TODO: implement me
// break;
way_function(profile_table, way, result, relations);
break;
case 2:
way_function(profile_table, way, result);
break;

3
unit_tests/mocks/mock_scripting_environment.hpp
View File

@ -36,9 +36,10 @@ class MockScriptingEnvironment : public extractor::ScriptingEnvironment
void ProcessElements(const osmium::memory::Buffer &,
const extractor::RestrictionParser &,
const extractor::ExtractionRelationContainer &,
std::vector<std::pair<const osmium::Node &, extractor::ExtractionNode>> &,
std::vector<std::pair<const osmium::Way &, extractor::ExtractionWay>> &,
std::vector<std::pair<const osmium::Relation &, extractor::ExtractionRelation>> &resulting_relations,
std::vector<std::pair<const osmium::Relation &, extractor::ExtractionRelation>> &,
std::vector<extractor::InputConditionalTurnRestriction> &) override final
{
}