Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Enables the use of multiple segment-speed-files on the osrm-contract

Browse Source 
command line, and exposes the file name used for each edge in the debug
tiles.
This commit is contained in:
Daniel Patterson 2016-03-14 23:03:19 -07:00 committed by Patrick Niklaus
parent 80f008684d
commit 11b356e55f
15 changed files with 442 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/contractor/contractor.hpp
View File

@ -83,9 +83,11 @@ class Contractor
util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list,
const std::string &edge_segment_lookup_path,
const std::string &edge_penalty_path,
const std::string &segment_speed_path,
const std::vector<std::string> &segment_speed_path,
const std::string &nodes_filename,
const std::string &geometry_filename,
const std::string &datasource_names_filename,
const std::string &datasource_indexes_filename,
const std::string &rtree_leaf_filename);
};
}

10
include/contractor/contractor_config.hpp
View File

@ -53,6 +53,8 @@ struct ContractorConfig
node_based_graph_path = osrm_input_path.string() + ".nodes";
geometry_path = osrm_input_path.string() + ".geometry";
rtree_leaf_path = osrm_input_path.string() + ".fileIndex";
datasource_names_path = osrm_input_path.string() + ".datasource_names";
datasource_indexes_path = osrm_input_path.string() + ".datasource_indexes";
}
boost::filesystem::path config_file_path;
@ -78,11 +80,9 @@ struct ContractorConfig
//(e.g. 0.8 contracts 80 percent of the hierarchy, leaving a core of 20%)
double core_factor;
std::string segment_speed_lookup_path;
#ifdef DEBUG_GEOMETRY
std::string debug_geometry_path;
#endif
std::vector<std::string> segment_speed_lookup_paths;
std::string datasource_indexes_path;
std::string datasource_names_path;
};
}
}

8
include/engine/datafacade/datafacade_base.hpp
View File

@ -76,6 +76,14 @@ class BaseDataFacade
virtual void GetUncompressedWeights(const EdgeID id,
std::vector<EdgeWeight> &result_weights) const = 0;
// Returns the data source ids that were used to supply the edge
// weights. Will return an empty array when only the base profile is used.
virtual void GetUncompressedDatasources(const EdgeID id,
std::vector<uint8_t> &data_sources) const = 0;
// Gets the name of a datasource
virtual std::string GetDatasourceName(const uint8_t datasource_name_id) const = 0;
virtual extractor::guidance::TurnInstruction
GetTurnInstructionForEdgeID(const unsigned id) const = 0;

83
include/engine/datafacade/internal_datafacade.hpp
View File

@ -77,6 +77,8 @@ class InternalDataFacade final : public BaseDataFacade
util::ShM<extractor::CompressedEdgeContainer::CompressedEdge, false>::vector m_geometry_list;
util::ShM<bool, false>::vector m_is_core_node;
util::ShM<unsigned, false>::vector m_segment_weights;
util::ShM<uint8_t, false>::vector m_datasource_list;
util::ShM<std::string, false>::vector m_datasource_names;
boost::thread_specific_ptr<InternalRTree> m_static_rtree;
boost::thread_specific_ptr<InternalGeospatialQuery> m_geospatial_query;
@ -213,6 +215,34 @@ class InternalDataFacade final : public BaseDataFacade
}
}
void LoadDatasourceInfo(const boost::filesystem::path &datasource_names_file,
const boost::filesystem::path &datasource_indexes_file)
{
std::ifstream datasources_stream(datasource_indexes_file.c_str(), std::ios::binary);
if (datasources_stream)
{
std::size_t number_of_datasources = 0;
datasources_stream.read(reinterpret_cast<char *>(&number_of_datasources),
sizeof(std::size_t));
if (number_of_datasources > 0)
{
m_datasource_list.resize(number_of_datasources);
datasources_stream.read(reinterpret_cast<char *>(&(m_datasource_list[0])),
number_of_datasources * sizeof(uint8_t));
}
}
std::ifstream datasourcenames_stream(datasource_names_file.c_str(), std::ios::binary);
if (datasourcenames_stream)
{
std::string name;
while (std::getline(datasourcenames_stream, name))
{
m_datasource_names.push_back(name);
}
}
}
void LoadRTree()
{
BOOST_ASSERT_MSG(!m_coordinate_list->empty(), "coordinates must be loaded before r-tree");
@ -260,6 +290,14 @@ class InternalDataFacade final : public BaseDataFacade
return it->second;
};
const auto optional_file_for = [&server_paths, &end_it](const std::string &path)
{
const auto it = server_paths.find(path);
if (it == end_it)
throw util::exception("no valid " + path + " file given in ini file");
return it->second;
};
ram_index_path = file_for("ramindex");
file_index_path = file_for("fileindex");
@ -275,6 +313,10 @@ class InternalDataFacade final : public BaseDataFacade
util::SimpleLogger().Write() << "loading geometries";
LoadGeometries(file_for("geometries"));
util::SimpleLogger().Write() << "loading datasource info";
LoadDatasourceInfo(optional_file_for("datasource_names"),
optional_file_for("datasource_indexes"));
util::SimpleLogger().Write() << "loading timestamp";
LoadTimestamp(file_for("timestamp"));
@ -578,6 +620,47 @@ class InternalDataFacade final : public BaseDataFacade
});
}
// Returns the data source ids that were used to supply the edge
// weights.
virtual void
GetUncompressedDatasources(const EdgeID id,
std::vector<uint8_t> &result_datasources) const override final
{
const unsigned begin = m_geometry_indices.at(id);
const unsigned end = m_geometry_indices.at(id + 1);
result_datasources.clear();
result_datasources.reserve(end - begin);
// If there was no datasource info, return an array of 0's.
if (m_datasource_list.empty())
{
for (unsigned i = 0; i < end - begin; ++i)
{
result_datasources.push_back(0);
}
}
else
{
std::for_each(m_datasource_list.begin() + begin, m_datasource_list.begin() + end,
[&](const uint8_t &datasource_id)
{
result_datasources.push_back(datasource_id);
});
}
}
virtual std::string GetDatasourceName(const uint8_t datasource_name_id) const override final
{
if (m_datasource_names.empty() || datasource_name_id > m_datasource_names.size())
{
if (datasource_name_id == 0)
return "lua profile";
return "UNKNOWN";
}
return m_datasource_names[datasource_name_id];
}
std::string GetTimestamp() const override final { return m_timestamp; }
};
}

83
include/engine/datafacade/shared_datafacade.hpp
View File

@ -79,6 +79,11 @@ class SharedDataFacade final : public BaseDataFacade
util::ShM<unsigned, true>::vector m_geometry_indices;
util::ShM<extractor::CompressedEdgeContainer::CompressedEdge, true>::vector m_geometry_list;
util::ShM<bool, true>::vector m_is_core_node;
util::ShM<uint8_t, true>::vector m_datasource_list;
util::ShM<char, true>::vector m_datasource_name_data;
util::ShM<std::size_t, true>::vector m_datasource_name_offsets;
util::ShM<std::size_t, true>::vector m_datasource_name_lengths;
boost::thread_specific_ptr<std::pair<unsigned, std::shared_ptr<SharedRTree>>> m_static_rtree;
boost::thread_specific_ptr<SharedGeospatialQuery> m_geospatial_query;
@ -223,6 +228,34 @@ class SharedDataFacade final : public BaseDataFacade
geometry_list(geometries_list_ptr,
data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_LIST]);
m_geometry_list = std::move(geometry_list);
auto datasources_list_ptr = data_layout->GetBlockPtr<uint8_t>(
shared_memory, storage::SharedDataLayout::DATASOURCES_LIST);
typename util::ShM<uint8_t, true>::vector datasources_list(
datasources_list_ptr,
data_layout->num_entries[storage::SharedDataLayout::DATASOURCES_LIST]);
m_datasource_list = std::move(datasources_list);
auto datasource_name_data_ptr = data_layout->GetBlockPtr<char>(
shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_DATA);
typename util::ShM<char, true>::vector datasource_name_data(
datasource_name_data_ptr,
data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_DATA]);
m_datasource_name_data = std::move(datasource_name_data);
auto datasource_name_offsets_ptr = data_layout->GetBlockPtr<std::size_t>(
shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_OFFSETS);
typename util::ShM<std::size_t, true>::vector datasource_name_offsets(
datasource_name_offsets_ptr,
data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_OFFSETS]);
m_datasource_name_offsets = std::move(datasource_name_offsets);
auto datasource_name_lengths_ptr = data_layout->GetBlockPtr<std::size_t>(
shared_memory, storage::SharedDataLayout::DATASOURCE_NAME_LENGTHS);
typename util::ShM<std::size_t, true>::vector datasource_name_lengths(
datasource_name_lengths_ptr,
data_layout->num_entries[storage::SharedDataLayout::DATASOURCE_NAME_LENGTHS]);
m_datasource_name_lengths = std::move(datasource_name_lengths);
}
public:
@ -619,6 +652,56 @@ class SharedDataFacade final : public BaseDataFacade
virtual std::size_t GetCoreSize() const override final { return m_is_core_node.size(); }
// Returns the data source ids that were used to supply the edge
// weights.
virtual void
GetUncompressedDatasources(const EdgeID id,
std::vector<uint8_t> &result_datasources) const override final
{
const unsigned begin = m_geometry_indices.at(id);
const unsigned end = m_geometry_indices.at(id + 1);
result_datasources.clear();
result_datasources.reserve(end - begin);
// If there was no datasource info, return an array of 0's.
if (m_datasource_list.empty())
{
for (unsigned i = 0; i < end - begin; ++i)
{
result_datasources.push_back(0);
}
}
else
{
std::for_each(m_datasource_list.begin() + begin, m_datasource_list.begin() + end,
[&](const uint8_t &datasource_id)
{
result_datasources.push_back(datasource_id);
});
}
}
virtual std::string GetDatasourceName(const uint8_t datasource_name_id) const override final
{
std::string result;
if (m_datasource_name_offsets.empty() ||
datasource_name_id > m_datasource_name_offsets.size())
{
if (datasource_name_id == 0)
return "lua profile";
return "UNKNOWN";
}
std::copy(m_datasource_name_data.begin() + m_datasource_name_offsets[datasource_name_id],
m_datasource_name_data.begin() + m_datasource_name_offsets[datasource_name_id] +
m_datasource_name_lengths[datasource_name_id],
std::back_inserter(result));
return result;
}
std::string GetTimestamp() const override final { return m_timestamp; }
};
}

2
include/engine/engine_config.hpp
View File

@ -52,6 +52,8 @@ struct EngineConfig
{"coredata", base.string() + ".core"},
{"geometries", base.string() + ".geometry"},
{"timestamp", base.string() + ".timestamp"},
{"datasource_names", base.string() + ".datasource_names"},
{"datasource_indexes", base.string() + ".datasource_indexes"},
{"namesdata", base.string() + ".names"}}
{
}

9
include/extractor/edge_based_graph_factory.hpp
View File

@ -109,20 +109,11 @@ class EdgeBasedGraphFactory
void CompressGeometry();
unsigned RenumberEdges();
void GenerateEdgeExpandedNodes();
#ifdef DEBUG_GEOMETRY
void GenerateEdgeExpandedEdges(const std::string &original_edge_data_filename,
lua_State *lua_state,
const std::string &edge_segment_lookup_filename,
const std::string &edge_fixed_penalties_filename,
const bool generate_edge_lookup,
const std::string &debug_turns_path);
#else
void GenerateEdgeExpandedEdges(const std::string &original_edge_data_filename,
lua_State *lua_state,
const std::string &edge_segment_lookup_filename,
const std::string &edge_fixed_penalties_filename,
const bool generate_edge_lookup);
#endif
void InsertEdgeBasedNode(const NodeID u, const NodeID v);

4
include/storage/shared_datatype.hpp
View File

@ -37,6 +37,10 @@ struct SharedDataLayout
TIMESTAMP,
FILE_INDEX_PATH,
CORE_MARKER,
DATASOURCES_LIST,
DATASOURCE_NAME_DATA,
DATASOURCE_NAME_OFFSETS,
DATASOURCE_NAME_LENGTHS,
NUM_BLOCKS
};

4
include/util/routed_options.hpp
View File

@ -53,6 +53,10 @@ populate_base_path(std::unordered_map<std::string, boost::filesystem::path> &ser
BOOST_ASSERT(server_paths.find("namesdata") != server_paths.end());
server_paths["timestamp"] = base_string + ".timestamp";
BOOST_ASSERT(server_paths.find("timestamp") != server_paths.end());
server_paths["datasource_indexes"] = base_string + ".datasource_indexes";
BOOST_ASSERT(server_paths.find("timestamp") != server_paths.end());
server_paths["datasource_names"] = base_string + ".datasource_names";
BOOST_ASSERT(server_paths.find("timestamp") != server_paths.end());
}
// check if files are give and whether they exist at all

114
src/contractor/contractor.cpp
View File

@ -71,8 +71,9 @@ int Contractor::Run()
std::size_t max_edge_id = LoadEdgeExpandedGraph(
config.edge_based_graph_path, edge_based_edge_list, config.edge_segment_lookup_path,
config.edge_penalty_path, config.segment_speed_lookup_path, config.node_based_graph_path,
config.geometry_path, config.rtree_leaf_path);
config.edge_penalty_path, config.segment_speed_lookup_paths, config.node_based_graph_path,
config.geometry_path, config.datasource_names_path, config.datasource_indexes_path,
config.rtree_leaf_path);
// Contracting the edge-expanded graph
@ -127,15 +128,17 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list,
const std::string &edge_segment_lookup_filename,
const std::string &edge_penalty_filename,
const std::string &segment_speed_filename,
const std::vector<std::string> &segment_speed_filenames,
const std::string &nodes_filename,
const std::string &geometry_filename,
const std::string &datasource_names_filename,
const std::string &datasource_indexes_filename,
const std::string &rtree_leaf_filename)
{
util::SimpleLogger().Write() << "Opening " << edge_based_graph_filename;
boost::filesystem::ifstream input_stream(edge_based_graph_filename, std::ios::binary);
const bool update_edge_weights = segment_speed_filename != "";
const bool update_edge_weights = !segment_speed_filenames.empty();
boost::filesystem::ifstream edge_segment_input_stream;
boost::filesystem::ifstream edge_fixed_penalties_input_stream;
@ -167,22 +170,36 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
util::SimpleLogger().Write() << "Reading " << number_of_edges
<< " edges from the edge based graph";
std::unordered_map<std::pair<OSMNodeID, OSMNodeID>, unsigned> segment_speed_lookup;
std::unordered_map<std::pair<OSMNodeID, OSMNodeID>, std::pair<unsigned, uint8_t>>
segment_speed_lookup;
if (update_edge_weights)
{
util::SimpleLogger().Write()
<< "Segment speed data supplied, will update edge weights from "
<< segment_speed_filename;
io::CSVReader<3> csv_in(segment_speed_filename);
csv_in.set_header("from_node", "to_node", "speed");
uint64_t from_node_id{};
uint64_t to_node_id{};
unsigned speed{};
while (csv_in.read_row(from_node_id, to_node_id, speed))
uint8_t file_id = 1;
for (auto segment_speed_filename : segment_speed_filenames)
{
segment_speed_lookup[std::make_pair(OSMNodeID(from_node_id), OSMNodeID(to_node_id))] =
speed;
util::SimpleLogger().Write()
<< "Segment speed data supplied, will update edge weights from "
<< segment_speed_filename;
io::CSVReader<3> csv_in(segment_speed_filename);
csv_in.set_header("from_node", "to_node", "speed");
uint64_t from_node_id{};
uint64_t to_node_id{};
unsigned speed{};
while (csv_in.read_row(from_node_id, to_node_id, speed))
{
segment_speed_lookup[std::make_pair(OSMNodeID(from_node_id),
OSMNodeID(to_node_id))] =
std::make_pair(speed, file_id);
}
++file_id;
// Check for overflow
if (file_id == 0)
{
throw util::exception(
"Sorry, there's a limit of 254 segment speed files, you supplied too many");
}
}
std::vector<extractor::QueryNode> internal_to_external_node_map;
@ -241,6 +258,14 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
}
}
// This is a list of the "data source id" for every segment in the compressed
// geometry container. We assume that everything so far has come from the
// profile (data source 0). Here, we replace the 0's with the index of the
// CSV file that supplied the value that gets used for that segment, then
// we write out this list so that it can be returned by the debugging
// vector tiles later on.
std::vector<uint8_t> m_geometry_datasource(m_geometry_list.size(), 0);
// Now, we iterate over all the segments stored in the StaticRTree, updating
// the packed geometry weights in the `.geometries` file (note: we do not
// update the RTree itself, we just use the leaf nodes to iterate over all segments)
@ -297,12 +322,16 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
segment_speed_lookup.find(std::make_pair(u->node_id, v->node_id));
if (forward_speed_iter != segment_speed_lookup.end())
{
int new_segment_weight = std::max(
1, static_cast<int>(std::floor(
(segment_length * 10.) / (forward_speed_iter->second / 3.6) +
.5)));
int new_segment_weight =
std::max(1, static_cast<int>(std::floor(
(segment_length * 10.) /
(forward_speed_iter->second.first / 3.6) +
.5)));
m_geometry_list[forward_begin + leaf_object.fwd_segment_position]
.weight = new_segment_weight;
m_geometry_datasource[forward_begin +
leaf_object.fwd_segment_position] =
forward_speed_iter->second.second;
}
}
if (leaf_object.reverse_packed_geometry_id != SPECIAL_EDGEID)
@ -338,12 +367,15 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
segment_speed_lookup.find(std::make_pair(u->node_id, v->node_id));
if (reverse_speed_iter != segment_speed_lookup.end())
{
int new_segment_weight = std::max(
1, static_cast<int>(std::floor(
(segment_length * 10.) / (reverse_speed_iter->second / 3.6) +
.5)));
int new_segment_weight =
std::max(1, static_cast<int>(std::floor(
(segment_length * 10.) /
(reverse_speed_iter->second.first / 3.6) +
.5)));
m_geometry_list[reverse_begin + rev_segment_position].weight =
new_segment_weight;
m_geometry_datasource[reverse_begin + rev_segment_position] =
reverse_speed_iter->second.second;
}
}
}
@ -370,6 +402,34 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
number_of_compressed_geometries *
sizeof(extractor::CompressedEdgeContainer::CompressedEdge));
}
{
std::ofstream datasource_stream(datasource_indexes_filename, std::ios::binary);
if (!datasource_stream)
{
throw util::exception("Failed to open " + datasource_indexes_filename +
" for writing");
}
auto number_of_datasource_entries = m_geometry_datasource.size();
datasource_stream.write(reinterpret_cast<const char *>(&number_of_datasource_entries),
sizeof(number_of_datasource_entries));
datasource_stream.write(reinterpret_cast<char *>(&(m_geometry_datasource[0])),
number_of_datasource_entries * sizeof(uint8_t));
}
{
std::ofstream datasource_stream(datasource_names_filename, std::ios::binary);
if (!datasource_stream)
{
throw util::exception("Failed to open " + datasource_names_filename +
" for writing");
}
datasource_stream << "lua profile" << std::endl;
for (auto const &name : segment_speed_filenames)
{
datasource_stream << name << std::endl;
}
}
}
// TODO: can we read this in bulk? util::DeallocatingVector isn't necessarily
@ -413,9 +473,9 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
// This sets the segment weight using the same formula as the
// EdgeBasedGraphFactory for consistency. The *why* of this formula
// is lost in the annals of time.
int new_segment_weight =
std::max(1, static_cast<int>(std::floor(
(segment_length * 10.) / (speed_iter->second / 3.6) + .5)));
int new_segment_weight = std::max(
1, static_cast<int>(std::floor(
(segment_length * 10.) / (speed_iter->second.first / 3.6) + .5)));
new_weight += new_segment_weight;
}
else

42
src/engine/plugins/tile.cpp
View File

@ -191,6 +191,8 @@ Status TilePlugin::HandleRequest(const api::TileParameters &parameters, std::str
xyz2mercator(parameters.x, parameters.y, parameters.z, min_lon, min_lat, max_lon, max_lat);
const bbox tile_bbox{min_lon, min_lat, max_lon, max_lat};
uint8_t max_datasource_id = 0;
// Protobuf serialized blocks when objects go out of scope, hence
// the extra scoping below.
protozero::pbf_writer tile_writer{pbf_buffer};
@ -221,12 +223,20 @@ Status TilePlugin::HandleRequest(const api::TileParameters &parameters, std::str
int forward_weight = 0;
int reverse_weight = 0;
uint8_t forward_datasource = 0;
uint8_t reverse_datasource = 0;
if (edge.forward_packed_geometry_id != SPECIAL_EDGEID)
{
std::vector<EdgeWeight> forward_weight_vector;
facade.GetUncompressedWeights(edge.forward_packed_geometry_id,
forward_weight_vector);
forward_weight = forward_weight_vector[edge.fwd_segment_position];
std::vector<uint8_t> forward_datasource_vector;
facade.GetUncompressedDatasources(edge.forward_packed_geometry_id,
forward_datasource_vector);
forward_datasource = forward_datasource_vector[edge.fwd_segment_position];
}
if (edge.reverse_packed_geometry_id != SPECIAL_EDGEID)
@ -239,8 +249,20 @@ Status TilePlugin::HandleRequest(const api::TileParameters &parameters, std::str
reverse_weight = reverse_weight_vector[reverse_weight_vector.size() -
edge.fwd_segment_position - 1];
std::vector<uint8_t> reverse_datasource_vector;
facade.GetUncompressedDatasources(edge.reverse_packed_geometry_id,
reverse_datasource_vector);
reverse_datasource =
reverse_datasource_vector[reverse_datasource_vector.size() -
edge.fwd_segment_position - 1];
}
// Keep track of the highest datasource seen so that we don't write unnecessary
// data to the layer attribute values
max_datasource_id = std::max(max_datasource_id, forward_datasource);
max_datasource_id = std::max(max_datasource_id, reverse_datasource);
// If this is a valid forward edge, go ahead and add it to the tile
if (forward_weight != 0 && edge.forward_edge_based_node_id != SPECIAL_NODEID)
{
@ -296,7 +318,10 @@ Status TilePlugin::HandleRequest(const api::TileParameters &parameters, std::str
field.add_element(
std::min(speed, 127u)); // save the speed value, capped at 127
field.add_element(1); // "is_small" tag key offset
field.add_element(edge.component.is_tiny ? 0 : 1); // is_small feature
field.add_element(128 +
(edge.component.is_tiny ? 0 : 1)); // is_small feature
field.add_element(2); // "datasource" tag key offset
field.add_element(130 + forward_datasource); // datasource value offset
}
{
// Encode the geometry for the feature
@ -346,7 +371,10 @@ Status TilePlugin::HandleRequest(const api::TileParameters &parameters, std::str
field.add_element(
std::min(speed, 127u)); // save the speed value, capped at 127
field.add_element(1); // "is_small" tag key offset
field.add_element(edge.component.is_tiny ? 0 : 1); // is_small feature
field.add_element(128 +
(edge.component.is_tiny ? 0 : 1)); // is_small feature
field.add_element(2); // "datasource" tag key offset
field.add_element(130 + reverse_datasource); // datasource value offset
}
{
protozero::packed_field_uint32 geometry(feature_writer, 4);
@ -361,6 +389,7 @@ Status TilePlugin::HandleRequest(const api::TileParameters &parameters, std::str
// earlier
layer_writer.add_string(3, "speed");
layer_writer.add_string(3, "is_small");
layer_writer.add_string(3, "datasource");
// Now, we write out the possible speed value arrays and possible is_tiny
// values. Field type 4 is the "values" field. It's a variable type field,
@ -384,6 +413,15 @@ Status TilePlugin::HandleRequest(const api::TileParameters &parameters, std::str
// Attribute value 7 == bool type
values_writer.add_bool(7, false);
}
for (std::size_t i = 0; i <= max_datasource_id; i++)
{
{
// Writing field type 4 == variant type
protozero::pbf_writer values_writer(layer_writer, 4);
// Attribute value 1 == string type
values_writer.add_string(1, facade.GetDatasourceName(i));
}
}
}
return Status::Ok;

7
src/extractor/extractor.cpp
View File

@ -528,12 +528,7 @@ Extractor::BuildEdgeExpandedGraph(std::vector<QueryNode> &internal_to_external_n
edge_based_graph_factory.Run(config.edge_output_path, lua_state,
config.edge_segment_lookup_path, config.edge_penalty_path,
config.generate_edge_lookup
#ifdef DEBUG_GEOMETRY
,
config.debug_turns_path
#endif
);
config.generate_edge_lookup);
lua_close(lua_state);

99
src/storage/storage.cpp
View File

@ -131,6 +131,14 @@ int Storage::Run()
{
throw util::exception("no core file found");
}
if (paths.find("datasource_indexes") == paths.end())
{
throw util::exception("no datasource_indexes file found");
}
if (paths.find("datasource_names") == paths.end())
{
throw util::exception("no datasource_names file found");
}
auto paths_iterator = paths.find("hsgrdata");
BOOST_ASSERT(paths.end() != paths_iterator);
@ -170,6 +178,14 @@ int Storage::Run()
BOOST_ASSERT(paths.end() != paths_iterator);
BOOST_ASSERT(!paths_iterator->second.empty());
const boost::filesystem::path &core_marker_path = paths_iterator->second;
paths_iterator = paths.find("datasource_indexes");
BOOST_ASSERT(paths.end() != paths_iterator);
BOOST_ASSERT(!paths_iterator->second.empty());
const boost::filesystem::path &datasource_indexes_path = paths_iterator->second;
paths_iterator = paths.find("datasource_names");
BOOST_ASSERT(paths.end() != paths_iterator);
BOOST_ASSERT(!paths_iterator->second.empty());
const boost::filesystem::path &datasource_names_path = paths_iterator->second;
// determine segment to use
bool segment2_in_use = SharedMemory::RegionExists(LAYOUT_2);
@ -322,6 +338,45 @@ int Storage::Run()
geometry_input_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned));
shared_layout_ptr->SetBlockSize<extractor::CompressedEdgeContainer::CompressedEdge>(
SharedDataLayout::GEOMETRIES_LIST, number_of_compressed_geometries);
// load datasource sizes. This file is optional, and it's non-fatal if it doesn't
// exist.
std::ifstream geometry_datasource_input_stream(datasource_indexes_path.c_str(),
std::ios::binary);
if (geometry_datasource_input_stream)
{
std::size_t number_of_compressed_datasources = 0;
geometry_datasource_input_stream.read(
reinterpret_cast<char *>(&number_of_compressed_datasources), sizeof(std::size_t));
shared_layout_ptr->SetBlockSize<uint8_t>(SharedDataLayout::DATASOURCES_LIST,
number_of_compressed_datasources);
}
// Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't
// exist
std::ifstream datasource_names_input_stream(datasource_names_path.c_str(), std::ios::binary);
std::vector<char> m_datasource_name_data;
std::vector<std::size_t> m_datasource_name_offsets;
std::vector<std::size_t> m_datasource_name_lengths;
if (datasource_names_input_stream)
{
std::string name;
while (std::getline(datasource_names_input_stream, name))
{
m_datasource_name_offsets.push_back(m_datasource_name_data.size());
std::copy(name.c_str(), name.c_str() + name.size(),
std::back_inserter(m_datasource_name_data));
m_datasource_name_lengths.push_back(name.size());
}
shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::DATASOURCE_NAME_DATA,
m_datasource_name_data.size());
shared_layout_ptr->SetBlockSize<std::size_t>(SharedDataLayout::DATASOURCE_NAME_OFFSETS,
m_datasource_name_offsets.size());
shared_layout_ptr->SetBlockSize<std::size_t>(SharedDataLayout::DATASOURCE_NAME_LENGTHS,
m_datasource_name_lengths.size());
}
// allocate shared memory block
util::SimpleLogger().Write() << "allocating shared memory of "
<< shared_layout_ptr->GetSizeOfLayout() << " bytes";
@ -435,6 +490,50 @@ int Storage::Run()
shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_LIST));
}
// load datasource information (if it exists)
if (geometry_datasource_input_stream)
{
uint8_t *datasources_list_ptr = shared_layout_ptr->GetBlockPtr<uint8_t, true>(
shared_memory_ptr, SharedDataLayout::DATASOURCES_LIST);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCES_LIST) > 0)
{
geometry_datasource_input_stream.read(
reinterpret_cast<char *>(datasources_list_ptr),
shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCES_LIST));
}
}
// load datasource name information (if it exists)
if (!m_datasource_name_data.empty())
{
char *datasource_name_data_ptr = shared_layout_ptr->GetBlockPtr<char, true>(
shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_DATA);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_DATA) > 0)
{
std::cout << "Copying "
<< (m_datasource_name_data.end() - m_datasource_name_data.begin())
<< " chars into name data ptr\n";
std::copy(m_datasource_name_data.begin(), m_datasource_name_data.end(),
datasource_name_data_ptr);
}
auto datasource_name_offsets_ptr = shared_layout_ptr->GetBlockPtr<std::size_t, true>(
shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_OFFSETS);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_OFFSETS) > 0)
{
std::copy(m_datasource_name_offsets.begin(), m_datasource_name_offsets.end(),
datasource_name_offsets_ptr);
}
auto datasource_name_lengths_ptr = shared_layout_ptr->GetBlockPtr<std::size_t, true>(
shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_LENGTHS);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_LENGTHS) > 0)
{
std::copy(m_datasource_name_lengths.begin(), m_datasource_name_lengths.end(),
datasource_name_lengths_ptr);
}
}
// Loading list of coordinates
util::Coordinate *coordinates_ptr = shared_layout_ptr->GetBlockPtr<util::Coordinate, true>(
shared_memory_ptr, SharedDataLayout::COORDINATE_LIST);

7
src/tools/contract.cpp
View File

@ -39,9 +39,10 @@ return_code parseArguments(int argc, char *argv[], contractor::ContractorConfig
"core,k",
boost::program_options::value<double>(&contractor_config.core_factor)->default_value(1.0),
"Percentage of the graph (in vertices) to contract [0..1]")(
"segment-speed-file",
boost::program_options::value<std::string>(&contractor_config.segment_speed_lookup_path),
"Lookup file containing nodeA,nodeB,speed data to adjust edge weights")(
"segment-speed-file", boost::program_options::value<std::vector<std::string>>(
&contractor_config.segment_speed_lookup_paths)
->composing(),
"Lookup files containing nodeA, nodeB, speed data to adjust edge weights")(
"level-cache,o", boost::program_options::value<bool>(&contractor_config.use_cached_priority)
->default_value(false),
"Use .level file to retain the contaction level for each node from the last run.");

20
src/tools/store.cpp
View File

@ -38,7 +38,13 @@ bool generateDataStoreOptions(const int argc, const char *argv[], storage::DataP
"namesdata", boost::program_options::value<boost::filesystem::path>(&paths["namesdata"]),
".names file")("timestamp",
boost::program_options::value<boost::filesystem::path>(&paths["timestamp"]),
".timestamp file");
".timestamp file")(
"datasource_names",
boost::program_options::value<boost::filesystem::path>(&paths["datasource_names"]),
".datasource_names file")(
"datasource_indexes",
boost::program_options::value<boost::filesystem::path>(&paths["datasource_indexes"]),
".datasource_indexes file");
// hidden options, will be allowed on command line but will not be shown to the user
boost::program_options::options_description hidden_options("Hidden options");
@ -145,6 +151,18 @@ bool generateDataStoreOptions(const int argc, const char *argv[], storage::DataP
path_iterator->second = base_string + ".timestamp";
}
path_iterator = paths.find("datasource_indexes");
if (path_iterator != paths.end())
{
path_iterator->second = base_string + ".datasource_indexes";
}
path_iterator = paths.find("datasource_names");
if (path_iterator != paths.end())
{
path_iterator->second = base_string + ".datasource_names";
}
path_iterator = paths.find("hsgrdata");
if (path_iterator == paths.end() || path_iterator->second.string().empty() ||
!boost::filesystem::is_regular_file(path_iterator->second))