Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Include datasources in .geometries file and refactor .datasource_names

Browse Source
This commit is contained in:
Patrick Niklaus
2017-03-09 17:01:04 +00:00
committed by Patrick Niklaus
parent ffd6311e7d
commit a636e8cc13
14 changed files with 166 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/extractor/compressed_edge_container.cpp
+6
View File
@@ -221,6 +221,7 @@ void CompressedEdgeContainer::InitializeBothwayVector()
segment_data->rev_weights.reserve(m_compressed_oneway_geometries.size());
segment_data->fwd_durations.reserve(m_compressed_oneway_geometries.size());
segment_data->rev_durations.reserve(m_compressed_oneway_geometries.size());
segment_data->datasources.reserve(m_compressed_oneway_geometries.size());
}
unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID r_edge_id)
@@ -238,11 +239,14 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID
const auto &first_node = reverse_bucket.back();
constexpr DatasourceID LUA_SOURCE = 0;
segment_data->nodes.emplace_back(first_node.node_id);
segment_data->fwd_weights.emplace_back(INVALID_EDGE_WEIGHT);
segment_data->rev_weights.emplace_back(first_node.weight);
segment_data->fwd_durations.emplace_back(INVALID_EDGE_WEIGHT);
segment_data->rev_durations.emplace_back(first_node.duration);
segment_data->datasources.emplace_back(LUA_SOURCE);
for (std::size_t i = 0; i < forward_bucket.size() - 1; ++i)
{
@@ -256,6 +260,7 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID
segment_data->rev_weights.emplace_back(rev_node.weight);
segment_data->fwd_durations.emplace_back(fwd_node.duration);
segment_data->rev_durations.emplace_back(rev_node.duration);
segment_data->datasources.emplace_back(LUA_SOURCE);
}
const auto &last_node = forward_bucket.back();
@@ -265,6 +270,7 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID
segment_data->rev_weights.emplace_back(INVALID_EDGE_WEIGHT);
segment_data->fwd_durations.emplace_back(last_node.duration);
segment_data->rev_durations.emplace_back(INVALID_EDGE_WEIGHT);
segment_data->datasources.emplace_back(LUA_SOURCE);
return zipped_geometry_id;
}
src/storage/storage.cpp
+13 -103
View File
@@ -3,6 +3,7 @@
#include "extractor/compressed_edge_container.hpp"
#include "extractor/edge_based_edge.hpp"
#include "extractor/guidance/turn_instruction.hpp"
#include "extractor/io.hpp"
#include "extractor/original_edge_data.hpp"
#include "extractor/profile_properties.hpp"
#include "extractor/query_node.hpp"
@@ -339,40 +340,13 @@ void Storage::PopulateLayout(DataLayout &layout)
number_of_compressed_geometries);
layout.SetBlockSize<EdgeWeight>(DataLayout::GEOMETRIES_REV_DURATION_LIST,
number_of_compressed_geometries);
layout.SetBlockSize<DatasourceID>(DataLayout::DATASOURCES_LIST,
number_of_compressed_geometries);
}
// load datasource sizes. This file is optional, and it's non-fatal if it doesn't exist.
if (boost::filesystem::exists(config.datasource_indexes_path))
// Load datasource name sizes.
{
io::FileReader reader(config.datasource_indexes_path, io::FileReader::HasNoFingerprint);
const auto number_of_datasources = reader.ReadElementCount64();
layout.SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST, number_of_datasources);
}
else
{
layout.SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST, 0);
}
// Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't exist
if (boost::filesystem::exists(config.datasource_names_path))
{
io::FileReader reader(config.datasource_names_path, io::FileReader::HasNoFingerprint);
const serialization::DatasourceNamesData datasource_names_data =
serialization::readDatasourceNames(reader);
layout.SetBlockSize<char>(DataLayout::DATASOURCE_NAME_DATA,
datasource_names_data.names.size());
layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_OFFSETS,
datasource_names_data.offsets.size());
layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_LENGTHS,
datasource_names_data.lengths.size());
}
else
{
layout.SetBlockSize<char>(DataLayout::DATASOURCE_NAME_DATA, 0);
layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_OFFSETS, 0);
layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_LENGTHS, 0);
layout.SetBlockSize<extractor::Datasources>(DataLayout::DATASOURCES_NAMES, 1);
}
{
@@ -689,81 +663,17 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
BOOST_ASSERT(geometry_node_lists_count ==
layout.num_entries[DataLayout::GEOMETRIES_REV_DURATION_LIST]);
geometry_input_file.ReadInto(geometries_rev_duration_list_ptr, geometry_node_lists_count);
const auto datasource_list_ptr =
layout.GetBlockPtr<DatasourceID, true>(memory_ptr, DataLayout::DATASOURCES_LIST);
BOOST_ASSERT(geometry_node_lists_count == layout.num_entries[DataLayout::DATASOURCES_LIST]);
geometry_input_file.ReadInto(datasource_list_ptr, geometry_node_lists_count);
}
if (boost::filesystem::exists(config.datasource_indexes_path))
{
io::FileReader geometry_datasource_file(config.datasource_indexes_path,
io::FileReader::HasNoFingerprint);
const auto number_of_compressed_datasources = geometry_datasource_file.ReadElementCount64();
// load datasource information (if it exists)
const auto datasources_list_ptr =
layout.GetBlockPtr<uint8_t, true>(memory_ptr, DataLayout::DATASOURCES_LIST);
if (number_of_compressed_datasources > 0)
{
serialization::readDatasourceIndexes(
geometry_datasource_file, datasources_list_ptr, number_of_compressed_datasources);
}
}
else
{
layout.GetBlockPtr<uint8_t, true>(memory_ptr, DataLayout::DATASOURCES_LIST);
}
if (boost::filesystem::exists(config.datasource_names_path))
{
io::FileReader datasource_names_file(config.datasource_names_path,
io::FileReader::HasNoFingerprint);
const auto datasource_names_data =
serialization::readDatasourceNames(datasource_names_file);
// load datasource name information (if it exists)
const auto datasource_name_data_ptr =
layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::DATASOURCE_NAME_DATA);
if (layout.GetBlockSize(DataLayout::DATASOURCE_NAME_DATA) > 0)
{
BOOST_ASSERT(std::distance(datasource_names_data.names.begin(),
datasource_names_data.names.end()) *
sizeof(decltype(datasource_names_data.names)::value_type) <=
layout.GetBlockSize(DataLayout::DATASOURCE_NAME_DATA));
std::copy(datasource_names_data.names.begin(),
datasource_names_data.names.end(),
datasource_name_data_ptr);
}
const auto datasource_name_offsets_ptr =
layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_OFFSETS);
if (layout.GetBlockSize(DataLayout::DATASOURCE_NAME_OFFSETS) > 0)
{
BOOST_ASSERT(std::distance(datasource_names_data.offsets.begin(),
datasource_names_data.offsets.end()) *
sizeof(decltype(datasource_names_data.offsets)::value_type) <=
layout.GetBlockSize(DataLayout::DATASOURCE_NAME_OFFSETS));
std::copy(datasource_names_data.offsets.begin(),
datasource_names_data.offsets.end(),
datasource_name_offsets_ptr);
}
const auto datasource_name_lengths_ptr =
layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_LENGTHS);
if (layout.GetBlockSize(DataLayout::DATASOURCE_NAME_LENGTHS) > 0)
{
BOOST_ASSERT(std::distance(datasource_names_data.lengths.begin(),
datasource_names_data.lengths.end()) *
sizeof(decltype(datasource_names_data.lengths)::value_type) <=
layout.GetBlockSize(DataLayout::DATASOURCE_NAME_LENGTHS));
std::copy(datasource_names_data.lengths.begin(),
datasource_names_data.lengths.end(),
datasource_name_lengths_ptr);
}
}
else
{
layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::DATASOURCE_NAME_DATA);
layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_OFFSETS);
layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_LENGTHS);
const auto datasources_names_ptr = layout.GetBlockPtr<extractor::Datasources, true>(
memory_ptr, DataLayout::DATASOURCES_NAMES);
extractor::io::read(config.datasource_names_path, *datasources_names_ptr);
}
// Loading list of coordinates
src/storage/storage_config.cpp
+3 -3
View File
@@ -32,7 +32,6 @@ StorageConfig::StorageConfig(const boost::filesystem::path &base)
turn_weight_penalties_path{base.string() + ".turn_weight_penalties"},
turn_duration_penalties_path{base.string() + ".turn_duration_penalties"},
datasource_names_path{base.string() + ".datasource_names"},
datasource_indexes_path{base.string() + ".datasource_indexes"},
names_data_path{base.string() + ".names"}, properties_path{base.string() + ".properties"},
intersection_class_path{base.string() + ".icd"}, turn_lane_data_path{base.string() + ".tld"},
turn_lane_description_path{base.string() + ".tls"},
@@ -54,7 +53,8 @@ bool StorageConfig::IsValid() const
turn_duration_penalties_path,
names_data_path,
properties_path,
intersection_class_path}))
intersection_class_path,
datasource_names_path}))
{
return false;
}
@@ -62,7 +62,7 @@ bool StorageConfig::IsValid() const
// TODO: add algorithm checks
// CH files
CheckFileList({hsgr_data_path, core_data_path, datasource_names_path, datasource_indexes_path});
CheckFileList({hsgr_data_path, core_data_path});
// MLD files
CheckFileList({mld_partition_path, mld_storage_path, edge_based_graph_path});
src/updater/updater.cpp
+9 -44
View File
@@ -262,10 +262,6 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
auto segment_speed_lookup = csv::readSegmentValues(config.segment_speed_lookup_paths);
auto turn_penalty_lookup = csv::readTurnValues(config.turn_penalty_lookup_paths);
// If we update the edge weights, this file will hold the datasource information for each
// segment; the other files will also be conditionally filled concurrently if we make an update
std::vector<uint8_t> geometry_datasource;
std::vector<extractor::QueryNode> internal_to_external_node_map;
extractor::SegmentDataContainer segment_data;
@@ -294,14 +290,6 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
// Here, we have to update the compressed geometry weights
// First, we need the external-to-internal node lookup table
// This is a list of the "data source id" for every segment in the compressed
// geometry container. We assume that everything so far has come from the
// profile (data source 0). Here, we replace the 0's with the index of the
// CSV file that supplied the value that gets used for that segment, then
// we write out this list so that it can be returned by the debugging
// vector tiles later on.
geometry_datasource.resize(segment_data.GetNumberOfSegments(), 0);
// Now, we iterate over all the segments stored in the StaticRTree, updating
// the packed geometry weights in the `.geometries` file (note: we do not
// update the RTree itself, we just use the leaf nodes to iterate over all segments)
@@ -364,8 +352,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
segment_data.ForwardWeight(geometry_id, segment_offset) = new_segment_weight;
segment_data.ForwardDuration(geometry_id, segment_offset) =
new_segment_duration;
geometry_datasource[segment_data.GetOffset(geometry_id, segment_offset) + 1] =
value->source;
segment_data.ForwardDatasource(geometry_id, segment_offset) = value->source;
fwd_source = value->source;
}
@@ -385,8 +372,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
segment_data.ReverseWeight(geometry_id, segment_offset) = new_segment_weight;
segment_data.ReverseDuration(geometry_id, segment_offset) =
new_segment_duration;
geometry_datasource[segment_data.GetOffset(geometry_id, segment_offset)] =
value->source;
segment_data.ReverseDatasource(geometry_id, segment_offset) = value->source;
rev_source = value->source;
}
@@ -430,44 +416,23 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
extractor::io::write(config.geometry_path, segment_data);
};
const auto save_datasource_indexes = [&] {
std::ofstream datasource_stream(config.datasource_indexes_path, std::ios::binary);
if (!datasource_stream)
{
const std::string message{"Failed to open " + config.datasource_indexes_path +
" for writing"};
throw util::exception(message + SOURCE_REF);
}
std::uint64_t number_of_datasource_entries = geometry_datasource.size();
datasource_stream.write(reinterpret_cast<const char *>(&number_of_datasource_entries),
sizeof(number_of_datasource_entries));
if (number_of_datasource_entries > 0)
{
datasource_stream.write(reinterpret_cast<char *>(&(geometry_datasource[0])),
number_of_datasource_entries * sizeof(uint8_t));
}
};
const auto save_datastore_names = [&] {
std::ofstream datasource_stream(config.datasource_names_path, std::ios::binary);
if (!datasource_stream)
{
const std::string message{"Failed to open " + config.datasource_names_path +
" for writing"};
throw util::exception(message + SOURCE_REF);
}
datasource_stream << "lua profile" << std::endl;
extractor::Datasources sources;
DatasourceID source = 0;
sources.SetSourceName(source++, "lua profile");
// Only write the filename, without path or extension.
// This prevents information leakage, and keeps names short
// for rendering in the debug tiles.
for (auto const &name : config.segment_speed_lookup_paths)
{
datasource_stream << boost::filesystem::path(name).stem().string() << std::endl;
sources.SetSourceName(source++, boost::filesystem::path(name).stem().string());
}
extractor::io::write(config.datasource_names_path, sources);
};
tbb::parallel_invoke(maybe_save_geometries, save_datasource_indexes, save_datastore_names);
tbb::parallel_invoke(maybe_save_geometries, save_datastore_names);
std::vector<TurnPenalty> turn_weight_penalties;
std::vector<TurnPenalty> turn_duration_penalties;