Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Store node ID's and weights separately in CompressedEdgeContainer

Browse Source 
Removes CompressedEdges from the extractor and shared data format by
directly serializing vectors of node ID's, forward weights and reverse
weights for each node-based-edge

Refs #2575
This commit is contained in:
jakepruitt 2016-10-06 21:06:33 -07:00 committed by Jake Pruitt
parent ecee13bffa
commit 42271d99b2
7 changed files with 224 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
include/engine/datafacade/internal_datafacade.hpp
View File

@ -86,7 +86,9 @@ class InternalDataFacade final : public BaseDataFacade
util::ShM<extractor::TravelMode, false>::vector m_travel_mode_list;
util::ShM<char, false>::vector m_names_char_list;
util::ShM<unsigned, false>::vector m_geometry_indices;
util::ShM<extractor::CompressedEdgeContainer::CompressedEdge, false>::vector m_geometry_list;
util::ShM<NodeID, false>::vector m_geometry_node_list;
util::ShM<EdgeWeight, false>::vector m_geometry_fwd_weight_list;
util::ShM<EdgeWeight, false>::vector m_geometry_rev_weight_list;
util::ShM<bool, false>::vector m_is_core_node;
util::ShM<unsigned, false>::vector m_segment_weights;
util::ShM<uint8_t, false>::vector m_datasource_list;
@ -268,14 +270,32 @@ class InternalDataFacade final : public BaseDataFacade
geometry_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned));
std::cout << "m_geometry_indices: " << std::endl;
std::for_each(m_geometry_indices.begin(),
m_geometry_indices.end(),
[](const auto &index) {
std::cout << index << " ";
});
std::cout << std::endl;
std::cout << "number_of_compressed_geometries: " << number_of_compressed_geometries << std::endl;
BOOST_ASSERT(m_geometry_indices.back() == number_of_compressed_geometries);
m_geometry_list.resize(number_of_compressed_geometries);
m_geometry_node_list.resize(number_of_compressed_geometries);
m_geometry_fwd_weight_list.resize(number_of_compressed_geometries);
m_geometry_rev_weight_list.resize(number_of_compressed_geometries);
if (number_of_compressed_geometries > 0)
{
geometry_stream.read((char *)&(m_geometry_list[0]),
geometry_stream.read((char *)&(m_geometry_node_list[0]),
number_of_compressed_geometries *
sizeof(extractor::CompressedEdgeContainer::CompressedEdge));
sizeof(NodeID));
geometry_stream.read((char *)&(m_geometry_fwd_weight_list[0]),
number_of_compressed_geometries *
sizeof(EdgeWeight));
geometry_stream.read((char *)&(m_geometry_rev_weight_list[0]),
number_of_compressed_geometries *
sizeof(EdgeWeight));
}
}
@ -714,8 +734,10 @@ class InternalDataFacade final : public BaseDataFacade
* both forward and reverse segments along the same bi-
* directional edge. The m_geometry_indices stores
* refences to where to find the beginning of the bi-
* directional edge in the m_geometry_list vector.
* */
* directional edge in the m_geometry_node_list vector. For
* forward geometries of bi-directional edges, edges 2 to
* n of that edge need to be read.
*/
const unsigned begin = m_geometry_indices.at(id);
const unsigned end = m_geometry_indices.at(id + 1);
@ -723,10 +745,10 @@ class InternalDataFacade final : public BaseDataFacade
result_nodes.reserve(end - begin);
std::for_each(m_geometry_list.begin() + begin,
m_geometry_list.begin() + end,
[&](const osrm::extractor::CompressedEdgeContainer::CompressedEdge &edge) {
result_nodes.emplace_back(edge.node_id);
std::for_each(m_geometry_node_list.begin() + begin,
m_geometry_node_list.begin() + end,
[&](const NodeID &node_id) {
result_nodes.emplace_back(node_id);
});
return result_nodes;
@ -739,19 +761,19 @@ class InternalDataFacade final : public BaseDataFacade
* both forward and reverse segments along the same bi-
* directional edge. The m_geometry_indices stores
* refences to where to find the beginning of the bi-
* directional edge in the m_geometry_list vector.
* directional edge in the m_geometry_node_list vector.
* */
const unsigned begin = m_geometry_indices.at(id);
const unsigned end = m_geometry_indices.at(id + 1);
const signed begin = m_geometry_indices.at(id);
const signed end = m_geometry_indices.at(id + 1);
std::vector<NodeID> result_nodes;
result_nodes.reserve(end - begin);
std::for_each(m_geometry_list.rbegin() + (m_geometry_list.size() - end),
m_geometry_list.rbegin() + (m_geometry_list.size() - begin),
[&](const osrm::extractor::CompressedEdgeContainer::CompressedEdge &edge) {
result_nodes.emplace_back(edge.node_id);
std::for_each(m_geometry_node_list.rbegin() + (m_geometry_node_list.size() - end),
m_geometry_node_list.rbegin() + (m_geometry_node_list.size() - begin),
[&](const NodeID &node_id) {
result_nodes.emplace_back(node_id);
});
return result_nodes;
@ -765,20 +787,18 @@ class InternalDataFacade final : public BaseDataFacade
* both forward and reverse segments along the same bi-
* directional edge. The m_geometry_indices stores
* refences to where to find the beginning of the bi-
* directional edge in the m_geometry_list vector. For
* forward weights of bi-directional edges, edges 2 to
* n of that edge need to be read.
*/
* directional edge in the m_geometry_fwd_weight_list vector.
* */
const unsigned begin = m_geometry_indices.at(id) + 1;
const unsigned end = m_geometry_indices.at(id + 1);
std::vector<EdgeWeight> result_weights;
result_weights.reserve(end - begin);
std::for_each(m_geometry_list.begin() + begin,
m_geometry_list.begin() + end,
[&](const osrm::extractor::CompressedEdgeContainer::CompressedEdge &edge) {
result_weights.emplace_back(edge.forward_weight);
std::for_each(m_geometry_fwd_weight_list.begin() + begin,
m_geometry_fwd_weight_list.begin() + end,
[&](const EdgeWeight &forward_weight) {
result_weights.emplace_back(forward_weight);
});
return result_weights;
@ -792,20 +812,20 @@ class InternalDataFacade final : public BaseDataFacade
* both forward and reverse segments along the same bi-
* directional edge. The m_geometry_indices stores
* refences to where to find the beginning of the bi-
* directional edge in the m_geometry_list vector. For
* directional edge in the m_geometry_rev_weight_list vector. For
* reverse weights of bi-directional edges, edges 1 to
* n-1 of that edge need to be read in reverse.
*/
const unsigned begin = m_geometry_indices.at(id);
const unsigned end = m_geometry_indices.at(id + 1) - 1;
const signed begin = m_geometry_indices.at(id);
const signed end = m_geometry_indices.at(id + 1) - 1;
std::vector<EdgeWeight> result_weights;
result_weights.reserve(end - begin);
std::for_each(m_geometry_list.rbegin() + (m_geometry_list.size() - end),
m_geometry_list.rbegin() + (m_geometry_list.size() - begin),
[&](const osrm::extractor::CompressedEdgeContainer::CompressedEdge &edge) {
result_weights.emplace_back(edge.reverse_weight);
std::for_each(m_geometry_rev_weight_list.rbegin() + (m_geometry_rev_weight_list.size() - end),
m_geometry_rev_weight_list.rbegin() + (m_geometry_rev_weight_list.size() - begin),
[&](const EdgeWeight &reverse_weight) {
result_weights.emplace_back(reverse_weight);
});
return result_weights;

74
include/engine/datafacade/shared_datafacade.hpp
View File

@ -91,7 +91,9 @@ class SharedDataFacade final : public BaseDataFacade
util::ShM<char, true>::vector m_names_char_list;
util::ShM<unsigned, true>::vector m_name_begin_indices;
util::ShM<unsigned, true>::vector m_geometry_indices;
util::ShM<extractor::CompressedEdgeContainer::CompressedEdge, true>::vector m_geometry_list;
util::ShM<NodeID, true>::vector m_geometry_node_list;
util::ShM<EdgeWeight, true>::vector m_geometry_fwd_weight_list;
util::ShM<EdgeWeight, true>::vector m_geometry_rev_weight_list;
util::ShM<bool, true>::vector m_is_core_node;
util::ShM<uint8_t, true>::vector m_datasource_list;
util::ShM<std::uint32_t, true>::vector m_lane_description_offsets;
@ -327,13 +329,29 @@ class SharedDataFacade final : public BaseDataFacade
data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_INDEX]);
m_geometry_indices = std::move(geometry_begin_indices);
auto geometries_list_ptr =
data_layout->GetBlockPtr<extractor::CompressedEdgeContainer::CompressedEdge>(
shared_memory, storage::SharedDataLayout::GEOMETRIES_LIST);
util::ShM<extractor::CompressedEdgeContainer::CompressedEdge, true>::vector geometry_list(
geometries_list_ptr,
data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_LIST]);
m_geometry_list = std::move(geometry_list);
auto geometries_node_list_ptr =
data_layout->GetBlockPtr<NodeID>(
shared_memory, storage::SharedDataLayout::GEOMETRIES_NODE_LIST);
util::ShM<NodeID, true>::vector geometry_node_list(
geometries_node_list_ptr,
data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_NODE_LIST]);
m_geometry_node_list = std::move(geometry_node_list);
auto geometries_fwd_weight_list_ptr =
data_layout->GetBlockPtr<EdgeWeight>(
shared_memory, storage::SharedDataLayout::GEOMETRIES_FWD_WEIGHT_LIST);
util::ShM<EdgeWeight, true>::vector geometry_fwd_weight_list(
geometries_fwd_weight_list_ptr,
data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_FWD_WEIGHT_LIST]);
m_geometry_fwd_weight_list = std::move(geometry_fwd_weight_list);
auto geometries_rev_weight_list_ptr =
data_layout->GetBlockPtr<EdgeWeight>(
shared_memory, storage::SharedDataLayout::GEOMETRIES_REV_WEIGHT_LIST);
util::ShM<EdgeWeight, true>::vector geometry_rev_weight_list(
geometries_rev_weight_list_ptr,
data_layout->num_entries[storage::SharedDataLayout::GEOMETRIES_REV_WEIGHT_LIST]);
m_geometry_rev_weight_list = std::move(geometry_rev_weight_list);
auto datasources_list_ptr = data_layout->GetBlockPtr<uint8_t>(
shared_memory, storage::SharedDataLayout::DATASOURCES_LIST);
@ -532,7 +550,7 @@ class SharedDataFacade final : public BaseDataFacade
* both forward and reverse segments along the same bi-
* directional edge. The m_geometry_indices stores
* refences to where to find the beginning of the bi-
* directional edge in the m_geometry_list vector. For
* directional edge in the m_geometry_node_list vector. For
* forward geometries of bi-directional edges, edges 2 to
* n of that edge need to be read.
*/
@ -543,10 +561,10 @@ class SharedDataFacade final : public BaseDataFacade
result_nodes.reserve(end - begin);
std::for_each(m_geometry_list.begin() + begin,
m_geometry_list.begin() + end,
[&](const osrm::extractor::CompressedEdgeContainer::CompressedEdge &edge) {
result_nodes.emplace_back(edge.node_id);
std::for_each(m_geometry_node_list.begin() + begin,
m_geometry_node_list.begin() + end,
[&](const NodeID &node_id) {
result_nodes.emplace_back(node_id);
});
return result_nodes;
@ -559,7 +577,7 @@ class SharedDataFacade final : public BaseDataFacade
* both forward and reverse segments along the same bi-
* directional edge. The m_geometry_indices stores
* refences to where to find the beginning of the bi-
* directional edge in the m_geometry_list vector.
* directional edge in the m_geometry_node_list vector.
* */
const signed begin = m_geometry_indices.at(id);
const signed end = m_geometry_indices.at(id + 1);
@ -568,10 +586,10 @@ class SharedDataFacade final : public BaseDataFacade
result_nodes.reserve(end - begin);
std::for_each(m_geometry_list.rbegin() + (m_geometry_list.size() - end),
m_geometry_list.rbegin() + (m_geometry_list.size() - begin),
[&](const osrm::extractor::CompressedEdgeContainer::CompressedEdge &edge) {
result_nodes.emplace_back(edge.node_id);
std::for_each(m_geometry_node_list.rbegin() + (m_geometry_node_list.size() - end),
m_geometry_node_list.rbegin() + (m_geometry_node_list.size() - begin),
[&](const NodeID &node_id) {
result_nodes.emplace_back(node_id);
});
return result_nodes;
@ -585,7 +603,7 @@ class SharedDataFacade final : public BaseDataFacade
* both forward and reverse segments along the same bi-
* directional edge. The m_geometry_indices stores
* refences to where to find the beginning of the bi-
* directional edge in the m_geometry_list vector.
* directional edge in the m_geometry_fwd_weight_list vector.
* */
const unsigned begin = m_geometry_indices.at(id) + 1;
const unsigned end = m_geometry_indices.at(id + 1);
@ -593,10 +611,10 @@ class SharedDataFacade final : public BaseDataFacade
std::vector<EdgeWeight> result_weights;
result_weights.reserve(end - begin);
std::for_each(m_geometry_list.begin() + begin,
m_geometry_list.begin() + end,
[&](const osrm::extractor::CompressedEdgeContainer::CompressedEdge &edge) {
result_weights.emplace_back(edge.forward_weight);
std::for_each(m_geometry_fwd_weight_list.begin() + begin,
m_geometry_fwd_weight_list.begin() + end,
[&](const EdgeWeight &forward_weight) {
result_weights.emplace_back(forward_weight);
});
return result_weights;
@ -610,7 +628,7 @@ class SharedDataFacade final : public BaseDataFacade
* both forward and reverse segments along the same bi-
* directional edge. The m_geometry_indices stores
* refences to where to find the beginning of the bi-
* directional edge in the m_geometry_list vector. For
* directional edge in the m_geometry_rev_weight_list vector. For
* reverse weights of bi-directional edges, edges 1 to
* n-1 of that edge need to be read in reverse.
*/
@ -620,10 +638,10 @@ class SharedDataFacade final : public BaseDataFacade
std::vector<EdgeWeight> result_weights;
result_weights.reserve(end - begin);
std::for_each(m_geometry_list.rbegin() + (m_geometry_list.size() - end),
m_geometry_list.rbegin() + (m_geometry_list.size() - begin),
[&](const osrm::extractor::CompressedEdgeContainer::CompressedEdge &edge) {
result_weights.emplace_back(edge.reverse_weight);
std::for_each(m_geometry_rev_weight_list.rbegin() + (m_geometry_rev_weight_list.size() - end),
m_geometry_rev_weight_list.rbegin() + (m_geometry_rev_weight_list.size() - begin),
[&](const EdgeWeight &reverse_weight) {
result_weights.emplace_back(reverse_weight);
});
return result_weights;

14
include/extractor/compressed_edge_container.hpp
View File

@ -23,16 +23,7 @@ class CompressedEdgeContainer
EdgeWeight weight; // the weight of the edge leading to this node
};
struct CompressedEdge
{
public:
NodeID node_id;
EdgeWeight forward_weight;
EdgeWeight reverse_weight;
};
using OnewayEdgeBucket = std::vector<OnewayCompressedEdge>;
using EdgeBucket = std::vector<CompressedEdge>;
CompressedEdgeContainer();
void CompressEdge(const EdgeID surviving_edge_id,
@ -67,7 +58,10 @@ class CompressedEdgeContainer
void IncreaseFreeList();
std::vector<OnewayEdgeBucket> m_compressed_oneway_geometries;
std::vector<EdgeBucket> m_compressed_geometries;
std::vector<unsigned> m_compressed_geometry_index;
std::vector<NodeID> m_compressed_geometry_nodes;
std::vector<EdgeWeight> m_compressed_geometry_fwd_weights;
std::vector<EdgeWeight> m_compressed_geometry_rev_weights;
std::vector<unsigned> m_free_list;
std::unordered_map<EdgeID, unsigned> m_edge_id_to_list_index_map;
std::unordered_map<EdgeID, unsigned> m_forward_edge_id_to_zipped_index_map;

8
include/storage/shared_datatype.hpp
View File

@ -30,7 +30,9 @@ const constexpr char *block_id_to_name[] = {"NAME_OFFSETS",
"ENTRY_CLASSID",
"R_SEARCH_TREE",
"GEOMETRIES_INDEX",
"GEOMETRIES_LIST",
"GEOMETRIES_NODE_LIST",
"GEOMETRIES_FWD_WEIGHT_LIST",
"GEOMETRIES_REV_WEIGHT_LIST",
"HSGR_CHECKSUM",
"TIMESTAMP",
"FILE_INDEX_PATH",
@ -70,7 +72,9 @@ struct SharedDataLayout
ENTRY_CLASSID,
R_SEARCH_TREE,
GEOMETRIES_INDEX,
GEOMETRIES_LIST,
GEOMETRIES_NODE_LIST,
GEOMETRIES_FWD_WEIGHT_LIST,
GEOMETRIES_REV_WEIGHT_LIST,
HSGR_CHECKSUM,
TIMESTAMP,
FILE_INDEX_PATH,

63
src/contractor/contractor.cpp
View File

@ -488,7 +488,9 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
std::vector<extractor::QueryNode> internal_to_external_node_map;
std::vector<unsigned> m_geometry_indices;
std::vector<extractor::CompressedEdgeContainer::CompressedEdge> m_geometry_list;
std::vector<NodeID> m_geometry_node_list;
std::vector<EdgeWeight> m_geometry_fwd_weight_list;
std::vector<EdgeWeight> m_geometry_rev_weight_list;
const auto maybe_load_internal_to_external_node_map = [&] {
if (!(update_edge_weights || update_turn_penalties))
@ -534,13 +536,23 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
geometry_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned));
BOOST_ASSERT(m_geometry_indices.back() == number_of_compressed_geometries);
m_geometry_list.resize(number_of_compressed_geometries);
m_geometry_node_list.resize(number_of_compressed_geometries);
m_geometry_fwd_weight_list.resize(number_of_compressed_geometries);
m_geometry_rev_weight_list.resize(number_of_compressed_geometries);
if (number_of_compressed_geometries > 0)
{
geometry_stream.read((char *)&(m_geometry_list[0]),
geometry_stream.read((char *)&(m_geometry_node_list[0]),
number_of_compressed_geometries *
sizeof(extractor::CompressedEdgeContainer::CompressedEdge));
sizeof(NodeID));
geometry_stream.read((char *)&(m_geometry_fwd_weight_list[0]),
number_of_compressed_geometries *
sizeof(EdgeWeight));
geometry_stream.read((char *)&(m_geometry_rev_weight_list[0]),
number_of_compressed_geometries *
sizeof(EdgeWeight));
}
};
@ -561,7 +573,7 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
// CSV file that supplied the value that gets used for that segment, then
// we write out this list so that it can be returned by the debugging
// vector tiles later on.
m_geometry_datasource.resize(m_geometry_list.size(), 0);
m_geometry_datasource.resize(m_geometry_fwd_weight_list.size(), 0);
// Now, we iterate over all the segments stored in the StaticRTree, updating
// the packed geometry weights in the `.geometries` file (note: we do not
@ -595,16 +607,14 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
const unsigned forward_begin =
m_geometry_indices.at(leaf_object.packed_geometry_id);
const auto current_segment =
&(m_geometry_list[forward_begin + leaf_object.fwd_segment_position]);
const auto current_fwd_weight =
m_geometry_fwd_weight_list[forward_begin + leaf_object.fwd_segment_position];
u = &(
internal_to_external_node_map[m_geometry_list[forward_begin +
leaf_object.fwd_segment_position]
.node_id]);
u = &(internal_to_external_node_map
[m_geometry_node_list[forward_begin +
leaf_object.fwd_segment_position]]);
v = &(internal_to_external_node_map
[m_geometry_list[forward_begin + leaf_object.fwd_segment_position + 1]
.node_id]);
[m_geometry_node_list[forward_begin + leaf_object.fwd_segment_position + 1]]);
const double segment_length = util::coordinate_calculation::greatCircleDistance(
util::Coordinate{u->lon, u->lat}, util::Coordinate{v->lon, v->lat});
@ -616,11 +626,11 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
const auto new_segment_weight = getNewWeight(forward_speed_iter,
segment_length,
segment_speed_filenames,
current_segment->forward_weight,
current_fwd_weight,
log_edge_updates_factor);
m_geometry_list[forward_begin + 1 + leaf_object.fwd_segment_position]
.forward_weight = new_segment_weight;
m_geometry_fwd_weight_list[forward_begin + 1 + leaf_object.fwd_segment_position] =
new_segment_weight;
m_geometry_datasource[forward_begin + 1 + leaf_object.fwd_segment_position] =
forward_speed_iter->speed_source.source;
@ -633,6 +643,9 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
counters[LUA_SOURCE] += 1;
}
const auto current_rev_weight =
m_geometry_rev_weight_list[forward_begin + leaf_object.fwd_segment_position];
const auto reverse_speed_iter =
find(segment_speed_lookup, Segment{v->node_id, u->node_id});
if (reverse_speed_iter != segment_speed_lookup.end())
@ -640,10 +653,10 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
const auto new_segment_weight = getNewWeight(reverse_speed_iter,
segment_length,
segment_speed_filenames,
current_segment->reverse_weight,
current_rev_weight,
log_edge_updates_factor);
m_geometry_list[forward_begin + leaf_object.fwd_segment_position]
.reverse_weight = new_segment_weight;
m_geometry_rev_weight_list[forward_begin + leaf_object.fwd_segment_position] =
new_segment_weight;
m_geometry_datasource[forward_begin + leaf_object.fwd_segment_position] =
reverse_speed_iter->speed_source.source;
@ -694,15 +707,21 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
throw util::exception("Failed to open " + geometry_filename + " for writing");
}
const unsigned number_of_indices = m_geometry_indices.size();
const unsigned number_of_compressed_geometries = m_geometry_list.size();
const unsigned number_of_compressed_geometries = m_geometry_node_list.size();
geometry_stream.write(reinterpret_cast<const char *>(&number_of_indices), sizeof(unsigned));
geometry_stream.write(reinterpret_cast<char *>(&(m_geometry_indices[0])),
number_of_indices * sizeof(unsigned));
geometry_stream.write(reinterpret_cast<const char *>(&number_of_compressed_geometries),
sizeof(unsigned));
geometry_stream.write(reinterpret_cast<char *>(&(m_geometry_list[0])),
geometry_stream.write(reinterpret_cast<char *>(&(m_geometry_node_list[0])),
number_of_compressed_geometries *
sizeof(extractor::CompressedEdgeContainer::CompressedEdge));
sizeof(NodeID));
geometry_stream.write(reinterpret_cast<char *>(&(m_geometry_fwd_weight_list[0])),
number_of_compressed_geometries *
sizeof(EdgeWeight));
geometry_stream.write(reinterpret_cast<char *>(&(m_geometry_rev_weight_list[0])),
number_of_compressed_geometries *
sizeof(EdgeWeight));
};
const auto save_datasource_indexes = [&] {

78
src/extractor/compressed_edge_container.cpp
View File

@ -61,7 +61,7 @@ unsigned CompressedEdgeContainer::GetZippedPositionForForwardID(const EdgeID edg
{
auto map_iterator = m_forward_edge_id_to_zipped_index_map.find(edge_id);
BOOST_ASSERT(map_iterator != m_forward_edge_id_to_zipped_index_map.end());
BOOST_ASSERT(map_iterator->second < m_compressed_geometries.size());
BOOST_ASSERT(map_iterator->second < m_compressed_geometry_nodes.size());
return map_iterator->second;
}
@ -69,48 +69,34 @@ unsigned CompressedEdgeContainer::GetZippedPositionForReverseID(const EdgeID edg
{
auto map_iterator = m_reverse_edge_id_to_zipped_index_map.find(edge_id);
BOOST_ASSERT(map_iterator != m_reverse_edge_id_to_zipped_index_map.end());
BOOST_ASSERT(map_iterator->second < m_compressed_geometries.size());
BOOST_ASSERT(map_iterator->second < m_compressed_geometry_nodes.size());
return map_iterator->second;
}
void CompressedEdgeContainer::SerializeInternalVector(const std::string &path) const
{
boost::filesystem::fstream geometry_out_stream(path, std::ios::binary | std::ios::out);
const unsigned compressed_geometries = m_compressed_geometries.size() + 1;
BOOST_ASSERT(std::numeric_limits<unsigned>::max() != compressed_geometries);
geometry_out_stream.write((char *)&compressed_geometries, sizeof(unsigned));
const unsigned compressed_geometry_indices = m_compressed_geometry_index.size() + 1;
const unsigned compressed_geometries = m_compressed_geometry_nodes.size();
BOOST_ASSERT(std::numeric_limits<unsigned>::max() != compressed_geometry_indices);
geometry_out_stream.write((char *)&compressed_geometry_indices, sizeof(unsigned));
// write indices array
unsigned prefix_sum_of_list_indices = 0;
for (const auto &elem : m_compressed_geometries)
{
geometry_out_stream.write((char *)&prefix_sum_of_list_indices, sizeof(unsigned));
geometry_out_stream.write((char *)(m_compressed_geometry_index.data()), sizeof(unsigned) * m_compressed_geometry_index.size());
const std::vector<CompressedEdge> &current_vector = elem;
const unsigned unpacked_size = current_vector.size();
BOOST_ASSERT(std::numeric_limits<unsigned>::max() != unpacked_size);
prefix_sum_of_list_indices += unpacked_size;
}
// sentinel element
geometry_out_stream.write((char *)&prefix_sum_of_list_indices, sizeof(unsigned));
geometry_out_stream.write((char *)&(compressed_geometries), sizeof(unsigned));
// number of geometry entries to follow, it is the (inclusive) prefix sum
geometry_out_stream.write((char *)&prefix_sum_of_list_indices, sizeof(unsigned));
// number of geometry entries to follow
geometry_out_stream.write((char *)&(compressed_geometries), sizeof(unsigned));
unsigned control_sum = 0;
// write compressed geometries
for (auto &elem : m_compressed_geometries)
{
const std::vector<CompressedEdge> &current_vector = elem;
const unsigned unpacked_size = current_vector.size();
control_sum += unpacked_size;
BOOST_ASSERT(std::numeric_limits<unsigned>::max() != unpacked_size);
for (const auto &current_node : current_vector)
{
geometry_out_stream.write((char *)&(current_node), sizeof(CompressedEdge));
}
}
BOOST_ASSERT(control_sum == prefix_sum_of_list_indices);
// write compressed geometry node id's
geometry_out_stream.write((char *)(m_compressed_geometry_nodes.data()), sizeof(NodeID) * m_compressed_geometry_nodes.size());
// write compressed geometry forward weights
geometry_out_stream.write((char *)(m_compressed_geometry_fwd_weights.data()), sizeof(EdgeWeight) * m_compressed_geometry_fwd_weights.size());
// write compressed geometry reverse weights
geometry_out_stream.write((char *)(m_compressed_geometry_rev_weights.data()), sizeof(EdgeWeight) * m_compressed_geometry_rev_weights.size());
}
// Adds info for a compressed edge to the container. edge_id_2
@ -250,7 +236,10 @@ void CompressedEdgeContainer::AddUncompressedEdge(const EdgeID edge_id,
void CompressedEdgeContainer::InitializeBothwayVector()
{
m_compressed_geometries.reserve(m_compressed_oneway_geometries.size() / 2);
m_compressed_geometry_index.reserve(m_compressed_oneway_geometries.size());
m_compressed_geometry_nodes.reserve(m_compressed_oneway_geometries.size());
m_compressed_geometry_fwd_weights.reserve(m_compressed_oneway_geometries.size());
m_compressed_geometry_rev_weights.reserve(m_compressed_oneway_geometries.size());
}
unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID r_edge_id)
@ -260,14 +249,17 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID
BOOST_ASSERT(forward_bucket.size() == reverse_bucket.size());
const unsigned zipped_geometry_id = m_compressed_geometries.size();
const unsigned zipped_geometry_id = m_compressed_geometry_index.size();
m_forward_edge_id_to_zipped_index_map[f_edge_id] = zipped_geometry_id;
m_reverse_edge_id_to_zipped_index_map[r_edge_id] = zipped_geometry_id;
std::vector<CompressedEdge> zipped_edge_bucket;
m_compressed_geometry_index.emplace_back(m_compressed_geometry_nodes.size());
const auto &first_node = reverse_bucket.back();
zipped_edge_bucket.emplace_back(
CompressedEdge{first_node.node_id, INVALID_EDGE_WEIGHT, first_node.weight});
m_compressed_geometry_nodes.emplace_back(first_node.node_id);
m_compressed_geometry_fwd_weights.emplace_back(INVALID_EDGE_WEIGHT);
m_compressed_geometry_rev_weights.emplace_back(first_node.weight);
for (std::size_t i = 0; i < forward_bucket.size() - 1; ++i)
{
@ -276,14 +268,16 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID
BOOST_ASSERT(fwd_node.node_id == rev_node.node_id);
zipped_edge_bucket.emplace_back(
CompressedEdge{fwd_node.node_id, fwd_node.weight, rev_node.weight});
m_compressed_geometry_nodes.emplace_back(fwd_node.node_id);
m_compressed_geometry_fwd_weights.emplace_back(fwd_node.weight);
m_compressed_geometry_rev_weights.emplace_back(rev_node.weight);
}
const auto &last_node = forward_bucket.back();
zipped_edge_bucket.emplace_back(
CompressedEdge{last_node.node_id, last_node.weight, INVALID_EDGE_WEIGHT});
m_compressed_geometries.emplace_back(std::move(zipped_edge_bucket));
m_compressed_geometry_nodes.emplace_back(last_node.node_id);
m_compressed_geometry_fwd_weights.emplace_back(last_node.weight);
m_compressed_geometry_rev_weights.emplace_back(INVALID_EDGE_WEIGHT);
return zipped_geometry_id;
}

48
src/storage/storage.cpp
View File

@ -329,8 +329,12 @@ Storage::ReturnCode Storage::Run(int max_wait)
boost::iostreams::seek(
geometry_input_stream, number_of_geometries_indices * sizeof(unsigned), BOOST_IOS::cur);
geometry_input_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned));
shared_layout_ptr->SetBlockSize<extractor::CompressedEdgeContainer::CompressedEdge>(
SharedDataLayout::GEOMETRIES_LIST, number_of_compressed_geometries);
shared_layout_ptr->SetBlockSize<NodeID>(
SharedDataLayout::GEOMETRIES_NODE_LIST, number_of_compressed_geometries);
shared_layout_ptr->SetBlockSize<EdgeWeight>(
SharedDataLayout::GEOMETRIES_FWD_WEIGHT_LIST, number_of_compressed_geometries);
shared_layout_ptr->SetBlockSize<EdgeWeight>(
SharedDataLayout::GEOMETRIES_REV_WEIGHT_LIST, number_of_compressed_geometries);
// load datasource sizes. This file is optional, and it's non-fatal if it doesn't
// exist.
@ -601,19 +605,45 @@ Storage::ReturnCode Storage::Run(int max_wait)
(char *)geometries_index_ptr,
shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_INDEX));
}
extractor::CompressedEdgeContainer::CompressedEdge *geometries_list_ptr =
shared_layout_ptr->GetBlockPtr<extractor::CompressedEdgeContainer::CompressedEdge, true>(
shared_memory_ptr, SharedDataLayout::GEOMETRIES_LIST);
NodeID *geometries_node_id_list_ptr =
shared_layout_ptr->GetBlockPtr<NodeID, true>(
shared_memory_ptr, SharedDataLayout::GEOMETRIES_NODE_LIST);
geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned));
BOOST_ASSERT(temporary_value ==
shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_LIST]);
shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_NODE_LIST]);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_LIST) > 0)
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_NODE_LIST) > 0)
{
geometry_input_stream.read(
(char *)geometries_list_ptr,
shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_LIST));
(char *)geometries_node_id_list_ptr,
shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_NODE_LIST));
}
EdgeWeight *geometries_fwd_weight_list_ptr =
shared_layout_ptr->GetBlockPtr<EdgeWeight, true>(
shared_memory_ptr, SharedDataLayout::GEOMETRIES_FWD_WEIGHT_LIST);
BOOST_ASSERT(temporary_value ==
shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_FWD_WEIGHT_LIST]);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_FWD_WEIGHT_LIST) > 0)
{
geometry_input_stream.read(
(char *)geometries_fwd_weight_list_ptr,
shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_FWD_WEIGHT_LIST));
}
EdgeWeight *geometries_rev_weight_list_ptr =
shared_layout_ptr->GetBlockPtr<EdgeWeight, true>(
shared_memory_ptr, SharedDataLayout::GEOMETRIES_REV_WEIGHT_LIST);
BOOST_ASSERT(temporary_value ==
shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_REV_WEIGHT_LIST]);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_REV_WEIGHT_LIST) > 0)
{
geometry_input_stream.read(
(char *)geometries_rev_weight_list_ptr,
shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_REV_WEIGHT_LIST));
}
// load datasource information (if it exists)