Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Refactor compressed geometry in own abstraction with read/write

Browse Source
This commit is contained in:
Patrick Niklaus 2017-03-09 00:06:06 +00:00 committed by Patrick Niklaus
parent 83820bf82c
commit 1b5ab37dfd
5 changed files with 228 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
include/extractor/compressed_edge_container.hpp
View File

@ -1,6 +1,8 @@
#ifndef GEOMETRY_COMPRESSOR_HPP_
#define GEOMETRY_COMPRESSOR_HPP_
#include "extractor/segment_data_container.hpp"
#include "util/typedefs.hpp"
#include <unordered_map>
@ -48,7 +50,6 @@ class CompressedEdgeContainer
bool HasZippedEntryForForwardID(const EdgeID edge_id) const;
bool HasZippedEntryForReverseID(const EdgeID edge_id) const;
void PrintStatistics() const;
void SerializeInternalVector(const std::string &path) const;
unsigned GetPositionForID(const EdgeID edge_id) const;
unsigned GetZippedPositionForForwardID(const EdgeID edge_id) const;
unsigned GetZippedPositionForReverseID(const EdgeID edge_id) const;
@ -58,21 +59,19 @@ class CompressedEdgeContainer
NodeID GetLastEdgeTargetID(const EdgeID edge_id) const;
NodeID GetLastEdgeSourceID(const EdgeID edge_id) const;
// Invalidates the internal storage
SegmentDataContainer ToSegmentData();
private:
int free_list_maximum = 0;
void IncreaseFreeList();
std::vector<OnewayEdgeBucket> m_compressed_oneway_geometries;
std::vector<unsigned> m_compressed_geometry_index;
std::vector<NodeID> m_compressed_geometry_nodes;
std::vector<EdgeWeight> m_compressed_geometry_fwd_weights;
std::vector<EdgeWeight> m_compressed_geometry_rev_weights;
std::vector<EdgeWeight> m_compressed_geometry_fwd_durations;
std::vector<EdgeWeight> m_compressed_geometry_rev_durations;
std::vector<unsigned> m_free_list;
std::unordered_map<EdgeID, unsigned> m_edge_id_to_list_index_map;
std::unordered_map<EdgeID, unsigned> m_forward_edge_id_to_zipped_index_map;
std::unordered_map<EdgeID, unsigned> m_reverse_edge_id_to_zipped_index_map;
SegmentDataContainer segment_data;
};
}
}

62
include/extractor/io.hpp Normal file
View File

@ -0,0 +1,62 @@
#ifndef OSRM_EXTRACTOR_IO_HPP
#define OSRM_EXTRACTOR_IO_HPP
#include "extractor/segment_data_container.hpp"
#include "storage/io.hpp"
namespace osrm
{
namespace extractor
{
namespace io
{
template <>
void read(const boost::filesystem::path &path, SegmentDataContainer &segment_data)
{
const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
storage::io::FileReader reader{path, fingerprint};
auto num_indices = reader.ReadElementCount32();
segment_data.index.resize(num_indices);
reader.ReadInto(segment_data.index.data(), num_indices);
auto num_entries = reader.ReadElementCount32();
segment_data.nodes.resize(num_entries);
segment_data.fwd_weights.resize(num_entries);
segment_data.rev_weights.resize(num_entries);
segment_data.fwd_durations.resize(num_entries);
segment_data.rev_durations.resize(num_entries);
reader.ReadInto(segment_data.nodes.data(), segment_data.nodes.size());
reader.ReadInto(segment_data.fwd_weights.data(), segment_data.fwd_weights.size());
reader.ReadInto(segment_data.rev_weights.data(), segment_data.rev_weights.size());
reader.ReadInto(segment_data.fwd_durations.data(), segment_data.fwd_durations.size());
reader.ReadInto(segment_data.rev_durations.data(), segment_data.rev_durations.size());
}
template <> void write(const boost::filesystem::path &path, const SegmentDataContainer &segment_data)
{
const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
storage::io::FileWriter writer{path, fingerprint};
// FIXME this _should_ just be size and the senitel below need to be removed
writer.WriteElementCount32(segment_data.index.size() + 1);
writer.WriteFrom(segment_data.index.data(), segment_data.index.size());
// FIMXE remove unnecessary senitel
writer.WriteElementCount32(segment_data.nodes.size());
writer.WriteElementCount32(segment_data.nodes.size());
writer.WriteFrom(segment_data.nodes.data(), segment_data.nodes.size());
writer.WriteFrom(segment_data.fwd_weights.data(), segment_data.fwd_weights.size());
writer.WriteFrom(segment_data.rev_weights.data(), segment_data.rev_weights.size());
writer.WriteFrom(segment_data.fwd_durations.data(), segment_data.fwd_durations.size());
writer.WriteFrom(segment_data.rev_durations.data(), segment_data.rev_durations.size());
}
}
}
}
#endif

126
include/extractor/segment_data_container.hpp Normal file
View File

@ -0,0 +1,126 @@
#ifndef OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_
#define OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/typedefs.hpp"
#include <boost/range/adaptor/reversed.hpp>
#include <boost/range/iterator_range.hpp>
#include <boost/filesystem/path.hpp>
#include <unordered_map>
#include <string>
#include <vector>
namespace osrm
{
namespace extractor
{
class CompressedEdgeContainer;
namespace detail
{
template <bool UseShareMemory> class SegmentDataContainerImpl;
}
namespace io
{
template <bool UseShareMemory>
inline void read(const boost::filesystem::path &path, detail::SegmentDataContainerImpl<UseShareMemory> &segment_data);
template <bool UseShareMemory>
inline void write(const boost::filesystem::path &path, const detail::SegmentDataContainerImpl<UseShareMemory> &segment_data);
}
namespace detail
{
template <bool UseShareMemory> class SegmentDataContainerImpl
{
template <typename T> using Vector = typename util::ShM<T, UseShareMemory>::vector;
friend CompressedEdgeContainer;
public:
using SegmentID = std::uint32_t;
SegmentDataContainerImpl() = default;
SegmentDataContainerImpl(Vector<std::uint32_t> index_,
Vector<NodeID> nodes_,
Vector<EdgeWeight> fwd_weights_,
Vector<EdgeWeight> rev_weights_,
Vector<EdgeWeight> fwd_durations_,
Vector<EdgeWeight> rev_durations_)
: index(std::move(index_)), nodes(std::move(nodes_)), fwd_weights(std::move(fwd_weights_)),
rev_weights(std::move(rev_weights_)), fwd_durations(std::move(fwd_durations_)),
rev_durations(std::move(rev_durations_))
{
}
auto GetForwardGeometry(const SegmentID id) const
{
const auto begin = nodes.begin() + index.at(id);
const auto end = nodes.begin() + index.at(id + 1);
return boost::make_iterator_range(begin, end);
}
auto GetReverseGeometry(const SegmentID id) const
{
return boost::adaptors::reverse(GetForwardGeometry(id));
}
auto GetForwardDurations(const SegmentID id) const
{
const auto begin = fwd_durations.begin() + index.at(id) + 1;
const auto end = fwd_durations.begin() + index.at(id + 1);
return boost::make_iterator_range(begin, end);
}
auto GetReverseDurations(const SegmentID id) const
{
const auto begin = rev_durations.begin() + index.at(id);
const auto end = rev_durations.begin() + index.at(id + 1) - 1;
return boost::adaptors::reverse(boost::make_iterator_range(begin, end));
}
auto GetForwardWeights(const SegmentID id) const
{
const auto begin = fwd_weights.begin() + index.at(id) + 1;
const auto end = fwd_weights.begin() + index.at(id + 1);
return boost::make_iterator_range(begin, end);
}
auto GetReverseWeights(const SegmentID id) const
{
const auto begin = rev_weights.begin() + index.at(id);
const auto end = rev_weights.begin() + index.at(id + 1) - 1;
return boost::adaptors::reverse(boost::make_iterator_range(begin, end));
}
friend void io::read<UseShareMemory>(const boost::filesystem::path &path,
detail::SegmentDataContainerImpl<UseShareMemory> &segment_data);
friend void io::write<UseShareMemory>(const boost::filesystem::path &path,
const detail::SegmentDataContainerImpl<UseShareMemory> &segment_data);
private:
Vector<std::uint32_t> index;
Vector<NodeID> nodes;
Vector<EdgeWeight> fwd_weights;
Vector<EdgeWeight> rev_weights;
Vector<EdgeWeight> fwd_durations;
Vector<EdgeWeight> rev_durations;
};
}
using SegmentDataView = detail::SegmentDataContainerImpl<true>;
using SegmentDataContainer = detail::SegmentDataContainerImpl<false>;
}
}
#endif

94
src/extractor/compressed_edge_container.cpp
View File

@ -61,7 +61,7 @@ unsigned CompressedEdgeContainer::GetZippedPositionForForwardID(const EdgeID edg
{
auto map_iterator = m_forward_edge_id_to_zipped_index_map.find(edge_id);
BOOST_ASSERT(map_iterator != m_forward_edge_id_to_zipped_index_map.end());
BOOST_ASSERT(map_iterator->second < m_compressed_geometry_nodes.size());
BOOST_ASSERT(map_iterator->second < segment_data.nodes.size());
return map_iterator->second;
}
@ -69,48 +69,10 @@ unsigned CompressedEdgeContainer::GetZippedPositionForReverseID(const EdgeID edg
{
auto map_iterator = m_reverse_edge_id_to_zipped_index_map.find(edge_id);
BOOST_ASSERT(map_iterator != m_reverse_edge_id_to_zipped_index_map.end());
BOOST_ASSERT(map_iterator->second < m_compressed_geometry_nodes.size());
BOOST_ASSERT(map_iterator->second < segment_data.nodes.size());
return map_iterator->second;
}
void CompressedEdgeContainer::SerializeInternalVector(const std::string &path) const
{
boost::filesystem::fstream geometry_out_stream(path, std::ios::binary | std::ios::out);
const unsigned compressed_geometry_indices = m_compressed_geometry_index.size() + 1;
const unsigned compressed_geometries = m_compressed_geometry_nodes.size();
BOOST_ASSERT(std::numeric_limits<unsigned>::max() != compressed_geometry_indices);
geometry_out_stream.write((char *)&compressed_geometry_indices, sizeof(unsigned));
geometry_out_stream.write((char *)(m_compressed_geometry_index.data()),
sizeof(unsigned) * m_compressed_geometry_index.size());
// sentinel element
geometry_out_stream.write((char *)&(compressed_geometries), sizeof(unsigned));
// number of geometry entries to follow
geometry_out_stream.write((char *)&(compressed_geometries), sizeof(unsigned));
// write compressed geometry node id's
geometry_out_stream.write((char *)(m_compressed_geometry_nodes.data()),
sizeof(NodeID) * m_compressed_geometry_nodes.size());
// write compressed geometry forward weights
geometry_out_stream.write((char *)(m_compressed_geometry_fwd_weights.data()),
sizeof(EdgeWeight) * m_compressed_geometry_fwd_weights.size());
// write compressed geometry reverse weights
geometry_out_stream.write((char *)(m_compressed_geometry_rev_weights.data()),
sizeof(EdgeWeight) * m_compressed_geometry_rev_weights.size());
// write compressed geometry forward durations
geometry_out_stream.write((char *)(m_compressed_geometry_fwd_durations.data()),
sizeof(EdgeWeight) * m_compressed_geometry_fwd_durations.size());
// write compressed geometry reverse durations
geometry_out_stream.write((char *)(m_compressed_geometry_rev_durations.data()),
sizeof(EdgeWeight) * m_compressed_geometry_rev_durations.size());
}
// Adds info for a compressed edge to the container. edge_id_2
// has been removed from the graph, so we have to save These edges/nodes
// have already been trimmed from the graph, this function just stores
@ -252,12 +214,12 @@ void CompressedEdgeContainer::AddUncompressedEdge(const EdgeID edge_id,
void CompressedEdgeContainer::InitializeBothwayVector()
{
m_compressed_geometry_index.reserve(m_compressed_oneway_geometries.size());
m_compressed_geometry_nodes.reserve(m_compressed_oneway_geometries.size());
m_compressed_geometry_fwd_weights.reserve(m_compressed_oneway_geometries.size());
m_compressed_geometry_rev_weights.reserve(m_compressed_oneway_geometries.size());
m_compressed_geometry_fwd_durations.reserve(m_compressed_oneway_geometries.size());
m_compressed_geometry_rev_durations.reserve(m_compressed_oneway_geometries.size());
segment_data.index.reserve(m_compressed_oneway_geometries.size());
segment_data.nodes.reserve(m_compressed_oneway_geometries.size());
segment_data.fwd_weights.reserve(m_compressed_oneway_geometries.size());
segment_data.rev_weights.reserve(m_compressed_oneway_geometries.size());
segment_data.fwd_durations.reserve(m_compressed_oneway_geometries.size());
segment_data.rev_durations.reserve(m_compressed_oneway_geometries.size());
}
unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID r_edge_id)
@ -267,19 +229,19 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID
BOOST_ASSERT(forward_bucket.size() == reverse_bucket.size());
const unsigned zipped_geometry_id = m_compressed_geometry_index.size();
const unsigned zipped_geometry_id = segment_data.index.size();
m_forward_edge_id_to_zipped_index_map[f_edge_id] = zipped_geometry_id;
m_reverse_edge_id_to_zipped_index_map[r_edge_id] = zipped_geometry_id;
m_compressed_geometry_index.emplace_back(m_compressed_geometry_nodes.size());
segment_data.index.emplace_back(segment_data.nodes.size());
const auto &first_node = reverse_bucket.back();
m_compressed_geometry_nodes.emplace_back(first_node.node_id);
m_compressed_geometry_fwd_weights.emplace_back(INVALID_EDGE_WEIGHT);
m_compressed_geometry_rev_weights.emplace_back(first_node.weight);
m_compressed_geometry_fwd_durations.emplace_back(INVALID_EDGE_WEIGHT);
m_compressed_geometry_rev_durations.emplace_back(first_node.duration);
segment_data.nodes.emplace_back(first_node.node_id);
segment_data.fwd_weights.emplace_back(INVALID_EDGE_WEIGHT);
segment_data.rev_weights.emplace_back(first_node.weight);
segment_data.fwd_durations.emplace_back(INVALID_EDGE_WEIGHT);
segment_data.rev_durations.emplace_back(first_node.duration);
for (std::size_t i = 0; i < forward_bucket.size() - 1; ++i)
{
@ -288,20 +250,20 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID
BOOST_ASSERT(fwd_node.node_id == rev_node.node_id);
m_compressed_geometry_nodes.emplace_back(fwd_node.node_id);
m_compressed_geometry_fwd_weights.emplace_back(fwd_node.weight);
m_compressed_geometry_rev_weights.emplace_back(rev_node.weight);
m_compressed_geometry_fwd_durations.emplace_back(fwd_node.duration);
m_compressed_geometry_rev_durations.emplace_back(rev_node.duration);
segment_data.nodes.emplace_back(fwd_node.node_id);
segment_data.fwd_weights.emplace_back(fwd_node.weight);
segment_data.rev_weights.emplace_back(rev_node.weight);
segment_data.fwd_durations.emplace_back(fwd_node.duration);
segment_data.rev_durations.emplace_back(rev_node.duration);
}
const auto &last_node = forward_bucket.back();
m_compressed_geometry_nodes.emplace_back(last_node.node_id);
m_compressed_geometry_fwd_weights.emplace_back(last_node.weight);
m_compressed_geometry_rev_weights.emplace_back(INVALID_EDGE_WEIGHT);
m_compressed_geometry_fwd_durations.emplace_back(last_node.duration);
m_compressed_geometry_rev_durations.emplace_back(INVALID_EDGE_WEIGHT);
segment_data.nodes.emplace_back(last_node.node_id);
segment_data.fwd_weights.emplace_back(last_node.weight);
segment_data.rev_weights.emplace_back(INVALID_EDGE_WEIGHT);
segment_data.fwd_durations.emplace_back(last_node.duration);
segment_data.rev_durations.emplace_back(INVALID_EDGE_WEIGHT);
return zipped_geometry_id;
}
@ -364,5 +326,11 @@ NodeID CompressedEdgeContainer::GetLastEdgeSourceID(const EdgeID edge_id) const
BOOST_ASSERT(bucket.size() >= 2);
return bucket[bucket.size() - 2].node_id;
}
SegmentDataContainer CompressedEdgeContainer::ToSegmentData()
{
return std::move(segment_data);
}
}
}

5
src/extractor/extractor.cpp
View File

@ -5,10 +5,11 @@
#include "extractor/extraction_node.hpp"
#include "extractor/extraction_way.hpp"
#include "extractor/extractor_callbacks.hpp"
#include "extractor/io.hpp"
#include "extractor/raster_source.hpp"
#include "extractor/restriction_parser.hpp"
#include "extractor/scripting_environment.hpp"
#include "extractor/raster_source.hpp"
#include "storage/io.hpp"
#include "util/exception.hpp"
#include "util/exception_utils.hpp"
@ -513,7 +514,7 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment,
});
WriteTurnLaneData(config.turn_lane_descriptions_file_name);
compressed_edge_container.SerializeInternalVector(config.geometry_output_path);
io::write(config.geometry_output_path, compressed_edge_container.ToSegmentData());
edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list);
edge_based_graph_factory.GetEdgeBasedNodes(node_based_edge_list);