Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

reformat geometry compressor according to new guidelines

Browse Source
This commit is contained in:
Dennis Luxen 2014-04-11 16:25:25 -04:00
parent 4f85fd28cf
commit a4d6e5c9cc
2 changed files with 80 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

149
Contractor/GeometryCompressor.cpp
View File

@ -29,13 +29,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "../Util/SimpleLogger.h" #include "../Util/SimpleLogger.h"
#include <boost/assert.hpp> #include <boost/assert.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/foreach.hpp> #include <boost/foreach.hpp>
#include <fstream>
#include <limits> #include <limits>
int current_free_list_maximum = 0; int current_free_list_maximum = 0;
int UniqueNumber () { return ++current_free_list_maximum; } int UniqueNumber() { return ++current_free_list_maximum; }
GeometryCompressor::GeometryCompressor() GeometryCompressor::GeometryCompressor()
{ {
@ -46,7 +47,7 @@ GeometryCompressor::GeometryCompressor()
void GeometryCompressor::IncreaseFreeList() void GeometryCompressor::IncreaseFreeList()
{ {
m_compressed_geometries.resize(m_compressed_geometries.size() + 100); m_compressed_geometries.resize(m_compressed_geometries.size() + 100);
for(unsigned i = 100; i > 0; --i) for (unsigned i = 100; i > 0; --i)
{ {
m_free_list.push_back(current_free_list_maximum); m_free_list.push_back(current_free_list_maximum);
++current_free_list_maximum; ++current_free_list_maximum;
@ -62,88 +63,68 @@ unsigned GeometryCompressor::GetPositionForID(const EdgeID edge_id) const
{ {
boost::unordered_map<EdgeID, unsigned>::const_iterator map_iterator; boost::unordered_map<EdgeID, unsigned>::const_iterator map_iterator;
map_iterator = m_edge_id_to_list_index_map.find(edge_id); map_iterator = m_edge_id_to_list_index_map.find(edge_id);
BOOST_ASSERT( map_iterator != m_edge_id_to_list_index_map.end() ); BOOST_ASSERT(map_iterator != m_edge_id_to_list_index_map.end());
BOOST_ASSERT( map_iterator->second < m_compressed_geometries.size() ); BOOST_ASSERT(map_iterator->second < m_compressed_geometries.size());
return map_iterator->second; return map_iterator->second;
} }
void GeometryCompressor::SerializeInternalVector(const std::string & path) const void GeometryCompressor::SerializeInternalVector(const std::string &path) const
{ {
std::ofstream geometry_out_stream( path.c_str(), std::ios::binary ); boost::filesystem::fstream geometry_out_stream(path, std::ios::binary|std::ios::out);
const unsigned number_of_compressed_geometries = m_compressed_geometries.size()+1; const unsigned number_of_compressed_geometries = m_compressed_geometries.size() + 1;
BOOST_ASSERT( UINT_MAX != number_of_compressed_geometries ); BOOST_ASSERT(UINT_MAX != number_of_compressed_geometries);
geometry_out_stream.write( geometry_out_stream.write((char *)&number_of_compressed_geometries, sizeof(unsigned));
(char*)&number_of_compressed_geometries,
sizeof(unsigned)
);
SimpleLogger().Write(logDEBUG) << "number_of_compressed_geometries: " << number_of_compressed_geometries;
// write indices array // write indices array
unsigned prefix_sum_of_list_indices = 0; unsigned prefix_sum_of_list_indices = 0;
for (unsigned i = 0; i < m_compressed_geometries.size(); ++i) for (unsigned i = 0; i < m_compressed_geometries.size(); ++i)
{ {
geometry_out_stream.write( geometry_out_stream.write((char *)&prefix_sum_of_list_indices, sizeof(unsigned));
(char*)&prefix_sum_of_list_indices,
sizeof(unsigned)
);
const std::vector<CompressedNode> & current_vector = m_compressed_geometries.at(i); const std::vector<CompressedNode> &current_vector = m_compressed_geometries.at(i);
const unsigned unpacked_size = current_vector.size(); const unsigned unpacked_size = current_vector.size();
BOOST_ASSERT( UINT_MAX != unpacked_size ); BOOST_ASSERT(UINT_MAX != unpacked_size);
prefix_sum_of_list_indices += unpacked_size; prefix_sum_of_list_indices += unpacked_size;
} }
// sentinel element // sentinel element
geometry_out_stream.write( geometry_out_stream.write((char *)&prefix_sum_of_list_indices, sizeof(unsigned));
(char*)&prefix_sum_of_list_indices,
sizeof(unsigned)
);
// number of geometry entries to follow, it is the (inclusive) prefix sum // number of geometry entries to follow, it is the (inclusive) prefix sum
geometry_out_stream.write( geometry_out_stream.write((char *)&prefix_sum_of_list_indices, sizeof(unsigned));
(char*)&prefix_sum_of_list_indices,
sizeof(unsigned)
);
SimpleLogger().Write(logDEBUG) << "number of geometry nodes: " << prefix_sum_of_list_indices;
unsigned control_sum = 0; unsigned control_sum = 0;
// write compressed geometries // write compressed geometries
for (unsigned i = 0; i < m_compressed_geometries.size(); ++i) for (unsigned i = 0; i < m_compressed_geometries.size(); ++i)
{ {
const std::vector<CompressedNode> & current_vector = m_compressed_geometries[i]; const std::vector<CompressedNode> &current_vector = m_compressed_geometries[i];
const unsigned unpacked_size = current_vector.size(); const unsigned unpacked_size = current_vector.size();
control_sum += unpacked_size; control_sum += unpacked_size;
BOOST_ASSERT( UINT_MAX != unpacked_size ); BOOST_ASSERT(UINT_MAX != unpacked_size);
BOOST_FOREACH (const CompressedNode current_node, current_vector) BOOST_FOREACH (const CompressedNode current_node, current_vector)
{ {
geometry_out_stream.write( geometry_out_stream.write((char *)&(current_node.first), sizeof(NodeID));
(char*)&(current_node.first),
sizeof(NodeID)
);
} }
} }
BOOST_ASSERT( control_sum == prefix_sum_of_list_indices ); BOOST_ASSERT(control_sum == prefix_sum_of_list_indices);
// all done, let's close the resource // all done, let's close the resource
geometry_out_stream.close(); geometry_out_stream.close();
} }
void GeometryCompressor::CompressEdge( void GeometryCompressor::CompressEdge(const EdgeID edge_id_1,
const EdgeID edge_id_1, const EdgeID edge_id_2,
const EdgeID edge_id_2, const NodeID via_node_id,
const NodeID via_node_id, const NodeID target_node_id,
const NodeID target_node_id, const EdgeWeight weight1,
const EdgeWeight weight1, const EdgeWeight weight2)
const EdgeWeight weight2 {
) { // remove super-trivial geometries
//TODO: remove super-trivial geometries BOOST_ASSERT(SPECIAL_EDGEID != edge_id_1);
BOOST_ASSERT(SPECIAL_EDGEID != edge_id_2);
BOOST_ASSERT( SPECIAL_EDGEID != edge_id_1 ); BOOST_ASSERT(SPECIAL_NODEID != via_node_id);
BOOST_ASSERT( SPECIAL_EDGEID != edge_id_2 ); BOOST_ASSERT(SPECIAL_NODEID != target_node_id);
BOOST_ASSERT( SPECIAL_NODEID != via_node_id ); BOOST_ASSERT(std::numeric_limits<unsigned>::max() != weight1);
BOOST_ASSERT( SPECIAL_NODEID != target_node_id ); BOOST_ASSERT(std::numeric_limits<unsigned>::max() != weight2);
BOOST_ASSERT( std::numeric_limits<unsigned>::max() != weight1 );
BOOST_ASSERT( std::numeric_limits<unsigned>::max() != weight2 );
// append list of removed edge_id plus via node to surviving edge id: // append list of removed edge_id plus via node to surviving edge id:
// <surv_1, .. , surv_n, via_node_id, rem_1, .. rem_n // <surv_1, .. , surv_n, via_node_id, rem_1, .. rem_n
@ -162,53 +143,52 @@ void GeometryCompressor::CompressEdge(
// SimpleLogger().Write() << "increased free list"; // SimpleLogger().Write() << "increased free list";
IncreaseFreeList(); IncreaseFreeList();
} }
BOOST_ASSERT( !m_free_list.empty() ); BOOST_ASSERT(!m_free_list.empty());
// SimpleLogger().Write() << "free list size: " << m_free_list.size(); // SimpleLogger().Write() << "free list size: " << m_free_list.size();
m_edge_id_to_list_index_map[edge_id_1] = m_free_list.back(); m_edge_id_to_list_index_map[edge_id_1] = m_free_list.back();
m_free_list.pop_back(); m_free_list.pop_back();
} }
const unsigned edge_bucket_id1 = m_edge_id_to_list_index_map[edge_id_1]; const unsigned edge_bucket_id1 = m_edge_id_to_list_index_map[edge_id_1];
BOOST_ASSERT( edge_bucket_id1 == GetPositionForID(edge_id_1)); BOOST_ASSERT(edge_bucket_id1 == GetPositionForID(edge_id_1));
BOOST_ASSERT( edge_bucket_id1 < m_compressed_geometries.size() ); BOOST_ASSERT(edge_bucket_id1 < m_compressed_geometries.size());
std::vector<CompressedNode> & edge_bucket_list1 = m_compressed_geometries[edge_bucket_id1]; std::vector<CompressedNode> &edge_bucket_list1 = m_compressed_geometries[edge_bucket_id1];
if (edge_bucket_list1.empty()) if (edge_bucket_list1.empty())
{ {
edge_bucket_list1.push_back( std::make_pair(via_node_id, weight1) ); edge_bucket_list1.push_back(std::make_pair(via_node_id, weight1));
} }
BOOST_ASSERT( 0 < edge_bucket_list1.size() ); BOOST_ASSERT(0 < edge_bucket_list1.size());
BOOST_ASSERT( !edge_bucket_list1.empty() ); BOOST_ASSERT(!edge_bucket_list1.empty());
if (HasEntryForID(edge_id_2)) if (HasEntryForID(edge_id_2))
{ {
// second edge is not atomic anymore // second edge is not atomic anymore
const unsigned list_to_remove_index = GetPositionForID(edge_id_2); const unsigned list_to_remove_index = GetPositionForID(edge_id_2);
BOOST_ASSERT( list_to_remove_index < m_compressed_geometries.size() ); BOOST_ASSERT(list_to_remove_index < m_compressed_geometries.size());
std::vector<CompressedNode> & edge_bucket_list2 = m_compressed_geometries[list_to_remove_index]; std::vector<CompressedNode> &edge_bucket_list2 =
m_compressed_geometries[list_to_remove_index];
// found an existing list, append it to the list of edge_id_1 // found an existing list, append it to the list of edge_id_1
edge_bucket_list1.insert( edge_bucket_list1.insert(
edge_bucket_list1.end(), edge_bucket_list1.end(), edge_bucket_list2.begin(), edge_bucket_list2.end());
edge_bucket_list2.begin(),
edge_bucket_list2.end()
);
//remove the list of edge_id_2 // remove the list of edge_id_2
m_edge_id_to_list_index_map.erase(edge_id_2); m_edge_id_to_list_index_map.erase(edge_id_2);
BOOST_ASSERT( m_edge_id_to_list_index_map.end() == m_edge_id_to_list_index_map.find(edge_id_2) ); BOOST_ASSERT(m_edge_id_to_list_index_map.end() ==
m_edge_id_to_list_index_map.find(edge_id_2));
edge_bucket_list2.clear(); edge_bucket_list2.clear();
BOOST_ASSERT( 0 == edge_bucket_list2.size() ); BOOST_ASSERT(0 == edge_bucket_list2.size());
m_free_list.push_back(list_to_remove_index); m_free_list.push_back(list_to_remove_index);
BOOST_ASSERT( list_to_remove_index == m_free_list.back() ); BOOST_ASSERT(list_to_remove_index == m_free_list.back());
} }
else else
{ {
// we are certain that the second edge is atomic. // we are certain that the second edge is atomic.
edge_bucket_list1.push_back( std::make_pair(target_node_id, weight2) ); edge_bucket_list1.push_back(std::make_pair(target_node_id, weight2));
} }
} }
@ -216,28 +196,31 @@ void GeometryCompressor::PrintStatistics() const
{ {
const uint64_t compressed_edges = m_compressed_geometries.size(); const uint64_t compressed_edges = m_compressed_geometries.size();
BOOST_ASSERT(0 == compressed_edges % 2); BOOST_ASSERT(0 == compressed_edges % 2);
BOOST_ASSERT( m_compressed_geometries.size() + m_free_list.size() > 0 ); BOOST_ASSERT(m_compressed_geometries.size() + m_free_list.size() > 0);
uint64_t number_of_compressed_geometries = 0; uint64_t number_of_compressed_geometries = 0;
uint64_t longest_chain_length = 0; uint64_t longest_chain_length = 0;
BOOST_FOREACH(const std::vector<CompressedNode> & current_vector, m_compressed_geometries) BOOST_FOREACH (const std::vector<CompressedNode> &current_vector, m_compressed_geometries)
{ {
number_of_compressed_geometries += current_vector.size(); number_of_compressed_geometries += current_vector.size();
longest_chain_length = std::max(longest_chain_length, (uint64_t)current_vector.size()); longest_chain_length = std::max(longest_chain_length, (uint64_t)current_vector.size());
} }
SimpleLogger().Write() << "Geometry successfully removed:" SimpleLogger().Write() << "Geometry successfully removed:"
"\n compressed edges: " << compressed_edges << "\n compressed edges: " << compressed_edges
"\n compressed geometries: " << number_of_compressed_geometries << << "\n compressed geometries: " << number_of_compressed_geometries
"\n longest chain length: " << longest_chain_length << << "\n longest chain length: " << longest_chain_length
"\n cmpr ratio: " << ((float)compressed_edges/std::max(number_of_compressed_geometries, (uint64_t)1) ) << << "\n cmpr ratio: "
"\n avg chain length: " << (float)number_of_compressed_geometries/std::max((uint64_t)1, compressed_edges); << ((float)compressed_edges /
std::max(number_of_compressed_geometries, (uint64_t)1))
<< "\n avg chain length: "
<< (float)number_of_compressed_geometries /
std::max((uint64_t)1, compressed_edges);
} }
const std::vector<GeometryCompressor::CompressedNode> & GeometryCompressor::GetBucketReference( const std::vector<GeometryCompressor::CompressedNode> &
const EdgeID edge_id GeometryCompressor::GetBucketReference(const EdgeID edge_id) const
) const
{ {
const unsigned index = m_edge_id_to_list_index_map.at( edge_id ); const unsigned index = m_edge_id_to_list_index_map.at(edge_id);
return m_compressed_geometries.at( index ); return m_compressed_geometries.at(index);
} }

29
Contractor/GeometryCompressor.h
View File

@ -28,38 +28,37 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "../typedefs.h" #include "../typedefs.h"
#include <boost/unordered_map.hpp> #include <boost/unordered_map.hpp>
#include <boost/filesystem.hpp>
#include <vector> #include <vector>
#ifndef GEOMETRY_COMPRESSOR_H #ifndef GEOMETRY_COMPRESSOR_H
#define GEOMETRY_COMPRESSOR_H #define GEOMETRY_COMPRESSOR_H
class GeometryCompressor { class GeometryCompressor
public: {
public:
typedef std::pair<NodeID, EdgeWeight> CompressedNode; typedef std::pair<NodeID, EdgeWeight> CompressedNode;
GeometryCompressor(); GeometryCompressor();
void CompressEdge( void CompressEdge(const EdgeID surviving_edge_id,
const EdgeID surviving_edge_id, const EdgeID removed_edge_id,
const EdgeID removed_edge_id, const NodeID via_node_id,
const NodeID via_node_id, const NodeID target_node,
const NodeID target_node, const EdgeWeight weight1,
const EdgeWeight weight1, const EdgeWeight weight2);
const EdgeWeight weight2
);
bool HasEntryForID(const EdgeID edge_id) const; bool HasEntryForID(const EdgeID edge_id) const;
void PrintStatistics() const; void PrintStatistics() const;
void SerializeInternalVector(const std::string & path) const; void SerializeInternalVector(const std::string &path) const;
unsigned GetPositionForID(const EdgeID edge_id) const; unsigned GetPositionForID(const EdgeID edge_id) const;
const std::vector<GeometryCompressor::CompressedNode> & GetBucketReference(const EdgeID edge_id) const; const std::vector<GeometryCompressor::CompressedNode> &
GetBucketReference(const EdgeID edge_id) const;
private: private:
void IncreaseFreeList(); void IncreaseFreeList();
std::vector<std::vector<CompressedNode> > m_compressed_geometries; std::vector<std::vector<CompressedNode> > m_compressed_geometries;
std::vector<unsigned> m_free_list; std::vector<unsigned> m_free_list;
boost::unordered_map<EdgeID, unsigned> m_edge_id_to_list_index_map; boost::unordered_map<EdgeID, unsigned> m_edge_id_to_list_index_map;
}; };
#endif //GEOMETRY_COMPRESSOR_H #endif // GEOMETRY_COMPRESSOR_H