Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

implement loading of compressed geometries

Browse Source
This commit is contained in:
Dennis Luxen 2014-04-17 16:59:08 +02:00
parent 7a6a5f6612
commit 9b3dab8055
5 changed files with 75 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
DataStructures/SharedMemoryVectorWrapper.h
View File

@ -140,13 +140,15 @@ public:
SharedMemoryWrapper() :
m_ptr(NULL),
m_size(0)
{ }
{
SimpleLogger().Write(logDEBUG) << "generated std::vector<bool> of size " << m_size;
}
SharedMemoryWrapper(unsigned * ptr, std::size_t size) :
m_ptr(ptr),
m_size(size)
{
SimpleLogger().Write(logDEBUG) << "generated std::vector<bool> of size " << size;
SimpleLogger().Write(logDEBUG) << "generated std::vector<bool> of size " << m_size;
}
void swap( SharedMemoryWrapper<bool> & other ) {

16
Descriptors/DescriptionFactory.cpp
View File

@ -68,10 +68,10 @@ double DescriptionFactory::GetBearing(const FixedPointCoordinate & A, const Fixe
void DescriptionFactory::SetStartSegment(const PhantomNode & source, const bool source_traversed_in_reverse)
{
int fwd_weight = source.forward_weight;
int rev_weight = source.reverse_weight;
int fwd_offset = source.forward_offset;
int rev_offset = source.reverse_offset;
// int fwd_weight = source.forward_weight;
// int rev_weight = source.reverse_weight;
// int fwd_offset = source.forward_offset;
// int rev_offset = source.reverse_offset;
// SimpleLogger().Write(logDEBUG) << "df source, traversed in reverse: " << (source_traversed_in_reverse ? "y" : "n") << ", location: " << source.location << ", fwd_weight: " << fwd_weight << ", fwd_offset: " << fwd_offset << ", rev_weight: " << rev_weight << ", rev_offset: " << rev_offset;
// SimpleLogger().Write(logDEBUG) << "duration of first segment: " << (source_traversed_in_reverse ? source.GetReverseWeightPlusOffset() : source.GetForwardWeightPlusOffset());
start_phantom = source;
@ -83,10 +83,10 @@ void DescriptionFactory::SetStartSegment(const PhantomNode & source, const bool
void DescriptionFactory::SetEndSegment(const PhantomNode & target, const bool target_traversed_in_reverse)
{
int fwd_weight = target.forward_weight;
int rev_weight = target.reverse_weight;
int fwd_offset = target.forward_offset;
int rev_offset = target.reverse_offset;
// int fwd_weight = target.forward_weight;
// int rev_weight = target.reverse_weight;
// int fwd_offset = target.forward_offset;
// int rev_offset = target.reverse_offset;
// SimpleLogger().Write(logDEBUG) << "df target, traversed in reverse: " << (target_traversed_in_reverse ? "y" : "n") << ", location: " << target.location << ", fwd_weight: " << fwd_weight << ", fwd_offset: " << fwd_offset << ", rev_weight: " << rev_weight << ", rev_offset: " << rev_offset;
// SimpleLogger().Write(logDEBUG) << "duration of last segment: " << (target_traversed_in_reverse ? target.GetReverseWeightPlusOffset() : target.GetForwardWeightPlusOffset());

4
Server/DataStructures/SharedDataFacade.h
View File

@ -210,7 +210,7 @@ private:
void LoadGeometries()
{
unsigned * geometries_compressed_ptr = (unsigned *)(
shared_memory + data_layout->GetGeometriesCompressedOffset()
shared_memory + data_layout->GetGeometriesIndexListOffset()
);
typename ShM<bool, true>::vector egde_is_compressed(
geometries_compressed_ptr,
@ -219,7 +219,7 @@ private:
m_egde_is_compressed.swap(egde_is_compressed);
unsigned * geometries_index_ptr = (unsigned *)(
shared_memory + data_layout->GetGeometriesIndicesOffset()
shared_memory + data_layout->GetGeometriesIndicatorOffset()
);
typename ShM<unsigned, true>::vector geometry_begin_indices(
geometries_index_ptr,

21
Server/DataStructures/SharedDataType.h
View File

@ -116,11 +116,12 @@ struct SharedDataLayout {
(coordinate_list_size * sizeof(FixedPointCoordinate) ) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode) ) +
(geometries_compression/32) + 1 +
(geometries_compression/32 + 1) * sizeof(unsigned) +
(geometries_index_list_size * sizeof(unsigned) ) +
(geometries_list_size * sizeof(unsigned) ) +
sizeof(checksum) +
1024*sizeof(char);
PrintInformation();
return result;
}
@ -170,6 +171,7 @@ struct SharedDataLayout {
(via_node_list_size * sizeof(NodeID) ) +
(graph_node_list_size * sizeof(QueryGraph::_StrNode)) +
(graph_edge_list_size * sizeof(QueryGraph::_StrEdge));
SimpleLogger().Write(logDEBUG) << "GetTimeStampOffset: " << result;
return result;
}
uint64_t GetCoordinateListOffset() const {
@ -181,6 +183,7 @@ struct SharedDataLayout {
(graph_node_list_size * sizeof(QueryGraph::_StrNode)) +
(graph_edge_list_size * sizeof(QueryGraph::_StrEdge)) +
(timestamp_length * sizeof(char) );
SimpleLogger().Write(logDEBUG) << "GetCoordinateListOffset: " << result;
return result;
}
uint64_t GetTurnInstructionListOffset() const {
@ -193,6 +196,7 @@ struct SharedDataLayout {
(graph_edge_list_size * sizeof(QueryGraph::_StrEdge)) +
(timestamp_length * sizeof(char) ) +
(coordinate_list_size * sizeof(FixedPointCoordinate));
SimpleLogger().Write(logDEBUG) << "GetTurnInstructionListOffset: " << result;
return result;
}
uint64_t GetRSearchTreeOffset() const {
@ -206,9 +210,10 @@ struct SharedDataLayout {
(timestamp_length * sizeof(char) ) +
(coordinate_list_size * sizeof(FixedPointCoordinate) ) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass));
SimpleLogger().Write(logDEBUG) << "GetRSearchTreeOffset: " << result;
return result;
}
uint64_t GetGeometriesIndicesOffset() const {
uint64_t GetGeometriesIndicatorOffset() const {
uint64_t result =
(name_index_list_size * sizeof(unsigned) ) +
(name_char_list_size * sizeof(char) ) +
@ -220,10 +225,11 @@ struct SharedDataLayout {
(coordinate_list_size * sizeof(FixedPointCoordinate) ) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode) );
SimpleLogger().Write(logDEBUG) << "GetGeometriesIndicatorOffset: " << result;
return result;
}
uint64_t GetGeometriesCompressedOffset() const
uint64_t GetGeometriesIndexListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned) ) +
@ -236,7 +242,8 @@ struct SharedDataLayout {
(coordinate_list_size * sizeof(FixedPointCoordinate) ) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode) ) +
(geometries_compression/32) + 1;
(geometries_compression/32 + 1) * sizeof(unsigned);
SimpleLogger().Write(logDEBUG) << "GetGeometriesCompressedOffset: " << result;
return result;
}
@ -252,8 +259,9 @@ struct SharedDataLayout {
(coordinate_list_size * sizeof(FixedPointCoordinate) ) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode) ) +
(geometries_compression/32) + 1 +
(geometries_compression/32 + 1) * sizeof(unsigned) +
(geometries_index_list_size * sizeof(unsigned) );
SimpleLogger().Write(logDEBUG) << "GetGeometryListOffset: " << result;
return result;
}
uint64_t GetChecksumOffset() const {
@ -268,9 +276,10 @@ struct SharedDataLayout {
(coordinate_list_size * sizeof(FixedPointCoordinate) ) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode) ) +
(geometries_compression/32) + 1 +
(geometries_compression/32 + 1) * sizeof(unsigned) +
(geometries_index_list_size * sizeof(unsigned) ) +
(geometries_list_size * sizeof(unsigned) );
SimpleLogger().Write(logDEBUG) << "GetChecksumOffset: " << result;
return result;
}
};

50
datastore.cpp
View File

@ -44,8 +44,9 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <sys/mman.h>
#endif
#include <boost/integer.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/integer.hpp>
#include <boost/iostreams/seek.hpp>
#include <string>
#include <vector>
@ -137,6 +138,10 @@ int main( const int argc, const char * argv[] ) {
BOOST_ASSERT(!paths_iterator->second.empty());
const boost::filesystem::path & names_data_path = paths_iterator->second;
paths_iterator = server_paths.find("geometries");
BOOST_ASSERT(server_paths.end() != paths_iterator);
BOOST_ASSERT(!paths_iterator->second.empty());
const boost::filesystem::path & geometries_data_path = paths_iterator->second;
// get the shared memory segment to use
bool use_first_segment = SharedMemory::RegionExists( LAYOUT_2 );
@ -277,6 +282,22 @@ int main( const int argc, const char * argv[] ) {
shared_layout_ptr->coordinate_list_size = coordinate_list_size;
// load geometries sizes
boost::filesystem::ifstream geometry_input_stream(
geometries_data_path,
std::ios::binary
);
unsigned number_of_geometries_indices = 0;
unsigned number_of_compressed_geometries = 0;
geometry_input_stream.read((char *)&number_of_geometries_indices, sizeof(unsigned));
shared_layout_ptr->geometries_index_list_size = number_of_geometries_indices;
boost::iostreams::seek(geometry_input_stream, number_of_geometries_indices*sizeof(unsigned), BOOST_IOS::cur);
geometry_input_stream.read( (char *)&number_of_compressed_geometries, sizeof(unsigned));
shared_layout_ptr->geometries_list_size = number_of_compressed_geometries;
SimpleLogger().Write(logDEBUG) << "number_of_geometries_indices : " << number_of_geometries_indices << ", number_of_compressed_geometries: " << number_of_compressed_geometries;
// allocate shared memory block
SimpleLogger().Write() << "allocating shared memory of " << shared_layout_ptr->GetSizeOfLayout() << " bytes";
@ -317,8 +338,8 @@ int main( const int argc, const char * argv[] ) {
shared_memory_ptr + shared_layout_ptr->GetTurnInstructionListOffset()
);
unsigned * geometries_index_ptr = (unsigned *)(
shared_memory + shared_layout_ptr->GetGeometriesIndicesOffset()
unsigned * geometries_indicator_ptr = (unsigned *)(
shared_memory_ptr + shared_layout_ptr->GetGeometriesIndicatorOffset()
);
OriginalEdgeData current_edge_data;
@ -334,11 +355,32 @@ int main( const int argc, const char * argv[] ) {
const unsigned bucket = i / 32;
const unsigned offset = i % 32;
geometries_index_ptr[bucket] |= (1 << offset);
unsigned value = ((0 == offset) ? 0 : geometries_indicator_ptr[bucket]);
geometries_indicator_ptr[bucket] = (value | (1 << offset));
}
edges_input_stream.close();
//TODO: load compressed geometry
unsigned * geometries_index_ptr = (unsigned *)(
shared_memory_ptr + shared_layout_ptr->GetGeometriesIndexListOffset()
);
boost::iostreams::seek(geometry_input_stream, sizeof(unsigned), BOOST_IOS::beg);
geometry_input_stream.read(
(char *)geometries_index_ptr,
shared_layout_ptr->geometries_index_list_size*sizeof(unsigned)
);
unsigned * geometries_list_ptr = (unsigned *)(
shared_memory_ptr + shared_layout_ptr->GetGeometryListOffset()
);
boost::iostreams::seek(geometry_input_stream, sizeof(unsigned), BOOST_IOS::cur);
geometry_input_stream.read(
(char *)geometries_list_ptr,
shared_layout_ptr->geometries_list_size*sizeof(unsigned)
);
// Loading list of coordinates
FixedPointCoordinate * coordinates_ptr = (FixedPointCoordinate *)(
shared_memory_ptr + shared_layout_ptr->GetCoordinateListOffset()