Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Integrate RangeTable into server

Browse Source
This commit is contained in:
Patrick Niklaus 2014-06-03 00:07:34 +02:00
parent f90ce77da4
commit 7a7d0c09d9
3 changed files with 52 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
DataStructures/RangeTable.h
View File

@ -8,19 +8,21 @@
#include <xmmintrin.h> #include <xmmintrin.h>
#endif #endif
/* /*
* These pre-declarations are needed because parsing C++ is hard * These pre-declarations are needed because parsing C++ is hard
* and otherwise the compiler gets confused. * and otherwise the compiler gets confused.
*/ */
template<unsigned BLOCK_SIZE=16> class RangeTable; template<unsigned BLOCK_SIZE=16, bool USE_SHARED_MEMORY = false> class RangeTable;
template<unsigned BLOCK_SIZE> template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
std::ostream& operator<<(std::ostream &out, const RangeTable<BLOCK_SIZE> &table); std::ostream& operator<<(std::ostream &out, const RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table);
template<unsigned BLOCK_SIZE> template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
std::istream& operator>>(std::istream &in, RangeTable<BLOCK_SIZE> &table); std::istream& operator>>(std::istream &in, RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table);
#include "SharedMemoryFactory.h"
#include "SharedMemoryVectorWrapper.h"
/** /**
* Stores adjacent ranges in a compressed format. * Stores adjacent ranges in a compressed format.
@ -31,10 +33,10 @@ std::istream& operator>>(std::istream &in, RangeTable<BLOCK_SIZE> &table);
* But each block consists of an absolute value and BLOCK_SIZE differential values. * But each block consists of an absolute value and BLOCK_SIZE differential values.
* So the effective block size is sizeof(unsigned) + BLOCK_SIZE. * So the effective block size is sizeof(unsigned) + BLOCK_SIZE.
*/ */
template<unsigned BLOCK_SIZE> template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
class RangeTable class RangeTable
{ {
private: public:
union BlockT union BlockT
{ {
unsigned char uint8_blocks[BLOCK_SIZE]; unsigned char uint8_blocks[BLOCK_SIZE];
@ -45,22 +47,23 @@ private:
#endif #endif
}; };
// contains offset for each differential block typedef typename ShM<BlockT, USE_SHARED_MEMORY>::vector BlockContainerT;
std::vector<unsigned> block_offsets; typedef typename ShM<unsigned, USE_SHARED_MEMORY>::vector OffsetContainerT;
// blocks of differential encoded offsets, should be aligned
std::vector<BlockT> diff_blocks;
unsigned sum_lengths;
inline unsigned PrefixSumAtIndex(int index, const BlockT& block) const;
public:
friend std::ostream& operator<< <>(std::ostream &out, const RangeTable &table); friend std::ostream& operator<< <>(std::ostream &out, const RangeTable &table);
friend std::istream& operator>> <>(std::istream &in, RangeTable &table); friend std::istream& operator>> <>(std::istream &in, RangeTable &table);
RangeTable() {} RangeTable() {}
// for loading from shared memory
explicit RangeTable(OffsetContainerT& external_offsets, BlockContainerT& external_blocks)
{
block_offsets.swap(external_offsets);
diff_blocks.swap(external_blocks);
}
// construct table from length vector // construct table from length vector
RangeTable(std::vector<unsigned> lengths) explicit RangeTable(std::vector<unsigned> lengths)
{ {
unsigned number_of_blocks = (lengths.size() + 1) / (BLOCK_SIZE + 1); unsigned number_of_blocks = (lengths.size() + 1) / (BLOCK_SIZE + 1);
if (lengths.size() % (BLOCK_SIZE + 1) != 0) if (lengths.size() % (BLOCK_SIZE + 1) != 0)
@ -159,14 +162,22 @@ public:
end_idx = block_offsets[block_idx + 1]; end_idx = block_offsets[block_idx + 1];
} }
//std::cout << block_idx << " / " << internal_idx << " : " << begin_idx << " - " << end_idx << std::endl;
BOOST_ASSERT(begin_idx < sum_lengths && end_idx <= sum_lengths); BOOST_ASSERT(begin_idx < sum_lengths && end_idx <= sum_lengths);
} }
private:
inline unsigned PrefixSumAtIndex(int index, const BlockT& block) const;
// contains offset for each differential block
OffsetContainerT block_offsets;
// blocks of differential encoded offsets, should be aligned
BlockContainerT diff_blocks;
unsigned sum_lengths;
}; };
#ifdef OSRM_USE_SSE #ifdef OSRM_USE_SSE
// For blocksize 16 we can use SSE instructions // For blocksize 16 we can use SSE instructions
// FIXME only implemented for non-shared memory
template<> template<>
unsigned RangeTable<16>::PrefixSumAtIndex(int index, const BlockT& block) const unsigned RangeTable<16>::PrefixSumAtIndex(int index, const BlockT& block) const
{ {
@ -206,8 +217,8 @@ unsigned RangeTable<16>::PrefixSumAtIndex(int index, const BlockT& block) const
} }
#endif #endif
template<unsigned BLOCK_SIZE> template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
unsigned RangeTable<BLOCK_SIZE>::PrefixSumAtIndex(int index, const BlockT& block) const unsigned RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY>::PrefixSumAtIndex(int index, const BlockT& block) const
{ {
unsigned sum = 0; unsigned sum = 0;
for (int i = 0; i <= index; i++) for (int i = 0; i <= index; i++)
@ -216,8 +227,8 @@ unsigned RangeTable<BLOCK_SIZE>::PrefixSumAtIndex(int index, const BlockT& block
return sum; return sum;
} }
template<unsigned BLOCK_SIZE> template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
std::ostream& operator<<(std::ostream &out, const RangeTable<BLOCK_SIZE> &table) std::ostream& operator<<(std::ostream &out, const RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table)
{ {
// write number of block // write number of block
unsigned number_of_blocks = table.diff_blocks.size(); unsigned number_of_blocks = table.diff_blocks.size();
@ -232,23 +243,22 @@ std::ostream& operator<<(std::ostream &out, const RangeTable<BLOCK_SIZE> &table)
return out; return out;
} }
template<unsigned BLOCK_SIZE> template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
std::istream& operator>>(std::istream &in, RangeTable<BLOCK_SIZE> &table) std::istream& operator>>(std::istream &in, RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table)
{ {
// write number of block // read number of block
unsigned number_of_blocks; unsigned number_of_blocks;
in.read((char *) &number_of_blocks, sizeof(unsigned)); in.read((char *) &number_of_blocks, sizeof(unsigned));
// read total length
in.read((char *) &table.sum_lengths, sizeof(unsigned));
table.block_offsets.resize(number_of_blocks); table.block_offsets.resize(number_of_blocks);
table.diff_blocks.resize(number_of_blocks); table.diff_blocks.resize(number_of_blocks);
// write total length
in.read((char *) &table.sum_lengths, sizeof(unsigned));
// read block offsets // read block offsets
in.read((char *) table.block_offsets.data(), sizeof(unsigned) * number_of_blocks); in.read((char *) table.block_offsets.data(), sizeof(unsigned) * number_of_blocks);
// read blocks // read blocks
in.read((char *) table.diff_blocks.data(), BLOCK_SIZE * number_of_blocks); in.read((char *) table.diff_blocks.data(), BLOCK_SIZE * number_of_blocks);
return in; return in;
} }

6
Extractor/ExtractionContainers.cpp
View File

@ -395,15 +395,19 @@ void ExtractionContainers::PrepareData(const std::string &output_file_name,
std::string name_file_streamName = (output_file_name + ".names"); std::string name_file_streamName = (output_file_name + ".names");
boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary); boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary);
unsigned total_length = 0;
std::vector<unsigned> name_lengths; std::vector<unsigned> name_lengths;
for (const std::string &temp_string : name_list) for (const std::string &temp_string : name_list)
{ {
name_lengths.push_back(temp_string.length()); const unsigned string_length = std::min(temp_string.length(), 255lu);
name_lengths.push_back(string_length);
total_length += string_length;
} }
RangeTable<> table(name_lengths); RangeTable<> table(name_lengths);
name_file_stream << table; name_file_stream << table;
name_file_stream.write((char*) &total_length, sizeof(unsigned));
// write all chars consecutively // write all chars consecutively
for (const std::string &temp_string : name_list) for (const std::string &temp_string : name_list)
{ {

19
Server/DataStructures/InternalDataFacade.h
View File

@ -38,6 +38,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "../../DataStructures/SharedMemoryVectorWrapper.h" #include "../../DataStructures/SharedMemoryVectorWrapper.h"
#include "../../DataStructures/StaticGraph.h" #include "../../DataStructures/StaticGraph.h"
#include "../../DataStructures/StaticRTree.h" #include "../../DataStructures/StaticRTree.h"
#include "../../DataStructures/RangeTable.h"
#include "../../Util/BoostFileSystemFix.h" #include "../../Util/BoostFileSystemFix.h"
#include "../../Util/GraphLoader.h" #include "../../Util/GraphLoader.h"
#include "../../Util/ProgramOptions.h" #include "../../Util/ProgramOptions.h"
@ -66,13 +67,13 @@ template <class EdgeDataT> class InternalDataFacade : public BaseDataFacade<Edge
ShM<unsigned, false>::vector m_name_ID_list; ShM<unsigned, false>::vector m_name_ID_list;
ShM<TurnInstruction, false>::vector m_turn_instruction_list; ShM<TurnInstruction, false>::vector m_turn_instruction_list;
ShM<char, false>::vector m_names_char_list; ShM<char, false>::vector m_names_char_list;
ShM<unsigned, false>::vector m_name_begin_indices;
ShM<bool, false>::vector m_egde_is_compressed; ShM<bool, false>::vector m_egde_is_compressed;
ShM<unsigned, false>::vector m_geometry_indices; ShM<unsigned, false>::vector m_geometry_indices;
ShM<unsigned, false>::vector m_geometry_list; ShM<unsigned, false>::vector m_geometry_list;
std::shared_ptr<StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, false>::vector, false>> std::shared_ptr<StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, false>::vector, false>>
m_static_rtree; m_static_rtree;
RangeTable<16, false> m_name_table;
void LoadTimestamp(const boost::filesystem::path &timestamp_path) void LoadTimestamp(const boost::filesystem::path &timestamp_path)
{ {
@ -203,16 +204,12 @@ template <class EdgeDataT> class InternalDataFacade : public BaseDataFacade<Edge
void LoadStreetNames(const boost::filesystem::path &names_file) void LoadStreetNames(const boost::filesystem::path &names_file)
{ {
boost::filesystem::ifstream name_stream(names_file, std::ios::binary); boost::filesystem::ifstream name_stream(names_file, std::ios::binary);
unsigned number_of_names = 0;
name_stream >> m_name_table;
unsigned number_of_chars = 0; unsigned number_of_chars = 0;
name_stream.read((char *)&number_of_names, sizeof(unsigned));
name_stream.read((char *)&number_of_chars, sizeof(unsigned)); name_stream.read((char *)&number_of_chars, sizeof(unsigned));
BOOST_ASSERT_MSG(0 != number_of_names, "name file broken");
BOOST_ASSERT_MSG(0 != number_of_chars, "name file broken"); BOOST_ASSERT_MSG(0 != number_of_chars, "name file broken");
m_name_begin_indices.resize(number_of_names);
name_stream.read((char *)&m_name_begin_indices[0], number_of_names * sizeof(unsigned));
m_names_char_list.resize(number_of_chars + 1); //+1 gives sentinel element m_names_char_list.resize(number_of_chars + 1); //+1 gives sentinel element
name_stream.read((char *)&m_names_char_list[0], number_of_chars * sizeof(char)); name_stream.read((char *)&m_names_char_list[0], number_of_chars * sizeof(char));
BOOST_ASSERT_MSG(0 != m_names_char_list.size(), "could not load any names"); BOOST_ASSERT_MSG(0 != m_names_char_list.size(), "could not load any names");
@ -384,9 +381,9 @@ template <class EdgeDataT> class InternalDataFacade : public BaseDataFacade<Edge
result = ""; result = "";
return; return;
} }
BOOST_ASSERT_MSG(name_id < m_name_begin_indices.size(), "name id too high"); unsigned begin_index;
const unsigned begin_index = m_name_begin_indices[name_id]; unsigned end_index;
const unsigned end_index = m_name_begin_indices[name_id + 1]; m_name_table.GetRange(name_id, begin_index, end_index);
BOOST_ASSERT_MSG(begin_index < m_names_char_list.size(), "begin index of name too high"); BOOST_ASSERT_MSG(begin_index < m_names_char_list.size(), "begin index of name too high");
BOOST_ASSERT_MSG(end_index < m_names_char_list.size(), "end index of name too high"); BOOST_ASSERT_MSG(end_index < m_names_char_list.size(), "end index of name too high");