Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Integrate RangeTable into server

Browse Source
This commit is contained in:
Patrick Niklaus 2014-06-03 00:07:34 +02:00
parent f90ce77da4
commit 7a7d0c09d9
3 changed files with 52 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
DataStructures/RangeTable.h
View File

@ -8,19 +8,21 @@
#include <xmmintrin.h>
#endif
/*
* These pre-declarations are needed because parsing C++ is hard
* and otherwise the compiler gets confused.
*/
template<unsigned BLOCK_SIZE=16> class RangeTable;
template<unsigned BLOCK_SIZE=16, bool USE_SHARED_MEMORY = false> class RangeTable;
template<unsigned BLOCK_SIZE>
std::ostream& operator<<(std::ostream &out, const RangeTable<BLOCK_SIZE> &table);
template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
std::ostream& operator<<(std::ostream &out, const RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table);
template<unsigned BLOCK_SIZE>
std::istream& operator>>(std::istream &in, RangeTable<BLOCK_SIZE> &table);
template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
std::istream& operator>>(std::istream &in, RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table);
#include "SharedMemoryFactory.h"
#include "SharedMemoryVectorWrapper.h"
/**
* Stores adjacent ranges in a compressed format.
@ -31,10 +33,10 @@ std::istream& operator>>(std::istream &in, RangeTable<BLOCK_SIZE> &table);
* But each block consists of an absolute value and BLOCK_SIZE differential values.
* So the effective block size is sizeof(unsigned) + BLOCK_SIZE.
*/
template<unsigned BLOCK_SIZE>
template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
class RangeTable
{
private:
public:
union BlockT
{
unsigned char uint8_blocks[BLOCK_SIZE];
@ -45,22 +47,23 @@ private:
#endif
};
// contains offset for each differential block
std::vector<unsigned> block_offsets;
// blocks of differential encoded offsets, should be aligned
std::vector<BlockT> diff_blocks;
unsigned sum_lengths;
typedef typename ShM<BlockT, USE_SHARED_MEMORY>::vector BlockContainerT;
typedef typename ShM<unsigned, USE_SHARED_MEMORY>::vector OffsetContainerT;
inline unsigned PrefixSumAtIndex(int index, const BlockT& block) const;
public:
friend std::ostream& operator<< <>(std::ostream &out, const RangeTable &table);
friend std::istream& operator>> <>(std::istream &in, RangeTable &table);
RangeTable() {}
// for loading from shared memory
explicit RangeTable(OffsetContainerT& external_offsets, BlockContainerT& external_blocks)
{
block_offsets.swap(external_offsets);
diff_blocks.swap(external_blocks);
}
// construct table from length vector
RangeTable(std::vector<unsigned> lengths)
explicit RangeTable(std::vector<unsigned> lengths)
{
unsigned number_of_blocks = (lengths.size() + 1) / (BLOCK_SIZE + 1);
if (lengths.size() % (BLOCK_SIZE + 1) != 0)
@ -159,14 +162,22 @@ public:
end_idx = block_offsets[block_idx + 1];
}
//std::cout << block_idx << " / " << internal_idx << " : " << begin_idx << " - " << end_idx << std::endl;
BOOST_ASSERT(begin_idx < sum_lengths && end_idx <= sum_lengths);
}
private:
inline unsigned PrefixSumAtIndex(int index, const BlockT& block) const;
// contains offset for each differential block
OffsetContainerT block_offsets;
// blocks of differential encoded offsets, should be aligned
BlockContainerT diff_blocks;
unsigned sum_lengths;
};
#ifdef OSRM_USE_SSE
// For blocksize 16 we can use SSE instructions
// FIXME only implemented for non-shared memory
template<>
unsigned RangeTable<16>::PrefixSumAtIndex(int index, const BlockT& block) const
{
@ -206,8 +217,8 @@ unsigned RangeTable<16>::PrefixSumAtIndex(int index, const BlockT& block) const
}
#endif
template<unsigned BLOCK_SIZE>
unsigned RangeTable<BLOCK_SIZE>::PrefixSumAtIndex(int index, const BlockT& block) const
template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
unsigned RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY>::PrefixSumAtIndex(int index, const BlockT& block) const
{
unsigned sum = 0;
for (int i = 0; i <= index; i++)
@ -216,8 +227,8 @@ unsigned RangeTable<BLOCK_SIZE>::PrefixSumAtIndex(int index, const BlockT& block
return sum;
}
template<unsigned BLOCK_SIZE>
std::ostream& operator<<(std::ostream &out, const RangeTable<BLOCK_SIZE> &table)
template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
std::ostream& operator<<(std::ostream &out, const RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table)
{
// write number of block
unsigned number_of_blocks = table.diff_blocks.size();
@ -232,23 +243,22 @@ std::ostream& operator<<(std::ostream &out, const RangeTable<BLOCK_SIZE> &table)
return out;
}
template<unsigned BLOCK_SIZE>
std::istream& operator>>(std::istream &in, RangeTable<BLOCK_SIZE> &table)
template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
std::istream& operator>>(std::istream &in, RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table)
{
// write number of block
// read number of block
unsigned number_of_blocks;
in.read((char *) &number_of_blocks, sizeof(unsigned));
// read total length
in.read((char *) &table.sum_lengths, sizeof(unsigned));
table.block_offsets.resize(number_of_blocks);
table.diff_blocks.resize(number_of_blocks);
// write total length
in.read((char *) &table.sum_lengths, sizeof(unsigned));
// read block offsets
in.read((char *) table.block_offsets.data(), sizeof(unsigned) * number_of_blocks);
// read blocks
in.read((char *) table.diff_blocks.data(), BLOCK_SIZE * number_of_blocks);
return in;
}

6
Extractor/ExtractionContainers.cpp
View File

@ -395,15 +395,19 @@ void ExtractionContainers::PrepareData(const std::string &output_file_name,
std::string name_file_streamName = (output_file_name + ".names");
boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary);
unsigned total_length = 0;
std::vector<unsigned> name_lengths;
for (const std::string &temp_string : name_list)
{
name_lengths.push_back(temp_string.length());
const unsigned string_length = std::min(temp_string.length(), 255lu);
name_lengths.push_back(string_length);
total_length += string_length;
}
RangeTable<> table(name_lengths);
name_file_stream << table;
name_file_stream.write((char*) &total_length, sizeof(unsigned));
// write all chars consecutively
for (const std::string &temp_string : name_list)
{

19
Server/DataStructures/InternalDataFacade.h
View File

@ -38,6 +38,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "../../DataStructures/SharedMemoryVectorWrapper.h"
#include "../../DataStructures/StaticGraph.h"
#include "../../DataStructures/StaticRTree.h"
#include "../../DataStructures/RangeTable.h"
#include "../../Util/BoostFileSystemFix.h"
#include "../../Util/GraphLoader.h"
#include "../../Util/ProgramOptions.h"
@ -66,13 +67,13 @@ template <class EdgeDataT> class InternalDataFacade : public BaseDataFacade<Edge
ShM<unsigned, false>::vector m_name_ID_list;
ShM<TurnInstruction, false>::vector m_turn_instruction_list;
ShM<char, false>::vector m_names_char_list;
ShM<unsigned, false>::vector m_name_begin_indices;
ShM<bool, false>::vector m_egde_is_compressed;
ShM<unsigned, false>::vector m_geometry_indices;
ShM<unsigned, false>::vector m_geometry_list;
std::shared_ptr<StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, false>::vector, false>>
m_static_rtree;
RangeTable<16, false> m_name_table;
void LoadTimestamp(const boost::filesystem::path &timestamp_path)
{
@ -203,16 +204,12 @@ template <class EdgeDataT> class InternalDataFacade : public BaseDataFacade<Edge
void LoadStreetNames(const boost::filesystem::path &names_file)
{
boost::filesystem::ifstream name_stream(names_file, std::ios::binary);
unsigned number_of_names = 0;
name_stream >> m_name_table;
unsigned number_of_chars = 0;
name_stream.read((char *)&number_of_names, sizeof(unsigned));
name_stream.read((char *)&number_of_chars, sizeof(unsigned));
BOOST_ASSERT_MSG(0 != number_of_names, "name file broken");
BOOST_ASSERT_MSG(0 != number_of_chars, "name file broken");
m_name_begin_indices.resize(number_of_names);
name_stream.read((char *)&m_name_begin_indices[0], number_of_names * sizeof(unsigned));
m_names_char_list.resize(number_of_chars + 1); //+1 gives sentinel element
name_stream.read((char *)&m_names_char_list[0], number_of_chars * sizeof(char));
BOOST_ASSERT_MSG(0 != m_names_char_list.size(), "could not load any names");
@ -384,9 +381,9 @@ template <class EdgeDataT> class InternalDataFacade : public BaseDataFacade<Edge
result = "";
return;
}
BOOST_ASSERT_MSG(name_id < m_name_begin_indices.size(), "name id too high");
const unsigned begin_index = m_name_begin_indices[name_id];
const unsigned end_index = m_name_begin_indices[name_id + 1];
unsigned begin_index;
unsigned end_index;
m_name_table.GetRange(name_id, begin_index, end_index);
BOOST_ASSERT_MSG(begin_index < m_names_char_list.size(), "begin index of name too high");
BOOST_ASSERT_MSG(end_index < m_names_char_list.size(), "end index of name too high");