diff --git a/DataStructures/RangeTable.h b/DataStructures/RangeTable.h index 2b9be7d67..7dfb09f54 100644 --- a/DataStructures/RangeTable.h +++ b/DataStructures/RangeTable.h @@ -8,19 +8,21 @@ #include #endif - /* * These pre-declarations are needed because parsing C++ is hard * and otherwise the compiler gets confused. */ -template class RangeTable; +template class RangeTable; -template -std::ostream& operator<<(std::ostream &out, const RangeTable &table); +template +std::ostream& operator<<(std::ostream &out, const RangeTable &table); -template -std::istream& operator>>(std::istream &in, RangeTable &table); +template +std::istream& operator>>(std::istream &in, RangeTable &table); + +#include "SharedMemoryFactory.h" +#include "SharedMemoryVectorWrapper.h" /** * Stores adjacent ranges in a compressed format. @@ -31,10 +33,10 @@ std::istream& operator>>(std::istream &in, RangeTable &table); * But each block consists of an absolute value and BLOCK_SIZE differential values. * So the effective block size is sizeof(unsigned) + BLOCK_SIZE. */ -template +template class RangeTable { -private: +public: union BlockT { unsigned char uint8_blocks[BLOCK_SIZE]; @@ -45,22 +47,23 @@ private: #endif }; - // contains offset for each differential block - std::vector block_offsets; - // blocks of differential encoded offsets, should be aligned - std::vector diff_blocks; - unsigned sum_lengths; + typedef typename ShM::vector BlockContainerT; + typedef typename ShM::vector OffsetContainerT; - inline unsigned PrefixSumAtIndex(int index, const BlockT& block) const; - -public: friend std::ostream& operator<< <>(std::ostream &out, const RangeTable &table); friend std::istream& operator>> <>(std::istream &in, RangeTable &table); RangeTable() {} + // for loading from shared memory + explicit RangeTable(OffsetContainerT& external_offsets, BlockContainerT& external_blocks) + { + block_offsets.swap(external_offsets); + diff_blocks.swap(external_blocks); + } + // construct table from length vector - RangeTable(std::vector lengths) + explicit RangeTable(std::vector lengths) { unsigned number_of_blocks = (lengths.size() + 1) / (BLOCK_SIZE + 1); if (lengths.size() % (BLOCK_SIZE + 1) != 0) @@ -159,14 +162,22 @@ public: end_idx = block_offsets[block_idx + 1]; } - //std::cout << block_idx << " / " << internal_idx << " : " << begin_idx << " - " << end_idx << std::endl; - BOOST_ASSERT(begin_idx < sum_lengths && end_idx <= sum_lengths); } +private: + + inline unsigned PrefixSumAtIndex(int index, const BlockT& block) const; + + // contains offset for each differential block + OffsetContainerT block_offsets; + // blocks of differential encoded offsets, should be aligned + BlockContainerT diff_blocks; + unsigned sum_lengths; }; #ifdef OSRM_USE_SSE // For blocksize 16 we can use SSE instructions +// FIXME only implemented for non-shared memory template<> unsigned RangeTable<16>::PrefixSumAtIndex(int index, const BlockT& block) const { @@ -206,8 +217,8 @@ unsigned RangeTable<16>::PrefixSumAtIndex(int index, const BlockT& block) const } #endif -template -unsigned RangeTable::PrefixSumAtIndex(int index, const BlockT& block) const +template +unsigned RangeTable::PrefixSumAtIndex(int index, const BlockT& block) const { unsigned sum = 0; for (int i = 0; i <= index; i++) @@ -216,8 +227,8 @@ unsigned RangeTable::PrefixSumAtIndex(int index, const BlockT& block return sum; } -template -std::ostream& operator<<(std::ostream &out, const RangeTable &table) +template +std::ostream& operator<<(std::ostream &out, const RangeTable &table) { // write number of block unsigned number_of_blocks = table.diff_blocks.size(); @@ -232,23 +243,22 @@ std::ostream& operator<<(std::ostream &out, const RangeTable &table) return out; } -template -std::istream& operator>>(std::istream &in, RangeTable &table) +template +std::istream& operator>>(std::istream &in, RangeTable &table) { - // write number of block + // read number of block unsigned number_of_blocks; in.read((char *) &number_of_blocks, sizeof(unsigned)); + // read total length + in.read((char *) &table.sum_lengths, sizeof(unsigned)); table.block_offsets.resize(number_of_blocks); table.diff_blocks.resize(number_of_blocks); - // write total length - in.read((char *) &table.sum_lengths, sizeof(unsigned)); // read block offsets in.read((char *) table.block_offsets.data(), sizeof(unsigned) * number_of_blocks); // read blocks in.read((char *) table.diff_blocks.data(), BLOCK_SIZE * number_of_blocks); - return in; } diff --git a/Extractor/ExtractionContainers.cpp b/Extractor/ExtractionContainers.cpp index 70174e7d9..2760a8e9e 100644 --- a/Extractor/ExtractionContainers.cpp +++ b/Extractor/ExtractionContainers.cpp @@ -395,15 +395,19 @@ void ExtractionContainers::PrepareData(const std::string &output_file_name, std::string name_file_streamName = (output_file_name + ".names"); boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary); + unsigned total_length = 0; std::vector name_lengths; for (const std::string &temp_string : name_list) { - name_lengths.push_back(temp_string.length()); + const unsigned string_length = std::min(temp_string.length(), 255lu); + name_lengths.push_back(string_length); + total_length += string_length; } RangeTable<> table(name_lengths); name_file_stream << table; + name_file_stream.write((char*) &total_length, sizeof(unsigned)); // write all chars consecutively for (const std::string &temp_string : name_list) { diff --git a/Server/DataStructures/InternalDataFacade.h b/Server/DataStructures/InternalDataFacade.h index 3c63bd461..316ab24ea 100644 --- a/Server/DataStructures/InternalDataFacade.h +++ b/Server/DataStructures/InternalDataFacade.h @@ -38,6 +38,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../../DataStructures/SharedMemoryVectorWrapper.h" #include "../../DataStructures/StaticGraph.h" #include "../../DataStructures/StaticRTree.h" +#include "../../DataStructures/RangeTable.h" #include "../../Util/BoostFileSystemFix.h" #include "../../Util/GraphLoader.h" #include "../../Util/ProgramOptions.h" @@ -66,13 +67,13 @@ template class InternalDataFacade : public BaseDataFacade::vector m_name_ID_list; ShM::vector m_turn_instruction_list; ShM::vector m_names_char_list; - ShM::vector m_name_begin_indices; ShM::vector m_egde_is_compressed; ShM::vector m_geometry_indices; ShM::vector m_geometry_list; std::shared_ptr::vector, false>> m_static_rtree; + RangeTable<16, false> m_name_table; void LoadTimestamp(const boost::filesystem::path ×tamp_path) { @@ -203,16 +204,12 @@ template class InternalDataFacade : public BaseDataFacade> m_name_table; + unsigned number_of_chars = 0; - name_stream.read((char *)&number_of_names, sizeof(unsigned)); name_stream.read((char *)&number_of_chars, sizeof(unsigned)); - BOOST_ASSERT_MSG(0 != number_of_names, "name file broken"); BOOST_ASSERT_MSG(0 != number_of_chars, "name file broken"); - - m_name_begin_indices.resize(number_of_names); - name_stream.read((char *)&m_name_begin_indices[0], number_of_names * sizeof(unsigned)); - m_names_char_list.resize(number_of_chars + 1); //+1 gives sentinel element name_stream.read((char *)&m_names_char_list[0], number_of_chars * sizeof(char)); BOOST_ASSERT_MSG(0 != m_names_char_list.size(), "could not load any names"); @@ -384,9 +381,9 @@ template class InternalDataFacade : public BaseDataFacade