Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Merge branch 'TheMarex-diffencoding' into develop

Browse Source
This commit is contained in:
Dennis Luxen 2014-06-15 12:10:18 +02:00
commit 5b6518d4a9
7 changed files with 551 additions and 360 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

231
DataStructures/RangeTable.h Normal file
View File

@ -0,0 +1,231 @@
#ifndef __RANGE_TABLE_H__
#define __RANGE_TABLE_H__
#include "SharedMemoryFactory.h"
#include "SharedMemoryVectorWrapper.h"
#include <boost/range/irange.hpp>
#include <fstream>
#include <vector>
#include <array>
/*
* These pre-declarations are needed because parsing C++ is hard
* and otherwise the compiler gets confused.
*/
template<unsigned BLOCK_SIZE=16, bool USE_SHARED_MEMORY = false> class RangeTable;
template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
std::ostream& operator<<(std::ostream &out, const RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table);
template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
std::istream& operator>>(std::istream &in, RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table);
/**
* Stores adjacent ranges in a compressed format.
*
* Maximum supported length of a range is 255.
*
* Note: BLOCK_SIZE is the number of differential encodoed values.
* But each block consists of an absolute value and BLOCK_SIZE differential values.
* So the effective block size is sizeof(unsigned) + BLOCK_SIZE.
*/
template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
class RangeTable
{
public:
typedef std::array<unsigned char, BLOCK_SIZE> BlockT;
typedef typename ShM<BlockT, USE_SHARED_MEMORY>::vector BlockContainerT;
typedef typename ShM<unsigned, USE_SHARED_MEMORY>::vector OffsetContainerT;
typedef decltype(boost::irange(0u,0u)) RangeT;
friend std::ostream& operator<< <>(std::ostream &out, const RangeTable &table);
friend std::istream& operator>> <>(std::istream &in, RangeTable &table);
RangeTable() {}
// for loading from shared memory
explicit RangeTable(OffsetContainerT& external_offsets, BlockContainerT& external_blocks, const unsigned sum_lengths)
: sum_lengths(sum_lengths)
{
block_offsets.swap(external_offsets);
diff_blocks.swap(external_blocks);
}
// construct table from length vector
explicit RangeTable(const std::vector<unsigned>& lengths)
{
const unsigned number_of_blocks = [&lengths]() {
unsigned num = (lengths.size() + 1) / (BLOCK_SIZE + 1);
if ((lengths.size() + 1) % (BLOCK_SIZE + 1) != 0)
{
num += 1;
}
return num;
}();
block_offsets.reserve(number_of_blocks);
diff_blocks.reserve(number_of_blocks);
unsigned last_length = 0;
unsigned lengths_prefix_sum = 0;
unsigned block_idx = 0;
unsigned block_counter = 0;
BlockT block;
unsigned block_sum = 0;
for (const unsigned l : lengths)
{
// first entry of a block: encode absolute offset
if (block_idx == 0)
{
block_offsets.push_back(lengths_prefix_sum);
block_sum = 0;
}
else
{
block[block_idx - 1] = last_length;
block_sum += last_length;
}
BOOST_ASSERT((block_idx == 0 && block_offsets[block_counter] == lengths_prefix_sum)
|| lengths_prefix_sum == (block_offsets[block_counter]+block_sum));
// block is full
if (BLOCK_SIZE == block_idx)
{
diff_blocks.push_back(block);
block_counter++;
}
// we can only store strings with length 255
BOOST_ASSERT(l <= 255);
lengths_prefix_sum += l;
last_length = l;
block_idx = (block_idx + 1) % (BLOCK_SIZE + 1);
}
// Last block can't be finished because we didn't add the sentinel
BOOST_ASSERT (block_counter == (number_of_blocks - 1));
// one block missing: starts with guard value
if (0 == block_idx)
{
// the last value is used as sentinel
block_offsets.push_back(lengths_prefix_sum);
block_idx = (block_idx + 1) % BLOCK_SIZE;
}
while (0 != block_idx)
{
block[block_idx - 1] = last_length;
last_length = 0;
block_idx = (block_idx + 1) % (BLOCK_SIZE + 1);
}
diff_blocks.push_back(block);
BOOST_ASSERT(diff_blocks.size() == number_of_blocks && block_offsets.size() == number_of_blocks);
sum_lengths = lengths_prefix_sum;
}
inline RangeT GetRange(const unsigned id) const
{
BOOST_ASSERT(id < block_offsets.size() + diff_blocks.size() * BLOCK_SIZE);
// internal_idx 0 is implicitly stored in block_offsets[block_idx]
const unsigned internal_idx = id % (BLOCK_SIZE + 1);
const unsigned block_idx = id / (BLOCK_SIZE + 1);
BOOST_ASSERT(block_idx < diff_blocks.size());
unsigned begin_idx = 0;
unsigned end_idx = 0;
begin_idx = block_offsets[block_idx];
const BlockT& block = diff_blocks[block_idx];
if (internal_idx > 0)
{
begin_idx += PrefixSumAtIndex(internal_idx - 1, block);
}
// next index inside current block
if (internal_idx < BLOCK_SIZE)
{
// note internal_idx - 1 is the *current* index for uint8_blocks
end_idx = begin_idx + block[internal_idx];
}
else
{
BOOST_ASSERT(block_idx < block_offsets.size() - 1);
end_idx = block_offsets[block_idx + 1];
}
BOOST_ASSERT(begin_idx < sum_lengths && end_idx <= sum_lengths);
BOOST_ASSERT(begin_idx <= end_idx);
return boost::irange(begin_idx, end_idx);
}
private:
inline unsigned PrefixSumAtIndex(int index, const BlockT& block) const;
// contains offset for each differential block
OffsetContainerT block_offsets;
// blocks of differential encoded offsets, should be aligned
BlockContainerT diff_blocks;
unsigned sum_lengths;
};
template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
unsigned RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY>::PrefixSumAtIndex(int index, const BlockT& block) const
{
// this loop looks inefficent, but a modern compiler
// will emit nice SIMD here, at least for sensible block sizes. (I checked.)
unsigned sum = 0;
for (int i = 0; i <= index; ++i)
{
sum += block[i];
}
return sum;
}
template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
std::ostream& operator<<(std::ostream &out, const RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table)
{
// write number of block
const unsigned number_of_blocks = table.diff_blocks.size();
out.write((char *) &number_of_blocks, sizeof(unsigned));
// write total length
out.write((char *) &table.sum_lengths, sizeof(unsigned));
// write block offsets
out.write((char *) table.block_offsets.data(), sizeof(unsigned) * table.block_offsets.size());
// write blocks
out.write((char *) table.diff_blocks.data(), BLOCK_SIZE * table.diff_blocks.size());
return out;
}
template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
std::istream& operator>>(std::istream &in, RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table)
{
// read number of block
unsigned number_of_blocks;
in.read((char *) &number_of_blocks, sizeof(unsigned));
// read total length
in.read((char *) &table.sum_lengths, sizeof(unsigned));
table.block_offsets.resize(number_of_blocks);
table.diff_blocks.resize(number_of_blocks);
// read block offsets
in.read((char *) table.block_offsets.data(), sizeof(unsigned) * number_of_blocks);
// read blocks
in.read((char *) table.diff_blocks.data(), BLOCK_SIZE * number_of_blocks);
return in;
}
#endif

2
DataStructures/StaticRTree.h
View File

@ -766,7 +766,9 @@ class StaticRTree
}
const uint64_t seek_pos = sizeof(uint64_t) + leaf_id * sizeof(LeafNode);
thread_local_rtree_stream->seekg(seek_pos);
BOOST_ASSERT_MSG(thread_local_rtree_stream->good(), "Seeking to position in leaf file failed.");
thread_local_rtree_stream->read((char *)&result_node, sizeof(LeafNode));
BOOST_ASSERT_MSG(thread_local_rtree_stream->good(), "Reading from leaf file failed.");
}
inline bool EdgesAreEquivalent(const FixedPointCoordinate &a,

31
Extractor/ExtractionContainers.cpp
View File

@ -30,6 +30,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "../Util/OSRMException.h"
#include "../Util/SimpleLogger.h"
#include "../Util/TimingUtil.h"
#include "../DataStructures/RangeTable.h"
#include <boost/assert.hpp>
#include <boost/filesystem.hpp>
@ -64,6 +65,7 @@ void ExtractionContainers::PrepareData(const std::string &output_file_name,
{
unsigned number_of_used_nodes = 0;
unsigned number_of_used_edges = 0;
std::cout << "[extractor] Sorting used nodes ... " << std::flush;
TIMER_START(sorting_used_nodes);
stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), stxxl_memory);
@ -395,32 +397,23 @@ void ExtractionContainers::PrepareData(const std::string &output_file_name,
std::string name_file_streamName = (output_file_name + ".names");
boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary);
// write number of names
const unsigned number_of_names = name_list.size() + 1;
name_file_stream.write((char *)&(number_of_names), sizeof(unsigned));
// compute total number of chars
unsigned total_number_of_chars = 0;
unsigned total_length = 0;
std::vector<unsigned> name_lengths;
for (const std::string &temp_string : name_list)
{
total_number_of_chars += temp_string.length();
const unsigned string_length = std::min(static_cast<unsigned>(temp_string.length()), 255u);
name_lengths.push_back(string_length);
total_length += string_length;
}
// write total number of chars
name_file_stream.write((char *)&(total_number_of_chars), sizeof(unsigned));
// write prefixe sums
unsigned name_lengths_prefix_sum = 0;
for (const std::string &temp_string : name_list)
{
name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
name_lengths_prefix_sum += temp_string.length();
}
// duplicate on purpose!
name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
RangeTable<> table(name_lengths);
name_file_stream << table;
name_file_stream.write((char*) &total_length, sizeof(unsigned));
// write all chars consecutively
for (const std::string &temp_string : name_list)
{
const unsigned string_length = temp_string.length();
const unsigned string_length = std::min(static_cast<unsigned>(temp_string.length()), 255u);
name_file_stream.write(temp_string.c_str(), string_length);
}

31
Server/DataStructures/InternalDataFacade.h
View File

@ -38,6 +38,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "../../DataStructures/SharedMemoryVectorWrapper.h"
#include "../../DataStructures/StaticGraph.h"
#include "../../DataStructures/StaticRTree.h"
#include "../../DataStructures/RangeTable.h"
#include "../../Util/BoostFileSystemFix.h"
#include "../../Util/GraphLoader.h"
#include "../../Util/ProgramOptions.h"
@ -66,13 +67,13 @@ template <class EdgeDataT> class InternalDataFacade : public BaseDataFacade<Edge
ShM<unsigned, false>::vector m_name_ID_list;
ShM<TurnInstruction, false>::vector m_turn_instruction_list;
ShM<char, false>::vector m_names_char_list;
ShM<unsigned, false>::vector m_name_begin_indices;
ShM<bool, false>::vector m_egde_is_compressed;
ShM<unsigned, false>::vector m_geometry_indices;
ShM<unsigned, false>::vector m_geometry_list;
std::shared_ptr<StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, false>::vector, false>>
m_static_rtree;
RangeTable<16, false> m_name_table;
void LoadTimestamp(const boost::filesystem::path &timestamp_path)
{
@ -203,16 +204,12 @@ template <class EdgeDataT> class InternalDataFacade : public BaseDataFacade<Edge
void LoadStreetNames(const boost::filesystem::path &names_file)
{
boost::filesystem::ifstream name_stream(names_file, std::ios::binary);
unsigned number_of_names = 0;
name_stream >> m_name_table;
unsigned number_of_chars = 0;
name_stream.read((char *)&number_of_names, sizeof(unsigned));
name_stream.read((char *)&number_of_chars, sizeof(unsigned));
BOOST_ASSERT_MSG(0 != number_of_names, "name file broken");
BOOST_ASSERT_MSG(0 != number_of_chars, "name file broken");
m_name_begin_indices.resize(number_of_names);
name_stream.read((char *)&m_name_begin_indices[0], number_of_names * sizeof(unsigned));
m_names_char_list.resize(number_of_chars + 1); //+1 gives sentinel element
name_stream.read((char *)&m_names_char_list[0], number_of_chars * sizeof(char));
BOOST_ASSERT_MSG(0 != m_names_char_list.size(), "could not load any names");
@ -384,18 +381,16 @@ template <class EdgeDataT> class InternalDataFacade : public BaseDataFacade<Edge
result = "";
return;
}
BOOST_ASSERT_MSG(name_id < m_name_begin_indices.size(), "name id too high");
const unsigned begin_index = m_name_begin_indices[name_id];
const unsigned end_index = m_name_begin_indices[name_id + 1];
BOOST_ASSERT_MSG(begin_index < m_names_char_list.size(), "begin index of name too high");
BOOST_ASSERT_MSG(end_index < m_names_char_list.size(), "end index of name too high");
auto range = m_name_table.GetRange(name_id);
BOOST_ASSERT_MSG(begin_index <= end_index, "string ends before begin");
result.clear();
result.resize(end_index - begin_index);
std::copy(m_names_char_list.begin() + begin_index,
m_names_char_list.begin() + end_index,
result.begin());
if (range.begin() != range.end())
{
result.resize(range.back() - range.front() + 1);
std::copy(m_names_char_list.begin() + range.front(),
m_names_char_list.begin() + range.back() + 1,
result.begin());
}
}
virtual unsigned GetGeometryIndexForEdgeID(const unsigned id) const

120
Server/DataStructures/SharedDataFacade.h
View File

@ -33,6 +33,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "BaseDataFacade.h"
#include "SharedDataType.h"
#include "../../DataStructures/RangeTable.h"
#include "../../DataStructures/StaticGraph.h"
#include "../../DataStructures/StaticRTree.h"
#include "../../Util/BoostFileSystemFix.h"
@ -51,6 +52,7 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
typedef StaticGraph<EdgeData, true> QueryGraph;
typedef typename StaticGraph<EdgeData, true>::NodeArrayEntry GraphNode;
typedef typename StaticGraph<EdgeData, true>::EdgeArrayEntry GraphEdge;
typedef typename RangeTable<16, true>::BlockT NameIndexBlock;
typedef typename QueryGraph::InputEdge InputEdge;
typedef typename super::RTreeLeaf RTreeLeaf;
typedef typename StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>::TreeNode
@ -84,43 +86,51 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
std::shared_ptr<StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>>
m_static_rtree;
std::shared_ptr<RangeTable<16, true>> m_name_table;
void LoadChecksum()
{
m_check_sum = data_layout->checksum;
m_check_sum = *data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::HSGR_CHECKSUM);
SimpleLogger().Write() << "set checksum: " << m_check_sum;
}
void LoadTimestamp()
{
char *timestamp_ptr = shared_memory + data_layout->GetTimeStampOffset();
m_timestamp.resize(data_layout->timestamp_length);
char *timestamp_ptr = data_layout->GetBlockPtr<char>(shared_memory, SharedDataLayout::TIMESTAMP);
m_timestamp.resize(data_layout->GetBlockSize(SharedDataLayout::TIMESTAMP));
std::copy(
timestamp_ptr, timestamp_ptr + data_layout->timestamp_length, m_timestamp.begin());
timestamp_ptr,
timestamp_ptr + data_layout->GetBlockSize(SharedDataLayout::TIMESTAMP),
m_timestamp.begin());
}
void LoadRTree(const boost::filesystem::path &file_index_path)
{
BOOST_ASSERT_MSG(!m_coordinate_list->empty(), "coordinates must be loaded before r-tree");
RTreeNode *tree_ptr = (RTreeNode *)(shared_memory + data_layout->GetRSearchTreeOffset());
RTreeNode *tree_ptr = data_layout->GetBlockPtr<RTreeNode>(shared_memory, SharedDataLayout::R_SEARCH_TREE);
m_static_rtree =
std::make_shared<StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>>(
tree_ptr, data_layout->r_search_tree_size, file_index_path, m_coordinate_list);
tree_ptr,
data_layout->num_entries[SharedDataLayout::R_SEARCH_TREE],
file_index_path,
m_coordinate_list);
}
void LoadGraph()
{
m_number_of_nodes = data_layout->graph_node_list_size;
m_number_of_nodes = data_layout->num_entries[SharedDataLayout::GRAPH_NODE_LIST];
GraphNode *graph_nodes_ptr =
(GraphNode *)(shared_memory + data_layout->GetGraphNodeListOffset());
data_layout->GetBlockPtr<GraphNode>(shared_memory, SharedDataLayout::GRAPH_NODE_LIST);
GraphEdge *graph_edges_ptr =
(GraphEdge *)(shared_memory + data_layout->GetGraphEdgeListOffset());
data_layout->GetBlockPtr<GraphEdge>(shared_memory, SharedDataLayout::GRAPH_EDGE_LIST);
typename ShM<GraphNode, true>::vector node_list(graph_nodes_ptr,
data_layout->graph_node_list_size);
data_layout->num_entries[SharedDataLayout::GRAPH_NODE_LIST]);
typename ShM<GraphEdge, true>::vector edge_list(graph_edges_ptr,
data_layout->graph_edge_list_size);
data_layout->num_entries[SharedDataLayout::GRAPH_EDGE_LIST]);
m_query_graph.reset(new QueryGraph(node_list, edge_list));
}
@ -128,63 +138,62 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
{
FixedPointCoordinate *coordinate_list_ptr =
(FixedPointCoordinate *)(shared_memory + data_layout->GetCoordinateListOffset());
data_layout->GetBlockPtr<FixedPointCoordinate>(shared_memory, SharedDataLayout::COORDINATE_LIST);
m_coordinate_list = std::make_shared<ShM<FixedPointCoordinate, true>::vector>(
coordinate_list_ptr, data_layout->coordinate_list_size);
coordinate_list_ptr, data_layout->num_entries[SharedDataLayout::COORDINATE_LIST]);
TurnInstruction *turn_instruction_list_ptr =
(TurnInstruction *)(shared_memory + data_layout->GetTurnInstructionListOffset());
data_layout->GetBlockPtr<TurnInstruction>(shared_memory, SharedDataLayout::TURN_INSTRUCTION);
typename ShM<TurnInstruction, true>::vector turn_instruction_list(
turn_instruction_list_ptr, data_layout->turn_instruction_list_size);
turn_instruction_list_ptr, data_layout->num_entries[SharedDataLayout::TURN_INSTRUCTION]);
m_turn_instruction_list.swap(turn_instruction_list);
unsigned *name_id_list_ptr =
(unsigned *)(shared_memory + data_layout->GetNameIDListOffset());
unsigned *name_id_list_ptr = data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::NAME_ID_LIST);
typename ShM<unsigned, true>::vector name_id_list(name_id_list_ptr,
data_layout->name_id_list_size);
data_layout->num_entries[SharedDataLayout::NAME_ID_LIST]);
m_name_ID_list.swap(name_id_list);
}
void LoadViaNodeList()
{
NodeID *via_node_list_ptr = (NodeID *)(shared_memory + data_layout->GetViaNodeListOffset());
NodeID *via_node_list_ptr = data_layout->GetBlockPtr<NodeID>(shared_memory, SharedDataLayout::VIA_NODE_LIST);
typename ShM<NodeID, true>::vector via_node_list(via_node_list_ptr,
data_layout->via_node_list_size);
data_layout->num_entries[SharedDataLayout::VIA_NODE_LIST]);
m_via_node_list.swap(via_node_list);
}
void LoadNames()
{
unsigned *street_names_index_ptr =
(unsigned *)(shared_memory + data_layout->GetNameIndexOffset());
typename ShM<unsigned, true>::vector name_begin_indices(street_names_index_ptr,
data_layout->name_index_list_size);
m_name_begin_indices.swap(name_begin_indices);
unsigned *offsets_ptr = data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::NAME_OFFSETS);
NameIndexBlock *blocks_ptr = data_layout->GetBlockPtr<NameIndexBlock>(shared_memory, SharedDataLayout::NAME_BLOCKS);
typename ShM<unsigned, true>::vector name_offsets(offsets_ptr,
data_layout->num_entries[SharedDataLayout::NAME_OFFSETS]);
typename ShM<NameIndexBlock, true>::vector name_blocks(blocks_ptr,
data_layout->num_entries[SharedDataLayout::NAME_BLOCKS]);
char *names_list_ptr = (char *)(shared_memory + data_layout->GetNameListOffset());
char *names_list_ptr = data_layout->GetBlockPtr<char>(shared_memory, SharedDataLayout::NAME_CHAR_LIST);
typename ShM<char, true>::vector names_char_list(names_list_ptr,
data_layout->name_char_list_size);
data_layout->num_entries[SharedDataLayout::NAME_CHAR_LIST]);
m_name_table = std::make_shared<RangeTable<16, true>>(name_offsets, name_blocks, names_char_list.size());
m_names_char_list.swap(names_char_list);
}
void LoadGeometries()
{
unsigned *geometries_compressed_ptr =
(unsigned *)(shared_memory + data_layout->GetGeometriesIndicatorOffset());
unsigned *geometries_compressed_ptr = data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::GEOMETRIES_INDICATORS);
typename ShM<bool, true>::vector egde_is_compressed(geometries_compressed_ptr,
data_layout->geometries_indicators);
data_layout->num_entries[SharedDataLayout::GEOMETRIES_INDICATORS]);
m_egde_is_compressed.swap(egde_is_compressed);
unsigned *geometries_index_ptr =
(unsigned *)(shared_memory + data_layout->GetGeometriesIndexListOffset());
unsigned *geometries_index_ptr = data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::GEOMETRIES_INDEX);
typename ShM<unsigned, true>::vector geometry_begin_indices(
geometries_index_ptr, data_layout->geometries_index_list_size);
geometries_index_ptr, data_layout->num_entries[SharedDataLayout::GEOMETRIES_INDEX]);
m_geometry_indices.swap(geometry_begin_indices);
unsigned *geometries_list_ptr =
(unsigned *)(shared_memory + data_layout->GetGeometryListOffset());
unsigned *geometries_list_ptr = data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::GEOMETRIES_LIST);
typename ShM<unsigned, true>::vector geometry_list(geometries_list_ptr,
data_layout->geometries_list_size);
data_layout->num_entries[SharedDataLayout::GEOMETRIES_LIST]);
m_geometry_list.swap(geometry_list);
}
@ -219,21 +228,28 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
m_layout_memory.reset(SharedMemoryFactory::Get(CURRENT_LAYOUT));
data_layout = (SharedDataLayout *)(m_layout_memory->Ptr());
boost::filesystem::path ram_index_path(data_layout->ram_index_file_name);
if (!boost::filesystem::exists(ram_index_path))
{
throw OSRMException("no leaf index file given. "
"Is any data loaded into shared memory?");
}
m_large_memory.reset(SharedMemoryFactory::Get(CURRENT_DATA));
shared_memory = (char *)(m_large_memory->Ptr());
std::ofstream out("debug.bin");
out.write(shared_memory, data_layout->GetSizeOfLayout());
out.close();
const char* file_index_ptr = data_layout->GetBlockPtr<char>(shared_memory, SharedDataLayout::FILE_INDEX_PATH);
boost::filesystem::path file_index_path(file_index_ptr);
if (!boost::filesystem::exists(file_index_path))
{
SimpleLogger().Write(logDEBUG) << "Leaf file name " << file_index_path.string();
throw OSRMException("Could not load leaf index file."
"Is any data loaded into shared memory?");
}
LoadGraph();
LoadChecksum();
LoadNodeAndEdgeInformation();
LoadGeometries();
LoadRTree(ram_index_path);
LoadRTree(file_index_path);
LoadTimestamp();
LoadViaNodeList();
LoadNames();
@ -339,18 +355,16 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
result = "";
return;
}
BOOST_ASSERT_MSG(name_id < m_name_begin_indices.size(), "name id too high");
const unsigned begin_index = m_name_begin_indices[name_id];
const unsigned end_index = m_name_begin_indices[name_id + 1];
BOOST_ASSERT_MSG(begin_index <= m_names_char_list.size(), "begin index of name too high");
BOOST_ASSERT_MSG(end_index <= m_names_char_list.size(), "end index of name too high");
auto range = m_name_table->GetRange(name_id);
BOOST_ASSERT_MSG(begin_index <= end_index, "string ends before begin");
result.clear();
result.resize(end_index - begin_index);
std::copy(m_names_char_list.begin() + begin_index,
m_names_char_list.begin() + end_index,
result.begin());
if (range.begin() != range.end())
{
result.resize(range.back() - range.front() + 1);
std::copy(m_names_char_list.begin() + range.front(),
m_names_char_list.begin() + range.back() + 1,
result.begin());
}
}
std::string GetTimestamp() const { return m_timestamp; }

317
Server/DataStructures/SharedDataType.h
View File

@ -28,230 +28,145 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef SHARED_DATA_TYPE_H_
#define SHARED_DATA_TYPE_H_
#include "BaseDataFacade.h"
#include "../../DataStructures/QueryEdge.h"
#include "../../DataStructures/StaticGraph.h"
#include "../../DataStructures/StaticRTree.h"
#include "../../DataStructures/TurnInstructions.h"
#include "../../typedefs.h"
#include <osrm/Coordinate.h>
#include "../../Util/SimpleLogger.h"
#include <cstdint>
typedef BaseDataFacade<QueryEdge::EdgeData>::RTreeLeaf RTreeLeaf;
typedef StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>::TreeNode RTreeNode;
typedef StaticGraph<QueryEdge::EdgeData> QueryGraph;
#include <array>
// Added at the start and end of each block as sanity check
constexpr char CANARY[] = "OSRM";
struct SharedDataLayout
{
uint64_t name_index_list_size;
uint64_t name_char_list_size;
uint64_t name_id_list_size;
uint64_t via_node_list_size;
uint64_t graph_node_list_size;
uint64_t graph_edge_list_size;
uint64_t coordinate_list_size;
uint64_t turn_instruction_list_size;
uint64_t r_search_tree_size;
uint64_t geometries_index_list_size;
uint64_t geometries_list_size;
uint64_t geometries_indicators;
enum BlockID {
NAME_OFFSETS = 0,
NAME_BLOCKS,
NAME_CHAR_LIST,
NAME_ID_LIST,
VIA_NODE_LIST,
GRAPH_NODE_LIST,
GRAPH_EDGE_LIST,
COORDINATE_LIST,
TURN_INSTRUCTION,
R_SEARCH_TREE,
GEOMETRIES_INDEX,
GEOMETRIES_LIST,
GEOMETRIES_INDICATORS,
HSGR_CHECKSUM,
TIMESTAMP,
FILE_INDEX_PATH,
NUM_BLOCKS
};
unsigned checksum;
unsigned timestamp_length;
char ram_index_file_name[1024];
std::array<uint64_t, NUM_BLOCKS> num_entries;
std::array<uint64_t, NUM_BLOCKS> entry_size;
SharedDataLayout()
: name_index_list_size(0), name_char_list_size(0), name_id_list_size(0),
via_node_list_size(0), graph_node_list_size(0), graph_edge_list_size(0),
coordinate_list_size(0), turn_instruction_list_size(0), r_search_tree_size(0),
geometries_index_list_size(0), geometries_list_size(0), geometries_indicators(0),
checksum(0), timestamp_length(0)
: num_entries()
, entry_size()
{
ram_index_file_name[0] = '\0';
}
void PrintInformation() const
{
SimpleLogger().Write(logDEBUG) << "-";
SimpleLogger().Write(logDEBUG) << "name_index_list_size: " << name_index_list_size;
SimpleLogger().Write(logDEBUG) << "name_char_list_size: " << name_char_list_size;
SimpleLogger().Write(logDEBUG) << "name_id_list_size: " << name_id_list_size;
SimpleLogger().Write(logDEBUG) << "via_node_list_size: " << via_node_list_size;
SimpleLogger().Write(logDEBUG) << "graph_node_list_size: " << graph_node_list_size;
SimpleLogger().Write(logDEBUG) << "graph_edge_list_size: " << graph_edge_list_size;
SimpleLogger().Write(logDEBUG) << "timestamp_length: " << timestamp_length;
SimpleLogger().Write(logDEBUG) << "coordinate_list_size: " << coordinate_list_size;
SimpleLogger().Write(logDEBUG)
<< "turn_instruction_list_size: " << turn_instruction_list_size;
SimpleLogger().Write(logDEBUG) << "r_search_tree_size: " << r_search_tree_size;
SimpleLogger().Write(logDEBUG) << "geometries_indicators: " << geometries_indicators
<< "/" << ((geometries_indicators / 8) + 1);
SimpleLogger().Write(logDEBUG)
<< "geometries_index_list_size: " << geometries_index_list_size;
SimpleLogger().Write(logDEBUG) << "geometries_list_size: " << geometries_list_size;
SimpleLogger().Write(logDEBUG) << "checksum: " << checksum;
SimpleLogger().Write(logDEBUG) << "sizeof(checksum): " << sizeof(checksum);
SimpleLogger().Write(logDEBUG) << "ram index file name: " << ram_index_file_name;
SimpleLogger().Write(logDEBUG) << "name_offsets_size: " << num_entries[NAME_OFFSETS];
SimpleLogger().Write(logDEBUG) << "name_blocks_size: " << num_entries[NAME_BLOCKS];
SimpleLogger().Write(logDEBUG) << "name_char_list_size: " << num_entries[NAME_CHAR_LIST];
SimpleLogger().Write(logDEBUG) << "name_id_list_size: " << num_entries[NAME_ID_LIST];
SimpleLogger().Write(logDEBUG) << "via_node_list_size: " << num_entries[VIA_NODE_LIST];
SimpleLogger().Write(logDEBUG) << "graph_node_list_size: " << num_entries[GRAPH_NODE_LIST];
SimpleLogger().Write(logDEBUG) << "graph_edge_list_size: " << num_entries[GRAPH_EDGE_LIST];
SimpleLogger().Write(logDEBUG) << "timestamp_length: " << num_entries[TIMESTAMP];
SimpleLogger().Write(logDEBUG) << "coordinate_list_size: " << num_entries[COORDINATE_LIST];
SimpleLogger().Write(logDEBUG) << "turn_instruction_list_size: " << num_entries[TURN_INSTRUCTION];
SimpleLogger().Write(logDEBUG) << "r_search_tree_size: " << num_entries[R_SEARCH_TREE];
SimpleLogger().Write(logDEBUG) << "geometries_indicators: " << num_entries[GEOMETRIES_INDICATORS]
<< "/" << ((num_entries[GEOMETRIES_INDICATORS] / 8) + 1);
SimpleLogger().Write(logDEBUG) << "geometries_index_list_size: " << num_entries[GEOMETRIES_INDEX];
SimpleLogger().Write(logDEBUG) << "geometries_list_size: " << num_entries[GEOMETRIES_LIST];
SimpleLogger().Write(logDEBUG) << "sizeof(checksum): " << entry_size[HSGR_CHECKSUM];
SimpleLogger().Write(logDEBUG) << "NAME_OFFSETS " << ": " << GetBlockSize(NAME_OFFSETS );
SimpleLogger().Write(logDEBUG) << "NAME_BLOCKS " << ": " << GetBlockSize(NAME_BLOCKS );
SimpleLogger().Write(logDEBUG) << "NAME_CHAR_LIST " << ": " << GetBlockSize(NAME_CHAR_LIST );
SimpleLogger().Write(logDEBUG) << "NAME_ID_LIST " << ": " << GetBlockSize(NAME_ID_LIST );
SimpleLogger().Write(logDEBUG) << "VIA_NODE_LIST " << ": " << GetBlockSize(VIA_NODE_LIST );
SimpleLogger().Write(logDEBUG) << "GRAPH_NODE_LIST " << ": " << GetBlockSize(GRAPH_NODE_LIST );
SimpleLogger().Write(logDEBUG) << "GRAPH_EDGE_LIST " << ": " << GetBlockSize(GRAPH_EDGE_LIST );
SimpleLogger().Write(logDEBUG) << "COORDINATE_LIST " << ": " << GetBlockSize(COORDINATE_LIST );
SimpleLogger().Write(logDEBUG) << "TURN_INSTRUCTION " << ": " << GetBlockSize(TURN_INSTRUCTION );
SimpleLogger().Write(logDEBUG) << "R_SEARCH_TREE " << ": " << GetBlockSize(R_SEARCH_TREE );
SimpleLogger().Write(logDEBUG) << "GEOMETRIES_INDEX " << ": " << GetBlockSize(GEOMETRIES_INDEX );
SimpleLogger().Write(logDEBUG) << "GEOMETRIES_LIST " << ": " << GetBlockSize(GEOMETRIES_LIST );
SimpleLogger().Write(logDEBUG) << "GEOMETRIES_INDICATORS" << ": " << GetBlockSize(GEOMETRIES_INDICATORS);
SimpleLogger().Write(logDEBUG) << "HSGR_CHECKSUM " << ": " << GetBlockSize(HSGR_CHECKSUM );
SimpleLogger().Write(logDEBUG) << "TIMESTAMP " << ": " << GetBlockSize(TIMESTAMP );
SimpleLogger().Write(logDEBUG) << "FILE_INDEX_PATH " << ": " << GetBlockSize(FILE_INDEX_PATH );
}
uint64_t GetSizeOfLayout() const
template<typename T>
inline void SetBlockSize(BlockID bid, uint64_t entries)
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode)) +
(geometries_indicators / 32 + 1) * sizeof(unsigned) +
(geometries_index_list_size * sizeof(unsigned)) +
(geometries_list_size * sizeof(unsigned)) + sizeof(checksum) + 1024 * sizeof(char);
num_entries[bid] = entries;
entry_size[bid] = sizeof(T);
}
inline uint64_t GetBlockSize(BlockID bid) const
{
// special encoding
if (bid == GEOMETRIES_INDICATORS)
{
return (num_entries[GEOMETRIES_INDICATORS]/32 + 1) * entry_size[GEOMETRIES_INDICATORS];
}
return num_entries[bid] * entry_size[bid];
}
inline uint64_t GetSizeOfLayout() const
{
return GetBlockOffset(NUM_BLOCKS) + NUM_BLOCKS*2*sizeof(CANARY);
}
inline uint64_t GetBlockOffset(BlockID bid) const
{
uint64_t result = sizeof(CANARY);
for (unsigned i = 0; i < bid; i++)
{
result += GetBlockSize((BlockID) i) + 2*sizeof(CANARY);
}
return result;
}
uint64_t GetNameIndexOffset() const { return 0; }
uint64_t GetNameListOffset() const
template<typename T, bool WRITE_CANARY=false>
inline T* GetBlockPtr(char* shared_memory, BlockID bid)
{
uint64_t result = (name_index_list_size * sizeof(unsigned));
return result;
}
uint64_t GetNameIDListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char));
return result;
}
uint64_t GetViaNodeListOffset() const
{
uint64_t result = (name_index_list_size * sizeof(unsigned)) +
(name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned));
return result;
}
uint64_t GetGraphNodeListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID));
return result;
}
uint64_t GetGraphEdgeListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry));
return result;
}
uint64_t GetTimeStampOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry));
return result;
}
uint64_t GetCoordinateListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char));
return result;
}
uint64_t GetTurnInstructionListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate));
return result;
}
uint64_t GetRSearchTreeOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass));
return result;
}
uint64_t GetGeometriesIndicatorOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode));
return result;
}
T* ptr = (T*)(shared_memory + GetBlockOffset(bid));
if (WRITE_CANARY)
{
char* start_canary_ptr = shared_memory + GetBlockOffset(bid) - sizeof(CANARY);
char* end_canary_ptr = shared_memory + GetBlockOffset(bid) + GetBlockSize(bid);
std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
}
else
{
char* start_canary_ptr = shared_memory + GetBlockOffset(bid) - sizeof(CANARY);
char* end_canary_ptr = shared_memory + GetBlockOffset(bid) + GetBlockSize(bid);
bool start_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
bool end_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
if (!start_canary_alive)
{
throw OSRMException("Start canary of block corrupted.");
}
if (!end_canary_alive)
{
throw OSRMException("End canary of block corrupted.");
}
}
uint64_t GetGeometriesIndexListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode)) +
(geometries_indicators / 32 + 1) * sizeof(unsigned);
return result;
}
uint64_t GetGeometryListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode)) +
(geometries_indicators / 32 + 1) * sizeof(unsigned) +
(geometries_index_list_size * sizeof(unsigned));
return result;
}
uint64_t GetChecksumOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode)) +
(geometries_indicators / 32 + 1) * sizeof(unsigned) +
(geometries_index_list_size * sizeof(unsigned)) +
(geometries_list_size * sizeof(unsigned));
return result;
return ptr;
}
};

179
datastore.cpp
View File

@ -26,7 +26,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "DataStructures/OriginalEdgeData.h"
#include "DataStructures/RangeTable.h"
#include "DataStructures/QueryEdge.h"
#include "DataStructures/SharedMemoryFactory.h"
#include "DataStructures/SharedMemoryVectorWrapper.h"
#include "DataStructures/StaticGraph.h"
#include "DataStructures/StaticRTree.h"
#include "DataStructures/TurnInstructions.h"
#include "Server/DataStructures/BaseDataFacade.h"
#include "Server/DataStructures/SharedDataType.h"
#include "Server/DataStructures/SharedBarriers.h"
#include "Util/BoostFileSystemFix.h"
@ -35,6 +42,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "Util/FingerPrint.h"
#include "typedefs.h"
#include <osrm/Coordinate.h>
typedef BaseDataFacade<QueryEdge::EdgeData>::RTreeLeaf RTreeLeaf;
typedef StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>::TreeNode RTreeNode;
typedef StaticGraph<QueryEdge::EdgeData> QueryGraph;
#ifdef __linux__
#include <sys/mman.h>
#endif
@ -161,7 +174,7 @@ int main(const int argc, const char *argv[])
BOOST_ASSERT(!paths_iterator->second.empty());
const boost::filesystem::path index_file_path_absolute =
boost::filesystem::portable_canonical(paths_iterator->second);
const std::string &file_index_file_name = index_file_path_absolute.string();
const std::string &file_index_path = index_file_path_absolute.string();
paths_iterator = server_paths.find("nodesdata");
BOOST_ASSERT(server_paths.end() != paths_iterator);
BOOST_ASSERT(!paths_iterator->second.empty());
@ -201,37 +214,34 @@ int main(const int argc, const char *argv[])
SharedDataLayout *shared_layout_ptr = static_cast<SharedDataLayout *>(layout_memory->Ptr());
shared_layout_ptr = new (layout_memory->Ptr()) SharedDataLayout();
std::copy(file_index_file_name.begin(),
(file_index_file_name.length() <= 1024 ? file_index_file_name.end()
: file_index_file_name.begin() + 1023),
shared_layout_ptr->ram_index_file_name);
// add zero termination
unsigned end_of_string_index = std::min((std::size_t)1023, file_index_file_name.length());
shared_layout_ptr->ram_index_file_name[end_of_string_index] = '\0';
shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::FILE_INDEX_PATH, file_index_path.length() + 1);
// collect number of elements to store in shared memory object
SimpleLogger().Write() << "load names from: " << names_data_path;
// number of entries in name index
boost::filesystem::ifstream name_stream(names_data_path, std::ios::binary);
unsigned name_index_size = 0;
name_stream.read((char *)&name_index_size, sizeof(unsigned));
shared_layout_ptr->name_index_list_size = name_index_size;
SimpleLogger().Write() << "size: " << name_index_size;
BOOST_ASSERT_MSG(0 != shared_layout_ptr->name_index_list_size, "name file broken");
unsigned name_blocks = 0;
name_stream.read((char *)&name_blocks, sizeof(unsigned));
shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::NAME_OFFSETS, name_blocks);
shared_layout_ptr->SetBlockSize<typename RangeTable<16, true>::BlockT>(SharedDataLayout::NAME_BLOCKS, name_blocks);
SimpleLogger().Write() << "name offsets size: " << name_blocks;
BOOST_ASSERT_MSG(0 != name_blocks, "name file broken");
unsigned number_of_chars = 0;
name_stream.read((char *)&number_of_chars, sizeof(unsigned));
shared_layout_ptr->name_char_list_size = number_of_chars;
shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::NAME_CHAR_LIST, number_of_chars);
// Loading information for original edges
boost::filesystem::ifstream edges_input_stream(edges_data_path, std::ios::binary);
unsigned number_of_original_edges = 0;
edges_input_stream.read((char *)&number_of_original_edges, sizeof(unsigned));
shared_layout_ptr->via_node_list_size = number_of_original_edges;
shared_layout_ptr->name_id_list_size = number_of_original_edges;
shared_layout_ptr->turn_instruction_list_size = number_of_original_edges;
shared_layout_ptr->geometries_indicators = number_of_original_edges;
// note: settings this all to the same size is correct, we extract them from the same struct
shared_layout_ptr->SetBlockSize<NodeID>(SharedDataLayout::VIA_NODE_LIST, number_of_original_edges);
shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::NAME_ID_LIST, number_of_original_edges);
shared_layout_ptr->SetBlockSize<TurnInstruction>(SharedDataLayout::TURN_INSTRUCTION, number_of_original_edges);
// note: there are 32 geometry indicators in one unsigned block
shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::GEOMETRIES_INDICATORS, number_of_original_edges);
boost::filesystem::ifstream hsgr_input_stream(hsgr_path, std::ios::binary);
@ -250,26 +260,26 @@ int main(const int argc, const char *argv[])
// load checksum
unsigned checksum = 0;
hsgr_input_stream.read((char *)&checksum, sizeof(unsigned));
shared_layout_ptr->checksum = checksum;
shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::HSGR_CHECKSUM, 1);
// load graph node size
unsigned number_of_graph_nodes = 0;
hsgr_input_stream.read((char *)&number_of_graph_nodes, sizeof(unsigned));
BOOST_ASSERT_MSG((0 != number_of_graph_nodes), "number of nodes is zero");
shared_layout_ptr->graph_node_list_size = number_of_graph_nodes;
shared_layout_ptr->SetBlockSize<QueryGraph::NodeArrayEntry>(SharedDataLayout::GRAPH_NODE_LIST, number_of_graph_nodes);
// load graph edge size
unsigned number_of_graph_edges = 0;
hsgr_input_stream.read((char *)&number_of_graph_edges, sizeof(unsigned));
BOOST_ASSERT_MSG(0 != number_of_graph_edges, "number of graph edges is zero");
shared_layout_ptr->graph_edge_list_size = number_of_graph_edges;
shared_layout_ptr->SetBlockSize<QueryGraph::EdgeArrayEntry>(SharedDataLayout::GRAPH_EDGE_LIST, number_of_graph_edges);
// load rsearch tree size
boost::filesystem::ifstream tree_node_file(ram_index_path, std::ios::binary);
uint32_t tree_size = 0;
tree_node_file.read((char *)&tree_size, sizeof(uint32_t));
shared_layout_ptr->r_search_tree_size = tree_size;
shared_layout_ptr->SetBlockSize<RTreeNode>(SharedDataLayout::R_SEARCH_TREE, tree_size);
// load timestamp size
std::string m_timestamp;
@ -295,13 +305,13 @@ int main(const int argc, const char *argv[])
{
m_timestamp.resize(25);
}
shared_layout_ptr->timestamp_length = m_timestamp.length();
shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::TIMESTAMP, m_timestamp.length());
// load coordinate size
boost::filesystem::ifstream nodes_input_stream(nodes_data_path, std::ios::binary);
unsigned coordinate_list_size = 0;
nodes_input_stream.read((char *)&coordinate_list_size, sizeof(unsigned));
shared_layout_ptr->coordinate_list_size = coordinate_list_size;
shared_layout_ptr->SetBlockSize<FixedPointCoordinate>(SharedDataLayout::COORDINATE_LIST, coordinate_list_size);
// load geometries sizes
std::ifstream geometry_input_stream(geometries_data_path.string().c_str(), std::ios::binary);
@ -309,11 +319,11 @@ int main(const int argc, const char *argv[])
unsigned number_of_compressed_geometries = 0;
geometry_input_stream.read((char *)&number_of_geometries_indices, sizeof(unsigned));
shared_layout_ptr->geometries_index_list_size = number_of_geometries_indices;
shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::GEOMETRIES_INDEX, number_of_geometries_indices);
boost::iostreams::seek(
geometry_input_stream, number_of_geometries_indices * sizeof(unsigned), BOOST_IOS::cur);
geometry_input_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned));
shared_layout_ptr->geometries_list_size = number_of_compressed_geometries;
shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::GEOMETRIES_LIST, number_of_compressed_geometries);
// allocate shared memory block
SimpleLogger().Write() << "allocating shared memory of "
@ -323,35 +333,70 @@ int main(const int argc, const char *argv[])
char *shared_memory_ptr = static_cast<char *>(shared_memory->Ptr());
// read actual data into shared memory object //
// hsgr checksum
unsigned* checksum_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr,
SharedDataLayout::HSGR_CHECKSUM);
*checksum_ptr = checksum;
// ram index file name
char* file_index_path_ptr = shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr,
SharedDataLayout::FILE_INDEX_PATH);
// make sure we have 0 ending
std::fill(file_index_path_ptr,
file_index_path_ptr +
shared_layout_ptr->GetBlockSize(SharedDataLayout::FILE_INDEX_PATH),
0);
std::copy(file_index_path.begin(), file_index_path.end(), file_index_path_ptr);
// Loading street names
unsigned *name_index_ptr =
(unsigned *)(shared_memory_ptr + shared_layout_ptr->GetNameIndexOffset());
if (shared_layout_ptr->name_index_list_size > 0)
unsigned *name_offsets_ptr =
shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr,
SharedDataLayout::NAME_OFFSETS);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_OFFSETS) > 0)
{
name_stream.read((char *)name_index_ptr,
shared_layout_ptr->name_index_list_size * sizeof(unsigned));
name_stream.read((char *)name_offsets_ptr,
shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_OFFSETS));
}
char *name_char_ptr = shared_memory_ptr + shared_layout_ptr->GetNameListOffset();
if (shared_layout_ptr->name_char_list_size > 0)
unsigned *name_blocks_ptr =
shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr,
SharedDataLayout::NAME_BLOCKS);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_BLOCKS) > 0)
{
name_stream.read(name_char_ptr, shared_layout_ptr->name_char_list_size * sizeof(char));
name_stream.read((char *)name_blocks_ptr,
shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_BLOCKS));
}
char *name_char_ptr =
shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr,
SharedDataLayout::NAME_CHAR_LIST);
unsigned temp_length;
name_stream.read((char*) &temp_length, sizeof(unsigned));
BOOST_ASSERT_MSG(temp_length == shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST),
"Name file corrupted!");
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST) > 0)
{
name_stream.read(name_char_ptr, shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST));
}
name_stream.close();
// load original edge information
NodeID *via_node_ptr =
(NodeID *)(shared_memory_ptr + shared_layout_ptr->GetViaNodeListOffset());
NodeID *via_node_ptr = shared_layout_ptr->GetBlockPtr<NodeID, true>(shared_memory_ptr,
SharedDataLayout::VIA_NODE_LIST);
unsigned *name_id_ptr =
(unsigned *)(shared_memory_ptr + shared_layout_ptr->GetNameIDListOffset());
unsigned *name_id_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr,
SharedDataLayout::NAME_ID_LIST);
TurnInstruction *turn_instructions_ptr =
(TurnInstruction *)(shared_memory_ptr +
shared_layout_ptr->GetTurnInstructionListOffset());
shared_layout_ptr->GetBlockPtr<TurnInstruction, true>(shared_memory_ptr,
SharedDataLayout::TURN_INSTRUCTION);
unsigned *geometries_indicator_ptr =
(unsigned *)(shared_memory_ptr + shared_layout_ptr->GetGeometriesIndicatorOffset());
unsigned *geometries_indicator_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr,
SharedDataLayout::GEOMETRIES_INDICATORS);
OriginalEdgeData current_edge_data;
for (unsigned i = 0; i < number_of_original_edges; ++i)
@ -382,33 +427,32 @@ int main(const int argc, const char *argv[])
// load compressed geometry
unsigned temporary_value;
unsigned *geometries_index_ptr =
(unsigned *)(shared_memory_ptr + shared_layout_ptr->GetGeometriesIndexListOffset());
shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr, SharedDataLayout::GEOMETRIES_INDEX);
geometry_input_stream.seekg(0, geometry_input_stream.beg);
geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned));
BOOST_ASSERT(temporary_value == shared_layout_ptr->geometries_index_list_size);
if (shared_layout_ptr->geometries_index_list_size > 0)
BOOST_ASSERT(temporary_value == shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_INDEX]);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_INDEX) > 0)
{
geometry_input_stream.read((char *)geometries_index_ptr,
shared_layout_ptr->geometries_index_list_size *
sizeof(unsigned));
shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_INDEX));
}
unsigned *geometries_list_ptr =
(unsigned *)(shared_memory_ptr + shared_layout_ptr->GetGeometryListOffset());
shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr, SharedDataLayout::GEOMETRIES_LIST);
geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned));
BOOST_ASSERT(temporary_value == shared_layout_ptr->geometries_list_size);
BOOST_ASSERT(temporary_value == shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_LIST]);
if (shared_layout_ptr->geometries_list_size > 0)
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_LIST) > 0)
{
geometry_input_stream.read((char *)geometries_list_ptr,
shared_layout_ptr->geometries_list_size * sizeof(unsigned));
shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_LIST));
}
// Loading list of coordinates
FixedPointCoordinate *coordinates_ptr =
(FixedPointCoordinate *)(shared_memory_ptr +
shared_layout_ptr->GetCoordinateListOffset());
shared_layout_ptr->GetBlockPtr<FixedPointCoordinate, true>(shared_memory_ptr,
SharedDataLayout::COORDINATE_LIST);
NodeInfo current_node;
for (unsigned i = 0; i < coordinate_list_size; ++i)
@ -419,40 +463,37 @@ int main(const int argc, const char *argv[])
nodes_input_stream.close();
// store timestamp
char *timestamp_ptr =
static_cast<char *>(shared_memory_ptr + shared_layout_ptr->GetTimeStampOffset());
char *timestamp_ptr = shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr, SharedDataLayout::TIMESTAMP);
std::copy(m_timestamp.c_str(), m_timestamp.c_str() + m_timestamp.length(), timestamp_ptr);
// store search tree portion of rtree
char *rtree_ptr =
static_cast<char *>(shared_memory_ptr + shared_layout_ptr->GetRSearchTreeOffset());
char *rtree_ptr = shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr, SharedDataLayout::R_SEARCH_TREE);
if (tree_size > 0)
{
tree_node_file.read(rtree_ptr, sizeof(RTreeNode) * tree_size);
tree_node_file.read(rtree_ptr, sizeof(RTreeNode) * tree_size);
}
tree_node_file.close();
// load the nodes of the search graph
QueryGraph::NodeArrayEntry *graph_node_list_ptr =
(QueryGraph::NodeArrayEntry *)(shared_memory_ptr +
shared_layout_ptr->GetGraphNodeListOffset());
if (shared_layout_ptr->graph_node_list_size > 0)
shared_layout_ptr->GetBlockPtr<QueryGraph::NodeArrayEntry, true>(shared_memory_ptr,
SharedDataLayout::GRAPH_NODE_LIST);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_NODE_LIST) > 0)
{
hsgr_input_stream.read((char *)graph_node_list_ptr,
shared_layout_ptr->graph_node_list_size *
sizeof(QueryGraph::NodeArrayEntry));
shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_NODE_LIST));
}
// load the edges of the search graph
QueryGraph::EdgeArrayEntry *graph_edge_list_ptr =
(QueryGraph::EdgeArrayEntry *)(shared_memory_ptr +
shared_layout_ptr->GetGraphEdgeListOffset());
if (shared_layout_ptr->graph_edge_list_size > 0)
shared_layout_ptr->GetBlockPtr<QueryGraph::EdgeArrayEntry, true>(shared_memory_ptr,
SharedDataLayout::GRAPH_EDGE_LIST);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST) > 0)
{
hsgr_input_stream.read((char *)graph_edge_list_ptr,
shared_layout_ptr->graph_edge_list_size *
sizeof(QueryGraph::EdgeArrayEntry));
shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST));
}
hsgr_input_stream.close();