Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Initial support for SharedDataFacade

Browse Source 
SharedDataLayout was refactored to include canary values at the
boundaries of each memory block. This makes it easy to detect overruns
and block-size mismatches between osrm-datastore and the
SharedDataFacade.
This commit is contained in:
Patrick Niklaus 2014-06-05 17:18:28 +02:00
parent 7a7d0c09d9
commit 807f1d7c1c
7 changed files with 320 additions and 338 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
DataStructures/RangeTable.h
View File

@ -1,6 +1,9 @@
#ifndef __RANGE_TABLE_H__
#define __RANGE_TABLE_H__
#include <boost/range/irange.hpp>
#include <fstream>
#include <vector>
#if defined(__GNUC__) && defined(__SSE2__)
@ -8,6 +11,9 @@
#include <xmmintrin.h>
#endif
#include "SharedMemoryFactory.h"
#include "SharedMemoryVectorWrapper.h"
/*
* These pre-declarations are needed because parsing C++ is hard
* and otherwise the compiler gets confused.
@ -21,9 +27,6 @@ std::ostream& operator<<(std::ostream &out, const RangeTable<BLOCK_SIZE, USE_SHA
template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY>
std::istream& operator>>(std::istream &in, RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table);
#include "SharedMemoryFactory.h"
#include "SharedMemoryVectorWrapper.h"
/**
* Stores adjacent ranges in a compressed format.
*
@ -49,6 +52,7 @@ public:
typedef typename ShM<BlockT, USE_SHARED_MEMORY>::vector BlockContainerT;
typedef typename ShM<unsigned, USE_SHARED_MEMORY>::vector OffsetContainerT;
typedef decltype(boost::irange(0u,0u)) RangeT;
friend std::ostream& operator<< <>(std::ostream &out, const RangeTable &table);
friend std::istream& operator>> <>(std::istream &in, RangeTable &table);
@ -56,7 +60,8 @@ public:
RangeTable() {}
// for loading from shared memory
explicit RangeTable(OffsetContainerT& external_offsets, BlockContainerT& external_blocks)
explicit RangeTable(OffsetContainerT& external_offsets, BlockContainerT& external_blocks, unsigned sum_lengths)
: sum_lengths(sum_lengths)
{
block_offsets.swap(external_offsets);
diff_blocks.swap(external_blocks);
@ -134,7 +139,7 @@ public:
sum_lengths = lengths_prefix_sum;
}
inline void GetRange(const unsigned id, unsigned& begin_idx, unsigned& end_idx) const
inline RangeT GetRange(const unsigned id) const
{
BOOST_ASSERT(id < block_offsets.size() + diff_blocks.size() * BLOCK_SIZE);
// internal_idx 0 is implicitly stored in block_offsets[block_idx]
@ -143,6 +148,8 @@ public:
BOOST_ASSERT(block_idx < diff_blocks.size());
unsigned begin_idx = 0;
unsigned end_idx = 0;
begin_idx = block_offsets[block_idx];
const BlockT& block = diff_blocks[block_idx];
if (internal_idx > 0)
@ -163,6 +170,9 @@ public:
}
BOOST_ASSERT(begin_idx < sum_lengths && end_idx <= sum_lengths);
BOOST_ASSERT(begin_idx <= end_idx);
return boost::irange(begin_idx, end_idx);
}
private:

2
DataStructures/StaticRTree.h
View File

@ -766,7 +766,9 @@ class StaticRTree
}
const uint64_t seek_pos = sizeof(uint64_t) + leaf_id * sizeof(LeafNode);
thread_local_rtree_stream->seekg(seek_pos);
BOOST_ASSERT_MSG(thread_local_rtree_stream->good(), "Seeking to position in leaf file failed.");
thread_local_rtree_stream->read((char *)&result_node, sizeof(LeafNode));
BOOST_ASSERT_MSG(thread_local_rtree_stream->good(), "Reading from leaf file failed.");
}
inline bool EdgesAreEquivalent(const FixedPointCoordinate &a,

2
Extractor/ExtractionContainers.cpp
View File

@ -30,6 +30,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "../Util/OSRMException.h"
#include "../Util/SimpleLogger.h"
#include "../Util/TimingUtil.h"
#include "../DataStructures/RangeTable.h"
#include <boost/assert.hpp>
#include <boost/filesystem.hpp>
@ -64,6 +65,7 @@ void ExtractionContainers::PrepareData(const std::string &output_file_name,
{
unsigned number_of_used_nodes = 0;
unsigned number_of_used_edges = 0;
std::cout << "[extractor] Sorting used nodes ... " << std::flush;
TIMER_START(sorting_used_nodes);
stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), stxxl_memory);

18
Server/DataStructures/InternalDataFacade.h
View File

@ -381,18 +381,16 @@ template <class EdgeDataT> class InternalDataFacade : public BaseDataFacade<Edge
result = "";
return;
}
unsigned begin_index;
unsigned end_index;
m_name_table.GetRange(name_id, begin_index, end_index);
BOOST_ASSERT_MSG(begin_index < m_names_char_list.size(), "begin index of name too high");
BOOST_ASSERT_MSG(end_index < m_names_char_list.size(), "end index of name too high");
auto range = m_name_table.GetRange(name_id);
BOOST_ASSERT_MSG(begin_index <= end_index, "string ends before begin");
result.clear();
result.resize(end_index - begin_index);
std::copy(m_names_char_list.begin() + begin_index,
m_names_char_list.begin() + end_index,
result.begin());
if (range.begin() != range.end())
{
result.resize(range.back() - range.front());
std::copy(m_names_char_list.begin() + range.front(),
m_names_char_list.begin() + range.back(),
result.begin());
}
}
virtual unsigned GetGeometryIndexForEdgeID(const unsigned id) const

120
Server/DataStructures/SharedDataFacade.h
View File

@ -33,6 +33,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "BaseDataFacade.h"
#include "SharedDataType.h"
#include "../../DataStructures/RangeTable.h"
#include "../../DataStructures/StaticGraph.h"
#include "../../DataStructures/StaticRTree.h"
#include "../../Util/BoostFileSystemFix.h"
@ -51,6 +52,7 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
typedef StaticGraph<EdgeData, true> QueryGraph;
typedef typename StaticGraph<EdgeData, true>::NodeArrayEntry GraphNode;
typedef typename StaticGraph<EdgeData, true>::EdgeArrayEntry GraphEdge;
typedef typename RangeTable<16, true>::BlockT NameIndexBlock;
typedef typename QueryGraph::InputEdge InputEdge;
typedef typename super::RTreeLeaf RTreeLeaf;
typedef typename StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>::TreeNode
@ -84,43 +86,51 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
std::shared_ptr<StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>>
m_static_rtree;
std::shared_ptr<RangeTable<16, true>> m_name_table;
void LoadChecksum()
{
m_check_sum = data_layout->checksum;
m_check_sum = *data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::HSGR_CHECKSUM);
SimpleLogger().Write() << "set checksum: " << m_check_sum;
}
void LoadTimestamp()
{
char *timestamp_ptr = shared_memory + data_layout->GetTimeStampOffset();
m_timestamp.resize(data_layout->timestamp_length);
char *timestamp_ptr = data_layout->GetBlockPtr<char>(shared_memory, SharedDataLayout::TIMESTAMP);
m_timestamp.resize(data_layout->GetBlockSize(SharedDataLayout::TIMESTAMP));
std::copy(
timestamp_ptr, timestamp_ptr + data_layout->timestamp_length, m_timestamp.begin());
timestamp_ptr,
timestamp_ptr + data_layout->GetBlockSize(SharedDataLayout::TIMESTAMP),
m_timestamp.begin());
}
void LoadRTree(const boost::filesystem::path &file_index_path)
{
BOOST_ASSERT_MSG(!m_coordinate_list->empty(), "coordinates must be loaded before r-tree");
RTreeNode *tree_ptr = (RTreeNode *)(shared_memory + data_layout->GetRSearchTreeOffset());
RTreeNode *tree_ptr = data_layout->GetBlockPtr<RTreeNode>(shared_memory, SharedDataLayout::R_SEARCH_TREE);
m_static_rtree =
std::make_shared<StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>>(
tree_ptr, data_layout->r_search_tree_size, file_index_path, m_coordinate_list);
tree_ptr,
data_layout->num_entries[SharedDataLayout::R_SEARCH_TREE],
file_index_path,
m_coordinate_list);
}
void LoadGraph()
{
m_number_of_nodes = data_layout->graph_node_list_size;
m_number_of_nodes = data_layout->num_entries[SharedDataLayout::GRAPH_NODE_LIST];
GraphNode *graph_nodes_ptr =
(GraphNode *)(shared_memory + data_layout->GetGraphNodeListOffset());
data_layout->GetBlockPtr<GraphNode>(shared_memory, SharedDataLayout::GRAPH_NODE_LIST);
GraphEdge *graph_edges_ptr =
(GraphEdge *)(shared_memory + data_layout->GetGraphEdgeListOffset());
data_layout->GetBlockPtr<GraphEdge>(shared_memory, SharedDataLayout::GRAPH_EDGE_LIST);
typename ShM<GraphNode, true>::vector node_list(graph_nodes_ptr,
data_layout->graph_node_list_size);
data_layout->num_entries[SharedDataLayout::GRAPH_NODE_LIST]);
typename ShM<GraphEdge, true>::vector edge_list(graph_edges_ptr,
data_layout->graph_edge_list_size);
data_layout->num_entries[SharedDataLayout::GRAPH_EDGE_LIST]);
m_query_graph.reset(new QueryGraph(node_list, edge_list));
}
@ -128,63 +138,62 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
{
FixedPointCoordinate *coordinate_list_ptr =
(FixedPointCoordinate *)(shared_memory + data_layout->GetCoordinateListOffset());
data_layout->GetBlockPtr<FixedPointCoordinate>(shared_memory, SharedDataLayout::COORDINATE_LIST);
m_coordinate_list = std::make_shared<ShM<FixedPointCoordinate, true>::vector>(
coordinate_list_ptr, data_layout->coordinate_list_size);
coordinate_list_ptr, data_layout->num_entries[SharedDataLayout::COORDINATE_LIST]);
TurnInstruction *turn_instruction_list_ptr =
(TurnInstruction *)(shared_memory + data_layout->GetTurnInstructionListOffset());
data_layout->GetBlockPtr<TurnInstruction>(shared_memory, SharedDataLayout::TURN_INSTRUCTION);
typename ShM<TurnInstruction, true>::vector turn_instruction_list(
turn_instruction_list_ptr, data_layout->turn_instruction_list_size);
turn_instruction_list_ptr, data_layout->num_entries[SharedDataLayout::TURN_INSTRUCTION]);
m_turn_instruction_list.swap(turn_instruction_list);
unsigned *name_id_list_ptr =
(unsigned *)(shared_memory + data_layout->GetNameIDListOffset());
unsigned *name_id_list_ptr = data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::NAME_ID_LIST);
typename ShM<unsigned, true>::vector name_id_list(name_id_list_ptr,
data_layout->name_id_list_size);
data_layout->num_entries[SharedDataLayout::NAME_ID_LIST]);
m_name_ID_list.swap(name_id_list);
}
void LoadViaNodeList()
{
NodeID *via_node_list_ptr = (NodeID *)(shared_memory + data_layout->GetViaNodeListOffset());
NodeID *via_node_list_ptr = data_layout->GetBlockPtr<NodeID>(shared_memory, SharedDataLayout::VIA_NODE_LIST);
typename ShM<NodeID, true>::vector via_node_list(via_node_list_ptr,
data_layout->via_node_list_size);
data_layout->num_entries[SharedDataLayout::VIA_NODE_LIST]);
m_via_node_list.swap(via_node_list);
}
void LoadNames()
{
unsigned *street_names_index_ptr =
(unsigned *)(shared_memory + data_layout->GetNameIndexOffset());
typename ShM<unsigned, true>::vector name_begin_indices(street_names_index_ptr,
data_layout->name_index_list_size);
m_name_begin_indices.swap(name_begin_indices);
unsigned *offsets_ptr = data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::NAME_OFFSETS);
NameIndexBlock *blocks_ptr = data_layout->GetBlockPtr<NameIndexBlock>(shared_memory, SharedDataLayout::NAME_BLOCKS);
typename ShM<unsigned, true>::vector name_offsets(offsets_ptr,
data_layout->num_entries[SharedDataLayout::NAME_OFFSETS]);
typename ShM<NameIndexBlock, true>::vector name_blocks(blocks_ptr,
data_layout->num_entries[SharedDataLayout::NAME_BLOCKS]);
char *names_list_ptr = (char *)(shared_memory + data_layout->GetNameListOffset());
char *names_list_ptr = data_layout->GetBlockPtr<char>(shared_memory, SharedDataLayout::NAME_CHAR_LIST);
typename ShM<char, true>::vector names_char_list(names_list_ptr,
data_layout->name_char_list_size);
data_layout->num_entries[SharedDataLayout::NAME_CHAR_LIST]);
m_name_table = std::make_shared<RangeTable<16, true>>(name_offsets, name_blocks, names_char_list.size());
m_names_char_list.swap(names_char_list);
}
void LoadGeometries()
{
unsigned *geometries_compressed_ptr =
(unsigned *)(shared_memory + data_layout->GetGeometriesIndicatorOffset());
unsigned *geometries_compressed_ptr = data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::GEOMETRIES_INDICATORS);
typename ShM<bool, true>::vector egde_is_compressed(geometries_compressed_ptr,
data_layout->geometries_indicators);
data_layout->num_entries[SharedDataLayout::GEOMETRIES_INDICATORS]);
m_egde_is_compressed.swap(egde_is_compressed);
unsigned *geometries_index_ptr =
(unsigned *)(shared_memory + data_layout->GetGeometriesIndexListOffset());
unsigned *geometries_index_ptr = data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::GEOMETRIES_INDEX);
typename ShM<unsigned, true>::vector geometry_begin_indices(
geometries_index_ptr, data_layout->geometries_index_list_size);
geometries_index_ptr, data_layout->num_entries[SharedDataLayout::GEOMETRIES_INDEX]);
m_geometry_indices.swap(geometry_begin_indices);
unsigned *geometries_list_ptr =
(unsigned *)(shared_memory + data_layout->GetGeometryListOffset());
unsigned *geometries_list_ptr = data_layout->GetBlockPtr<unsigned>(shared_memory, SharedDataLayout::GEOMETRIES_LIST);
typename ShM<unsigned, true>::vector geometry_list(geometries_list_ptr,
data_layout->geometries_list_size);
data_layout->num_entries[SharedDataLayout::GEOMETRIES_LIST]);
m_geometry_list.swap(geometry_list);
}
@ -219,21 +228,28 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
m_layout_memory.reset(SharedMemoryFactory::Get(CURRENT_LAYOUT));
data_layout = (SharedDataLayout *)(m_layout_memory->Ptr());
boost::filesystem::path ram_index_path(data_layout->ram_index_file_name);
if (!boost::filesystem::exists(ram_index_path))
{
throw OSRMException("no leaf index file given. "
"Is any data loaded into shared memory?");
}
m_large_memory.reset(SharedMemoryFactory::Get(CURRENT_DATA));
shared_memory = (char *)(m_large_memory->Ptr());
std::ofstream out("debug.bin");
out.write(shared_memory, data_layout->GetSizeOfLayout());
out.close();
const char* file_index_ptr = data_layout->GetBlockPtr<char>(shared_memory, SharedDataLayout::FILE_INDEX_PATH);
boost::filesystem::path file_index_path(file_index_ptr);
if (!boost::filesystem::exists(file_index_path))
{
SimpleLogger().Write(logDEBUG) << "Leaf file name " << file_index_path.string();
throw OSRMException("Could not load leaf index file."
"Is any data loaded into shared memory?");
}
LoadGraph();
LoadChecksum();
LoadNodeAndEdgeInformation();
LoadGeometries();
LoadRTree(ram_index_path);
LoadRTree(file_index_path);
LoadTimestamp();
LoadViaNodeList();
LoadNames();
@ -339,18 +355,16 @@ template <class EdgeDataT> class SharedDataFacade : public BaseDataFacade<EdgeDa
result = "";
return;
}
BOOST_ASSERT_MSG(name_id < m_name_begin_indices.size(), "name id too high");
const unsigned begin_index = m_name_begin_indices[name_id];
const unsigned end_index = m_name_begin_indices[name_id + 1];
BOOST_ASSERT_MSG(begin_index <= m_names_char_list.size(), "begin index of name too high");
BOOST_ASSERT_MSG(end_index <= m_names_char_list.size(), "end index of name too high");
auto range = m_name_table->GetRange(name_id);
BOOST_ASSERT_MSG(begin_index <= end_index, "string ends before begin");
result.clear();
result.resize(end_index - begin_index);
std::copy(m_names_char_list.begin() + begin_index,
m_names_char_list.begin() + end_index,
result.begin());
if (range.begin() != range.end())
{
result.resize(range.back() - range.front());
std::copy(m_names_char_list.begin() + range.front(),
m_names_char_list.begin() + range.back(),
result.begin());
}
}
std::string GetTimestamp() const { return m_timestamp; }

317
Server/DataStructures/SharedDataType.h
View File

@ -28,230 +28,145 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef SHARED_DATA_TYPE_H_
#define SHARED_DATA_TYPE_H_
#include "BaseDataFacade.h"
#include "../../DataStructures/QueryEdge.h"
#include "../../DataStructures/StaticGraph.h"
#include "../../DataStructures/StaticRTree.h"
#include "../../DataStructures/TurnInstructions.h"
#include "../../typedefs.h"
#include <osrm/Coordinate.h>
#include "../../Util/SimpleLogger.h"
#include <cstdint>
typedef BaseDataFacade<QueryEdge::EdgeData>::RTreeLeaf RTreeLeaf;
typedef StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>::TreeNode RTreeNode;
typedef StaticGraph<QueryEdge::EdgeData> QueryGraph;
#include <array>
// Added at the start and end of each block as sanity check
constexpr char CANARY[] = "OSRM";
struct SharedDataLayout
{
uint64_t name_index_list_size;
uint64_t name_char_list_size;
uint64_t name_id_list_size;
uint64_t via_node_list_size;
uint64_t graph_node_list_size;
uint64_t graph_edge_list_size;
uint64_t coordinate_list_size;
uint64_t turn_instruction_list_size;
uint64_t r_search_tree_size;
uint64_t geometries_index_list_size;
uint64_t geometries_list_size;
uint64_t geometries_indicators;
enum BlockID {
NAME_OFFSETS = 0,
NAME_BLOCKS,
NAME_CHAR_LIST,
NAME_ID_LIST,
VIA_NODE_LIST,
GRAPH_NODE_LIST,
GRAPH_EDGE_LIST,
COORDINATE_LIST,
TURN_INSTRUCTION,
R_SEARCH_TREE,
GEOMETRIES_INDEX,
GEOMETRIES_LIST,
GEOMETRIES_INDICATORS,
HSGR_CHECKSUM,
TIMESTAMP,
FILE_INDEX_PATH,
NUM_BLOCKS
};
unsigned checksum;
unsigned timestamp_length;
char ram_index_file_name[1024];
std::array<uint64_t, NUM_BLOCKS> num_entries;
std::array<uint64_t, NUM_BLOCKS> entry_size;
SharedDataLayout()
: name_index_list_size(0), name_char_list_size(0), name_id_list_size(0),
via_node_list_size(0), graph_node_list_size(0), graph_edge_list_size(0),
coordinate_list_size(0), turn_instruction_list_size(0), r_search_tree_size(0),
geometries_index_list_size(0), geometries_list_size(0), geometries_indicators(0),
checksum(0), timestamp_length(0)
: num_entries()
, entry_size()
{
ram_index_file_name[0] = '\0';
}
void PrintInformation() const
{
SimpleLogger().Write(logDEBUG) << "-";
SimpleLogger().Write(logDEBUG) << "name_index_list_size: " << name_index_list_size;
SimpleLogger().Write(logDEBUG) << "name_char_list_size: " << name_char_list_size;
SimpleLogger().Write(logDEBUG) << "name_id_list_size: " << name_id_list_size;
SimpleLogger().Write(logDEBUG) << "via_node_list_size: " << via_node_list_size;
SimpleLogger().Write(logDEBUG) << "graph_node_list_size: " << graph_node_list_size;
SimpleLogger().Write(logDEBUG) << "graph_edge_list_size: " << graph_edge_list_size;
SimpleLogger().Write(logDEBUG) << "timestamp_length: " << timestamp_length;
SimpleLogger().Write(logDEBUG) << "coordinate_list_size: " << coordinate_list_size;
SimpleLogger().Write(logDEBUG)
<< "turn_instruction_list_size: " << turn_instruction_list_size;
SimpleLogger().Write(logDEBUG) << "r_search_tree_size: " << r_search_tree_size;
SimpleLogger().Write(logDEBUG) << "geometries_indicators: " << geometries_indicators
<< "/" << ((geometries_indicators / 8) + 1);
SimpleLogger().Write(logDEBUG)
<< "geometries_index_list_size: " << geometries_index_list_size;
SimpleLogger().Write(logDEBUG) << "geometries_list_size: " << geometries_list_size;
SimpleLogger().Write(logDEBUG) << "checksum: " << checksum;
SimpleLogger().Write(logDEBUG) << "sizeof(checksum): " << sizeof(checksum);
SimpleLogger().Write(logDEBUG) << "ram index file name: " << ram_index_file_name;
SimpleLogger().Write(logDEBUG) << "name_offsets_size: " << num_entries[NAME_OFFSETS];
SimpleLogger().Write(logDEBUG) << "name_blocks_size: " << num_entries[NAME_BLOCKS];
SimpleLogger().Write(logDEBUG) << "name_char_list_size: " << num_entries[NAME_CHAR_LIST];
SimpleLogger().Write(logDEBUG) << "name_id_list_size: " << num_entries[NAME_ID_LIST];
SimpleLogger().Write(logDEBUG) << "via_node_list_size: " << num_entries[VIA_NODE_LIST];
SimpleLogger().Write(logDEBUG) << "graph_node_list_size: " << num_entries[GRAPH_NODE_LIST];
SimpleLogger().Write(logDEBUG) << "graph_edge_list_size: " << num_entries[GRAPH_EDGE_LIST];
SimpleLogger().Write(logDEBUG) << "timestamp_length: " << num_entries[TIMESTAMP];
SimpleLogger().Write(logDEBUG) << "coordinate_list_size: " << num_entries[COORDINATE_LIST];
SimpleLogger().Write(logDEBUG) << "turn_instruction_list_size: " << num_entries[TURN_INSTRUCTION];
SimpleLogger().Write(logDEBUG) << "r_search_tree_size: " << num_entries[R_SEARCH_TREE];
SimpleLogger().Write(logDEBUG) << "geometries_indicators: " << num_entries[GEOMETRIES_INDICATORS]
<< "/" << ((num_entries[GEOMETRIES_INDICATORS] / 8) + 1);
SimpleLogger().Write(logDEBUG) << "geometries_index_list_size: " << num_entries[GEOMETRIES_INDEX];
SimpleLogger().Write(logDEBUG) << "geometries_list_size: " << num_entries[GEOMETRIES_LIST];
SimpleLogger().Write(logDEBUG) << "sizeof(checksum): " << entry_size[HSGR_CHECKSUM];
SimpleLogger().Write(logDEBUG) << "NAME_OFFSETS " << ": " << GetBlockSize(NAME_OFFSETS );
SimpleLogger().Write(logDEBUG) << "NAME_BLOCKS " << ": " << GetBlockSize(NAME_BLOCKS );
SimpleLogger().Write(logDEBUG) << "NAME_CHAR_LIST " << ": " << GetBlockSize(NAME_CHAR_LIST );
SimpleLogger().Write(logDEBUG) << "NAME_ID_LIST " << ": " << GetBlockSize(NAME_ID_LIST );
SimpleLogger().Write(logDEBUG) << "VIA_NODE_LIST " << ": " << GetBlockSize(VIA_NODE_LIST );
SimpleLogger().Write(logDEBUG) << "GRAPH_NODE_LIST " << ": " << GetBlockSize(GRAPH_NODE_LIST );
SimpleLogger().Write(logDEBUG) << "GRAPH_EDGE_LIST " << ": " << GetBlockSize(GRAPH_EDGE_LIST );
SimpleLogger().Write(logDEBUG) << "COORDINATE_LIST " << ": " << GetBlockSize(COORDINATE_LIST );
SimpleLogger().Write(logDEBUG) << "TURN_INSTRUCTION " << ": " << GetBlockSize(TURN_INSTRUCTION );
SimpleLogger().Write(logDEBUG) << "R_SEARCH_TREE " << ": " << GetBlockSize(R_SEARCH_TREE );
SimpleLogger().Write(logDEBUG) << "GEOMETRIES_INDEX " << ": " << GetBlockSize(GEOMETRIES_INDEX );
SimpleLogger().Write(logDEBUG) << "GEOMETRIES_LIST " << ": " << GetBlockSize(GEOMETRIES_LIST );
SimpleLogger().Write(logDEBUG) << "GEOMETRIES_INDICATORS" << ": " << GetBlockSize(GEOMETRIES_INDICATORS);
SimpleLogger().Write(logDEBUG) << "HSGR_CHECKSUM " << ": " << GetBlockSize(HSGR_CHECKSUM );
SimpleLogger().Write(logDEBUG) << "TIMESTAMP " << ": " << GetBlockSize(TIMESTAMP );
SimpleLogger().Write(logDEBUG) << "FILE_INDEX_PATH " << ": " << GetBlockSize(FILE_INDEX_PATH );
}
uint64_t GetSizeOfLayout() const
template<typename T>
inline void SetBlockSize(BlockID bid, uint64_t entries)
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode)) +
(geometries_indicators / 32 + 1) * sizeof(unsigned) +
(geometries_index_list_size * sizeof(unsigned)) +
(geometries_list_size * sizeof(unsigned)) + sizeof(checksum) + 1024 * sizeof(char);
num_entries[bid] = entries;
entry_size[bid] = sizeof(T);
}
inline uint64_t GetBlockSize(BlockID bid) const
{
// special encoding
if (bid == GEOMETRIES_INDICATORS)
{
return (num_entries[GEOMETRIES_INDICATORS]/32 + 1) * entry_size[GEOMETRIES_INDICATORS];
}
return num_entries[bid] * entry_size[bid];
}
inline uint64_t GetSizeOfLayout() const
{
return GetBlockOffset(NUM_BLOCKS) + NUM_BLOCKS*2*sizeof(CANARY);
}
inline uint64_t GetBlockOffset(BlockID bid) const
{
uint64_t result = sizeof(CANARY);
for (unsigned i = 0; i < bid; i++)
{
result += GetBlockSize((BlockID) i) + 2*sizeof(CANARY);
}
return result;
}
uint64_t GetNameIndexOffset() const { return 0; }
uint64_t GetNameListOffset() const
template<typename T, bool WRITE_CANARY=false>
inline T* GetBlockPtr(char* shared_memory, BlockID bid)
{
uint64_t result = (name_index_list_size * sizeof(unsigned));
return result;
}
uint64_t GetNameIDListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char));
return result;
}
uint64_t GetViaNodeListOffset() const
{
uint64_t result = (name_index_list_size * sizeof(unsigned)) +
(name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned));
return result;
}
uint64_t GetGraphNodeListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID));
return result;
}
uint64_t GetGraphEdgeListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry));
return result;
}
uint64_t GetTimeStampOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry));
return result;
}
uint64_t GetCoordinateListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char));
return result;
}
uint64_t GetTurnInstructionListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate));
return result;
}
uint64_t GetRSearchTreeOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass));
return result;
}
uint64_t GetGeometriesIndicatorOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode));
return result;
}
T* ptr = (T*)(shared_memory + GetBlockOffset(bid));
if (WRITE_CANARY)
{
char* start_canary_ptr = shared_memory + GetBlockOffset(bid) - sizeof(CANARY);
char* end_canary_ptr = shared_memory + GetBlockOffset(bid) + GetBlockSize(bid);
std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
}
else
{
char* start_canary_ptr = shared_memory + GetBlockOffset(bid) - sizeof(CANARY);
char* end_canary_ptr = shared_memory + GetBlockOffset(bid) + GetBlockSize(bid);
bool start_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
bool end_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
if (!start_canary_alive)
{
throw OSRMException("Start canary of block corrupted.");
}
if (!end_canary_alive)
{
throw OSRMException("End canary of block corrupted.");
}
}
uint64_t GetGeometriesIndexListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode)) +
(geometries_indicators / 32 + 1) * sizeof(unsigned);
return result;
}
uint64_t GetGeometryListOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode)) +
(geometries_indicators / 32 + 1) * sizeof(unsigned) +
(geometries_index_list_size * sizeof(unsigned));
return result;
}
uint64_t GetChecksumOffset() const
{
uint64_t result =
(name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) +
(name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) +
(graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) +
(graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) +
(timestamp_length * sizeof(char)) +
(coordinate_list_size * sizeof(FixedPointCoordinate)) +
(turn_instruction_list_size * sizeof(TurnInstructionsClass)) +
(r_search_tree_size * sizeof(RTreeNode)) +
(geometries_indicators / 32 + 1) * sizeof(unsigned) +
(geometries_index_list_size * sizeof(unsigned)) +
(geometries_list_size * sizeof(unsigned));
return result;
return ptr;
}
};

179
datastore.cpp
View File

@ -26,7 +26,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "DataStructures/OriginalEdgeData.h"
#include "DataStructures/RangeTable.h"
#include "DataStructures/QueryEdge.h"
#include "DataStructures/SharedMemoryFactory.h"
#include "DataStructures/SharedMemoryVectorWrapper.h"
#include "DataStructures/StaticGraph.h"
#include "DataStructures/StaticRTree.h"
#include "DataStructures/TurnInstructions.h"
#include "Server/DataStructures/BaseDataFacade.h"
#include "Server/DataStructures/SharedDataType.h"
#include "Server/DataStructures/SharedBarriers.h"
#include "Util/BoostFileSystemFix.h"
@ -35,6 +42,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "Util/FingerPrint.h"
#include "typedefs.h"
#include <osrm/Coordinate.h>
typedef BaseDataFacade<QueryEdge::EdgeData>::RTreeLeaf RTreeLeaf;
typedef StaticRTree<RTreeLeaf, ShM<FixedPointCoordinate, true>::vector, true>::TreeNode RTreeNode;
typedef StaticGraph<QueryEdge::EdgeData> QueryGraph;
#ifdef __linux__
#include <sys/mman.h>
#endif
@ -161,7 +174,7 @@ int main(const int argc, const char *argv[])
BOOST_ASSERT(!paths_iterator->second.empty());
const boost::filesystem::path index_file_path_absolute =
boost::filesystem::portable_canonical(paths_iterator->second);
const std::string &file_index_file_name = index_file_path_absolute.string();
const std::string &file_index_path = index_file_path_absolute.string();
paths_iterator = server_paths.find("nodesdata");
BOOST_ASSERT(server_paths.end() != paths_iterator);
BOOST_ASSERT(!paths_iterator->second.empty());
@ -201,37 +214,34 @@ int main(const int argc, const char *argv[])
SharedDataLayout *shared_layout_ptr = static_cast<SharedDataLayout *>(layout_memory->Ptr());
shared_layout_ptr = new (layout_memory->Ptr()) SharedDataLayout();
std::copy(file_index_file_name.begin(),
(file_index_file_name.length() <= 1024 ? file_index_file_name.end()
: file_index_file_name.begin() + 1023),
shared_layout_ptr->ram_index_file_name);
// add zero termination
unsigned end_of_string_index = std::min((std::size_t)1023, file_index_file_name.length());
shared_layout_ptr->ram_index_file_name[end_of_string_index] = '\0';
shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::FILE_INDEX_PATH, file_index_path.length() + 1);
// collect number of elements to store in shared memory object
SimpleLogger().Write() << "load names from: " << names_data_path;
// number of entries in name index
boost::filesystem::ifstream name_stream(names_data_path, std::ios::binary);
unsigned name_index_size = 0;
name_stream.read((char *)&name_index_size, sizeof(unsigned));
shared_layout_ptr->name_index_list_size = name_index_size;
SimpleLogger().Write() << "size: " << name_index_size;
BOOST_ASSERT_MSG(0 != shared_layout_ptr->name_index_list_size, "name file broken");
unsigned name_blocks = 0;
name_stream.read((char *)&name_blocks, sizeof(unsigned));
shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::NAME_OFFSETS, name_blocks);
shared_layout_ptr->SetBlockSize<typename RangeTable<16, true>::BlockT>(SharedDataLayout::NAME_BLOCKS, name_blocks);
SimpleLogger().Write() << "name offsets size: " << name_blocks;
BOOST_ASSERT_MSG(0 != name_blocks, "name file broken");
unsigned number_of_chars = 0;
name_stream.read((char *)&number_of_chars, sizeof(unsigned));
shared_layout_ptr->name_char_list_size = number_of_chars;
shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::NAME_CHAR_LIST, number_of_chars);
// Loading information for original edges
boost::filesystem::ifstream edges_input_stream(edges_data_path, std::ios::binary);
unsigned number_of_original_edges = 0;
edges_input_stream.read((char *)&number_of_original_edges, sizeof(unsigned));
shared_layout_ptr->via_node_list_size = number_of_original_edges;
shared_layout_ptr->name_id_list_size = number_of_original_edges;
shared_layout_ptr->turn_instruction_list_size = number_of_original_edges;
shared_layout_ptr->geometries_indicators = number_of_original_edges;
// note: settings this all to the same size is correct, we extract them from the same struct
shared_layout_ptr->SetBlockSize<NodeID>(SharedDataLayout::VIA_NODE_LIST, number_of_original_edges);
shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::NAME_ID_LIST, number_of_original_edges);
shared_layout_ptr->SetBlockSize<TurnInstruction>(SharedDataLayout::TURN_INSTRUCTION, number_of_original_edges);
// note: there are 32 geometry indicators in one unsigned block
shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::GEOMETRIES_INDICATORS, number_of_original_edges);
boost::filesystem::ifstream hsgr_input_stream(hsgr_path, std::ios::binary);
@ -250,26 +260,26 @@ int main(const int argc, const char *argv[])
// load checksum
unsigned checksum = 0;
hsgr_input_stream.read((char *)&checksum, sizeof(unsigned));
shared_layout_ptr->checksum = checksum;
shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::HSGR_CHECKSUM, 1);
// load graph node size
unsigned number_of_graph_nodes = 0;
hsgr_input_stream.read((char *)&number_of_graph_nodes, sizeof(unsigned));
BOOST_ASSERT_MSG((0 != number_of_graph_nodes), "number of nodes is zero");
shared_layout_ptr->graph_node_list_size = number_of_graph_nodes;
shared_layout_ptr->SetBlockSize<QueryGraph::NodeArrayEntry>(SharedDataLayout::GRAPH_NODE_LIST, number_of_graph_nodes);
// load graph edge size
unsigned number_of_graph_edges = 0;
hsgr_input_stream.read((char *)&number_of_graph_edges, sizeof(unsigned));
BOOST_ASSERT_MSG(0 != number_of_graph_edges, "number of graph edges is zero");
shared_layout_ptr->graph_edge_list_size = number_of_graph_edges;
shared_layout_ptr->SetBlockSize<QueryGraph::EdgeArrayEntry>(SharedDataLayout::GRAPH_EDGE_LIST, number_of_graph_edges);
// load rsearch tree size
boost::filesystem::ifstream tree_node_file(ram_index_path, std::ios::binary);
uint32_t tree_size = 0;
tree_node_file.read((char *)&tree_size, sizeof(uint32_t));
shared_layout_ptr->r_search_tree_size = tree_size;
shared_layout_ptr->SetBlockSize<RTreeNode>(SharedDataLayout::R_SEARCH_TREE, tree_size);
// load timestamp size
std::string m_timestamp;
@ -295,13 +305,13 @@ int main(const int argc, const char *argv[])
{
m_timestamp.resize(25);
}
shared_layout_ptr->timestamp_length = m_timestamp.length();
shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::TIMESTAMP, m_timestamp.length());
// load coordinate size
boost::filesystem::ifstream nodes_input_stream(nodes_data_path, std::ios::binary);
unsigned coordinate_list_size = 0;
nodes_input_stream.read((char *)&coordinate_list_size, sizeof(unsigned));
shared_layout_ptr->coordinate_list_size = coordinate_list_size;
shared_layout_ptr->SetBlockSize<FixedPointCoordinate>(SharedDataLayout::COORDINATE_LIST, coordinate_list_size);
// load geometries sizes
std::ifstream geometry_input_stream(geometries_data_path.string().c_str(), std::ios::binary);
@ -309,11 +319,11 @@ int main(const int argc, const char *argv[])
unsigned number_of_compressed_geometries = 0;
geometry_input_stream.read((char *)&number_of_geometries_indices, sizeof(unsigned));
shared_layout_ptr->geometries_index_list_size = number_of_geometries_indices;
shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::GEOMETRIES_INDEX, number_of_geometries_indices);
boost::iostreams::seek(
geometry_input_stream, number_of_geometries_indices * sizeof(unsigned), BOOST_IOS::cur);
geometry_input_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned));
shared_layout_ptr->geometries_list_size = number_of_compressed_geometries;
shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::GEOMETRIES_LIST, number_of_compressed_geometries);
// allocate shared memory block
SimpleLogger().Write() << "allocating shared memory of "
@ -323,35 +333,70 @@ int main(const int argc, const char *argv[])
char *shared_memory_ptr = static_cast<char *>(shared_memory->Ptr());
// read actual data into shared memory object //
// hsgr checksum
unsigned* checksum_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr,
SharedDataLayout::HSGR_CHECKSUM);
*checksum_ptr = checksum;
// ram index file name
char* file_index_path_ptr = shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr,
SharedDataLayout::FILE_INDEX_PATH);
// make sure we have 0 ending
std::fill(file_index_path_ptr,
file_index_path_ptr +
shared_layout_ptr->GetBlockSize(SharedDataLayout::FILE_INDEX_PATH),
0);
std::copy(file_index_path.begin(), file_index_path.end(), file_index_path_ptr);
// Loading street names
unsigned *name_index_ptr =
(unsigned *)(shared_memory_ptr + shared_layout_ptr->GetNameIndexOffset());
if (shared_layout_ptr->name_index_list_size > 0)
unsigned *name_offsets_ptr =
shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr,
SharedDataLayout::NAME_OFFSETS);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_OFFSETS) > 0)
{
name_stream.read((char *)name_index_ptr,
shared_layout_ptr->name_index_list_size * sizeof(unsigned));
name_stream.read((char *)name_offsets_ptr,
shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_OFFSETS));
}
char *name_char_ptr = shared_memory_ptr + shared_layout_ptr->GetNameListOffset();
if (shared_layout_ptr->name_char_list_size > 0)
unsigned *name_blocks_ptr =
shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr,
SharedDataLayout::NAME_BLOCKS);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_BLOCKS) > 0)
{
name_stream.read(name_char_ptr, shared_layout_ptr->name_char_list_size * sizeof(char));
name_stream.read((char *)name_blocks_ptr,
shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_BLOCKS));
}
char *name_char_ptr =
shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr,
SharedDataLayout::NAME_CHAR_LIST);
unsigned temp_length;
name_stream.read((char*) &temp_length, sizeof(unsigned));
BOOST_ASSERT_MSG(temp_length == shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST),
"Name file corrupted!");
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST) > 0)
{
name_stream.read(name_char_ptr, shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST));
}
name_stream.close();
// load original edge information
NodeID *via_node_ptr =
(NodeID *)(shared_memory_ptr + shared_layout_ptr->GetViaNodeListOffset());
NodeID *via_node_ptr = shared_layout_ptr->GetBlockPtr<NodeID, true>(shared_memory_ptr,
SharedDataLayout::VIA_NODE_LIST);
unsigned *name_id_ptr =
(unsigned *)(shared_memory_ptr + shared_layout_ptr->GetNameIDListOffset());
unsigned *name_id_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr,
SharedDataLayout::NAME_ID_LIST);
TurnInstruction *turn_instructions_ptr =
(TurnInstruction *)(shared_memory_ptr +
shared_layout_ptr->GetTurnInstructionListOffset());
shared_layout_ptr->GetBlockPtr<TurnInstruction, true>(shared_memory_ptr,
SharedDataLayout::TURN_INSTRUCTION);
unsigned *geometries_indicator_ptr =
(unsigned *)(shared_memory_ptr + shared_layout_ptr->GetGeometriesIndicatorOffset());
unsigned *geometries_indicator_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr,
SharedDataLayout::GEOMETRIES_INDICATORS);
OriginalEdgeData current_edge_data;
for (unsigned i = 0; i < number_of_original_edges; ++i)
@ -382,33 +427,32 @@ int main(const int argc, const char *argv[])
// load compressed geometry
unsigned temporary_value;
unsigned *geometries_index_ptr =
(unsigned *)(shared_memory_ptr + shared_layout_ptr->GetGeometriesIndexListOffset());
shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr, SharedDataLayout::GEOMETRIES_INDEX);
geometry_input_stream.seekg(0, geometry_input_stream.beg);
geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned));
BOOST_ASSERT(temporary_value == shared_layout_ptr->geometries_index_list_size);
if (shared_layout_ptr->geometries_index_list_size > 0)
BOOST_ASSERT(temporary_value == shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_INDEX]);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_INDEX) > 0)
{
geometry_input_stream.read((char *)geometries_index_ptr,
shared_layout_ptr->geometries_index_list_size *
sizeof(unsigned));
shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_INDEX));
}
unsigned *geometries_list_ptr =
(unsigned *)(shared_memory_ptr + shared_layout_ptr->GetGeometryListOffset());
shared_layout_ptr->GetBlockPtr<unsigned, true>(shared_memory_ptr, SharedDataLayout::GEOMETRIES_LIST);
geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned));
BOOST_ASSERT(temporary_value == shared_layout_ptr->geometries_list_size);
BOOST_ASSERT(temporary_value == shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_LIST]);
if (shared_layout_ptr->geometries_list_size > 0)
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_LIST) > 0)
{
geometry_input_stream.read((char *)geometries_list_ptr,
shared_layout_ptr->geometries_list_size * sizeof(unsigned));
shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_LIST));
}
// Loading list of coordinates
FixedPointCoordinate *coordinates_ptr =
(FixedPointCoordinate *)(shared_memory_ptr +
shared_layout_ptr->GetCoordinateListOffset());
shared_layout_ptr->GetBlockPtr<FixedPointCoordinate, true>(shared_memory_ptr,
SharedDataLayout::COORDINATE_LIST);
NodeInfo current_node;
for (unsigned i = 0; i < coordinate_list_size; ++i)
@ -419,40 +463,37 @@ int main(const int argc, const char *argv[])
nodes_input_stream.close();
// store timestamp
char *timestamp_ptr =
static_cast<char *>(shared_memory_ptr + shared_layout_ptr->GetTimeStampOffset());
char *timestamp_ptr = shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr, SharedDataLayout::TIMESTAMP);
std::copy(m_timestamp.c_str(), m_timestamp.c_str() + m_timestamp.length(), timestamp_ptr);
// store search tree portion of rtree
char *rtree_ptr =
static_cast<char *>(shared_memory_ptr + shared_layout_ptr->GetRSearchTreeOffset());
char *rtree_ptr = shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr, SharedDataLayout::R_SEARCH_TREE);
if (tree_size > 0)
{
tree_node_file.read(rtree_ptr, sizeof(RTreeNode) * tree_size);
tree_node_file.read(rtree_ptr, sizeof(RTreeNode) * tree_size);
}
tree_node_file.close();
// load the nodes of the search graph
QueryGraph::NodeArrayEntry *graph_node_list_ptr =
(QueryGraph::NodeArrayEntry *)(shared_memory_ptr +
shared_layout_ptr->GetGraphNodeListOffset());
if (shared_layout_ptr->graph_node_list_size > 0)
shared_layout_ptr->GetBlockPtr<QueryGraph::NodeArrayEntry, true>(shared_memory_ptr,
SharedDataLayout::GRAPH_NODE_LIST);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_NODE_LIST) > 0)
{
hsgr_input_stream.read((char *)graph_node_list_ptr,
shared_layout_ptr->graph_node_list_size *
sizeof(QueryGraph::NodeArrayEntry));
shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_NODE_LIST));
}
// load the edges of the search graph
QueryGraph::EdgeArrayEntry *graph_edge_list_ptr =
(QueryGraph::EdgeArrayEntry *)(shared_memory_ptr +
shared_layout_ptr->GetGraphEdgeListOffset());
if (shared_layout_ptr->graph_edge_list_size > 0)
shared_layout_ptr->GetBlockPtr<QueryGraph::EdgeArrayEntry, true>(shared_memory_ptr,
SharedDataLayout::GRAPH_EDGE_LIST);
if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST) > 0)
{
hsgr_input_stream.read((char *)graph_edge_list_ptr,
shared_layout_ptr->graph_edge_list_size *
sizeof(QueryGraph::EdgeArrayEntry));
shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST));
}
hsgr_input_stream.close();