From b87a98bbda8672fdff7e312d94be067945cc92cc Mon Sep 17 00:00:00 2001 From: Dennis Luxen Date: Thu, 26 Sep 2013 18:19:51 +0200 Subject: [PATCH] street name file is now more canonical --- DataStructures/SharedMemoryFactory.h | 3 - DataStructures/SharedMemoryVectorWrapper.h | 4 +- DataStructures/StaticRTree.h | 1 - Extractor/ExtractionContainers.cpp | 22 ++- Server/DataStructures/InternalDataFacade.h | 41 ++-- Server/DataStructures/SharedDataFacade.h | 114 +++++------ Server/DataStructures/SharedDataType.h | 211 +++++++++++++++++++-- 7 files changed, 294 insertions(+), 102 deletions(-) diff --git a/DataStructures/SharedMemoryFactory.h b/DataStructures/SharedMemoryFactory.h index 4218250c7..3f71279c6 100644 --- a/DataStructures/SharedMemoryFactory.h +++ b/DataStructures/SharedMemoryFactory.h @@ -37,9 +37,7 @@ struct OSRMLockFile { boost::filesystem::path operator()() { boost::filesystem::path temp_dir = boost::filesystem::temp_directory_path(); - // SimpleLogger().Write(logDEBUG) << "creating lock file in " << temp_dir; boost::filesystem::path lock_file = temp_dir / "osrm.lock"; - // SimpleLogger().Write(logDEBUG) << "locking at " << lock_file; return lock_file; } }; @@ -66,7 +64,6 @@ class SharedMemory : boost::noncopyable { boost::interprocess::xsi_shared_memory::remove(m_shmid); } } - }; public: diff --git a/DataStructures/SharedMemoryVectorWrapper.h b/DataStructures/SharedMemoryVectorWrapper.h index 1679efa99..89d549c05 100644 --- a/DataStructures/SharedMemoryVectorWrapper.h +++ b/DataStructures/SharedMemoryVectorWrapper.h @@ -106,12 +106,12 @@ public: std::size_t size() const { return m_size; } - DataT & operator[](const int index) { + DataT & operator[](const unsigned index) { BOOST_ASSERT_MSG(index < m_size, "invalid size"); return m_ptr[index]; } - const DataT & operator[](const int index) const { + const DataT & operator[](const unsigned index) const { BOOST_ASSERT_MSG(index < m_size, "invalid size"); return m_ptr[index]; } diff --git a/DataStructures/StaticRTree.h b/DataStructures/StaticRTree.h index 306b974de..9ff8e4f92 100644 --- a/DataStructures/StaticRTree.h +++ b/DataStructures/StaticRTree.h @@ -36,7 +36,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "SharedMemoryFactory.h" #include "SharedMemoryVectorWrapper.h" -#include "../Server/DataStructures/SharedDataType.h" #include "../Util/OSRMException.h" #include "../Util/SimpleLogger.h" #include "../Util/TimingUtil.h" diff --git a/Extractor/ExtractionContainers.cpp b/Extractor/ExtractionContainers.cpp index 58f00252e..2665c410c 100644 --- a/Extractor/ExtractionContainers.cpp +++ b/Extractor/ExtractionContainers.cpp @@ -297,8 +297,21 @@ void ExtractionContainers::PrepareData(const std::string & output_file_name, con std::ios::binary ); - const unsigned number_of_ways = name_list.size()+1; - name_file_stream.write((char *)&(number_of_ways), sizeof(unsigned)); + //write number of names + const unsigned number_of_names = name_list.size()+1; + name_file_stream.write((char *)&(number_of_names), sizeof(unsigned)); + + //compute total number of chars + unsigned total_number_of_chars = 0; + BOOST_FOREACH(const std::string & str, name_list) { + total_number_of_chars += strlen(str.c_str()); + } + //write total number of chars + name_file_stream.write( + (char *)&(total_number_of_chars), + sizeof(unsigned) + ); + //write prefixe sums unsigned name_lengths_prefix_sum = 0; BOOST_FOREACH(const std::string & str, name_list) { name_file_stream.write( @@ -307,12 +320,13 @@ void ExtractionContainers::PrepareData(const std::string & output_file_name, con ); name_lengths_prefix_sum += strlen(str.c_str()); } + //duplicate on purpose! name_file_stream.write( (char *)&(name_lengths_prefix_sum), sizeof(unsigned) ); - //duplicate on purpose! - name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned)); + + //write all chars consecutively BOOST_FOREACH(const std::string & str, name_list) { const unsigned lengthOfRawString = strlen(str.c_str()); name_file_stream.write(str.c_str(), lengthOfRawString); diff --git a/Server/DataStructures/InternalDataFacade.h b/Server/DataStructures/InternalDataFacade.h index 33f251bd7..efba74fdd 100644 --- a/Server/DataStructures/InternalDataFacade.h +++ b/Server/DataStructures/InternalDataFacade.h @@ -47,10 +47,10 @@ private: InternalDataFacade() { } - unsigned m_check_sum; - unsigned m_number_of_nodes; - QueryGraph * m_query_graph; - std::string m_timestamp; + unsigned m_check_sum; + unsigned m_number_of_nodes; + QueryGraph * m_query_graph; + std::string m_timestamp; ShM::vector m_coordinate_list; ShM::vector m_via_node_list; @@ -59,7 +59,7 @@ private: ShM::vector m_names_char_list; ShM::vector m_name_begin_indices; - StaticRTree * m_static_rtree; + StaticRTree * m_static_rtree; void LoadTimestamp(const boost::filesystem::path & timestamp_path) { @@ -168,19 +168,28 @@ private: const boost::filesystem::path & names_file ) { boost::filesystem::ifstream name_stream(names_file, std::ios::binary); - unsigned size = 0; - name_stream.read((char *)&size, sizeof(unsigned)); - BOOST_ASSERT_MSG(0 != size, "name file broken"); + unsigned number_of_names = 0; + unsigned number_of_chars = 0; + name_stream.read((char *)&number_of_names, sizeof(unsigned)); + name_stream.read((char *)&number_of_chars, sizeof(unsigned)); + BOOST_ASSERT_MSG(0 != number_of_names, "name file broken"); + BOOST_ASSERT_MSG(0 != number_of_chars, "name file broken"); - m_name_begin_indices.resize(size); - name_stream.read((char*)&m_name_begin_indices[0], size*sizeof(unsigned)); - name_stream.read((char *)&size, sizeof(unsigned)); - BOOST_ASSERT_MSG(0 != size, "name file broken"); - - m_names_char_list.resize(size+1); //+1 is sentinel/dummy element - name_stream.read((char *)&m_names_char_list[0], size*sizeof(char)); - BOOST_ASSERT_MSG(0 != m_names_char_list.size(), "could not load any names"); + m_name_begin_indices.resize(number_of_names); + name_stream.read( + (char*)&m_name_begin_indices[0], + number_of_names*sizeof(unsigned) + ); + m_names_char_list.resize(number_of_chars+1); //+1 gives sentinel element + name_stream.read( + (char *)&m_names_char_list[0], + number_of_chars*sizeof(char) + ); + BOOST_ASSERT_MSG( + 0 != m_names_char_list.size(), + "could not load any names" + ); name_stream.close(); } public: diff --git a/Server/DataStructures/SharedDataFacade.h b/Server/DataStructures/SharedDataFacade.h index 2084eeca6..aefb2fc64 100644 --- a/Server/DataStructures/SharedDataFacade.h +++ b/Server/DataStructures/SharedDataFacade.h @@ -24,6 +24,7 @@ or see http://www.gnu.org/licenses/agpl.txt. //implements all data storage when shared memory is _NOT_ used #include "BaseDataFacade.h" +#include "SharedDataType.h" #include "../../DataStructures/StaticGraph.h" #include "../../DataStructures/StaticRTree.h" @@ -46,6 +47,9 @@ private: typedef typename super::RTreeLeaf RTreeLeaf; typedef typename StaticRTree::TreeNode RTreeNode; + SharedDataLayout * data_layout; + char * shared_memory; + unsigned m_check_sum; unsigned m_number_of_nodes; QueryGraph * m_query_graph; @@ -62,46 +66,46 @@ private: SharedDataFacade() { } void LoadTimestamp() { - uint64_t timestamp_size = *static_cast( - SharedMemoryFactory::Get(TIMESTAMP_SIZE)->Ptr() + char * timestamp_ptr = shared_memory + data_layout->GetTimeStampOffset(); + m_timestamp.resize(data_layout->timestamp_length); + std::copy( + timestamp_ptr, + timestamp_ptr+data_layout->timestamp_length, + m_timestamp.begin() ); - timestamp_size = std::min(timestamp_size, (uint64_t)25); - SharedMemory * search_tree = SharedMemoryFactory::Get(TIMESTAMP); - char * tree_ptr = static_cast( search_tree->Ptr() ); - m_timestamp.resize(timestamp_size); - std::copy(tree_ptr, tree_ptr+timestamp_size, m_timestamp.begin()); } void LoadRTree( const boost::filesystem::path & file_index_path ) { - uint64_t tree_size = *static_cast( - SharedMemoryFactory::Get(R_SEARCH_TREE_SIZE)->Ptr() + RTreeNode * tree_ptr = (RTreeNode *)( + shared_memory + data_layout->GetRSearchTreeOffset() ); - SharedMemory * search_tree = SharedMemoryFactory::Get(R_SEARCH_TREE); - RTreeNode * tree_ptr = static_cast( search_tree->Ptr() ); m_static_rtree = new StaticRTree( tree_ptr, - tree_size, + data_layout->r_search_tree_size, file_index_path ); } void LoadGraph() { - m_number_of_nodes = *static_cast( - SharedMemoryFactory::Get(GRAPH_NODE_LIST_SIZE)->Ptr() + m_number_of_nodes = data_layout->graph_node_list_size; + GraphNode * graph_nodes_ptr = (GraphNode *)( + shared_memory + data_layout->GetGraphNodeListOffset() ); - SharedMemory * graph_nodes = SharedMemoryFactory::Get(GRAPH_NODE_LIST); - GraphNode * graph_nodes_ptr = static_cast( graph_nodes->Ptr() ); - uint64_t number_of_edges = *static_cast( - SharedMemoryFactory::Get(GRAPH_EDGE_LIST_SIZE)->Ptr() + GraphEdge * graph_edges_ptr = (GraphEdge *)( + shared_memory + data_layout->GetGraphEdgeListOffsett() ); - SharedMemory * graph_edges = SharedMemoryFactory::Get(GRAPH_EDGE_LIST); - GraphEdge * graph_edges_ptr = static_cast( graph_edges->Ptr() ); - typename ShM::vector node_list(graph_nodes_ptr, m_number_of_nodes); - typename ShM::vector edge_list(graph_edges_ptr, number_of_edges); + typename ShM::vector node_list( + graph_nodes_ptr, + data_layout->graph_node_list_size + ); + typename ShM::vector edge_list( + graph_edges_ptr, + data_layout->graph_edge_list_size + ); m_query_graph = new QueryGraph( node_list , edge_list @@ -110,70 +114,62 @@ private: } void LoadNodeAndEdgeInformation() { - uint64_t number_of_coordinates = *static_cast( - SharedMemoryFactory::Get(COORDINATE_LIST_SIZE)->Ptr() - ); - FixedPointCoordinate * coordinate_list_ptr = static_cast( - SharedMemoryFactory::Get(COORDINATE_LIST)->Ptr() + + FixedPointCoordinate * coordinate_list_ptr = (FixedPointCoordinate *)( + shared_memory + data_layout->GetCoordinateListOffset() ); typename ShM::vector coordinate_list( coordinate_list_ptr, - number_of_coordinates + data_layout->coordinate_list_size ); m_coordinate_list.swap( coordinate_list ); - uint64_t number_of_turn_instructions = *static_cast( - SharedMemoryFactory::Get(TURN_INSTRUCTION_LIST_SIZE)->Ptr() + TurnInstruction * turn_instruction_list_ptr = (TurnInstruction *)( + shared_memory + data_layout->GetTurnInstructionListOffset() ); - - TurnInstruction * turn_instruction_list_ptr = static_cast( - SharedMemoryFactory::Get(TURN_INSTRUCTION_LIST)->Ptr() - ); - typename ShM::vector turn_instruction_list( turn_instruction_list_ptr, - number_of_turn_instructions + data_layout->turn_instruction_list_size ); - m_turn_instruction_list.swap(turn_instruction_list); + + unsigned * name_id_list_ptr = (unsigned *)( + shared_memory + data_layout->GetNameIDListOffset() + ); + typename ShM::vector name_id_list( + name_id_list_ptr, + data_layout->name_id_list_size + ); + m_name_ID_list.swap(name_id_list); } void LoadViaNodeList() { - uint64_t number_of_via_nodes = * static_cast ( - SharedMemoryFactory::Get(VIA_NODE_LIST_SIZE)->Ptr() - ); - NodeID * via_node_list_ptr = static_cast( - SharedMemoryFactory::Get(VIA_NODE_LIST)->Ptr() + NodeID * via_node_list_ptr = (NodeID *)( + shared_memory + data_layout->GetViaNodeListOffset() ); typename ShM::vector via_node_list( via_node_list_ptr, - number_of_via_nodes + data_layout->via_node_list_size ); m_via_node_list.swap(via_node_list); } void LoadNames() { - uint64_t street_names_index_size = * static_cast ( - SharedMemoryFactory::Get(NAME_INDEX_SIZE)->Ptr() - ); - unsigned * street_names_index_ptr = static_cast( - SharedMemoryFactory::Get(NAMES_INDEX)->Ptr() + unsigned * street_names_index_ptr = (unsigned *)( + shared_memory + data_layout->GetNameIndexOffset() ); typename ShM::vector name_begin_indices( street_names_index_ptr, - street_names_index_size + data_layout->names_index_size ); m_name_begin_indices.swap(m_name_begin_indices); - uint64_t names_list_size = * static_cast( - SharedMemoryFactory::Get(NAMES_LIST_SIZE)->Ptr() - ); - char * names_list_ptr = static_cast( - SharedMemoryFactory::Get(NAMES_LIST)->Ptr() + char * names_list_ptr = (char *)( + shared_memory + data_layout->GetNameListOffset() ); typename ShM::vector names_char_list( names_list_ptr, - names_list_size + data_layout->names_list_size ); m_names_char_list.swap(names_char_list); } @@ -194,6 +190,14 @@ public: base_path ); + data_layout = (SharedDataLayout *)( + SharedMemoryFactory::Get(LAYOUT_1)->Ptr() + ); + + shared_memory = (char *)( + SharedMemoryFactory::Get(DATA_1)->Ptr() + ); + //load data SimpleLogger().Write() << "loading graph data"; LoadGraph(); diff --git a/Server/DataStructures/SharedDataType.h b/Server/DataStructures/SharedDataType.h index abecdefcf..ec20acc3e 100644 --- a/Server/DataStructures/SharedDataType.h +++ b/Server/DataStructures/SharedDataType.h @@ -21,28 +21,197 @@ or see http://www.gnu.org/licenses/agpl.txt. #ifndef SHARED_DATA_TYPE_H_ #define SHARED_DATA_TYPE_H_ +#include "BaseDataFacade.h" + +#include "../../DataStructures/Coordinate.h" +#include "../../DataStructures/QueryEdge.h" +#include "../../DataStructures/StaticGraph.h" +#include "../../DataStructures/StaticRTree.h" +#include "../../DataStructures/TurnInstructions.h" + +#include "../../typedefs.h" + +#include + +typedef BaseDataFacade::RTreeLeaf RTreeLeaf; +typedef StaticRTree::TreeNode RTreeNode; +typedef StaticGraph QueryGraph; + +struct SharedDataLayout { + uint64_t names_index_size; + uint64_t names_list_size; + uint64_t name_id_list_size; + uint64_t via_node_list_size; + uint64_t graph_node_list_size; + uint64_t graph_edge_list_size; + uint64_t coordinate_list_size; + uint64_t turn_instruction_list_size; + uint64_t r_search_tree_size; + + unsigned checksum; + unsigned timestamp_length; + + SharedDataLayout() : + names_index_size(0), + names_list_size(0), + name_id_list_size(0), + via_node_list_size(0), + graph_node_list_size(0), + graph_edge_list_size(0), + coordinate_list_size(0), + turn_instruction_list_size(0), + r_search_tree_size(0), + checksum(0), + timestamp_length(0) + { } + + uint64_t GetSizeOfLayout() const { + uint64_t result = + (names_index_size * sizeof(unsigned) ) + + (names_list_size * sizeof(char) ) + + (name_id_list_size * sizeof(unsigned) ) + + (via_node_list_size * sizeof(NodeID) ) + + (graph_node_list_size * sizeof(QueryGraph::_StrNode)) + + (graph_edge_list_size * sizeof(QueryGraph::_StrEdge)) + + (timestamp_length * sizeof(char) ) + + (coordinate_list_size * sizeof(FixedPointCoordinate)) + + (turn_instruction_list_size * sizeof(TurnInstructions) ) + + (r_search_tree_size * sizeof(RTreeNode) ) + + sizeof(checksum); + return result; + } + + uint64_t GetNameIndexOffset() const { + return 0; + } + uint64_t GetNameListOffset() const { + uint64_t result = + (names_index_size * sizeof(unsigned) ); + return result; + } + uint64_t GetNameIDListOffset() const { + uint64_t result = + (names_index_size * sizeof(unsigned) ) + + (names_list_size * sizeof(char) ); + return result; + } + uint64_t GetViaNodeListOffset() const { + uint64_t result = + (names_index_size * sizeof(unsigned) ) + + (names_list_size * sizeof(char) ) + + (name_id_list_size * sizeof(unsigned) ); + return result; + } + uint64_t GetGraphNodeListOffset() const { + uint64_t result = + (names_index_size * sizeof(unsigned) ) + + (names_list_size * sizeof(char) ) + + (name_id_list_size * sizeof(unsigned) ) + + (via_node_list_size * sizeof(NodeID) ); + return result; + } + uint64_t GetGraphEdgeListOffsett() const { + uint64_t result = + (names_index_size * sizeof(unsigned) ) + + (names_list_size * sizeof(char) ) + + (name_id_list_size * sizeof(unsigned) ) + + (via_node_list_size * sizeof(NodeID) ) + + (graph_node_list_size * sizeof(QueryGraph::_StrNode)) ; + return result; + } + uint64_t GetTimeStampOffset() const { + uint64_t result = + (names_index_size * sizeof(unsigned) ) + + (names_list_size * sizeof(char) ) + + (name_id_list_size * sizeof(unsigned) ) + + (via_node_list_size * sizeof(NodeID) ) + + (graph_node_list_size * sizeof(QueryGraph::_StrNode)) + + (graph_edge_list_size * sizeof(QueryGraph::_StrEdge)); + return result; + } + uint64_t GetCoordinateListOffset() const { + uint64_t result = + (names_index_size * sizeof(unsigned) ) + + (names_list_size * sizeof(char) ) + + (name_id_list_size * sizeof(unsigned) ) + + (via_node_list_size * sizeof(NodeID) ) + + (graph_node_list_size * sizeof(QueryGraph::_StrNode)) + + (graph_edge_list_size * sizeof(QueryGraph::_StrEdge)) + + (timestamp_length * sizeof(char) ); + return result; + } + uint64_t GetTurnInstructionListOffset() const { + uint64_t result = + (names_index_size * sizeof(unsigned) ) + + (names_list_size * sizeof(char) ) + + (name_id_list_size * sizeof(unsigned) ) + + (via_node_list_size * sizeof(NodeID) ) + + (graph_node_list_size * sizeof(QueryGraph::_StrNode)) + + (graph_edge_list_size * sizeof(QueryGraph::_StrEdge)) + + (timestamp_length * sizeof(char) ) + + (coordinate_list_size * sizeof(FixedPointCoordinate)); + return result; + } + uint64_t GetRSearchTreeOffset() const { + uint64_t result = + (names_index_size * sizeof(unsigned) ) + + (names_list_size * sizeof(char) ) + + (name_id_list_size * sizeof(unsigned) ) + + (via_node_list_size * sizeof(NodeID) ) + + (graph_node_list_size * sizeof(QueryGraph::_StrNode)) + + (graph_edge_list_size * sizeof(QueryGraph::_StrEdge)) + + (timestamp_length * sizeof(char) ) + + (coordinate_list_size * sizeof(FixedPointCoordinate)) + + (turn_instruction_list_size * sizeof(TurnInstructions) ); + return result; + } + uint64_t GetChecksumOffset() const { + uint64_t result = + (names_index_size * sizeof(unsigned) ) + + (names_list_size * sizeof(char) ) + + (name_id_list_size * sizeof(unsigned) ) + + (via_node_list_size * sizeof(NodeID) ) + + (graph_node_list_size * sizeof(QueryGraph::_StrNode)) + + (graph_edge_list_size * sizeof(QueryGraph::_StrEdge)) + + (timestamp_length * sizeof(char) ) + + (coordinate_list_size * sizeof(FixedPointCoordinate)) + + (turn_instruction_list_size * sizeof(TurnInstructions) ) + + (r_search_tree_size * sizeof(RTreeNode) ); + return result; + } +}; + enum SharedDataType { - NAMES_INDEX = 0, - NAME_INDEX_SIZE, - NAMES_LIST, - NAMES_LIST_SIZE, - NAME_ID_LIST, - NAME_ID_LIST_SIZE, - VIA_NODE_LIST, - VIA_NODE_LIST_SIZE, - GRAPH_NODE_LIST, - GRAPH_NODE_LIST_SIZE, - GRAPH_EDGE_LIST, - GRAPH_EDGE_LIST_SIZE, - CHECK_SUM, - TIMESTAMP, - TIMESTAMP_SIZE, - COORDINATE_LIST, - COORDINATE_LIST_SIZE, - TURN_INSTRUCTION_LIST, - TURN_INSTRUCTION_LIST_SIZE, - R_SEARCH_TREE, - R_SEARCH_TREE_SIZE + // NAMES_INDEX = 0, + // NAME_INDEX_SIZE, + // NAMES_LIST, + // NAMES_LIST_SIZE, + // NAME_ID_LIST, + // NAME_ID_LIST_SIZE, + // VIA_NODE_LIST, + // VIA_NODE_LIST_SIZE, + // GRAPH_NODE_LIST, + // GRAPH_NODE_LIST_SIZE, + // GRAPH_EDGE_LIST, + // GRAPH_EDGE_LIST_SIZE, + // CHECK_SUM, + // TIMESTAMP, + // TIMESTAMP_SIZE, + // COORDINATE_LIST, + // COORDINATE_LIST_SIZE, + // TURN_INSTRUCTION_LIST, + // TURN_INSTRUCTION_LIST_SIZE, + // R_SEARCH_TREE, + // R_SEARCH_TREE_SIZE + + LAYOUT_1, + DATA_1, + LAYOUT_2, + DATA_2, + LAYOUT_3, + DATA_3, + LAYOUT_4, + DATA_5 }; #endif /* SHARED_DATA_TYPE_H_ */