street name file is now more canonical

DataStructures/SharedMemoryFactory.h

@@ -37,9 +37,7 @@ struct OSRMLockFile {
 	boost::filesystem::path operator()() {
 		boost::filesystem::path temp_dir =
 			boost::filesystem::temp_directory_path();
-	    // SimpleLogger().Write(logDEBUG) << "creating lock file in " << temp_dir;
 	    boost::filesystem::path lock_file = temp_dir / "osrm.lock";
-	    // SimpleLogger().Write(logDEBUG) << "locking at " << lock_file;
 	    return lock_file;
 	}
 };
@@ -66,7 +64,6 @@ class SharedMemory : boost::noncopyable {
 				boost::interprocess::xsi_shared_memory::remove(m_shmid);
 			}
 		}
-
 	};
 
 public:

DataStructures/SharedMemoryVectorWrapper.h

@@ -106,12 +106,12 @@ public:
 
     std::size_t size() const { return m_size; }
 
-    DataT & operator[](const int index) {
+    DataT & operator[](const unsigned index) {
         BOOST_ASSERT_MSG(index < m_size, "invalid size");
         return m_ptr[index];
     }
 
-    const DataT & operator[](const int index) const {
+    const DataT & operator[](const unsigned index) const {
         BOOST_ASSERT_MSG(index < m_size, "invalid size");
         return m_ptr[index];
     }

DataStructures/StaticRTree.h

@@ -36,7 +36,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "SharedMemoryFactory.h"
 #include "SharedMemoryVectorWrapper.h"
 
-#include "../Server/DataStructures/SharedDataType.h"
 #include "../Util/OSRMException.h"
 #include "../Util/SimpleLogger.h"
 #include "../Util/TimingUtil.h"

Extractor/ExtractionContainers.cpp

@@ -297,8 +297,21 @@ void ExtractionContainers::PrepareData(const std::string & output_file_name, con
             std::ios::binary
         );
 
-        const unsigned number_of_ways = name_list.size()+1;
+        //write number of names
-        name_file_stream.write((char *)&(number_of_ways), sizeof(unsigned));
+        const unsigned number_of_names = name_list.size()+1;
+        name_file_stream.write((char *)&(number_of_names), sizeof(unsigned));
+
+        //compute total number of chars
+        unsigned total_number_of_chars = 0;
+        BOOST_FOREACH(const std::string & str, name_list) {
+            total_number_of_chars += strlen(str.c_str());
+        }
+        //write total number of chars
+        name_file_stream.write(
+            (char *)&(total_number_of_chars),
+            sizeof(unsigned)
+        );
+        //write prefixe sums
         unsigned name_lengths_prefix_sum = 0;
         BOOST_FOREACH(const std::string & str, name_list) {
             name_file_stream.write(
@@ -307,12 +320,13 @@ void ExtractionContainers::PrepareData(const std::string & output_file_name, con
             );
             name_lengths_prefix_sum += strlen(str.c_str());
         }
+        //duplicate on purpose!
         name_file_stream.write(
             (char *)&(name_lengths_prefix_sum),
             sizeof(unsigned)
         );
-        //duplicate on purpose!
+
-        name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
+        //write all chars consecutively
         BOOST_FOREACH(const std::string & str, name_list) {
             const unsigned lengthOfRawString = strlen(str.c_str());
             name_file_stream.write(str.c_str(), lengthOfRawString);

Server/DataStructures/InternalDataFacade.h

@@ -168,19 +168,28 @@ private:
         const boost::filesystem::path & names_file
     ) {
         boost::filesystem::ifstream name_stream(names_file, std::ios::binary);
-        unsigned size = 0;
+        unsigned number_of_names = 0;
-        name_stream.read((char *)&size, sizeof(unsigned));
+        unsigned number_of_chars = 0;
-        BOOST_ASSERT_MSG(0 != size, "name file broken");
+        name_stream.read((char *)&number_of_names, sizeof(unsigned));
+        name_stream.read((char *)&number_of_chars, sizeof(unsigned));
+        BOOST_ASSERT_MSG(0 != number_of_names, "name file broken");
+        BOOST_ASSERT_MSG(0 != number_of_chars, "name file broken");
 
-        m_name_begin_indices.resize(size);
+        m_name_begin_indices.resize(number_of_names);
-        name_stream.read((char*)&m_name_begin_indices[0], size*sizeof(unsigned));
+        name_stream.read(
-        name_stream.read((char *)&size, sizeof(unsigned));
+            (char*)&m_name_begin_indices[0],
-        BOOST_ASSERT_MSG(0 != size, "name file broken");
+            number_of_names*sizeof(unsigned)
-
+        );
-        m_names_char_list.resize(size+1); //+1 is sentinel/dummy element
-        name_stream.read((char *)&m_names_char_list[0], size*sizeof(char));
-        BOOST_ASSERT_MSG(0 != m_names_char_list.size(), "could not load any names");
 
+        m_names_char_list.resize(number_of_chars+1); //+1 gives sentinel element
+        name_stream.read(
+            (char *)&m_names_char_list[0],
+            number_of_chars*sizeof(char)
+        );
+        BOOST_ASSERT_MSG(
+            0 != m_names_char_list.size(),
+            "could not load any names"
+        );
         name_stream.close();
     }
 public:

Server/DataStructures/SharedDataFacade.h

@@ -24,6 +24,7 @@ or see http://www.gnu.org/licenses/agpl.txt.
 //implements all data storage when shared memory is _NOT_ used
 
 #include "BaseDataFacade.h"
+#include "SharedDataType.h"
 
 #include "../../DataStructures/StaticGraph.h"
 #include "../../DataStructures/StaticRTree.h"
@@ -46,6 +47,9 @@ private:
     typedef typename super::RTreeLeaf                       RTreeLeaf;
     typedef typename StaticRTree<RTreeLeaf, true>::TreeNode RTreeNode;
 
+    SharedDataLayout * data_layout;
+    char             * shared_memory;
+
     unsigned                                m_check_sum;
     unsigned                                m_number_of_nodes;
     QueryGraph                            * m_query_graph;
@@ -62,46 +66,46 @@ private:
     SharedDataFacade() { }
 
     void LoadTimestamp() {
-        uint64_t timestamp_size = *static_cast<uint64_t *>(
+        char * timestamp_ptr = shared_memory + data_layout->GetTimeStampOffset();
-            SharedMemoryFactory::Get(TIMESTAMP_SIZE)->Ptr()
+        m_timestamp.resize(data_layout->timestamp_length);
+        std::copy(
+            timestamp_ptr,
+            timestamp_ptr+data_layout->timestamp_length,
+            m_timestamp.begin()
         );
-        timestamp_size = std::min(timestamp_size, (uint64_t)25);
-        SharedMemory * search_tree = SharedMemoryFactory::Get(TIMESTAMP);
-        char * tree_ptr = static_cast<char *>( search_tree->Ptr() );
-        m_timestamp.resize(timestamp_size);
-        std::copy(tree_ptr, tree_ptr+timestamp_size, m_timestamp.begin());
     }
 
     void LoadRTree(
         const boost::filesystem::path & file_index_path
     ) {
-        uint64_t tree_size = *static_cast<uint64_t *>(
+        RTreeNode * tree_ptr = (RTreeNode *)(
-            SharedMemoryFactory::Get(R_SEARCH_TREE_SIZE)->Ptr()
+            shared_memory + data_layout->GetRSearchTreeOffset()
         );
-        SharedMemory * search_tree = SharedMemoryFactory::Get(R_SEARCH_TREE);
-        RTreeNode * tree_ptr = static_cast<RTreeNode *>( search_tree->Ptr() );
         m_static_rtree = new StaticRTree<RTreeLeaf, true>(
             tree_ptr,
-            tree_size,
+            data_layout->r_search_tree_size,
             file_index_path
         );
     }
 
     void LoadGraph() {
-        m_number_of_nodes = *static_cast<uint64_t *>(
+        m_number_of_nodes = data_layout->graph_node_list_size;
-            SharedMemoryFactory::Get(GRAPH_NODE_LIST_SIZE)->Ptr()
+        GraphNode * graph_nodes_ptr = (GraphNode *)(
+            shared_memory + data_layout->GetGraphNodeListOffset()
         );
-        SharedMemory * graph_nodes = SharedMemoryFactory::Get(GRAPH_NODE_LIST);
-        GraphNode * graph_nodes_ptr = static_cast<GraphNode *>( graph_nodes->Ptr() );
 
-        uint64_t number_of_edges = *static_cast<uint64_t *>(
+        GraphEdge * graph_edges_ptr = (GraphEdge *)(
-            SharedMemoryFactory::Get(GRAPH_EDGE_LIST_SIZE)->Ptr()
+            shared_memory + data_layout->GetGraphEdgeListOffsett()
         );
-        SharedMemory * graph_edges = SharedMemoryFactory::Get(GRAPH_EDGE_LIST);
-        GraphEdge * graph_edges_ptr = static_cast<GraphEdge *>( graph_edges->Ptr() );
 
-        typename ShM<GraphNode, true>::vector node_list(graph_nodes_ptr, m_number_of_nodes);
+        typename ShM<GraphNode, true>::vector node_list(
-        typename ShM<GraphEdge, true>::vector edge_list(graph_edges_ptr, number_of_edges);
+            graph_nodes_ptr,
+            data_layout->graph_node_list_size
+        );
+        typename ShM<GraphEdge, true>::vector edge_list(
+            graph_edges_ptr,
+            data_layout->graph_edge_list_size
+        );
         m_query_graph = new QueryGraph(
             node_list ,
             edge_list
@@ -110,70 +114,62 @@ private:
     }
 
     void LoadNodeAndEdgeInformation() {
-        uint64_t number_of_coordinates = *static_cast<uint64_t *>(
+
-            SharedMemoryFactory::Get(COORDINATE_LIST_SIZE)->Ptr()
+        FixedPointCoordinate * coordinate_list_ptr = (FixedPointCoordinate *)(
-        );
+            shared_memory + data_layout->GetCoordinateListOffset()
-        FixedPointCoordinate * coordinate_list_ptr = static_cast<FixedPointCoordinate *>(
-            SharedMemoryFactory::Get(COORDINATE_LIST)->Ptr()
         );
         typename ShM<FixedPointCoordinate, true>::vector coordinate_list(
                 coordinate_list_ptr,
-                number_of_coordinates
+                data_layout->coordinate_list_size
         );
         m_coordinate_list.swap( coordinate_list );
 
-        uint64_t number_of_turn_instructions = *static_cast<uint64_t *>(
+        TurnInstruction * turn_instruction_list_ptr = (TurnInstruction *)(
-            SharedMemoryFactory::Get(TURN_INSTRUCTION_LIST_SIZE)->Ptr()
+            shared_memory + data_layout->GetTurnInstructionListOffset()
         );
-
-        TurnInstruction * turn_instruction_list_ptr = static_cast<TurnInstruction *>(
-            SharedMemoryFactory::Get(TURN_INSTRUCTION_LIST)->Ptr()
-        );
-
         typename ShM<TurnInstruction, true>::vector turn_instruction_list(
             turn_instruction_list_ptr,
-            number_of_turn_instructions
+            data_layout->turn_instruction_list_size
         );
-
         m_turn_instruction_list.swap(turn_instruction_list);
+
+        unsigned * name_id_list_ptr = (unsigned *)(
+            shared_memory + data_layout->GetNameIDListOffset()
+        );
+        typename ShM<unsigned, true>::vector name_id_list(
+            name_id_list_ptr,
+            data_layout->name_id_list_size
+        );
+        m_name_ID_list.swap(name_id_list);
     }
 
     void LoadViaNodeList() {
-        uint64_t number_of_via_nodes = * static_cast<uint64_t *> (
+        NodeID * via_node_list_ptr = (NodeID *)(
-            SharedMemoryFactory::Get(VIA_NODE_LIST_SIZE)->Ptr()
+            shared_memory + data_layout->GetViaNodeListOffset()
-        );
-        NodeID * via_node_list_ptr = static_cast<NodeID *>(
-            SharedMemoryFactory::Get(VIA_NODE_LIST)->Ptr()
         );
         typename ShM<NodeID, true>::vector via_node_list(
             via_node_list_ptr,
-            number_of_via_nodes
+            data_layout->via_node_list_size
         );
         m_via_node_list.swap(via_node_list);
     }
 
     void LoadNames() {
-        uint64_t street_names_index_size = * static_cast<uint64_t *> (
+        unsigned * street_names_index_ptr = (unsigned *)(
-            SharedMemoryFactory::Get(NAME_INDEX_SIZE)->Ptr()
+            shared_memory + data_layout->GetNameIndexOffset()
-        );
-        unsigned * street_names_index_ptr = static_cast<unsigned *>(
-            SharedMemoryFactory::Get(NAMES_INDEX)->Ptr()
         );
         typename ShM<unsigned, true>::vector name_begin_indices(
             street_names_index_ptr,
-            street_names_index_size
+            data_layout->names_index_size
         );
         m_name_begin_indices.swap(m_name_begin_indices);
 
-        uint64_t names_list_size = * static_cast<uint64_t *>(
+        char * names_list_ptr = (char *)(
-            SharedMemoryFactory::Get(NAMES_LIST_SIZE)->Ptr()
+            shared_memory + data_layout->GetNameListOffset()
-        );
-        char * names_list_ptr = static_cast<char *>(
-            SharedMemoryFactory::Get(NAMES_LIST)->Ptr()
         );
         typename ShM<char, true>::vector names_char_list(
             names_list_ptr,
-            names_list_size
+            data_layout->names_list_size
         );
         m_names_char_list.swap(names_char_list);
     }
@@ -194,6 +190,14 @@ public:
                 base_path
         );
 
+        data_layout = (SharedDataLayout *)(
+            SharedMemoryFactory::Get(LAYOUT_1)->Ptr()
+        );
+
+        shared_memory = (char *)(
+            SharedMemoryFactory::Get(DATA_1)->Ptr()
+        );
+
         //load data
         SimpleLogger().Write() << "loading graph data";
         LoadGraph();

Server/DataStructures/SharedDataType.h

@@ -21,28 +21,197 @@ or see http://www.gnu.org/licenses/agpl.txt.
 #ifndef SHARED_DATA_TYPE_H_
 #define SHARED_DATA_TYPE_H_
 
+#include "BaseDataFacade.h"
+
+#include "../../DataStructures/Coordinate.h"
+#include "../../DataStructures/QueryEdge.h"
+#include "../../DataStructures/StaticGraph.h"
+#include "../../DataStructures/StaticRTree.h"
+#include "../../DataStructures/TurnInstructions.h"
+
+#include "../../typedefs.h"
+
+#include <boost/integer.hpp>
+
+typedef BaseDataFacade<QueryEdge::EdgeData>::RTreeLeaf RTreeLeaf;
+typedef StaticRTree<RTreeLeaf, true>::TreeNode RTreeNode;
+typedef StaticGraph<QueryEdge::EdgeData> QueryGraph;
+
+struct SharedDataLayout {
+    uint64_t names_index_size;
+    uint64_t names_list_size;
+    uint64_t name_id_list_size;
+    uint64_t via_node_list_size;
+    uint64_t graph_node_list_size;
+    uint64_t graph_edge_list_size;
+    uint64_t coordinate_list_size;
+    uint64_t turn_instruction_list_size;
+    uint64_t r_search_tree_size;
+
+    unsigned checksum;
+    unsigned timestamp_length;
+
+    SharedDataLayout() :
+        names_index_size(0),
+        names_list_size(0),
+        name_id_list_size(0),
+        via_node_list_size(0),
+        graph_node_list_size(0),
+        graph_edge_list_size(0),
+        coordinate_list_size(0),
+        turn_instruction_list_size(0),
+        r_search_tree_size(0),
+        checksum(0),
+        timestamp_length(0)
+    { }
+
+    uint64_t GetSizeOfLayout() const {
+        uint64_t result =
+            (names_index_size           * sizeof(unsigned)            ) +
+            (names_list_size            * sizeof(char)                ) +
+            (name_id_list_size          * sizeof(unsigned)            ) +
+            (via_node_list_size         * sizeof(NodeID)              ) +
+            (graph_node_list_size       * sizeof(QueryGraph::_StrNode)) +
+            (graph_edge_list_size       * sizeof(QueryGraph::_StrEdge)) +
+            (timestamp_length           * sizeof(char)                ) +
+            (coordinate_list_size       * sizeof(FixedPointCoordinate)) +
+            (turn_instruction_list_size * sizeof(TurnInstructions)    ) +
+            (r_search_tree_size         * sizeof(RTreeNode)           ) +
+            sizeof(checksum);
+        return result;
+    }
+
+    uint64_t GetNameIndexOffset() const {
+        return 0;
+    }
+    uint64_t GetNameListOffset() const {
+        uint64_t result =
+            (names_index_size           * sizeof(unsigned)            );
+        return result;
+    }
+    uint64_t GetNameIDListOffset() const {
+        uint64_t result =
+            (names_index_size           * sizeof(unsigned)            ) +
+            (names_list_size            * sizeof(char)                );
+        return result;
+    }
+    uint64_t GetViaNodeListOffset() const {
+        uint64_t result =
+            (names_index_size           * sizeof(unsigned)            ) +
+            (names_list_size            * sizeof(char)                ) +
+            (name_id_list_size          * sizeof(unsigned)            );
+        return result;
+    }
+    uint64_t GetGraphNodeListOffset() const {
+        uint64_t result =
+            (names_index_size           * sizeof(unsigned)            ) +
+            (names_list_size            * sizeof(char)                ) +
+            (name_id_list_size          * sizeof(unsigned)            ) +
+            (via_node_list_size         * sizeof(NodeID)              );
+        return result;
+    }
+    uint64_t GetGraphEdgeListOffsett() const {
+        uint64_t result =
+            (names_index_size           * sizeof(unsigned)            ) +
+            (names_list_size            * sizeof(char)                ) +
+            (name_id_list_size          * sizeof(unsigned)            ) +
+            (via_node_list_size         * sizeof(NodeID)              ) +
+            (graph_node_list_size       * sizeof(QueryGraph::_StrNode)) ;
+        return result;
+    }
+    uint64_t GetTimeStampOffset() const {
+        uint64_t result =
+            (names_index_size           * sizeof(unsigned)            ) +
+            (names_list_size            * sizeof(char)                ) +
+            (name_id_list_size          * sizeof(unsigned)            ) +
+            (via_node_list_size         * sizeof(NodeID)              ) +
+            (graph_node_list_size       * sizeof(QueryGraph::_StrNode)) +
+            (graph_edge_list_size       * sizeof(QueryGraph::_StrEdge));
+        return result;
+    }
+    uint64_t GetCoordinateListOffset() const {
+        uint64_t result =
+            (names_index_size           * sizeof(unsigned)            ) +
+            (names_list_size            * sizeof(char)                ) +
+            (name_id_list_size          * sizeof(unsigned)            ) +
+            (via_node_list_size         * sizeof(NodeID)              ) +
+            (graph_node_list_size       * sizeof(QueryGraph::_StrNode)) +
+            (graph_edge_list_size       * sizeof(QueryGraph::_StrEdge)) +
+            (timestamp_length           * sizeof(char)                );
+        return result;
+    }
+    uint64_t GetTurnInstructionListOffset() const {
+        uint64_t result =
+            (names_index_size           * sizeof(unsigned)            ) +
+            (names_list_size            * sizeof(char)                ) +
+            (name_id_list_size          * sizeof(unsigned)            ) +
+            (via_node_list_size         * sizeof(NodeID)              ) +
+            (graph_node_list_size       * sizeof(QueryGraph::_StrNode)) +
+            (graph_edge_list_size       * sizeof(QueryGraph::_StrEdge)) +
+            (timestamp_length           * sizeof(char)                ) +
+            (coordinate_list_size       * sizeof(FixedPointCoordinate));
+        return result;
+    }
+    uint64_t GetRSearchTreeOffset() const {
+        uint64_t result =
+            (names_index_size           * sizeof(unsigned)            ) +
+            (names_list_size            * sizeof(char)                ) +
+            (name_id_list_size          * sizeof(unsigned)            ) +
+            (via_node_list_size         * sizeof(NodeID)              ) +
+            (graph_node_list_size       * sizeof(QueryGraph::_StrNode)) +
+            (graph_edge_list_size       * sizeof(QueryGraph::_StrEdge)) +
+            (timestamp_length           * sizeof(char)                ) +
+            (coordinate_list_size       * sizeof(FixedPointCoordinate)) +
+            (turn_instruction_list_size * sizeof(TurnInstructions)    );
+        return result;
+    }
+    uint64_t GetChecksumOffset() const {
+        uint64_t result =
+            (names_index_size           * sizeof(unsigned)            ) +
+            (names_list_size            * sizeof(char)                ) +
+            (name_id_list_size          * sizeof(unsigned)            ) +
+            (via_node_list_size         * sizeof(NodeID)              ) +
+            (graph_node_list_size       * sizeof(QueryGraph::_StrNode)) +
+            (graph_edge_list_size       * sizeof(QueryGraph::_StrEdge)) +
+            (timestamp_length           * sizeof(char)                ) +
+            (coordinate_list_size       * sizeof(FixedPointCoordinate)) +
+            (turn_instruction_list_size * sizeof(TurnInstructions)    ) +
+            (r_search_tree_size         * sizeof(RTreeNode)           );
+        return result;
+    }
+};
+
 enum SharedDataType {
-    NAMES_INDEX = 0,
+    // NAMES_INDEX = 0,
-    NAME_INDEX_SIZE,
+    // NAME_INDEX_SIZE,
-    NAMES_LIST,
+    // NAMES_LIST,
-    NAMES_LIST_SIZE,
+    // NAMES_LIST_SIZE,
-    NAME_ID_LIST,
+    // NAME_ID_LIST,
-    NAME_ID_LIST_SIZE,
+    // NAME_ID_LIST_SIZE,
-    VIA_NODE_LIST,
+    // VIA_NODE_LIST,
-    VIA_NODE_LIST_SIZE,
+    // VIA_NODE_LIST_SIZE,
-    GRAPH_NODE_LIST,
+    // GRAPH_NODE_LIST,
-    GRAPH_NODE_LIST_SIZE,
+    // GRAPH_NODE_LIST_SIZE,
-    GRAPH_EDGE_LIST,
+    // GRAPH_EDGE_LIST,
-    GRAPH_EDGE_LIST_SIZE,
+    // GRAPH_EDGE_LIST_SIZE,
-    CHECK_SUM,
+    // CHECK_SUM,
-    TIMESTAMP,
+    // TIMESTAMP,
-    TIMESTAMP_SIZE,
+    // TIMESTAMP_SIZE,
-    COORDINATE_LIST,
+    // COORDINATE_LIST,
-    COORDINATE_LIST_SIZE,
+    // COORDINATE_LIST_SIZE,
-    TURN_INSTRUCTION_LIST,
+    // TURN_INSTRUCTION_LIST,
-    TURN_INSTRUCTION_LIST_SIZE,
+    // TURN_INSTRUCTION_LIST_SIZE,
-    R_SEARCH_TREE,
+    // R_SEARCH_TREE,
-    R_SEARCH_TREE_SIZE
+    // R_SEARCH_TREE_SIZE
+
+    LAYOUT_1,
+    DATA_1,
+    LAYOUT_2,
+    DATA_2,
+    LAYOUT_3,
+    DATA_3,
+    LAYOUT_4,
+    DATA_5
 };
 
 #endif /* SHARED_DATA_TYPE_H_ */