Also support loading core information into shared memory

contractor/contractor.hpp

@@ -549,12 +549,19 @@ class Contractor
 
         if (remaining_nodes.size() > 2)
         {
+            // TODO: for small cores a sorted array of core ids might also work good
             for (const auto& node : remaining_nodes)
             {
                 auto orig_id = orig_node_id_from_new_node_id_map[node.id];
                 is_core_node[orig_id] = true;
             }
         }
+        else
+        {
+            // in this case we don't need core markers since we fully contracted
+            // the graph
+            is_core_node.clear();
+        }
 
         SimpleLogger().Write() << "[core] " << remaining_nodes.size() << " nodes " << contractor_graph->GetNumberOfEdges() << " edges." << std::endl;
 

datastore.cpp

@@ -164,6 +164,10 @@ int main(const int argc, const char *argv[])
         {
             throw osrm::exception("no geometry file found");
         }
+        if (server_paths.find("core") == server_paths.end())
+        {
+            throw osrm::exception("no core file found");
+        }
 
         ServerPaths::const_iterator paths_iterator = server_paths.find("hsgrdata");
         BOOST_ASSERT(server_paths.end() != paths_iterator);
@@ -199,6 +203,10 @@ int main(const int argc, const char *argv[])
         BOOST_ASSERT(server_paths.end() != paths_iterator);
         BOOST_ASSERT(!paths_iterator->second.empty());
         const boost::filesystem::path &geometries_data_path = paths_iterator->second;
+        paths_iterator = server_paths.find("core");
+        BOOST_ASSERT(server_paths.end() != paths_iterator);
+        BOOST_ASSERT(!paths_iterator->second.empty());
+        const boost::filesystem::path &core_marker_path = paths_iterator->second;
 
         // determine segment to use
         bool segment2_in_use = SharedMemory::RegionExists(LAYOUT_2);
@@ -329,6 +337,13 @@ int main(const int argc, const char *argv[])
         }
         shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::TIMESTAMP, m_timestamp.length());
 
+        // load core marker size
+        boost::filesystem::ifstream core_marker_file(core_marker_path, std::ios::binary);
+
+        uint32_t number_of_core_markers = 0;
+        core_marker_file.read((char *)&number_of_core_markers, sizeof(uint32_t));
+        shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::CORE_MARKER, number_of_core_markers);
+
         // load coordinate size
         boost::filesystem::ifstream nodes_input_stream(nodes_data_path, std::ios::binary);
         unsigned coordinate_list_size = 0;
@@ -509,6 +524,35 @@ int main(const int argc, const char *argv[])
         }
         tree_node_file.close();
 
+        // load core markers
+        std::vector<char> unpacked_core_markers(number_of_core_markers);
+        core_marker_file.read((char *)unpacked_core_markers.data(), sizeof(char)*number_of_core_markers);
+
+        unsigned *core_marker_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(
+            shared_memory_ptr, SharedDataLayout::CORE_MARKER);
+
+        for (auto i = 0u; i < number_of_core_markers; ++i)
+        {
+            BOOST_ASSERT(unpacked_core_markers[i] == 0 || unpacked_core_markers[i] == 1);
+
+            if (unpacked_core_markers[i] == 1)
+            {
+                const unsigned bucket = i / 32;
+                const unsigned offset = i % 32;
+                const unsigned value = [&]
+                {
+                    unsigned return_value = 0;
+                    if (0 != offset)
+                    {
+                        return_value = core_marker_ptr[bucket];
+                    }
+                    return return_value;
+                }();
+
+                core_marker_ptr[bucket] = (value | (1 << offset));
+            }
+        }
+
         // load the nodes of the search graph
         QueryGraph::NodeArrayEntry *graph_node_list_ptr =
             shared_layout_ptr->GetBlockPtr<QueryGraph::NodeArrayEntry, true>(

server/data_structures/internal_datafacade.hpp

@@ -181,6 +181,12 @@ template <class EdgeDataT> class InternalDataFacade final : public BaseDataFacad
         std::vector<char> unpacked_core_markers(number_of_markers);
         core_stream.read((char *)unpacked_core_markers.data(), sizeof(char)*number_of_markers);
 
+        // in this case we have nothing to do
+        if (number_of_markers <= 0)
+        {
+            return;
+        }
+
         m_is_core_node.resize(number_of_markers);
         for (auto i = 0u; i < number_of_markers; ++i)
         {
@@ -493,9 +499,16 @@ template <class EdgeDataT> class InternalDataFacade final : public BaseDataFacad
     }
 
     virtual bool IsCoreNode(const NodeID id) const override final
+    {
+        if (m_is_core_node.size() > 0)
         {
             return m_is_core_node[id];
         }
+        else
+        {
+            return false;
+        }
+    }
 
     virtual void GetUncompressedGeometry(const unsigned id,
                                          std::vector<unsigned> &result_nodes) const override final

server/data_structures/shared_datafacade.hpp

@@ -194,6 +194,21 @@ template <class EdgeDataT> class SharedDataFacade final : public BaseDataFacade<
         m_names_char_list.swap(names_char_list);
     }
 
+    void LoadCoreInformation()
+    {
+        if (data_layout->num_entries[SharedDataLayout::CORE_MARKER] <= 0)
+        {
+            return;
+        }
+
+        unsigned *core_marker_ptr = data_layout->GetBlockPtr<unsigned>(
+            shared_memory, SharedDataLayout::CORE_MARKER);
+        typename ShM<bool, true>::vector is_core_node(
+            core_marker_ptr,
+            data_layout->num_entries[SharedDataLayout::CORE_MARKER]);
+        m_is_core_node.swap(is_core_node);
+    }
+
     void LoadGeometries()
     {
         unsigned *geometries_compressed_ptr = data_layout->GetBlockPtr<unsigned>(
@@ -269,6 +284,7 @@ template <class EdgeDataT> class SharedDataFacade final : public BaseDataFacade<
             LoadTimestamp();
             LoadViaNodeList();
             LoadNames();
+            LoadCoreInformation();
 
             data_layout->PrintInformation();
 
@@ -450,7 +466,11 @@ template <class EdgeDataT> class SharedDataFacade final : public BaseDataFacade<
 
     bool IsCoreNode(const NodeID id) const override final
     {
-        //return m_is_core_node[id];
+        if (m_is_core_node.size() > 0)
+        {
+            return m_is_core_node.at(id);
+        }
+
         return false;
     }
 

server/data_structures/shared_datatype.hpp

@@ -62,6 +62,7 @@ struct SharedDataLayout
         HSGR_CHECKSUM,
         TIMESTAMP,
         FILE_INDEX_PATH,
+        CORE_MARKER,
         NUM_BLOCKS
     };
 
@@ -72,40 +73,6 @@ struct SharedDataLayout
 
     void PrintInformation() const
     {
-        SimpleLogger().Write(logDEBUG) << "-";
-        SimpleLogger().Write(logDEBUG)
-            << "name_offsets_size:          " << num_entries[NAME_OFFSETS];
-        SimpleLogger().Write(logDEBUG)
-            << "name_blocks_size:           " << num_entries[NAME_BLOCKS];
-        SimpleLogger().Write(logDEBUG)
-            << "name_char_list_size:        " << num_entries[NAME_CHAR_LIST];
-        SimpleLogger().Write(logDEBUG)
-            << "name_id_list_size:          " << num_entries[NAME_ID_LIST];
-        SimpleLogger().Write(logDEBUG)
-            << "via_node_list_size:         " << num_entries[VIA_NODE_LIST];
-        SimpleLogger().Write(logDEBUG)
-            << "graph_node_list_size:       " << num_entries[GRAPH_NODE_LIST];
-        SimpleLogger().Write(logDEBUG)
-            << "graph_edge_list_size:       " << num_entries[GRAPH_EDGE_LIST];
-        SimpleLogger().Write(logDEBUG) << "timestamp_length:           " << num_entries[TIMESTAMP];
-        SimpleLogger().Write(logDEBUG)
-            << "coordinate_list_size:       " << num_entries[COORDINATE_LIST];
-        SimpleLogger().Write(logDEBUG)
-            << "turn_instruction_list_size: " << num_entries[TURN_INSTRUCTION];
-        SimpleLogger().Write(logDEBUG)
-            << "travel_mode_list_size:      " << num_entries[TRAVEL_MODE];
-        SimpleLogger().Write(logDEBUG)
-            << "r_search_tree_size:         " << num_entries[R_SEARCH_TREE];
-        SimpleLogger().Write(logDEBUG)
-            << "geometries_indicators:      " << num_entries[GEOMETRIES_INDICATORS] << "/"
-            << ((num_entries[GEOMETRIES_INDICATORS] / 8) + 1);
-        SimpleLogger().Write(logDEBUG)
-            << "geometries_index_list_size: " << num_entries[GEOMETRIES_INDEX];
-        SimpleLogger().Write(logDEBUG)
-            << "geometries_list_size:       " << num_entries[GEOMETRIES_LIST];
-        SimpleLogger().Write(logDEBUG)
-            << "sizeof(checksum):           " << entry_size[HSGR_CHECKSUM];
-
         SimpleLogger().Write(logDEBUG) << "NAME_OFFSETS         "
                                        << ": " << GetBlockSize(NAME_OFFSETS);
         SimpleLogger().Write(logDEBUG) << "NAME_BLOCKS          "
@@ -140,6 +107,8 @@ struct SharedDataLayout
                                        << ": " << GetBlockSize(TIMESTAMP);
         SimpleLogger().Write(logDEBUG) << "FILE_INDEX_PATH      "
                                        << ": " << GetBlockSize(FILE_INDEX_PATH);
+        SimpleLogger().Write(logDEBUG) << "CORE_MARKER          "
+                                       << ": " << GetBlockSize(CORE_MARKER);
     }
 
     template <typename T> inline void SetBlockSize(BlockID bid, uint64_t entries)
@@ -150,11 +119,11 @@ struct SharedDataLayout
 
     inline uint64_t GetBlockSize(BlockID bid) const
     {
-        // special encoding
+        // special bit encoding
-        if (bid == GEOMETRIES_INDICATORS)
+        if (bid == GEOMETRIES_INDICATORS || bid == CORE_MARKER)
         {
-            return (num_entries[GEOMETRIES_INDICATORS] / 32 + 1) *
+            return (num_entries[bid] / 32 + 1) *
-                   entry_size[GEOMETRIES_INDICATORS];
+                   entry_size[bid];
         }
 
         return num_entries[bid] * entry_size[bid];

util/datastore_options.hpp

@@ -69,6 +69,8 @@ bool GenerateDataStoreOptions(const int argc, const char *argv[], ServerPaths &p
         ".ramIndex file")(
         "fileindex", boost::program_options::value<boost::filesystem::path>(&paths["fileindex"]),
         ".fileIndex file")(
+        "core", boost::program_options::value<boost::filesystem::path>(&paths["core"]),
+        ".core file")(
         "namesdata", boost::program_options::value<boost::filesystem::path>(&paths["namesdata"]),
         ".names file")("timestamp",
                        boost::program_options::value<boost::filesystem::path>(&paths["timestamp"]),
@@ -130,6 +132,8 @@ bool GenerateDataStoreOptions(const int argc, const char *argv[], ServerPaths &p
                                     !paths.find("ramindex")->second.string().empty()) ||
                                    (paths.find("fileindex") != paths.end() &&
                                     !paths.find("fileindex")->second.string().empty()) ||
+                                   (paths.find("core") != paths.end() &&
+                                    !paths.find("core")->second.string().empty()) ||
                                    (paths.find("timestamp") != paths.end() &&
                                     !paths.find("timestamp")->second.string().empty());
 
@@ -199,6 +203,12 @@ bool GenerateDataStoreOptions(const int argc, const char *argv[], ServerPaths &p
             path_iterator->second = base_string + ".fileIndex";
         }
 
+        path_iterator = paths.find("core");
+        if (path_iterator != paths.end())
+        {
+            path_iterator->second = base_string + ".core";
+        }
+
         path_iterator = paths.find("namesdata");
         if (path_iterator != paths.end())
         {