Enables the use of multiple segment-speed-files on the osrm-contract

command line, and exposes the file name used for each edge in the debug
tiles.

src/contractor/contractor.cpp

@@ -71,8 +71,9 @@ int Contractor::Run()
 
     std::size_t max_edge_id = LoadEdgeExpandedGraph(
         config.edge_based_graph_path, edge_based_edge_list, config.edge_segment_lookup_path,
-        config.edge_penalty_path, config.segment_speed_lookup_path, config.node_based_graph_path,
-        config.geometry_path, config.rtree_leaf_path);
+        config.edge_penalty_path, config.segment_speed_lookup_paths, config.node_based_graph_path,
+        config.geometry_path, config.datasource_names_path, config.datasource_indexes_path,
+        config.rtree_leaf_path);
 
     // Contracting the edge-expanded graph
 
@@ -127,15 +128,17 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
     util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list,
     const std::string &edge_segment_lookup_filename,
     const std::string &edge_penalty_filename,
-    const std::string &segment_speed_filename,
+    const std::vector<std::string> &segment_speed_filenames,
     const std::string &nodes_filename,
     const std::string &geometry_filename,
+    const std::string &datasource_names_filename,
+    const std::string &datasource_indexes_filename,
     const std::string &rtree_leaf_filename)
 {
     util::SimpleLogger().Write() << "Opening " << edge_based_graph_filename;
     boost::filesystem::ifstream input_stream(edge_based_graph_filename, std::ios::binary);
 
-    const bool update_edge_weights = segment_speed_filename != "";
+    const bool update_edge_weights = !segment_speed_filenames.empty();
 
     boost::filesystem::ifstream edge_segment_input_stream;
     boost::filesystem::ifstream edge_fixed_penalties_input_stream;
@@ -167,22 +170,36 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
     util::SimpleLogger().Write() << "Reading " << number_of_edges
                                  << " edges from the edge based graph";
 
-    std::unordered_map<std::pair<OSMNodeID, OSMNodeID>, unsigned> segment_speed_lookup;
+    std::unordered_map<std::pair<OSMNodeID, OSMNodeID>, std::pair<unsigned, uint8_t>>
+        segment_speed_lookup;
 
     if (update_edge_weights)
     {
-        util::SimpleLogger().Write()
-            << "Segment speed data supplied, will update edge weights from "
-            << segment_speed_filename;
-        io::CSVReader<3> csv_in(segment_speed_filename);
-        csv_in.set_header("from_node", "to_node", "speed");
-        uint64_t from_node_id{};
-        uint64_t to_node_id{};
-        unsigned speed{};
-        while (csv_in.read_row(from_node_id, to_node_id, speed))
+        uint8_t file_id = 1;
+        for (auto segment_speed_filename : segment_speed_filenames)
         {
-            segment_speed_lookup[std::make_pair(OSMNodeID(from_node_id), OSMNodeID(to_node_id))] =
-                speed;
+            util::SimpleLogger().Write()
+                << "Segment speed data supplied, will update edge weights from "
+                << segment_speed_filename;
+            io::CSVReader<3> csv_in(segment_speed_filename);
+            csv_in.set_header("from_node", "to_node", "speed");
+            uint64_t from_node_id{};
+            uint64_t to_node_id{};
+            unsigned speed{};
+            while (csv_in.read_row(from_node_id, to_node_id, speed))
+            {
+                segment_speed_lookup[std::make_pair(OSMNodeID(from_node_id),
+                                                    OSMNodeID(to_node_id))] =
+                    std::make_pair(speed, file_id);
+            }
+            ++file_id;
+
+            // Check for overflow
+            if (file_id == 0)
+            {
+                throw util::exception(
+                    "Sorry, there's a limit of 254 segment speed files, you supplied too many");
+            }
         }
 
         std::vector<extractor::QueryNode> internal_to_external_node_map;
@@ -241,6 +258,14 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
             }
         }
 
+        // This is a list of the "data source id" for every segment in the compressed
+        // geometry container.  We assume that everything so far has come from the
+        // profile (data source 0).  Here, we replace the 0's with the index of the
+        // CSV file that supplied the value that gets used for that segment, then
+        // we write out this list so that it can be returned by the debugging
+        // vector tiles later on.
+        std::vector<uint8_t> m_geometry_datasource(m_geometry_list.size(), 0);
+
         // Now, we iterate over all the segments stored in the StaticRTree, updating
         // the packed geometry weights in the `.geometries` file (note: we do not
         // update the RTree itself, we just use the leaf nodes to iterate over all segments)
@@ -297,12 +322,16 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
                             segment_speed_lookup.find(std::make_pair(u->node_id, v->node_id));
                         if (forward_speed_iter != segment_speed_lookup.end())
                         {
-                            int new_segment_weight = std::max(
-                                1, static_cast<int>(std::floor(
-                                       (segment_length * 10.) / (forward_speed_iter->second / 3.6) +
-                                       .5)));
+                            int new_segment_weight =
+                                std::max(1, static_cast<int>(std::floor(
+                                                (segment_length * 10.) /
+                                                    (forward_speed_iter->second.first / 3.6) +
+                                                .5)));
                             m_geometry_list[forward_begin + leaf_object.fwd_segment_position]
                                 .weight = new_segment_weight;
+                            m_geometry_datasource[forward_begin +
+                                                  leaf_object.fwd_segment_position] =
+                                forward_speed_iter->second.second;
                         }
                     }
                     if (leaf_object.reverse_packed_geometry_id != SPECIAL_EDGEID)
@@ -338,12 +367,15 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
                             segment_speed_lookup.find(std::make_pair(u->node_id, v->node_id));
                         if (reverse_speed_iter != segment_speed_lookup.end())
                         {
-                            int new_segment_weight = std::max(
-                                1, static_cast<int>(std::floor(
-                                       (segment_length * 10.) / (reverse_speed_iter->second / 3.6) +
-                                       .5)));
+                            int new_segment_weight =
+                                std::max(1, static_cast<int>(std::floor(
+                                                (segment_length * 10.) /
+                                                    (reverse_speed_iter->second.first / 3.6) +
+                                                .5)));
                             m_geometry_list[reverse_begin + rev_segment_position].weight =
                                 new_segment_weight;
+                            m_geometry_datasource[reverse_begin + rev_segment_position] =
+                                reverse_speed_iter->second.second;
                         }
                     }
                 }
@@ -370,6 +402,34 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
                                   number_of_compressed_geometries *
                                       sizeof(extractor::CompressedEdgeContainer::CompressedEdge));
         }
+
+        {
+            std::ofstream datasource_stream(datasource_indexes_filename, std::ios::binary);
+            if (!datasource_stream)
+            {
+                throw util::exception("Failed to open " + datasource_indexes_filename +
+                                      " for writing");
+            }
+            auto number_of_datasource_entries = m_geometry_datasource.size();
+            datasource_stream.write(reinterpret_cast<const char *>(&number_of_datasource_entries),
+                                    sizeof(number_of_datasource_entries));
+            datasource_stream.write(reinterpret_cast<char *>(&(m_geometry_datasource[0])),
+                                    number_of_datasource_entries * sizeof(uint8_t));
+        }
+
+        {
+            std::ofstream datasource_stream(datasource_names_filename, std::ios::binary);
+            if (!datasource_stream)
+            {
+                throw util::exception("Failed to open " + datasource_names_filename +
+                                      " for writing");
+            }
+            datasource_stream << "lua profile" << std::endl;
+            for (auto const &name : segment_speed_filenames)
+            {
+                datasource_stream << name << std::endl;
+            }
+        }
     }
 
     // TODO: can we read this in bulk?  util::DeallocatingVector isn't necessarily
@@ -413,9 +473,9 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
                     // This sets the segment weight using the same formula as the
                     // EdgeBasedGraphFactory for consistency.  The *why* of this formula
                     // is lost in the annals of time.
-                    int new_segment_weight =
-                        std::max(1, static_cast<int>(std::floor(
-                                        (segment_length * 10.) / (speed_iter->second / 3.6) + .5)));
+                    int new_segment_weight = std::max(
+                        1, static_cast<int>(std::floor(
+                               (segment_length * 10.) / (speed_iter->second.first / 3.6) + .5)));
                     new_weight += new_segment_weight;
                 }
                 else

src/engine/plugins/tile.cpp

@@ -191,6 +191,8 @@ Status TilePlugin::HandleRequest(const api::TileParameters &parameters, std::str
     xyz2mercator(parameters.x, parameters.y, parameters.z, min_lon, min_lat, max_lon, max_lat);
     const bbox tile_bbox{min_lon, min_lat, max_lon, max_lat};
 
+    uint8_t max_datasource_id = 0;
+
     // Protobuf serialized blocks when objects go out of scope, hence
     // the extra scoping below.
     protozero::pbf_writer tile_writer{pbf_buffer};
@@ -221,12 +223,20 @@ Status TilePlugin::HandleRequest(const api::TileParameters &parameters, std::str
                 int forward_weight = 0;
                 int reverse_weight = 0;
 
+                uint8_t forward_datasource = 0;
+                uint8_t reverse_datasource = 0;
+
                 if (edge.forward_packed_geometry_id != SPECIAL_EDGEID)
                 {
                     std::vector<EdgeWeight> forward_weight_vector;
                     facade.GetUncompressedWeights(edge.forward_packed_geometry_id,
                                                   forward_weight_vector);
                     forward_weight = forward_weight_vector[edge.fwd_segment_position];
+
+                    std::vector<uint8_t> forward_datasource_vector;
+                    facade.GetUncompressedDatasources(edge.forward_packed_geometry_id,
+                                                      forward_datasource_vector);
+                    forward_datasource = forward_datasource_vector[edge.fwd_segment_position];
                 }
 
                 if (edge.reverse_packed_geometry_id != SPECIAL_EDGEID)
@@ -239,8 +249,20 @@ Status TilePlugin::HandleRequest(const api::TileParameters &parameters, std::str
 
                     reverse_weight = reverse_weight_vector[reverse_weight_vector.size() -
                                                            edge.fwd_segment_position - 1];
+
+                    std::vector<uint8_t> reverse_datasource_vector;
+                    facade.GetUncompressedDatasources(edge.reverse_packed_geometry_id,
+                                                      reverse_datasource_vector);
+                    reverse_datasource =
+                        reverse_datasource_vector[reverse_datasource_vector.size() -
+                                                  edge.fwd_segment_position - 1];
                 }
 
+                // Keep track of the highest datasource seen so that we don't write unnecessary
+                // data to the layer attribute values
+                max_datasource_id = std::max(max_datasource_id, forward_datasource);
+                max_datasource_id = std::max(max_datasource_id, reverse_datasource);
+
                 // If this is a valid forward edge, go ahead and add it to the tile
                 if (forward_weight != 0 && edge.forward_edge_based_node_id != SPECIAL_NODEID)
                 {
@@ -296,7 +318,10 @@ Status TilePlugin::HandleRequest(const api::TileParameters &parameters, std::str
                         field.add_element(
                             std::min(speed, 127u)); // save the speed value, capped at 127
                         field.add_element(1);       // "is_small" tag key offset
-                        field.add_element(edge.component.is_tiny ? 0 : 1); // is_small feature
+                        field.add_element(128 +
+                                          (edge.component.is_tiny ? 0 : 1)); // is_small feature
+                        field.add_element(2);                        // "datasource" tag key offset
+                        field.add_element(130 + forward_datasource); // datasource value offset
                     }
                     {
                         // Encode the geometry for the feature
@@ -346,7 +371,10 @@ Status TilePlugin::HandleRequest(const api::TileParameters &parameters, std::str
                         field.add_element(
                             std::min(speed, 127u)); // save the speed value, capped at 127
                         field.add_element(1);       // "is_small" tag key offset
-                        field.add_element(edge.component.is_tiny ? 0 : 1); // is_small feature
+                        field.add_element(128 +
+                                          (edge.component.is_tiny ? 0 : 1)); // is_small feature
+                        field.add_element(2);                        // "datasource" tag key offset
+                        field.add_element(130 + reverse_datasource); // datasource value offset
                     }
                     {
                         protozero::packed_field_uint32 geometry(feature_writer, 4);
@@ -361,6 +389,7 @@ Status TilePlugin::HandleRequest(const api::TileParameters &parameters, std::str
         // earlier
         layer_writer.add_string(3, "speed");
         layer_writer.add_string(3, "is_small");
+        layer_writer.add_string(3, "datasource");
 
         // Now, we write out the possible speed value arrays and possible is_tiny
         // values.  Field type 4 is the "values" field.  It's a variable type field,
@@ -384,6 +413,15 @@ Status TilePlugin::HandleRequest(const api::TileParameters &parameters, std::str
             // Attribute value 7 == bool type
             values_writer.add_bool(7, false);
         }
+        for (std::size_t i = 0; i <= max_datasource_id; i++)
+        {
+            {
+                // Writing field type 4 == variant type
+                protozero::pbf_writer values_writer(layer_writer, 4);
+                // Attribute value 1 == string type
+                values_writer.add_string(1, facade.GetDatasourceName(i));
+            }
+        }
     }
 
     return Status::Ok;

src/extractor/extractor.cpp

@@ -528,12 +528,7 @@ Extractor::BuildEdgeExpandedGraph(std::vector<QueryNode> &internal_to_external_n
 
     edge_based_graph_factory.Run(config.edge_output_path, lua_state,
                                  config.edge_segment_lookup_path, config.edge_penalty_path,
-                                 config.generate_edge_lookup
-#ifdef DEBUG_GEOMETRY
-                                 ,
-                                 config.debug_turns_path
-#endif
-                                 );
+                                 config.generate_edge_lookup);
 
     lua_close(lua_state);
 

src/storage/storage.cpp

@@ -131,6 +131,14 @@ int Storage::Run()
     {
         throw util::exception("no core file found");
     }
+    if (paths.find("datasource_indexes") == paths.end())
+    {
+        throw util::exception("no datasource_indexes file found");
+    }
+    if (paths.find("datasource_names") == paths.end())
+    {
+        throw util::exception("no datasource_names file found");
+    }
 
     auto paths_iterator = paths.find("hsgrdata");
     BOOST_ASSERT(paths.end() != paths_iterator);
@@ -170,6 +178,14 @@ int Storage::Run()
     BOOST_ASSERT(paths.end() != paths_iterator);
     BOOST_ASSERT(!paths_iterator->second.empty());
     const boost::filesystem::path &core_marker_path = paths_iterator->second;
+    paths_iterator = paths.find("datasource_indexes");
+    BOOST_ASSERT(paths.end() != paths_iterator);
+    BOOST_ASSERT(!paths_iterator->second.empty());
+    const boost::filesystem::path &datasource_indexes_path = paths_iterator->second;
+    paths_iterator = paths.find("datasource_names");
+    BOOST_ASSERT(paths.end() != paths_iterator);
+    BOOST_ASSERT(!paths_iterator->second.empty());
+    const boost::filesystem::path &datasource_names_path = paths_iterator->second;
 
     // determine segment to use
     bool segment2_in_use = SharedMemory::RegionExists(LAYOUT_2);
@@ -322,6 +338,45 @@ int Storage::Run()
     geometry_input_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned));
     shared_layout_ptr->SetBlockSize<extractor::CompressedEdgeContainer::CompressedEdge>(
         SharedDataLayout::GEOMETRIES_LIST, number_of_compressed_geometries);
+
+    // load datasource sizes.  This file is optional, and it's non-fatal if it doesn't
+    // exist.
+    std::ifstream geometry_datasource_input_stream(datasource_indexes_path.c_str(),
+                                                   std::ios::binary);
+    if (geometry_datasource_input_stream)
+    {
+        std::size_t number_of_compressed_datasources = 0;
+        geometry_datasource_input_stream.read(
+            reinterpret_cast<char *>(&number_of_compressed_datasources), sizeof(std::size_t));
+        shared_layout_ptr->SetBlockSize<uint8_t>(SharedDataLayout::DATASOURCES_LIST,
+                                                 number_of_compressed_datasources);
+    }
+
+    // Load datasource name sizes.  This file is optional, and it's non-fatal if it doesn't
+    // exist
+    std::ifstream datasource_names_input_stream(datasource_names_path.c_str(), std::ios::binary);
+    std::vector<char> m_datasource_name_data;
+    std::vector<std::size_t> m_datasource_name_offsets;
+    std::vector<std::size_t> m_datasource_name_lengths;
+    if (datasource_names_input_stream)
+    {
+        std::string name;
+        while (std::getline(datasource_names_input_stream, name))
+        {
+            m_datasource_name_offsets.push_back(m_datasource_name_data.size());
+            std::copy(name.c_str(), name.c_str() + name.size(),
+                      std::back_inserter(m_datasource_name_data));
+            m_datasource_name_lengths.push_back(name.size());
+        }
+
+        shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::DATASOURCE_NAME_DATA,
+                                              m_datasource_name_data.size());
+        shared_layout_ptr->SetBlockSize<std::size_t>(SharedDataLayout::DATASOURCE_NAME_OFFSETS,
+                                                m_datasource_name_offsets.size());
+        shared_layout_ptr->SetBlockSize<std::size_t>(SharedDataLayout::DATASOURCE_NAME_LENGTHS,
+                                                m_datasource_name_lengths.size());
+    }
+
     // allocate shared memory block
     util::SimpleLogger().Write() << "allocating shared memory of "
                                  << shared_layout_ptr->GetSizeOfLayout() << " bytes";
@@ -435,6 +490,50 @@ int Storage::Run()
             shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_LIST));
     }
 
+    // load datasource information (if it exists)
+    if (geometry_datasource_input_stream)
+    {
+        uint8_t *datasources_list_ptr = shared_layout_ptr->GetBlockPtr<uint8_t, true>(
+            shared_memory_ptr, SharedDataLayout::DATASOURCES_LIST);
+        if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCES_LIST) > 0)
+        {
+            geometry_datasource_input_stream.read(
+                reinterpret_cast<char *>(datasources_list_ptr),
+                shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCES_LIST));
+        }
+    }
+
+    // load datasource name information (if it exists)
+    if (!m_datasource_name_data.empty())
+    {
+        char *datasource_name_data_ptr = shared_layout_ptr->GetBlockPtr<char, true>(
+            shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_DATA);
+        if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_DATA) > 0)
+        {
+            std::cout << "Copying "
+                      << (m_datasource_name_data.end() - m_datasource_name_data.begin())
+                      << " chars into name data ptr\n";
+            std::copy(m_datasource_name_data.begin(), m_datasource_name_data.end(),
+                      datasource_name_data_ptr);
+        }
+
+        auto datasource_name_offsets_ptr = shared_layout_ptr->GetBlockPtr<std::size_t, true>(
+            shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_OFFSETS);
+        if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_OFFSETS) > 0)
+        {
+            std::copy(m_datasource_name_offsets.begin(), m_datasource_name_offsets.end(),
+                      datasource_name_offsets_ptr);
+        }
+
+        auto datasource_name_lengths_ptr = shared_layout_ptr->GetBlockPtr<std::size_t, true>(
+            shared_memory_ptr, SharedDataLayout::DATASOURCE_NAME_LENGTHS);
+        if (shared_layout_ptr->GetBlockSize(SharedDataLayout::DATASOURCE_NAME_LENGTHS) > 0)
+        {
+            std::copy(m_datasource_name_lengths.begin(), m_datasource_name_lengths.end(),
+                      datasource_name_lengths_ptr);
+        }
+    }
+
     // Loading list of coordinates
     util::Coordinate *coordinates_ptr = shared_layout_ptr->GetBlockPtr<util::Coordinate, true>(
         shared_memory_ptr, SharedDataLayout::COORDINATE_LIST);

src/tools/contract.cpp

@@ -39,9 +39,10 @@ return_code parseArguments(int argc, char *argv[], contractor::ContractorConfig
         "core,k",
         boost::program_options::value<double>(&contractor_config.core_factor)->default_value(1.0),
         "Percentage of the graph (in vertices) to contract [0..1]")(
-        "segment-speed-file",
-        boost::program_options::value<std::string>(&contractor_config.segment_speed_lookup_path),
-        "Lookup file containing nodeA,nodeB,speed data to adjust edge weights")(
+        "segment-speed-file", boost::program_options::value<std::vector<std::string>>(
+                                  &contractor_config.segment_speed_lookup_paths)
+                                  ->composing(),
+        "Lookup files containing nodeA, nodeB, speed data to adjust edge weights")(
         "level-cache,o", boost::program_options::value<bool>(&contractor_config.use_cached_priority)
                              ->default_value(false),
         "Use .level file to retain the contaction level for each node from the last run.");

src/tools/store.cpp

@@ -38,7 +38,13 @@ bool generateDataStoreOptions(const int argc, const char *argv[], storage::DataP
         "namesdata", boost::program_options::value<boost::filesystem::path>(&paths["namesdata"]),
         ".names file")("timestamp",
                        boost::program_options::value<boost::filesystem::path>(&paths["timestamp"]),
-                       ".timestamp file");
+                       ".timestamp file")(
+        "datasource_names",
+        boost::program_options::value<boost::filesystem::path>(&paths["datasource_names"]),
+        ".datasource_names file")(
+        "datasource_indexes",
+        boost::program_options::value<boost::filesystem::path>(&paths["datasource_indexes"]),
+        ".datasource_indexes file");
 
     // hidden options, will be allowed on command line but will not be shown to the user
     boost::program_options::options_description hidden_options("Hidden options");
@@ -145,6 +151,18 @@ bool generateDataStoreOptions(const int argc, const char *argv[], storage::DataP
         path_iterator->second = base_string + ".timestamp";
     }
 
+    path_iterator = paths.find("datasource_indexes");
+    if (path_iterator != paths.end())
+    {
+        path_iterator->second = base_string + ".datasource_indexes";
+    }
+
+    path_iterator = paths.find("datasource_names");
+    if (path_iterator != paths.end())
+    {
+        path_iterator->second = base_string + ".datasource_names";
+    }
+
     path_iterator = paths.find("hsgrdata");
     if (path_iterator == paths.end() || path_iterator->second.string().empty() ||
         !boost::filesystem::is_regular_file(path_iterator->second))