expose mandatory / optional / output files in io_config; config files to use new io_config constructor

include/contractor/contractor_config.hpp

@@ -44,7 +44,8 @@ struct ContractorConfig final : storage::IOConfig
 {
     ContractorConfig() : requested_num_threads(0) {}
 
-    ContractorConfig(const boost::filesystem::path &base) : requested_num_threads(0), IOConfig(base)
+    ContractorConfig(const boost::filesystem::path &base) : requested_num_threads(0),
+      IOConfig({".osrm", }, {}, {".osrm.level", ".osrm.core", ".osrm.hsgr", ".osrm.enw"})
     {
     }
 
@@ -56,6 +57,9 @@ struct ContractorConfig final : storage::IOConfig
         updater_config.UseDefaultOutputNames();
     }
 
+    // TODO override IsValid to also check updater_config validity
+    // TODO remove direct access to osrm_path to allow passing osrm_path to underlying configs
+
     updater::UpdaterConfig updater_config;
 
     bool use_cached_priority;

include/customizer/customizer_config.hpp

@@ -16,7 +16,10 @@ namespace customizer
 
 struct CustomizationConfig final : storage::IOConfig
 {
-    CustomizationConfig() : requested_num_threads(0) {}
+    CustomizationConfig() : requested_num_threads(0),
+      IOConfig({".osrm", }, {}, {".osrm.ebg", ".osrm.partition", ".osrm.cells", ".osrm.mldgr"})
+    {
+    }
 
     void UseDefaultOutputNames()
     {

include/extractor/extractor_config.hpp

@@ -42,12 +42,19 @@ namespace extractor
 
 struct ExtractorConfig final : storage::IOConfig
 {
-    ExtractorConfig() noexcept : requested_num_threads(0) {}
+    ExtractorConfig() noexcept : requested_num_threads(0),
+      IOConfig({"", }, {}, {".osrm", ".osrm.restrictions", ".osrm.names", ".osrm.tls", ".osrm.tld", ".osrm.timestamp", ".osrm.geometry"
+        , ".osrm.nbg_nodes", ".osrm.ebg_nodes", ".osrm.edges", ".osrm.ebg", ".osrm.ramIndex", ".osrm.fileIndex", ".osrm.turn_duration_penalties"
+        , ".osrm.turn_weight_penalties", ".osrm.turn_penalties_index", ".osrm.enw", ".osrm.properties", ".osrm.icd", ".osrm.cnbg"
+        , ".osrm.cnbg_to_ebg"})
+    {
+    }
 
     void UseDefaultOutputNames()
     {
         std::string basepath = input_path.string();
 
+        // TODO move the known_extensions to IsValid
         auto pos = std::string::npos;
         std::array<std::string, 5> known_extensions{
             {".osm.bz2", ".osm.pbf", ".osm.xml", ".pbf", ".osm"}};

include/partition/partition_config.hpp

@@ -18,7 +18,8 @@ struct PartitionConfig final : storage::IOConfig
     PartitionConfig()
         : requested_num_threads(0), balance(1.2), boundary_factor(0.25), num_optimizing_cuts(10),
           small_component_size(1000),
-          max_cell_sizes{128, 128 * 32, 128 * 32 * 16, 128 * 32 * 16 * 32}
+          max_cell_sizes{128, 128 * 32, 128 * 32 * 16, 128 * 32 * 16 * 32},
+          IOConfig({".osrm", }, {}, {".osrm.ebg", ".osrm.cnbg", ".osrm.cnbg_to_ebg", ".osrm.partition", ".osrm.cells"})
       {
       }
 

include/storage/io_config.hpp

@@ -1,6 +1,7 @@
 #ifndef OSRM_IO_CONFIG_HPP
 #define OSRM_IO_CONFIG_HPP
 
+#include <boost/algorithm/string/predicate.hpp>
 #include <boost/filesystem/path.hpp>
 
 namespace osrm
@@ -9,13 +10,75 @@ namespace storage
 {
 struct IOConfig
 {
-    IOConfig() = default;
+    IOConfig(std::vector<boost::filesystem::path> _required_input_files,
+      std::vector<boost::filesystem::path> _optional_input_files,
+      std::vector<boost::filesystem::path> _output_files)
+      : required_input_files(_required_input_files),
+      optional_input_files(_optional_input_files),
+      output_files(_output_files)
+    {}
 
-    IOConfig(const boost::filesystem::path &base) : osrm_path(base) { UseDefaultOutputNames(); }
+    // Infer the base path from the path of the .osrm file
-
+    void UseDefaultOutputNames(const boost::filesystem::path &base)
-    // Infer the output names from the path of the .osrm file
-    void UseDefaultOutputNames()
     {
+        // potentially strip off the .osrm extension for
+        // determining the base path
+        const std::string ext = ".osrm";
+        const auto pos = base.find(ext);
+        if (pos != std::string::npos)
+        {
+            base.replace(pos, ext.size(), "");
+        }
+        else
+        {
+            // unknown extension or no extension
+        }
+
+        base_path = base;
+    }
+
+    bool IsValid()
+    {
+        bool success = true;
+        for (auto &path : required_input_files)
+        {
+            if (!boost::filesystem::is_regular_file(path))
+            {
+                util::Log(logWARNING) << "Missing/Broken File: " << path.string();
+                success = false;
+            }
+        }
+        return success;
+    }
+
+    boost::filesystem::path GetPath(const std::string &fileName)
+    {
+        if(!IsConfigured(fileName, required_input_files)
+          && !IsConfigured(fileName, optional_input_files)
+          && !IsConfigured(fileName, output_files)) {
+            util::Log(logERROR) << "Missing File: " << fileName;
+            return 0; // TODO throw
+        }
+
+        return { base_path.string() + fileName };
+    }
+
+    private:
+      bool IsConfigured(const std::string &fileName, const std::vector<boost::filesystem::path> &paths) {
+        for (auto &path : paths) {
+            if(boost::algorithm::ends_with(path.string(), fileName)) {
+                return true;
+            }
+        }
+      }
+
+      std::vector<boost::filesystem::path> required_input_files;
+      std::vector<boost::filesystem::path> optional_input_files;
+      std::vector<boost::filesystem::path> output_files;
+
+      /*
+      boost::filesystem::path base_path;
+
       ram_index_path = {osrm_path.string() + ".ramIndex"};
       file_index_path = {osrm_path.string() + ".fileIndex"};
       hsgr_data_path = {osrm_path.string() + ".hsgr"};
@@ -45,10 +108,9 @@ struct IOConfig
       cnbg_ebg_mapping_path = {osrm_path.string() + ".cnbg_to_ebg"};
       turn_restrictions_path = {osrm_path.string() + ".restrictions"};
       intersection_class_data_path = {osrm_path.string() + ".icd"};
-    }
+      */
 
       boost::filesystem::path osrm_path;
-
       boost::filesystem::path ram_index_path;
       boost::filesystem::path file_index_path;
       boost::filesystem::path hsgr_data_path;

include/storage/storage_config.hpp

@@ -44,7 +44,12 @@ namespace storage
  */
 struct StorageConfig final : IOConfig
 {
-    StorageConfig() = default;
+    StorageConfig() :
+      IOConfig({".osrm.ramIndex", ".osrm.fileIndex", ".osrm.hsgr", ".osrm.nodes", ".osrm.edges", ".osrm.core",
+        ".osrm.geometry", ".osrm.timestamp", ".osrm.turn_weight_penalties", ".osrm.turn_duration_penalties", ".osrm.turn_penalties_index", ".osrm.datasource_names",
+        ".osrm.names", ".osrm.properties", ".osrm.icd", ".osrm.tld", ".osrm.tls", ".osrm.partition", ".osrm.cells", ".osrm.mldgr"}, {}, {})
+    {
+    }
 
     /**
      * Constructs a storage configuration setting paths based on a base path.

include/updater/updater_config.hpp

@@ -44,7 +44,10 @@ namespace updater
 struct UpdaterConfig final : storage::IOConfig
 {
     UpdaterConfig() {}
-    UpdaterConfig(const boost::filesystem::path &base) : IOConfig(base) {}
+    UpdaterConfig(const boost::filesystem::path &base) : IOConfig(
+      { ".osrm.ebg", ".osrm.turn_weight_penalties", ".osrm.turn_duration_penalties", ".osrm.turn_penalties_index", ".osrm.nbg_nodes", ".osrm.ebg_nodes",
+        ".osrm.edges", ".osrm.geometry", ".osrm.fileIndex", ".osrm.datasource_names", ".osrm.properties", ".osrm.restrictions", }, {}, {}
+    ) {}
 
     double log_edge_updates_factor;
     std::time_t valid_now;

src/storage/storage_config.cpp

@@ -23,28 +23,5 @@ bool CheckFileList(const std::vector<boost::filesystem::path> &files)
     return success;
 }
 }
-
-bool StorageConfig::IsValid() const
-{
-    // Common files
-    if (!CheckFileList({ram_index_path,
-                        file_index_path,
-                        node_based_nodes_data_path,
-                        edge_based_nodes_data_path,
-                        edges_data_path,
-                        geometries_path,
-                        timestamp_path,
-                        turn_weight_penalties_path,
-                        turn_duration_penalties_path,
-                        names_data_path,
-                        properties_path,
-                        intersection_class_path,
-                        datasource_names_path}))
-    {
-        return false;
-    }
-
-    return true;
-}
 }
 }