diff --git a/CMakeLists.txt b/CMakeLists.txt index 30edd0042..7c3396fa5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -119,6 +119,7 @@ configure_file( file(GLOB UtilGlob src/util/*.cpp src/util/*/*.cpp) file(GLOB ExtractorGlob src/extractor/*.cpp src/extractor/*/*.cpp) file(GLOB PartitionerGlob src/partition/*.cpp) +file(GLOB CustomizerGlob src/customize/*.cpp) file(GLOB ContractorGlob src/contractor/*.cpp) file(GLOB StorageGlob src/storage/*.cpp) file(GLOB ServerGlob src/server/*.cpp src/server/**/*.cpp) @@ -127,6 +128,7 @@ file(GLOB EngineGlob src/engine/*.cpp src/engine/**/*.cpp) add_library(UTIL OBJECT ${UtilGlob}) add_library(EXTRACTOR OBJECT ${ExtractorGlob}) add_library(PARTITIONER OBJECT ${PartitionerGlob}) +add_library(CUSTOMIZER OBJECT ${CustomizerGlob}) add_library(CONTRACTOR OBJECT ${ContractorGlob}) add_library(STORAGE OBJECT ${StorageGlob}) add_library(ENGINE OBJECT ${EngineGlob}) @@ -136,12 +138,14 @@ set_target_properties(UTIL PROPERTIES LINKER_LANGUAGE CXX) add_executable(osrm-extract src/tools/extract.cpp) add_executable(osrm-partition src/tools/partition.cpp) +add_executable(osrm-customize src/tools/customize.cpp) add_executable(osrm-contract src/tools/contract.cpp) add_executable(osrm-routed src/tools/routed.cpp $ $) add_executable(osrm-datastore src/tools/store.cpp $) add_library(osrm src/osrm/osrm.cpp $ $ $) add_library(osrm_extract $ $) add_library(osrm_partition $ $) +add_library(osrm_customize $ $) add_library(osrm_contract $ $) add_library(osrm_store $ $) @@ -593,6 +597,7 @@ set(BOOST_ENGINE_LIBRARIES target_link_libraries(osrm-datastore osrm_store ${Boost_PROGRAM_OPTIONS_LIBRARY}) target_link_libraries(osrm-extract osrm_extract ${Boost_PROGRAM_OPTIONS_LIBRARY}) target_link_libraries(osrm-partition osrm_partition ${Boost_PROGRAM_OPTIONS_LIBRARY}) +target_link_libraries(osrm-customize osrm_customize ${Boost_PROGRAM_OPTIONS_LIBRARY}) target_link_libraries(osrm-contract osrm_contract ${Boost_PROGRAM_OPTIONS_LIBRARY}) target_link_libraries(osrm-routed osrm ${Boost_PROGRAM_OPTIONS_LIBRARY} ${OPTIONAL_SOCKET_LIBS} ${ZLIB_LIBRARY}) @@ -615,6 +620,12 @@ set(PARTITIONER_LIBRARIES ${MAYBE_RT_LIBRARY} ${MAYBE_COVERAGE_LIBRARIES} ${ZLIB_LIBRARY}) + set(CUSTOMIZER_LIBRARIES + ${BOOST_ENGINE_LIBRARIES} + ${CMAKE_THREAD_LIBS_INIT} + ${TBB_LIBRARIES} + ${MAYBE_RT_LIBRARY} + ${MAYBE_COVERAGE_LIBRARIES}) set(CONTRACTOR_LIBRARIES ${BOOST_BASE_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} @@ -647,6 +658,7 @@ target_link_libraries(osrm ${ENGINE_LIBRARIES}) target_link_libraries(osrm_contract ${CONTRACTOR_LIBRARIES}) target_link_libraries(osrm_extract ${EXTRACTOR_LIBRARIES}) target_link_libraries(osrm_partition ${PARTITIONER_LIBRARIES}) +target_link_libraries(osrm_customize ${CUSTOMIZER_LIBRARIES}) target_link_libraries(osrm_store ${STORAGE_LIBRARIES}) # BUILD_COMPONENTS diff --git a/include/customizer/cell_customizer.hpp b/include/customizer/cell_customizer.hpp index bdf7ac19c..67c69fe12 100644 --- a/include/customizer/cell_customizer.hpp +++ b/include/customizer/cell_customizer.hpp @@ -10,7 +10,7 @@ namespace osrm { -namespace customizer +namespace customize { class CellCustomizer diff --git a/include/customizer/customizer.hpp b/include/customizer/customizer.hpp new file mode 100644 index 000000000..db42d3d01 --- /dev/null +++ b/include/customizer/customizer.hpp @@ -0,0 +1,20 @@ +#ifndef OSRM_CUSTOMIZE_CUSTOMIZER_HPP +#define OSRM_CUSTOMIZE_CUSTOMIZER_HPP + +#include "customizer/customizer_config.hpp" + +namespace osrm +{ +namespace customize +{ + +class Customizer +{ + public: + int Run(const CustomizationConfig &config); +}; + +} // namespace customize +} // namespace osrm + +#endif // OSRM_CUSTOMIZE_CUSTOMIZER_HPP diff --git a/include/customizer/customizer_config.hpp b/include/customizer/customizer_config.hpp new file mode 100644 index 000000000..1d8bad984 --- /dev/null +++ b/include/customizer/customizer_config.hpp @@ -0,0 +1,49 @@ +#ifndef OSRM_CUSTOMIZE_CUSTOMIZER_CONFIG_HPP +#define OSRM_CUSTOMIZE_CUSTOMIZER_CONFIG_HPP + +#include + +#include +#include + +namespace osrm +{ +namespace customize +{ + +struct CustomizationConfig +{ + CustomizationConfig() : requested_num_threads(0) {} + + void UseDefaults() + { + std::string basepath = base_path.string(); + + const std::string ext = ".osrm"; + const auto pos = basepath.find(ext); + if (pos != std::string::npos) + { + basepath.replace(pos, ext.size(), ""); + } + else + { + // unknown extension + } + + edge_based_graph_path = basepath + ".osrm.ebg"; + mld_partition_path = basepath + ".osrm.partition"; + mld_storage_path = basepath + ".osrm.cells"; + } + + // might be changed to the node based graph at some point + boost::filesystem::path base_path; + boost::filesystem::path edge_based_graph_path; + boost::filesystem::path mld_partition_path; + boost::filesystem::path mld_storage_path; + + unsigned requested_num_threads; +}; +} +} + +#endif // OSRM_CUSTOMIZE_CUSTOMIZER_CONFIG_HPP diff --git a/include/partition/cell_storage.hpp b/include/partition/cell_storage.hpp index 875a670cc..5937a3552 100644 --- a/include/partition/cell_storage.hpp +++ b/include/partition/cell_storage.hpp @@ -1,5 +1,5 @@ -#ifndef OSRM_UTIL_CELL_STORAGE_HPP -#define OSRM_UTIL_CELL_STORAGE_HPP +#ifndef OSRM_CUSTOMIZE_CELL_STORAGE_HPP +#define OSRM_CUSTOMIZE_CELL_STORAGE_HPP #include "partition/multi_level_partition.hpp" @@ -177,7 +177,7 @@ template class CellStorageImpl } }; - std::size_t LevelIDToIndex(LevelID level) const { return level - 1; } + std::size_t LevelIDToIndex(partition::LevelID level) const { return level - 1; } public: using Cell = CellImpl; @@ -186,11 +186,11 @@ template class CellStorageImpl CellStorageImpl() {} template > - CellStorageImpl(const MultiLevelPartition &partition, const GraphT &base_graph) + CellStorageImpl(const partition::MultiLevelPartition &partition, const GraphT &base_graph) { // pre-allocate storge for CellData so we can have random access to it by cell id unsigned number_of_cells = 0; - for (LevelID level = 1u; level < partition.GetNumberOfLevels(); ++level) + for (partition::LevelID level = 1u; level < partition.GetNumberOfLevels(); ++level) { level_to_cell_offset.push_back(number_of_cells); number_of_cells += partition.GetNumberOfCells(level); @@ -198,12 +198,12 @@ template class CellStorageImpl level_to_cell_offset.push_back(number_of_cells); cells.resize(number_of_cells); - std::vector> level_source_boundary; - std::vector> level_destination_boundary; + std::vector> level_source_boundary; + std::vector> level_destination_boundary; std::size_t number_of_unconneced = 0; - for (LevelID level = 1u; level < partition.GetNumberOfLevels(); ++level) + for (partition::LevelID level = 1u; level < partition.GetNumberOfLevels(); ++level) { auto level_offset = level_to_cell_offset[LevelIDToIndex(level)]; @@ -212,7 +212,7 @@ template class CellStorageImpl for (auto node = 0u; node < base_graph.GetNumberOfNodes(); ++node) { - const CellID cell_id = partition.GetCell(level, node); + const partition::CellID cell_id = partition.GetCell(level, node); bool is_source_node = false; bool is_destination_node = false; bool is_boundary_node = false; @@ -297,7 +297,8 @@ template class CellStorageImpl // to a different cell. if (number_of_unconneced > 0) { - util::Log(logWARNING) << "Node needs to either have incoming or outgoing edges in cell"; + util::Log(logWARNING) << "Node needs to either have incoming or outgoing edges in cell." + << " Number of unconnected nodes is " << number_of_unconneced; } // Set weight offsets and calculate total storage size @@ -323,7 +324,7 @@ template class CellStorageImpl { } - ConstCell GetCell(LevelID level, CellID id) const + ConstCell GetCell(partition::LevelID level, partition::CellID id) const { const auto level_index = LevelIDToIndex(level); BOOST_ASSERT(level_index < level_to_cell_offset.size()); @@ -334,7 +335,8 @@ template class CellStorageImpl cells[cell_index], weights.data(), source_boundary.data(), destination_boundary.data()}; } - template > Cell GetCell(LevelID level, CellID id) + template > + Cell GetCell(partition::LevelID level, partition::CellID id) { const auto level_index = LevelIDToIndex(level); BOOST_ASSERT(level_index < level_to_cell_offset.size()); @@ -359,4 +361,4 @@ template class CellStorageImpl } } -#endif +#endif // OSRM_CUSTOMIZE_CELL_STORAGE_HPP diff --git a/include/partition/io.hpp b/include/partition/io.hpp index 7ba34cfbe..520371b93 100644 --- a/include/partition/io.hpp +++ b/include/partition/io.hpp @@ -1,8 +1,8 @@ #ifndef OSRM_PARTITION_IO_HPP #define OSRM_PARTITION_IO_HPP -#include "partition/cell_storage.hpp" #include "partition/multi_level_partition.hpp" +#include "partition/cell_storage.hpp" #include "storage/io.hpp" @@ -13,8 +13,17 @@ namespace partition namespace io { -template <> -inline void write(const boost::filesystem::path &path, const partition::MultiLevelPartition &mlp) +template <> inline void read(const boost::filesystem::path &path, MultiLevelPartition &mlp) +{ + const auto fingerprint = storage::io::FileReader::VerifyFingerprint; + storage::io::FileReader reader{path, fingerprint}; + + reader.ReadInto(mlp.level_data); + reader.DeserializeVector(mlp.partition); + reader.DeserializeVector(mlp.cell_to_children); +} + +template <> inline void write(const boost::filesystem::path &path, const MultiLevelPartition &mlp) { const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; storage::io::FileWriter writer{path, fingerprint}; @@ -24,8 +33,7 @@ inline void write(const boost::filesystem::path &path, const partition::MultiLev writer.SerializeVector(mlp.cell_to_children); } -template <> -inline void write(const boost::filesystem::path &path, const partition::CellStorage &storage) +template <> inline void write(const boost::filesystem::path &path, const CellStorage &storage) { const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; storage::io::FileWriter writer{path, fingerprint}; @@ -36,6 +44,7 @@ inline void write(const boost::filesystem::path &path, const partition::CellStor writer.SerializeVector(storage.cells); writer.SerializeVector(storage.level_to_cell_offset); } + } } } diff --git a/include/partition/multi_level_partition.hpp b/include/partition/multi_level_partition.hpp index 0158c214e..188fbbef9 100644 --- a/include/partition/multi_level_partition.hpp +++ b/include/partition/multi_level_partition.hpp @@ -33,14 +33,13 @@ using MultiLevelPartitionView = detail::MultiLevelPartitionImpl; namespace io { template +void read(const boost::filesystem::path &file, + detail::MultiLevelPartitionImpl &mlp); +template void write(const boost::filesystem::path &file, const detail::MultiLevelPartitionImpl &mlp); } -namespace detail -{ -} - using LevelID = std::uint8_t; using CellID = std::uint32_t; using PartitionID = std::uint64_t; @@ -141,6 +140,8 @@ template class MultiLevelPartitionImpl final return cell_to_children[offset + cell + 1]; } + friend void io::read(const boost::filesystem::path &file, + MultiLevelPartitionImpl &mlp); friend void io::write(const boost::filesystem::path &file, const MultiLevelPartitionImpl &mlp); diff --git a/src/customize/customizer.cpp b/src/customize/customizer.cpp new file mode 100644 index 000000000..44fb8a9c9 --- /dev/null +++ b/src/customize/customizer.cpp @@ -0,0 +1,47 @@ +#include "customizer/customizer.hpp" +#include "customizer/cell_customizer.hpp" +#include "partition/edge_based_graph_reader.hpp" +#include "partition/io.hpp" + +#include "partition/cell_storage.hpp" +#include "partition/io.hpp" +#include "partition/multi_level_partition.hpp" +#include "util/log.hpp" +#include "util/timing_util.hpp" + +namespace osrm +{ +namespace customize +{ + +int Customizer::Run(const CustomizationConfig &config) +{ + TIMER_START(loading_data); + auto edge_based_graph = partition::LoadEdgeBasedGraph(config.edge_based_graph_path.string()); + util::Log() << "Loaded edge based graph for mapping partition ids: " + << edge_based_graph->GetNumberOfEdges() << " edges, " + << edge_based_graph->GetNumberOfNodes() << " nodes"; + + osrm::partition::MultiLevelPartition mlp; + partition::io::read(config.mld_partition_path, mlp); + + partition::CellStorage storage(mlp, *edge_based_graph); + TIMER_STOP(loading_data); + util::Log() << "Loading partition data took " << TIMER_SEC(loading_data) << " seconds"; + + TIMER_START(cell_customize); + CellCustomizer customizer(mlp); + customizer.Customize(*edge_based_graph, storage); + TIMER_STOP(cell_customize); + util::Log() << "Cells customization took " << TIMER_SEC(cell_customize) << " seconds"; + + TIMER_START(writing_mld_data); + partition::io::write(config.mld_storage_path, storage); + TIMER_STOP(writing_mld_data); + util::Log() << "MLD customization writing took " << TIMER_SEC(writing_mld_data) << " seconds"; + + return 0; +} + +} // namespace customize +} // namespace osrm diff --git a/src/partition/partitioner.cpp b/src/partition/partitioner.cpp index b0f9d3b7a..671d3a068 100644 --- a/src/partition/partitioner.cpp +++ b/src/partition/partitioner.cpp @@ -181,14 +181,8 @@ int Partitioner::Run(const PartitionConfig &config) TIMER_STOP(packed_mlp); util::Log() << "MultiLevelPartition constructed in " << TIMER_SEC(packed_mlp) << " seconds"; - TIMER_START(cell_storage); - CellStorage storage(mlp, *edge_based_graph); - TIMER_STOP(cell_storage); - util::Log() << "CellStorage constructed in " << TIMER_SEC(cell_storage) << " seconds"; - TIMER_START(writing_mld_data); io::write(config.mld_partition_path, mlp); - io::write(config.mld_storage_path, storage); TIMER_STOP(writing_mld_data); util::Log() << "MLD data writing took " << TIMER_SEC(writing_mld_data) << " seconds"; diff --git a/src/tools/customize.cpp b/src/tools/customize.cpp new file mode 100644 index 000000000..e06e366a4 --- /dev/null +++ b/src/tools/customize.cpp @@ -0,0 +1,152 @@ +#include "customizer/customizer.hpp" + +#include "util/log.hpp" +#include "util/meminfo.hpp" +#include "util/version.hpp" + +#include + +#include +#include + +#include + +using namespace osrm; + +enum class return_code : unsigned +{ + ok, + fail, + exit +}; + +return_code +parseArguments(int argc, char *argv[], customize::CustomizationConfig &customization_config) +{ + // declare a group of options that will be allowed only on command line + boost::program_options::options_description generic_options("Options"); + generic_options.add_options()("version,v", "Show version")("help,h", "Show this help message"); + + // declare a group of options that will be allowed both on command line + boost::program_options::options_description config_options("Configuration"); + config_options.add_options() + // + ("threads,t", + boost::program_options::value(&customization_config.requested_num_threads) + ->default_value(tbb::task_scheduler_init::default_num_threads()), + "Number of threads to use"); + + // hidden options, will be allowed on command line, but will not be + // shown to the user + boost::program_options::options_description hidden_options("Hidden options"); + hidden_options.add_options()( + "input,i", + boost::program_options::value(&customization_config.base_path), + "Input file in .osrm format"); + + // positional option + boost::program_options::positional_options_description positional_options; + positional_options.add("input", 1); + + // combine above options for parsing + boost::program_options::options_description cmdline_options; + cmdline_options.add(generic_options).add(config_options).add(hidden_options); + + const auto *executable = argv[0]; + boost::program_options::options_description visible_options( + boost::filesystem::path(executable).filename().string() + " [options]"); + visible_options.add(generic_options).add(config_options); + + // parse command line options + boost::program_options::variables_map option_variables; + try + { + boost::program_options::store(boost::program_options::command_line_parser(argc, argv) + .options(cmdline_options) + .positional(positional_options) + .run(), + option_variables); + } + catch (const boost::program_options::error &e) + { + util::Log(logERROR) << e.what(); + return return_code::fail; + } + + if (option_variables.count("version")) + { + std::cout << OSRM_VERSION << std::endl; + return return_code::exit; + } + + if (option_variables.count("help")) + { + std::cout << visible_options; + return return_code::exit; + } + + boost::program_options::notify(option_variables); + + if (!option_variables.count("input")) + { + std::cout << visible_options; + return return_code::exit; + } + + return return_code::ok; +} + +int main(int argc, char *argv[]) try +{ + util::LogPolicy::GetInstance().Unmute(); + customize::CustomizationConfig customization_config; + + const auto result = parseArguments(argc, argv, customization_config); + + if (return_code::fail == result) + { + return EXIT_FAILURE; + } + + if (return_code::exit == result) + { + return EXIT_SUCCESS; + } + + // set the default in/output names + customization_config.UseDefaults(); + + if (1 > customization_config.requested_num_threads) + { + util::Log(logERROR) << "Number of threads must be 1 or larger"; + return EXIT_FAILURE; + } + + if (!boost::filesystem::is_regular_file(customization_config.base_path)) + { + util::Log(logERROR) << "Input file " << customization_config.base_path.string() + << " not found!"; + return EXIT_FAILURE; + } + + tbb::task_scheduler_init init(customization_config.requested_num_threads); + + auto exitcode = customize::Customizer().Run(customization_config); + + util::DumpMemoryStats(); + + return exitcode; +} +catch (const std::bad_alloc &e) +{ + util::Log(logERROR) << "[exception] " << e.what(); + util::Log(logERROR) << "Please provide more memory or consider using a larger swapfile"; + return EXIT_FAILURE; +} +#ifdef _WIN32 +catch (const std::exception &e) +{ + util::Log(logERROR) << "[exception] " << e.what() << std::endl; + return EXIT_FAILURE; +} +#endif diff --git a/unit_tests/util/cell_customization.cpp b/unit_tests/util/cell_customization.cpp index ededd37df..d77fddb25 100644 --- a/unit_tests/util/cell_customization.cpp +++ b/unit_tests/util/cell_customization.cpp @@ -2,7 +2,6 @@ #include #include "customizer/cell_customizer.hpp" -#include "partition/cell_storage.hpp" #include "partition/multi_level_partition.hpp" #include "util/static_graph.hpp" @@ -21,7 +20,7 @@ } while (0) using namespace osrm; -using namespace osrm::customizer; +using namespace osrm::customize; using namespace osrm::partition; using namespace osrm::util;