Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Added osrm-customizer tool

Browse Source
This commit is contained in:
Michael Krasnyk 2017-03-06 15:50:04 +01:00 committed by Patrick Niklaus
parent bc2e06502e
commit 3f6ae245f6
11 changed files with 316 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
CMakeLists.txt
View File

@ -119,6 +119,7 @@ configure_file(
file(GLOB UtilGlob src/util/*.cpp src/util/*/*.cpp)
file(GLOB ExtractorGlob src/extractor/*.cpp src/extractor/*/*.cpp)
file(GLOB PartitionerGlob src/partition/*.cpp)
file(GLOB CustomizerGlob src/customize/*.cpp)
file(GLOB ContractorGlob src/contractor/*.cpp)
file(GLOB StorageGlob src/storage/*.cpp)
file(GLOB ServerGlob src/server/*.cpp src/server/**/*.cpp)
@ -127,6 +128,7 @@ file(GLOB EngineGlob src/engine/*.cpp src/engine/**/*.cpp)
add_library(UTIL OBJECT ${UtilGlob})
add_library(EXTRACTOR OBJECT ${ExtractorGlob})
add_library(PARTITIONER OBJECT ${PartitionerGlob})
add_library(CUSTOMIZER OBJECT ${CustomizerGlob})
add_library(CONTRACTOR OBJECT ${ContractorGlob})
add_library(STORAGE OBJECT ${StorageGlob})
add_library(ENGINE OBJECT ${EngineGlob})
@ -136,12 +138,14 @@ set_target_properties(UTIL PROPERTIES LINKER_LANGUAGE CXX)
add_executable(osrm-extract src/tools/extract.cpp)
add_executable(osrm-partition src/tools/partition.cpp)
add_executable(osrm-customize src/tools/customize.cpp)
add_executable(osrm-contract src/tools/contract.cpp)
add_executable(osrm-routed src/tools/routed.cpp $<TARGET_OBJECTS:SERVER> $<TARGET_OBJECTS:UTIL>)
add_executable(osrm-datastore src/tools/store.cpp $<TARGET_OBJECTS:UTIL>)
add_library(osrm src/osrm/osrm.cpp $<TARGET_OBJECTS:ENGINE> $<TARGET_OBJECTS:UTIL> $<TARGET_OBJECTS:STORAGE>)
add_library(osrm_extract $<TARGET_OBJECTS:EXTRACTOR> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_partition $<TARGET_OBJECTS:PARTITIONER> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_customize $<TARGET_OBJECTS:CUSTOMIZER> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_contract $<TARGET_OBJECTS:CONTRACTOR> $<TARGET_OBJECTS:UTIL>)
add_library(osrm_store $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:UTIL>)
@ -593,6 +597,7 @@ set(BOOST_ENGINE_LIBRARIES
target_link_libraries(osrm-datastore osrm_store ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries(osrm-extract osrm_extract ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries(osrm-partition osrm_partition ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries(osrm-customize osrm_customize ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries(osrm-contract osrm_contract ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries(osrm-routed osrm ${Boost_PROGRAM_OPTIONS_LIBRARY} ${OPTIONAL_SOCKET_LIBS} ${ZLIB_LIBRARY})
@ -615,6 +620,12 @@ set(PARTITIONER_LIBRARIES
${MAYBE_RT_LIBRARY}
${MAYBE_COVERAGE_LIBRARIES}
${ZLIB_LIBRARY})
set(CUSTOMIZER_LIBRARIES
${BOOST_ENGINE_LIBRARIES}
${CMAKE_THREAD_LIBS_INIT}
${TBB_LIBRARIES}
${MAYBE_RT_LIBRARY}
${MAYBE_COVERAGE_LIBRARIES})
set(CONTRACTOR_LIBRARIES
${BOOST_BASE_LIBRARIES}
${CMAKE_THREAD_LIBS_INIT}
@ -647,6 +658,7 @@ target_link_libraries(osrm ${ENGINE_LIBRARIES})
target_link_libraries(osrm_contract ${CONTRACTOR_LIBRARIES})
target_link_libraries(osrm_extract ${EXTRACTOR_LIBRARIES})
target_link_libraries(osrm_partition ${PARTITIONER_LIBRARIES})
target_link_libraries(osrm_customize ${CUSTOMIZER_LIBRARIES})
target_link_libraries(osrm_store ${STORAGE_LIBRARIES})
# BUILD_COMPONENTS

2
include/customizer/cell_customizer.hpp
View File

@ -10,7 +10,7 @@
namespace osrm
{
namespace customizer
namespace customize
{
class CellCustomizer

20
include/customizer/customizer.hpp Normal file
View File

@ -0,0 +1,20 @@
#ifndef OSRM_CUSTOMIZE_CUSTOMIZER_HPP
#define OSRM_CUSTOMIZE_CUSTOMIZER_HPP
#include "customizer/customizer_config.hpp"
namespace osrm
{
namespace customize
{
class Customizer
{
public:
int Run(const CustomizationConfig &config);
};
} // namespace customize
} // namespace osrm
#endif // OSRM_CUSTOMIZE_CUSTOMIZER_HPP

49
include/customizer/customizer_config.hpp Normal file
View File

@ -0,0 +1,49 @@
#ifndef OSRM_CUSTOMIZE_CUSTOMIZER_CONFIG_HPP
#define OSRM_CUSTOMIZE_CUSTOMIZER_CONFIG_HPP
#include <boost/filesystem/path.hpp>
#include <array>
#include <string>
namespace osrm
{
namespace customize
{
struct CustomizationConfig
{
CustomizationConfig() : requested_num_threads(0) {}
void UseDefaults()
{
std::string basepath = base_path.string();
const std::string ext = ".osrm";
const auto pos = basepath.find(ext);
if (pos != std::string::npos)
{
basepath.replace(pos, ext.size(), "");
}
else
{
// unknown extension
}
edge_based_graph_path = basepath + ".osrm.ebg";
mld_partition_path = basepath + ".osrm.partition";
mld_storage_path = basepath + ".osrm.cells";
}
// might be changed to the node based graph at some point
boost::filesystem::path base_path;
boost::filesystem::path edge_based_graph_path;
boost::filesystem::path mld_partition_path;
boost::filesystem::path mld_storage_path;
unsigned requested_num_threads;
};
}
}
#endif // OSRM_CUSTOMIZE_CUSTOMIZER_CONFIG_HPP

28
include/partition/cell_storage.hpp
View File

@ -1,5 +1,5 @@
#ifndef OSRM_UTIL_CELL_STORAGE_HPP
#define OSRM_UTIL_CELL_STORAGE_HPP
#ifndef OSRM_CUSTOMIZE_CELL_STORAGE_HPP
#define OSRM_CUSTOMIZE_CELL_STORAGE_HPP
#include "partition/multi_level_partition.hpp"
@ -177,7 +177,7 @@ template <bool UseShareMemory> class CellStorageImpl
}
};
std::size_t LevelIDToIndex(LevelID level) const { return level - 1; }
std::size_t LevelIDToIndex(partition::LevelID level) const { return level - 1; }
public:
using Cell = CellImpl<EdgeWeight>;
@ -186,11 +186,11 @@ template <bool UseShareMemory> class CellStorageImpl
CellStorageImpl() {}
template <typename GraphT, typename = std::enable_if<!UseShareMemory>>
CellStorageImpl(const MultiLevelPartition &partition, const GraphT &base_graph)
CellStorageImpl(const partition::MultiLevelPartition &partition, const GraphT &base_graph)
{
// pre-allocate storge for CellData so we can have random access to it by cell id
unsigned number_of_cells = 0;
for (LevelID level = 1u; level < partition.GetNumberOfLevels(); ++level)
for (partition::LevelID level = 1u; level < partition.GetNumberOfLevels(); ++level)
{
level_to_cell_offset.push_back(number_of_cells);
number_of_cells += partition.GetNumberOfCells(level);
@ -198,12 +198,12 @@ template <bool UseShareMemory> class CellStorageImpl
level_to_cell_offset.push_back(number_of_cells);
cells.resize(number_of_cells);
std::vector<std::pair<CellID, NodeID>> level_source_boundary;
std::vector<std::pair<CellID, NodeID>> level_destination_boundary;
std::vector<std::pair<partition::CellID, NodeID>> level_source_boundary;
std::vector<std::pair<partition::CellID, NodeID>> level_destination_boundary;
std::size_t number_of_unconneced = 0;
for (LevelID level = 1u; level < partition.GetNumberOfLevels(); ++level)
for (partition::LevelID level = 1u; level < partition.GetNumberOfLevels(); ++level)
{
auto level_offset = level_to_cell_offset[LevelIDToIndex(level)];
@ -212,7 +212,7 @@ template <bool UseShareMemory> class CellStorageImpl
for (auto node = 0u; node < base_graph.GetNumberOfNodes(); ++node)
{
const CellID cell_id = partition.GetCell(level, node);
const partition::CellID cell_id = partition.GetCell(level, node);
bool is_source_node = false;
bool is_destination_node = false;
bool is_boundary_node = false;
@ -297,7 +297,8 @@ template <bool UseShareMemory> class CellStorageImpl
// to a different cell.
if (number_of_unconneced > 0)
{
util::Log(logWARNING) << "Node needs to either have incoming or outgoing edges in cell";
util::Log(logWARNING) << "Node needs to either have incoming or outgoing edges in cell."
<< " Number of unconnected nodes is " << number_of_unconneced;
}
// Set weight offsets and calculate total storage size
@ -323,7 +324,7 @@ template <bool UseShareMemory> class CellStorageImpl
{
}
ConstCell GetCell(LevelID level, CellID id) const
ConstCell GetCell(partition::LevelID level, partition::CellID id) const
{
const auto level_index = LevelIDToIndex(level);
BOOST_ASSERT(level_index < level_to_cell_offset.size());
@ -334,7 +335,8 @@ template <bool UseShareMemory> class CellStorageImpl
cells[cell_index], weights.data(), source_boundary.data(), destination_boundary.data()};
}
template <typename = std::enable_if<!UseShareMemory>> Cell GetCell(LevelID level, CellID id)
template <typename = std::enable_if<!UseShareMemory>>
Cell GetCell(partition::LevelID level, partition::CellID id)
{
const auto level_index = LevelIDToIndex(level);
BOOST_ASSERT(level_index < level_to_cell_offset.size());
@ -359,4 +361,4 @@ template <bool UseShareMemory> class CellStorageImpl
}
}
#endif
#endif // OSRM_CUSTOMIZE_CELL_STORAGE_HPP

19
include/partition/io.hpp
View File

@ -1,8 +1,8 @@
#ifndef OSRM_PARTITION_IO_HPP
#define OSRM_PARTITION_IO_HPP
#include "partition/cell_storage.hpp"
#include "partition/multi_level_partition.hpp"
#include "partition/cell_storage.hpp"
#include "storage/io.hpp"
@ -13,8 +13,17 @@ namespace partition
namespace io
{
template <>
inline void write(const boost::filesystem::path &path, const partition::MultiLevelPartition &mlp)
template <> inline void read(const boost::filesystem::path &path, MultiLevelPartition &mlp)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.ReadInto<MultiLevelPartition::LevelData>(mlp.level_data);
reader.DeserializeVector(mlp.partition);
reader.DeserializeVector(mlp.cell_to_children);
}
template <> inline void write(const boost::filesystem::path &path, const MultiLevelPartition &mlp)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
@ -24,8 +33,7 @@ inline void write(const boost::filesystem::path &path, const partition::MultiLev
writer.SerializeVector(mlp.cell_to_children);
}
template <>
inline void write(const boost::filesystem::path &path, const partition::CellStorage &storage)
template <> inline void write(const boost::filesystem::path &path, const CellStorage &storage)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
@ -36,6 +44,7 @@ inline void write(const boost::filesystem::path &path, const partition::CellStor
writer.SerializeVector(storage.cells);
writer.SerializeVector(storage.level_to_cell_offset);
}
}
}
}

9
include/partition/multi_level_partition.hpp
View File

@ -33,14 +33,13 @@ using MultiLevelPartitionView = detail::MultiLevelPartitionImpl<true>;
namespace io
{
template <bool UseShareMemory>
void read(const boost::filesystem::path &file,
detail::MultiLevelPartitionImpl<UseShareMemory> &mlp);
template <bool UseShareMemory>
void write(const boost::filesystem::path &file,
const detail::MultiLevelPartitionImpl<UseShareMemory> &mlp);
}
namespace detail
{
}
using LevelID = std::uint8_t;
using CellID = std::uint32_t;
using PartitionID = std::uint64_t;
@ -141,6 +140,8 @@ template <bool UseShareMemory> class MultiLevelPartitionImpl final
return cell_to_children[offset + cell + 1];
}
friend void io::read<UseShareMemory>(const boost::filesystem::path &file,
MultiLevelPartitionImpl &mlp);
friend void io::write<UseShareMemory>(const boost::filesystem::path &file,
const MultiLevelPartitionImpl &mlp);

47
src/customize/customizer.cpp Normal file
View File

@ -0,0 +1,47 @@
#include "customizer/customizer.hpp"
#include "customizer/cell_customizer.hpp"
#include "partition/edge_based_graph_reader.hpp"
#include "partition/io.hpp"
#include "partition/cell_storage.hpp"
#include "partition/io.hpp"
#include "partition/multi_level_partition.hpp"
#include "util/log.hpp"
#include "util/timing_util.hpp"
namespace osrm
{
namespace customize
{
int Customizer::Run(const CustomizationConfig &config)
{
TIMER_START(loading_data);
auto edge_based_graph = partition::LoadEdgeBasedGraph(config.edge_based_graph_path.string());
util::Log() << "Loaded edge based graph for mapping partition ids: "
<< edge_based_graph->GetNumberOfEdges() << " edges, "
<< edge_based_graph->GetNumberOfNodes() << " nodes";
osrm::partition::MultiLevelPartition mlp;
partition::io::read(config.mld_partition_path, mlp);
partition::CellStorage storage(mlp, *edge_based_graph);
TIMER_STOP(loading_data);
util::Log() << "Loading partition data took " << TIMER_SEC(loading_data) << " seconds";
TIMER_START(cell_customize);
CellCustomizer customizer(mlp);
customizer.Customize(*edge_based_graph, storage);
TIMER_STOP(cell_customize);
util::Log() << "Cells customization took " << TIMER_SEC(cell_customize) << " seconds";
TIMER_START(writing_mld_data);
partition::io::write(config.mld_storage_path, storage);
TIMER_STOP(writing_mld_data);
util::Log() << "MLD customization writing took " << TIMER_SEC(writing_mld_data) << " seconds";
return 0;
}
} // namespace customize
} // namespace osrm

6
src/partition/partitioner.cpp
View File

@ -181,14 +181,8 @@ int Partitioner::Run(const PartitionConfig &config)
TIMER_STOP(packed_mlp);
util::Log() << "MultiLevelPartition constructed in " << TIMER_SEC(packed_mlp) << " seconds";
TIMER_START(cell_storage);
CellStorage storage(mlp, *edge_based_graph);
TIMER_STOP(cell_storage);
util::Log() << "CellStorage constructed in " << TIMER_SEC(cell_storage) << " seconds";
TIMER_START(writing_mld_data);
io::write(config.mld_partition_path, mlp);
io::write(config.mld_storage_path, storage);
TIMER_STOP(writing_mld_data);
util::Log() << "MLD data writing took " << TIMER_SEC(writing_mld_data) << " seconds";

152
src/tools/customize.cpp Normal file
View File

@ -0,0 +1,152 @@
#include "customizer/customizer.hpp"
#include "util/log.hpp"
#include "util/meminfo.hpp"
#include "util/version.hpp"
#include <tbb/task_scheduler_init.h>
#include <boost/filesystem.hpp>
#include <boost/program_options.hpp>
#include <iostream>
using namespace osrm;
enum class return_code : unsigned
{
ok,
fail,
exit
};
return_code
parseArguments(int argc, char *argv[], customize::CustomizationConfig &customization_config)
{
// declare a group of options that will be allowed only on command line
boost::program_options::options_description generic_options("Options");
generic_options.add_options()("version,v", "Show version")("help,h", "Show this help message");
// declare a group of options that will be allowed both on command line
boost::program_options::options_description config_options("Configuration");
config_options.add_options()
//
("threads,t",
boost::program_options::value<unsigned int>(&customization_config.requested_num_threads)
->default_value(tbb::task_scheduler_init::default_num_threads()),
"Number of threads to use");
// hidden options, will be allowed on command line, but will not be
// shown to the user
boost::program_options::options_description hidden_options("Hidden options");
hidden_options.add_options()(
"input,i",
boost::program_options::value<boost::filesystem::path>(&customization_config.base_path),
"Input file in .osrm format");
// positional option
boost::program_options::positional_options_description positional_options;
positional_options.add("input", 1);
// combine above options for parsing
boost::program_options::options_description cmdline_options;
cmdline_options.add(generic_options).add(config_options).add(hidden_options);
const auto *executable = argv[0];
boost::program_options::options_description visible_options(
boost::filesystem::path(executable).filename().string() + " <input.osrm> [options]");
visible_options.add(generic_options).add(config_options);
// parse command line options
boost::program_options::variables_map option_variables;
try
{
boost::program_options::store(boost::program_options::command_line_parser(argc, argv)
.options(cmdline_options)
.positional(positional_options)
.run(),
option_variables);
}
catch (const boost::program_options::error &e)
{
util::Log(logERROR) << e.what();
return return_code::fail;
}
if (option_variables.count("version"))
{
std::cout << OSRM_VERSION << std::endl;
return return_code::exit;
}
if (option_variables.count("help"))
{
std::cout << visible_options;
return return_code::exit;
}
boost::program_options::notify(option_variables);
if (!option_variables.count("input"))
{
std::cout << visible_options;
return return_code::exit;
}
return return_code::ok;
}
int main(int argc, char *argv[]) try
{
util::LogPolicy::GetInstance().Unmute();
customize::CustomizationConfig customization_config;
const auto result = parseArguments(argc, argv, customization_config);
if (return_code::fail == result)
{
return EXIT_FAILURE;
}
if (return_code::exit == result)
{
return EXIT_SUCCESS;
}
// set the default in/output names
customization_config.UseDefaults();
if (1 > customization_config.requested_num_threads)
{
util::Log(logERROR) << "Number of threads must be 1 or larger";
return EXIT_FAILURE;
}
if (!boost::filesystem::is_regular_file(customization_config.base_path))
{
util::Log(logERROR) << "Input file " << customization_config.base_path.string()
<< " not found!";
return EXIT_FAILURE;
}
tbb::task_scheduler_init init(customization_config.requested_num_threads);
auto exitcode = customize::Customizer().Run(customization_config);
util::DumpMemoryStats();
return exitcode;
}
catch (const std::bad_alloc &e)
{
util::Log(logERROR) << "[exception] " << e.what();
util::Log(logERROR) << "Please provide more memory or consider using a larger swapfile";
return EXIT_FAILURE;
}
#ifdef _WIN32
catch (const std::exception &e)
{
util::Log(logERROR) << "[exception] " << e.what() << std::endl;
return EXIT_FAILURE;
}
#endif

3
unit_tests/util/cell_customization.cpp
View File

@ -2,7 +2,6 @@
#include <boost/test/unit_test.hpp>
#include "customizer/cell_customizer.hpp"
#include "partition/cell_storage.hpp"
#include "partition/multi_level_partition.hpp"
#include "util/static_graph.hpp"
@ -21,7 +20,7 @@
} while (0)
using namespace osrm;
using namespace osrm::customizer;
using namespace osrm::customize;
using namespace osrm::partition;
using namespace osrm::util;