Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Initial interation on writing out MLD partition/cell data

Browse Source
This commit is contained in:
Michael Krasnyk
2017-02-23 16:39:29 +01:00
committed by Patrick Niklaus
parent 757e7ca936
commit ff0a98196f
15 changed files with 415 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/partition/partitioner.cpp
+86
View File
@@ -6,18 +6,22 @@
#include "partition/node_based_graph_to_edge_based_graph_mapping_reader.hpp"
#include "partition/recursive_bisection.hpp"
#include "util/cell_storage.hpp"
#include "util/coordinate.hpp"
#include "util/geojson_debug_logger.hpp"
#include "util/geojson_debug_policies.hpp"
#include "util/integer_range.hpp"
#include "util/json_container.hpp"
#include "util/log.hpp"
#include "util/multi_level_partition.hpp"
#include <algorithm>
#include <iterator>
#include <unordered_set>
#include <vector>
#include <boost/assert.hpp>
#include <boost/container/small_vector.hpp>
#include "util/geojson_debug_logger.hpp"
#include "util/geojson_debug_policies.hpp"
@@ -181,6 +185,88 @@ int Partitioner::Run(const PartitionConfig &config)
}
}
// FIXME The CellID computation code need to be replaced by a more sophisticated method
{
BOOST_ASSERT(edge_based_partition_ids.size() == edge_based_graph->GetNumberOfNodes());
using namespace osrm::partition;
// find bit size of bisection ids
int first_nonzero_position = sizeof(BisectionID) * CHAR_BIT;
for (auto id : edge_based_partition_ids)
{
first_nonzero_position = id == 0 ? first_nonzero_position
: std::min(first_nonzero_position, __builtin_ctz(id));
}
BOOST_ASSERT(first_nonzero_position != sizeof(BisectionID) * CHAR_BIT);
// split bisection id bits into groups starting from SCC and stop at level 1
BOOST_ASSERT(recursive_bisection.SCCDepth() != 0);
int mask_from = sizeof(BisectionID) * CHAR_BIT - recursive_bisection.SCCDepth();
boost::container::small_vector<BisectionID, 8> level_masks;
for (int mask_to = sizeof(BisectionID) * CHAR_BIT; mask_to > first_nonzero_position;
mask_to = mask_from, mask_from -= 3) // TODO: find better grouping
{
auto bit = std::max(first_nonzero_position, mask_from);
level_masks.push_back(((1u << (sizeof(BisectionID) * CHAR_BIT - bit)) - 1) << bit);
}
util::Log() << "Bisection IDs split for SCC depth " << recursive_bisection.SCCDepth()
<< " and first non-zero bit position " << first_nonzero_position
<< " number of levels is " << level_masks.size();
for (auto x : level_masks)
std::cout << std::setw(8) << std::hex << x << std::dec << "\n";
// collect cell ids as masked bisection ids
std::vector<std::vector<osrm::util::CellID>> partitions(
level_masks.size(), std::vector<osrm::util::CellID>(edge_based_partition_ids.size()));
std::vector<std::unordered_set<osrm::util::CellID>> partition_sets(level_masks.size());
for (std::size_t index = 0; index < edge_based_partition_ids.size(); ++index)
{
auto bisection_id = edge_based_partition_ids[index];
for (std::size_t level = 0; level < level_masks.size(); ++level)
{
osrm::util::CellID cell_id =
bisection_id & level_masks[level_masks.size() - 1 - level];
partitions[level][index] = cell_id;
partition_sets[level].insert(cell_id);
}
}
std::vector<std::uint32_t> level_to_num_cells;
std::transform(partition_sets.begin(),
partition_sets.end(),
std::back_inserter(level_to_num_cells),
[](const std::unordered_set<osrm::util::CellID> &partition_set) {
return partition_set.size();
});
std::cout << "# of cell on levels\n";
for (std::size_t level = 0; level < partition_sets.size(); ++level)
{
std::cout << level_to_num_cells[level] << ": ";
for (auto x : partition_sets[level])
std::cout << " " << x;
std::cout << "\n";
}
TIMER_START(packed_mlp);
osrm::util::PackedMultiLevelPartition<false> mlp{partitions, level_to_num_cells};
TIMER_STOP(packed_mlp);
util::Log() << "PackedMultiLevelPartition constructed in " << TIMER_SEC(packed_mlp)
<< " seconds";
TIMER_START(cell_storage);
osrm::util::CellStorage<false> storage(mlp, *edge_based_graph);
TIMER_STOP(cell_storage);
util::Log() << "CellStorage constructed in " << TIMER_SEC(cell_storage) << " seconds";
TIMER_START(writing_mld_data);
mlp.Write(config.mld_partition_path);
storage.Write(config.mld_storage_path);
TIMER_STOP(writing_mld_data);
util::Log() << "MLD data writing took " << TIMER_SEC(writing_mld_data) << " seconds";
}
return 0;
}
src/partition/recursive_bisection.cpp
+2
View File
@@ -105,5 +105,7 @@ const std::vector<BisectionID> &RecursiveBisection::BisectionIDs() const
return internal_state.BisectionIDs();
}
std::uint32_t RecursiveBisection::SCCDepth() const { return internal_state.SCCDepth(); }
} // namespace partition
} // namespace osrm
src/storage/storage.cpp
+45
View File
@@ -12,12 +12,14 @@
#include "storage/shared_memory.hpp"
#include "storage/shared_monitor.hpp"
#include "engine/datafacade/datafacade_base.hpp"
#include "util/cell_storage.hpp"
#include "util/coordinate.hpp"
#include "util/exception.hpp"
#include "util/exception_utils.hpp"
#include "util/fingerprint.hpp"
#include "util/io.hpp"
#include "util/log.hpp"
#include "util/multi_level_partition.hpp"
#include "util/packed_vector.hpp"
#include "util/range_table.hpp"
#include "util/shared_memory_vector_wrapper.hpp"
@@ -393,6 +395,27 @@ void Storage::PopulateLayout(DataLayout &layout)
layout.SetBlockSize<util::guidance::LaneTupleIdPair>(DataLayout::TURN_LANE_DATA,
lane_tuple_count);
}
{
// Loading MLD Data
if (boost::filesystem::exists(config.mld_partition_path))
{
auto mld_partition_size = util::PackedMultiLevelPartition<true>().GetRequiredMemorySize(
config.mld_partition_path);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_PARTITION, mld_partition_size);
}
else
layout.SetBlockSize<char>(DataLayout::MLD_CELL_PARTITION, 0);
if (boost::filesystem::exists(config.mld_storage_path))
{
auto mld_cell_storage_size =
util::CellStorage<true>().GetRequiredMemorySize(config.mld_storage_path);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_STORAGE, mld_cell_storage_size);
}
else
layout.SetBlockSize<char>(DataLayout::MLD_CELL_STORAGE, 0);
}
}
void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
@@ -840,6 +863,28 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
std::copy(entry_class_table.begin(), entry_class_table.end(), entry_class_ptr);
}
}
{
// Loading MLD Data
const auto mld_partition_ptr =
layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::MLD_CELL_PARTITION);
const auto mld_partition_size = layout.GetBlockSize(DataLayout::MLD_CELL_PARTITION);
if (boost::filesystem::exists(config.mld_partition_path))
{
util::PackedMultiLevelPartition<true>().Read(
config.mld_partition_path, mld_partition_ptr, mld_partition_ptr + mld_partition_size);
}
const auto mld_cell_storage_ptr =
layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::MLD_CELL_STORAGE);
const auto mld_cell_storage_size = layout.GetBlockSize(DataLayout::MLD_CELL_STORAGE);
if (boost::filesystem::exists(config.mld_storage_path))
{
util::CellStorage<true>().Read(config.mld_storage_path,
mld_cell_storage_ptr,
mld_cell_storage_ptr + mld_cell_storage_size);
}
}
}
}
}
src/storage/storage_config.cpp
+3 -1
View File
@@ -19,7 +19,9 @@ StorageConfig::StorageConfig(const boost::filesystem::path &base)
datasource_indexes_path{base.string() + ".datasource_indexes"},
names_data_path{base.string() + ".names"}, properties_path{base.string() + ".properties"},
intersection_class_path{base.string() + ".icd"}, turn_lane_data_path{base.string() + ".tld"},
turn_lane_description_path{base.string() + ".tls"}
turn_lane_description_path{base.string() + ".tls"},
mld_partition_path{base.string() + ".mld_partition"},
mld_storage_path{base.string() + ".mld_storage"}
{
}