Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

No shared_memory_wrapper, (De)SerializeVector

Browse Source
This commit is contained in:
Patrick Niklaus
2017-04-04 22:00:17 +00:00
committed by Patrick Niklaus
parent dce0ce0e17
commit d94017dfae
24 changed files with 292 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/contractor/contractor.cpp
+9 -10
View File
@@ -1,6 +1,6 @@
#include "contractor/contractor.hpp"
#include "contractor/files.hpp"
#include "contractor/crc32_processor.hpp"
#include "contractor/files.hpp"
#include "contractor/graph_contractor.hpp"
#include "contractor/graph_contractor_adaptors.hpp"
@@ -47,9 +47,9 @@ int Contractor::Run()
util::Log() << "Reading node weights.";
std::vector<EdgeWeight> node_weights;
{
storage::io::FileReader node_file(config.node_file_path,
storage::io::FileReader reader(config.node_file_path,
storage::io::FileReader::VerifyFingerprint);
node_file.DeserializeVector(node_weights);
storage::serialization::read(reader, node_weights);
}
util::Log() << "Done reading node weights.";
@@ -115,10 +115,10 @@ void Contractor::WriteNodeLevels(std::vector<float> &&in_node_levels) const
{
std::vector<float> node_levels(std::move(in_node_levels));
storage::io::FileWriter node_level_file(config.level_output_path,
storage::io::FileWriter::HasNoFingerprint);
storage::io::FileWriter writer(config.level_output_path,
storage::io::FileWriter::HasNoFingerprint);
node_level_file.SerializeVector(node_levels);
storage::serialization::write(writer, node_levels);
}
void Contractor::WriteCoreNodeMarker(std::vector<bool> &&in_is_core_node) const
@@ -138,9 +138,8 @@ void Contractor::WriteCoreNodeMarker(std::vector<bool> &&in_is_core_node) const
core_marker_output_file.WriteFrom(unpacked_bool_flags.data(), count);
}
void
Contractor::WriteContractedGraph(unsigned max_node_id,
util::DeallocatingVector<QueryEdge> contracted_edge_list)
void Contractor::WriteContractedGraph(unsigned max_node_id,
util::DeallocatingVector<QueryEdge> contracted_edge_list)
{
// Sorting contracted edges in a way that the static query graph can read some in in-place.
tbb::parallel_sort(contracted_edge_list.begin(), contracted_edge_list.end());
@@ -150,7 +149,7 @@ Contractor::WriteContractedGraph(unsigned max_node_id,
RangebasedCRC32 crc32_calculator;
const unsigned checksum = crc32_calculator(contracted_edge_list);
QueryGraph query_graph {max_node_id+1, contracted_edge_list};
QueryGraph query_graph{max_node_id + 1, contracted_edge_list};
files::writeGraph(config.graph_output_path, checksum, query_graph);
}
src/extractor/edge_based_graph_factory.cpp
+20 -23
View File
@@ -48,8 +48,7 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory(
std::vector<std::uint32_t> &turn_lane_offsets,
std::vector<guidance::TurnLaneType::Mask> &turn_lane_masks,
guidance::LaneDescriptionMap &lane_description_map)
: m_max_edge_id(0), m_coordinates(coordinates),
m_osm_node_ids(osm_node_ids),
: m_max_edge_id(0), m_coordinates(coordinates), m_osm_node_ids(osm_node_ids),
m_node_based_graph(std::move(node_based_graph)),
m_restriction_map(std::move(restriction_map)), m_barrier_nodes(barrier_nodes),
m_traffic_lights(traffic_lights), m_compressed_edge_container(compressed_edge_container),
@@ -540,16 +539,14 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
const bool isTrivial = m_compressed_edge_container.IsTrivial(incoming_edge);
const auto &from_node =
isTrivial
? m_osm_node_ids[node_along_road_entering]
: m_osm_node_ids[m_compressed_edge_container.GetLastEdgeSourceID(
incoming_edge)];
isTrivial ? m_osm_node_ids[node_along_road_entering]
: m_osm_node_ids[m_compressed_edge_container.GetLastEdgeSourceID(
incoming_edge)];
const auto &via_node =
m_osm_node_ids[m_compressed_edge_container.GetLastEdgeTargetID(
incoming_edge)];
const auto &to_node =
m_osm_node_ids[m_compressed_edge_container.GetFirstEdgeTargetID(
turn.eid)];
m_osm_node_ids[m_compressed_edge_container.GetFirstEdgeTargetID(turn.eid)];
lookup::TurnIndexBlock turn_index_block = {from_node, via_node, to_node};
@@ -562,32 +559,32 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
// write weight penalties per turn
BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size());
{
storage::io::FileWriter turn_weight_penalties_file(
turn_weight_penalties_filename, storage::io::FileWriter::HasNoFingerprint);
turn_weight_penalties_file.SerializeVector(turn_weight_penalties);
storage::io::FileWriter writer(turn_weight_penalties_filename,
storage::io::FileWriter::HasNoFingerprint);
storage::serialization::write(writer, turn_weight_penalties);
}
{
storage::io::FileWriter turn_duration_penalties_file(
turn_duration_penalties_filename, storage::io::FileWriter::HasNoFingerprint);
turn_duration_penalties_file.SerializeVector(turn_duration_penalties);
storage::io::FileWriter writer(turn_duration_penalties_filename,
storage::io::FileWriter::HasNoFingerprint);
storage::serialization::write(writer, turn_duration_penalties);
}
util::Log() << "Created " << entry_class_hash.size() << " entry classes and "
<< bearing_class_hash.size() << " Bearing Classes";
util::Log() << "Writing Turn Lane Data to File...";
{
storage::io::FileWriter writer(turn_lane_data_filename,
storage::io::FileWriter::HasNoFingerprint);
storage::io::FileWriter turn_lane_data_file(turn_lane_data_filename,
storage::io::FileWriter::HasNoFingerprint);
std::vector<util::guidance::LaneTupleIdPair> lane_data(lane_data_map.size());
// extract lane data sorted by ID
for (auto itr : lane_data_map)
lane_data[itr.second] = itr.first;
turn_lane_data_file.SerializeVector(lane_data);
std::vector<util::guidance::LaneTupleIdPair> lane_data(lane_data_map.size());
// extract lane data sorted by ID
for (auto itr : lane_data_map)
lane_data[itr.second] = itr.first;
storage::serialization::write(writer, lane_data);
}
util::Log() << "done.";
files::writeTurnData(turn_data_filename, turn_data_container);
src/extractor/extractor.cpp
+18 -18
View File
@@ -281,11 +281,11 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
util::Log() << "Saving edge-based node weights to file.";
TIMER_START(timer_write_node_weights);
storage::io::FileWriter edge_file(config.edge_based_node_weights_output_path,
storage::io::FileWriter::GenerateFingerprint);
edge_file.SerializeVector(edge_based_node_weights);
{
storage::io::FileWriter writer(config.edge_based_node_weights_output_path,
storage::io::FileWriter::GenerateFingerprint);
storage::serialization::write(writer, edge_based_node_weights);
}
TIMER_STOP(timer_write_node_weights);
util::Log() << "Done writing. (" << TIMER_SEC(timer_write_node_weights) << ")";
@@ -299,7 +299,8 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
TIMER_STOP(rtree);
util::Log() << "Writing node map ...";
files::writeNodes<storage::Ownership::Container>(config.node_output_path, coordinates, osm_node_ids);
files::writeNodes<storage::Ownership::Container>(
config.node_output_path, coordinates, osm_node_ids);
WriteEdgeBasedGraph(config.edge_graph_output_path, max_edge_id, edge_based_edge_list);
@@ -509,9 +510,8 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment,
};
compressed_node_based_graph_writing = std::async(std::launch::async, [&] {
WriteCompressedNodeBasedGraph(config.compressed_node_based_graph_output_path,
*node_based_graph,
coordinates);
WriteCompressedNodeBasedGraph(
config.compressed_node_based_graph_output_path, *node_based_graph, coordinates);
});
WriteTurnLaneData(config.turn_lane_descriptions_file_name);
@@ -612,11 +612,11 @@ void Extractor::WriteIntersectionClassificationData(
const std::vector<util::guidance::BearingClass> &bearing_classes,
const std::vector<util::guidance::EntryClass> &entry_classes) const
{
storage::io::FileWriter file(output_file_name, storage::io::FileWriter::GenerateFingerprint);
storage::io::FileWriter writer(output_file_name, storage::io::FileWriter::GenerateFingerprint);
util::Log() << "Writing Intersection Classification Data";
TIMER_START(write_edges);
file.SerializeVector(node_based_intersection_classes);
storage::serialization::write(writer, node_based_intersection_classes);
// create range table for vectors:
std::vector<unsigned> bearing_counts;
@@ -630,17 +630,17 @@ void Extractor::WriteIntersectionClassificationData(
}
util::RangeTable<> bearing_class_range_table(bearing_counts);
bearing_class_range_table.Write(file);
bearing_class_range_table.Write(writer);
file.WriteOne(total_bearings);
writer.WriteOne(total_bearings);
for (const auto &bearing_class : bearing_classes)
{
const auto &bearings = bearing_class.getAvailableBearings();
file.WriteFrom(bearings.data(), bearings.size());
writer.WriteFrom(bearings.data(), bearings.size());
}
file.SerializeVector(entry_classes);
storage::serialization::write(writer, entry_classes);
TIMER_STOP(write_edges);
util::Log() << "ok, after " << TIMER_SEC(write_edges) << "s for "
@@ -659,9 +659,9 @@ void Extractor::WriteTurnLaneData(const std::string &turn_lane_file) const
util::Log() << "Writing turn lane masks...";
TIMER_START(turn_lane_timer);
storage::io::FileWriter file(turn_lane_file, storage::io::FileWriter::HasNoFingerprint);
file.SerializeVector(turn_lane_offsets);
file.SerializeVector(turn_lane_masks);
storage::io::FileWriter writer(turn_lane_file, storage::io::FileWriter::HasNoFingerprint);
storage::serialization::write(writer, turn_lane_offsets);
storage::serialization::write(writer, turn_lane_masks);
TIMER_STOP(turn_lane_timer);
util::Log() << "done (" << TIMER_SEC(turn_lane_timer) << ")";
src/storage/storage.cpp
+10 -8
View File
@@ -34,10 +34,10 @@
#include "util/log.hpp"
#include "util/packed_vector.hpp"
#include "util/range_table.hpp"
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/static_graph.hpp"
#include "util/static_rtree.hpp"
#include "util/typedefs.hpp"
#include "util/vector_view.hpp"
#ifdef __linux__
#include <sys/mman.h>
@@ -63,7 +63,7 @@ namespace storage
{
using RTreeLeaf = engine::datafacade::BaseDataFacade::RTreeLeaf;
using RTreeNode = util:: StaticRTree<RTreeLeaf, storage::Ownership::View>::TreeNode;
using RTreeNode = util::StaticRTree<RTreeLeaf, storage::Ownership::View>::TreeNode;
using QueryGraph = util::StaticGraph<contractor::QueryEdge::EdgeData>;
using EdgeBasedGraph = util::StaticGraph<extractor::EdgeBasedEdge::EdgeData>;
@@ -361,7 +361,7 @@ void Storage::PopulateLayout(DataLayout &layout)
io::FileReader::VerifyFingerprint);
std::vector<BearingClassID> bearing_class_id_table;
intersection_file.DeserializeVector(bearing_class_id_table);
serialization::read(intersection_file, bearing_class_id_table);
layout.SetBlockSize<BearingClassID>(DataLayout::BEARING_CLASSID,
bearing_class_id_table.size());
@@ -386,7 +386,7 @@ void Storage::PopulateLayout(DataLayout &layout)
layout.SetBlockSize<DiscreteBearing>(DataLayout::BEARING_VALUES, num_bearings);
std::vector<util::guidance::EntryClass> entry_class_table;
intersection_file.DeserializeVector(entry_class_table);
serialization::read(intersection_file, entry_class_table);
layout.SetBlockSize<util::guidance::EntryClass>(DataLayout::ENTRY_CLASS,
entry_class_table.size());
@@ -690,11 +690,13 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
layout.GetBlockPtr<util::Coordinate, true>(memory_ptr, DataLayout::COORDINATE_LIST);
const auto osmnodeid_ptr =
layout.GetBlockPtr<std::uint64_t, true>(memory_ptr, DataLayout::OSM_NODE_ID_LIST);
util::vector_view<util::Coordinate> coordinates(coordinates_ptr, layout.num_entries[DataLayout::COORDINATE_LIST]);
util::vector_view<util::Coordinate> coordinates(
coordinates_ptr, layout.num_entries[DataLayout::COORDINATE_LIST]);
util::PackedVectorView<OSMNodeID> osm_node_ids;
osm_node_ids.reset(osmnodeid_ptr, layout.num_entries[DataLayout::OSM_NODE_ID_LIST]);
extractor::files::readNodes<storage::Ownership::View>(config.nodes_data_path, coordinates, osm_node_ids);
extractor::files::readNodes<storage::Ownership::View>(
config.nodes_data_path, coordinates, osm_node_ids);
}
// load turn weight penalties
@@ -790,7 +792,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
io::FileReader::VerifyFingerprint);
std::vector<BearingClassID> bearing_class_id_table;
intersection_file.DeserializeVector(bearing_class_id_table);
serialization::read(intersection_file, bearing_class_id_table);
const auto bearing_blocks = intersection_file.ReadElementCount32();
intersection_file.Skip<std::uint32_t>(1); // sum_lengths
@@ -808,7 +810,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
intersection_file.ReadInto(bearing_class_table.data(), num_bearings);
std::vector<util::guidance::EntryClass> entry_class_table;
intersection_file.DeserializeVector(entry_class_table);
serialization::read(intersection_file, entry_class_table);
// load intersection classes
if (!bearing_class_id_table.empty())
src/updater/updater.cpp
+6 -7
View File
@@ -6,7 +6,6 @@
#include "extractor/edge_based_graph_factory.hpp"
#include "extractor/files.hpp"
#include "extractor/node_based_edge.hpp"
#include "extractor/seralization.hpp"
#include "storage/io.hpp"
@@ -460,14 +459,14 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
const auto load_turn_weight_penalties = [&] {
using storage::io::FileReader;
FileReader file(config.turn_weight_penalties_path, FileReader::HasNoFingerprint);
file.DeserializeVector(turn_weight_penalties);
FileReader reader(config.turn_weight_penalties_path, FileReader::HasNoFingerprint);
storage::serialization::read(reader, turn_weight_penalties);
};
const auto load_turn_duration_penalties = [&] {
using storage::io::FileReader;
FileReader file(config.turn_duration_penalties_path, FileReader::HasNoFingerprint);
file.DeserializeVector(turn_duration_penalties);
FileReader reader(config.turn_duration_penalties_path, FileReader::HasNoFingerprint);
storage::serialization::read(reader, turn_duration_penalties);
};
const auto load_profile_properties = [&] {
@@ -648,8 +647,8 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
if (update_turn_penalties)
{
const auto save_penalties = [](const auto &filename, const auto &data) -> void {
storage::io::FileWriter file(filename, storage::io::FileWriter::HasNoFingerprint);
file.SerializeVector(data);
storage::io::FileWriter writer(filename, storage::io::FileWriter::HasNoFingerprint);
storage::serialization::write(writer, data);
};
tbb::parallel_invoke(