Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Port .cnbg file to tar format

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-21 18:17:24 +00:00
parent 8152dcfb4c
commit c410c200bd
8 changed files with 87 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
include/extractor/compressed_node_based_graph_edge.hpp Normal file
View File

@ -0,0 +1,21 @@
#ifndef OSRM_EXTRACTOR_COMPRESSED_NODE_BASED_GRAPH_EDGE_HPP
#define OSRM_EXTRACTOR_COMPRESSED_NODE_BASED_GRAPH_EDGE_HPP
#include "util/typedefs.hpp"
namespace osrm
{
namespace extractor
{
// We encode the cnbg graph only using its topology as edge list
struct CompressedNodeBasedGraphEdge
{
NodeID source;
NodeID target;
};
}
}
#endif // OSRM_EXTRACTOR_COMPRESSED_NODE_BASED_GRAPH_EDGE_HPP

5
include/extractor/extractor.hpp
View File

@ -99,11 +99,6 @@ class Extractor
const std::vector<util::Coordinate> &coordinates);
std::shared_ptr<RestrictionMap> LoadRestrictionMap();
// Writes compressed node based graph and its embedding into a file for osrm-partition to use.
static void WriteCompressedNodeBasedGraph(const std::string &path,
const util::NodeBasedDynamicGraph &graph,
const std::vector<util::Coordinate> &coordiantes);
void WriteConditionalRestrictions(
const std::string &path,
std::vector<ConditionalTurnRestriction> &conditional_turn_restrictions);

32
include/extractor/files.hpp
View File

@ -94,6 +94,7 @@ void writeEdgeBasedGraph(const boost::filesystem::path &path,
writer.WriteFrom("/common/connectivity_checksum", connectivity_checksum);
}
// reads .osrm.ebg file
template <typename EdgeBasedEdgeVector>
void readEdgeBasedGraph(const boost::filesystem::path &path,
EdgeID &number_of_edge_based_nodes,
@ -125,6 +126,19 @@ inline void readNodes(const boost::filesystem::path &path,
util::serialization::read(reader, "/common/osm_node_ids", osm_node_ids);
}
// reads only coordinates from .osrm.nbg_nodes
template <typename CoordinatesT>
inline void readNodeCoordinates(const boost::filesystem::path &path,
CoordinatesT &coordinates)
{
static_assert(std::is_same<typename CoordinatesT::value_type, util::Coordinate>::value, "");
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
storage::serialization::read(reader, "/common/coordinates", coordinates);
}
// writes .osrm.nbg_nodes
template <typename CoordinatesT, typename PackedOSMIDsT>
inline void writeNodes(const boost::filesystem::path &path,
@ -476,6 +490,24 @@ void readRamIndex(const boost::filesystem::path &path, RTreeT &rtree)
util::serialization::read(reader, "/common/rtree", rtree);
}
template <typename EdgeListT>
void writeCompressedNodeBasedGraph(const boost::filesystem::path &path, const EdgeListT &edge_list)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
storage::serialization::write(writer, "/extractor/cnbg", edge_list);
}
template <typename EdgeListT>
void readCompressedNodeBasedGraph(const boost::filesystem::path &path, EdgeListT &edge_list)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
storage::serialization::read(reader, "/extractor/cnbg", edge_list);
}
}
}
}

60
include/partitioner/compressed_node_based_graph_reader.hpp
View File

@ -1,60 +0,0 @@
#ifndef OSRM_PARTITIONER_COMPRESSED_NODE_BASED_GRAPH_READER_HPP
#define OSRM_PARTITIONER_COMPRESSED_NODE_BASED_GRAPH_READER_HPP
#include "storage/io.hpp"
#include "util/coordinate.hpp"
#include "util/typedefs.hpp"
#include <string>
#include <vector>
namespace osrm
{
namespace partitioner
{
struct CompressedNodeBasedGraphEdge
{
NodeID source;
NodeID target;
};
struct CompressedNodeBasedGraph
{
CompressedNodeBasedGraph(storage::io::FileReader &reader)
{
// Reads: | Fingerprint | #e | #n | edges | coordinates |
// - uint64: number of edges (from, to) pairs
// - uint64: number of nodes and therefore also coordinates
// - (uint32_t, uint32_t): num_edges * edges
// - (int32_t, int32_t: num_nodes * coordinates (lon, lat)
//
// Gets written in Extractor::WriteCompressedNodeBasedGraph
const auto num_edges = reader.ReadElementCount64();
const auto num_nodes = reader.ReadElementCount64();
edges.resize(num_edges);
coordinates.resize(num_nodes);
reader.ReadInto(edges);
reader.ReadInto(coordinates);
}
std::vector<CompressedNodeBasedGraphEdge> edges;
std::vector<util::Coordinate> coordinates;
};
inline CompressedNodeBasedGraph LoadCompressedNodeBasedGraph(const std::string &path)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader(path, fingerprint);
CompressedNodeBasedGraph graph{reader};
return graph;
}
} // ns partition
} // ns osrm
#endif

1
include/partitioner/partitioner_config.hpp
View File

@ -21,6 +21,7 @@ struct PartitionerConfig final : storage::IOConfig
{".osrm.ebg",
".osrm.cnbg",
".osrm.cnbg_to_ebg",
".osrm.nbg_nodes",
".osrm.partition",
".osrm.cells",
".osrm.maneuver_overrides"}),

12
src/benchmarks/static_rtree.cpp
View File

@ -33,15 +33,6 @@ constexpr int32_t WORLD_MAX_LON = 180 * COORDINATE_PRECISION;
using RTreeLeaf = extractor::EdgeBasedNodeSegment;
using BenchStaticRTree = util::StaticRTree<RTreeLeaf, storage::Ownership::Container>;
std::vector<util::Coordinate> loadCoordinates(const boost::filesystem::path &nodes_file)
{
std::vector<util::Coordinate> coords;
extractor::PackedOSMIDs nodes;
extractor::files::readNodes(nodes_file, coords, nodes);
return coords;
}
template <typename QueryT>
void benchmarkQuery(const std::vector<util::Coordinate> &queries,
const std::string &name,
@ -99,7 +90,8 @@ int main(int argc, char **argv)
const char *file_path = argv[2];
const char *nodes_path = argv[3];
auto coords = osrm::benchmarks::loadCoordinates(nodes_path);
std::vector<osrm::util::Coordinate> coords;
osrm::extractor::files::readNodeCoordinates(nodes_path, coords);
osrm::benchmarks::BenchStaticRTree rtree(file_path, coords);
osrm::extractor::files::readRamIndex(ram_path, rtree);

68
src/extractor/extractor.cpp
View File

@ -14,6 +14,7 @@
#include "extractor/restriction_parser.hpp"
#include "extractor/scripting_environment.hpp"
#include "extractor/name_table.hpp"
#include "extractor/compressed_node_based_graph_edge.hpp"
#include "guidance/files.hpp"
#include "guidance/guidance_processing.hpp"
@ -168,6 +169,26 @@ void SetExcludableClasses(const ExtractorCallbacks::ClassesMap &classes_map,
}
}
}
std::vector<CompressedNodeBasedGraphEdge> toEdgeList(const util::NodeBasedDynamicGraph& graph)
{
std::vector<CompressedNodeBasedGraphEdge> edges;
edges.reserve(graph.GetNumberOfEdges());
// For all nodes iterate over its edges and dump (from, to) pairs
for (const NodeID from_node : util::irange(0u, graph.GetNumberOfNodes()))
{
for (const EdgeID edge : graph.GetAdjacentEdgeRange(from_node))
{
const auto to_node = graph.GetTarget(edge);
edges.push_back({from_node, to_node});
}
}
return edges;
}
}
/**
@ -267,10 +288,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
compressed_node_based_graph_writing.wait();
};
compressed_node_based_graph_writing = std::async(std::launch::async, [&] {
WriteCompressedNodeBasedGraph(
config.GetPath(".osrm.cnbg").string(), node_based_graph, coordinates);
});
files::writeCompressedNodeBasedGraph(config.GetPath(".osrm.cnbg").string(), toEdgeList(node_based_graph));
node_based_graph_factory.GetCompressedEdges().PrintStatistics();
@ -814,48 +832,6 @@ void Extractor::BuildRTree(std::vector<EdgeBasedNodeSegment> edge_based_node_seg
util::Log() << "finished r-tree construction in " << TIMER_SEC(construction) << " seconds";
}
void Extractor::WriteCompressedNodeBasedGraph(const std::string &path,
const util::NodeBasedDynamicGraph &graph,
const std::vector<util::Coordinate> &coordinates)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
// Writes: | Fingerprint | #e | #n | edges | coordinates |
// - uint64: number of edges (from, to) pairs
// - uint64: number of nodes and therefore also coordinates
// - (uint32_t, uint32_t): num_edges * edges
// - (int32_t, int32_t: num_nodes * coordinates (lon, lat)
const auto num_edges = graph.GetNumberOfEdges();
const auto num_nodes = graph.GetNumberOfNodes();
BOOST_ASSERT_MSG(num_nodes == coordinates.size(), "graph and embedding out of sync");
writer.WriteElementCount64(num_edges);
writer.WriteElementCount64(num_nodes);
// For all nodes iterate over its edges and dump (from, to) pairs
for (const NodeID from_node : util::irange(0u, num_nodes))
{
for (const EdgeID edge : graph.GetAdjacentEdgeRange(from_node))
{
const auto to_node = graph.GetTarget(edge);
writer.WriteFrom(from_node);
writer.WriteFrom(to_node);
}
}
// FIXME this is unneccesary: We have this data
for (const auto &qnode : coordinates)
{
writer.WriteFrom(qnode.lon);
writer.WriteFrom(qnode.lat);
}
}
template <typename Map> auto convertIDMapToVector(const Map &map)
{
std::vector<typename Map::key_type> result(map.size());

20
src/partitioner/partitioner.cpp
View File

@ -2,7 +2,6 @@
#include "partitioner/bisection_graph.hpp"
#include "partitioner/bisection_to_partition.hpp"
#include "partitioner/cell_storage.hpp"
#include "partitioner/compressed_node_based_graph_reader.hpp"
#include "partitioner/edge_based_graph_reader.hpp"
#include "partitioner/files.hpp"
#include "partitioner/multi_level_partition.hpp"
@ -10,6 +9,7 @@
#include "partitioner/remove_unconnected.hpp"
#include "partitioner/renumber.hpp"
#include "extractor/compressed_node_based_graph_edge.hpp"
#include "extractor/files.hpp"
#include "util/coordinate.hpp"
@ -40,19 +40,17 @@ namespace partitioner
{
auto getGraphBisection(const PartitionerConfig &config)
{
auto compressed_node_based_graph =
LoadCompressedNodeBasedGraph(config.GetPath(".osrm.cnbg").string());
std::vector<extractor::CompressedNodeBasedGraphEdge> edges;
extractor::files::readCompressedNodeBasedGraph(config.GetPath(".osrm.cnbg"), edges);
groupEdgesBySource(begin(edges), end(edges));
util::Log() << "Loaded compressed node based graph: "
<< compressed_node_based_graph.edges.size() << " edges, "
<< compressed_node_based_graph.coordinates.size() << " nodes";
std::vector<util::Coordinate> coordinates;
extractor::files::readNodeCoordinates(config.GetPath(".osrm.nbg_nodes"), coordinates);
groupEdgesBySource(begin(compressed_node_based_graph.edges),
end(compressed_node_based_graph.edges));
util::Log() << "Loaded compressed node based graph: " << edges.size() << " edges, "
<< coordinates.size() << " nodes";
auto graph =
makeBisectionGraph(compressed_node_based_graph.coordinates,
adaptToBisectionEdge(std::move(compressed_node_based_graph.edges)));
auto graph = makeBisectionGraph(coordinates, adaptToBisectionEdge(std::move(edges)));
util::Log() << " running partition: " << config.max_cell_sizes.front() << " " << config.balance
<< " " << config.boundary_factor << " " << config.num_optimizing_cuts << " "