Fix platform-independent data in data files
This commit is contained in:
parent
6e4f6fec91
commit
12d4832037
@ -78,7 +78,7 @@ class Contractor
|
|||||||
private:
|
private:
|
||||||
ContractorConfig config;
|
ContractorConfig config;
|
||||||
|
|
||||||
std::size_t
|
EdgeID
|
||||||
LoadEdgeExpandedGraph(const std::string &edge_based_graph_path,
|
LoadEdgeExpandedGraph(const std::string &edge_based_graph_path,
|
||||||
util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list,
|
util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list,
|
||||||
const std::string &edge_segment_lookup_path,
|
const std::string &edge_segment_lookup_path,
|
||||||
|
@ -251,9 +251,9 @@ class InternalDataFacade final : public BaseDataFacade
|
|||||||
}
|
}
|
||||||
BOOST_ASSERT(datasources_stream);
|
BOOST_ASSERT(datasources_stream);
|
||||||
|
|
||||||
std::size_t number_of_datasources = 0;
|
std::uint64_t number_of_datasources = 0;
|
||||||
datasources_stream.read(reinterpret_cast<char *>(&number_of_datasources),
|
datasources_stream.read(reinterpret_cast<char *>(&number_of_datasources),
|
||||||
sizeof(std::size_t));
|
sizeof(number_of_datasources));
|
||||||
if (number_of_datasources > 0)
|
if (number_of_datasources > 0)
|
||||||
{
|
{
|
||||||
m_datasource_list.resize(number_of_datasources);
|
m_datasource_list.resize(number_of_datasources);
|
||||||
|
@ -104,7 +104,7 @@ class EdgeBasedGraphFactory
|
|||||||
//! list of edge based nodes (compressed segments)
|
//! list of edge based nodes (compressed segments)
|
||||||
std::vector<EdgeBasedNode> m_edge_based_node_list;
|
std::vector<EdgeBasedNode> m_edge_based_node_list;
|
||||||
util::DeallocatingVector<EdgeBasedEdge> m_edge_based_edge_list;
|
util::DeallocatingVector<EdgeBasedEdge> m_edge_based_edge_list;
|
||||||
unsigned m_max_edge_id;
|
EdgeID m_max_edge_id;
|
||||||
|
|
||||||
const std::vector<QueryNode> &m_node_info_list;
|
const std::vector<QueryNode> &m_node_info_list;
|
||||||
std::shared_ptr<util::NodeBasedDynamicGraph> m_node_based_graph;
|
std::shared_ptr<util::NodeBasedDynamicGraph> m_node_based_graph;
|
||||||
|
@ -54,7 +54,7 @@ class Extractor
|
|||||||
private:
|
private:
|
||||||
ExtractorConfig config;
|
ExtractorConfig config;
|
||||||
|
|
||||||
std::pair<std::size_t, std::size_t>
|
std::pair<std::size_t, EdgeID>
|
||||||
BuildEdgeExpandedGraph(lua_State *lua_state,
|
BuildEdgeExpandedGraph(lua_State *lua_state,
|
||||||
const ProfileProperties &profile_properties,
|
const ProfileProperties &profile_properties,
|
||||||
std::vector<QueryNode> &internal_to_external_node_map,
|
std::vector<QueryNode> &internal_to_external_node_map,
|
||||||
@ -79,7 +79,7 @@ class Extractor
|
|||||||
std::vector<QueryNode> &internal_to_external_node_map);
|
std::vector<QueryNode> &internal_to_external_node_map);
|
||||||
|
|
||||||
void WriteEdgeBasedGraph(const std::string &output_file_filename,
|
void WriteEdgeBasedGraph(const std::string &output_file_filename,
|
||||||
const size_t max_edge_id,
|
const EdgeID max_edge_id,
|
||||||
util::DeallocatingVector<EdgeBasedEdge> const &edge_based_edge_list);
|
util::DeallocatingVector<EdgeBasedEdge> const &edge_based_edge_list);
|
||||||
|
|
||||||
void WriteIntersectionClassificationData(
|
void WriteIntersectionClassificationData(
|
||||||
|
@ -24,8 +24,8 @@ template <typename EdgeDataT> class DynamicGraph
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
using EdgeData = EdgeDataT;
|
using EdgeData = EdgeDataT;
|
||||||
using NodeIterator = unsigned;
|
using NodeIterator = std::uint32_t;
|
||||||
using EdgeIterator = unsigned;
|
using EdgeIterator = std::uint32_t;
|
||||||
using EdgeRange = range<EdgeIterator>;
|
using EdgeRange = range<EdgeIterator>;
|
||||||
|
|
||||||
class InputEdge
|
class InputEdge
|
||||||
|
@ -64,7 +64,7 @@ using NodeBasedDynamicGraph = DynamicGraph<NodeBasedEdgeData>;
|
|||||||
/// Since DynamicGraph expects directed edges, we need to insert
|
/// Since DynamicGraph expects directed edges, we need to insert
|
||||||
/// two edges for undirected edges.
|
/// two edges for undirected edges.
|
||||||
inline std::shared_ptr<NodeBasedDynamicGraph>
|
inline std::shared_ptr<NodeBasedDynamicGraph>
|
||||||
NodeBasedDynamicGraphFromEdges(std::size_t number_of_nodes,
|
NodeBasedDynamicGraphFromEdges(NodeID number_of_nodes,
|
||||||
const std::vector<extractor::NodeBasedEdge> &input_edge_list)
|
const std::vector<extractor::NodeBasedEdge> &input_edge_list)
|
||||||
{
|
{
|
||||||
auto edges_list = directedEdgesFromCompressed<NodeBasedDynamicGraph::InputEdge>(
|
auto edges_list = directedEdgesFromCompressed<NodeBasedDynamicGraph::InputEdge>(
|
||||||
@ -84,8 +84,7 @@ NodeBasedDynamicGraphFromEdges(std::size_t number_of_nodes,
|
|||||||
|
|
||||||
tbb::parallel_sort(edges_list.begin(), edges_list.end());
|
tbb::parallel_sort(edges_list.begin(), edges_list.end());
|
||||||
|
|
||||||
auto graph = std::make_shared<NodeBasedDynamicGraph>(
|
auto graph = std::make_shared<NodeBasedDynamicGraph>(number_of_nodes, edges_list);
|
||||||
static_cast<NodeBasedDynamicGraph::NodeIterator>(number_of_nodes), edges_list);
|
|
||||||
|
|
||||||
return graph;
|
return graph;
|
||||||
}
|
}
|
||||||
|
@ -54,10 +54,10 @@ static const OSMWayID MIN_OSM_WAYID = OSMWayID(std::numeric_limits<std::uint32_t
|
|||||||
using OSMNodeID_weak = std::uint64_t;
|
using OSMNodeID_weak = std::uint64_t;
|
||||||
using OSMEdgeID_weak = std::uint64_t;
|
using OSMEdgeID_weak = std::uint64_t;
|
||||||
|
|
||||||
using NodeID = unsigned int;
|
using NodeID = std::uint32_t;
|
||||||
using EdgeID = unsigned int;
|
using EdgeID = std::uint32_t;
|
||||||
using NameID = std::uint32_t;
|
using NameID = std::uint32_t;
|
||||||
using EdgeWeight = int;
|
using EdgeWeight = std::int32_t;
|
||||||
|
|
||||||
using BearingClassID = std::uint32_t;
|
using BearingClassID = std::uint32_t;
|
||||||
static const BearingClassID INVALID_BEARING_CLASSID = std::numeric_limits<std::uint32_t>::max();
|
static const BearingClassID INVALID_BEARING_CLASSID = std::numeric_limits<std::uint32_t>::max();
|
||||||
@ -67,13 +67,13 @@ using DiscreteBearing = std::uint16_t;
|
|||||||
using EntryClassID = std::uint16_t;
|
using EntryClassID = std::uint16_t;
|
||||||
static const EntryClassID INVALID_ENTRY_CLASSID = std::numeric_limits<std::uint16_t>::max();
|
static const EntryClassID INVALID_ENTRY_CLASSID = std::numeric_limits<std::uint16_t>::max();
|
||||||
|
|
||||||
static const NodeID SPECIAL_NODEID = std::numeric_limits<unsigned>::max();
|
static const NodeID SPECIAL_NODEID = std::numeric_limits<NodeID>::max();
|
||||||
static const NodeID SPECIAL_SEGMENTID = std::numeric_limits<int>::max();
|
static const NodeID SPECIAL_SEGMENTID = std::numeric_limits<NodeID>::max() >> 1;
|
||||||
static const EdgeID SPECIAL_EDGEID = std::numeric_limits<unsigned>::max();
|
static const EdgeID SPECIAL_EDGEID = std::numeric_limits<EdgeID>::max();
|
||||||
static const unsigned INVALID_NAMEID = std::numeric_limits<unsigned>::max();
|
static const NameID INVALID_NAMEID = std::numeric_limits<NameID>::max();
|
||||||
static const unsigned EMPTY_NAMEID = 0;
|
static const NameID EMPTY_NAMEID = 0;
|
||||||
static const unsigned INVALID_COMPONENTID = 0;
|
static const unsigned INVALID_COMPONENTID = 0;
|
||||||
static const EdgeWeight INVALID_EDGE_WEIGHT = std::numeric_limits<int>::max();
|
static const EdgeWeight INVALID_EDGE_WEIGHT = std::numeric_limits<EdgeWeight>::max();
|
||||||
|
|
||||||
struct SegmentID
|
struct SegmentID
|
||||||
{
|
{
|
||||||
|
@ -93,7 +93,7 @@ int Contractor::Run()
|
|||||||
|
|
||||||
util::DeallocatingVector<extractor::EdgeBasedEdge> edge_based_edge_list;
|
util::DeallocatingVector<extractor::EdgeBasedEdge> edge_based_edge_list;
|
||||||
|
|
||||||
std::size_t max_edge_id = LoadEdgeExpandedGraph(config.edge_based_graph_path,
|
EdgeID max_edge_id = LoadEdgeExpandedGraph(config.edge_based_graph_path,
|
||||||
edge_based_edge_list,
|
edge_based_edge_list,
|
||||||
config.edge_segment_lookup_path,
|
config.edge_segment_lookup_path,
|
||||||
config.edge_penalty_path,
|
config.edge_penalty_path,
|
||||||
@ -349,7 +349,7 @@ parse_turn_penalty_lookup_from_csv_files(const std::vector<std::string> &turn_pe
|
|||||||
}
|
}
|
||||||
} // anon ns
|
} // anon ns
|
||||||
|
|
||||||
std::size_t Contractor::LoadEdgeExpandedGraph(
|
EdgeID Contractor::LoadEdgeExpandedGraph(
|
||||||
std::string const &edge_based_graph_filename,
|
std::string const &edge_based_graph_filename,
|
||||||
util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list,
|
util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list,
|
||||||
const std::string &edge_segment_lookup_filename,
|
const std::string &edge_segment_lookup_filename,
|
||||||
@ -392,12 +392,10 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
|
|||||||
input_stream.read((char *)&fingerprint_loaded, sizeof(util::FingerPrint));
|
input_stream.read((char *)&fingerprint_loaded, sizeof(util::FingerPrint));
|
||||||
fingerprint_loaded.TestContractor(fingerprint_valid);
|
fingerprint_loaded.TestContractor(fingerprint_valid);
|
||||||
|
|
||||||
// TODO std::size_t can vary on systems. Our files are not transferable, but we might want to
|
std::uint64_t number_of_edges = 0;
|
||||||
// consider using a fixed size type for I/O
|
EdgeID max_edge_id = SPECIAL_EDGEID;
|
||||||
std::size_t number_of_edges = 0;
|
input_stream.read((char *)&number_of_edges, sizeof(number_of_edges));
|
||||||
std::size_t max_edge_id = SPECIAL_EDGEID;
|
input_stream.read((char *)&max_edge_id, sizeof(max_edge_id));
|
||||||
input_stream.read((char *)&number_of_edges, sizeof(std::size_t));
|
|
||||||
input_stream.read((char *)&max_edge_id, sizeof(std::size_t));
|
|
||||||
|
|
||||||
edge_based_edge_list.resize(number_of_edges);
|
edge_based_edge_list.resize(number_of_edges);
|
||||||
util::SimpleLogger().Write() << "Reading " << number_of_edges
|
util::SimpleLogger().Write() << "Reading " << number_of_edges
|
||||||
@ -686,7 +684,7 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
|
|||||||
{
|
{
|
||||||
throw util::exception("Failed to open " + datasource_indexes_filename + " for writing");
|
throw util::exception("Failed to open " + datasource_indexes_filename + " for writing");
|
||||||
}
|
}
|
||||||
auto number_of_datasource_entries = m_geometry_datasource.size();
|
std::uint64_t number_of_datasource_entries = m_geometry_datasource.size();
|
||||||
datasource_stream.write(reinterpret_cast<const char *>(&number_of_datasource_entries),
|
datasource_stream.write(reinterpret_cast<const char *>(&number_of_datasource_entries),
|
||||||
sizeof(number_of_datasource_entries));
|
sizeof(number_of_datasource_entries));
|
||||||
if (number_of_datasource_entries > 0)
|
if (number_of_datasource_entries > 0)
|
||||||
@ -861,8 +859,8 @@ Contractor::WriteContractedGraph(unsigned max_node_id,
|
|||||||
const util::FingerPrint fingerprint = util::FingerPrint::GetValid();
|
const util::FingerPrint fingerprint = util::FingerPrint::GetValid();
|
||||||
boost::filesystem::ofstream hsgr_output_stream(config.graph_output_path, std::ios::binary);
|
boost::filesystem::ofstream hsgr_output_stream(config.graph_output_path, std::ios::binary);
|
||||||
hsgr_output_stream.write((char *)&fingerprint, sizeof(util::FingerPrint));
|
hsgr_output_stream.write((char *)&fingerprint, sizeof(util::FingerPrint));
|
||||||
const unsigned max_used_node_id = [&contracted_edge_list] {
|
const NodeID max_used_node_id = [&contracted_edge_list] {
|
||||||
unsigned tmp_max = 0;
|
NodeID tmp_max = 0;
|
||||||
for (const QueryEdge &edge : contracted_edge_list)
|
for (const QueryEdge &edge : contracted_edge_list)
|
||||||
{
|
{
|
||||||
BOOST_ASSERT(SPECIAL_NODEID != edge.source);
|
BOOST_ASSERT(SPECIAL_NODEID != edge.source);
|
||||||
@ -928,7 +926,7 @@ Contractor::WriteContractedGraph(unsigned max_node_id,
|
|||||||
|
|
||||||
// serialize all edges
|
// serialize all edges
|
||||||
util::SimpleLogger().Write() << "Building edge array";
|
util::SimpleLogger().Write() << "Building edge array";
|
||||||
int number_of_used_edges = 0;
|
std::size_t number_of_used_edges = 0;
|
||||||
|
|
||||||
util::StaticGraph<EdgeData>::EdgeArrayEntry current_edge;
|
util::StaticGraph<EdgeData>::EdgeArrayEntry current_edge;
|
||||||
for (const auto edge : util::irange<std::size_t>(0UL, contracted_edge_list.size()))
|
for (const auto edge : util::irange<std::size_t>(0UL, contracted_edge_list.size()))
|
||||||
@ -970,7 +968,7 @@ Contractor::WriteContractedGraph(unsigned max_node_id,
|
|||||||
\brief Build contracted graph.
|
\brief Build contracted graph.
|
||||||
*/
|
*/
|
||||||
void Contractor::ContractGraph(
|
void Contractor::ContractGraph(
|
||||||
const unsigned max_edge_id,
|
const EdgeID max_edge_id,
|
||||||
util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list,
|
util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list,
|
||||||
util::DeallocatingVector<QueryEdge> &contracted_edge_list,
|
util::DeallocatingVector<QueryEdge> &contracted_edge_list,
|
||||||
std::vector<EdgeWeight> &&node_weights,
|
std::vector<EdgeWeight> &&node_weights,
|
||||||
|
@ -83,7 +83,7 @@ void EdgeBasedGraphFactory::GetEdgeBasedNodeWeights(std::vector<EdgeWeight> &out
|
|||||||
swap(m_edge_based_node_weights, output_node_weights);
|
swap(m_edge_based_node_weights, output_node_weights);
|
||||||
}
|
}
|
||||||
|
|
||||||
unsigned EdgeBasedGraphFactory::GetHighestEdgeID() { return m_max_edge_id; }
|
EdgeID EdgeBasedGraphFactory::GetHighestEdgeID() { return m_max_edge_id; }
|
||||||
|
|
||||||
void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeID node_v)
|
void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeID node_v)
|
||||||
{
|
{
|
||||||
|
@ -167,7 +167,7 @@ void ExtractionContainers::PrepareNodes()
|
|||||||
// handle > uint32_t actual usable nodes. This should be OK for a while
|
// handle > uint32_t actual usable nodes. This should be OK for a while
|
||||||
// because we usually route on a *lot* less than 2^32 of the OSM
|
// because we usually route on a *lot* less than 2^32 of the OSM
|
||||||
// graph nodes.
|
// graph nodes.
|
||||||
std::size_t internal_id = 0;
|
std::uint64_t internal_id = 0;
|
||||||
|
|
||||||
// compute the intersection of nodes that were referenced and nodes we actually have
|
// compute the intersection of nodes that were referenced and nodes we actually have
|
||||||
while (node_iter != all_nodes_list_end && ref_iter != used_node_id_list_end)
|
while (node_iter != all_nodes_list_end && ref_iter != used_node_id_list_end)
|
||||||
@ -479,11 +479,11 @@ void ExtractionContainers::WriteEdges(std::ofstream &file_out_stream) const
|
|||||||
std::cout << "[extractor] Writing used edges ... " << std::flush;
|
std::cout << "[extractor] Writing used edges ... " << std::flush;
|
||||||
TIMER_START(write_edges);
|
TIMER_START(write_edges);
|
||||||
// Traverse list of edges and nodes in parallel and set target coord
|
// Traverse list of edges and nodes in parallel and set target coord
|
||||||
std::size_t used_edges_counter = 0;
|
std::uint64_t used_edges_counter = 0;
|
||||||
unsigned used_edges_counter_buffer = 0;
|
std::uint32_t used_edges_counter_buffer = 0;
|
||||||
|
|
||||||
auto start_position = file_out_stream.tellp();
|
auto start_position = file_out_stream.tellp();
|
||||||
file_out_stream.write((char *)&used_edges_counter_buffer, sizeof(unsigned));
|
file_out_stream.write((char *)&used_edges_counter_buffer, sizeof(used_edges_counter_buffer));
|
||||||
|
|
||||||
for (const auto &edge : all_edges_list)
|
for (const auto &edge : all_edges_list)
|
||||||
{
|
{
|
||||||
@ -508,10 +508,10 @@ void ExtractionContainers::WriteEdges(std::ofstream &file_out_stream) const
|
|||||||
|
|
||||||
std::cout << "[extractor] setting number of edges ... " << std::flush;
|
std::cout << "[extractor] setting number of edges ... " << std::flush;
|
||||||
|
|
||||||
used_edges_counter_buffer = boost::numeric_cast<unsigned>(used_edges_counter);
|
used_edges_counter_buffer = boost::numeric_cast<std::uint32_t>(used_edges_counter);
|
||||||
|
|
||||||
file_out_stream.seekp(start_position);
|
file_out_stream.seekp(start_position);
|
||||||
file_out_stream.write((char *)&used_edges_counter_buffer, sizeof(unsigned));
|
file_out_stream.write((char *)&used_edges_counter_buffer, sizeof(used_edges_counter_buffer));
|
||||||
std::cout << "ok" << std::endl;
|
std::cout << "ok" << std::endl;
|
||||||
|
|
||||||
util::SimpleLogger().Write() << "Processed " << used_edges_counter << " edges";
|
util::SimpleLogger().Write() << "Processed " << used_edges_counter << " edges";
|
||||||
|
@ -475,7 +475,7 @@ Extractor::LoadNodeBasedGraph(std::unordered_set<NodeID> &barrier_nodes,
|
|||||||
/**
|
/**
|
||||||
\brief Building an edge-expanded graph from node-based input and turn restrictions
|
\brief Building an edge-expanded graph from node-based input and turn restrictions
|
||||||
*/
|
*/
|
||||||
std::pair<std::size_t, std::size_t>
|
std::pair<std::size_t, EdgeID>
|
||||||
Extractor::BuildEdgeExpandedGraph(lua_State *lua_state,
|
Extractor::BuildEdgeExpandedGraph(lua_State *lua_state,
|
||||||
const ProfileProperties &profile_properties,
|
const ProfileProperties &profile_properties,
|
||||||
std::vector<QueryNode> &internal_to_external_node_map,
|
std::vector<QueryNode> &internal_to_external_node_map,
|
||||||
@ -600,7 +600,7 @@ void Extractor::BuildRTree(std::vector<EdgeBasedNode> node_based_edge_list,
|
|||||||
|
|
||||||
void Extractor::WriteEdgeBasedGraph(
|
void Extractor::WriteEdgeBasedGraph(
|
||||||
std::string const &output_file_filename,
|
std::string const &output_file_filename,
|
||||||
size_t const max_edge_id,
|
EdgeID const max_edge_id,
|
||||||
util::DeallocatingVector<EdgeBasedEdge> const &edge_based_edge_list)
|
util::DeallocatingVector<EdgeBasedEdge> const &edge_based_edge_list)
|
||||||
{
|
{
|
||||||
|
|
||||||
@ -613,9 +613,9 @@ void Extractor::WriteEdgeBasedGraph(
|
|||||||
<< std::flush;
|
<< std::flush;
|
||||||
TIMER_START(write_edges);
|
TIMER_START(write_edges);
|
||||||
|
|
||||||
size_t number_of_used_edges = edge_based_edge_list.size();
|
std::uint64_t number_of_used_edges = edge_based_edge_list.size();
|
||||||
file_out_stream.write((char *)&number_of_used_edges, sizeof(size_t));
|
file_out_stream.write((char *)&number_of_used_edges, sizeof(number_of_used_edges));
|
||||||
file_out_stream.write((char *)&max_edge_id, sizeof(size_t));
|
file_out_stream.write((char *)&max_edge_id, sizeof(max_edge_id));
|
||||||
|
|
||||||
for (const auto &edge : edge_based_edge_list)
|
for (const auto &edge : edge_based_edge_list)
|
||||||
{
|
{
|
||||||
|
Loading…
Reference in New Issue
Block a user