Fix platform-independent data in data files

This commit is contained in:
Michael Krasnyk 2016-06-11 17:23:29 +02:00
parent 6e4f6fec91
commit 12d4832037
No known key found for this signature in database
GPG Key ID: 3854C9454FE00649
11 changed files with 52 additions and 55 deletions

View File

@ -78,7 +78,7 @@ class Contractor
private: private:
ContractorConfig config; ContractorConfig config;
std::size_t EdgeID
LoadEdgeExpandedGraph(const std::string &edge_based_graph_path, LoadEdgeExpandedGraph(const std::string &edge_based_graph_path,
util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list, util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list,
const std::string &edge_segment_lookup_path, const std::string &edge_segment_lookup_path,

View File

@ -251,9 +251,9 @@ class InternalDataFacade final : public BaseDataFacade
} }
BOOST_ASSERT(datasources_stream); BOOST_ASSERT(datasources_stream);
std::size_t number_of_datasources = 0; std::uint64_t number_of_datasources = 0;
datasources_stream.read(reinterpret_cast<char *>(&number_of_datasources), datasources_stream.read(reinterpret_cast<char *>(&number_of_datasources),
sizeof(std::size_t)); sizeof(number_of_datasources));
if (number_of_datasources > 0) if (number_of_datasources > 0)
{ {
m_datasource_list.resize(number_of_datasources); m_datasource_list.resize(number_of_datasources);

View File

@ -104,7 +104,7 @@ class EdgeBasedGraphFactory
//! list of edge based nodes (compressed segments) //! list of edge based nodes (compressed segments)
std::vector<EdgeBasedNode> m_edge_based_node_list; std::vector<EdgeBasedNode> m_edge_based_node_list;
util::DeallocatingVector<EdgeBasedEdge> m_edge_based_edge_list; util::DeallocatingVector<EdgeBasedEdge> m_edge_based_edge_list;
unsigned m_max_edge_id; EdgeID m_max_edge_id;
const std::vector<QueryNode> &m_node_info_list; const std::vector<QueryNode> &m_node_info_list;
std::shared_ptr<util::NodeBasedDynamicGraph> m_node_based_graph; std::shared_ptr<util::NodeBasedDynamicGraph> m_node_based_graph;

View File

@ -54,7 +54,7 @@ class Extractor
private: private:
ExtractorConfig config; ExtractorConfig config;
std::pair<std::size_t, std::size_t> std::pair<std::size_t, EdgeID>
BuildEdgeExpandedGraph(lua_State *lua_state, BuildEdgeExpandedGraph(lua_State *lua_state,
const ProfileProperties &profile_properties, const ProfileProperties &profile_properties,
std::vector<QueryNode> &internal_to_external_node_map, std::vector<QueryNode> &internal_to_external_node_map,
@ -79,7 +79,7 @@ class Extractor
std::vector<QueryNode> &internal_to_external_node_map); std::vector<QueryNode> &internal_to_external_node_map);
void WriteEdgeBasedGraph(const std::string &output_file_filename, void WriteEdgeBasedGraph(const std::string &output_file_filename,
const size_t max_edge_id, const EdgeID max_edge_id,
util::DeallocatingVector<EdgeBasedEdge> const &edge_based_edge_list); util::DeallocatingVector<EdgeBasedEdge> const &edge_based_edge_list);
void WriteIntersectionClassificationData( void WriteIntersectionClassificationData(

View File

@ -24,8 +24,8 @@ template <typename EdgeDataT> class DynamicGraph
{ {
public: public:
using EdgeData = EdgeDataT; using EdgeData = EdgeDataT;
using NodeIterator = unsigned; using NodeIterator = std::uint32_t;
using EdgeIterator = unsigned; using EdgeIterator = std::uint32_t;
using EdgeRange = range<EdgeIterator>; using EdgeRange = range<EdgeIterator>;
class InputEdge class InputEdge

View File

@ -64,7 +64,7 @@ using NodeBasedDynamicGraph = DynamicGraph<NodeBasedEdgeData>;
/// Since DynamicGraph expects directed edges, we need to insert /// Since DynamicGraph expects directed edges, we need to insert
/// two edges for undirected edges. /// two edges for undirected edges.
inline std::shared_ptr<NodeBasedDynamicGraph> inline std::shared_ptr<NodeBasedDynamicGraph>
NodeBasedDynamicGraphFromEdges(std::size_t number_of_nodes, NodeBasedDynamicGraphFromEdges(NodeID number_of_nodes,
const std::vector<extractor::NodeBasedEdge> &input_edge_list) const std::vector<extractor::NodeBasedEdge> &input_edge_list)
{ {
auto edges_list = directedEdgesFromCompressed<NodeBasedDynamicGraph::InputEdge>( auto edges_list = directedEdgesFromCompressed<NodeBasedDynamicGraph::InputEdge>(
@ -84,8 +84,7 @@ NodeBasedDynamicGraphFromEdges(std::size_t number_of_nodes,
tbb::parallel_sort(edges_list.begin(), edges_list.end()); tbb::parallel_sort(edges_list.begin(), edges_list.end());
auto graph = std::make_shared<NodeBasedDynamicGraph>( auto graph = std::make_shared<NodeBasedDynamicGraph>(number_of_nodes, edges_list);
static_cast<NodeBasedDynamicGraph::NodeIterator>(number_of_nodes), edges_list);
return graph; return graph;
} }

View File

@ -54,10 +54,10 @@ static const OSMWayID MIN_OSM_WAYID = OSMWayID(std::numeric_limits<std::uint32_t
using OSMNodeID_weak = std::uint64_t; using OSMNodeID_weak = std::uint64_t;
using OSMEdgeID_weak = std::uint64_t; using OSMEdgeID_weak = std::uint64_t;
using NodeID = unsigned int; using NodeID = std::uint32_t;
using EdgeID = unsigned int; using EdgeID = std::uint32_t;
using NameID = std::uint32_t; using NameID = std::uint32_t;
using EdgeWeight = int; using EdgeWeight = std::int32_t;
using BearingClassID = std::uint32_t; using BearingClassID = std::uint32_t;
static const BearingClassID INVALID_BEARING_CLASSID = std::numeric_limits<std::uint32_t>::max(); static const BearingClassID INVALID_BEARING_CLASSID = std::numeric_limits<std::uint32_t>::max();
@ -67,13 +67,13 @@ using DiscreteBearing = std::uint16_t;
using EntryClassID = std::uint16_t; using EntryClassID = std::uint16_t;
static const EntryClassID INVALID_ENTRY_CLASSID = std::numeric_limits<std::uint16_t>::max(); static const EntryClassID INVALID_ENTRY_CLASSID = std::numeric_limits<std::uint16_t>::max();
static const NodeID SPECIAL_NODEID = std::numeric_limits<unsigned>::max(); static const NodeID SPECIAL_NODEID = std::numeric_limits<NodeID>::max();
static const NodeID SPECIAL_SEGMENTID = std::numeric_limits<int>::max(); static const NodeID SPECIAL_SEGMENTID = std::numeric_limits<NodeID>::max() >> 1;
static const EdgeID SPECIAL_EDGEID = std::numeric_limits<unsigned>::max(); static const EdgeID SPECIAL_EDGEID = std::numeric_limits<EdgeID>::max();
static const unsigned INVALID_NAMEID = std::numeric_limits<unsigned>::max(); static const NameID INVALID_NAMEID = std::numeric_limits<NameID>::max();
static const unsigned EMPTY_NAMEID = 0; static const NameID EMPTY_NAMEID = 0;
static const unsigned INVALID_COMPONENTID = 0; static const unsigned INVALID_COMPONENTID = 0;
static const EdgeWeight INVALID_EDGE_WEIGHT = std::numeric_limits<int>::max(); static const EdgeWeight INVALID_EDGE_WEIGHT = std::numeric_limits<EdgeWeight>::max();
struct SegmentID struct SegmentID
{ {

View File

@ -93,7 +93,7 @@ int Contractor::Run()
util::DeallocatingVector<extractor::EdgeBasedEdge> edge_based_edge_list; util::DeallocatingVector<extractor::EdgeBasedEdge> edge_based_edge_list;
std::size_t max_edge_id = LoadEdgeExpandedGraph(config.edge_based_graph_path, EdgeID max_edge_id = LoadEdgeExpandedGraph(config.edge_based_graph_path,
edge_based_edge_list, edge_based_edge_list,
config.edge_segment_lookup_path, config.edge_segment_lookup_path,
config.edge_penalty_path, config.edge_penalty_path,
@ -349,7 +349,7 @@ parse_turn_penalty_lookup_from_csv_files(const std::vector<std::string> &turn_pe
} }
} // anon ns } // anon ns
std::size_t Contractor::LoadEdgeExpandedGraph( EdgeID Contractor::LoadEdgeExpandedGraph(
std::string const &edge_based_graph_filename, std::string const &edge_based_graph_filename,
util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list, util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list,
const std::string &edge_segment_lookup_filename, const std::string &edge_segment_lookup_filename,
@ -392,12 +392,10 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
input_stream.read((char *)&fingerprint_loaded, sizeof(util::FingerPrint)); input_stream.read((char *)&fingerprint_loaded, sizeof(util::FingerPrint));
fingerprint_loaded.TestContractor(fingerprint_valid); fingerprint_loaded.TestContractor(fingerprint_valid);
// TODO std::size_t can vary on systems. Our files are not transferable, but we might want to std::uint64_t number_of_edges = 0;
// consider using a fixed size type for I/O EdgeID max_edge_id = SPECIAL_EDGEID;
std::size_t number_of_edges = 0; input_stream.read((char *)&number_of_edges, sizeof(number_of_edges));
std::size_t max_edge_id = SPECIAL_EDGEID; input_stream.read((char *)&max_edge_id, sizeof(max_edge_id));
input_stream.read((char *)&number_of_edges, sizeof(std::size_t));
input_stream.read((char *)&max_edge_id, sizeof(std::size_t));
edge_based_edge_list.resize(number_of_edges); edge_based_edge_list.resize(number_of_edges);
util::SimpleLogger().Write() << "Reading " << number_of_edges util::SimpleLogger().Write() << "Reading " << number_of_edges
@ -686,7 +684,7 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
{ {
throw util::exception("Failed to open " + datasource_indexes_filename + " for writing"); throw util::exception("Failed to open " + datasource_indexes_filename + " for writing");
} }
auto number_of_datasource_entries = m_geometry_datasource.size(); std::uint64_t number_of_datasource_entries = m_geometry_datasource.size();
datasource_stream.write(reinterpret_cast<const char *>(&number_of_datasource_entries), datasource_stream.write(reinterpret_cast<const char *>(&number_of_datasource_entries),
sizeof(number_of_datasource_entries)); sizeof(number_of_datasource_entries));
if (number_of_datasource_entries > 0) if (number_of_datasource_entries > 0)
@ -861,8 +859,8 @@ Contractor::WriteContractedGraph(unsigned max_node_id,
const util::FingerPrint fingerprint = util::FingerPrint::GetValid(); const util::FingerPrint fingerprint = util::FingerPrint::GetValid();
boost::filesystem::ofstream hsgr_output_stream(config.graph_output_path, std::ios::binary); boost::filesystem::ofstream hsgr_output_stream(config.graph_output_path, std::ios::binary);
hsgr_output_stream.write((char *)&fingerprint, sizeof(util::FingerPrint)); hsgr_output_stream.write((char *)&fingerprint, sizeof(util::FingerPrint));
const unsigned max_used_node_id = [&contracted_edge_list] { const NodeID max_used_node_id = [&contracted_edge_list] {
unsigned tmp_max = 0; NodeID tmp_max = 0;
for (const QueryEdge &edge : contracted_edge_list) for (const QueryEdge &edge : contracted_edge_list)
{ {
BOOST_ASSERT(SPECIAL_NODEID != edge.source); BOOST_ASSERT(SPECIAL_NODEID != edge.source);
@ -928,7 +926,7 @@ Contractor::WriteContractedGraph(unsigned max_node_id,
// serialize all edges // serialize all edges
util::SimpleLogger().Write() << "Building edge array"; util::SimpleLogger().Write() << "Building edge array";
int number_of_used_edges = 0; std::size_t number_of_used_edges = 0;
util::StaticGraph<EdgeData>::EdgeArrayEntry current_edge; util::StaticGraph<EdgeData>::EdgeArrayEntry current_edge;
for (const auto edge : util::irange<std::size_t>(0UL, contracted_edge_list.size())) for (const auto edge : util::irange<std::size_t>(0UL, contracted_edge_list.size()))
@ -970,7 +968,7 @@ Contractor::WriteContractedGraph(unsigned max_node_id,
\brief Build contracted graph. \brief Build contracted graph.
*/ */
void Contractor::ContractGraph( void Contractor::ContractGraph(
const unsigned max_edge_id, const EdgeID max_edge_id,
util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list, util::DeallocatingVector<extractor::EdgeBasedEdge> &edge_based_edge_list,
util::DeallocatingVector<QueryEdge> &contracted_edge_list, util::DeallocatingVector<QueryEdge> &contracted_edge_list,
std::vector<EdgeWeight> &&node_weights, std::vector<EdgeWeight> &&node_weights,

View File

@ -83,7 +83,7 @@ void EdgeBasedGraphFactory::GetEdgeBasedNodeWeights(std::vector<EdgeWeight> &out
swap(m_edge_based_node_weights, output_node_weights); swap(m_edge_based_node_weights, output_node_weights);
} }
unsigned EdgeBasedGraphFactory::GetHighestEdgeID() { return m_max_edge_id; } EdgeID EdgeBasedGraphFactory::GetHighestEdgeID() { return m_max_edge_id; }
void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeID node_v) void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeID node_v)
{ {

View File

@ -167,7 +167,7 @@ void ExtractionContainers::PrepareNodes()
// handle > uint32_t actual usable nodes. This should be OK for a while // handle > uint32_t actual usable nodes. This should be OK for a while
// because we usually route on a *lot* less than 2^32 of the OSM // because we usually route on a *lot* less than 2^32 of the OSM
// graph nodes. // graph nodes.
std::size_t internal_id = 0; std::uint64_t internal_id = 0;
// compute the intersection of nodes that were referenced and nodes we actually have // compute the intersection of nodes that were referenced and nodes we actually have
while (node_iter != all_nodes_list_end && ref_iter != used_node_id_list_end) while (node_iter != all_nodes_list_end && ref_iter != used_node_id_list_end)
@ -479,11 +479,11 @@ void ExtractionContainers::WriteEdges(std::ofstream &file_out_stream) const
std::cout << "[extractor] Writing used edges ... " << std::flush; std::cout << "[extractor] Writing used edges ... " << std::flush;
TIMER_START(write_edges); TIMER_START(write_edges);
// Traverse list of edges and nodes in parallel and set target coord // Traverse list of edges and nodes in parallel and set target coord
std::size_t used_edges_counter = 0; std::uint64_t used_edges_counter = 0;
unsigned used_edges_counter_buffer = 0; std::uint32_t used_edges_counter_buffer = 0;
auto start_position = file_out_stream.tellp(); auto start_position = file_out_stream.tellp();
file_out_stream.write((char *)&used_edges_counter_buffer, sizeof(unsigned)); file_out_stream.write((char *)&used_edges_counter_buffer, sizeof(used_edges_counter_buffer));
for (const auto &edge : all_edges_list) for (const auto &edge : all_edges_list)
{ {
@ -508,10 +508,10 @@ void ExtractionContainers::WriteEdges(std::ofstream &file_out_stream) const
std::cout << "[extractor] setting number of edges ... " << std::flush; std::cout << "[extractor] setting number of edges ... " << std::flush;
used_edges_counter_buffer = boost::numeric_cast<unsigned>(used_edges_counter); used_edges_counter_buffer = boost::numeric_cast<std::uint32_t>(used_edges_counter);
file_out_stream.seekp(start_position); file_out_stream.seekp(start_position);
file_out_stream.write((char *)&used_edges_counter_buffer, sizeof(unsigned)); file_out_stream.write((char *)&used_edges_counter_buffer, sizeof(used_edges_counter_buffer));
std::cout << "ok" << std::endl; std::cout << "ok" << std::endl;
util::SimpleLogger().Write() << "Processed " << used_edges_counter << " edges"; util::SimpleLogger().Write() << "Processed " << used_edges_counter << " edges";

View File

@ -475,7 +475,7 @@ Extractor::LoadNodeBasedGraph(std::unordered_set<NodeID> &barrier_nodes,
/** /**
\brief Building an edge-expanded graph from node-based input and turn restrictions \brief Building an edge-expanded graph from node-based input and turn restrictions
*/ */
std::pair<std::size_t, std::size_t> std::pair<std::size_t, EdgeID>
Extractor::BuildEdgeExpandedGraph(lua_State *lua_state, Extractor::BuildEdgeExpandedGraph(lua_State *lua_state,
const ProfileProperties &profile_properties, const ProfileProperties &profile_properties,
std::vector<QueryNode> &internal_to_external_node_map, std::vector<QueryNode> &internal_to_external_node_map,
@ -600,7 +600,7 @@ void Extractor::BuildRTree(std::vector<EdgeBasedNode> node_based_edge_list,
void Extractor::WriteEdgeBasedGraph( void Extractor::WriteEdgeBasedGraph(
std::string const &output_file_filename, std::string const &output_file_filename,
size_t const max_edge_id, EdgeID const max_edge_id,
util::DeallocatingVector<EdgeBasedEdge> const &edge_based_edge_list) util::DeallocatingVector<EdgeBasedEdge> const &edge_based_edge_list)
{ {
@ -613,9 +613,9 @@ void Extractor::WriteEdgeBasedGraph(
<< std::flush; << std::flush;
TIMER_START(write_edges); TIMER_START(write_edges);
size_t number_of_used_edges = edge_based_edge_list.size(); std::uint64_t number_of_used_edges = edge_based_edge_list.size();
file_out_stream.write((char *)&number_of_used_edges, sizeof(size_t)); file_out_stream.write((char *)&number_of_used_edges, sizeof(number_of_used_edges));
file_out_stream.write((char *)&max_edge_id, sizeof(size_t)); file_out_stream.write((char *)&max_edge_id, sizeof(max_edge_id));
for (const auto &edge : edge_based_edge_list) for (const auto &edge : edge_based_edge_list)
{ {