Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Consolidate file reading through the new FileReader class/interface. (#3321)

Browse Source
This commit is contained in:
Daniel Patterson
2016-11-30 19:08:01 -08:00
committed by GitHub
parent ef087f963d
commit 5a311012af
11 changed files with 438 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/benchmarks/static_rtree.cpp
+5 -4
View File
@@ -2,6 +2,7 @@
#include "extractor/edge_based_node.hpp"
#include "extractor/query_node.hpp"
#include "mocks/mock_datafacade.hpp"
#include "storage/io.hpp"
#include "engine/geospatial_query.hpp"
#include "util/coordinate.hpp"
#include "util/timing_util.hpp"
@@ -31,15 +32,15 @@ using BenchStaticRTree =
std::vector<util::Coordinate> loadCoordinates(const boost::filesystem::path &nodes_file)
{
boost::filesystem::ifstream nodes_input_stream(nodes_file, std::ios::binary);
osrm::storage::io::FileReader nodes_path_file_reader(
nodes_file, osrm::storage::io::FileReader::HasNoFingerprint);
extractor::QueryNode current_node;
std::uint64_t coordinate_count = 0;
nodes_input_stream.read((char *)&coordinate_count, sizeof(std::uint64_t));
unsigned coordinate_count = nodes_path_file_reader.ReadElementCount32();
std::vector<util::Coordinate> coords(coordinate_count);
for (unsigned i = 0; i < coordinate_count; ++i)
{
nodes_input_stream.read((char *)&current_node, sizeof(extractor::QueryNode));
nodes_path_file_reader.ReadInto(&current_node, 1);
coords[i] = util::Coordinate(current_node.lon, current_node.lat);
}
return coords;
src/contractor/contractor.cpp
+68 -87
View File
@@ -6,6 +6,7 @@
#include "extractor/edge_based_graph_factory.hpp"
#include "extractor/node_based_edge.hpp"
#include "storage/io.hpp"
#include "storage/io.hpp"
#include "util/exception.hpp"
#include "util/graph_loader.hpp"
@@ -299,9 +300,8 @@ parse_segment_lookup_from_csv_files(const std::vector<std::string> &segment_spee
const auto file_id = idx + 1; // starts at one, zero means we assigned the weight
const auto filename = segment_speed_filenames[idx];
std::ifstream segment_speed_file{filename, std::ios::binary};
if (!segment_speed_file)
throw util::exception{"Unable to open segment speed file " + filename};
storage::io::FileReader segment_speed_file_reader(
filename, storage::io::FileReader::HasNoFingerprint);
SegmentSpeedSourceFlatMap local;
@@ -309,30 +309,33 @@ parse_segment_lookup_from_csv_files(const std::vector<std::string> &segment_spee
std::uint64_t to_node_id{};
unsigned speed{};
for (std::string line; std::getline(segment_speed_file, line);)
{
using namespace boost::spirit::qi;
std::for_each(
segment_speed_file_reader.GetLineIteratorBegin(),
segment_speed_file_reader.GetLineIteratorEnd(),
[&](const std::string &line) {
auto it = begin(line);
const auto last = end(line);
using namespace boost::spirit::qi;
// The ulong_long -> uint64_t will likely break on 32bit platforms
const auto ok =
parse(it,
last, //
(ulong_long >> ',' >> ulong_long >> ',' >> uint_ >> *(',' >> *char_)), //
from_node_id,
to_node_id,
speed); //
auto it = begin(line);
const auto last = end(line);
if (!ok || it != last)
throw util::exception{"Segment speed file " + filename + " malformed"};
// The ulong_long -> uint64_t will likely break on 32bit platforms
const auto ok =
parse(it,
last, //
(ulong_long >> ',' >> ulong_long >> ',' >> uint_ >> *(',' >> *char_)), //
from_node_id,
to_node_id,
speed); //
SegmentSpeedSource val{{OSMNodeID{from_node_id}, OSMNodeID{to_node_id}},
{speed, static_cast<std::uint8_t>(file_id)}};
if (!ok || it != last)
throw util::exception{"Segment speed file " + filename + " malformed"};
local.push_back(std::move(val));
}
SegmentSpeedSource val{{OSMNodeID{from_node_id}, OSMNodeID{to_node_id}},
{speed, static_cast<std::uint8_t>(file_id)}};
local.push_back(std::move(val));
});
util::SimpleLogger().Write() << "Loaded speed file " << filename << " with " << local.size()
<< " speeds";
@@ -387,10 +390,8 @@ parse_turn_penalty_lookup_from_csv_files(const std::vector<std::string> &turn_pe
const auto file_id = idx + 1; // starts at one, zero means we assigned the weight
const auto filename = turn_penalty_filenames[idx];
std::ifstream turn_penalty_file{filename, std::ios::binary};
if (!turn_penalty_file)
throw util::exception{"Unable to open turn penalty file " + filename};
storage::io::FileReader turn_penalty_file_reader(filename,
storage::io::FileReader::HasNoFingerprint);
TurnPenaltySourceFlatMap local;
std::uint64_t from_node_id{};
@@ -398,31 +399,34 @@ parse_turn_penalty_lookup_from_csv_files(const std::vector<std::string> &turn_pe
std::uint64_t to_node_id{};
double penalty{};
for (std::string line; std::getline(turn_penalty_file, line);)
{
using namespace boost::spirit::qi;
std::for_each(
turn_penalty_file_reader.GetLineIteratorBegin(),
turn_penalty_file_reader.GetLineIteratorEnd(),
[&](const std::string &line) {
auto it = begin(line);
const auto last = end(line);
using namespace boost::spirit::qi;
// The ulong_long -> uint64_t will likely break on 32bit platforms
const auto ok = parse(it,
last, //
(ulong_long >> ',' >> ulong_long >> ',' >> ulong_long >> ',' >>
double_ >> *(',' >> *char_)), //
from_node_id,
via_node_id,
to_node_id,
penalty); //
auto it = begin(line);
const auto last = end(line);
if (!ok || it != last)
throw util::exception{"Turn penalty file " + filename + " malformed"};
// The ulong_long -> uint64_t will likely break on 32bit platforms
const auto ok = parse(it,
last, //
(ulong_long >> ',' >> ulong_long >> ',' >> ulong_long >>
',' >> double_ >> *(',' >> *char_)), //
from_node_id,
via_node_id,
to_node_id,
penalty); //
TurnPenaltySource val{
{OSMNodeID{from_node_id}, OSMNodeID{via_node_id}, OSMNodeID{to_node_id}},
{penalty, static_cast<std::uint8_t>(file_id)}};
local.push_back(std::move(val));
}
if (!ok || it != last)
throw util::exception{"Turn penalty file " + filename + " malformed"};
TurnPenaltySource val{
{OSMNodeID{from_node_id}, OSMNodeID{via_node_id}, OSMNodeID{to_node_id}},
{penalty, static_cast<std::uint8_t>(file_id)}};
local.push_back(std::move(val));
});
util::SimpleLogger().Write() << "Loaded penalty file " << filename << " with "
<< local.size() << " turn penalties";
@@ -568,44 +572,24 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
if (!(update_edge_weights || update_turn_penalties))
return;
boost::filesystem::ifstream nodes_input_stream(nodes_filename, std::ios::binary);
storage::io::FileReader nodes_file(nodes_filename,
storage::io::FileReader::HasNoFingerprint);
if (!nodes_input_stream)
{
throw util::exception("Failed to open " + nodes_filename);
}
nodes_file.DeserializeVector(internal_to_external_node_map);
std::uint64_t number_of_nodes = 0;
nodes_input_stream.read((char *)&number_of_nodes, sizeof(std::uint64_t));
internal_to_external_node_map.resize(number_of_nodes);
// Load all the query nodes into a vector
nodes_input_stream.read(reinterpret_cast<char *>(&(internal_to_external_node_map[0])),
number_of_nodes * sizeof(extractor::QueryNode));
};
const auto maybe_load_geometries = [&] {
if (!(update_edge_weights || update_turn_penalties))
return;
std::ifstream geometry_stream(geometry_filename, std::ios::binary);
if (!geometry_stream)
{
throw util::exception("Failed to open " + geometry_filename);
}
unsigned number_of_indices = 0;
unsigned number_of_compressed_geometries = 0;
geometry_stream.read((char *)&number_of_indices, sizeof(unsigned));
storage::io::FileReader geometry_file(geometry_filename,
storage::io::FileReader::HasNoFingerprint);
const auto number_of_indices = geometry_file.ReadElementCount32();
m_geometry_indices.resize(number_of_indices);
if (number_of_indices > 0)
{
geometry_stream.read((char *)&(m_geometry_indices[0]),
number_of_indices * sizeof(unsigned));
}
geometry_file.ReadInto(m_geometry_indices.data(), number_of_indices);
geometry_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned));
const auto number_of_compressed_geometries = geometry_file.ReadElementCount32();
BOOST_ASSERT(m_geometry_indices.back() == number_of_compressed_geometries);
m_geometry_node_list.resize(number_of_compressed_geometries);
@@ -614,14 +598,11 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
if (number_of_compressed_geometries > 0)
{
geometry_stream.read((char *)&(m_geometry_node_list[0]),
number_of_compressed_geometries * sizeof(NodeID));
geometry_stream.read((char *)&(m_geometry_fwd_weight_list[0]),
number_of_compressed_geometries * sizeof(EdgeWeight));
geometry_stream.read((char *)&(m_geometry_rev_weight_list[0]),
number_of_compressed_geometries * sizeof(EdgeWeight));
geometry_file.ReadInto(m_geometry_node_list.data(), number_of_compressed_geometries);
geometry_file.ReadInto(m_geometry_fwd_weight_list.data(),
number_of_compressed_geometries);
geometry_file.ReadInto(m_geometry_rev_weight_list.data(),
number_of_compressed_geometries);
}
};
@@ -940,12 +921,12 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
void Contractor::ReadNodeLevels(std::vector<float> &node_levels) const
{
boost::filesystem::ifstream order_input_stream(config.level_output_path, std::ios::binary);
storage::io::FileReader order_file(config.level_output_path,
storage::io::FileReader::HasNoFingerprint);
unsigned level_size;
order_input_stream.read((char *)&level_size, sizeof(unsigned));
const auto level_size = order_file.ReadElementCount32();
node_levels.resize(level_size);
order_input_stream.read((char *)node_levels.data(), sizeof(float) * node_levels.size());
order_file.ReadInto(node_levels);
}
void Contractor::WriteNodeLevels(std::vector<float> &&in_node_levels) const
src/extractor/extractor.cpp
+1
View File
@@ -10,6 +10,7 @@
#include "extractor/raster_source.hpp"
#include "storage/io.hpp"
#include "storage/io.hpp"
#include "util/graph_loader.hpp"
#include "util/io.hpp"
#include "util/name_table.hpp"
src/storage/storage.cpp
+28 -26
View File
@@ -7,6 +7,7 @@
#include "extractor/query_node.hpp"
#include "extractor/travel_mode.hpp"
#include "storage/io.hpp"
#include "storage/serialization.hpp"
#include "storage/shared_barriers.hpp"
#include "storage/shared_datatype.hpp"
#include "storage/shared_memory.hpp"
@@ -308,7 +309,7 @@ void Storage::PopulateLayout(DataLayout &layout)
{
io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::HasNoFingerprint);
const auto hsgr_header = io::readHSGRHeader(hsgr_file);
const auto hsgr_header = serialization::readHSGRHeader(hsgr_file);
layout.SetBlockSize<unsigned>(DataLayout::HSGR_CHECKSUM, 1);
layout.SetBlockSize<QueryGraph::NodeArrayEntry>(DataLayout::GRAPH_NODE_LIST,
hsgr_header.number_of_nodes);
@@ -326,7 +327,7 @@ void Storage::PopulateLayout(DataLayout &layout)
{
// allocate space in shared memory for profile properties
const auto properties_size = io::readPropertiesCount();
const auto properties_size = serialization::readPropertiesCount();
layout.SetBlockSize<extractor::ProfileProperties>(DataLayout::PROPERTIES, properties_size);
}
@@ -390,8 +391,8 @@ void Storage::PopulateLayout(DataLayout &layout)
io::FileReader datasource_names_file(config.datasource_names_path,
io::FileReader::HasNoFingerprint);
const io::DatasourceNamesData datasource_names_data =
io::readDatasourceNames(datasource_names_file);
const serialization::DatasourceNamesData datasource_names_data =
serialization::readDatasourceNames(datasource_names_file);
layout.SetBlockSize<char>(DataLayout::DATASOURCE_NAME_DATA,
datasource_names_data.names.size());
@@ -454,7 +455,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Load the HSGR file
{
io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::HasNoFingerprint);
auto hsgr_header = io::readHSGRHeader(hsgr_file);
auto hsgr_header = serialization::readHSGRHeader(hsgr_file);
unsigned *checksum_ptr =
layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
*checksum_ptr = hsgr_header.checksum;
@@ -469,11 +470,11 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
layout.GetBlockPtr<QueryGraph::EdgeArrayEntry, true>(memory_ptr,
DataLayout::GRAPH_EDGE_LIST);
io::readHSGR(hsgr_file,
graph_node_list_ptr,
hsgr_header.number_of_nodes,
graph_edge_list_ptr,
hsgr_header.number_of_edges);
serialization::readHSGR(hsgr_file,
graph_node_list_ptr,
hsgr_header.number_of_nodes,
graph_edge_list_ptr,
hsgr_header.number_of_edges);
}
// store the filename of the on-disk portion of the RTree
@@ -606,16 +607,16 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
const auto entry_class_id_ptr =
layout.GetBlockPtr<EntryClassID, true>(memory_ptr, DataLayout::ENTRY_CLASSID);
io::readEdges(edges_input_file,
via_geometry_ptr,
name_id_ptr,
turn_instructions_ptr,
lane_data_id_ptr,
travel_mode_ptr,
entry_class_id_ptr,
pre_turn_bearing_ptr,
post_turn_bearing_ptr,
number_of_original_edges);
serialization::readEdges(edges_input_file,
via_geometry_ptr,
name_id_ptr,
turn_instructions_ptr,
lane_data_id_ptr,
travel_mode_ptr,
entry_class_id_ptr,
pre_turn_bearing_ptr,
post_turn_bearing_ptr,
number_of_original_edges);
}
// load compressed geometry
@@ -659,7 +660,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
layout.GetBlockPtr<uint8_t, true>(memory_ptr, DataLayout::DATASOURCES_LIST);
if (number_of_compressed_datasources > 0)
{
io::readDatasourceIndexes(
serialization::readDatasourceIndexes(
geometry_datasource_file, datasources_list_ptr, number_of_compressed_datasources);
}
}
@@ -669,7 +670,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
io::FileReader datasource_names_file(config.datasource_names_path,
io::FileReader::HasNoFingerprint);
const auto datasource_names_data = io::readDatasourceNames(datasource_names_file);
const auto datasource_names_data =
serialization::readDatasourceNames(datasource_names_file);
// load datasource name information (if it exists)
const auto datasource_name_data_ptr =
@@ -724,10 +726,10 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
osmnodeid_list.reset(osmnodeid_ptr, layout.num_entries[DataLayout::OSM_NODE_ID_LIST]);
io::readNodes(nodes_file,
coordinates_ptr,
osmnodeid_list,
layout.num_entries[DataLayout::COORDINATE_LIST]);
serialization::readNodes(nodes_file,
coordinates_ptr,
osmnodeid_list,
layout.num_entries[DataLayout::COORDINATE_LIST]);
}
// store timestamp
src/util/name_table.cpp
+6 -16
View File
@@ -15,27 +15,20 @@ namespace util
NameTable::NameTable(const std::string &filename)
{
boost::filesystem::ifstream name_stream(filename, std::ios::binary);
storage::io::FileReader name_stream_file_reader(filename,
storage::io::FileReader::HasNoFingerprint);
if (!name_stream)
throw exception("Failed to open " + filename + " for reading.");
// name_stream >> m_name_table;
name_stream >> m_name_table;
m_name_table.ReadARangeTable(name_stream_file_reader);
if (!name_stream)
throw exception("Unable to deserialize RangeTable for NameTable");
unsigned number_of_chars = 0;
name_stream.read(reinterpret_cast<char *>(&number_of_chars), sizeof(number_of_chars));
if (!name_stream)
throw exception("Encountered invalid file, failed to read number of contained chars");
unsigned number_of_chars = name_stream_file_reader.ReadElementCount32();
m_names_char_list.resize(number_of_chars + 1); //+1 gives sentinel element
m_names_char_list.back() = 0;
if (number_of_chars > 0)
{
name_stream.read(reinterpret_cast<char *>(&m_names_char_list[0]),
number_of_chars * sizeof(m_names_char_list[0]));
name_stream_file_reader.ReadInto(&m_names_char_list[0], number_of_chars);
}
else
{
@@ -43,9 +36,6 @@ NameTable::NameTable(const std::string &filename)
<< "list of street names is empty in construction of name table from: \"" << filename
<< "\"";
}
if (!name_stream)
throw exception("Failed to read " + std::to_string(number_of_chars) + " characters from " +
filename);
}
std::string NameTable::GetNameForID(const unsigned name_id) const