Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Include datasources in .geometries file and refactor .datasource_names

Browse Source
This commit is contained in:
Patrick Niklaus
2017-03-09 17:01:04 +00:00
committed by Patrick Niklaus
parent ffd6311e7d
commit a636e8cc13
14 changed files with 166 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
+30 -109
View File
@@ -8,6 +8,7 @@
#include "engine/algorithm.hpp"
#include "engine/geospatial_query.hpp"
#include "extractor/datasources.hpp"
#include "extractor/guidance/turn_instruction.hpp"
#include "extractor/guidance/turn_lane_types.hpp"
#include "extractor/profile_properties.hpp"
@@ -205,6 +206,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
std::string m_timestamp;
extractor::ProfileProperties *m_profile_properties;
extractor::Datasources *m_datasources;
unsigned m_check_sum;
util::ShM<util::Coordinate, true>::vector m_coordinate_list;
@@ -219,7 +221,6 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
util::NameTable m_names_table;
util::ShM<unsigned, true>::vector m_name_begin_indices;
util::ShM<bool, true>::vector m_is_core_node;
util::ShM<DatasourceID, true>::vector m_datasource_list;
util::ShM<std::uint32_t, true>::vector m_lane_description_offsets;
util::ShM<extractor::guidance::TurnLaneType::Mask, true>::vector m_lane_description_masks;
util::ShM<TurnPenalty, true>::vector m_turn_weight_penalties;
@@ -467,39 +468,21 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
geometries_rev_duration_list_ptr,
data_layout.num_entries[storage::DataLayout::GEOMETRIES_REV_DURATION_LIST]);
auto datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>(
memory_block, storage::DataLayout::DATASOURCES_LIST);
util::ShM<DatasourceID, true>::vector datasources_list(
datasources_list_ptr, data_layout.num_entries[storage::DataLayout::DATASOURCES_LIST]);
segment_data = extractor::SegmentDataView{std::move(geometry_begin_indices),
std::move(geometry_node_list),
std::move(geometry_fwd_weight_list),
std::move(geometry_rev_weight_list),
std::move(geometry_fwd_duration_list),
std::move(geometry_rev_duration_list)};
std::move(geometry_rev_duration_list),
std::move(datasources_list)};
auto datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>(
memory_block, storage::DataLayout::DATASOURCES_LIST);
util::ShM<DatasourceID, true>::vector datasources_list(
datasources_list_ptr, data_layout.num_entries[storage::DataLayout::DATASOURCES_LIST]);
m_datasource_list = std::move(datasources_list);
auto datasource_name_data_ptr =
data_layout.GetBlockPtr<char>(memory_block, storage::DataLayout::DATASOURCE_NAME_DATA);
util::ShM<char, true>::vector datasource_name_data(
datasource_name_data_ptr,
data_layout.num_entries[storage::DataLayout::DATASOURCE_NAME_DATA]);
m_datasource_name_data = std::move(datasource_name_data);
auto datasource_name_offsets_ptr = data_layout.GetBlockPtr<std::size_t>(
memory_block, storage::DataLayout::DATASOURCE_NAME_OFFSETS);
util::ShM<std::size_t, true>::vector datasource_name_offsets(
datasource_name_offsets_ptr,
data_layout.num_entries[storage::DataLayout::DATASOURCE_NAME_OFFSETS]);
m_datasource_name_offsets = std::move(datasource_name_offsets);
auto datasource_name_lengths_ptr = data_layout.GetBlockPtr<std::size_t>(
memory_block, storage::DataLayout::DATASOURCE_NAME_LENGTHS);
util::ShM<std::size_t, true>::vector datasource_name_lengths(
datasource_name_lengths_ptr,
data_layout.num_entries[storage::DataLayout::DATASOURCE_NAME_LENGTHS]);
m_datasource_name_lengths = std::move(datasource_name_lengths);
m_datasources = data_layout.GetBlockPtr<extractor::Datasources>(
memory_block, storage::DataLayout::DATASOURCES_NAMES);
}
void InitializeIntersectionClassPointers(storage::DataLayout &data_layout, char *memory_block)
@@ -611,6 +594,24 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
return std::vector<EdgeWeight>{range.begin(), range.end()};
}
// Returns the data source ids that were used to supply the edge
// weights.
virtual std::vector<DatasourceID>
GetUncompressedForwardDatasources(const EdgeID id) const override final
{
auto range = segment_data.GetForwardDatasources(id);
return std::vector<DatasourceID>{range.begin(), range.end()};
}
// Returns the data source ids that were used to supply the edge
// weights.
virtual std::vector<DatasourceID>
GetUncompressedReverseDatasources(const EdgeID id) const override final
{
auto range = segment_data.GetReverseDatasources(id);
return std::vector<DatasourceID>{range.begin(), range.end()};
}
virtual GeometryID GetGeometryIndexForEdgeID(const EdgeID id) const override final
{
return m_via_geometry_list.at(id);
@@ -781,89 +782,9 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
return m_name_table.GetDestinationsForID(id);
}
// Returns the data source ids that were used to supply the edge
// weights.
virtual std::vector<DatasourceID>
GetUncompressedForwardDatasources(const EdgeID id) const override final
{
/*
* Data sources for geometries are stored in one place for
* both forward and reverse segments along the same bi-
* directional edge. The indices stores
* refences to where to find the beginning of the bi-
* directional edge in the list vector. For
* forward datasources of bi-directional edges, edges 2 to
* n of that edge need to be read.
*/
const auto begin = segment_data.GetOffset(id, 0) + 1;
const auto end = segment_data.GetOffset(id + 1, 0);
std::vector<DatasourceID> result_datasources;
result_datasources.reserve(end - begin);
// If there was no datasource info, return an array of 0's.
if (m_datasource_list.empty())
{
result_datasources.resize(end - begin, 0);
}
else
{
std::copy(m_datasource_list.begin() + begin,
m_datasource_list.begin() + end,
std::back_inserter(result_datasources));
}
return result_datasources;
}
// Returns the data source ids that were used to supply the edge
// weights.
virtual std::vector<DatasourceID>
GetUncompressedReverseDatasources(const EdgeID id) const override final
{
/*
* Datasources for geometries are stored in one place for
* both forward and reverse segments along the same bi-
* directional edge. The indices stores
* refences to where to find the beginning of the bi-
* directional edge in the list vector. For
* reverse datasources of bi-directional edges, edges 1 to
* n-1 of that edge need to be read in reverse.
*/
const auto begin = segment_data.GetOffset(id, 0);
const auto end = segment_data.GetOffset(id + 1, 0) - 1;
std::vector<DatasourceID> result_datasources;
result_datasources.reserve(end - begin);
// If there was no datasource info, return an array of 0's.
if (m_datasource_list.empty())
{
result_datasources.resize(end - begin, 0);
}
else
{
std::reverse_copy(m_datasource_list.begin() + begin,
m_datasource_list.begin() + end,
std::back_inserter(result_datasources));
}
return result_datasources;
}
StringView GetDatasourceName(const DatasourceID id) const override final
{
BOOST_ASSERT(m_datasource_name_offsets.size() >= 1);
BOOST_ASSERT(m_datasource_name_offsets.size() > id);
auto first = m_datasource_name_data.begin() + m_datasource_name_offsets[id];
auto last = m_datasource_name_data.begin() + m_datasource_name_offsets[id] +
m_datasource_name_lengths[id];
// These iterators are useless: they're InputIterators onto a contiguous block of memory.
// Deref to get to the first element, then Addressof to get the memory address of the it.
const std::size_t len = &*last - &*first;
return StringView{&*first, len};
return m_datasources->GetSourceName(id);
}
std::string GetTimestamp() const override final { return m_timestamp; }
include/extractor/datasources.hpp
+48
View File
@@ -0,0 +1,48 @@
#ifndef OSRM_EXTRACT_DATASOURCES_HPP
#define OSRM_EXTRACT_DATASOURCES_HPP
#include <cstdint>
#include <util/string_view.hpp>
#include <util/typedefs.hpp>
namespace osrm
{
namespace extractor
{
class Datasources
{
static constexpr const std::uint8_t MAX_NUM_SOURES = 255;
static constexpr const std::uint8_t MAX_LENGTH_NAME = 255;
public:
Datasources()
{
std::fill(lengths.begin(), lengths.end(), 0);
std::fill(sources.begin(), sources.end(), '\0');
}
util::StringView GetSourceName(DatasourceID id) const
{
auto begin = sources.data() + (MAX_LENGTH_NAME * id);
return util::StringView{begin, lengths[id]};
}
void SetSourceName(DatasourceID id, const std::string &name)
{
BOOST_ASSERT(name.size() < MAX_LENGTH_NAME);
lengths[id] = std::min<std::uint32_t>(name.size(), MAX_LENGTH_NAME);
auto out = sources.data() + (MAX_LENGTH_NAME * id);
std::copy(name.begin(), name.begin() + lengths[id], out);
}
private:
std::array<std::uint32_t, MAX_NUM_SOURES> lengths;
std::array<char, MAX_LENGTH_NAME * MAX_NUM_SOURES> sources;
};
}
}
#endif
include/extractor/io.hpp
+27
View File
@@ -1,10 +1,13 @@
#ifndef OSRM_EXTRACTOR_IO_HPP
#define OSRM_EXTRACTOR_IO_HPP
#include "extractor/datasources.hpp"
#include "extractor/segment_data_container.hpp"
#include "storage/io.hpp"
#include <boost/assert.hpp>
namespace osrm
{
namespace extractor
@@ -12,6 +15,22 @@ namespace extractor
namespace io
{
void read(const boost::filesystem::path &path, Datasources &sources)
{
const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.ReadInto(sources);
}
void write(const boost::filesystem::path &path, Datasources &sources)
{
const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.WriteFrom(sources);
}
template <> void read(const boost::filesystem::path &path, SegmentDataContainer &segment_data)
{
const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
@@ -27,12 +46,14 @@ template <> void read(const boost::filesystem::path &path, SegmentDataContainer
segment_data.rev_weights.resize(num_entries);
segment_data.fwd_durations.resize(num_entries);
segment_data.rev_durations.resize(num_entries);
segment_data.datasources.resize(num_entries);
reader.ReadInto(segment_data.nodes);
reader.ReadInto(segment_data.fwd_weights);
reader.ReadInto(segment_data.rev_weights);
reader.ReadInto(segment_data.fwd_durations);
reader.ReadInto(segment_data.rev_durations);
reader.ReadInto(segment_data.datasources);
}
template <>
@@ -45,11 +66,17 @@ void write(const boost::filesystem::path &path, const SegmentDataContainer &segm
writer.WriteFrom(segment_data.index);
writer.WriteElementCount32(segment_data.nodes.size());
BOOST_ASSERT(segment_data.fwd_weights.size() == segment_data.nodes.size());
BOOST_ASSERT(segment_data.rev_weights.size() == segment_data.nodes.size());
BOOST_ASSERT(segment_data.fwd_durations.size() == segment_data.nodes.size());
BOOST_ASSERT(segment_data.rev_durations.size() == segment_data.nodes.size());
BOOST_ASSERT(segment_data.datasources.size() == segment_data.nodes.size());
writer.WriteFrom(segment_data.nodes);
writer.WriteFrom(segment_data.fwd_weights);
writer.WriteFrom(segment_data.rev_weights);
writer.WriteFrom(segment_data.fwd_durations);
writer.WriteFrom(segment_data.rev_durations);
writer.WriteFrom(segment_data.datasources);
}
}
}
include/extractor/segment_data_container.hpp
+26 -6
View File
@@ -55,10 +55,11 @@ template <bool UseShareMemory> class SegmentDataContainerImpl
Vector<EdgeWeight> fwd_weights_,
Vector<EdgeWeight> rev_weights_,
Vector<EdgeWeight> fwd_durations_,
Vector<EdgeWeight> rev_durations_)
Vector<EdgeWeight> rev_durations_,
Vector<DatasourceID> datasources_)
: index(std::move(index_)), nodes(std::move(nodes_)), fwd_weights(std::move(fwd_weights_)),
rev_weights(std::move(rev_weights_)), fwd_durations(std::move(fwd_durations_)),
rev_durations(std::move(rev_durations_))
rev_durations(std::move(rev_durations_)), datasources(std::move(datasources_))
{
}
@@ -79,11 +80,13 @@ template <bool UseShareMemory> class SegmentDataContainerImpl
{
return rev_weights[index[id] + offset];
}
// TODO we only need this for the datasource file since it breaks this
// abstraction, but uses this index
auto GetOffset(const DirectionalGeometryID id, const SegmentOffset offset) const
auto &ForwardDatasource(const DirectionalGeometryID id, const SegmentOffset offset)
{
return index[id] + offset;
return datasources[index[id] + 1 + offset];
}
auto &ReverseDatasource(const DirectionalGeometryID id, const SegmentOffset offset)
{
return datasources[index[id] + offset];
}
auto GetForwardGeometry(const DirectionalGeometryID id) const
@@ -131,6 +134,22 @@ template <bool UseShareMemory> class SegmentDataContainerImpl
return boost::adaptors::reverse(boost::make_iterator_range(begin, end));
}
auto GetForwardDatasources(const DirectionalGeometryID id) const
{
const auto begin = datasources.cbegin() + index.at(id) + 1;
const auto end = datasources.cbegin() + index.at(id + 1);
return boost::make_iterator_range(begin, end);
}
auto GetReverseDatasources(const DirectionalGeometryID id) const
{
const auto begin = datasources.cbegin() + index.at(id);
const auto end = datasources.cbegin() + index.at(id + 1) - 1;
return boost::adaptors::reverse(boost::make_iterator_range(begin, end));
}
auto GetNumberOfSegments() const { return fwd_weights.size(); }
friend void
@@ -147,6 +166,7 @@ template <bool UseShareMemory> class SegmentDataContainerImpl
Vector<EdgeWeight> rev_weights;
Vector<EdgeWeight> fwd_durations;
Vector<EdgeWeight> rev_durations;
Vector<DatasourceID> datasources;
};
}
include/storage/serialization.hpp
-33
View File
@@ -70,16 +70,6 @@ inline void readHSGR(io::FileReader &input_file,
input_file.ReadInto(edge_buffer, number_of_edges);
}
// Loads datasource_indexes from .datasource_indexes into memory
// Needs to be called after readElementCount() to get the correct offset in the stream
inline void readDatasourceIndexes(io::FileReader &datasource_indexes_file,
uint8_t *datasource_buffer,
const std::uint64_t number_of_datasource_indexes)
{
BOOST_ASSERT(datasource_buffer);
datasource_indexes_file.ReadInto(datasource_buffer, number_of_datasource_indexes);
}
// Loads edge data from .edge files into memory which includes its
// geometry, name ID, turn instruction, lane data ID, travel mode, entry class ID
// Needs to be called after readElementCount() to get the correct offset in the stream
@@ -134,29 +124,6 @@ void readNodes(io::FileReader &nodes_file,
BOOST_ASSERT(coordinate_list[i].IsValid());
}
}
// Reads datasource names out of .datasource_names files and metadata such as
// the length and offset of each name
struct DatasourceNamesData
{
std::vector<char> names;
std::vector<std::size_t> offsets;
std::vector<std::size_t> lengths;
};
inline DatasourceNamesData readDatasourceNames(io::FileReader &datasource_names_file)
{
DatasourceNamesData datasource_names_data;
std::vector<std::string> lines = datasource_names_file.ReadLines();
for (const auto &name : lines)
{
datasource_names_data.offsets.push_back(datasource_names_data.names.size());
datasource_names_data.lengths.push_back(name.size());
std::copy(name.c_str(),
name.c_str() + name.size(),
std::back_inserter(datasource_names_data.names));
}
return datasource_names_data;
}
}
}
}
include/storage/shared_datatype.hpp
+2 -6
View File
@@ -40,9 +40,7 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
"FILE_INDEX_PATH",
"CH_CORE_MARKER",
"DATASOURCES_LIST",
"DATASOURCE_NAME_DATA",
"DATASOURCE_NAME_OFFSETS",
"DATASOURCE_NAME_LENGTHS",
"DATASOURCES_NAMES",
"PROPERTIES",
"BEARING_CLASSID",
"BEARING_OFFSETS",
@@ -94,9 +92,7 @@ struct DataLayout
FILE_INDEX_PATH,
CH_CORE_MARKER,
DATASOURCES_LIST,
DATASOURCE_NAME_DATA,
DATASOURCE_NAME_OFFSETS,
DATASOURCE_NAME_LENGTHS,
DATASOURCES_NAMES,
PROPERTIES,
BEARING_CLASSID,
BEARING_OFFSETS,
include/updater/updater_config.hpp
-2
View File
@@ -51,7 +51,6 @@ struct UpdaterConfig final
geometry_path = osrm_input_path.string() + ".geometry";
rtree_leaf_path = osrm_input_path.string() + ".fileIndex";
datasource_names_path = osrm_input_path.string() + ".datasource_names";
datasource_indexes_path = osrm_input_path.string() + ".datasource_indexes";
profile_properties_path = osrm_input_path.string() + ".properties";
}
@@ -71,7 +70,6 @@ struct UpdaterConfig final
std::vector<std::string> segment_speed_lookup_paths;
std::vector<std::string> turn_penalty_lookup_paths;
std::string datasource_indexes_path;
std::string datasource_names_path;
std::string profile_properties_path;
};