Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Include datasources in .geometries file and refactor .datasource_names

Browse Source
This commit is contained in:
Patrick Niklaus 2017-03-09 17:01:04 +00:00 committed by Patrick Niklaus
parent ffd6311e7d
commit a636e8cc13
14 changed files with 166 additions and 349 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -2,6 +2,8 @@
- Changes from 5.6
- Internals
- Shared memory notification via conditional variables on Linux or semaphore queue on OS X and Windows with a limit of 128 OSRM Engine instances
- Files
- .osrm.datasource_index file was removed. Data is now part of .osrm.geometries.
# 5.6.3
- Changes from 5.6.0

38
features/options/contract/datasources.feature
View File

@ -1,38 +0,0 @@
@prepare @options @files
Feature: osrm-contract command line options: datasources
# expansions:
# {processed_file} => path to .osrm file
Background:
Given the profile "testbot"
Given the extract extra arguments "--generate-edge-lookup"
And the node map
"""
a b
"""
And the ways
| nodes |
| ab |
And the speed file
"""
1,2,50
2,1,50
2,3,50
3,2,50
1,4,50
4,1,50
"""
And the data has been extracted
Scenario: osrm-contract - Passing base file
When I run "osrm-contract --segment-speed-file {speeds_file} {processed_file}"
Then datasource names should contain "lua profile,27_osrmcontract_passing_base_file_speeds"
And it should exit successfully
Scenario: osrm-contract - Passing base file
Given the speed file
"""
"""
And the data has been extracted
When I run "osrm-contract --segment-speed-file {speeds_file} {processed_file}"
Then it should exit successfully

5
features/step_definitions/options.js
View File

@ -103,11 +103,6 @@ module.exports = function () {
assert.equal(this.stderr.split('\n').length - 1, parseInt(lines));
});
this.Then(/^datasource names should contain "(.+)"$/, (expectedData) => {
const actualData = fs.readFileSync(this.processedCacheFile + '.datasource_names', {encoding:'UTF-8'}).trim().split('\n').join(',');
assert.equal(actualData, expectedData);
});
this.Given(/^the query options$/, (table, callback) => {
table.raw().forEach(tuple => {
this.queryParams[tuple[0]] = tuple[1];

139
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
View File

@ -8,6 +8,7 @@
#include "engine/algorithm.hpp"
#include "engine/geospatial_query.hpp"
#include "extractor/datasources.hpp"
#include "extractor/guidance/turn_instruction.hpp"
#include "extractor/guidance/turn_lane_types.hpp"
#include "extractor/profile_properties.hpp"
@ -205,6 +206,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
std::string m_timestamp;
extractor::ProfileProperties *m_profile_properties;
extractor::Datasources *m_datasources;
unsigned m_check_sum;
util::ShM<util::Coordinate, true>::vector m_coordinate_list;
@ -219,7 +221,6 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
util::NameTable m_names_table;
util::ShM<unsigned, true>::vector m_name_begin_indices;
util::ShM<bool, true>::vector m_is_core_node;
util::ShM<DatasourceID, true>::vector m_datasource_list;
util::ShM<std::uint32_t, true>::vector m_lane_description_offsets;
util::ShM<extractor::guidance::TurnLaneType::Mask, true>::vector m_lane_description_masks;
util::ShM<TurnPenalty, true>::vector m_turn_weight_penalties;
@ -467,39 +468,21 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
geometries_rev_duration_list_ptr,
data_layout.num_entries[storage::DataLayout::GEOMETRIES_REV_DURATION_LIST]);
auto datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>(
memory_block, storage::DataLayout::DATASOURCES_LIST);
util::ShM<DatasourceID, true>::vector datasources_list(
datasources_list_ptr, data_layout.num_entries[storage::DataLayout::DATASOURCES_LIST]);
segment_data = extractor::SegmentDataView{std::move(geometry_begin_indices),
std::move(geometry_node_list),
std::move(geometry_fwd_weight_list),
std::move(geometry_rev_weight_list),
std::move(geometry_fwd_duration_list),
std::move(geometry_rev_duration_list)};
std::move(geometry_rev_duration_list),
std::move(datasources_list)};
auto datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>(
memory_block, storage::DataLayout::DATASOURCES_LIST);
util::ShM<DatasourceID, true>::vector datasources_list(
datasources_list_ptr, data_layout.num_entries[storage::DataLayout::DATASOURCES_LIST]);
m_datasource_list = std::move(datasources_list);
auto datasource_name_data_ptr =
data_layout.GetBlockPtr<char>(memory_block, storage::DataLayout::DATASOURCE_NAME_DATA);
util::ShM<char, true>::vector datasource_name_data(
datasource_name_data_ptr,
data_layout.num_entries[storage::DataLayout::DATASOURCE_NAME_DATA]);
m_datasource_name_data = std::move(datasource_name_data);
auto datasource_name_offsets_ptr = data_layout.GetBlockPtr<std::size_t>(
memory_block, storage::DataLayout::DATASOURCE_NAME_OFFSETS);
util::ShM<std::size_t, true>::vector datasource_name_offsets(
datasource_name_offsets_ptr,
data_layout.num_entries[storage::DataLayout::DATASOURCE_NAME_OFFSETS]);
m_datasource_name_offsets = std::move(datasource_name_offsets);
auto datasource_name_lengths_ptr = data_layout.GetBlockPtr<std::size_t>(
memory_block, storage::DataLayout::DATASOURCE_NAME_LENGTHS);
util::ShM<std::size_t, true>::vector datasource_name_lengths(
datasource_name_lengths_ptr,
data_layout.num_entries[storage::DataLayout::DATASOURCE_NAME_LENGTHS]);
m_datasource_name_lengths = std::move(datasource_name_lengths);
m_datasources = data_layout.GetBlockPtr<extractor::Datasources>(
memory_block, storage::DataLayout::DATASOURCES_NAMES);
}
void InitializeIntersectionClassPointers(storage::DataLayout &data_layout, char *memory_block)
@ -611,6 +594,24 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
return std::vector<EdgeWeight>{range.begin(), range.end()};
}
// Returns the data source ids that were used to supply the edge
// weights.
virtual std::vector<DatasourceID>
GetUncompressedForwardDatasources(const EdgeID id) const override final
{
auto range = segment_data.GetForwardDatasources(id);
return std::vector<DatasourceID>{range.begin(), range.end()};
}
// Returns the data source ids that were used to supply the edge
// weights.
virtual std::vector<DatasourceID>
GetUncompressedReverseDatasources(const EdgeID id) const override final
{
auto range = segment_data.GetReverseDatasources(id);
return std::vector<DatasourceID>{range.begin(), range.end()};
}
virtual GeometryID GetGeometryIndexForEdgeID(const EdgeID id) const override final
{
return m_via_geometry_list.at(id);
@ -781,89 +782,9 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
return m_name_table.GetDestinationsForID(id);
}
// Returns the data source ids that were used to supply the edge
// weights.
virtual std::vector<DatasourceID>
GetUncompressedForwardDatasources(const EdgeID id) const override final
{
/*
* Data sources for geometries are stored in one place for
* both forward and reverse segments along the same bi-
* directional edge. The indices stores
* refences to where to find the beginning of the bi-
* directional edge in the list vector. For
* forward datasources of bi-directional edges, edges 2 to
* n of that edge need to be read.
*/
const auto begin = segment_data.GetOffset(id, 0) + 1;
const auto end = segment_data.GetOffset(id + 1, 0);
std::vector<DatasourceID> result_datasources;
result_datasources.reserve(end - begin);
// If there was no datasource info, return an array of 0's.
if (m_datasource_list.empty())
{
result_datasources.resize(end - begin, 0);
}
else
{
std::copy(m_datasource_list.begin() + begin,
m_datasource_list.begin() + end,
std::back_inserter(result_datasources));
}
return result_datasources;
}
// Returns the data source ids that were used to supply the edge
// weights.
virtual std::vector<DatasourceID>
GetUncompressedReverseDatasources(const EdgeID id) const override final
{
/*
* Datasources for geometries are stored in one place for
* both forward and reverse segments along the same bi-
* directional edge. The indices stores
* refences to where to find the beginning of the bi-
* directional edge in the list vector. For
* reverse datasources of bi-directional edges, edges 1 to
* n-1 of that edge need to be read in reverse.
*/
const auto begin = segment_data.GetOffset(id, 0);
const auto end = segment_data.GetOffset(id + 1, 0) - 1;
std::vector<DatasourceID> result_datasources;
result_datasources.reserve(end - begin);
// If there was no datasource info, return an array of 0's.
if (m_datasource_list.empty())
{
result_datasources.resize(end - begin, 0);
}
else
{
std::reverse_copy(m_datasource_list.begin() + begin,
m_datasource_list.begin() + end,
std::back_inserter(result_datasources));
}
return result_datasources;
}
StringView GetDatasourceName(const DatasourceID id) const override final
{
BOOST_ASSERT(m_datasource_name_offsets.size() >= 1);
BOOST_ASSERT(m_datasource_name_offsets.size() > id);
auto first = m_datasource_name_data.begin() + m_datasource_name_offsets[id];
auto last = m_datasource_name_data.begin() + m_datasource_name_offsets[id] +
m_datasource_name_lengths[id];
// These iterators are useless: they're InputIterators onto a contiguous block of memory.
// Deref to get to the first element, then Addressof to get the memory address of the it.
const std::size_t len = &*last - &*first;
return StringView{&*first, len};
return m_datasources->GetSourceName(id);
}
std::string GetTimestamp() const override final { return m_timestamp; }

48
include/extractor/datasources.hpp Normal file
View File

@ -0,0 +1,48 @@
#ifndef OSRM_EXTRACT_DATASOURCES_HPP
#define OSRM_EXTRACT_DATASOURCES_HPP
#include <cstdint>
#include <util/string_view.hpp>
#include <util/typedefs.hpp>
namespace osrm
{
namespace extractor
{
class Datasources
{
static constexpr const std::uint8_t MAX_NUM_SOURES = 255;
static constexpr const std::uint8_t MAX_LENGTH_NAME = 255;
public:
Datasources()
{
std::fill(lengths.begin(), lengths.end(), 0);
std::fill(sources.begin(), sources.end(), '\0');
}
util::StringView GetSourceName(DatasourceID id) const
{
auto begin = sources.data() + (MAX_LENGTH_NAME * id);
return util::StringView{begin, lengths[id]};
}
void SetSourceName(DatasourceID id, const std::string &name)
{
BOOST_ASSERT(name.size() < MAX_LENGTH_NAME);
lengths[id] = std::min<std::uint32_t>(name.size(), MAX_LENGTH_NAME);
auto out = sources.data() + (MAX_LENGTH_NAME * id);
std::copy(name.begin(), name.begin() + lengths[id], out);
}
private:
std::array<std::uint32_t, MAX_NUM_SOURES> lengths;
std::array<char, MAX_LENGTH_NAME * MAX_NUM_SOURES> sources;
};
}
}
#endif

27
include/extractor/io.hpp
View File

@ -1,10 +1,13 @@
#ifndef OSRM_EXTRACTOR_IO_HPP
#define OSRM_EXTRACTOR_IO_HPP
#include "extractor/datasources.hpp"
#include "extractor/segment_data_container.hpp"
#include "storage/io.hpp"
#include <boost/assert.hpp>
namespace osrm
{
namespace extractor
@ -12,6 +15,22 @@ namespace extractor
namespace io
{
void read(const boost::filesystem::path &path, Datasources &sources)
{
const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.ReadInto(sources);
}
void write(const boost::filesystem::path &path, Datasources &sources)
{
const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.WriteFrom(sources);
}
template <> void read(const boost::filesystem::path &path, SegmentDataContainer &segment_data)
{
const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
@ -27,12 +46,14 @@ template <> void read(const boost::filesystem::path &path, SegmentDataContainer
segment_data.rev_weights.resize(num_entries);
segment_data.fwd_durations.resize(num_entries);
segment_data.rev_durations.resize(num_entries);
segment_data.datasources.resize(num_entries);
reader.ReadInto(segment_data.nodes);
reader.ReadInto(segment_data.fwd_weights);
reader.ReadInto(segment_data.rev_weights);
reader.ReadInto(segment_data.fwd_durations);
reader.ReadInto(segment_data.rev_durations);
reader.ReadInto(segment_data.datasources);
}
template <>
@ -45,11 +66,17 @@ void write(const boost::filesystem::path &path, const SegmentDataContainer &segm
writer.WriteFrom(segment_data.index);
writer.WriteElementCount32(segment_data.nodes.size());
BOOST_ASSERT(segment_data.fwd_weights.size() == segment_data.nodes.size());
BOOST_ASSERT(segment_data.rev_weights.size() == segment_data.nodes.size());
BOOST_ASSERT(segment_data.fwd_durations.size() == segment_data.nodes.size());
BOOST_ASSERT(segment_data.rev_durations.size() == segment_data.nodes.size());
BOOST_ASSERT(segment_data.datasources.size() == segment_data.nodes.size());
writer.WriteFrom(segment_data.nodes);
writer.WriteFrom(segment_data.fwd_weights);
writer.WriteFrom(segment_data.rev_weights);
writer.WriteFrom(segment_data.fwd_durations);
writer.WriteFrom(segment_data.rev_durations);
writer.WriteFrom(segment_data.datasources);
}
}
}

32
include/extractor/segment_data_container.hpp
View File

@ -55,10 +55,11 @@ template <bool UseShareMemory> class SegmentDataContainerImpl
Vector<EdgeWeight> fwd_weights_,
Vector<EdgeWeight> rev_weights_,
Vector<EdgeWeight> fwd_durations_,
Vector<EdgeWeight> rev_durations_)
Vector<EdgeWeight> rev_durations_,
Vector<DatasourceID> datasources_)
: index(std::move(index_)), nodes(std::move(nodes_)), fwd_weights(std::move(fwd_weights_)),
rev_weights(std::move(rev_weights_)), fwd_durations(std::move(fwd_durations_)),
rev_durations(std::move(rev_durations_))
rev_durations(std::move(rev_durations_)), datasources(std::move(datasources_))
{
}
@ -79,11 +80,13 @@ template <bool UseShareMemory> class SegmentDataContainerImpl
{
return rev_weights[index[id] + offset];
}
// TODO we only need this for the datasource file since it breaks this
// abstraction, but uses this index
auto GetOffset(const DirectionalGeometryID id, const SegmentOffset offset) const
auto &ForwardDatasource(const DirectionalGeometryID id, const SegmentOffset offset)
{
return index[id] + offset;
return datasources[index[id] + 1 + offset];
}
auto &ReverseDatasource(const DirectionalGeometryID id, const SegmentOffset offset)
{
return datasources[index[id] + offset];
}
auto GetForwardGeometry(const DirectionalGeometryID id) const
@ -131,6 +134,22 @@ template <bool UseShareMemory> class SegmentDataContainerImpl
return boost::adaptors::reverse(boost::make_iterator_range(begin, end));
}
auto GetForwardDatasources(const DirectionalGeometryID id) const
{
const auto begin = datasources.cbegin() + index.at(id) + 1;
const auto end = datasources.cbegin() + index.at(id + 1);
return boost::make_iterator_range(begin, end);
}
auto GetReverseDatasources(const DirectionalGeometryID id) const
{
const auto begin = datasources.cbegin() + index.at(id);
const auto end = datasources.cbegin() + index.at(id + 1) - 1;
return boost::adaptors::reverse(boost::make_iterator_range(begin, end));
}
auto GetNumberOfSegments() const { return fwd_weights.size(); }
friend void
@ -147,6 +166,7 @@ template <bool UseShareMemory> class SegmentDataContainerImpl
Vector<EdgeWeight> rev_weights;
Vector<EdgeWeight> fwd_durations;
Vector<EdgeWeight> rev_durations;
Vector<DatasourceID> datasources;
};
}

33
include/storage/serialization.hpp
View File

@ -70,16 +70,6 @@ inline void readHSGR(io::FileReader &input_file,
input_file.ReadInto(edge_buffer, number_of_edges);
}
// Loads datasource_indexes from .datasource_indexes into memory
// Needs to be called after readElementCount() to get the correct offset in the stream
inline void readDatasourceIndexes(io::FileReader &datasource_indexes_file,
uint8_t *datasource_buffer,
const std::uint64_t number_of_datasource_indexes)
{
BOOST_ASSERT(datasource_buffer);
datasource_indexes_file.ReadInto(datasource_buffer, number_of_datasource_indexes);
}
// Loads edge data from .edge files into memory which includes its
// geometry, name ID, turn instruction, lane data ID, travel mode, entry class ID
// Needs to be called after readElementCount() to get the correct offset in the stream
@ -134,29 +124,6 @@ void readNodes(io::FileReader &nodes_file,
BOOST_ASSERT(coordinate_list[i].IsValid());
}
}
// Reads datasource names out of .datasource_names files and metadata such as
// the length and offset of each name
struct DatasourceNamesData
{
std::vector<char> names;
std::vector<std::size_t> offsets;
std::vector<std::size_t> lengths;
};
inline DatasourceNamesData readDatasourceNames(io::FileReader &datasource_names_file)
{
DatasourceNamesData datasource_names_data;
std::vector<std::string> lines = datasource_names_file.ReadLines();
for (const auto &name : lines)
{
datasource_names_data.offsets.push_back(datasource_names_data.names.size());
datasource_names_data.lengths.push_back(name.size());
std::copy(name.c_str(),
name.c_str() + name.size(),
std::back_inserter(datasource_names_data.names));
}
return datasource_names_data;
}
}
}
}

8
include/storage/shared_datatype.hpp
View File

@ -40,9 +40,7 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
"FILE_INDEX_PATH",
"CH_CORE_MARKER",
"DATASOURCES_LIST",
"DATASOURCE_NAME_DATA",
"DATASOURCE_NAME_OFFSETS",
"DATASOURCE_NAME_LENGTHS",
"DATASOURCES_NAMES",
"PROPERTIES",
"BEARING_CLASSID",
"BEARING_OFFSETS",
@ -94,9 +92,7 @@ struct DataLayout
FILE_INDEX_PATH,
CH_CORE_MARKER,
DATASOURCES_LIST,
DATASOURCE_NAME_DATA,
DATASOURCE_NAME_OFFSETS,
DATASOURCE_NAME_LENGTHS,
DATASOURCES_NAMES,
PROPERTIES,
BEARING_CLASSID,
BEARING_OFFSETS,

2
include/updater/updater_config.hpp
View File

@ -51,7 +51,6 @@ struct UpdaterConfig final
geometry_path = osrm_input_path.string() + ".geometry";
rtree_leaf_path = osrm_input_path.string() + ".fileIndex";
datasource_names_path = osrm_input_path.string() + ".datasource_names";
datasource_indexes_path = osrm_input_path.string() + ".datasource_indexes";
profile_properties_path = osrm_input_path.string() + ".properties";
}
@ -71,7 +70,6 @@ struct UpdaterConfig final
std::vector<std::string> segment_speed_lookup_paths;
std::vector<std::string> turn_penalty_lookup_paths;
std::string datasource_indexes_path;
std::string datasource_names_path;
std::string profile_properties_path;
};

6
src/extractor/compressed_edge_container.cpp
View File

@ -221,6 +221,7 @@ void CompressedEdgeContainer::InitializeBothwayVector()
segment_data->rev_weights.reserve(m_compressed_oneway_geometries.size());
segment_data->fwd_durations.reserve(m_compressed_oneway_geometries.size());
segment_data->rev_durations.reserve(m_compressed_oneway_geometries.size());
segment_data->datasources.reserve(m_compressed_oneway_geometries.size());
}
unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID r_edge_id)
@ -238,11 +239,14 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID
const auto &first_node = reverse_bucket.back();
constexpr DatasourceID LUA_SOURCE = 0;
segment_data->nodes.emplace_back(first_node.node_id);
segment_data->fwd_weights.emplace_back(INVALID_EDGE_WEIGHT);
segment_data->rev_weights.emplace_back(first_node.weight);
segment_data->fwd_durations.emplace_back(INVALID_EDGE_WEIGHT);
segment_data->rev_durations.emplace_back(first_node.duration);
segment_data->datasources.emplace_back(LUA_SOURCE);
for (std::size_t i = 0; i < forward_bucket.size() - 1; ++i)
{
@ -256,6 +260,7 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID
segment_data->rev_weights.emplace_back(rev_node.weight);
segment_data->fwd_durations.emplace_back(fwd_node.duration);
segment_data->rev_durations.emplace_back(rev_node.duration);
segment_data->datasources.emplace_back(LUA_SOURCE);
}
const auto &last_node = forward_bucket.back();
@ -265,6 +270,7 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID
segment_data->rev_weights.emplace_back(INVALID_EDGE_WEIGHT);
segment_data->fwd_durations.emplace_back(last_node.duration);
segment_data->rev_durations.emplace_back(INVALID_EDGE_WEIGHT);
segment_data->datasources.emplace_back(LUA_SOURCE);
return zipped_geometry_id;
}

116
src/storage/storage.cpp
View File

@ -3,6 +3,7 @@
#include "extractor/compressed_edge_container.hpp"
#include "extractor/edge_based_edge.hpp"
#include "extractor/guidance/turn_instruction.hpp"
#include "extractor/io.hpp"
#include "extractor/original_edge_data.hpp"
#include "extractor/profile_properties.hpp"
#include "extractor/query_node.hpp"
@ -339,40 +340,13 @@ void Storage::PopulateLayout(DataLayout &layout)
number_of_compressed_geometries);
layout.SetBlockSize<EdgeWeight>(DataLayout::GEOMETRIES_REV_DURATION_LIST,
number_of_compressed_geometries);
layout.SetBlockSize<DatasourceID>(DataLayout::DATASOURCES_LIST,
number_of_compressed_geometries);
}
// load datasource sizes. This file is optional, and it's non-fatal if it doesn't exist.
if (boost::filesystem::exists(config.datasource_indexes_path))
// Load datasource name sizes.
{
io::FileReader reader(config.datasource_indexes_path, io::FileReader::HasNoFingerprint);
const auto number_of_datasources = reader.ReadElementCount64();
layout.SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST, number_of_datasources);
}
else
{
layout.SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST, 0);
}
// Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't exist
if (boost::filesystem::exists(config.datasource_names_path))
{
io::FileReader reader(config.datasource_names_path, io::FileReader::HasNoFingerprint);
const serialization::DatasourceNamesData datasource_names_data =
serialization::readDatasourceNames(reader);
layout.SetBlockSize<char>(DataLayout::DATASOURCE_NAME_DATA,
datasource_names_data.names.size());
layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_OFFSETS,
datasource_names_data.offsets.size());
layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_LENGTHS,
datasource_names_data.lengths.size());
}
else
{
layout.SetBlockSize<char>(DataLayout::DATASOURCE_NAME_DATA, 0);
layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_OFFSETS, 0);
layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_LENGTHS, 0);
layout.SetBlockSize<extractor::Datasources>(DataLayout::DATASOURCES_NAMES, 1);
}
{
@ -689,81 +663,17 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
BOOST_ASSERT(geometry_node_lists_count ==
layout.num_entries[DataLayout::GEOMETRIES_REV_DURATION_LIST]);
geometry_input_file.ReadInto(geometries_rev_duration_list_ptr, geometry_node_lists_count);
const auto datasource_list_ptr =
layout.GetBlockPtr<DatasourceID, true>(memory_ptr, DataLayout::DATASOURCES_LIST);
BOOST_ASSERT(geometry_node_lists_count == layout.num_entries[DataLayout::DATASOURCES_LIST]);
geometry_input_file.ReadInto(datasource_list_ptr, geometry_node_lists_count);
}
if (boost::filesystem::exists(config.datasource_indexes_path))
{
io::FileReader geometry_datasource_file(config.datasource_indexes_path,
io::FileReader::HasNoFingerprint);
const auto number_of_compressed_datasources = geometry_datasource_file.ReadElementCount64();
// load datasource information (if it exists)
const auto datasources_list_ptr =
layout.GetBlockPtr<uint8_t, true>(memory_ptr, DataLayout::DATASOURCES_LIST);
if (number_of_compressed_datasources > 0)
{
serialization::readDatasourceIndexes(
geometry_datasource_file, datasources_list_ptr, number_of_compressed_datasources);
}
}
else
{
layout.GetBlockPtr<uint8_t, true>(memory_ptr, DataLayout::DATASOURCES_LIST);
}
if (boost::filesystem::exists(config.datasource_names_path))
{
io::FileReader datasource_names_file(config.datasource_names_path,
io::FileReader::HasNoFingerprint);
const auto datasource_names_data =
serialization::readDatasourceNames(datasource_names_file);
// load datasource name information (if it exists)
const auto datasource_name_data_ptr =
layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::DATASOURCE_NAME_DATA);
if (layout.GetBlockSize(DataLayout::DATASOURCE_NAME_DATA) > 0)
{
BOOST_ASSERT(std::distance(datasource_names_data.names.begin(),
datasource_names_data.names.end()) *
sizeof(decltype(datasource_names_data.names)::value_type) <=
layout.GetBlockSize(DataLayout::DATASOURCE_NAME_DATA));
std::copy(datasource_names_data.names.begin(),
datasource_names_data.names.end(),
datasource_name_data_ptr);
}
const auto datasource_name_offsets_ptr =
layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_OFFSETS);
if (layout.GetBlockSize(DataLayout::DATASOURCE_NAME_OFFSETS) > 0)
{
BOOST_ASSERT(std::distance(datasource_names_data.offsets.begin(),
datasource_names_data.offsets.end()) *
sizeof(decltype(datasource_names_data.offsets)::value_type) <=
layout.GetBlockSize(DataLayout::DATASOURCE_NAME_OFFSETS));
std::copy(datasource_names_data.offsets.begin(),
datasource_names_data.offsets.end(),
datasource_name_offsets_ptr);
}
const auto datasource_name_lengths_ptr =
layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_LENGTHS);
if (layout.GetBlockSize(DataLayout::DATASOURCE_NAME_LENGTHS) > 0)
{
BOOST_ASSERT(std::distance(datasource_names_data.lengths.begin(),
datasource_names_data.lengths.end()) *
sizeof(decltype(datasource_names_data.lengths)::value_type) <=
layout.GetBlockSize(DataLayout::DATASOURCE_NAME_LENGTHS));
std::copy(datasource_names_data.lengths.begin(),
datasource_names_data.lengths.end(),
datasource_name_lengths_ptr);
}
}
else
{
layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::DATASOURCE_NAME_DATA);
layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_OFFSETS);
layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_LENGTHS);
const auto datasources_names_ptr = layout.GetBlockPtr<extractor::Datasources, true>(
memory_ptr, DataLayout::DATASOURCES_NAMES);
extractor::io::read(config.datasource_names_path, *datasources_names_ptr);
}
// Loading list of coordinates

6
src/storage/storage_config.cpp
View File

@ -32,7 +32,6 @@ StorageConfig::StorageConfig(const boost::filesystem::path &base)
turn_weight_penalties_path{base.string() + ".turn_weight_penalties"},
turn_duration_penalties_path{base.string() + ".turn_duration_penalties"},
datasource_names_path{base.string() + ".datasource_names"},
datasource_indexes_path{base.string() + ".datasource_indexes"},
names_data_path{base.string() + ".names"}, properties_path{base.string() + ".properties"},
intersection_class_path{base.string() + ".icd"}, turn_lane_data_path{base.string() + ".tld"},
turn_lane_description_path{base.string() + ".tls"},
@ -54,7 +53,8 @@ bool StorageConfig::IsValid() const
turn_duration_penalties_path,
names_data_path,
properties_path,
intersection_class_path}))
intersection_class_path,
datasource_names_path}))
{
return false;
}
@ -62,7 +62,7 @@ bool StorageConfig::IsValid() const
// TODO: add algorithm checks
// CH files
CheckFileList({hsgr_data_path, core_data_path, datasource_names_path, datasource_indexes_path});
CheckFileList({hsgr_data_path, core_data_path});
// MLD files
CheckFileList({mld_partition_path, mld_storage_path, edge_based_graph_path});

53
src/updater/updater.cpp
View File

@ -262,10 +262,6 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
auto segment_speed_lookup = csv::readSegmentValues(config.segment_speed_lookup_paths);
auto turn_penalty_lookup = csv::readTurnValues(config.turn_penalty_lookup_paths);
// If we update the edge weights, this file will hold the datasource information for each
// segment; the other files will also be conditionally filled concurrently if we make an update
std::vector<uint8_t> geometry_datasource;
std::vector<extractor::QueryNode> internal_to_external_node_map;
extractor::SegmentDataContainer segment_data;
@ -294,14 +290,6 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
// Here, we have to update the compressed geometry weights
// First, we need the external-to-internal node lookup table
// This is a list of the "data source id" for every segment in the compressed
// geometry container. We assume that everything so far has come from the
// profile (data source 0). Here, we replace the 0's with the index of the
// CSV file that supplied the value that gets used for that segment, then
// we write out this list so that it can be returned by the debugging
// vector tiles later on.
geometry_datasource.resize(segment_data.GetNumberOfSegments(), 0);
// Now, we iterate over all the segments stored in the StaticRTree, updating
// the packed geometry weights in the `.geometries` file (note: we do not
// update the RTree itself, we just use the leaf nodes to iterate over all segments)
@ -364,8 +352,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
segment_data.ForwardWeight(geometry_id, segment_offset) = new_segment_weight;
segment_data.ForwardDuration(geometry_id, segment_offset) =
new_segment_duration;
geometry_datasource[segment_data.GetOffset(geometry_id, segment_offset) + 1] =
value->source;
segment_data.ForwardDatasource(geometry_id, segment_offset) = value->source;
fwd_source = value->source;
}
@ -385,8 +372,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
segment_data.ReverseWeight(geometry_id, segment_offset) = new_segment_weight;
segment_data.ReverseDuration(geometry_id, segment_offset) =
new_segment_duration;
geometry_datasource[segment_data.GetOffset(geometry_id, segment_offset)] =
value->source;
segment_data.ReverseDatasource(geometry_id, segment_offset) = value->source;
rev_source = value->source;
}
@ -430,44 +416,23 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
extractor::io::write(config.geometry_path, segment_data);
};
const auto save_datasource_indexes = [&] {
std::ofstream datasource_stream(config.datasource_indexes_path, std::ios::binary);
if (!datasource_stream)
{
const std::string message{"Failed to open " + config.datasource_indexes_path +
" for writing"};
throw util::exception(message + SOURCE_REF);
}
std::uint64_t number_of_datasource_entries = geometry_datasource.size();
datasource_stream.write(reinterpret_cast<const char *>(&number_of_datasource_entries),
sizeof(number_of_datasource_entries));
if (number_of_datasource_entries > 0)
{
datasource_stream.write(reinterpret_cast<char *>(&(geometry_datasource[0])),
number_of_datasource_entries * sizeof(uint8_t));
}
};
const auto save_datastore_names = [&] {
std::ofstream datasource_stream(config.datasource_names_path, std::ios::binary);
if (!datasource_stream)
{
const std::string message{"Failed to open " + config.datasource_names_path +
" for writing"};
throw util::exception(message + SOURCE_REF);
}
datasource_stream << "lua profile" << std::endl;
extractor::Datasources sources;
DatasourceID source = 0;
sources.SetSourceName(source++, "lua profile");
// Only write the filename, without path or extension.
// This prevents information leakage, and keeps names short
// for rendering in the debug tiles.
for (auto const &name : config.segment_speed_lookup_paths)
{
datasource_stream << boost::filesystem::path(name).stem().string() << std::endl;
sources.SetSourceName(source++, boost::filesystem::path(name).stem().string());
}
extractor::io::write(config.datasource_names_path, sources);
};
tbb::parallel_invoke(maybe_save_geometries, save_datasource_indexes, save_datastore_names);
tbb::parallel_invoke(maybe_save_geometries, save_datastore_names);
std::vector<TurnPenalty> turn_weight_penalties;
std::vector<TurnPenalty> turn_duration_penalties;