Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

wip

Browse Source
This commit is contained in:
Siarhei Fedartsou 2022-11-26 21:37:18 +01:00
parent 38bb017057
commit 14d3d50263
14 changed files with 391 additions and 444 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
features/step_definitions/data.js
View File

@ -5,6 +5,7 @@ var path = require('path');
var fs = require('fs');
var d3 = require('d3-queue');
var OSM = require('../lib/osm');
const parquet = require('parquetjs');
module.exports = function () {
this.Given(/^the profile "([^"]*)"$/, (profile, callback) => {
@ -257,6 +258,61 @@ module.exports = function () {
this.environment = Object.assign({OSRM_RASTER_SOURCE: this.rasterCacheFile}, this.environment);
});
this.Given(/^the parquet speed file$/, (data, callback) => {
console.log(this.speedsCacheFile);
const schema = new parquet.ParquetSchema({
from: { type: 'INT64', encoding: 'PLAIN' },
to: { type: 'INT64', encoding: 'PLAIN' },
speed: { type: 'DOUBLE', encoding: 'PLAIN' },
rate: { type: 'DOUBLE', encoding: 'PLAIN', optional: true },
});
parquet.ParquetWriter.openFile(schema, this.speedsCacheFile).then((writer) => {
const promises = [];
data.split('\n').forEach((line) => {
const entries = line.split(',');
const from = Number.parseInt(entries[0]);
const to = Number.parseInt(entries[1]);
const speed = Number.parseFloat(entries[2]);
const rate = entries[3] ? Number.parseFloat(entries[3]) : undefined;
console.log('writing', from, to, speed, rate);
promises.push(writer.appendRow({ from, to, speed, rate }));
});
Promise.all(promises).then(() => {
writer.close().then(() => {
callback();
});
});
});
});
this.Given(/^the parquet turn penalty file$/, (data, callback) => {
const schema = new parquet.ParquetSchema({
from: { type: 'INT64' },
via: { type: 'INT64' },
to: { type: 'INT64' },
duration: { type: 'DOUBLE' },
weight: { type: 'DOUBLE' },
});
parquet.ParquetWriter.openFile(schema, this.penaltiesCacheFile).then((writer) => {
const promises = [];
data.split('\n').forEach((line) => {
const entries = line.split(',');
const from = Number.parseInt(entries[0]);
const via = Number.parseInt(entries[1]);
const to = Number.parseInt(entries[2]);
const duration = Number.parseFloat(entries[3]);
const weight = Number.parseFloat(entries[4]);
promises.push(writer.appendRow({ from, via, to, duration, weight }));
});
Promise.all(promises).then(() => {
writer.close().then(() => {
callback();
});
});
});
});
this.Given(/^the speed file$/, (data, callback) => {
// TODO: Don't overwrite if it exists
fs.writeFile(this.speedsCacheFile, data, callback);

363
features/testbot/weight.feature
View File

@ -1,292 +1,5 @@
@testbot
Feature: Weight tests
Background:
Given the profile "testbot"
Given a grid size of 10 meters
Given the query options
| geometries | geojson |
Scenario: Weight details
Given the query options
| annotations | weight |
Given the node map
"""
s
·
a---b---c
|
d
|··t
e
"""
And the ways
| nodes |
| abc |
| cde |
When I route I should get
| waypoints | route | a:weight |
| s,t | abc,cde | 1.1:2:2:0.9 |
When I route I should get
| waypoints | route | times | weight_name | weights |
| s,t | abc,cde | 6s,0s | duration | 6,0 |
# FIXME include/engine/guidance/assemble_geometry.hpp:95
Scenario: Start and target on the same and adjacent edge
Given the query options
| annotations | distance,duration,weight,nodes,speed |
Given the node map
"""
a-------b-------c
· · ·
s t e
"""
And the ways
| nodes |
| abc |
When I route I should get
| waypoints | route | distances | weights | times | a:distance | a:duration | a:weight | a:speed |
| s,t | abc,abc | 20m,0m | 2,0 | 2s,0s | 20.034626629 | 2 | 2 | 10 |
| t,s | abc,abc | 20m,0m | 2,0 | 2s,0s | 20.034626629 | 2 | 2 | 10 |
| s,e | abc,abc | 40m,0m | 3.9,0 | 3.9s,0s | 29.940636463:10.017313314 | 3:0.9 | 3:0.9 | 10:11.1 |
| e,s | abc,abc | 40m,0m | 3.9,0 | 3.9s,0s | 10.017313314:29.940636463 | 0.9:3 | 0.9:3 | 11.1:10 |
Scenario: Step weights -- way_function: fail if no weight or weight_per_meter property
Given the profile file
"""
local functions = require('testbot')
functions.setup_testbot = functions.setup
functions.setup = function()
local profile = functions.setup_testbot()
profile.properties.traffic_signal_penalty = 0
profile.properties.u_turn_penalty = 0
profile.properties.weight_name = 'steps'
return profile
end
functions.process_way = function(profile, way, result)
result.forward_mode = mode.driving
result.backward_mode = mode.driving
result.forward_speed = 42
result.backward_speed = 42
end
return functions
"""
And the node map
"""
a---b
"""
And the ways
| nodes |
| ab |
And the data has been saved to disk
When I try to run "osrm-extract {osm_file} --profile {profile_file}"
Then stderr should contain "There are no edges"
And it should exit with an error
Scenario: Step weights -- way_function: second way wins
Given the profile file
"""
local functions = require('testbot')
functions.setup_testbot = functions.setup
functions.setup = function()
local profile = functions.setup_testbot()
profile.properties.traffic_signal_penalty = 0
profile.properties.u_turn_penalty = 0
profile.properties.weight_name = 'steps'
return profile
end
functions.process_way = function(profile, way, result)
result.forward_mode = mode.driving
result.backward_mode = mode.driving
result.duration = 42
result.weight = 35
end
return functions
"""
Given the node map
"""
a---b---c---d---e---f---g---h
"""
And the ways
| nodes |
| abcdef |
| abcdefgh |
When I route I should get
| waypoints | route | distance | weights | times |
| a,f | , | 100m | 25,0 | 30s,0s |
| f,a | , | 100m | 25,0 | 30s,0s |
| a,h | , | 140m +-1 | 35,0 | 42s,0s |
| h,a | , | 140m +-1 | 35,0 | 42s,0s |
Scenario: Step weights -- way_function: higher weight_per_meter is preferred
Given the profile file
"""
local functions = require('testbot')
functions.setup_testbot = functions.setup
functions.setup = function()
local profile = functions.setup_testbot()
profile.properties.traffic_signal_penalty = 0
profile.properties.u_turn_penalty = 0
profile.properties.weight_name = 'steps'
return profile
end
functions.process_way = function(profile, way, result)
result.forward_mode = mode.driving
result.backward_mode = mode.driving
result.duration = 42
result.forward_rate = 1
result.backward_rate = 0.5
end
return functions
"""
Given the node map
"""
a---b---c---d---e---f---g---h
"""
And the ways
| nodes |
| abcdefgh |
| abcdef |
| fgh |
When I route I should get
| waypoints | route | distance | weights | times |
| a,f | , | 100m | 99.8,0 | 30s,0s |
| f,a | , | 100m | 199.9,0 | 30s,0s |
| a,h | , | 140m | 139.8,0 | 42s,0s |
| h,a | , | 140m | 280.1,0 | 42s,0s |
| f,h | , | 40.1m | 40,0 | 12s,0s |
| h,f | , | 40.1m | 80.2,0 | 12s,0s |
Scenario: Step weights -- segment_function
Given the profile file
"""
local functions = require('testbot')
functions.setup_testbot = functions.setup
functions.setup = function()
local profile = functions.setup_testbot()
profile.properties.traffic_signal_penalty = 0
profile.properties.u_turn_penalty = 0
profile.properties.weight_name = 'steps'
return profile
end
functions.process_way = function(profile, way, result)
result.forward_mode = mode.driving
result.backward_mode = mode.driving
result.weight = 42
result.duration = 3
end
functions.process_segment = function(profile, segment)
segment.weight = 1
segment.duration = 11
end
return functions
"""
Given the node map
"""
a---b---c---d---e---f---g---h
"""
And the ways
| nodes |
| abcdefgh |
| abcdef |
| fgh |
When I route I should get
| waypoints | route | distance | weights | times |
| a,f | , | 100m | 5,0 | 55s,0s |
| f,a | , | 100m | 5,0 | 55s,0s |
| a,h | , | 140m +-1 | 7,0 | 77s,0s |
| h,a | , | 140m +-1 | 7,0 | 77s,0s |
| f,h | , | 40m +-1 | 2,0 | 22s,0s |
| h,f | , | 40m +-1 | 2,0 | 22s,0s |
Scenario: Step weights -- segment_function and turn_function with weight precision
Given the profile file
"""
local functions = require('testbot')
functions.setup_testbot = functions.setup
functions.setup = function()
local profile = functions.setup_testbot()
profile.properties.traffic_signal_penalty = 0
profile.properties.u_turn_penalty = 0
profile.properties.weight_name = 'steps'
profile.properties.weight_precision = 3
return profile
end
functions.process_way = function(profile, way, result)
result.forward_mode = mode.driving
result.backward_mode = mode.driving
result.weight = 42
result.duration = 3
end
functions.process_segment = function(profile, segment)
segment.weight = 1.11
segment.duration = 100
end
functions.process_turn = function(profile, turn)
print (turn.angle)
turn.weight = 2 + turn.angle / 100
turn.duration = turn.angle
end
return functions
"""
Given the node map
"""
a---b---c---d
⋮
e
"""
And the ways
| nodes |
| abcd |
| ce |
When I route I should get
| waypoints | route | distance | weights | times |
| a,c | , | 40m +-.1 | 2.22,0 | 200s,0s |
| a,e | ,, | 60m +-.1 | 5.12,1.11,0 | 290s,100s,0s |
| e,a | ,, | 60m +-.1 | 2.21,2.22,0 | 10s,200s,0s |
| e,d | ,, | 40m +-.1 | 4.01,1.11,0 | 190s,100s,0s |
| d,e | ,, | 40m +-.1 | 2.21,1.11,0 | 10s,100s,0s |
@traffic @speed
Scenario: Step weights -- segment_function with speed and turn updates
Given the profile file
@ -327,17 +40,17 @@ Feature: Weight tests
| nodes |
| abcd |
| ce |
And the speed file
And the parquet speed file
"""
1,2,36.999,42
2,1,36,42
"""
And the turn penalty file
And the parquet turn penalty file
"""
2,3,5,25.5,16.7
"""
And the contract extra arguments "--segment-speed-file {speeds_file} --turn-penalty-file {penalties_file}"
And the customize extra arguments "--segment-speed-file {speeds_file} --turn-penalty-file {penalties_file}"
And the contract extra arguments "--speed-and-turn-penalty-format parquet --segment-speed-file {speeds_file} --turn-penalty-file {penalties_file}"
And the customize extra arguments "--speed-and-turn-penalty-format parquet --segment-speed-file {speeds_file} --turn-penalty-file {penalties_file}"
When I route I should get
| waypoints | route | distance | weights | times |
@ -345,70 +58,4 @@ Feature: Weight tests
| a,e | ,, | 60m | 27.2,10,0 | 38.4s,11s,0s |
| d,e | ,, | 40m | 10,10,0 | 11s,11s,0s |
@traffic @speed
Scenario: Step weights -- segment_function with speed and turn updates with fallback to durations
Given the profile file "testbot" initialized with
"""
profile.properties.weight_precision = 3
"""
And the node map
"""
a---b---c---d
.
e
"""
And the ways
| nodes |
| abcd |
| ce |
And the speed file
"""
1,2,24
2,1,24
"""
And the turn penalty file
"""
2,3,5,1
"""
And the contract extra arguments "--segment-speed-file {speeds_file} --turn-penalty-file {penalties_file}"
And the customize extra arguments "--segment-speed-file {speeds_file} --turn-penalty-file {penalties_file}"
When I route I should get
| waypoints | route | distance | weights | times |
| a,d | abcd,abcd | 60m | 7,0 | 7s,0s |
| a,e | abcd,ce,ce | 60m | 5.997,2.001,0 | 6s,2s,0s |
| d,e | abcd,ce,ce | 40m | 2.003,2.001,0 | 2s,2s,0s |
@traffic @speed
Scenario: Updating speeds without affecting weights.
Given the profile file "testbot" initialized with
"""
profile.properties.weight_precision = 3
"""
And the node map
"""
a-----------b
\ /
c----d
"""
And the ways
| nodes | highway | maxspeed |
| ab | living_street | 5 |
| acdb | motorway | 100 |
# Note the comma on the last column - this indicates 'keep existing weight value'
And the speed file
"""
1,2,100,
1,3,5,,junk
3,4,5,,
4,2,5,
"""
And the contract extra arguments "--segment-speed-file {speeds_file}"
And the customize extra arguments "--segment-speed-file {speeds_file}"
When I route I should get
| waypoints | route | distance | weights | times |
| a,b | acdb,acdb | 78.3m | 11.742,0 | 56.4s,0s |

12
include/updater/csv_source.hpp → include/updater/data_source.hpp
View File

@ -1,5 +1,5 @@
#ifndef OSRM_UPDATER_CSV_SOURCE_HPP
#define OSRM_UPDATER_CSV_SOURCE_HPP
#ifndef OSRM_UPDATER_DATA_SOURCE_HPP
#define OSRM_UPDATER_DATA_SOURCE_HPP
#include "updater/source.hpp"
#include "updater/updater_config.hpp"
@ -8,11 +8,11 @@ namespace osrm
{
namespace updater
{
namespace csv
namespace data
{
SegmentLookupTable readSegmentValues(const std::vector<std::string> &paths, UpdaterConfig::SpeedAndTurnPenaltyFormat format);
TurnLookupTable readTurnValues(const std::vector<std::string> &paths, UpdaterConfig::SpeedAndTurnPenaltyFormat format);
} // namespace csv
SegmentLookupTable readSegmentValues(const std::vector<std::string> &paths, SpeedAndTurnPenaltyFormat format);
TurnLookupTable readTurnValues(const std::vector<std::string> &paths, SpeedAndTurnPenaltyFormat format);
} // namespace data
} // namespace updater
} // namespace osrm

12
include/updater/file_parser.hpp
View File

@ -41,16 +41,6 @@ template <typename Key, typename Value> struct FilesParser
// Operator returns a lambda function that maps input Key to boost::optional<Value>.
auto operator()(const std::vector<std::string> &csv_filenames) const
{
// std::shared_ptr<arrow::io::ReadableFile> infile;
// PARQUET_ASSIGN_OR_THROW(
// infile,
// arrow::io::ReadableFile::Open("test.parquet"));
// parquet::StreamReader os{parquet::ParquetFileReader::Open(infile)};
// (void)os;
try
{
tbb::spin_mutex mutex;
@ -59,7 +49,7 @@ template <typename Key, typename Value> struct FilesParser
// TODO: do we need this `1 + ` here?
auto local = ParseFile(csv_filenames[idx], 1 + idx);
{ // Merge local CSV results into a flat global vector
{ // Merge local results into a flat global vector
tbb::spin_mutex::scoped_lock _{mutex};
lookup.insert(end(lookup),
std::make_move_iterator(begin(local)),

77
include/updater/parquet_file_parser.hpp
View File

@ -3,6 +3,7 @@
#include "file_parser.hpp"
#include <optional>
#include <parquet/arrow/reader.h>
#include <parquet/exception.h>
#include <parquet/stream_reader.h>
#include <arrow/io/file.h>
@ -26,71 +27,73 @@ namespace osrm
namespace updater
{
// Functor to parse a list of CSV files using "key,value,comment" grammar.
// Key and Value structures must be a model of Random Access Sequence.
// Also the Value structure must have source member that will be filled
// with the corresponding file index in the CSV filenames vector.
template <typename Key, typename Value> struct ParquetFilesParser : public FilesParser<Key, Value>
{
private:
// Parse a single CSV file and return result as a vector<Key, Value>
// Parse a single Parquet file and return result as a vector<Key, Value>
std::vector<std::pair<Key, Value>> ParseFile(const std::string &filename, std::size_t file_id) const final
{
// TODO: error handling
std::shared_ptr<arrow::io::ReadableFile> infile;
try {
std::shared_ptr<arrow::io::ReadableFile> infile;
PARQUET_ASSIGN_OR_THROW(
infile,
arrow::io::ReadableFile::Open(filename));
PARQUET_ASSIGN_OR_THROW(
infile,
arrow::io::ReadableFile::Open(filename));
parquet::StreamReader os{parquet::ParquetFileReader::Open(infile)};
parquet::StreamReader os{parquet::ParquetFileReader::Open(infile)};
std::vector<std::pair<Key, Value>> result;
while ( !os.eof() )
{
result.emplace_back(ReadKey(os), ReadValue(os, file_id));
// ...
std::vector<std::pair<Key, Value>> result;
while ( !os.eof() )
{
result.emplace_back(ReadKeyValue(os, file_id));
}
return result;
} catch (const std::exception &e) {
throw util::exception(e.what() + SOURCE_REF);
}
return result;
// (void)os;
}
Key ReadKey(parquet::StreamReader &os) const
std::pair<Key, Value> ReadKeyValue(parquet::StreamReader &os, std::size_t file_id) const
{
Key key;
ReadKey(os, key);
return key;
}
Value ReadValue(parquet::StreamReader &os, std::size_t file_id) const
{
Value value;
ReadValue(os, value);
Read(os, key);
Read(os, value);
value.source = file_id;
return value;
os >> parquet::EndRow;
return {key, value};
}
void ReadKey(parquet::StreamReader &os, Turn& turn) const {
os >> turn.from >> turn.via >> turn.to >> parquet::EndRow;
void Read(parquet::StreamReader &os, Turn& turn) const {
int64_t from, via, to;
os >> from >> via >> to;
turn.from = from;
turn.via = via;
turn.to = to;
}
void ReadValue(parquet::StreamReader &os, PenaltySource& penalty_source) const {
os >> penalty_source.duration >> penalty_source.weight >> parquet::EndRow;
void Read(parquet::StreamReader &os, PenaltySource& penalty_source) const {
os >> penalty_source.duration >> penalty_source.weight;
}
void ReadKey(parquet::StreamReader &os, Segment& segment) const {
os >> segment.from >> segment.to >> parquet::EndRow;
void Read(parquet::StreamReader &os, Segment& segment) const {
int64_t from;
int64_t to;
os >> from >> to;
segment.from = from;
segment.to = to;
//std::cerr << from << " " << to<< std::endl;
//os >> segment.from >> segment.to >> parquet::EndRow;
}
void ReadValue(parquet::StreamReader &os, SpeedSource& speed_source) const {
void Read(parquet::StreamReader &os, SpeedSource& speed_source) const {
std::optional<double> rate;
os >> speed_source.speed >> rate >> parquet::EndRow;
os >> speed_source.speed >> rate;
// TODO: boost::optional
if (rate) {
speed_source.rate = *rate;
}
// os >> turn.weight >> turn.duration >> turn.pre_turn_bearing >> turn.post_turn_bearing >> turn.source >> parquet::EndRow;
}
};

46
include/updater/updater_config.hpp
View File

@ -35,12 +35,19 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "storage/io_config.hpp"
#include "storage/storage_config.hpp"
#include <boost/program_options.hpp>
namespace osrm
{
namespace updater
{
enum class SpeedAndTurnPenaltyFormat {
CSV,
PARQUET
};
struct UpdaterConfig final : storage::IOConfig
{
UpdaterConfig()
@ -69,16 +76,45 @@ struct UpdaterConfig final : storage::IOConfig
double log_edge_updates_factor = 0.0;
std::time_t valid_now;
enum class SpeedAndTurnPenaltyFormat {
CSV,
PARQUET
} speed_and_turn_penalty_format = SpeedAndTurnPenaltyFormat::CSV;
SpeedAndTurnPenaltyFormat speed_and_turn_penalty_format = SpeedAndTurnPenaltyFormat::CSV;
std::vector<std::string> segment_speed_lookup_paths;
std::vector<std::string> turn_penalty_lookup_paths;
std::string tz_file_path;
};
inline std::istream& operator>> (std::istream &in, SpeedAndTurnPenaltyFormat& format) {
std::string token;
in >> token;
std::transform(token.begin(), token.end(), token.begin(), [](auto c){ return std::tolower(c); });
if (token == "csv") {
format = SpeedAndTurnPenaltyFormat::CSV;
} else if (token == "parquet") {
format = SpeedAndTurnPenaltyFormat::PARQUET;
} else {
throw boost::program_options::validation_error{boost::program_options::validation_error::invalid_option_value};
}
return in;
}
inline std::ostream& operator<< (std::ostream &out, SpeedAndTurnPenaltyFormat format) {
switch (format) {
case SpeedAndTurnPenaltyFormat::CSV:
out << "csv";
break;
case SpeedAndTurnPenaltyFormat::PARQUET:
out << "parquet";
break;
}
return out;
}
} // namespace updater
} // namespace osrm

210
package-lock.json generated
View File

@ -10,7 +10,8 @@
"hasInstallScript": true,
"license": "BSD-2-Clause",
"dependencies": {
"@mapbox/node-pre-gyp": "^1.0.10"
"@mapbox/node-pre-gyp": "^1.0.10",
"parquetjs": "^0.11.2"
},
"devDependencies": {
"@babel/cli": "^7.18.10",
@ -4286,7 +4287,6 @@
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
"dev": true,
"funding": [
{
"type": "github",
@ -4423,6 +4423,14 @@
"integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=",
"dev": true
},
"node_modules/brotli": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.3.tgz",
"integrity": "sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg==",
"dependencies": {
"base64-js": "^1.1.2"
}
},
"node_modules/browser-pack": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/browser-pack/-/browser-pack-6.1.0.tgz",
@ -4777,6 +4785,14 @@
"integrity": "sha512-3pwjAK0qHSDN9+YAF4fJknsSruP7mpjdWzUSruIJD/JCH77pEh0SorEyb3xVaKkfwk2tzjOt2D8scJ0KAdfXLA==",
"dev": true
},
"node_modules/bson": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz",
"integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg==",
"engines": {
"node": ">=0.6.19"
}
},
"node_modules/buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.2.1.tgz",
@ -10856,6 +10872,11 @@
"insert-module-globals": "bin/cmd.js"
}
},
"node_modules/int53": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/int53/-/int53-0.2.4.tgz",
"integrity": "sha512-a5jlKftS7HUOhkUyYD7j2sJ/ZnvWiNlZS1ldR+g1ifQ+/UuZXIE+YTc/lK1qGj/GwAU5F8Z0e1eVq2t1J5Ob2g=="
},
"node_modules/internal-slot": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.3.tgz",
@ -12020,6 +12041,22 @@
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
},
"node_modules/lzo": {
"version": "0.4.11",
"resolved": "https://registry.npmjs.org/lzo/-/lzo-0.4.11.tgz",
"integrity": "sha512-apQHNoW2Alg72FMqaC/7pn03I7umdgSVFt2KRkCXXils4Z9u3QBh1uOtl2O5WmZIDLd9g6Lu4lIdOLmiSTFVCQ==",
"hasInstallScript": true,
"optional": true,
"dependencies": {
"bindings": "~1.2.1"
}
},
"node_modules/lzo/node_modules/bindings": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz",
"integrity": "sha512-u4cBQNepWxYA55FunZSM7wMi55yQaN0otnhhilNoWHq0MfOfJeQx0v0mRRpolGOExPjZcl6FtB0BB8Xkb88F0g==",
"optional": true
},
"node_modules/magic-string": {
"version": "0.22.5",
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz",
@ -12772,6 +12809,11 @@
}
}
},
"node_modules/node-int64": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz",
"integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw=="
},
"node_modules/node-releases": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.6.tgz",
@ -12960,6 +13002,14 @@
"node": ">= 0.4"
}
},
"node_modules/object-stream": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/object-stream/-/object-stream-0.0.1.tgz",
"integrity": "sha512-+NPJnRvX9RDMRY9mOWOo/NDppBjbZhXirNNSu2IBnuNboClC9h1ZGHXgHBLDbJMHsxeJDq922aVmG5xs24a/cA==",
"engines": {
"node": ">=0.10"
}
},
"node_modules/object-visit": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz",
@ -13275,6 +13325,26 @@
"path-platform": "~0.11.15"
}
},
"node_modules/parquetjs": {
"version": "0.11.2",
"resolved": "https://registry.npmjs.org/parquetjs/-/parquetjs-0.11.2.tgz",
"integrity": "sha512-Y6FOc3Oi2AxY4TzJPz7fhICCR8tQNL3p+2xGQoUAMbmlJBR7+JJmMrwuyMjIpDiM7G8Wj/8oqOH4UDUmu4I5ZA==",
"dependencies": {
"brotli": "^1.3.0",
"bson": "^1.0.4",
"int53": "^0.2.4",
"object-stream": "0.0.1",
"snappyjs": "^0.6.0",
"thrift": "^0.11.0",
"varint": "^5.0.0"
},
"engines": {
"node": ">=7.6"
},
"optionalDependencies": {
"lzo": "^0.4.0"
}
},
"node_modules/parse-asn1": {
"version": "5.1.6",
"resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz",
@ -14102,7 +14172,6 @@
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz",
"integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=",
"dev": true,
"engines": {
"node": ">=0.6.0",
"teleport": ">=0.2.0"
@ -15345,6 +15414,11 @@
"node": ">=0.10.0"
}
},
"node_modules/snappyjs": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/snappyjs/-/snappyjs-0.6.1.tgz",
"integrity": "sha512-YIK6I2lsH072UE0aOFxxY1dPDCS43I5ktqHpeAsuLNYWkE5pGxRGWfDM4/vSUfNzXjC1Ivzt3qx31PCLmc9yqg=="
},
"node_modules/sort-keys": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.2.tgz",
@ -16950,6 +17024,19 @@
"integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=",
"dev": true
},
"node_modules/thrift": {
"version": "0.11.0",
"resolved": "https://registry.npmjs.org/thrift/-/thrift-0.11.0.tgz",
"integrity": "sha512-UpsBhOC45a45TpeHOXE4wwYwL8uD2apbHTbtBvkwtUU4dNwCjC7DpQTjw2Q6eIdfNtw+dKthdwq94uLXTJPfFw==",
"dependencies": {
"node-int64": "^0.4.0",
"q": "^1.5.0",
"ws": ">= 2.2.3"
},
"engines": {
"node": ">= 4.1.0"
}
},
"node_modules/through": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
@ -18417,6 +18504,11 @@
"spdx-expression-parse": "~1.0.0"
}
},
"node_modules/varint": {
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz",
"integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow=="
},
"node_modules/vendors": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/vendors/-/vendors-1.0.4.tgz",
@ -18800,6 +18892,26 @@
"node": ">=0.10.0"
}
},
"node_modules/ws": {
"version": "8.11.0",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
"integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==",
"engines": {
"node": ">=10.0.0"
},
"peerDependencies": {
"bufferutil": "^4.0.1",
"utf-8-validate": "^5.0.2"
},
"peerDependenciesMeta": {
"bufferutil": {
"optional": true
},
"utf-8-validate": {
"optional": true
}
}
},
"node_modules/x-is-string": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/x-is-string/-/x-is-string-0.1.0.tgz",
@ -22262,8 +22374,7 @@
"base64-js": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
"dev": true
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="
},
"bcrypt-pbkdf": {
"version": "1.0.2",
@ -22376,6 +22487,14 @@
"integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=",
"dev": true
},
"brotli": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.3.tgz",
"integrity": "sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg==",
"requires": {
"base64-js": "^1.1.2"
}
},
"browser-pack": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/browser-pack/-/browser-pack-6.1.0.tgz",
@ -22658,6 +22777,11 @@
}
}
},
"bson": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz",
"integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg=="
},
"buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.2.1.tgz",
@ -27614,6 +27738,11 @@
"xtend": "^4.0.0"
}
},
"int53": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/int53/-/int53-0.2.4.tgz",
"integrity": "sha512-a5jlKftS7HUOhkUyYD7j2sJ/ZnvWiNlZS1ldR+g1ifQ+/UuZXIE+YTc/lK1qGj/GwAU5F8Z0e1eVq2t1J5Ob2g=="
},
"internal-slot": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.3.tgz",
@ -28504,6 +28633,23 @@
}
}
},
"lzo": {
"version": "0.4.11",
"resolved": "https://registry.npmjs.org/lzo/-/lzo-0.4.11.tgz",
"integrity": "sha512-apQHNoW2Alg72FMqaC/7pn03I7umdgSVFt2KRkCXXils4Z9u3QBh1uOtl2O5WmZIDLd9g6Lu4lIdOLmiSTFVCQ==",
"optional": true,
"requires": {
"bindings": "~1.2.1"
},
"dependencies": {
"bindings": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz",
"integrity": "sha512-u4cBQNepWxYA55FunZSM7wMi55yQaN0otnhhilNoWHq0MfOfJeQx0v0mRRpolGOExPjZcl6FtB0BB8Xkb88F0g==",
"optional": true
}
}
},
"magic-string": {
"version": "0.22.5",
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz",
@ -29134,6 +29280,11 @@
"whatwg-url": "^5.0.0"
}
},
"node-int64": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz",
"integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw=="
},
"node-releases": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.6.tgz",
@ -29282,6 +29433,11 @@
"integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==",
"dev": true
},
"object-stream": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/object-stream/-/object-stream-0.0.1.tgz",
"integrity": "sha512-+NPJnRvX9RDMRY9mOWOo/NDppBjbZhXirNNSu2IBnuNboClC9h1ZGHXgHBLDbJMHsxeJDq922aVmG5xs24a/cA=="
},
"object-visit": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz",
@ -29530,6 +29686,21 @@
"path-platform": "~0.11.15"
}
},
"parquetjs": {
"version": "0.11.2",
"resolved": "https://registry.npmjs.org/parquetjs/-/parquetjs-0.11.2.tgz",
"integrity": "sha512-Y6FOc3Oi2AxY4TzJPz7fhICCR8tQNL3p+2xGQoUAMbmlJBR7+JJmMrwuyMjIpDiM7G8Wj/8oqOH4UDUmu4I5ZA==",
"requires": {
"brotli": "^1.3.0",
"bson": "^1.0.4",
"int53": "^0.2.4",
"lzo": "^0.4.0",
"object-stream": "0.0.1",
"snappyjs": "^0.6.0",
"thrift": "^0.11.0",
"varint": "^5.0.0"
}
},
"parse-asn1": {
"version": "5.1.6",
"resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz",
@ -30248,8 +30419,7 @@
"q": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz",
"integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=",
"dev": true
"integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc="
},
"qs": {
"version": "6.10.1",
@ -31271,6 +31441,11 @@
"kind-of": "^3.2.0"
}
},
"snappyjs": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/snappyjs/-/snappyjs-0.6.1.tgz",
"integrity": "sha512-YIK6I2lsH072UE0aOFxxY1dPDCS43I5ktqHpeAsuLNYWkE5pGxRGWfDM4/vSUfNzXjC1Ivzt3qx31PCLmc9yqg=="
},
"sort-keys": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.2.tgz",
@ -32519,6 +32694,16 @@
"integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=",
"dev": true
},
"thrift": {
"version": "0.11.0",
"resolved": "https://registry.npmjs.org/thrift/-/thrift-0.11.0.tgz",
"integrity": "sha512-UpsBhOC45a45TpeHOXE4wwYwL8uD2apbHTbtBvkwtUU4dNwCjC7DpQTjw2Q6eIdfNtw+dKthdwq94uLXTJPfFw==",
"requires": {
"node-int64": "^0.4.0",
"q": "^1.5.0",
"ws": ">= 2.2.3"
}
},
"through": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
@ -33771,6 +33956,11 @@
"spdx-expression-parse": "~1.0.0"
}
},
"varint": {
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz",
"integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow=="
},
"vendors": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/vendors/-/vendors-1.0.4.tgz",
@ -34085,6 +34275,12 @@
"mkdirp": "^0.5.1"
}
},
"ws": {
"version": "8.11.0",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
"integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==",
"requires": {}
},
"x-is-string": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/x-is-string/-/x-is-string-0.1.0.tgz",

11
package.json
View File

@ -4,7 +4,8 @@
"private": false,
"description": "The Open Source Routing Machine is a high performance routing engine written in C++ designed to run on OpenStreetMap data.",
"dependencies": {
"@mapbox/node-pre-gyp": "^1.0.10"
"@mapbox/node-pre-gyp": "^1.0.10",
"parquetjs": "^0.11.2"
},
"browserify": {
"transform": [
@ -58,6 +59,7 @@
"jsonpath": "^1.1.1",
"mkdirp": "^0.5.6",
"node-addon-api": "^5.0.0",
"node-cmake": "^2.5.1",
"node-timeout": "0.0.4",
"polyline": "^0.2.0",
"request": "^2.88.2",
@ -65,12 +67,13 @@
"tape": "^4.16.0",
"turf": "^3.0.14",
"uglify-js": "^3.17.0",
"xmlbuilder": "^4.2.1",
"node-cmake": "^2.5.1"
"xmlbuilder": "^4.2.1"
},
"main": "lib/index.js",
"binary": {
"napi_versions": [8],
"napi_versions": [
8
],
"module_name": "node_osrm",
"module_path": "./lib/binding_napi_v{napi_build_version}/",
"host": "https://github.com",

2
src/tools/contract.cpp
View File

@ -49,7 +49,7 @@ return_code parseArguments(int argc,
"core,k",
boost::program_options::value<double>(&contractor_config.core_factor)->default_value(1.0),
"DEPRECATED: Will always be 1.0. Percentage of the graph (in vertices) to contract "
"[0..1].")("segment-speed-file",
"[0..1].") ("speed-and-turn-penalty-format", boost::program_options::value<updater::SpeedAndTurnPenaltyFormat>(&contractor_config.updater_config.speed_and_turn_penalty_format)->default_value(updater::SpeedAndTurnPenaltyFormat::CSV))("segment-speed-file",
boost::program_options::value<std::vector<std::string>>(
&contractor_config.updater_config.segment_speed_lookup_paths)
->composing(),

5
src/tools/customize.cpp
View File

@ -1,4 +1,5 @@
#include "customizer/customizer.hpp"
#include "updater/updater_config.hpp"
#include "osrm/exception.hpp"
#include "util/log.hpp"
@ -20,6 +21,7 @@ enum class return_code : unsigned
exit
};
return_code parseArguments(int argc,
char *argv[],
std::string &verbosity,
@ -39,7 +41,8 @@ return_code parseArguments(int argc,
("threads,t",
boost::program_options::value<unsigned int>(&customization_config.requested_num_threads)
->default_value(std::thread::hardware_concurrency()),
"Number of threads to use")(
"Number of threads to use")
("speed-and-turn-penalty-format", boost::program_options::value<updater::SpeedAndTurnPenaltyFormat>(&customization_config.updater_config.speed_and_turn_penalty_format)->default_value(updater::SpeedAndTurnPenaltyFormat::CSV))(
"segment-speed-file",
boost::program_options::value<std::vector<std::string>>(
&customization_config.updater_config.segment_speed_lookup_paths)

22
src/updater/csv_source.cpp → src/updater/data_source.cpp
View File

@ -1,4 +1,4 @@
#include "updater/csv_source.hpp"
#include "updater/data_source.hpp"
#include "updater/csv_file_parser.hpp"
#include "updater/parquet_file_parser.hpp"
@ -31,40 +31,40 @@ namespace osrm
{
namespace updater
{
namespace csv
namespace data
{
namespace {
std::unique_ptr<FilesParser<Segment, SpeedSource>> makeSegmentParser(UpdaterConfig::SpeedAndTurnPenaltyFormat format) {
std::unique_ptr<FilesParser<Segment, SpeedSource>> makeSegmentParser(SpeedAndTurnPenaltyFormat format) {
switch (format) {
case UpdaterConfig::SpeedAndTurnPenaltyFormat::CSV:
case SpeedAndTurnPenaltyFormat::CSV:
{
static const auto value_if_blank = std::numeric_limits<double>::quiet_NaN();
const qi::real_parser<double, qi::ureal_policies<double>> unsigned_double;
return std::make_unique<CSVFilesParser<Segment, SpeedSource>>(qi::ulong_long >> ',' >> qi::ulong_long,
unsigned_double >> -(',' >> (qi::double_ | qi::attr(value_if_blank))));
}
case UpdaterConfig::SpeedAndTurnPenaltyFormat::PARQUET:
case SpeedAndTurnPenaltyFormat::PARQUET:
return std::make_unique<ParquetFilesParser<Segment, SpeedSource>>();
}
}
std::unique_ptr<FilesParser<Turn, PenaltySource>> makeTurnParser(UpdaterConfig::SpeedAndTurnPenaltyFormat format) {
std::unique_ptr<FilesParser<Turn, PenaltySource>> makeTurnParser(SpeedAndTurnPenaltyFormat format) {
switch (format) {
case UpdaterConfig::SpeedAndTurnPenaltyFormat::CSV:
case SpeedAndTurnPenaltyFormat::CSV:
{
return std::make_unique<CSVFilesParser<Turn, PenaltySource>>(qi::ulong_long >> ',' >> qi::ulong_long >> ',' >>
qi::ulong_long,
qi::double_ >> -(',' >> qi::double_));
}
case UpdaterConfig::SpeedAndTurnPenaltyFormat::PARQUET:
case SpeedAndTurnPenaltyFormat::PARQUET:
return std::make_unique<ParquetFilesParser<Turn, PenaltySource>>();
}
}
} // namespace
SegmentLookupTable readSegmentValues(const std::vector<std::string> &paths, UpdaterConfig::SpeedAndTurnPenaltyFormat format)
SegmentLookupTable readSegmentValues(const std::vector<std::string> &paths, SpeedAndTurnPenaltyFormat format)
{
auto parser = makeSegmentParser(format);
@ -83,11 +83,11 @@ SegmentLookupTable readSegmentValues(const std::vector<std::string> &paths, Upda
return result;
}
TurnLookupTable readTurnValues(const std::vector<std::string> &paths, UpdaterConfig::SpeedAndTurnPenaltyFormat format)
TurnLookupTable readTurnValues(const std::vector<std::string> &paths, SpeedAndTurnPenaltyFormat format)
{
auto parser = makeTurnParser(format);
return (*parser)(paths);
}
} // namespace csv
} // namespace data
} // namespace updater
} // namespace osrm

6
src/updater/updater.cpp
View File

@ -1,5 +1,5 @@
#include "updater/updater.hpp"
#include "updater/csv_source.hpp"
#include "updater/data_source.hpp"
#include "extractor/compressed_edge_container.hpp"
#include "extractor/edge_based_graph_factory.hpp"
@ -618,7 +618,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
tbb::concurrent_vector<GeometryID> updated_segments;
if (update_edge_weights)
{
auto segment_speed_lookup = csv::readSegmentValues(config.segment_speed_lookup_paths, config.speed_and_turn_penalty_format);
auto segment_speed_lookup = data::readSegmentValues(config.segment_speed_lookup_paths, config.speed_and_turn_penalty_format);
TIMER_START(segment);
updated_segments = updateSegmentData(config,
@ -633,7 +633,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
util::Log() << "Updating segment data took " << TIMER_MSEC(segment) << "ms.";
}
auto turn_penalty_lookup = csv::readTurnValues(config.turn_penalty_lookup_paths, config.speed_and_turn_penalty_format);
auto turn_penalty_lookup = data::readTurnValues(config.turn_penalty_lookup_paths, config.speed_and_turn_penalty_format);
if (update_turn_penalties)
{
auto updated_turn_penalties = updateTurnPenalties(config,

13
unit_tests/updater/parquet.cpp Normal file
View File

@ -0,0 +1,13 @@
#include <boost/test/tools/old/interface.hpp>
#include <updater/data_source.hpp>
#include <boost/test/unit_test.hpp>
using namespace osrm;
using namespace osrm::updater;
BOOST_AUTO_TEST_CASE(parquet_readSegmentValues)
{
boost::filesystem::path test_path(TEST_DATA_DIR "/speeds_file.parquet");
SegmentLookupTable segment_lookup_table = data::readSegmentValues({test_path.string()}, SpeedAndTurnPenaltyFormat::PARQUET);
BOOST_CHECK_EQUAL(segment_lookup_table.lookup.size(), 2);
}

BIN
unit_tests/updater/speeds_file.parquet Normal file
View File

Binary file not shown.