diff --git a/features/step_definitions/data.js b/features/step_definitions/data.js index 0d9b76e8a..cb57eb04a 100644 --- a/features/step_definitions/data.js +++ b/features/step_definitions/data.js @@ -5,6 +5,7 @@ var path = require('path'); var fs = require('fs'); var d3 = require('d3-queue'); var OSM = require('../lib/osm'); +const parquet = require('parquetjs'); module.exports = function () { this.Given(/^the profile "([^"]*)"$/, (profile, callback) => { @@ -257,6 +258,61 @@ module.exports = function () { this.environment = Object.assign({OSRM_RASTER_SOURCE: this.rasterCacheFile}, this.environment); }); + this.Given(/^the parquet speed file$/, (data, callback) => { + console.log(this.speedsCacheFile); + const schema = new parquet.ParquetSchema({ + from: { type: 'INT64', encoding: 'PLAIN' }, + to: { type: 'INT64', encoding: 'PLAIN' }, + speed: { type: 'DOUBLE', encoding: 'PLAIN' }, + rate: { type: 'DOUBLE', encoding: 'PLAIN', optional: true }, + }); + parquet.ParquetWriter.openFile(schema, this.speedsCacheFile).then((writer) => { + const promises = []; + data.split('\n').forEach((line) => { + const entries = line.split(','); + const from = Number.parseInt(entries[0]); + const to = Number.parseInt(entries[1]); + const speed = Number.parseFloat(entries[2]); + const rate = entries[3] ? Number.parseFloat(entries[3]) : undefined; + console.log('writing', from, to, speed, rate); + promises.push(writer.appendRow({ from, to, speed, rate })); + }); + Promise.all(promises).then(() => { + writer.close().then(() => { + callback(); + }); + }); + }); + }); + + this.Given(/^the parquet turn penalty file$/, (data, callback) => { + const schema = new parquet.ParquetSchema({ + from: { type: 'INT64' }, + via: { type: 'INT64' }, + to: { type: 'INT64' }, + duration: { type: 'DOUBLE' }, + weight: { type: 'DOUBLE' }, + }); + parquet.ParquetWriter.openFile(schema, this.penaltiesCacheFile).then((writer) => { + const promises = []; + data.split('\n').forEach((line) => { + const entries = line.split(','); + const from = Number.parseInt(entries[0]); + const via = Number.parseInt(entries[1]); + const to = Number.parseInt(entries[2]); + const duration = Number.parseFloat(entries[3]); + const weight = Number.parseFloat(entries[4]); + promises.push(writer.appendRow({ from, via, to, duration, weight })); + }); + Promise.all(promises).then(() => { + writer.close().then(() => { + callback(); + }); + }); + }); + }); + + this.Given(/^the speed file$/, (data, callback) => { // TODO: Don't overwrite if it exists fs.writeFile(this.speedsCacheFile, data, callback); diff --git a/features/testbot/weight.feature b/features/testbot/weight.feature index d5ae35206..e6a8c95f1 100644 --- a/features/testbot/weight.feature +++ b/features/testbot/weight.feature @@ -1,292 +1,5 @@ @testbot Feature: Weight tests - - Background: - Given the profile "testbot" - Given a grid size of 10 meters - Given the query options - | geometries | geojson | - - Scenario: Weight details - Given the query options - | annotations | weight | - - Given the node map - """ - s - · - a---b---c - | - d - |··t - e - """ - - And the ways - | nodes | - | abc | - | cde | - - When I route I should get - | waypoints | route | a:weight | - | s,t | abc,cde | 1.1:2:2:0.9 | - - When I route I should get - | waypoints | route | times | weight_name | weights | - | s,t | abc,cde | 6s,0s | duration | 6,0 | - - # FIXME include/engine/guidance/assemble_geometry.hpp:95 - Scenario: Start and target on the same and adjacent edge - Given the query options - | annotations | distance,duration,weight,nodes,speed | - - Given the node map - """ - a-------b-------c - · · · - s t e - """ - - And the ways - | nodes | - | abc | - - When I route I should get - | waypoints | route | distances | weights | times | a:distance | a:duration | a:weight | a:speed | - | s,t | abc,abc | 20m,0m | 2,0 | 2s,0s | 20.034626629 | 2 | 2 | 10 | - | t,s | abc,abc | 20m,0m | 2,0 | 2s,0s | 20.034626629 | 2 | 2 | 10 | - | s,e | abc,abc | 40m,0m | 3.9,0 | 3.9s,0s | 29.940636463:10.017313314 | 3:0.9 | 3:0.9 | 10:11.1 | - | e,s | abc,abc | 40m,0m | 3.9,0 | 3.9s,0s | 10.017313314:29.940636463 | 0.9:3 | 0.9:3 | 11.1:10 | - - - Scenario: Step weights -- way_function: fail if no weight or weight_per_meter property - Given the profile file - """ - local functions = require('testbot') - functions.setup_testbot = functions.setup - - functions.setup = function() - local profile = functions.setup_testbot() - profile.properties.traffic_signal_penalty = 0 - profile.properties.u_turn_penalty = 0 - profile.properties.weight_name = 'steps' - return profile - end - - functions.process_way = function(profile, way, result) - result.forward_mode = mode.driving - result.backward_mode = mode.driving - result.forward_speed = 42 - result.backward_speed = 42 - end - - return functions - """ - And the node map - """ - a---b - """ - And the ways - | nodes | - | ab | - And the data has been saved to disk - - When I try to run "osrm-extract {osm_file} --profile {profile_file}" - Then stderr should contain "There are no edges" - And it should exit with an error - - Scenario: Step weights -- way_function: second way wins - Given the profile file - """ - local functions = require('testbot') - functions.setup_testbot = functions.setup - - functions.setup = function() - local profile = functions.setup_testbot() - profile.properties.traffic_signal_penalty = 0 - profile.properties.u_turn_penalty = 0 - profile.properties.weight_name = 'steps' - return profile - end - - functions.process_way = function(profile, way, result) - result.forward_mode = mode.driving - result.backward_mode = mode.driving - result.duration = 42 - result.weight = 35 - end - - return functions - """ - - Given the node map - """ - a---b---c---d---e---f---g---h - """ - - And the ways - | nodes | - | abcdef | - | abcdefgh | - - When I route I should get - | waypoints | route | distance | weights | times | - | a,f | , | 100m | 25,0 | 30s,0s | - | f,a | , | 100m | 25,0 | 30s,0s | - | a,h | , | 140m +-1 | 35,0 | 42s,0s | - | h,a | , | 140m +-1 | 35,0 | 42s,0s | - - Scenario: Step weights -- way_function: higher weight_per_meter is preferred - Given the profile file - """ - local functions = require('testbot') - functions.setup_testbot = functions.setup - - functions.setup = function() - local profile = functions.setup_testbot() - profile.properties.traffic_signal_penalty = 0 - profile.properties.u_turn_penalty = 0 - profile.properties.weight_name = 'steps' - return profile - end - - functions.process_way = function(profile, way, result) - result.forward_mode = mode.driving - result.backward_mode = mode.driving - result.duration = 42 - result.forward_rate = 1 - result.backward_rate = 0.5 - end - - return functions - """ - - Given the node map - """ - a---b---c---d---e---f---g---h - """ - - And the ways - | nodes | - | abcdefgh | - | abcdef | - | fgh | - - When I route I should get - | waypoints | route | distance | weights | times | - | a,f | , | 100m | 99.8,0 | 30s,0s | - | f,a | , | 100m | 199.9,0 | 30s,0s | - | a,h | , | 140m | 139.8,0 | 42s,0s | - | h,a | , | 140m | 280.1,0 | 42s,0s | - | f,h | , | 40.1m | 40,0 | 12s,0s | - | h,f | , | 40.1m | 80.2,0 | 12s,0s | - - Scenario: Step weights -- segment_function - Given the profile file - """ - local functions = require('testbot') - functions.setup_testbot = functions.setup - - functions.setup = function() - local profile = functions.setup_testbot() - profile.properties.traffic_signal_penalty = 0 - profile.properties.u_turn_penalty = 0 - profile.properties.weight_name = 'steps' - return profile - end - - functions.process_way = function(profile, way, result) - result.forward_mode = mode.driving - result.backward_mode = mode.driving - result.weight = 42 - result.duration = 3 - end - - functions.process_segment = function(profile, segment) - segment.weight = 1 - segment.duration = 11 - end - - return functions - """ - - Given the node map - """ - a---b---c---d---e---f---g---h - """ - - And the ways - | nodes | - | abcdefgh | - | abcdef | - | fgh | - - When I route I should get - | waypoints | route | distance | weights | times | - | a,f | , | 100m | 5,0 | 55s,0s | - | f,a | , | 100m | 5,0 | 55s,0s | - | a,h | , | 140m +-1 | 7,0 | 77s,0s | - | h,a | , | 140m +-1 | 7,0 | 77s,0s | - | f,h | , | 40m +-1 | 2,0 | 22s,0s | - | h,f | , | 40m +-1 | 2,0 | 22s,0s | - - - Scenario: Step weights -- segment_function and turn_function with weight precision - Given the profile file - """ - local functions = require('testbot') - functions.setup_testbot = functions.setup - - functions.setup = function() - local profile = functions.setup_testbot() - profile.properties.traffic_signal_penalty = 0 - profile.properties.u_turn_penalty = 0 - profile.properties.weight_name = 'steps' - profile.properties.weight_precision = 3 - return profile - end - - functions.process_way = function(profile, way, result) - result.forward_mode = mode.driving - result.backward_mode = mode.driving - result.weight = 42 - result.duration = 3 - end - - functions.process_segment = function(profile, segment) - segment.weight = 1.11 - segment.duration = 100 - end - - functions.process_turn = function(profile, turn) - print (turn.angle) - turn.weight = 2 + turn.angle / 100 - turn.duration = turn.angle - end - - return functions - """ - - Given the node map - """ - a---b---c---d - ⋮ - e - """ - - And the ways - | nodes | - | abcd | - | ce | - - When I route I should get - | waypoints | route | distance | weights | times | - | a,c | , | 40m +-.1 | 2.22,0 | 200s,0s | - | a,e | ,, | 60m +-.1 | 5.12,1.11,0 | 290s,100s,0s | - | e,a | ,, | 60m +-.1 | 2.21,2.22,0 | 10s,200s,0s | - | e,d | ,, | 40m +-.1 | 4.01,1.11,0 | 190s,100s,0s | - | d,e | ,, | 40m +-.1 | 2.21,1.11,0 | 10s,100s,0s | - @traffic @speed Scenario: Step weights -- segment_function with speed and turn updates Given the profile file @@ -327,17 +40,17 @@ Feature: Weight tests | nodes | | abcd | | ce | - And the speed file + And the parquet speed file """ 1,2,36.999,42 2,1,36,42 """ - And the turn penalty file + And the parquet turn penalty file """ 2,3,5,25.5,16.7 """ - And the contract extra arguments "--segment-speed-file {speeds_file} --turn-penalty-file {penalties_file}" - And the customize extra arguments "--segment-speed-file {speeds_file} --turn-penalty-file {penalties_file}" + And the contract extra arguments "--speed-and-turn-penalty-format parquet --segment-speed-file {speeds_file} --turn-penalty-file {penalties_file}" + And the customize extra arguments "--speed-and-turn-penalty-format parquet --segment-speed-file {speeds_file} --turn-penalty-file {penalties_file}" When I route I should get | waypoints | route | distance | weights | times | @@ -345,70 +58,4 @@ Feature: Weight tests | a,e | ,, | 60m | 27.2,10,0 | 38.4s,11s,0s | | d,e | ,, | 40m | 10,10,0 | 11s,11s,0s | - @traffic @speed - Scenario: Step weights -- segment_function with speed and turn updates with fallback to durations - Given the profile file "testbot" initialized with - """ - profile.properties.weight_precision = 3 - """ - - And the node map - """ - a---b---c---d - . - e - """ - And the ways - | nodes | - | abcd | - | ce | - And the speed file - """ - 1,2,24 - 2,1,24 - """ - And the turn penalty file - """ - 2,3,5,1 - """ - And the contract extra arguments "--segment-speed-file {speeds_file} --turn-penalty-file {penalties_file}" - And the customize extra arguments "--segment-speed-file {speeds_file} --turn-penalty-file {penalties_file}" - - When I route I should get - | waypoints | route | distance | weights | times | - | a,d | abcd,abcd | 60m | 7,0 | 7s,0s | - | a,e | abcd,ce,ce | 60m | 5.997,2.001,0 | 6s,2s,0s | - | d,e | abcd,ce,ce | 40m | 2.003,2.001,0 | 2s,2s,0s | - - @traffic @speed - Scenario: Updating speeds without affecting weights. - Given the profile file "testbot" initialized with - """ - profile.properties.weight_precision = 3 - """ - - And the node map - """ - a-----------b - \ / - c----d - """ - And the ways - | nodes | highway | maxspeed | - | ab | living_street | 5 | - | acdb | motorway | 100 | - - # Note the comma on the last column - this indicates 'keep existing weight value' - And the speed file - """ - 1,2,100, - 1,3,5,,junk - 3,4,5,, - 4,2,5, - """ - And the contract extra arguments "--segment-speed-file {speeds_file}" - And the customize extra arguments "--segment-speed-file {speeds_file}" - - When I route I should get - | waypoints | route | distance | weights | times | - | a,b | acdb,acdb | 78.3m | 11.742,0 | 56.4s,0s | + \ No newline at end of file diff --git a/include/updater/csv_source.hpp b/include/updater/data_source.hpp similarity index 56% rename from include/updater/csv_source.hpp rename to include/updater/data_source.hpp index 89d60726b..54551a675 100644 --- a/include/updater/csv_source.hpp +++ b/include/updater/data_source.hpp @@ -1,5 +1,5 @@ -#ifndef OSRM_UPDATER_CSV_SOURCE_HPP -#define OSRM_UPDATER_CSV_SOURCE_HPP +#ifndef OSRM_UPDATER_DATA_SOURCE_HPP +#define OSRM_UPDATER_DATA_SOURCE_HPP #include "updater/source.hpp" #include "updater/updater_config.hpp" @@ -8,11 +8,11 @@ namespace osrm { namespace updater { -namespace csv +namespace data { -SegmentLookupTable readSegmentValues(const std::vector &paths, UpdaterConfig::SpeedAndTurnPenaltyFormat format); -TurnLookupTable readTurnValues(const std::vector &paths, UpdaterConfig::SpeedAndTurnPenaltyFormat format); -} // namespace csv +SegmentLookupTable readSegmentValues(const std::vector &paths, SpeedAndTurnPenaltyFormat format); +TurnLookupTable readTurnValues(const std::vector &paths, SpeedAndTurnPenaltyFormat format); +} // namespace data } // namespace updater } // namespace osrm diff --git a/include/updater/file_parser.hpp b/include/updater/file_parser.hpp index 09f61a372..2ed2449f7 100644 --- a/include/updater/file_parser.hpp +++ b/include/updater/file_parser.hpp @@ -41,16 +41,6 @@ template struct FilesParser // Operator returns a lambda function that maps input Key to boost::optional. auto operator()(const std::vector &csv_filenames) const { - -// std::shared_ptr infile; - -// PARQUET_ASSIGN_OR_THROW( -// infile, -// arrow::io::ReadableFile::Open("test.parquet")); - -// parquet::StreamReader os{parquet::ParquetFileReader::Open(infile)}; -// (void)os; - try { tbb::spin_mutex mutex; @@ -59,7 +49,7 @@ template struct FilesParser // TODO: do we need this `1 + ` here? auto local = ParseFile(csv_filenames[idx], 1 + idx); - { // Merge local CSV results into a flat global vector + { // Merge local results into a flat global vector tbb::spin_mutex::scoped_lock _{mutex}; lookup.insert(end(lookup), std::make_move_iterator(begin(local)), diff --git a/include/updater/parquet_file_parser.hpp b/include/updater/parquet_file_parser.hpp index ce52d34c4..7e46b3431 100644 --- a/include/updater/parquet_file_parser.hpp +++ b/include/updater/parquet_file_parser.hpp @@ -3,6 +3,7 @@ #include "file_parser.hpp" #include #include +#include #include #include @@ -26,71 +27,73 @@ namespace osrm namespace updater { -// Functor to parse a list of CSV files using "key,value,comment" grammar. -// Key and Value structures must be a model of Random Access Sequence. -// Also the Value structure must have source member that will be filled -// with the corresponding file index in the CSV filenames vector. template struct ParquetFilesParser : public FilesParser { private: - // Parse a single CSV file and return result as a vector + // Parse a single Parquet file and return result as a vector std::vector> ParseFile(const std::string &filename, std::size_t file_id) const final { - // TODO: error handling - std::shared_ptr infile; + try { + std::shared_ptr infile; - PARQUET_ASSIGN_OR_THROW( - infile, - arrow::io::ReadableFile::Open(filename)); + PARQUET_ASSIGN_OR_THROW( + infile, + arrow::io::ReadableFile::Open(filename)); - parquet::StreamReader os{parquet::ParquetFileReader::Open(infile)}; + parquet::StreamReader os{parquet::ParquetFileReader::Open(infile)}; - std::vector> result; - while ( !os.eof() ) - { - result.emplace_back(ReadKey(os), ReadValue(os, file_id)); - // ... + std::vector> result; + while ( !os.eof() ) + { + result.emplace_back(ReadKeyValue(os, file_id)); + } + return result; + } catch (const std::exception &e) { + throw util::exception(e.what() + SOURCE_REF); } - return result; -// (void)os; - } - Key ReadKey(parquet::StreamReader &os) const + std::pair ReadKeyValue(parquet::StreamReader &os, std::size_t file_id) const { Key key; - ReadKey(os, key); - return key; - } - - Value ReadValue(parquet::StreamReader &os, std::size_t file_id) const - { Value value; - ReadValue(os, value); + Read(os, key); + Read(os, value); value.source = file_id; - return value; + os >> parquet::EndRow; + return {key, value}; } - void ReadKey(parquet::StreamReader &os, Turn& turn) const { - os >> turn.from >> turn.via >> turn.to >> parquet::EndRow; + void Read(parquet::StreamReader &os, Turn& turn) const { + int64_t from, via, to; + os >> from >> via >> to; + turn.from = from; + turn.via = via; + turn.to = to; } - void ReadValue(parquet::StreamReader &os, PenaltySource& penalty_source) const { - os >> penalty_source.duration >> penalty_source.weight >> parquet::EndRow; + void Read(parquet::StreamReader &os, PenaltySource& penalty_source) const { + os >> penalty_source.duration >> penalty_source.weight; } - void ReadKey(parquet::StreamReader &os, Segment& segment) const { - os >> segment.from >> segment.to >> parquet::EndRow; + void Read(parquet::StreamReader &os, Segment& segment) const { + int64_t from; + int64_t to; + os >> from >> to; + + segment.from = from; + segment.to = to; + //std::cerr << from << " " << to<< std::endl; + //os >> segment.from >> segment.to >> parquet::EndRow; } - void ReadValue(parquet::StreamReader &os, SpeedSource& speed_source) const { + void Read(parquet::StreamReader &os, SpeedSource& speed_source) const { std::optional rate; - os >> speed_source.speed >> rate >> parquet::EndRow; + os >> speed_source.speed >> rate; // TODO: boost::optional if (rate) { speed_source.rate = *rate; } - // os >> turn.weight >> turn.duration >> turn.pre_turn_bearing >> turn.post_turn_bearing >> turn.source >> parquet::EndRow; } }; diff --git a/include/updater/updater_config.hpp b/include/updater/updater_config.hpp index 945aa6f5f..dcc46c810 100644 --- a/include/updater/updater_config.hpp +++ b/include/updater/updater_config.hpp @@ -35,12 +35,19 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "storage/io_config.hpp" #include "storage/storage_config.hpp" +#include namespace osrm { namespace updater { + + enum class SpeedAndTurnPenaltyFormat { + CSV, + PARQUET + }; + struct UpdaterConfig final : storage::IOConfig { UpdaterConfig() @@ -69,16 +76,45 @@ struct UpdaterConfig final : storage::IOConfig double log_edge_updates_factor = 0.0; std::time_t valid_now; - - enum class SpeedAndTurnPenaltyFormat { - CSV, - PARQUET - } speed_and_turn_penalty_format = SpeedAndTurnPenaltyFormat::CSV; +SpeedAndTurnPenaltyFormat speed_and_turn_penalty_format = SpeedAndTurnPenaltyFormat::CSV; std::vector segment_speed_lookup_paths; std::vector turn_penalty_lookup_paths; std::string tz_file_path; }; + + +inline std::istream& operator>> (std::istream &in, SpeedAndTurnPenaltyFormat& format) { + std::string token; + in >> token; + + std::transform(token.begin(), token.end(), token.begin(), [](auto c){ return std::tolower(c); }); + + if (token == "csv") { + format = SpeedAndTurnPenaltyFormat::CSV; + } else if (token == "parquet") { + format = SpeedAndTurnPenaltyFormat::PARQUET; + } else { + throw boost::program_options::validation_error{boost::program_options::validation_error::invalid_option_value}; + } + return in; +} + + +inline std::ostream& operator<< (std::ostream &out, SpeedAndTurnPenaltyFormat format) { + switch (format) { + case SpeedAndTurnPenaltyFormat::CSV: + out << "csv"; + break; + case SpeedAndTurnPenaltyFormat::PARQUET: + out << "parquet"; + break; + } + return out; +} + + + } // namespace updater } // namespace osrm diff --git a/package-lock.json b/package-lock.json index 457213005..c66c173a1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,8 @@ "hasInstallScript": true, "license": "BSD-2-Clause", "dependencies": { - "@mapbox/node-pre-gyp": "^1.0.10" + "@mapbox/node-pre-gyp": "^1.0.10", + "parquetjs": "^0.11.2" }, "devDependencies": { "@babel/cli": "^7.18.10", @@ -4286,7 +4287,6 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, "funding": [ { "type": "github", @@ -4423,6 +4423,14 @@ "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=", "dev": true }, + "node_modules/brotli": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.3.tgz", + "integrity": "sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg==", + "dependencies": { + "base64-js": "^1.1.2" + } + }, "node_modules/browser-pack": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/browser-pack/-/browser-pack-6.1.0.tgz", @@ -4777,6 +4785,14 @@ "integrity": "sha512-3pwjAK0qHSDN9+YAF4fJknsSruP7mpjdWzUSruIJD/JCH77pEh0SorEyb3xVaKkfwk2tzjOt2D8scJ0KAdfXLA==", "dev": true }, + "node_modules/bson": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz", + "integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg==", + "engines": { + "node": ">=0.6.19" + } + }, "node_modules/buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.2.1.tgz", @@ -10856,6 +10872,11 @@ "insert-module-globals": "bin/cmd.js" } }, + "node_modules/int53": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/int53/-/int53-0.2.4.tgz", + "integrity": "sha512-a5jlKftS7HUOhkUyYD7j2sJ/ZnvWiNlZS1ldR+g1ifQ+/UuZXIE+YTc/lK1qGj/GwAU5F8Z0e1eVq2t1J5Ob2g==" + }, "node_modules/internal-slot": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.3.tgz", @@ -12020,6 +12041,22 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, + "node_modules/lzo": { + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/lzo/-/lzo-0.4.11.tgz", + "integrity": "sha512-apQHNoW2Alg72FMqaC/7pn03I7umdgSVFt2KRkCXXils4Z9u3QBh1uOtl2O5WmZIDLd9g6Lu4lIdOLmiSTFVCQ==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "bindings": "~1.2.1" + } + }, + "node_modules/lzo/node_modules/bindings": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz", + "integrity": "sha512-u4cBQNepWxYA55FunZSM7wMi55yQaN0otnhhilNoWHq0MfOfJeQx0v0mRRpolGOExPjZcl6FtB0BB8Xkb88F0g==", + "optional": true + }, "node_modules/magic-string": { "version": "0.22.5", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz", @@ -12772,6 +12809,11 @@ } } }, + "node_modules/node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==" + }, "node_modules/node-releases": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.6.tgz", @@ -12960,6 +13002,14 @@ "node": ">= 0.4" } }, + "node_modules/object-stream": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/object-stream/-/object-stream-0.0.1.tgz", + "integrity": "sha512-+NPJnRvX9RDMRY9mOWOo/NDppBjbZhXirNNSu2IBnuNboClC9h1ZGHXgHBLDbJMHsxeJDq922aVmG5xs24a/cA==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/object-visit": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz", @@ -13275,6 +13325,26 @@ "path-platform": "~0.11.15" } }, + "node_modules/parquetjs": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/parquetjs/-/parquetjs-0.11.2.tgz", + "integrity": "sha512-Y6FOc3Oi2AxY4TzJPz7fhICCR8tQNL3p+2xGQoUAMbmlJBR7+JJmMrwuyMjIpDiM7G8Wj/8oqOH4UDUmu4I5ZA==", + "dependencies": { + "brotli": "^1.3.0", + "bson": "^1.0.4", + "int53": "^0.2.4", + "object-stream": "0.0.1", + "snappyjs": "^0.6.0", + "thrift": "^0.11.0", + "varint": "^5.0.0" + }, + "engines": { + "node": ">=7.6" + }, + "optionalDependencies": { + "lzo": "^0.4.0" + } + }, "node_modules/parse-asn1": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", @@ -14102,7 +14172,6 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=", - "dev": true, "engines": { "node": ">=0.6.0", "teleport": ">=0.2.0" @@ -15345,6 +15414,11 @@ "node": ">=0.10.0" } }, + "node_modules/snappyjs": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/snappyjs/-/snappyjs-0.6.1.tgz", + "integrity": "sha512-YIK6I2lsH072UE0aOFxxY1dPDCS43I5ktqHpeAsuLNYWkE5pGxRGWfDM4/vSUfNzXjC1Ivzt3qx31PCLmc9yqg==" + }, "node_modules/sort-keys": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.2.tgz", @@ -16950,6 +17024,19 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, + "node_modules/thrift": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/thrift/-/thrift-0.11.0.tgz", + "integrity": "sha512-UpsBhOC45a45TpeHOXE4wwYwL8uD2apbHTbtBvkwtUU4dNwCjC7DpQTjw2Q6eIdfNtw+dKthdwq94uLXTJPfFw==", + "dependencies": { + "node-int64": "^0.4.0", + "q": "^1.5.0", + "ws": ">= 2.2.3" + }, + "engines": { + "node": ">= 4.1.0" + } + }, "node_modules/through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -18417,6 +18504,11 @@ "spdx-expression-parse": "~1.0.0" } }, + "node_modules/varint": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz", + "integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow==" + }, "node_modules/vendors": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/vendors/-/vendors-1.0.4.tgz", @@ -18800,6 +18892,26 @@ "node": ">=0.10.0" } }, + "node_modules/ws": { + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz", + "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/x-is-string": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/x-is-string/-/x-is-string-0.1.0.tgz", @@ -22262,8 +22374,7 @@ "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" }, "bcrypt-pbkdf": { "version": "1.0.2", @@ -22376,6 +22487,14 @@ "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=", "dev": true }, + "brotli": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.3.tgz", + "integrity": "sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg==", + "requires": { + "base64-js": "^1.1.2" + } + }, "browser-pack": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/browser-pack/-/browser-pack-6.1.0.tgz", @@ -22658,6 +22777,11 @@ } } }, + "bson": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz", + "integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg==" + }, "buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.2.1.tgz", @@ -27614,6 +27738,11 @@ "xtend": "^4.0.0" } }, + "int53": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/int53/-/int53-0.2.4.tgz", + "integrity": "sha512-a5jlKftS7HUOhkUyYD7j2sJ/ZnvWiNlZS1ldR+g1ifQ+/UuZXIE+YTc/lK1qGj/GwAU5F8Z0e1eVq2t1J5Ob2g==" + }, "internal-slot": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.3.tgz", @@ -28504,6 +28633,23 @@ } } }, + "lzo": { + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/lzo/-/lzo-0.4.11.tgz", + "integrity": "sha512-apQHNoW2Alg72FMqaC/7pn03I7umdgSVFt2KRkCXXils4Z9u3QBh1uOtl2O5WmZIDLd9g6Lu4lIdOLmiSTFVCQ==", + "optional": true, + "requires": { + "bindings": "~1.2.1" + }, + "dependencies": { + "bindings": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz", + "integrity": "sha512-u4cBQNepWxYA55FunZSM7wMi55yQaN0otnhhilNoWHq0MfOfJeQx0v0mRRpolGOExPjZcl6FtB0BB8Xkb88F0g==", + "optional": true + } + } + }, "magic-string": { "version": "0.22.5", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz", @@ -29134,6 +29280,11 @@ "whatwg-url": "^5.0.0" } }, + "node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==" + }, "node-releases": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.6.tgz", @@ -29282,6 +29433,11 @@ "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", "dev": true }, + "object-stream": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/object-stream/-/object-stream-0.0.1.tgz", + "integrity": "sha512-+NPJnRvX9RDMRY9mOWOo/NDppBjbZhXirNNSu2IBnuNboClC9h1ZGHXgHBLDbJMHsxeJDq922aVmG5xs24a/cA==" + }, "object-visit": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz", @@ -29530,6 +29686,21 @@ "path-platform": "~0.11.15" } }, + "parquetjs": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/parquetjs/-/parquetjs-0.11.2.tgz", + "integrity": "sha512-Y6FOc3Oi2AxY4TzJPz7fhICCR8tQNL3p+2xGQoUAMbmlJBR7+JJmMrwuyMjIpDiM7G8Wj/8oqOH4UDUmu4I5ZA==", + "requires": { + "brotli": "^1.3.0", + "bson": "^1.0.4", + "int53": "^0.2.4", + "lzo": "^0.4.0", + "object-stream": "0.0.1", + "snappyjs": "^0.6.0", + "thrift": "^0.11.0", + "varint": "^5.0.0" + } + }, "parse-asn1": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", @@ -30248,8 +30419,7 @@ "q": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", - "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=", - "dev": true + "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=" }, "qs": { "version": "6.10.1", @@ -31271,6 +31441,11 @@ "kind-of": "^3.2.0" } }, + "snappyjs": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/snappyjs/-/snappyjs-0.6.1.tgz", + "integrity": "sha512-YIK6I2lsH072UE0aOFxxY1dPDCS43I5ktqHpeAsuLNYWkE5pGxRGWfDM4/vSUfNzXjC1Ivzt3qx31PCLmc9yqg==" + }, "sort-keys": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.2.tgz", @@ -32519,6 +32694,16 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, + "thrift": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/thrift/-/thrift-0.11.0.tgz", + "integrity": "sha512-UpsBhOC45a45TpeHOXE4wwYwL8uD2apbHTbtBvkwtUU4dNwCjC7DpQTjw2Q6eIdfNtw+dKthdwq94uLXTJPfFw==", + "requires": { + "node-int64": "^0.4.0", + "q": "^1.5.0", + "ws": ">= 2.2.3" + } + }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -33771,6 +33956,11 @@ "spdx-expression-parse": "~1.0.0" } }, + "varint": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz", + "integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow==" + }, "vendors": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/vendors/-/vendors-1.0.4.tgz", @@ -34085,6 +34275,12 @@ "mkdirp": "^0.5.1" } }, + "ws": { + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz", + "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==", + "requires": {} + }, "x-is-string": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/x-is-string/-/x-is-string-0.1.0.tgz", diff --git a/package.json b/package.json index 6e464a71a..8e294a5bc 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "private": false, "description": "The Open Source Routing Machine is a high performance routing engine written in C++ designed to run on OpenStreetMap data.", "dependencies": { - "@mapbox/node-pre-gyp": "^1.0.10" + "@mapbox/node-pre-gyp": "^1.0.10", + "parquetjs": "^0.11.2" }, "browserify": { "transform": [ @@ -58,6 +59,7 @@ "jsonpath": "^1.1.1", "mkdirp": "^0.5.6", "node-addon-api": "^5.0.0", + "node-cmake": "^2.5.1", "node-timeout": "0.0.4", "polyline": "^0.2.0", "request": "^2.88.2", @@ -65,12 +67,13 @@ "tape": "^4.16.0", "turf": "^3.0.14", "uglify-js": "^3.17.0", - "xmlbuilder": "^4.2.1", - "node-cmake": "^2.5.1" + "xmlbuilder": "^4.2.1" }, "main": "lib/index.js", "binary": { - "napi_versions": [8], + "napi_versions": [ + 8 + ], "module_name": "node_osrm", "module_path": "./lib/binding_napi_v{napi_build_version}/", "host": "https://github.com", diff --git a/src/tools/contract.cpp b/src/tools/contract.cpp index 289ec671b..2288ab0f3 100644 --- a/src/tools/contract.cpp +++ b/src/tools/contract.cpp @@ -49,7 +49,7 @@ return_code parseArguments(int argc, "core,k", boost::program_options::value(&contractor_config.core_factor)->default_value(1.0), "DEPRECATED: Will always be 1.0. Percentage of the graph (in vertices) to contract " - "[0..1].")("segment-speed-file", + "[0..1].") ("speed-and-turn-penalty-format", boost::program_options::value(&contractor_config.updater_config.speed_and_turn_penalty_format)->default_value(updater::SpeedAndTurnPenaltyFormat::CSV))("segment-speed-file", boost::program_options::value>( &contractor_config.updater_config.segment_speed_lookup_paths) ->composing(), diff --git a/src/tools/customize.cpp b/src/tools/customize.cpp index f77b88795..4ae8ad6fb 100644 --- a/src/tools/customize.cpp +++ b/src/tools/customize.cpp @@ -1,4 +1,5 @@ #include "customizer/customizer.hpp" +#include "updater/updater_config.hpp" #include "osrm/exception.hpp" #include "util/log.hpp" @@ -20,6 +21,7 @@ enum class return_code : unsigned exit }; + return_code parseArguments(int argc, char *argv[], std::string &verbosity, @@ -39,7 +41,8 @@ return_code parseArguments(int argc, ("threads,t", boost::program_options::value(&customization_config.requested_num_threads) ->default_value(std::thread::hardware_concurrency()), - "Number of threads to use")( + "Number of threads to use") + ("speed-and-turn-penalty-format", boost::program_options::value(&customization_config.updater_config.speed_and_turn_penalty_format)->default_value(updater::SpeedAndTurnPenaltyFormat::CSV))( "segment-speed-file", boost::program_options::value>( &customization_config.updater_config.segment_speed_lookup_paths) diff --git a/src/updater/csv_source.cpp b/src/updater/data_source.cpp similarity index 85% rename from src/updater/csv_source.cpp rename to src/updater/data_source.cpp index 43b5f4488..456cd8ccb 100644 --- a/src/updater/csv_source.cpp +++ b/src/updater/data_source.cpp @@ -1,4 +1,4 @@ -#include "updater/csv_source.hpp" +#include "updater/data_source.hpp" #include "updater/csv_file_parser.hpp" #include "updater/parquet_file_parser.hpp" @@ -31,40 +31,40 @@ namespace osrm { namespace updater { -namespace csv +namespace data { namespace { -std::unique_ptr> makeSegmentParser(UpdaterConfig::SpeedAndTurnPenaltyFormat format) { +std::unique_ptr> makeSegmentParser(SpeedAndTurnPenaltyFormat format) { switch (format) { - case UpdaterConfig::SpeedAndTurnPenaltyFormat::CSV: + case SpeedAndTurnPenaltyFormat::CSV: { static const auto value_if_blank = std::numeric_limits::quiet_NaN(); const qi::real_parser> unsigned_double; return std::make_unique>(qi::ulong_long >> ',' >> qi::ulong_long, unsigned_double >> -(',' >> (qi::double_ | qi::attr(value_if_blank)))); } - case UpdaterConfig::SpeedAndTurnPenaltyFormat::PARQUET: + case SpeedAndTurnPenaltyFormat::PARQUET: return std::make_unique>(); } } -std::unique_ptr> makeTurnParser(UpdaterConfig::SpeedAndTurnPenaltyFormat format) { +std::unique_ptr> makeTurnParser(SpeedAndTurnPenaltyFormat format) { switch (format) { - case UpdaterConfig::SpeedAndTurnPenaltyFormat::CSV: + case SpeedAndTurnPenaltyFormat::CSV: { return std::make_unique>(qi::ulong_long >> ',' >> qi::ulong_long >> ',' >> qi::ulong_long, qi::double_ >> -(',' >> qi::double_)); } - case UpdaterConfig::SpeedAndTurnPenaltyFormat::PARQUET: + case SpeedAndTurnPenaltyFormat::PARQUET: return std::make_unique>(); } } } // namespace -SegmentLookupTable readSegmentValues(const std::vector &paths, UpdaterConfig::SpeedAndTurnPenaltyFormat format) +SegmentLookupTable readSegmentValues(const std::vector &paths, SpeedAndTurnPenaltyFormat format) { auto parser = makeSegmentParser(format); @@ -83,11 +83,11 @@ SegmentLookupTable readSegmentValues(const std::vector &paths, Upda return result; } -TurnLookupTable readTurnValues(const std::vector &paths, UpdaterConfig::SpeedAndTurnPenaltyFormat format) +TurnLookupTable readTurnValues(const std::vector &paths, SpeedAndTurnPenaltyFormat format) { auto parser = makeTurnParser(format); return (*parser)(paths); } -} // namespace csv +} // namespace data } // namespace updater } // namespace osrm diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index bfc3df59a..6b8ea8919 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -1,5 +1,5 @@ #include "updater/updater.hpp" -#include "updater/csv_source.hpp" +#include "updater/data_source.hpp" #include "extractor/compressed_edge_container.hpp" #include "extractor/edge_based_graph_factory.hpp" @@ -618,7 +618,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e tbb::concurrent_vector updated_segments; if (update_edge_weights) { - auto segment_speed_lookup = csv::readSegmentValues(config.segment_speed_lookup_paths, config.speed_and_turn_penalty_format); + auto segment_speed_lookup = data::readSegmentValues(config.segment_speed_lookup_paths, config.speed_and_turn_penalty_format); TIMER_START(segment); updated_segments = updateSegmentData(config, @@ -633,7 +633,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e util::Log() << "Updating segment data took " << TIMER_MSEC(segment) << "ms."; } - auto turn_penalty_lookup = csv::readTurnValues(config.turn_penalty_lookup_paths, config.speed_and_turn_penalty_format); + auto turn_penalty_lookup = data::readTurnValues(config.turn_penalty_lookup_paths, config.speed_and_turn_penalty_format); if (update_turn_penalties) { auto updated_turn_penalties = updateTurnPenalties(config, diff --git a/unit_tests/updater/parquet.cpp b/unit_tests/updater/parquet.cpp new file mode 100644 index 000000000..35a99ef2c --- /dev/null +++ b/unit_tests/updater/parquet.cpp @@ -0,0 +1,13 @@ +#include +#include +#include + +using namespace osrm; +using namespace osrm::updater; + +BOOST_AUTO_TEST_CASE(parquet_readSegmentValues) +{ + boost::filesystem::path test_path(TEST_DATA_DIR "/speeds_file.parquet"); + SegmentLookupTable segment_lookup_table = data::readSegmentValues({test_path.string()}, SpeedAndTurnPenaltyFormat::PARQUET); + BOOST_CHECK_EQUAL(segment_lookup_table.lookup.size(), 2); +} \ No newline at end of file diff --git a/unit_tests/updater/speeds_file.parquet b/unit_tests/updater/speeds_file.parquet new file mode 100644 index 000000000..bb71d8e9a Binary files /dev/null and b/unit_tests/updater/speeds_file.parquet differ