Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

wip

Browse Source
This commit is contained in:
Siarhei Fedartsou 2022-11-26 15:07:00 +01:00
parent ae08c5c1e7
commit 4046f9dfd3
2 changed files with 22 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
CMakeLists.txt
View File

@ -507,6 +507,7 @@ if(ENABLE_CONAN)
OPTIONS boost:filesystem_version=3 # https://stackoverflow.com/questions/73392648/error-with-boost-filesystem-version-in-cmake OPTIONS boost:filesystem_version=3 # https://stackoverflow.com/questions/73392648/error-with-boost-filesystem-version-in-cmake
onetbb:shared=${TBB_SHARED} onetbb:shared=${TBB_SHARED}
boost:without_stacktrace=True # Apple Silicon cross-compilation fails without it boost:without_stacktrace=True # Apple Silicon cross-compilation fails without it
arrow:parquet=True
BUILD missing BUILD missing
) )
# explicitly say Conan to use x86 dependencies if build for x86 platforms (https://github.com/conan-io/cmake-conan/issues/141) # explicitly say Conan to use x86 dependencies if build for x86 platforms (https://github.com/conan-io/cmake-conan/issues/141)
@ -524,6 +525,8 @@ if(ENABLE_CONAN)
add_dependency_includes(${CONAN_INCLUDE_DIRS_EXPAT}) add_dependency_includes(${CONAN_INCLUDE_DIRS_EXPAT})
add_dependency_includes(${CONAN_INCLUDE_DIRS_LUA}) add_dependency_includes(${CONAN_INCLUDE_DIRS_LUA})
add_dependency_includes(${CONAN_INCLUDE_DIRS_TBB}) add_dependency_includes(${CONAN_INCLUDE_DIRS_TBB})
add_dependency_includes(${CONAN_INCLUDE_DIRS_ARROW})
# message(FATAL_ERROR "SDSDF ${CONAN_INCLUDE_DIRS_ARROW}")
set(Boost_USE_STATIC_LIBS ON) set(Boost_USE_STATIC_LIBS ON)
find_package(Boost REQUIRED COMPONENTS ${BOOST_COMPONENTS}) find_package(Boost REQUIRED COMPONENTS ${BOOST_COMPONENTS})
@ -545,6 +548,9 @@ if(ENABLE_CONAN)
set(LUA_LIBRARIES ${lua_LIBRARIES}) set(LUA_LIBRARIES ${lua_LIBRARIES})
find_package(TBB REQUIRED) find_package(TBB REQUIRED)
#find_package(Parquet REQUIRED)
find_package(Arrow REQUIRED)
set(ARROW_LIBRARIES ${arrow_LIBRARIES})
# note: we avoid calling find_package(Osmium ...) here to ensure that the # note: we avoid calling find_package(Osmium ...) here to ensure that the
@ -678,7 +684,9 @@ set(UPDATER_LIBRARIES
${TBB_LIBRARIES} ${TBB_LIBRARIES}
${MAYBE_RT_LIBRARY} ${MAYBE_RT_LIBRARY}
${MAYBE_COVERAGE_LIBRARIES} ${MAYBE_COVERAGE_LIBRARIES}
${ZLIB_LIBRARY}) ${ZLIB_LIBRARY}
${ARROW_LIBRARIES}
)
set(CONTRACTOR_LIBRARIES set(CONTRACTOR_LIBRARIES
${BOOST_BASE_LIBRARIES} ${BOOST_BASE_LIBRARIES}
${CMAKE_THREAD_LIBS_INIT} ${CMAKE_THREAD_LIBS_INIT}

13
include/updater/csv_file_parser.hpp
View File

@ -1,5 +1,8 @@
#ifndef OSRM_UPDATER_CSV_FILE_PARSER_HPP #ifndef OSRM_UPDATER_CSV_FILE_PARSER_HPP
#define OSRM_UPDATER_CSV_FILE_PARSER_HPP #define OSRM_UPDATER_CSV_FILE_PARSER_HPP
#include <parquet/arrow/reader.h>
#include <parquet/stream_reader.h>
#include <arrow/io/file.h>
#include "updater/source.hpp" #include "updater/source.hpp"
@ -44,6 +47,16 @@ template <typename Key, typename Value> struct CSVFilesParser
// Operator returns a lambda function that maps input Key to boost::optional<Value>. // Operator returns a lambda function that maps input Key to boost::optional<Value>.
auto operator()(const std::vector<std::string> &csv_filenames) const auto operator()(const std::vector<std::string> &csv_filenames) const
{ {
std::shared_ptr<arrow::io::ReadableFile> infile;
PARQUET_ASSIGN_OR_THROW(
infile,
arrow::io::ReadableFile::Open("test.parquet"));
parquet::StreamReader os{parquet::ParquetFileReader::Open(infile)};
(void)os;
try try
{ {
tbb::spin_mutex mutex; tbb::spin_mutex mutex;