Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Merge commit '788bc67faa7738cf7c6b2a192ecf3e3567d1c20e' into develop

Browse Source
This commit is contained in:
Patrick Niklaus
2015-08-28 12:42:03 +02:00
parent 8e1f70865e 788bc67faa
commit 048be2da2c
150 changed files with 12325 additions and 4321 deletions
Download Patch File Download Diff File Expand all files Collapse all files
third_party/libosmium/include/osmium/index/bool_vector.hpp
Vendored
+83
View File
@@ -0,0 +1,83 @@
#ifndef OSMIUM_INDEX_BOOL_VECTOR_HPP
#define OSMIUM_INDEX_BOOL_VECTOR_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <type_traits>
#include <vector>
namespace osmium {
namespace index {
/**
* Index storing one bit for each Id. The index automatically scales
* with the Ids stored. Default value is 'false'. Storage uses
* std::vector<bool> and needs a minimum of memory if the Ids are
* dense.
*/
template <typename T>
class BoolVector {
static_assert(std::is_unsigned<T>::value, "Needs unsigned type");
std::vector<bool> m_bits;
public:
BoolVector() = default;
BoolVector(const BoolVector&) = default;
BoolVector(BoolVector&&) = default;
BoolVector& operator=(const BoolVector&) = default;
BoolVector& operator=(BoolVector&&) = default;
~BoolVector() = default;
void set(T id, bool value = true) {
if (m_bits.size() <= id) {
m_bits.resize(id + 1024 * 1024);
}
m_bits[id] = value;
}
bool get(T id) const {
return id < m_bits.size() && m_bits[id];
}
}; // class BoolVector
} // namespace index
} // namespace osmium
#endif // OSMIUM_INDEX_BOOL_VECTOR_HPP
third_party/libosmium/include/osmium/index/detail/create_map_with_fd.hpp
Vendored
-2
View File
@@ -39,8 +39,6 @@ DEALINGS IN THE SOFTWARE.
#include <fcntl.h>
#include <stdexcept>
#include <string>
#include <sys/stat.h>
#include <sys/types.h>
#include <vector>
namespace osmium {
third_party/libosmium/include/osmium/index/detail/mmap_vector_anon.hpp
Vendored
+4 -17
View File
@@ -35,9 +35,6 @@ DEALINGS IN THE SOFTWARE.
#ifdef __linux__
#include <cstddef>
#include <osmium/index/detail/typed_mmap.hpp>
#include <osmium/index/detail/mmap_vector_base.hpp>
namespace osmium {
@@ -45,26 +42,16 @@ namespace osmium {
namespace detail {
/**
* This class looks and behaves like STL vector, but uses mmap internally.
* This class looks and behaves like STL vector, but uses mmap
* internally.
*/
template <typename T>
class mmap_vector_anon : public mmap_vector_base<T, mmap_vector_anon> {
class mmap_vector_anon : public mmap_vector_base<T> {
public:
mmap_vector_anon() :
mmap_vector_base<T, osmium::detail::mmap_vector_anon>(
-1,
osmium::detail::mmap_vector_size_increment,
0,
osmium::detail::typed_mmap<T>::map(osmium::detail::mmap_vector_size_increment)) {
}
void reserve(size_t new_capacity) {
if (new_capacity > this->capacity()) {
this->data(osmium::detail::typed_mmap<T>::remap(this->data(), this->capacity(), new_capacity));
this->m_capacity = new_capacity;
}
mmap_vector_base<T>() {
}
}; // class mmap_vector_anon
third_party/libosmium/include/osmium/index/detail/mmap_vector_base.hpp
Vendored
+44 -48
View File
@@ -34,11 +34,10 @@ DEALINGS IN THE SOFTWARE.
*/
#include <cstddef>
#include <new>
#include <new> // IWYU pragma: keep
#include <stdexcept>
#include <osmium/index/detail/typed_mmap.hpp>
#include <osmium/util/compatibility.hpp>
#include <osmium/util/memory_mapping.hpp>
namespace osmium {
@@ -48,40 +47,29 @@ namespace osmium {
/**
* This is a base class for implementing classes that look like
* STL vector but use mmap internally. This class can not be used
* on it's own. Use the derived classes mmap_vector_anon or
* mmap_vector_file.
* STL vector but use mmap internally. Do not use this class itself,
* use the derived classes mmap_vector_anon or mmap_vector_file.
*/
template <typename T, template <typename> class TDerived>
template <typename T>
class mmap_vector_base {
protected:
int m_fd;
size_t m_capacity;
size_t m_size;
T* m_data;
explicit mmap_vector_base(int fd, size_t capacity, size_t size, T* data) noexcept :
m_fd(fd),
m_capacity(capacity),
m_size(size),
m_data(data) {
}
explicit mmap_vector_base(int fd, size_t capacity, size_t size) :
m_fd(fd),
m_capacity(capacity),
m_size(size),
m_data(osmium::detail::typed_mmap<T>::grow_and_map(capacity, m_fd)) {
}
void data(T* data) {
m_data = data;
}
osmium::util::TypedMemoryMapping<T> m_mapping;
public:
explicit mmap_vector_base(int fd, size_t capacity, size_t size = 0) :
m_size(size),
m_mapping(capacity, osmium::util::MemoryMapping::mapping_mode::write_shared, fd) {
}
explicit mmap_vector_base(size_t capacity = mmap_vector_size_increment) :
m_size(0),
m_mapping(capacity) {
}
typedef T value_type;
typedef T& reference;
typedef const T& const_reference;
@@ -90,12 +78,14 @@ namespace osmium {
typedef T* iterator;
typedef const T* const_iterator;
~mmap_vector_base() {
osmium::detail::typed_mmap<T>::unmap(m_data, m_capacity);
~mmap_vector_base() = default;
void close() {
m_mapping.unmap();
}
size_t capacity() const noexcept {
return m_capacity;
return m_mapping.size();
}
size_t size() const noexcept {
@@ -106,23 +96,23 @@ namespace osmium {
return m_size == 0;
}
const T* data() const noexcept {
return m_data;
const T* data() const {
return m_mapping.begin();
}
T* data() noexcept {
return m_data;
T* data() {
return m_mapping.begin();
}
T& operator[](size_t n) {
return m_data[n];
return data()[n];
}
T at(size_t n) const {
if (n >= m_size) {
throw std::out_of_range("out of range");
}
return m_data[n];
return data()[n];
}
void clear() noexcept {
@@ -134,16 +124,22 @@ namespace osmium {
}
void push_back(const T& value) {
if (m_size >= m_capacity) {
if (m_size >= capacity()) {
resize(m_size+1);
}
m_data[m_size] = value;
data()[m_size] = value;
++m_size;
}
void reserve(size_t new_capacity) {
if (new_capacity > capacity()) {
m_mapping.resize(new_capacity);
}
}
void resize(size_t new_size) {
if (new_size > capacity()) {
static_cast<TDerived<T>*>(this)->reserve(new_size + osmium::detail::mmap_vector_size_increment);
reserve(new_size + osmium::detail::mmap_vector_size_increment);
}
if (new_size > size()) {
new (data() + size()) T[new_size - size()];
@@ -152,27 +148,27 @@ namespace osmium {
}
iterator begin() noexcept {
return m_data;
return data();
}
iterator end() noexcept {
return m_data + m_size;
return data() + m_size;
}
const_iterator begin() const noexcept {
return m_data;
return data();
}
const_iterator end() const noexcept {
return m_data + m_size;
return data() + m_size;
}
const_iterator cbegin() noexcept {
return m_data;
const_iterator cbegin() const noexcept {
return data();
}
const_iterator cend() noexcept {
return m_data + m_size;
const_iterator cend() const noexcept {
return data() + m_size;
}
}; // class mmap_vector_base
third_party/libosmium/include/osmium/index/detail/mmap_vector_file.hpp
Vendored
+7 -22
View File
@@ -33,11 +33,9 @@ DEALINGS IN THE SOFTWARE.
*/
#include <cstddef>
#include <osmium/index/detail/typed_mmap.hpp>
#include <osmium/index/detail/mmap_vector_base.hpp>
#include <osmium/index/detail/tmpfile.hpp>
#include <osmium/util/file.hpp>
namespace osmium {
@@ -48,32 +46,19 @@ namespace osmium {
* internally.
*/
template <typename T>
class mmap_vector_file : public mmap_vector_base<T, mmap_vector_file> {
class mmap_vector_file : public mmap_vector_base<T> {
public:
explicit mmap_vector_file() :
mmap_vector_base<T, osmium::detail::mmap_vector_file>(
explicit mmap_vector_file() : mmap_vector_base<T>(
osmium::detail::create_tmp_file(),
osmium::detail::mmap_vector_size_increment,
0) {
osmium::detail::mmap_vector_size_increment) {
}
explicit mmap_vector_file(int fd) :
mmap_vector_base<T, osmium::detail::mmap_vector_file>(
explicit mmap_vector_file(int fd) : mmap_vector_base<T>(
fd,
osmium::detail::typed_mmap<T>::file_size(fd) == 0 ?
osmium::detail::mmap_vector_size_increment :
osmium::detail::typed_mmap<T>::file_size(fd),
osmium::detail::typed_mmap<T>::file_size(fd)) {
}
void reserve(size_t new_capacity) {
if (new_capacity > this->capacity()) {
typed_mmap<T>::unmap(this->data(), this->capacity());
this->data(typed_mmap<T>::grow_and_map(new_capacity, this->m_fd));
this->m_capacity = new_capacity;
}
osmium::util::file_size(fd) / sizeof(T),
osmium::util::file_size(fd) / sizeof(T)) {
}
}; // class mmap_vector_file
third_party/libosmium/include/osmium/index/detail/typed_mmap.hpp
Vendored
-229
View File
@@ -1,229 +0,0 @@
#ifndef OSMIUM_INDEX_DETAIL_TYPED_MMAP_HPP
#define OSMIUM_INDEX_DETAIL_TYPED_MMAP_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <cerrno>
#include <cstddef>
#include <stdexcept>
#include <system_error>
#include <sys/stat.h>
#ifndef _WIN32
# include <sys/mman.h>
#else
# include <mmap_for_windows.hpp>
#endif
#ifndef _MSC_VER
# include <unistd.h>
#else
# define ftruncate _chsize
#endif
// for bsd systems
#ifndef MAP_ANONYMOUS
# define MAP_ANONYMOUS MAP_ANON
#endif
#include <osmium/util/cast.hpp>
namespace osmium {
/**
* @brief Namespace for Osmium internal use
*/
namespace detail {
/**
* This is a helper class for working with memory mapped files and
* anonymous shared memory. It wraps the necessary system calls
* adding:
* - error checking: all functions throw exceptions where needed
* - internal casts and size calculations allow use with user defined
* type T instead of void*
*
* This class only contains static functions. It should never be
* instantiated.
*
* @tparam T Type of objects we want to store.
*/
template <typename T>
class typed_mmap {
public:
/**
* Create anonymous private memory mapping with enough space for size
* objects of type T.
*
* Note that no constructor is called for any of the objects in this memory!
*
* @param size Number of objects of type T that should fit into this memory
* @returns Pointer to mapped memory
* @throws std::system_error If mmap(2) failed
*/
static T* map(size_t size) {
void* addr = ::mmap(nullptr, sizeof(T) * size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
if (addr == MAP_FAILED) {
throw std::system_error(errno, std::system_category(), "mmap failed");
}
#pragma GCC diagnostic pop
return reinterpret_cast<T*>(addr);
}
/**
* Create shared memory mapping of a file with enough space for size
* objects of type T. The file must already have at least the
* required size.
*
* Note that no constructor is called for any of the objects in this memory!
*
* @param size Number of objects of type T that should fit into this memory
* @param fd File descriptor
* @param write True if data should be writable
* @returns Pointer to mapped memory
* @throws std::system_error If mmap(2) failed
*/
static T* map(size_t size, int fd, bool write = false) {
int prot = PROT_READ;
if (write) {
prot |= PROT_WRITE;
}
void* addr = ::mmap(nullptr, sizeof(T) * size, prot, MAP_SHARED, fd, 0);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
if (addr == MAP_FAILED) {
throw std::system_error(errno, std::system_category(), "mmap failed");
}
#pragma GCC diagnostic pop
return reinterpret_cast<T*>(addr);
}
// mremap(2) is only available on linux systems
#ifdef __linux__
/**
* Grow memory mapping created with map().
*
* Note that no constructor is called for any of the objects in this memory!
*
* @param data Pointer to current mapping (as returned by typed_mmap())
* @param old_size Number of objects currently stored in this memory
* @param new_size Number of objects we want to have space for
* @throws std::system_error If mremap(2) call failed
*/
static T* remap(T* data, size_t old_size, size_t new_size) {
void* addr = ::mremap(reinterpret_cast<void*>(data), sizeof(T) * old_size, sizeof(T) * new_size, MREMAP_MAYMOVE);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
if (addr == MAP_FAILED) {
throw std::system_error(errno, std::system_category(), "mremap failed");
}
#pragma GCC diagnostic pop
return reinterpret_cast<T*>(addr);
}
#endif
/**
* Release memory from map() call.
*
* Note that no destructor is called for the objects in this memory!
*
* @param data Pointer to the data
* @param size Number of objects of type T stored
* @throws std::system_error If munmap(2) call failed
*/
static void unmap(T* data, size_t size) {
if (::munmap(reinterpret_cast<void*>(data), sizeof(T) * size) != 0) {
throw std::system_error(errno, std::system_category(), "munmap failed");
}
}
/**
* Get number of objects of type T that would fit into a file.
*
* @param fd File descriptor
* @returns Number of objects of type T in this file
* @throws std::system_error If fstat(2) call failed
* @throws std::length_error If size of the file isn't a multiple of sizeof(T)
*/
static size_t file_size(int fd) {
struct stat s;
if (fstat(fd, &s) < 0) {
throw std::system_error(errno, std::system_category(), "fstat failed");
}
if (static_cast<size_t>(s.st_size) % sizeof(T) != 0) {
throw std::length_error("file size has to be multiple of object size");
}
return static_cast<size_t>(s.st_size) / sizeof(T);
}
/**
* Grow file so there is enough space for at least new_size objects
* of type T. If the file is large enough already, nothing is done.
* The file is never shrunk.
*
* @param new_size Number of objects of type T that should fit into this file
* @param fd File descriptor
* @throws std::system_error If ftruncate(2) call failed
*/
static void grow_file(size_t new_size, int fd) {
if (file_size(fd) < new_size) {
if (::ftruncate(fd, static_cast_with_assert<off_t>(sizeof(T) * new_size)) < 0) {
throw std::system_error(errno, std::system_category(), "ftruncate failed");
}
}
}
/**
* Grow file to given size (if it is smaller) and mmap it.
*
* @param size Number of objects of type T that should fit into this file
* @param fd File descriptor
* @throws Errors thrown by grow_file() or map()
*/
static T* grow_and_map(size_t size, int fd) {
grow_file(size, fd);
return map(size, fd, true);
}
}; // class typed_mmap
} // namespace detail
} // namespace osmium
#endif // OSMIUM_INDEX_DETAIL_TYPED_MMAP_HPP
third_party/libosmium/include/osmium/index/detail/vector_map.hpp
Vendored
+9 -1
View File
@@ -68,7 +68,7 @@ namespace osmium {
m_vector(fd) {
}
~VectorBasedDenseMap() {}
~VectorBasedDenseMap() = default;
void reserve(const size_t size) override final {
m_vector.reserve(size);
@@ -97,6 +97,10 @@ namespace osmium {
return m_vector.size();
}
size_t byte_size() const {
return m_vector.size() * sizeof(element_type);
}
size_t used_memory() const override final {
return sizeof(TValue) * size();
}
@@ -106,6 +110,10 @@ namespace osmium {
m_vector.shrink_to_fit();
}
void dump_as_array(const int fd) override final {
osmium::io::detail::reliable_write(fd, reinterpret_cast<const char*>(m_vector.data()), byte_size());
}
iterator begin() {
return m_vector.begin();
}
third_party/libosmium/include/osmium/index/detail/vector_multimap.hpp
Vendored
+34
View File
@@ -67,6 +67,16 @@ namespace osmium {
public:
VectorBasedSparseMultimap() :
m_vector() {
}
explicit VectorBasedSparseMultimap(int fd) :
m_vector(fd) {
}
~VectorBasedSparseMultimap() = default;
void set(const TId id, const TValue value) override final {
m_vector.push_back(element_type(id, value));
}
@@ -141,6 +151,30 @@ namespace osmium {
osmium::io::detail::reliable_write(fd, reinterpret_cast<const char*>(m_vector.data()), byte_size());
}
iterator begin() {
return m_vector.begin();
}
iterator end() {
return m_vector.end();
}
const_iterator cbegin() const {
return m_vector.cbegin();
}
const_iterator cend() const {
return m_vector.cend();
}
const_iterator begin() const {
return m_vector.cbegin();
}
const_iterator end() const {
return m_vector.cend();
}
}; // class VectorBasedSparseMultimap
} // namespace multimap
third_party/libosmium/include/osmium/index/index.hpp
Vendored
+1 -1
View File
@@ -67,7 +67,7 @@ namespace osmium {
template <typename TKey>
OSMIUM_NORETURN void not_found_error(TKey key) {
std::stringstream s;
s << "id " << key << " no found";
s << "id " << key << " not found";
throw not_found(s.str());
}
third_party/libosmium/include/osmium/index/map.hpp
Vendored
+14 -2
View File
@@ -148,7 +148,11 @@ namespace osmium {
}
virtual void dump_as_list(const int /*fd*/) {
std::runtime_error("can't dump as list");
throw std::runtime_error("can't dump as list");
}
virtual void dump_as_array(const int /*fd*/) {
throw std::runtime_error("can't dump as array");
}
}; // class Map
@@ -195,6 +199,10 @@ namespace osmium {
return m_callbacks.emplace(map_type_name, func).second;
}
bool has_map_type(const std::string& map_type_name) const {
return m_callbacks.count(map_type_name);
}
std::vector<std::string> map_types() const {
std::vector<std::string> result;
@@ -242,9 +250,13 @@ namespace osmium {
});
}
#define OSMIUM_CONCATENATE_DETAIL_(x, y) x##y
#define OSMIUM_CONCATENATE_(x, y) OSMIUM_CONCATENATE_DETAIL_(x, y)
#define OSMIUM_MAKE_UNIQUE_(x) OSMIUM_CONCATENATE_(x, __COUNTER__)
#define REGISTER_MAP(id, value, klass, name) \
namespace { \
const bool registered_index_map_##name = osmium::index::register_map<id, value, klass>(#name); \
const bool OSMIUM_MAKE_UNIQUE_(registered_index_map_##name) = osmium::index::register_map<id, value, klass>(#name); \
}
} // namespace index
third_party/libosmium/include/osmium/index/map/dense_mmap_array.hpp
Vendored
+1 -1
View File
@@ -35,7 +35,7 @@ DEALINGS IN THE SOFTWARE.
#ifdef __linux__
#include <osmium/index/detail/mmap_vector_anon.hpp>
#include <osmium/index/detail/mmap_vector_anon.hpp> // IWYU pragma: keep
#include <osmium/index/detail/vector_map.hpp>
#define OSMIUM_HAS_INDEX_MAP_DENSE_MMAP_ARRAY
third_party/libosmium/include/osmium/index/map/sparse_mem_map.hpp
Vendored
+1 -1
View File
@@ -33,7 +33,7 @@ DEALINGS IN THE SOFTWARE.
*/
#include <algorithm>
#include <algorithm> // IWYU pragma: keep (for std::copy)
#include <cstddef>
#include <iterator>
#include <map>