Create public facing libraries for extractor, contractor and datastore

New libraries libosrm_extract, libosrm_contract, libosrm_store
2016-01-07 19:19:55 +01:00
parent b36145e3c4
commit 439eb9da3d
68 changed files with 3266 additions and 3104 deletions
@@ -0,0 +1,39 @@
+#ifndef SHARED_BARRIERS_HPP
+#define SHARED_BARRIERS_HPP
+
+#include <boost/interprocess/sync/named_mutex.hpp>
+#include <boost/interprocess/sync/named_condition.hpp>
+
+namespace osrm
+{
+namespace storage
+{
+struct SharedBarriers
+{
+
+    SharedBarriers()
+        : pending_update_mutex(boost::interprocess::open_or_create, "pending_update"),
+          update_mutex(boost::interprocess::open_or_create, "update"),
+          query_mutex(boost::interprocess::open_or_create, "query"),
+          no_running_queries_condition(boost::interprocess::open_or_create, "no_running_queries"),
+          update_ongoing(false), number_of_queries(0)
+    {
+    }
+
+    // Mutex to protect access to the boolean variable
+    boost::interprocess::named_mutex pending_update_mutex;
+    boost::interprocess::named_mutex update_mutex;
+    boost::interprocess::named_mutex query_mutex;
+
+    // Condition that no update is running
+    boost::interprocess::named_condition no_running_queries_condition;
+
+    // Is there an ongoing update?
+    bool update_ongoing;
+    // Is there any query?
+    int number_of_queries;
+};
+}
+}
+
+#endif // SHARED_BARRIERS_HPP
@@ -0,0 +1,132 @@
+#ifndef SHARED_DATA_TYPE_HPP
+#define SHARED_DATA_TYPE_HPP
+
+#include "util/osrm_exception.hpp"
+#include "util/simple_logger.hpp"
+
+#include <cstdint>
+
+#include <array>
+
+namespace osrm
+{
+namespace storage
+{
+
+// Added at the start and end of each block as sanity check
+const constexpr char CANARY[] = "OSRM";
+
+struct SharedDataLayout
+{
+    enum BlockID
+    {
+        NAME_OFFSETS = 0,
+        NAME_BLOCKS,
+        NAME_CHAR_LIST,
+        NAME_ID_LIST,
+        VIA_NODE_LIST,
+        GRAPH_NODE_LIST,
+        GRAPH_EDGE_LIST,
+        COORDINATE_LIST,
+        TURN_INSTRUCTION,
+        TRAVEL_MODE,
+        R_SEARCH_TREE,
+        GEOMETRIES_INDEX,
+        GEOMETRIES_LIST,
+        GEOMETRIES_INDICATORS,
+        HSGR_CHECKSUM,
+        TIMESTAMP,
+        FILE_INDEX_PATH,
+        CORE_MARKER,
+        NUM_BLOCKS
+    };
+
+    std::array<uint64_t, NUM_BLOCKS> num_entries;
+    std::array<uint64_t, NUM_BLOCKS> entry_size;
+
+    SharedDataLayout() : num_entries(), entry_size() {}
+
+    template <typename T> inline void SetBlockSize(BlockID bid, uint64_t entries)
+    {
+        num_entries[bid] = entries;
+        entry_size[bid] = sizeof(T);
+    }
+
+    inline uint64_t GetBlockSize(BlockID bid) const
+    {
+        // special bit encoding
+        if (bid == GEOMETRIES_INDICATORS || bid == CORE_MARKER)
+        {
+            return (num_entries[bid] / 32 + 1) * entry_size[bid];
+        }
+
+        return num_entries[bid] * entry_size[bid];
+    }
+
+    inline uint64_t GetSizeOfLayout() const
+    {
+        return GetBlockOffset(NUM_BLOCKS) + NUM_BLOCKS * 2 * sizeof(CANARY);
+    }
+
+    inline uint64_t GetBlockOffset(BlockID bid) const
+    {
+        uint64_t result = sizeof(CANARY);
+        for (auto i = 0; i < bid; i++)
+        {
+            result += GetBlockSize((BlockID)i) + 2 * sizeof(CANARY);
+        }
+        return result;
+    }
+
+    template <typename T, bool WRITE_CANARY = false>
+    inline T *GetBlockPtr(char *shared_memory, BlockID bid)
+    {
+        T *ptr = (T *)(shared_memory + GetBlockOffset(bid));
+        if (WRITE_CANARY)
+        {
+            char *start_canary_ptr = shared_memory + GetBlockOffset(bid) - sizeof(CANARY);
+            char *end_canary_ptr = shared_memory + GetBlockOffset(bid) + GetBlockSize(bid);
+            std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
+            std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
+        }
+        else
+        {
+            char *start_canary_ptr = shared_memory + GetBlockOffset(bid) - sizeof(CANARY);
+            char *end_canary_ptr = shared_memory + GetBlockOffset(bid) + GetBlockSize(bid);
+            bool start_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
+            bool end_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
+            if (!start_canary_alive)
+            {
+                throw util::exception("Start canary of block corrupted.");
+            }
+            if (!end_canary_alive)
+            {
+                throw util::exception("End canary of block corrupted.");
+            }
+        }
+
+        return ptr;
+    }
+};
+
+enum SharedDataType
+{
+    CURRENT_REGIONS,
+    LAYOUT_1,
+    DATA_1,
+    LAYOUT_2,
+    DATA_2,
+    LAYOUT_NONE,
+    DATA_NONE
+};
+
+struct SharedDataTimestamp
+{
+    SharedDataType layout;
+    SharedDataType data;
+    unsigned timestamp;
+};
+}
+}
+
+#endif /* SHARED_DATA_TYPE_HPP */
@@ -0,0 +1,358 @@
+#ifndef SHARED_MEMORY_HPP
+#define SHARED_MEMORY_HPP
+
+#include "util/osrm_exception.hpp"
+#include "util/simple_logger.hpp"
+
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/fstream.hpp>
+#include <boost/interprocess/mapped_region.hpp>
+#ifndef WIN32
+#include <boost/interprocess/xsi_shared_memory.hpp>
+#else
+#include <boost/interprocess/shared_memory_object.hpp>
+#endif
+
+#ifdef __linux__
+#include <sys/ipc.h>
+#include <sys/shm.h>
+#endif
+
+// #include <cstring>
+#include <cstdint>
+
+#include <algorithm>
+#include <exception>
+
+namespace osrm
+{
+namespace storage
+{
+
+struct OSRMLockFile
+{
+    boost::filesystem::path operator()()
+    {
+        boost::filesystem::path temp_dir = boost::filesystem::temp_directory_path();
+        boost::filesystem::path lock_file = temp_dir / "osrm.lock";
+        return lock_file;
+    }
+};
+
+#ifndef WIN32
+class SharedMemory
+{
+
+    // Remove shared memory on destruction
+    class shm_remove
+    {
+      private:
+        int m_shmid;
+        bool m_initialized;
+
+      public:
+        void SetID(int shmid)
+        {
+            m_shmid = shmid;
+            m_initialized = true;
+        }
+
+        shm_remove() : m_shmid(INT_MIN), m_initialized(false) {}
+        shm_remove(const shm_remove &) = delete;
+        ~shm_remove()
+        {
+            if (m_initialized)
+            {
+                util::SimpleLogger().Write(logDEBUG) << "automatic memory deallocation";
+                if (!boost::interprocess::xsi_shared_memory::remove(m_shmid))
+                {
+                    util::SimpleLogger().Write(logDEBUG) << "could not deallocate id " << m_shmid;
+                }
+            }
+        }
+    };
+
+  public:
+    void *Ptr() const { return region.get_address(); }
+
+    SharedMemory() = delete;
+    SharedMemory(const SharedMemory &) = delete;
+    SharedMemory &operator=(const SharedMemory &) = delete;
+
+    template <typename IdentifierT>
+    SharedMemory(const boost::filesystem::path &lock_file,
+                 const IdentifierT id,
+                 const uint64_t size = 0,
+                 bool read_write = false,
+                 bool remove_prev = true)
+        : key(lock_file.string().c_str(), id)
+    {
+        if (0 == size)
+        { // read_only
+            shm = boost::interprocess::xsi_shared_memory(boost::interprocess::open_only, key);
+
+            region = boost::interprocess::mapped_region(
+                shm,
+                (read_write ? boost::interprocess::read_write : boost::interprocess::read_only));
+        }
+        else
+        { // writeable pointer
+            // remove previously allocated mem
+            if (remove_prev)
+            {
+                Remove(key);
+            }
+            shm = boost::interprocess::xsi_shared_memory(boost::interprocess::open_or_create, key,
+                                                         size);
+#ifdef __linux__
+            if (-1 == shmctl(shm.get_shmid(), SHM_LOCK, nullptr))
+            {
+                if (ENOMEM == errno)
+                {
+                    util::SimpleLogger().Write(logWARNING) << "could not lock shared memory to RAM";
+                }
+            }
+#endif
+            region = boost::interprocess::mapped_region(shm, boost::interprocess::read_write);
+
+            remover.SetID(shm.get_shmid());
+            util::SimpleLogger().Write(logDEBUG) << "writeable memory allocated " << size
+                                                 << " bytes";
+        }
+    }
+
+    template <typename IdentifierT> static bool RegionExists(const IdentifierT id)
+    {
+        bool result = true;
+        try
+        {
+            OSRMLockFile lock_file;
+            boost::interprocess::xsi_key key(lock_file().string().c_str(), id);
+            result = RegionExists(key);
+        }
+        catch (...)
+        {
+            result = false;
+        }
+        return result;
+    }
+
+    template <typename IdentifierT> static bool Remove(const IdentifierT id)
+    {
+        OSRMLockFile lock_file;
+        boost::interprocess::xsi_key key(lock_file().string().c_str(), id);
+        return Remove(key);
+    }
+
+  private:
+    static bool RegionExists(const boost::interprocess::xsi_key &key)
+    {
+        bool result = true;
+        try
+        {
+            boost::interprocess::xsi_shared_memory shm(boost::interprocess::open_only, key);
+        }
+        catch (...)
+        {
+            result = false;
+        }
+        return result;
+    }
+
+    static bool Remove(const boost::interprocess::xsi_key &key)
+    {
+        bool ret = false;
+        try
+        {
+            util::SimpleLogger().Write(logDEBUG) << "deallocating prev memory";
+            boost::interprocess::xsi_shared_memory xsi(boost::interprocess::open_only, key);
+            ret = boost::interprocess::xsi_shared_memory::remove(xsi.get_shmid());
+        }
+        catch (const boost::interprocess::interprocess_exception &e)
+        {
+            if (e.get_error_code() != boost::interprocess::not_found_error)
+            {
+                throw;
+            }
+        }
+        return ret;
+    }
+
+    boost::interprocess::xsi_key key;
+    boost::interprocess::xsi_shared_memory shm;
+    boost::interprocess::mapped_region region;
+    shm_remove remover;
+};
+#else
+// Windows - specific code
+class SharedMemory
+{
+    SharedMemory(const SharedMemory &) = delete;
+    SharedMemory &operator=(const SharedMemory &) = delete;
+    // Remove shared memory on destruction
+    class shm_remove
+    {
+      private:
+        shm_remove(const shm_remove &) = delete;
+        shm_remove &operator=(const shm_remove &) = delete;
+        char *m_shmid;
+        bool m_initialized;
+
+      public:
+        void SetID(char *shmid)
+        {
+            m_shmid = shmid;
+            m_initialized = true;
+        }
+
+        shm_remove() : m_shmid("undefined"), m_initialized(false) {}
+
+        ~shm_remove()
+        {
+            if (m_initialized)
+            {
+                util::SimpleLogger().Write(logDEBUG) << "automatic memory deallocation";
+                if (!boost::interprocess::shared_memory_object::remove(m_shmid))
+                {
+                    util::SimpleLogger().Write(logDEBUG) << "could not deallocate id " << m_shmid;
+                }
+            }
+        }
+    };
+
+  public:
+    void *Ptr() const { return region.get_address(); }
+
+    SharedMemory(const boost::filesystem::path &lock_file,
+                 const int id,
+                 const uint64_t size = 0,
+                 bool read_write = false,
+                 bool remove_prev = true)
+    {
+        sprintf(key, "%s.%d", "osrm.lock", id);
+        if (0 == size)
+        { // read_only
+            shm = boost::interprocess::shared_memory_object(
+                boost::interprocess::open_only, key,
+                read_write ? boost::interprocess::read_write : boost::interprocess::read_only);
+            region = boost::interprocess::mapped_region(
+                shm, read_write ? boost::interprocess::read_write : boost::interprocess::read_only);
+        }
+        else
+        { // writeable pointer
+            // remove previously allocated mem
+            if (remove_prev)
+            {
+                Remove(key);
+            }
+            shm = boost::interprocess::shared_memory_object(boost::interprocess::open_or_create,
+                                                            key, boost::interprocess::read_write);
+            shm.truncate(size);
+            region = boost::interprocess::mapped_region(shm, boost::interprocess::read_write);
+
+            remover.SetID(key);
+            util::SimpleLogger().Write(logDEBUG) << "writeable memory allocated " << size
+                                                 << " bytes";
+        }
+    }
+
+    static bool RegionExists(const int id)
+    {
+        bool result = true;
+        try
+        {
+            char k[500];
+            build_key(id, k);
+            result = RegionExists(k);
+        }
+        catch (...)
+        {
+            result = false;
+        }
+        return result;
+    }
+
+    static bool Remove(const int id)
+    {
+        char k[500];
+        build_key(id, k);
+        return Remove(k);
+    }
+
+  private:
+    static void build_key(int id, char *key) { sprintf(key, "%s.%d", "osrm.lock", id); }
+
+    static bool RegionExists(const char *key)
+    {
+        bool result = true;
+        try
+        {
+            boost::interprocess::shared_memory_object shm(boost::interprocess::open_only, key,
+                                                          boost::interprocess::read_write);
+        }
+        catch (...)
+        {
+            result = false;
+        }
+        return result;
+    }
+
+    static bool Remove(char *key)
+    {
+        bool ret = false;
+        try
+        {
+            util::SimpleLogger().Write(logDEBUG) << "deallocating prev memory";
+            ret = boost::interprocess::shared_memory_object::remove(key);
+        }
+        catch (const boost::interprocess::interprocess_exception &e)
+        {
+            if (e.get_error_code() != boost::interprocess::not_found_error)
+            {
+                throw;
+            }
+        }
+        return ret;
+    }
+
+    char key[500];
+    boost::interprocess::shared_memory_object shm;
+    boost::interprocess::mapped_region region;
+    shm_remove remover;
+};
+#endif
+
+template <typename IdentifierT, typename LockFileT = OSRMLockFile>
+SharedMemory *makeSharedMemory(const IdentifierT &id,
+    const uint64_t size = 0,
+    bool read_write = false,
+    bool remove_prev = true)
+{
+  try
+  {
+    LockFileT lock_file;
+    if (!boost::filesystem::exists(lock_file()))
+    {
+      if (0 == size)
+      {
+        throw util::exception("lock file does not exist, exiting");
+      }
+      else
+      {
+        boost::filesystem::ofstream ofs(lock_file());
+      }
+    }
+    return new SharedMemory(lock_file(), id, size, read_write, remove_prev);
+  }
+  catch (const boost::interprocess::interprocess_exception &e)
+  {
+    util::SimpleLogger().Write(logWARNING) << "caught exception: " << e.what() << ", code "
+      << e.get_error_code();
+    throw util::exception(e.what());
+  }
+}
+
+}
+}
+
+#endif // SHARED_MEMORY_HPP
@@ -0,0 +1,25 @@
+#ifndef STORAGE_HPP
+#define STORAGE_HPP
+
+#include <boost/filesystem/path.hpp>
+
+#include <unordered_map>
+#include <string>
+
+namespace osrm
+{
+namespace storage
+{
+using DataPaths = std::unordered_map<std::string, boost::filesystem::path>;
+class Storage
+{
+public:
+    Storage(const DataPaths& data_paths);
+    int Run();
+private:
+    DataPaths paths;
+};
+}
+}
+
+#endif