Load data in two separate data regions

src/engine/datafacade/mmap_memory_allocator.cpp

@@ -1,5 +1,7 @@
 #include "engine/datafacade/mmap_memory_allocator.hpp"
 
+#include "storage/io.hpp"
+#include "storage/serialization.hpp"
 #include "storage/storage.hpp"
 
 #include "util/log.hpp"
@@ -22,31 +24,43 @@ MMapMemoryAllocator::MMapMemoryAllocator(const storage::StorageConfig &config,
     if (!boost::filesystem::exists(memory_file))
     {
         storage::DataLayout initial_layout;
-        storage.PopulateLayout(initial_layout);
+        storage.PopulateStaticLayout(initial_layout);
+        storage.PopulateUpdatableLayout(initial_layout);
 
         auto data_size = initial_layout.GetSizeOfLayout();
-        auto total_size = data_size + sizeof(storage::DataLayout);
+
+        storage::io::BufferWriter writer;
+        storage::serialization::write(writer, initial_layout);
+        auto encoded_layout = writer.GetBuffer();
+
+        auto total_size = data_size + encoded_layout.size();
 
         mapped_memory = util::mmapFile<char>(memory_file, mapped_memory_file, total_size);
 
-        data_layout = reinterpret_cast<storage::DataLayout *>(mapped_memory.data());
-        *data_layout = initial_layout;
-        storage.PopulateData(*data_layout, GetMemory());
+        std::copy(encoded_layout.begin(), encoded_layout.end(), mapped_memory.data());
+
+        index = storage::SharedDataIndex(
+            {{mapped_memory.data() + encoded_layout.size(), std::move(initial_layout)}});
+
+        storage.PopulateStaticData(index);
+        storage.PopulateUpdatableData(index);
     }
     else
     {
         mapped_memory = util::mmapFile<char>(memory_file, mapped_memory_file);
-        data_layout = reinterpret_cast<storage::DataLayout *>(mapped_memory.data());
+
+        storage::DataLayout layout;
+        storage::io::BufferReader reader(mapped_memory.data());
+        storage::serialization::read(reader, layout);
+        auto layout_size = reader.GetPosition();
+
+        index = storage::SharedDataIndex({{mapped_memory.data() + layout_size, std::move(layout)}});
     }
 }
 
 MMapMemoryAllocator::~MMapMemoryAllocator() {}
 
-storage::DataLayout &MMapMemoryAllocator::GetLayout() { return *data_layout; }
-char *MMapMemoryAllocator::GetMemory()
-{
-    return mapped_memory.data() + sizeof(storage::DataLayout);
-}
+const storage::SharedDataIndex &MMapMemoryAllocator::GetIndex() { return index; }
 
 } // namespace datafacade
 } // namespace engine

src/engine/datafacade/process_memory_allocator.cpp

@@ -15,17 +15,22 @@ ProcessMemoryAllocator::ProcessMemoryAllocator(const storage::StorageConfig &con
     storage::Storage storage(config);
 
     // Calculate the layout/size of the memory block
-    storage.PopulateLayout(internal_layout);
+    storage::DataLayout layout;
+    storage.PopulateStaticLayout(layout);
+    storage.PopulateUpdatableLayout(layout);
 
     // Allocate the memory block, then load data from files into it
-    internal_memory = std::make_unique<char[]>(internal_layout.GetSizeOfLayout());
-    storage.PopulateData(internal_layout, internal_memory.get());
+    internal_memory = std::make_unique<char[]>(layout.GetSizeOfLayout());
+
+    index = storage::SharedDataIndex({{internal_memory.get(), std::move(layout)}});
+
+    storage.PopulateStaticData(index);
+    storage.PopulateUpdatableData(index);
 }
 
 ProcessMemoryAllocator::~ProcessMemoryAllocator() {}
 
-const storage::DataLayout &ProcessMemoryAllocator::GetLayout() { return internal_layout; }
-char *ProcessMemoryAllocator::GetMemory() { return internal_memory.get(); }
+const storage::SharedDataIndex &ProcessMemoryAllocator::GetIndex() { return index; }
 
 } // namespace datafacade
 } // namespace engine

src/engine/datafacade/shared_memory_allocator.cpp

@@ -13,27 +13,32 @@ namespace engine
 namespace datafacade
 {
 
-SharedMemoryAllocator::SharedMemoryAllocator(storage::SharedRegionRegister::ShmKey data_shm_key)
+SharedMemoryAllocator::SharedMemoryAllocator(
+    const std::vector<storage::SharedRegionRegister::ShmKey> &shm_keys)
 {
-    util::Log(logDEBUG) << "Loading new data for region " << (int)data_shm_key;
+    std::vector<storage::SharedDataIndex::AllocatedRegion> regions;
 
-    BOOST_ASSERT(storage::SharedMemory::RegionExists(data_shm_key));
-    m_large_memory = storage::makeSharedMemory(data_shm_key);
+    for (const auto shm_key : shm_keys)
+    {
+        util::Log(logDEBUG) << "Loading new data for region " << (int)shm_key;
+        BOOST_ASSERT(storage::SharedMemory::RegionExists(shm_key));
+        auto mem = storage::makeSharedMemory(shm_key);
 
-    storage::io::BufferReader reader(reinterpret_cast<char *>(m_large_memory->Ptr()),
-                                     m_large_memory->Size());
-    storage::serialization::read(reader, data_layout);
-    layout_size = reader.GetPosition();
-    util::Log(logDEBUG) << "Data layout has size " << layout_size;
+        storage::io::BufferReader reader(reinterpret_cast<char *>(mem->Ptr()), mem->Size());
+        storage::DataLayout layout;
+        storage::serialization::read(reader, layout);
+        auto layout_size = reader.GetPosition();
+
+        regions.push_back({reinterpret_cast<char *>(mem->Ptr()) + layout_size, std::move(layout)});
+        memory_regions.push_back(std::move(mem));
+    }
+
+    index = storage::SharedDataIndex{std::move(regions)};
 }
 
 SharedMemoryAllocator::~SharedMemoryAllocator() {}
 
-const storage::DataLayout &SharedMemoryAllocator::GetLayout() { return data_layout; }
-char *SharedMemoryAllocator::GetMemory()
-{
-    return reinterpret_cast<char *>(m_large_memory->Ptr()) + layout_size;
-}
+const storage::SharedDataIndex &SharedMemoryAllocator::GetIndex() { return index; }
 
 } // namespace datafacade
 } // namespace engine