Refactor loading code for timestamp file

This commit is contained in:
Huyen Chau Nguyen 2016-10-18 11:21:41 -07:00
parent ab1a9271c8
commit 316ef305de
3 changed files with 31 additions and 11 deletions

View File

@ -13,6 +13,7 @@
#include "extractor/original_edge_data.hpp" #include "extractor/original_edge_data.hpp"
#include "extractor/profile_properties.hpp" #include "extractor/profile_properties.hpp"
#include "extractor/query_node.hpp" #include "extractor/query_node.hpp"
#include "storage/io.hpp"
#include "storage/storage_config.hpp" #include "storage/storage_config.hpp"
#include "storage/io.hpp" #include "storage/io.hpp"
#include "engine/geospatial_query.hpp" #include "engine/geospatial_query.hpp"
@ -71,7 +72,6 @@ class InternalDataFacade final : public BaseDataFacade
InternalDataFacade() {} InternalDataFacade() {}
unsigned m_check_sum; unsigned m_check_sum;
unsigned m_number_of_nodes;
std::unique_ptr<QueryGraph> m_query_graph; std::unique_ptr<QueryGraph> m_query_graph;
std::string m_timestamp; std::string m_timestamp;
@ -141,12 +141,17 @@ class InternalDataFacade final : public BaseDataFacade
void LoadTimestamp(const boost::filesystem::path &timestamp_path) void LoadTimestamp(const boost::filesystem::path &timestamp_path)
{ {
util::SimpleLogger().Write() << "Loading Timestamp"; util::SimpleLogger().Write() << "Loading Timestamp";
boost::filesystem::ifstream timestamp_stream(timestamp_path); boost::filesystem::ifstream timestamp_stream(timestamp_path);
if (!timestamp_stream) if (!timestamp_stream)
{ {
throw util::exception("Could not open " + timestamp_path.string() + " for reading."); throw util::exception("Could not open " + timestamp_path.string() + " for reading.");
} }
getline(timestamp_stream, m_timestamp);
auto timestamp_size = storage::io::readTimestampSize(timestamp_stream);
char *timestamp_ptr = new char[timestamp_size]();
storage::io::readTimestamp(timestamp_stream, timestamp_ptr, timestamp_size);
m_timestamp = std::string(timestamp_ptr);
} }
void LoadGraph(const boost::filesystem::path &hsgr_path) void LoadGraph(const boost::filesystem::path &hsgr_path)

View File

@ -48,8 +48,7 @@ inline HSGRHeader readHSGRHeader(boost::filesystem::ifstream &input_stream)
return header; return header;
} }
// Needs to be called after getHSGRSize() to get the correct offset in the stream // Needs to be called after HSGRHeader() to get the correct offset in the stream
//
template <typename NodeT, typename EdgeT> template <typename NodeT, typename EdgeT>
void readHSGR(boost::filesystem::ifstream &input_stream, void readHSGR(boost::filesystem::ifstream &input_stream,
NodeT *node_buffer, NodeT *node_buffer,
@ -61,6 +60,21 @@ void readHSGR(boost::filesystem::ifstream &input_stream,
input_stream.read(reinterpret_cast<char *>(edge_buffer), number_of_edges * sizeof(EdgeT)); input_stream.read(reinterpret_cast<char *>(edge_buffer), number_of_edges * sizeof(EdgeT));
} }
// Returns the size of the timestamp in a file
inline std::uint32_t readTimestampSize(boost::filesystem::ifstream &timestamp_input_stream)
{
timestamp_input_stream.seekg (0, timestamp_input_stream.end);
auto length = timestamp_input_stream.tellg();
timestamp_input_stream.seekg (0, timestamp_input_stream.beg);
return length;
}
// Reads the timestamp in a file
inline void readTimestamp(boost::filesystem::ifstream &timestamp_input_stream, char *timestamp, std::size_t timestamp_length)
{
timestamp_input_stream.read(timestamp, timestamp_length * sizeof(char));
}
} }
} }
} }

View File

@ -272,11 +272,14 @@ Storage::ReturnCode Storage::Run(int max_wait)
// load profile properties // load profile properties
shared_layout_ptr->SetBlockSize<extractor::ProfileProperties>(SharedDataLayout::PROPERTIES, 1); shared_layout_ptr->SetBlockSize<extractor::ProfileProperties>(SharedDataLayout::PROPERTIES, 1);
// load timestamp size // read timestampsize
boost::filesystem::ifstream timestamp_stream(config.timestamp_path); boost::filesystem::ifstream timestamp_stream(config.timestamp_path);
std::string m_timestamp; if (!timestamp_stream)
getline(timestamp_stream, m_timestamp); {
shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::TIMESTAMP, m_timestamp.length()); throw util::exception("Could not open " + config.timestamp_path.string() + " for reading.");
}
std::size_t timestamp_size = io::readTimestampSize(timestamp_stream);
shared_layout_ptr->SetBlockSize<char>(SharedDataLayout::TIMESTAMP, timestamp_size);
// load core marker size // load core marker size
boost::filesystem::ifstream core_marker_file(config.core_data_path, std::ios::binary); boost::filesystem::ifstream core_marker_file(config.core_data_path, std::ios::binary);
@ -659,7 +662,7 @@ Storage::ReturnCode Storage::Run(int max_wait)
// store timestamp // store timestamp
char *timestamp_ptr = char *timestamp_ptr =
shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr, SharedDataLayout::TIMESTAMP); shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr, SharedDataLayout::TIMESTAMP);
std::copy(m_timestamp.c_str(), m_timestamp.c_str() + m_timestamp.length(), timestamp_ptr); io::readTimestamp(timestamp_stream, timestamp_ptr, timestamp_size);
// store search tree portion of rtree // store search tree portion of rtree
char *rtree_ptr = shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr, char *rtree_ptr = shared_layout_ptr->GetBlockPtr<char, true>(shared_memory_ptr,
@ -771,7 +774,6 @@ Storage::ReturnCode Storage::Run(int max_wait)
static_cast<SharedDataTimestamp *>(data_type_memory->Ptr()); static_cast<SharedDataTimestamp *>(data_type_memory->Ptr());
{ {
boost::interprocess::scoped_lock<boost::interprocess::named_upgradable_mutex> boost::interprocess::scoped_lock<boost::interprocess::named_upgradable_mutex>
current_regions_exclusive_lock; current_regions_exclusive_lock;
@ -801,7 +803,6 @@ Storage::ReturnCode Storage::Run(int max_wait)
} }
util::SimpleLogger().Write() << "Ok."; util::SimpleLogger().Write() << "Ok.";
data_timestamp_ptr->layout = layout_region; data_timestamp_ptr->layout = layout_region;
data_timestamp_ptr->data = data_region; data_timestamp_ptr->data = data_region;
data_timestamp_ptr->timestamp += 1; data_timestamp_ptr->timestamp += 1;