Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Get offset of data inside tar file

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-22 12:15:40 +00:00
parent 8e800c48bc
commit a52213c885
4 changed files with 39 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
include/storage/tar.hpp
View File

@ -110,7 +110,13 @@ class FileReader
} }
} }
using TarEntry = std::tuple<std::string, std::size_t>; struct FileEntry
{
std::string name;
std::size_t size;
std::size_t offset;
};
template <typename OutIter> void List(OutIter out) template <typename OutIter> void List(OutIter out)
{ {
mtar_header_t header; mtar_header_t header;
@ -118,7 +124,12 @@ class FileReader
{ {
if (header.type == MTAR_TREG) if (header.type == MTAR_TREG)
{ {
*out++ = TarEntry{header.name, header.size}; mtar_read_data(&handle, nullptr, 0);
auto offset = handle.pos;
// seek back to the header
handle.remaining_data = 0;
mtar_seek(&handle, handle.last_header);
*out++ = FileEntry{header.name, header.size, offset};
} }
mtar_next(&handle); mtar_next(&handle);
} }

15
src/extractor/edge_based_graph_factory.cpp
View File

@ -423,8 +423,9 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
std::size_t node_based_edge_counter = 0; std::size_t node_based_edge_counter = 0;
storage::io::FileWriter turn_penalties_index_file(turn_penalties_index_filename, storage::tar::FileWriter turn_penalties_index_file(
storage::io::FileWriter::HasNoFingerprint); turn_penalties_index_filename, storage::tar::FileWriter::GenerateFingerprint);
turn_penalties_index_file.WriteFrom("/extractor/turn_index", (char *)nullptr, 0);
SuffixTable street_name_suffix_table(scripting_environment); SuffixTable street_name_suffix_table(scripting_environment);
const auto &turn_lanes_data = transformTurnLaneMapIntoArrays(lane_description_map); const auto &turn_lanes_data = transformTurnLaneMapIntoArrays(lane_description_map);
@ -1029,8 +1030,9 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
// Buffer writes to reduce syscall count // Buffer writes to reduce syscall count
if (turn_indexes_write_buffer.size() >= TURN_INDEX_WRITE_BUFFER_SIZE) if (turn_indexes_write_buffer.size() >= TURN_INDEX_WRITE_BUFFER_SIZE)
{ {
turn_penalties_index_file.WriteFrom(turn_indexes_write_buffer.data(), turn_penalties_index_file.ContinueFrom("/extractor/turn_index",
turn_indexes_write_buffer.size()); turn_indexes_write_buffer.data(),
turn_indexes_write_buffer.size());
turn_indexes_write_buffer.clear(); turn_indexes_write_buffer.clear();
} }
@ -1093,8 +1095,9 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
// Flush the turn_indexes_write_buffer if it's not empty // Flush the turn_indexes_write_buffer if it's not empty
if (!turn_indexes_write_buffer.empty()) if (!turn_indexes_write_buffer.empty())
{ {
turn_penalties_index_file.WriteFrom(turn_indexes_write_buffer.data(), turn_penalties_index_file.ContinueFrom("/extractor/turn_index",
turn_indexes_write_buffer.size()); turn_indexes_write_buffer.data(),
turn_indexes_write_buffer.size());
turn_indexes_write_buffer.clear(); turn_indexes_write_buffer.clear();
} }
} }

12
src/storage/storage.cpp
View File

@ -74,20 +74,16 @@ template <typename OutIter> void readBlocks(const boost::filesystem::path &path,
{ {
tar::FileReader reader(path, tar::FileReader::VerifyFingerprint); tar::FileReader reader(path, tar::FileReader::VerifyFingerprint);
std::vector<tar::FileReader::TarEntry> entries; std::vector<tar::FileReader::FileEntry> entries;
reader.List(std::back_inserter(entries)); reader.List(std::back_inserter(entries));
for (const auto &entry : entries) for (const auto &entry : entries)
{ {
std::string name; const auto name_end = entry.name.rfind(".meta");
std::uint64_t size;
std::tie(name, size) = entry;
const auto name_end = name.rfind(".meta");
if (name_end == std::string::npos) if (name_end == std::string::npos)
{ {
auto number_of_elements = reader.ReadElementCount64(name); auto number_of_elements = reader.ReadElementCount64(entry.name);
*out++ = NamedBlock{name, Block{number_of_elements, size}}; *out++ = NamedBlock{entry.name, Block{number_of_elements, entry.size}};
} }
} }
} }

23
unit_tests/storage/tar.cpp
View File

@ -14,19 +14,22 @@ BOOST_AUTO_TEST_CASE(list_tar_file)
storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar", storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar",
storage::tar::FileReader::HasNoFingerprint); storage::tar::FileReader::HasNoFingerprint);
std::vector<storage::tar::FileReader::TarEntry> file_list; std::vector<storage::tar::FileReader::FileEntry> file_list;
reader.List(std::back_inserter(file_list)); reader.List(std::back_inserter(file_list));
auto reference_0 = storage::tar::FileReader::TarEntry{"foo_1.txt", 4}; auto reference_0 = storage::tar::FileReader::FileEntry{"foo_1.txt", 4, 0x00000200};
auto reference_1 = storage::tar::FileReader::TarEntry{"bla/foo_2.txt", 4}; auto reference_1 = storage::tar::FileReader::FileEntry{"bla/foo_2.txt", 4, 0x00000600};
auto reference_2 = storage::tar::FileReader::TarEntry{"foo_3.txt", 4}; auto reference_2 = storage::tar::FileReader::FileEntry{"foo_3.txt", 4, 0x00000a00};
BOOST_CHECK_EQUAL(std::get<0>(file_list[0]), std::get<0>(reference_0)); BOOST_CHECK_EQUAL(file_list[0].name, reference_0.name);
BOOST_CHECK_EQUAL(std::get<1>(file_list[0]), std::get<1>(reference_0)); BOOST_CHECK_EQUAL(file_list[1].name, reference_1.name);
BOOST_CHECK_EQUAL(std::get<0>(file_list[1]), std::get<0>(reference_1)); BOOST_CHECK_EQUAL(file_list[2].name, reference_2.name);
BOOST_CHECK_EQUAL(std::get<1>(file_list[1]), std::get<1>(reference_1)); BOOST_CHECK_EQUAL(file_list[0].size, reference_0.size);
BOOST_CHECK_EQUAL(std::get<0>(file_list[2]), std::get<0>(reference_2)); BOOST_CHECK_EQUAL(file_list[1].size, reference_1.size);
BOOST_CHECK_EQUAL(std::get<1>(file_list[2]), std::get<1>(reference_2)); BOOST_CHECK_EQUAL(file_list[2].size, reference_2.size);
BOOST_CHECK_EQUAL(file_list[0].offset, reference_0.offset);
BOOST_CHECK_EQUAL(file_list[1].offset, reference_1.offset);
BOOST_CHECK_EQUAL(file_list[2].offset, reference_2.offset);
} }
BOOST_AUTO_TEST_CASE(read_tar_file) BOOST_AUTO_TEST_CASE(read_tar_file)