Skip to content

Commit

Permalink
feat(metadata_freezer): log metadata freezing time
Browse files Browse the repository at this point in the history
  • Loading branch information
mhx committed Nov 27, 2024
1 parent 4f20bb9 commit 2a7fccc
Show file tree
Hide file tree
Showing 3 changed files with 50 additions and 7 deletions.
23 changes: 21 additions & 2 deletions include/dwarfs/writer/internal/metadata_freezer.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,14 @@
#pragma once

#include <cstdint>
#include <memory>
#include <utility>
#include <vector>

namespace dwarfs {

class logger;

namespace thrift::metadata {
class metadata;
}
Expand All @@ -35,8 +38,24 @@ namespace writer::internal {

class metadata_freezer {
public:
static std::pair<std::vector<uint8_t>, std::vector<uint8_t>>
freeze(const thrift::metadata::metadata& data);
metadata_freezer(logger& lgr);
~metadata_freezer();

std::pair<std::vector<uint8_t>, std::vector<uint8_t>>
freeze(thrift::metadata::metadata const& data) const {
return impl_->freeze(data);
}

class impl {
public:
virtual ~impl() = default;

virtual std::pair<std::vector<uint8_t>, std::vector<uint8_t>>
freeze(thrift::metadata::metadata const& data) const = 0;
};

private:
std::unique_ptr<impl> impl_;
};

} // namespace writer::internal
Expand Down
32 changes: 28 additions & 4 deletions src/writer/internal/metadata_freezer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
#include <thrift/lib/cpp2/frozen/FrozenUtil.h>
#include <thrift/lib/cpp2/protocol/Serializer.h>

#include <dwarfs/logger.h>

#include <dwarfs/writer/internal/metadata_freezer.h>

#include <dwarfs/gen-cpp2/metadata_layouts.h>
Expand Down Expand Up @@ -59,11 +61,33 @@ freeze_to_buffer(const T& x) {
return {schema_buffer, data_buffer};
}

template <typename LoggerPolicy>
class metadata_freezer_ : public metadata_freezer::impl {
public:
explicit metadata_freezer_(logger& lgr)
: LOG_PROXY_INIT(lgr) {}

std::pair<std::vector<uint8_t>, std::vector<uint8_t>>
freeze(thrift::metadata::metadata const& data) const override {
auto ti = LOG_TIMED_VERBOSE;
auto rv = freeze_to_buffer(data);

ti << "freezing metadata to " << rv.second.size() << " bytes...";

return rv;
}

private:
LOG_PROXY_DECL(LoggerPolicy);
};

} // namespace

std::pair<std::vector<uint8_t>, std::vector<uint8_t>>
metadata_freezer::freeze(const thrift::metadata::metadata& data) {
return freeze_to_buffer(data);
}
metadata_freezer::metadata_freezer(logger& lgr)
: impl_{
make_unique_logging_object<impl, metadata_freezer_, logger_policies>(
lgr)} {}

metadata_freezer::~metadata_freezer() = default;

} // namespace dwarfs::writer::internal
2 changes: 1 addition & 1 deletion src/writer/scanner.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -887,7 +887,7 @@ void scanner_<LoggerPolicy>::scan(
mdb.set_total_hardlink_size(prog.hardlink_size);
mdb.gather_global_entry_data(ge_data);

auto [schema, data] = metadata_freezer::freeze(mdb.build());
auto [schema, data] = metadata_freezer(LOG_GET_LOGGER).freeze(mdb.build());

LOG_VERBOSE << "uncompressed metadata size: " << size_with_unit(data.size());

Expand Down

0 comments on commit 2a7fccc

Please sign in to comment.