Merge pull request #123 from NuiCpp/feat/bin2hpp_compression

Added base64 and compressed index.hpp encoding.

.github/workflows/macos_13.yml

@@ -29,6 +29,13 @@ jobs:
       - name: Install Brew Dependencies
         run: brew install cmake llvm@16 boost git openssl@3 cryptopp curl ninja make
 
+      - name: Tool Version Dump
+        run: |
+            clang++ --version
+            cmake --version
+            ninja --version
+            brew list --versions boost
+
       - name: Configure CMake
         run: cmake -B ${{github.workspace}}/build/clang_${{env.BUILD_TYPE}} -G"Ninja" -DNUI_ENABLE_TESTS=on -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DNUI_BUILD_EXAMPLES=on -DCMAKE_CXX_EXTENSIONS=on -DCMAKE_CXX_COMPILER=/usr/local/opt/llvm@16/bin/clang++ -DCMAKE_C_COMPILER=/usr/local/opt/llvm@16/bin/clang -DNUI_NPM=npm -DNUI_NODE=node -DCMAKE_CXX_STANDARD=20
         env:

.github/workflows/ubuntu_20.yml

@@ -31,6 +31,12 @@ jobs:
           # OPTIONAL: Specify a platform version
           platform_version: 20.04
 
+      - name: Tool Version Dump
+        run: |
+            clang++ --version
+            cmake --version
+            ninja --version
+
       - name: Setup clang
         uses: egor-tensin/setup-clang@v1
         with:
@@ -40,7 +46,7 @@ jobs:
       - name: Configure CMake
         run: cmake -B ${{github.workspace}}/build/clang_${{env.BUILD_TYPE}} -G"Ninja" -DNUI_ENABLE_TESTS=on -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DNUI_BUILD_EXAMPLES=on -DCMAKE_CXX_EXTENSIONS=on -DCMAKE_CXX_COMPILER=c++ -DCMAKE_C_COMPILER=cc -DCMAKE_CXX_STANDARD=20
         env:
-          BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }}
+          Boost_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }}
 
       - name: Build
         run: cmake --build ${{github.workspace}}/build/clang_${{env.BUILD_TYPE}} --config ${{env.BUILD_TYPE}}

.github/workflows/ubuntu_22.yml

@@ -34,13 +34,19 @@ jobs:
           # OPTIONAL: Specify a platform version
           platform_version: 22.04
 
+      - name: Tool Version Dump
+        run: |
+          clang++ --version
+          cmake --version
+          ninja --version
+
       - name: Symlink xlocale
         run: sudo ln -s /usr/include/locale.h /usr/include/xlocale.h
 
       - name: Configure CMake
         run: cmake -B ${{github.workspace}}/build/clang_${{env.BUILD_TYPE}} -G"Ninja" -DNUI_ENABLE_TESTS=on -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DNUI_BUILD_EXAMPLES=on -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_CXX_EXTENSIONS=on -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -DCMAKE_LINKER=lld -DCMAKE_CXX_STANDARD=20
         env:
-          BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }}
+          Boost_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }}
 
       - name: Build
         run: cmake --build ${{github.workspace}}/build/clang_${{env.BUILD_TYPE}} --config ${{env.BUILD_TYPE}}

cmake/backend/emscripten.cmake

@@ -40,7 +40,7 @@ function(nui_add_emscripten_target)
     cmake_parse_arguments(
         NUI_ADD_EMSCRIPTEN_TARGET_ARGS
         "DISABLE_BIN2HPP;DISABLE_PARCEL_ADAPTER;ENABLE_TAILWIND;ENABLE_DOTENV"
-        "TARGET;PREJS;SOURCE_DIR"
+        "TARGET;PREJS;SOURCE_DIR;BIN2HPP_ENCODING"
         "CMAKE_OPTIONS"
         ${ARGN}
     )
@@ -99,8 +99,14 @@ function(nui_add_emscripten_target)
         set(BUILD_COMMAND BUILD_COMMAND cmake -E copy "${SOURCE_DIR}/package.json" "${CMAKE_BINARY_DIR}/module_${NUI_ADD_EMSCRIPTEN_TARGET_ARGS_TARGET}/package.json")
     endif()
 
+    if (NUI_ADD_EMSCRIPTEN_TARGET_ARGS_BIN2HPP_ENCODING)
+        set(BIN2HPP_ENCODING ${NUI_ADD_EMSCRIPTEN_TARGET_ARGS_BIN2HPP_ENCODING})
+    else()
+        set(BIN2HPP_ENCODING "raw")
+    endif()
+
     if (ENABLE_BIN2HPP AND ${ENABLE_BIN2HPP})
-        set(BIN2HPP_COMMAND COMMAND $<TARGET_FILE:bin2hpp> "on" "${CMAKE_BINARY_DIR}/module_${NUI_ADD_EMSCRIPTEN_TARGET_ARGS_TARGET}/bin/index.html" "${CMAKE_BINARY_DIR}/module_${NUI_ADD_EMSCRIPTEN_TARGET_ARGS_TARGET}/include/index.hpp" index)
+        set(BIN2HPP_COMMAND COMMAND $<TARGET_FILE:bin2hpp> "on" "${CMAKE_BINARY_DIR}/module_${NUI_ADD_EMSCRIPTEN_TARGET_ARGS_TARGET}/bin/index.html" "${CMAKE_BINARY_DIR}/module_${NUI_ADD_EMSCRIPTEN_TARGET_ARGS_TARGET}/include/index.hpp" index ${BIN2HPP_ENCODING})
     else()
         set(BIN2HPP_COMMAND COMMAND cmake -E true)
     endif()

cmake/dependencies/boost.cmake

@@ -1,5 +1,5 @@
 if (CMAKE_VERSION VERSION_LESS "3.30")
-    find_package(Boost 1.81.0 REQUIRED COMPONENTS system)
+    find_package(Boost 1.81.0 REQUIRED COMPONENTS system iostreams)
 else()
-    find_package(Boost CONFIG 1.81.0 REQUIRED COMPONENTS system)
+    find_package(Boost CONFIG 1.81.0 REQUIRED COMPONENTS system iostreams)
 endif()

cmake/dependencies/roar.cmake

@@ -1,6 +1,6 @@
 option(NUI_FETCH_ROAR "Fetch roar" ON)
 set(NUI_ROAR_REPOSITORY "https://github.com/5cript/roar.git" CACHE STRING "roar repository")
-set(NUI_ROAR_TAG "4ac9a877b70fbb821de2edbb7dd3fcad1b0a7f08" CACHE STRING "roar tag")
+set(NUI_ROAR_TAG "2781a88fcd1fce9af1a697673e26fd3ad55817e9" CACHE STRING "roar tag")
 
 if(NUI_FETCH_ROAR)
     include(FetchContent)

tools/bin2hpp/CMakeLists.txt

@@ -2,10 +2,14 @@ cmake_minimum_required(VERSION 3.16)
 
 project(bin2hpp VERSION 0.1.0)
 
-add_executable(bin2hpp main.cpp)
+add_executable(bin2hpp main.cpp raw_encoder.cpp base64_encoder.cpp compressed_encoder.cpp)
 target_compile_features(bin2hpp PRIVATE cxx_std_20)
 
+include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/dependencies/boost.cmake")
+
 set_target_properties(bin2hpp
     PROPERTIES
     RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools_bin"
-)
+)
+
+target_link_libraries(bin2hpp PRIVATE roar Boost::iostreams)

tools/bin2hpp/base64_encoder.cpp

@@ -0,0 +1,58 @@
+#include "base64_encoder.hpp"
+#include "raw_encoder.hpp"
+#include "constants.hpp"
+
+#include <roar/utility/base64.hpp>
+
+#include <iostream>
+#include <sstream>
+
+std::ostream& Base64Encoder::header(std::ostream& output) const
+{
+    output <<
+        R"(#pragma once
+
+#include <roar/utility/base64.hpp>
+
+#include <string_view>
+#include <string>
+
+static const std::string_view )"
+           << name_ << R"(_data[] = {)";
+
+    return output;
+}
+
+std::ostream& Base64Encoder::content(std::ostream& output, std::istream& input) const
+{
+    std::stringstream readStream;
+    readStream << input.rdbuf();
+
+    auto encoded = Roar::base64Encode(std::move(readStream).str());
+
+    std::cout << "Encoded size: " << encoded.size() << "\n";
+
+    std::stringstream ss(std::move(encoded));
+
+    RawEncoder rawEncoder(name_);
+    return rawEncoder.content(output, ss);
+}
+
+std::ostream& Base64Encoder::index(std::ostream& output) const
+{
+    output << "};\n\n";
+    output << "static std::string " << name_ << "()\n";
+    output << "{\n";
+    output << "\tstatic std::string memo;\n";
+    output << "\tif(!memo.empty())\n";
+    output << "\t\treturn memo;\n";
+    output << "\tfor (std::size_t i = 0; i != sizeof(" << name_ << "_data) / sizeof(std::string_view)"
+           << "; ++i) {\n";
+    output << "\t\tmemo += " << name_ << "_data[i];\n";
+    output << "\t}\n\n";
+    output << "\tmemo = Roar::base64Decode(memo);\n";
+    output << "\treturn memo;\n";
+    output << "}\n";
+
+    return output;
+}

tools/bin2hpp/base64_encoder.hpp

@@ -0,0 +1,13 @@
+#pragma once
+
+#include "encoder.hpp"
+
+class Base64Encoder : public Encoder
+{
+  public:
+    using Encoder::Encoder;
+
+    std::ostream& header(std::ostream& output) const override;
+    std::ostream& content(std::ostream& output, std::istream& input) const override;
+    std::ostream& index(std::ostream& output) const override;
+};

tools/bin2hpp/compressed_encoder.cpp

@@ -0,0 +1,74 @@
+#include "compressed_encoder.hpp"
+#include "raw_encoder.hpp"
+#include "constants.hpp"
+
+#include <roar/utility/base64.hpp>
+#include <boost/iostreams/filtering_stream.hpp>
+#include <boost/iostreams/copy.hpp>
+#include <boost/iostreams/filter/gzip.hpp>
+
+#include <iostream>
+#include <sstream>
+
+std::ostream& CompressedEncoder::header(std::ostream& output) const
+{
+    output <<
+        R"(#pragma once
+
+#include <roar/utility/base64.hpp>
+#include <boost/iostreams/filtering_stream.hpp>
+#include <boost/iostreams/copy.hpp>
+#include <boost/iostreams/filter/gzip.hpp>
+
+#include <string_view>
+#include <string>
+#include <sstream>
+
+static const std::string_view )"
+           << name_ << R"(_data[] = {)";
+
+    return output;
+}
+
+std::ostream& CompressedEncoder::content(std::ostream& output, std::istream& input) const
+{
+    boost::iostreams::filtering_streambuf<boost::iostreams::input> in;
+    in.push(boost::iostreams::gzip_compressor());
+    in.push(input);
+
+    std::stringstream compressedRaw;
+    boost::iostreams::copy(in, compressedRaw);
+
+    auto encoded = Roar::base64Encode(std::move(compressedRaw).str());
+    std::stringstream ss{std::move(encoded)};
+
+    RawEncoder rawEncoder(name_);
+    return rawEncoder.content(output, ss);
+}
+
+std::ostream& CompressedEncoder::index(std::ostream& output) const
+{
+    output << "};\n\n";
+    output << "static std::string " << name_ << "()\n";
+    output << "{\n";
+    output << "\tstatic std::string memo;\n";
+    output << "\tif(!memo.empty())\n";
+    output << "\t\treturn memo;\n\n";
+    output << "\tstd::string compressed;\n";
+    output << "\tfor (std::size_t i = 0; i != sizeof(" << name_ << "_data) / sizeof(std::string_view)"
+           << "; ++i) {\n";
+    output << "\t\tcompressed += " << name_ << "_data[i];\n";
+    output << "\t}\n\n";
+    output << "\tcompressed = Roar::base64Decode(compressed);\n";
+    output << "\tstd::stringstream compressedStream{compressed};\n";
+    output << "\tboost::iostreams::filtering_streambuf<boost::iostreams::input> in;\n";
+    output << "\tin.push(boost::iostreams::gzip_decompressor());\n";
+    output << "\tin.push(compressedStream);\n";
+    output << "\tstd::stringstream decompressed;\n";
+    output << "\tboost::iostreams::copy(in, decompressed);\n";
+    output << "\tmemo = std::move(decompressed).str();\n";
+    output << "\treturn memo;\n";
+    output << "}\n";
+
+    return output;
+}

tools/bin2hpp/compressed_encoder.hpp

@@ -0,0 +1,13 @@
+#pragma once
+
+#include "encoder.hpp"
+
+class CompressedEncoder : public Encoder
+{
+  public:
+    using Encoder::Encoder;
+
+    std::ostream& header(std::ostream& output) const override;
+    std::ostream& content(std::ostream& output, std::istream& input) const override;
+    std::ostream& index(std::ostream& output) const override;
+};

tools/bin2hpp/constants.hpp

@@ -0,0 +1,3 @@
+#pragma once
+
+constexpr std::size_t lineWidth = 120;

tools/bin2hpp/encoder.hpp

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <iosfwd>
+#include <string>
+
+class Encoder
+{
+  public:
+    Encoder(std::string name)
+        : name_(std::move(name))
+    {}
+
+    virtual std::ostream& header(std::ostream& output) const = 0;
+    virtual std::ostream& content(std::ostream& output, std::istream& input) const = 0;
+    virtual std::ostream& index(std::ostream& output) const = 0;
+
+    virtual ~Encoder() = default;
+    Encoder(Encoder const&) = default;
+    Encoder(Encoder&&) = default;
+    Encoder& operator=(Encoder const&) = default;
+    Encoder& operator=(Encoder&&) = default;
+
+  protected:
+    std::string name_;
+};