diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..889e683 --- /dev/null +++ b/.gitignore @@ -0,0 +1,116 @@ +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# Build and install directories +build +test + +# MacOS +.DS_Store + +# Files +*.json +*.log + +# Symlinks to executables +encoder +decoder + +# CMake junk +CMakeCache.txt +/CMakeFiles/ +cmake_install.cmake +Makefile + +# Editor Junk +/.vscode/ +/local_backup/ +.idea/.gitignore +.idea/.name +.idea/dev-huffman.iml +.idea/misc.xml +.idea/modules.xml +.idea/vcs.xml +cmake-build-debug/build.ninja +cmake-build-debug/.cmake/api/v1/query/cache-v2 +cmake-build-debug/.cmake/api/v1/query/cmakeFiles-v1 +cmake-build-debug/.cmake/api/v1/query/codemodel-v2 +cmake-build-debug/.cmake/api/v1/query/toolchains-v1 +cmake-build-debug/CMakeFiles/clion-environment.txt +cmake-build-debug/CMakeFiles/clion-log.txt +cmake-build-debug/CMakeFiles/cmake.check_cache +cmake-build-debug/CMakeFiles/rules.ninja +cmake-build-debug/CMakeFiles/TargetDirectories.txt +cmake-build-debug/CMakeFiles/3.24.3/CMakeCCompiler.cmake +cmake-build-debug/CMakeFiles/3.24.3/CMakeCXXCompiler.cmake +cmake-build-debug/CMakeFiles/3.24.3/CMakeDetermineCompilerABI_C.bin +cmake-build-debug/CMakeFiles/3.24.3/CMakeDetermineCompilerABI_CXX.bin +cmake-build-debug/CMakeFiles/3.24.3/CMakeSystem.cmake +cmake-build-debug/CMakeFiles/3.24.3/CompilerIdC/CMakeCCompilerId.c +cmake-build-debug/CMakeFiles/3.24.3/CompilerIdCXX/CMakeCXXCompilerId.cpp +cmake-build-release/.ninja_deps +cmake-build-release/.ninja_log +cmake-build-release/build.ninja +cmake-build-release/dbgPrettyPrint.diag +cmake-build-release/decomp +cmake-build-release/huffman +cmake-build-release/.cmake/api/v1/query/cache-v2 +cmake-build-release/.cmake/api/v1/query/cmakeFiles-v1 +cmake-build-release/.cmake/api/v1/query/codemodel-v2 +cmake-build-release/.cmake/api/v1/query/toolchains-v1 +cmake-build-release/CMakeFiles/clion-environment.txt +cmake-build-release/CMakeFiles/clion-log.txt +cmake-build-release/CMakeFiles/cmake.check_cache +cmake-build-release/CMakeFiles/rules.ninja +cmake-build-release/CMakeFiles/TargetDirectories.txt +cmake-build-release/CMakeFiles/3.24.3/CMakeCCompiler.cmake +cmake-build-release/CMakeFiles/3.24.3/CMakeCXXCompiler.cmake +cmake-build-release/CMakeFiles/3.24.3/CMakeDetermineCompilerABI_C.bin +cmake-build-release/CMakeFiles/3.24.3/CMakeDetermineCompilerABI_CXX.bin +cmake-build-release/CMakeFiles/3.24.3/CMakeSystem.cmake +cmake-build-release/CMakeFiles/3.24.3/CompilerIdC/CMakeCCompilerId.c +cmake-build-release/CMakeFiles/3.24.3/CompilerIdCXX/CMakeCXXCompilerId.cpp +src/dbgPrettyPrint.diag +src/decoder_main +src/encodeMapResultOutput.bin +src/HuffmanD +src/dbgPrettyPrint.diag +src/decoder_main +src/encodeMapResultOutput.bin +src/HuffmanD +dbgPrettyPrint.diag +decompress +de +encodeMapResultOutput.bin +main +outputFull.bin diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..74e6b69 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "sample-models"] + path = sample-models + url = git@github.com:UKTechArena/sample-models.git diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..2994bbb --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,29 @@ +# $COPYRIGHT + +cmake_minimum_required(VERSION 3.0.0) +project(example-template VERSION 0.1.0) + +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +include_directories( + "${PROJECT_SOURCE_DIR}/include" +) + +list(APPEND samples encoder) +list(APPEND samples decoder) +# list(APPEND samples serialize) + + +file(GLOB_RECURSE core_sources ${PROJECT_SOURCE_DIR}/src/core/*.cpp) +foreach(sample ${samples}) + set(exec ${sample}) + file(GLOB_RECURSE ${sample}_sources ${PROJECT_SOURCE_DIR}/src/${sample}/*.cpp) + add_executable(${exec} + ${PROJECT_SOURCE_DIR}/src/${sample}_main.cpp + ${core_sources} + ${${sample}_sources}) + target_compile_options(${exec} PUBLIC -std=gnu++17 -Wall -O3) + install(TARGETS ${exec} DESTINATION bin) +endforeach(sample) \ No newline at end of file diff --git a/NOTES.md b/NOTES.md new file mode 100644 index 0000000..5e617cf --- /dev/null +++ b/NOTES.md @@ -0,0 +1,34 @@ +It seems that the whole system is tested using syslink `./decoder` and `./encoder`. So it's cool to use python, just don't forget to add shebang and give it `x` permission. + +Command to run the test in `scripts/test_non_ai.sh` +```bash +./encoder -i $input_file_path -o $encoded_file_path > /dev/null 2>&1 +``` + +Build_non_ai is modified to make it work with python. Check the content of it and it's easy to understand. + +## Performance of GZIP (Baseline) +``` +Average compression ratio: 0.585738 percent +Average decompression time: 27.1818 milliseconds +Average image quality metric 0 decibels + +Weighted compression score (35%): 20.5008 +Weighted time score (25%): 24.3205 +Weighted quality score (20%): 0 + +Total score (80%): 44.8213 +``` + +## Performance of the command of `cp` +``` +Average compression ratio: 0 percent +Average decompression time: 1.18182 milliseconds +Average image quality metric 0 decibels + +Weighted compression score (35%): 0 +Weighted time score (25%): 24.9705 +Weighted quality score (20%): 0 + +Total score (80%): 24.9705 +``` \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..06e8a39 --- /dev/null +++ b/README.md @@ -0,0 +1,252 @@ +# LZSS Understanding and Improving +Original Rules: https://github.com/UKTechArena + +Original Team Project: https://github.com/UKTechArena/crazy-thursday +## Score +``` +Average compression ratio: 0.83859 percent +Average decompression time: 24.6667 milliseconds +Average image quality metric 100 decibels + +Weighted compression score (35%): 29.3506 +Weighted time score (25%): 24.3833 +Weighted quality score (20%): 20 + +Total score (80%): 73.7339 +``` +## LZSS Compression Algorithm (Explained by a Demo) +``` +First byte of the Literal run: flag L4-L0, which flag indicates which level is used now. + 000 // level 1 + *(ubyte*)OUTPUT_ |= (1 << 7); // level 2 + *(ubyte*)OUTPUT_ |= (1 << 6); // level 3 + *(ubyte*)OUTPUT_ |= (1 << 5); // level 4 + + *(ubyte*)OUTPUT_ |= (1 << 5); // level 5 + *(ubyte*)OUTPUT_ |= (1 << 7); +Reuse: +``` +``` +***Level 1: Supported Windows Size 8192 2^13 (0-8191) + Windows Size 0-8191 +Original Match-> Encoded Len + 1-2 Bytes -> 1 Bytes (Literal Run) + 3-8 Bytes -> 2 Bytes (Short Match) + 9-264 Bytes -> 3 Bytes (Long Match) +| Instruction type | Decode[0] | Decode[1] | Decode[2] | Decode[3] | +|------------------|-------------|------------|--------------|--------------| +| Literal run | 000 L4-L0 | | | | +| Short match | M2-M0 W12-W8| W7-W0 | | | +| Long match | 111 W12-W8| M7-M0 | W7-W0 | | + + Demo +Original: + ------------ + I am Sam \n + Sam I am \n + That + ------------ +Encoded: +-------------------------------------------------------------------------- + 28 I am Sam \n + 20 03 00 20 40 0C 04 \nThat +-------------------------------------------------------------------------- + |short match| |literal run| |short match| |literal run| +High 3bits Match length 001+2 1 Match length 010+2=4 5 + Offset 3 Copy space(20) Offset 12 Copy \nThat + Sam [space] I am \nThat + + +That Sam-I-am! +That Sam-I-am! +I do not like +that Sam-I-am! +Do you like green eggs and ham? +I do not like them, Sam-I-am. +I do not like green eggs and ham. +``` +``` +***** Level 2: Extended Windows & Infinite Match Length + Supported Windows Size 65535 + 8191 (0-65535 + 8191 ) + Windows Size 0-8190 (8191 as flag) +Original Data-> Encoded Len + 1-2 Bytes -> 1 Bytes (Literal Run) + 3-8 Bytes -> 2 Bytes (Short Match) + 9+ Bytes -> 3 Bytes (Long Match) + +* distance >= MAX_L2_DISTANCE 8191 ( 13 bits of 1 for the high ) + 5-8 Bytes -> 4 Bytes (Extended Windows Size: Short Match) + Windows Size 8191 - (65535 + 8191 - 1) (shouldn' be all 1, to avoid unable to detect end!) + 9-264 Bytes -> 4+ Bytes (Extended Windows Size: Long Match) + Depends on Match length + +| Instruction type | Decode[0] | Decode[1] | Decode[2] | Decode[3] | Decode[4] | +|------------------|-------------|------------|--------------|--------------|--------------| +| Literal run | 000 L4-L0 | | | | +| Short match | M2-M0 W12-W8| W7-W0 | | | +| Long match | 111 W12-W8| M7-M0... | W7-W0 | | +| Extended Windows*| ----------------------------------------------------------------------| +| Short match | M2-M0 11111 | 1111 1111 | W15-W8 | W7-W0 | +| Long match | 111 11111 | M7-M0... | 1111 1111 | W15-W8 | W7-W0 | + ...Depends on Match length + +``` +``` +******* Level 3: Extra Windows Size + Supported Windows Size 65535 + 8191 - 1 (0-65535 + 8191 - 1) +Windows Size 0-8190 + Original Data-> Encoded Len + 1-2 Bytes -> 1 Bytes (Literal Run) + 3-8 Bytes -> 2 Bytes (Short Match) + 9+ Bytes -> 3 Bytes (Long Match) + +* distance >= MAX_L2_DISTANCE 8191 ( 13 bits of 1 for the high ) + 5-8 Bytes -> 4 Bytes (Extended Windows Size: Short Match) + Windows Size 8191 - (65535 + 8191 - 2) (shouldn' be all 1, to avoid unable to detect end!) + 9-264 Bytes -> 4+ Bytes (Extended Windows Size: Long Match) + Depends on Match length + +* distance >= MAX_L3_DISTANCE 8191 ( 13 bits of 1 for the high ) + 5-8 Bytes -> 4 Bytes (Extended Windows Size: Short Match) + Windows Size (65535 + 8191 - 1) - (65535 + 65535 + 8191 - 2) (shouldn' be all 1, to avoid unable to detect end!) + 9-264 Bytes -> 4+ Bytes (Extended Windows Size: Long Match) + Depends on Match length + +| Instruction type | Decode[0] | Decode[1] | Decode[2] | Decode[3] | Decode[4] | Decode[5] | Decode[6] | +|------------------|-------------|------------|--------------|-----------|-----------|-----------|-----------| +| Literal run | 000 L4-L0 | | | | | | | +| Short match | M2-M0 W12-W8| W7-W0 | | | | | | +| Long match | 111 W12-W8| M7-M0... | W7-W0 | | | | | +| Extended Windows*| ----------------------------------------------------------------------------------------| +| Short match | M2-M0 11111 | 1111 1111 | W15-W8 | W7-W0 | | | | +| Long match | 111 11111 | M7-M0... | 1111 1111 | W15-W8 | W7-W0 | | | +| Extended Windows2| ----------------------------------------------------------------------------------------| +| Extra Short match| M2-M0 11111 | 1111 1111 | 1111 1111 | 1111 1111 | W15-W8 | W7-W0 | | +| Extra Long match | 111 11111 | M7-M0... | 1111 1111 | 1111 1111 | 1111 1111 | W15-W8 | W7-W0 | + ...Depends on Match length +``` +``` +** level4: Ultra Windows Size + Larger Windows Size +``` +``` +** level5: Direct Match +| Instruction type | Decode[0] | Decode[1] | Decode[2] | Decode[3] | Decode[4] | Decode[5] | Decode[6] | +|------------------|-------------|------------|--------------|-----------|-----------|-----------|-----------| +| Literal run | 000 L4-L0 | | | | | | | +| Short match | M2-M0 W12-W8| W7-W0 | | | | | | + +| Long match | 111 W12-W8| M7-M0... | W7-W0 | | | | | +| Direct match | 110 W12-W8| D15-D8 | D7-D0 | W7-W0 | | | +| Long match | 111 W12-W8| 1111 1111 | W7-W0 | | | | | +| Extended Windows*| ----------------------------------------------------------------------------------------| +| Short match | M2-M0 11111 | 1111 1111 | W15-W8 | W7-W0 | | | | +| Long match | 111 11111 | M7-M0... | 1111 1111 | W15-W8 | W7-W0 | | | +| Extended Windows2| ----------------------------------------------------------------------------------------| +| Extra Short match| M2-M0 11111 | 1111 1111 | 1111 1111 | 1111 1111 | W15-W8 | W7-W0 | | +| Extra Long match | 111 11111 | M7-M0... | 1111 1111 | 1111 1111 | 1111 1111 | W15-W8 | W7-W0 | + + ...Depends on Match length + + Original Data -> Encoded Len +| Literal run | 1-2 Bytes -> 1 Bytes (Literal Run) +| 000 L4-L0 | | | | | | | + + +| Short match | 3-7 Bytes -> 2 Bytes (Short Match) [001-101 +6] +| M2-M0 W12-W8| W7-W0 | | | | | | +------------------------------------------------- -1*125977 (110) ------------------------------------------------- +| Long match 1| 8-262 Bytes -> 3 Bytes / 1 Bytes for Match_Len [00-FE +255] +| 111 W12-W8| M7-M0... | W7-W0 | | | | | +match_len>260 30 0000 +match_len>260+256. 146146 +match_len>260+256+256 8181 + Max 61042 (238) +match_len>65797 0 +| Direct match| 263-65797 Bytes -> 4 Bytes / 2 Bytes for Match_Len [262+2^16-1] +2^16-1 +| 110 W12-W8| D15-D8 | D7-D0 | W7-W0 | | | +-----------------------------------len>260+256---- +i*146146 -[i:1-238]--------------------------------------------- +| Long match 2| 65798+ Bytes -> 4+ Bytes / 2+ Bytes for Match_Len +| 111 W12-W8| 1111 1111 | M15-M8... | M7-M0... ｜ W7-W0 | | | | +``` +``` +*** Level6: Direct Long Match For 34MB +-----------------------------------len>260+256---- +i*146146 -[i:1-238]--------------------------------------------- +| Long match 2| 65798+ Bytes -> 4+ Bytes / 2-5 Bytes for Match_Len +Explain in detail: First three rows: 256*3-1 + Stop when not 1111 1111 +| 111 W12-W8| 1111 1111 | M15-M8... | M7-M0... ｜ W7-W0 | | | | +| 111 W12-W8| 1111 1111 | M23-M16.. | M15-M8... | M7-M0... ｜ W7-W0 | | | +| 111 W12-W8| 1111 1111 | M31-M24.. | M23-M16.. | M15-M8...| M7-M0... ｜ W7-W0 | | +| 111 W12-W8| 1111 1111 | 0000 0000 | D31-D24 | D23-D16 | D15-D8 | D7-D0 | W7-W0 | +``` +``` +*** Level6: Direct Long Match For 34MB +-----------------------------------len>260+256---- +i*146146 -[i:1-238]--------------------------------------------- +| Long match 2| 65798+ Bytes -> 4+ Bytes / 2-5 Bytes for Match_Len +Explain in detail: First three rows: 256*3-1 + Stop when not 1111 1111 +| 111 W12-W8| 1111 1111 | M15-M8... | M7-M0... ｜ W7-W0 | | | | +| 111 W12-W8| 1111 1111 | M23-M16.. | M15-M8... | M7-M0... ｜ W7-W0 | | | +| 111 W12-W8| 1111 1111 | 0000 0000 | D23-D16 | D15-D8 | D7-D0 | W7-W0 | +``` + +## Update +Compress / Decompress\ +Using namespace std; +Record time spent: +``` +chrono::time_point begin_time= + std::chrono::system_clock::now(); + //sleep(10); + auto end_time = std::chrono::system_clock::now(); + chrono::duration duration_mili = end_time - begin_time; + + printf("PrintDuration : duration_mili duration = %ld ms", (long)duration_mili.count()); +``` +Updated Concurrent implementation + Unit Reading of I/O + Multinputle Threads of Decompression + + +### Optimization +Ongoing TESTING here +### WHAT Have Been Done +Implemented from LZSS, using Object Oriented Programming. + +Branch prediction [[likely]] + +#define LEVEL1_MAX 65536 + IN + int Compress_LZ(const void* INPUT, int length, void* OUTPUT) { + if (length < LEVEL1_MAX) return Compress_level1(INPUT, length, OUTPUT); + return Compress_level2(INPUT, length, OUTPUT); + } + +Two Interfaces: + LZSS_Comp_once(); + LZSS_Compression(Compress_Twice); + Defined in LZSS_helper.h: bool Compress_Twice; + +Changed the I/O (tested) and removed anything unrelated to the algorithm + +Deleted duplicate v, vn, vt + +Treating every input in compression as ubyte (uint8_t) +## How to build and test +### Creating Your Repository +Please look at the original document. + +Run the `scripts/build_non_ai.sh` script to build the executables. + +``` +bash scripts/build_non_ai.sh +``` + +### Testing the Executables + +Use `scripts/test_obj.sh` instead of `scripts/test_non_ai.sh` script runs all the sample models through the encoder and decoder. + +``` +bash scripts/test_non_ai.sh +``` diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..9364c28 --- /dev/null +++ b/build.sh @@ -0,0 +1 @@ +bash scripts/build_non_ai.sh diff --git a/include/LZSS.md b/include/LZSS.md new file mode 100644 index 0000000..4e58fa3 --- /dev/null +++ b/include/LZSS.md @@ -0,0 +1,188 @@ +# LZSS Understanding and Improving + +## LZSS Compression Algorithm (Explained by a Demo) +``` +First byte of the Literal run: flag L4-L0, which flag indicates which level is used now. + 000 // level 1 + *(ubyte*)OUTPUT_ |= (1 << 7); // level 2 + *(ubyte*)OUTPUT_ |= (1 << 6); // level 3 + *(ubyte*)OUTPUT_ |= (1 << 5); // level 4 + + *(ubyte*)OUTPUT_ |= (1 << 5); // level 5 + *(ubyte*)OUTPUT_ |= (1 << 7); +Reuse: +***Level 1: Supported Windows Size 8192 2^13 (0-8191) + Windows Size 0-8191 +Original Match-> Encoded Len + 1-2 Bytes -> 1 Bytes (Literal Run) + 3-8 Bytes -> 2 Bytes (Short Match) + 9-264 Bytes -> 3 Bytes (Long Match) +| Instruction type | Decode[0] | Decode[1] | Decode[2] | Decode[3] | +|------------------|-------------|------------|--------------|--------------| +| Literal run | 000 L4-L0 | | | | +| Short match | M2-M0 W12-W8| W7-W0 | | | +| Long match | 111 W12-W8| M7-M0 | W7-W0 | | + + Demo +Original: + ------------ + I am Sam \n + Sam I am \n + That + ------------ +Encoded: +-------------------------------------------------------------------------- + 28 I am Sam \n + 20 03 00 20 40 0C 04 \nThat +-------------------------------------------------------------------------- + |short match| |literal run| |short match| |literal run| +High 3bits Match length 001+2 1 Match length 010+2=4 5 + Offset 3 Copy space(20) Offset 12 Copy \nThat + Sam [space] I am \nThat + + +That Sam-I-am! +That Sam-I-am! +I do not like +that Sam-I-am! +Do you like green eggs and ham? +I do not like them, Sam-I-am. +I do not like green eggs and ham. + +***** Level 2: Extended Windows & Infinite Match Length + Supported Windows Size 65535 + 8191 (0-65535 + 8191 ) + Windows Size 0-8190 (8191 as flag) +Original Data-> Encoded Len + 1-2 Bytes -> 1 Bytes (Literal Run) + 3-8 Bytes -> 2 Bytes (Short Match) + 9+ Bytes -> 3 Bytes (Long Match) + +* distance >= MAX_L2_DISTANCE 8191 ( 13 bits of 1 for the high ) + 5-8 Bytes -> 4 Bytes (Extended Windows Size: Short Match) + Windows Size 8191 - (65535 + 8191 - 1) (shouldn' be all 1, to avoid unable to detect end!) + 9-264 Bytes -> 4+ Bytes (Extended Windows Size: Long Match) + Depends on Match length + +| Instruction type | Decode[0] | Decode[1] | Decode[2] | Decode[3] | Decode[4] | +|------------------|-------------|------------|--------------|--------------|--------------| +| Literal run | 000 L4-L0 | | | | +| Short match | M2-M0 W12-W8| W7-W0 | | | +| Long match | 111 W12-W8| M7-M0... | W7-W0 | | +| Extended Windows*| ----------------------------------------------------------------------| +| Short match | M2-M0 11111 | 1111 1111 | W15-W8 | W7-W0 | +| Long match | 111 11111 | M7-M0... | 1111 1111 | W15-W8 | W7-W0 | + ...Depends on Match length + + +******* Level 3: Extra Windows Size + Supported Windows Size 65535 + 8191 - 1 (0-65535 + 8191 - 1) +Windows Size 0-8190 + Original Data-> Encoded Len + 1-2 Bytes -> 1 Bytes (Literal Run) + 3-8 Bytes -> 2 Bytes (Short Match) + 9+ Bytes -> 3 Bytes (Long Match) + +* distance >= MAX_L2_DISTANCE 8191 ( 13 bits of 1 for the high ) + 5-8 Bytes -> 4 Bytes (Extended Windows Size: Short Match) + Windows Size 8191 - (65535 + 8191 - 2) (shouldn' be all 1, to avoid unable to detect end!) + 9-264 Bytes -> 4+ Bytes (Extended Windows Size: Long Match) + Depends on Match length + +* distance >= MAX_L3_DISTANCE 8191 ( 13 bits of 1 for the high ) + 5-8 Bytes -> 4 Bytes (Extended Windows Size: Short Match) + Windows Size (65535 + 8191 - 1) - (65535 + 65535 + 8191 - 2) (shouldn' be all 1, to avoid unable to detect end!) + 9-264 Bytes -> 4+ Bytes (Extended Windows Size: Long Match) + Depends on Match length + +| Instruction type | Decode[0] | Decode[1] | Decode[2] | Decode[3] | Decode[4] | Decode[5] | Decode[6] | +|------------------|-------------|------------|--------------|-----------|-----------|-----------|-----------| +| Literal run | 000 L4-L0 | | | | | | | +| Short match | M2-M0 W12-W8| W7-W0 | | | | | | +| Long match | 111 W12-W8| M7-M0... | W7-W0 | | | | | +| Extended Windows*| ----------------------------------------------------------------------------------------| +| Short match | M2-M0 11111 | 1111 1111 | W15-W8 | W7-W0 | | | | +| Long match | 111 11111 | M7-M0... | 1111 1111 | W15-W8 | W7-W0 | | | +| Extended Windows2| ----------------------------------------------------------------------------------------| +| Extra Short match| M2-M0 11111 | 1111 1111 | 1111 1111 | 1111 1111 | W15-W8 | W7-W0 | | +| Extra Long match | 111 11111 | M7-M0... | 1111 1111 | 1111 1111 | 1111 1111 | W15-W8 | W7-W0 | + ...Depends on Match length + +** level4: Ultra Windows Size + Larger Windows Size + +** level5: Direct Match +| Instruction type | Decode[0] | Decode[1] | Decode[2] | Decode[3] | Decode[4] | Decode[5] | Decode[6] | +|------------------|-------------|------------|--------------|-----------|-----------|-----------|-----------| +| Literal run | 000 L4-L0 | | | | | | | +| Short match | M2-M0 W12-W8| W7-W0 | | | | | | + +| Long match | 111 W12-W8| M7-M0... | W7-W0 | | | | | +| Direct match | 110 W12-W8| D15-D8 | D7-D0 | W7-W0 | | | +| Long match | 111 W12-W8| 1111 1111 | W7-W0 | | | | | +| Extended Windows*| ----------------------------------------------------------------------------------------| +| Short match | M2-M0 11111 | 1111 1111 | W15-W8 | W7-W0 | | | | +| Long match | 111 11111 | M7-M0... | 1111 1111 | W15-W8 | W7-W0 | | | +| Extended Windows2| ----------------------------------------------------------------------------------------| +| Extra Short match| M2-M0 11111 | 1111 1111 | 1111 1111 | 1111 1111 | W15-W8 | W7-W0 | | +| Extra Long match | 111 11111 | M7-M0... | 1111 1111 | 1111 1111 | 1111 1111 | W15-W8 | W7-W0 | + + ...Depends on Match length + + Original Data -> Encoded Len +| Literal run | 1-2 Bytes -> 1 Bytes (Literal Run) +| 000 L4-L0 | | | | | | | + + +| Short match | 3-7 Bytes -> 2 Bytes (Short Match) [001-101 +6] +| M2-M0 W12-W8| W7-W0 | | | | | | +------------------------------------------------- -1*125977 (110) ------------------------------------------------- +| Long match 1| 8-262 Bytes -> 3 Bytes / 1 Bytes for Match_Len [00-FE +255] +| 111 W12-W8| M7-M0... | W7-W0 | | | | | +match_len>260 30 0000 +match_len>260+256. 146146 +match_len>260+256+256 8181 + Max 61042 (238) +match_len>65797 0 +| Direct match| 263-65797 Bytes -> 4 Bytes / 2 Bytes for Match_Len [262+2^16-1] +2^16-1 +| 110 W12-W8| D15-D8 | D7-D0 | W7-W0 | | | +-----------------------------------len>260+256---- +i*146146 -[i:1-238]--------------------------------------------- +| Long match 2| 65798+ Bytes -> 4+ Bytes / 2+ Bytes for Match_Len +| 111 W12-W8| 1111 1111 | M15-M8... | M7-M0... ｜ W7-W0 | | | | + +*** Level6: Direct Long Match For 34MB +-----------------------------------len>260+256---- +i*146146 -[i:1-238]--------------------------------------------- +| Long match 2| 65798+ Bytes -> 4+ Bytes / 2-5 Bytes for Match_Len +Explain in detail: First three rows: 256*3-1 + Stop when not 1111 1111 +| 111 W12-W8| 1111 1111 | M15-M8... | M7-M0... ｜ W7-W0 | | | | +| 111 W12-W8| 1111 1111 | M23-M16.. | M15-M8... | M7-M0... ｜ W7-W0 | | | +| 111 W12-W8| 1111 1111 | M31-M24.. | M23-M16.. | M15-M8...| M7-M0... ｜ W7-W0 | | +| 111 W12-W8| 1111 1111 | 0000 0000 | D31-D24 | D23-D16 | D15-D8 | D7-D0 | W7-W0 | + +*** Level6: Direct Long Match For 34MB +-----------------------------------len>260+256---- +i*146146 -[i:1-238]--------------------------------------------- +| Long match 2| 65798+ Bytes -> 4+ Bytes / 2-5 Bytes for Match_Len +Explain in detail: First three rows: 256*3-1 + Stop when not 1111 1111 +| 111 W12-W8| 1111 1111 | M15-M8... | M7-M0... ｜ W7-W0 | | | | +| 111 W12-W8| 1111 1111 | M23-M16.. | M15-M8... | M7-M0... ｜ W7-W0 | | | +| 111 W12-W8| 1111 1111 | 0000 0000 | D23-D16 | D15-D8 | D7-D0 | W7-W0 | +``` + +## Update +Compress / Decompress\ +Using namespace std; +Record time spent: +``` +chrono::time_point begin_time= + std::chrono::system_clock::now(); + //sleep(10); + auto end_time = std::chrono::system_clock::now(); + chrono::duration duration_mili = end_time - begin_time; + + printf("PrintDuration : duration_mili duration = %ld ms", (long)duration_mili.count()); +``` +Updated Concurrent implementation + Unit Reading of I/O + Multinputle Threads of Decompression + diff --git a/include/LZSS_Compression.h b/include/LZSS_Compression.h new file mode 100644 index 0000000..c475604 --- /dev/null +++ b/include/LZSS_Compression.h @@ -0,0 +1,641 @@ +/* + + ----------------------------------------------------------------- + + Reference: + + RFC 1951 / DEFLATE Compressed Data Format Specification + + Ziv J., Lempel A., "A Universal Algorithm for sequenceuential Data + Compression", IEEE Transactions on Information Theory, Vol. 23, + No. 3, pp. 337-343. + + Lempel–Ziv–Storer–Szymanski + https://en.wikipedia.org/wiki/Lempel–Ziv–Storer–Szymanski#cite_note-1 + https://en.wikipedia.org/wiki/LZ77_and_LZ78#LZ77 + + Data Compression - Lecture 10 - Lempel-Ziv Schemes + https://www.youtube.com/watch?v=VDXBnmr8AY0&list=PLU4IQLU9e_OpnkbCS_to64F_vw5yyg4HB&index=3 + + ------------------------------------------------------------------------------------------ +*/ +#include "LZSS_helper.h" +class HashTable{ + public: + uint table[HASH_SIZE]; // sequence (every 3 B) -> hash + HashTable(){ + for (uint hash = 0; hash < HASH_SIZE; ++hash){ + table[hash] = 0; + } + } + + /** + * @brief Creating the hash index for the sequence given + * @param seq 3 bytes sequence given + * @return uint16_t the hash index + */ + static uint16_t hash_func(uint seq) { + uint hash = (seq * 2654435769LL) >> (32 - 14); //HASH_LOG=14 + return hash & Hash_Key_Masks; + } +}; + +class LZSS_Encoder{ + private: + const void* INPUT_; + int LENGTH_; //length of the block of data to be compressed + void* OUTPUT_; + const ubyte* input; // pointer to input data + ubyte* output ; // pointer to output data + + const ubyte* input_start; + const ubyte* input_bound; + const ubyte* input_limit; //? + + HashTable Hash; //initialized + uint hash; // 0 - {(hash_size/1<<14)-1} + const ubyte* pivot ;// first of unwritten byte + const ubyte* ref; //pointer to ubyte (starting point of three bytes) of the same three bytes given + uint sequence;// a sequence of 3 bytes from the third of the input data given.(For the first two, no need to implement) + + uint distance; //offset from current position to the reference + uint cmp; //cmp with sequence given + uint match_len; + uint last_num_bytes;//last several literals + int length_after; + uint Window_Num; //level4 + public: + LZSS_Encoder(const void* INPUT, int length, void* OUTPUT_):INPUT_(INPUT), input((const ubyte*)INPUT),Hash(){ + this->LENGTH_=length; + this->OUTPUT_=OUTPUT_; + length_after=0; + output = (ubyte*)OUTPUT_; + pivot = input; + input_start = input; + input_bound = input + LENGTH_ - 4; /* because readU32 */ + input_limit = input + LENGTH_ - 12 - 1; //? + + input += 2;//Omit the first two char + Window_Num=2; + } + ~LZSS_Encoder(){} + int Compress(){ + /* for short block, choose level1 */ + if (LENGTH_ < LEVEL1_MAX) { + this->level3(); + return length_after; + } + /* else length > LEVEL1_MAX */ + this->level3(); + return length_after; + } + int Compress(int level){ + if (level==1) { + this->level1(); + return length_after; + } + else if (level==2){ + this->level2(); + return length_after; + } + else if (level==3){ + this->level3(); + return length_after; + } + return 0; + } + private: + /* Copies the values of (count+1) bytes (MAX OF 256 bytes) from the location pointed by source to + the memory block pointed by destination.*/ + void copy(ubyte* destination, const ubyte* source, uint count) { + const ull* src = (const ull*)source;//64Bytes + ull* dest = (ull*)destination; + *dest++ = *src++; //0-7 + if(count<16){ //8-15 + *dest++ = *src++; + }else if(count>=24){//24-31 + *dest++ = *src++; + *dest++ = *src++; + *dest++ = *src++; + }else{//16-23 + *dest++ = *src++; + *dest++ = *src++; + } + } + /** + * @brief *destination++ = COPY_MAX - 1; + * @param runs numbers of bytes to be written + * @param source start point + * @param destination + * @return uint8_t* + */ + ubyte* Literals_Output(uint runs, const ubyte* source, ubyte* destination) { + while (runs >0 ) { //COPY_MAX 32*8 + uint this_run =runs; + if(runs>32) { + this_run=32; + } + *destination++ = this_run - 1; + this->copy(destination, source,this_run); + source += this_run; + destination += this_run; + runs -= this_run; + } + return destination; + } + /** + * @brief return updated output (from short and long match) + * + * @param len + * @param distance + * @param output + * @return uint8_t* + */ + ubyte* Match_Output1(uint len, uint distance, ubyte* output) { + --distance; + uint index=0; + if (len > 264 - 2)[[unlikely]] //Max of match length 256+8 + while (len > 264 - 2) { + output[index++] = (7 << 5) + (distance >> 8); + output[index++] = 264 - 2 - 7 - 2; + output[index++] = (distance & 0xff); + len -= 264 - 2; + } + if (len < 7) {//short match + output[index++] = (len << 5) + (distance >> 8); + output[index++] = (distance & 0xff); + } else {//len==7 long match + output[index++] = (7 << 5) + (distance >> 8); + output[index++] = len - 7; + output[index++] = (distance & 0xff); + } + return &output[index]; + } + + ubyte* Match_Output2(uint len, uint distance, ubyte* output) { + --distance; + uint index=0; + if (distance < MAX_L2_Length) { + if (len < 7) { + output[index++] = (len << 5) + (distance >> 8); + output[index++] = (distance & 0xff); + } else { + output[index++] = (7 << 5) + (distance >> 8); + for (len -= 7; len >= 0xff; len -= 0xff) output[index++] = 0xff; + output[index++] = len; + output[index++] = (distance & 0xff); + } + } else { + /* still far away... */ + if (len < 7) {//Level 2 Extended Windows* Short match + distance -= MAX_L2_Length; + output[index++] = (len << 5) + 0x1f; //0b111 11111 + output[index++] = 0xff; + output[index++] = distance >> 8; + output[index++] = distance & 0xff; + } else {//Level 2 Extended Windows* Long match + distance -= MAX_L2_Length; + output[index++] = (7 << 5) + 0x1f; //0b111 11111 + for (len -= 7; len >= 0xff; len -= 0xff) output[index++] = 0xff; + output[index++] = len; //M7-M0... + output[index++] = 0xff; //0b 1111 1111 + output[index++] = distance >> 8; //W15-W8 + output[index++] = distance & 0xff; // W7-W0 + } + } + return &output[index]; + } + + ubyte* Match_Output3(uint len, uint distance, ubyte* output) { + --distance; + uint index=0; + if (distance < MAX_L2_Length) { + if (len < 7) { + output[index++] = (len << 5) + (distance >> 8); + output[index++] = (distance & 0xff); + } else { + output[index++] = (7 << 5) + (distance >> 8); + for (len -= 7; len >= 0xff; len -= 0xff) output[index++] = 0xff; + output[index++] = len; + output[index++] = (distance & 0xff); + } + } else if(distance> 8; + output[index++] = distance & 0xff; + } else {//Level 2 Extended Windows* Long match + distance -= MAX_L2_Length; + output[index++] = (7 << 5) + 0x1f; + for (len -= 7; len >= 0xff; len -= 0xff) output[index++] = 0xff; + output[index++] = len; + output[index++] = 0xff; + output[index++] = distance >> 8; + output[index++] = distance & 0xff; + } + } else{ //level3 + if (len < 7) {//Level 3 Extended Windows* Short match + distance -= MAX_L2_Length; + distance -= 65535; + output[index++] = (len << 5) + 0x1f; + output[index++] = 0xff; + output[index++] = 0xff; //0b 1111 1111 + output[index++] = 0xff; //0b 1111 1111 + output[index++] = distance >> 8; + output[index++] = distance & 0xff; + } else {//Level 3 Extended Windows* Long match + distance -= MAX_L2_Length; + distance -= 65535; + output[index++] = (7 << 5) + 0x1f; + for (len -= 7; len >= 0xff; len -= 0xff) output[index++] = 0xff; + output[index++] = len; + output[index++] = 0xff; + output[index++] = 0xff; //0b 1111 1111 + output[index++] = 0xff; //0b 1111 1111 + output[index++] = distance >> 8; + output[index++] = distance & 0xff; + } + } + return &output[index]; + } + ubyte* Match_Output4(uint len, uint distance, ubyte* output) { //Window_Num=n + --distance; + uint index=0; + if (distance < MAX_L2_Length) { + if (len < 7) { + output[index++] = (len << 5) + (distance >> 8); + output[index++] = (distance & 0xff); + } else { + output[index++] = (7 << 5) + (distance >> 8); + for (len -= 7; len >= 0xff; len -= 0xff) output[index++] = 0xff; + output[index++] = len; + output[index++] = (distance & 0xff); + } + } + for(uint i=1;i<=Window_Num;i++){ + if (distance>= MAX_L2_Length&& distance > 8; + output[index++] = distance & 0xff; + break; + } else { + output[index++] = (7 << 5) + 0x1f; + for (len -= 7; len >= 0xff; len -= 0xff) output[index++] = 0xff; + output[index++] = len; + output[index++] = 0xff; + for(uint j=2;j<=i;j++){ + output[index++] = 0xff; //0b 1111 1111 + output[index++] = 0xff; //0b 1111 1111 + } + output[index++] = distance >> 8; + output[index++] = distance & 0xff; + } + break; + } + } + return &output[index]; + } + ubyte* Match_Output5(uint len, uint distance, ubyte* output) { //Window_Num=2 + --distance; + uint index=0; + if (distance < MAX_L2_Length) { + if (len < 6) { + output[index++] = (len << 5) + (distance >> 8); + output[index++] = (distance & 0xff); + } else if(len<6+255){ //not 1111 1111 + output[index++] = (7 << 5) + (distance >> 8); //110 + // for (len -= 6; len >= 0xff; len -= 0xff) output[index++] = 0xff; + len-=6; + output[index++] = len; + output[index++] = (distance & 0xff); + }else if(len<6+255+(1<<16)){ + uint flag=6; + len-=6; + len-=255; + output[index++] = (flag << 5) + (distance >> 8); + output[index++]=(len>>8); //D15-D0 + output[index++]=(len&0xff); + output[index++] = (distance & 0xff); + + }else if(len>=6+255+(1<<16)){ + output[index++] = (7 << 5) + (distance >> 8); //110 + output[index++] = 0xff; //1111 1111 + for (len -= 6; len >= 0xff; len -= 0xff) output[index++] = 0xff; + output[index++] = len; + output[index++] = (distance & 0xff); + } + } else if(distance6 + distance -= MAX_L2_Length; + output[index++] = (len << 5) + 0x1f; + output[index++] = 0xff; + output[index++] = distance >> 8; + output[index++] = distance & 0xff; + } else if(len<6+255){//Level 2 Extended Windows* Long match + distance -= MAX_L2_Length; + output[index++] = (7 << 5) + 0x1f; //110 + // for (len -= 6; len >= 0xff; len -= 0xff) output[index++] = 0xff; + len-=6; + output[index++] = len; + output[index++] = 0xff; + output[index++] = distance >> 8; + output[index++] = distance & 0xff; + }else if(len<6+255+(1<<16)){ + uint flag=6; + len-=6; + len-=255; + output[index++] = (flag << 5) + (distance >> 8); + output[index++]=(len>>8); + output[index++]=(len&0xff); + output[index++] = (distance & 0xff); + + }else if(len>=6+255+(1<<16)){ + output[index++] = (7 << 5) + (distance >> 8); //110 + output[index++] = 0xff; //1111 1111 + for (len -= 6; len >= 0xff; len -= 0xff) output[index++] = 0xff; + len-=6; + output[index++] = len; + output[index++] = (distance & 0xff); + } + } else{ //level3 + if (len < 6) {//Level 3 Extended Windows* Short match + distance -= MAX_L2_Length; + distance -= 65535; + output[index++] = (len << 5) + 0x1f; + output[index++] = 0xff; + output[index++] = 0xff; //0b 1111 1111 + output[index++] = 0xff; //0b 1111 1111 + output[index++] = distance >> 8; + output[index++] = distance & 0xff; + } else if(len<6+255){ //Level 3 Extended Windows* Long match //not 1111 1111 + distance -= MAX_L2_Length; + distance -= 65535; + output[index++] = (7 << 5) + (distance >> 8); //110 + // for (len -= 6; len >= 0xff; len -= 0xff) output[index++] = 0xff; + len-=6; + output[index++] = len; + output[index++] = 0xff; + output[index++] = 0xff; //0b 1111 1111 + output[index++] = 0xff; //0b 1111 1111 + output[index++] = (distance & 0xff); + }else if(len<6+255+(1<<16)){ + distance -= MAX_L2_Length; + distance -= 65535; + uint flag=6; + len-=6; + len-=255; + output[index++] = (flag << 5) + (distance >> 8); + output[index++]=(len>>8); //D15-D0 + output[index++]=(len&0xff); + output[index++] = 0xff; + output[index++] = 0xff; //0b 1111 1111 + output[index++] = 0xff; //0b 1111 1111 + output[index++] = (distance & 0xff); + + }else if(len>=6+255+(1<<16)){ + distance -= MAX_L2_Length; + distance -= 65535; + output[index++] = (7 << 5) + (distance >> 8); //110 + output[index++] = 0xff; //1111 1111 + for (len -= 6; len >= 0xff; len -= 0xff) output[index++] = 0xff; + output[index++] = len; + output[index++] = 0xff; + output[index++] = 0xff; //0b 1111 1111 + output[index++] = 0xff; //0b 1111 1111 + output[index++] = (distance & 0xff); + } + } + return &output[index]; + } + bool Matched_First_Every3Bytes(uint windows_size){ + sequence = read_uint32(input) & 0xffffff; //take every three bytes as a sequence + //(p[2] << 16) | (p[1] << 8) | p[0] from current input store consecutive 3 bytes. + hash = HashTable::hash_func(sequence);//create the hash index + ref = input_start + Hash.table[hash]; //input_start+ uint16 + Hash.table[hash] = input - input_start;//update the current index to hash table + distance = input - ref; //offset + cmp = (__builtin_expect(!!(distance < windows_size), 1)) ? read_uint32(ref) + & 0xffffff : 0x1000000; + //check distance < WINDOW SIZE + //GNU Extension //likely + return(sequence==cmp); + } + + // The bytes before are matched, there exists a ref already ,so no need to update + void Update_Hash(){ + input += match_len; //for the match boundary, the second byte of the last matched sequence + sequence = read_uint32(input); // update hash table + hash = HashTable::hash_func(sequence & 0xffffff); + Hash.table[hash] = input++ - input_start;//the third byte of the last matched sequence + sequence >>= 8; // & 0xffffff + hash = HashTable::hash_func(sequence); + Hash.table[hash] = input++ - input_start;//first of unwritten byte + } + + void level1() { + // literal copy + while (input < input_limit)[[likely]] { + // find potential match + while(input <= input_limit) [[likely]]{ + if (Matched_First_Every3Bytes(WINDOW_SIZE_L1)) break; + ++input; //move on 1 byte to take a new sequence + } + if (input >= input_limit) [[unlikely]]break; + //write literal run part to the output + output = Literals_Output(input - pivot, pivot, output); + // copy(input-pivot+1) bytes from pivot to output + + //write short/long match part to the output + match_len = match_cmp(ref + 3, input + 3, input_bound); + //ignore compared part and compare until input_bound (EOF) + output = Match_Output1( match_len, distance, output); + Update_Hash(); + pivot = input; // -> updated first of unwritten byte + }// end of while + + last_num_bytes = (ubyte*)INPUT_ + LENGTH_ - pivot;// the rest of unmatched part + output = Literals_Output(last_num_bytes, pivot, output);//copy from pivot to EOF to output + + length_after= output - (ubyte*)OUTPUT_; + } + + void level2() { + while (input < input_limit)[[likely]] { + // potential match + while(input <= input_limit)[[likely]]{ + if(Matched_First_Every3Bytes(65535 + MAX_L2_Length )) break; + ++input; + } // if not match to the previous, continue to looping + + if (input >= input_limit) [[unlikely]]break; + // if match to the previous one(>=3) + + /* far, needs at least 5-byte match */ + if (distance >= MAX_L2_Length) { + if (ref[3] != input[3] || ref[4] != input[4]) { + ++input; + continue; + } + } + output = Literals_Output(input - pivot, pivot, output); + + match_len = match_cmp(ref + 3, input + 3, input_bound); + output = Match_Output2( match_len, distance, output); + + Update_Hash(); + + pivot = input; + } + + last_num_bytes = (ubyte*)INPUT_ + LENGTH_ - pivot; + output = Literals_Output (last_num_bytes, pivot, output); + + // id of this level + *(ubyte*)OUTPUT_ |= (1 << 7); // default is all 0s + + length_after= output - (ubyte*)OUTPUT_; + } + void level3() { + while (input < input_limit)[[likely]] { + // potential match + while(input <= input_limit)[[likely]]{ + if(Matched_First_Every3Bytes(65535 +65535 + MAX_L2_Length )) break; + ++input; + } // if not match to the previous, continue to looping + + if (input >= input_limit) [[unlikely]]break; + // if match to the previous one(>=3) + + /* far, needs at least 5-byte match */ + if (distance >= MAX_L2_Length) { + if (ref[3] != input[3] || ref[4] != input[4]) { + ++input; + continue; + } + } + /* far, needs at least 7-byte match */ + if (distance >= MAX_L2_Length+65535) { + if (ref[3] != input[3] || ref[4] != input[4]||ref[5] != input[5]||ref[6] != input[6]) { + ++input; //invalid + continue; //continue finding the next matched + } + } + + output = Literals_Output(input - pivot, pivot, output); + + match_len = match_cmp(ref + 3, input + 3, input_bound); + output = Match_Output3( match_len, distance, output); + + Update_Hash(); + + pivot = input; + } + + last_num_bytes = (ubyte*)INPUT_ + LENGTH_ - pivot; + output = Literals_Output (last_num_bytes, pivot, output); + + // id of this level + *(ubyte*)OUTPUT_ |= (1 << 6); // default is all 0s + + length_after= output - (ubyte*)OUTPUT_; + + } + void level4() { //Window_Num + while (input < input_limit)[[likely]] { + // potential match + while(input <= input_limit)[[likely]]{ + if(Matched_First_Every3Bytes(Window_Num*65535 + MAX_L2_Length )) break; + ++input; + } // if not match to the previous, continue to looping + if (input >= input_limit) [[unlikely]]break; + bool Invalid_Match=false; + for(uint i=0;i<=Window_Num;i++){ + /* far, needs at least 7-byte match */ + if (distance >= MAX_L2_Length+i*65535) { + if (ref[3+i] != input[3+i] || ref[4+i] != input[4+i]) { + Invalid_Match=true; //invalid + break; //continue finding the next matched + } + } + } + if(Invalid_Match){ + ++input; + continue; + } + output = Literals_Output(input - pivot, pivot, output); + + match_len = match_cmp(ref + 3, input + 3, input_bound); + + output = Match_Output4(match_len, distance, output); + + Update_Hash(); + + pivot = input; + } + + last_num_bytes = (ubyte*)INPUT_ + LENGTH_ - pivot; + output = Literals_Output (last_num_bytes, pivot, output); + + // id of this level + *(ubyte*)OUTPUT_ |= ( 1<< 5); // default is all 0s + + length_after= output - (ubyte*)OUTPUT_; + } + void level5() { //Direct Match + while (input < input_limit)[[likely]] { + // potential match + while(input <= input_limit)[[likely]]{ + if(Matched_First_Every3Bytes(65535 +65535 + MAX_L2_Length )) break; + ++input; + } // if not match to the previous, continue to looping + + if (input >= input_limit) [[unlikely]]break; + // if match to the previous one(>=3) + + /* far, needs at least 5-byte match */ + if (distance >= MAX_L2_Length) { + if (ref[3] != input[3] || ref[4] != input[4]) { + ++input; + continue; + } + } + /* far, needs at least 7-byte match */ + if (distance >= MAX_L2_Length+65535) { + if (ref[3] != input[3] || ref[4] != input[4]||ref[5] != input[5]||ref[6] != input[6]) { + ++input; //invalid + continue; //continue finding the next matched + } + } + + output = Literals_Output(input - pivot, pivot, output); + + match_len = match_cmp(ref + 3, input + 3, input_bound); + output = Match_Output5( match_len, distance, output); + + Update_Hash(); + + pivot = input; + } + + last_num_bytes = (ubyte*)INPUT_ + LENGTH_ - pivot; + output = Literals_Output (last_num_bytes, pivot, output); + + // id of this level + *(ubyte*)OUTPUT_ |= (1 << 7); // default is all 0s + *(ubyte*)OUTPUT_ |= (1 << 5); // default is all 0s + length_after= output - (ubyte*)OUTPUT_; + + } +}; diff --git a/include/LZSS_Decompression.h b/include/LZSS_Decompression.h new file mode 100644 index 0000000..1a7ae4a --- /dev/null +++ b/include/LZSS_Decompression.h @@ -0,0 +1,259 @@ +/* + + ----------------------------------------------------------------- + + Reference: + + RFC 1951 / DEFLATE Compressed Data Format Specification + + Ziv J., Lempel A., "A Universal Algorithm for sequenceuential Data + Compression", IEEE Transactions on Information Theory, Vol. 23, + No. 3, pp. 337-343. + + Lempel–Ziv–Storer–Szymanski + https://en.wikipedia.org/wiki/Lempel–Ziv–Storer–Szymanski#cite_note-1 + https://en.wikipedia.org/wiki/LZ77_and_LZ78#LZ77 + + Data Compression - Lecture 10 - Lempel-Ziv Schemes + https://www.youtube.com/watch?v=VDXBnmr8AY0&list=PLU4IQLU9e_OpnkbCS_to64F_vw5yyg4HB&index=3 + + ------------------------------------------------------------------------------------------ +*/ +#include "LZSS_helper.h" + +class LZSS_Decoder{ + private: + const void* INPUT_; + int LENGTH_; //length of the block of data to be Decompressed + void* OUTPUT_; + const ubyte* input; // pointer to input data + const ubyte* input_limit; // input data size (max) + ubyte* output ; // pointer to output data + uint ctrl; // current position's value + const ubyte* ref; + uint match_len; + ull offset;// distance from current position to references + int length_after; + uint Window_Num; //level4 + public: + LZSS_Decoder(const void* INPUT, int length, void* OUTPUT):INPUT_(INPUT), input((const ubyte*)INPUT){ + this->LENGTH_=length; + input_limit = input + length; + ctrl = (*input++) & 31;//start with literal run, 000 L4-L0 lower 5 bits -> length of the literal run *(input++) + this->OUTPUT_=OUTPUT; + output = (ubyte*)OUTPUT_; + length_after=0; //default / failed + Window_Num=2; + } + ~LZSS_Decoder(){} + int Decompress(){ + // default level1 compression + int MSB_7=(*(ubyte*)INPUT_) >>7; //level=2 + int MSB_6=((*(ubyte*)INPUT_) >>6)&1; //level=3 + int MSB_5=((*(ubyte*)INPUT_) >>5)&1; //level=4 + int level = MSB_7+( MSB_6<<1) +MSB_5*3 + 1; + if (level <= 1) { + level1(); + } + else if (level == 2) { + level2(); + }else if(level==4){ + level4(); + } + else if(level>=3){ + level3(); + } + //level !=2| level !=1, return 0 + return length_after; + } + + private: + /* Copies the values of count bytes from the location pointed by source to + the memory block pointed by destination. Copying takes place as if an + intermediate buffer were used, allowing the destination and source to overlap. + 64-bit implementation for speed improvements. + memmove=copy8 */ + void mem_move(ubyte* destination, const ubyte* source, uint count) { + if ((count > 4) && (destination >= source + count)) { + memmove(destination, source, count); + } else { + for(uint i=0;i= source + count || destination + count <= source) { + // memcpy(destination, source, count); + // } else { + // do { + // *destination++ = *source++; + // } while(--count); + // } + } + void level1() { + while (input <= input_limit - 2)[[likely]] { + if (ctrl >= 32) { //>=2 bytes //Long match && Short match + //The 3 most-significant bits of opcode[0], + // M, determines the match length. + match_len = (ctrl >> 5) - 1; + offset = (ctrl & 31) << 8;//update offset to the high bytes of Opcode[0] + + //. Note that R is a back reference, + //i.e. the value of 0 corresponds + // the last byte in the output buffer, + //1 is the second to last byte, and so forth. + ref = output - offset - 1; + if (match_len == 7 - 1) { // Long match 111 + match_len += *input++; //The value of opcode[1], M, determines the match length. + //The value of 0 indicates a 9-byte match, 1 indicates a 10-byte match + } + //Long match && Short match + ref -= *input++;//update offset with Opcode[1] for short match + //update offset with Opcode[2] fro Long Match + match_len += 3;//Short match：(M+2) The value of 1 indicates a 3-byte match, 2 indicates a 4-byte match and so on. + //The minimum match length is 3 and the maximum match length is 8. + this->mem_move(output, ref, match_len); + output += match_len;//move on + } else { //Literal run + ctrl++;//The minimum literal run is 1 and the maximum literal run is 32. + memcpy(output, input, ctrl); + input += ctrl;//have been copied; move on + output += ctrl; + } + ctrl = *input++; + } + length_after =output - (ubyte*)OUTPUT_; + } + + void level2() { + while (input < input_limit)[[likely]] { + if (ctrl >= 0b00100000) { //Not lieteral run Level 2 Extended Windows* Short match + match_len = (ctrl >> 5) - 1; // M2-M0 high 3 bits + offset = (ctrl & 31) << 8; // W12-W8 low 5 bits + ref = output - offset - 1; // reference (copied) of existed decoded output + ubyte code; //current position's value + if (match_len == 7 - 1) do {//Level 2 Long match: read in all of the len + code = *input++; //...... M7-M0 + match_len += code; + } while (code == 255);//Len left or Windows size not full + code = *input++; //W7-W0 + //For Extended Windows | Short match code=255 + ref -= code; + match_len += 3; + + /* match from 16-bit distance */ + if (code == 255) [[unlikely]]//level 2 Extended Windows + if (offset == (31 << 8))[[likely]] { //Distinguish with level1 + offset = (*input++) << 8; // 1111 1111 | W15-W8 | W7-W0 + offset += *input++; + ref = output - offset - MAX_L2_Length - 1; + } + this->mem_move(output, ref, match_len); + output += match_len; + } else { //literal run + ctrl++; //start with 0 + memcpy(output, input, ctrl); + input += ctrl; + output += ctrl; + } + ctrl = *input++; + } + length_after=output - (ubyte*)OUTPUT_; + } + void level3() { + while (input < input_limit)[[likely]] { + if (ctrl >= 32) { //Level 2 Extended Windows* Short match + match_len = (ctrl >> 5) - 1; + // is_longMatch=false; + offset = (ctrl & 31) << 8; + ref = output - offset - 1; + ubyte code; + if (match_len == 7 - 1) do {//Level 2 Extended Windows* Long match + // is_longMatch=true; + code = *input++; + match_len += code; + } while (code == 255);//Len left or Windows size not full + code = *input++; //|- + ref -= code; + match_len += 3; +// | Extended Windows*| ----------------------------------------------------------------------------------------| +// | Short match | M2-M0 11111 |- 1111 1111 |$ W15-W8 | W7-W0 | | | | +// | Long match | 111 11111 | M7-M0... |- 1111 1111 |$ W15-W8 | W7-W0 | | | +// | Extended Windows2| ----------------------------------------------------------------------------------------| +// | Extra Short match| M2-M0 11111 |- 1111 1111 |$ 1111 1111 | 1111 1111 | W15-W8 | W7-W0 | | +// | Extra Long match | 111 11111 | M7-M0... |- 1111 1111 |$ 1111 1111 | 1111 1111 | W15-W8 | W7-W0 | + /* match from 32-bit distance */ + if (code == 255) [[unlikely]]{ + if (offset == (31 << 8))[[likely]] { //level 2-3 + offset = (*input++) << 8; //|$ + offset += *input++; // W7-W0 + if(offset != ((1<<16)-1))[[likely]]{ //level 2 1111 1111 | W15-W8 | W7-W0 + ref = output - offset - MAX_L2_Length - 1; + } + else{ //level3 1111 1111 | 1111 1111 | 1111 1111 | W15-W8 | W7-W0 + offset = (*input++) << 8; /// W15-W8 + offset += *input++;// W7-W0 + ref = output - offset -65535- MAX_L2_Length-1; + //ref-(ubyte*)OUTPUT_; + } + } + } + this->mem_move(output, ref, match_len); + output += match_len; + } else { + ctrl++; + memcpy(output, input, ctrl); + input += ctrl; + output += ctrl; + } + ctrl = *input++; + } + length_after=output - (ubyte*)OUTPUT_; + + } + void level4() { + // bool is_longMatch; + while (input < input_limit)[[likely]] { + if (ctrl >= 32) { //Level 2 Extended Windows* Short match + match_len = (ctrl >> 5) - 1; + // is_longMatch=false; + offset = (ctrl & 31) << 8; + ref = output - offset - 1; + ubyte code; + if (match_len == 7 - 1) do {//Level 2 Extended Windows* Long match + // is_longMatch=true; + code = *input++; + match_len += code; + } while (code == 255);//Len left or Windows size not full + code = *input++; //|- + ref -= code; + match_len += 3; + + if (code == 255) [[unlikely]]{ + if (offset == (31 << 8))[[likely]] { + for(uint index=1;index<=Window_Num;index++){//level 3+ + //level3 1111 1111 | 1111 1111 | 1111 1111 | W15-W8 | W7-W0 + offset = (*input++) << 8; /// W15-W8 + offset += *input++;// W7-W0 + if(offset != 0xffff)[[likely]]{ //level 2 1111 1111 | W15-W8 | W7-W0 + ref = output - offset -65535*(index-1) - MAX_L2_Length-1; + break; + } + } + } + } + this->mem_move(output, ref, match_len); + output += match_len; + } else { + ctrl++; + memcpy(output, input, ctrl); + input += ctrl; + output += ctrl; + } + ctrl = *input++; + } + length_after=output - (ubyte*)OUTPUT_; + + } + +}; diff --git a/include/LZSS_helper.h b/include/LZSS_helper.h new file mode 100644 index 0000000..0022b90 --- /dev/null +++ b/include/LZSS_helper.h @@ -0,0 +1,65 @@ +/* + + I/O part of Compression and Decompression + + ----------------------------------------------------------------- + + Reference: + + RFC 1951 / DEFLATE Compressed Data Format Specification + + Ziv J., Lempel A., "A Universal Algorithm for sequenceuential Data + Compression", IEEE Transactions on Information Theory, Vol. 23, + No. 3, pp. 337-343. + + Lempel–Ziv–Storer–Szymanski + https://en.wikipedia.org/wiki/Lempel–Ziv–Storer–Szymanski#cite_note-1 + https://en.wikipedia.org/wiki/LZ77_and_LZ78#LZ77 + + Data Compression - Lecture 10 - Lempel-Ziv Schemes + https://www.youtube.com/watch?v=VDXBnmr8AY0&list=PLU4IQLU9e_OpnkbCS_to64F_vw5yyg4HB&index=3 + + ------------------------------------------------------------------------------------------ +*/ +#include +#include +using namespace jls; +#define WINDOW_SIZE_L1 8192 +#define MAX_L2_Length 8191 +#define MAX_L3_Length 65535 +#define LEVEL1_MAX 8192*8 //65536 +#define HASH_SIZE (1 << 14) //HASH_LOG 14 +#define Hash_Key_Masks (HASH_SIZE-1) +bool Compress_Twice=true; + +#define read_uint64(ptr) ((ull*)(ptr))[0] +#define read_uint32(ptr) ((uint*)(ptr))[0] + +// // Return a 8-byte unsigned integer from the current stream +// ull read_uint64(const void* ptr) { +// const ull* ptr_to64=(const ull*)ptr; +// return ptr_to64[0]; +// } + +// // Return a 4-byte unsigned integer from the current stream +// uint read_uint32(const void* ptr) { +// const uint* ptr_to32=(const uint*)ptr; +// return ptr_to32[0]; +// } + +//return the same len + 1 from the start of p; if no matched return 1 +uint match_cmp(const ubyte* ref, const ubyte* input, const ubyte* bound) { + const ubyte* start = ref; + //check whether 64 Bytes are the same or not + if (read_uint64(ref) == read_uint64(input)) { + ref += 8,input += 8; + } + //check whether 32 Bytes are the same or not + if (read_uint32(ref) == read_uint32(input)) { + ref += 4,input += 4; + } + while (input < bound){ + if (*ref++ != *input++) break; + } + return ref - start; +} diff --git a/include/serialize.h b/include/serialize.h new file mode 100644 index 0000000..6f8f626 --- /dev/null +++ b/include/serialize.h @@ -0,0 +1,540 @@ +#include +#include +#include +#include +#include +using namespace std; // fuck you and your code +namespace jls { + // declarations + using ubyte = uint8_t; + using uint=uint32_t; + using ull = uint64_t; + using tiii = tuple; + using tbbb = tuple; + string int_to_str_div_1M(int); + ull three_pack(int, int, int); + ull three_pack(tiii); + tiii three_unpack(ull); + void push_int24(vector&, int); + void push_int(vector&, int); + void push_int64(vector &v, ull); + struct node_v; + struct node_vt; + struct node_vn; + struct node_f; + struct delta_f; + class fixed_byte_stream { + protected: + unsigned char* vstr; + int ptr; + int maxSize; + public: + fixed_byte_stream(unsigned char *buf, int length) { + vstr = buf; + ptr = 0; + maxSize = length; + } + int get_remaining_size() const { + return maxSize - ptr; + } + void clear() { + ptr = 0; + } + friend fixed_byte_stream& operator >> (fixed_byte_stream&, char &b); + friend fixed_byte_stream& operator >> (fixed_byte_stream&, ubyte &b); + friend fixed_byte_stream& operator >> (fixed_byte_stream&, int &x); + friend fixed_byte_stream& operator >> (fixed_byte_stream&, ull &x); + friend fixed_byte_stream& operator >> (fixed_byte_stream&, node_v &v); + friend fixed_byte_stream& operator >> (fixed_byte_stream&, node_vt &vt); + friend fixed_byte_stream& operator >> (fixed_byte_stream&, node_vn &vn); + friend fixed_byte_stream& operator >> (fixed_byte_stream&, node_f &f); + friend fixed_byte_stream& operator >> (fixed_byte_stream&, delta_f &df); + int get_int24() { + int v = 0; + for (int i = 0; i < 3; i++) { + v <<= 8; + v |= vstr[ptr++]; + } + return v; + } + }; + const int FLAG_BYTES = 1; + const ubyte FLAGS[] = { + 0x00 + }; + const int NUM_BLOCKS = 4; + const unsigned int msk_pack = 0x1fffff; + + + // Implementations + string int_to_str_div_1M(int x) { + string ret(""); + bool is_neg = (x < 0); + if (is_neg) { x = -x; } + while (x) { + int m = x % 10; + ret += char(m + '0'); + x /= 10; + } + while (ret.size() < 7) { + ret += "0"; + } + ret.insert(6, "."); + if (is_neg) { + ret += '-'; + } + reverse(ret.begin(), ret.end()); + return ret; + } + int write_pos_int_less_than_10M(char *buf, int x) { + int offset = 0; + if (x >= 1000000) { buf[offset++] = (x / 1000000) % 10 + '0'; } + if (x >= 100000) { buf[offset++] = (x / 100000) % 10 + '0'; } + if (x >= 10000) { buf[offset++] = (x / 10000) % 10 + '0'; } + if (x >= 1000) { buf[offset++] = (x / 1000) % 10 + '0'; } + if (x >= 100) { buf[offset++] = (x / 100) % 10 + '0'; } + if (x >= 10) { buf[offset++] = (x / 10) % 10 + '0'; } + buf[offset++] = (x % 10) + '0'; + return offset; + } + int write_int_to_str_div_1M(char *buf, int x) { + int offset = 0; + if (x == 0) { + buf[offset] = '0'; + return 1; + } + if (x < 0) { + buf[offset++] = '-'; + x = -x; + } + if (x >= 1000000000) { + buf[offset++] = '0' + (x / 1000000000) % 10; + } + if (x >= 100000000) { + buf[offset++] = '0' + (x / 100000000) % 10; + } + if (x >= 10000000) { + buf[offset++] = '0' + (x / 10000000) % 10; + } + buf[offset++] = '0' + (x / 1000000) % 10; + buf[offset++] = '.'; + buf[offset++] = '0' + (x / 100000) % 10; + buf[offset++] = '0' + (x / 10000) % 10; + buf[offset++] = '0' + (x / 1000) % 10; + buf[offset++] = '0' + (x / 100) % 10; + buf[offset++] = '0' + (x / 10) % 10; + buf[offset++] = '0' + x % 10; + return offset; + } + ull three_pack(int a, int b, int c) { + // assert (a >= -(1<<20) && a <= (1<<20)-1); + // assert (b >= -(1<<20) && b <= (1<<20)-1); + // assert (c >= -(1<<20) && c <= (1<<20)-1); + return (ull(a & msk_pack) << 42 | ull(b & msk_pack) << 21 | (c & msk_pack)); + } + ull three_pack(tiii t) { + return three_pack(get<0>(t), get<1>(t), get<2>(t)); + } + tiii three_unpack(ull x) { + int a = int(x >> 42) & msk_pack; + int b = int(x >> 21) & msk_pack; + int c = x & msk_pack; + if (a >> 20 == 1) { + a |= ~msk_pack; // 111111111110(0)*20 + } + if (b >> 20 == 1) { + b |= ~msk_pack; + } + if (c >> 20 == 1) { + c |= ~msk_pack; + } + return make_tuple(a, b, c); + } + void push_int24(vector &v, int x) { + v.push_back((x >> 16) & (0xFF)); + v.push_back((x >> 8) & (0xFF)); + v.push_back(x & (0xFF)); + } + void push_int(vector &v, int x) { + v.push_back((x >> 24) & (0xFF)); + v.push_back((x >> 16) & (0xFF)); + v.push_back((x >> 8) & (0xFF)); + v.push_back(x & (0xFF)); + } + void push_int64(vector &v, ull x) { + v.push_back((x >> 56) & (0xFF)); + v.push_back((x >> 48) & (0xFF)); + v.push_back((x >> 40) & (0xFF)); + v.push_back((x >> 32) & (0xFF)); + v.push_back((x >> 24) & (0xFF)); + v.push_back((x >> 16) & (0xFF)); + v.push_back((x >> 8) & (0xFF)); + v.push_back(x & (0xFF)); + } + fixed_byte_stream& operator >> (fixed_byte_stream &fbs, char &b) { + b = fbs.vstr[fbs.ptr++]; + return fbs; + } + fixed_byte_stream& operator >> (fixed_byte_stream &fbs, ubyte &b) { + b = fbs.vstr[fbs.ptr++]; + return fbs; + } + fixed_byte_stream& operator >> (fixed_byte_stream &fbs, int &v) { + v = 0; + for (int i = 0; i < 4; i++) { + v <<= 8; + v |= fbs.vstr[fbs.ptr++]; + } + return fbs; + } + fixed_byte_stream& operator >> (fixed_byte_stream &fbs, ull &v) { + v = 0; + for (int i = 0; i < 8; i++) { + v <<= 8; + v |= fbs.vstr[fbs.ptr++]; + } + return fbs; + } + struct node_v { + int x, y, z; + node_v(): x(0), y(0), z(0) {} + node_v(int _x, int _y, int _z): x(_x), y(_y), z(_z) {} + string dump_output_obj() const { + string ret("v "); + ret += int_to_str_div_1M(x); + ret += " "; + ret += int_to_str_div_1M(y); + ret += " "; + ret += int_to_str_div_1M(z); + return ret; + } + int write_obj(char *buf) const { + int offset = 0; + buf[offset++] = 'v'; + buf[offset++] = ' '; + offset += write_int_to_str_div_1M(buf+offset, x); + buf[offset++] = ' '; + offset += write_int_to_str_div_1M(buf+offset, y); + buf[offset++] = ' '; + offset += write_int_to_str_div_1M(buf+offset, z); + return offset; + } + void to_stream(vector &st) { + push_int(st, x); + push_int(st, y); + push_int(st, z); + } + bool operator == (const node_v &b) const { + return (x == b.x && y == b.y && z == b.z); + } + bool operator < (const node_v &b) const { + if (x < b.x) { + return true; + } + if (b.x < x) { + return false; + } + if (y < b.y) { + return true; + } + if (b.y < y) { + return false; + } + if (z < b.z) { + return true; + } + if (b.z < z) { + return false; + } + return false; + } + }; + fixed_byte_stream& operator >> (fixed_byte_stream &fbs, node_v &v) { + int x, y, z; + fbs >> x >> y >> z; + v = node_v(x,y,z); + return fbs; + } + struct node_vt { + int x, y; + node_vt(): x(0), y(0) {} + node_vt(int _x, int _y): x(_x), y(_y) {} + string dump_output_obj() const { + string ret("vt "); + ret += int_to_str_div_1M(x); + ret += " "; + ret += int_to_str_div_1M(y); + return ret; + } + int write_obj(char *buf) const { + int offset = 0; + buf[offset++] = 'v'; + buf[offset++] = 't'; + buf[offset++] = ' '; + offset += write_int_to_str_div_1M(buf+offset, x); + buf[offset++] = ' '; + offset += write_int_to_str_div_1M(buf+offset, y); + return offset; + } + void to_stream(vector &st) { + push_int(st, x); + push_int(st, y); + } + bool operator == (const node_vt &b) const { + return x == b.x && y == b.y; + } + bool operator<(const node_vt &b) const { + if (x < b.x) { + return true; + } + if (b.x < x) { + return false; + } + if (y < b.y) { + return true; + } + if (b.y < y) { + return false; + } + return false; + } + }; + fixed_byte_stream& operator >> (fixed_byte_stream &fbs, node_vt &vt) { + int x, y; + fbs >> x >> y; + vt = node_vt(x,y); + return fbs; + } + struct node_vn { + int x, y, z; + node_vn(): x(0), y(0), z(0) {} + node_vn(int _x, int _y, int _z): x(_x), y(_y), z(_z) {} + string dump_output_obj() const { + string ret("vn "); + ret += int_to_str_div_1M(x); + ret += " "; + ret += int_to_str_div_1M(y); + ret += " "; + ret += int_to_str_div_1M(z); + return ret; + } + int write_obj(char *buf) const { + int offset = 0; + buf[offset++] = 'v'; + buf[offset++] = 'n'; + buf[offset++] = ' '; + offset += write_int_to_str_div_1M(buf+offset, x); + buf[offset++] = ' '; + offset += write_int_to_str_div_1M(buf+offset, y); + buf[offset++] = ' '; + offset += write_int_to_str_div_1M(buf+offset, z); + return offset; + } + void to_stream(vector &st) { + push_int64(st, three_pack(x, y, z)); + } + bool operator == (const node_vn &b) const { + return (x == b.x) && (y == b.y) && (z == b.z); + } + bool operator < (const node_vn &b) const { + if (x < b.x) { + return true; + } + if (b.x < x) { + return false; + } + if (y < b.y) { + return true; + } + if (b.y < y) { + return false; + } + if (z < b.z) { + return true; + } + if (b.z < z) { + return false; + } + return false; + } + }; + fixed_byte_stream& operator >> (fixed_byte_stream &fbs, node_vn &vn) { + ull t; + fbs >> t; + auto [a, b, c] = three_unpack(t); + vn = node_vn(a, b, c); + return fbs; + } + struct node_f { + tiii f[3]; + node_f() { + f[0] = make_tuple(0,0,0); + f[1] = make_tuple(0,0,0); + f[2] = make_tuple(0,0,0); + } + node_f(tiii _a, tiii _b, tiii _c) { + f[0] = _a; + f[1] = _b; + f[2] = _c; + } + string to_str(const tiii &x) const { + string ret; + auto [a, b, c] = x; // C++17 + if (a) { // if a != 0 + ret += to_string(a); + } + ret += "/"; + if (b) { // if a != 0 + ret += to_string(b); + } + ret += "/"; + if (c) { // if a != 0 + ret += to_string(c); + } + return ret; + } + int write_str(char *buf, const tiii &x) const { + int offset = 0; + auto [p, q, r] = x; + offset += write_pos_int_less_than_10M(buf+offset, p); + buf[offset++] = '/'; + if (q) { + offset += write_pos_int_less_than_10M(buf+offset, q); + } + buf[offset++] = '/'; + offset += write_pos_int_less_than_10M(buf+offset, r); + return offset; + } + string dump_output_obj() const { + string ret("f "); + ret += to_str(f[0]); + ret += " "; + ret += to_str(f[1]); + ret += " "; + ret += to_str(f[2]); + return ret; + } + int write_obj(char *buf) const { + int offset = 0; + buf[offset++] = 'f'; + buf[offset++] = ' '; + offset += write_str(buf+offset, f[0]); + buf[offset++] = ' '; + offset += write_str(buf+offset, f[1]); + buf[offset++] = ' '; + offset += write_str(buf+offset, f[2]); + return offset; + } + void to_stream(vector &st) { + push_int64(st, three_pack(f[0])); + push_int64(st, three_pack(f[1])); + push_int64(st, three_pack(f[2])); + } + }; + fixed_byte_stream& operator >> (fixed_byte_stream &fbs, node_f &f) { + ull t1, t2, t3; + fbs >> t1 >> t2 >> t3; + f = node_f(three_unpack(t1), three_unpack(t2), three_unpack(t3)); + return fbs; + } + struct delta_f { + char dv[3] = {0,0,0}, dvt[3] = {0,0,0}, dvn[3] = {0,0,0}; + int pv[3] = {0,0,0}, pvt[3] = {0,0,0}, pvn[3] = {0,0,0}; + delta_f() {} + delta_f(node_f pivot) { + for (int i = 0; i < 3; i++) { + dv[i] = dvt[i] = dvn[i] = -128; + pv[i] = get<0>(pivot.f[i]); + pvt[i] = get<1>(pivot.f[i]); + pvn[i] = get<2>(pivot.f[i]); + } + } + delta_f(node_f f1, node_f f2) { + for (int i = 0; i < 3; i++) { + int delv = get<0>(f2.f[i]) - get<0>(f1.f[i]); + int delvt = get<1>(f2.f[i]) - get<1>(f1.f[i]); + int delvn = get<2>(f2.f[i]) - get<2>(f1.f[i]); + if (delv >= -127 && delv <= 127) { + dv[i] = delv; + } else { + dv[i] = -128; + pv[i] = get<0>(f2.f[i]); + } + if (delvt >= -127 && delvt <= 127) { + dvt[i] = delvt; + } else { + dvt[i] = -128; + pvt[i] = get<1>(f2.f[i]); + } + if (delvn >= -127 && delvn <= 127) { + dvn[i] = delvn; + } else { + dvn[i] = -128; + pvn[i] = get<2>(f2.f[i]); + } + } + } + void to_stream(vector &st) { + for (int i = 0; i < 3; i++) { + st.push_back(dv[i]); + if (dv[i] == -128) { + push_int24(st, pv[i]); + } + st.push_back(dvt[i]); + if (dvt[i] == -128) { + push_int24(st, pvt[i]); + } + st.push_back(dvn[i]); + if (dvn[i] == -128) { + push_int24(st, pvn[i]); + } + } + } + }; + node_f operator + (const node_f &f1, const delta_f &df) { + node_f res; + for (int i = 0; i < 3; i++) { + int v, vt, vn; + v = (df.dv[i] != -128 ? get<0>(f1.f[i]) +df.dv[i] : df.pv[i]); + vt = (df.dvt[i] != -128 ? get<1>(f1.f[i]) +df.dvt[i] : df.pvt[i]); + vn = (df.dvn[i] != -128 ? get<2>(f1.f[i]) +df.dvn[i] : df.pvn[i]); + res.f[i] = make_tuple(v, vt, vn); + } + return res; + } + node_f& operator += (node_f &f1, const delta_f &df) { + for (int i = 0; i < 3; i++) { + int v, vt, vn; + v = (df.dv[i] != -128 ? get<0>(f1.f[i]) + df.dv[i] : df.pv[i]); + vt = (df.dvt[i] != -128 ? get<1>(f1.f[i]) + df.dvt[i] : df.pvt[i]); + vn = (df.dvn[i] != -128 ? get<2>(f1.f[i]) + df.dvn[i] : df.pvn[i]); + f1.f[i] = make_tuple(v, vt, vn); + } + return f1; + } + + fixed_byte_stream& operator >> (fixed_byte_stream &fbs, delta_f &df) { + for (int i = 0; i < 3; i++) { + char ch; + fbs >> ch; + df.dv[i] = ch; + if (ch == -128) { + int x = fbs.get_int24(); + df.pv[i] = x; + } + fbs >> ch; + df.dvt[i] = ch; + if (ch == -128) { + int x = fbs.get_int24(); + df.pvt[i] = x; + } + fbs >> ch; + df.dvn[i] = ch; + if (ch == -128) { + int x = fbs.get_int24(); + df.pvn[i] = x; + } + } + return fbs; + } +} \ No newline at end of file diff --git a/output_location b/output_location new file mode 100644 index 0000000..41a610f Binary files /dev/null and b/output_location differ diff --git a/scripts/.gitignore b/scripts/.gitignore new file mode 100644 index 0000000..fa82646 --- /dev/null +++ b/scripts/.gitignore @@ -0,0 +1,2 @@ +.DS_Store +*.json diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..cc74df5 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,22 @@ +# Scripts +Shared scripts used in the CI/CD pipeline. + +You can also run these scripts locally. + +## Dependancies + +The Python dependencies are in the requirements.txt file. + +Mac: +``` +brew install XQuartz +``` +Restart terminal after install! + +Ubuntu: +``` +sudo apt-get install cmake +sudo apt-get install build-essential libssl-dev +sudo apt-get install jq +sudo apt-get install xvfb +``` diff --git a/scripts/build_ai.sh b/scripts/build_ai.sh new file mode 100755 index 0000000..18d8781 --- /dev/null +++ b/scripts/build_ai.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# Install requirements +pip install -r requirements.txt +pip install -r scripts/requirements.txt diff --git a/scripts/build_draco.sh b/scripts/build_draco.sh new file mode 100755 index 0000000..1237f68 --- /dev/null +++ b/scripts/build_draco.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Install requirements +pip install -r scripts/requirements.txt + +# Create build directory +mkdir -p build +cd ./build + +# Configure CMake build +cmake ../draco -DDRACO_TRANSCODER_SUPPORTED=ON -DCMAKE_BUILD_TYPE=Release + +# Build with CMake +cmake --build . -- -j4 + +# Create links to executables +ln -sf ./build/draco_encoder ../encoder +ln -sf ./build/draco_decoder ../decoder diff --git a/scripts/build_non_ai.sh b/scripts/build_non_ai.sh new file mode 100755 index 0000000..d8e35a7 --- /dev/null +++ b/scripts/build_non_ai.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Install requirements +pip install -r scripts/requirements.txt + +# Create build directory +mkdir -p build +cd ./build + +cmake .. -DCMAKE_BUILD_TYPE=Release +cmake --build . -- -j10 + +# Create links to executables +ln -sf ./build/bin/encoder ../encoder +ln -sf ./build/bin/decoder ../decoder diff --git a/scripts/iqm.py b/scripts/iqm.py new file mode 100644 index 0000000..6255b1c --- /dev/null +++ b/scripts/iqm.py @@ -0,0 +1,30 @@ + +import argparse +import numpy as np +import imageio.v3 as iio + +from sewar.full_ref import uqi +from sewar.full_ref import psnr + +def image_quality_metric( + ref_image, + mod_image +): + ref_im = iio.imread(ref_image) + mod_im = iio.imread(mod_image) + #return uqi(ref_im, mod_im) + return psnr(ref_im, mod_im) + +if __name__ == '__main__': + # Read and parse arguments from command line + parser = argparse.ArgumentParser() + + parser.add_argument("-r", "--Ref", help = "Original Image File") + parser.add_argument("-m", "--Mod", help = "Modified Image File") + + args = parser.parse_args() + + quality = image_quality_metric(args.Ref,args.Mod) + if quality is np.inf: + quality = 100 + print(quality) diff --git a/scripts/iqm.sh b/scripts/iqm.sh new file mode 100755 index 0000000..e94d9b2 --- /dev/null +++ b/scripts/iqm.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +input_file_path='' +decoded_file_path='' + +print_usage() { + printf "Usage: ..." +} + +while getopts 'i:d:v' flag; do + case "${flag}" in + i) input_file_path="${OPTARG}" ;; + d) decoded_file_path="${OPTARG}" ;; + *) print_usage + exit 1 ;; + esac +done + +input_render_file_path="$input_file_path.png" +decoded_render_file_path="$decoded_file_path.png" + +python3 ./scripts/offscreen_render.py -i $input_file_path -o $input_render_file_path 2> /dev/null +python3 ./scripts/offscreen_render.py -i $decoded_file_path -o $decoded_render_file_path 2> /dev/null + +python3 ./scripts/iqm.py -r $input_render_file_path -m $decoded_render_file_path diff --git a/scripts/offscreen_render.py b/scripts/offscreen_render.py new file mode 100644 index 0000000..8211f83 --- /dev/null +++ b/scripts/offscreen_render.py @@ -0,0 +1,59 @@ + +import argparse +import numpy as np +import trimesh +import os +from pyglet import gl +from pyvirtualdisplay import Display + +if __name__ == '__main__': + # Read and parse arguments from command line + parser = argparse.ArgumentParser() + + parser.add_argument("-i", "--Input", help = "Input GLTF/Obj File") + parser.add_argument("-o", "--Output", help = "Output PNG File") + + args = parser.parse_args() + + # print logged messages + # trimesh.util.attach_to_log() + + # Comment this out to use native display + display = Display(visible=0, size=(1920, 1080)) + display.start() + + if args.Input: + # print("Displaying Input as: % s" % args.Input) + mesh = trimesh.load(args.Input, force='mesh') + scene = mesh.scene() + # mesh.show(flags={'wireframe': True}) + + window_conf = gl.Config(double_buffer=True, depth_size=24) + + # a 45 degree homogeneous rotation matrix around + # the Y axis at the scene centroid + rotate = trimesh.transformations.rotation_matrix( + angle=np.radians(10.0), + direction=[0, 1, 0], + point=scene.centroid) + + # rotate the camera view transform + camera_old, _geometry = scene.graph[scene.camera.name] + camera_new = np.dot(rotate, camera_old) + + # apply the new transform + scene.graph[scene.camera.name] = camera_new + + # saving an image requires an opengl context, so if -nw + # is passed don't save the image + try: + if args.Output: + # print("Displaying Output as: % s" % args.Output) + # save a render of the object as a png + png = scene.save_image(resolution=[1920, 1080], window_conf=window_conf) + os.makedirs(os.path.dirname(args.Output), exist_ok=True) + with open(args.Output, 'wb') as f: + f.write(png) + f.close() + except BaseException as E: + print("unable to save image", str(E)) diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 0000000..1c380a4 --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,6 @@ +imageio +sewar +numpy +trimesh +pyglet +pyvirtualdisplay diff --git a/scripts/submit_score.sh b/scripts/submit_score.sh new file mode 100755 index 0000000..7280d53 --- /dev/null +++ b/scripts/submit_score.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +NEW_SCORE=${1:-0} +REPO_NAME=${GITHUB_REPOSITORY#*/} +PLAYER_NAME=(${REPO_NAME//[-_]/ }) +PLAYER_NAME=${PLAYER_NAME[*]^} + +echo '' +echo "Team: $PLAYER_NAME" +echo "New score: $NEW_SCORE" + +curl -s https://keepthescore.co/api/$LEADERBOARD_TOKEN/board/ > board.json +PLAYER_ID=$(jq ".players[] | select(.name == \"$PLAYER_NAME\") | .id" board.json) + +# Create player +if [[ -z "$PLAYER_ID" ]]; then + curl -s -H "Content-Type: application/json" -X POST -d "{\"name\": \"$PLAYER_NAME\"}" https://keepthescore.co/api/$LEADERBOARD_TOKEN/player/ > players.json + curl -s https://keepthescore.co/api/$LEADERBOARD_TOKEN/board/ > board.json + PLAYER_ID=$(jq ".players[] | select(.name == \"$PLAYER_NAME\") | .id" board.json) +fi + +OLD_SCORE=$(jq ".players[] | select(.id == $PLAYER_ID) | .score" board.json) +echo "Old score: $OLD_SCORE" + +SCORE_DIFF=$(echo | awk "{print $NEW_SCORE - $OLD_SCORE}") +SCORE_DIFF=${SCORE_DIFF:-0} +echo "Score diff: $SCORE_DIFF" + +curl -s -H "Content-Type: application/json" -X POST -d "{ \"player_id\": $PLAYER_ID, \"score\": $SCORE_DIFF}" https://keepthescore.co/api/$LEADERBOARD_TOKEN/score/ > players.json +UPDATED_SCORE=$(jq ".players[] | select(.id == $PLAYER_ID) | .score" players.json) +echo "Updated score: $UPDATED_SCORE" diff --git a/scripts/test_ai.sh b/scripts/test_ai.sh new file mode 100755 index 0000000..f2be97d --- /dev/null +++ b/scripts/test_ai.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +# Remove output from previous run +rm -rf ./test + +# Loop through sample models +jq -r '.[].name' ./sample-models/model-index.json | while read i; do + + input_file_path="./sample-models/$i/obj/$i.obj" + encoded_file_path="./test/encoded/$i/" + decoded_file_path="./test/decoded/$i/$i.obj" + input_render_file_path="./test/rendered/$i/$i.input.png" + decoded_render_file_path="./test/rendered/$i/$i.decoded.png" + + if [[ -f $input_file_path ]]; then + # Run encoder + python3 encode.py -i $input_file_path -o $encoded_file_path > /dev/null 2>&1 + + # Calculate compression ratio + if [[ -f $encoded_file_path ]] || [[ -d $encoded_file_path ]]; then + input_file_size=$(ls -l $input_file_path | awk '{ total += $5 }; END { print total }') + # echo $input_file_size + encoded_file_size=$(ls -l $encoded_file_path | awk '{ total += $5 }; END { print total }') + # echo $encoded_file_size + RATIO=$(echo | awk "{print 1 - ($encoded_file_size / $input_file_size)}") + echo "Encoded $encoded_file_path $RATIO" | tee -a test/compression.log + else + echo "Failed $encoded_file_path" | tee -a test/compression.log + fi + + # Run decoder + { time python3 decode.py -i $encoded_file_path -o $decoded_file_path > /dev/null 2>&1 ; } 2> time.log + + # Calculate decompression time + if [[ -f $decoded_file_path ]]; then + TIME=$(awk '/user|sys/ {split($2,a,/m|s/); sum+=(a[1] * 60000) + (a[2] * 1000)} END { print sum }' time.log) + echo "Decoded $decoded_file_path $TIME" | tee -a test/decompression.log + else + echo "Failed $decoded_file_path" | tee -a test/decompression.log + fi + + # Calculate image quality + if [[ -f $input_file_path ]] && [[ -f $decoded_file_path ]]; then + python3 scripts/offscreen_render.py -i $input_file_path -o $input_render_file_path 2> /dev/null + python3 scripts/offscreen_render.py -i $decoded_file_path -o $decoded_render_file_path 2> /dev/null + + IQM=0 + if [[ -f $input_render_file_path ]] && [[ -f $decoded_render_file_path ]]; then + IQM=$(python3 scripts/iqm.py -r $input_render_file_path -m $decoded_render_file_path) + fi + + if [[ ! -z "$IQM" ]]; then + echo "Quality $decoded_render_file_path $IQM" | tee -a test/quality.log + else + echo "Failed $decoded_render_file_path" | tee -a test/quality.log + fi + fi + + echo '' + + fi +done + +# Calculate score + +# Average compression percentage +AVG_COMPRESSION_RATIO=0 +if [[ -f 'test/compression.log' ]]; then + AVG_COMPRESSION_RATIO=$(awk '/Encoded/ {sum+=$3} END { print sum/NR}' test/compression.log) + echo "Average compression ratio: $AVG_COMPRESSION_RATIO percent" +fi + +# Average time in milliseconds +AVG_DECOMPRESSION_TIME=0 +if [[ -f 'test/decompression.log' ]]; then + AVG_DECOMPRESSION_TIME=$(awk '/Decoded/ {sum+=$3} END { print sum/NR }' test/decompression.log) + echo "Average decompression time: $AVG_DECOMPRESSION_TIME milliseconds" +fi + +# Average quality metric in decibels +AVG_IMAGE_QUALITY_METRIC=0 +if [[ -f 'test/quality.log' ]]; then + AVG_IMAGE_QUALITY_METRIC=$(awk '/Quality/ {sum+=$3} END { print sum/NR}' test/quality.log) + echo "Average image quality metric $AVG_IMAGE_QUALITY_METRIC decibels" +fi + +echo '' + +# Weighted compression score +WEIGHTED_COMPRESSION_SCORE=$(echo | awk "{ print ($AVG_COMPRESSION_RATIO * 100 * 0.35)}") +echo "Weighted compression score (35%): $WEIGHTED_COMPRESSION_SCORE" + +# Weighted time score +WEIGHTED_TIME_SCORE=$(echo | awk "{ print ((1000 - $AVG_DECOMPRESSION_TIME) / 1000 * 100 * 0.25)}") +echo "Weighted time score (25%): $WEIGHTED_TIME_SCORE" + +# Weighted quality score +WEIGHTED_QUALITY_SCORE=$(echo | awk "{ print ($AVG_IMAGE_QUALITY_METRIC * 0.20)}") +echo "Weighted quality score (20%): $WEIGHTED_QUALITY_SCORE" + +echo '' + +# Total score +TOTAL_SCORE=$(echo | awk "{ print $WEIGHTED_COMPRESSION_SCORE + $WEIGHTED_TIME_SCORE + $WEIGHTED_QUALITY_SCORE}") +echo "Total score (80%): $TOTAL_SCORE" + +# The last 20% for the presentation will be awarded manually by the judges + +# Submit score +if [[ "$GITHUB_REF_NAME" == "main" ]]; then + ./scripts/submit_score.sh $TOTAL_SCORE +fi diff --git a/scripts/test_draco.sh b/scripts/test_draco.sh new file mode 100755 index 0000000..c68ef58 --- /dev/null +++ b/scripts/test_draco.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +# Remove output from previous run +rm -rf ./test + +# Loop through sample models +jq -r '.[].name' ./sample-models/model-index.json | while read i; do + + input_file_path="./sample-models/$i/obj/$i.obj" + encoded_file_path="./test/encoded/$i/$i.drc" + decoded_file_path="./test/decoded/$i/$i.obj" + input_render_file_path="./test/rendered/$i/$i.input.png" + decoded_render_file_path="./test/rendered/$i/$i.decoded.png" + + if [[ -f $input_file_path ]]; then + # Run encoder + ./encoder -i $input_file_path -o $encoded_file_path > /dev/null 2>&1 + + # Calculate compression ratio + if [[ -f $encoded_file_path ]]; then + input_file_size=$(ls -l $input_file_path | awk '{ total += $5 }; END { print total }') + # echo $input_file_size + encoded_file_size=$(ls -l $encoded_file_path | awk '{ total += $5 }; END { print total }') + # echo $encoded_file_size + RATIO=$(echo | awk "{print 1 - ($encoded_file_size / $input_file_size)}") + echo "Encoded $encoded_file_path $RATIO" | tee -a test/compression.log + else + echo "Failed $encoded_file_path" | tee -a test/compression.log + fi + + # Run decoder + { time ./decoder -i $encoded_file_path -o $decoded_file_path > /dev/null 2>&1 ; } 2> test/time.log + + # Calculate decompression time + if [[ -f $decoded_file_path ]]; then + TIME=$(awk '/user|sys/ {split($2,a,/m|s/); sum+=(a[1] * 60000) + (a[2] * 1000)} END { print sum }' test/time.log) + echo "Decoded $decoded_file_path $TIME" | tee -a test/decompression.log + else + echo "Failed $decoded_file_path" | tee -a test/decompression.log + fi + + # Calculate image quality + if [[ -f $input_file_path ]] && [[ -f $decoded_file_path ]]; then + python3 scripts/offscreen_render.py -i $input_file_path -o $input_render_file_path 2> /dev/null + python3 scripts/offscreen_render.py -i $decoded_file_path -o $decoded_render_file_path 2> /dev/null + + IQM=0 + if [[ -f $input_render_file_path ]] && [[ -f $decoded_render_file_path ]]; then + IQM=$(python3 scripts/iqm.py -r $input_render_file_path -m $decoded_render_file_path) + fi + + if [[ ! -z "$IQM" ]]; then + echo "Quality $decoded_render_file_path $IQM" | tee -a test/quality.log + else + echo "Failed $decoded_render_file_path" | tee -a test/quality.log + fi + fi + + echo '' + + fi +done + +# Calculate score + +# Average compression percentage +AVG_COMPRESSION_RATIO=0 +if [[ -f 'test/compression.log' ]]; then + AVG_COMPRESSION_RATIO=$(awk '/Encoded/ {sum+=$3} END { print sum/NR}' test/compression.log) + echo "Average compression ratio: $AVG_COMPRESSION_RATIO percent" +fi + +# Average time in milliseconds +AVG_DECOMPRESSION_TIME=0 +if [[ -f 'test/decompression.log' ]]; then + AVG_DECOMPRESSION_TIME=$(awk '/Decoded/ {sum+=$3} END { print sum/NR }' test/decompression.log) + echo "Average decompression time: $AVG_DECOMPRESSION_TIME milliseconds" +fi + +# Average quality metric in decibels +AVG_IMAGE_QUALITY_METRIC=0 +if [[ -f 'test/quality.log' ]]; then + AVG_IMAGE_QUALITY_METRIC=$(awk '/Quality/ {sum+=$3} END { print sum/NR}' test/quality.log) + echo "Average image quality metric $AVG_IMAGE_QUALITY_METRIC decibels" +fi + +echo '' + +# Weighted compression score +WEIGHTED_COMPRESSION_SCORE=$(echo | awk "{ print ($AVG_COMPRESSION_RATIO * 100 * 0.35)}") +echo "Weighted compression score (35%): $WEIGHTED_COMPRESSION_SCORE" + +# Weighted time score +WEIGHTED_TIME_SCORE=$(echo | awk "{ print ((1000 - $AVG_DECOMPRESSION_TIME) / 1000 * 100 * 0.25)}") +echo "Weighted time score (25%): $WEIGHTED_TIME_SCORE" + +# Weighted quality score +WEIGHTED_QUALITY_SCORE=$(echo | awk "{ print ($AVG_IMAGE_QUALITY_METRIC * 0.20)}") +echo "Weighted quality score (20%): $WEIGHTED_QUALITY_SCORE" + +echo '' + +# Total score +TOTAL_SCORE=$(echo | awk "{ print $WEIGHTED_COMPRESSION_SCORE + $WEIGHTED_TIME_SCORE + $WEIGHTED_QUALITY_SCORE}") +echo "Total score (80%): $TOTAL_SCORE" + +# The last 20% for the presentation will be awarded manually by the judges + +# Submit score +if [[ "$GITHUB_REF_NAME" == "main" ]]; then + ./scripts/submit_score.sh $TOTAL_SCORE +fi diff --git a/scripts/test_non_ai.sh b/scripts/test_non_ai.sh new file mode 100755 index 0000000..9100b3a --- /dev/null +++ b/scripts/test_non_ai.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# Remove output from previous run +rm -rf ./test + +# Loop through sample models +jq -r '.[].name' ./sample-models/model-index.json | while read i; do + + input_file_path="./sample-models/$i/obj/$i.obj" + encoded_file_path="./test/encoded/$i/$i.obj" + decoded_file_path="./test/decoded/$i/$i.obj" + input_render_file_path="./test/rendered/$i/$i.input.png" + decoded_render_file_path="./test/rendered/$i/$i.decoded.png" + + if [[ -f $input_file_path ]]; then + # Create folders + mkdir -p ./test/encoded/$i + mkdir -p ./test/decoded/$i + mkdir -p ./test/rendered/$i + + # Run encoder + ./encoder -i $input_file_path -o $encoded_file_path > /dev/null 2>&1 + + # Calculate compression ratio + if [[ -f $encoded_file_path ]]; then + input_file_size=$(ls -l $input_file_path | awk '{ total += $5 }; END { print total }') + # echo $input_file_size + encoded_file_size=$(ls -l $encoded_file_path | awk '{ total += $5 }; END { print total }') + # echo $encoded_file_size + RATIO=$(echo | awk "{print ($input_file_size - $encoded_file_size) / $input_file_size }") + echo "Encoded $encoded_file_path $RATIO" | tee -a test/compression.log + else + echo "Failed $encoded_file_path" | tee -a test/compression.log + fi + + # Run decoder + { time ./decoder -i $encoded_file_path -o $decoded_file_path > /dev/null 2>&1 ; } 2> time.log + + # Calculate decompression time + if [[ -f $decoded_file_path ]]; then + TIME=$(awk '/user|sys/ {split($2,a,/m|s/); sum+=(a[1] * 60000) + (a[2] * 1000)} END { print sum }' time.log) + echo "Decoded $decoded_file_path $TIME" | tee -a test/decompression.log + else + echo "Failed $decoded_file_path" | tee -a test/decompression.log + fi + + # Calculate image quality + if [[ -f $input_file_path ]] && [[ -f $decoded_file_path ]]; then + python3 scripts/offscreen_render.py -i $input_file_path -o $input_render_file_path 2> /dev/null + python3 scripts/offscreen_render.py -i $decoded_file_path -o $decoded_render_file_path 2> /dev/null + + IQM=0 + if [[ -f $input_render_file_path ]] && [[ -f $decoded_render_file_path ]]; then + IQM=$(python3 scripts/iqm.py -r $input_render_file_path -m $decoded_render_file_path) + fi + + if [[ ! -z "$IQM" ]]; then + echo "Quality $decoded_render_file_path $IQM" | tee -a test/quality.log + else + echo "Failed $decoded_render_file_path" | tee -a test/quality.log + fi + fi + + echo '' + + fi +done + +# Calculate score + +# Average compression percentage +AVG_COMPRESSION_RATIO=0 +if [[ -f 'test/compression.log' ]]; then + AVG_COMPRESSION_RATIO=$(awk '/Encoded/ {sum+=$3} END { print sum/NR}' test/compression.log) + echo "Average compression ratio: $AVG_COMPRESSION_RATIO percent" +fi + +# Average time in milliseconds +AVG_DECOMPRESSION_TIME=0 +if [[ -f 'test/decompression.log' ]]; then + AVG_DECOMPRESSION_TIME=$(awk '/Decoded/ {sum+=$3} END { print sum/NR }' test/decompression.log) + echo "Average decompression time: $AVG_DECOMPRESSION_TIME milliseconds" +fi + +# Average quality metric in decibels +AVG_IMAGE_QUALITY_METRIC=0 +if [[ -f 'test/quality.log' ]]; then + AVG_IMAGE_QUALITY_METRIC=$(awk '/Quality/ {sum+=$3} END { print sum/NR}' test/quality.log) + echo "Average image quality metric $AVG_IMAGE_QUALITY_METRIC decibels" +fi + +echo '' + +# Weighted compression score +WEIGHTED_COMPRESSION_SCORE=$(echo | awk "{ print ($AVG_COMPRESSION_RATIO * 100 * 0.35)}") +echo "Weighted compression score (35%): $WEIGHTED_COMPRESSION_SCORE" + +# Weighted time score +WEIGHTED_TIME_SCORE=$(echo | awk "{ print ((1000 - $AVG_DECOMPRESSION_TIME) / 1000 * 100 * 0.25)}") +echo "Weighted time score (25%): $WEIGHTED_TIME_SCORE" + +# Weighted quality score +WEIGHTED_QUALITY_SCORE=$(echo | awk "{ print ($AVG_IMAGE_QUALITY_METRIC * 0.20)}") +echo "Weighted quality score (20%): $WEIGHTED_QUALITY_SCORE" + +echo '' + +# Total score +TOTAL_SCORE=$(echo | awk "{ print $WEIGHTED_COMPRESSION_SCORE + $WEIGHTED_TIME_SCORE + $WEIGHTED_QUALITY_SCORE}") +echo "Total score (80%): $TOTAL_SCORE" + +# The last 20% for the presentation will be awarded manually by the judges + +# Submit score +if [[ "$GITHUB_REF_NAME" == "main" ]]; then + ./scripts/submit_score.sh $TOTAL_SCORE +fi diff --git a/src/HuffmanC.cpp b/src/HuffmanC.cpp new file mode 100644 index 0000000..ce062f2 --- /dev/null +++ b/src/HuffmanC.cpp @@ -0,0 +1,460 @@ +/* + * This is written based on : https://www.youtube.com/watch?v=dM6us854Jk0 + * And also : https://www.youtube.com/watch?v=apcCVfXfcqE (This guys is cool ;D) + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; +using namespace std::chrono; + +// Change here AND the definition of counterMap to control compress rate +const int windowSz = 2; +ofstream dbgPrint("dbgPrettyPrint.diag"); + +// Auto indentation +class VecCompressor +{ +public: + VecCompressor() {} + void putValue(int value) + { + value <<= bitPos - 1; + tempChar += value, bitPos--; + if (bitPos == 0) + ret.push_back((char)tempChar), tempChar = 0, bitPos = 8; + } + const vector &returnResult() + { + ret.push_back((char)tempChar); + return ret; + } + +private: + vector ret; + unsigned char tempChar = 0; + int bitPos = 8; +}; + +template +void printAsBinary(T input) +{ + bitset<64> x(input); + for (int i = 0; i < 64; i++) + { + if (i % 8 == 0 && i != 0) + cout << ' '; + cout << x[i]; + } + // cout << endl; +} + +unsigned long long charToUll(vector::iterator start, vector::iterator end) +{ + unsigned long long ret = 0L; + for (auto it = start; it != end - 1; it++) + { + ret += (unsigned char)*it, ret <<= 8; + } + ret += (unsigned char)*(end - 1); + return ret; +} + +vector ullToChar(unsigned long long in) +{ + vector ret(8); + for (int i = 0; i < 8; i++) + { + ret[i] = (char)(in & 0b11111111); + in >>= 8; + } + reverse(ret.begin(), ret.end()); + return ret; +} + +class Node +{ +public: + Node(bool flag, unsigned long long int val, Node *left, Node *right) + : flag(flag), val(val), left(left), right(right) + { + } + + Node(bool flag, unsigned long long int val) + : flag(flag), val(val) + { + } + +public: + bool flag; // mark if it's end node, 1 if it's end node, 0 is not. REMOVABLE? + unsigned long long val; // node val + Node *left = nullptr; + Node *right = nullptr; +}; + +class HuffmanEncoder +{ +public: + explicit HuffmanEncoder(const vector &unprocessedArray) + : unprocessedStream(unprocessedArray) + { + } + + void putMap() + { + for (int i = 0; i < 10; i++) + { + cout << (int)(unsigned char)unprocessedStream[i] << ' '; + } + cout << endl; + cout << "<-- putMap Begin --> " << endl; + int windowSzTrack = 0; + unsigned long long ret = 0L; + int tracer = 0; + for (const char &i : unprocessedStream) + { + if (tracer < 10) + { + cerr << (int)(unsigned char)i << ' '; + tracer++; + } + if (windowSzTrack < windowSz - 1) + { + ret += (unsigned char)i, ret <<= 8; + windowSzTrack++; + } + else + { + ret += (unsigned char)i; + if (tracer < 10) + { + cerr << " is " << ret << ' '; + } + counterMap[ret]++, ret = 0, windowSzTrack = 0; + } + } + if (ret) + counterMap[ret]++; + + // cerr << "=============" << endl; + + cout << "<-- putMap End --> " << endl; + } + + Node *buildTree() + { + auto cmpIntMinHeap = [](const pair &lhs, const pair &rhs) + { + return lhs.second > rhs.second; + }; + priority_queue, vector>, decltype(cmpIntMinHeap)> pqii( + cmpIntMinHeap); // create min heap to store data in counterMap + for (const auto iteratorTuple : counterMap) + pqii.push(iteratorTuple); + auto cmpNodeMinHeap = [](const Node *lhs, const Node *rhs) + { return lhs->val > rhs->val; }; + priority_queue, decltype(cmpNodeMinHeap)> pqNode(cmpNodeMinHeap); + Node *currentMax = nullptr; + // Code Block here, so you don't mess up with global variables + { + auto top = pqii.top(); + pqii.pop(); + auto top2 = pqii.top(); + pqii.pop(); + Node *LHSNode = new Node(true, top.first); + Node *RHSNode = new Node(true, top2.first); + Node *newNode = new Node(false, top.second + top2.second, LHSNode, RHSNode); + pqNode.push(newNode); + // Maintain a max node to check if we need to build a new Node + currentMax = newNode; + } + + while (!pqii.empty()) + { + auto top = pqii.top(); + pqii.pop(); + // If current item's count is larger than the count of the largest node, we want to create a new node to store it. + if (currentMax->val > top.second && !pqii.empty()) + { + auto top2 = pqii.top(); + pqii.pop(); + Node *LHSNode = new Node(true, top.first); + Node *RHSNode = new Node(true, top2.first); + Node *newNode = new Node(false, top.second + top2.second, LHSNode, RHSNode); + pqNode.push(newNode); + if (currentMax->val < newNode->val) + { + currentMax = newNode; + } + } + else + { + Node *LHSNode = new Node(true, top.first); + Node *parentNodeOfBoth = new Node(false, pqNode.top()->val + top.second, LHSNode, pqNode.top()); + // As we need to combine all trees later, this step is to remove all trees that already have a parent. + pqNode.pop(); + pqNode.push(parentNodeOfBoth); + if (currentMax->val < parentNodeOfBoth->val) + { + currentMax = parentNodeOfBoth; + } + } + } + // + while (!pqNode.empty() && pqNode.size() != 1) + { + auto top1 = pqNode.top(); + pqNode.pop(); + auto top2 = pqNode.top(); + pqNode.pop(); + Node *parentNode = new Node(false, top1->val + top2->val, top1, top2); + pqNode.push(parentNode); + } + return pqNode.top(); + } + + void print2DUtil(Node *root, int space) + { + // Base case + if (root == nullptr) + return; + + space += 2; + + print2DUtil(root->right, space); + + dbgPrint << endl; + for (int i = 2; i < space; i++) + dbgPrint << " "; + dbgPrint << root->val << "\n"; + + print2DUtil(root->left, space); + } + + // Wrapper over print2DUtil() + void print2D(Node *root) + { + dbgPrint << "Sum of all Nodes: " << root->val << endl; + // Pass initial space count as 0 + print2DUtil(root, 0); + } + + void _getTreeContent(Node *currentNode, const string &nodePath, int depth) + { + if (currentNode->flag) + { + encodeCodeMap[currentNode->val] = nodePath; + // encodeTreeMaxDepth = max(encodeTreeMaxDepth, depth); + return; + } + _getTreeContent(currentNode->left, nodePath + "0", depth + 1); + _getTreeContent(currentNode->right, nodePath + "1", depth + 1); + } + // Wrapper over _getTreeContent + void getTreeContent() + { + _getTreeContent(this->head, "", 1); + // assert(encodeTreeMaxDepth != -1 && "encodeTreeMaxDepth is still -1, what the fuck?"); + } + + // Read every $windowsSz window and output a result in the `encodedOutput` vec + vector substitute() + { + for (int i = 10; i > 0; i--) + cout << (int)(unsigned char)*(unprocessedStream.end() - i) << ' '; + VecCompressor vecCompressor; + int windowSzTrack = 0; + unsigned long long ret = 0L; + int tracer = 0; + int thisIsAVeryLongValueSoIDontFuckUp = 0; + for (const char &i : unprocessedStream) + { + if (tracer < 10) + { + // cout << (int)(unsigned char)i << ' '; + tracer++; + } + if (windowSzTrack < windowSz - 1) + { + ret += (unsigned char)i, ret <<= 8; + windowSzTrack++; + } + else + { + ret += (unsigned char)i; + // ret is the current window's value + // cerr << ret << ' '; + assert(encodeCodeMap.count(ret) && "WTF where is my content?"); + auto encodeCode = encodeCodeMap[ret]; + for (auto &a : encodeCode) + { + if (thisIsAVeryLongValueSoIDontFuckUp < 10) + cout << '[' << a << ']' << ' ', thisIsAVeryLongValueSoIDontFuckUp++; + vecCompressor.putValue(a - '0'); + } + ret = 0, windowSzTrack = 0; + } + } + if (ret) + { + auto encodeCode = encodeCodeMap[ret]; + cout << encodeCode << endl; + for (auto &a : encodeCode) + { + vecCompressor.putValue(a - '0'); + } + } + auto result = vecCompressor.returnResult(); + cout << "<->" << endl; + for (int i = 10; i > 0; i--) + cout << (int)(unsigned char)*(result.end()-i) << ' '; + cout << endl; + return result; + } + + vector compress() + { + // for (int i = 0; i < 10; i++) { + // cout << (int)(unsigned char)unprocessedStream[i] << ' '; + // } + // cout << endl; + putMap(); + ofstream dbgFile("dbg.log"); + for (auto &p : counterMap) + dbgFile << p.first << ' ' << p.second << endl; + this->head = buildTree(); + // print2D(this->head); + getTreeContent(); + ofstream encodeCodeMapDBG; + encodeCodeMapDBG.open("encodeCodeMapDbg.log"); + // encodeCodeMapDBG << "Max depth of tree: " << encodeTreeMaxDepth << endl; + for (auto &p : encodeCodeMap) + { + encodeCodeMapDBG << p.first << ' ' << p.second << endl; + // // Print encode path as binary: L is 0, R is 1 + // // bitset<64> x(p.second); + // // for (int i = 0; i < 64; i++) { + // // if (i % 8 == 0 && i != 0) + // // encodeCodeMapDBG << ' '; + // // encodeCodeMapDBG << x[i]; + // // } + // // encodeCodeMapDBG << endl; + } + // indentConst = ((int)encodeTreeMaxDepth / 8) + 1; + storeMap(); + vector result = substitute(); + // for (int i = 0; i < 10; i++) + // cout << '*' << (int)(unsigned char)result[i] << '*' << ' '; + return result; + } + + vector storeMap() + { + vector encodeMapResultOutput; + unsigned short count = 0; + for (const auto &p : encodeCodeMap) + { + if ((int)p.first < count) + cout << "Oops " << p.first << ' ' << count << endl; + count = p.first; + assert(count <= 65535 && count >= 0); + char *dataCount = reinterpret_cast(&count); + + unsigned char prefix_zero = 0; + for (const auto &c : p.second) + { + if (c == '0') + prefix_zero++; + else + break; + } + + encodeMapResultOutput.insert(encodeMapResultOutput.end(), dataCount, dataCount + sizeof(unsigned short)); + string currentString = p.second; + int currentResult = 0; + for (char i : currentString) + { + currentResult <<= 1, currentResult += (i - '0'); + } + + encodeMapResultOutput.push_back((char)prefix_zero); + char *data = reinterpret_cast(¤tResult); + encodeMapResultOutput.insert(encodeMapResultOutput.end(), data, data + 4); + auto *returnBackDebug = reinterpret_cast(data); + assert(currentString.size() < 32); + assert(currentResult == *returnBackDebug); + } + ofstream outfile("encodeMapResultOutput.bin", ios::out | ios::binary); + outfile.write(&encodeMapResultOutput[0], encodeMapResultOutput.size()); + return encodeMapResultOutput; + } + + void saveAsFile(const string &fileName) + { + ofstream outfile(fileName, ios::out | ios::binary); + vector outputBufferVec; + auto mapResult = storeMap(); + unsigned int lengthOfMapResult = mapResult.size(); + char *lengthOfMapResultData = reinterpret_cast(&lengthOfMapResult); + outputBufferVec.insert(outputBufferVec.end(), lengthOfMapResultData, lengthOfMapResultData + sizeof(int)); + outputBufferVec.insert(outputBufferVec.end(), mapResult.begin(), mapResult.end()); + auto literal = compress(); + outputBufferVec.insert(outputBufferVec.end(), literal.begin(), literal.end()); + cout << "Map Result: " << lengthOfMapResult << " Literal Len: " << literal.size() << endl; + outfile.write(&outputBufferVec[0], outputBufferVec.size()); + } + + vector outputWholeVec() + { + vector outputBufferVec; + auto mapResult = storeMap(); + int lengthOfMapResult = mapResult.size(); + char *lengthOfMapResultData = reinterpret_cast(&lengthOfMapResult); + outputBufferVec.insert(outputBufferVec.end(), lengthOfMapResultData, lengthOfMapResultData + sizeof(int)); + outputBufferVec.insert(outputBufferVec.end(), mapResult.begin(), mapResult.end()); + auto literal = compress(); + outputBufferVec.insert(outputBufferVec.end(), literal.begin(), literal.end()); + cout << "Map Result: " << lengthOfMapResult << " Literal Len: " << literal.size() << endl; + return outputBufferVec; + } + +private: + const vector unprocessedStream; + map counterMap; // track occurrence of each element in window + Node *head = nullptr; + map encodeCodeMap; + // int encodeTreeMaxDepth = -1; + + // Sequence output + // vector& encodedOutput; +}; + +// int main() +// { +// ifstream inStream("huffmanC.obj", ios::in | ios::binary); +// std::vector inputVecUnprocessed((std::istreambuf_iterator(inStream)), istreambuf_iterator()); +// // inputVecUnprocessed.reserve(inStream.tellg()); + +// // std::copy(istream_iterator(inStream), istream_iterator(), +// // back_inserter(inputVecUnprocessed)); + +// HuffmanEncoder huffmanEncoder(inputVecUnprocessed); +// auto result = huffmanEncoder.compress(); +// auto resultVector = huffmanEncoder.outputWholeVec(); + +// ofstream outStream("outStream.obj", ios::binary | ios::out); +// outStream.write((char *)&resultVector[0], resultVector.size()); + +// // ofstream outDecompStream("outDecomped.obj", ios::binary | ios::out); +// // outDecompStream.write((char*)&deCompedRes[0], deCompedRes.size()); +// } diff --git a/src/HuffmanD.cpp b/src/HuffmanD.cpp new file mode 100644 index 0000000..202f877 --- /dev/null +++ b/src/HuffmanD.cpp @@ -0,0 +1,256 @@ +#include + +using namespace std; +using namespace std::chrono; + +class Node +{ +public: + Node(bool flag, unsigned long long int val, Node *left, Node *right) + : flag(flag), val(val), left(left), right(right) + { + } + + Node(bool flag, unsigned long long int val) + : flag(flag), val(val) + { + } + +public: + bool flag; // mark if it's end node, 1 if it's end node, 0 is not. REMOVABLE? + unsigned long long val; // node val + Node *left = nullptr; + Node *right = nullptr; +}; + +class HuffmanDecoder +{ +private: + vector inputStream; + Node *head = new Node(false, NULL); + +public: + explicit HuffmanDecoder(const vector &inputStream) + : inputStream(inputStream) + { + } + + pair&, vector&> splitContent() + { + // Length of first section + int *lengthOfEncodeMapPtr = reinterpret_cast(&inputStream[0]); + int lengthOfEncodeMap = *lengthOfEncodeMapPtr; + // Remove header + inputStream.erase(inputStream.begin(), inputStream.begin() + 4); + // Dictionary + vector encodeCodeMapVector(inputStream.begin(), inputStream.begin() + lengthOfEncodeMap); + inputStream.erase(inputStream.begin(), inputStream.begin() + lengthOfEncodeMap); + // inputStream: Encoded data + return {encodeCodeMapVector, inputStream}; + } + + map& getEncodeMapTree(vector& original) + { + // assert(original.size() % 7 == 0 && "Bruh you fed me the WRONG VECTOR"); + map ret; + // [01010101 0101] + + for (int i = 0; i < original.size(); i += 7) + { + auto *originalCode = reinterpret_cast(&original[i]); + char prefix_zeros = original[i + 2]; + auto *outputStringCode = reinterpret_cast(&original[i + 3]); + int outputStringCodeLiteral = *outputStringCode; + string result; + + while (outputStringCodeLiteral) + { + result.push_back((outputStringCodeLiteral & 1) + '0'); + outputStringCodeLiteral >>= 1; + } + for (int j = 0; j < prefix_zeros; j++) + result.push_back('0'); + reverse(result.begin(), result.end()); + ret[(unsigned long long)*originalCode] = result; + } + + // while (!original.empty()) + // { + // vector currentBuffer(original.begin(), original.begin() + 7); + // original.erase(original.begin(), original.begin() + 7); + + // auto *originalCode = reinterpret_cast(¤tBuffer[0]); + // char prefix_zeros = currentBuffer[2]; + // auto *outputStringCode = reinterpret_cast(¤tBuffer[3]); + // int outputStringCodeLiteral = *outputStringCode; + // string result; + // while (outputStringCodeLiteral) + // { + // // result.insert(result.begin(), ((outputStringCodeLiteral & 1) + '0')); + // result.push_back((outputStringCodeLiteral & 1) + '0'); + // outputStringCodeLiteral >>= 1; + // } + // for (int i = 0; i < prefix_zeros; i++) + // result.push_back('0'); + // reverse(result.begin(), result.end()); + // ret[(unsigned long long)*originalCode] = result; + // } + return ret; + } + + void _rebuildTree(Node *currentNode, const string ¤tRoute, const unsigned long long &finalValue) + { + if (currentRoute.empty()) + { + assert(currentNode->val == NULL); + currentNode->flag = true, currentNode->val = finalValue; + return; + } + string newRoute(currentRoute.begin() + 1, currentRoute.end()); + assert(newRoute != currentRoute && &newRoute != ¤tRoute); + if (currentRoute[0] - '0') + { + if (currentNode->right == nullptr) + { + Node *newNode = new Node(false, NULL); + currentNode->right = newNode; + _rebuildTree(currentNode->right, newRoute, finalValue); + } + else + _rebuildTree(currentNode->right, newRoute, finalValue); + } + else + { + if (currentNode->left == nullptr) + { + Node *newNode = new Node(false, NULL); + currentNode->left = newNode; + _rebuildTree(currentNode->left, newRoute, finalValue); + } + else + _rebuildTree(currentNode->left, newRoute, finalValue); + } + } + + void rebuildTree(const map &inputMap) + { + for (const auto &p : inputMap) + { + _rebuildTree(head, p.second, p.first); + + Node *currentHead = head; + // '1' - '0' = 1 + // for (int i = 0; iright; + else + currentHead = currentHead->left; + + assert(currentHead != nullptr); + } + assert(currentHead != nullptr && currentHead->val == p.first); + } + } + + vector& literal(vector &input) const + { + + vector ret; + + // optimize + // char[0b10101010] -> int[1 0 1 0 1 0 1 0] + // vector inputButBool; + // for (auto a : input) + // { + // auto currentU8 = (unsigned char)*reinterpret_cast(&a); + // for (int i = 0; i < 8; i++) + // { + // inputButBool.push_back(currentU8 & 0b10000000), currentU8 <<= 1; + // } + // } + // Node *currentHead = head; + // for (const auto &a : inputButBool) + // { + // if (a) + // currentHead = currentHead->right; + // else + // currentHead = currentHead->left; + // if (currentHead->flag) + // { + // auto result = currentHead->val; + // ret.push_back((char)(unsigned char)((result & 0xFF00) >> 8)); + // ret.push_back((char)(unsigned char)(result & 0xFF)); + // currentHead = head; + // } + // } + Node *currentHead = head; + for (auto &a : input) + { + auto currentU8 = (unsigned char)*reinterpret_cast(&a); + for (int i = 0; i < 8; i++) + { + const bool a = (currentU8 & (1 << (7 - i))); + if (a) + currentHead = currentHead->right; + else + currentHead = currentHead->left; + if (currentHead->flag) + { + auto result = currentHead->val; + ret.push_back((char)(unsigned char)((result & 0xFF00) >> 8)); + ret.push_back((char)(unsigned char)(result & 0xFF)); + currentHead = head; + } + } + } + + return ret; + } + + vector& decompress() + { + + auto splittedContent = splitContent(); + + // multithread + // ~70ms + auto encodeMapTree = getEncodeMapTree(splittedContent.first); + + rebuildTree(encodeMapTree); + + // 37 + auto result = literal(splittedContent.second); + if (result.back() == 0) + result.pop_back(); + return result; + } +}; + +// int main(int argc, char *argv[]) +// { + +// // Read in + +// string input_location = "outStream.obj", output_location = "rebuildStream.obj"; +// for (int i = 0; i < argc; i++) +// { +// if (!strcmp(argv[i], "-i")) +// input_location = argv[++i]; +// else if (!strcmp(argv[i], "-o")) +// output_location = argv[++i]; +// } + +// ifstream inStream(input_location, ios::in | ios::binary); +// std::vector inputFile((std::istreambuf_iterator(inStream)), istreambuf_iterator()); + +// // start + +// HuffmanDecoder huffmanDecoder(inputFile); +// auto result = huffmanDecoder.decompress(); +// ofstream rebuildStream(output_location, ios::binary | ios::out); +// rebuildStream.write((char *)&result[0], result.size()); + +// return 0; +// } \ No newline at end of file diff --git a/src/decoder_main.cpp b/src/decoder_main.cpp new file mode 100644 index 0000000..f07fdae --- /dev/null +++ b/src/decoder_main.cpp @@ -0,0 +1,325 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +// header for read in using C +#include +#include + +#include "serialize.h" +#include "HuffmanD.cpp" +// header for decompression +#include +#include "LZSS_Decompression.h" +//-- +using namespace std; +using namespace jls; +// Yes, I know this is bad code. This is for testing so nobody should care. +const bool debug = false; +string debug_output_location; ofstream ferr; + +string input_location; +string output_location; +string file_contents; +char *res; +char ores[50000004]; +int res_len; +extern bool Compress_Twice; +fixed_byte_stream fbs(NULL, 0); +int fread_bias = 0; +// vector vs; +// vector vts; +// vector vns; +// vector fs; + +/* ubyte -> numbers of occurence of that input */ +int input_pro[257]; +const int input_pro_size=256; +const bool debug_pro=false; + +void readin() { + ifstream fin(input_location, ios::binary | ios::in); + file_contents = string(istreambuf_iterator(fin), istreambuf_iterator()); + res = file_contents.data(); + res_len = file_contents.length(); + fin.close(); +} + +void reconstruct_and_output(char *data, int len, string outfile) { + if (outfile == "") { + outfile = output_location; + } + fbs = fixed_byte_stream((unsigned char*) data, len); + ofstream fout(outfile); + ubyte flags[FLAG_BYTES] = {}; + for (int i = 0; i < FLAG_BYTES; i++) { + fbs >> flags[i]; + } + int p = 0; + int nblocks; + fbs >> nblocks; + // assert (nblocks == NUM_BLOCKS); + int nv; + fbs >> nv; + for (int i = 0; i < nv; i++) { + node_v v; + fbs >> v; + p += v.write_obj(ores + p); + ores[p++] = '\n'; + } + int nvt; + fbs >> nvt; + for (int i = 0; i < nvt; i++) { + node_vt vt; + fbs >> vt; + p += vt.write_obj(ores + p); + ores[p++] = '\n'; + } + int nvn; + fbs >> nvn; + for (int i = 0; i < nvn; i++) { + node_vn vn; + fbs >> vn; + p += vn.write_obj(ores + p); + ores[p++] = '\n'; + } + int nf; + fbs >> nf; + node_f f1; + fbs >> f1; + p += f1.write_obj(ores + p); + ores[p++] = '\n'; + for (int i = 1; i < nf; i++) { + delta_f df; + fbs >> df; + f1 += df; + p += f1.write_obj(ores + p); + ores[p++] = '\n'; + } + fout.write(ores, p); + fout.close(); +} + +void read_chunk_header(FILE* f, long long* size, + long long* extra) { + unsigned char buffer[8]; + fread(buffer, 1, 8, f); + *size = read_uint32(buffer ) & 0xffffffff; + *extra = read_uint32(buffer+4) & 0xffffffff; +} +void read_chunk_header(FILE* f, long long* size) { + unsigned char buffer[4]; + fread(buffer, 1, 4, f); + *size = read_uint32(buffer) & 0xffffffff; +} +void read_chunk_header(vector &input, long long *size, + long long *extra) +{ + unsigned char buffer[8]; + // fread(buffer, 1, 8, f); + copy(input.begin() + fread_bias, input.begin() + fread_bias + 8, buffer); + fread_bias += 8; + *size = read_uint32(buffer) & 0xffffffff; + *extra = read_uint32(buffer + 4) & 0xffffffff; +} +void read_chunk_header(vector &input, long long *size) +{ + unsigned char buffer[4]; + // fread(buffer, 1, 4, f); + copy(input.begin() + fread_bias, input.begin() + fread_bias + 4, buffer); + fread_bias += 4; + *size = read_uint32(buffer) & 0xffffffff; +} +/** + Decompress a block of compressed data and returns the size of the + output data. If error occurs, e.g. the compressed data is + corrupted or the OUTPUT buffer is not large enough, then 0 + will be returned. + */ + +void LZSS_Decompression(bool Compress_Twice){ + FILE* infile =fopen(input_location.c_str(),"rb"); + long long numbytes=0,numbytes2=0,chunk_size=0; + /* Get the number of bytes */ + // fseek(infile, 0L, SEEK_END); + // const long long numbytes = ftell(infile); + + // /* reset the file position indicator to + // the beginning of the file */ + // fseek(infile, 0L, SEEK_SET); + read_chunk_header(infile, &numbytes, &numbytes2); + + unsigned char* buffer = new unsigned char[numbytes]; + unsigned char* buffer2 = new unsigned char[numbytes2]; + + // FILE *f=fopen(output_location.c_str(),"wb"); + // fwrite(buffer, 1, numbytes , f); + // fclose(f); + if(Compress_Twice){ + long long chunk_extra; + read_chunk_header(infile, &chunk_extra); + fread(buffer, sizeof(char), numbytes, infile); + unsigned char* buffer3 = new unsigned char[chunk_extra]; + LZSS_Decoder Lzss_Decoder(buffer, numbytes, buffer2); + chunk_size=Lzss_Decoder.Decompress(); + LZSS_Decoder Lzss_Decoder2(buffer2, numbytes2, buffer3); + long long chunk_size2 =Lzss_Decoder2.Decompress(); + res = (char*) buffer3; + res_len = chunk_size2; + reconstruct_and_output(res, res_len, output_location); + //Reconstruct before being deleted + delete[]buffer3; + } + else{ + fread(buffer, sizeof(char), numbytes, infile); + LZSS_Decoder Lzss_Decoder(buffer, numbytes, buffer2); + chunk_size=Lzss_Decoder.Decompress(); + res = (char*) buffer2; + res_len = chunk_size; + reconstruct_and_output(res, res_len, output_location); + //Reconstruct before being deleted + } + //chunk_extra no longer in use + //cout< huffDin((std::istreambuf_iterator(inStream)), istreambuf_iterator()); + vector &infile = huffDin; + if (__builtin_expect(huffDin.back() == 2, 0)) + { + huffDin.pop_back(); + HuffmanDecoder huffmanDecoder(huffDin); + infile = huffmanDecoder.decompress(); + } + else + { + huffDin.pop_back(); + } + long long numbytes = 0, numbytes2 = 0, chunk_size = 0; + /* Get the number of bytes */ + // fseek(infile, 0L, SEEK_END); + // const long long numbytes = ftell(infile); + + // /* reset the file position indicator to + // the beginning of the file */ + // fseek(infile, 0L, SEEK_SET); + read_chunk_header(infile, &numbytes, &numbytes2); + + unsigned char *buffer = new unsigned char[numbytes]; + unsigned char *buffer2 = new unsigned char[numbytes2]; + + // FILE *f=fopen(output_location.c_str(),"wb"); + // fwrite(buffer, 1, numbytes , f); + // fclose(f); + if (Compress_Twice) + { + long long chunk_extra; + read_chunk_header(infile, &chunk_extra); + // fread(buffer, sizeof(char), numbytes, infile); + copy(infile.begin() + fread_bias, infile.begin() + fread_bias + numbytes, buffer); + fread_bias += numbytes; + unsigned char *buffer3 = new unsigned char[chunk_extra]; + LZSS_Decoder Lzss_Decoder(buffer, numbytes, buffer2); + + chunk_size = Lzss_Decoder.Decompress(); + LZSS_Decoder Lzss_Decoder2(buffer2, numbytes2, buffer3); + long long chunk_size2 = Lzss_Decoder2.Decompress(); + res = (char *)buffer3; + res_len = chunk_size2; + reconstruct_and_output(res, res_len, output_location); + // Reconstruct before being deleted + } + else + { + // fread(buffer, sizeof(char), numbytes, infile); + + copy(infile.begin() + fread_bias, infile.begin() + fread_bias + numbytes, buffer); + fread_bias += numbytes; + + LZSS_Decoder Lzss_Decoder(buffer, numbytes, buffer2); + + chunk_size = Lzss_Decoder.Decompress(); + res = (char *)buffer2; + res_len = chunk_size; + reconstruct_and_output(res, res_len, output_location); + // Reconstruct before being deleted + } + // chunk_extra no longer in use + // cout< +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +// header for read in using C +#include +#include +#include "serialize.h" +// header for compression +#include "LZSS_Compression.h" +#include "HuffmanC.cpp" +using namespace std; +using namespace jls; + +string input_location, output_location; + +vector res; + +vector vs; + +vector vts; +vector vns; +vector fs; + +/* ubyte -> numbers of occurence of that input */ +map input_pro; +const bool debug_pro=false; +extern bool Compress_Twice; +/** + * @brief Reads in a 6 decimal place float as string and returns an integer that is exactly that float * 1e6. + * + * @param str the string representing the float. + * @return int + */ +int strfloat_mult1M_to_int(string str) { + int ret = 0; + int n = str.length(); + int pos_of_dp = str.find('.'); + if (n - pos_of_dp != 7) { + cerr << "WARNING: Encountered non-6-dp number: " << str << endl; + } + for (int i = 0; i < n; i++) { + if (isdigit(str[i])) { + ret = ret * 10 + (str[i] - '0'); + } + } + if (str[0] == '-') { + ret *= -1; + } + return ret; +} + +/** + * @brief Converts a face string ("[integer]/[integer]/[integer]") and returns a tuple of those three integers. + * + * @param str + * @return tiii + */ +tiii face_to_tuple(string str) { + int p1 = str.find('/'); + int a = stoi(str.substr(0, p1)); + int p2 = str.find('/', p1 + 1); + + int b; + if (p2 - p1 > 1) { + b = stoi(str.substr(p1 + 1, p2 - p1 - 1)); + } + else { + b = 0; + } + int c = stoi(str.substr(p2 + 1)); + return make_tuple(a, b, c); +} + +void readin() { + ifstream fin(input_location); + string node_type; + while (fin >> node_type) { + if (node_type == "v") { + string s1, s2, s3; + fin >> s1 >> s2 >> s3; + vs.push_back(node_v( + strfloat_mult1M_to_int(s1), + strfloat_mult1M_to_int(s2), + strfloat_mult1M_to_int(s3))); + } + else if (node_type == "vn") { + string s1, s2, s3; + fin >> s1 >> s2 >> s3; + vns.push_back(node_vn( + strfloat_mult1M_to_int(s1), + strfloat_mult1M_to_int(s2), + strfloat_mult1M_to_int(s3))); + } + else if (node_type == "vt") { + string s1, s2; + fin >> s1 >> s2; + vts.push_back(node_vt( + strfloat_mult1M_to_int(s1), + strfloat_mult1M_to_int(s2))); + } + else if (node_type == "f") { + string s1, s2, s3; + fin >> s1 >> s2 >> s3; + fs.push_back(node_f( + face_to_tuple(s1), + face_to_tuple(s2), + face_to_tuple(s3))); + } + } + fin.close(); +} + +void obj_output(string outfile = "") { + if (outfile == "") { + outfile = output_location + ".obj"; + } + ofstream fout(outfile); + for (auto x : vs) { + fout << x.dump_output_obj() << "\n"; + } + for (auto x : vts) { + fout << x.dump_output_obj() << "\n"; + } + for (auto x : vns) { + fout << x.dump_output_obj() << "\n"; + } + for (auto x : fs) { + fout << x.dump_output_obj() << "\n"; + } + fout << flush; + return; +} + +void generate_output() { + // Add flags + for (int i = 0; i < FLAG_BYTES; i++) { + res.push_back(FLAGS[i]); + } + push_int(res, NUM_BLOCKS); + push_int(res, vs.size()); + for (auto v : vs) { + v.to_stream(res); + } + push_int(res, vts.size()); + for (auto vt : vts) { + vt.to_stream(res); + } + push_int(res, vns.size()); + for (auto vn : vns) { + vn.to_stream(res); + } + int nfs = fs.size(); + push_int(res, nfs); + fs[0].to_stream(res); + for (int i = 1; i < nfs; i++) { + delta_f df(fs[i-1], fs[i]); + df.to_stream(res); + } +} + + +void debug_writeout(string outfile = output_location) { + ofstream bout(outfile, ios::out | ios::binary); + bout.write((char *)&res[0], res.size()); + bout << flush; + bout.close(); + return; +} + + void remove_duplicate_v() { + map v_map; // node_vn -> id of first occurrence + vector id_map(vs.size()); // id of node_vn in old array -> id in new array + vector new_vs; + for (int i = 0; i < (int)vs.size(); i++) { + const node_v &v = vs[i]; + auto iter = v_map.find(v); + int new_ind = 0; + if (iter == v_map.end()) { + v_map[v] = new_ind = new_vs.size(); + new_vs.push_back(v); + } else { + new_ind = iter -> second; + } + id_map[i] = new_ind; + } + for (int j = 0; j < (int)fs.size(); j++) { + node_f &f = fs[j]; + for (int k = 0; k < 3; k++) { + f.f[k] = make_tuple(id_map[get<0>(f.f[k])-1]+1, get<1>(f.f[k]), get<2>(f.f[k])); + } + } + vs = new_vs; +} + +void remove_duplicate_vt() { + map vt_map; // node_vn -> id of first occurrence + vector id_map(vts.size()); // id of node_vn in old array -> id in new array + vector new_vts; + for (int i = 0; i < (int)vts.size(); i++) { + const node_vt &vt = vts[i]; + auto iter = vt_map.find(vt); + int new_ind = 0; + if (iter == vt_map.end()) { + vt_map[vt] = new_ind = new_vts.size(); + new_vts.push_back(vt); + } else { + new_ind = iter -> second; + } + id_map[i] = new_ind; + } + for (int j = 0; j < (int)fs.size(); j++) { + node_f &f = fs[j]; + for (int k = 0; k < 3; k++) { + if (get<1>(f.f[k]) != 0) { + f.f[k] = make_tuple(get<0>(f.f[k]), id_map[get<1>(f.f[k])-1]+1, get<2>(f.f[k])); + } + } + } + vts = new_vts; +} + +void remove_duplicate_vn() { + map vn_map; // node_vn -> id of first occurrence + vector id_map(vns.size()); // id of node_vn in old array -> id in new array + vector new_vns; + for (int i = 0; i < (int)vns.size(); i++) { + const node_vn &vn = vns[i]; + auto iter = vn_map.find(vn); + int new_ind = 0; + if (iter == vn_map.end()) { + vn_map[vn] = new_ind = new_vns.size(); + new_vns.push_back(vn); + } else { + new_ind = iter -> second; + } + id_map[i] = new_ind; + } + for (int j = 0; j < (int)fs.size(); j++) { + node_f &f = fs[j]; + for (int k = 0; k < 3; k++) { + f.f[k] = make_tuple(get<0>(f.f[k]), get<1>(f.f[k]), id_map[get<2>(f.f[k])-1]+1); + } + } + vns = new_vns; +} + void write_chunk_header(FILE* f, unsigned long long size, unsigned long long extra) { + unsigned char buffer[8]; + buffer[0] = size & 255; + buffer[1] = (size >> 8) & 255; + buffer[2] = (size >> 16) & 255; + buffer[3] = (size >> 24) & 255; + + buffer[4] = extra & 255; + buffer[5] = (extra >> 8) & 255; + buffer[6] = (extra >> 16) & 255; + buffer[7] = (extra >> 24) & 255; + + fwrite(buffer, 8, 1, f); +} +void write_chunk_header(FILE* f, unsigned long long size) { + unsigned char buffer[4]; + buffer[0] = size & 255; + buffer[1] = (size >> 8) & 255; + buffer[2] = (size >> 16) & 255; + buffer[3] = (size >> 24) & 255; + + fwrite(buffer, 4, 1, f); +} +void write_chunk_header(vector &input, unsigned long long size, unsigned long long extra) +{ + unsigned char buffer[8]; + buffer[0] = size & 255; + buffer[1] = (size >> 8) & 255; + buffer[2] = (size >> 16) & 255; + buffer[3] = (size >> 24) & 255; + + buffer[4] = extra & 255; + buffer[5] = (extra >> 8) & 255; + buffer[6] = (extra >> 16) & 255; + buffer[7] = (extra >> 24) & 255; + + // fwrite(buffer, 8, 1, f); + copy(buffer, buffer + 8, back_inserter(input)); +} +void write_chunk_header(vector &input, unsigned long long size) +{ + unsigned char buffer[4]; + buffer[0] = size & 255; + buffer[1] = (size >> 8) & 255; + buffer[2] = (size >> 16) & 255; + buffer[3] = (size >> 24) & 255; + + // fwrite(buffer, 4, 1, f); + copy(buffer, buffer + 4, back_inserter(input)); +} +/** + Compress the data in the INPUT buffer and returns the size of + output data. The size of INPUT buffer is specified by length. The + minimum INPUT buffer size is 16. +*/ + +void LZSS_Compression(bool Compress_Twice){ + // int compress_level=2; //default using better + unsigned char* buffer = new unsigned char[res.size()]; + unsigned char* buffer2 = new unsigned char[res.size()]; + unsigned char* buffer3 = new unsigned char[res.size()]; + copy(res.begin(), res.end(), buffer); + // FILE *f=fopen("output_location","wb"); + // fwrite(buffer, 1, res.size(), f); + // fclose(f); + // long long chunk_size =LZSS_compress_level(compress_level, buffer, res.size(), buffer2); + FILE *f=fopen(output_location.c_str(),"wb"); + LZSS_Encoder Lzss_Encoder(buffer, res.size(), buffer2); + + long long chunk_size =Lzss_Encoder.Compress(); + + if(Compress_Twice){ + LZSS_Encoder Lzss_Encoder2(buffer2, chunk_size, buffer3); + // long long chunk_size2 =Compress_LZ(buffer2, chunk_size, buffer3); + long long chunk_size2 =Lzss_Encoder2.Compress(3); + write_chunk_header(f, chunk_size2,chunk_size); + write_chunk_header(f, res.size()); + fwrite(buffer3, 1, chunk_size2, f); + delete [] buffer3; + } + else{ + write_chunk_header(f, chunk_size,res.size() ); + fwrite(buffer2, 1, chunk_size, f); + } + fclose(f); + // static_cast(res.data())); + // debug_writeout(); + // obj_output(); + delete [] buffer; + delete [] buffer2; +} +void LZSS_Comp_once(){ + // int compress_level=2; //default using better + unsigned char* buffer = new unsigned char[res.size()]; + unsigned char* buffer2 = new unsigned char[res.size()]; + copy(res.begin(), res.end(), buffer); + // FILE *f=fopen("output_location","wb"); + // fwrite(buffer, 1, res.size(), f); + // fclose(f); + // long long chunk_size =fastlz_compress_level(compress_level, buffer + // , res.size(), buffer2); + LZSS_Encoder Lzss_Encoder(buffer, res.size(), buffer2); + // long long chunk_size =Compress_LZ(buffer + // , res.size(), buffer2); + long long chunk_size =Lzss_Encoder.Compress(); + FILE *f=fopen(output_location.c_str(),"wb"); + + write_chunk_header(f, chunk_size,res.size() ); + //cout<(res.data())); + // debug_writeout(); + // obj_output(); + + delete [] buffer; + delete [] buffer2; +} +void LZSS_Compression_with_Huffman(bool Compress_Twice) +{ + // int compress_level=2; //default using better + unsigned char *buffer = new unsigned char[res.size()]; + unsigned char *buffer2 = new unsigned char[res.size()]; + unsigned char *buffer3 = new unsigned char[res.size()]; + copy(res.begin(), res.end(), buffer); + // FILE *f=fopen("output_location","wb"); + // fwrite(buffer, 1, res.size(), f); + // fclose(f); + // long long chunk_size =LZSS_compress_level(compress_level, buffer, res.size(), buffer2); + // FILE *f = fopen(output_location.c_str(), "wb"); + vector huffIn; + LZSS_Encoder Lzss_Encoder(buffer, res.size(), buffer2); + + long long chunk_size = Lzss_Encoder.Compress(); + // cout<(res.data())); + // debug_writeout(); + // obj_output(); + HuffmanEncoder huffmanEncoder(huffIn); + auto lengthHuffin = huffIn.size(); + huffmanEncoder.compress(); + vector output = huffmanEncoder.outputWholeVec(); + if (lengthHuffin < output.size()) + { + huffIn.push_back(1); + ofstream outStream(output_location, ios::binary | ios::out); + outStream.write((char *)&huffIn[0], huffIn.size()); + } + else + { + output.push_back(2); + ofstream outStream(output_location, ios::binary | ios::out); + outStream.write((char *)&output[0], output.size()); + } + +} +int main(int argc, char **argv) { + for (int i = 0; i < argc; i++) { + if (!strcmp(argv[i], "-i")) { + input_location = argv[++i]; + } + else if (!strcmp(argv[i], "-o")) { + output_location = argv[++i]; + } + } + readin(); + remove_duplicate_v(); + remove_duplicate_vn(); + remove_duplicate_vt(); + generate_output(); + //LZSS_Comp_once(); + LZSS_Compression(Compress_Twice); + //LZSS_Compression_with_Huffman(Compress_Twice); + // debug_writeout(); + return 0; +} \ No newline at end of file diff --git a/src/serialize_specs.md b/src/serialize_specs.md new file mode 100644 index 0000000..22e0482 --- /dev/null +++ b/src/serialize_specs.md @@ -0,0 +1,45 @@ +# Obj Serialization Specification + +First, we define an optimization technique `3pack` that packs three 21-bit ints `a, b, c` into one 64-bit integer `s`. +```c++ +s = (a << 42) || (b << 21) || c; +``` + +This way, we can store three ints (`-1048576 ~ 1048575`) in 8 bytes. + +## General structure of the file + +The first part is reserved for one-time flags and block length numbers. (Note: there are no line breaks. They are only here to aid readability.) + +``` +[Flags] (Size TBD, probably 8 bits or 16 bits) +Number of Blocks (Size: 1 byte) +Length of Block 1 (in number of entries, int32), Block 1 +Length of Block 2 (in number of entries, int32), Block 2 +... +``` + +## 4 Types of structures (Blocks) + +### v + +Each v has 3 elements, each of which have 6 d.p., and may be too large to 3pack. + +Therefore each v value is multiplied by 1e6, and then it would correspond to a 32-bit integer. (float32 is not enough) + +### vt + +Each vt has 2 elements, each of which is between 0 and 1 and has 6 sig figs (or in the case of 1, 7 sig figs). + +Therefore each vt value is multiplied by 1e6, and then it would correspond to a 32-bit integer. (int16 is not sufficient) + +### vn + +Each vn has 3 elements, each of which is between -1 and 1 and has 6 sig figs. + +Therefore each vn value is multiplied by 1e6, and then 3packed into one 64-bit integer. + +### f + +Each f has three elements. Each element is in the form `v/[vt]/vn`. They can be 3packed into one 64-bit integer. + diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..e0ae6d4 --- /dev/null +++ b/test.sh @@ -0,0 +1 @@ +bash scripts/test_non_ai.sh