Merge pull request #781 from Performance Network branch

Accelerator branch
ton-blockchain · Oct 19, 2023 · 3466631 · 3466631
2 parents 7f815fc + 80a9ad7
commit 3466631
Show file tree

Hide file tree

Showing 165 changed files with 7,094 additions and 1,727 deletions.
diff --git a/.github/workflows/macos-11.7-compile.yml b/.github/workflows/macos-11.7-compile.yml
@@ -28,7 +28,7 @@ jobs:
           mkdir build
           cd build
           cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=11.7 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=Release ..
-          ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator
+          ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver
 
       - name: Find & copy binaries
         run: |
@@ -52,6 +52,7 @@ jobs:
           cp build/utils/json2tlo artifacts/
           cp build/adnl/adnl-proxy artifacts/
           cp build/emulator/*emulator.* artifacts/
+          cp build/utils/proxy-liteserver artifacts/
           chmod +x artifacts/*
           rsync -r crypto/smartcont artifacts/
           rsync -r crypto/fift/lib artifacts/

diff --git a/.github/workflows/macos-12.6-compile.yml b/.github/workflows/macos-12.6-compile.yml
@@ -39,7 +39,7 @@ jobs:
           mkdir build
           cd build
           cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=12.6 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=Release ..
-          ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator
+          ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver
 
       - name: Find & copy binaries
         run: |
@@ -63,6 +63,7 @@ jobs:
           cp build/utils/json2tlo artifacts/
           cp build/adnl/adnl-proxy artifacts/
           cp build/emulator/*emulator.* artifacts/
+          cp build/utils/proxy-liteserver artifacts/
           chmod +x artifacts/*
           rsync -r crypto/smartcont artifacts/
           rsync -r crypto/fift/lib artifacts/

diff --git a/.github/workflows/ubuntu-22.04-compile.yml b/.github/workflows/ubuntu-22.04-compile.yml
@@ -40,12 +40,12 @@ jobs:
         cd build
         
         cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
-        ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state emulator
+        ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state emulator proxy-liteserver tps-counter
 
     - name: Find & copy binaries
       run: |
         mkdir artifacts
-        cp  build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/tonlib/libtonlibjson.so.0.5 build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.* artifacts
+        cp  build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/tonlib/libtonlibjson.so.0.5 build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.* build/utils/proxy-liteserver build/tps-counter/tps-counter artifacts
         chmod +x artifacts/*
         cp -R crypto/smartcont artifacts/
         cp -R crypto/fift/lib artifacts/

diff --git a/.github/workflows/ubuntu-compile.yml b/.github/workflows/ubuntu-compile.yml
@@ -44,12 +44,12 @@ jobs:
         buildPath=`pwd`
 
         cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$buildPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$buildPath/openssl_1_1_1/libcrypto.a -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
-        ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator
+        ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver tps-counter
 
     - name: Find & copy binaries
       run: |
         mkdir artifacts-${{ matrix.os }}
-        cp build-${{ matrix.os }}/storage/storage-daemon/storage-daemon build-${{ matrix.os }}/storage/storage-daemon/storage-daemon-cli build-${{ matrix.os }}/crypto/fift build-${{ matrix.os }}/crypto/tlbc build-${{ matrix.os }}/crypto/func build-${{ matrix.os }}/crypto/create-state build-${{ matrix.os }}/validator-engine-console/validator-engine-console build-${{ matrix.os }}/tonlib/tonlib-cli build-${{ matrix.os }}/tonlib/libtonlibjson.so.0.5 build-${{ matrix.os }}/http/http-proxy build-${{ matrix.os }}/rldp-http-proxy/rldp-http-proxy build-${{ matrix.os }}/dht-server/dht-server build-${{ matrix.os }}/lite-client/lite-client build-${{ matrix.os }}/validator-engine/validator-engine build-${{ matrix.os }}/utils/generate-random-id build-${{ matrix.os }}/utils/json2tlo build-${{ matrix.os }}/adnl/adnl-proxy build-${{ matrix.os }}/emulator/libemulator.* artifacts-${{ matrix.os }}
+        cp build-${{ matrix.os }}/storage/storage-daemon/storage-daemon build-${{ matrix.os }}/storage/storage-daemon/storage-daemon-cli build-${{ matrix.os }}/crypto/fift build-${{ matrix.os }}/crypto/tlbc build-${{ matrix.os }}/crypto/func build-${{ matrix.os }}/crypto/create-state build-${{ matrix.os }}/validator-engine-console/validator-engine-console build-${{ matrix.os }}/tonlib/tonlib-cli build-${{ matrix.os }}/tonlib/libtonlibjson.so.0.5 build-${{ matrix.os }}/http/http-proxy build-${{ matrix.os }}/rldp-http-proxy/rldp-http-proxy build-${{ matrix.os }}/dht-server/dht-server build-${{ matrix.os }}/lite-client/lite-client build-${{ matrix.os }}/validator-engine/validator-engine build-${{ matrix.os }}/utils/generate-random-id build-${{ matrix.os }}/utils/json2tlo build-${{ matrix.os }}/adnl/adnl-proxy build-${{ matrix.os }}/emulator/libemulator.* build-${{ matrix.os }}/utils/proxy-liteserver build-${{ matrix.os }}/tps-counter/tps-counter artifacts-${{ matrix.os }}
         chmod +x artifacts-${{ matrix.os }}/*
         cp -R crypto/smartcont artifacts-${{ matrix.os }}
         cp -R crypto/fift/lib artifacts-${{ matrix.os }}

diff --git a/.github/workflows/win-2019-compile.yml b/.github/workflows/win-2019-compile.yml
@@ -73,7 +73,7 @@ jobs:
           mkdir build
           cd build
           cmake -DCMAKE_BUILD_TYPE=Release -DSODIUM_USE_STATIC_LIBS=1 -DSECP256K1_INCLUDE_DIR=%root%\secp256k1\include -DSECP256K1_LIBRARY=%root%\secp256k1\bin\x64\Release\v142\static\secp256k1.lib -DREADLINE_INCLUDE_DIR=%root%\readline-5.0-1-lib\include\readline -DREADLINE_LIBRARY=%root%\readline-5.0-1-lib\lib\readline.lib -DPORTABLE=1 -DZLIB_FOUND=1 -DMHD_FOUND=1 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1.1j/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1.1j/lib/libcrypto_static.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" ..
-          cmake --build . --target storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator --config Release
+          cmake --build . --target storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver --config Release
 
       - name: Show executables
         run: |
@@ -92,7 +92,7 @@ jobs:
           mkdir artifacts\smartcont
           mkdir artifacts\lib
 
-          for %%I in (build\storage\storage-daemon\Release\storage-daemon.exe build\storage\storage-daemon\Release\storage-daemon-cli.exe  build\blockchain-explorer\blockchain-explorer.exe build\crypto\Release\fift.exe build\crypto\Release\tlbc.exe build\crypto\Release\func.exe build\crypto\Release\create-state.exe build\validator-engine-console\Release\validator-engine-console.exe build\tonlib\Release\tonlib-cli.exe build\tonlib\Release\tonlibjson.dll build\http\Release\http-proxy.exe build\rldp-http-proxy\Release\rldp-http-proxy.exe build\dht-server\Release\dht-server.exe build\lite-client\Release\lite-client.exe build\validator-engine\Release\validator-engine.exe build\utils\Release\generate-random-id.exe build\utils\Release\json2tlo.exe build\adnl\Release\adnl-proxy.exe build\emulator\Release\emulator.dll) do copy %%I artifacts\
+          for %%I in (build\storage\storage-daemon\Release\storage-daemon.exe build\storage\storage-daemon\Release\storage-daemon-cli.exe  build\blockchain-explorer\blockchain-explorer.exe build\crypto\Release\fift.exe build\crypto\Release\tlbc.exe build\crypto\Release\func.exe build\crypto\Release\create-state.exe build\validator-engine-console\Release\validator-engine-console.exe build\tonlib\Release\tonlib-cli.exe build\tonlib\Release\tonlibjson.dll build\http\Release\http-proxy.exe build\rldp-http-proxy\Release\rldp-http-proxy.exe build\dht-server\Release\dht-server.exe build\lite-client\Release\lite-client.exe build\validator-engine\Release\validator-engine.exe build\utils\Release\generate-random-id.exe build\utils\Release\json2tlo.exe build\adnl\Release\adnl-proxy.exe build\emulator\Release\emulator.dll build\utils\proxy-liteserver.exe) do copy %%I artifacts\
           xcopy /e /k /h /i crypto\smartcont artifacts\smartcont
           xcopy /e /k /h /i crypto\fift\lib artifacts\lib
 

diff --git a/CMake/FindLZ4.cmake b/CMake/FindLZ4.cmake
@@ -0,0 +1,39 @@
+###############################################################################
+# Find LZ4
+#
+# This sets the following variables:
+# LZ4_FOUND        - True if LZ4 was found.
+# LZ4_INCLUDE_DIRS - Directories containing the LZ4 include files.
+# LZ4_LIBRARIES    - Libraries needed to use LZ4.
+# LZ4_LIBRARY      - Library needed to use LZ4.
+# LZ4_LIBRARY_DIRS - Library needed to use LZ4.
+
+find_package(PkgConfig REQUIRED)
+
+# If found, LZ$_* variables will be defined
+pkg_check_modules(LZ4 REQUIRED liblz4)
+
+if(NOT LZ4_FOUND)
+  find_path(LZ4_INCLUDE_DIR lz4.h
+    HINTS "${LZ4_ROOT}" "$ENV{LZ4_ROOT}"
+    PATHS "$ENV{PROGRAMFILES}/lz4" "$ENV{PROGRAMW6432}/lz4"
+    PATH_SUFFIXES include)
+
+  find_library(LZ4_LIBRARY
+    NAMES lz4 lz4_static
+    HINTS "${LZ4_ROOT}" "$ENV{LZ4_ROOT}"
+    PATHS "$ENV{PROGRAMFILES}/lz4" "$ENV{PROGRAMW6432}/lz4"
+    PATH_SUFFIXES lib)
+
+  if(LZ4_LIBRARY)
+    set(LZ4_LIBRARIES ${LZ4_LIBRARY})
+    get_filename_component(LZ4_LIBRARY_DIRS ${LZ4_LIBRARY} DIRECTORY)
+  endif()
+else()
+  find_library(LZ4_LIBRARY
+    NAMES lz4 lz4_static
+    PATHS ${LZ4_LIBRARY_DIRS}
+    NO_DEFAULT_PATH)
+endif()
+
+mark_as_advanced(LZ4_LIBRARY LZ4_INCLUDE_DIRS LZ4_LIBRARY_DIRS LZ4_LIBRARIES)
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -424,6 +424,7 @@ add_subdirectory(dht-server)
 add_subdirectory(utils)
 add_subdirectory(http)
 add_subdirectory(rldp-http-proxy)
+add_subdirectory(tps-counter)
 endif()
 #END internal
 

diff --git a/adnl/adnl-message.cpp b/adnl/adnl-message.cpp
@@ -46,6 +46,9 @@ AdnlMessage::AdnlMessage(tl_object_ptr<ton_api::adnl_Message> message) {
           [&](ton_api::adnl_message_part &msg) {
             message_ = adnlmessage::AdnlMessagePart{msg.hash_, static_cast<td::uint32>(msg.total_size_),
                                                     static_cast<td::uint32>(msg.offset_), std::move(msg.data_)};
+          },
+          [&](ton_api::adnl_message_queryError &msg) {
+            message_ = adnlmessage::AdnlMessageQueryError{msg.query_id_};
           }));
 }
 

diff --git a/adnl/adnl-message.h b/adnl/adnl-message.h
@@ -170,6 +170,24 @@ class AdnlMessageAnswer {
   td::BufferSlice data_;
 };
 
+class AdnlMessageQueryError {
+ public:
+  explicit AdnlMessageQueryError(AdnlQueryId query_id) : query_id_(query_id) {
+  }
+  const auto &query_id() const {
+    return query_id_;
+  }
+  td::uint32 size() const {
+    return 36;
+  }
+  tl_object_ptr<ton_api::adnl_Message> tl() const {
+    return create_tl_object<ton_api::adnl_message_queryError>(query_id_);
+  }
+
+ private:
+  AdnlQueryId query_id_;
+};
+
 class AdnlMessagePart {
  public:
   AdnlMessagePart(td::Bits256 hash, td::uint32 total_size, td::uint32 offset, td::BufferSlice data)
@@ -220,7 +238,8 @@ class AdnlMessage {
  private:
   td::Variant<Empty, adnlmessage::AdnlMessageCreateChannel, adnlmessage::AdnlMessageConfirmChannel,
               adnlmessage::AdnlMessageCustom, adnlmessage::AdnlMessageNop, adnlmessage::AdnlMessageReinit,
-              adnlmessage::AdnlMessageQuery, adnlmessage::AdnlMessageAnswer, adnlmessage::AdnlMessagePart>
+              adnlmessage::AdnlMessageQuery, adnlmessage::AdnlMessageAnswer, adnlmessage::AdnlMessagePart,
+              adnlmessage::AdnlMessageQueryError>
       message_{Empty{}};
 
  public:

diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp
@@ -534,10 +534,14 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQuery &mess
                                        flags = static_cast<td::uint32>(0)](td::Result<td::BufferSlice> R) {
     if (R.is_error()) {
       LOG(WARNING) << "failed to answer query: " << R.move_as_error();
+      td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_message,
+                              OutboundAdnlMessage{adnlmessage::AdnlMessageQueryError{query_id}, flags});
     } else {
       auto data = R.move_as_ok();
       if (data.size() > Adnl::huge_packet_max_size()) {
         LOG(WARNING) << "dropping too big answer query: size=" << data.size();
+        td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_message,
+                                OutboundAdnlMessage{adnlmessage::AdnlMessageQueryError{query_id}, flags});
       } else {
         td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_message,
                                 OutboundAdnlMessage{adnlmessage::AdnlMessageAnswer{query_id, std::move(data)}, flags});
@@ -619,6 +623,18 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessagePart &messa
   }
 }
 
+void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQueryError &message) {
+  auto Q = out_queries_.find(message.query_id());
+
+  if (Q == out_queries_.end()) {
+    VLOG(ADNL_NOTICE) << this << ": dropping IN query error: unknown query id " << message.query_id();
+    return;
+  }
+
+  td::actor::send_closure_later(Q->second, &AdnlQuery::set_error, td::Status::Error("adnl query rejected"));
+  out_queries_.erase(Q);
+}
+
 void AdnlPeerPairImpl::delete_query(AdnlQueryId id) {
   auto Q = out_queries_.find(id);
 

diff --git a/adnl/adnl-peer.hpp b/adnl/adnl-peer.hpp
@@ -104,6 +104,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair {
   void process_message(const adnlmessage::AdnlMessageQuery &message);
   void process_message(const adnlmessage::AdnlMessageAnswer &message);
   void process_message(const adnlmessage::AdnlMessagePart &message);
+  void process_message(const adnlmessage::AdnlMessageQueryError &message);
   void process_message(const AdnlMessage::Empty &message) {
     UNREACHABLE();
   }

diff --git a/adnl/adnl-query.cpp b/adnl/adnl-query.cpp
@@ -25,7 +25,7 @@ namespace ton {
 namespace adnl {
 
 void AdnlQuery::alarm() {
-  set_error(td::Status::Error(ErrorCode::timeout, "adnl query timeout"));
+  set_error(td::Status::Error(ErrorCode::timeout, PSTRING() << "timeout for adnl query " << name_));
 }
 void AdnlQuery::result(td::BufferSlice data) {
   promise_.set_value(std::move(data));

diff --git a/blockchain-explorer/CMakeLists.txt b/blockchain-explorer/CMakeLists.txt
@@ -17,11 +17,11 @@ if (NIX)
   find_package(PkgConfig REQUIRED)
   pkg_check_modules(MHD libmicrohttpd)
   target_include_directories(blockchain-explorer PUBLIC ${MHD_INCLUDE_DIRS} ${MHD_STATIC_INCLUDE_DIRS})
-  target_link_libraries(blockchain-explorer tdutils tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ton_block ${MHD_LIBRARIES} ${MHD_STATIC_LIBRARIES})
+  target_link_libraries(blockchain-explorer tdutils tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ton_block lite-client-common ${MHD_LIBRARIES} ${MHD_STATIC_LIBRARIES})
 else()
   find_package(MHD)
   target_include_directories(blockchain-explorer PUBLIC ${MHD_INCLUDE_DIRS})
-  target_link_libraries(blockchain-explorer tdutils tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ton_block ${MHD_LIBRARIES})
+  target_link_libraries(blockchain-explorer tdutils tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ton_block lite-client-common ${MHD_LIBRARIES})
 endif()
 
 install(TARGETS blockchain-explorer RUNTIME DESTINATION bin)