Use trunk indicator to prevent loader threads bomb (#68)

shotit · Jul 18, 2023 · 7a8f0f7 · 7a8f0f7
2 parents 873a01e + a79390e
commit 7a8f0f7
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 9 deletions.
diff --git a/README.md b/README.md
@@ -4,13 +4,13 @@
 [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/shotit/shotit-worker/docker-image.yml?branch=main&style=flat-square)](https://github.com/shotit/shotit-worker/actions)
 [![GitHub release](https://img.shields.io/github/release/shotit/shotit-worker.svg)](https://github.com/shotit/shotit-worker/releases/latest)
 [![Watcher Docker](https://img.shields.io/docker/pulls/lesliewong007/shotit-worker-watcher?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-watcher)
-[![Watcher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-watcher/v0.9.13?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-watcher)
+[![Watcher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-watcher/v0.9.14?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-watcher)
 [![Hasher Docker](https://img.shields.io/docker/pulls/lesliewong007/shotit-worker-hasher?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-hasher)
-[![Hasher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-hasher/v0.9.13?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-hasher)
+[![Hasher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-hasher/v0.9.14?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-hasher)
 [![Loader Docker](https://img.shields.io/docker/pulls/lesliewong007/shotit-worker-loader?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-loader)
-[![Loader Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-loader/v0.9.13?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-loader)
+[![Loader Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-loader/v0.9.14?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-loader)
 [![Searcher Docker](https://img.shields.io/docker/pulls/lesliewong007/shotit-worker-searcher?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-searcher)
-[![Searcher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-searcher/v0.9.13?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-searcher)
+[![Searcher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-searcher/v0.9.14?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-searcher)
 
 Backend workers for [shotit](https://github.com/shotit/shotit). Four core workers of shotit: watcher, hasher, loader and searcher.
 

diff --git a/loader.js b/loader.js
@@ -129,6 +129,7 @@ const getPrimaryKey = (str) => {
 /**
  * If isMainThread, spawn a new worker to load hash vectors, and when the
  * loading is done, terminate the worker.
+ * Use trunk indicator to prevent loader thread being overwhelming, eight at most one batch
  * @param {*} data
  * @returns
  */
@@ -138,13 +139,16 @@ const messageHandle = async (data) => {
     let worker = new Worker(__filename, { workerData: data.toString() });
     console.log("Spawn new Worker: ", worker.threadId);
     const resolve = (payload) => {
-      ws.send(payload);
+      const { trunk } = JSON.parse(payload);
+      if (trunk === "true") {
+        ws.send(payload);
+      }
       worker.terminate();
       worker = null;
     };
     worker.on("message", resolve);
   } else {
-    const { file } = JSON.parse(data);
+    const { file, trunk } = JSON.parse(data); // from workerData
 
     console.log(`Downloading ${file}.xml.xz`);
     const [imdbID, fileName] = file.split("/");
@@ -326,12 +330,12 @@ const messageHandle = async (data) => {
         // console.log("Load done", performance.now() - startTime);
 
         await fetch(`${TRACE_API_URL}/loaded/${imdbID}/${encodeURIComponent(fileName)}`, {
-          headers: { "x-trace-secret": TRACE_API_SECRET },
+          headers: { "x-trace-secret": TRACE_API_SECRET, "x-trunk-load": trunk },
         });
         // ws.send(data);
         console.log(`Loaded ${file}`);
         milvusClient.closeConnection();
-        parentPort.postMessage(data);
+        parentPort.postMessage(data); // from workerData
       } catch (error) {
         console.log(error);
         console.log("Reconnecting in 60 seconds");

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "shotit-worker",
-  "version": "0.9.13",
+  "version": "0.9.14",
   "description": "Four core workers of shotit: watcher, hasher, loader and searcher",
   "main": "",
   "type": "module",