From 00d01a9fb431bc3d134e0ba164d75040b8b45284 Mon Sep 17 00:00:00 2001 From: Leslie-Wong-H <79917148leslie@gmail.com> Date: Wed, 19 Jul 2023 06:11:15 +0800 Subject: [PATCH 1/2] Use trunk indicator to prevent loader threads bomb --- loader.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/loader.js b/loader.js index 2cef432..4cb3d93 100644 --- a/loader.js +++ b/loader.js @@ -129,6 +129,7 @@ const getPrimaryKey = (str) => { /** * If isMainThread, spawn a new worker to load hash vectors, and when the * loading is done, terminate the worker. + * Use trunk indicator to prevent loader thread being overwhelming, eight at most one batch * @param {*} data * @returns */ @@ -138,13 +139,16 @@ const messageHandle = async (data) => { let worker = new Worker(__filename, { workerData: data.toString() }); console.log("Spawn new Worker: ", worker.threadId); const resolve = (payload) => { - ws.send(payload); + const { trunk } = JSON.parse(payload); + if (trunk === "true") { + ws.send(payload); + } worker.terminate(); worker = null; }; worker.on("message", resolve); } else { - const { file } = JSON.parse(data); + const { file, trunk } = JSON.parse(data); // from workerData console.log(`Downloading ${file}.xml.xz`); const [imdbID, fileName] = file.split("/"); @@ -326,12 +330,12 @@ const messageHandle = async (data) => { // console.log("Load done", performance.now() - startTime); await fetch(`${TRACE_API_URL}/loaded/${imdbID}/${encodeURIComponent(fileName)}`, { - headers: { "x-trace-secret": TRACE_API_SECRET }, + headers: { "x-trace-secret": TRACE_API_SECRET, "x-trunk-load": trunk }, }); // ws.send(data); console.log(`Loaded ${file}`); milvusClient.closeConnection(); - parentPort.postMessage(data); + parentPort.postMessage(data); // from workerData } catch (error) { console.log(error); console.log("Reconnecting in 60 seconds"); From a79390e0c7178a1fdf016e48e5a836a4832e4097 Mon Sep 17 00:00:00 2001 From: Leslie-Wong-H <79917148leslie@gmail.com> Date: Wed, 19 Jul 2023 06:23:22 +0800 Subject: [PATCH 2/2] Build: release shotit-worker v0.9.14 --- README.md | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a70f3ef..4230b07 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,13 @@ [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/shotit/shotit-worker/docker-image.yml?branch=main&style=flat-square)](https://github.com/shotit/shotit-worker/actions) [![GitHub release](https://img.shields.io/github/release/shotit/shotit-worker.svg)](https://github.com/shotit/shotit-worker/releases/latest) [![Watcher Docker](https://img.shields.io/docker/pulls/lesliewong007/shotit-worker-watcher?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-watcher) -[![Watcher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-watcher/v0.9.13?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-watcher) +[![Watcher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-watcher/v0.9.14?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-watcher) [![Hasher Docker](https://img.shields.io/docker/pulls/lesliewong007/shotit-worker-hasher?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-hasher) -[![Hasher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-hasher/v0.9.13?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-hasher) +[![Hasher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-hasher/v0.9.14?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-hasher) [![Loader Docker](https://img.shields.io/docker/pulls/lesliewong007/shotit-worker-loader?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-loader) -[![Loader Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-loader/v0.9.13?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-loader) +[![Loader Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-loader/v0.9.14?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-loader) [![Searcher Docker](https://img.shields.io/docker/pulls/lesliewong007/shotit-worker-searcher?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-searcher) -[![Searcher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-searcher/v0.9.13?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-searcher) +[![Searcher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-searcher/v0.9.14?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-searcher) Backend workers for [shotit](https://github.com/shotit/shotit). Four core workers of shotit: watcher, hasher, loader and searcher. diff --git a/package.json b/package.json index c730f26..0a6752a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "shotit-worker", - "version": "0.9.13", + "version": "0.9.14", "description": "Four core workers of shotit: watcher, hasher, loader and searcher", "main": "", "type": "module",