diff --git a/README.md b/README.md index a70f3ef..4230b07 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,13 @@ [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/shotit/shotit-worker/docker-image.yml?branch=main&style=flat-square)](https://github.com/shotit/shotit-worker/actions) [![GitHub release](https://img.shields.io/github/release/shotit/shotit-worker.svg)](https://github.com/shotit/shotit-worker/releases/latest) [![Watcher Docker](https://img.shields.io/docker/pulls/lesliewong007/shotit-worker-watcher?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-watcher) -[![Watcher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-watcher/v0.9.13?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-watcher) +[![Watcher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-watcher/v0.9.14?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-watcher) [![Hasher Docker](https://img.shields.io/docker/pulls/lesliewong007/shotit-worker-hasher?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-hasher) -[![Hasher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-hasher/v0.9.13?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-hasher) +[![Hasher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-hasher/v0.9.14?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-hasher) [![Loader Docker](https://img.shields.io/docker/pulls/lesliewong007/shotit-worker-loader?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-loader) -[![Loader Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-loader/v0.9.13?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-loader) +[![Loader Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-loader/v0.9.14?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-loader) [![Searcher Docker](https://img.shields.io/docker/pulls/lesliewong007/shotit-worker-searcher?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-searcher) -[![Searcher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-searcher/v0.9.13?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-searcher) +[![Searcher Docker Image Size](https://img.shields.io/docker/image-size/lesliewong007/shotit-worker-searcher/v0.9.14?style=flat-square)](https://hub.docker.com/r/lesliewong007/shotit-worker-searcher) Backend workers for [shotit](https://github.com/shotit/shotit). Four core workers of shotit: watcher, hasher, loader and searcher. diff --git a/loader.js b/loader.js index 2cef432..4cb3d93 100644 --- a/loader.js +++ b/loader.js @@ -129,6 +129,7 @@ const getPrimaryKey = (str) => { /** * If isMainThread, spawn a new worker to load hash vectors, and when the * loading is done, terminate the worker. + * Use trunk indicator to prevent loader thread being overwhelming, eight at most one batch * @param {*} data * @returns */ @@ -138,13 +139,16 @@ const messageHandle = async (data) => { let worker = new Worker(__filename, { workerData: data.toString() }); console.log("Spawn new Worker: ", worker.threadId); const resolve = (payload) => { - ws.send(payload); + const { trunk } = JSON.parse(payload); + if (trunk === "true") { + ws.send(payload); + } worker.terminate(); worker = null; }; worker.on("message", resolve); } else { - const { file } = JSON.parse(data); + const { file, trunk } = JSON.parse(data); // from workerData console.log(`Downloading ${file}.xml.xz`); const [imdbID, fileName] = file.split("/"); @@ -326,12 +330,12 @@ const messageHandle = async (data) => { // console.log("Load done", performance.now() - startTime); await fetch(`${TRACE_API_URL}/loaded/${imdbID}/${encodeURIComponent(fileName)}`, { - headers: { "x-trace-secret": TRACE_API_SECRET }, + headers: { "x-trace-secret": TRACE_API_SECRET, "x-trunk-load": trunk }, }); // ws.send(data); console.log(`Loaded ${file}`); milvusClient.closeConnection(); - parentPort.postMessage(data); + parentPort.postMessage(data); // from workerData } catch (error) { console.log(error); console.log("Reconnecting in 60 seconds"); diff --git a/package.json b/package.json index 4740205..d49607d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "shotit-worker", - "version": "0.9.13", + "version": "0.9.14", "description": "Four core workers of shotit: watcher, hasher, loader and searcher", "main": "", "type": "module",