Init package and start bullmq workers

crawler/crawler.ts

@@ -0,0 +1,6 @@
+import logger from "@remember/shared/logger";
+import { Job } from "bullmq";
+
+export default async function runCrawler(job: Job) {
+    logger.info(`[Crawler] Got a new job: ${job.name}`);
+}

crawler/main.ts

@@ -0,0 +1,17 @@
+import { Worker } from "bullmq";
+
+import {
+    LinkCrawlerQueue,
+    queueConnectionDetails,
+} from "@remember/shared/queues";
+import logger from "@remember/shared/logger";
+import runCrawler from "./crawler";
+
+logger.info("Starting crawler worker ...");
+
+const crawlerWorker = new Worker(LinkCrawlerQueue.name, runCrawler, {
+    connection: queueConnectionDetails,
+    autorun: false,
+});
+
+await Promise.all([crawlerWorker]);

crawler/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "@remember/crawler",
+  "version": "0.1.0",
+  "private": true,
+  "dependencies": {
+    "@remember/shared": "workspace:*"
+  }
+}

docker-compose.yml

@@ -0,0 +1,11 @@
+version: "3.8"
+name: remember
+services:
+  redis:
+    image: redis:7.2-alpine
+    ports:
+      - 6379:6379
+    volumes:
+      - redis:/data
+volumes:
+  redis:

package.json

@@ -3,15 +3,25 @@
   "version": "0.1.0",
   "private": true,
   "workspaces": [
-    "web"
+    "web",
+    "crawler",
+    "shared"
   ],
   "dependencies": {
     "@next/eslint-plugin-next": "^14.1.0",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^6.21.0",
+    "bullmq": "^5.1.9",
     "class-variance-authority": "^0.7.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-react": "^7.33.2",
-    "eslint-plugin-react-hooks": "^4.6.0"
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "winston": "^3.11.0"
+  },
+  "devDependencies": {
+    "typescript": "^5",
+    "@types/node": "^20",
+    "eslint": "^8.56.0",
+    "eslint-config-next": "14.1.0"
   }
 }

shared/index.ts

@@ -0,0 +1 @@
+export * as Queues from './queues.ts';

shared/logger.ts

@@ -0,0 +1,15 @@
+import winston from "winston";
+
+const logger = winston.createLogger({
+    level: process.env.LOG_LEVEL || "debug",
+    format: winston.format.combine(
+        winston.format.timestamp(),
+        winston.format.colorize(),
+        winston.format.printf(
+            (info) => `${info.timestamp} ${info.level}: ${info.message}`,
+        ),
+    ),
+    transports: [new winston.transports.Console()],
+});
+
+export default logger;

shared/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "@remember/shared",
+  "version": "0.1.0",
+  "private": true,
+  "dependencies": {
+  },
+  "main": "index.ts"
+}

shared/queues.ts

@@ -0,0 +1,10 @@
+import { Queue } from "bullmq";
+
+export const queueConnectionDetails = {
+    host: process.env.REDIS_HOST || "localhost",
+    port: parseInt(process.env.REDIS_PORT || "6379"),
+};
+
+export const LinkCrawlerQueue = new Queue("link_crawler_queue", { connection: queueConnectionDetails });
+
+

web/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "remember",
+  "name": "@remember/web",
   "version": "0.1.0",
   "private": true,
   "scripts": {
@@ -26,14 +26,10 @@
     "zod": "^3.22.4"
   },
   "devDependencies": {
-    "typescript": "^5",
-    "@types/node": "^20",
     "@types/react": "^18",
     "@types/react-dom": "^18",
     "autoprefixer": "^10.0.1",
     "postcss": "^8",
-    "tailwindcss": "^3.3.0",
-    "eslint": "^8.56.0",
-    "eslint-config-next": "14.1.0"
+    "tailwindcss": "^3.3.0"
   }
 }

web/tsconfig.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "https://json.schemastore.org/tsconfig",
   "compilerOptions": {
     "lib": ["dom", "dom.iterable", "esnext"],
     "allowJs": true,