Merge pull request #50 from /issues/48

feat: Allow shared_buffers and work_mem to be set

.env

@@ -15,3 +15,8 @@ MONGO_URI=mongodb://localhost:27017
 #REDIS_URL=redis://localhost:6379
 # Log levels are: debug, verbose, log (default), warn, error
 LOG_LEVEL=debug
+QUERY_DATA_DIR=./data/pgdata
+# This is the default. Use 16G in production
+QUERY_POSTGRES_SHARED_BUFFERS=128MB
+# This is the default. Use 1G in production
+QUERY_POSTGRES_WORK_MEM=4MB

.github/workflows/container-deploy.yml

@@ -28,26 +28,39 @@ jobs:
         run: |
           echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV
           echo "SSH_USERNAME=off" >> $GITHUB_ENV
+
           # deploy target
           echo "SSH_HOST=10.1.0.200" >> $GITHUB_ENV
+
           # configurations
           echo "COMMON_NET_NAME=po_webnet" >> $GITHUB_ENV
           echo "MONGO_URI=mongodb://10.1.0.200:27017" >> $GITHUB_ENV
           echo "REDIS_URL=redis://redis:6379" >> $GITHUB_ENV
+
+          # Postgres config
           echo "POSTGRES_SHM_SIZE=256m" >> $GITHUB_ENV
+          echo "QUERY_POSTGRES_SHARED_BUFFERS=4GB" >> $GITHUB_ENV
+          echo "QUERY_POSTGRES_WORK_MEM=256MB" >> $GITHUB_ENV
       - name: Set various variable for production deployment
         if: matrix.env == 'off-query-org'
         run: |
           echo "SSH_PROXY_HOST=off1.openfoodfacts.org" >> $GITHUB_ENV
           echo "SSH_USERNAME=off" >> $GITHUB_ENV
+
           # deploy target
           echo "SSH_HOST=10.1.0.115" >> $GITHUB_ENV
+
           # configurations
           echo "COMMON_NET_NAME=common_net" >> $GITHUB_ENV
+
           # mongodb and redis (through stunnel)
           echo "MONGO_URI=mongodb://10.1.0.102:27017" >> $GITHUB_ENV
           echo "REDIS_URL=redis://10.1.0.122:6379" >> $GITHUB_ENV
+
+          # Postgres config
           echo "POSTGRES_SHM_SIZE=1g" >> $GITHUB_ENV
+          echo "QUERY_POSTGRES_SHARED_BUFFERS=16GB" >> $GITHUB_ENV
+          echo "QUERY_POSTGRES_WORK_MEM=1GB" >> $GITHUB_ENV
       - name: Wait for container build workflow
         uses: tomchv/wait-my-workflow@v1.1.0
         id: wait-build
@@ -126,7 +139,14 @@ jobs:
             echo "COMMON_NET_NAME=${{ env.COMMON_NET_NAME }}" >> .env
             echo "MONGO_URI=${{ env.MONGO_URI }}" >> .env
             echo "REDIS_URL=${{ env.REDIS_URL }}" >> .env
+
+            # Postgres config
             echo "POSTGRES_SHM_SIZE=${{ env.POSTGRES_SHM_SIZE }}" >> .env
+            echo "QUERY_POSTGRES_SHARED_BUFFERS=${{ env.QUERY_POSTGRES_SHARED_BUFFERS }}" >> .env
+            echo "QUERY_POSTGRES_WORK_MEM=${{ env.QUERY_POSTGRES_WORK_MEM }}" >> .env
+
+            # Use docker volume for now
+            echo "QUERY_DATA_DIR=dbdata" >> .env
             echo "LOG_LEVEL=log" >> .env
 
 

Makefile

@@ -13,12 +13,15 @@ ifneq (,$(wildcard ./${ENV_FILE}))
     export
 endif
 
+create_folders:
+	mkdir -p ${QUERY_DATA_DIR}
+
 # Use this to start both the query service and associated database in Docker
-up:
+up: create_folders
 	docker compose up -d --build
 
 # This task starts a Postgres database in Docker and then prepares the local environment for development
-dev:
+dev: create_folders
 	docker compose up -d query_postgres
 	npm install
 	npm run migration:up

docker-compose.yml

@@ -1,14 +1,15 @@
 services:
   query_postgres:
     image: postgres:12-alpine
+    command: postgres -c shared_buffers=${QUERY_POSTGRES_SHARED_BUFFERS} -c work_mem=${QUERY_POSTGRES_WORK_MEM} -c listen_addresses='*'
     restart: ${RESTART_POLICY:-always}
     environment:
       - POSTGRES_USER
       - POSTGRES_PASSWORD
       - POSTGRES_DB
     shm_size: ${POSTGRES_SHM_SIZE}
     volumes:
-      - dbdata:/var/lib/postgresql/data
+      - ${QUERY_DATA_DIR}:/var/lib/postgresql/data
     networks:
       - common_net
 
@@ -38,5 +39,7 @@ networks:
     name: ${COMMON_NET_NAME}
     external: true
 
+# This is here for legacy reasons as QUERY_DATA_DIR didn't used to be configurable
+# Set QUERY_DATA_DIR to "dbdata" to continue to use this volume
 volumes:
   dbdata: