From 90b8aaedbea921efa93d655f512941919b8c2ec2 Mon Sep 17 00:00:00 2001 From: Ori Newman Date: Wed, 1 Nov 2023 12:02:36 +0200 Subject: [PATCH] Some fixes to RPC and docker --- api/tsconfig.json | 5 ++-- docker-build.sh | 4 +-- docker-compose.yaml | 12 ++++++++ docker-run.sh | 33 ++++++++++++++++++---- processing/Dockerfile | 2 +- processing/infrastructure/config/config.go | 6 ++-- processing/main.go | 2 +- processing/processing/processing.go | 21 +++++++------- web/Dockerfile | 4 +-- 9 files changed, 64 insertions(+), 25 deletions(-) diff --git a/api/tsconfig.json b/api/tsconfig.json index 614e763..2780b14 100644 --- a/api/tsconfig.json +++ b/api/tsconfig.json @@ -11,9 +11,10 @@ "allowSyntheticDefaultImports": true, "module": "esnext", "moduleResolution": "node", - "outDir": "dist" + "outDir": "dist", + "sourceMap": true }, "include": [ "src" ] -} +} \ No newline at end of file diff --git a/docker-build.sh b/docker-build.sh index 99476c2..a8a2afc 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -28,7 +28,7 @@ if [ true = "${REQUIRED_VARIABLE_NOT_SET}" ]; then fi # Build processing -docker build -f processing/Dockerfile -t kaspa-graph-inspector-processing:latest --build-arg KASPAD_VERSION="${KASPAD_VERSION}" processing +docker build -f processing/Dockerfile -t kaspa-graph-inspector-processing:latest --build-arg KASPAD_VERSION="${KASPAD_VERSION}" --build-arg KASPAD_REPOSITORY="${KASPAD_REPOSITORY}" processing # Build api docker build -f api/Dockerfile -t kaspa-graph-inspector-api:latest api @@ -37,4 +37,4 @@ docker build -f api/Dockerfile -t kaspa-graph-inspector-api:latest api REACT_APP_API_ADDRESS="${API_ADDRESS}:${API_PORT}" REACT_APP_KATNIP_ADDRESS="${KATNIP_ADDRESS}" REACT_APP_KASPA_LIVE_ADDRESS="${KASPA_LIVE_ADDRESS}" -docker build -f web/Dockerfile --build-arg REACT_APP_API_ADDRESS="${REACT_APP_API_ADDRESS}" --build-arg REACT_APP_KATNIP_ADDRESS="${REACT_APP_KATNIP_ADDRESS}" --build-arg REACT_APP_KASPA_LIVE_ADDRESS="${REACT_APP_KASPA_LIVE_ADDRESS}" -t kaspa-graph-inspector-web:latest web +docker build -f web/Dockerfile --build-arg REACT_APP_API_ADDRESS="${REACT_APP_API_ADDRESS}" --build-arg REACT_APP_KATNIP_ADDRESS="${REACT_APP_KATNIP_ADDRESS}" --build-arg REACT_APP_KASPA_LIVE_ADDRESS="${REACT_APP_KASPA_LIVE_ADDRESS}" --build-arg REACT_APP_EXPLORER_ADDRESS=explorer.kaspa.org -t kaspa-graph-inspector-web:latest web diff --git a/docker-compose.yaml b/docker-compose.yaml index 511b68d..d6387f4 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -9,6 +9,17 @@ services: user: root network_mode: host + persistent-postgres: + image: "postgres:13.2" + environment: + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_DB=${POSTGRES_DB} + user: root + network_mode: host + volumes: + - ${KGI_POSTGRES_MOUNT:-/tmp/kgi-postgres-mount}:/var/lib/postgresql/data + processing: image: "kaspa-graph-inspector-processing:latest" depends_on: @@ -16,6 +27,7 @@ services: command: - "/app/processing" - "--connection-string=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/${POSTGRES_DB}?sslmode=disable" + - "--rpcserver=${KGI_RPCSERVER}" network_mode: host user: root restart: always diff --git a/docker-run.sh b/docker-run.sh index 3a1beca..42fd16b 100755 --- a/docker-run.sh +++ b/docker-run.sh @@ -2,6 +2,20 @@ set -e +export KGI_RPCSERVER=${KGI_RPCSERVER-"localhost"} +export POSTGRES_USER=${POSTGRES_USER-postgres} +export POSTGRES_PASSWORD=${POSTGRES_PASSWORD-postgres} +export POSTGRES_DB=${POSTGRES_DB-postgres} +export API_ADDRESS=${API_ADDRESS-"localhost"} +export KATNIP_ADDRESS=${KATNIP_ADDRESS-explorer.kaspa.org} +export API_PORT=${API_PORT-4575} +export WEB_PORT=${WEB_PORT-8080} +export KASPAD_VERSION=${KASPAD_VERSION-increased-route-capacity-20000} +export KASPAD_REPOSITORY=${KASPAD_REPOSITORY-github.com/someone235/kaspad} +export KASPA_LIVE_ADDRESS=${KASPA_LIVE_ADDRESS-kaspa.live} +export REACT_APP_EXPLORER_ADDRESS=${REACT_APP_EXPLORER_ADDRESS-explorer.kaspa.org} +export KGI_POSTGRES_MOUNT=${KGI_POSTGRES_MOUNT-"~/.kgi-postgres"} + # Verify that all the required environment variables are set declare -A REQUIRED_VARIABLES REQUIRED_VARIABLES["POSTGRES_USER"]="${POSTGRES_USER}" @@ -34,16 +48,25 @@ fi # Build kaspa-graph-inspector ./docker-build.sh +function docker_compose() { + if ! command -v docker-compose &> /dev/null + then + docker compose "$@" + else + docker-compose "$@" + fi +} + # Start postgres -docker-compose up -d postgres +docker_compose up -d persistent-postgres # Wait for postgres to finish initializing sleep 10s # Start processing, api, and web -docker-compose up -d processing -docker-compose up -d api -docker-compose up -d web +docker_compose up -d processing +docker_compose up -d api +docker_compose up -d web # Print logs for all services -docker-compose logs -f +docker_compose logs -f diff --git a/processing/Dockerfile b/processing/Dockerfile index 4f5d143..f0a9191 100644 --- a/processing/Dockerfile +++ b/processing/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.16-alpine AS build +FROM golang:1.19-alpine AS build # gcc and libc (musl-dev) are required by kaspad # git is required to build kaspad by commit-hash diff --git a/processing/infrastructure/config/config.go b/processing/infrastructure/config/config.go index bff0bd9..8d3515a 100644 --- a/processing/infrastructure/config/config.go +++ b/processing/infrastructure/config/config.go @@ -41,6 +41,7 @@ type Flags struct { Resync bool `long:"resync" description:"Force to resync all available node blocks with the PostgrSQL database -- Use if some recently added blocks have missing parents"` ClearDB bool `long:"clear-db" description:"Clear the PostgrSQL database and sync from scratch"` LogLevel string `short:"d" long:"loglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify =,=,... to set the log level for individual subsystems -- Use show to list available subsystems"` + RPCServer string `short:"s" long:"rpcserver" description:"RPC server to connect to"` kaspaConfigPackage.NetworkFlags } @@ -64,8 +65,9 @@ func cleanAndExpandPath(path string) string { func defaultFlags() *Flags { return &Flags{ - AppDir: defaultDataDir, - LogLevel: defaultLogLevel, + AppDir: defaultDataDir, + LogLevel: defaultLogLevel, + RPCServer: "localhost", } } diff --git a/processing/main.go b/processing/main.go index 32b9b11..2e3a851 100644 --- a/processing/main.go +++ b/processing/main.go @@ -32,7 +32,7 @@ func main() { } defer database.Close() - rpcAddress, err := config.NetParams().NormalizeRPCServerAddress("localhost") + rpcAddress, err := config.NetParams().NormalizeRPCServerAddress(config.RPCServer) if err != nil { panic(err) } diff --git a/processing/processing/processing.go b/processing/processing/processing.go index 42fae5e..17c225f 100644 --- a/processing/processing/processing.go +++ b/processing/processing/processing.go @@ -229,7 +229,9 @@ func (p *Processing) ResyncDatabase() error { log.Infof("Time %s", time.Unix(rpcBlock.Block.Header.Timestamp/1000, 0)) - log.Infof("Progress %d%%", (100*(rpcBlock.Block.Header.DAAScore-rpcPruning.Block.Header.DAAScore))/(dagInfo.VirtualDAAScore-rpcPruning.Block.Header.DAAScore)) + if dagInfo.VirtualDAAScore-rpcPruning.Block.Header.DAAScore != 0 { + log.Infof("Progress %d%%", (100*(rpcBlock.Block.Header.DAAScore-rpcPruning.Block.Header.DAAScore))/(dagInfo.VirtualDAAScore-rpcPruning.Block.Header.DAAScore)) + } } hashes, err := hashesFromStrings(getBlocks.BlockHashes) @@ -559,20 +561,19 @@ func (p *Processing) processBlock(databaseTransaction *pg.Tx, block *externalapi return err } - selectedParentID, err := p.database.BlockIDByHash(databaseTransaction, selectedParent) - if err != nil { - return errors.Wrapf(err, "Could not get selected parent block ID for block %s", - selectedParent) - } blockID, err := p.database.BlockIDByHash(databaseTransaction, blockHash) if err != nil { // enhanced error description return errors.Wrapf(err, "Could not get id for block %s", blockHash) } - err = p.database.UpdateBlockSelectedParent(databaseTransaction, blockID, selectedParentID) - if err != nil { - // enhanced error description - return errors.Wrapf(err, "Could not update selected parent for block %s", blockHash) + + selectedParentID, err := p.database.BlockIDByHash(databaseTransaction, selectedParent) + if err == nil { + err = p.database.UpdateBlockSelectedParent(databaseTransaction, blockID, selectedParentID) + if err != nil { + // enhanced error description + return errors.Wrapf(err, "Could not update selected parent for block %s", blockHash) + } } mergeSetReds, err := hashesFromStrings(rpcBlock.Block.VerboseData.MergeSetRedsHashes) diff --git a/web/Dockerfile b/web/Dockerfile index 725b834..7a444ec 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -9,9 +9,9 @@ RUN npm install COPY . . ARG REACT_APP_API_ADDRESS -ARG REACT_APP_KATNIP_ADDRESS +ARG REACT_APP_EXPLORER_ADDRESS ARG REACT_APP_KASPA_LIVE_ADDRESS -RUN REACT_APP_API_ADDRESS=${REACT_APP_API_ADDRESS} REACT_APP_KATNIP_ADDRESS=${REACT_APP_KATNIP_ADDRESS} REACT_APP_KASPA_LIVE_ADDRESS=${REACT_APP_KASPA_LIVE_ADDRESS} npm run build +RUN REACT_APP_API_ADDRESS=${REACT_APP_API_ADDRESS} REACT_APP_EXPLORER_ADDRESS=${REACT_APP_EXPLORER_ADDRESS} REACT_APP_KASPA_LIVE_ADDRESS=${REACT_APP_KASPA_LIVE_ADDRESS} npm run build FROM node:15.8.0-alpine WORKDIR /app