feat: Kava Protonet Data Reset Chain CI

.github/scripts/exit-standby-all-chain-nodes.sh

@@ -2,7 +2,7 @@
 set -x
 
 # get all the node's ec2 instance ids for the specified chain id
-chain_node_instance_ids=$(aws ec2 describe-instances --filters "Name=tag:KavaChainId,Values=$CHAIN_ID" | jq -r '[.Reservations | .[] | .Instances | .[] | .InstanceId] | join(" ")')
+chain_node_instance_ids=$(aws ec2 describe-instances --filters "Name=tag:$CHAIN_TAG_NAME,Values=$CHAIN_ID" | jq -r '[.Reservations | .[] | .Instances | .[] | .InstanceId] | join(" ")')
 
 for chain_node_instance_id in ${chain_node_instance_ids}
 do

.github/scripts/put-all-chain-nodes-on-standby.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 set -x
 
-# get all the node's ec2 instance ids for the specified chain id
-chain_node_instance_ids=$(aws ec2 describe-instances --filters "Name=tag:KavaChainId,Values=$CHAIN_ID" | jq -r '[.Reservations | .[] | .Instances | .[] | .InstanceId] | join(" ")')
+# get all the node's ec2 instance ids for the specified chain id KavaChainId
+chain_node_instance_ids=$(aws ec2 describe-instances --filters "Name=tag:$CHAIN_TAG_NAME,Values=$CHAIN_ID" | jq -r '[.Reservations | .[] | .Instances | .[] | .InstanceId] | join(" ")')
 
 for chain_node_instance_id in ${chain_node_instance_ids}
 do

.github/scripts/seed-protonet.sh

@@ -7,7 +7,7 @@ set -ex
 # broadcast mode of `block` in order to
 # minimize the chance tx's fail due to an
 # account sequence number mismatch
-AVG_SECONDS_BETWEEN_BLOCKS=6.5
+AVG_SECONDS_BETWEEN_BLOCKS=10
 
 # configure kava binary to talk to the desired chain endpoint
 kava config node "${CHAIN_API_URL}"

.github/workflows/cd-protonet.yml

@@ -5,6 +5,8 @@ name: Continuous Deployment (Protonet)
 #    workflows: [Continuous Integration (Kava Master)]
 #    types:
 #      - completed
+on:
+  push:
 
 jobs:
   # in order:
@@ -15,22 +17,26 @@ jobs:
   # reset application database state (only done on internal testnet)
   reset-chain-to-zero-state:
     # only start cd pipeline if last ci run was successful
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
-    uses: ./.github/workflows/cd-reset-internal-testnet.yml
+#    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    uses: ./.github/workflows/cd-reset-protonet.yml
     with:
       aws-region: us-east-1
+      chain-tag-name: ChainId
       chain-id: proto_2221-17000
-      ssm-document-name: kava-testnet-internal-node-update
-      playbook-name: reset-protonet-playbook.yml
-      playbook-infrastructure-branch: master
+      auto_scaling_group_names: kava-protonet-iavl-v1-validator-api-node,kava-protonet-iavl-v1-rpc-api-node
+      ssm-document-name: kava-protonet-iavl-v1-instance-update
+      playbook-name: reset-chain.yml
+      playbook-infrastructure-branch: shef-protonet-validator-network-ephemeral
     secrets: inherit
 
   # start kava with new binary and genesis state on api, peer and seed nodes, place nodes in service once they start and are synched to live
   start-chain-api:
-    uses: ./.github/workflows/cd-start-chain.yml
+    uses: ./.github/workflows/cd-start-protonet-chain.yml
     with:
       aws-region: us-east-1
+      chain-tag-name: ChainId
       chain-id: proto_2221-17000
+      auto_scaling_group_names: kava-protonet-iavl-v1-validator-api-node,kava-protonet-iavl-v1-rpc-api-node
       ssm-document-name: kava-testnet-internal-node-update
       playbook-name: start-chain-api-playbook.yml
       playbook-infrastructure-branch: master
@@ -45,7 +51,7 @@ jobs:
       chain-id: proto_2221-17000
       seed-script-filename: seed-protonet.sh
       erc20-deployer-network-name: protonet
-      genesis_validator_addresses: "kavavaloper14w4avgdvqrlpww6l5dhgj4egfn6ln7gmtp7r2m"
+      genesis_validator_addresses: "kavavaloper1jaw3g097lq9jdrnscchspr2233yhpsxdlq7ula"
       kava_version_filepath:  ./ci/env/kava-protonet/KAVA.VERSION
     secrets: inherit
     needs: [start-chain-api]
@@ -57,4 +63,4 @@ jobs:
       metric-name: kava.deploys.testnet.proto
       namespace: Kava/ContinuousDeployment
     secrets: inherit
-    needs: [seed-chain-state]
+    needs: [seed-chain-state]

.github/workflows/cd-reset-internal-testnet.yml

@@ -40,6 +40,7 @@ jobs:
       - name: take the chain offline
         run: bash ${GITHUB_WORKSPACE}/.github/scripts/put-all-chain-nodes-on-standby.sh
         env:
+          CHAIN_TAG_NAME: ${{ inputs.chain-tag-name }}
           CHAIN_ID: ${{ inputs.chain-id }}
           AWS_REGION: ${{ inputs.aws-region }}
           AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_KEY_ID }}

.github/workflows/cd-reset-protonet.yml

@@ -0,0 +1,92 @@
+name: Reset Internal Testnet
+
+on:
+  workflow_call:
+    inputs:
+      auto_scaling_group_names:
+        required: true
+        type: string
+        description: 'Comma-separated list of auto-scaling group names'
+      chain-tag-name:
+        required: false
+        type: string
+        default: 'KavaChainId'
+      chain-id:
+        required: true
+        type: string
+      aws-region:
+        required: true
+        type: string
+      ssm-document-name:
+        required: true
+        type: string
+      playbook-name:
+        required: true
+        type: string
+      playbook-infrastructure-branch:
+        required: true
+        type: string
+    secrets:
+      CI_AWS_KEY_ID:
+        required: true
+      CI_AWS_KEY_SECRET:
+        required: true
+      KAVA_PRIVATE_GITHUB_ACCESS_TOKEN:
+        required: true
+
+# in order:
+# enter standby (prevents autoscaling group from killing node during deploy)
+# stop kava
+# download updated binary and genesis
+# reset application database state (only done on internal testnet)
+jobs:
+  place-chain-nodes-on-standby:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout repo from current commit
+        uses: actions/checkout@v4
+      - name: take the chain offline
+        run: bash ${GITHUB_WORKSPACE}/.github/scripts/put-all-chain-nodes-on-standby.sh
+        env:
+          CHAIN_TAG_NAME: ${{ inputs.chain-tag-name }}
+          CHAIN_ID: ${{ inputs.chain-id }}
+          AWS_REGION: ${{ inputs.aws-region }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_KEY_SECRET }}
+      - name: checkout infrastructure repo
+        uses: actions/checkout@v4
+        with:
+          repository: Kava-Labs/infrastructure
+          token: ${{ secrets.KAVA_PRIVATE_GITHUB_ACCESS_TOKEN }}
+          path: infrastructure
+          ref: master
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: go.mod
+      - name: build kava node updater
+        run: cd infrastructure/cli/kava-node-updater && make install && cd ../../../
+      - name: run reset playbook on all chain nodes
+        run: |
+          IFS=',' read -r -a auto_scaling_group_names <<< "$AUTO_SCALING_GROUP_NAMES"
+          for auto_scaling_group_name in "${auto_scaling_group_names[@]}"; do
+            kava-node-updater \
+              --debug \
+              --max-retries=2 \
+              --aws-ssm-document-name="$SSM_DOCUMENT_NAME" \
+              --infrastructure-git-pointer="$PLAYBOOK_INFRASTRUCTURE_BRANCH" \
+              --update-playbook-filename="$PLAYBOOK_NAME" \
+              --autoscaling-group-name="$auto_scaling_group_name" \
+              --max-upgrade-batch-size=0 \
+              --node-types="" \
+              --wait-for-node-sync-after-upgrade=true
+          done
+        env:
+          SSM_DOCUMENT_NAME: ${{ inputs.ssm-document-name }}
+          PLAYBOOK_NAME: ${{ inputs.playbook-name }}
+          AUTO_SCALING_GROUP_NAMES: ${{ inputs.auto_scaling_group_names }}
+          AWS_REGION: ${{ inputs.aws-region }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_KEY_SECRET }}
+          AWS_SDK_LOAD_CONFIG: 1
+          PLAYBOOK_INFRASTRUCTURE_BRANCH: ${{ inputs.playbook-infrastructure-branch }}

.github/workflows/cd-start-chain.yml

@@ -3,6 +3,10 @@ name: Start Chain
 on:
   workflow_call:
     inputs:
+      chain-tag-name:
+        required: false
+        type: string
+        default: 'KavaChainId'
       chain-id:
         required: true
         type: string
@@ -36,6 +40,7 @@ jobs:
       - name: take the chain offline
         run: bash ${GITHUB_WORKSPACE}/.github/scripts/put-all-chain-nodes-on-standby.sh
         env:
+          CHAIN_TAG_NAME: ${{ inputs.chain-tag-name }}
           CHAIN_ID: ${{ inputs.chain-id }}
           AWS_REGION: ${{ inputs.aws-region }}
           AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_KEY_ID }}

.github/workflows/cd-start-protonet-chain.yml

@@ -0,0 +1,90 @@
+name: Start Chain
+
+on:
+  workflow_call:
+    inputs:
+      auto_scaling_group_names:
+        required: true
+        type: string
+        description: 'Comma-separated list of auto-scaling group names'
+      chain-tag-name:
+        required: false
+        type: string
+        default: 'KavaChainId'
+      chain-id:
+        required: true
+        type: string
+      aws-region:
+        required: true
+        type: string
+      ssm-document-name:
+        required: true
+        type: string
+      playbook-name:
+        required: true
+        type: string
+      playbook-infrastructure-branch:
+        required: true
+        type: string
+    secrets:
+      CI_AWS_KEY_ID:
+        required: true
+      CI_AWS_KEY_SECRET:
+        required: true
+      KAVA_PRIVATE_GITHUB_ACCESS_TOKEN:
+        required: true
+
+jobs:
+  # start kava, allow nodes to start processing requests from users once they are synced to live
+  serve-traffic:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout repo from current commit
+        uses: actions/checkout@v4
+      - name: take the chain offline
+        run: bash ${GITHUB_WORKSPACE}/.github/scripts/put-all-chain-nodes-on-standby.sh
+        env:
+          CHAIN_TAG_NAME: ${{ inputs.chain-tag-name }}
+          CHAIN_ID: ${{ inputs.chain-id }}
+          AWS_REGION: ${{ inputs.aws-region }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_KEY_SECRET }}
+      - name: checkout infrastructure repo
+        uses: actions/checkout@v4
+        with:
+          repository: Kava-Labs/infrastructure
+          token: ${{ secrets.KAVA_PRIVATE_GITHUB_ACCESS_TOKEN }}
+          path: infrastructure
+          ref: master
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: go.mod
+      - name: build kava node updater
+        run: cd infrastructure/cli/kava-node-updater && make install && cd ../../../
+      - name: run start-chain playbook on all chain nodes
+        run: |
+          IFS=',' read -r -a auto_scaling_group_names <<< "$AUTO_SCALING_GROUP_NAMES"
+          for auto_scaling_group_name in "${auto_scaling_group_names[@]}"; do
+            kava-node-updater \
+              --debug \
+              --max-retries=2 \
+              --aws-ssm-document-name="$SSM_DOCUMENT_NAME" \
+              --infrastructure-git-pointer="$PLAYBOOK_INFRASTRUCTURE_BRANCH" \
+              --update-playbook-filename="$PLAYBOOK_NAME" \
+              --autoscaling-group-name="$auto_scaling_group_name" \
+              --max-upgrade-batch-size=0 \
+              --node-types="" \
+              --wait-for-node-sync-after-upgrade=true
+          done
+        env:
+          SSM_DOCUMENT_NAME: ${{ inputs.ssm-document-name }}
+          PLAYBOOK_NAME: ${{ inputs.playbook-name }}
+          AUTO_SCALING_GROUP_NAMES: ${{ inputs.auto_scaling_group_names }}
+          AWS_REGION: ${{ inputs.aws-region }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_KEY_SECRET }}
+          AWS_SDK_LOAD_CONFIG: 1
+          PLAYBOOK_INFRASTRUCTURE_BRANCH: ${{ inputs.playbook-infrastructure-branch }}
+      - name: bring the chain online
+        run: bash ${GITHUB_WORKSPACE}/.github/scripts/exit-standby-all-chain-nodes.sh

ci/env/kava-protonet/KAVA.VERSION

@@ -1 +1 @@
-master
+shef-protonet-validator-network

ci/env/kava-protonet/README.md

@@ -0,0 +1,11 @@
+Genesis Updates
+===============
+- Added the God Committee section to the `app_state.committee.committees` section 
+- Added dev wallet `kava1vlpsrmdyuywvaqrv7rx6xga224sqfwz3fyfhwq` to `app_state.auth.accounts` for genesis account
+- Added dev wallet balances to `app_state.bank.balances` to enable this account to fund various other accounts
+- Updated `app_state.issuance.params` with different kava assets that will be needed in `seed-protonet.sh` script
+
+Summary
+=======
+- We decided to move critical data sections into the genesis.json file to ensure the node that starts up has all relevant state it needs BEFORE we execute any scripts to manipulate state.
+- The genesis.json contains the initial dev wallet, genesis account and associated balances to said account which is then used to pre-fund various ancillary kava balances (ex: community) to serve whatever testing purpose to protonet