sandbox

#!/usr/bin/env bash
set -euo pipefail

#####################
# Script constants. #
#####################
DEFAULT_CONFIG='release'
SANDBOX_DIR=$(dirname "$0")
source "$SANDBOX_DIR/.env"
CLEAN_FILE="$SANDBOX_DIR/.clean"
ACTIVE_CONFIG_FILE="$SANDBOX_DIR/.active_config"
SANDBOX_LOG="$SANDBOX_DIR/sandbox.log"
FAST_CATCHUP_URL='https://algorand-catchpoints.s3.us-east-2.amazonaws.com/channel/CHANNEL/latest.catchpoint'

APORT="${ALGOD_PORT:-4001}"
KPORT="${KMD_PORT:-4002}"
IPORT="${INDEXER_PORT:-8980}"

# Global flags
USE_FAST_CATCHUP=1
INTERACTIVE_MODE=0
VERBOSE_MODE=0

#########################
# Config file variables #
#########################
export ALGOD_CHANNEL=""
export ALGOD_URL=""
export ALGOD_BRANCH=""
export ALGOD_SHA=""
export NETWORK=""
export NETWORK_BOOTSTRAP_URL=""
export NETWORK_GENESIS_FILE=""
export NODE_ARCHIVAL=""
export INDEXER_URL=""
export INDEXER_BRANCH=""
export INDEXER_SHA=""
export INDEXER_DISABLED=""

##################################################
# Determine the correct docker command (or fail) #
##################################################
if [ -x "$(command -v docker-compose)" ]; then
  DOCKER_COMPOSE="docker-compose"
elif [ "$(docker compose help &>/dev/null; echo "$?")" -eq 16 ]; then
  DOCKER_COMPOSE="docker compose"
else
  echo 'Error: docker compose is not installed.' >&2
  exit 1
fi

######################################
# Compatibility with Windows / MSYS2 #
######################################

# All commands that use a TTY/pseudo-TTY
# need to be prefixed by "$WINDOWS_PTY_PREFIX"
WINDOWS_PTY_PREFIX=
if [ "$(uname)" == "Darwin" ]; then
  true
elif [ "$(expr substr $(uname -s) 1 10)" == "MINGW32_NT" ] || \
     [ "$(expr substr $(uname -s) 1 10)" == "MINGW64_NT" ]; then
  WINDOWS_PTY_PREFIX="winpty"
fi

#########################
# Helpers and utilities #
#########################

# Console colorized print helpers
default="\033[0m"

red="\033[0;31m"
green="\033[0;32m"
blue="\033[0;34m"
teal="\033[0;36m"

# bold
Bgreen="\033[1;32m"


function printc () {
  printf "$1$2${default}\n"
}

function statusline () {
  printc "${Bgreen}" "\n$1"
}

function err_noexit () {
  printf "${red}$1${default}\n"
}

function err () {
  err_noexit "$1"
  exit 1
}

# Overwrite the current line on the terminal
function overwrite() {
  echo -e "\r\033[1A\033[0K$@";
}

# Yes/no prompt around the clean command
function ask_clean () {
  if ask "$1"; then
    sandbox clean
  else
    exit 1
  fi
}

# Interactive yes/no prompt
function ask () {
    # https://djm.me/ask
    local prompt default reply

    if [ "${2:-}" = "Y" ]; then
        prompt="Y/n"
        default=Y
    elif [ "${2:-}" = "N" ]; then
        prompt="y/N"
        default=N
    else
        prompt="y/n"
        default=
    fi

    while true; do

        # Ask the question (not using "read -p" as it uses stderr not stdout)
        echo -n "$1 [$prompt] "

        # Read the answer (use /dev/tty in case stdin is redirected from somewhere else)
        read reply </dev/tty

        # Default?
        if [ -z "$reply" ]; then
            reply=$default
        fi

        # Check if the reply is valid
        case "$reply" in
            Y*|y*) return 0 ;;
            N*|n*) return 1 ;;
        esac

    done
}

# Spinner - https://stackoverflow.com/a/20369590
# Show a spinner for long running commands:
#   (command) & spinner
function spinner()
{
  local pid=$!
  local delay=0.75
  local spinstr='|/-\'
  while [ "$(ps a | awk '{print $1}' | grep $pid)" ]; do
    local temp=${spinstr#?}
    overwrite " [${spinstr::1}] "
    local spinstr=$temp${spinstr%"$temp"}
    sleep $delay
  done
  overwrite ""
}

# Progress bar - https://stackoverflow.com/a/28044986
# 1. Create progress_bar function
# 1.1 Input is prefixString($1) currentState($2) and totalState($3)
function progress_bar {
  local progress
  local done
  local left
  local fill
  local empty

  # Process data
  progress=$(( $(( ${2} * 100 / ${3} * 100)) / 100))
  done=$(( $(( ${progress} * 4 )) / 10 ))
  left=$(( 40 - $done ))

  # Build progressbar string lengths
  fill=$(printf "%${done}s")
  empty=$(printf "%${left}s")

  # 1.2 Build progressbar strings and print the progress_bar line
  # 1.2.1 Output example:
  # 1.2.1.1 Progress : [****************************************] 100%
  overwrite "$1 : [${fill// /▇}${empty// / }] [$2/$3] ${progress}%"
}

sandbox () {
  status_helper () {
    statusline "algod - goal node status"
    goal_helper node status
    statusline "indexer - health"
    curl -s "localhost:$IPORT/health?pretty"
  }

  # docker-compose alias
  # must only be used when no pseudo-TTY is required
  # in particular, when using 
  #   dc ... exec ...
  # the flag -T needs to be passed
  # Concretely, to execute non-interactive commands:
  #   dc exec -T algod goal node status
  # and if the command is interactive, dc_pty needs to be used:
  #   dc_pty algod exec //bin/bash
  # This is to ensure compatibility with Windows Msys
  dc () {
    VERBOSE_FLAG=""
    if [ $VERBOSE_MODE == 1 ]; then
      # --verbose is universally available since tag https://github.com/docker/compose-cli/releases/tag/v1.0.18 pr https://github.com/docker/compose-cli/pull/1830
      VERBOSE_FLAG=" --verbose"
    fi

    $DOCKER_COMPOSE$VERBOSE_FLAG -f "$SANDBOX_DIR/docker-compose.yml" "$@"
  }

  # dc_pty needs to be used instead of dc any time
  # a pseudo-TTY is required.
  # See comment of dc ()
  # Do NOT used when the command input/output needs to be piped
  # or inside command substituions as this will fail on Windows Msys
  dc_pty () {
    # note: cannot be replaced by $WINDOWS_PTY_PREFIX dc "$@"
    #       because winpty requires an executable as argument
    # TODO: can windows docker handle --verbose?
    $WINDOWS_PTY_PREFIX $DOCKER_COMPOSE -f "$SANDBOX_DIR/docker-compose.yml" "$@"
  }

  rebuild_if_needed () {
    if [ -f "$CLEAN_FILE" ]; then
      rm "$CLEAN_FILE"
      echo ".clean file found in sandbox directory. Rebuilding images..."
      echo "* docker compose build --no-cache"
      dc build --no-cache
    elif [ $FRESH_INSTALL -eq 0 ]; then
      echo ".clean file NOT FOUND. Sandbox images will NOT be rebuilt."
    fi
  }

  # A shortcut for goal commands on docker-compose
  # It assumes the command is non-interactive and does not use a pseudo-TTY
  # (option -T)
  goal_helper () {
    dc exec -T algod goal "$@"
  }

  # A shortcut for tealdbg commands on docker-compose
  # It assumes the command is non-interactive and does not use a pseudo-TTY
  # (option -T)
  tealdbg_helper () {
    if [[ "$*" == *--listen* ]]
    then
        dc exec -T algod tealdbg "$@"
    else
      if [[ "$*" == *debug* ]]
      then
          echo "tealdbg debug command called without --listen option therefore sandbox attached the option automatically!"
          dc exec -T algod tealdbg  "$@" --listen 0.0.0.0
          
      else
          dc exec -T algod tealdbg "$@"
      fi        
    fi    
  }

  # wait until indexer/algod health endpoints report 200
  wait_for_services() {
    local ATTEMPTS_REMAINING=60
    local INDEXER_STATUS="000"
    local ALGOD_STATUS="000"

    # wait for startup
    while [[ $ATTEMPTS_REMAINING -gt 0 ]]; do
      INDEXER_STATUS=$(curl -sL -w "%{http_code}\n" "localhost:$IPORT/health" -o /dev/null || true)
      ALGOD_STATUS=$(curl -sL -w "%{http_code}\n" "localhost:$APORT/health" -o /dev/null || true)
      if [ "$ALGOD_STATUS" == "200" ] && [ "$INDEXER_STATUS" == "200" ]; then
        ATTEMPTS_REMAINING=0
      else
        ((ATTEMPTS_REMAINING--))
        sleep 1
      fi
    done

    if [ "$ALGOD_STATUS" != "200" ] || [ "$INDEXER_STATUS" != "200" ]; then
      echo "the following did not start:"
      if [ "$ALGOD_STATUS" != "200" ]; then
        echo "* algorand node"
      fi
      if [ "$INDEXER_STATUS" != "200" ]; then
        echo "* indexer node"
      fi
      err "One or more services failed to start."
    fi
  }

  version_helper () {
    statusline "algod version"
    dc exec -T algod goal -v
    statusline "Indexer version"
    INDEXER_VERSION=$(dc exec -T indexer cmd/algorand-indexer/algorand-indexer daemon -v) && echo ${INDEXER_VERSION} || curl -s "localhost:$IPORT/health?pretty"
    statusline "Postgres version"
    dc exec -T indexer-db postgres -V
  }

  # Given a network name attempts to fetch a catchpoint and catchup.
  perform_fast_catchup () {
    goal_helper node catchup "$1"

    if [ $? -ne 0 ]; then
      err "There was a problem starting fast catchup."
    fi

    # Newline for the progress bar to use.
    echo ""
    DONE=false
    STARTED=false
    while [ $DONE == "false" ]; do
      sleep 0.1

      local RES
      local TOTAL
      local PROGRESS

      RES="$(status_helper)"
      TOTAL=1000
      PROGRESS=0

      # If progress has been made, update the progress.
      if [[ "$RES" == *"Catchpoint total accounts"* ]]; then
        STARTED=true
        TOTAL=$(echo $RES | grep -o 'Catchpoint total accounts: [[:digit:]]*' | cut -d':' -f 2 )
        PROGRESS=$(echo $RES | grep -o 'Catchpoint accounts processed: [[:digit:]]*' | cut -d':' -f 2 )
      elif [ $STARTED == "true" ]; then
        DONE=true
        PROGRESS=$TOTAL
      fi

      if [ $TOTAL == $PROGRESS ]; then
        DONE=true
      else
        progress_bar "Processing accounts:" "$PROGRESS" "$TOTAL"
      fi
    done

    overwrite "* Account processing complete."

    # Newline for the progress bar to use.
    echo ""

    DONE=false
    STARTED=false
    while [ $DONE == "false" ]; do
      sleep 0.1

      local RES
      local TOTAL
      local PROGRESS

      RES="$(status_helper)"
      TOTAL=1000
      PROGRESS=0

      # If progress has been made, update the progress.
      if [[ "$RES" == *"Catchpoint downloaded blocks"* ]]; then
        STARTED=true
        TOTAL=$(echo $RES | grep -o 'Catchpoint total blocks: [[:digit:]]*' | cut -d ':' -f 2 )
        PROGRESS=$(echo $RES | grep -o 'Catchpoint downloaded blocks: [[:digit:]]*' | cut -d ':' -f 2 )
      elif [ $STARTED == "true" ]; then
        DONE=true
        PROGRESS=$TOTAL
      fi

      if [ $TOTAL == $PROGRESS ]; then
        DONE=true
      else
        progress_bar "Downloading blocks:" "$PROGRESS" "$TOTAL"
      fi
    done

    # Clear progress bar line and print success text.
    overwrite "* Blocks downloaded."

    sleep 1
  }

  clean () {
    echo "* docker compose down"
    dc down -t 0                                   || true
    echo "* docker rmi sandbox_algod sandbox_indexer"
    docker rmi sandbox_algod sandbox_indexer                   || true
    echo "* docker compose rm -f"
    dc rm -f                                       || true
    echo "* docker rmi $(docker images -f "dangling=true" -q)"
    docker rmi $(docker images -f "dangling=true" -q)          || true
    rm "$ACTIVE_CONFIG_FILE" > /dev/null 2>&1                  || true
    echo 'clean function executed' > "$CLEAN_FILE";
  }

  # Enter attaches users to a shell in the desired container
  enter () {
    CONTAINER=${2:-}
    if [ -z "$CONTAINER" ]; then
        err "'enter' requires a container. Available containers: algod, indexer, indexer-db"
    fi
    case $CONTAINER in
      algod)
        statusline "Entering /bin/bash session in the algod container..."
        dc_pty exec algod //bin/bash
        return
        ;;
      indexer)
        statusline "Entering /bin/bash session in the indexer container..."
        dc_pty exec indexer //bin/bash
        return
        ;;
      indexer-db)
        statusline "Entering /bin/bash session in the indexer-db container..."
        dc_pty exec indexer-db //bin/bash
        return
        ;;
    esac

    err "Cannot enter '$CONTAINER'. Available containers: algod, indexer, indexer-db"
  }


  # Logs streams the logs from the container to the shell
  logs () {
    if [[ $# -gt 1 && $2 == "raw" ]]; then
      dc exec -T algod tail -f node.log
    else
      # We need a PTY with carpenter to have colors
      dc_pty exec algod carpenter -D
    fi
  }

  set_catchpoint () {
    # TODO: Might be useful to allow providing the catchpoint with '-c'
    CATCHPOINT=$(curl -s ${FAST_CATCHUP_URL/CHANNEL/$1})
    # If the catchpoint wasn't returned there is a multiline error.
    if [[ "$(echo $CATCHPOINT | wc -l | tr -d ' ')" != "1" ]]; then
      CATCHPOINT=""
    fi
  }

  # Start the algorand node
  up () {
      FRESH_INSTALL=1

      # Grab active config if there is one
      ACTIVE_CONFIG=""
      if [ -f "$ACTIVE_CONFIG_FILE" ]; then
        ACTIVE_CONFIG=$(cat $ACTIVE_CONFIG_FILE)
      fi

      # Initialize new config, accounting for no-argument up command.
      if [ "$1" = "" ]; then
        if [ ! -z $ACTIVE_CONFIG ]; then
          statusline "Bringing up existing sandbox: '$ACTIVE_CONFIG'"
          FRESH_INSTALL=0
          NEW_CONFIG="$ACTIVE_CONFIG"
        else
          statusline "Starting default sandbox: $DEFAULT_CONFIG"
          NEW_CONFIG="$DEFAULT_CONFIG"
        fi
      else
        statusline "Starting sandbox for: $1"
        NEW_CONFIG="$1"
      fi
      CONFIG_FILE="$SANDBOX_DIR/config.$NEW_CONFIG"

      # Verify config exists
      if [ ! -f "$CONFIG_FILE" ]; then
        SANDBOX_CONFIG_OPTIONS=$(ls "$SANDBOX_DIR"/config.* | sed 's/^.*config\./ /'| paste -sd',' -)
        err "Could not find config file for '$NEW_CONFIG'.\nValid options:$SANDBOX_CONFIG_OPTIONS"
      fi

      # Handle mismatched argument + active config
      if [ ! -z "$ACTIVE_CONFIG" ] && [ "$NEW_CONFIG" != "$ACTIVE_CONFIG" ]; then
        err_noexit "Sandbox was already started for '$ACTIVE_CONFIG'."
        ask_clean "Would you like to reset the local sandbox with '$NEW_CONFIG'?"
        # If we get here the active config was cleared out.
        up "$@"
        return
      fi

      #################
      # START NETWORK #
      #################

      # network configured with the environment exported by $CONFIG_FILE
      echo "$NEW_CONFIG" > "$ACTIVE_CONFIG_FILE"
      source "$CONFIG_FILE"
      if [ $INTERACTIVE_MODE == 1 ]; then
        rebuild_if_needed
        dc up
      elif [ $VERBOSE_MODE == 0 ]; then
        echo "see sandbox.log for detailed progress, or use -v."
        echo "" # The spinner will rewrite this line.
        rebuild_if_needed >> "$SANDBOX_LOG" 2>&1               & spinner
        echo "* docker compose up -d"   >> "$SANDBOX_LOG"
        dc up -d                        >> "$SANDBOX_LOG" 2>&1 & spinner
        overwrite "* docker containers started!"
      else
        rebuild_if_needed
        echo "* running: docker compose up [--verbose] -d"
        dc up -d
      fi

      echo -e "* waiting for services to initialize."
      wait_for_services
      echo "* services ready!"

      version_helper
      status_helper

      if [ -z $NETWORK_GENESIS_FILE ]; then
        #####################
        # PRINT WALLET INFO #
        #####################
        goal_helper wallet -f unencrypted-default-wallet > /dev/null 2>&1

        ACCOUNTS_OUTPUT=$(goal_helper account list)
        for acct in $(echo "$ACCOUNTS_OUTPUT" | cut -f 3 |tr -s ' '); do
          ACCOUNTS+=($acct)
        done

        statusline "Available accounts"
        printc $default "~$ ${blue}./sandbox ${teal}goal account list"
        echo "$ACCOUNTS_OUTPUT"
        statusline "Example command to send between two accounts:"
        printc $default "~$ ${blue}./sandbox ${teal}goal clerk send -a 123456789 -f ${ACCOUNTS[1]} -t ${ACCOUNTS[2]}"
        statusline "Soon after sending the transaction it will appear in indexer:"
        printc $default "~$ ${blue}curl ${teal}\"localhost:$IPORT/v2/transactions?pretty\""
      else
        ################################
        # FAST CATCHUP (if applicable) #
        ################################
        if [ $USE_FAST_CATCHUP == 1 ] && [ $FRESH_INSTALL == 1 ] && [ ! -z "$NETWORK" ]; then
          set_catchpoint $NETWORK

          if [ -z $CATCHPOINT ]; then
            err_noexit "Fast catchup is not available for $NETWORK, continuing without catchup."
          else
            statusline "Starting fast-catchup with catchpoint: $CATCHPOINT"

            perform_fast_catchup $CATCHPOINT

            statusline "Fast-catchup complete! Printing status..."
            status_helper
          fi
        else
          statusline "Skipping fast catchup"
        fi
      fi
  }

  help () {
      cat <<-EOF
sandbox commands:
  up    [config]  -> start the sandbox environment.
  down            -> tear down the sandbox environment.
  reset           -> reset the containers to their initial state.
  clean           -> stops and deletes containers and data directory.
  test            -> runs some tests to demonstrate usage.
  enter [algod||indexer||indexer-db]
                  -> enter the sandbox container.
  version         -> print binary versions.
  copyTo <file>   -> copy <file> into the algod. Useful for offline transactions, offline LogicSigs & TEAL work.
  copyFrom <file> -> copy <file> from the algod. Useful for offline transactions, offline LogicSigs & TEAL work.

algorand commands:
  logs           -> stream algorand logs with the carpenter utility.
  status         -> get node status.
  goal (args)    -> run goal command like 'goal node status'.
  tealdbg (args) -> run tealdbg command.

special flags for 'up' command:
  -v|--verbose           -> display verbose output when starting standbox.
  -s|--skip-fast-catchup -> skip catchup when connecting to real network.
  -i|--interactive       -> start docker compose in interactive mode.
EOF
  }

  if [ $# -eq 0 ]; then
    help
    exit 1
  fi

  case $1 in
    start|up|resume)
      # Remove "up"
      shift

      # Process flags
      PARAMS=()
      while (( "$#" )); do
        case "$1" in
          -v|--verbose)
            VERBOSE_MODE=1
            ;;
          -s|--skip-fast-catchup)
            USE_FAST_CATCHUP=0
            ;;
          -i|--interactive)
            statusline "Note: interactive mode is incompatible with fast-catchup."
            INTERACTIVE_MODE=1
            ;;
          *) # preserve positional arguments
            PARAMS+=("$1")
            ;;
        esac
        shift
      done

      # Call up with remaining parameters
      up "${PARAMS[@]-}"
      ;;

    stop|down|pause)
      statusline "Stopping sandbox containers..."
      dc stop -t 0
      ;;

    restart|reset)
      # Make sure fast-catchup runs when resetting a real network.
      if [ -f "$ACTIVE_CONFIG_FILE" ]; then
        ACTIVE_CONFIG=$(cat $ACTIVE_CONFIG_FILE)
        # up only runs fast-catchup for fresh installs.
        # removing the active config file will trick it.
        rm "$ACTIVE_CONFIG_FILE" > /dev/null 2>&1 || true
      else
        err "No active sandbox to reset."
      fi
      sandbox down
      dc rm -f
      sandbox up "$ACTIVE_CONFIG" "$@"
      ;;

    clean)
      statusline "Cleaning up sandbox environment..."
      clean >> "$SANDBOX_LOG" 2>&1
      ;;

    test)
      statusline "Test command forwarding..."
      printc $default "~$ ${blue}docker exec -it algod uname -a"
      dc exec -T algod uname -a

      TOKEN=$(cat "$SANDBOX_DIR/config/token")
      statusline "Test Algod REST API..."
      printc $default "~$ ${blue}curl ${teal}localhost:$APORT/v2/status?pretty -H \"X-Algo-API-Token: $TOKEN\""
      curl "localhost:$APORT/v2/status?pretty" -H "X-Algo-API-Token: $TOKEN"

      statusline "Test KMD REST API..."
      printc $default "~$ ${blue}curl ${teal}localhost:$KPORT/v1/wallets -H \"X-KMD-API-Token: $TOKEN\""
      curl "localhost:$KPORT/v1/wallets" -H "X-KMD-API-Token: $TOKEN"

      statusline "\nTest Indexer REST API..."
      printc $default "~$ ${blue}curl ${teal}\"localhost:$IPORT/health?pretty\""
      curl "localhost:$IPORT/health?pretty"
      ;;

    enter)
      enter "$@"
      ;;

    logs)
      logs "$@"
      ;;

    status)
      status_helper
      ;;

    version)
      version_helper
      ;;

    goal)
      shift
      goal_helper "$@"
      ;;

    tealdbg)
      shift
      tealdbg_helper "$@"
      ;;

    copyTo|cpt)
      shift
      statusline "Now copying $1 to Algod container in /opt/data/$1"
      docker cp "$1" "$(dc ps -q algod):/opt/data/$(basename $1)"
      ;;

    copyFrom|cpf)
      shift
      statusline "Now copying  /opt/data/$1 from Algod container to $SANDBOX_DIR/" 
      docker cp "$(dc ps -q algod):/opt/data/$(basename $1)" "$1"
      ;;

    *)

      help
      ;;
  esac
}

##############
# Entrypoint #
##############

sandbox "$@"