diff --git a/sidecar.sh b/sidecar.sh index b32cc31..c485e5a 100755 --- a/sidecar.sh +++ b/sidecar.sh @@ -9,7 +9,7 @@ function log(){ } populate_notebook() { - MANIFEST=$1 + MANIFEST_FILE=$1 shift FOLDER=$1 shift @@ -17,7 +17,7 @@ populate_notebook() { manifest_ls="!gen3 drs-pull ls manifest.json" jq --arg cmd "$manifest_ls" '.cells[1].source |= $cmd' "$FOLDER/data.ipynb" > "$FOLDER/data.tmp" && mv "$FOLDER/data.tmp" "$FOLDER/data.ipynb" jq --arg cmd "$manifest_pull" '.cells[3].source |= $cmd' "$FOLDER/data.ipynb" > "$FOLDER/data.tmp" && mv "$FOLDER/data.tmp" "$FOLDER/data.ipynb" - echo $MANIFEST | jq -c '.[]' | while read j; do + echo $MANIFEST_FILE | jq -c '.[]' | while read j; do obj=$(echo $j | jq -r .object_id) filename=$(echo $j | jq -r .file_name) filesize=$(echo $j | jq -r .file_size) @@ -41,29 +41,29 @@ populate_notebook() { function populate() { log "querying manifest service at $GEN3_ENDPOINT/manifests/" - MANIFESTS=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/manifests/") + MANIFEST_FILE=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/manifests/") log "querying manifest service at $GEN3_ENDPOINT/metadata/" - METADATA=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/metadata/") + METADATA_FILE=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/metadata/") - while [ -z "$MANIFESTS" ] || [ -z "$METADATA" ]; do - if [ -z "$MANIFESTS" ]; then + while [ -z "$MANIFEST_FILE" ] && [ -z "$METADATA_FILE" ]; do + if [ -z "$MANIFEST_FILE" ]; then log "Unable to get manifests from '$GEN3_ENDPOINT/manifests/'" - log $MANIFESTS + log $MANIFEST_FILE fi - if [ -z "$METADATA" ]; then + if [ -z "$METADATA_FILE" ]; then log "Unable to get metadata from '$GEN3_ENDPOINT/metadata/'" - log $METADATA + log $METADATA_FILE fi log "sleeping for 15 seconds before trying again.." sleep 15 - MANIFESTS=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/manifests/") - METADATA=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/metadata/") + MANIFEST_FILE=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/manifests/") + METADATA_FILE=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/metadata/") done log "successfully retrieved manifests and metadata for user" process_files() { - local data=$1 - local base_dir=$2 + local base_dir=$1 + local data=$2 echo $data | jq -c '.[]' | while read i; do FILENAME=$(echo "${i}" | jq -r .filename) @@ -78,20 +78,22 @@ function populate() { chown -R 1000:100 $FOLDER if ["$base_dir" == "manifests"];then - MANIFEST=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/manifests/file/$FILENAME") - echo "${MANIFEST}" > $FOLDER/manifest.json + MANIFEST_FILE=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/manifests/file/$FILENAME") + echo "${MANIFEST_FILE}" > $FOLDER/manifest.json log "Creating notebook for $FILENAME" cp ./template_manifest.json $FOLDER/data.ipynb - populate_notebook "$MANIFEST" "$FOLDER" + populate_notebook "$MANIFEST_FILE" "$FOLDER" elif ["$base_dir" == "metadata"];then - METADATA=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/manifests/metadata/$FILENAME") - echo "${METADATA}" > $FOLDER/metadata.json + METADATA_FILE=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/manifests/metadata/$FILENAME") + echo "${METADATA_FILE}" > $FOLDER/metadata.json fi fi done } - echo $MANIFESTS | jq -c '.manifests' | process_files manifests - echo $METADATA | jq -c '.external_file_metadata' | process_files metadata + # echo $MANIFEST_FILE | jq -c '.manifests' | process_files manifests + process_files "manifests" "$(echo $MANIFEST_FILE | jq -c '.manifests')" + # echo $METADATA_FILE | jq -c '.external_file_metadata' | process_files metadata + process_files "metadata" "$(echo $METADATA_FILE | jq -c '.external_file_metadata')" # Make sure notebook user has write access to the folders chown -R 1000:100 /data @@ -126,10 +128,6 @@ function get_access_token() { export ACCESS_TOKEN="$ACCESS_TOKEN" } -function get_manifest_service_access_token() { - log "Getting access token" - ACCESS_TOKEN=$(curl -s -H "Content-Type: application/json" -X POST "https://$GEN3_ENDPOINT/user/credentials/api/access_token/" -d "{ \"api_key\": \"${API_KEY}\" }" | jq -r .access_token) -} function mount_hatchery_files() { log "Mounting Hatchery files" @@ -179,7 +177,7 @@ function main() { while true; do populate # If the access token expires, fetch a new access token and try again - if [[ $(echo "$MANIFESTS" | jq -r '.error') = "Please log in." ]]; then + if [[ $(echo "$MANIFEST_FILE" | jq -r '.error') = "Please log in." ]]; then echo "Session Expired. Trying again with new access token" get_access_token else