diff --git a/scripts/delete-local-queues.sh b/scripts/delete-local-queues.sh new file mode 100755 index 0000000..3b94fa2 --- /dev/null +++ b/scripts/delete-local-queues.sh @@ -0,0 +1,7 @@ +#!/bin/zsh + +aws sqs delete-queue --queue-url http://localhost:4566/000000000000/transcription-service-task-queue-DEV.fifo --endpoint-url http://localhost:4566 +aws sqs delete-queue --queue-url http://localhost:4566/000000000000/transcription-service-output-queue-DEV --endpoint-url http://localhost:4566 +aws sqs delete-queue --queue-url http://localhost:4566/000000000000/giant-output-dead-letter-queue-DEV.fifo --endpoint-url http://localhost:4566 +aws sqs delete-queue --queue-url http://localhost:4566/000000000000/transcription-service-task-dead-letter-queue-DEV.fifo --endpoint-url http://localhost:4566 +aws sqs delete-queue --queue-url http://localhost:4566/000000000000/giant-output-queue-DEV.fifo --endpoint-url http://localhost:4566 \ No newline at end of file diff --git a/scripts/purge-local-queue.sh b/scripts/purge-local-queue.sh deleted file mode 100755 index 5fed108..0000000 --- a/scripts/purge-local-queue.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/zsh - -aws sqs purge-queue --queue-url http://localhost:4566/000000000000/transcription-service-task-queue-DEV.fifo --endpoint-url http://localhost:4566 -aws sqs purge-queue --queue-url http://localhost:4566/000000000000/giant-output-queue-DEV.fifo --endpoint-url http://localhost:4566 \ No newline at end of file diff --git a/scripts/purge-local-queues.sh b/scripts/purge-local-queues.sh new file mode 100755 index 0000000..0292c59 --- /dev/null +++ b/scripts/purge-local-queues.sh @@ -0,0 +1,7 @@ +#!/bin/zsh + +aws sqs purge-queue --queue-url http://localhost:4566/000000000000/transcription-service-task-queue-DEV.fifo --endpoint-url http://localhost:4566 +aws sqs purge-queue --queue-url http://localhost:4566/000000000000/transcription-service-output-queue-DEV --endpoint-url http://localhost:4566 +aws sqs purge-queue --queue-url http://localhost:4566/000000000000/giant-output-dead-letter-queue-DEV.fifo --endpoint-url http://localhost:4566 +aws sqs purge-queue --queue-url http://localhost:4566/000000000000/transcription-service-task-dead-letter-queue-DEV.fifo --endpoint-url http://localhost:4566 +aws sqs purge-queue --queue-url http://localhost:4566/000000000000/giant-output-queue-DEV.fifo --endpoint-url http://localhost:4566 \ No newline at end of file diff --git a/scripts/setup.sh b/scripts/setup.sh index 683b635..9fb6efa 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -25,12 +25,33 @@ fi docker-compose up -d APP_NAME="transcription-service" # If the queue already exists this command appears to still work and returns the existing queue url -QUEUE_URL=$(aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name=$APP_NAME-task-queue-DEV.fifo --attributes "FifoQueue=true,ContentBasedDeduplication=true" | jq .QueueUrl) + +######### +##### task dead letter queue +######### +DEAD_LETTER_QUEUE_URL=$(aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name=$APP_NAME-task-dead-letter-queue-DEV.fifo --attributes "FifoQueue=true,ContentBasedDeduplication=true" | jq .QueueUrl) +# We don't install the localstack dns so need to replace the endpoint with localhost +DEAD_LETTER_QUEUE_URL_LOCALHOST=${DEAD_LETTER_QUEUE_URL/sqs.eu-west-1.localhost.localstack.cloud/localhost} + +echo "Created queue in localstack, url: ${DEAD_LETTER_QUEUE_URL_LOCALHOST}" + +######### +##### task queue +######### +QUEUE_URL=$(aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name=$APP_NAME-task-queue-DEV.fifo \ + --attributes '{ + "FifoQueue": "true", + "ContentBasedDeduplication": "true", + "RedrivePolicy": "{\"deadLetterTargetArn\":\"arn:aws:sqs:us-east-1:000000000000:transcription-service-task-dead-letter-queue-DEV.fifo\",\"maxReceiveCount\":\"3\"}" + }' | jq .QueueUrl) # We don't install the localstack dns so need to replace the endpoint with localhost QUEUE_URL_LOCALHOST=${QUEUE_URL/sqs.eu-west-1.localhost.localstack.cloud/localhost} echo "Created queue in localstack, url: ${QUEUE_URL_LOCALHOST}" +######### +##### output queue +######### OUTPUT_QUEUE_URL=$(aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name=$APP_NAME-output-queue-DEV | jq .QueueUrl) # We don't install the localstack dns so need to replace the endpoint with localhost OUTPUT_QUEUE_URL_LOCALHOST=${OUTPUT_QUEUE_URL/sqs.eu-west-1.localhost.localstack.cloud/localhost} @@ -38,18 +59,38 @@ OUTPUT_QUEUE_URL_LOCALHOST=${OUTPUT_QUEUE_URL/sqs.eu-west-1.localhost.localstack echo "Created queue in localstack, url: ${OUTPUT_QUEUE_URL_LOCALHOST}" # ########### -# Create output queue for Giant: +# Creating output queue for Giant: # Giant is a service that uses transcription service to transcribe its audio/video files. # Giant pushes messages to the transcription input queue 'transcription-service-task-queue-DEV.fifo' -# and transcription worker pushes the giant transcripts into the giant output queue 'giant-output-queue-DEV'. +# and transcription worker pushes the resulting transcripts into the giant output queue 'giant-output-queue-DEV.fifo'. # Since creating multiple localstack containers could add complication, and localstack is -# only needed for local running, the giant output queue is created in the localstack created for transcription service. +# only needed for local running, the giant output queue is created in the transcription service localstack. # ########### -OUTPUT_QUEUE_URL=$(aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name=giant-output-queue-DEV.fifo --attributes "FifoQueue=true,ContentBasedDeduplication=true" | jq .QueueUrl) + +######### +##### giant output dead letter queue +######### +GIANT_OUTPUT_DEAD_LETTER_QUEUE_URL=$(aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name=giant-output-dead-letter-queue-DEV.fifo --attributes "FifoQueue=true,ContentBasedDeduplication=true" | jq .QueueUrl) # We don't install the localstack dns so need to replace the endpoint with localhost -OUTPUT_QUEUE_URL_LOCALHOST=${OUTPUT_QUEUE_URL/sqs.eu-west-1.localhost.localstack.cloud/localhost} +GIANT_OUTPUT_DEAD_LETTER_QUEUE_URL_LOCALHOST=${GIANT_OUTPUT_DEAD_LETTER_QUEUE_URL/sqs.eu-west-1.localhost.localstack.cloud/localhost} -echo "Created queue in localstack, url: ${OUTPUT_QUEUE_URL_LOCALHOST}" +echo "Created queue in localstack, url: ${GIANT_OUTPUT_DEAD_LETTER_QUEUE_URL_LOCALHOST}" + +######### +##### giant output queue +######### +GIANT_OUTPUT_QUEUE_URL=$(aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name=giant-output-queue-DEV.fifo \ + --attributes '{ + "FifoQueue": "true", + "ContentBasedDeduplication": "true", + "RedrivePolicy": "{\"deadLetterTargetArn\":\"arn:aws:sqs:us-east-1:000000000000:giant-output-dead-letter-queue-DEV.fifo\",\"maxReceiveCount\":\"3\"}" + }' | jq .QueueUrl) + + +# We don't install the localstack dns so need to replace the endpoint with localhost +GIANT_OUTPUT_QUEUE_URL_LOCALHOST=${GIANT_OUTPUT_QUEUE_URL/sqs.eu-west-1.localhost.localstack.cloud/localhost} + +echo "Created queue in localstack, url: ${GIANT_OUTPUT_QUEUE_URL_LOCALHOST}" DYNAMODB_ARN=$(aws --endpoint-url=http://localhost:4566 dynamodb create-table \ --table-name ${APP_NAME}-DEV \