#!/bin/bash

# Stop and report the error if ANYTHING breaks.
set -e

#
# This script contains various functions for setting up and provisioning a Bigscreen Arch5 server fleet from Jenkins. 
# See the pipeline files in Jenkins/pipelines for more info on how these functions are used.
#
# Usage (in a Jenkins shell command):
#    #!/bin/bash
#    . /path/to/this_file.sh
#    init_server_fleet_config_files "test" "v2"
#

jq-replace() {
    SRC=$1
    KEY=$2
    VAL=$3

    tmp=$(mktemp)
    jq ".$KEY = \"$VAL\"" "$SRC" > "$tmp" && mv "$tmp" "$SRC"
}

HOME_FOLDER="/home/ubuntu/networks"
JENKINS_BUILD_PATH=`pwd`
DEFAULT_AWS_REGION="us-west-2"
DEFAULT_REDIS_CLUSTER_FOLDER="terraform/redis-cluster"

init_beyond_config_files() {
    NETWORK_NAME=$1
    REGION=$DEFAULT_AWS_REGION

    # 1. Make the destination folders for configurating the server fleet
    NETWORK_DIR="$HOME_FOLDER/$NETWORK_NAME"
    mkdir -p "$NETWORK_DIR/.beyond"
    mkdir -p "$NETWORK_DIR/.beyond/terraform"
}

terraform_deploy_beyond_databases() {
    NETWORK_NAME=$1
    REGION=$DEFAULT_AWS_REGION
    # "version" is a reserved word in terraform, so we use "version_name" instead.
    VERSION_NAME="v1"

    # Copy the terraform scripts
    NETWORK_DIR="$HOME_FOLDER/$NETWORK_NAME"
    cp -r $JENKINS_BUILD_PATH/devops/terraform/v6/databases "$NETWORK_DIR/.beyond/terraform"
    cd "$NETWORK_DIR/.beyond/terraform/databases"
    terraform init

    # This is an AWS friendly name - note the dash instead of an underscore.
    DB_INSTANCE_CLASS="db.t3.micro"
    DB_ALLOCATED_STORAGE_GB=100

    # Run terraform apply
    if [ "$NETWORK_NAME" = "main" ];
    then
        DB_INSTANCE_CLASS="db.t3.xlarge"
        DB_ALLOCATED_STORAGE_GB=1000
    fi

    POSTGRES_USERNAME="postgres_admin"
    POSTGRES_PASSWORD="postgres_admin_password"

    terraform plan -input=false \
        -var="network_name=$NETWORK_NAME" \
        -var="version_name=$VERSION_NAME" \
        -var="db_instance_class=$DB_INSTANCE_CLASS" \
        -var="db_allocated_storage=$DB_ALLOCATED_STORAGE_GB" \
        -var="db_postgres_username=$POSTGRES_USERNAME" \
        -var="db_postgres_password=$POSTGRES_PASSWORD"

    terraform apply -input=false -auto-approve \
        -var="network_name=$NETWORK_NAME" \
        -var="version_name=$VERSION_NAME" \
        -var="db_instance_class=$DB_INSTANCE_CLASS" \
        -var="db_allocated_storage=$DB_ALLOCATED_STORAGE_GB" \
        -var="db_postgres_username=$POSTGRES_USERNAME" \
        -var="db_postgres_password=$POSTGRES_PASSWORD"
}

init_cloud_database_config_files() {
    NETWORK_NAME=$1
    REGION=$DEFAULT_AWS_REGION

    # 1. Make the destination folders for configurating the server fleet
    NETWORK_DIR="$HOME_FOLDER/$NETWORK_NAME"
    mkdir -p "$NETWORK_DIR/.cloud_database"
    mkdir -p "$NETWORK_DIR/.cloud_database/terraform"
}

terraform_deploy_cloud_databases() {
    NETWORK_NAME=$1
    REGION=$DEFAULT_AWS_REGION
    # "version" is a reserved word in terraform, so we use "version_name" instead.
    VERSION_NAME="v1"

    # Copy the terraform scripts
    NETWORK_DIR="$HOME_FOLDER/$NETWORK_NAME"
    cp -r $JENKINS_BUILD_PATH/devops/terraform/v6/cloud-database "$NETWORK_DIR/.cloud_database/terraform"
    cd "$NETWORK_DIR/.cloud_database/terraform/cloud-database"
    terraform init

    # This is an AWS friendly name - note the dash instead of an underscore.
    DB_INSTANCE_CLASS="db.t3.micro"
    DB_ALLOCATED_STORAGE_GB=100

    # Run terraform apply
    if [ "$NETWORK_NAME" = "main" ];
    then
        DB_INSTANCE_CLASS="db.t3.xlarge"
        DB_ALLOCATED_STORAGE_GB=1000
    fi

    POSTGRES_USERNAME="postgres_cloud_admin"
    POSTGRES_PASSWORD="postgres_cloud_admin_password"

    FULL_NETWORK_NAME="${NETWORK_NAME}cloud"

    terraform plan -input=false \
        -var="network_name=$FULL_NETWORK_NAME" \
        -var="version_name=$VERSION_NAME" \
        -var="db_instance_class=$DB_INSTANCE_CLASS" \
        -var="db_allocated_storage=$DB_ALLOCATED_STORAGE_GB" \
        -var="db_postgres_username=$POSTGRES_USERNAME" \
        -var="db_postgres_password=$POSTGRES_PASSWORD"

    terraform apply -input=false -auto-approve \
        -var="network_name=$FULL_NETWORK_NAME" \
        -var="version_name=$VERSION_NAME" \
        -var="db_instance_class=$DB_INSTANCE_CLASS" \
        -var="db_allocated_storage=$DB_ALLOCATED_STORAGE_GB" \
        -var="db_postgres_username=$POSTGRES_USERNAME" \
        -var="db_postgres_password=$POSTGRES_PASSWORD"
}

create_server_fleet_iam_users() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    
    TMP_OUTPUT_FILE=$(mktemp)

    # Create an IAM for this server fleet using the templates
    #
    # 1. Does the user already exist for this fleet?
    IAM_USER_NAME="$NETWORK_NAME-$FLEET_NAME-cloud-user"
    set +e
    aws iam get-user --user-name $IAM_USER_NAME
    IAM_USER_RESULT=$?
    set -e

    if [ $RESULT -eq 0 ]
    then
        echo "IAM user already exists."
    else
        echo "IAM user doesn't exist...creating key..."
        aws iam create-user --user-name test-violin-dynamo-db --permissions-boundary arn:aws:iam::164780942622:policy/test-dynamo-db
        aws iam create-access-key --user-name test-violin-dynamo-db
    fi
}

# Generate the config files for all the server and applications in the fleet.
#
# Requires the jenkins process to have an AWS credentials file at:
# /var/lib/jenkins/.aws
#
# The default IAM must have the ability to create entities on EC2.
init_server_fleet_config_files() {
    NETWORK_NAME=$1
    FLEET_NAME=$2

    if [[ -z $3 ]]
    then
        REGION=$DEFAULT_AWS_REGION
    else
        REGION=$3
    fi

    # TODO: if we've already built the servers via terraform, then bail out.
    # if there was an error, either rollback or create another fleet.

    # 1. Make the destination folders for configurating the server fleet
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    mkdir -p "$NETWORK_FLEET_DIR"
    mkdir -p "$NETWORK_FLEET_DIR/aws"
    mkdir -p "$NETWORK_FLEET_DIR/digitalocean"
    mkdir -p "$NETWORK_FLEET_DIR/config"
    mkdir -p "$NETWORK_FLEET_DIR/keys"
    mkdir -p "$NETWORK_FLEET_DIR/terraform"

    # 2. Use the AWS CLI to create a pem file.
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    TMP_OUTPUT_FILE=$(mktemp)

    # TODO - make the AWS profile configurable?
    # aws --region $DEFAULT_AWS_REGION --profile JenkinsServerBuilder ec2 create-key-pair --key-name $KEY_NAME > "$TMP_OUTPUT_FILE"
    set +e
    aws --region $DEFAULT_AWS_REGION ec2 describe-key-pairs --key-name $KEY_NAME
    RESULT=$?
    set -e

    if [ $RESULT -eq 0 ]
    then
        echo "Key already exists."
    else
        echo "Key doesn't exist...creating key..."
        aws --region $DEFAULT_AWS_REGION ec2 create-key-pair --key-name $KEY_NAME > "$TMP_OUTPUT_FILE"
        jq --raw-output ".KeyMaterial" "$TMP_OUTPUT_FILE" > "$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
        rm "$TMP_OUTPUT_FILE"
        chmod 400 "$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    fi

    # 3. Create a base config from the template and copy it to the config folder.
    FLEET_CONFIG_FILE="$NETWORK_FLEET_DIR/config/config.json"
    cp "$HOME_FOLDER/$NETWORK_NAME/.config/"$NETWORK_NAME"_config.json" "$FLEET_CONFIG_FILE"

    # 4. Replace the values in the config file with the generated ones.
    NETWORK_FLEET_NAME="$NETWORK_NAME-$FLEET_NAME"
    # Convert to lower case
    URL_PREFIX="${NETWORK_FLEET_NAME,,}"

    jq-replace "$FLEET_CONFIG_FILE" "NETWORK_NAME" "$NETWORK_NAME"
    jq-replace "$FLEET_CONFIG_FILE" "SERVER_FLEET_NAME" $NETWORK_NAME"_"$FLEET_NAME
    jq-replace "$FLEET_CONFIG_FILE" "PUBLIC_EMAIL_URL" "https://$URL_PREFIX-website.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "WEBAPP_ARDA_HOST" "https://arda.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "BIGSCREEN_API_URL"  "https://$URL_PREFIX-api.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "BIGSCREEN_ADMIN_API_URL" "https://$URL_PREFIX-admin-api.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "BIGSCREEN_CLOUD_API_URL" "https://$URL_PREFIX-cloud-api.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "CLOUD_WEBSOCKET_SERVER_NGINX_SERVER_NAME" "$URL_PREFIX-cloud.bigscreencloud.com"

    jq-replace "$FLEET_CONFIG_FILE" "WEBAPP_ARDA_NGINX_SERVER_NAME" "$URL_PREFIX-arda.bigscreencloud.com"
    
    # 5. copy the network firebase config files from the common key folder
    cp "$HOME_FOLDER/$NETWORK_NAME/.keys/firebase_"$NETWORK_NAME".json" "$NETWORK_FLEET_DIR/keys/firebase_"$NETWORK_NAME"_"$FLEET_NAME".json"
    cp "$HOME_FOLDER/$NETWORK_NAME/.keys/firebase_readonly_"$NETWORK_NAME".json" "$NETWORK_FLEET_DIR/keys/firebase_readonly_"$NETWORK_NAME"_"$FLEET_NAME".json"
    cp "$HOME_FOLDER/$NETWORK_NAME/.keys/firebase_"$NETWORK_NAME"_analytics.json" "$NETWORK_FLEET_DIR/keys/firebase_"$NETWORK_NAME"_"$FLEET_NAME"_analytics.json"

    # 6. access token private key
    cp "$HOME_FOLDER/$NETWORK_NAME/.keys/"$NETWORK_NAME"_access_token.key" "$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key"

    # 7. access token public key
    cp "$HOME_FOLDER/$NETWORK_NAME/.keys/"$NETWORK_NAME"_access_token.key.pub" "$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    # 8. brightcove token pem
    cp "$HOME_FOLDER/$NETWORK_NAME/.keys/"$NETWORK_NAME"_brightcove_token.pem" "$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_brightcove_token.pem"

    # 9. Allowed apps files
    cp "$HOME_FOLDER/$NETWORK_NAME/.keys/"$NETWORK_NAME"_allowed_apps.json" "$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json"

    # 10. Generate the digital ocean key pair and upload to digital ocean using API.
    DIGITAL_OCEAN_TOKEN=$(cat $NETWORK_FLEET_DIR/digitalocean/$NETWORK_NAME"-"$FLEET_NAME.pat)
    DIGITAL_OCEAN_KEY_NAME="$NETWORK_NAME-$FLEET_NAME-media-server"

    # Get existing keys from the API matching the key name, return 1 if the key exists, 0 if it doesn't.
    EXISTING_SSH_KEYS=( $(curl -s -X GET "https://api.digitalocean.com/v2/account/keys" -H "Authorization: Bearer $DIGITAL_OCEAN_TOKEN" | jq -r ".ssh_keys | map(select(.name == \""$DIGITAL_OCEAN_KEY_NAME"\")) | length > 0 | if . then 1 else 0 end") )

    if [ $EXISTING_SSH_KEYS -eq 1 ]
    then
        echo "Key already exists."
    else
        echo "Key doesn't exist...creating key..."
        ssh-keygen -t rsa -b 4096 -C "$DIGITAL_OCEAN_KEY_NAME" -f "$NETWORK_FLEET_DIR/digitalocean/$DIGITAL_OCEAN_KEY_NAME" -N ""
        chmod 400 "$NETWORK_FLEET_DIR/digitalocean/$DIGITAL_OCEAN_KEY_NAME"
        chmod 400 "$NETWORK_FLEET_DIR/digitalocean/$DIGITAL_OCEAN_KEY_NAME.pub"
        DIGITAL_OCEAN_PUBLIC_KEY=$(cat "$NETWORK_FLEET_DIR/digitalocean/$DIGITAL_OCEAN_KEY_NAME.pub")

        # Use curl to POST PUBLIC_KEY_JSON to digital ocean.
        curl -X POST \
            -H "Content-Type: application/json" \
            -H "Authorization: Bearer $DIGITAL_OCEAN_TOKEN" \
            -d "{\"name\":\"$DIGITAL_OCEAN_KEY_NAME\",\"public_key\":\"$DIGITAL_OCEAN_PUBLIC_KEY\"}" \
            "https://api.digitalocean.com/v2/account/keys"
    fi

    # 10. Run the generate script to generate the remainder of the configuration files.
    cd $JENKINS_BUILD_PATH/devops/ServerEnvironmentBuilder 
    npm install
    node app_v6.js \
        --default-config-file "$HOME_FOLDER/$NETWORK_NAME/.config/default_config.json" \
        --fleet-config-file "$FLEET_CONFIG_FILE" \
        --allowed-apps-file "$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json" \
        --output-folder-path "$NETWORK_FLEET_DIR/config"
    cd $JENKINS_BUILD_PATH
}

# Requires the jenkins process to have an AWS credentials file at:
# /var/lib/jenkins/.aws
#
# The default IAM must have the ability to create entities on EC2.
terraform_deploy_fleet() {
    NETWORK_NAME=$1
    FLEET_NAME=$2

    if [[ -z $3 ]]
    then
        REGION=$DEFAULT_AWS_REGION
    else
        REGION=$3
    fi

    # Copy the terraform scripts
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    cp -r $JENKINS_BUILD_PATH/devops/terraform/v6/fleet "$NETWORK_FLEET_DIR/terraform"
    cd "$NETWORK_FLEET_DIR/terraform/fleet"
    terraform init

    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    # This is an AWS friendly name - note the dash instead of an underscore.
    ELB_CERTIFICATE_ARN="arn:aws:acm:us-west-2:164780942622:certificate/61f977f7-27ff-4538-861c-dfb6535a8f80"
    API_SERVER_INSTANCE_TYPE="c5.large"
    ADMIN_API_SERVER_INSTANCE_TYPE="c5.large"
    CLOUD_SERVER_INSTANCE_TYPE="c5.large"
    WEBAPP_INSTANCE_TYPE="t3.medium"
    USE_PLACEMENT_GROUP_FOR_WEBAPPS="0"

    # Run terraform apply
    if [ "$NETWORK_NAME" = "main" ];
    then
        CLOUD_MEDIA_SERVER_COUNT=4
        ELB_CERTIFICATE_ARN="arn:aws:acm:us-west-2:164780942622:certificate/61f977f7-27ff-4538-861c-dfb6535a8f80"
        API_SERVER_INSTANCE_TYPE="c5.4xlarge"
        ADMIN_API_SERVER_INSTANCE_TYPE="c5.large"
        CLOUD_SERVER_INSTANCE_TYPE="c5.4xlarge"
        WEBAPP_INSTANCE_TYPE="c5.large"
        USE_PLACEMENT_GROUP_FOR_WEBAPPS="1"
    fi

    terraform plan -input=false \
        -var="network_name=$NETWORK_NAME" \
        -var="fleet_name=$FLEET_NAME" \
        -var="api_key_name=$KEY_NAME" \
        -var="api_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="api_instance_type=$API_SERVER_INSTANCE_TYPE" \
        -var="admin_api_key_name=$KEY_NAME" \
        -var="admin_api_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="admin_api_instance_type=$ADMIN_API_SERVER_INSTANCE_TYPE" \
        -var="cloud_websocket_key_name=$KEY_NAME" \
        -var="cloud_websocket_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="cloud_websocket_instance_type=$CLOUD_SERVER_INSTANCE_TYPE" \
        -var="cloud_api_key_name=$KEY_NAME" \
        -var="cloud_api_instance_type=$CLOUD_SERVER_INSTANCE_TYPE" \
        -var="cloud_api_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="webapps_key_name=$KEY_NAME" \
        -var="webapps_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="webapps_instance_type=$WEBAPP_INSTANCE_TYPE" \
        -var="use_placement_group_for_webapps"=$USE_PLACEMENT_GROUP_FOR_WEBAPPS

    terraform apply -input=false -auto-approve \
        -var="network_name=$NETWORK_NAME" \
        -var="fleet_name=$FLEET_NAME" \
        -var="api_key_name=$KEY_NAME" \
        -var="api_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="api_instance_type=$API_SERVER_INSTANCE_TYPE" \
        -var="admin_api_key_name=$KEY_NAME" \
        -var="admin_api_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="admin_api_instance_type=$ADMIN_API_SERVER_INSTANCE_TYPE" \
        -var="cloud_websocket_key_name=$KEY_NAME" \
        -var="cloud_websocket_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="cloud_websocket_instance_type=$CLOUD_SERVER_INSTANCE_TYPE" \
        -var="cloud_api_key_name=$KEY_NAME" \
        -var="cloud_api_instance_type=$CLOUD_SERVER_INSTANCE_TYPE" \
        -var="cloud_api_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="webapps_key_name=$KEY_NAME" \
        -var="webapps_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="webapps_instance_type=$WEBAPP_INSTANCE_TYPE" \
        -var="use_placement_group_for_webapps"=$USE_PLACEMENT_GROUP_FOR_WEBAPPS
}

terraform_deploy_redis_cluster() {
    NETWORK_NAME=$1
    FLEET_NAME=$2

    if [[ -z $3 ]]
    then
        REGION=$DEFAULT_AWS_REGION
    else
        REGION=$3
    fi

    # Copy the terraform scripts
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"

    # Get the security group ids from cloud_api so they can access this redis cluster
    cd $NETWORK_FLEET_DIR/terraform/fleet
    terraform refresh
    API_SECURITY_GROUP_ID=`terraform output -json api | jq .\"security_group_id\" --raw-output`
    ADMIN_API_SECURITY_GROUP_ID=`terraform output -json admin_api | jq .\"security_group_id\" --raw-output`
    CLOUD_API_SECURITY_GROUP_ID=`terraform output -json cloud_api | jq .\"security_group_id\" --raw-output`
    CLOUD_WEBSOCKET_SECURITY_GROUP_ID=`terraform output -json cloud_websocket | jq .\"security_group_id\" --raw-output`

    # copy and init terraform for the redis-cluster module.
    cp -r $JENKINS_BUILD_PATH/devops/terraform/v6/redis-cluster "$NETWORK_FLEET_DIR/terraform"
    cd "$NETWORK_FLEET_DIR/$DEFAULT_REDIS_CLUSTER_FOLDER"
    terraform init

    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    
    # Redis cluster node must be a cache.m* node type.
    REDIS_NODE_TYPE="cache.m5.large"
    REPLICAS_PER_NODE_GROUP=0
    NUM_NODE_GROUPS=1

    # Run terraform apply
    if [ "$NETWORK_NAME" = "main" ];
    then
        REPLICAS_PER_NODE_GROUP=0
        NUM_NODE_GROUPS=3
        REDIS_NODE_TYPE="cache.m5.2xlarge"
    fi

    terraform plan -input=false \
        -var="network_name=$NETWORK_NAME" \
        -var="fleet_name=$FLEET_NAME" \
        -var="replicas_per_node_group=$REPLICAS_PER_NODE_GROUP" \
        -var="num_node_groups=$NUM_NODE_GROUPS" \
        -var="api_security_group_id=$API_SECURITY_GROUP_ID" \
        -var="admin_api_security_group_id=$ADMIN_API_SECURITY_GROUP_ID" \
        -var="cloud_api_security_group_id=$CLOUD_API_SECURITY_GROUP_ID" \
        -var="cloud_websocket_security_group_id=$CLOUD_WEBSOCKET_SECURITY_GROUP_ID" \
        -var="redis_node_type=$REDIS_NODE_TYPE"

    terraform apply -input=false -auto-approve \
        -var="network_name=$NETWORK_NAME" \
        -var="fleet_name=$FLEET_NAME" \
        -var="replicas_per_node_group=$REPLICAS_PER_NODE_GROUP" \
        -var="num_node_groups=$NUM_NODE_GROUPS" \
        -var="api_security_group_id=$API_SECURITY_GROUP_ID" \
        -var="admin_api_security_group_id=$ADMIN_API_SECURITY_GROUP_ID" \
        -var="cloud_api_security_group_id=$CLOUD_API_SECURITY_GROUP_ID" \
        -var="cloud_websocket_security_group_id=$CLOUD_WEBSOCKET_SECURITY_GROUP_ID" \
        -var="redis_node_type=$REDIS_NODE_TYPE"
}

deploy_media_server_group() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    GROUP_NAME=$3
    CLOUD_MEDIA_SERVER_COUNT=$4

    if [ "$CLOUD_MEDIA_SERVER_COUNT" -lt 1 ]; then
        echo "CLOUD_MEDIA_SERVER_COUNT must be beween 1 and 15"
        exit 1
    fi

    if [ "$CLOUD_MEDIA_SERVER_COUNT" -gt 15 ]; then
        echo "CLOUD_MEDIA_SERVER_COUNT must be beween 1 and 15"
        exit 1
    fi

    # Copy the terraform scripts
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    cp -r $JENKINS_BUILD_PATH/devops/terraform/v6/media-server-aws "$NETWORK_FLEET_DIR/terraform/$GROUP_NAME-media-server"
    cd "$NETWORK_FLEET_DIR/terraform/$GROUP_NAME-media-server"
    terraform init

    CLOUD_MEDIA_KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    CLOUD_MEDIA_SERVER_NAME="$NETWORK_NAME"-"$FLEET_NAME"-cloud-media-"$GROUP_NAME"
    CLOUD_MEDIA_SERVER_INSTANCE_TYPE="c5.large"

    terraform plan -input=false \
        -var="network_name=$NETWORK_NAME" \
        -var="fleet_name=$FLEET_NAME" \
        -var="cloud_media_server_count=$CLOUD_MEDIA_SERVER_COUNT" \
        -var="cloud_media_key_name=$CLOUD_MEDIA_KEY_NAME" \
        -var="cloud_media_server_name=$CLOUD_MEDIA_SERVER_NAME" \
        -var="cloud_media_instance_type=$CLOUD_MEDIA_SERVER_INSTANCE_TYPE"

    terraform apply -input=false -auto-approve \
        -var="network_name=$NETWORK_NAME" \
        -var="fleet_name=$FLEET_NAME" \
        -var="cloud_media_server_count=$CLOUD_MEDIA_SERVER_COUNT" \
        -var="cloud_media_key_name=$CLOUD_MEDIA_KEY_NAME" \
        -var="cloud_media_server_name=$CLOUD_MEDIA_SERVER_NAME" \
        -var="cloud_media_instance_type=$CLOUD_MEDIA_SERVER_INSTANCE_TYPE"
}

server_fleet_destroy() {
    NETWORK_NAME=$1
    FLEET_NAME=$2

    # Copy the terraform scripts
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    cd "$NETWORK_FLEET_DIR/terraform/fleet"
    terraform destroy -auto-approve

    # Use the AWS CLI to delete the key pair.
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    
    # TODO - make the AWS profile configurable?
    # aws --profile JenkinsServerBuilder ec2 delete-key-pair --key-name $KEY_NAME
    aws --region $DEFAULT_AWS_REGION ec2 delete-key-pair --key-name $KEY_NAME
}

provision_servers() {
    NETWORK_NAME=$1
    FLEET_NAME=$2

    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME

    cd $NETWORK_FLEET_DIR/terraform/fleet
    terraform refresh

    API_DNS=`terraform output -json api | jq .\"api_dns\" --raw-output`
    ADMIN_API_DNS=`terraform output -json admin_api | jq .\"admin_api_dns\" --raw-output`
    CLOUD_API_DNS=`terraform output -json cloud_api | jq .\"cloud_api_dns\" --raw-output`
    CLOUD_WEBSOCKET_DNS=`terraform output -json cloud_websocket | jq .\"cloud_websocket_dns\" --raw-output`
    WEBAPPS_DNS=`terraform output -json webapps | jq .\"webapps_dns\" --raw-output`

    USER_NAME='ubuntu'

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # API nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" API server\n"
    API_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_api.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $API_NGINX_CONFIG_FILE $USER_NAME@$API_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$API_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-enabled/default && sudo service nginx restart"

    # Admin API nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" Admin API server\n"
    ADMIN_API_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_admin_api.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $ADMIN_API_NGINX_CONFIG_FILE $USER_NAME@$ADMIN_API_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$ADMIN_API_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-enabled/default && sudo service nginx restart"

    # Cloud API nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" Cloud API server\n"
    CLOUD_API_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_api.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $CLOUD_API_NGINX_CONFIG_FILE $USER_NAME@$CLOUD_API_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$CLOUD_API_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-enabled/default && sudo service nginx restart"

    # Cloud Websocket nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" Cloud Websocket server\n"
    CLOUD_WEBSOCKET_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_websocket_server.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $CLOUD_WEBSOCKET_NGINX_CONFIG_FILE $USER_NAME@$CLOUD_WEBSOCKET_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$CLOUD_WEBSOCKET_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-enabled/default && sudo service nginx restart"
    
    # Web apps nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" Web apps server\n"
    WEBAPPS_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_webapps.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $WEBAPPS_NGINX_CONFIG_FILE $USER_NAME@$WEBAPPS_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$WEBAPPS_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-enabled/default && sudo service nginx restart"
}

dump_dns() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME

    cd $NETWORK_FLEET_DIR/terraform/fleet
    terraform refresh

    API_DNS=`terraform output -json api | jq .\"api_dns\" --raw-output`
    ADMIN_API_DNS=`terraform output -json admin_api | jq .\"admin_api_dns\" --raw-output`
    CLOUD_API_DNS=`terraform output -json cloud_api | jq .\"cloud_api_dns\" --raw-output`
    CLOUD_WEBSOCKET_DNS=`terraform output -json cloud_websocket | jq .\"cloud_websocket_dns\" --raw-output`
    WEBAPPS_DNS=`terraform output -json webapps | jq .\"webapps_dns\" --raw-output`

    printf "API\t"$API_DNS"\n"
    printf "Admin API\t"$ADMIN_API_DNS"\n"
    printf "Cloud API\t"$CLOUD_API_DNS"\n"
    printf "Cloud WS\t"$CLOUD_WEBSOCKET_DNS"\n"
    printf "WebApps\t"$WEBAPP_ARDA_DNS"\n"
}

# Setup DNS on Cloudflare
assign_dns() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    SERVER_FLEET_URL_SUFFIX=$3
    CLOUDFLARE_API_KEY=$4
    CLOUDFLARE_ZONE=$5

    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    
    # Replace the values in the config file with the generated ones.
    NETWORK_FLEET_NAME="$NETWORK_NAME-$FLEET_NAME"
    # Convert to lower case
    URL_PREFIX="${NETWORK_FLEET_NAME,,}"

    cd $NETWORK_FLEET_DIR/terraform/fleet
    terraform refresh

    API_ELB_DNS=`terraform output -json api | jq .\"api_elb_dns\" --raw-output`
    ADMIN_API_ELB_DNS=`terraform output -json admin_api | jq .\"admin_api_elb_dns\" --raw-output`
    CLOUD_API_ELB_DNS=`terraform output -json cloud_api | jq .\"cloud_api_elb_dns\" --raw-output`
    CLOUD_WEBSOCKET_ELB_DNS=`terraform output -json cloud_websocket | jq .\"cloud_websocket_elb_dns\" --raw-output`
    WEBAPPS_ELB_DNS=`terraform output -json webapps | jq .\"webapps_elb_dns\" --raw-output`

    # API
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-api","content":"'$API_ELB_DNS'","ttl":120,"proxied":false}'

    # Admin API
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-admin-api","content":"'$ADMIN_API_ELB_DNS'","ttl":120,"proxied":false}'
    
    # Cloud API
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-cloud-api","content":"'$CLOUD_API_ELB_DNS'","ttl":120,"proxied":false}'

    # Cloud web socket
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-cloud","content":"'$CLOUD_WEBSOCKET_ELB_DNS'","ttl":120,"proxied":false}'

    # Arda webapp - points to webapp server
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-arda","content":"'$WEBAPPS_ELB_DNS'","ttl":120,"proxied":false}'
}

# Run yarn installers. Run this first in the jenkins script.
#
# Throw an error if yarn fails.
verify_cloud_installers() {
    echo "Build Cloud Libs"
    cd $JENKINS_BUILD_PATH/cloud/lib && yarn install
    echo "Build API Root"
    cd $JENKINS_BUILD_PATH/cloud/api && yarn install
    echo "Build Cloud Root"
    cd $JENKINS_BUILD_PATH/cloud/cloud && yarn install
    echo "Building Cloud WS Server"
    cd $JENKINS_BUILD_PATH/cloud/cloud/ws_server && yarn install && yarn build
    cd $JENKINS_BUILD_PATH
}

# This is an internal function that sets up common global variables for building and deploying a nodejs application on 
# an AWS EC2 instance.
#
# Most functions within this file will be called from a Jenkins pipeline, so some variables are loaded via the Jenkins
# bootstrapper (e.g. $BUILD_NUMBER)
#
# Inputs:
# NETWORK_NAME - 'dev', 'test', or 'main'
# FLEET_NAME - the designated fleet name.
# APP_NAME - the application name.
# BUILD_PATH - the build location on the local jenkins folder.
ec2_preamble() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    APP_NAME=$3
    BUILD_PATH=$4
    SERVER_NAME=$5
    DNS_OVERRIDE=$6

    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    SERVER_PUBLIC_DNS=``
    
    if [ -n "$SERVER_NAME" ]; then
        SERVER_PUBLIC_DNS=`cd $NETWORK_FLEET_DIR/terraform/fleet && terraform output -json $SERVER_NAME | jq .\""$SERVER_NAME"_dns\" --raw-output`
    else
        SERVER_PUBLIC_DNS=`cd $NETWORK_FLEET_DIR/terraform/fleet && terraform output -json $APP_NAME | jq .\""$APP_NAME"_dns\" --raw-output`
    fi

    # Check if DNS_OVERRIDE is present
    if [ -n "$DNS_OVERRIDE" ]; then
        SERVER_PUBLIC_DNS=$DNS_OVERRIDE
    fi

    # May use redis-cluster names with different suffixes if we need to deploy a new one on the same fleet.
    if [ "$NETWORK_NAME" = "main" ] && [ "$FLEET_NAME" = "ocean" ];
    then
        DEFAULT_REDIS_CLUSTER_FOLDER="terraform/redis-cluster"
    else
        DEFAULT_REDIS_CLUSTER_FOLDER="terraform/redis-cluster"
    fi
    
    FOLDER_NAME="$NETWORK_NAME"_"$FLEET_NAME"_"$APP_NAME"_"$BUILD_NUMBER"
    USER_NAME='ubuntu'
    KEY_PATH="/home/ubuntu/.keys"
    DEST_PATH="/home/ubuntu/"$FOLDER_NAME""

    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH
    
    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"
}

build_api() {
    ec2_preamble $1 $2 "api" "$JENKINS_BUILD_PATH/cloud/api"

    # Config files to copy
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_api.env"

    # Key files from local jenkins to server
    FIREBASE_JSON_FILE="$NETWORK_FLEET_DIR/keys/firebase_"$NETWORK_NAME"_"$FLEET_NAME".json"
    FIREBASE_ANALYTICS_JSON_FILE="$NETWORK_FLEET_DIR/keys/firebase_"$NETWORK_NAME"_"$FLEET_NAME"_analytics.json"
    ACCESS_TOKEN_PRIVATE_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"
    BRIGHTCOVE_TOKEN_FILE="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_brightcove_token.pem"
    ALLOWED_APPS_DATA_FILE="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json"

    # Get the elasticache address and append to the env file.
    REDIS_CONFIGURATION_ENDPOINT=`cd $NETWORK_FLEET_DIR/$DEFAULT_REDIS_CLUSTER_FOLDER && terraform output -json redis | jq .\"configuration_endpoint_address\" --raw-output`
    FINAL_ENV_FILE="./"$NETWORK_NAME"_"$FLEET_NAME"_api.env"
    cp $ENV_FILE $FINAL_ENV_FILE
    printf "\nREDIS_CONFIGURATION_ENDPOINT='"$REDIS_CONFIGURATION_ENDPOINT"'" >> $FINAL_ENV_FILE

    # .env file goes to the app path
    rsync -avz -e "$SSH_COMMAND" --progress $FINAL_ENV_FILE $RSYNC_DEST/.env

    # Key files go into the home .keys folder.
    rsync -avz -e "$SSH_COMMAND" --progress $ALLOWED_APPS_DATA_FILE $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json
    rsync -avz -e "$SSH_COMMAND" --progress $FIREBASE_JSON_FILE $RSYNC_KEY_DEST/firebase_"$NETWORK_NAME"_"$FLEET_NAME".json
    rsync -avz -e "$SSH_COMMAND" --progress $FIREBASE_ANALYTICS_JSON_FILE $RSYNC_KEY_DEST/firebase_"$NETWORK_NAME"_"$FLEET_NAME"_analytics.json
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PRIVATE_KEY $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY  $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub
    rsync -avz -e "$SSH_COMMAND" --progress $BRIGHTCOVE_TOKEN_FILE  $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_brightcove_token.pem

    # TODO - simply this list of install directives.
    cd $JENKINS_BUILD_PATH/cloud && yarn install
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_api clean && rm -rf ./apps/api/dist/node_modules
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_api build
    cd $JENKINS_BUILD_PATH/cloud && mv ./node_modules ./apps/api/dist
    # if preprocessor folder exists, copy it to the dist folder
    if [ -d "$JENKINS_BUILD_PATH/cloud/api/src/fabricator/preprocessor" ]; then
        cd $JENKINS_BUILD_PATH/cloud && cp -r ./api/src/fabricator/preprocessor ./apps/api/dist/api/src/fabricator/preprocessor
    fi
    cd $JENKINS_BUILD_PATH/cloud/apps/api/dist && tar -czf api.tgz ./apps ./api ./auth ./lib ./node_modules ./*.pem

    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/apps/api/dist/api.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf api.tgz && rm api.tgz"

    # Prepare node_modules - link to the various folders
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && rm -rf lib && rm -rf auth && rm -rf api && rm -rf cloud"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../lib lib"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../auth auth"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../api api"

    # Install python libs
    if [ -d "$JENKINS_BUILD_PATH/cloud/api/src/fabricator/preprocessor" ]; then
        $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/api/src/fabricator/preprocessor && python3 -m venv .venv"
        $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/api/src/fabricator/preprocessor && source .venv/bin/activate && pip install -r requirements.txt && deactivate"
    fi
    
    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop all && pm2 delete all"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_api_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/api/api.js -i max --env production --update-env"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}

build_admin_api() {
    ec2_preamble $1 $2 "admin_api" "$JENKINS_BUILD_PATH/cloud/api" "admin_api" $3

    # Config files to copy
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_admin_api.env"

    # Key files from local jenkins to server
    FIREBASE_JSON_FILE="$NETWORK_FLEET_DIR/keys/firebase_"$NETWORK_NAME"_"$FLEET_NAME".json"
    FIREBASE_ANALYTICS_JSON_FILE="$NETWORK_FLEET_DIR/keys/firebase_"$NETWORK_NAME"_"$FLEET_NAME"_analytics.json"
    ACCESS_TOKEN_PRIVATE_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"
    BRIGHTCOVE_TOKEN_FILE="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_brightcove_token.pem"
    ALLOWED_APPS_DATA_FILE="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json"

    # Get the elasticache address and append to the env file.
    REDIS_CONFIGURATION_ENDPOINT=`cd $NETWORK_FLEET_DIR/$DEFAULT_REDIS_CLUSTER_FOLDER && terraform output -json redis | jq .\"configuration_endpoint_address\" --raw-output`
    FINAL_ENV_FILE="./"$NETWORK_NAME"_"$FLEET_NAME"_admin_api.env"
    cp $ENV_FILE $FINAL_ENV_FILE
    printf "\nREDIS_CONFIGURATION_ENDPOINT='"$REDIS_CONFIGURATION_ENDPOINT"'" >> $FINAL_ENV_FILE

    # .env file goes to the app path
    rsync -avz -e "$SSH_COMMAND" --progress $FINAL_ENV_FILE $RSYNC_DEST/.env

    # Key files go into the home .keys folder.
    rsync -avz -e "$SSH_COMMAND" --progress $ALLOWED_APPS_DATA_FILE $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json
    rsync -avz -e "$SSH_COMMAND" --progress $FIREBASE_JSON_FILE $RSYNC_KEY_DEST/firebase_"$NETWORK_NAME"_"$FLEET_NAME".json
    rsync -avz -e "$SSH_COMMAND" --progress $FIREBASE_ANALYTICS_JSON_FILE $RSYNC_KEY_DEST/firebase_"$NETWORK_NAME"_"$FLEET_NAME"_analytics.json
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PRIVATE_KEY $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY  $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub
    rsync -avz -e "$SSH_COMMAND" --progress $BRIGHTCOVE_TOKEN_FILE  $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_brightcove_token.pem

    cd $JENKINS_BUILD_PATH/cloud && yarn install
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_admin_api clean && rm -rf ./apps/admin_api/dist/node_modules
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_admin_api build
    cd $JENKINS_BUILD_PATH/cloud && mv ./node_modules ./apps/admin_api/dist
    # if preprocessor folder exists, copy it to the dist folder
    if [ -d "$JENKINS_BUILD_PATH/cloud/api/src/fabricator/preprocessor" ]; then
        cd $JENKINS_BUILD_PATH/cloud && cp -r ./api/src/fabricator/preprocessor ./apps/admin_api/dist/api/src/fabricator/preprocessor
    fi
    if [ -d "$JENKINS_BUILD_PATH/cloud/api/src/fabricator/fonts" ]; then
        cd $JENKINS_BUILD_PATH/cloud && cp -r ./api/src/fabricator/fonts ./apps/admin_api/dist/api/src/fabricator/fonts
    fi
    cd $JENKINS_BUILD_PATH/cloud/apps/admin_api/dist && tar -czf admin_api.tgz ./apps ./api ./auth ./lib ./node_modules ./*.pem

    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/apps/admin_api/dist/admin_api.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf admin_api.tgz && rm admin_api.tgz"

    # Prepare node_modules
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && rm -rf lib && rm -rf auth && rm -rf api && rm -rf cloud"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../lib lib"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../auth auth"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../api api"

    # Install python libs
    if [ -d "$JENKINS_BUILD_PATH/cloud/api/src/fabricator/preprocessor" ]; then
        $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/api/src/fabricator/preprocessor && python3 -m venv .venv"
        $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/api/src/fabricator/preprocessor && source .venv/bin/activate && pip install -r requirements.txt && deactivate"
    fi

    # Run db_migrate
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && node apps/admin_api/beyond_db_migrate.js"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && node apps/admin_api/oauth_db_setup.js"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop all && pm2 delete all"
    set -e

    if [ "$NETWORK_NAME" = "main" ];
    then
        $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app1 -- 3001"
        $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app2 -- 3002"
        $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app3 -- 3003"
        $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app4 -- 3004"
        $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app5 -- 3005"
        $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app6 -- 3006"
        $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app7 -- 3007"
        $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app8 -- 3008"
        $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app9 -- 3009"
        $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app10 -- 3010"
        
        # Disable these until we've verified stability of the first 10 instances, to avoid potential issues with all 16 instances going down at once.
        #$SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app11 -- 3011"
        #$SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app12 -- 3012"
        #$SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app13 -- 3013"
        #$SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app14 -- 3014"
        #$SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app15 -- 3015"
        #$SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app16 -- 3016"
    else
        $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app1 -- 3001"
        $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js --env production --name app2 -- 3002"
    fi
    
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 save"
}

# Looper is compiled and deployed to the jenkins server.
build_looper() {
    NETWORK_NAME=$1
    FLEET_NAME=$2

    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"

    cd $JENKINS_BUILD_PATH/cloud && yarn install
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_factory_looper clean && rm -rf ./apps/looper/dist/node_modules
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_factory_looper build

    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_webapp_arda.env"
    cp $ENV_FILE $JENKINS_BUILD_PATH/cloud/.env

    cd $JENKINS_BUILD_PATH/cloud && node ./apps/looper/dist/looper.js --program "test"
}

build_cloud_api() {
    ec2_preamble $1 $2 "cloud_api" "$JENKINS_BUILD_PATH/cloud/cloud"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_api.env"

    # Key files from local jenkins to server
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"
    FIREBASE_READONLY_JSON_FILE="$NETWORK_FLEET_DIR/keys/firebase_readonly_"$NETWORK_NAME"_"$FLEET_NAME".json"
    ALLOWED_APPS_DATA_FILE="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json"

    # Get the elasticache address and append to the env file.
    REDIS_CONFIGURATION_ENDPOINT=`cd $NETWORK_FLEET_DIR/$DEFAULT_REDIS_CLUSTER_FOLDER && terraform output -json redis | jq .\"configuration_endpoint_address\" --raw-output`
    FINAL_ENV_FILE="./"$NETWORK_NAME"_"$FLEET_NAME"_cloud_api.env"
    cp $ENV_FILE $FINAL_ENV_FILE
    printf "\nREDIS_CONFIGURATION_ENDPOINT='"$REDIS_CONFIGURATION_ENDPOINT"'" >> $FINAL_ENV_FILE

    rsync -avz -e "$SSH_COMMAND" --progress $FINAL_ENV_FILE $RSYNC_DEST/.env
    rsync -avz -e "$SSH_COMMAND" --progress $FIREBASE_READONLY_JSON_FILE $RSYNC_KEY_DEST/firebase_readonly_"$NETWORK_NAME"_"$FLEET_NAME".json
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"
    rsync -avz -e "$SSH_COMMAND" --progress $ALLOWED_APPS_DATA_FILE "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json"
    
    cd $JENKINS_BUILD_PATH/cloud && yarn install
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_api clean && rm -rf ./cloud/cloud_api/dist/node_modules
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_api build
    cd $JENKINS_BUILD_PATH/cloud && mv ./node_modules ./cloud/cloud_api/dist
    cd $JENKINS_BUILD_PATH/cloud/cloud/cloud_api/dist && tar -czf cloud_api.tgz ./cloud ./api ./auth ./lib ./node_modules

    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/cloud/cloud_api/dist/cloud_api.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf cloud_api.tgz && rm cloud_api.tgz"

    # Prepare node_modules
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && rm -rf lib && rm -rf auth && rm -rf api && rm -rf cloud"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../lib lib"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../auth auth"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../api api"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../cloud cloud"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop all && pm2 delete all"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_cloud_api_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start cloud/cloud_api/cloud_api.js -i max"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}

build_cloud_api_next() {
    ec2_preamble $1 $2 "cloud_api" "$JENKINS_BUILD_PATH/cloud/cloud"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_api.env"

    # Key files from local jenkins to server
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"
    FIREBASE_READONLY_JSON_FILE="$NETWORK_FLEET_DIR/keys/firebase_readonly_"$NETWORK_NAME"_"$FLEET_NAME".json"
    ALLOWED_APPS_DATA_FILE="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json"

    # Get the elasticache address and append to the env file.
    REDIS_CONFIGURATION_ENDPOINT=`cd $NETWORK_FLEET_DIR/$DEFAULT_REDIS_CLUSTER_FOLDER && terraform output -json redis | jq .\"configuration_endpoint_address\" --raw-output`
    FINAL_ENV_FILE="./"$NETWORK_NAME"_"$FLEET_NAME"_cloud_api.env"
    cp $ENV_FILE $FINAL_ENV_FILE
    printf "\nREDIS_CONFIGURATION_ENDPOINT='"$REDIS_CONFIGURATION_ENDPOINT"'" >> $FINAL_ENV_FILE

    rsync -avz -e "$SSH_COMMAND" --progress $FINAL_ENV_FILE $RSYNC_DEST/.env
    rsync -avz -e "$SSH_COMMAND" --progress $FIREBASE_READONLY_JSON_FILE $RSYNC_KEY_DEST/firebase_readonly_"$NETWORK_NAME"_"$FLEET_NAME".json
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"
    rsync -avz -e "$SSH_COMMAND" --progress $ALLOWED_APPS_DATA_FILE "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json"
    
    cd $JENKINS_BUILD_PATH/cloud && yarn install
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_api clean && rm -rf ./cloud/cloud_api/dist/node_modules
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_api build
    cd $JENKINS_BUILD_PATH/cloud && mv ./node_modules ./cloud/cloud_api/dist
    cd $JENKINS_BUILD_PATH/cloud/cloud/cloud_api/dist && tar -czf cloud_api.tgz ./cloud ./api ./auth ./lib ./node_modules ./*.pem

    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/cloud/cloud_api/dist/cloud_api.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf cloud_api.tgz && rm cloud_api.tgz"

    # Prepare node_modules
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && rm -rf lib && rm -rf auth && rm -rf api && rm -rf cloud"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../lib lib"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../auth auth"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../api api"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../cloud cloud"

    # Run db_migrate
    #$SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && node apps/admin_api/beyond_db_migrate.js"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop all && pm2 delete all"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_cloud_api_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start cloud/cloud_api/cloud_api.js -i max"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}

# Run an cloud setup script on the cloud_api server.
run_cloud_setup() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    SERVER_PUBLIC_DNS=`cd $NETWORK_FLEET_DIR/terraform/fleet && terraform output -json cloud_api | jq .\"cloud_api_dns\" --raw-output`
    
    USER_NAME='ubuntu'
    DEST_PATH="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_setup_$BUILD_NUMBER"
    BUILD_PATH=$JENKINS_BUILD_PATH/cloud/cloud
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Copy config files from local jenkins to server
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_api.env"
    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/.env

    cd $JENKINS_BUILD_PATH/cloud && yarn install
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_setup clean && rm -rf ./cloud/cloud_setup/dist/node_modules
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_setup build
    cd $JENKINS_BUILD_PATH/cloud && mv ./node_modules ./cloud/cloud_setup/dist
    cd $JENKINS_BUILD_PATH/cloud/cloud/cloud_setup/dist && tar -czf cloud_setup.tgz ./cloud ./api ./auth ./lib ./node_modules

    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/cloud/cloud_setup/dist/cloud_setup.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf cloud_setup.tgz && rm cloud_setup.tgz"

    # Prepare node_modules
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && rm -rf lib && rm -rf auth && rm -rf api && rm -rf cloud"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../lib lib"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../auth auth"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../api api"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../cloud cloud"

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && node cloud/cloud_setup/cloud_setup.js"
}

# Run an cloud setup script on the cloud_api server.
run_cloud_setup_next() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    SERVER_PUBLIC_DNS=`cd $NETWORK_FLEET_DIR/terraform/fleet && terraform output -json cloud_api | jq .\"cloud_api_dns\" --raw-output`
    
    USER_NAME='ubuntu'
    DEST_PATH="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_setup_$BUILD_NUMBER"
    BUILD_PATH=$JENKINS_BUILD_PATH/cloud/cloud
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH
    KEY_PATH="/home/ubuntu/.keys"
    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"

    # Copy config files from local jenkins to server
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_api.env"
    
    # Get the elasticache address and append to the env file.
    REDIS_CONFIGURATION_ENDPOINT=`cd $NETWORK_FLEET_DIR/$DEFAULT_REDIS_CLUSTER_FOLDER && terraform output -json redis | jq .\"configuration_endpoint_address\" --raw-output`
    FINAL_ENV_FILE="./"$NETWORK_NAME"_"$FLEET_NAME"_cloud_api.env"
    cp $ENV_FILE $FINAL_ENV_FILE
    printf "\nREDIS_CONFIGURATION_ENDPOINT='"$REDIS_CONFIGURATION_ENDPOINT"'" >> $FINAL_ENV_FILE


    rsync -avz -e "$SSH_COMMAND" --progress $FINAL_ENV_FILE $RSYNC_DEST/.env
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    cd $JENKINS_BUILD_PATH/cloud && yarn install
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_setup clean && rm -rf ./cloud/cloud_setup/dist/node_modules
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_setup build
    cd $JENKINS_BUILD_PATH/cloud && mv ./node_modules ./cloud/cloud_setup/dist
    cd $JENKINS_BUILD_PATH/cloud/cloud/cloud_setup/dist && tar -czf cloud_setup.tgz ./cloud ./api ./auth ./lib ./node_modules ./*.pem

    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/cloud/cloud_setup/dist/cloud_setup.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf cloud_setup.tgz && rm cloud_setup.tgz"

    # Prepare node_modules
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && rm -rf lib && rm -rf auth && rm -rf api && rm -rf cloud"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../lib lib"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../auth auth"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../api api"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../cloud cloud"

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && node cloud/cloud_setup/cloud_setup.js"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && node cloud/cloud_setup/cloud_db_setup.js"
}

build_cloud_websocket() {
    ec2_preamble $1 $2 "cloud_websocket" "$JENKINS_BUILD_PATH/cloud/cloud"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_websocket_server.env"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"
    FIREBASE_READONLY_JSON_FILE="$NETWORK_FLEET_DIR/keys/firebase_readonly_"$NETWORK_NAME"_"$FLEET_NAME".json"
    
    # Get the elasticache address and append to the env file.
    REDIS_CONFIGURATION_ENDPOINT=`cd $NETWORK_FLEET_DIR/$DEFAULT_REDIS_CLUSTER_FOLDER && terraform output -json redis | jq .\"configuration_endpoint_address\" --raw-output`
    FINAL_ENV_FILE="./"$NETWORK_NAME"_"$FLEET_NAME"_cloud_websocket_server.env"
    cp $ENV_FILE $FINAL_ENV_FILE
    printf "\nREDIS_CONFIGURATION_ENDPOINT='"$REDIS_CONFIGURATION_ENDPOINT"'" >> $FINAL_ENV_FILE

    rsync -avz -e "$SSH_COMMAND" --progress $FINAL_ENV_FILE $RSYNC_DEST/.env
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"
    rsync -avz -e "$SSH_COMMAND" --progress $FIREBASE_READONLY_JSON_FILE $RSYNC_KEY_DEST/firebase_readonly_"$NETWORK_NAME"_"$FLEET_NAME".json

    cd $JENKINS_BUILD_PATH/cloud && yarn install
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_ws_server clean && rm -rf ./cloud/ws_server/dist/node_modules
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_ws_server build
    cd $JENKINS_BUILD_PATH/cloud && mv ./node_modules ./cloud/ws_server/dist
    cd $JENKINS_BUILD_PATH/cloud/cloud/ws_server/dist && tar -czf ws_server.tgz ./cloud ./api ./auth ./lib ./node_modules

    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/cloud/ws_server/dist/ws_server.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf ws_server.tgz && rm ws_server.tgz"

    # Prepare node_modules
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && rm -rf lib && rm -rf auth && rm -rf api && rm -rf cloud"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../lib lib"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../auth auth"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../api api"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../cloud cloud"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop all && pm2 delete all"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_cloud_websocket_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start cloud/ws_server/ws_server.js -i max"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}


build_cloud_media_v1() {
    # TODO: replace this stuff with ec2_preamble
    # Difficulty: SERVER_PUBLIC_DNS is inserted by another function running in a loop.
    NETWORK_NAME=$1
    FLEET_NAME=$2
    SERVER_PUBLIC_DNS=$3
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    
    USER_NAME='ubuntu'
    KEY_PATH="/home/ubuntu/.keys"
    DEST_PATH="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_media_server_$BUILD_NUMBER"

    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_media_server.env"

    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/.env

    cd $JENKINS_BUILD_PATH/cloud && yarn install
    cd $JENKINS_BUILD_PATH/cloud/cloud/media_server && yarn add mediasoup@3.10.8
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_media_server clean && rm -rf ./cloud/media_server/dist/node_modules
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_media_server build
    cd $JENKINS_BUILD_PATH/cloud && mv ./node_modules ./cloud/media_server/dist
    cd $JENKINS_BUILD_PATH/cloud/cloud/media_server/dist && tar -czf media_server.tgz ./cloud ./api ./auth ./lib ./node_modules

    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/cloud/media_server/dist/media_server.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf media_server.tgz && rm media_server.tgz"

    # HACK - stop the target process first so CPU doesn't get starved.a
    #
    # TODO - this will stop the build if there's no existing media_server process.
    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop all && pm2 delete all"
    set -e

    # Prepare node_modules
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && rm -rf lib && rm -rf auth && rm -rf api && rm -rf cloud"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../lib lib"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../auth auth"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../api api"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../cloud cloud"


    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_media_server_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start cloud/media_server/media_server.js -i max"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}


build_cloud_media_v2() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    SERVER_PUBLIC_DNS=$3
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    
    USER_NAME='ubuntu'
    KEY_PATH="/home/ubuntu/.keys"
    DEST_PATH="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_media_server_$BUILD_NUMBER"

    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_media_server.env"

    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/.env

    # Copy the git repo files.
    cd $JENKINS_BUILD_PATH/cloud
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/media_server $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/src $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/package.json $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/yarn.lock $RSYNC_DEST/

    # Copy the media soup prebuilt archive
    rsync -avz -e "$SSH_COMMAND" --progress /home/ubuntu/archives/node_modules_mediasoup.ubuntu.3.9.7.tgz $RSYNC_DEST/media_server/mediasoup.tgz

    # HACK - stop the target process first so CPU doesn't get starved.
    #
    # TODO - this will stop the build if there's no existing media_server process.
    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop all && pm2 delete all"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && yarn install"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/media_server && yarn install"

    # Unzip the media soup archive.
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/media_server && tar -xf mediasoup.tgz && rm mediasoup.tgz"

    # Run the typescript builder.
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/media_server && yarn build"

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_media_server_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start media_server/dist/media_server/media_server.js -i max"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}


build_cloud_media_v3() {
    # TODO: replace this stuff with ec2_preamble
    # Difficulty: SERVER_PUBLIC_DNS is inserted by another function running in a loop.
    NETWORK_NAME=$1
    FLEET_NAME=$2
    SERVER_PUBLIC_DNS=$3
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    
    USER_NAME='ubuntu'
    KEY_PATH="/home/ubuntu/.keys"
    DEST_PATH="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_media_server_next_$BUILD_NUMBER"

    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_media_server.env"

    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/.env

    cd $JENKINS_BUILD_PATH/cloud && yarn install
    cd $JENKINS_BUILD_PATH/cloud/cloud/media_server_next && yarn add mediasoup@3.10.8
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_media_server_next clean && rm -rf ./cloud/media_server_next/dist/node_modules
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_media_server_next build
    cd $JENKINS_BUILD_PATH/cloud && mv ./node_modules ./cloud/media_server_next/dist
    cd $JENKINS_BUILD_PATH/cloud/cloud/media_server_next/dist && tar -czf media_server_next.tgz ./cloud ./api ./auth ./lib ./node_modules

    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/cloud/media_server_next/dist/media_server_next.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf media_server_next.tgz && rm media_server_next.tgz"

    # HACK - stop the target process first so CPU doesn't get starved.a
    #
    # TODO - this will stop the build if there's no existing media_server process.
    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop all && pm2 delete all"
    set -e

    # Prepare node_modules
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && rm -rf lib && rm -rf auth && rm -rf api && rm -rf cloud"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../lib lib"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../auth auth"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../api api"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../cloud cloud"

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start cloud/media_server_next/media_server_next.js -i max"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}


restart_server() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    SERVER_PUBLIC_DNS=$3
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    USER_NAME='ubuntu'
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "pm2 restart all"
}


build_media_server_group() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    GROUP_NAME=$3

    # EC2 Instance lookup to get public DNS
    MEDIA_SERVER_DNS_NAMES=( $(aws --region $DEFAULT_AWS_REGION ec2 describe-instances --no-paginate --filters Name=tag:Name,Values=*"$NETWORK_NAME"-"$FLEET_NAME"-cloud-media-"$GROUP_NAME" | jq -r '.Reservations[].Instances[].PublicDnsName' ) )

    for MEDIA_SERVER_DNS_NAME in "${MEDIA_SERVER_DNS_NAMES[@]}"
    do
        :
        build_cloud_media_v1 $NETWORK_NAME $FLEET_NAME $MEDIA_SERVER_DNS_NAME
    done
}


build_media_server_group_v3() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    GROUP_NAME=$3

    # EC2 Instance lookup to get public DNS
    MEDIA_SERVER_DNS_NAMES=( $(aws --region $DEFAULT_AWS_REGION ec2 describe-instances --no-paginate --filters Name=tag:Name,Values=*"$NETWORK_NAME"-"$FLEET_NAME"-cloud-media-"$GROUP_NAME" | jq -r '.Reservations[].Instances[].PublicDnsName' ) )

    for MEDIA_SERVER_DNS_NAME in "${MEDIA_SERVER_DNS_NAMES[@]}"
    do
        :
        build_cloud_media_v3 $NETWORK_NAME $FLEET_NAME $MEDIA_SERVER_DNS_NAME
    done
}


restart_media_server_group() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    GROUP_NAME=$3

    # EC2 Instance lookup to get public DNS
    MEDIA_SERVER_DNS_NAMES=( $(aws --region $DEFAULT_AWS_REGION ec2 describe-instances --no-paginate --filters Name=tag:Name,Values=*"$NETWORK_NAME"-"$FLEET_NAME"-cloud-media-"$GROUP_NAME" | jq -r '.Reservations[].Instances[].PublicDnsName' ) )

    for MEDIA_SERVER_DNS_NAME in "${MEDIA_SERVER_DNS_NAMES[@]}"
    do
        :
        restart_server $NETWORK_NAME $FLEET_NAME $MEDIA_SERVER_DNS_NAME
    done
}


build_webapp_arda() {
    ec2_preamble $1 $2 "webapp_arda" "$JENKINS_BUILD_PATH/cloud/webapps" "webapps" $3

    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/arda/views $RSYNC_DEST/arda
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/arda/arda.js $RSYNC_DEST/arda
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/arda/.babelrc $RSYNC_DEST/arda
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/arda/public $RSYNC_DEST/arda
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/src/server $RSYNC_DEST/src
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/package.json $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/yarn.lock $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/.yarnrc.yml $RSYNC_DEST/

    # Copy config files from local jenkins to server
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_webapp_arda.env"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/arda/.env
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && yarn install"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/arda && pm2 stop arda && pm2 delete arda"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/arda && pm2 start arda.js"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/arda && pm2 cleardump && pm2 save"
}

# Builds the website static files and then syncs to S3.
# The website is first built on the webapps server, then we copy the result back to sync it.
build_website_s3() {
    # If NETWORK_NAME is `main`, and TARGET_BUCKET is set to "live", then we're building into the `LIVE` bucket, otherwise, it's the preview bucket.
    # This value is ignored on other networks.
    TARGET_BUCKET=${3:-DEFAULT}

    # HACK - keep "webapps" flag here even though its irrelevant to the website build, we need it for the final param.
    if [[ -z $4 ]]
    then
        ec2_preamble $1 $2 "website" "$JENKINS_BUILD_PATH/website" "webapps" 
    else
        ec2_preamble $1 $2 "website" "$JENKINS_BUILD_PATH/website" "webapps" $4
    fi

    # Copy config files from local jenkins to server
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_website_s3.env"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/.env

    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/src $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/public $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/package.json $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/yarn.lock $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && yarn install"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && DISABLE_ESLINT_PLUGIN=true SKIP_PREFLIGHT_CHECK=true yarn build"

    # Remove the .map files.
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/build && find ./ -type f -name '*.map' -exec rm {} +"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/build && tar -czf website.tgz ./*"

    set +e
    rsync -avz -e "$SSH_COMMAND" --progress $RSYNC_DEST/build/website.tgz $JENKINS_BUILD_PATH/website.tgz
    mkdir -p $JENKINS_BUILD_PATH/built_website
    cd $JENKINS_BUILD_PATH && tar -xf website.tgz -C ./built_website

    # main network gets some cache controls, while test/dev networks get none.
    if [ "$NETWORK_NAME" = "main" ];
    then
        if [ "$TARGET_BUCKET" == "live" ];
        then
            aws s3 sync $JENKINS_BUILD_PATH/built_website "s3://main-melon-bigscreen-website" --exclude "*" --include "*.html" --cache-control "max-age=60"
            aws s3 sync $JENKINS_BUILD_PATH/built_website "s3://main-melon-bigscreen-website" --exclude "*.html" --include "*" --cache-control "max-age=60"
        else
            aws s3 sync $JENKINS_BUILD_PATH/built_website "s3://main-bigscreen-website-preview" --exclude "*" --include "*.html" --cache-control "no-cache"
            aws s3 sync $JENKINS_BUILD_PATH/built_website "s3://main-bigscreen-website-preview" --exclude "*.html" --include "*" --cache-control "no-cache"
        fi
    else
        aws s3 sync $JENKINS_BUILD_PATH/built_website "s3://"$NETWORK_NAME"-bigscreen-website" --exclude "*" --include "*.html" --cache-control "no-cache"
        aws s3 sync $JENKINS_BUILD_PATH/built_website "s3://"$NETWORK_NAME"-bigscreen-website" --exclude "*.html" --include "*" --cache-control "no-cache"
    fi
    set -e
}

build_cloud_browser_s3() {
    TARGET_FLEET=${2:-DEFAULT}

    # If NETWORK_NAME is `main`, and TARGET_BUCKET is set to "live", then we're building into the `LIVE` bucket, otherwise, it's the preview bucket.
    # This value is ignored on other networks.
    TARGET_BUCKET=${3:-DEFAULT}

    # HACK - keep "webapps" here because even though its irrelevant to the website build.
    if [[ -z $4 ]]
    then
        ec2_preamble $1 $2 "website" "$JENKINS_BUILD_PATH/website" "webapps" 
    else
        ec2_preamble $1 $2 "website" "$JENKINS_BUILD_PATH/website" "webapps" $4
    fi

    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/cloud-browser-static/src $RSYNC_DEST/cloud-browser-static/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/cloud-browser-static/package.json $RSYNC_DEST/cloud-browser-static/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/cloud-browser-static/build.js $RSYNC_DEST/cloud-browser-static/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/cloud-browser-static/yarn.lock $RSYNC_DEST/cloud-browser-static/

    # Copy config files from local jenkins to server
    CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_website.config.js"

    rsync -avz -e "$SSH_COMMAND" --progress $CONFIG_FILE $RSYNC_DEST/cloud-browser-static/src/api/config.js

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/cloud-browser-static && yarn install && yarn build"

    set +e
    mkdir -p $JENKINS_BUILD_PATH/website/cloud-browser-static/dist
    rsync -avz -e "$SSH_COMMAND" --progress $RSYNC_DEST/cloud-browser-static/dist/index.html $JENKINS_BUILD_PATH/website/cloud-browser-static/dist/index.html

    # main network gets some cache controls, while test/dev networks get none.
    if [ "$NETWORK_NAME" = "main" ];
    then
        if [ "$TARGET_BUCKET" == "live" ];
        then
            if [ "$TARGET_FLEET" == "rice" ];
            then
                aws s3 sync $JENKINS_BUILD_PATH/website/cloud-browser-static/dist "s3://bigscreen-cloud-browser-rice" --exclude "*" --include "*.html" --cache-control "max-age=60"
            else
                aws s3 sync $JENKINS_BUILD_PATH/website/cloud-browser-static/dist "s3://bigscreen-cloud-browser" --exclude "*" --include "*.html" --cache-control "max-age=60"
            fi
        else
            if [ "$TARGET_FLEET" == "rice" ];
            then
                aws s3 sync $JENKINS_BUILD_PATH/website/cloud-browser-static/dist "s3://bigscreen-cloud-browser-preview-rice" --exclude "*" --include "*.html" --cache-control "no-cache"
            else
                #aws s3 sync $JENKINS_BUILD_PATH/website/cloud-browser-static/dist "s3://bigscreen-cloud-browser-preview" --exclude "*" --include "*.html" --cache-control "no-cache"
                :
            fi
        fi
    fi
    set -e
}

send_discord_success() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    MESSAGE=$3
    URL=$4

    sh `pwd`/devops/Jenkins/deps/discord.sh \
        --webhook-url="https://discordapp.com/api/webhooks/516791347613532207/YVB-QQcWfwzN5sDunKgNuSmWl3KI1aAEi_QMiX93BMDdTtOmrCxQaiqKGdOHj85g_Yfv" \
        --username "Jenkins" \
        --avatar "https://icon-library.com/images/jenkins-icon/jenkins-icon-15.jpg" \
        --text "$NETWORK_NAME-$FLEET_NAME build succeeded" \
        --title "Build succeeded" \
        --color "3066993" \
        --url "$URL" \
        --description "$MESSAGE" \
        --timestamp 
}

# Deploys a number of pre-built images running the user simulator which start stress testing the target server fleet.
#
# Use higher server counts for more load.
begin_load_test() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    SERVER_COUNT=$3

    if [ "$SERVER_COUNT" -lt 1 ]; then
        echo "SERVER_COUNT must be beween 1 and 100"
        exit 1
    fi

    if [ "$SERVER_COUNT" -gt 100 ]; then
        echo "SERVER_COUNT must be beween 1 and 100"
        exit 1
    fi

    # Copy the terraform scripts
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    cp -r $JENKINS_BUILD_PATH/devops/terraform/UserSimulatorLoadTest "$NETWORK_FLEET_DIR/terraform/UserSimulatorLoadTest"
    cd "$NETWORK_FLEET_DIR/terraform/UserSimulatorLoadTest"
    terraform init

    SERVER_INSTANCE_TYPE="t3.xlarge"

    terraform plan -input=false \
        -var="server_count=$SERVER_COUNT" \
        -var="key_name=UserSimulatorLoadTest" \
        -var="instance_type=$SERVER_INSTANCE_TYPE"

    terraform apply -input=false -auto-approve \
        -var="server_count=$SERVER_COUNT" \
        -var="key_name=UserSimulatorLoadTest" \
        -var="instance_type=$SERVER_INSTANCE_TYPE"
}

end_load_test() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    cd "$NETWORK_FLEET_DIR/terraform/UserSimulatorLoadTest"
    terraform destroy -auto-approve
    rm -rf "$NETWORK_FLEET_DIR/terraform/UserSimulatorLoadTest"
}

update_fusion_server() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
}


#
# Digital Ocean Stuff
#

build_media_server_image_if_needed() {
    NETWORK_NAME=$1
    FLEET_NAME=$2

    MEDIA_SERVER_ARCHIVE="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_media_server_next.tgz"

    if [ ! -f "$MEDIA_SERVER_ARCHIVE" ]; then
        cd $JENKINS_BUILD_PATH/cloud && yarn install
        cd $JENKINS_BUILD_PATH/cloud/cloud/media_server_next && yarn add mediasoup@3.10.8
        cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_media_server_next clean && rm -rf ./cloud/media_server_next/dist/node_modules
        cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_media_server_next build
        cd $JENKINS_BUILD_PATH/cloud && mv ./node_modules ./cloud/media_server_next/dist
        cd $JENKINS_BUILD_PATH/cloud/cloud/media_server_next/dist && tar -czf media_server_next.tgz ./cloud ./api ./auth ./lib ./node_modules
        cp $JENKINS_BUILD_PATH/cloud/cloud/media_server_next/dist/media_server_next.tgz $MEDIA_SERVER_ARCHIVE
    fi
}

build_cloud_media_digitalocean() {
    # TODO: replace this stuff with ec2_preamble
    # Difficulty: SERVER_PUBLIC_DNS is inserted by another function running in a loop.
    NETWORK_NAME=$1
    FLEET_NAME=$2
    SERVER_PUBLIC_DNS=$3
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    
    USER_NAME='root'
    KEY_PATH="/home/root/.keys"
    DEST_PATH="/home/root/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_media_server_$BUILD_NUMBER"

    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH

    DIGITAL_OCEAN_KEY_NAME="$NETWORK_NAME-$FLEET_NAME-media-server"
    # Local key files.
    LOCAL_SSH_KEY_FILE="$NETWORK_FLEET_DIR/digitalocean/$DIGITAL_OCEAN_KEY_NAME"
    SSH_COMMAND="ssh -i $LOCAL_SSH_KEY_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_media_server_digitalocean.env"

    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/.env

    MEDIA_SERVER_ARCHIVE="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_media_server_next.tgz"

    if [ ! -f "$MEDIA_SERVER_ARCHIVE" ]; then
        # Throw an error!
        echo "Media server archive not found!"
        exit 1
    fi

    rsync -avz -e "$SSH_COMMAND" --progress $MEDIA_SERVER_ARCHIVE $RSYNC_DEST/media_server_next.tgz

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf media_server_next.tgz && rm media_server_next.tgz"

    # HACK - stop the target process first so CPU doesn't get starved.a
    #
    # TODO - this will stop the build if there's no existing media_server process.
    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop all && pm2 delete all"
    set -e

    # Prepare node_modules
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && rm -rf lib && rm -rf auth && rm -rf api && rm -rf cloud"
    
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../lib lib"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../auth auth"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../api api"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../cloud cloud"

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start cloud/media_server_next/media_server_next.js -i 4"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}

deploy_media_server_digitalocean() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    GROUP_NAME=$3
    SERVER_ID=$4

    # Copy the terraform scripts
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    cp -r $JENKINS_BUILD_PATH/devops/terraform/v6/media-server-digitalocean "$NETWORK_FLEET_DIR/terraform/$GROUP_NAME-media-server-digitalocean-$SERVER_ID"
    cd "$NETWORK_FLEET_DIR/terraform/$GROUP_NAME-media-server-digitalocean-$SERVER_ID"
    terraform init

    CLOUD_MEDIA_KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    CLOUD_MEDIA_SERVER_NAME="$NETWORK_NAME"-"$FLEET_NAME"-media-server-digitalocean-"$GROUP_NAME"

    DIGITAL_OCEAN_REGION="sfo3"
    DIGITAL_OCEAN_TOKEN=$(cat $NETWORK_FLEET_DIR/digitalocean/$NETWORK_NAME"-"$FLEET_NAME.pat)
    DIGITAL_OCEAN_KEY_NAME="$NETWORK_NAME-$FLEET_NAME-media-server"
    DIGITAL_OCEAN_SSH_KEY_FILE_PATH="$NETWORK_FLEET_DIR/digitalocean/$DIGITAL_OCEAN_KEY_NAME"

    terraform plan -input=false \
        -var="network_name=$NETWORK_NAME" \
        -var="fleet_name=$FLEET_NAME" \
        -var="id=$SERVER_ID" \
        -var="server_name=$CLOUD_MEDIA_SERVER_NAME" \
        -var="digitalocean_region=$DIGITAL_OCEAN_REGION" \
        -var="digitalocean_token=$DIGITAL_OCEAN_TOKEN" \
        -var="digitalocean_ssh_key_name=$DIGITAL_OCEAN_KEY_NAME" \
        -var="digitalocean_ssh_private_key_file=$DIGITAL_OCEAN_SSH_KEY_FILE_PATH"

    terraform apply -input=false -auto-approve \
        -var="network_name=$NETWORK_NAME" \
        -var="fleet_name=$FLEET_NAME" \
        -var="id=$SERVER_ID" \
        -var="server_name=$CLOUD_MEDIA_SERVER_NAME" \
        -var="digitalocean_region=$DIGITAL_OCEAN_REGION" \
        -var="digitalocean_token=$DIGITAL_OCEAN_TOKEN" \
        -var="digitalocean_ssh_key_name=$DIGITAL_OCEAN_KEY_NAME" \
        -var="digitalocean_ssh_private_key_file=$DIGITAL_OCEAN_SSH_KEY_FILE_PATH"

    # Sleep for 30 seconds to wait for the server to come up.
    sleep 60

    # SSH into the new server and prepare it for things.
    FULL_SERVER_NAME=$CLOUD_MEDIA_SERVER_NAME""$SERVER_ID
    MEDIA_SERVER_DNS_NAME=( $(curl -s -X GET "https://api.digitalocean.com/v2/droplets?name=$FULL_SERVER_NAME" -H "Authorization: Bearer $DIGITAL_OCEAN_TOKEN" | jq -r ".droplets[].networks.v4[] | select(.type == \"public\") | .ip_address") )

    if [ ${#MEDIA_SERVER_DNS_NAME[@]} -ne 1 ]; then
        echo "Error: Number of items returned is not 1"
        exit 1
    fi

    SERVER_PUBLIC_DNS=${MEDIA_SERVER_DNS_NAME[0]}
    LOCAL_SSH_KEY_FILE="$NETWORK_FLEET_DIR/digitalocean/$DIGITAL_OCEAN_KEY_NAME"
    SSH_COMMAND="ssh -i $LOCAL_SSH_KEY_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
    USER_NAME='root'

    # HACK HACK HACK
    #
    # A lot of the weird shit in here (e.g. the sleeps) is because of Digital Ocean's unstable default Ubuntu distro for 20.04.
    # Specifically, apt-get would acquire a lock and wouldn't release it for, seemingly, a second. If another apt command was
    # executed it would error out because the lock was unavailable.
    #
    # After switching to 22.04 the problems have gone away.

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "sysctl -p"
    set -e
        
    # Sleep for 5 seconds
    sleep 5

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "apt-get update"
    set -e

    # Sleep for 5 seconds
    sleep 5

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "echo 'root hard nofile 1048576' | sudo tee -a /etc/security/limits.conf"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "echo 'root soft nofile 1048576' | sudo tee -a /etc/security/limits.conf"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "curl -sL https://deb.nodesource.com/setup_18.x -o nodesource_setup.sh"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "bash nodesource_setup.sh"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "apt-get install -y gcc g++ make build-essential"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "apt-get install -y nodejs"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "npm install pm2@latest -g"
    set -e

    # Sleep for 5 seconds
    sleep 5

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "pm2 startup"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "sysctl -p"
    set -e

    # Sleep for 5 seconds
    sleep 5

    build_cloud_media_digitalocean $NETWORK_NAME $FLEET_NAME $SERVER_PUBLIC_DNS
}

deploy_media_servers_digitalocean() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    GROUP_NAME=$3
    CLOUD_MEDIA_SERVER_COUNT=$4

    if [ "$CLOUD_MEDIA_SERVER_COUNT" -lt 1 ]; then
        echo "CLOUD_MEDIA_SERVER_COUNT must be beween 1 and 15"
        exit 1
    fi

    if [ "$CLOUD_MEDIA_SERVER_COUNT" -gt 15 ]; then
        echo "CLOUD_MEDIA_SERVER_COUNT must be beween 1 and 15"
        exit 1
    fi

    # As far as I can tell, the digital ocean terraform provider doesn't support creating multiple instances in one go.
    for i in $(seq 1 $CLOUD_MEDIA_SERVER_COUNT)
    do
        :
        deploy_media_server_digitalocean $NETWORK_NAME $FLEET_NAME $GROUP_NAME $i
    done
}


build_media_server_group_digitalocean() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    GROUP_NAME=$3

    SERVER_NAME=$NETWORK_NAME"-"$FLEET_NAME"-media-server-"$GROUP_NAME
    TAG_NAME=$NETWORK_NAME"-"$FLEET_NAME"-media-server-digitalocean-"$GROUP_NAME
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"

    DIGITAL_OCEAN_TOKEN=$(cat $NETWORK_FLEET_DIR/digitalocean/$NETWORK_NAME"-"$FLEET_NAME.pat)
    DIGITAL_OCEAN_KEY_NAME="$NETWORK_NAME-$FLEET_NAME-media-server"

    # Get the list of IP addresses for the media servers.
    MEDIA_SERVER_DNS_NAMES=( $(curl -s -X GET "https://api.digitalocean.com/v2/droplets?tag_name=$TAG_NAME" -H "Authorization: Bearer $DIGITAL_OCEAN_TOKEN" | jq -r ".droplets[].networks.v4[] | select(.type == \"public\") | .ip_address") )

    for MEDIA_SERVER_DNS_NAME in "${MEDIA_SERVER_DNS_NAMES[@]}"
    do
        :
        build_cloud_media_digitalocean $NETWORK_NAME $FLEET_NAME $MEDIA_SERVER_DNS_NAME
    done
}