#!/bin/bash

# Stop and report the error if ANYTHING breaks.
set -e

#
# This script contains various functions for setting up and provisioning a Bigscreen Arch5 server fleet from Jenkins. 
# See the pipeline files in Jenkins/pipelines for more info on how these functions are used.
#
# Usage (in a Jenkins shell command):
#    #!/bin/bash
#    . /path/to/this_file.sh
#    init_server_fleet_config_files "test" "v2"
#

jq-replace() {
    SRC=$1
    KEY=$2
    VAL=$3

    tmp=$(mktemp)
    jq ".$KEY = \"$VAL\"" "$SRC" > "$tmp" && mv "$tmp" "$SRC"
}

HOME_FOLDER="/home/ubuntu/networks"
JENKINS_BUILD_PATH=`pwd`
DEFAULT_AWS_REGION="us-west-2"

create_server_fleet_iam_users() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    
    TMP_OUTPUT_FILE=$(mktemp)

    # Create an IAM for this server fleet using the templates
    #
    # 1. Does the user already exist for this fleet?
    IAM_USER_NAME="$NETWORK_NAME-$FLEET_NAME-cloud-user"
    set +e
    aws iam get-user --user-name $IAM_USER_NAME
    IAM_USER_RESULT=$?
    set -e

    if [ $RESULT -eq 0 ]
    then
        echo "IAM user already exists."
    else
        echo "IAM user doesn't exist...creating key..."
        aws iam create-user --user-name test-violin-dynamo-db --permissions-boundary arn:aws:iam::164780942622:policy/test-dynamo-db
        aws iam create-access-key --user-name test-violin-dynamo-db
    fi
}

# Generate the config files for all the server and applications in the fleet.
#
# Requires the jenkins process to have an AWS credentials file at:
# /var/lib/jenkins/.aws
#
# The default IAM must have the ability to create entities on EC2.
init_server_fleet_config_files() {
    NETWORK_NAME=$1
    FLEET_NAME=$2

    if [[ -z $3 ]]
    then
        REGION=$DEFAULT_AWS_REGION
    else
        REGION=$3
    fi

    # TODO: if we've already built the servers via terraform, then bail out.
    # if there was an error, either rollback or create another fleet.

    # 1. Make the destination folders for configurating the server fleet
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    mkdir -p "$NETWORK_FLEET_DIR"
    mkdir -p "$NETWORK_FLEET_DIR/aws"
    mkdir -p "$NETWORK_FLEET_DIR/config"
    mkdir -p "$NETWORK_FLEET_DIR/keys"
    mkdir -p "$NETWORK_FLEET_DIR/terraform"

    # 2. Use the AWS CLI to create a pem file.
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    TMP_OUTPUT_FILE=$(mktemp)

    # TODO - make the AWS profile configurable?
    # aws --region $DEFAULT_AWS_REGION --profile JenkinsServerBuilder ec2 create-key-pair --key-name $KEY_NAME > "$TMP_OUTPUT_FILE"
    set +e
    aws --region $DEFAULT_AWS_REGION ec2 describe-key-pairs --key-name $KEY_NAME
    RESULT=$?
    set -e

    if [ $RESULT -eq 0 ]
    then
        echo "Key already exists."
    else
        echo "Key doesn't exist...creating key..."
        aws --region $DEFAULT_AWS_REGION ec2 create-key-pair --key-name $KEY_NAME > "$TMP_OUTPUT_FILE"
        jq --raw-output ".KeyMaterial" "$TMP_OUTPUT_FILE" > "$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
        rm "$TMP_OUTPUT_FILE"
        chmod 400 "$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    fi

    # 3. Create a base config from the template and copy it to the config folder.
    FLEET_CONFIG_FILE="$NETWORK_FLEET_DIR/config/config.json"
    cp "$HOME_FOLDER/$NETWORK_NAME/.config/"$NETWORK_NAME"_config.json" "$FLEET_CONFIG_FILE"

    # 4. Replace the values in the config file with the generated ones.
    NETWORK_FLEET_NAME="$NETWORK_NAME-$FLEET_NAME"
    # Convert to lower case
    URL_PREFIX="${NETWORK_FLEET_NAME,,}"

    jq-replace "$FLEET_CONFIG_FILE" "NETWORK_NAME" "$NETWORK_NAME"
    jq-replace "$FLEET_CONFIG_FILE" "SERVER_FLEET_NAME" $NETWORK_NAME"_"$FLEET_NAME
    jq-replace "$FLEET_CONFIG_FILE" "PUBLIC_EMAIL_URL" "https://$URL_PREFIX-website.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "WEBAPP_ADMIN_HOST" "https://$URL_PREFIX-admin.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "WEBAPP_FABRICATOR_HOST"  "https://$URL_PREFIX-fabricator.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "WEBAPP_MODS_HOST" "https://$URL_PREFIX-mods.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "WEBSITE_HOST"  "https://$URL_PREFIX-website.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "BIGSCREEN_API_URL"  "https://$URL_PREFIX-api.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "BIGSCREEN_ADMIN_API_URL" "https://$URL_PREFIX-admin-api.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "BIGSCREEN_CLOUD_API_URL" "https://$URL_PREFIX-cloud-api.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "CLOUD_WEBSOCKET_SERVER_NGINX_SERVER_NAME" "$URL_PREFIX-cloud.bigscreencloud.com"
    
    # 5. copy the network firebase config files from the common key folder
    cp "$HOME_FOLDER/$NETWORK_NAME/.keys/firebase_"$NETWORK_NAME".json" "$NETWORK_FLEET_DIR/keys/firebase_"$NETWORK_NAME"_"$FLEET_NAME".json"
    cp "$HOME_FOLDER/$NETWORK_NAME/.keys/firebase_"$NETWORK_NAME"_analytics.json" "$NETWORK_FLEET_DIR/keys/firebase_"$NETWORK_NAME"_"$FLEET_NAME"_analytics.json"

    # 6. access token private key
    cp "$HOME_FOLDER/$NETWORK_NAME/.keys/"$NETWORK_NAME"_access_token.key" "$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key"

    # 7. access token public key
    cp "$HOME_FOLDER/$NETWORK_NAME/.keys/"$NETWORK_NAME"_access_token.key.pub" "$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    # 8. brightcove token pem
    cp "$HOME_FOLDER/$NETWORK_NAME/.keys/"$NETWORK_NAME"_brightcove_token.pem" "$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_brightcove_token.pem"

    # 9. Run the generate script to generate the remainder of the configuration files.
    cd $JENKINS_BUILD_PATH/devops/ServerEnvironmentBuilder 
    npm install
    node app_v3.js \
        --default-config-file "$HOME_FOLDER/$NETWORK_NAME/.config/default_config.json" \
        --fleet-config-file "$FLEET_CONFIG_FILE" \
        --output-folder-path "$NETWORK_FLEET_DIR/config"
    cd $JENKINS_BUILD_PATH
}

# Requires the jenkins process to have an AWS credentials file at:
# /var/lib/jenkins/.aws
#
# The default IAM must have the ability to create entities on EC2.
terraform_deploy_fleet() {
    NETWORK_NAME=$1
    FLEET_NAME=$2

    if [[ -z $3 ]]
    then
        REGION=$DEFAULT_AWS_REGION
    else
        REGION=$3
    fi

    # Copy the terraform scripts
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    cp -r $JENKINS_BUILD_PATH/devops/terraform/all "$NETWORK_FLEET_DIR/terraform"
    cd "$NETWORK_FLEET_DIR/terraform/all"
    terraform init

    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    AWS_FRIENDLY_NAME=$NETWORK_NAME"-"$FLEET_NAME
    ELB_CERTIFICATE_ARN="arn:aws:acm:us-west-2:164780942622:certificate/61f977f7-27ff-4538-861c-dfb6535a8f80"
    API_SERVER_INSTANCE_TYPE="c5.2xlarge"
    CLOUD_SERVER_INSTANCE_TYPE="c5.2xlarge"
    WEBSITE_INSTANCE_TYPE="c5.large"
    WEBAPP_INSTANCE_TYPE="c5.large"

    # Run terraform apply
    if [ "$NETWORK_NAME" = "main" ];
    then
        CLOUD_MEDIA_SERVER_COUNT=4
        ELB_CERTIFICATE_ARN="arn:aws:acm:us-west-2:164780942622:certificate/61f977f7-27ff-4538-861c-dfb6535a8f80"
        API_SERVER_INSTANCE_TYPE="c5.4xlarge"
        CLOUD_SERVER_INSTANCE_TYPE="c5.4xlarge"
        WEBSITE_INSTANCE_TYPE="c5.xlarge"
        WEBAPP_INSTANCE_TYPE="c5.large"
    fi

    terraform plan -input=false \
        -var="stage=$AWS_FRIENDLY_NAME" \
        -var="api_key_name=$KEY_NAME" \
        -var="api_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="api_instance_type=$API_SERVER_INSTANCE_TYPE" \
        -var="admin_api_key_name=$KEY_NAME" \
        -var="admin_api_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="admin_api_instance_type=$API_SERVER_INSTANCE_TYPE" \
        -var="cloud_worker_key_name=$KEY_NAME" \
        -var="cloud_worker_instance_type=$CLOUD_SERVER_INSTANCE_TYPE" \
        -var="cloud_websocket_key_name=$KEY_NAME" \
        -var="cloud_websocket_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="cloud_websocket_instance_type=$CLOUD_SERVER_INSTANCE_TYPE" \
        -var="cloud_api_key_name=$KEY_NAME" \
        -var="cloud_api_instance_type=$CLOUD_SERVER_INSTANCE_TYPE" \
        -var="cloud_api_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="webapp_admin_key_name=$KEY_NAME" \
        -var="webapp_admin_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="webapp_admin_instance_type=$WEBAPP_INSTANCE_TYPE" \
        -var="webapp_fabricator_key_name=$KEY_NAME" \
        -var="webapp_fabricator_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="webapp_fabricator_instance_type=$WEBAPP_INSTANCE_TYPE" \
        -var="webapp_mods_key_name=$KEY_NAME" \
        -var="webapp_mods_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="webapp_mods_instance_type=$WEBAPP_INSTANCE_TYPE" \
        -var="website_key_name=$KEY_NAME" \
        -var="website_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="website_preview_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="website_instance_type=$WEBSITE_INSTANCE_TYPE"

    terraform apply -input=false -auto-approve \
        -var="stage=$AWS_FRIENDLY_NAME" \
        -var="api_key_name=$KEY_NAME" \
        -var="api_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="api_instance_type=$API_SERVER_INSTANCE_TYPE" \
        -var="admin_api_key_name=$KEY_NAME" \
        -var="admin_api_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="admin_api_instance_type=$API_SERVER_INSTANCE_TYPE" \
        -var="cloud_worker_key_name=$KEY_NAME" \
        -var="cloud_worker_instance_type=$CLOUD_SERVER_INSTANCE_TYPE" \
        -var="cloud_websocket_key_name=$KEY_NAME" \
        -var="cloud_websocket_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="cloud_websocket_instance_type=$CLOUD_SERVER_INSTANCE_TYPE" \
        -var="cloud_api_key_name=$KEY_NAME" \
        -var="cloud_api_instance_type=$CLOUD_SERVER_INSTANCE_TYPE" \
        -var="cloud_api_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="webapp_admin_key_name=$KEY_NAME" \
        -var="webapp_admin_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="webapp_admin_instance_type=$WEBAPP_INSTANCE_TYPE" \
        -var="webapp_fabricator_key_name=$KEY_NAME" \
        -var="webapp_fabricator_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="webapp_fabricator_instance_type=$WEBAPP_INSTANCE_TYPE" \
        -var="webapp_mods_key_name=$KEY_NAME" \
        -var="webapp_mods_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="webapp_mods_instance_type=$WEBAPP_INSTANCE_TYPE" \
        -var="website_key_name=$KEY_NAME" \
        -var="website_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="website_preview_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="website_instance_type=$WEBSITE_INSTANCE_TYPE"
}

terraform_deploy_redis_cluster() {
    NETWORK_NAME=$1
    FLEET_NAME=$2

    if [[ -z $3 ]]
    then
        REGION=$DEFAULT_AWS_REGION
    else
        REGION=$3
    fi

    # Copy the terraform scripts
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"

    # Get the security group ids from cloud_worker and cloud_api so they can access this redis cluster
    cd $NETWORK_FLEET_DIR/terraform/all
    terraform refresh
    API_SECURITY_GROUP_ID=`terraform output -json api | jq .\"security_group_id\" --raw-output`
    ADMIN_API_SECURITY_GROUP_ID=`terraform output -json admin_api | jq .\"security_group_id\" --raw-output`
    CLOUD_API_SECURITY_GROUP_ID=`terraform output -json cloud_api | jq .\"security_group_id\" --raw-output`
    CLOUD_WEBSOCKET_SECURITY_GROUP_ID=`terraform output -json cloud_websocket | jq .\"security_group_id\" --raw-output`
    CLOUD_WORKER_SECURITY_GROUP_ID=`terraform output -json cloud_worker | jq .\"security_group_id\" --raw-output`

    # copy and init terraform for the redis-cluster module.
    cp -r $JENKINS_BUILD_PATH/devops/terraform/redis-cluster "$NETWORK_FLEET_DIR/terraform"
    cd "$NETWORK_FLEET_DIR/terraform/redis-cluster"
    terraform init

    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    AWS_FRIENDLY_NAME=$NETWORK_NAME"-"$FLEET_NAME
    
    # Redis cluster node must be a cache.m* node type.
    REDIS_NODE_TYPE="cache.m5.large"
    REPLICAS_PER_NODE_GROUP=1
    NUM_NODE_GROUPS=1

    # Run terraform apply
    if [ "$NETWORK_NAME" = "main" ];
    then
        REPLICAS_PER_NODE_GROUP=3
        NUM_NODE_GROUPS=2
        REDIS_NODE_TYPE="cache.m5.4xlarge"
    fi

    terraform plan -input=false \
        -var="stage=$AWS_FRIENDLY_NAME" \
        -var="replicas_per_node_group=$REPLICAS_PER_NODE_GROUP" \
        -var="num_node_groups=$NUM_NODE_GROUPS" \
        -var="redis_stage=$AWS_FRIENDLY_NAME" \
        -var="api_security_group_id=$API_SECURITY_GROUP_ID" \
        -var="admin_api_security_group_id=$ADMIN_API_SECURITY_GROUP_ID" \
        -var="cloud_api_security_group_id=$CLOUD_API_SECURITY_GROUP_ID" \
        -var="cloud_websocket_security_group_id=$CLOUD_WEBSOCKET_SECURITY_GROUP_ID" \
        -var="cloud_worker_security_group_id=$CLOUD_WORKER_SECURITY_GROUP_ID" \
        -var="redis_node_type=$REDIS_NODE_TYPE"

    terraform apply -input=false -auto-approve \
        -var="stage=$AWS_FRIENDLY_NAME" \
        -var="replicas_per_node_group=$REPLICAS_PER_NODE_GROUP" \
        -var="num_node_groups=$NUM_NODE_GROUPS" \
        -var="redis_stage=$AWS_FRIENDLY_NAME" \
        -var="api_security_group_id=$API_SECURITY_GROUP_ID" \
        -var="admin_api_security_group_id=$ADMIN_API_SECURITY_GROUP_ID" \
        -var="cloud_api_security_group_id=$CLOUD_API_SECURITY_GROUP_ID" \
        -var="cloud_websocket_security_group_id=$CLOUD_WEBSOCKET_SECURITY_GROUP_ID" \
        -var="cloud_worker_security_group_id=$CLOUD_WORKER_SECURITY_GROUP_ID" \
        -var="redis_node_type=$REDIS_NODE_TYPE"
}

deploy_media_server_group() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    GROUP_NAME=$3
    CLOUD_MEDIA_SERVER_COUNT=$4

    if [ "$CLOUD_MEDIA_SERVER_COUNT" -lt 1 ]; then
        echo "CLOUD_MEDIA_SERVER_COUNT must be beween 1 and 15"
        exit 1
    fi

    if [ "$CLOUD_MEDIA_SERVER_COUNT" -gt 15 ]; then
        echo "CLOUD_MEDIA_SERVER_COUNT must be beween 1 and 15"
        exit 1
    fi

    # Copy the terraform scripts
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    cp -r $JENKINS_BUILD_PATH/devops/terraform/media-only "$NETWORK_FLEET_DIR/terraform/$GROUP_NAME-media-only"
    cd "$NETWORK_FLEET_DIR/terraform/$GROUP_NAME-media-only"
    terraform init

    NETWORK_FLEET_NAME=$NETWORK_NAME"-"$FLEET_NAME
    CLOUD_MEDIA_KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    CLOUD_MEDIA_SERVER_NAME="$NETWORK_NAME"-"$FLEET_NAME"-cloud-media-"$GROUP_NAME"
    CLOUD_MEDIA_SERVER_INSTANCE_TYPE="c5.large"

    terraform plan -input=false \
        -var="network_fleet=$NETWORK_FLEET_NAME" \
        -var="cloud_media_server_count=$CLOUD_MEDIA_SERVER_COUNT" \
        -var="cloud_media_key_name=$CLOUD_MEDIA_KEY_NAME" \
        -var="cloud_media_server_name=$CLOUD_MEDIA_SERVER_NAME" \
        -var="cloud_media_instance_type=$CLOUD_MEDIA_SERVER_INSTANCE_TYPE"

    terraform apply -input=false -auto-approve \
        -var="network_fleet=$NETWORK_FLEET_NAME" \
        -var="cloud_media_server_count=$CLOUD_MEDIA_SERVER_COUNT" \
        -var="cloud_media_key_name=$CLOUD_MEDIA_KEY_NAME" \
        -var="cloud_media_server_name=$CLOUD_MEDIA_SERVER_NAME" \
        -var="cloud_media_instance_type=$CLOUD_MEDIA_SERVER_INSTANCE_TYPE"
}

server_fleet_destroy() {
    NETWORK_NAME=$1
    FLEET_NAME=$2

    # Copy the terraform scripts
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    cd "$NETWORK_FLEET_DIR/terraform/all"
    terraform destroy -auto-approve

    # Use the AWS CLI to delete the key pair.
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    
    # TODO - make the AWS profile configurable?
    # aws --profile JenkinsServerBuilder ec2 delete-key-pair --key-name $KEY_NAME
    aws --region $DEFAULT_AWS_REGION ec2 delete-key-pair --key-name $KEY_NAME
}

provision_servers() {
    NETWORK_NAME=$1
    FLEET_NAME=$2

    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME

    cd $NETWORK_FLEET_DIR/terraform/all
    terraform refresh

    API_DNS=`terraform output -json api | jq .\"api_dns\" --raw-output`
    ADMIN_API_DNS=`terraform output -json admin_api | jq .\"admin_api_dns\" --raw-output`
    CLOUD_API_DNS=`terraform output -json cloud_api | jq .\"cloud_api_dns\" --raw-output`
    CLOUD_WEBSOCKET_DNS=`terraform output -json cloud_websocket | jq .\"cloud_websocket_dns\" --raw-output`
    WEBAPP_ADMIN_DNS=`terraform output -json webapp_admin | jq .\"webapp_admin_dns\" --raw-output`
    WEBAPP_FABRICATOR_DNS=`terraform output -json webapp_fabricator | jq .\"webapp_fabricator_dns\" --raw-output`
    WEBAPP_MODS_DNS=`terraform output -json webapp_mods | jq .\"webapp_mods_dns\" --raw-output`
    WEBSITE_DNS=`terraform output -json website | jq .\"website_dns\" --raw-output`

    USER_NAME='ubuntu'

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # API nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" API server\n"
    API_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_api.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $API_NGINX_CONFIG_FILE $USER_NAME@$API_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$API_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-enabled/default && sudo service nginx restart"

    # Admin API nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" Admin API server\n"
    ADMIN_API_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_admin_api.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $ADMIN_API_NGINX_CONFIG_FILE $USER_NAME@$ADMIN_API_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$ADMIN_API_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-enabled/default && sudo service nginx restart"

    # Cloud API nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" Cloud API server\n"
    CLOUD_API_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_api_server.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $CLOUD_API_NGINX_CONFIG_FILE $USER_NAME@$CLOUD_API_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$CLOUD_API_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-enabled/default && sudo service nginx restart"

    # Cloud Websocket nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" Cloud Websocket server\n"
    CLOUD_WEBSOCKET_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_websocket_server.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $CLOUD_WEBSOCKET_NGINX_CONFIG_FILE $USER_NAME@$CLOUD_WEBSOCKET_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$CLOUD_WEBSOCKET_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-enabled/default && sudo service nginx restart"
    
    # Admin web app nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" Admin web app server\n"
    WEBAPP_ADMIN_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_webapp_admin.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $WEBAPP_ADMIN_NGINX_CONFIG_FILE $USER_NAME@$WEBAPP_ADMIN_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$WEBAPP_ADMIN_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-enabled/default && sudo service nginx restart"
    
    # Fabricator web app nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" Fabricator web app server\n"
    WEBAPP_FABRICATOR_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_webapp_fabricator.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $WEBAPP_FABRICATOR_NGINX_CONFIG_FILE $USER_NAME@$WEBAPP_FABRICATOR_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$WEBAPP_FABRICATOR_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-enabled/default && sudo service nginx restart"
        
    # Mods web app nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" Mods web app server\n"
    WEBAPP_MODS_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_webapp_mods.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $WEBAPP_MODS_NGINX_CONFIG_FILE $USER_NAME@$WEBAPP_MODS_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$WEBAPP_MODS_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-enabled/default && sudo service nginx restart"

    # Website nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" Website server\n"
    WEBSITE_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_website.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $WEBSITE_NGINX_CONFIG_FILE $USER_NAME@$WEBSITE_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$WEBSITE_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-enabled/default && sudo service nginx restart"
}

dump_dns() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME

    cd $NETWORK_FLEET_DIR/terraform/all
    terraform refresh

    API_DNS=`terraform output -json api | jq .\"api_dns\" --raw-output`
    ADMIN_API_DNS=`terraform output -json admin_api | jq .\"admin_api_dns\" --raw-output`
    CLOUD_API_DNS=`terraform output -json cloud_api | jq .\"cloud_api_dns\" --raw-output`
    CLOUD_WORKER_DNS=`terraform output -json cloud_worker | jq .\"cloud_worker_dns\" --raw-output`
    CLOUD_WEBSOCKET_DNS=`terraform output -json cloud_websocket | jq .\"cloud_websocket_dns\" --raw-output`
    WEBAPP_ADMIN_DNS=`terraform output -json webapp_admin | jq .\"webapp_admin_dns\" --raw-output`
    WEBAPP_FABRICATOR_DNS=`terraform output -json webapp_fabricator | jq .\"webapp_fabricator_dns\" --raw-output`
    WEBAPP_MODS_DNS=`terraform output -json webapp_mods | jq .\"webapp_mods_dns\" --raw-output`
    WEBSITE_DNS=`terraform output -json website | jq .\"website_dns\" --raw-output`

    printf "API\t"$API_DNS"\n"
    printf "Admin API\t"$ADMIN_API_DNS"\n"
    printf "Cloud API\t"$CLOUD_API_DNS"\n"
    printf "Cloud Worker\t"$CLOUD_WEBSOCKET_DNS"\n"
    printf "Cloud WS\t"$CLOUD_WEBSOCKET_DNS"\n"
    printf "WebApp Admin\t"$WEBAPP_ADMIN_DNS"\n"
    printf "WebApp Fabricator\t"$WEBAPP_FABRICATOR_DNS"\n"
    printf "WebApp Mods\t"$WEBAPP_MODS_DNS"\n"
    printf "Website\t"$WEBSITE_DNS"\n"
}

# Setup DNS on Cloudflare
assign_dns() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    SERVER_FLEET_URL_SUFFIX=$3
    CLOUDFLARE_API_KEY=$4
    CLOUDFLARE_ZONE=$5

    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    
    # Replace the values in the config file with the generated ones.
    NETWORK_FLEET_NAME="$NETWORK_NAME-$FLEET_NAME"
    # Convert to lower case
    URL_PREFIX="${NETWORK_FLEET_NAME,,}"

    cd $NETWORK_FLEET_DIR/terraform/all
    terraform refresh

    API_ELB_DNS=`terraform output -json api | jq .\"api_elb_dns\" --raw-output`
    ADMIN_API_ELB_DNS=`terraform output -json admin_api | jq .\"admin_api_elb_dns\" --raw-output`
    CLOUD_API_ELB_DNS=`terraform output -json cloud_api | jq .\"cloud_api_elb_dns\" --raw-output`
    CLOUD_WEBSOCKET_ELB_DNS=`terraform output -json cloud_websocket | jq .\"cloud_websocket_elb_dns\" --raw-output`
    WEBAPP_ADMIN_ELB_DNS=`terraform output -json webapp_admin | jq .\"webapp_admin_elb_dns\" --raw-output`
    WEBAPP_FABRICATOR_ELB_DNS=`terraform output -json webapp_fabricator | jq .\"webapp_fabricator_elb_dns\" --raw-output`
    WEBAPP_MODS_ELB_DNS=`terraform output -json webapp_mods | jq .\"webapp_mods_elb_dns\" --raw-output`
    WEBSITE_ELB_DNS=`terraform output -json website | jq .\"website_elb_dns\" --raw-output`
    WEBSITE_PREVIEW_ELB_DNS=`terraform output -json website | jq .\"website_preview_elb_dns\" --raw-output`

    # API
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-api","content":"'$API_ELB_DNS'","ttl":120,"proxied":false}'

    # Admin API
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-admin-api","content":"'$ADMIN_API_ELB_DNS'","ttl":120,"proxied":false}'
    
    # Cloud API
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-cloud-api","content":"'$CLOUD_API_ELB_DNS'","ttl":120,"proxied":false}'

    # Cloud web socket
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-cloud","content":"'$CLOUD_WEBSOCKET_ELB_DNS'","ttl":120,"proxied":false}'

    # Admin webapp
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-admin","content":"'$WEBAPP_ADMIN_ELB_DNS'","ttl":120,"proxied":false}'
    
    # Fabricator webapp
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-fabricator","content":"'$WEBAPP_FABRICATOR_ELB_DNS'","ttl":120,"proxied":false}'
    
    # Mods webapp
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-mods","content":"'$WEBAPP_MODS_ELB_DNS'","ttl":120,"proxied":false}'

    # Website Test
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-website","content":"'$WEBSITE_ELB_DNS'","ttl":120,"proxied":false}'
}

# Run yarn installers. Run this first in the jenkins script.
#
# Throw an error if yarn fails.
verify_cloud_installers() {
    echo "Build Cloud Libs"
    cd $JENKINS_BUILD_PATH/cloud/lib && yarn install
    echo "Build API Root"
    cd $JENKINS_BUILD_PATH/cloud/api && yarn install
    echo "Build Cloud Root"
    cd $JENKINS_BUILD_PATH/cloud/cloud && yarn install
    echo "Building Cloud API"
    cd $JENKINS_BUILD_PATH/cloud/cloud/cloud_api && yarn install && yarn build
    echo "Building Cloud Worker"
    cd $JENKINS_BUILD_PATH/cloud/cloud/cloud_worker && yarn install && yarn build
    echo "Building Cloud WS Server"
    cd $JENKINS_BUILD_PATH/cloud/cloud/ws_server && yarn install && yarn build
    cd $JENKINS_BUILD_PATH
}

# This is an internal function that sets up common global variables for building and deploying a nodejs application on 
# an AWS EC2 instance.
#
# Most functions within this file will be called from a Jenkins pipeline, so some variables are loaded via the Jenkins
# bootstrapper (e.g. $BUILD_NUMBER)
#
# Inputs:
# NETWORK_NAME - 'dev', 'test', or 'main'
# FLEET_NAME - the designated fleet name.
# APP_NAME - the application name.
# BUILD_PATH - the build location on the local jenkins folder.
ec2_preamble() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    APP_NAME=$3
    BUILD_PATH=$4

    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    SERVER_PUBLIC_DNS=`cd $NETWORK_FLEET_DIR/terraform/all && terraform output -json $APP_NAME | jq .\""$APP_NAME"_dns\" --raw-output`
    
    FOLDER_NAME="$NETWORK_NAME"_"$FLEET_NAME"_"$APP_NAME"_"$BUILD_NUMBER"
    USER_NAME='ubuntu'
    KEY_PATH="/home/ubuntu/.keys"
    DEST_PATH="/home/ubuntu/"$FOLDER_NAME""

    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH
    
    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"
}

build_api() {
    ec2_preamble $1 $2 "api" "$JENKINS_BUILD_PATH/cloud/api"

    # Config files to copy
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_api.env"
    ALLOWED_APPS_DATA_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json"

    # Key files from local jenkins to server
    FIREBASE_KEY_FILE="$NETWORK_FLEET_DIR/keys/firebase_"$NETWORK_NAME"_"$FLEET_NAME".json"
    FIREBASE_ANALYTICS_JSON_FILE="$NETWORK_FLEET_DIR/keys/firebase_"$NETWORK_NAME"_"$FLEET_NAME"_analytics.json"
    ACCESS_TOKEN_PRIVATE_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"
    BRIGHTCOVE_TOKEN_FILE="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_brightcove_token.pem"

    # Get the elasticache address and append to the env file.
    REDIS_CONFIGURATION_ENDPOINT=`cd $NETWORK_FLEET_DIR/terraform/redis-cluster && terraform output -json redis | jq .\"configuration_endpoint_address\" --raw-output`
    FINAL_ENV_FILE="./"$NETWORK_NAME"_"$FLEET_NAME"_api.env"
    cp $ENV_FILE $FINAL_ENV_FILE
    printf "\nREDIS_CONFIGURATION_ENDPOINT='"$REDIS_CONFIGURATION_ENDPOINT"'" >> $FINAL_ENV_FILE

    # .env file goes to the app path
    rsync -avz -e "$SSH_COMMAND" --progress $FINAL_ENV_FILE $RSYNC_DEST/.env

    # Key files go into the home .keys folder.
    rsync -avz -e "$SSH_COMMAND" --progress $ALLOWED_APPS_DATA_FILE $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json
    rsync -avz -e "$SSH_COMMAND" --progress $FIREBASE_KEY_FILE $RSYNC_KEY_DEST/firebase_"$NETWORK_NAME"_"$FLEET_NAME".json
    rsync -avz -e "$SSH_COMMAND" --progress $FIREBASE_ANALYTICS_JSON_FILE $RSYNC_KEY_DEST/firebase_"$NETWORK_NAME"_"$FLEET_NAME"_analytics.json
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PRIVATE_KEY $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY  $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub
    rsync -avz -e "$SSH_COMMAND" --progress $BRIGHTCOVE_TOKEN_FILE  $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_brightcove_token.pem

    # TODO - simply this list of install directives.
    cd $JENKINS_BUILD_PATH/cloud && yarn install
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_api clean && rm -rf ./apps/api/dist/node_modules
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_api build
    cd $JENKINS_BUILD_PATH/cloud && mv ./node_modules ./apps/api/dist
    cd $JENKINS_BUILD_PATH/cloud/apps/api/dist && tar -czf api.tgz ./apps ./api ./auth ./lib ./node_modules

    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/apps/api/dist/api.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf api.tgz && rm api.tgz"

    # Prepare node_modules - link to the various folders
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && rm -rf lib && rm -rf auth && rm -rf api && rm -rf cloud"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../lib lib"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../auth auth"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../api api"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop all && pm2 delete all"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_api_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/api/api.js -i max --env production --update-env"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}

build_admin_api() {
    ec2_preamble $1 $2 "admin_api" "$JENKINS_BUILD_PATH/cloud/api"

    # Config files to copy
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_admin_api.env"
    ALLOWED_APPS_DATA_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json"

    # Key files from local jenkins to server
    FIREBASE_KEY_FILE="$NETWORK_FLEET_DIR/keys/firebase_"$NETWORK_NAME"_"$FLEET_NAME".json"
    FIREBASE_ANALYTICS_JSON_FILE="$NETWORK_FLEET_DIR/keys/firebase_"$NETWORK_NAME"_"$FLEET_NAME"_analytics.json"
    ACCESS_TOKEN_PRIVATE_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"
    BRIGHTCOVE_TOKEN_FILE="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_brightcove_token.pem"

    # Get the elasticache address and append to the env file.
    REDIS_CONFIGURATION_ENDPOINT=`cd $NETWORK_FLEET_DIR/terraform/redis-cluster && terraform output -json redis | jq .\"configuration_endpoint_address\" --raw-output`
    FINAL_ENV_FILE="./"$NETWORK_NAME"_"$FLEET_NAME"_admin_api.env"
    cp $ENV_FILE $FINAL_ENV_FILE
    printf "\nREDIS_CONFIGURATION_ENDPOINT='"$REDIS_CONFIGURATION_ENDPOINT"'" >> $FINAL_ENV_FILE

    # .env file goes to the app path
    rsync -avz -e "$SSH_COMMAND" --progress $FINAL_ENV_FILE $RSYNC_DEST/.env

    # Key files go into the home .keys folder.
    rsync -avz -e "$SSH_COMMAND" --progress $ALLOWED_APPS_DATA_FILE $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json
    rsync -avz -e "$SSH_COMMAND" --progress $FIREBASE_KEY_FILE $RSYNC_KEY_DEST/firebase_"$NETWORK_NAME"_"$FLEET_NAME".json
    rsync -avz -e "$SSH_COMMAND" --progress $FIREBASE_ANALYTICS_JSON_FILE $RSYNC_KEY_DEST/firebase_"$NETWORK_NAME"_"$FLEET_NAME"_analytics.json
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PRIVATE_KEY $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY  $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub
    rsync -avz -e "$SSH_COMMAND" --progress $BRIGHTCOVE_TOKEN_FILE  $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_brightcove_token.pem

    cd $JENKINS_BUILD_PATH/cloud && yarn install
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_admin_api clean && rm -rf ./apps/admin_api/dist/node_modules
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_admin_api build
    cd $JENKINS_BUILD_PATH/cloud && mv ./node_modules ./apps/admin_api/dist
    cd $JENKINS_BUILD_PATH/cloud/apps/admin_api/dist && tar -czf admin_api.tgz ./apps ./api ./auth ./lib ./node_modules

    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/apps/admin_api/dist/admin_api.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf admin_api.tgz && rm admin_api.tgz"

    # Prepare node_modules
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && rm -rf lib && rm -rf auth && rm -rf api && rm -rf cloud"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../lib lib"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../auth auth"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../api api"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop all && pm2 delete all"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_admin_api_$BUILD_NUMBER"
    # TODO: fix the deep dist paths
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start apps/admin_api/admin_api.js -i max --env production --update-env"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}

build_cloud_api() {
    ec2_preamble $1 $2 "cloud_api" "$JENKINS_BUILD_PATH/cloud/cloud"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_api_server.env"
    ALLOWED_APPS_DATA_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json"

    # Key files from local jenkins to server
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"
    FIREBASE_KEY_FILE="$NETWORK_FLEET_DIR/keys/firebase_"$NETWORK_NAME"_"$FLEET_NAME".json"

    # Get the elasticache address and append to the env file.
    REDIS_CONFIGURATION_ENDPOINT=`cd $NETWORK_FLEET_DIR/terraform/redis-cluster && terraform output -json redis | jq .\"configuration_endpoint_address\" --raw-output`
    FINAL_ENV_FILE="./"$NETWORK_NAME"_"$FLEET_NAME"_cloud_api_server.env"
    cp $ENV_FILE $FINAL_ENV_FILE
    printf "\nREDIS_CONFIGURATION_ENDPOINT='"$REDIS_CONFIGURATION_ENDPOINT"'" >> $FINAL_ENV_FILE

    rsync -avz -e "$SSH_COMMAND" --progress $FINAL_ENV_FILE $RSYNC_DEST/.env
    rsync -avz -e "$SSH_COMMAND" --progress $FIREBASE_KEY_FILE $RSYNC_KEY_DEST/firebase_"$NETWORK_NAME"_"$FLEET_NAME".json
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"
    rsync -avz -e "$SSH_COMMAND" --progress $ALLOWED_APPS_DATA_FILE "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json"
    
    cd $JENKINS_BUILD_PATH/cloud && yarn install
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_api clean && rm -rf ./cloud/cloud_api/dist/node_modules
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_api build
    cd $JENKINS_BUILD_PATH/cloud && mv ./node_modules ./cloud/cloud_api/dist
    cd $JENKINS_BUILD_PATH/cloud/cloud/cloud_api/dist && tar -czf cloud_api.tgz ./cloud ./api ./auth ./lib ./node_modules

    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/cloud/cloud_api/dist/cloud_api.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf cloud_api.tgz && rm cloud_api.tgz"

    # Prepare node_modules
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && rm -rf lib && rm -rf auth && rm -rf api && rm -rf cloud"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../lib lib"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../auth auth"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../api api"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../cloud cloud"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop all && pm2 delete all"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_cloud_api_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start cloud/cloud_api/cloud_api.js -i max"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}

# Run an cloud setup script on the cloud_worker server.
run_cloud_setup() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    SERVER_PUBLIC_DNS=`cd $NETWORK_FLEET_DIR/terraform/all && terraform output -json cloud_worker | jq .\"cloud_worker_dns\" --raw-output`
    
    USER_NAME='ubuntu'
    DEST_PATH="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_setup_$BUILD_NUMBER"
    BUILD_PATH=$JENKINS_BUILD_PATH/cloud/cloud
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Copy config files from local jenkins to server
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_worker.env"
    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/.env

    cd $JENKINS_BUILD_PATH/cloud && yarn install
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace @bigscreen/lib build
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_setup clean
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_setup build
    cd $JENKINS_BUILD_PATH/cloud && tar -czf cloud_setup.tgz ./api/dist ./api/package.json ./cloud/dist ./cloud/package.json ./lib/dist ./lib/package.json ./auth/dist ./auth/package.json ./cloud/cloud_setup/dist ./node_modules ./package.json ./yarn.lock

    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/built.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf built.tgz && rm built.tgz"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && node cloud/cloud_setup/dist/cloud_setup.js"
}

build_cloud_websocket() {
    ec2_preamble $1 $2 "cloud_websocket" "$JENKINS_BUILD_PATH/cloud/cloud"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_websocket_server.env"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"
    
    # Get the elasticache address and append to the env file.
    REDIS_CONFIGURATION_ENDPOINT=`cd $NETWORK_FLEET_DIR/terraform/redis-cluster && terraform output -json redis | jq .\"configuration_endpoint_address\" --raw-output`
    FINAL_ENV_FILE="./"$NETWORK_NAME"_"$FLEET_NAME"_cloud_websocket_server.env"
    cp $ENV_FILE $FINAL_ENV_FILE
    printf "\nREDIS_CONFIGURATION_ENDPOINT='"$REDIS_CONFIGURATION_ENDPOINT"'" >> $FINAL_ENV_FILE

    rsync -avz -e "$SSH_COMMAND" --progress $FINAL_ENV_FILE $RSYNC_DEST/.env
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    cd $JENKINS_BUILD_PATH/cloud && yarn install
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_ws_server clean && rm -rf ./cloud/ws_server/dist/node_modules
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_ws_server build
    cd $JENKINS_BUILD_PATH/cloud && mv ./node_modules ./cloud/ws_server/dist
    cd $JENKINS_BUILD_PATH/cloud/cloud/ws_server/dist && tar -czf ws_server.tgz ./cloud ./api ./auth ./lib ./node_modules

    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/cloud/ws_server/dist/ws_server.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf ws_server.tgz && rm ws_server.tgz"

    # Prepare node_modules
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && rm -rf lib && rm -rf auth && rm -rf api && rm -rf cloud"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../lib lib"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../auth auth"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../api api"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../cloud cloud"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop all && pm2 delete all"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_cloud_websocket_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start cloud/ws_server/ws_server.js -i max"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}

build_cloud_worker() {
    ec2_preamble $1 $2 "cloud_worker" "$JENKINS_BUILD_PATH/cloud/cloud"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_worker.env"

    # Get the elasticache address and append to the env file.
    REDIS_CONFIGURATION_ENDPOINT=`cd $NETWORK_FLEET_DIR/terraform/redis-cluster && terraform output -json redis | jq .\"configuration_endpoint_address\" --raw-output`
    FINAL_ENV_FILE="./"$NETWORK_NAME"_"$FLEET_NAME"_cloud_worker.env"
    cp $ENV_FILE $FINAL_ENV_FILE
    printf "\nREDIS_CONFIGURATION_ENDPOINT='"$REDIS_CONFIGURATION_ENDPOINT"'" >> $FINAL_ENV_FILE

    rsync -avz -e "$SSH_COMMAND" --progress $FINAL_ENV_FILE $RSYNC_DEST/.env

    cd $JENKINS_BUILD_PATH/cloud && yarn install
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_worker clean && rm -rf ./cloud/cloud_worker/dist/node_modules
    cd $JENKINS_BUILD_PATH/cloud && yarn workspace bigscreen_cloud_worker build
    cd $JENKINS_BUILD_PATH/cloud && mv ./node_modules ./cloud/cloud_worker/dist
    cd $JENKINS_BUILD_PATH/cloud/cloud/cloud_worker/dist && tar -czf cloud_worker.tgz ./cloud ./api ./auth ./lib ./node_modules

    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/cloud/cloud_worker/dist/cloud_worker.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf cloud_worker.tgz && rm cloud_worker.tgz"

    # Prepare node_modules
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && rm -rf lib && rm -rf auth && rm -rf api && rm -rf cloud"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../lib lib"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../auth auth"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../api api"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/node_modules/\@bigscreen && ln -s ../../cloud cloud"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop all && pm2 delete all"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_cloud_worker_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start cloud/cloud_worker/cloud_worker.js -i max"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}


build_cloud_media_v1() {
    # TODO: replace this stuff with ec2_preamble
    # Difficulty: SERVER_PUBLIC_DNS is inserted by another function running in a loop.
    NETWORK_NAME=$1
    FLEET_NAME=$2
    SERVER_PUBLIC_DNS=$3
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    
    USER_NAME='ubuntu'
    KEY_PATH="/home/ubuntu/.keys"
    DEST_PATH="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_media_server_$BUILD_NUMBER"

    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_media_server.env"

    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/.env

    # Copy the git repo files.
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH/cloud"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH/lib"

    cd $JENKINS_BUILD_PATH/cloud
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/cloud/media_server $RSYNC_DEST/cloud/
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/cloud/src $RSYNC_DEST/cloud/
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/cloud/package.json $RSYNC_DEST/cloud/
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/cloud/yarn.lock $RSYNC_DEST/cloud/
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/lib $RSYNC_DEST
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/api $RSYNC_DEST

    # HACK - stop the target process first so CPU doesn't get starved.a
    #
    # TODO - this will stop the build if there's no existing media_server process.
    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop all && pm2 delete all"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/lib && yarn install"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/api && yarn install"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/cloud && yarn install"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/cloud/media_server && yarn install"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/cloud/media_server && yarn build"

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_media_server_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start cloud/media_server/dist/cloud/media_server/media_server.js -i max"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}


build_cloud_media_v2() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    SERVER_PUBLIC_DNS=$3
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    
    USER_NAME='ubuntu'
    KEY_PATH="/home/ubuntu/.keys"
    DEST_PATH="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_media_server_$BUILD_NUMBER"

    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_media_server.env"

    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/.env

    # Copy the git repo files.
    cd $JENKINS_BUILD_PATH/cloud
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/media_server $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/src $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/package.json $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/yarn.lock $RSYNC_DEST/

    # Copy the media soup prebuilt archive
    rsync -avz -e "$SSH_COMMAND" --progress /home/ubuntu/archives/node_modules_mediasoup.ubuntu.3.9.7.tgz $RSYNC_DEST/media_server/mediasoup.tgz

    # HACK - stop the target process first so CPU doesn't get starved.
    #
    # TODO - this will stop the build if there's no existing media_server process.
    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop all && pm2 delete all"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && yarn install"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/media_server && yarn install"

    # Unzip the media soup archive.
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/media_server && tar -xf mediasoup.tgz && rm mediasoup.tgz"

    # Run the typescript builder.
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/media_server && yarn build"

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_media_server_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start media_server/dist/media_server/media_server.js -i max"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}


build_media_server_group() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    GROUP_NAME=$3

    # EC2 Instance lookup to get public DNS
    MEDIA_SERVER_DNS_NAMES=( $(aws --region $DEFAULT_AWS_REGION ec2 describe-instances --no-paginate --filters Name=tag:Name,Values=*"$NETWORK_NAME"-"$FLEET_NAME"-cloud-media-"$GROUP_NAME" | jq -r '.Reservations[].Instances[].PublicDnsName' ) )

    for MEDIA_SERVER_DNS_NAME in "${MEDIA_SERVER_DNS_NAMES[@]}"
    do
        :
        build_cloud_media_v1 $NETWORK_NAME $FLEET_NAME $MEDIA_SERVER_DNS_NAME
    done
}


build_webapp_admin() {
    ec2_preamble $1 $2 "webapp_admin" "$JENKINS_BUILD_PATH/cloud/webapps"

    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/admin/views $RSYNC_DEST/admin
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/admin/app.js $RSYNC_DEST/admin
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/admin/.babelrc $RSYNC_DEST/admin
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/admin/package.json $RSYNC_DEST/admin
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/admin/public $RSYNC_DEST/admin
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/src/server $RSYNC_DEST/src
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/package.json $RSYNC_DEST/

    # Copy config files from local jenkins to server
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_webapp_admin.env"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/admin/.env
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && yarn install"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/admin && yarn install"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/admin && pm2 stop all && pm2 delete all"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/admin && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do $FOLDER_NAME"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/admin && pm2 start app.js"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/admin && pm2 cleardump && pm2 save"
}


build_webapp_fabricator() {
    ec2_preamble $1 $2 "webapp_fabricator" "$JENKINS_BUILD_PATH/cloud/webapps"

    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/fabricator/views $RSYNC_DEST/fabricator
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/fabricator/fabricator.js $RSYNC_DEST/fabricator
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/fabricator/.babelrc $RSYNC_DEST/fabricator
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/fabricator/package.json $RSYNC_DEST/fabricator
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/fabricator/public $RSYNC_DEST/fabricator
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/src/server $RSYNC_DEST/src
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/package.json $RSYNC_DEST/

    # Copy config files from local jenkins to server
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_webapp_fabricator.env"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/fabricator/.env
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && yarn install"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/fabricator && yarn install"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/fabricator && pm2 stop all && pm2 delete all"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/fabricator && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do $FOLDER_NAME"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/fabricator && pm2 start fabricator.js"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/fabricator && pm2 cleardump && pm2 save"
}


build_webapp_mods() {
    ec2_preamble $1 $2 "webapp_mods" "$JENKINS_BUILD_PATH/cloud/webapps"

    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/mods/views $RSYNC_DEST/mods
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/mods/mods.js $RSYNC_DEST/mods
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/mods/.babelrc $RSYNC_DEST/mods
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/mods/package.json $RSYNC_DEST/mods
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/mods/public $RSYNC_DEST/mods
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/src/server $RSYNC_DEST/src
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/package.json $RSYNC_DEST/

    # Copy config files from local jenkins to server
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_webapp_mods.env"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/mods/.env
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && yarn install"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/mods && yarn install"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/mods && pm2 stop all && pm2 delete all"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/mods && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do $FOLDER_NAME"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/mods && pm2 start mods.js"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/mods && pm2 cleardump && pm2 save"
}


build_website() {
    ec2_preamble $1 $2 "website" "$JENKINS_BUILD_PATH/cloud/webapps/website"

    # Copy config files from local jenkins to server
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_website.env"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/.env
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    #Express
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/src $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/app.js $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/.babelrc $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/package.json $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/ecosystem.config.js $RSYNC_DEST/

    #React
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/app/src $RSYNC_DEST/app/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/app/public $RSYNC_DEST/app/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/app/.babelrc $RSYNC_DEST/app/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/app/package.json $RSYNC_DEST/app/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && npm install"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/app && yarn install"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/app && yarn build"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop all && pm2 delete all"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start ecosystem.config.js --env production --update-env"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$FLEET_NAME"_website_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}


send_discord_success() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    MESSAGE=$3
    URL=$4

    sh `pwd`/devops/Jenkins/deps/discord.sh \
        --webhook-url="https://discordapp.com/api/webhooks/516791347613532207/YVB-QQcWfwzN5sDunKgNuSmWl3KI1aAEi_QMiX93BMDdTtOmrCxQaiqKGdOHj85g_Yfv" \
        --username "Jenkins" \
        --avatar "https://icon-library.com/images/jenkins-icon/jenkins-icon-15.jpg" \
        --text "$NETWORK_NAME-$FLEET_NAME build succeeded" \
        --title "Build succeeded" \
        --color "3066993" \
        --url "$URL" \
        --description "$MESSAGE" \
        --timestamp 
}

# Deploys a number of pre-built images running the user simulator which start stress testing the target server fleet.
#
# Use higher server counts for more load.
begin_load_test() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    SERVER_COUNT=$3

    if [ "$SERVER_COUNT" -lt 1 ]; then
        echo "SERVER_COUNT must be beween 1 and 100"
        exit 1
    fi

    if [ "$SERVER_COUNT" -gt 100 ]; then
        echo "SERVER_COUNT must be beween 1 and 100"
        exit 1
    fi

    # Copy the terraform scripts
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    cp -r $JENKINS_BUILD_PATH/devops/terraform/UserSimulatorLoadTest "$NETWORK_FLEET_DIR/terraform/UserSimulatorLoadTest"
    cd "$NETWORK_FLEET_DIR/terraform/UserSimulatorLoadTest"
    terraform init

    SERVER_INSTANCE_TYPE="t3.large"

    terraform plan -input=false \
        -var="server_count=$SERVER_COUNT" \
        -var="key_name=UserSimulatorLoadTest" \
        -var="instance_type=$SERVER_INSTANCE_TYPE"

    terraform apply -input=false -auto-approve \
        -var="server_count=$SERVER_COUNT" \
        -var="key_name=UserSimulatorLoadTest" \
        -var="instance_type=$SERVER_INSTANCE_TYPE"
}

end_load_test() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    cd "$NETWORK_FLEET_DIR/terraform/UserSimulatorLoadTest"
    terraform destroy -auto-approve
    rm -rf "$NETWORK_FLEET_DIR/terraform/UserSimulatorLoadTest"
}

update_fusion_server() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
}