#!/bin/bash

# Stop and report the error if ANYTHING breaks.
set -e

#
# This script contains various functions for setting up and provisioning a Bigscreen Arch5 server fleet from Jenkins.
#
# Usage (in a Jenkins shell command):
#    #!/bin/bash
#    . /path/to/this_file.sh
#    init_server_fleet_config_files "test" "v2"
#

jq-replace() {
    SRC=$1
    KEY=$2
    VAL=$3

    tmp=$(mktemp)
    jq ".$KEY = \"$VAL\"" "$SRC" > "$tmp" && mv "$tmp" "$SRC"
}

HOME_FOLDER="/home/ubuntu/networks"
JENKINS_BUILD_PATH=`pwd`
DEFAULT_AWS_REGION="us-west-2"

# Requires the jenkins process to have an AWS credentials file at:
# /var/lib/jenkins/.aws
#
# The default IAM must have the ability to create entities on EC2.
init_server_fleet_config_files() {
    NETWORK_NAME=$1
    FLEET_NAME=$2

    if [[ -z $3 ]]
    then
        REGION=$DEFAULT_AWS_REGION
    else
        REGION=$3
    fi

    # TODO: if we've already built the servers via terraform, then bail out.
    # if there was an error, either rollback or create another fleet.

    # Make the destination folders for configurating the server fleet
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    mkdir -p "$NETWORK_FLEET_DIR"
    mkdir -p "$NETWORK_FLEET_DIR/aws"
    mkdir -p "$NETWORK_FLEET_DIR/config"
    mkdir -p "$NETWORK_FLEET_DIR/keys"
    mkdir -p "$NETWORK_FLEET_DIR/terraform"

    # Use the AWS CLI to create a pem file.
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    TMP_OUTPUT_FILE=$(mktemp)

    # TODO - make the AWS profile configurable?
    # aws --region $DEFAULT_AWS_REGION --profile JenkinsServerBuilder ec2 create-key-pair --key-name $KEY_NAME > "$TMP_OUTPUT_FILE"
    set +e
    aws --region $DEFAULT_AWS_REGION ec2 describe-key-pairs --key-name $KEY_NAME
    RESULT=$?
    set -e

    if [ $RESULT -eq 0 ]
    then
        echo "Key already exists."
    else
        echo "Key doesn't exist...creating key..."
        aws --region $DEFAULT_AWS_REGION ec2 create-key-pair --key-name $KEY_NAME > "$TMP_OUTPUT_FILE"
        jq --raw-output ".KeyMaterial" "$TMP_OUTPUT_FILE" > "$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
        rm "$TMP_OUTPUT_FILE"
        chmod 400 "$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    fi

    # Create a base config from the template and copy it to the config folder.
    FLEET_CONFIG_FILE="$NETWORK_FLEET_DIR/config/config.json"
    cp "$HOME_FOLDER/$NETWORK_NAME/.config/"$NETWORK_NAME"_config.json" "$FLEET_CONFIG_FILE"

    # Replace the values in the config file with the generated ones.
    NETWORK_FLEET_NAME="$NETWORK_NAME-$FLEET_NAME"
    # Convert to lower case
    URL_PREFIX="${NETWORK_FLEET_NAME,,}"

    jq-replace "$FLEET_CONFIG_FILE" "SERVER_FLEET_NAME" $NETWORK_NAME"_"$FLEET_NAME
    jq-replace "$FLEET_CONFIG_FILE" "PUBLIC_EMAIL_URL"    "https://$URL_PREFIX-website.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "MOD_PANEL_WEB_HOST"  "https://$URL_PREFIX-mod.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "DASHBOARD_WEB_HOST"  "https://$URL_PREFIX-website.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "BIGSCREEN_ACCOUNTS_URL"  "https://$URL_PREFIX-accounts.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "BIGSCREEN_MODERATOR_URL" "https://$URL_PREFIX-moderator.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "CLOUD_HTTP_MODERATOR_NGINX_SERVER_NAME" "$URL_PREFIX-moderator.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "CLOUD_HTTP_REALTIME_NGINX_SERVER_NAME" "$URL_PREFIX-realtime.bigscreencloud.com"
    jq-replace "$FLEET_CONFIG_FILE" "CLOUD_WEBSOCKET_SERVER_NGINX_SERVER_NAME" "$URL_PREFIX-cloud.bigscreencloud.com"
    
    # Generate the key files
    # 1. copy the network firebase config files from the common key folder
    cp "$HOME_FOLDER/$NETWORK_NAME/.keys/firebase_"$NETWORK_NAME".json" "$NETWORK_FLEET_DIR/keys/firebase_"$NETWORK_NAME"_"$FLEET_NAME".json"
    cp "$HOME_FOLDER/$NETWORK_NAME/.keys/firebase_"$NETWORK_NAME"_analytics.json" "$NETWORK_FLEET_DIR/keys/firebase_"$NETWORK_NAME"_"$FLEET_NAME"_analytics.json"

    # access token private key
    cp "$HOME_FOLDER/$NETWORK_NAME/.keys/"$NETWORK_NAME"_access_token.key" "$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key"

    # access token public key
    cp "$HOME_FOLDER/$NETWORK_NAME/.keys/"$NETWORK_NAME"_access_token.key.pub" "$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    # brightcove token pem
    cp "$HOME_FOLDER/$NETWORK_NAME/.keys/"$NETWORK_NAME"_brightcove_token.pem" "$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_brightcove_token.pem"

    # 2. run the generate script.
    cd $JENKINS_BUILD_PATH/devops/ServerEnvironmentBuilder 
    npm install
    node app.js \
        --default-config-file "$HOME_FOLDER/$NETWORK_NAME/.config/default_config.json" \
        --fleet-config-file "$FLEET_CONFIG_FILE" \
        --output-folder-path "$NETWORK_FLEET_DIR/config"
    cd $JENKINS_BUILD_PATH
}

# Requires the jenkins process to have an AWS credentials file at:
# /var/lib/jenkins/.aws
#
# The default IAM must have the ability to create entities on EC2.
terraform_deploy_fleet() {
    NETWORK_NAME=$1
    FLEET_NAME=$2

    if [[ -z $3 ]]
    then
        REGION=$DEFAULT_AWS_REGION
    else
        REGION=$3
    fi

    # Copy the terraform scripts
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    cp -r $JENKINS_BUILD_PATH/devops/terraform/all "$NETWORK_FLEET_DIR/terraform"
    cd "$NETWORK_FLEET_DIR/terraform/all"
    terraform init

    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    AWS_FRIENDLY_NAME=$NETWORK_NAME"-"$FLEET_NAME
    ELB_CERTIFICATE_ARN="arn:aws:acm:us-west-2:164780942622:certificate/61f977f7-27ff-4538-861c-dfb6535a8f80"
    ACCOUNTS_SERVER_INSTANCE_TYPE="c5.2xlarge"
    CLOUD_SERVER_INSTANCE_TYPE="c5.2xlarge"
    WEBSITE_INSTANCE_TYPE="c5.large"
    MODPANEL_INSTANCE_TYPE="c5.large"

    # Run terraform apply
    if [ "$NETWORK_NAME" = "main" ];
    then
        CLOUD_MEDIA_SERVER_COUNT=4
        ELB_CERTIFICATE_ARN="arn:aws:acm:us-west-2:164780942622:certificate/61f977f7-27ff-4538-861c-dfb6535a8f80"
        ACCOUNTS_SERVER_INSTANCE_TYPE="c5.4xlarge"
        CLOUD_SERVER_INSTANCE_TYPE="c5.4xlarge"
        WEBSITE_INSTANCE_TYPE="c5.xlarge"
        MODPANEL_INSTANCE_TYPE="c5.large"
    fi

    terraform plan -input=false \
        -var="stage=$AWS_FRIENDLY_NAME" \
        -var="accounts_key_name=$KEY_NAME" \
        -var="accounts_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="cloud_app_key_name=$KEY_NAME" \
        -var="cloud_websocket_key_name=$KEY_NAME" \
        -var="cloud_websocket_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="cloud_http_key_name=$KEY_NAME" \
        -var="moderator_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="realtime_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="mod_panel_key_name=$KEY_NAME" \
        -var="mod_panel_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="website_key_name=$KEY_NAME" \
        -var="website_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="website_preview_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="accounts_instance_type=$ACCOUNTS_SERVER_INSTANCE_TYPE" \
        -var="cloud_app_instance_type=$CLOUD_SERVER_INSTANCE_TYPE" \
        -var="cloud_http_instance_type=$CLOUD_SERVER_INSTANCE_TYPE" \
        -var="cloud_websocket_instance_type=$CLOUD_SERVER_INSTANCE_TYPE" \
        -var="mod_panel_instance_type=$MODPANEL_INSTANCE_TYPE" \
        -var="website_instance_type=$WEBSITE_INSTANCE_TYPE" \

    terraform apply -input=false -auto-approve \
        -var="stage=$AWS_FRIENDLY_NAME" \
        -var="accounts_key_name=$KEY_NAME" \
        -var="accounts_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="cloud_app_key_name=$KEY_NAME" \
        -var="cloud_websocket_key_name=$KEY_NAME" \
        -var="cloud_websocket_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="cloud_http_key_name=$KEY_NAME" \
        -var="moderator_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="realtime_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="mod_panel_key_name=$KEY_NAME" \
        -var="mod_panel_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="website_key_name=$KEY_NAME" \
        -var="website_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="website_preview_elb_certificate_arn=$ELB_CERTIFICATE_ARN" \
        -var="accounts_instance_type=$ACCOUNTS_SERVER_INSTANCE_TYPE" \
        -var="cloud_app_instance_type=$CLOUD_SERVER_INSTANCE_TYPE" \
        -var="cloud_http_instance_type=$CLOUD_SERVER_INSTANCE_TYPE" \
        -var="cloud_websocket_instance_type=$CLOUD_SERVER_INSTANCE_TYPE" \
        -var="mod_panel_instance_type=$MODPANEL_INSTANCE_TYPE" \
        -var="website_instance_type=$WEBSITE_INSTANCE_TYPE"
}

terraform_deploy_redis_cluster() {
    NETWORK_NAME=$1
    FLEET_NAME=$2

    if [[ -z $3 ]]
    then
        REGION=$DEFAULT_AWS_REGION
    else
        REGION=$3
    fi

    # Copy the terraform scripts
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"

    # Get the security group ids from cloud_app and cloud_http so they can access this redis cluster
    cd $NETWORK_FLEET_DIR/terraform/all
    terraform refresh
    CLOUD_APP_SECURITY_GROUP_ID=`terraform output -json cloud_app | jq .\"security_group_id\" --raw-output`
    CLOUD_HTTP_SECURITY_GROUP_ID=`terraform output -json cloud_http | jq .\"security_group_id\" --raw-output`

    # copy and init terraform for the redis-cluster module.
    cp -r $JENKINS_BUILD_PATH/devops/terraform/redis-cluster "$NETWORK_FLEET_DIR/terraform"
    cd "$NETWORK_FLEET_DIR/terraform/redis-cluster"
    terraform init

    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    AWS_FRIENDLY_NAME=$NETWORK_NAME"-"$FLEET_NAME
    
    # Redis cluster node must be a cache.m* node type.
    REDIS_NODE_TYPE="cache.m5.large"
    REPLICAS_PER_NODE_GROUP=1
    NUM_NODE_GROUPS=1

    # Run terraform apply
    if [ "$NETWORK_NAME" = "main" ];
    then
        REPLICAS_PER_NODE_GROUP=3
        NUM_NODE_GROUPS=2
        REDIS_NODE_TYPE="cache.m5.4xlarge"
    fi

    terraform plan -input=false \
        -var="stage=$AWS_FRIENDLY_NAME" \
        -var="replicas_per_node_group=$REPLICAS_PER_NODE_GROUP" \
        -var="num_node_groups=$NUM_NODE_GROUPS" \
        -var="redis_stage=$AWS_FRIENDLY_NAME" \
        -var="cloud_app_security_group_id=$CLOUD_APP_SECURITY_GROUP_ID" \
        -var="cloud_http_security_group_id=$CLOUD_HTTP_SECURITY_GROUP_ID" \
        -var="redis_node_type=$REDIS_NODE_TYPE"

    terraform apply -input=false -auto-approve \
        -var="stage=$AWS_FRIENDLY_NAME" \
        -var="replicas_per_node_group=$REPLICAS_PER_NODE_GROUP" \
        -var="num_node_groups=$NUM_NODE_GROUPS" \
        -var="redis_stage=$AWS_FRIENDLY_NAME" \
        -var="cloud_app_security_group_id=$CLOUD_APP_SECURITY_GROUP_ID" \
        -var="cloud_http_security_group_id=$CLOUD_HTTP_SECURITY_GROUP_ID" \
        -var="redis_node_type=$REDIS_NODE_TYPE"
}

deploy_media_server_group() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    GROUP_NAME=$3
    CLOUD_MEDIA_SERVER_COUNT=$4

    if [ "$CLOUD_MEDIA_SERVER_COUNT" -lt 1 ]; then
        echo "CLOUD_MEDIA_SERVER_COUNT must be beween 1 and 15"
        exit 1
    fi

    if [ "$CLOUD_MEDIA_SERVER_COUNT" -gt 15 ]; then
        echo "CLOUD_MEDIA_SERVER_COUNT must be beween 1 and 15"
        exit 1
    fi

    # Copy the terraform scripts
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    cp -r $JENKINS_BUILD_PATH/devops/terraform/media-only "$NETWORK_FLEET_DIR/terraform/$GROUP_NAME-media-only"
    cd "$NETWORK_FLEET_DIR/terraform/$GROUP_NAME-media-only"
    terraform init

    NETWORK_FLEET_NAME=$NETWORK_NAME"-"$FLEET_NAME
    CLOUD_MEDIA_KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    CLOUD_MEDIA_SERVER_NAME="$NETWORK_NAME"-"$FLEET_NAME"-cloud-media-"$GROUP_NAME"
    CLOUD_MEDIA_SERVER_INSTANCE_TYPE="c5.large"

    terraform plan -input=false \
        -var="network_fleet=$NETWORK_FLEET_NAME" \
        -var="cloud_media_server_count=$CLOUD_MEDIA_SERVER_COUNT" \
        -var="cloud_media_key_name=$CLOUD_MEDIA_KEY_NAME" \
        -var="cloud_media_server_name=$CLOUD_MEDIA_SERVER_NAME" \
        -var="cloud_media_instance_type=$CLOUD_MEDIA_SERVER_INSTANCE_TYPE"

    terraform apply -input=false -auto-approve \
        -var="network_fleet=$NETWORK_FLEET_NAME" \
        -var="cloud_media_server_count=$CLOUD_MEDIA_SERVER_COUNT" \
        -var="cloud_media_key_name=$CLOUD_MEDIA_KEY_NAME" \
        -var="cloud_media_server_name=$CLOUD_MEDIA_SERVER_NAME" \
        -var="cloud_media_instance_type=$CLOUD_MEDIA_SERVER_INSTANCE_TYPE"
}

server_fleet_destroy() {
    NETWORK_NAME=$1
    FLEET_NAME=$2

    # Copy the terraform scripts
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    cd "$NETWORK_FLEET_DIR/terraform/all"
    terraform destroy -auto-approve

    # Use the AWS CLI to delete the key pair.
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    
    # TODO - make the AWS profile configurable?
    # aws --profile JenkinsServerBuilder ec2 delete-key-pair --key-name $KEY_NAME
    aws --region $DEFAULT_AWS_REGION ec2 delete-key-pair --key-name $KEY_NAME
}

provision_servers() {
    NETWORK_NAME=$1
    FLEET_NAME=$2

    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME

    cd $NETWORK_FLEET_DIR/terraform/all
    terraform refresh

    ACCOUNTS_DNS=`terraform output -json accounts | jq .\"accounts_dns\" --raw-output`
    CLOUDAPP_DNS=`terraform output -json cloud_app | jq .\"cloud_app_dns\" --raw-output`
    CLOUDHTTP_DNS=`terraform output -json cloud_http | jq .\"cloud_http_dns\" --raw-output`
    CLOUDWEBSOCKET_DNS=`terraform output -json cloud_websocket | jq .\"cloud_websocket_dns\" --raw-output`
    MODPANEL_DNS=`terraform output -json mod_panel | jq .\"mod_panel_dns\" --raw-output`
    WEBSITE_DNS=`terraform output -json website | jq .\"website_dns\" --raw-output`

    USER_NAME='ubuntu'

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Accounts nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" Accounts server\n"
    ACCOUNTS_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_accounts.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $ACCOUNTS_NGINX_CONFIG_FILE $USER_NAME@$ACCOUNTS_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$ACCOUNTS_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-available/default && sudo service nginx restart"

    # Cloud HTTP nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" Cloud HTTP server\n"
    CLOUD_HTTP_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_http_server.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $CLOUD_HTTP_NGINX_CONFIG_FILE $USER_NAME@$CLOUDHTTP_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$CLOUDHTTP_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-available/default && sudo service nginx restart"

    # Cloud Websocket nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" Cloud Websocket server\n"
    CLOUD_WEBSOCKET_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_websocket_server.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $CLOUD_WEBSOCKET_NGINX_CONFIG_FILE $USER_NAME@$CLOUDWEBSOCKET_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$CLOUDWEBSOCKET_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-available/default && sudo service nginx restart"
    
    # Mod Panel nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" Mod Panel server\n"
    MOD_PANEL_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_modpanel.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $MOD_PANEL_NGINX_CONFIG_FILE $USER_NAME@$WEBSITE_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$MODPANEL_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-available/default && sudo service nginx restart"

    # Website nginx
    printf "Provisioning "$NETWORK_NAME-$FLEET_NAME" Website server\n"
    WEBSITE_NGINX_CONFIG_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_website.nginx.conf"
    rsync -avz -e "$SSH_COMMAND" --progress $WEBSITE_NGINX_CONFIG_FILE $USER_NAME@$MODPANEL_DNS:/home/ubuntu/nginx.conf
    $SSH_COMMAND $USER_NAME@$WEBSITE_DNS "sudo mv /home/ubuntu/nginx.conf /etc/nginx/sites-available/default && sudo service nginx restart"
}

dump_dns() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME

    cd $NETWORK_FLEET_DIR/terraform/all
    terraform refresh

    ACCOUNTS_DNS=`terraform output -json accounts | jq .\"accounts_dns\" --raw-output`
    CLOUDAPP_DNS=`terraform output -json cloud_app | jq .\"cloud_app_dns\" --raw-output`
    CLOUDHTTP_DNS=`terraform output -json cloud_http | jq .\"cloud_http_dns\" --raw-output`
    CLOUDWEBSOCKET_DNS=`terraform output -json cloud_websocket | jq .\"cloud_websocket_dns\" --raw-output`
    MODPANEL_DNS=`terraform output -json mod_panel | jq .\"mod_panel_dns\" --raw-output`
    WEBSITE_DNS=`terraform output -json website | jq .\"website_dns\" --raw-output`

    printf "Accounts\t"$ACCOUNTS_DNS"\n"
    printf "Cloud App\t"$CLOUDAPP_DNS"\n"
    printf "Cloud HTTP\t"$CLOUDHTTP_DNS"\n"
    printf "Cloud WS\t"$CLOUDWEBSOCKET_DNS"\n"
    printf "Mod Panel\t"$MODPANEL_DNS"\n"
    printf "Website\t"$WEBSITE_DNS"\n"
}

# Setup DNS on Cloudflare
assign_dns() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    SERVER_FLEET_URL_SUFFIX=$3
    CLOUDFLARE_API_KEY=$4
    CLOUDFLARE_ZONE=$5

    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    
    # Replace the values in the config file with the generated ones.
    NETWORK_FLEET_NAME="$NETWORK_NAME-$FLEET_NAME"
    # Convert to lower case
    URL_PREFIX="${NETWORK_FLEET_NAME,,}"

    cd $NETWORK_FLEET_DIR/terraform/all
    terraform refresh

    ACCOUNTS_ELB_DNS=`terraform output -json accounts | jq .\"accounts_elb_dns\" --raw-output`
    MODERATOR_ELB_DNS=`terraform output -json cloud_http | jq .\"moderator_elb_dns\" --raw-output`
    REALTIME_ELB_DNS=`terraform output -json cloud_http | jq .\"realtime_elb_dns\" --raw-output`
    CLOUD_WEBSOCKET_ELB_DNS=`terraform output -json cloud_websocket | jq .\"cloud_websocket_elb_dns\" --raw-output`
    MODPANEL_ELB_DNS=`terraform output -json mod_panel | jq .\"mod_panel_elb_dns\" --raw-output`
    WEBSITE_ELB_DNS=`terraform output -json website | jq .\"website_elb_dns\" --raw-output`
    WEBSITE_PREVIEW_ELB_DNS=`terraform output -json website | jq .\"website_preview_elb_dns\" --raw-output`

    # Accounts
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-accounts","content":"'$ACCOUNTS_ELB_DNS'","ttl":120,"proxied":false}'
    
    # Moderator
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-moderator","content":"'$MODERATOR_ELB_DNS'","ttl":120,"proxied":false}'
    
    # Realtime
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-realtime","content":"'$REALTIME_ELB_DNS'","ttl":120,"proxied":false}'

    # Cloud web socket
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-cloud","content":"'$CLOUD_WEBSOCKET_ELB_DNS'","ttl":120,"proxied":false}'

    # Mod panel
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-mod","content":"'$MODPANEL_ELB_DNS'","ttl":120,"proxied":false}'

    # Website Test
    curl -X POST "https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE/dns_records" \
        -H "Authorization: Bearer $CLOUDFLARE_API_KEY" \
        -H "Content-Type: application/json" \
        --data '{"type":"CNAME","name":"'$URL_PREFIX'-website","content":"'$WEBSITE_ELB_DNS'","ttl":120,"proxied":false}'
}

# Run yarn installers. Run this first in the jenkins script.
#
# Throw an error if yarn fails.
test_cloud_installers() {
    cd $JENKINS_BUILD_PATH/cloud && yarn install
    cd $JENKINS_BUILD_PATH/cloud/cloud-app && yarn install && yarn build
    cd $JENKINS_BUILD_PATH/cloud/http-server && yarn install && yarn build
    cd $JENKINS_BUILD_PATH/cloud/web-socket-server && yarn install && yarn build
    cd $JENKINS_BUILD_PATH
}

build_accounts() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    SERVER_PUBLIC_DNS=`cd $NETWORK_FLEET_DIR/terraform/all && terraform output -json accounts | jq .\"accounts_dns\" --raw-output`
    
    USER_NAME='ubuntu'
    KEY_PATH="/home/ubuntu/.keys"
    DEST_PATH="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_accounts_$BUILD_NUMBER"
    BUILD_PATH=$JENKINS_BUILD_PATH/accounts

    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"

    # Config files to copy
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_accounts.env"
    ALLOWED_APPS_DATA_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json"

    # Key files from local jenkins to server
    FIREBASE_KEY_FILE="$NETWORK_FLEET_DIR/keys/firebase_"$NETWORK_NAME"_"$FLEET_NAME".json"
    FIREBASE_ANALYTICS_KEY_FILE="$NETWORK_FLEET_DIR/keys/firebase_"$NETWORK_NAME"_"$FLEET_NAME"_analytics.json"
    ACCESS_TOKEN_PRIVATE_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"
    BRIGHTCOVE_TOKEN_FILE="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_brightcove_token.pem"

    # .env file goes to the app path
    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/.env

    # Key files go into the home .keys folder.
    rsync -avz -e "$SSH_COMMAND" --progress $ALLOWED_APPS_DATA_FILE $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_allowed_apps.json
    rsync -avz -e "$SSH_COMMAND" --progress $FIREBASE_KEY_FILE $RSYNC_KEY_DEST/firebase_"$NETWORK_NAME"_"$FLEET_NAME".json
    rsync -avz -e "$SSH_COMMAND" --progress $FIREBASE_ANALYTICS_KEY_FILE $RSYNC_KEY_DEST/firebase_"$NETWORK_NAME"_"$FLEET_NAME"_analytics.json
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PRIVATE_KEY $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY  $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub
    rsync -avz -e "$SSH_COMMAND" --progress $BRIGHTCOVE_TOKEN_FILE  $RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_brightcove_token.pem

    # Copy the git repo files.
    cd $BUILD_PATH
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/src $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/package.json $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/package-lock.json $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/api.js $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/ecosystem.config.js $RSYNC_DEST/

    # Start the new process
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && npm install"

    # pm2 stop will return an error if there's no process available, so we ignore it (this pattern is repeated throughout this script)
    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop api && pm2 delete api"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start ecosystem.config.js --env production --update-env"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"accounts$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}


build_cloud_app() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    SERVER_PUBLIC_DNS=`cd $NETWORK_FLEET_DIR/terraform/all && terraform output -json cloud_app | jq .\"cloud_app_dns\" --raw-output`
    
    USER_NAME='ubuntu'
    KEY_PATH="/home/ubuntu/.keys"
    DEST_PATH="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_app_$BUILD_NUMBER"
    BUILD_PATH=$JENKINS_BUILD_PATH/cloud

    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_app.env"

    # Get the elasticache address and append to the env file.
    REDIS_CONFIGURATION_ENDPOINT=`cd $NETWORK_FLEET_DIR/terraform/redis-cluster && terraform output -json redis | jq .\"configuration_endpoint_address\" --raw-output`
    FINAL_ENV_FILE="./"$NETWORK_NAME"_"$FLEET_NAME"_cloud_app.env"
    cp $ENV_FILE $FINAL_ENV_FILE
    printf "\nREDIS_CONFIGURATION_ENDPOINT='"$REDIS_CONFIGURATION_ENDPOINT"'" >> $FINAL_ENV_FILE

    rsync -avz -e "$SSH_COMMAND" --progress $FINAL_ENV_FILE $RSYNC_DEST/.env

    cd $BUILD_PATH && yarn install
    cd $BUILD_PATH/cloud-app && yarn install && yarn build
    cd $BUILD_PATH && tar -czf built.tgz ./node_modules ./cloud-app/dist ./cloud-app/ecosystem.config.js ./cloud-app/node_modules ./cloud-app/package.json ./cloud-app/yarn.lock

    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/built.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf built.tgz && rm built.tgz"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop cloud-app && pm2 delete cloud-app"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_cloud_app_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start cloud-app/dist/cloud-app/cloud-app.js -i max"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}


build_cloud_http() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    SERVER_PUBLIC_DNS=`cd $NETWORK_FLEET_DIR/terraform/all && terraform output -json cloud_http | jq .\"cloud_http_dns\" --raw-output`
    
    USER_NAME='ubuntu'
    KEY_PATH="/home/ubuntu/.keys"
    DEST_PATH="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_http_$BUILD_NUMBER"
    BUILD_PATH=$JENKINS_BUILD_PATH/cloud

    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_http_server.env"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    # Get the elasticache address and append to the env file.
    REDIS_CONFIGURATION_ENDPOINT=`cd $NETWORK_FLEET_DIR/terraform/redis-cluster && terraform output -json redis | jq .\"configuration_endpoint_address\" --raw-output`
    FINAL_ENV_FILE="./"$NETWORK_NAME"_"$FLEET_NAME"_cloud_http_server.env"
    cp $ENV_FILE $FINAL_ENV_FILE
    printf "\nREDIS_CONFIGURATION_ENDPOINT='"$REDIS_CONFIGURATION_ENDPOINT"'" >> $FINAL_ENV_FILE

    rsync -avz -e "$SSH_COMMAND" --progress $FINAL_ENV_FILE $RSYNC_DEST/.env
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    cd $BUILD_PATH && yarn install
    cd $BUILD_PATH/http-server && yarn install && yarn build
    cd $BUILD_PATH && tar -czf built.tgz ./http-server/dist ./http-server/ecosystem.config.js ./http-server/node_modules ./http-server/package.json ./http-server/yarn.lock

    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/built.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf built.tgz && rm built.tgz"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop http-server && pm2 delete http-server"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_cloud_http_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start http-server/dist/http-server/http-server.js -i max"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}


build_cloud_media_v1() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    SERVER_PUBLIC_DNS=$3
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    
    USER_NAME='ubuntu'
    KEY_PATH="/home/ubuntu/.keys"
    DEST_PATH="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_media_server_$BUILD_NUMBER"

    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_media_server.env"

    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/.env

    # Copy the git repo files.
    cd $JENKINS_BUILD_PATH/cloud
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/media-server $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/src $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/package.json $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/yarn.lock $RSYNC_DEST/

    # HACK - stop the target process first so CPU doesn't get starved.
    #
    # TODO - this will stop the build if there's no existing media-server process.
    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop media-server && pm2 delete media-server"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && yarn install"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/media-server && yarn install"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/media-server && npm link mediasoup"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/media-server && yarn build"

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"CloudMediaServer$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start media-server/dist/media-server/media-server.js -i max"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}


build_cloud_media_v2() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    SERVER_PUBLIC_DNS=$3
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    
    USER_NAME='ubuntu'
    KEY_PATH="/home/ubuntu/.keys"
    DEST_PATH="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_media_server_$BUILD_NUMBER"

    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_media_server.env"

    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/.env

    # Copy the git repo files.
    cd $JENKINS_BUILD_PATH/cloud
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/media-server $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/src $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/package.json $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $JENKINS_BUILD_PATH/cloud/yarn.lock $RSYNC_DEST/

    # Copy the media soup prebuilt archive
    rsync -avz -e "$SSH_COMMAND" --progress /home/ubuntu/archives/node_modules_mediasoup.ubuntu.3.9.7.tgz $RSYNC_DEST/media-server/mediasoup.tgz

    # HACK - stop the target process first so CPU doesn't get starved.
    #
    # TODO - this will stop the build if there's no existing media-server process.
    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop media-server && pm2 delete media-server"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && yarn install"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/media-server && yarn install"

    # Unzip the media soup archive.
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/media-server && tar -xf mediasoup.tgz && rm mediasoup.tgz"

    # Run the typescript builder.
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/media-server && yarn build"

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"CloudMediaServer$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start media-server/dist/media-server/media-server.js -i max"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}


build_media_server_group() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    GROUP_NAME=$3

    # EC2 Instance lookup to get public DNS
    MEDIA_SERVER_DNS_NAMES=( $(aws --region $DEFAULT_AWS_REGION ec2 describe-instances --no-paginate --filters Name=tag:Name,Values=*"$NETWORK_NAME"-"$FLEET_NAME"-cloud-media-"$GROUP_NAME" | jq -r '.Reservations[].Instances[].PublicDnsName' ) )

    for MEDIA_SERVER_DNS_NAME in "${MEDIA_SERVER_DNS_NAMES[@]}"
    do
        :
        build_cloud_media_v1 $NETWORK_NAME $FLEET_NAME $MEDIA_SERVER_DNS_NAME
    done
}


build_cloud_websocket() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    SERVER_PUBLIC_DNS=`cd $NETWORK_FLEET_DIR/terraform/all && terraform output -json cloud_websocket | jq .\"cloud_websocket_dns\" --raw-output`
    
    USER_NAME='ubuntu'
    KEY_PATH="/home/ubuntu/.keys"
    DEST_PATH="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_websocket_$BUILD_NUMBER"
    BUILD_PATH=$JENKINS_BUILD_PATH/cloud

    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_websocket_server.env"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"
    
    # Get the elasticache address and append to the env file.
    REDIS_CONFIGURATION_ENDPOINT=`cd $NETWORK_FLEET_DIR/terraform/redis-cluster && terraform output -json redis | jq .\"configuration_endpoint_address\" --raw-output`
    FINAL_ENV_FILE="./"$NETWORK_NAME"_"$FLEET_NAME"_cloud_websocket_server.env"
    cp $ENV_FILE $FINAL_ENV_FILE
    printf "\nREDIS_CONFIGURATION_ENDPOINT='"$REDIS_CONFIGURATION_ENDPOINT"'" >> $FINAL_ENV_FILE

    rsync -avz -e "$SSH_COMMAND" --progress $FINAL_ENV_FILE $RSYNC_DEST/.env
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    cd $BUILD_PATH && yarn install
    cd $BUILD_PATH/web-socket-server && yarn install && yarn build
    cd $BUILD_PATH && tar -czf built.tgz ./web-socket-server/dist ./web-socket-server/ecosystem.config.js ./web-socket-server/node_modules ./web-socket-server/package.json ./web-socket-server/yarn.lock

    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/built.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf built.tgz && rm built.tgz"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop web-socket-server && pm2 delete web-socket-server"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_cloud_websocket_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start web-socket-server/dist/web-socket-server/web-socket-server.js"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}


build_cloud_websocket_and_run_in_clustered_mode_EXPERIMENTAL() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    SERVER_PUBLIC_DNS=`cd $NETWORK_FLEET_DIR/terraform/all && terraform output -json cloud_websocket | jq .\"cloud_websocket_dns\" --raw-output`
    
    USER_NAME='ubuntu'
    KEY_PATH="/home/ubuntu/.keys"
    DEST_PATH="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_websocket_$BUILD_NUMBER"
    BUILD_PATH=$JENKINS_BUILD_PATH/cloud

    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"

    # Copy config files from local jenkins to server
    #
    # Note - see upload_$FLEET_NAME_keys_to_jenkins.sh to upload the dev keys to jenkins.
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_cloud_websocket_server.env"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"
    
    # Get the elasticache address and append to the env file.
    REDIS_CONFIGURATION_ENDPOINT=`cd $NETWORK_FLEET_DIR/terraform/redis-cluster && terraform output -json redis | jq .\"configuration_endpoint_address\" --raw-output`
    FINAL_ENV_FILE="./"$NETWORK_NAME"_"$FLEET_NAME"_cloud_websocket_server.env"
    cp $ENV_FILE $FINAL_ENV_FILE
    printf "\nREDIS_CONFIGURATION_ENDPOINT='"$REDIS_CONFIGURATION_ENDPOINT"'" >> $FINAL_ENV_FILE

    rsync -avz -e "$SSH_COMMAND" --progress $FINAL_ENV_FILE $RSYNC_DEST/.env
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    cd $BUILD_PATH && yarn install
    cd $BUILD_PATH/web-socket-server && yarn install && yarn build
    cd $BUILD_PATH && tar -czf built.tgz ./web-socket-server/dist ./web-socket-server/ecosystem.config.js ./web-socket-server/node_modules ./web-socket-server/package.json ./web-socket-server/yarn.lock

    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/built.tgz $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && tar -xf built.tgz && rm built.tgz"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop web-socket-server && pm2 delete web-socket-server"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_cloud_websocket_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start web-socket-server/dist/web-socket-server/web-socket-server.js -i max"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}


build_mod_panel() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    SERVER_PUBLIC_DNS=`cd $NETWORK_FLEET_DIR/terraform/all && terraform output -json mod_panel | jq .\"mod_panel_dns\" --raw-output`
    
    USER_NAME='ubuntu'
    KEY_PATH="/home/ubuntu/.keys"
    DEST_PATH="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_modpanel_$BUILD_NUMBER"
    BUILD_PATH=$JENKINS_BUILD_PATH/modpanel

    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"

    # Copy config files from local jenkins to server
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_modpanel.env"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/.env
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/src $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/views $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/app.js $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/.babelrc $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/package.json $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/public $RSYNC_DEST/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && yarn install"

    set +e
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop app && pm2 delete app"
    set -e

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$NETWORK_NAME"_"$FLEET_NAME"_mod_panel_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start app.js"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}


build_website() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    NETWORK_FLEET_DIR="$HOME_FOLDER/$NETWORK_NAME/$FLEET_NAME"
    KEY_NAME=$NETWORK_NAME"_"$FLEET_NAME
    SERVER_PUBLIC_DNS=`cd $NETWORK_FLEET_DIR/terraform/all && terraform output -json website | jq .\"website_dns\" --raw-output`
    
    USER_NAME='ubuntu'
    KEY_PATH="/home/ubuntu/.keys"
    DEST_PATH="/home/ubuntu/"$NETWORK_NAME"_"$FLEET_NAME"_website_$BUILD_NUMBER"
    BUILD_PATH=$JENKINS_BUILD_PATH/website

    RSYNC_KEY_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$KEY_PATH
    RSYNC_DEST=$USER_NAME@$SERVER_PUBLIC_DNS:$DEST_PATH

    # Local key files.
    AWS_KEY_PAIR_FILE="$NETWORK_FLEET_DIR/aws/$KEY_NAME.pem"
    SSH_COMMAND="ssh -i $AWS_KEY_PAIR_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

    # Make the new app destination
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $DEST_PATH"

    # Make the key path
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "mkdir -p $KEY_PATH"

    # Copy config files from local jenkins to server
    ENV_FILE="$NETWORK_FLEET_DIR/config/"$NETWORK_NAME"_"$FLEET_NAME"_website.env"
    ACCESS_TOKEN_PUBLIC_KEY="$NETWORK_FLEET_DIR/keys/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    rsync -avz -e "$SSH_COMMAND" --progress $ENV_FILE $RSYNC_DEST/.env
    rsync -avz -e "$SSH_COMMAND" --progress $ACCESS_TOKEN_PUBLIC_KEY "$RSYNC_KEY_DEST/"$NETWORK_NAME"_"$FLEET_NAME"_access_token.key.pub"

    #Express
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/src $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/app.js $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/.babelrc $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/package.json $RSYNC_DEST/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/ecosystem.config.js $RSYNC_DEST/

    #React
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/app/src $RSYNC_DEST/app/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/app/public $RSYNC_DEST/app/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/app/.babelrc $RSYNC_DEST/app/
    rsync -avz -e "$SSH_COMMAND" --progress $BUILD_PATH/app/package.json $RSYNC_DEST/app/

    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && npm install"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/app && yarn install"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH/app && yarn build"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 stop app && pm2 delete app"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 start ecosystem.config.js --env production --update-env"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 link e5fvqnpr8zq7lpm 5suget3t92ds3do "$FLEET_NAME"_website_$BUILD_NUMBER"
    $SSH_COMMAND $USER_NAME@$SERVER_PUBLIC_DNS "cd $DEST_PATH && pm2 cleardump && pm2 save"
}


send_discord_success() {
    NETWORK_NAME=$1
    FLEET_NAME=$2
    MESSAGE=$3
    URL=$4

    sh `pwd`/devops/Jenkins/deps/discord.sh \
        --webhook-url="https://discordapp.com/api/webhooks/516791347613532207/YVB-QQcWfwzN5sDunKgNuSmWl3KI1aAEi_QMiX93BMDdTtOmrCxQaiqKGdOHj85g_Yfv" \
        --username "Jenkins" \
        --avatar "https://icon-library.com/images/jenkins-icon/jenkins-icon-15.jpg" \
        --text "$NETWORK_NAME-$FLEET_NAME build succeeded" \
        --title "Build succeeded" \
        --color "3066993" \
        --url "$URL" \
        --description "$MESSAGE" \
        --timestamp 
}