#!/bin/bash -
#===============================================================================
#
#          FILE: Ctrl
#
#         USAGE: Ctrl <Command>
#
#   DESCRIPTION: Live Server Controller
#
#       OPTIONS: All options: run this script without any command will show
#
#  REQUIREMENTS: 1. A correct bin/hosts (check "Naming" on wiki)
#                2. A correct bin/rsync.exclude for gmsync
#                3. DBbackup script
#                4. Customized function file(customized_function)
#
#         NOTES: ALL CONFIG INSIDE ~/.gamerc, DO NOT EDIT THIS FILE.
#
#          BUGS:  ---
#        AUTHOR: rickz (Rick Zhang), xlrickz@gmail.com
#       COMPANY: X-LEGEND Entertainment Corp.
#       CREATED: Mon Mar  7 15:30:55 CST 2011
#      REVISION: 2.0
#
#          TODO:
#
#===============================================================================

#set -o nounset                              # Treat unset variables as an error
set -m                                       # Enable job control

#############################################################################
#                                                                           #
#                           DO NOT EDIT THIS FILE                           #
#                            EDIT ~/.gamerc ONLY                            #
#                                                                           #
#############################################################################
[ -r ~/.gamerc ] && source ~/.gamerc || exec echo "Error: Read ~/.gamerc failed."

# What to do?
ACTION="$1"


# The customized function file
NANO_SECOND="$(date +%N)"
CUSTFUNC="/tmp/customized_function.$NANO_SECOND"
sed  -n '/() { # .* -> .*/,/^}$/p' $(dirname $0)/customized_function > $CUSTFUNC
echo >> $CUSTFUNC
sed  -n '/() {[[:blank:]]*$/,/^}$/p' $(dirname $0)/customized_function >> $CUSTFUNC
echo >> $CUSTFUNC

# Log on begin
TTY="$(awk -F/dev/ '{print $2}' <<< "$SSH_TTY")"
SOURCEIP="$(try_to_get_full_hostname $(awk '{print $1}' <<< "$SSH_CLIENT"))"
LOGIN_USER=$(who -m|awk '{print $1}')
DATE_NOW="$(date +%Y/%m/%dT%H:%M:%S)"
[ "$ACTION" == "" ] && DO="NOTHING-TO-DO" || DO="$(head -c 14 <<< "$ACTION                 ")"
echo -e "$DATE_NOW\tBegin\t$DO\tAns:$ANSWER\tLogin:$LOGIN_USER\tUser:$USER\tTTY:$TTY\tIP:$SOURCEIP\tARG:$*" >> "$CTRL_LOGFILE"

ulogger $0 Begin $DO <<< $*

# Detect the ACTION right or not
grep -a -h "^[A-Za-z0-9_-]\+ *(" $CUSTFUNC $0|awk '{print $1}'|grep -a -q "^$ACTION$" || ACTION="show_help"
[ "$ACTION" == "" ] && ACTION="show_help"
DESCRIPTION="$(grep -a "^$ACTION .*" $0 $CUSTFUNC|tail -1|awk -F"# " '{print $2}'|awk -F'..argument' '{print $1}'|tr -d '\n')"

# Ctrl can only run on TEST or CTRL or DATA
if ! egrep -a -q "CTRL|TEST|DATA" <<< "$HOST_NAME" ; then
    ulogger_echo "bash: Ctrl: Permission denied"
    echo "Ctrl can only run on TEST or CTRL" | ulogger
    exit
fi

#############################
# Built-in script functions #
#############################
show_cmd () {
  for CMD_TYPE in $(grep -a "^[A-Za-z0-9_-]\+ (.*) { # .*->" $CUSTFUNC $0|awk -F" # " '{print $2}'|awk -F" -> " '{print $1}'|sort -r|uniq) ; do
	echo "$CMD_TYPE:" | colorize yellow black
	for CMD in $(grep -a -h "^[A-Za-z0-9_-]\+ (.*) { # .*->" $CUSTFUNC $0|grep -a " # $CMD_TYPE -> "|awk '{print $1}'|sort|uniq) ; do
		DESCRIPTION="$(grep -a "^$CMD () { # " $CUSTFUNC $0|head -1|awk -F" # " '{print $2}'|awk -F" -> " '{print $2}')"
		echo -n "$CMD" | colorize cyan black
		echo " -> $DESCRIPTION"
	done
	echo
  done
}

show_help () {
	echo $"Usage: $0 <Command>

Commands:
$(show_cmd)
"
}

IsTodayMDay () {
   DayOfWeek="$(date +%u)"
   for server in $(print_hosts MS) ; do
	let $(ssh $server "cd \$WORKING_DIRECTORY;grep -a -h MaintenanceDate= servers*/setup.ini")
	[ "$MaintenanceDate" != "$DayOfWeek" ] && return 1 || return 0
   done
}

ChangeMDayToToday () { ## Change Maintenance Day to Today
   for server in $(print_hosts MS) ; do
	ssh $server "cd \$WORKING_DIRECTORY;sed -i -e \"s/MaintenanceDate=.*/MaintenanceDate=$(date +%u)/g\" servers*/setup.ini"
   done
}

LIVEBackup () {
   echo -n "Backing up the LIVE server binary to LIVEBackup ... " | colorize magenta
   rm -rf $WORKING_DIRECTORY/common-LIVEBackup/*
   rsync -a CTRL:$WORKING_DIRECTORY/common/. $WORKING_DIRECTORY/common-LIVEBackup/ \
   && echo "OK" | colorize green \
   || echo "FAILED" | colorize red
}

#####################
# Command functions #
#####################
itemmall () { # Maintenance -> Update Promotion Files/Tables from TEST Server to LIVE Server (argument: table names)

   TABLES_TO_APPLY="$ARG2 $ARG3 $ARG4 $ARG5 $ARG6 $ARG7"
   grep -a -q "[A-Za-z]" <<< "$TABLES_TO_APPLY" && PROMOTION_TABLES_APPLY_TO_LIVE="$TABLES_TO_APPLY"

   if [ "$PROMOTION_DATA_APPLY_TO_LIVE_MODE" == "2" ] ; then # 2 = File(on TEST)  -->  Table(on LIVE)
	echo "Mode 2: File(on TEST)  -->  Table(on LIVE)" | colorize green
	for SQL_FILE in $PROMOTION_FILES_APPLY_TO_LIVE ; do
	        echo "File $SQL_FILE(on TEST)  -->  Table(on LIVE):" | colorize yellow
		ssh $TEST_SERVER_USER@TEST "cat ~/common-$SOURCE_OF_SYNC_TO_LIVE/$SQL_FILE" | grep -a -v "GRANT ALL" | sed 's/PUBLIC/PUBLIC\;/g' | ssh accountdb "psql $ACCOUNT_DB_NAME" 2>&1 | uniq -c
	done

   elif [ "$PROMOTION_DATA_APPLY_TO_LIVE_MODE" == "3" ] ; then # 3 = File(on LIVE)  -->  Table(on LIVE)
	echo "Mode 3: File(on LIVE)  -->  Table(on LIVE)" | colorize green
	for SQL_FILE in $PROMOTION_FILES_APPLY_TO_LIVE ; do
		echo "File $SQL_FILE(on LIVE)  -->  Table(on LIVE):" | colorize yellow
		ssh accountdb "cat ~/servers1/$SQL_FILE" | grep -a -v "GRANT ALL" | sed 's/PUBLIC/PUBLIC\;/g' | ssh accountdb "psql $ACCOUNT_DB_NAME" 2>&1 | uniq -c
	done

   else
	echo "Mode 1: Table(on TEST) -->  Table(on LIVE)" | colorize green
	for TABLE in $PROMOTION_TABLES_APPLY_TO_LIVE ; do # 1 = Table(on TEST) -->  Table(on LIVE)
		echo "Syncing table $TABLE(on TEST) -->  Table(on LIVE):" | colorize yellow black
		ssh $TEST_SERVER_USER@TEST pg_dump --clean --attribute-inserts -t $TABLE $ACCOUNT_DB_NAME | grep -a -v "SET lock_timeout" | sed '/^\\unrestrict /d' | sed '/^\\restrict /d' | ssh accountdb "psql $ACCOUNT_DB_NAME" | uniq -c \
		&& echo "Table $TABLE done." | colorize green \
		|| echo "Table $TABLE failed." | colorize red
		online_reload_cgi ${TABLE}
	done
   fi
}

synctranslate () { # Maintenance -> Sync translation ONLY to live servers
   sendscript allall <<< "rsync -av --exclude-from=bin/rsync.exclude TEST:common-$SOURCE_OF_SYNC_TO_LIVE/Data/db/[Tt]_* common/Data/db/."
   eval $LIVE_TRANSLATE_CDN_SYNC_CMDLINE

   if [ -e $HOME/ibackup/ibackup.list ] ; then
       grep -a -q "^translate" $HOME/ibackup/ibackup.list \
       || echo -e "translate\t$TRANSLATED_INI_UPLOAD_DIR" >> $HOME/ibackup/ibackup.list
       ssh TEST "ibackup backup translate 'Restore to sync translate only to LIVE at $(date +%Y%m%dT%H%M)' 2>&1" | grep -a -v "Processing changed file "
   fi
}


gmsync_pre () { # Maintenance -> Sync server to live before maintenance (argument: "binary_only")
   DATETIME="$(/bin/date +%Y%m%dT%H%M)"
   [ ! -d "$WORKING_DIRECTORY/live_pre_sync" ] && mkdir -p $WORKING_DIRECTORY/live_pre_sync
   cd $WORKING_DIRECTORY/live_pre_sync
   SERVER_DIFF_FILE="$WORKING_DIRECTORY/live_pre_sync/server_diff.txt.$DATETIME"
   if grep -a -qi "binary_only" <<< "$ALLARGS" ; then
	rsync -i -avn --exclude-from=$WORKING_DIRECTORY/bin/rsync.exclude --out-format="FILE:%n" ~/common-TEST/ CTRL:~/common_pre/ | grep -a "^FILE:" | grep -a -v "/$" | sed 's/^FILE:/\//g' > $SERVER_DIFF_FILE
	for ip in $(cat /etc/hosts | sed -n '/\# PLEASE/,/# DO/p' | egrep -a 'CTRL|TICKET|ACCTDBS|ACCOUNTDB|LOGIN|HTTPA|GDS|MS|GAMEDB|SOCIETYS|SOCIETYDB|RANKS|CHATS|WS|WORLDDB|LWEBSPS|WWEBSPS' | awk -F" " '{print $1}'); do
		echo ${ip} | colorize blue black
		print_servername ${ip} | colorize yellow black
		if grep -a -qi "check" <<< "$ALLARGS" ; then
			ssh ${ip} "source ~/.gamerc; rsync -avz --exclude-from=bin/rsync.exclude $TEST_SERVER_USER@TEST:common-$SOURCE_OF_SYNC_TO_LIVE/bin/. common_pre/bin/"
		else
			ssh ${ip} "source ~/.gamerc; rsync -aHAXxv --numeric-ids --delete --progress --quiet --exclude-from=$WORKING_DIRECTORY/bin/rsync.exclude -e \"ssh -T -o Compression=no -x\" $TEST_SERVER_USER@TEST:$WORKING_DIRECTORY/common-$SOURCE_OF_SYNC_TO_LIVE/bin/. $WORKING_DIRECTORY/common_pre/bin/" &
		fi
		sleep 1
	done
    if grep -qi "check" <<< "$ALLARGS" ; then
        parameter_text="gmsync_pre binary_only check"
    else
        parameter_text="gmsync_pre binary_only"
    fi
   else
	rsync -i -avn --exclude-from=$WORKING_DIRECTORY/bin/rsync.exclude --out-format="FILE:%n" ~/common-TEST/ CTRL:~/common_pre/ | grep -a "^FILE:" | grep -a -v "/$" | sed 's/^FILE:/\//g' > $SERVER_DIFF_FILE
	GAME_TYPE_LOWER=$(echo ${GAME_TYPE} | tr '[:upper:]' '[:lower:]')
	for ip in $(cat /etc/hosts | sed -n '/\# PLEASE/,/# DO/p' | egrep -a 'CTRL|TICKET|ACCTDBS|ACCOUNTDB|LOGIN|HTTPA|GDS|MS|GAMEDB|SOCIETYS|SOCIETYDB|RANKS|CHATS|WS|WORLDDB|LWEBSPS|WWEBSPS' | awk -F" " '{print $1}'); do
		echo ${ip} | colorize blue black
		print_servername ${ip} | colorize yellow black
		if grep -a -qi "check" <<< "$ALLARGS" ; then
			ssh ${ip} "source ~/.gamerc; rsync -avz --exclude-from=bin/rsync.exclude $TEST_SERVER_USER@TEST:common-$SOURCE_OF_SYNC_TO_LIVE/. common_pre/"
		else
			ssh ${ip} "source ~/.gamerc; rsync -aHAXxv --numeric-ids --delete --progress --quiet --exclude-from=$WORKING_DIRECTORY/bin/rsync.exclude -e \"ssh -T -o Compression=no -x\" $TEST_SERVER_USER@TEST:$WORKING_DIRECTORY/common-$SOURCE_OF_SYNC_TO_LIVE/. $WORKING_DIRECTORY/common_pre/; cd $WORKING_DIRECTORY/common_pre/Data/db; if [ -h S_GMToolCmds.ini ]; then unlink S_GMToolCmds.ini; fi; cd $WORKING_DIRECTORY/common_pre/Data/db; ln -s ${WORKING_DIRECTORY}/bin/S_RootCmds.ini_live_${GAME_TYPE_LOWER} s_rootcmds.ini" &
		fi
		sleep 1
	done
    if grep -qi "check" <<< "$ALLARGS" ; then
        parameter_text="gmsync_pre check"
    else
        parameter_text="gmsync_pre"
    fi
   fi
   automatic_send_chatbot_text_only_by_thread_tag "server_sync_pre" "Server 預先同步完成 ( ${parameter_text} )" minute 10
}

gmsync () { # Maintenance -> Sync test to live (argument: "binary_only", "no_cdn_sync", "local")
   [ "$SOURCE_OF_SYNC_TO_LIVE" == "LIVEBackup" ] || LIVEBackup
   DATETIME="$(/bin/date +%Y%m%dT%H%M)"
   [ ! -d "$WORKING_DIRECTORY/live_sync" ] && mkdir -p $WORKING_DIRECTORY/live_sync
   cd $WORKING_DIRECTORY/live_sync
   SERVER_DIFF_FILE="$WORKING_DIRECTORY/live_sync/server_diff.txt.$DATETIME"
   if grep -a -qi "binary_only" <<< "$ALLARGS" ; then
	rsync -i -avn --exclude-from=$WORKING_DIRECTORY/bin/rsync.exclude --out-format="FILE:%n" ~/common-TEST/ CTRL:~/common/ | grep -a "^FILE:" | grep -a -v "/$" | sed 's/^FILE:/\//g' > $SERVER_DIFF_FILE
	for ip in $(cat /etc/hosts | sed -n '/\# PLEASE/,/# DO/p' | egrep -a 'CTRL|TICKET|ACCTDBS|ACCOUNTDB|LOGIN|HTTPA|GDS|MS|GAMEDB|SOCIETYS|SOCIETYDB|RANKS|CHATS|WS|WORLDDB|LWEBSPS|WWEBSPS' | awk -F" " '{print $1}'); do
		echo ${ip} | colorize blue black
		print_servername ${ip} | colorize yellow black
		if grep -a -qi "local" <<< "$ALLARGS" ; then
			ssh ${ip} "source ~/.gamerc; rsync -avz --exclude-from=bin/rsync.exclude common_pre/bin/. common/bin/"
            parameter_text="gmsync binary_only local"
		else
			ssh ${ip} "source ~/.gamerc; rsync -avz --exclude-from=bin/rsync.exclude $TEST_SERVER_USER@TEST:common-$SOURCE_OF_SYNC_TO_LIVE/bin/. common/bin/"
            parameter_text="gmsync binary_only"
		fi
	done
    if grep -a -qi "local" <<< "$ALLARGS" ; then
        parameter_text="gmsync binary_only local"
    else
        parameter_text="gmsync binary_only"
    fi
   else
	rsync -i -avn --exclude-from=$WORKING_DIRECTORY/bin/rsync.exclude --out-format="FILE:%n" ~/common-TEST/ CTRL:~/common/ | grep -a "^FILE:" | grep -a -v "/$" | sed 's/^FILE:/\//g' > $SERVER_DIFF_FILE
	for ip in $(cat /etc/hosts | sed -n '/\# PLEASE/,/# DO/p' | egrep -a 'CTRL|TICKET|ACCTDBS|ACCOUNTDB|LOGIN|HTTPA|GDS|MS|GAMEDB|SOCIETYS|SOCIETYDB|RANKS|CHATS|WS|WORLDDB|LWEBSPS|WWEBSPS' | awk -F" " '{print $1}'); do
		echo ${ip} | colorize blue black
		print_servername ${ip} | colorize yellow black
		if grep -a -qi "local" <<< "$ALLARGS" ; then
            ssh ${ip} "rm -rf common_old; mv common common_old; mv common_pre common"
		else
			ssh ${ip} "source ~/.gamerc; rsync -avz --exclude-from=bin/rsync.exclude $TEST_SERVER_USER@TEST:common-$SOURCE_OF_SYNC_TO_LIVE/. common/"
		fi
	done
    if grep -a -qi "local" <<< "$ALLARGS" ; then
        parameter_text="gmsync local"
    else
        parameter_text="gmsync"
    fi
   fi
   automatic_send_chatbot_text_only_by_thread_tag "server_sync" "Server 全部同步完成 ( ${parameter_text} )" minute 10

   if grep -a -qi "no_cdn_sync" <<< "$ALLARGS" ; then
	echo "Okay, sync client patch to LIVE disabled"
   else
	eval $LIVE_PATCH_CDN_SYNC_CMDLINE
	ssh TEST "$CLIENT_PATCH_BACKUP_COMMAND_LINE 'Restore to sync to LIVE at $(date +%Y%m%dT%H%M)' 2>&1" | grep -a -v "Processing changed file "
   fi
}

RollbackSync () { # Maintenance -> Rollback the server binary from lastest LIVEBackup
   echo
   echo "DANGER, THIS WILL SYNC FROM LIVEBackup(AUTOMATICALLY BACKUP ONCE BEFORE gmsync) TO LIVE SERVERS" | colorize red
   echo "WARNING: NOT INCLUDE DATABASE" | colorize yellow
   echo
   CAPTCHA="$RANDOM"
   read -p "Please type $CAPTCHA to confirm you want to do this: " CAPTCHA_A
   [ "$CAPTCHA" == "$CAPTCHA_A" ] || exec echo "Error: Wrong answer."

   sendscript allall <<< "rsync -av --exclude-from=bin/rsync.exclude $TEST_SERVER_USER@TEST:common-LIVEBackup/. common/"
}

dbpatch () { # Maintenance -> Patch the databases
if [ "$ARG2" == "fast" ] ; then
    ## 1) ACCOUNTDB
    ssh -o BatchMode=yes -o ConnectTimeout=5 ACCOUNTDB "$HOME/bin/do_dbpatch_for_current_machine_n1 ACCOUNTDB" &
    echo "ACCOUNTDB" | colorize orange blue
    # 等 ACCOUNTDB 達到 result==2
    wait_until_dbpatch_result_eq2 ACCOUNTDB ACCOUNTDB

    ## 2) GAMEDB
    for machine in $(print_hosts_uniq GAMEDB); do
        ssh -o BatchMode=yes -o ConnectTimeout=5 ${machine} "$HOME/bin/do_dbpatch_for_current_machine_n1 GAMEDB" &
    done
    GAMEDB_MACHINES=()
    for machine in $(print_hosts_uniq GAMEDB); do
        GAMEDB_MACHINES+=(${machine})
    done
    echo "${GAMEDB_MACHINES[@]}" | colorize orange blue
    # 等 GAMEDB 組達到 result==2
    wait_until_dbpatch_result_eq2 GAMEDB "${GAMEDB_MACHINES[@]}"

    ## 3) WORLDDB
    for machine in $(print_hosts_uniq WORLDDB); do
        ssh -o BatchMode=yes -o ConnectTimeout=5 ${machine} "$HOME/bin/do_dbpatch_for_current_machine_n1 WORLDDB" &
    done
    WORLDDB_MACHINES=()
    for machine in $(print_hosts_uniq WORLDDB); do
        WORLDDB_MACHINES+=(${machine})
    done
    echo "${WORLDDB_MACHINES[@]}" | colorize orange blue
    # 等 WORLDDB 組達到 result==2
    wait_until_dbpatch_result_eq2 WORLDDB "${WORLDDB_MACHINES[@]}"

    echo "Ctrl dbpatch fast Finished" | colorize white red

    automatic_send_chatbot_text_only_by_thread_tag "dbpatch" "Database Schema 快速更新完成 ( dbpatch )" minute 10
else
    dbpatch_precheck SOCIETYDB
    dbpatch_precheck WORLDDB
    dbpatch_precheck GAMEDB
    dbpatch_precheck ACCOUNTDB
    
    worlddb_flag=$(cat /tmp/worlddb_flag)
    gamedb_flag=$(cat /tmp/gamedb_flag)
    accountdb_flag=$(cat /tmp/accountdb_flag)
    societydb_flag=$(cat /tmp/societydb_flag)
    
    if ! [ "$DO_NOT_RUN_DBBACKUP_BEFORE_DBPATCH" == "1" ] ; then
        echo yes | $0 DBbackup
    fi
    
    if [ "$worlddb_flag" -eq 1 ]; then
        for server in $(print_hosts WORLDDB) ; do
            print_servername $server
            ssh $server "sed -i -e 's/system (\"pg_dump/#system (\"pg_dump/g;s/^\(.*vacuum.*\)$/#\1/g' common/db/db_world_alter"
            ssh $server "cd servers$(get_set_id $server)/db;perl db_world_alter :$WORLD_DB_NAME$(get_world_id $server)"
        done
    else
        exit
    fi
    
    if [ "$gamedb_flag" -eq 1 ]; then
        for server in $(print_hosts GAMEDB) ; do
            print_servername $server
            if [ "$GAME_TYPE" == "N1" ]; then
                ssh $server "sed -i -e 's/system (\"pg_dump/#system (\"pg_dump/g;s/^\(.*vacuum.*\)$/#\1/g' common/db/db_game_alter"
                ssh $server "cd servers$(get_set_id $server)/db;perl db_game_alter :$GAME_DB_NAME$(get_set_id $server)"
            else
                ssh $server "sed -i -e 's/system (\"pg_dump/#system (\"pg_dump/g;s/^\(.*vacuum.*\)$/#\1/g' common/db/dbalter"
                ssh $server "cd servers$(get_set_id $server)/db;perl dbalter :$GAME_DB_NAME$(get_set_id $server)"
            fi
        done
    else
        exit
    fi
    
    if [ "$accountdb_flag" -eq 1 ]; then
       print_servername accountdb
       ssh accountdb "sed -i -e 's/system (\"pg_dump/#system (\"pg_dump/g;s/^\(.*vacuum.*\)$/#\1/g' common/db/db_account_alter"
       ssh accountdb "cd servers1/db;perl db_account_alter :$ACCOUNT_DB_NAME"
    else
       exit
    fi
    
    SOCIETYDB_EXIST=$(check_server_exist_from_hosts SOCIETYDB)
    if [ "$societydb_flag" -eq 1 ]; then
        if [ "$SOCIETYDB_EXIST" -eq 1 ]; then
            for server in $(print_hosts SOCIETYDB) ; do
                zone_category=$(get_zone_category $server)
                zone_id=$(get_zone_id $server)
                set_id=$(get_set_id $server)
                DBNAME="${GAME_TYPE}Society${set_id}"
                print_servername $server | colorize blue black		
                TABLE_COUNTS=$(ssh $server "psql -U postgres ${DBNAME} -c \"COPY (SELECT COUNT(*) FROM information_schema.tables WHERE table_schema NOT IN ('information_schema', 'pg_catalog') AND table_type = 'BASE TABLE') TO STDOUT\"")
                if [ "$TABLE_COUNTS" -gt "0" ]; then			
                    ssh $server "sed -i -e 's/system (\"pg_dump/#system (\"pg_dump/g;s/^\(.*vacuum.*\)$/#\1/g' common/db/db_society_alter"
                    ssh $server "cd servers$(get_set_id $server)/db;perl db_society_alter :$SOCIETY_DB_NAME$(get_set_id $server)"
                fi
            done
        fi
    fi
    
    if ! [ "$DO_NOT_RUN_DBVACUUM_AFTER_DBPATCH" == "1" ] ; then
        echo yes | $0 DBvacuum
    fi
    
    automatic_send_chatbot_text_only_by_thread_tag "dbpatch" "Database Schema 更新完成 ( dbpatch )" minute 10
fi
}

StopMonitor () { # Maintenance -> Stop the server monitor
   sendscript allall <<< 'source ~/.gamerc;rm -fv $WORKING_DIRECTORY/servers*/*/pid'
}

StartMonitor () { # Maintenance -> Start the server monitor
   sendscript allall <<< 'source ~/.gamerc;ls $WORKING_DIRECTORY/servers*/*/auto_start|sed "s/auto_start/pid/g;s/^/echo 65535 > /g"|sh'
}


LIVEStatus () { # ServerStatus -> Show status of LIVE servers
   sendscript allall nosave <<< "cat /proc/loadavg;ps -Ao pid,lstart,pcpu,s,cmd f|grep -a \"[A-Za-z]\+Server\""
}

LIVEBuild () { # Initialization -> Build the live servers (arguments: "nosync", "noreset")
   [ -f ~/.hosts ] && cat ~/.hosts ~/bin/hosts > /etc/hosts 2> /dev/null
   echo "Notice: /etc/hosts updated!" | colorize green

   echo -n "Checking any server process running ... " | colorize magenta
   SERVER_PROCESS="$(sendscript allall nosave nohostname <<< "source ~/.gamerc ; all_running_servers" | wc -l)"
   if [ "$SERVER_PROCESS" == "0" ] ; then
	echo "clean, good to run now." | colorize green
   else
	echo "NOT CLEAN." | colorize red black
	sendscript allall nohostname nosave <<< "ps -Ao pid,lstart,pcpu,s,cmd f|grep -a \"[A-Za-z]\+Server\""
	echo "                          ______                            " | colorize red black 5
	echo "                         /      \                           " | colorize red black 5
	echo "                        /        \                          " | colorize red black 5
	echo "                       |   STOP   |                         " | colorize red black 5
	echo "                        \        /                          " | colorize red black 5
	echo "                         \______/                           " | colorize red black 5
	echo "                                                            " | colorize red black 5
	echo
	echo "           $SERVER_PROCESS server process(es) still running." | colorize red black 5
	echo
	if grep -a -qi "noreset" <<< "$ALLARGS" ; then
		echo
		echo "If you are adding channel or set, you can just hit [Enter]"   | colorize yellow black
		echo "to continue."                                                 | colorize yellow black
		echo
		echo "If you are not adding channel or set, please hit Ctrl + C."   | colorize red black 5
		echo
		read
	else
		echo
		echo "     YOU ARE TRYING TO RUN LIVEBuild WITHOUT \"noreset\"    "    | colorize red black
		echo
		echo "LIVEBuild WITHOUT \"noreset\", MissionServer(s) WILL CRASH"   | colorize red black
		exit
	fi
   fi


   if [ "$NAT" == "1" ] || [ "$ONLY_ONE_NETWORK_INTERFACE" == "1" ] ; then
	[ "$DONT_GET_NAT_EXT_IP" == "1" ] || generate_NAT_IP_table
   fi

   echo "Syncing the scripts to all the LIVE servers ... " | colorize yellow
   syncscript
   echo

   echo "Make sure every server has same content of /etc/hosts ... " | colorize yellow
   sendscript allall nosave <<< "source ~/.gamerc ; rewrite_etc_hosts"
   echo

   sed -n -i -e  '/^[^awg]/p' ~/.ssh/known_hosts
   echo "Testing the connection from every server to TEST server ... " | colorize yellow
   sendscript allall <<< "ssh -o StrictHostKeyChecking=no TEST echo 'OK'"

   if ! grep -a -qi "noreset" <<< "$ALLARGS" ; then
	echo "Reset the minimum ports ... " | colorize yellow black
	sendscript allall <<< "rm -rfv .min_ports"
	echo "OK" | colorize green

	echo "Removing the auto_start files ... " | colorize yellow black
	sendscript allall <<< 'find ~/servers*|grep -a "/auto_start$"|xargs rm -f'
	echo "OK" | colorize green
   fi

   #echo -n "Saving last_start_time ... " | colorize yellow black
   #for set_number in $SET_NUMBER_LIST;do
   #     ssh gamedb$set_number "psql -q -t $GAME_DB_NAME$set_number <<< 'select last_start_time from serverstatus where id<0;'|awk '/[0-9]/ {print \$1}' 2> /dev/null > /tmp/gamedb$set_number-last_start_time"
   #done
   #echo "OK" | colorize green black

   sendscript allall <<< "mkdir -p common"
   grep -a -qi "nosync" <<< "$ALLARGS" || sendscript allall <<< "rsync -av --exclude-from=bin/rsync.exclude-build TEST:$WORKING_DIRECTORY/common-$SOURCE_OF_SYNC_TO_LIVE/. $WORKING_DIRECTORY/common/."

   grep -a -qi "noreset" <<< "$ALLARGS" && SKSERVER_BUILD_COMMAND="BuildIfNotExist" || SKSERVER_BUILD_COMMAND="Build"

   ssh -o StrictHostKeyChecking=no ACCOUNTDB "echo yes|SKServer $SKSERVER_BUILD_COMMAND SetDir 1"
   grep -a -qi "noreset" <<< "$ALLARGS" || PGPASSWORD="$LIVE_SERVER_DB_PASSWORD" psql -h ACCOUNTDB $ACCOUNT_DB_NAME <<< "delete from worlds;" 2>&1 | sed 's/^/worlds: /g'
   ssh -o StrictHostKeyChecking=no ACCOUNTDB "echo yes|SKServer $SKSERVER_BUILD_COMMAND ACCOUNTDB 1"

   if [ "$ENABLE_SOCIETY" -eq 1 ]; then
   	SOCIETYDB_EXIST=$(check_server_exist_from_hosts SOCIETYDB)
	if [ "$SOCIETYDB_EXIST" -eq 1 ]; then
		for server in $(print_hosts SOCIETYDB);do
			print_servername $server | colorize yellow black
			ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND SetDir $(get_set_id $server)"
			ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND SOCIETYDB $(get_set_id $server)"
		done
   	fi
   fi

   for server in $(print_hosts GAMEDB);do
	print_servername $server | colorize yellow black
	ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND SetDir $(get_set_id $server)"
	ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND GAMEDB $(get_set_id $server)"
   done

   for server in $(print_hosts WORLDDB);do
	print_servername $server | colorize yellow black
	ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND SetDir $(get_set_id $server)"
	ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND WORLDDB $(get_set_id $server) $(get_world_id $server)"
   done

   for server in TICKET ACCTDBS $(print_hosts LOGIN) $(print_hosts HTTPA); do
	print_servername $server | colorize yellow black
	ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND SetDir $(get_set_id $server)"
	ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND $(hostname2build $server)"
   done
   for server in $(print_hosts GDS) ; do
	print_servername $server | colorize yellow black
	ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND SetDir $(get_set_id $server)"
	ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND $(hostname2build $server)"
   done
   for server in $(print_hosts MS) ; do
	print_servername $server | colorize yellow black
	ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND SetDir $(get_set_id $server)"
	ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND $(hostname2build $server)"
   done
   for server in $(print_hosts SOCIETYS) $(print_hosts RANKS) $(print_hosts CHATS); do
	print_servername $server | colorize yellow black
	ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND SetDir $(get_set_id $server)"
	ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND $(hostname2build $server)"
   done
   for server in $(print_hosts WS) $(print_hosts WHS); do
	print_servername $server | colorize yellow black
	ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND SetDir $(get_set_id $server)"
	ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND $(hostname2build $server)"
   done
   for server in $(print_hosts LWEBSPS); do
    print_servername $server | colorize yellow black
    ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND SetDir $(get_set_id $server)"
    ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND $(hostname2build $server)"
   done
   for server in $(print_hosts WWEBSPS); do
    print_servername $server | colorize yellow black
    ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND SetDir $(get_set_id $server)"
    ssh -o StrictHostKeyChecking=no $server "echo yes|SKServer $SKSERVER_BUILD_COMMAND $(hostname2build $server)"
   done

   if [ "$ENABLE_SOCIETY" -eq 1 ]; then
	SOCIETYS_EXSIT=$(check_server_exist_from_hosts SOCIETYS)
	if [ "$SOCIETYS_EXSIT" -eq 1 ]; then
		for ip in $(cat /etc/hosts | sed -n '/\# PLEASE/,/# DO/p' | grep -a -v '#' | awk -F" " '{print $1}' | sed '/^$/d'); do
			echo ${ip} | colorize blue black
			print_servername ${ip} | colorize yellow black
			ssh ${ip} "update_society_ip EXT; update_society_ip INT; update_empty_society_port EXT; update_empty_society_port INT"
		done
	fi
   fi

   GDS_EXIST=$(check_server_exist_from_hosts GDS)
   if [ "$GDS_EXIST" -eq 1 ]; then
	for ip in $(cat /etc/hosts | sed -n '/\# PLEASE/,/# DO/p' | grep -a -v '#' | awk -F" " '{print $1}' | sed '/^$/d'); do
		echo ${ip} | colorize blue black
		print_servername ${ip} | colorize yellow black
		ssh ${ip} "update_empty_gds_port"
	done
   fi

   MS_EXIST=$(check_server_exist_from_hosts MS)
   if [ "$MS_EXIST" -eq 1 ]; then
	for ip in $(cat /etc/hosts | sed -n '/\# PLEASE/,/# DO/p' | grep -a -v '#' | awk -F" " '{print $1}' | sed '/^$/d'); do
		echo ${ip} | colorize blue black
		print_servername ${ip} | colorize yellow black
		ssh ${ip} "update_empty_ms_ip; update_empty_ms_port"
	done
   fi

   RANKS_EXIST=$(check_server_exist_from_hosts RANKS)
   if [ "$RANKS_EXIST" -eq 1 ]; then
	for ip in $(cat /etc/hosts | sed -n '/\# PLEASE/,/# DO/p' | grep -a -v '#' | awk -F" " '{print $1}' | sed '/^$/d'); do
		echo ${ip} | colorize blue black
		print_servername ${ip} | colorize yellow black
		ssh ${ip} "update_empty_ranks_port"
	done
   fi

   CHATS_EXIST=$(check_server_exist_from_hosts CHATS)
   if [ "$CHATS_EXIST" -eq 1 ]; then
	for ip in $(cat /etc/hosts | sed -n '/\# PLEASE/,/# DO/p' | grep -a -v '#' | awk -F" " '{print $1}' | sed '/^$/d'); do
		echo ${ip} | colorize blue black
		print_servername ${ip} | colorize yellow black
		ssh ${ip} "update_empty_chats_port"
	done
   fi

   echo "OK"
}

HealthyCheck () { # Maintenance -> Check the Disk/DeadLine/s_rootcmds.ini between LIVE and TEST

   # Disk Space Monitor
   #ssh TEST '~/bin/disk_space_monitor_notice_cron maintenance'
   ssh TEST '~/bin/disk_space_monitor_notice_cron_new maintenance'

   # Deadline Days Remaining Check
   #ssh TEST '~/bin/deadline_monitor_notice_cron maintenance'
   #ssh TEST '~/bin/deadline_monitor_notice_cron_new maintenance'

   # s_rootcmds.ini LIVE vs. TEST Check
   if [ "$MOBILE" == "1" ] ; then
       CHECK_FILE="s_rootcmds.ini"
   else
       CHECK_FILE="S_RootCmds.ini"
   fi

   echo "Checking ${CHECK_FILE} is different or not ... " | colorize magenta
   OUTPUT=$(source ~/.gamerc ; final_return_compare_test_srootcmds_ini)

   if [ -z "${OUTPUT}" ]; then
        echo "Looks good, ${CHECK_FILE} between LIVE and TEST is different." | colorize green black
   else
        #ulogger_echo "${OUTPUT} ${CHECK_FILE} is the same and we can not start live servers."
        echo "${OUTPUT}=>${CHECK_FILE} is the same and we can not start live servers." | ulogger
        echo '                         FETAL ERRORS!!                         ' | colorize red black 5
        echo "${OUTPUT}=> ${CHECK_FILE} between LIVE and TEST is the same. We can not start LIVE servers. Please check it!" | colorize red black

        # Log the time on the end
        DATE_NOW="$(date +%Y/%m/%dT%H:%M:%S)"
        echo -e "$DATE_NOW\t=End=\t$DO\tAns:x\tLogin:$LOGIN_USER\tUser:$USER\tTTY:$TTY\tIP:$SOURCEIP\tARG:LIVEStart" >> "$CTRL_LOGFILE"

        ulogger $0 =End= $DO x <<< $*

        exit
   fi

   if ! [ "$DISABLE_SETUP_INI_CHECK_AFTER_GMSYNC" == "1" ] ; then
        echo ""
        echo "As we did sync to live today, let's check any different in setup.ini between TEST and LIVE ... " | colorize green black
	for ip in $(cat /etc/hosts | sed -n '/\# PLEASE/,/# DO/p' | egrep -a 'TICKET|ACCTDBS|ACCOUNTDB|LOGIN|HTTPA|GDS|MS|GAMEDB|SOCIETYS|SOCIETYDB|RANKS|CHATS|WS|WORLDDB|LWEBSPS|WWEBSPS' | awk -F" " '{print $1}'); do
		echo ${ip} | colorize blue black
		print_servername ${ip} | colorize yellow black
		ssh ${ip} "source ~/.gamerc ; compare_test_setup_ini"
	done
   fi

   if ! [ "$DISABLE_SROOTCMD_INI_CHECK_AFTER_GMSYNC" == "1" ] ; then
        echo ""
        echo "As we did sync to live today, let's also check any different in ${CHECK_FILE} between TEST and LIVE ... " | colorize green black
        #sendscript all <<< "source ~/.gamerc ; compare_test_srootcmds_ini"
	for ip in $(cat /etc/hosts | sed -n '/\# PLEASE/,/# DO/p' | egrep -a 'MS|WS' | awk -F" " '{print $1}'); do
		echo ${ip} | colorize blue black
		print_servername ${ip} | colorize yellow black
		ssh ${ip} "source ~/.gamerc ; compare_test_srootcmds_ini"
	done
   fi
}

LIVEStart () { # Maintenance -> Start the live servers (argument: "fast")

   echo -n "Checking any server process running ... " | colorize magenta
   SERVER_PROCESS="$(sendscript allall nosave nohostname <<< "source ~/.gamerc ; all_running_servers" | wc -l)"
   if [ "$SERVER_PROCESS" == "0" ] ; then
	echo "clean, good to run now." | colorize green
   else
	echo
	echo "                          ______                            " | colorize red black 5
	echo "                         /      \                           " | colorize red black 5
	echo "                        /        \                          " | colorize red black 5
	echo "                       |   STOP   |                         " | colorize red black 5
	echo "                        \        /                          " | colorize red black 5
	echo "                         \______/                           " | colorize red black 5
	echo "                                                            " | colorize red black 5
	echo
	echo "           $SERVER_PROCESS server process(es) still running." | colorize red black 5
	echo
	sendscript allall nohostname nosave <<< "ps -Ao pid,lstart,pcpu,s,cmd f|grep -a \"[A-Za-z]\+Server\""
	echo
	echo "If you are adding channel or set, you can just hit [Enter]"   | colorize yellow black
	echo "to continue."                                                 | colorize yellow black
	echo
	echo "If you are on the maintenance, you should have a clean-start" | colorize red black
	echo "or the player may not be able to login."                      | colorize red black
	echo "Please do LIVEStop and LIVEStatus again to check if the"      | colorize red black
	echo "servers stopped correctly."                                   | colorize red black
	echo
	echo "Hit Ctrl+C if you want to abort LIVEStart."                   | colorize cyan black 5
	read
	PROCESS_IS_NOT_CLEAN="1"
   fi

   # Stop the LIVEDBbackup and pg_dump to prevent table lock
   killall LIVEDBbackup 2> /dev/null ; killall pg_dump 2> /dev/null

   if [ "${NTP_CHECK}" == "1" ] ; then
   	echo -n "Time Synchronization with NTP ... " | colorize yellow
   	if [ -z "$NTP_SERVER" ] ; then
   	     echo "\$NTP_SERVER is empty, no time-server setting." | colorize red
   	else
   	     if [ "$PROCESS_IS_NOT_CLEAN" == "1" ] ; then
   	         echo "Process is not clean, maybe you are adding a set or channel, so we won't run ntpdate this time." | colorize magenta
   	         sleep 5
   	     else
   	         echo
   	         # Test
   	         echo TEST | colorize yellow black
   	         NTP_RESULT="$(ssh $TEST_SERVER_USER@TEST /usr/sbin/ntpdate -u $NTP_SERVER 2>&1)";ulogger_echo "$NTP_RESULT"

   	         # Live
   	         sendscript allall <<< 'source ~/.gamerc;NTP_RESULT="$(/usr/sbin/ntpdate -u $NTP_SERVER 2>&1)";ulogger_echo "$NTP_RESULT"'
   	     fi
   	fi
   fi

   # Start from here
   if [ "$ARG2" == "fast" ] ; then
        ## 1) ACCOUNTDB
        ssh -o BatchMode=yes -o ConnectTimeout=5 ACCOUNTDB "$HOME/bin/SKServer start all"
        echo "ACCOUNTDB" | colorize orange blue
        # 等 ACCOUNTDB 達到 result==2
        wait_until_open_result_eq2 ACCOUNTDB

        ## 2) GDS
        # 啟動 GDS 組
        for machine in $(print_hosts_uniq GDS); do
            ssh -o BatchMode=yes -o ConnectTimeout=5 ${machine} "$HOME/bin/SKServer start all" &
        done
        GDS_MACHINES=()
        for machine in $(print_hosts_uniq GDS); do
            GDS_MACHINES+=(${machine})
        done
        echo "${GDS_MACHINES[@]}" | colorize orange blue
        # 等 GDS 組達到 result==2
        wait_until_open_result_eq2 "${GDS_MACHINES[@]}"

        ## 3) WS（依需求可先 sleep）
        sleep 30
        # 啟動 WS 組
        for machine in $(print_hosts_uniq WS); do
            ssh -o BatchMode=yes -o ConnectTimeout=5 ${machine} "$HOME/bin/SKServer start all" &
        done
        WS_MACHINES=()
        for machine in $(print_hosts_uniq WS); do
            WS_MACHINES+=(${machine})
        done
        echo "${WS_MACHINES[@]}" | colorize orange blue
        # 等 WS 組達到 result==2
        wait_until_open_result_eq2 "${WS_MACHINES[@]}"

        sendscript allall nosave <<< "cat /proc/loadavg;ps -Ao pid,lstart,pcpu,s,cmd f|grep -a \"[A-Za-z]\+Server\"|grep -a -v SKServer"

        echo "Ctrl LIVEStart fast Finished" | colorize white red 
   else
	for server in TICKET ; do
		print_servername $server | colorize yellow black
		grep -a -q TICKET <<< "$server" && ssh $server "log_rotate 0 TicketServer 2> /dev/null" &
		ssh $server "cd servers$(get_set_id $server);SKServer start $(hostname2server $server)"
		sleep 3
	done

	for server in ACCTDBS ; do
		print_servername $server | colorize yellow black
		ssh $server "cd servers$(get_set_id $server);SKServer start $(hostname2server $server)"
		sleep 3
	done

	for server in LGS ; do
		print_servername $server | colorize yellow black
		ssh $server "cd servers$(get_set_id $server);SKServer start all"
		sleep 3
	done

	sleep 10

	for server in $(print_hosts GDS) ; do
		print_servername $server | colorize yellow black
		ssh $server "cd servers$(get_set_id $server);SKServer start all"
		sleep 5
	done

	sleep 10

	for server in $(print_hosts WS) ; do
		print_servername $server | colorize yellow black
		ssh $server "cd servers$(get_set_id $server);SKServer start all"
	done
	wait

	sendscript allall nosave <<< "cat /proc/loadavg;ps -Ao pid,lstart,pcpu,s,cmd f|grep -a \"[A-Za-z]\+Server\"|grep -a -v SKServer"
   fi

    automatic_send_chatbot_text_only_by_thread_tag "live_start" "遊戲伺服器已開啟完成\n對內了，請營運 5 ~ 10 分鐘後，嘗試看看登入正式機" minute 10
}

LIVEStop () { # Maintenance -> Stop the live servers (argument: "fast")
   if [ "$ARG2" == "fast" ] ; then
      echo "Removing the pid files ... "
      for ip in $(cat /etc/hosts | sed -n '/\# PLEASE/,/# DO/p' | egrep -a 'CTRL|TICKET|ACCTDBS|ACCOUNTDB|LOGIN|HTTPA|GDS|MS|GAMEDB|SOCIETYS|SOCIETYDB|RANKS|CHATS|WS|WORLDDB|LWEBSPS|WWEBSPS' | awk -F" " '{print $1}'); do
         echo ${ip} | colorize blue black
         print_servername ${ip} | colorize yellow black
         ssh ${ip} "rm -f $WORKING_DIRECTORY/server*/*Server*/pid" 2>/dev/null
      done

	#echo -n "Sending stop signal to all RankServer ... " | colorize magenta
	#sendscript allranks <<< "ps -ef|grep -v grep|grep -a 'RankServer[0-9]'|awk -F' ' '{print \"kill -15 \"\$2}'|sh" > /dev/null 2>&1
	sleep 10
	echo "OK" | colorize green

        ## 1) WS
        # 關閉 WS 組
        for machine in $(print_hosts_uniq WS); do
            ssh -o BatchMode=yes -o ConnectTimeout=5 ${machine} "$HOME/bin/SKServer stop all" &
        done
        WS_MACHINES=()
        for machine in $(print_hosts_uniq WS); do
            WS_MACHINES+=(${machine})
        done
        echo "${WS_MACHINES[@]}" | colorize orange blue
        # 等 WS 組達到 result==2
        wait_until_close_result_eq2 "${WS_MACHINES[@]}"

        ## 2) GDS
        sleep 15
        # 關閉 GDS 組
        for machine in $(print_hosts_uniq GDS); do
            ssh -o BatchMode=yes -o ConnectTimeout=5 ${machine} "$HOME/bin/SKServer stop all" &
        done
        GDS_MACHINES=()
        for machine in $(print_hosts_uniq GDS); do
            GDS_MACHINES+=(${machine})
        done
        echo "${GDS_MACHINES[@]}" | colorize orange blue
        # 等 GDS 組達到 result==2
        wait_until_close_result_eq2 "${GDS_MACHINES[@]}"

        ## 1) ACCOUNTDB
        sleep 15
        ssh -o BatchMode=yes -o ConnectTimeout=5 ACCOUNTDB "$HOME/bin/SKServer stop all"
        echo "ACCOUNTDB" | colorize orange blue
        # 等 ACCOUNTDB 達到 result==2
        wait_until_close_result_eq2 ACCOUNTDB

        echo "Ctrl LIVEStop fast Finished" | colorize white red
   else
	echo "Removing the pid files ... "
	sendscript allall <<< "rm -f $WORKING_DIRECTORY/server*/*Server*/pid" 2>/dev/null

	echo -n "Sending stop signal to all ChatServer ... " | colorize magenta
	#sendscript allchats <<< "ps -ef|grep -v grep|grep -a 'ChatServer[0-9]'|awk -F' ' '{print \"kill -15 \"\$2}'|sh" > /dev/null 2>&1
	sendscript allchats <<< "ps -ef|grep -v grep|grep -a 'ChatServer[0-9]'|awk -F' ' '{print \"kill -USR1 \"\$2}'|sh" > /dev/null 2>&1
	sleep 10
	echo "OK" | colorize green

	echo -n "Sending stop signal to all RankServer ... " | colorize magenta
	sendscript allranks <<< "ps -ef|grep -v grep|grep -a 'RankServer[0-9]'|awk -F' ' '{print \"kill -15 \"\$2}'|sh" > /dev/null 2>&1
	sleep 10
	echo "OK" | colorize green

        SOCIETYS_EXIST=$(check_server_exist_from_hosts SOCIETYS)
        if [ "$SOCIETYS_EXIST" -eq 1 ]; then
		echo -n "Sending stop signal to SocietyServer ... " | colorize magenta
		sendscript allsocietys <<< "ps -Ao cmd|grep -a 'SocietyServer'|awk -F'/' '{print \"killall -USR1 \"\$2}'|sh" > /dev/null 2>&1
		#sendscript allsocietys <<< "ps -ef|grep -v grep|grep -a 'SocietyServer[0-9]'|awk -F' ' '{print \"kill -15 \"\$2}'|sh" > /dev/null 2>&1
	fi

	echo -n "Sending stop signal to all WHTTPAServer ... " | colorize magenta
	sendscript allwhs <<< "ps -Ao cmd|grep -a 'WHTTPAServer[0-9]'|awk -F'/' '{print \"killall -USR1 \"\$2}'|sh" > /dev/null 2>&1
	for server in $(print_hosts WHS) ; do
		print_servername $server | colorize yellow black
		ssh $server "SKServer stop $(hostname2server $server stop)"
	done
	sleep 10
	echo "OK" | colorize green

	for server in $(print_hosts WS) ; do
		[ "$STOP_SIGNAL_SENT_TO_ALL_WORLDSERVER" != "1" ] \
		&& WHTTPASERVERS_STILL_RUNNING="$(sendscript allall nosave nohostname <<< "ps -Ao pid,lstart,pcpu,s,cmd f|grep -a '\.[/]WHTTPAServer'" | wc -l)"

		if [ "$WHTTPASERVERS_STILL_RUNNING" == "0" ] && [ "$STOP_SIGNAL_SENT_TO_ALL_WORLDSERVER" != "1" ] ; then
			echo -n "All WHTTPAServers have been stopped. Sending stop signal to all WorldServer ... " | colorize magenta
			sendscript allws <<< "ps -Ao cmd|grep -a 'WorldServer[0-9]'|awk -F'/' '{print \"killall -USR1 \"\$2}'|sh" > /dev/null 2>&1
			echo "OK" | colorize green
			STOP_SIGNAL_SENT_TO_ALL_WORLDSERVER="1"
		fi
           
		print_servername $server | colorize yellow black
		ssh $server "SKServer stop $(hostname2server $server stop)"
	done

      	echo -n "Sending stop signal to all MissionServer ... " | colorize magenta
      	sendscript allms <<< "ps -Ao cmd|grep -a 'MissionServer[0-9]'|awk -F'/' '{print \"killall -USR1 \"\$2}'|sh" > /dev/null 2>&1

   sleep 10
   echo "OK" | colorize green

	SOCIETYS_EXIST=$(check_server_exist_from_hosts SOCIETYS)
        if [ "$SOCIETYS_EXIST" -eq 1 ]; then
		echo -n "Sending stop signal to SocietyServer ... " | colorize magenta
		sendscript allsocietys <<< "ps -Ao cmd|grep -a 'SocietyServer'|awk -F'/' '{print \"killall -USR1 \"\$2}'|sh" > /dev/null 2>&1
		#sendscript allsocietys <<< "ps -ef|grep -v grep|grep -a 'SocietyServer[0-9]'|awk -F' ' '{print \"kill -15 \"\$2}'|sh" > /dev/null 2>&1
	fi

	for server in $(print_hosts MS) ; do
		print_servername $server | colorize yellow black
		ssh $server "SKServer stop $(hostname2server $server stop)"
		sleep 3
	done

	for server in $(print_hosts GDS) ; do
		print_servername $server | colorize yellow black
		ssh $server "cd servers$(get_set_id $server);SKServer stop $(hostname2server $server)"
		sleep 3
	done

    for server in $(print_hosts WWEBSPS) ; do
        print_servername $server | colorize yellow black
        ssh $server "cd servers$(get_set_id $server);SKServer stop $(hostname2server $server)"
        sleep 3
    done 

	for server in $(print_hosts HTTPA) $(print_hosts LOGIN) ACCTDBS TICKET $(print_hosts LWEBSPS) ; do
	print_servername $server | colorize yellow black
	ssh $server "cd servers1;SKServer stop $(hostname2server $server stop)"
   done

   echo -n "Cleaning table currentuser on $MEMBER_DB_NAME ... " | colorize yellow black
   if [ "$(sendscript allms nosave nohostname <<< "ps -Ao cmd|grep -a 'MissionServer[0-9]'"|wc -l)" == "0" ] ; then
	ssh accountdb clean_currentuser \
	&& echo "OK" | colorize green \
	|| echo "FAILED, Unable to call function clean_currentuser." | colorize red
   else
	echo "FAILED, There are some MissionServer still running:" | colorize red
	sendscript allgamedb <<< "ps -Ao cmd|grep -a 'MissionServer[0-9]'|egrep -a -v 'level.log'"
   fi
  fi

   ssh accountdb "psql $ACCOUNT_DB_NAME <<< 'UPDATE worlds set online_user = 0;'"

   [ "1" == "$(sendscript allws nohostname <<< "source ~/.gamerc ; [ -f $APEXITEMSERVER_PATH/$APEXITEMSERVER_BIN ] && echo 1" | sort | uniq)" ] \
   && APEXStop

    automatic_send_chatbot_text_only_by_thread_tag "live_stop" "遊戲伺服器 已關閉完成" minute 10
}

LIVEStartSet () { # Maintenance -> Start the live servers by set (arguments: set_id)
	SET_IDS_ARR="$ARG2 $ARG3 $ARG4 $ARG5 $ARG6 $ARG7 $ARG8 $ARG9 $ARG10 $ARG11 $ARG12 $ARG13 $ARG14 $ARG15 $ARG16 $ARG17 $ARG18 $ARG19 $ARG20 $ARG21 $ARG22 $ARG23 $ARG24 $ARG25 $ARG26 $ARG27 $ARG28 $ARG29 $ARG30 $ARG31 $ARG32 $ARG33 $ARG34 $ARG35 $ARG36 $ARG37 $ARG38 $ARG39 $ARG40 $ARG41 $ARG42 $ARG43 $ARG44 $ARG45 $ARG46 $ARG47 $ARG48 $ARG49 $ARG50 $ARG51 $ARG52 $ARG53 $ARG54 $ARG55 $ARG56 $ARG57 $ARG58 $ARG59 $ARG60 $ARG61 $ARG62 $ARG63 $ARG64 $ARG65 $ARG66 $ARG67 $ARG68 $ARG69 $ARG70 $ARG71 $ARG72 $ARG73 $ARG74 $ARG75 $ARG76 $ARG77 $ARG78 $ARG79 $ARG80 $ARG81 $ARG82 $ARG83 $ARG84 $ARG85 $ARG86 $ARG87 $ARG88 $ARG89 $ARG90 $ARG91 $ARG92 $ARG93 $ARG94 $ARG95 $ARG96 $ARG97 $ARG98 $ARG99 $ARG100"
	for set_id in $SET_IDS_ARR; do
        automatic_send_chatbot_text_only_by_thread_tag "live_start" "遊戲伺服器 Set ${set_id} 啟動中" minute 10
		sendscript set${set_id}ms <<< "SKServer start all"
	done

	sleep  10

	for set_id in $SET_IDS_ARR; do
		sendscript set${set_id}ws <<< "SKServer start all"
	done

    automatic_send_chatbot_text_only_by_thread_tag "live_start" "遊戲伺服器 已啟動完成 ( by Set )" minute 10
}

LIVEStopSet () { # Maintenance -> Stop the live servers by set (arguments: set_id)
	SET_IDS_ARR="$ARG2 $ARG3 $ARG4 $ARG5 $ARG6 $ARG7 $ARG8 $ARG9 $ARG10 $ARG11 $ARG12 $ARG13 $ARG14 $ARG15 $ARG16 $ARG17 $ARG18 $ARG19 $ARG20 $ARG21 $ARG22 $ARG23 $ARG24 $ARG25 $ARG26 $ARG27 $ARG28 $ARG29 $ARG30 $ARG31 $ARG32 $ARG33 $ARG34 $ARG35 $ARG36 $ARG37 $ARG38 $ARG39 $ARG40 $ARG41 $ARG42 $ARG43 $ARG44 $ARG45 $ARG46 $ARG47 $ARG48 $ARG49 $ARG50 $ARG51 $ARG52 $ARG53 $ARG54 $ARG55 $ARG56 $ARG57 $ARG58 $ARG59 $ARG60 $ARG61 $ARG62 $ARG63 $ARG64 $ARG65 $ARG66 $ARG67 $ARG68 $ARG69 $ARG70 $ARG71 $ARG72 $ARG73 $ARG74 $ARG75 $ARG76 $ARG77 $ARG78 $ARG79 $ARG80 $ARG81 $ARG82 $ARG83 $ARG84 $ARG85 $ARG86 $ARG87 $ARG88 $ARG89 $ARG90 $ARG91 $ARG92 $ARG93 $ARG94 $ARG95 $ARG96 $ARG97 $ARG98 $ARG99 $ARG100"
	for set_id in $SET_IDS_ARR; do
        automatic_send_chatbot_text_only_by_thread_tag "live_stop" "遊戲伺服器 Set ${set_id} 關閉中" minute 10
		sendscript set${set_id}ws <<< "SKServer stop all"
	done

	sleep  10

	for set_id in $SET_IDS_ARR; do
		sendscript set${set_id}chats <<< "ps -ef|grep -v grep|grep -a 'ChatServer[0-9]'|awk -F' ' '{print \"kill -15 \"\$2}'|sh" > /dev/null 2>&1
		sendscript set${set_id}ranks <<< "ps -ef|grep -v grep|grep -a 'RankServer[0-9]'|awk -F' ' '{print \"kill -15 \"\$2}'|sh" > /dev/null 2>&1
		sleep 5
		sendscript set${set_id}ms <<< "SKServer stop all"
	done

    automatic_send_chatbot_text_only_by_thread_tag "live_stop" "遊戲伺服器 已關閉完成 ( by Set )" minute 10
}

LIVEStartWorld () { # Maintenance -> Start the live servers by world
    automatic_send_chatbot_text_only_by_thread_tag "live_start_world" "遊戲伺服器 WorldServers 開始啟動" minute 10

    # 啟動 WS 組
    for machine in $(print_hosts_uniq WS); do
        ssh -o BatchMode=yes -o ConnectTimeout=5 ${machine} "$HOME/bin/SKServer start all" &
    done
    WS_MACHINES=()
    for machine in $(print_hosts_uniq WS); do
        WS_MACHINES+=(${machine})
    done
    echo "${WS_MACHINES[@]}" | colorize orange blue
    # 等 WS 組達到 result==2
    wait_until_open_result_eq2 "${WS_MACHINES[@]}"
    sendscript allws nosave <<< "cat /proc/loadavg;ps -Ao pid,lstart,pcpu,s,cmd f|grep -a \"[A-Za-z]\+Server\"|grep -a -v SKServer"

    echo "Ctrl LIVEStartWorld fast Finished" | colorize white red

    automatic_send_chatbot_text_only_by_thread_tag "live_start_world" "遊戲伺服器 WorldServers 已啟動完成" minute 10
}

LIVEStopWorld () { # Maintenance -> Stop the live servers by world
    automatic_send_chatbot_text_only_by_thread_tag "live_stop_world" "遊戲伺服器 WorldServers 開始關閉" minute 10

    # 關閉 WS 組
    for machine in $(print_hosts_uniq WS); do
        ssh -o BatchMode=yes -o ConnectTimeout=5 ${machine} "$HOME/bin/SKServer stop all" &
    done
    WS_MACHINES=()
    for machine in $(print_hosts_uniq WS); do
        WS_MACHINES+=(${machine})
    done
    echo "${WS_MACHINES[@]}" | colorize orange blue
    # 等 WS 組達到 result==2
    wait_until_close_result_eq2 "${WS_MACHINES[@]}"
    sendscript allws nosave <<< "cat /proc/loadavg;ps -Ao pid,lstart,pcpu,s,cmd f|grep -a \"[A-Za-z]\+Server\"|grep -a -v SKServer"

    echo "Ctrl LIVEStopWorld fast Finished" | colorize white red

    automatic_send_chatbot_text_only_by_thread_tag "live_stop_world" "遊戲伺服器 WorldServers 已關閉完成" minute 10
}

CheckAlive () { # ServerStatus -> Check the LIVE servers running status
   sendscript allall <<< "SKServer CheckAlive all"
}

LGStop () { # Maintenance -> Stop login servers
   for login in $(print_hosts LGS);do
	ssh $login "cd servers1;SKServer stop LoginServer*"
   done
}
LGStart () { # Maintenance -> Start login servers
   for login in $(print_hosts LGS);do
	ssh $login "cd servers1;SKServer start LoginServer*"
   done
}
LGRestart () { # Maintenance -> Restart login servers
   for login in $(print_hosts LGS);do
	ssh $login "cd servers1;SKServer stop LoginServer*;SKServer start LoginServer*"
   done
}

LGBlock () { # Maintenance -> Block the players by iptables firewall ( Login )
   for login in $(print_hosts LGS);do
	echo $login | colorize yellow black
	if [ "$LGFW_MODE" == "1" ] ; then
		ssh $login "sudo iptables-restore < $IPTABLES_LGFW_ON_CONFIG_FILE; sudo iptables -L -v -n"
	elif [ "$LGFW_MODE" == "2" ] ; then
		ssh $login "for PORT in $LOGIN_SERVER_PORTS ;do
			for IP in $LOGIN_SERVER_ALLOW_IP ;do
				sudo iptables -v -A INPUT -p tcp -s \$IP --dport \$PORT -j ACCEPT
			done
			sudo iptables -v -A INPUT -p tcp --dport \$PORT -j DROP
   		done
		sudo iptables -L -v -n"
	elif [ "$LGFW_MODE" == "3" ] ; then
		ssh -t $login "sudo su -l -c '/etc/init.d/iptables start'"
		ssh -t $login "sudo su -l -c '/etc/init.d/iptables status'"
	fi
   done
   $HOME/bin/check_live_iptables_and_notify
}

LGAllow () { # Maintenance -> Allow the players by iptables firewall ( Login )
   for login in $(print_hosts LGS);do
	echo $login | colorize yellow black
	if [ "$LGFW_MODE" == "1" ] ; then
		ssh $login "sudo iptables-restore < $IPTABLES_LGFW_OFF_CONFIG_FILE; sudo iptables -L -v -n"
	elif [ "$LGFW_MODE" == "2" ] ; then
		ssh $login "for PORT in $LOGIN_SERVER_PORTS ;do
			for IP in $LOGIN_SERVER_ALLOW_IP ;do
				sudo iptables -v -D INPUT -p tcp -s \$IP --dport \$PORT -j ACCEPT
			done
			sudo iptables -v -D INPUT -p tcp --dport \$PORT -j DROP
   		done
		sudo iptables -L -v -n"
	elif [ "$LGFW_MODE" == "3" ] ; then
		ssh -t $login "sudo su -l -c '/etc/init.d/iptables stop'; sudo iptables -L -v -n'"
	fi
   done
   $HOME/bin/check_live_iptables_and_notify
}

WSBlock () { # Maintenance -> Block the players by iptables firewall ( World )
   MACHINE=$ARG2

   if [ ! -z "$ARG2" ]; then
                echo $MACHINE | colorize yellow black
                if [ "$LGFW_MODE" == "1" ] ; then
                        ssh $MACHINE "sudo iptables-restore < $IPTABLES_LGFW_ON_CONFIG_FILE; sudo iptables -L -v -n"
                elif [ "$LGFW_MODE" == "2" ] ; then
                        PORT_START=$(echo $WORLD_SERVER_PORTS_RANGE | cut -d'-' -f1)
                        PORT_END=$(echo $WORLD_SERVER_PORTS_RANGE | cut -d'-' -f2)
                        ssh $MACHINE "sudo iptables_port_range ADD $PORT_START $PORT_END"
                elif [ "$LGFW_MODE" == "3" ] ; then
                        ssh -t $MACHINE "sudo su -l -c '/etc/init.d/iptables start'"
                        ssh -t $MACHINE "sudo su -l -c '/etc/init.d/iptables status'"
                fi
   else
                for WS in $(print_hosts WS);do
                        echo $WS | colorize yellow black
                        if [ "$LGFW_MODE" == "1" ] ; then
                                ssh $WS "sudo iptables-restore < $IPTABLES_LGFW_ON_CONFIG_FILE; sudo iptables -L -v -n"
                        elif [ "$LGFW_MODE" == "2" ] ; then
                                PORT_START=$(echo $WORLD_SERVER_PORTS_RANGE | cut -d'-' -f1)
                                PORT_END=$(echo $WORLD_SERVER_PORTS_RANGE | cut -d'-' -f2)
                                ssh $WS "sudo iptables_port_range ADD $PORT_START $PORT_END"
                        elif [ "$LGFW_MODE" == "3" ] ; then
                                ssh -t $WS "sudo su -l -c '/etc/init.d/iptables start'"
                                ssh -t $WS "sudo su -l -c '/etc/init.d/iptables status'"
                        fi
                done
   fi
}

WSAllow () { # Maintenance -> Allow the players by iptables firewall ( World )
   MACHINE=$ARG2

   if [ ! -z "$ARG2" ]; then
                echo $MACHINE | colorize yellow black
                if [ "$LGFW_MODE" == "1" ] ; then
                        ssh $MACHINE "sudo iptables-restore < $IPTABLES_LGFW_OFF_CONFIG_FILE; sudo iptables -L -v -n"
                elif [ "$LGFW_MODE" == "2" ] ; then
                        PORT_START=$(echo $WORLD_SERVER_PORTS_RANGE | cut -d'-' -f1)
                        PORT_END=$(echo $WORLD_SERVER_PORTS_RANGE | cut -d'-' -f2)
                        ssh $MACHINE "sudo iptables_port_range REMOVE $PORT_START $PORT_END"
                elif [ "$LGFW_MODE" == "3" ] ; then
                        ssh -t $MACHINE "sudo su -l -c '/etc/init.d/iptables stop'; sudo iptables -L -v -n'"
                fi
   else
                for WS in $(print_hosts WS);do
                        echo $WS | colorize yellow black
                        if [ "$LGFW_MODE" == "1" ] ; then
                                ssh $WS "sudo iptables-restore < $IPTABLES_LGFW_OFF_CONFIG_FILE; sudo iptables -L -v -n"
                        elif [ "$LGFW_MODE" == "2" ] ; then
                                PORT_START=$(echo $WORLD_SERVER_PORTS_RANGE | cut -d'-' -f1)
                                PORT_END=$(echo $WORLD_SERVER_PORTS_RANGE | cut -d'-' -f2)
                                ssh $WS "sudo iptables_port_range REMOVE $PORT_START $PORT_END"
                        elif [ "$LGFW_MODE" == "3" ] ; then
                                ssh -t $WS "sudo su -l -c '/etc/init.d/iptables stop'; sudo iptables -L -v -n'"
                        fi
                done
   fi
}

DBbackup () { # Maintenance -> Backup databases (argument: "accountdb_only")
   print_servername accountdb | colorize yellow black
   ssh accountdb "DBbackup $ACCOUNT_DB_NAME" &
   ssh accountdb "DBbackup $MEMBER_DB_NAME" &
   if [ "$ENABLE_SOCIETY" -eq 1 ]; then
	SOCIETYDB_EXIST=$(check_server_exist_from_hosts SOCIETYDB)
	if [ "$SOCIETYDB_EXIST" -eq 1 ]; then
		[ "$ARG2" == "accountdb_only" ] || for server in $(print_hosts SOCIETYDB) ; do
		print_servername $server | colorize yellow black
		ssh $server "DBbackup $SOCIETY_DB_NAME$(get_set_id $server)" &
		done
	fi
   fi

   [ "$ARG2" == "accountdb_only" ] || for server in $(print_hosts GAMEDB) ; do
	print_servername $server | colorize yellow black
	ssh $server "DBbackup $GAME_DB_NAME$(get_set_id $server)" &
   done

   if [ "$GAME_TYPE" == "N1" ]; then
	   for server in $(print_hosts WORLDDB) ; do
		print_servername $server | colorize yellow black
		ssh $server "DBbackup $WORLD_DB_NAME$(get_world_id $server)" &
           done
   fi
   wait

    automatic_send_chatbot_text_only_by_thread_tag "dbbackup" "遊戲資料庫 備分完成" minute 10 
}

DBbackupSet () { # Maintenance -> Backup databases by set (argument: set_id)
        SET_IDS_ARR="$ARG2 $ARG3 $ARG4 $ARG5 $ARG6 $ARG7 $ARG8 $ARG9 $ARG10 $ARG11 $ARG12 $ARG13 $ARG14 $ARG15 $ARG16 $ARG17 $ARG18 $ARG19 $ARG20 $ARG21 $ARG22 $ARG23 $ARG24 $ARG25 $ARG26 $ARG27 $ARG28 $ARG29 $ARG30 $ARG31 $ARG32 $ARG33 $ARG34 $ARG35 $ARG36 $ARG37 $ARG38 $ARG39 $ARG40 $ARG41 $ARG42 $ARG43 $ARG44 $ARG45 $ARG46 $ARG47 $ARG48 $ARG49 $ARG50 $ARG51 $ARG52 $ARG53 $ARG54 $ARG55 $ARG56 $ARG57 $ARG58 $ARG59 $ARG60 $ARG61 $ARG62 $ARG63 $ARG64 $ARG65 $ARG66 $ARG67 $ARG68 $ARG69 $ARG70 $ARG71 $ARG72 $ARG73 $ARG74 $ARG75 $ARG76 $ARG77 $ARG78 $ARG79 $ARG80 $ARG81 $ARG82 $ARG83 $ARG84 $ARG85 $ARG86 $ARG87 $ARG88 $ARG89 $ARG90 $ARG91 $ARG92 $ARG93 $ARG94 $ARG95 $ARG96 $ARG97 $ARG98 $ARG99 $ARG100"

	if [ "$ENABLE_SOCIETY" -eq 1 ]; then
		SOCIETYDB_EXIST=$(check_server_exist_from_hosts SOCIETYDB)
		if [ "$SOCIETYDB_EXIST" -eq 1 ]; then
			for set_id in $SET_IDS_ARR ; do
                automatic_send_chatbot_text_only_by_thread_tag "dbbackup" "遊戲資料庫 Set ${set_id} 備份中" minute 10
				print_servername SOCIETYDB${set_id} | colorize yellow black
				ssh SOCIETYDB${set_id} "DBbackup $SOCIETY_DB_NAME${set_id}" &
			done
		fi
   	fi

	for set_id in $SET_IDS_ARR ; do
		print_servername GAMEDB${set_id} | colorize yellow black
		ssh GAMEDB${set_id} "DBbackup $GAME_DB_NAME${set_id}" &
	done

	for server in $(print_hosts WORLDDB) ; do
		set_id=$(get_set_id $server)
		world_id=$(get_world_id $server)
		if [[ " $SET_IDS_ARR " =~ [[:space:]]$set_id[[:space:]] || " $SET_IDS_ARR " =~ ^$set_id[[:space:]] || " $SET_IDS_ARR " =~ [[:space:]]$set_id$ ]]; then
			print_servername $server | colorize yellow black
			ssh $server "DBbackup $WORLD_DB_NAME${world_id}" &
		fi
	done

	wait

    automatic_send_chatbot_text_only_by_thread_tag "dbbackup" "遊戲資料庫 備分完成 ( by Set )" minute 10
}

DBbackupWorld () { # Maintenance -> Backup databases by world (argument: world_id)
        WORLD_IDS_ARR="$ARG2 $ARG3 $ARG4 $ARG5 $ARG6 $ARG7 $ARG8 $ARG9 $ARG10 $ARG11 $ARG12 $ARG13 $ARG14 $ARG15 $ARG16 $ARG17 $ARG18 $ARG19 $ARG20 $ARG21 $ARG22 $ARG23 $ARG24 $ARG25 $ARG26 $ARG27 $ARG28 $ARG29 $ARG30 $ARG31 $ARG32 $ARG33 $ARG34 $ARG35 $ARG36 $ARG37 $ARG38 $ARG39 $ARG40 $ARG41 $ARG42 $ARG43 $ARG44 $ARG45 $ARG46 $ARG47 $ARG48 $ARG49 $ARG50 $ARG51 $ARG52 $ARG53 $ARG54 $ARG55 $ARG56 $ARG57 $ARG58 $ARG59 $ARG60 $ARG61 $ARG62 $ARG63 $ARG64 $ARG65 $ARG66 $ARG67 $ARG68 $ARG69 $ARG70 $ARG71 $ARG72 $ARG73 $ARG74 $ARG75 $ARG76 $ARG77 $ARG78 $ARG79 $ARG80 $ARG81 $ARG82 $ARG83 $ARG84 $ARG85 $ARG86 $ARG87 $ARG88 $ARG89 $ARG90 $ARG91 $ARG92 $ARG93 $ARG94 $ARG95 $ARG96 $ARG97 $ARG98 $ARG99 $ARG100"

	for world_id in $WORLD_IDS_ARR ; do
        automatic_send_chatbot_text_only_by_thread_tag "dbbackup" "遊戲資料庫 World ${world_id} 備份中" minute 10
		print_servername WORLDDB${world_id} | colorize yellow black
		ssh WORLDDB${world_id} "DBbackup $WORLD_DB_NAME${world_id}" &
	done

	wait

    automatic_send_chatbot_text_only_by_thread_tag "dbbackup" "遊戲資料庫 備分完成 ( by World )" minute 10
}

GMList () { # ServerStatus -> The list of GM characters
   sendscript allgamedb <<< 'source ~/.gamerc ; psql -P border=2 -P footer=0 $GAME_DB_NAME$(get_set_id $HOST_NAME) <<< "select id, given_name, privilege, account_name from player_characters where privilege <> 0 order by id;"'
}

GMCmds () { # ServerStatus -> List all the commands that used by GM characters
   [ "$ARG2" == "" ] \
   && exec echo "Usage: $0 GMCmds <Set number>" \
   || sendscript set${ARG2}wz nosave nohostname <<< 'xcat `find server*|grep -a GMCommand.*`|tr -d "\r";cat server*/Zone*/GMCommand*|tr -d "\r"'|sort -t , -k 2
}

GMToolCmds () { # ServerStatus -> List all the commands that used by GM Tool
   ssh accountdb 'psql $ACCOUNT_DB_NAME -c "copy (select time,gm_id,cmd_type,cmd_detail,reason from gm_tool_log order by time) to stdout csv;"'
}

GMToolAccounts () { # ServerStatus -> List all the GM Tool accounts
   ssh accountdb "psql -P border=2 -P footer=0 $ACCOUNT_DB_NAME <<< 'select * from gm_tool_accounts;'"
}

IP () { # ServerStatus -> Show all IP and ports list
   if grep -a -q csv <<< "$ARG2" ; then

	for server in $(print_hosts GAMEDB) ; do
		ssh $server "psql $GAME_DB_NAME$(get_set_id $server) postgres -c 'copy (select id,name,ext_address,ext_port,int_address,int_port from serverstatus order by ext_address) to stdout with csv header;'"|sed "s/^\([^a-z]\)/Set $(get_set_id $server),\1/g;s/id,/set,id,/g"
	done

	echo
	echo or
	echo
	sendscript allall <<< "ifconfig|grep -a 'inet addr'|grep -a -v 127.0.0.1"|awk -F: '{print $1,$2}'|sed 's/ \+inet addr //g;s/Bcast//g'|sed ':a;N;$!ba;s/\n//g;s/\([A-Z]\{2,\}\)/|\1/g;s/ \+/,/g;s/_|/_/g'|tr '|' '\n'

   else

	echo "/etc/hosts:" | colorize yellow black
	sed -n '/BELOW/,/ABOVE/p' /etc/hosts|egrep -a "^[0-9]|^##"
	echo

        for LOGIN_SERVER_HOSTNAME in $(print_hosts LOGIN) ; do
  	    if [ "$NAT" == "1" ] || [ "$ONLY_ONE_NETWORK_INTERFACE" == "1" ] ; then
		    LOGIN_SERVER_EXT_IP="$(get_nat_ext_ip $(print_serverip $LOGIN_SERVER_HOSTNAME))"
	    else
		    LOGIN_SERVER_EXT_IP="$(ssh $LOGIN_SERVER_HOSTNAME "another_ip $(print_serverip $LOGIN_SERVER_HOSTNAME)")"
	    fi

	    echo "$LOGIN_SERVER_HOSTNAME: LoginServer ports(External IP: $LOGIN_SERVER_EXT_IP)" | colorize yellow black
	    ssh $LOGIN_SERVER_HOSTNAME "grep -a -h LoginServerPort */*/setup.ini"|sort|uniq
	    echo
        done

	echo "Connects.ini for LIVE server:" | colorize yellow black
	NUM_OF_STREAM=1
        for LOGIN_SERVER_HOSTNAME in $(print_hosts LOGIN) ; do
  	    if [ "$NAT" == "1" ] || [ "$ONLY_ONE_NETWORK_INTERFACE" == "1" ] ; then
		    LOGIN_SERVER_EXT_IP="$(get_nat_ext_ip $(print_serverip $LOGIN_SERVER_HOSTNAME))"
	    else
		    LOGIN_SERVER_EXT_IP="$(ssh $LOGIN_SERVER_HOSTNAME "another_ip $(print_serverip $LOGIN_SERVER_HOSTNAME)")"
	    fi

	    for LOGIN_PORT in $(ssh $LOGIN_SERVER_HOSTNAME "grep -a -h LoginServerPort */*/setup.ini"|sort|uniq|awk -F= '{print $NF}') ; do
		    echo "Stream$(seq -w 99|head -$NUM_OF_STREAM|tail -1)=$LOGIN_SERVER_EXT_IP,$LOGIN_PORT"
                    NUM_OF_STREAM=$(($NUM_OF_STREAM+1))
	    done
        done
	echo

	for server in $(print_hosts GAMEDB) ; do
		print_servername $server | colorize yellow black
		ssh $server "psql $GAME_DB_NAME$(get_set_id $server) postgres -c 'select id,name,ext_address,ext_port,int_address,int_port from serverstatus where int_port>0 and ext_port>0' -P border=2 -P border=2"
	done
   fi
}

DBvacuum () { # Maintenance -> Vacuum all the databases on live server
   SERVER_PROCESS="$(sendscript allall nosave nohostname <<< "source ~/.gamerc;all_running_servers|grep -a -v TicketServer"|wc -l)"
   if [ "$SERVER_PROCESS" == "0" ] ; then
	echo "Clean, no any server running." | colorize green
	sendscript alldb -p <<< 'psql -U postgres <<< "vacuum analyze;vacuum full;" 2>&1 | tee /tmp/VacuumDB.log'
   else
	echo "      YOU ARE TRYING TO DO VACUUM ON THE RUNNING SERVERS     " | colorize red black
	echo "  THIS WILL CAUSE DATA LOSS AND LOCKS TABLE ON THE DATABASE  " | colorize red black
	exit
   fi
}

Channels () { # ServerStatus -> Show channels and current players
   PGPASSWORD="$LIVE_SERVER_DB_PASSWORD" psql -h accountdb -P border=2 -P footer=0 $ACCOUNT_DB_NAME <<< "select * from worlds order by id;select sum(online_user) from worlds;"
}

DiskSpace () { # ServerStatus -> Report the disk spece usage
   sendscript allall <<< 'df -h .|grep -a "% /"|awk "{print \"Used:\"\$(NF-3)\"(\"\$(NF-1)\"), Free:\"\$(NF-2)}"'
}

WSTraffic () { # ServerStatus -> Show current traffic on all WS servers
   sendscript allws <<< "traffic $WS_EXT_IF"
}

DBConnCheck () { # ServerStatus -> Check the database connections between servers
   echo "Checking connect to ACCOUNTDB:" | colorize cyan
   sendscript all nohostname <<< "source ~/.gamerc;echo -en 'Connect from '\$HOST_NAME' to ACCOUNTDB ...                        '|head -c 55|colorize yellow;PGPASSWORD='$LIVE_SERVER_DB_PASSWORD' psql -t -h ACCOUNTDB $ACCOUNT_DB_NAME spiritking <<< '\q' && echo 'Return '\$?', OK'|colorize green || echo 'Return '\$?', FAILED, maybe unable to connect or no permission.'|colorize red"
   echo "Checking connect to GAMEDBs:" | colorize cyan
   for world in $WORLD_NUMBER_LIST;do
    set=$((world / 100))
	sendscript ws${world} nohostname <<< "source ~/.gamerc;echo -en 'Connect from '\$HOST_NAME' to GAMEDB$set ...                      '|head -c 55|colorize yellow;PGPASSWORD='$LIVE_SERVER_DB_PASSWORD' psql -t -h GAMEDB$set $GAME_DB_NAME$set spiritking <<< '\q' && echo 'Return '\$?', OK'|colorize green || echo 'Return '\$?', FAILED, maybe unable to connect or no permission.'|colorize red"
   done
   echo "Checking connect to WORLDDBs:" | colorize cyan
   for world in $WORLD_NUMBER_LIST;do
	sendscript ws${world} nohostname <<< "source ~/.gamerc;echo -en 'Connect from '\$HOST_NAME' to WORLDDB${world} ...                      '|head -c 55|colorize yellow;PGPASSWORD='$LIVE_SERVER_DB_PASSWORD' psql -t -h WORLDDB${world} $WORLD_DB_NAME${world} spiritking <<< '\q' && echo 'Return '\$?', OK'|colorize green || echo 'Return '\$?', FAILED, maybe unable to connect or no permission.'|colorize red"
   done
   if [ "${ENABLE_SOCIETY}" -eq 1 ]; then
    echo "Checking connect to SOCIETYDBs:" | colorize cyan
    for set in $SET_NUMBER_LIST;do
	    sendscript SocietyServer${set} nohostname <<< "source ~/.gamerc;echo -en 'Connect from '\$HOST_NAME' to SOCIETYDB$set ...                      '|head -c 55|colorize yellow;PGPASSWORD='$LIVE_SERVER_DB_PASSWORD' psql -t -h SOCIETYDB$set $SOCIETY_DB_NAME$set spiritking <<< '\q' && echo 'Return '\$?', OK'|colorize green || echo 'Return '\$?', FAILED, maybe unable to connect or no permission.'|colorize red"
    done
   fi
}

demofunc () { ## DEMO -> demo function, doing nothing
   echo "demo, doing nothing"
}

GCPFW () { # CloudServices -> Firewall Setting for GCP
   echo "Allow ports: "
   sendscript allall nohosname <<< "cat ~/.ports_allow 2> /dev/null" | sort | uniq | awk "{printf \"tcp:\"\$1\"; \"}END{print \"\"}"
   echo "Internal only ports(GMTools and FastCGI): "
   sendscript allall nohosname <<< "cat ~/.ports_internal_only 2> /dev/null" | sort | uniq | awk "{printf \"tcp:\"\$1\"; \"}END{print \"\"}"
}

PMStart () { # PerfMonitor -> Start Perf and Create Snapshot every 10 seconds
   echo "Perf Monitor Start"
   sendscript allall <<< "cd /tmp/; nohup $HOME/bin/open_perf &"
}

PMStop () { # PerfMonitor -> Stop Perf and Stop Snapshot Catching
   echo "Perf Monitor Stop"
   sendscript allall <<< "cd /tmp/; nohup $HOME/bin/close_perf &"
}

# Read customized function if exist
[ -f "$CUSTFUNC" ] && source $CUSTFUNC


###################################
# Start run the Command from here #
###################################
if [ "$ACTION" == "show_help" ] ; then
	[ -f ~/.hosts ] && cat ~/.hosts ~/bin/hosts > /etc/hosts 2> /dev/null
	echo "Notice: /etc/hosts updated!" | colorize green
	show_help
else
	if grep -a -q "$ACTION" <<< "$DONT_ASK_YES_NO" ; then
		$ACTION
	else
		yesno2 $ACTION "$DESCRIPTION"
	fi
fi

rm -f $CUSTFUNC

# Log the time on the end
DATE_NOW="$(date +%Y/%m/%dT%H:%M:%S)"
echo -e "$DATE_NOW\t=End=\t$DO\tAns:$ANSWER\tLogin:$LOGIN_USER\tUser:$USER\tTTY:$TTY\tIP:$SOURCEIP\tARG:$*" >> "$CTRL_LOGFILE"

logger $0 =End= $DO $ANSWER <<< $*
