#!/bin/bash

#  bru_restart.sh -> restart_cpu.sh
#
#  The purpose of this script library is to aid in the deployment of the
#  most recent update to the buildroot image that various LinuxRT and
#  Centos7 CPUs throughout the SLAC facilities are using.
#
#  It will be sourced by the main "driver" script, buildroot_update, to
#  aid in restarting the CPUs that need to be rebooted in order for
#  the update to take effect (or to reboot to revert should the update
#  not work on a given CPU for some reason).

#  There's a Confluence page associated with this and it's as follows:
#
#       https://confluence.slac.stanford.edu/display/~jmpdude/buildroot_update
#
#  Note: The original idea of this, as noted above, changed. At the moment,
#  this is now a standalone script.

#----------------------------------------------------------------------------
#  Constants
#----------------------------------------------------------------------------

#
# Note: Functions designed to be called from outside this file are
#       prepended with "BRU_RESTART_" while the rest are not.
#

# A given CPU node's IPMI node is the same name as the CPU itself plus
# "-mgt" appeneded to it. Since we attempt to use the IPMI node to
# reboot with, we need to make sure the CPU's IPMI tool node is
# available.
#
readonly CPU_IPMI_TOOL_CPU_NODE_EXT="-mgt"

readonly CPU_IPMI_TOOL_CMD_WAIT_SECONDS="30"
readonly CPU_IMPM_TOOL_CMD_STATUS="status"
readonly CPU_IPMI_TOOL_CMD_OFF="off"
readonly CPU_IPMI_TOOL_CMD_ON="on"

# If we're not passed how many seconds to wait for a given node, wait
# for two and half minutes (it takes longer on CentOS7 CPUs than it does
# for the LinuxRT ones).
#
readonly MAX_NODE_WAIT_SECONDS=150
readonly MIN_NODE_WAIT_SECONDS=5

readonly NODE_WAIT_SECONDS_PING=${MIN_NODE_WAIT_SECONDS}
readonly NODE_WAIT_SECONDS_REBOOT=${MAX_NODE_WAIT_SECONDS}

#----------------------------------------------------------------------------
#  Functions
#----------------------------------------------------------------------------

#
# Note: Functions designed to be called from outside this file are
#       prepended with "bru_restart_" while the rest are not.
#

#
# TODO: All of the timeout_wait_for_xxx() functions could be refactored
#       such that the core waiting functionality is the same but each
#       of the individual items being wait for is parameterized.
#

#----------------------------------------------------------------------------
#  timeout_wait_for_sh($1=cpu node, $2=max seconds to wait)
#
#  Using ssh, wait until either the node we're looking for shows up
#  or we time out waiting for it show up.
#----------------------------------------------------------------------------
timeout_wait_for_sh()
{
    # Assume failure.
    #
    local result=1

    # Get arguments.
    #
    local ip_address="$1"
    local seconds="$2"

    if [ -z "$seconds" ]; then
        seconds=$MAX_NODE_WAIT_SECONDS
    fi

    local before_ssh_seconds after_ssh_seconds elapsed_seconds sleep_seconds
    local delay_seconds=5
    local saved_seconds=$seconds

    # Wait until the node at the IP address shows up or we time out.
    #
    while true; do

        # Use seconds as a countdown as we only ssh for a second
        # each time.
        #
        if [ "$seconds" -le 0 ]; then
            break
        fi

        # Print out how long we've been at this on the same line.
        #
        printf "\rwaiting, seconds<%03d>" $(( $saved_seconds - $seconds ))

        before_ssh_seconds=$(date +%s)

        # Attempt to ssh into the node we're waiting for (ssh'ing into
        # it, simply asking for the date).
        #
        ssh -o ConnectTimeout=$delay_seconds -o ConnectionAttempts=1 -o StrictHostKeyChecking=no ${IOC_OWNER}@${ip_address} 'date' 1>/dev/null 2>&1
        result=$?

        after_ssh_seconds=$(date +%s)

        # Stop if the node can be ssh'd into.
        #
        if [ $result -eq 0 ]; then
            break
        fi

        # Determine how long we've been at this.
        #
        elapsed_seconds=$(( $after_ssh_seconds - $before_ssh_seconds ))

        # Sleep the remainder if the last ssh came back in less than the delay seconds.
        #
        if [ $elapsed_seconds -lt $delay_seconds ]; then
            sleep_seconds=$(( $delay_seconds - $elapsed_seconds ))
            sleep $sleep_seconds
        fi

        let seconds-=$delay_seconds

    done

    # Let the seconds we've been at this persist.
    #
    printf "\r\n"

    return $result

}

#----------------------------------------------------------------------------
#  timeout_wait_for_ip($1=ip address, $2=max seconds to wait)
#
#  Using ping, wait until either the node we're waiting for shows up
#  or we time out waiting for it show up.
#----------------------------------------------------------------------------
timeout_wait_for_ip()
{
    # Assume failure.
    #
    local result=1

    # Get arguments.
    #
    local ip_address="$1"
    local seconds="$2"

    if [ -z "$seconds" ]; then
        seconds=$MAX_NODE_WAIT_SECONDS
    fi

    local saved_seconds=$seconds

    # Wait until the node at the IP address shows up or we time out.
    #
    while true; do

        # Use seconds as a countdown as we only ping for a second each
        # time.
        #
        if [ "$seconds" -le 0 ]; then
            break
        fi

        # Ping the IP address we're waiting for.
        #
        ping -w 1 "$ip_address" >/dev/null
        result=$?

        # Print out how long we've been at this on the same line.
        #
        let seconds-=1

        printf "\rwaiting, seconds<%03d>" $(( $saved_seconds - $seconds ))

        # Stop if the node is pingable.
        #
        if [ $result -eq 0 ]; then
            break
        fi

        # Sleep to let us be interrupted easily since we're in a loop.
        #
        let seconds-=1
        sleep 1

    done

    # Let the seconds we've been at this persist.
    #
    printf "\r\n"

    return $result
}

#----------------------------------------------------------------------------
#  is_node_available($1=node to be ping'd for availability, $2=max seconds to wait)
#
#  This function should probably be sourced from somewhere else. But, for
#  now, it's here. It first does an nslookup to see whether the
#  passed-in node even has a reachable address and, if it does, it pings
#  it for availability.
#
#  It returns true (bash 0) if the node is available or false (bash
#  non-zero) otherwise.
#----------------------------------------------------------------------------
is_node_available()
{
    # Assume failure.
    #
    local result=1

    # Get arguments.
    #
    local node="$1"
    local seconds="$2"

    # The output of nslookup returns the address of the DNS it's using
    # as well as the address of the node we requested. The node we
    # requested will be the 2nd one. So, we grep the nslookup for
    # "Address:" but we want the 2nd one (record) of those.
    #
    local ip_address=$(nslookup "${node}" | grep 'Address:' | awk 'NR==2 {print $2}')

    # If we got an IP address...
    #
    if [ -n "$ip_address" ]; then

        # ...see if it's a available.
        #
        timeout_wait_for_ip $ip_address $seconds
        result=$?
    fi

    return $result
}

#----------------------------------------------------------------------------
#  wait_ipmiCmd_complete($1=CPU to send ipmiCmd to, $2=ipmiCmd)
#
#  Issue the specified ipmiCmd to the requested CPU's IPMI node, waiting
#  until the status from that command comes back and says that it's now
#  in the state we requested, or it times out.
#
#  We're using the ipmiCmd tool. It does NOT check to see whether the
#  CPU's IPMI node is available and, if it's not, it will hang (which is
#  why the top-level call checks that first before calling this).
#----------------------------------------------------------------------------
wait_ipmiCmd_complete()
{
    # Assume failure.
    #
    local result=1

    local cpu_node=$1
    local cmd=$2

    # Seconds to wait for command to complete.
    #
    local seconds=${CPU_IPMI_TOOL_CMD_WAIT_SECONDS}
    local status

    # Spin the cursor while waiting.
    #
    local spin="-\|/"
    local spun=false
    local i=0

    # The format for commands for ipmiCmd is:
    #
    #   ipcmCmd -c <cpu-node> -f <command>
    #
    # Here, we'll be issuing the on, off, or status commands.
    #
    # First, issue the specified command, piping its output to
    # /dev/null and exiting if the command failed.
    #
    echo "cpu<$cpu_node> IPMI cmd<$cmd>?"

    ipmiCmd -c $cpu_node -f $cmd 1>/dev/null 2>&1

    if [ $? -ne 0 ]; then
        echo "cpu<$cpu_node> IPMI cmd<$cmd>: failed, aborting"
        return $result
    else
        echo "cpu<$cpu_node> IPMI cmd<$cmd>: succeeded, waiting to complete"
    fi

    # Now, issue the status command, waiting until it either completes
    # successfully or times out.
    #
    while true; do

        # Get the current status, piping the error output to /dev/null.
        #
        status=$(ipmiCmd -c $cpu_node -f ${CPU_IMPM_TOOL_CMD_STATUS} 2>/dev/null)

        # If we couldn't get the status...
        #
        if [ $? -ne 0 ]; then

            # ...quit.
            #
            break
        fi

        # Read the status, checking to see if it matches the command we
        # just sent yet.
        #
        status=$(echo $status | awk '{sub(/^Chassis Power is /, ""); print}')

        if [ "$status" == "$cmd" ]; then
            result=0
            break

        else

            # Spin the cursor while we're waiting.
            #
            if [ $seconds -eq 0 ]; then
                break
            else
                let seconds-=1
            fi

            i=$(( (i+1) % 4 ))
            printf "\r${spin:$i:1}"
            spun=true

            sleep 1
        fi

    done

    if ( $spun ); then
        printf "\b"
    fi

    if [ $result -eq 0 ]; then
        echo "cpu<$cpu_node> IPMI cmd<$cmd>: succeeded, completed"
    else
        echo "cpu<$cpu_node> IPMI cmd<$cmd>: failed to complete, aborting"
    fi

    return $result
}

#----------------------------------------------------------------------------
#  reboot_cpu_node($1=CPU to be rebooted, $2=has_ipmi)
#
#  We're using the IPMI tool and, in particular, the ipmiCmd variant.
#  The ipmiCmd variant hides some of the details. It can do all sorts of
#  fancier things, like a full cycle. But, for our needs, we just want to
#  simulate a reboot (power off then power back on). We'll issue the
#  power off command first, check to see that it's indeed powered off.
#  And, once it's powered off, attempt to power it back on. If either
#  one of the commands fails, abort.
#----------------------------------------------------------------------------
reboot_cpu_node()
{
    local cpu_node=$1
    local has_ipmi=$2

    # If we have the IPMI tool available...
    #
    if ( $has_ipmi ); then

        # ...attempt to use it!
        #
        # The format for commands for ipmiCmd is:
        #
        #   ipcmCmd -c <cpu-node> -f <command>
        #
        # We'll only be using the off and on commands here. Basically, after
        # issuing a command, we'll wait for it to complete before moving on.
        #
        # Regardless of its current state, issue the "off" command first.
        #
        echo "cpu<$cpu_node> IPMI power off?"

        wait_ipmiCmd_complete $cpu_node ${CPU_IPMI_TOOL_CMD_OFF} >/dev/null

        # If that fails, quit now, saying that it failed.
        #
        if [ $? != 0 ]; then
            echo "cpu<$cpu_node> IPMI power off: failed, aborting"
            return 1
        else
            echo "cpu<$cpu_node> IPMI power off: succeeded"

            # Sleep for 2 seconds to give IPMI a chance to recover
            # (experience has shown that we can't power back on
            # immediately after a power off, perhaps there is a
            # mechanical process involved?).
            #
            sleep 2
        fi

        # Now, wait for it come back by issuing the "on" command.
        #
        echo "cpu<$cpu_node> IPMI power  on?"

        wait_ipmiCmd_complete $cpu_node ${CPU_IPMI_TOOL_CMD_ON} >/dev/null

        # Return its status.
        #
        if [ $? -ne 0 ]; then
            echo "cpu<$cpu_node> IPMI power  on: failed, aborting"
            return 1
        else
            echo "cpu<$cpu_node> IPMI power  on: succeeded"
            return 0
        fi

    else

        # Otherwise, attempt to reboot the CPU with the reboot command
        # itself. Note that using "reboot -f" causes ssh to hang because
        # the "-f" forces the reboot to bypass any of the communication
        # across the software stack that a reboot is occurring.
        #
        ssh ${IOC_OWNER}@${cpu_node} 'su root -c reboot' 1>/dev/null 2>&1

        # We need to about 15 seconds for the reboot to lose connection.
        #
        local delay_seconds=15
        if [ $? -eq 0 ]; then
            printf "waiting, seconds<%03d>\n" $delay_seconds
            sleep $delay_seconds
        else
            return 1
        fi

    fi
}

#----------------------------------------------------------------------------
#  bru_restart_reboot_cpu($1=CPU to be rebooted)
#
#  By the time we get here, we assume that the CPU (node) that we need
#  to reboot is already in the correct format for passing off to the
#  IPMI tool as well as will be recognized as a named node on the network.
#
#  At the moment, we attempt to use the IPMI tool first because there
#  was an issue where issuing the reboot command would actually cause a
#  kernel panic.
#
#  So, in order to reset the requested CPU, we check to see if the
#  requested CPU's IPMI tool is available. If it is, regardless of
#  whether the requested CPU is available, we attempt to reboot (power
#  cycle) the CPU.
#
#  If the IPMI tool is unavailable, we then fall back and attempt to
#  issue the reboot command on the CPU itself.
#
#  TODO: The string output is highly common, factor that out in some way.
#----------------------------------------------------------------------------
bru_restart_reboot_cpu()
{
    local has_ipmi=false
    local cpu_node=$1

    # Get the name of the CPU's IPMI tool node.
    #
    local cpu_node_mgt="${cpu_node}${CPU_IPMI_TOOL_CPU_NODE_EXT}"

    # If the CPU's IPMI tool node is available...
    #
    echo "cpu<$cpu_node> IPMI node<$cpu_node_mgt> available?"

    if ( is_node_available $cpu_node_mgt ${NODE_WAIT_SECONDS_PING} ); then

        # ...say so.
        #
        echo "cpu<$cpu_node> IPMI node<$cpu_node_mgt> available: yes"
        echo ""

        # And note that we have IPMI tool available.
        #
        has_ipmi=true

    else

        # Otherwise, note that we don't have the IPMI tool node available.
        #
        echo "cpu<$cpu_node> IPMI node<$cpu_node_mgt> available: no"
        echo ""
    fi

    # If the CPU is available...
    #
    echo "cpu<$cpu_node> available?"

    if ( is_node_available $cpu_node ${NODE_WAIT_SECONDS_PING} ); then

        # ...say so.
        #
        if ( $has_ipmi ); then
            echo "cpu<$cpu_node> available: yes, attempting to power cycle"
        else
            echo "cpu<$cpu_node> available: yes, attempting to reboot"
        fi

    else

        # Otherwise, if the IPMI tool node is available, attempt to power cycle it.
        #
        if ( $has_ipmi ); then
            echo "cpu<$cpu_node> available: no, attempting to power cycle"
        else
            echo "cpu<$cpu_node> available: no, aborting"
        fi
    fi

    # If we were able to reboot or power cycle the CPU...
    #
    if ( reboot_cpu_node $cpu_node $has_ipmi ); then

        # ...wait for it to come back and return whether it does or not.
        #
        echo ""
        echo "cpu<$cpu_node> up?"

        if ( is_node_available $cpu_node ${NODE_WAIT_SECONDS_REBOOT} ); then
            echo "cpu<$cpu_node> up: yes"
            echo "cpu<$cpu_node> sh?"

            if ( timeout_wait_for_sh $cpu_node ${NODE_WAIT_SECONDS_REBOOT} ); then
                echo "cpu<$cpu_node> sh: yes"
                return 0

            else

                echo "cpu<$cpu_node> sh, no, aborting"
                return 1

            fi

        else

            echo "cpu<$cpu_node> up: no, aborting"
            return 1

        fi

    else

        # Otherwise, return that we were unsuccessful.
        #
        echo ""
        echo "cpu<$cpu_node> failed to reboot, aborting"

        return 1
    fi

}

#----------------------------------------------------------------------------
#  Main
#----------------------------------------------------------------------------

usage()
{
    echo "Restarts (power cycles or reboots) a CPU"
    echo "Usage: $0 cpu_name"
    echo ""
    echo "Positional arguments:"
    echo "  cpu_name       The CPU nodename, e.g., cpu-b34-pm01"

    exit 1
}

main()
{
    # Parse for CPU argument and bail if we don't get one.
    #
    if [ "$#" -ne 1 ]; then
        usage
    fi

    # Call the restart function with the passed-in CPU name.
    #
    # TODO: It would be great to validate the passed-in CPU name.
    #
    bru_restart_reboot_cpu $1

    # If we succeeded, wait a few seconds for all of the SIOCs to finish
    # coming up.
    #
    # TODO: It would be REALLY great if we could actually check whether
    # or not the SLAC boot sequence has completed.
    #
    local delay_seconds=20
    if [ $? -eq 0 ]; then
        printf "waiting, seconds<%03d>\n" $delay_seconds
        sleep $delay_seconds
    fi

    # Clear out the possibly old SSH credentials for the previous buildroot.
    ssh-keygen -R $1

    # Issue an ssh request to the node to get upgraded credentials.
    ssh -o StrictHostKeyChecking=no ${IOC_OWNER}@$1 'date' 1>/dev/null 2>&1
}

# Execute main as a function to localize references.
#
main "$@"
exit 0


