#!/bin/bash

################################################################################
################################################################################
# Copyright 2023-2025 by NI SP Software GmbH, All rights reserved.
# Copyright 1999-2023 by Nice, srl., All rights reserved.
#
# This software includes confidential and proprietary information
# of NI SP Software GmbH ("Confidential Information").
# You shall not disclose such Confidential Information
# and shall use it only in accordance with the terms of
# the license agreement you entered into with NI SP Software.
################################################################################
################################################################################

################################################################################
# $EF_ROOT/plugins/lsfmc/bin/lsload - wrapper to lsf commands to create GridML
################################################################################


#-------------------------------------------------------------------------------
# Useful for debugging purposes
#-------------------------------------------------------------------------------
EF_PLUGIN_ACTION="LSF Host Load"
export EF_PLUGIN_ACTION

#-------------------------------------------------------------------------------
# eflsf_main() - main entry point of this script
#-------------------------------------------------------------------------------
eflsf_main(){
  # call initialization function
  eflsf_init "$@"

  case "${LSF_MULTICLUSTER}" in
    "false") eflsf_lsload "$@" ;;
    "true") eflsf_multicluster "$@" ;;
  esac
}

#-------------------------------------------------------------------------------
# cluster_start() - opens cluster-list tag and cluster tag
#-------------------------------------------------------------------------------
cluster_start(){
  if [ -z "${lsf_disable_cluster_listing}" ] ; then
    echo "<grid:cluster-list type=\"lsf\" ${EF_XMLNS_grid}>"
    echo "<grid:cluster name=\"$(ef_xml_escape_attribute -i "$1")\" type=\"lsf\">"
  fi
}

#-------------------------------------------------------------------------------
# cluster_end() - closes cluster-list tag and cluster tag
#-------------------------------------------------------------------------------
cluster_end(){
  if [ -z "${lsf_disable_cluster_listing}" ] ; then
    echo "</grid:cluster>"
    echo "</grid:cluster-list>"
  fi
}


#-------------------------------------------------------------------------------
# eflsf_multicluster() - Multicluster extensions to original lsload
#-------------------------------------------------------------------------------
eflsf_multicluster(){

  # read command line parameters with a regular expression which works on
  # strings like host, @cluster or host@cluster: \([^@]*\)@\{0,1\}\(.*\)
  # Read it as made of three different pieces:
  # \([^@]*\) = any character except @ repeated any number of times
  # @\{0,1\}       = zero or one instance of character @
  # (.*\)     = any character repeated any number of times
  _host=`echo "${@}" | sed 's/\([^@]*\)@\{0,1\}\(.*\)/\1/'`
  _cluster=`echo "${@}" | sed 's/\([^@]*\)@\{0,1\}\(.*\)/\2/'`

  # no command line parameters
  # => show all clusters
  if [ -z "${_host}" -a -z "${_cluster}" ] ; then
    if [ -n "${lsf_disable_cluster_listing}" ] ; then
      # No cluster list, just print the host list of the local cluster
      eflsf_lsload
    else
      echo "<grid:cluster-list type=\"lsf\" xmlns:grid=\"http://www.enginframe.com/2000/GRID\">"
      # loop among all clusters
      for _current_cluster in ${EF_LSFCLUSTERS} ; do
        # Tag <grid:cluster> is empty because it will be filled by an AJAX call
        echo "<grid:cluster name=\"$(ef_xml_escape_attribute -i "${_current_cluster}")\" type=\"lsf\"/>"
      done
      echo "</grid:cluster-list>"
    fi

  # command line parameters = hostname
  # => show details on a single host
  elif [ -n "${_host}" -a -z "${_cluster}" ] ; then
    cluster_start "${LSF_CLUSTER_ID}"
    eflsf_lsload "${_host}"
    cluster_end

  # command line parameters = hostname@mycluster or @mycluster
  # => show my cluster or details on a single host of my cluster
  elif [ "${_cluster}" = "${LSF_CLUSTER_ID}" ] ; then
    cluster_start "${LSF_CLUSTER_ID}"
    eflsf_lsload "${_host}"
    cluster_end

  # Else we need information on remote hosts/clusters.
  # If we are not running on LSF Master, then do lsrun to run on LSF master
  # This is necessary because only the masters are supposed to see each other.
  # Your actual MultiCluster could be configured in a different way,
  # but we cannot know it in advance.
  # This implies that $EF_ROOT/plugins/lsf MUST be shared at least between
  # EnginFrame Agent host and LSF Master.
  elif [ "${IamMaster}" = "false" ] ; then
    # We need to move to a directory that surely exists on LSF Master.
    # /tmp is the best choice, because it is surely present on any UNIX flavour
    # and if some day we will need a temporary working directory, it's the right place
    # Ok, spoolers should be there, but you never know...
    cd /tmp
    # unset that strange EF_LSF_DO_NOT_RECURSE beucase lsf propagates variables.
    unset EF_LSF_DO_NOT_RECURSE
    # STDIN is /dev/null because on some environment lsrun breaks
    # the script if you do not provide something to STDIN, even null.
    _out=`${EF_LSF_LSRUN} -m "${EF_MYLSFMASTER}" "${EF_ROOT}/plugins/lsf/bin/lsload" "$@" </dev/null 2>&1`
    if [ ! "$?" = "0" ] ; then
      _msg1="There was an error while executing lsrun on LSF Master ${EF_MYLSFMASTER}."
      _msg2="Please contact your EnginFrame Administrator."
      _msg3="The Standard Output was:"
      ef_error "`printf \"%s\n%s\n\n%s\n%s\" \"${_msg1}\" \"${_msg2}\" \"${_msg3}\" \"${_out}\"`" \
               "LSF Plugin Error" \
               "lsrun -m \"${EF_MYLSFMASTER}\" \"${EF_ROOT}/plugins/lsf/bin/lsload\" \"$@\""
      exit 1
    fi
    echo "${_out}"
    cd "${EF_SPOOLER}"
    exit 0
  
  # now we are surely running on LSF Master, so we can continue our if-then-elif

  # command line parameters = hostname@remotecluster
  # => show details on a remote host
  elif [ -n "${_host}" ] ; then
    cluster_start "${_cluster}"
    lsfmc_host "${_host}" "${_cluster}"
    cluster_end

  # command line parameters: @remotecluster
  # => show remote cluster info
  elif [ -z "${_host}" ] ; then
    cluster_start "${_cluster}"
    eflsf_lsload "${_cluster}"
    cluster_end
  fi

}

#-------------------------------------------------------------------------------
# lsfmc_host() - extract info about remote host
# we cannot use directly lsload from LSF plugin, because bhosts does not work
# for remote hosts as lsload or lshost (IMHO this is a bug of LSF)
#-------------------------------------------------------------------------------
lsfmc_host(){
  _myhost="$(ef_xml_escape_attribute -i "$1")"
  _mycluster="$2"

  eflsf_lsload "${_mycluster}" | $EF_AWK -v host="${_myhost}" '
  BEGIN {pattern="<grid:host type=\"lsf\" name=\"" host "\""}
  /<grid:host-list|<\/grid:host-list/ {print}
  { if (index($0,pattern)>0) {
      flag=1
    }
    if (index($0,"</grid:host>")>0) {
      if(flag==1) {
        print
      }
      flag=0
    }
    if (flag==1) {
      print
    }
  }'
}

#-------------------------------------------------------------------------------
# eflsf_init() - initialize everything
#-------------------------------------------------------------------------------
eflsf_init(){

  # Source common file
  . "${EF_ROOT}/plugins/lsf/bin/common"

  if [ -r "${LSF_ENVDIR}/lsf.shared" ]; then
    EF_LSFCLUSTERS=`$EF_AWK '

    # Match the end of cluster section and unset the flag
    $0 ~ /^[ \t]*End Cluster/ {
      flag=0
    }

    # if the flag is set, append the cluster name
    # if the row is not commented out, of course
    $1 !~ /^#/ {
      if (flag==1) printf $1 " "
    }

    # Match the beginning of cluster section and set the flag
    $0 ~ /^[ \t]*ClusterName/ {
      flag=1
    }

    ' "${LSF_ENVDIR}/lsf.shared"`
  fi

  # Where are we running?
  EF_MYHOSTNAME="`hostname`"

  # Am I running on the LSF MASTER?
  IamMaster="false"
  for _domain in "" ${LSF_STRIP_DOMAIN}; do
    if [ "${EF_MYHOSTNAME}${_domain}" = "${EF_MYLSFMASTER}" ] ; then
      IamMaster="true"
      break
    fi
  done

}

#-------------------------------------------------------------------------------
# eflsf_lsload() - original wrapper to LSF commands
#-------------------------------------------------------------------------------
eflsf_lsload(){

[ -z "${_csortby}" ] && _csortby="status"

if [ -z "${cluster_sort_order}" ];
then
cat <<EOF
<ef:session>
  <ef:option id="cluster_sort_order">ascending</ef:option>
</ef:session>
EOF
cluster_sort_order="ascending"
fi

if [ "${cluster_sortby}" = "${_csortby}" -a -n "${cluster_sortby}" ];
then
  if [ "${cluster_sort_order}" = "ascending" ];
  then
cat <<EOF
    <ef:session>
      <ef:option id="cluster_sort_order">descending</ef:option>
    </ef:session>
EOF
    cluster_sort_order="descending"
  else
cat <<EOF
    <ef:session>
      <ef:option id="cluster_sort_order">ascending</ef:option>
    </ef:session>
EOF
    cluster_sort_order="ascending"
  fi
else

if [ ! "x${_csortby}" = "x" ]; then
cat <<EOF
<ef:session>
  <ef:option id="cluster_sort_order">ascending</ef:option>
</ef:session>
<ef:session>
  <ef:option id="cluster_sortby">$(ef_xml_escape_content -i "${_csortby}")</ef:option>
</ef:session>
EOF
cluster_sortby=${_csortby}
cluster_sort_order="ascending"
fi
fi

if [ "${cluster_sortby}" = "name" ];
then
    if [ "${cluster_sort_order}" = "ascending" ];
    then
        ADD_SORT=" | sort"
    else
        REVERSE=1
        ADD_SORT=" | sort -r"
    fi
elif [ -n "${cluster_sortby}" ];
then
    if [ "${cluster_sortby}" != "status" -a "${cluster_sortby}" != "njobs" ];
    then
        if [ "${cluster_sort_order}" = "ascending" ];
        then
            ADD_SORT=" -R \"order[${cluster_sortby}]\""
        else
            ADD_SORT=" -R \"order[-${cluster_sortby}]\""
        fi
    fi
fi


EF_PARAMS=''

while [ -n "$1" ]
do
    Machine=`echo $1 | sed 's/@.[^ ]*//g' |$EF_AWK '{
        if ($1 !~ /\*/)
            print $1;
        else
            print substr ($1,index($1,"*")+1);
    }'`
    [ -n "$Machine" ] && EF_PARAMS="${EF_PARAMS} \"$Machine\""
    shift
done

[ -z "${_cluster}" ] && _cluster=${LSF_CLUSTER_ID}

# A variable that contains only space and tabs, runtime generated using printf
_st=`printf " \t"`

# A variable containing a SED regexp for matching one or more space and/or tabs
_sp="[${_st}][${_st}]*"

(
        eval ${EF_LSF_LSHOSTS} -w ${EF_PARAMS} ${ADD_SORT} | sed -n -e "/${_sp}Yes${_sp}/p" -e "/${_sp}Dyn${_sp}/p" | sed 's/(//g;s/)//g'
        eval ${EF_LSF_BHOSTS} ${EF_LSF_BHOSTS_X} -w ${EF_PARAMS} | sed 1d
        LSLOAD_OUTPUT="`eval \"${EF_LSF_LSLOAD} -l ${EF_PARAMS} ${ADD_SORT}\"`"
        # Be sure the column names are on the first line
        echo "${LSLOAD_OUTPUT}" | grep HOST_NAME
        echo "${LSLOAD_OUTPUT}" | grep -v HOST_NAME | sed 's/[*%]/ /g'
) 2>/dev/null | sed 's/'`printf '\r'`'$//' | $EF_AWK \
  -v cluster="${_cluster}" \
  -v "archmap=${EF_ROOT}/plugins/lsf/conf/grid.arch.mapping" \
  -v "statusmap=${EF_ROOT}/plugins/lsf/conf/grid.host.status.mapping" \
  -f "${EF_ROOT}/plugins/ef/lib/awk/utils.awk" \
  -f <( echo '
BEGIN {
    print "<grid:host-list type=\"lsf\" xmlns:grid=\"http://www.enginframe.com/2000/GRID\" cluster=\"" cluster "\">";
    # Read arch mapping file
    while (getline < archmap > 0) {
      split($0,fields,":")
      s=split(fields[2],label," ")
      for (i=1;i<=s;i++) {
        arch[label[i]]=fields[1]
      }
    }
    close(archmap);
    # Read status mapping file
    while (getline < statusmap > 0) {
      split($0,fields,":")
      s=split(fields[2],label," ")
      for (i=1;i<=s;i++) {
        status[label[i]]=fields[1]
      }
    }
    close(statusmap);
}

END {
    print "</grid:host-list>";
}
/^HOST_NAME/    {
        for (i = 1; i <= NF; i++) {
                name[i] = $i;
        }
        next;
}
{
        if (count[$1] == 2) {
                print info[$1];
                print "    <grid:status ef=\"" efStatus[$1] "\" grid=\"" gridStatus[$1] "\" base=\"" escapeXmlAttribute($2) "\" batch=\"" gridStatus[$1] "\"/>";
            for (i = 3; i <= NF; i++) {
            range="";
            if (max[name[i],$1]) range=" max=\"" max[name[i],$1] "\"";
                    print "    <grid:resource name=\"" escapeXmlAttribute(name[i]) "\"" range ">" escapeXmlContent($i) "</grid:resource>";
            }

            #GPUs
            gpucmd="${EF_ROOT}/plugins/lsf/bin/lsf_gpu_info.sh " $1
            gpucmd | getline gpu_info;
            close(gpucmd, "to");
            gpuCount=split(gpu_info,gpu,",")

            if(gpuCount > 1)
            {
              print "    <grid:resource name=\"gpu_avg_ut\"" range ">" gpu[1]"</grid:resource>";
              print "    <grid:resource name=\"gpu_avg_mut\"" range ">" gpu[2]"</grid:resource>";
              gpuIdx = 0;
              for (idx = 3; idx <= gpuCount; idx++)
              {
                print "    <grid:resource name=\"gpu_idx_ut_" gpuIdx "\"" range ">" gpu[idx] "</grid:resource>";
                print "    <grid:resource name=\"gpu_idx_mut_" gpuIdx "\"" range ">" gpu[idx+1] "</grid:resource>";
                idx++
                gpuIdx++;
              }
            }

	    print "  </grid:host>"
        }
    if (count[$1] == 1) {
                info[$1] = info[$1] "    <grid:usage running-jobs=\"" $6 "\"/>";
                if ($4 != "-") {
                  info[$1] = info[$1] "\n    <grid:job-slots>" $4 "</grid:job-slots>";
                }
        gridStatus[$1] = $2;
        if (status[$2] == "" ) {
                    efStatus[$1] = "unknown";
                }
                else {
            efStatus[$1] = status[$2];
                }
                count[$1]++;
        }
    if (count[$1] == 0) {
                myArch = arch[$2]
                if (myArch=="") myArch="unknown"
                info[$1] = "  <grid:host type=\"lsf\" name=\"" escapeXmlAttribute($1) "\" model=\"" $3 "\" arch=\"" $2 "\" arch-family=\"" myArch "\" ncpus=\"" $5 "\">\n";
        max["mem",$1] = $6;
        max["swp",$1] = $7;
        max["ut",$1] = 100;
                count[$1]++;
        }
}
')
}

#-------------------------------------------------------------------------------
# go to main
#-------------------------------------------------------------------------------
eflsf_main "$@"
