wirawan0
/
WP-hpc-tools
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Custom HPC software & tools from Wirawan. Primarily tailored toward ODU HPC sytems.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
34 Commits
1 Branch
0 Tags
76 KiB
 
 
 
Tag: Branch: Tree: 95034685ff
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '95034685ff'
${ noResults }
WP-hpc-tools/sge/show-node-status.py

487 lines
16 KiB
Raw Blame History

#!/usr/bin/env python
#
# Created: 20160830
# Wirawan Purwanto
"""
show-node-status.py
---------------------
Various tools to investigate node status in an SGE cluster.
This tool is a replacement and upgrade of the shell version of the tool
`node-slot-status.sh`.
Usage:
"""
import os
import re
import subprocess
import sys
class ParseError(RuntimeError):
pass
class ProgramError(RuntimeError):
pass
MYSELF = 'show-node-status.py'
def node_slot_stats_raw(qstat_f, show_disabled_nodes=False):
"""Prints the node stats from `qstat -f' in raw format:
- not printing disabled nodes
- not showing the computational jobs that are running on these nodes
"""
FNR = 0
for L in qstat_f:
FNR += 1
FLDS = L.split()
status_flags = FLDS[5] if (len(FLDS) > 5) else ""
if FNR == 1 and FLDS[0] == "queuename":
print(L)
continue
# Valid host status field
if re.search(r'^[A-Za-z]', L) and len(FLDS) in (5,6) \
and (show_disabled_nodes or ("d" not in status_flags)):
print(L)
def node_slot_stats(qstat_f, show_disabled_nodes=False):
"""Prints status of slot availability per machine type (defined as
host with the same base hostname (e.g. "c6-", or "c8-").
Originally implemented based on the naming of hosts on Turing cluster.
In SGE terminology, "slot" means a CPU core.
Example output:
MACHTYPE NODE CORES used free resv
c6 15 240 77 163 0
c8 40 768 569 199 0
cr 74 1480 988 492 0
crhimem 3 96 0 96 0
crphi 10 200 48 152 0
d430 49 1568 1292 276 0
d730 10 280 10 270 0
(changes depending on what's disabled and the load of the cluster)
FIXME: If a machine is covered by more than one queue, this will
cause the counts to be overestimated. Must register if a machine has
been encountered and not re-account that machine.
However this may not be the best approach as queues are overlapping
on machines. Since on Turing, the practice is not to further split a
machine to multiple queues (i.e. a 32-core node have all the 32
cores assignable to both main and timed-main queues, rather than
dedicating 16 for main and 16 for timed-main), we use a particular
way to avoid the double-counting:
- slots_resv: total number of reserved slots in a node (for whatever
the sysadmin designates) -- sum them up
- slots_used: total number of slots currently used (i.e.,
occupied by jobs) -- sum them up
- slots_tot: total number of slots in a node -- take the maximum
value encountered.
Had the nodes split-dedicated to a particular queue, we have to
take the sum of the values instead.
"""
from pprint import pprint
host_stats = collect_host_stats(qstat_f, show_disabled_nodes)
print_host_stats(host_stats)
def node_slot_stats_per_machine_type(qstat_f, show_disabled_nodes=False):
"""Prints status of slot availability per machine type (defined as
host with the same base hostname (e.g. "c6-", or "c8-").
Originally implemented based on the naming of hosts on Turing cluster.
In SGE terminology, "slot" means a CPU core.
Example output:
MACHTYPE NODE CORES used free resv
c6 15 240 77 163 0
c8 40 768 569 199 0
cr 74 1480 988 492 0
crhimem 3 96 0 96 0
crphi 10 200 48 152 0
d430 49 1568 1292 276 0
d730 10 280 10 270 0
(changes depending on what's disabled and the load of the cluster)
FIXME: If a machine is covered by more than one queue, this will
cause the counts to be overestimated. Must register if a machine has
been encountered and not re-account that machine.
However this may not be the best approach as queues are overlapping
on machines. Since on Turing, the practice is not to further split a
machine to multiple queues (i.e. a 32-core node have all the 32
cores assignable to both main and timed-main queues, rather than
dedicating 16 for main and 16 for timed-main), we use a particular
way to avoid the double-counting:
- slots_resv: total number of reserved slots in a node (for whatever
the sysadmin designates) -- sum them up
- slots_used: total number of slots currently used (i.e.,
occupied by jobs) -- sum them up
- slots_tot: total number of slots in a node -- take the maximum
value encountered.
Had the nodes split-dedicated to a particular queue, we have to
take the sum of the values instead.
"""
from pprint import pprint
host_stats = collect_host_stats(qstat_f, show_disabled_nodes)
#pprint(host_stats)
hosttype_stats = summarize_hosttype_stats(host_stats)
#pprint(hosttype_stats)
print_hosttype_stats(hosttype_stats)
def collect_host_stats(qstat_f, show_disabled_nodes=None):
"""Internal routine to collect node stats from `qstat -f` by
combining node status that were printed for each `queue@hostname`
combinations.
The result is a dict with hostname as the key."""
host_stats = {}
#host_list_by_kinds = {}
def host_get_stats_rec(hostname):
if hostname not in host_stats:
s = {
'slots_resv': 0,
'slots_used': 0,
'slots_tot': 0,
'queues': [],
}
host_stats[hostname] = s
return host_stats[hostname]
FNR = 0
for L in qstat_f:
FNR += 1
FLDS = L.split()
status_flags = FLDS[5] if (len(FLDS) > 5) else ""
if FNR == 1 and FLDS[0] == "queuename":
continue
# Valid host status field
if re.search(r'^[A-Za-z]', L) and len(FLDS) in (5,6):
# This line has a format like this:
# main@c8-014.cm.cluster BIP 0/10/16 9.98 linux-x64 d
# ^ queue & node name
queue_node, queue_type, core_usage_combo, node_load, os_arch \
= tuple(FLDS[0:5])
try:
node_load = float(node_load)
except ValueError:
node_load = 0
# status flags, see above
# skip disabled hosts
if ("d" in status_flags) and not show_disabled_nodes:
continue
#if (optPrintRaw != 0) print($0)
# Extract more useful info
m = re.search(r'^([^@]+)@([^-]+)-([^.]*)((?:\..*)?)$', queue_node)
if not m:
raise ParseError, \
"Invalid queue/host combo on line %D: %s" % (FNR, queue_node)
queue, hostkind, hostnum, hostdomain = m.groups()
hostname = hostkind + "-" + hostnum
slots_resv, slots_used, slots_tot = map(int, core_usage_combo.split("/"))
hoststat = host_get_stats_rec(hostname)
hoststat['slots_resv'] += slots_resv
hoststat['slots_used'] += slots_used
# FIXME assume same across queues; fix if not correct:
hoststat['slots_tot'] = max(hoststat['slots_tot'], slots_tot)
hoststat['os_arch'] = os_arch
# FIXME we assume all of same queue type; fix if not correct:
hoststat['queue_type'] = queue_type
hoststat['queues'].append(queue)
# FIXME we assume all have same load; fix if not correct:
hoststat['node_load'] = node_load
hoststat['status_flags'] = status_flags
#return host_list_by_kinds, host_stats
return host_stats
def node_load_ratio(node_load, slots_used):
"""Ratio of node load vs slots claimed to be used. This should be close to one
if the job uses the CPUs efficiently, or near zero if most jobs are interactive
(i.e. lots of idling)."""
return node_load / slots_used if slots_used != 0 \
else 0.0 if node_load < 0.75 \
else float('nan')
def print_host_stats(host_stats):
"""Prints the per-host statistics gathered by `collect_host_stats`.
"""
hostnames = sorted(host_stats.keys())
print("%-16s %5s %5s %5s %5s %7s %9s" \
% ("HOST", "CORES", "used", "free", "resv", "load", "load/used"))
for h in hostnames:
hs = host_stats[h]
print("%-16s %5d %5d %5d %5d %7.2f %9.3f%s" \
% (h,
hs['slots_tot'],
hs['slots_used'],
hs['slots_tot'] - hs['slots_used'] - hs['slots_resv'],
hs['slots_resv'],
hs['node_load'],
node_load_ratio(hs['node_load'], hs['slots_used']),
" "+hs['status_flags'] if hs['status_flags'] else ""
)
)
def summarize_hosttype_stats(host_stats):
"""Further summarize the host stats by the host type (denoted by the
prefix of the hostname before the dash character, i.e. "c8" for
"c8-003").
"""
hosttype_stats = {}
def hosttype_get_stats_rec(hosttype):
if hosttype not in hosttype_stats:
s = {
'hosts': [],
}
hosttype_stats[hosttype] = s
return hosttype_stats[hosttype]
for (hosttype, hostname) in [ (h.split('-')[0], h) for h in host_stats.keys() ]:
#print(hosttype, hostname)
hts = hosttype_get_stats_rec(hosttype)
hts['hosts'].append(hostname)
for hts in hosttype_stats.values():
#print hts
hts['host_count'] = len(hts['hosts'])
hts['slots_resv'] = sum(host_stats[h]['slots_resv'] for h in hts['hosts'])
hts['slots_tot'] = sum(host_stats[h]['slots_tot'] for h in hts['hosts'])
hts['slots_used'] = sum(host_stats[h]['slots_used'] for h in hts['hosts'])
hts['node_load'] = sum(host_stats[h]['node_load'] for h in hts['hosts'])
hts['os_arch'] = host_stats[hts['hosts'][0]]['slots_used']
return hosttype_stats
def print_hosttype_stats(hosttype_stats):
hosttypes = sorted(hosttype_stats.keys())
print("%-16s %5s %5s %5s %5s %5s %7s %9s" \
% ("MACHTYPE", "NODES", "CORES", "used", "free", "resv", "load", "load/used"))
for ht in hosttypes:
hts = hosttype_stats[ht]
print("%-16s %5d %5d %5d %5d %5d %7.2f %9.3f" \
% (ht, hts['host_count'],
hts['slots_tot'],
hts['slots_used'],
hts['slots_tot'] - hts['slots_used'] - hts['slots_resv'],
hts['slots_resv'],
hts['node_load'],
hts['node_load'] / hts['slots_used'] if hts['slots_used'] != 0
else 0.0 if hts['node_load'] < 0.75 else float('nan')
)
)
def help():
msg = """\
%(CMD)s - Shows node status from SGE information
The information is mainly drawn from `qstat -f` output.
Usage: one of the following:
%(CMD)s raw [qstat_file] [--save] [--show-disabled-nodes]
Shows the raw queue/node status
%(CMD)s hoststats [qstat_file] [--save] [--show-disabled-nodes]
Shows the present statistics for every node
%(CMD)s
%(CMD)s stats [qstat_file] [--save] [--show-disabled-nodes]
Shows the statistic summary per node type
""" \
% dict(CMD=MYSELF)
print(msg)
def main_default(argv):
"""Main default function:
- By default we invoke qstat -f and prints the analysis.
- If argv[1] is given, then we read in the file and
use that for the analysis.
"""
from time import localtime, strftime
from getopt import getopt, GetoptError
dtime = localtime()
dtimestr = strftime("%Y%m%d-%H%M", dtime)
# Read the command first--what do we want to do
if len(argv) < 2:
cmd = "stats"
elif argv[1] in ('--raw', 'raw'):
cmd = "raw"
elif argv[1] in ('--stats', 'stats', 'stat'):
cmd = "stats" # old stats, a.k.a. hosttype_stats
elif re.search(r'^(--)?host-?stat', argv[1]):
cmd = "hoststats"
elif argv[1] in ('--help', 'help', '-h'):
help()
return 0
else:
raise ValueError, "Unknown action: "+argv[1]
# Skip program name and first command:
cmdargs_in = argv[2:]
try:
cmdopts, cmdargs = getopt(cmdargs_in,
"dhs",
["show-disabled-nodes=", "include-disabled-nodes=",
"save",
"help"])
except GetoptError as err:
sys.stderr.writelines([str(err), "\n"])
return 2
# Process flag argument
show_disabled_nodes = False
save_qstat = False
for o,a in cmdopts:
if o in ('-h', '--help'):
help()
return 0
elif o in ('-d',):
show_disabled_nodes = True
elif o in ('--show-disabled-nodes', '--include-disabled-nodes'):
show_disabled_nodes = parse_int_or_bool(a)
elif o in ('-s', '--save'):
save_qstat = True
else:
raise ProgramError, "Unhandled option in main program: %s %s" % (o,a)
if len(cmdargs) > 0:
qstat_f_current = open(cmdargs[0], "r").read().splitlines()
else:
qstat_f_current = pipe_out(('qstat', '-f'), split=True)
if save_qstat:
with open("qstat-f-%s.txt" % dtimestr, "w") as F:
F.write("\n".join(qstat_f_current))
F.write("\n")
if cmd == "raw":
node_slot_stats_raw(qstat_f_current,
show_disabled_nodes=show_disabled_nodes,
)
elif cmd == "hoststats":
node_slot_stats(qstat_f_current,
show_disabled_nodes=show_disabled_nodes,
)
elif cmd == "stats":
node_slot_stats_per_machine_type(qstat_f_current,
show_disabled_nodes=show_disabled_nodes,
)
else:
raise ProgramError, "Missing support for command: "+cmd
return 0
# ---------------------------------------------------------------------------
# Support tools below
# ---------------------------------------------------------------------------
def pipe_out(args, split=False, shell=False):
"""Executes a shell command, piping out the stdout to python for parsing.
This is my customary shortcut for backtick operator.
The result is either a single string (if split==False) or a list of strings
with EOLs removed (if split==True)."""
retval = subprocess.Popen(args, stdout=subprocess.PIPE, shell=shell).communicate()[0]
if not split:
return retval
else:
return retval.splitlines()
# Internal variable: don't mess!
_str_fmt_heading_rx = None
def str_fmt_heading(fmt):
"""Replaces a printf-style formatting with one suitable for table heading:
all non-string conversions are replaced with string conversions,
preserving the minimum widths."""
# Originally from: $PWQMC77/scripts/cost.py and later Cr2_analysis_cbs.py .
#
#_str_fmt_heading_rx = None # only for development purposes
import re
global _str_fmt_heading_rx
if _str_fmt_heading_rx is None:
# Because of complicated regex, I verbosely write it out here:
_str_fmt_heading_rx = re.compile(r"""
(
% # % sign
(?:\([^)]+\))? # optional '(keyname)' mapping key
[-+#0 hlL]* # optional conversion flag
[0-9*]* # optional minimum field width
)
((?:\.[0-9]*)?) # optional precision
[^-+#*0 hlL0-9.%s] # not conv flag, dimensions, nor literal '%',
# nor 's' conversion specifiers
""", re.VERBOSE)
return _str_fmt_heading_rx.sub(r'\1s', fmt)
def parse_int_or_bool(S):
if isinstance(S, basestring):
S = S.strip().lower()
try:
return int(S)
except ValueError:
if S in ('true', 't', 'yes', 'y', 'on'):
return True
elif S in ('false', 'f', 'no', 'n', 'off', '-', ''):
return False
else:
raise ValueError, "Don't understand '%s' for boolean value" % S
else:
return S
def parse_bool(S):
if isinstance(S, basestring):
S = S.strip().lower()
if S in ('true', 't', 'yes', 'y', 'on', '1'):
return True
elif S in ('false', 'f', 'no', 'n', 'off', '0', '-', ''):
return False
else:
raise ValueError, "Don't understand '%s' for boolean value" % S
else:
return S
# stub main code
if __name__ == "__main__" and not "get_ipython" in globals():
main_default(sys.argv)