Custom HPC software & tools from Wirawan. Primarily tailored toward ODU HPC sytems.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

167 lines
4.7 KiB

#!/usr/bin/env python
#
# Created: 20160829
# Wirawan Purwanto
"""
show-usage-current.py
---------------------
Shows the instantaneous usage of the cluster per user at current time.
Based on qstat -f output.
Usage:
1) Using the current qstat -f data (saving that data to the
`qstat-f-<date>-<time>.txt` file:
python show-cluster-usage.py
2) Using a saved qstat -f output:
python show-cluster-usage.py <saved-qstat-f.txt>
"""
import os
import re
import subprocess
import sys
def analyze_cluster_usage_by_users(qstat_f):
"""Provides a summary analysis of cluster usage by users.
Input: `qstat_f` is a list (or iterable) of text lines yielded by
`qstat -f` command.
Output: total aggregate usage per user, given as dict with usernames
as the keys.
"""
usage = {}
for L in qstat_f:
if re.match(r'^ [0-9]+ ', L):
F = L.split()
# For running jobs there are possibly more than 8 fields,
# but we care only for these 8
(jobid, priority, jobname, user, status, Date, Time, numcores) = F[:8]
if status == "r":
try:
taskid = F[8]
xjobid = jobid + ":" + taskid
except IndexError:
xjobid = jobid
try:
urec = usage[user]
except KeyError:
urec = {
'user': user,
'jobids': set(),
'xjobids': set(),
'cores': 0,
}
usage[user] = urec
urec['jobids'].add(jobid)
urec['xjobids'].add(xjobid)
urec['cores'] += int(numcores)
return usage
def print_cluster_usage_by_users(usage):
"""Prints the instantaneous usage-per-user breakdown of the cluster.
Input: `usage` is the aggregated instantaneous cluster usage as reported
by the analyze_cluster_usage_by_users() function.
"""
cur_users = usage.keys()
# Sort based on total core usage, descending manner
cmp_usage = lambda u1, u2: -cmp(usage[u1]['cores'], usage[u2]['cores'])
cur_users_sorted = sorted(cur_users, cmp=cmp_usage)
fmt = "%-12s %8d %8d %8d"
print(str_fmt_heading(fmt) % ("user", "numcores", "numjobs", "numtasks"))
for u in cur_users_sorted:
urec = usage[u]
print(fmt % (urec['user'],
urec['cores'],
len(urec['jobids']),
len(urec['xjobids'])))
def main_default(save_qstat=True):
"""Main default function:
- By default we invoke qstat -f and prints the analysis.
- If argv[1] is given, then we read in the file and
use that for the analysis.
"""
from time import localtime, strftime
dtime = localtime()
dtimestr = strftime("%Y%m%d-%H%M", dtime)
if len(sys.argv) > 1:
qstat_f_current = open(sys.argv[1], "r").read().splitlines()
else:
qstat_f_current = pipe_out(('qstat', '-f'), split=True)
if save_qstat:
with open("qstat-f-%s.txt" % dtimestr, "w") as F:
F.write("\n".join(qstat_f_current))
F.write("\n")
summary = analyze_cluster_usage_by_users(qstat_f_current)
print_cluster_usage_by_users(summary)
# ---------------------------------------------------------------------------
# Support tools below
# ---------------------------------------------------------------------------
def pipe_out(args, split=False, shell=False):
"""Executes a shell command, piping out the stdout to python for parsing.
This is my customary shortcut for backtick operator.
The result is either a single string (if split==False) or a list of strings
with EOLs removed (if split==True)."""
retval = subprocess.Popen(args, stdout=subprocess.PIPE, shell=shell).communicate()[0]
if not split:
return retval
else:
return retval.splitlines()
# Internal variable: don't mess!
_str_fmt_heading_rx = None
def str_fmt_heading(fmt):
"""Replaces a printf-style formatting with one suitable for table heading:
all non-string conversions are replaced with string conversions,
preserving the minimum widths."""
# Originally from: $PWQMC77/scripts/cost.py and later Cr2_analysis_cbs.py .
#
#_str_fmt_heading_rx = None # only for development purposes
import re
global _str_fmt_heading_rx
if _str_fmt_heading_rx is None:
# Because of complicated regex, I verbosely write it out here:
_str_fmt_heading_rx = re.compile(r"""
(
% # % sign
(?:\([^)]+\))? # optional '(keyname)' mapping key
[-+#0 hlL]* # optional conversion flag
[0-9*]* # optional minimum field width
)
((?:\.[0-9]*)?) # optional precision
[^-+#*0 hlL0-9.%s] # not conv flag, dimensions, nor literal '%',
# nor 's' conversion specifiers
""", re.VERBOSE)
return _str_fmt_heading_rx.sub(r'\1s', fmt)
# stub main code
if __name__ == "__main__" and not "get_ipython" in globals():
main_default()