My tools of the trade for python programming.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

291 lines
8.2 KiB

# $Id: shell_tools.py,v 1.8 2010-10-25 14:41:39 wirawan Exp $
#
# wpylib.shell_tools
# Created: 20100106
# Wirawan Purwanto
#
# Simple and dirty tools like those I usually use in my shell
# scripts.
#
import glob
import os
import os.path
import sys
try:
import subprocess
has_subprocess = True
except:
if "has_subprocess" not in globals():
print >>sys.stderr, "Newer subprocess module does not exist, using older interfaces."
has_subprocess = False
# Files, directories, and filename utilities
def mcd(subdir):
# Assuming we have GNU coreutils' mkdir
mkdir("-p", subdir)
os.chdir(subdir)
def provide_link(dest, src):
"""Checks if file `dest' exists. If it does not, provide for it by means
of a softlink from `src'."""
if not os.path.exists(dest):
# strip trailing /'s just in case it exists
os.symlink(src, dest.rstrip("/"))
# Globbing utilities:
def sorted_glob(pathname):#, cmp=None, key=None, reverse=None):
"""Returns a sorted list of file names matching glob pattern `pathname'.
Added here to accomodate older python that do not have sorted() function."""
rslt = glob.glob(pathname)
rslt.sort() #cmp=cmp, key=key, reverse=reverse)
return rslt
# Environment variable utilities:
def env_push(name, new_value):
"""Temporarily modifies the value of an environment variable; saving the
original one in the function return value.
The original value can be restored to the environment variable by calling
env_pop.
Example:
oldpath = push_env('TOOL_HELPER', '/usr/bin/less')
execvp(os.P_WAIT, 'some_tool', ('some_tool', 'some_arg'))
pop_env('TOOL_HELPER', oldpath)
"""
old_value = os.environ.get(name, None)
os.environ[name] = new_value
return old_value
def env_pop(name, old_value):
"""Restores the original value of an environment variable that was modified
temporarily by env_push."""
if old_value == None:
del os.environ[name]
else:
os.environ[name] = old_value
def getenv(*names, **opts):
"""Tries to get a value from a list of environment variables.
The first found one will be used; if none is found, then a default will
be tried (use `default' parameter to specify this).
If no default is found, then a KeyError exception will be raised.
"""
for n in names:
if n in os.environ:
return os.environ[n]
if "default" in opts:
return opts["default"]
else:
raise KeyError, \
"Cannot find value among environment variables: %s" % (str(names))
# Low-level utilities:
def errchk(cmd, args, retcode):
"""Checking for error after the invocation of an external command."""
if retcode == 0: return
print >>sys.stderr, "Error executing ", cmd, " ".join(args)
if retcode < 0:
err = "Command %s was terminated by signal %d" % (cmd, -retcode)
else:
err = "Command %s returned %d" % (cmd, retcode)
raise RuntimeError, err
def quote_cmdline(seq):
"""Quotes the strings in seq for feeding to shell.
This is a severe protection to prevent:
- variable, command, or other substitutions
- shell expansions (parameter, wildcard)
- word splitting
- invocation of shell builtin (!!!)
"""
# Compared to other implementations:
# - Python 2.6's subprocess.py has list2cmdline, but I don't like it because
# it still allows the shell to interpret wildcards. We have to quote wildcards
# (*, [], {}, ?) and $ as well.
# - Python 3.3 has shlex.quote that performs similarly.
# The reverse operation can be done via shlex standard module.
rslt = []
for i in seq:
inew = '"' + i.replace("\\", "\\\\").replace('"', '\\"').replace('$', '\\$').replace('`', '\\`') + '"'
rslt.append(inew)
return " ".join(rslt)
"""
pipe_out and pipe_in defines unidirectional pipes for external programs.
Note the unconventional meaning: `pipe_in' means we drive the program with
python input as the stdin, `pipe_out' means we run the program and read in
the output into python.
"""
if has_subprocess:
def run(prg, args):
retcode = subprocess.call((prg,) + tuple(args))
errchk(prg, args, retcode)
return 0
def system(cmdline):
"""Similar to os.system(), except that errors are caught.
cmdline *must* be a string."""
retcode = subprocess.call(cmdline, shell=True)
errchk(cmdline, (), retcode)
return 0
def pipe_out(args, split=False, shell=False):
"""Executes a shell command, piping out the stdout to python for parsing.
This is my customary shortcut for backtick operator.
The result is either a single string (if split==False) or a list of strings
with EOLs removed (if split==True)."""
retval = subprocess.Popen(args, stdout=subprocess.PIPE, shell=shell).communicate()[0]
if not split:
return retval
else:
return retval.splitlines()
class pipe_in:
"""Executes a shell command, piping in the stdin from python for driving.
This is the reverse of pipe_out.
Commands are given through file-like write() or writelines() methods."""
def __init__(self, args, shell=False):
self.px = subprocess.Popen(args, stdin=subprocess.PIPE, shell=shell)
self.args = args
def write(self, line):
self.px.stdin.write(line)
def writelines(self, lines):
for line in lines:
self.write(line)
def flush(self):
self.px.stdin.flush()
def close(self):
self.px.stdin.close()
else:
def run(prg, args=()):
# Python < 2.4 does not have subprocess, so we use spawnvp
retcode = os.spawnvp(os.P_WAIT, prg, (prg,) + tuple(args))
errchk(prg, args, retcode)
return 0
def system(cmdline):
"""Similar to os.system(), except that errors are caught."""
retcode = os.system(cmdline)
errchk(cmdline, (), retcode)
return 0
def pipe_out(args, split=False, shell=False):
"""Executes a shell command, piping out the stdout to python for parsing.
This is my customary shortcut for backtick operator.
The result is either a single string (if split==False) or a list of strings
with EOLs removed (if split==True)."""
if shell or isinstance(args, basestring):
# BEWARE: args should be a string in this case
p = os.popen(args, "r")
else:
args = quote_cmdline(args)
p = os.popen(args, "r")
retval = p.read()
status = p.close()
if not split:
return retval
else:
return retval.splitlines()
def pipe_in(args, shell=False):
"""Executes a shell command, piping in the stdin from python for driving.
This is the reverse of pipe_out.
Commands are given through file-like write() or writelines() methods."""
if shell or isinstance(args, basestring):
# BEWARE: args should be a string in this case
p = os.popen(args, "w")
else:
args = quote_cmdline(args)
p = os.popen(args, "w")
return p
# coreutils
coreutils = """
base64 basename
cat chcon chgrp chmod chown cksum comm cp csplit cut
date dd df dir dircolors dirname du
echo env expand expr
factor false fmt fold
groups
head hostid
id install
join
link
ln logname ls
md5sum mkdir mkfifo mknod mv
nice nl nohup
od
paste pathchk pinky pr printenv printf ptx pwd
readlink rm rmdir runcon
seq sha1sum sha224sum sha256sum sha384sum sha512sum shred shuf sleep sort split stat stty sum sync
tac tail tee test touch touch tr true tsort tty
uname unexpand uniq unlink users
vdir
wc who whoami
yes
""".split()
# and other common utilities
CMD = coreutils
CMD += [ 'grep', 'less' ]
CMD += [ 'sh', 'bash' ]
CMD += [ 'gawk', 'sed', ]
CMD_NAME = {}
for n in CMD:
CMD_NAME[n] = n
s = """def %(cmd)s(*args): run(CMD_NAME['%(cmd)s'], args)"""
exec s % {'cmd': n }
def import_commands(namespace, cmds=None):
"""Safely import shell commands to a given namespace.
We should avoid importing names that belong to built-in functions,
therefore we added that check below."""
if cmds == None: cmds = CMD
#print namespace.keys()
#print namespace["__builtins__"]
my_namespace = globals()
dir = my_namespace['__builtins__']['dir']
#print dir(namespace["__builtins__"])
# Never clobber the built-in names:
try:
exclusions = dir(namespace["__builtins__"])
except:
exclusions = []
for n in cmds:
if n not in exclusions:
n_act = my_namespace[n]
namespace.setdefault(n, n_act)
"""
def cp(*args):
run('cp', args)
def mkdir(*args):
run('mkdir', args)
def mv(*args):
run('mv', args)
"""