From d159d57578b777c29da2b4f84855ab30fe5b1e44 Mon Sep 17 00:00:00 2001 From: Wirawan Purwanto Date: Tue, 22 Feb 2022 10:28:14 -0500 Subject: [PATCH] * Third attempt to fix lmod "module" for ipython ("fix3") (dated: 2020-06-09). * Supports both Turing and Wahab. * Allows both addition and deletion of paths from sys.path. * Author's note: Later on, I discovered that these complex steps do not appear to be necessary, somehow the lmod Python commands were able to take care of additions and deletions through the modification of os.environ['PYTHONPATH'] alone, so I am not sure if this fix is absolutely needed. (?) Sigs: # -rw------- 1 wpurwant users 3298 2020-06-09 20:07 lmod_python_fix3.py # 78abc7bae90bf23a42aa40285175ca48 lmod_python_fix3.py --- lmod/ipython/startup/lmod_python_fix.py | 117 +++++++++++++++++------- 1 file changed, 83 insertions(+), 34 deletions(-) diff --git a/lmod/ipython/startup/lmod_python_fix.py b/lmod/ipython/startup/lmod_python_fix.py index c72e300..00b04ac 100644 --- a/lmod/ipython/startup/lmod_python_fix.py +++ b/lmod/ipython/startup/lmod_python_fix.py @@ -1,39 +1,88 @@ # -*- python -*- +# modified "module" command for Wahab + #from __future__ import print_function #from subprocess import PIPE, Popen import os, sys -sys.path.append('/shared/apps/common/lmod/current/init') - -from env_modules_python import module as lmod_module - -def module(command, *arguments): - lmod_module(command, *arguments) - # BAD: This will cause alteration of PYTHONPATH in a way that may not be desirable. - sys_path_orig = sys.path - PYTHONPATH = os.environ.get('PYTHONPATH') - sys_path_new = PYTHONPATH.split(':') if PYTHONPATH is not None else [] - for p in sys_path_orig: - if p not in sys_path_new: - sys_path_new.append(p) - print(sys_path_new) - sys.path = sys_path_new - - manual_ld_library_dir = os.environ.get('LMOD_MANUAL_LD_LIBRARY_PATH') - if manual_ld_library_dir is None: - return - - for search_path in os.getenv('LD_LIBRARY_PATH').split(':')[::-1]: - if not os.path.isdir(search_path): - continue - if search_path == manual_ld_library_dir: - continue - - for f in os.listdir(search_path): - if '.so' in f: - src = f'{search_path}/{f}' - tgt = f'{manual_ld_library_dir}/{f}' - if os.path.islink(tgt): - os.unlink(tgt) - - os.symlink(src, tgt) +try: + with open("/etc/cluster", "r") as _cluster_F: + _cluster = next(_cluster_F).strip() +except: + sys.stderr.write("Unable to determine cluster, 'module' command will not be available.\n") +else: + if _cluster == "wahab": + sys.path.append('/shared/apps/common/lmod/current/init') + elif _cluster == "turing": + sys.path.append('/cm/shared/applications/lmod/lmod/init') + else: + sys.stderr.write("Unknown cluster, 'module' command will not be available.\n") + del _cluster + +try: + from env_modules_python import module as lmod_module +except: + pass +else: + def module(command, *arguments): + split_path_str = lambda P: P.split(':') if P is not None else [] + + PYTHONPATH_ORIG = os.environ.get('PYTHONPATH') + lmod_module(command, *arguments) + PYTHONPATH = os.environ.get('PYTHONPATH') + + # update sys.path + # Must account the *changes* in PYTHONPATH and reflect that to sys.path + # + # WARNING: This is not perfect! + # Due to the algorithm used below, the updated PYTHONPATH + # will not be reflected 100% in its order in sys.path + # + # 1) It assumes that additional paths are prepended. + # 2) It will completely delete a path that is remove + # as a result of module command + # 3) If there are simultaneous deletion(s) and adition(s), + # then the added paths will take precedence (unfortunately), + # which may result in unexpected behavior in corner cases, + # at least in principle. + # This should not be the case for "well behaved" HPC packages + # where order of loaded modules should NOT cause change in behavior. + + # Save the original in case needed for debugging + global sys_path_orig + sys_path_orig = sys.path + + python_path_orig = split_path_str(PYTHONPATH_ORIG) + python_path = split_path_str(PYTHONPATH) + + sys_path_added = set(python_path) - set(python_path_orig) + sys_path_deleted = set(python_path_orig) - set(python_path) + + # delete removed paths + sys_path_trimmed = [ p for p in sys_path_orig if p not in sys_path_deleted ] + + # prepend new paths + sys_path_prepended = [ p for p in python_path if p in sys_path_added ] + + sys.path = sys_path_prepended + sys_path_trimmed + + # Extra: if LMOD_MANUAL_LD_LIBRARY_PATH is specified, then + # the shared library symlinks are updated. + manual_ld_library_dir = os.environ.get('LMOD_MANUAL_LD_LIBRARY_PATH') + if manual_ld_library_dir is None: + return + + for search_path in os.getenv('LD_LIBRARY_PATH').split(':')[::-1]: + if not os.path.isdir(search_path): + continue + if search_path == manual_ld_library_dir: + continue + + for f in os.listdir(search_path): + if '.so' in f: + src = f'{search_path}/{f}' + tgt = f'{manual_ld_library_dir}/{f}' + if os.path.islink(tgt): + os.unlink(tgt) + + os.symlink(src, tgt)