From 9e7c5b3312f9528539b73954b318f7f9832641a9 Mon Sep 17 00:00:00 2001 From: Wirawan Purwanto Date: Tue, 20 Apr 2021 00:28:39 -0400 Subject: [PATCH] * Modularize & tidied up the nwbuild05 script. --- .../nwbuild05-OPENIB-internal-blas-lapack.sh | 112 ++++++++++++++---- 1 file changed, 87 insertions(+), 25 deletions(-) diff --git a/wahab/builds/nwchem/nwbuild05-OPENIB-internal-blas-lapack.sh b/wahab/builds/nwchem/nwbuild05-OPENIB-internal-blas-lapack.sh index e40c2f4..ff17371 100755 --- a/wahab/builds/nwchem/nwbuild05-OPENIB-internal-blas-lapack.sh +++ b/wahab/builds/nwchem/nwbuild05-OPENIB-internal-blas-lapack.sh @@ -1,18 +1,78 @@ #!/bin/bash # +# build05: +# * Use OPENIB ARMCI backend +# * Less fancy build: do not use external LAPACK or BLAS +# # 20210414 - First version, forked off nwbuild04-ARMCI-MPI-internal-blas-lapack.sh +preamble () { + # Site-specific preambles + module load ucx/1.9 openmpi/3.1.4 + SRC_ROOT_DIR=/cm/shared/containers/srcs/nwchem + GA_PACKAGE=ga-5.7.1 + SRC_TARBALL_NWCHEM=$SRC_ROOT_DIR/v7.0.0-release.tar.gz + SRC_TARBALL_GA=$SRC_ROOT_DIR/$GA_PACKAGE.tar.gz + #export NWCHEM_TOP=/opt/nwchem.build04c # FOR TESTINGS --> shortcut to the above path due to path cannot exceed 64 chars :( export NWCHEM_TOP=/scratch/wpurwant/BUILD/nwchem7/gcc7.3-openmpi3.1.4/nwchem.OPENIB # FOR TESTINGS - export NWCHEM_TOP=/scratch/wpurwant/.L/nwchem.OPENIB + #export NWCHEM_TOP=/scratch/wpurwant/.L/nwchem.OPENIB # temp shortlink (workaround)--not needed anymore + + # FOR PRODUCTION + export NWCHEM_TOP=/scratch/wpurwant/BUILD/nwchem7/gcc7.3-openmpi3.1.4/prod-nwchem.OPENIB + + if false; then # build production target--not used + export NWCHEM_TOP=/shared/apps/manual/nwchem/7.0.0/gcc7.3-openmpi3.1.4/BUILD-OPENIB + # This tweak was needed to build faster (files are actually located on scratch) + BUILD_TOP=/scratch/wpurwant/BUILD/nwchem7/gcc7.3-openmpi3.1.4/prod-nwchem.OPENIB + mkdir -p $BUILD_TOP + if [ ! -e $NWCHEM_TOP ]; then + ln -s $BUILD_TOP $NWCHEM_TOP + fi + fi + # Build tweaks -- these are VERY IMPORTANT to set correctly export NWCHEM_MODULES="all" # skip "python" export NWCHEM_TARGET=LINUX64 - if false; then + export USE_MPI=y + export USE_NOIO=y + export USE_SCALAPACK=n + export USE_PYTHON64=n + + export ARMCI_NETWORK=OPENIB + + # Set to "y" to FORCE internal BLAS & LAPACK: + export USE_INTERNALBLAS=y + export SCALAPACK_SIZE=8 + export BLAS_SIZE=8 + + #export SCALAPACK="-lmkl_scalapack_ilp64 -lmkl_gf_ilp64 -lmkl_gnu_thread -lmkl_core -lmkl_blacs_intelmpi_ilp64 -lgomp -lpthread -lm -ldl" + #export SCALAPACK_LIB="$SCALAPACK" + + #export BLASOPT="-lmkl_gf_ilp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl" + #export LAPACK_LIB=$BLASOPT + + # set OPT_COMPILER_NAME_WRAPPERS to yes if needing "proper" program names + # for the compiler front-ends: + OPT_COMPILER_NAME_WRAPPERS=no + + # automatically determined, no need to change + OPT_USE_PYTHON=$(if echo " $NWCHEM_MODULES " | grep -q " python "; then echo yes; else echo no; fi) + + # Needed to accommodate long path of nwchem source: set to capital "Y" only + # see src/utils/GNUmakefile around line 19 + export NWCHEM_LONG_PATHS=Y +} + +build_nwchem_native () { + # Function to build nwchem in native (non-container) settings. + # This script builds with GCC (hardcoded) + + if [ "x$OPT_COMPILER_NAME_WRAPPERS" = xyes ]; then COMPILER_DIR=$NWCHEM_TOP/compilers mkdir -p $COMPILER_DIR if ! test -e $COMPILER_DIR/gcc; then ln -sf $(which $CC) $COMPILER_DIR/gcc; fi @@ -20,7 +80,7 @@ if ! test -e $COMPILER_DIR/gfortran; then ln -sf $(which $FC) $COMPILER_DIR/gfortran; fi # Make this THE path to find gcc, g++, and gfortran: - PATH=$COMPILER_DIR:$PATH + export PATH="$COMPILER_DIR:$PATH" fi # Override the compiler's names to point to the CONDA-provided toolchains @@ -29,30 +89,15 @@ export FC=gfortran echo PATH=$PATH + ( + set -x mkdir -p $NWCHEM_TOP - test -d $NWCHEM_TOP/src || tar -C $NWCHEM_TOP -xf /cm/shared/containers/srcs/nwchem/v7.0.0-release.tar.gz --strip-components=1 - test -d $NWCHEM_TOP/src/tools/ga-5.7.1 || tar -C $NWCHEM_TOP/src/tools -xf /cm/shared/containers/srcs/nwchem/ga-5.7.1.tar.gz - - # scale down: do not use external LAPACK or BLAS - export USE_MPI=y - export USE_NOIO=y - export USE_SCALAPACK=n - export USE_PYTHON64=n - # FORCE: - export USE_INTERNALBLAS=y - - export ARMCI_NETWORK=OPENIB + test -d $NWCHEM_TOP/src || tar -C $NWCHEM_TOP -xf $SRC_TARBALL_NWCHEM --strip-components=1 + test -d $NWCHEM_TOP/src/tools/$GA_PACKAGE || tar -C $NWCHEM_TOP/src/tools -xf $SRC_TARBALL_GA + ) - export SCALAPACK_SIZE=8 - export BLAS_SIZE=8 - - #export SCALAPACK="-lmkl_scalapack_ilp64 -lmkl_gf_ilp64 -lmkl_gnu_thread -lmkl_core -lmkl_blacs_intelmpi_ilp64 -lgomp -lpthread -lm -ldl" - #export SCALAPACK_LIB="$SCALAPACK" - - #export BLASOPT="-lmkl_gf_ilp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl" - #export LAPACK_LIB=$BLASOPT - - if false; then + if [ "x$OPT_USE_PYTHON" = xyes ]; then + # FIXME. Not tried yet. export PYTHONHOME=/opt/conda export PYTHONVERSION=2.7 ( @@ -80,3 +125,20 @@ echo "PATH = $PATH" make +} + + +preamble + +if [ x"$1" = xpreamble ]; then + return + export + exit 1 +fi + +if [ x"$1" = x -o x"$1" = xbuild ]; then + build_nwchem_native +else + echo "Invalid subcommand: $1" >&2 + exit 2 +fi