WP-hpc-tools/wahab/builds/nwchem/nwbuild05-OPENIB-internal-blas-lapack.sh

#!/bin/bash
#
# build05:
# * Use OPENIB ARMCI backend
# * Less fancy build: do not use external LAPACK or BLAS
#
# 20210414 - First version, forked off nwbuild04-ARMCI-MPI-internal-blas-lapack.sh

preamble () {
    # Site-specific preambles

    module load ucx/1.9 openmpi/3.1.4

    SRC_ROOT_DIR=/cm/shared/containers/srcs/nwchem
    GA_PACKAGE=ga-5.7.1
    SRC_TARBALL_NWCHEM=$SRC_ROOT_DIR/v7.0.0-release.tar.gz
    SRC_TARBALL_GA=$SRC_ROOT_DIR/$GA_PACKAGE.tar.gz

    #export NWCHEM_TOP=/opt/nwchem.build04c
    # FOR TESTINGS --> shortcut to the above path due to path cannot exceed 64 chars :(
    export NWCHEM_TOP=/scratch/wpurwant/BUILD/nwchem7/gcc7.3-openmpi3.1.4/nwchem.OPENIB  # FOR TESTINGS
    #export NWCHEM_TOP=/scratch/wpurwant/.L/nwchem.OPENIB   # temp shortlink (workaround)--not needed anymore

    # FOR PRODUCTION
    export NWCHEM_TOP=/scratch/wpurwant/BUILD/nwchem7/gcc7.3-openmpi3.1.4/prod-nwchem.OPENIB

    if false; then # build production target--not used
        export NWCHEM_TOP=/shared/apps/manual/nwchem/7.0.0/gcc7.3-openmpi3.1.4/BUILD-OPENIB
        # This tweak was needed to build faster (files are actually located on scratch)
        BUILD_TOP=/scratch/wpurwant/BUILD/nwchem7/gcc7.3-openmpi3.1.4/prod-nwchem.OPENIB
        mkdir -p $BUILD_TOP
        if [ ! -e $NWCHEM_TOP ]; then
            ln -s $BUILD_TOP $NWCHEM_TOP
        fi
    fi

    # Build tweaks -- these are VERY IMPORTANT to set correctly
    export NWCHEM_MODULES="all"  # skip "python"
    export NWCHEM_TARGET=LINUX64

    export USE_MPI=y
    export USE_NOIO=y
    export USE_SCALAPACK=n
    export USE_PYTHON64=n

    export ARMCI_NETWORK=OPENIB

    # Set to "y" to FORCE internal BLAS & LAPACK:
    export USE_INTERNALBLAS=y
    export SCALAPACK_SIZE=8
    export BLAS_SIZE=8

    #export SCALAPACK="-lmkl_scalapack_ilp64 -lmkl_gf_ilp64 -lmkl_gnu_thread -lmkl_core -lmkl_blacs_intelmpi_ilp64 -lgomp -lpthread -lm -ldl"
    #export SCALAPACK_LIB="$SCALAPACK"

    #export BLASOPT="-lmkl_gf_ilp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl"
    #export LAPACK_LIB=$BLASOPT

    # set OPT_COMPILER_NAME_WRAPPERS to yes if needing "proper" program names
    # for the compiler front-ends:
    OPT_COMPILER_NAME_WRAPPERS=no

    # automatically determined, no need to change
    OPT_USE_PYTHON=$(if echo " $NWCHEM_MODULES " | grep -q " python "; then echo yes; else echo no; fi)

    # Needed to accommodate long path of nwchem source: set to capital "Y" only
    # see src/utils/GNUmakefile around line 19
    export NWCHEM_LONG_PATHS=Y
}

build_nwchem_native () {
    # Function to build nwchem in native (non-container) settings.
    # This script builds with GCC (hardcoded)

    if [ "x$OPT_COMPILER_NAME_WRAPPERS" = xyes ]; then
        COMPILER_DIR=$NWCHEM_TOP/compilers
        mkdir -p $COMPILER_DIR
        if ! test -e $COMPILER_DIR/gcc; then ln -sf $(which $CC) $COMPILER_DIR/gcc; fi
        if ! test -e $COMPILER_DIR/g++; then ln -sf $(which $CXX) $COMPILER_DIR/g++; fi
        if ! test -e $COMPILER_DIR/gfortran; then ln -sf $(which $FC) $COMPILER_DIR/gfortran; fi

        # Make this THE path to find gcc, g++, and gfortran:
        export PATH="$COMPILER_DIR:$PATH"
    fi

    # Override the compiler's names to point to the CONDA-provided toolchains
    export CC=gcc
    export CXX=g++
    export FC=gfortran
    echo PATH=$PATH

    (
    set -x
    mkdir -p $NWCHEM_TOP
    test -d $NWCHEM_TOP/src || tar -C $NWCHEM_TOP -xf $SRC_TARBALL_NWCHEM --strip-components=1
    test -d $NWCHEM_TOP/src/tools/$GA_PACKAGE || tar -C $NWCHEM_TOP/src/tools -xf $SRC_TARBALL_GA
    )

    if [ "x$OPT_USE_PYTHON" = xyes ]; then
        # FIXME. Not tried yet.
        export PYTHONHOME=/opt/conda
        export PYTHONVERSION=2.7
        (
            cd /opt/conda/bin
            ln -s python-config python2.7-config
        )
    fi

    unset CFLAGS
    unset CXXFLAGS
    unset CPPFLAGS
    unset FCFLAGS
    unset FFLAGS

    cd $NWCHEM_TOP/src

    make nwchem_config
    # Not needed anymore:
    #sed -i config/makefile.h \
    #    -e 's:-i8:-fdefault-integer-8:'

    echo "Compiler locations:"
    echo "CC = $CC => $(which $CC)"
    echo "FC = $FC => $(which $FC)"
    echo "PATH = $PATH"

    make
}


preamble

if [ x"$1" = xpreamble ]; then
    return
    export
    exit 1
fi

if [ x"$1" = x  -o  x"$1" = xbuild ]; then
    build_nwchem_native
else
    echo "Invalid subcommand: $1" >&2
    exit 2
fi