This is a corner cutting method. Perhaps a better way exists.
- Intel MKL 11 or newer scalapack cause strange segmentation faults with Openmpi 1.6-1.8. These segmentation faults do not appear with intel mpi.
- build_nwchem script overrides my settings, I'll not use it anymore.
- I edited config/makefile.h so that intel compiler settings are applied to mpif90
- QA tests in single cpu still run 10 times faster than my fastest binary, and my binary is much faster than the binary supplied with ubuntu for example. How do they achieve that performance?
- To Do Next: In order to compile nwchem for the mic card, I have to somehow cross compile ga, or make nwchem accept a cross compiled one.
The following settings work, and pass QA tests when compiled with
source settings.csh;cd src;make nwchem_config;make CC=mpicc FC=mpif90 CXX=mpicxx
settings.csh:
module purge
module load intelmpi/4.1.3.048-intel intel64/13.1up03
setenv NWCHEM_TARGET LINUX64
setenv NWCHEM_TOP /wherever nwchem sits, but name shortened from default/
setenv MPI_ROOT /wherever intel sits/intel/mpi/4.1.3.048/include64/
setenv LAPACK_LIB "-L$MKLROOT/lib/intel64 -lmkl_scalapack_ilp64 -lmkl_intel_ilp64 -lmkl_core -lmkl_intel_thread -lmkl_blacs_intelmpi_ilp64 -lpthread -lm"
setenv BLAS_LIB "-L$MKLROOT/lib/intel64 -lmkl_scalapack_ilp64 -lmkl_intel_ilp64 -lmkl_core -lmkl_intel_thread -lmkl_blacs_intelmpi_ilp64 -lpthread -lm"
setenv SCALAPACK_LIB "-L$MKLROOT/lib/intel64 -lmkl_scalapack_ilp64 -lmkl_intel_ilp64 -lmkl_core -lmkl_intel_thread -lmkl_blacs_intelmpi_ilp64 -lpthread -lm"
setenv BLASOPT "-L$MKLROOT/lib/intel64 -lmkl_scalapack_ilp64 -lmkl_intel_ilp64 -lmkl_core -lmkl_intel_thread -lmkl_blacs_intelmpi_ilp64 -lpthread -lm"
setenv USE_SCALAPACK_I8 y
setenv BLAS_SIZE 8
setenv LAPACK_SIZE 8
setenv SCALAPACK_SIZE 8
setenv FC ifort
setenv CC icc
setenv CXX icpc
setenv FXX ifort
setenv MPI_CXX icpc
setenv MPI_CC icc
setenv MPI_F90 ifort
setenv FOPTIMIZE "-O3 -openmp -static-intel -no-prec-div -funroll-loops"
setenv COPTIMIZE "-O3 -openmp -static-intel -no-prec-div -funroll-loops"
setenv FFLAGS " -i8 -openmp -static-intel -I$MKLROOT/include/intel64/ilp64 -I$MKLROOT/include"
setenv FOPTIONS " -i8 -openmp -static-intel -I$MKLROOT/include/intel64/ilp64 -I$MKLROOT/include"
setenv CFLAGS " -DMKL_ILP64 -openmp -static-intel -ftz -I$MKLROOT/include/intel64/ilp64 -I$MKLROOT/include"
setenv CFLAGS_FORGA " -DMKL_ILP64 -ftz -openmp -static-intel -I$MKLROOT/include/intel64/ilp64 -I$MKLROOT/include"
setenv COPTIONS " -DMKL_ILP64 -ftz -openmp -static-intel -I$MKLROOT/include/intel64/ilp64 -I$MKLROOT/include"
setenv CXXFLAGS " -DMKL_ILP64 -ftz -openmp -static-intel -I$MKLROOT/include/intel64/ilp64 -I$MKLROOT/include"
setenv CXXOPTIONS " -DMKL_ILP64 -ftz -openmp -static-intel -st-I$MKLROOT/include/intel64/ilp64 -I$MKLROOT/include"
setenv USE_OPENMP y
setenv NWCHEM_MODULES all
setenv USE_MPI y
setenv USE_MPIF y
setenv USE_MPIF4 y
setenv MPI_INCLUDE "-I$MPI_ROOT/include64 -I$MPI_ROOT/lib64"
setenv MPI_LIB -L$MPI_ROOT/lib
setenv LIBMPI "-lmpi_f90 -lmpi_f77 -lmpi -ldl -lm -lrt -lnsl -lutil"
setenv ARMCI_NETWORK OFA
setenv MSG_COMMS MPI
#Following settings are often true for IB Linux clusters
setenv IB_INCLUDE /usr/include
setenv IB_LIB /usr/lib64
setenv IB_LIB_NAME -libverbs
|