Using F2Py With OpenACC Gives Import Error In Python

I am writing a simple test code to see how I could wrap a fortran code containing openacc regions and call from python. Here's the code. module test use iso_c_binding, only: sp =

Solution 1:

I decided to check out what the executables that I directly create by compiling link to. So doing

ldd a.out 

gives me

linux-vdso.so.1 =>  (0x00007ffed24a0000)
libcublas.so.7.5 => /usr/local/cuda/lib64/libcublas.so.7.5 (0x00007f04c2d45000)
libcudart.so.7.5 => /usr/local/cuda/lib64/libcudart.so.7.5 (0x00007f04c2ae7000)
libgfortran.so.3 => /home//Experiments/Nvidia/OpenACC/OLCFHack15/gcc6/install/lib64/libgfortran.so.3 (0x00007f04c27c1000)
libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f04c24bb000)
libgomp.so.1 => /home//Experiments/Nvidia/OpenACC/OLCFHack15/gcc6/install/lib64/libgomp.so.1 (0x00007f04c228d000)
libgcc_s.so.1 => /home//Experiments/Nvidia/OpenACC/OLCFHack15/gcc6/install/lib64/libgcc_s.so.1 (0x00007f04c2077000)
libquadmath.so.0 => /home//Experiments/Nvidia/OpenACC/OLCFHack15/gcc6/install/lib64/libquadmath.so.0 (0x00007f04c1e38000)
libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f04c1c1a000)
libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f04c1855000)
librt.so.1 => /lib/x86_64-linux-gnu/librt.so.1 (0x00007f04c164d000)
libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f04c1449000)
libstdc++.so.6 => /home//Experiments/Nvidia/OpenACC/OLCFHack15/gcc6/install/lib64/libstdc++.so.6 (0x00007f04c10c9000)
/lib64/ld-linux-x86-64.so.2 (0x00007f04c4624000)  

whereas the module created by f2py using

f2py -c -m  --f90flags='-fopenacc -foffload=nvptx-none -foffload=-O3 -O3 - fPIC' hello hello.f90 

gives me

linux-vdso.so.1 =>  (0x00007ffeeef63000)
libpython3.5m.so.1.0 => not found
libgfortran.so.3 => /home//Experiments/Nvidia/OpenACC/OLCFHack15/gcc6/install/lib64/libgfortran.so.3 (0x00007f841918f000)
libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f8418e89000)
libgcc_s.so.1 => /home//Experiments/Nvidia/OpenACC/OLCFHack15/gcc6/install/lib64/libgcc_s.so.1 (0x00007f8418c73000)
libquadmath.so.0 => /home//Experiments/Nvidia/OpenACC/OLCFHack15/gcc6/install/lib64/libquadmath.so.0 (0x00007f8418a34000)
libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f841866f000)
/lib64/ld-linux-x86-64.so.2 (0x00007f84196be000)

Clearly libgomp was being linked in the executable but not in the f2py created object. So I modified the f2py command to

f2py -c -m  --f90flags='-fopenacc -foffload=nvptx-none -foffload=-O3 -O3 -fPIC' hello hello.f90 -L/usr/local/cuda/lib64 -lcublas -lcudart -lgomp

And now it compiles and I can import into python without getting that error.

Solution 2:

As I told you by mail I finally manage to call an acc_init and some kernels correctly through f2py. My code is not perfectly correct for now, I still have to work on it but here is the part of my CMakeLists.txt I use to call f2py (basically I added -L/data_local/sw/pgi/linuxpower/19.9/ -L/usr/lib64 -L. -laccapi -laccg -laccn -laccg2 -ldl -lcudadevice -lpthread -lpgc -lm to f2py calls following explanations in c - Linking a PGI OpenACC-enabled library with gcc ) :

  set(OPENACC_Fortran_FLAGS -Mnorpath)
  list(APPEND OPENACC_Fortran_FLAGS -Minfo)
  list(APPEND OPENACC_Fortran_FLAGS -acc)
  list(APPEND OPENACC_Fortran_FLAGS -ta=tesla:nordc)
  list(APPEND OPENACC_Fortran_FLAGS -ta=tesla:cc60,cc70)
  list(APPEND OPENACC_Fortran_FLAGS -Mextend)
  list(APPEND OPENACC_Fortran_FLAGS -Mbackslash)
  list(APPEND OPENACC_Fortran_FLAGS -Mcuda=cuda10.1)
  list(APPEND OPENACC_Fortran_FLAGS -lcuda)
  set(EXTRA_Fortran_FLAGS ${OPENACC_Fortran_FLAGS})

    ## BUILD python wrapper
    add_custom_command(OUTPUT ${SOURCE_LIST_WRAP}
        COMMAND f90wrap -m my_module ${SOURCE_LIST_TO_WRAP}
        DEPENDS ${SOURCE_LIST_TO_WRAP})
    add_custom_target(_my_module ALL
      DEPENDS ${SOURCE_LIST_WRAP}
      COMMAND CC=gcc f2py --fcompiler=pg --f90flags="${EXTRA_Fortran_FLAGS}" -m _my_module -c ${SOURCE_LIST_WRAP} -L. -lmy_kernels_lib -L/data_local/sw/pgi/linuxpower/19.9/ -L/usr/lib64 -L. -laccapi -laccg -laccn -laccg2 -ldl -lcudadevice -lpthread  -lpgc -lm )

In fact it appears that only -laccn is required for me:

add_custom_target(_my_module ALL
  DEPENDS ${SOURCE_LIST_WRAP}
  COMMAND CC=gcc f2py --fcompiler=pg  --f90flags="${EXTRA_Fortran_FLAGS}" -m _my_module -c ${SOURCE_LIST_WRAP} -L. -lmy_kernels_lib -laccn )

PS: This is correct with PGI 19.x with 20.x I needed to change the flags:

  if (CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 20)
    set(F2PY_ACC_LIBS -laccapi -laccn)
  else()
    set(F2PY_ACC_LIBS -lcudafor -lacccuda -lcudafor2  -lcudafor101 -lcudadevice)
  endif()