Update samples to SYCL 2023 specifications

Publications/GPU-Opt-Guide/CMakeLists.txt

@@ -1,35 +1,31 @@
 cmake_minimum_required(VERSION 3.21)
+option(BUILD_FORTRAN_EXAMPLES "Whether to build fortran examples" ON)
 set(CMAKE_C_COMPILER icx)
 set(CMAKE_CXX_COMPILER icpx)
-set(CMAKE_Fortran_COMPILER ifx)
+if (BUILD_FORTRAN_EXAMPLES)
+  set(CMAKE_Fortran_COMPILER ifx)
+endif()
 
 project(GPUOptGuide)
+set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)
 
 include(CheckLanguage)
 enable_testing()
 
 find_package(IntelDPCPP REQUIRED)
 
-check_language(Fortran)
-if(CMAKE_Fortran_COMPILER)
-  enable_language(Fortran)
-else()
-  message(STATUS "No Fortran support. Disabling Fortran tests. Install oneAPI HPC Toolkit.")
+if (BUILD_FOTRAN_EXAMPLES)
+  check_language(Fortran)
+  if(CMAKE_Fortran_COMPILER)
+    enable_language(Fortran)
+  else()
+    message(FATAL_ERROR "No Fortran support detected, but Fortran tests were requested. Install oneAPI HPC Toolkit.")
+  endif()
 endif()
 
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules/")
-include(SetLibraryPath)
-
-message(STATUS ${MKL_INCLUDE_DIR})
-message(STATUS ${MKL_LIBRARY_DIR})
-message(STATUS ${TBB_INCLUDE_DIR})
-message(STATUS ${TBB_LIBRARY_DIR})
-
-find_library(mkl_core NAMES mkl_core PATHS ${MKL_LIBRARY_DIR} REQUIRED)
-find_library(mkl_sycl NAMES mkl_sycl PATHS ${MKL_LIBRARY_DIR} REQUIRED)
-find_library(mkl_intel_ilp64 NAMES mkl_intel_ilp64 PATHS ${MKL_LIBRARY_DIR} REQUIRED)
-find_library(mkl_tbb_thread NAMES mkl_tbb_thread PATHS ${MKL_LIBRARY_DIR} REQUIRED)
-find_library(tbb NAMES tbb PATHS ${TBB_LIBRARY_DIR} REQUIRED)
+set(MKL_THREADING tbb_thread)
+set(DPCPP_COMPILER ON)
+find_package(MKL REQUIRED)
 
 string(CONCAT WARNING_CXX_FLAGS_STR
   "-Wall "
@@ -45,29 +41,20 @@ string(REPLACE " " ";" COMMON_CXX_FLAGS "${WARNING_CXX_FLAGS_STR}")
 function(add_example_with_mkl name)
   add_executable(${name} ${name}.cpp)
   target_compile_options(${name} PRIVATE ${COMMON_CXX_FLAGS})
-  target_compile_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64 -qmkl=parallel)
-  target_link_directories(${name}
-    PUBLIC ${MKL_LIBRARY_DIR} ${TBB_LIBRARY_DIR}
-  )
-  target_link_libraries(${name}
-    PRIVATE ${mkl_sycl}
-  )
-  target_link_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64 -qmkl=parallel -fsycl -lmkl_sycl -fsycl-device-code-split=per_kernel)
+  target_compile_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64 -qmkl)
+  target_link_libraries(${name} PRIVATE MKL::MKL_DPCPP)
+  target_link_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64 -qmkl -fsycl-device-code-split=per_kernel)
   add_test(NAME ${name} COMMAND ${name} ${ARGN})
 endfunction(add_example_with_mkl)
 
 function(add_fortran_example_with_mkl name)
   if(CMAKE_Fortran_COMPILER)
     add_executable(${name} ${name}.f)
     target_compile_options(${name} PRIVATE -warn all)
-    target_compile_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64 -qmkl=parallel -fpp -free)
-    target_link_directories(${name}
-      PUBLIC ${MKL_LIBRARY_DIR}
-    )
-    target_link_libraries(${name}
-      PRIVATE ${mkl_sycl}
-    )
-    target_link_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64 -qmkl=parallel -fsycl -lmkl_sycl)
+    target_compile_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64 -qmkl -fpp -free)
+    set_target_properties(${name} PROPERTIES LINKER_LANGUAGE Fortran)
+    target_link_libraries(${name} PUBLIC MKL::MKL_DPCPP)
+    target_link_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64)
     add_test(NAME ${name} COMMAND ${name} ${ARGN})
   endif()
 endfunction(add_fortran_example_with_mkl)
@@ -84,11 +71,22 @@ function(add_fortran_example name)
     add_executable(${name} ${name}.f90)
     target_compile_options(${name} PRIVATE -warn all)
     target_compile_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64)
+    set_target_properties(${name} PROPERTIES LINKER_LANGUAGE Fortran)
     target_link_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64)
     add_test(NAME ${name} COMMAND ${name} ${ARGN})
   endif()
 endfunction(add_fortran_example)
 
+function(add_mpi_example name)
+  if(MPI_FOUND)
+    add_executable(${name} ${name}.cpp)
+    target_compile_options(${name} PRIVATE -O3 -fiopenmp -fopenmp-targets=spir64)
+    target_link_options(${name} PRIVATE -O3 -fiopenmp -fopenmp-targets=spir64)
+    target_link_libraries(${name} PRIVATE MPI::MPI_CXX)
+    add_test(NAME ${name} COMMAND ${name} ${ARGN})
+  endif()
+endfunction(add_mpi_example)
+
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 
 add_subdirectory(matrix)
@@ -114,3 +112,5 @@ add_subdirectory(multiple-kernel-execution)
 add_subdirectory(work-group-size)
 add_subdirectory(registers)
 add_subdirectory(OpenMP)
+add_subdirectory(MPI)
+add_subdirectory(grf-mode-selection)

Publications/GPU-Opt-Guide/MPI/01_omp_mpich/CMakeLists.txt

@@ -0,0 +1 @@
+add_mpi_example(omp_mpich)

Publications/GPU-Opt-Guide/MPI/01_omp_mpich/omp_mpich.cpp

@@ -0,0 +1,93 @@
+//==============================================================
+// Copyright © 2022 Intel Corporation
+//
+// SPDX-License-Identifier: MIT
+// =============================================================
+// clang-format off
+#include <mpi.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <cmath>
+#include <omp.h>
+
+#define N 1000000
+#define STEPS 1000
+#define ABS(x) (x) > 0 ? (x) : -(x)
+
+int main (int argc, char **argv )
+{
+   int mpi_aware = 0;
+   if ( argc > 1 ) {
+       mpi_aware = 1;
+       printf("MPI device aware path enabled\n");
+   } // argc check
+
+   MPI_Init(NULL, NULL);
+   int rank,nranks;
+   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+   MPI_Comm_size(MPI_COMM_WORLD, &nranks);
+
+   int next_rank = ( rank + 1 ) % nranks;
+   int prev_rank = rank == 0 ? nranks-1 : rank-1;
+   printf("rank=%d next=%d prev=%d\n",rank,next_rank,prev_rank);
+
+#pragma omp target
+   ;
+
+   double buf1[N],buf2[N];
+   double *curr,*next,*tmp;
+
+   for ( int  i = 0; i < N; i++ ) {
+      buf1[i] = 0;
+      buf2[i] = 0;
+   }
+
+   MPI_Request psrq;
+   double start = omp_get_wtime();
+#pragma omp target data map(buf1,buf2)
+{
+  #pragma omp target data use_device_addr(buf1,buf2) if(mpi_aware)
+  {
+    curr = buf1;
+    next = buf2;
+  }
+  printf("curr=%p next=%p\n",curr,next);
+
+  for ( int step = 0; step < STEPS; step++ ) {
+     if ( rank == 0 && step % 100 == 0 ) printf("step: %d\n",step);
+
+#pragma omp target teams distribute parallel for
+     for ( int i = 0; i < N; i++ ) curr[i]++;
+
+     if ( nranks > 1 ) {
+        #pragma omp target update from(curr[0:N]) if(!mpi_aware)
+        MPI_Request srq;
+        MPI_Isend(curr,N,MPI_DOUBLE,next_rank,0,MPI_COMM_WORLD,&srq);
+	// we need to make sure that the MPI_Isend of the previous
+	// iteration finished before doing the MPI_Recv of this
+	// iteration
+	if ( step > 0 ) MPI_Wait(&psrq,MPI_STATUS_IGNORE);
+	psrq = srq;
+        MPI_Recv(next,N,MPI_DOUBLE,prev_rank,0,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
+        #pragma omp target update to(next[0:N]) if(!mpi_aware)
+     } // nranks
+
+     tmp = curr;
+     curr = next;
+     next = tmp;
+   }
+}
+
+   MPI_Barrier(MPI_COMM_WORLD);
+   double end = omp_get_wtime();
+   printf("rank %d total_time=%g\n",rank, end-start);
+
+   for ( int i = 0; i < N; i++ ) {
+       if ( buf1[i] != STEPS ) {
+         printf("Error in %d = %f\n",i,buf1[i]);
+         break;
+       }
+   }
+
+   return 0;
+}

Publications/GPU-Opt-Guide/MPI/CMakeLists.txt

@@ -0,0 +1,3 @@
+find_package(MPI)
+
+add_subdirectory(01_omp_mpich)

Publications/GPU-Opt-Guide/OpenMP/04_target_nowait/test_target_nowait_f.f90

@@ -11,7 +11,7 @@
 
       subroutine init(n, v1, v2)
       integer :: i, n
-      real :: v1(n), v2(n), vxv(n)
+      real :: v1(n), v2(n)
 
       do i = 1, n
          v1(i) = i * 0.25

Publications/GPU-Opt-Guide/OpenMP/22_mkl_dispatch/CMakeLists.txt

@@ -1,4 +1,3 @@
-
 include_directories(common)
 
 add_example_with_mkl(dgemm_target_variant_dispatch_c)

Publications/GPU-Opt-Guide/OpenMP/22_mkl_pad/CMakeLists.txt

@@ -0,0 +1,4 @@
+include_directories(common)
+
+add_example_with_mkl(dgemm_pad_c_01)
+add_fortran_example_with_mkl(dgemm_pad_f_01)