[OpenMP][AMDGPU] Add interop support for OpenMP AMD GPU plugin

openmp/libomptarget/include/OpenMP/InteropAPI.h

@@ -11,33 +11,130 @@
 #ifndef OMPTARGET_OPENMP_INTEROP_API_H
 #define OMPTARGET_OPENMP_INTEROP_API_H
 
-#include "omp.h"
+#define omp_interop_none 0
 
+#include "omp.h"
 #include "omptarget.h"
 
 extern "C" {
 
+/// TODO: Include the `omp.h` of the current build
+/* OpenMP 5.1 interop */
+typedef intptr_t omp_intptr_t;
+
+/* 0..omp_get_num_interop_properties()-1 are reserved for implementation-defined
+ * properties */
+typedef enum omp_interop_property {
+  omp_ipr_fr_id = -1,
+  omp_ipr_fr_name = -2,
+  omp_ipr_vendor = -3,
+  omp_ipr_vendor_name = -4,
+  omp_ipr_device_num = -5,
+  omp_ipr_platform = -6,
+  omp_ipr_device = -7,
+  omp_ipr_device_context = -8,
+  omp_ipr_targetsync = -9,
+  omp_ipr_first = -9
+} omp_interop_property_t;
+
+typedef enum omp_interop_rc {
+  omp_irc_no_value = 1,
+  omp_irc_success = 0,
+  omp_irc_empty = -1,
+  omp_irc_out_of_range = -2,
+  omp_irc_type_int = -3,
+  omp_irc_type_ptr = -4,
+  omp_irc_type_str = -5,
+  omp_irc_other = -6
+} omp_interop_rc_t;
+
+typedef enum omp_interop_fr {
+  omp_ifr_cuda = 1,
+  omp_ifr_cuda_driver = 2,
+  omp_ifr_opencl = 3,
+  omp_ifr_sycl = 4,
+  omp_ifr_hip = 5,
+  omp_ifr_level_zero = 6,
+  omp_ifr_amdhsa = 7,
+  omp_ifr_last = 8
+} omp_interop_fr_t;
+
+typedef enum omp_interop_backend_type_t {
+  // reserve 0
+  omp_interop_backend_type_cuda = 1,
+  omp_interop_backend_type_amdhsa = 7,
+  omp_interop_backend_type_invalid = 8
+} omp_interop_backend_type_t;
+
+typedef enum omp_foreign_runtime_ids {
+  invalid = 0,
+  cuda = 1,
+  cuda_driver = 2,
+  opencl = 3,
+  sycl = 4,
+  hip = 5,
+  level_zero = 6,
+  amdhsa = 7
+} omp_foreign_runtime_ids_t;
+
 typedef enum kmp_interop_type_t {
   kmp_interop_type_unknown = -1,
   kmp_interop_type_platform,
   kmp_interop_type_device,
   kmp_interop_type_tasksync,
 } kmp_interop_type_t;
 
+typedef void *omp_interop_t;
+
 /// The interop value type, aka. the interop object.
 typedef struct omp_interop_val_t {
   /// Device and interop-type are determined at construction time and fix.
-  omp_interop_val_t(intptr_t device_id, kmp_interop_type_t interop_type)
-      : interop_type(interop_type), device_id(device_id) {}
+  omp_interop_val_t(intptr_t device_id, kmp_interop_type_t interop_type,
+                    omp_foreign_runtime_ids_t vendor_id,
+                    intptr_t backend_type_id)
+      : interop_type(interop_type), device_id(device_id), vendor_id(vendor_id),
+        backend_type_id(backend_type_id) {}
   const char *err_str = nullptr;
   __tgt_async_info *async_info = nullptr;
   __tgt_device_info device_info;
   const kmp_interop_type_t interop_type;
   const intptr_t device_id;
-  const omp_foreign_runtime_ids_t vendor_id = cuda;
-  const intptr_t backend_type_id = omp_interop_backend_type_cuda_1;
+  omp_foreign_runtime_ids_t vendor_id;
+  intptr_t backend_type_id;
 } omp_interop_val_t;
 
+/// Retrieves the number of implementation-defined properties available for an
+/// omp_interop_t object.
+int __KAI_KMPC_CONVENTION omp_get_num_interop_properties(const omp_interop_t);
+
+/// Retrieves an integer property from an omp_interop_t object.
+omp_intptr_t __KAI_KMPC_CONVENTION omp_get_interop_int(const omp_interop_t,
+                                                       omp_interop_property_t,
+                                                       int *);
+
+/// Retrieves a pointer property from an omp_interop_t object.
+void *__KAI_KMPC_CONVENTION omp_get_interop_ptr(const omp_interop_t,
+                                                omp_interop_property_t, int *);
+
+/// Retrieve a string property from an omp_interop_t object.
+const char *__KAI_KMPC_CONVENTION omp_get_interop_str(const omp_interop_t,
+                                                      omp_interop_property_t,
+                                                      int *);
+
+/// Retrieve a property name from an omp_interop_t object.
+const char *__KAI_KMPC_CONVENTION omp_get_interop_name(const omp_interop_t,
+                                                       omp_interop_property_t);
+
+/// Retrieve a description of the type of a property associated with an
+/// omp_interop_t object.
+const char *__KAI_KMPC_CONVENTION
+omp_get_interop_type_desc(const omp_interop_t, omp_interop_property_t);
+
+/// Retrieve a description of the return code associated with an omp_interop_t
+/// object.
+extern const char *__KAI_KMPC_CONVENTION
+omp_get_interop_rc_desc(const omp_interop_t, omp_interop_rc_t);
+
 } // extern "C"
 
 #endif // OMPTARGET_OPENMP_INTEROP_API_H

openmp/libomptarget/include/OpenMP/omp.h

@@ -44,112 +44,6 @@ int omp_get_default_device(void) __attribute__((weak));
 
 ///}
 
-/// InteropAPI
-///
-///{
-
-/// TODO: Include the `omp.h` of the current build
-/* OpenMP 5.1 interop */
-typedef intptr_t omp_intptr_t;
-
-/* 0..omp_get_num_interop_properties()-1 are reserved for implementation-defined
- * properties */
-typedef enum omp_interop_property {
-  omp_ipr_fr_id = -1,
-  omp_ipr_fr_name = -2,
-  omp_ipr_vendor = -3,
-  omp_ipr_vendor_name = -4,
-  omp_ipr_device_num = -5,
-  omp_ipr_platform = -6,
-  omp_ipr_device = -7,
-  omp_ipr_device_context = -8,
-  omp_ipr_targetsync = -9,
-  omp_ipr_first = -9
-} omp_interop_property_t;
-
-#define omp_interop_none 0
-
-typedef enum omp_interop_rc {
-  omp_irc_no_value = 1,
-  omp_irc_success = 0,
-  omp_irc_empty = -1,
-  omp_irc_out_of_range = -2,
-  omp_irc_type_int = -3,
-  omp_irc_type_ptr = -4,
-  omp_irc_type_str = -5,
-  omp_irc_other = -6
-} omp_interop_rc_t;
-
-typedef enum omp_interop_fr {
-  omp_ifr_cuda = 1,
-  omp_ifr_cuda_driver = 2,
-  omp_ifr_opencl = 3,
-  omp_ifr_sycl = 4,
-  omp_ifr_hip = 5,
-  omp_ifr_level_zero = 6,
-  omp_ifr_last = 7
-} omp_interop_fr_t;
-
-typedef void *omp_interop_t;
-
-/*!
- * The `omp_get_num_interop_properties` routine retrieves the number of
- * implementation-defined properties available for an `omp_interop_t` object.
- */
-int __KAI_KMPC_CONVENTION omp_get_num_interop_properties(const omp_interop_t);
-/*!
- * The `omp_get_interop_int` routine retrieves an integer property from an
- * `omp_interop_t` object.
- */
-omp_intptr_t __KAI_KMPC_CONVENTION
-omp_get_interop_int(const omp_interop_t, omp_interop_property_t, int *);
-/*!
- * The `omp_get_interop_ptr` routine retrieves a pointer property from an
- * `omp_interop_t` object.
- */
-void *__KAI_KMPC_CONVENTION omp_get_interop_ptr(const omp_interop_t,
-                                                omp_interop_property_t, int *);
-/*!
- * The `omp_get_interop_str` routine retrieves a string property from an
- * `omp_interop_t` object.
- */
-const char *__KAI_KMPC_CONVENTION
-omp_get_interop_str(const omp_interop_t, omp_interop_property_t, int *);
-/*!
- * The `omp_get_interop_name` routine retrieves a property name from an
- * `omp_interop_t` object.
- */
-const char *__KAI_KMPC_CONVENTION omp_get_interop_name(const omp_interop_t,
-                                                       omp_interop_property_t);
-/*!
- * The `omp_get_interop_type_desc` routine retrieves a description of the type
- * of a property associated with an `omp_interop_t` object.
- */
-const char *__KAI_KMPC_CONVENTION
-omp_get_interop_type_desc(const omp_interop_t, omp_interop_property_t);
-/*!
- * The `omp_get_interop_rc_desc` routine retrieves a description of the return
- * code associated with an `omp_interop_t` object.
- */
-extern const char *__KAI_KMPC_CONVENTION
-omp_get_interop_rc_desc(const omp_interop_t, omp_interop_rc_t);
-
-typedef enum omp_interop_backend_type_t {
-  // reserve 0
-  omp_interop_backend_type_cuda_1 = 1,
-} omp_interop_backend_type_t;
-
-typedef enum omp_foreign_runtime_ids {
-  cuda = 1,
-  cuda_driver = 2,
-  opencl = 3,
-  sycl = 4,
-  hip = 5,
-  level_zero = 6,
-} omp_foreign_runtime_ids_t;
-
-///} InteropAPI
-
 } // extern "C"
 
 #endif // OMPTARGET_OPENMP_OMP_H

openmp/libomptarget/include/Shared/PluginAPI.h

@@ -17,6 +17,7 @@
 #include <cstddef>
 #include <cstdint>
 
+#include "OpenMP/InteropAPI.h"
 #include "Shared/APITypes.h"
 
 extern "C" {
@@ -165,6 +166,9 @@ void __tgt_rtl_set_info_flag(uint32_t);
 // Print the device information
 void __tgt_rtl_print_device_info(int32_t ID);
 
+// Set the runtime related information for interop object
+int32_t __tgt_rtl_set_interop_info(omp_interop_val_t *InteropPtr);
+
 // Event related interfaces. It is expected to use the interfaces in the
 // following way:
 // 1) Create an event on the target device (__tgt_rtl_create_event).

openmp/libomptarget/include/Shared/PluginAPI.inc

@@ -35,6 +35,7 @@ PLUGIN_API_HANDLE(synchronize);
 PLUGIN_API_HANDLE(query_async);
 PLUGIN_API_HANDLE(set_info_flag);
 PLUGIN_API_HANDLE(print_device_info);
+PLUGIN_API_HANDLE(set_interop_info);
 PLUGIN_API_HANDLE(create_event);
 PLUGIN_API_HANDLE(record_event);
 PLUGIN_API_HANDLE(wait_event);

openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp

@@ -2772,6 +2772,17 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
   bool useMultipleSdmaEngines() const { return OMPX_UseMultipleSdmaEngines; }
 
+  virtual Error setInteropInfo(omp_interop_val_t *InterOpPtr) override {
+    InterOpPtr->vendor_id = amdhsa;
+    InterOpPtr->backend_type_id = omp_interop_backend_type_amdhsa;
+
+    __tgt_device_info *DevInfo = &InterOpPtr->device_info;
+    DevInfo->Context = nullptr;
+    DevInfo->Device = &Agent;
+
+    return Plugin::success();
+  }
+
 private:
   using AMDGPUEventRef = AMDGPUResourceRef<AMDGPUEventTy>;
   using AMDGPUEventManagerTy = GenericDeviceResourceManagerTy<AMDGPUEventRef>;

openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h

@@ -19,6 +19,7 @@
 #include <shared_mutex>
 #include <vector>
 
+#include "OpenMP/InteropAPI.h"
 #include "Shared/Debug.h"
 #include "Shared/Environment.h"
 #include "Shared/EnvironmentVar.h"
@@ -850,6 +851,10 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
     return 0;
   }
 
+  virtual Error setInteropInfo(omp_interop_val_t *InterOpPtr) {
+    return Error::success();
+  }
+
   virtual Error getDeviceStackSize(uint64_t &V) = 0;
 
   /// Returns true if current plugin architecture is an APU
@@ -1059,7 +1064,6 @@ struct GenericPluginTy {
   /// we could not move this function into GenericDeviceTy.
   virtual Expected<bool> isELFCompatible(StringRef Image) const = 0;
 
-protected:
   /// Indicate whether a device id is valid.
   bool isValidDeviceId(int32_t DeviceId) const {
     return (DeviceId >= 0 && DeviceId < getNumDevices());

openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp

@@ -2040,6 +2040,21 @@ int32_t __tgt_rtl_init_plugin() {
   return OFFLOAD_SUCCESS;
 }
 
+int32_t __tgt_rtl_set_interop_info(omp_interop_val_t *InterOpPtr) {
+  assert(InterOpPtr && "Interop object is allocated");
+  int32_t DevId = InterOpPtr->device_id;
+
+  assert(PluginTy::get().isValidDeviceId(DevId) && "Device Id is valid");
+  if (auto Err = PluginTy::get().getDevice(DevId).setInteropInfo(InterOpPtr)) {
+    REPORT("Failure to determine the OpenMP interop object info for Device Id "
+           "%i\n",
+           DevId);
+    return OFFLOAD_FAIL;
+  }
+
+  return OFFLOAD_SUCCESS;
+}
+
 int32_t __tgt_rtl_is_valid_binary(__tgt_device_image *Image) {
   if (!PluginTy::isActive())
     return false;

openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp

@@ -1143,6 +1143,17 @@ struct CUDADeviceTy : public GenericDeviceTy {
   /// Returns the clock frequency for the given NVPTX device.
   uint64_t getClockFrequency() const override { return 1000000000; }
 
+  virtual Error setInteropInfo(omp_interop_val_t *InterOpPtr) override {
+    InterOpPtr->vendor_id = cuda;
+    InterOpPtr->backend_type_id = omp_interop_backend_type_cuda;
+
+    __tgt_device_info *DevInfo = &InterOpPtr->device_info;
+    DevInfo->Context = Context;
+    DevInfo->Device = Device;
+
+    return Plugin::success();
+  }
+
 private:
   using CUDAStreamManagerTy = GenericDeviceResourceManagerTy<CUDAStreamRef>;
   using CUDAEventManagerTy = GenericDeviceResourceManagerTy<CUDAEventRef>;