std: use futex for thread parking and Once on Apple platforms

joboet · joboet · commit 82fa41a8435d · 2024-03-09T13:04:28.000+01:00
diff --git a/library/std/src/sys/pal/unix/futex.rs b/library/std/src/sys/pal/unix/futex.rs
@@ -6,6 +6,10 @@
     target_os = "openbsd",
     target_os = "dragonfly",
     target_os = "fuchsia",
+    target_os = "macos",
+    target_os = "ios",
+    target_os = "tvos",
+    target_os = "watchos",
 ))]
 
 use crate::sync::atomic::AtomicU32;
@@ -135,6 +139,186 @@ pub fn futex_wake_all(futex: &AtomicU32) {
     };
 }
 
+/// With macOS version 14.4, Apple introduced a public futex API. Unfortunately,
+/// our minimum supported version is 10.12, so we need a fallback API. Luckily
+/// for us, the underlying syscalls have been available since exactly that
+/// version, so we just use those when needed. This is private API however,
+/// which means we need to take care to avoid breakage if the syscall is removed
+/// and to avoid apps being rejected from the App Store. To do this, we use weak
+/// linkage emulation for both the public and the private API.
+///
+/// See os/os_sync_wait_on_address.h (Apple has failed to upload the documentation
+/// to their website) for documentation of the public API and
+/// https://github.com/apple-oss-distributions/xnu/blob/1031c584a5e37aff177559b9f69dbd3c8c3fd30a/bsd/sys/ulock.h#L69
+/// for the header file of the private API, along with its usage in libpthread
+/// https://github.com/apple-oss-distributions/libpthread/blob/d8c4e3c212553d3e0f5d76bb7d45a8acd61302dc/src/pthread_cond.c#L463
+#[cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos",))]
+mod apple {
+    use crate::ffi::{c_int, c_void};
+    use crate::sys::pal::weak::weak;
+
+    pub const OS_CLOCK_MACH_ABSOLUTE_TIME: u32 = 32;
+    pub const OS_SYNC_WAIT_ON_ADDRESS_NONE: u32 = 0;
+    pub const OS_SYNC_WAKE_BY_ADDRESS_NONE: u32 = 0;
+
+    pub const UL_COMPARE_AND_WAIT: u32 = 1;
+    pub const ULF_WAKE_ALL: u32 = 0x100;
+    // The syscalls support directly returning errors instead of going through errno.
+    pub const ULF_NO_ERRNO: u32 = 0x1000000;
+
+    // These functions appeared with macOS 14.4, iOS 17.4, tvOS 17.4, watchOS 10.4, visionOS 1.1.
+    weak! {
+        pub fn os_sync_wait_on_address(*mut c_void, u64, usize, u32) -> c_int
+    }
+
+    weak! {
+        pub fn os_sync_wait_on_address_with_timeout(*mut c_void, u64, usize, u32, u32, u64) -> c_int
+    }
+
+    weak! {
+        pub fn os_sync_wake_by_address_any(*mut c_void, usize, u32) -> c_int
+    }
+
+    weak! {
+        pub fn os_sync_wake_by_address_all(*mut c_void, usize, u32) -> c_int
+    }
+
+    // This syscall appeared with macOS 11.0.
+    // It is used to support nanosecond precision for timeouts, among other features.
+    weak! {
+        pub fn __ulock_wait2(u32, *mut c_void, u64, u64, u64) -> c_int
+    }
+
+    // These syscalls appeared with macOS 10.12.
+    weak! {
+        pub fn __ulock_wait(u32, *mut c_void, u64, u32) -> c_int
+    }
+
+    weak! {
+        pub fn __ulock_wake(u32, *mut c_void, u64) -> c_int
+    }
+}
+
+#[cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos",))]
+pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool {
+    use crate::mem::size_of;
+    use apple::*;
+
+    let addr = futex.as_ptr().cast();
+    let value = expected as u64;
+    let size = size_of::<u32>();
+    if let Some(timeout) = timeout {
+        let timeout_ns = timeout.as_nanos().clamp(1, u64::MAX as u128) as u64;
+        let timeout_ms = timeout.as_micros().clamp(1, u32::MAX as u128) as u32;
+
+        if let Some(wait) = os_sync_wait_on_address_with_timeout.get() {
+            let r = unsafe {
+                wait(
+                    addr,
+                    value,
+                    size,
+                    OS_SYNC_WAIT_ON_ADDRESS_NONE,
+                    OS_CLOCK_MACH_ABSOLUTE_TIME,
+                    timeout_ns,
+                )
+            };
+
+            // We promote spurious wakeups (reported as EINTR) to normal ones for
+            // simplicity and because Apple's documentation is unclear as to what the
+            // unit for the deadline of `os_sync_wait_on_address_with_deadline` is,
+            // making it hard to support timeouts in a fashion similar to the Linux
+            // futex implementation.
+            r != -1 || super::os::errno() != libc::ETIMEDOUT
+        } else if let Some(wait) = __ulock_wait2.get() {
+            unsafe {
+                wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, value, timeout_ns, 0)
+                    != -libc::ETIMEDOUT
+            }
+        } else if let Some(wait) = __ulock_wait.get() {
+            unsafe {
+                wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, value, timeout_ms)
+                    != -libc::ETIMEDOUT
+            }
+        } else {
+            panic!("your system is below the minimum supported version of Rust");
+        }
+    } else {
+        if let Some(wait) = os_sync_wait_on_address.get() {
+            unsafe {
+                wait(addr, value, size, OS_SYNC_WAIT_ON_ADDRESS_NONE);
+            }
+        } else if let Some(wait) = __ulock_wait.get() {
+            unsafe {
+                wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, value, 0);
+            }
+        } else {
+            panic!("your system is below the minimum supported version of Rust");
+        }
+        true
+    }
+}
+
+#[cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos",))]
+pub fn futex_wake(futex: &AtomicU32) -> bool {
+    use crate::io::Error;
+    use crate::mem::size_of;
+    use apple::*;
+
+    let addr = futex.as_ptr().cast();
+    if let Some(wake) = os_sync_wake_by_address_any.get() {
+        unsafe { wake(addr, size_of::<u32>(), OS_SYNC_WAKE_BY_ADDRESS_NONE) == 0 }
+    } else if let Some(wake) = __ulock_wake.get() {
+        // __ulock_wake can get interrupted, so retry until either waking up a
+        // waiter or failing because there are no waiters (ENOENT).
+        loop {
+            let r = unsafe { wake(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, 0) };
+
+            if r >= 0 {
+                return true;
+            } else {
+                match -r {
+                    libc::ENOENT => return false,
+                    libc::EINTR => continue,
+                    err => panic!("__ulock_wake failed: {}", Error::from_raw_os_error(err)),
+                }
+            }
+        }
+    } else {
+        panic!("your system is below the minimum supported version of Rust");
+    }
+}
+
+#[cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos",))]
+pub fn futex_wake_all(futex: &AtomicU32) {
+    use crate::io::Error;
+    use crate::mem::size_of;
+    use apple::*;
+
+    let addr = futex.as_ptr().cast();
+
+    if let Some(wake) = os_sync_wake_by_address_all.get() {
+        unsafe {
+            wake(addr, size_of::<u32>(), OS_SYNC_WAKE_BY_ADDRESS_NONE);
+        }
+    } else if let Some(wake) = __ulock_wake.get() {
+        loop {
+            let r = unsafe { wake(UL_COMPARE_AND_WAIT | ULF_WAKE_ALL | ULF_NO_ERRNO, addr, 0) };
+
+            if r >= 0 {
+                return;
+            } else {
+                match -r {
+                    libc::ENOENT => return,
+                    libc::EINTR => continue,
+                    err => panic!("__ulock_wake failed: {}", Error::from_raw_os_error(err)),
+                }
+            }
+        }
+    } else {
+        panic!("your system is below the minimum supported version of Rust");
+    }
+}
+
 #[cfg(target_os = "openbsd")]
 pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool {
     use super::time::Timespec;
diff --git a/library/std/src/sys/pal/unix/thread_parking/darwin.rs b/library/std/src/sys/pal/unix/thread_parking/darwin.rs
diff --git a/library/std/src/sys/pal/unix/thread_parking/mod.rs b/library/std/src/sys/pal/unix/thread_parking/mod.rs
@@ -8,21 +8,14 @@
     target_os = "openbsd",
     target_os = "dragonfly",
     target_os = "fuchsia",
+    target_os = "macos",
+    target_os = "ios",
+    target_os = "tvos",
+    target_os = "watchos",
 )))]
 
 cfg_if::cfg_if! {
-    if #[cfg(all(
-        any(
-            target_os = "macos",
-            target_os = "ios",
-            target_os = "watchos",
-            target_os = "tvos",
-        ),
-        not(miri),
-    ))] {
-        mod darwin;
-        pub use darwin::Parker;
-    } else if #[cfg(target_os = "netbsd")] {
+    if #[cfg(target_os = "netbsd")] {
         mod netbsd;
         pub use netbsd::{current, park, park_timeout, unpark, ThreadId};
     } else {
diff --git a/library/std/src/sys_common/once/mod.rs b/library/std/src/sys_common/once/mod.rs
@@ -17,6 +17,10 @@ cfg_if::cfg_if! {
         target_os = "dragonfly",
         target_os = "fuchsia",
         target_os = "hermit",
+        target_os = "macos",
+        target_os = "ios",
+        target_os = "tvos",
+        target_os = "watchos",
     ))] {
         mod futex;
         pub use futex::{Once, OnceState};
diff --git a/library/std/src/sys_common/thread_parking/mod.rs b/library/std/src/sys_common/thread_parking/mod.rs
@@ -8,6 +8,10 @@ cfg_if::cfg_if! {
         target_os = "dragonfly",
         target_os = "fuchsia",
         target_os = "hermit",
+        target_os = "macos",
+        target_os = "ios",
+        target_os = "tvos",
+        target_os = "watchos",
     ))] {
         mod futex;
         pub use futex::Parker;
