portable_atomic/
utils.rs

Help
1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3#![cfg_attr(not(all(test, feature = "float")), allow(dead_code, unused_macros))]
4
5#[macro_use]
6#[path = "gen/utils.rs"]
7mod generated;
8
9use core::sync::atomic::Ordering;
10
11macro_rules! static_assert {
12    ($cond:expr $(,)?) => {{
13        let [] = [(); true as usize - $crate::utils::_assert_is_bool($cond) as usize];
14    }};
15}
16pub(crate) const fn _assert_is_bool(v: bool) -> bool {
17    v
18}
19
20macro_rules! static_assert_layout {
21    ($atomic_type:ty, $value_type:ty) => {
22        static_assert!(
23            core::mem::align_of::<$atomic_type>() == core::mem::size_of::<$atomic_type>()
24        );
25        static_assert!(core::mem::size_of::<$atomic_type>() == core::mem::size_of::<$value_type>());
26    };
27}
28
29// #[doc = concat!(...)] requires Rust 1.54
30macro_rules! doc_comment {
31    ($doc:expr, $($tt:tt)*) => {
32        #[doc = $doc]
33        $($tt)*
34    };
35}
36
37// Adapted from https://github.com/BurntSushi/memchr/blob/2.4.1/src/memchr/x86/mod.rs#L9-L71.
38/// # Safety
39///
40/// - the caller must uphold the safety contract for the function returned by $detect_body.
41/// - the memory pointed by the function pointer returned by $detect_body must be visible from any threads.
42///
43/// The second requirement is always met if the function pointer is to the function definition.
44/// (Currently, all uses of this macro in our code are in this case.)
45#[allow(unused_macros)]
46#[cfg(not(portable_atomic_no_outline_atomics))]
47#[cfg(any(
48    target_arch = "aarch64",
49    target_arch = "arm",
50    target_arch = "arm64ec",
51    target_arch = "powerpc64",
52    target_arch = "riscv32",
53    target_arch = "riscv64",
54    all(target_arch = "x86_64", not(any(target_env = "sgx", miri))),
55))]
56macro_rules! ifunc {
57    (unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)? { $($detect_body:tt)* }) => {{
58        type FnTy = unsafe fn($($arg_ty),*) $(-> $ret_ty)?;
59        static FUNC: core::sync::atomic::AtomicPtr<()>
60            = core::sync::atomic::AtomicPtr::new(detect as *mut ());
61        #[cold]
62        unsafe fn detect($($arg_pat: $arg_ty),*) $(-> $ret_ty)? {
63            let func: FnTy = { $($detect_body)* };
64            FUNC.store(func as *mut (), core::sync::atomic::Ordering::Relaxed);
65            // SAFETY: the caller must uphold the safety contract for the function returned by $detect_body.
66            unsafe { func($($arg_pat),*) }
67        }
68        // SAFETY: `FnTy` is a function pointer, which is always safe to transmute with a `*mut ()`.
69        // (To force the caller to use unsafe block for this macro, do not use
70        // unsafe block here.)
71        let func = {
72            core::mem::transmute::<*mut (), FnTy>(FUNC.load(core::sync::atomic::Ordering::Relaxed))
73        };
74        // SAFETY: the caller must uphold the safety contract for the function returned by $detect_body.
75        // (To force the caller to use unsafe block for this macro, do not use
76        // unsafe block here.)
77        func($($arg_pat),*)
78    }};
79}
80
81#[allow(unused_macros)]
82#[cfg(not(portable_atomic_no_outline_atomics))]
83#[cfg(any(
84    target_arch = "aarch64",
85    target_arch = "arm",
86    target_arch = "arm64ec",
87    target_arch = "powerpc64",
88    target_arch = "riscv32",
89    target_arch = "riscv64",
90    all(target_arch = "x86_64", not(any(target_env = "sgx", miri))),
91))]
92macro_rules! fn_alias {
93    (
94        $(#[$($fn_attr:tt)*])*
95        $vis:vis unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)?;
96        $(#[$($alias_attr:tt)*])*
97        $new:ident = $from:ident($($last_args:tt)*);
98        $($rest:tt)*
99    ) => {
100        $(#[$($fn_attr)*])*
101        $(#[$($alias_attr)*])*
102        $vis unsafe fn $new($($arg_pat: $arg_ty),*) $(-> $ret_ty)? {
103            // SAFETY: the caller must uphold the safety contract.
104            unsafe { $from($($arg_pat,)* $($last_args)*) }
105        }
106        fn_alias! {
107            $(#[$($fn_attr)*])*
108            $vis unsafe fn($($arg_pat: $arg_ty),*) $(-> $ret_ty)?;
109            $($rest)*
110        }
111    };
112    (
113        $(#[$($attr:tt)*])*
114        $vis:vis unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)?;
115    ) => {}
116}
117
118/// Make the given function const if the given condition is true.
119macro_rules! const_fn {
120    (
121        const_if: #[cfg($($cfg:tt)+)];
122        $(#[$($attr:tt)*])*
123        $vis:vis const $($rest:tt)*
124    ) => {
125        #[cfg($($cfg)+)]
126        $(#[$($attr)*])*
127        $vis const $($rest)*
128        #[cfg(not($($cfg)+))]
129        $(#[$($attr)*])*
130        $vis $($rest)*
131    };
132}
133
134/// Implements `core::fmt::Debug` and `serde::{Serialize, Deserialize}` (when serde
135/// feature is enabled) for atomic bool, integer, or float.
136macro_rules! impl_debug_and_serde {
137    // TODO(f16_and_f128): Implement serde traits for f16 & f128 once stabilized.
138    (AtomicF16) => {
139        impl_debug!(AtomicF16);
140    };
141    (AtomicF128) => {
142        impl_debug!(AtomicF128);
143    };
144    ($atomic_type:ident) => {
145        impl_debug!($atomic_type);
146        #[cfg(feature = "serde")]
147        #[cfg_attr(docsrs, doc(cfg(feature = "serde")))]
148        impl serde::ser::Serialize for $atomic_type {
149            #[allow(clippy::missing_inline_in_public_items)] // serde doesn't use inline on std atomic's Serialize/Deserialize impl
150            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
151            where
152                S: serde::ser::Serializer,
153            {
154                // https://github.com/serde-rs/serde/blob/v1.0.152/serde/src/ser/impls.rs#L958-L959
155                self.load(Ordering::Relaxed).serialize(serializer)
156            }
157        }
158        #[cfg(feature = "serde")]
159        #[cfg_attr(docsrs, doc(cfg(feature = "serde")))]
160        impl<'de> serde::de::Deserialize<'de> for $atomic_type {
161            #[allow(clippy::missing_inline_in_public_items)] // serde doesn't use inline on std atomic's Serialize/Deserialize impl
162            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
163            where
164                D: serde::de::Deserializer<'de>,
165            {
166                serde::de::Deserialize::deserialize(deserializer).map(Self::new)
167            }
168        }
169    };
170}
171macro_rules! impl_debug {
172    ($atomic_type:ident) => {
173        impl fmt::Debug for $atomic_type {
174            #[inline] // fmt is not hot path, but #[inline] on fmt seems to still be useful: https://github.com/rust-lang/rust/pull/117727
175            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
176                // std atomic types use Relaxed in Debug::fmt: https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L2188
177                fmt::Debug::fmt(&self.load(Ordering::Relaxed), f)
178            }
179        }
180    };
181}
182
183// We do not provide `nand` because it cannot be optimized on neither x86 nor MSP430.
184// https://godbolt.org/z/ahWejchbT
185macro_rules! impl_default_no_fetch_ops {
186    ($atomic_type:ident, bool) => {
187        impl $atomic_type {
188            #[inline]
189            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
190            pub(crate) fn and(&self, val: bool, order: Ordering) {
191                self.fetch_and(val, order);
192            }
193            #[inline]
194            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
195            pub(crate) fn or(&self, val: bool, order: Ordering) {
196                self.fetch_or(val, order);
197            }
198            #[inline]
199            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
200            pub(crate) fn xor(&self, val: bool, order: Ordering) {
201                self.fetch_xor(val, order);
202            }
203        }
204    };
205    ($atomic_type:ident, $int_type:ty) => {
206        impl $atomic_type {
207            #[inline]
208            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
209            pub(crate) fn add(&self, val: $int_type, order: Ordering) {
210                self.fetch_add(val, order);
211            }
212            #[inline]
213            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
214            pub(crate) fn sub(&self, val: $int_type, order: Ordering) {
215                self.fetch_sub(val, order);
216            }
217            #[inline]
218            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
219            pub(crate) fn and(&self, val: $int_type, order: Ordering) {
220                self.fetch_and(val, order);
221            }
222            #[inline]
223            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
224            pub(crate) fn or(&self, val: $int_type, order: Ordering) {
225                self.fetch_or(val, order);
226            }
227            #[inline]
228            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
229            pub(crate) fn xor(&self, val: $int_type, order: Ordering) {
230                self.fetch_xor(val, order);
231            }
232        }
233    };
234}
235macro_rules! impl_default_bit_opts {
236    ($atomic_type:ident, $int_type:ty) => {
237        impl $atomic_type {
238            #[inline]
239            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
240            pub(crate) fn bit_set(&self, bit: u32, order: Ordering) -> bool {
241                let mask = <$int_type>::wrapping_shl(1, bit);
242                self.fetch_or(mask, order) & mask != 0
243            }
244            #[inline]
245            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
246            pub(crate) fn bit_clear(&self, bit: u32, order: Ordering) -> bool {
247                let mask = <$int_type>::wrapping_shl(1, bit);
248                self.fetch_and(!mask, order) & mask != 0
249            }
250            #[inline]
251            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
252            pub(crate) fn bit_toggle(&self, bit: u32, order: Ordering) -> bool {
253                let mask = <$int_type>::wrapping_shl(1, bit);
254                self.fetch_xor(mask, order) & mask != 0
255            }
256        }
257    };
258}
259
260// This just outputs the input as is, but can be used like an item-level block by using it with cfg.
261macro_rules! items {
262    ($($tt:tt)*) => {
263        $($tt)*
264    };
265}
266
267#[allow(dead_code)]
268#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
269// Stable version of https://doc.rust-lang.org/nightly/std/hint/fn.assert_unchecked.html.
270// TODO: use real core::hint::assert_unchecked on 1.81+ https://github.com/rust-lang/rust/pull/123588
271#[inline(always)]
272#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
273pub(crate) unsafe fn assert_unchecked(cond: bool) {
274    if !cond {
275        if cfg!(debug_assertions) {
276            unreachable!()
277        } else {
278            // SAFETY: the caller promised `cond` is true.
279            unsafe { core::hint::unreachable_unchecked() }
280        }
281    }
282}
283
284// https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L3338
285#[inline]
286#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
287pub(crate) fn assert_load_ordering(order: Ordering) {
288    match order {
289        Ordering::Acquire | Ordering::Relaxed | Ordering::SeqCst => {}
290        Ordering::Release => panic!("there is no such thing as a release load"),
291        Ordering::AcqRel => panic!("there is no such thing as an acquire-release load"),
292        _ => unreachable!(),
293    }
294}
295// https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L3323
296#[inline]
297#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
298pub(crate) fn assert_store_ordering(order: Ordering) {
299    match order {
300        Ordering::Release | Ordering::Relaxed | Ordering::SeqCst => {}
301        Ordering::Acquire => panic!("there is no such thing as an acquire store"),
302        Ordering::AcqRel => panic!("there is no such thing as an acquire-release store"),
303        _ => unreachable!(),
304    }
305}
306// https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L3404
307#[inline]
308#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
309pub(crate) fn assert_compare_exchange_ordering(success: Ordering, failure: Ordering) {
310    match success {
311        Ordering::AcqRel
312        | Ordering::Acquire
313        | Ordering::Relaxed
314        | Ordering::Release
315        | Ordering::SeqCst => {}
316        _ => unreachable!(),
317    }
318    match failure {
319        Ordering::Acquire | Ordering::Relaxed | Ordering::SeqCst => {}
320        Ordering::Release => panic!("there is no such thing as a release failure ordering"),
321        Ordering::AcqRel => panic!("there is no such thing as an acquire-release failure ordering"),
322        _ => unreachable!(),
323    }
324}
325
326// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0418r2.html
327// https://github.com/rust-lang/rust/pull/98383
328#[allow(dead_code)]
329#[inline]
330pub(crate) fn upgrade_success_ordering(success: Ordering, failure: Ordering) -> Ordering {
331    match (success, failure) {
332        (Ordering::Relaxed, Ordering::Acquire) => Ordering::Acquire,
333        (Ordering::Release, Ordering::Acquire) => Ordering::AcqRel,
334        (_, Ordering::SeqCst) => Ordering::SeqCst,
335        _ => success,
336    }
337}
338
339/// Zero-extends the given 32-bit pointer to `MaybeUninit<u64>`.
340/// This is used for 64-bit architecture's 32-bit ABI (e.g., AArch64 ILP32 ABI).
341/// See ptr_reg! macro in src/gen/utils.rs for details.
342#[cfg(not(portable_atomic_no_asm_maybe_uninit))]
343#[cfg(target_pointer_width = "32")]
344#[allow(dead_code)]
345#[inline]
346pub(crate) fn zero_extend64_ptr(v: *mut ()) -> core::mem::MaybeUninit<u64> {
347    #[repr(C)]
348    struct ZeroExtended {
349        #[cfg(target_endian = "big")]
350        pad: *mut (),
351        v: *mut (),
352        #[cfg(target_endian = "little")]
353        pad: *mut (),
354    }
355    // SAFETY: we can safely transmute any 64-bit value to MaybeUninit<u64>.
356    unsafe { core::mem::transmute(ZeroExtended { v, pad: core::ptr::null_mut() }) }
357}
358
359#[allow(dead_code)]
360#[cfg(any(
361    target_arch = "aarch64",
362    target_arch = "arm64ec",
363    target_arch = "powerpc64",
364    target_arch = "riscv64",
365    target_arch = "s390x",
366    target_arch = "x86_64",
367))]
368/// A 128-bit value represented as a pair of 64-bit values.
369///
370/// This type is `#[repr(C)]`, both fields have the same in-memory representation
371/// and are plain old data types, so access to the fields is always safe.
372#[derive(Clone, Copy)]
373#[repr(C)]
374pub(crate) union U128 {
375    pub(crate) whole: u128,
376    pub(crate) pair: Pair<u64>,
377}
378#[allow(dead_code)]
379#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
380/// A 64-bit value represented as a pair of 32-bit values.
381///
382/// This type is `#[repr(C)]`, both fields have the same in-memory representation
383/// and are plain old data types, so access to the fields is always safe.
384#[derive(Clone, Copy)]
385#[repr(C)]
386pub(crate) union U64 {
387    pub(crate) whole: u64,
388    pub(crate) pair: Pair<u32>,
389}
390#[allow(dead_code)]
391#[derive(Clone, Copy)]
392#[repr(C)]
393pub(crate) struct Pair<T: Copy> {
394    // little endian order
395    #[cfg(any(
396        target_endian = "little",
397        target_arch = "aarch64",
398        target_arch = "arm",
399        target_arch = "arm64ec",
400    ))]
401    pub(crate) lo: T,
402    pub(crate) hi: T,
403    // big endian order
404    #[cfg(not(any(
405        target_endian = "little",
406        target_arch = "aarch64",
407        target_arch = "arm",
408        target_arch = "arm64ec",
409    )))]
410    pub(crate) lo: T,
411}
412
413#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
414type MinWord = u32;
415#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
416type RetInt = u32;
417// Adapted from https://github.com/taiki-e/atomic-maybe-uninit/blob/v0.3.6/src/utils.rs#L255.
418// Helper for implementing sub-word atomic operations using word-sized LL/SC loop or CAS loop.
419//
420// Refs: https://github.com/llvm/llvm-project/blob/llvmorg-20.1.0-rc1/llvm/lib/CodeGen/AtomicExpandPass.cpp#L799
421// (aligned_ptr, shift, mask)
422#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
423#[allow(dead_code)]
424#[inline]
425pub(crate) fn create_sub_word_mask_values<T>(ptr: *mut T) -> (*mut MinWord, RetInt, RetInt) {
426    #[cfg(portable_atomic_no_strict_provenance)]
427    use self::ptr::PtrExt as _;
428    use core::mem;
429    // RISC-V, MIPS, SPARC, LoongArch, Xtensa, BPF: shift amount of 32-bit shift instructions is 5 bits unsigned (0-31).
430    // PowerPC, C-SKY: shift amount of 32-bit shift instructions is 6 bits unsigned (0-63) and shift amount 32-63 means "clear".
431    // Arm: shift amount of 32-bit shift instructions is 8 bits unsigned (0-255).
432    // Hexagon: shift amount of 32-bit shift instructions is 7 bits signed (-64-63) and negative shift amount means "reverse the direction of the shift".
433    // (On s390x, we don't use the mask returned from this function.)
434    // (See also https://devblogs.microsoft.com/oldnewthing/20230904-00/?p=108704 for others)
435    const SHIFT_MASK: bool = !cfg!(any(
436        target_arch = "bpf",
437        target_arch = "loongarch64",
438        target_arch = "mips",
439        target_arch = "mips32r6",
440        target_arch = "mips64",
441        target_arch = "mips64r6",
442        target_arch = "riscv32",
443        target_arch = "riscv64",
444        target_arch = "s390x",
445        target_arch = "sparc",
446        target_arch = "sparc64",
447        target_arch = "xtensa",
448    ));
449    let ptr_mask = mem::size_of::<MinWord>() - 1;
450    let aligned_ptr = ptr.with_addr(ptr.addr() & !ptr_mask) as *mut MinWord;
451    let ptr_lsb = if SHIFT_MASK {
452        ptr.addr() & ptr_mask
453    } else {
454        // We use 32-bit wrapping shift instructions in asm on these platforms.
455        ptr.addr()
456    };
457    let shift = if cfg!(any(target_endian = "little", target_arch = "s390x")) {
458        ptr_lsb.wrapping_mul(8)
459    } else {
460        (ptr_lsb ^ (mem::size_of::<MinWord>() - mem::size_of::<T>())).wrapping_mul(8)
461    };
462    let mut mask: RetInt = (1 << (mem::size_of::<T>() * 8)) - 1; // !(0 as T) as RetInt
463    if SHIFT_MASK {
464        mask <<= shift;
465    }
466    (aligned_ptr, shift as RetInt, mask)
467}
468
469// This module provides core::ptr strict_provenance/exposed_provenance polyfill for pre-1.84 rustc.
470#[allow(dead_code)]
471pub(crate) mod ptr {
472    #[cfg(portable_atomic_no_strict_provenance)]
473    use core::mem;
474    #[cfg(not(portable_atomic_no_strict_provenance))]
475    #[allow(unused_imports)]
476    pub(crate) use core::ptr::{with_exposed_provenance, with_exposed_provenance_mut};
477
478    #[cfg(portable_atomic_no_strict_provenance)]
479    #[inline(always)]
480    #[must_use]
481    #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
482    pub(crate) fn with_exposed_provenance<T>(addr: usize) -> *const T {
483        addr as *const T
484    }
485    #[cfg(portable_atomic_no_strict_provenance)]
486    #[inline(always)]
487    #[must_use]
488    #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
489    pub(crate) fn with_exposed_provenance_mut<T>(addr: usize) -> *mut T {
490        addr as *mut T
491    }
492
493    #[cfg(portable_atomic_no_strict_provenance)]
494    pub(crate) trait PtrExt<T: ?Sized>: Copy {
495        #[must_use]
496        fn addr(self) -> usize;
497        #[must_use]
498        fn with_addr(self, addr: usize) -> Self
499        where
500            T: Sized;
501    }
502    #[cfg(portable_atomic_no_strict_provenance)]
503    impl<T: ?Sized> PtrExt<T> for *mut T {
504        #[inline(always)]
505        #[must_use]
506        fn addr(self) -> usize {
507            // A pointer-to-integer transmute currently has exactly the right semantics: it returns the
508            // address without exposing the provenance. Note that this is *not* a stable guarantee about
509            // transmute semantics, it relies on sysroot crates having special status.
510            // SAFETY: Pointer-to-integer transmutes are valid (if you are okay with losing the
511            // provenance).
512            #[allow(clippy::transmutes_expressible_as_ptr_casts)]
513            unsafe {
514                mem::transmute(self as *mut ())
515            }
516        }
517        #[allow(clippy::cast_possible_wrap)]
518        #[inline]
519        #[must_use]
520        fn with_addr(self, addr: usize) -> Self
521        where
522            T: Sized,
523        {
524            // This should probably be an intrinsic to avoid doing any sort of arithmetic, but
525            // meanwhile, we can implement it with `wrapping_offset`, which preserves the pointer's
526            // provenance.
527            let self_addr = self.addr() as isize;
528            let dest_addr = addr as isize;
529            let offset = dest_addr.wrapping_sub(self_addr);
530            (self as *mut u8).wrapping_offset(offset) as *mut T
531        }
532    }
533}
534
535// This module provides:
536// - core::ffi polyfill (c_* type aliases and CStr) for pre-1.64 rustc compatibility.
537//   (core::ffi::* (except c_void) requires Rust 1.64)
538// - safe abstraction (c! macro) for creating static C strings without runtime checks.
539//   (c"..." requires Rust 1.77)
540#[cfg(any(test, not(any(windows, target_arch = "x86", target_arch = "x86_64"))))]
541#[cfg(any(not(portable_atomic_no_asm), portable_atomic_unstable_asm))]
542#[allow(dead_code, non_camel_case_types, unused_macros)]
543#[macro_use]
544pub(crate) mod ffi {
545    pub(crate) type c_void = core::ffi::c_void;
546    // c_{,u}int is {i,u}16 on 16-bit targets, otherwise {i,u}32.
547    // https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/ffi/mod.rs#L156
548    #[cfg(target_pointer_width = "16")]
549    pub(crate) type c_int = i16;
550    #[cfg(target_pointer_width = "16")]
551    pub(crate) type c_uint = u16;
552    #[cfg(not(target_pointer_width = "16"))]
553    pub(crate) type c_int = i32;
554    #[cfg(not(target_pointer_width = "16"))]
555    pub(crate) type c_uint = u32;
556    // c_{,u}long is {i,u}64 on non-Windows 64-bit targets, otherwise {i,u}32.
557    // https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/ffi/mod.rs#L168
558    #[cfg(all(target_pointer_width = "64", not(windows)))]
559    pub(crate) type c_long = i64;
560    #[cfg(all(target_pointer_width = "64", not(windows)))]
561    pub(crate) type c_ulong = u64;
562    #[cfg(not(all(target_pointer_width = "64", not(windows))))]
563    pub(crate) type c_long = i32;
564    #[cfg(not(all(target_pointer_width = "64", not(windows))))]
565    pub(crate) type c_ulong = u32;
566    // c_size_t is currently always usize.
567    // https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/ffi/mod.rs#L76
568    pub(crate) type c_size_t = usize;
569    // c_char is u8 by default on non-Apple/non-Windows Arm/C-SKY/Hexagon/MSP430/PowerPC/RISC-V/s390x/Xtensa targets, otherwise i8 by default.
570    // See references in https://github.com/rust-lang/rust/issues/129945 for details.
571    #[cfg(all(
572        not(any(target_vendor = "apple", windows)),
573        any(
574            target_arch = "aarch64",
575            target_arch = "arm",
576            target_arch = "csky",
577            target_arch = "hexagon",
578            target_arch = "msp430",
579            target_arch = "powerpc",
580            target_arch = "powerpc64",
581            target_arch = "riscv32",
582            target_arch = "riscv64",
583            target_arch = "s390x",
584            target_arch = "xtensa",
585        ),
586    ))]
587    pub(crate) type c_char = u8;
588    #[cfg(not(all(
589        not(any(target_vendor = "apple", windows)),
590        any(
591            target_arch = "aarch64",
592            target_arch = "arm",
593            target_arch = "csky",
594            target_arch = "hexagon",
595            target_arch = "msp430",
596            target_arch = "powerpc",
597            target_arch = "powerpc64",
598            target_arch = "riscv32",
599            target_arch = "riscv64",
600            target_arch = "s390x",
601            target_arch = "xtensa",
602        ),
603    )))]
604    pub(crate) type c_char = i8;
605
606    // Static assertions for C type definitions.
607    #[cfg(test)]
608    const _: fn() = || {
609        let _: c_int = 0 as std::os::raw::c_int;
610        let _: c_uint = 0 as std::os::raw::c_uint;
611        let _: c_long = 0 as std::os::raw::c_long;
612        let _: c_ulong = 0 as std::os::raw::c_ulong;
613        #[cfg(unix)]
614        let _: c_size_t = 0 as libc::size_t; // std::os::raw::c_size_t is unstable
615        let _: c_char = 0 as std::os::raw::c_char;
616    };
617
618    #[repr(transparent)]
619    pub(crate) struct CStr([c_char]);
620    impl CStr {
621        #[inline]
622        #[must_use]
623        pub(crate) const fn as_ptr(&self) -> *const c_char {
624            self.0.as_ptr()
625        }
626        /// # Safety
627        ///
628        /// The provided slice **must** be nul-terminated and not contain any interior
629        /// nul bytes.
630        #[inline]
631        #[must_use]
632        pub(crate) unsafe fn from_bytes_with_nul_unchecked(bytes: &[u8]) -> &CStr {
633            // SAFETY: Casting to CStr is safe because *our* CStr is #[repr(transparent)]
634            // and its internal representation is a [u8] too. (Note that std's CStr
635            // is not #[repr(transparent)].)
636            // Dereferencing the obtained pointer is safe because it comes from a
637            // reference. Making a reference is then safe because its lifetime
638            // is bound by the lifetime of the given `bytes`.
639            unsafe { &*(bytes as *const [u8] as *const CStr) }
640        }
641        #[cfg(test)]
642        #[inline]
643        #[must_use]
644        pub(crate) fn to_bytes_with_nul(&self) -> &[u8] {
645            // SAFETY: Transmuting a slice of `c_char`s to a slice of `u8`s
646            // is safe on all supported targets.
647            #[allow(clippy::unnecessary_cast)] // triggered for targets that c_char is u8
648            unsafe {
649                &*(&self.0 as *const [c_char] as *const [u8])
650            }
651        }
652    }
653
654    macro_rules! c {
655        ($s:expr) => {{
656            const BYTES: &[u8] = concat!($s, "\0").as_bytes();
657            const _: () = static_assert!(crate::utils::ffi::_const_is_c_str(BYTES));
658            #[allow(unused_unsafe)]
659            // SAFETY: we've checked `BYTES` is a valid C string
660            unsafe {
661                crate::utils::ffi::CStr::from_bytes_with_nul_unchecked(BYTES)
662            }
663        }};
664    }
665
666    #[must_use]
667    pub(crate) const fn _const_is_c_str(bytes: &[u8]) -> bool {
668        #[cfg(portable_atomic_no_track_caller)]
669        {
670            // const_if_match/const_loop was stabilized (nightly-2020-06-30) 2 days before
671            // track_caller was stabilized (nightly-2020-07-02), so we reuse the cfg for
672            // track_caller here instead of emitting a cfg for const_if_match/const_loop.
673            // https://github.com/rust-lang/rust/pull/72437
674            // track_caller was stabilized 11 days after the oldest nightly version
675            // that uses this module, and is included in the same 1.46 stable release.
676            // The check here is insufficient in this case, but this is fine because this function
677            // is internal code that is not used to process input from the user and our CI checks
678            // all builtin targets and some custom targets with some versions of newer compilers.
679            !bytes.is_empty()
680        }
681        #[cfg(not(portable_atomic_no_track_caller))]
682        {
683            // Based on https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/ffi/c_str.rs#L417
684            // - bytes must be nul-terminated.
685            // - bytes must not contain any interior nul bytes.
686            if bytes.is_empty() {
687                return false;
688            }
689            let mut i = bytes.len() - 1;
690            if bytes[i] != 0 {
691                return false;
692            }
693            // Ending null byte exists, skip to the rest.
694            while i != 0 {
695                i -= 1;
696                if bytes[i] == 0 {
697                    return false;
698                }
699            }
700            true
701        }
702    }
703
704    #[allow(
705        clippy::alloc_instead_of_core,
706        clippy::std_instead_of_alloc,
707        clippy::std_instead_of_core,
708        clippy::undocumented_unsafe_blocks,
709        clippy::wildcard_imports
710    )]
711    #[cfg(test)]
712    mod tests {
713        #[test]
714        fn test_c_macro() {
715            #[track_caller]
716            fn t(s: &crate::utils::ffi::CStr, raw: &[u8]) {
717                assert_eq!(s.to_bytes_with_nul(), raw);
718            }
719            t(c!(""), b"\0");
720            t(c!("a"), b"a\0");
721            t(c!("abc"), b"abc\0");
722            t(c!(concat!("abc", "d")), b"abcd\0");
723        }
724
725        #[test]
726        fn test_is_c_str() {
727            #[track_caller]
728            fn t(bytes: &[u8]) {
729                assert_eq!(
730                    super::_const_is_c_str(bytes),
731                    std::ffi::CStr::from_bytes_with_nul(bytes).is_ok()
732                );
733            }
734            t(b"\0");
735            t(b"a\0");
736            t(b"abc\0");
737            t(b"");
738            t(b"a");
739            t(b"abc");
740            t(b"\0a");
741            t(b"\0a\0");
742            t(b"ab\0c\0");
743            t(b"\0\0");
744        }
745    }
746}
portable_atomic/utils.rs

portable_atomic/
utils.rs
