      1
      2
      3
      4
      5
      6
      7
      8
      9
     10
     11
     12
     13
     14
     15
     16
     17
     18
     19
     20
     21
     22
     23
     24
     25
     26
     27
     28
     29
     30
     31
     32
     33
     34
     35
     36
     37
     38
     39
     40
     41
     42
     43
     44
     45
     46
     47
     48
     49
     50
     51
     52
     53
     54
     55
     56
     57
     58
     59
     60
     61
     62
     63
     64
     65
     66
     67
     68
     69
     70
     71
     72
     73
     74
     75
     76
     77
     78
     79
     80
     81
     82
     83
     84
     85
     86
     87
     88
     89
     90
     91
     92
     93
     94
     95
     96
     97
     98
     99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116

    // Copyright 2016 Amanieu d'Antras
    //
    // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
    // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
    // http://opensource.org/licenses/MIT>, at your option. This file may not be
    // copied, modified, or distributed except according to those terms.

    use std::sync::atomic::AtomicUsize;

    // Extension trait to add lock elision primitives to atomic types
    pub trait AtomicElisionExt {
        type IntType;

        // Perform a compare_exchange and start a transaction
        fn elision_compare_exchange_acquire(
            &self,
            current: Self::IntType,
            new: Self::IntType,
        ) -> Result<Self::IntType, Self::IntType>;

        // Perform a fetch_sub and end a transaction
        fn elision_fetch_sub_release(&self, val: Self::IntType) -> Self::IntType;
    }

    // Indicates whether the target architecture supports lock elision
    #[inline]
    pub fn have_elision() -> bool {
        cfg!(all(
            feature = "nightly",
            any(target_arch = "x86", target_arch = "x86_64"),
        ))
    }

    // This implementation is never actually called because it is guarded by
    // have_elision().
    #[cfg(not(all(feature = "nightly", any(target_arch = "x86", target_arch = "x86_64"))))]
    impl AtomicElisionExt for AtomicUsize {
        type IntType = usize;

        #[inline]
        fn elision_compare_exchange_acquire(&self, _: usize, _: usize) -> Result<usize, usize> {
            unreachable!();
        }

        #[inline]
        fn elision_fetch_sub_release(&self, _: usize) -> usize {
            unreachable!();
        }
    }

    #[cfg(all(feature = "nightly", any(target_arch = "x86", target_arch = "x86_64")))]
    impl AtomicElisionExt for AtomicUsize {
        type IntType = usize;

        #[cfg(target_pointer_width = "32")]
        #[inline]
        fn elision_compare_exchange_acquire(&self, current: usize, new: usize) -> Result<usize, usize> {
            unsafe {
                let prev: usize;
                llvm_asm!("xacquire; lock; cmpxchgl $2, $1"
                          : "={eax}" (prev), "+*m" (self)
                          : "r" (new), "{eax}" (current)
                          : "memory"
                          : "volatile");
                if prev == current {
                    Ok(prev)
                } else {
                    Err(prev)
                }
            }
        }
        #[cfg(target_pointer_width = "64")]
        #[inline]
        fn elision_compare_exchange_acquire(&self, current: usize, new: usize) -> Result<usize, usize> {
            unsafe {
                let prev: usize;
                llvm_asm!("xacquire; lock; cmpxchgq $2, $1"
                          : "={rax}" (prev), "+*m" (self)
                          : "r" (new), "{rax}" (current)
                          : "memory"
                          : "volatile");
                if prev == current {
                    Ok(prev)
                } else {
                    Err(prev)
                }
            }
        }

        #[cfg(target_pointer_width = "32")]
        #[inline]
        fn elision_fetch_sub_release(&self, val: usize) -> usize {
            unsafe {
                let prev: usize;
                llvm_asm!("xrelease; lock; xaddl $2, $1"
                          : "=r" (prev), "+*m" (self)
                          : "0" (val.wrapping_neg())
                          : "memory"
                          : "volatile");
                prev
            }
        }
        #[cfg(target_pointer_width = "64")]
        #[inline]
        fn elision_fetch_sub_release(&self, val: usize) -> usize {
            unsafe {
                let prev: usize;
                llvm_asm!("xrelease; lock; xaddq $2, $1"
                          : "=r" (prev), "+*m" (self)
                          : "0" (val.wrapping_neg())
                          : "memory"
                          : "volatile");
                prev
            }
        }
    }