| ////////////////////////////////////////////////////////////////////////////// |
| // |
| // (C) Copyright Ion Gaztanaga 2006-2009 |
| // (C) Copyright Markus Schoepflin 2007 |
| // |
| // Distributed under the Boost Software License, Version 1.0. (See |
| // accompanying file LICENSE_1_0.txt or copy at |
| // http://www.boost.org/LICENSE_1_0.txt) |
| // |
| // See http://www.boost.org/libs/interprocess for documentation. |
| // |
| ////////////////////////////////////////////////////////////////////////////// |
| |
| #ifndef BOOST_INTERPROCESS_DETAIL_ATOMIC_HPP |
| #define BOOST_INTERPROCESS_DETAIL_ATOMIC_HPP |
| |
| #include <boost/interprocess/detail/config_begin.hpp> |
| #include <boost/interprocess/detail/workaround.hpp> |
| #include <boost/cstdint.hpp> |
| |
| namespace boost{ |
| namespace interprocess{ |
| namespace detail{ |
| |
| //! Atomically increment an boost::uint32_t by 1 |
| //! "mem": pointer to the object |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_inc32(volatile boost::uint32_t *mem); |
| |
| //! Atomically read an boost::uint32_t from memory |
| inline boost::uint32_t atomic_read32(volatile boost::uint32_t *mem); |
| |
| //! Atomically set an boost::uint32_t in memory |
| //! "mem": pointer to the object |
| //! "param": val value that the object will assume |
| inline void atomic_write32(volatile boost::uint32_t *mem, boost::uint32_t val); |
| |
| //! Compare an boost::uint32_t's value with "cmp". |
| //! If they are the same swap the value with "with" |
| //! "mem": pointer to the value |
| //! "with": what to swap it with |
| //! "cmp": the value to compare it to |
| //! Returns the old value of *mem |
| inline boost::uint32_t atomic_cas32 |
| (volatile boost::uint32_t *mem, boost::uint32_t with, boost::uint32_t cmp); |
| |
| } //namespace detail{ |
| } //namespace interprocess{ |
| } //namespace boost{ |
| |
| #if (defined BOOST_INTERPROCESS_WINDOWS) |
| |
| #include <boost/interprocess/detail/win32_api.hpp> |
| |
| namespace boost{ |
| namespace interprocess{ |
| namespace detail{ |
| |
| //! Atomically decrement an boost::uint32_t by 1 |
| //! "mem": pointer to the atomic value |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_dec32(volatile boost::uint32_t *mem) |
| { return winapi::interlocked_decrement(reinterpret_cast<volatile long*>(mem)) + 1; } |
| |
| //! Atomically increment an apr_uint32_t by 1 |
| //! "mem": pointer to the object |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_inc32(volatile boost::uint32_t *mem) |
| { return winapi::interlocked_increment(reinterpret_cast<volatile long*>(mem))-1; } |
| |
| //! Atomically read an boost::uint32_t from memory |
| inline boost::uint32_t atomic_read32(volatile boost::uint32_t *mem) |
| { return *mem; } |
| |
| //! Atomically set an boost::uint32_t in memory |
| //! "mem": pointer to the object |
| //! "param": val value that the object will assume |
| inline void atomic_write32(volatile boost::uint32_t *mem, boost::uint32_t val) |
| { winapi::interlocked_exchange(reinterpret_cast<volatile long*>(mem), val); } |
| |
| //! Compare an boost::uint32_t's value with "cmp". |
| //! If they are the same swap the value with "with" |
| //! "mem": pointer to the value |
| //! "with": what to swap it with |
| //! "cmp": the value to compare it to |
| //! Returns the old value of *mem |
| inline boost::uint32_t atomic_cas32 |
| (volatile boost::uint32_t *mem, boost::uint32_t with, boost::uint32_t cmp) |
| { return winapi::interlocked_compare_exchange(reinterpret_cast<volatile long*>(mem), with, cmp); } |
| |
| } //namespace detail{ |
| } //namespace interprocess{ |
| } //namespace boost{ |
| |
| #elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) |
| |
| namespace boost { |
| namespace interprocess { |
| namespace detail{ |
| |
| //! Compare an boost::uint32_t's value with "cmp". |
| //! If they are the same swap the value with "with" |
| //! "mem": pointer to the value |
| //! "with" what to swap it with |
| //! "cmp": the value to compare it to |
| //! Returns the old value of *mem |
| inline boost::uint32_t atomic_cas32 |
| (volatile boost::uint32_t *mem, boost::uint32_t with, boost::uint32_t cmp) |
| { |
| boost::uint32_t prev = cmp; |
| asm volatile( "lock\n\t" |
| "cmpxchg %3,%1" |
| : "=a" (prev), "=m" (*(mem)) |
| : "0" (prev), "r" (with) |
| : "memory", "cc"); |
| return prev; |
| /* |
| boost::uint32_t prev; |
| |
| asm volatile ("lock; cmpxchgl %1, %2" |
| : "=a" (prev) |
| : "r" (with), "m" (*(mem)), "0"(cmp)); |
| asm volatile("" : : : "memory"); |
| |
| return prev; |
| */ |
| } |
| |
| //! Atomically add 'val' to an boost::uint32_t |
| //! "mem": pointer to the object |
| //! "val": amount to add |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_add32 |
| (volatile boost::uint32_t *mem, boost::uint32_t val) |
| { |
| // int r = *pw; |
| // *mem += val; |
| // return r; |
| int r; |
| |
| asm volatile |
| ( |
| "lock\n\t" |
| "xadd %1, %0": |
| "+m"( *mem ), "=r"( r ): // outputs (%0, %1) |
| "1"( val ): // inputs (%2 == %1) |
| "memory", "cc" // clobbers |
| ); |
| |
| return r; |
| /* |
| asm volatile( "lock\n\t; xaddl %0,%1" |
| : "=r"(val), "=m"(*mem) |
| : "0"(val), "m"(*mem)); |
| asm volatile("" : : : "memory"); |
| |
| return val; |
| */ |
| } |
| |
| //! Atomically increment an apr_uint32_t by 1 |
| //! "mem": pointer to the object |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_inc32(volatile boost::uint32_t *mem) |
| { return atomic_add32(mem, 1); } |
| |
| //! Atomically decrement an boost::uint32_t by 1 |
| //! "mem": pointer to the atomic value |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_dec32(volatile boost::uint32_t *mem) |
| { return atomic_add32(mem, (boost::uint32_t)-1); } |
| |
| //! Atomically read an boost::uint32_t from memory |
| inline boost::uint32_t atomic_read32(volatile boost::uint32_t *mem) |
| { return *mem; } |
| |
| //! Atomically set an boost::uint32_t in memory |
| //! "mem": pointer to the object |
| //! "param": val value that the object will assume |
| inline void atomic_write32(volatile boost::uint32_t *mem, boost::uint32_t val) |
| { *mem = val; } |
| |
| } //namespace detail{ |
| } //namespace interprocess{ |
| } //namespace boost{ |
| |
| #elif defined(__GNUC__) && (defined(__PPC__) || defined(__ppc__)) |
| |
| namespace boost { |
| namespace interprocess { |
| namespace detail{ |
| |
| //! Atomically add 'val' to an boost::uint32_t |
| //! "mem": pointer to the object |
| //! "val": amount to add |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_add32(volatile boost::uint32_t *mem, boost::uint32_t val) |
| { |
| boost::uint32_t prev, temp; |
| |
| asm volatile ("0:\n\t" // retry local label |
| "lwarx %0,0,%2\n\t" // load prev and reserve |
| "add %1,%0,%3\n\t" // temp = prev + val |
| "stwcx. %1,0,%2\n\t" // conditionally store |
| "bne- 0b" // start over if we lost |
| // the reservation |
| //XXX find a cleaner way to define the temp |
| //it's not an output |
| : "=&r" (prev), "=&r" (temp) // output, temp |
| : "b" (mem), "r" (val) // inputs |
| : "memory", "cc"); // clobbered |
| return prev; |
| } |
| |
| //! Compare an boost::uint32_t's value with "cmp". |
| //! If they are the same swap the value with "with" |
| //! "mem": pointer to the value |
| //! "with" what to swap it with |
| //! "cmp": the value to compare it to |
| //! Returns the old value of *mem |
| inline boost::uint32_t atomic_cas32 |
| (volatile boost::uint32_t *mem, boost::uint32_t with, boost::uint32_t cmp) |
| { |
| boost::uint32_t prev; |
| |
| asm volatile ("0:\n\t" // retry local label |
| "lwarx %0,0,%1\n\t" // load prev and reserve |
| "cmpw %0,%3\n\t" // does it match cmp? |
| "bne- 1f\n\t" // ...no, bail out |
| "stwcx. %2,0,%1\n\t" // ...yes, conditionally |
| // store with |
| "bne- 0b\n\t" // start over if we lost |
| // the reservation |
| "1:" // exit local label |
| |
| : "=&r"(prev) // output |
| : "b" (mem), "r" (with), "r"(cmp) // inputs |
| : "memory", "cc"); // clobbered |
| return prev; |
| } |
| |
| //! Atomically increment an apr_uint32_t by 1 |
| //! "mem": pointer to the object |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_inc32(volatile boost::uint32_t *mem) |
| { return atomic_add32(mem, 1); } |
| |
| //! Atomically decrement an boost::uint32_t by 1 |
| //! "mem": pointer to the atomic value |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_dec32(volatile boost::uint32_t *mem) |
| { return atomic_add32(mem, boost::uint32_t(-1u)); } |
| |
| //! Atomically read an boost::uint32_t from memory |
| inline boost::uint32_t atomic_read32(volatile boost::uint32_t *mem) |
| { return *mem; } |
| |
| //! Atomically set an boost::uint32_t in memory |
| //! "mem": pointer to the object |
| //! "param": val value that the object will assume |
| inline void atomic_write32(volatile boost::uint32_t *mem, boost::uint32_t val) |
| { *mem = val; } |
| |
| } //namespace detail{ |
| } //namespace interprocess{ |
| } //namespace boost{ |
| |
| #elif defined(__GNUC__) && ( __GNUC__ * 100 + __GNUC_MINOR__ >= 401 ) |
| |
| namespace boost { |
| namespace interprocess { |
| namespace detail{ |
| |
| //! Atomically add 'val' to an boost::uint32_t |
| //! "mem": pointer to the object |
| //! "val": amount to add |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_add32 |
| (volatile boost::uint32_t *mem, boost::uint32_t val) |
| { return __sync_fetch_and_add(const_cast<boost::uint32_t *>(mem), val); } |
| |
| //! Atomically increment an apr_uint32_t by 1 |
| //! "mem": pointer to the object |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_inc32(volatile boost::uint32_t *mem) |
| { return atomic_add32(mem, 1); } |
| |
| //! Atomically decrement an boost::uint32_t by 1 |
| //! "mem": pointer to the atomic value |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_dec32(volatile boost::uint32_t *mem) |
| { return atomic_add32(mem, (boost::uint32_t)-1); } |
| |
| //! Atomically read an boost::uint32_t from memory |
| inline boost::uint32_t atomic_read32(volatile boost::uint32_t *mem) |
| { return *mem; } |
| |
| //! Compare an boost::uint32_t's value with "cmp". |
| //! If they are the same swap the value with "with" |
| //! "mem": pointer to the value |
| //! "with" what to swap it with |
| //! "cmp": the value to compare it to |
| //! Returns the old value of *mem |
| inline boost::uint32_t atomic_cas32 |
| (volatile boost::uint32_t *mem, boost::uint32_t with, boost::uint32_t cmp) |
| { return __sync_val_compare_and_swap(const_cast<boost::uint32_t *>(mem), cmp, with); } |
| |
| //! Atomically set an boost::uint32_t in memory |
| //! "mem": pointer to the object |
| //! "param": val value that the object will assume |
| inline void atomic_write32(volatile boost::uint32_t *mem, boost::uint32_t val) |
| { *mem = val; } |
| |
| } //namespace detail{ |
| } //namespace interprocess{ |
| } //namespace boost{ |
| |
| #elif (defined(sun) || defined(__sun)) |
| |
| #include <atomic.h> |
| |
| namespace boost{ |
| namespace interprocess{ |
| namespace detail{ |
| |
| //! Atomically add 'val' to an boost::uint32_t |
| //! "mem": pointer to the object |
| //! "val": amount to add |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_add32(volatile boost::uint32_t *mem, boost::uint32_t val) |
| { return atomic_add_32_nv(reinterpret_cast<volatile ::uint32_t*>(mem), (int32_t)val) - val; } |
| |
| //! Compare an boost::uint32_t's value with "cmp". |
| //! If they are the same swap the value with "with" |
| //! "mem": pointer to the value |
| //! "with" what to swap it with |
| //! "cmp": the value to compare it to |
| //! Returns the old value of *mem |
| inline boost::uint32_t atomic_cas32 |
| (volatile boost::uint32_t *mem, boost::uint32_t with, boost::uint32_t cmp) |
| { return atomic_cas_32(reinterpret_cast<volatile ::uint32_t*>(mem), cmp, with); } |
| |
| //! Atomically increment an apr_uint32_t by 1 |
| //! "mem": pointer to the object |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_inc32(volatile boost::uint32_t *mem) |
| { return atomic_add_32_nv(reinterpret_cast<volatile ::uint32_t*>(mem), 1) - 1; } |
| |
| //! Atomically decrement an boost::uint32_t by 1 |
| //! "mem": pointer to the atomic value |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_dec32(volatile boost::uint32_t *mem) |
| { return atomic_add_32_nv(reinterpret_cast<volatile ::uint32_t*>(mem), (boost::uint32_t)-1) + 1; } |
| |
| //! Atomically read an boost::uint32_t from memory |
| inline boost::uint32_t atomic_read32(volatile boost::uint32_t *mem) |
| { return *mem; } |
| |
| //! Atomically set an boost::uint32_t in memory |
| //! "mem": pointer to the object |
| //! "param": val value that the object will assume |
| inline void atomic_write32(volatile boost::uint32_t *mem, boost::uint32_t val) |
| { *mem = val; } |
| |
| } //namespace detail{ |
| } //namespace interprocess{ |
| } //namespace boost{ |
| |
| #elif defined(__osf__) && defined(__DECCXX) |
| |
| #include <machine/builtins.h> |
| #include <c_asm.h> |
| |
| namespace boost{ |
| namespace interprocess{ |
| namespace detail{ |
| |
| //! Atomically decrement a uint32_t by 1 |
| //! "mem": pointer to the atomic value |
| //! Returns the old value pointed to by mem |
| //! Acquire, memory barrier after decrement. |
| inline boost::uint32_t atomic_dec32(volatile boost::uint32_t *mem) |
| { boost::uint32_t old_val = __ATOMIC_DECREMENT_LONG(mem); __MB(); return old_val; } |
| |
| //! Atomically increment a uint32_t by 1 |
| //! "mem": pointer to the object |
| //! Returns the old value pointed to by mem |
| //! Release, memory barrier before increment. |
| inline boost::uint32_t atomic_inc32(volatile boost::uint32_t *mem) |
| { __MB(); return __ATOMIC_INCREMENT_LONG(mem); } |
| |
| // Rational for the implementation of the atomic read and write functions. |
| // |
| // 1. The Alpha Architecture Handbook requires that access to a byte, |
| // an aligned word, an aligned longword, or an aligned quadword is |
| // atomic. (See 'Alpha Architecture Handbook', version 4, chapter 5.2.2.) |
| // |
| // 2. The CXX User's Guide states that volatile quantities are accessed |
| // with single assembler instructions, and that a compilation error |
| // occurs when declaring a quantity as volatile which is not properly |
| // aligned. |
| |
| //! Atomically read an boost::uint32_t from memory |
| //! Acquire, memory barrier after load. |
| inline boost::uint32_t atomic_read32(volatile boost::uint32_t *mem) |
| { boost::uint32_t old_val = *mem; __MB(); return old_val; } |
| |
| //! Atomically set an boost::uint32_t in memory |
| //! "mem": pointer to the object |
| //! "param": val value that the object will assume |
| //! Release, memory barrier before store. |
| inline void atomic_write32(volatile boost::uint32_t *mem, boost::uint32_t val) |
| { __MB(); *mem = val; } |
| |
| //! Compare an boost::uint32_t's value with "cmp". |
| //! If they are the same swap the value with "with" |
| //! "mem": pointer to the value |
| //! "with" what to swap it with |
| //! "cmp": the value to compare it to |
| //! Returns the old value of *mem |
| //! Memory barrier between load and store. |
| inline boost::uint32_t atomic_cas32( |
| volatile boost::uint32_t *mem, boost::uint32_t with, boost::uint32_t cmp) |
| { |
| // Note: |
| // |
| // Branch prediction prefers backward branches, and the Alpha Architecture |
| // Handbook explicitely states that the loop should not be implemented like |
| // it is below. (See chapter 4.2.5.) Therefore the code should probably look |
| // like this: |
| // |
| // return asm( |
| // "10: ldl_l %v0,(%a0) ;" |
| // " cmpeq %v0,%a2,%t0 ;" |
| // " beq %t0,20f ;" |
| // " mb ;" |
| // " mov %a1,%t0 ;" |
| // " stl_c %t0,(%a0) ;" |
| // " beq %t0,30f ;" |
| // "20: ret ;" |
| // "30: br 10b;", |
| // mem, with, cmp); |
| // |
| // But as the compiler always transforms this into the form where a backward |
| // branch is taken on failure, we can as well implement it in the straight |
| // forward form, as this is what it will end up in anyway. |
| |
| return asm( |
| "10: ldl_l %v0,(%a0) ;" // load prev value from mem and lock mem |
| " cmpeq %v0,%a2,%t0 ;" // compare with given value |
| " beq %t0,20f ;" // if not equal, we're done |
| " mb ;" // memory barrier |
| " mov %a1,%t0 ;" // load new value into scratch register |
| " stl_c %t0,(%a0) ;" // store new value to locked mem (overwriting scratch) |
| " beq %t0,10b ;" // store failed because lock has been stolen, retry |
| "20: ", |
| mem, with, cmp); |
| } |
| |
| } //namespace detail{ |
| } //namespace interprocess{ |
| } //namespace boost{ |
| |
| #elif defined(__IBMCPP__) && (__IBMCPP__ >= 800) && defined(_AIX) |
| |
| #include <builtins.h> |
| |
| namespace boost { |
| namespace interprocess { |
| namespace detail{ |
| |
| //first define boost::uint32_t versions of __lwarx and __stwcx to avoid poluting |
| //all the functions with casts |
| |
| //! From XLC documenation : |
| //! This function can be used with a subsequent stwcxu call to implement a |
| //! read-modify-write on a specified memory location. The two functions work |
| //! together to ensure that if the store is successfully performed, no other |
| //! processor or mechanism can modify the target doubleword between the time |
| //! lwarxu function is executed and the time the stwcxu functio ncompletes. |
| //! "mem" : pointer to the object |
| //! Returns the value at pointed to by mem |
| inline boost::uint32_t lwarxu(volatile boost::uint32_t *mem) |
| { |
| return static_cast<boost::uint32_t>(__lwarx(reinterpret_cast<volatile int*>(mem))); |
| } |
| |
| //! "mem" : pointer to the object |
| //! "val" : the value to store |
| //! Returns true if the update of mem is successful and false if it is |
| //!unsuccessful |
| inline bool stwcxu(volatile boost::uint32_t* mem, boost::uint32_t val) |
| { |
| return (__stwcx(reinterpret_cast<volatile int*>(mem), static_cast<int>(val)) != 0); |
| } |
| |
| //! "mem": pointer to the object |
| //! "val": amount to add |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_add32 |
| (volatile boost::uint32_t *mem, boost::uint32_t val) |
| { |
| boost::uint32_t oldValue; |
| do |
| { |
| oldValue = lwarxu(mem); |
| }while (!stwcxu(mem, oldValue+val)); |
| return oldValue; |
| } |
| |
| //! Atomically increment an apr_uint32_t by 1 |
| //! "mem": pointer to the object |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_inc32(volatile boost::uint32_t *mem) |
| { return atomic_add32(mem, 1); } |
| |
| //! Atomically decrement an boost::uint32_t by 1 |
| //! "mem": pointer to the atomic value |
| //! Returns the old value pointed to by mem |
| inline boost::uint32_t atomic_dec32(volatile boost::uint32_t *mem) |
| { return atomic_add32(mem, (boost::uint32_t)-1); } |
| |
| //! Atomically read an boost::uint32_t from memory |
| inline boost::uint32_t atomic_read32(volatile boost::uint32_t *mem) |
| { return *mem; } |
| |
| //! Compare an boost::uint32_t's value with "cmp". |
| //! If they are the same swap the value with "with" |
| //! "mem": pointer to the value |
| //! "with" what to swap it with |
| //! "cmp": the value to compare it to |
| //! Returns the old value of *mem |
| inline boost::uint32_t atomic_cas32 |
| (volatile boost::uint32_t *mem, boost::uint32_t with, boost::uint32_t cmp) |
| { |
| boost::uint32_t oldValue; |
| boost::uint32_t valueToStore; |
| do |
| { |
| oldValue = lwarxu(mem); |
| } while (!stwcxu(mem, (oldValue == with) ? cmp : oldValue)); |
| |
| return oldValue; |
| } |
| |
| //! Atomically set an boost::uint32_t in memory |
| //! "mem": pointer to the object |
| //! "param": val value that the object will assume |
| inline void atomic_write32(volatile boost::uint32_t *mem, boost::uint32_t val) |
| { *mem = val; } |
| |
| } //namespace detail |
| } //namespace interprocess |
| } //namespace boost |
| |
| #else |
| |
| #error No atomic operations implemented for this platform, sorry! |
| |
| #endif |
| |
| namespace boost{ |
| namespace interprocess{ |
| namespace detail{ |
| |
| inline bool atomic_add_unless32 |
| (volatile boost::uint32_t *mem, boost::uint32_t value, boost::uint32_t unless_this) |
| { |
| boost::uint32_t old, c(atomic_read32(mem)); |
| while(c != unless_this && (old = atomic_cas32(mem, c + value, c)) != c){ |
| c = old; |
| } |
| return c != unless_this; |
| } |
| |
| } //namespace detail |
| } //namespace interprocess |
| } //namespace boost |
| |
| |
| #include <boost/interprocess/detail/config_end.hpp> |
| |
| #endif //BOOST_INTERPROCESS_DETAIL_ATOMIC_HPP |