compiler.h   compiler.h 
skipping to change at line 31 skipping to change at line 31
#include <stddef.h> /* for offsetof */ #include <stddef.h> /* for offsetof */
#define likely(x) __builtin_expect(!!(x), 1) #define likely(x) __builtin_expect(!!(x), 1)
#define unlikely(x) __builtin_expect(!!(x), 0) #define unlikely(x) __builtin_expect(!!(x), 0)
#define cmm_barrier() asm volatile("" : : : "memory") #define cmm_barrier() asm volatile("" : : : "memory")
/* /*
* Instruct the compiler to perform only a single access to a variable * Instruct the compiler to perform only a single access to a variable
* (prohibits merging and refetching). The compiler is also forbidden to re order * (prohibits merging and refetching). The compiler is also forbidden to re order
* successive instances of CAA_ACCESS_ONCE(), but only when the compiler is aware of * successive instances of CMM_ACCESS_ONCE(), but only when the compiler is aware of
* particular ordering. Compiler ordering can be ensured, for example, by * particular ordering. Compiler ordering can be ensured, for example, by
* putting two CAA_ACCESS_ONCE() in separate C statements. * putting two CMM_ACCESS_ONCE() in separate C statements.
* *
* This macro does absolutely -nothing- to prevent the CPU from reordering, * This macro does absolutely -nothing- to prevent the CPU from reordering,
* merging, or refetching absolutely anything at any time. Its main intend ed * merging, or refetching absolutely anything at any time. Its main intend ed
* use is to mediate communication between process-level code and irq/NMI * use is to mediate communication between process-level code and irq/NMI
* handlers, all running on the same CPU. * handlers, all running on the same CPU.
*/ */
#define CAA_ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x)) #define CMM_ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x))
#ifndef max #ifndef max
#define max(a,b) ((a)>(b)?(a):(b)) #define max(a,b) ((a)>(b)?(a):(b))
#endif #endif
#ifndef min #ifndef min
#define min(a,b) ((a)<(b)?(a):(b)) #define min(a,b) ((a)<(b)?(a):(b))
#endif #endif
#if defined(__SIZEOF_LONG__) #if defined(__SIZEOF_LONG__)
 End of changes. 3 change blocks. 
3 lines changed or deleted 3 lines changed or added


 system.h   system.h 
skipping to change at line 27 skipping to change at line 27
* provided the above notices are retained, and a notice that the code was * provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice. * modified is included with the above copyright notice.
*/ */
#include <urcu/compiler.h> #include <urcu/compiler.h>
#include <urcu/arch.h> #include <urcu/arch.h>
/* /*
* Identify a shared load. A cmm_smp_rmc() or cmm_smp_mc() should come befo re the load. * Identify a shared load. A cmm_smp_rmc() or cmm_smp_mc() should come befo re the load.
*/ */
#define _CAA_LOAD_SHARED(p) CAA_ACCESS_ONCE(p) #define _CMM_LOAD_SHARED(p) CMM_ACCESS_ONCE(p)
/* /*
* Load a data from shared memory, doing a cache flush if required. * Load a data from shared memory, doing a cache flush if required.
*/ */
#define CAA_LOAD_SHARED(p) \ #define CMM_LOAD_SHARED(p) \
({ \ ({ \
cmm_smp_rmc(); \ cmm_smp_rmc(); \
_CAA_LOAD_SHARED(p); \ _CMM_LOAD_SHARED(p); \
}) })
/* /*
* Identify a shared store. A cmm_smp_wmc() or cmm_smp_mc() should follow t he store. * Identify a shared store. A cmm_smp_wmc() or cmm_smp_mc() should follow t he store.
*/ */
#define _CAA_STORE_SHARED(x, v) ({ CAA_ACCESS_ONCE(x) = (v); }) #define _CMM_STORE_SHARED(x, v) ({ CMM_ACCESS_ONCE(x) = (v); })
/* /*
* Store v into x, where x is located in shared memory. Performs the requir ed * Store v into x, where x is located in shared memory. Performs the requir ed
* cache flush after writing. Returns v. * cache flush after writing. Returns v.
*/ */
#define CAA_STORE_SHARED(x, v) \ #define CMM_STORE_SHARED(x, v) \
({ \ ({ \
typeof(x) _v = _CAA_STORE_SHARED(x, v); \ typeof(x) _v = _CMM_STORE_SHARED(x, v); \
cmm_smp_wmc(); \ cmm_smp_wmc(); \
_v; \ _v; \
}) })
#endif /* _URCU_SYSTEM_H */ #endif /* _URCU_SYSTEM_H */
 End of changes. 6 change blocks. 
6 lines changed or deleted 6 lines changed or added


 uatomic_arch.h   uatomic_arch.h 
skipping to change at line 42 skipping to change at line 42
/* /*
* Derived from AO_compare_and_swap() and AO_test_and_set_full(). * Derived from AO_compare_and_swap() and AO_test_and_set_full().
*/ */
struct __uatomic_dummy { struct __uatomic_dummy {
unsigned long v[10]; unsigned long v[10];
}; };
#define __hp(x) ((struct __uatomic_dummy *)(x)) #define __hp(x) ((struct __uatomic_dummy *)(x))
#define _uatomic_set(addr, v) CAA_STORE_SHARED(*(addr), (v)) #define _uatomic_set(addr, v) CMM_STORE_SHARED(*(addr), (v))
/* cmpxchg */ /* cmpxchg */
static inline __attribute__((always_inline)) static inline __attribute__((always_inline))
unsigned long __uatomic_cmpxchg(void *addr, unsigned long old, unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
unsigned long _new, int len) unsigned long _new, int len)
{ {
switch (len) { switch (len) {
case 1: case 1:
{ {
skipping to change at line 84 skipping to change at line 84
{ {
unsigned int result = old; unsigned int result = old;
__asm__ __volatile__( __asm__ __volatile__(
"lock; cmpxchgl %2, %1" "lock; cmpxchgl %2, %1"
: "+a"(result), "+m"(*__hp(addr)) : "+a"(result), "+m"(*__hp(addr))
: "r"((unsigned int)_new) : "r"((unsigned int)_new)
: "memory"); : "memory");
return result; return result;
} }
#if (BITS_PER_LONG == 64) #if (CAA_BITS_PER_LONG == 64)
case 8: case 8:
{ {
unsigned long result = old; unsigned long result = old;
__asm__ __volatile__( __asm__ __volatile__(
"lock; cmpxchgq %2, %1" "lock; cmpxchgq %2, %1"
: "+a"(result), "+m"(*__hp(addr)) : "+a"(result), "+m"(*__hp(addr))
: "r"((unsigned long)_new) : "r"((unsigned long)_new)
: "memory"); : "memory");
return result; return result;
skipping to change at line 146 skipping to change at line 146
case 4: case 4:
{ {
unsigned int result; unsigned int result;
__asm__ __volatile__( __asm__ __volatile__(
"xchgl %0, %1" "xchgl %0, %1"
: "=r"(result), "+m"(*__hp(addr)) : "=r"(result), "+m"(*__hp(addr))
: "0" ((unsigned int)val) : "0" ((unsigned int)val)
: "memory"); : "memory");
return result; return result;
} }
#if (BITS_PER_LONG == 64) #if (CAA_BITS_PER_LONG == 64)
case 8: case 8:
{ {
unsigned long result; unsigned long result;
__asm__ __volatile__( __asm__ __volatile__(
"xchgq %0, %1" "xchgq %0, %1"
: "=r"(result), "+m"(*__hp(addr)) : "=r"(result), "+m"(*__hp(addr))
: "0" ((unsigned long)val) : "0" ((unsigned long)val)
: "memory"); : "memory");
return result; return result;
} }
skipping to change at line 209 skipping to change at line 209
{ {
unsigned int result = val; unsigned int result = val;
__asm__ __volatile__( __asm__ __volatile__(
"lock; xaddl %1, %0" "lock; xaddl %1, %0"
: "+m"(*__hp(addr)), "+r" (result) : "+m"(*__hp(addr)), "+r" (result)
: :
: "memory"); : "memory");
return result + (unsigned int)val; return result + (unsigned int)val;
} }
#if (BITS_PER_LONG == 64) #if (CAA_BITS_PER_LONG == 64)
case 8: case 8:
{ {
unsigned long result = val; unsigned long result = val;
__asm__ __volatile__( __asm__ __volatile__(
"lock; xaddq %1, %0" "lock; xaddq %1, %0"
: "+m"(*__hp(addr)), "+r" (result) : "+m"(*__hp(addr)), "+r" (result)
: :
: "memory"); : "memory");
return result + (unsigned long)val; return result + (unsigned long)val;
skipping to change at line 267 skipping to change at line 267
} }
case 4: case 4:
{ {
__asm__ __volatile__( __asm__ __volatile__(
"lock; addl %1, %0" "lock; addl %1, %0"
: "=m"(*__hp(addr)) : "=m"(*__hp(addr))
: "ir" ((unsigned int)val) : "ir" ((unsigned int)val)
: "memory"); : "memory");
return; return;
} }
#if (BITS_PER_LONG == 64) #if (CAA_BITS_PER_LONG == 64)
case 8: case 8:
{ {
__asm__ __volatile__( __asm__ __volatile__(
"lock; addq %1, %0" "lock; addq %1, %0"
: "=m"(*__hp(addr)) : "=m"(*__hp(addr))
: "er" ((unsigned long)val) : "er" ((unsigned long)val)
: "memory"); : "memory");
return; return;
} }
#endif #endif
skipping to change at line 321 skipping to change at line 321
} }
case 4: case 4:
{ {
__asm__ __volatile__( __asm__ __volatile__(
"lock; incl %0" "lock; incl %0"
: "=m"(*__hp(addr)) : "=m"(*__hp(addr))
: :
: "memory"); : "memory");
return; return;
} }
#if (BITS_PER_LONG == 64) #if (CAA_BITS_PER_LONG == 64)
case 8: case 8:
{ {
__asm__ __volatile__( __asm__ __volatile__(
"lock; incq %0" "lock; incq %0"
: "=m"(*__hp(addr)) : "=m"(*__hp(addr))
: :
: "memory"); : "memory");
return; return;
} }
#endif #endif
skipping to change at line 374 skipping to change at line 374
} }
case 4: case 4:
{ {
__asm__ __volatile__( __asm__ __volatile__(
"lock; decl %0" "lock; decl %0"
: "=m"(*__hp(addr)) : "=m"(*__hp(addr))
: :
: "memory"); : "memory");
return; return;
} }
#if (BITS_PER_LONG == 64) #if (CAA_BITS_PER_LONG == 64)
case 8: case 8:
{ {
__asm__ __volatile__( __asm__ __volatile__(
"lock; decq %0" "lock; decq %0"
: "=m"(*__hp(addr)) : "=m"(*__hp(addr))
: :
: "memory"); : "memory");
return; return;
} }
#endif #endif
} }
/* generate an illegal instruction. Cannot catch this with linker tr icks /* generate an illegal instruction. Cannot catch this with linker tr icks
* when optimizations are disabled. */ * when optimizations are disabled. */
__asm__ __volatile__("ud2"); __asm__ __volatile__("ud2");
return; return;
} }
#define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr)))) #define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr))))
#if ((BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH)) #if ((CAA_BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
extern int __rcu_cas_avail; extern int __rcu_cas_avail;
extern int __rcu_cas_init(void); extern int __rcu_cas_init(void);
#define UATOMIC_COMPAT(insn) \ #define UATOMIC_COMPAT(insn) \
((likely(__rcu_cas_avail > 0)) \ ((likely(__rcu_cas_avail > 0)) \
? (_uatomic_##insn) \ ? (_uatomic_##insn) \
: ((unlikely(__rcu_cas_avail < 0) \ : ((unlikely(__rcu_cas_avail < 0) \
? ((__rcu_cas_init() > 0) \ ? ((__rcu_cas_init() > 0) \
? (_uatomic_##insn) \ ? (_uatomic_##insn) \
: (compat_uatomic_##insn)) \ : (compat_uatomic_##insn)) \
 End of changes. 8 change blocks. 
8 lines changed or deleted 8 lines changed or added


 uatomic_generic.h   uatomic_generic.h 
skipping to change at line 32 skipping to change at line 32
*/ */
#include <urcu/compiler.h> #include <urcu/compiler.h>
#include <urcu/system.h> #include <urcu/system.h>
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#ifndef uatomic_set #ifndef uatomic_set
#define uatomic_set(addr, v) CAA_STORE_SHARED(*(addr), (v)) #define uatomic_set(addr, v) CMM_STORE_SHARED(*(addr), (v))
#endif #endif
#ifndef uatomic_read #ifndef uatomic_read
#define uatomic_read(addr) CAA_LOAD_SHARED(*(addr)) #define uatomic_read(addr) CMM_LOAD_SHARED(*(addr))
#endif #endif
#if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR
static inline __attribute__((always_inline)) static inline __attribute__((always_inline))
void _uatomic_link_error() void _uatomic_link_error()
{ {
#ifdef ILLEGAL_INSTR #ifdef ILLEGAL_INSTR
/* generate an illegal instruction. Cannot catch this with linker tr icks /* generate an illegal instruction. Cannot catch this with linker tr icks
* when optimizations are disabled. */ * when optimizations are disabled. */
__asm__ __volatile__(ILLEGAL_INSTR); __asm__ __volatile__(ILLEGAL_INSTR);
skipping to change at line 74 skipping to change at line 74
#ifdef UATOMIC_HAS_ATOMIC_BYTE #ifdef UATOMIC_HAS_ATOMIC_BYTE
case 1: case 1:
return __sync_val_compare_and_swap_1(addr, old, _new); return __sync_val_compare_and_swap_1(addr, old, _new);
#endif #endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT #ifdef UATOMIC_HAS_ATOMIC_SHORT
case 2: case 2:
return __sync_val_compare_and_swap_2(addr, old, _new); return __sync_val_compare_and_swap_2(addr, old, _new);
#endif #endif
case 4: case 4:
return __sync_val_compare_and_swap_4(addr, old, _new); return __sync_val_compare_and_swap_4(addr, old, _new);
#if (BITS_PER_LONG == 64) #if (CAA_BITS_PER_LONG == 64)
case 8: case 8:
return __sync_val_compare_and_swap_8(addr, old, _new); return __sync_val_compare_and_swap_8(addr, old, _new);
#endif #endif
} }
_uatomic_link_error(); _uatomic_link_error();
return 0; return 0;
} }
#define uatomic_cmpxchg(addr, old, _new) \ #define uatomic_cmpxchg(addr, old, _new) \
((__typeof__(*(addr))) _uatomic_cmpxchg((addr), (unsigned long)(old) ,\ ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), (unsigned long)(old) ,\
skipping to change at line 106 skipping to change at line 106
#ifdef UATOMIC_HAS_ATOMIC_BYTE #ifdef UATOMIC_HAS_ATOMIC_BYTE
case 1: case 1:
return __sync_add_and_fetch_1(addr, val); return __sync_add_and_fetch_1(addr, val);
#endif #endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT #ifdef UATOMIC_HAS_ATOMIC_SHORT
case 2: case 2:
return __sync_add_and_fetch_2(addr, val); return __sync_add_and_fetch_2(addr, val);
#endif #endif
case 4: case 4:
return __sync_add_and_fetch_4(addr, val); return __sync_add_and_fetch_4(addr, val);
#if (BITS_PER_LONG == 64) #if (CAA_BITS_PER_LONG == 64)
case 8: case 8:
return __sync_add_and_fetch_8(addr, val); return __sync_add_and_fetch_8(addr, val);
#endif #endif
} }
_uatomic_link_error(); _uatomic_link_error();
return 0; return 0;
} }
#define uatomic_add_return(addr, v) \ #define uatomic_add_return(addr, v) \
((__typeof__(*(addr))) _uatomic_add_return((addr), \ ((__typeof__(*(addr))) _uatomic_add_return((addr), \
skipping to change at line 162 skipping to change at line 162
case 4: case 4:
{ {
unsigned int old; unsigned int old;
do { do {
old = uatomic_read((unsigned int *)addr); old = uatomic_read((unsigned int *)addr);
} while (!__sync_bool_compare_and_swap_4(addr, old, val)); } while (!__sync_bool_compare_and_swap_4(addr, old, val));
return old; return old;
} }
#if (BITS_PER_LONG == 64) #if (CAA_BITS_PER_LONG == 64)
case 8: case 8:
{ {
unsigned long old; unsigned long old;
do { do {
old = uatomic_read((unsigned long *)addr); old = uatomic_read((unsigned long *)addr);
} while (!__sync_bool_compare_and_swap_8(addr, old, val)); } while (!__sync_bool_compare_and_swap_8(addr, old, val));
return old; return old;
} }
skipping to change at line 233 skipping to change at line 233
unsigned int old, oldt; unsigned int old, oldt;
oldt = uatomic_read((unsigned int *)addr); oldt = uatomic_read((unsigned int *)addr);
do { do {
old = oldt; old = oldt;
oldt = _uatomic_cmpxchg(addr, old, old + val, 4); oldt = _uatomic_cmpxchg(addr, old, old + val, 4);
} while (oldt != old); } while (oldt != old);
return old + val; return old + val;
} }
#if (BITS_PER_LONG == 64) #if (CAA_BITS_PER_LONG == 64)
case 8: case 8:
{ {
unsigned long old, oldt; unsigned long old, oldt;
oldt = uatomic_read((unsigned long *)addr); oldt = uatomic_read((unsigned long *)addr);
do { do {
old = oldt; old = oldt;
oldt = _uatomic_cmpxchg(addr, old, old + val, 8); oldt = _uatomic_cmpxchg(addr, old, old + val, 8);
} while (oldt != old); } while (oldt != old);
skipping to change at line 305 skipping to change at line 305
unsigned int old, oldt; unsigned int old, oldt;
oldt = uatomic_read((unsigned int *)addr); oldt = uatomic_read((unsigned int *)addr);
do { do {
old = oldt; old = oldt;
oldt = _uatomic_cmpxchg(addr, old, val, 4); oldt = _uatomic_cmpxchg(addr, old, val, 4);
} while (oldt != old); } while (oldt != old);
return old; return old;
} }
#if (BITS_PER_LONG == 64) #if (CAA_BITS_PER_LONG == 64)
case 8: case 8:
{ {
unsigned long old, oldt; unsigned long old, oldt;
oldt = uatomic_read((unsigned long *)addr); oldt = uatomic_read((unsigned long *)addr);
do { do {
old = oldt; old = oldt;
oldt = _uatomic_cmpxchg(addr, old, val, 8); oldt = _uatomic_cmpxchg(addr, old, val, 8);
} while (oldt != old); } while (oldt != old);
 End of changes. 7 change blocks. 
7 lines changed or deleted 7 lines changed or added


 urcu-bp-static.h   urcu-bp-static.h 
skipping to change at line 165 skipping to change at line 165
static inline int rcu_old_gp_ongoing(long *value) static inline int rcu_old_gp_ongoing(long *value)
{ {
long v; long v;
if (value == NULL) if (value == NULL)
return 0; return 0;
/* /*
* Make sure both tests below are done on the same version of *value * Make sure both tests below are done on the same version of *value
* to insure consistency. * to insure consistency.
*/ */
v = CAA_LOAD_SHARED(*value); v = CMM_LOAD_SHARED(*value);
return (v & RCU_GP_CTR_NEST_MASK) && return (v & RCU_GP_CTR_NEST_MASK) &&
((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE); ((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE);
} }
static inline void _rcu_read_lock(void) static inline void _rcu_read_lock(void)
{ {
long tmp; long tmp;
/* Check if registered */ /* Check if registered */
if (unlikely(!rcu_reader)) if (unlikely(!rcu_reader))
rcu_bp_register(); rcu_bp_register();
cmm_barrier(); /* Ensure the compiler does not reorder us with mute x */ cmm_barrier(); /* Ensure the compiler does not reorder us with mute x */
tmp = rcu_reader->ctr; tmp = rcu_reader->ctr;
/* /*
* rcu_gp_ctr is * rcu_gp_ctr is
* RCU_GP_COUNT | (~RCU_GP_CTR_PHASE or RCU_GP_CTR_PHASE) * RCU_GP_COUNT | (~RCU_GP_CTR_PHASE or RCU_GP_CTR_PHASE)
*/ */
if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) { if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
_CAA_STORE_SHARED(rcu_reader->ctr, _CAA_LOAD_SHARED(rcu_gp_c tr)); _CMM_STORE_SHARED(rcu_reader->ctr, _CMM_LOAD_SHARED(rcu_gp_c tr));
/* /*
* Set active readers count for outermost nesting level befo re * Set active readers count for outermost nesting level befo re
* accessing the pointer. * accessing the pointer.
*/ */
cmm_smp_mb(); cmm_smp_mb();
} else { } else {
_CAA_STORE_SHARED(rcu_reader->ctr, tmp + RCU_GP_COUNT); _CMM_STORE_SHARED(rcu_reader->ctr, tmp + RCU_GP_COUNT);
} }
} }
static inline void _rcu_read_unlock(void) static inline void _rcu_read_unlock(void)
{ {
/* /*
* Finish using rcu before decrementing the pointer. * Finish using rcu before decrementing the pointer.
*/ */
cmm_smp_mb(); cmm_smp_mb();
_CAA_STORE_SHARED(rcu_reader->ctr, rcu_reader->ctr - RCU_GP_COUNT); _CMM_STORE_SHARED(rcu_reader->ctr, rcu_reader->ctr - RCU_GP_COUNT);
cmm_barrier(); /* Ensure the compiler does not reorder us with mute x */ cmm_barrier(); /* Ensure the compiler does not reorder us with mute x */
} }
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#endif /* _URCU_BP_STATIC_H */ #endif /* _URCU_BP_STATIC_H */
 End of changes. 4 change blocks. 
4 lines changed or deleted 4 lines changed or added


 urcu-pointer-static.h   urcu-pointer-static.h 
skipping to change at line 52 skipping to change at line 52
* _rcu_dereference - reads (copy) a RCU-protected pointer to a local varia ble * _rcu_dereference - reads (copy) a RCU-protected pointer to a local varia ble
* into a RCU read-side critical section. The pointer can later be safely * into a RCU read-side critical section. The pointer can later be safely
* dereferenced within the critical section. * dereferenced within the critical section.
* *
* This ensures that the pointer copy is invariant thorough the whole criti cal * This ensures that the pointer copy is invariant thorough the whole criti cal
* section. * section.
* *
* Inserts memory barriers on architectures that require them (currently on ly * Inserts memory barriers on architectures that require them (currently on ly
* Alpha) and documents which pointers are protected by RCU. * Alpha) and documents which pointers are protected by RCU.
* *
* The compiler memory barrier in CAA_LOAD_SHARED() ensures that value-spec ulative * The compiler memory barrier in CMM_LOAD_SHARED() ensures that value-spec ulative
* optimizations (e.g. VSS: Value Speculation Scheduling) does not perform the * optimizations (e.g. VSS: Value Speculation Scheduling) does not perform the
* data read before the pointer read by speculating the value of the pointe r. * data read before the pointer read by speculating the value of the pointe r.
* Correct ordering is ensured because the pointer is read as a volatile ac cess. * Correct ordering is ensured because the pointer is read as a volatile ac cess.
* This acts as a global side-effect operation, which forbids reordering of * This acts as a global side-effect operation, which forbids reordering of
* dependent memory operations. Note that such concern about dependency-bre aking * dependent memory operations. Note that such concern about dependency-bre aking
* optimizations will eventually be taken care of by the "memory_order_cons ume" * optimizations will eventually be taken care of by the "memory_order_cons ume"
* addition to forthcoming C++ standard. * addition to forthcoming C++ standard.
* *
* Should match rcu_assign_pointer() or rcu_xchg_pointer(). * Should match rcu_assign_pointer() or rcu_xchg_pointer().
*/ */
#define _rcu_dereference(p) ({ \ #define _rcu_dereference(p) ({ \
typeof(p) _________p1 = CAA_LOAD_SHARED(p); \ typeof(p) _________p1 = CMM_LOAD_SHARED(p); \
cmm_smp_read_barrier_depends(); \ cmm_smp_read_barrier_depends(); \
(_________p1); \ (_________p1); \
}) })
/** /**
* _rcu_cmpxchg_pointer - same as rcu_assign_pointer, but tests if the poin ter * _rcu_cmpxchg_pointer - same as rcu_assign_pointer, but tests if the poin ter
* is as expected by "old". If succeeds, returns the previous pointer to th e * is as expected by "old". If succeeds, returns the previous pointer to th e
* data structure, which can be safely freed after waiting for a quiescent state * data structure, which can be safely freed after waiting for a quiescent state
* using synchronize_rcu(). If fails (unexpected value), returns old (which * using synchronize_rcu(). If fails (unexpected value), returns old (which
* should not be freed !). * should not be freed !).
 End of changes. 2 change blocks. 
2 lines changed or deleted 2 lines changed or added


 urcu-qsbr-static.h   urcu-qsbr-static.h 
skipping to change at line 162 skipping to change at line 162
uatomic_set(&gp_futex, 0); uatomic_set(&gp_futex, 0);
futex_noasync(&gp_futex, FUTEX_WAKE, 1, futex_noasync(&gp_futex, FUTEX_WAKE, 1,
NULL, NULL, 0); NULL, NULL, 0);
} }
} }
static inline int rcu_gp_ongoing(unsigned long *ctr) static inline int rcu_gp_ongoing(unsigned long *ctr)
{ {
unsigned long v; unsigned long v;
v = CAA_LOAD_SHARED(*ctr); v = CMM_LOAD_SHARED(*ctr);
return v && (v != rcu_gp_ctr); return v && (v != rcu_gp_ctr);
} }
static inline void _rcu_read_lock(void) static inline void _rcu_read_lock(void)
{ {
rcu_assert(rcu_reader.ctr); rcu_assert(rcu_reader.ctr);
} }
static inline void _rcu_read_unlock(void) static inline void _rcu_read_unlock(void)
{ {
} }
static inline void _rcu_quiescent_state(void) static inline void _rcu_quiescent_state(void)
{ {
cmm_smp_mb(); cmm_smp_mb();
_CAA_STORE_SHARED(rcu_reader.ctr, _CAA_LOAD_SHARED(rcu_gp_ctr)); _CMM_STORE_SHARED(rcu_reader.ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
cmm_smp_mb(); /* write rcu_reader.ctr before read futex */ cmm_smp_mb(); /* write rcu_reader.ctr before read futex */
wake_up_gp(); wake_up_gp();
cmm_smp_mb(); cmm_smp_mb();
} }
static inline void _rcu_thread_offline(void) static inline void _rcu_thread_offline(void)
{ {
cmm_smp_mb(); cmm_smp_mb();
CAA_STORE_SHARED(rcu_reader.ctr, 0); CMM_STORE_SHARED(rcu_reader.ctr, 0);
cmm_smp_mb(); /* write rcu_reader.ctr before read futex */ cmm_smp_mb(); /* write rcu_reader.ctr before read futex */
wake_up_gp(); wake_up_gp();
cmm_barrier(); /* Ensure the compiler does not reorder us with mute x */ cmm_barrier(); /* Ensure the compiler does not reorder us with mute x */
} }
static inline void _rcu_thread_online(void) static inline void _rcu_thread_online(void)
{ {
cmm_barrier(); /* Ensure the compiler does not reorder us with mute x */ cmm_barrier(); /* Ensure the compiler does not reorder us with mute x */
_CAA_STORE_SHARED(rcu_reader.ctr, CAA_LOAD_SHARED(rcu_gp_ctr)); _CMM_STORE_SHARED(rcu_reader.ctr, CMM_LOAD_SHARED(rcu_gp_ctr));
cmm_smp_mb(); cmm_smp_mb();
} }
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#endif /* _URCU_QSBR_STATIC_H */ #endif /* _URCU_QSBR_STATIC_H */
 End of changes. 4 change blocks. 
4 lines changed or deleted 4 lines changed or added


 urcu-static.h   urcu-static.h 
skipping to change at line 253 skipping to change at line 253
} }
static inline int rcu_gp_ongoing(unsigned long *ctr) static inline int rcu_gp_ongoing(unsigned long *ctr)
{ {
unsigned long v; unsigned long v;
/* /*
* Make sure both tests below are done on the same version of *value * Make sure both tests below are done on the same version of *value
* to insure consistency. * to insure consistency.
*/ */
v = CAA_LOAD_SHARED(*ctr); v = CMM_LOAD_SHARED(*ctr);
return (v & RCU_GP_CTR_NEST_MASK) && return (v & RCU_GP_CTR_NEST_MASK) &&
((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE); ((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE);
} }
static inline void _rcu_read_lock(void) static inline void _rcu_read_lock(void)
{ {
unsigned long tmp; unsigned long tmp;
cmm_barrier(); /* Ensure the compiler does not reorder us with mute x */ cmm_barrier(); /* Ensure the compiler does not reorder us with mute x */
tmp = rcu_reader.ctr; tmp = rcu_reader.ctr;
/* /*
* rcu_gp_ctr is * rcu_gp_ctr is
* RCU_GP_COUNT | (~RCU_GP_CTR_PHASE or RCU_GP_CTR_PHASE) * RCU_GP_COUNT | (~RCU_GP_CTR_PHASE or RCU_GP_CTR_PHASE)
*/ */
if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) { if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
_CAA_STORE_SHARED(rcu_reader.ctr, _CAA_LOAD_SHARED(rcu_gp_ct r)); _CMM_STORE_SHARED(rcu_reader.ctr, _CMM_LOAD_SHARED(rcu_gp_ct r));
/* /*
* Set active readers count for outermost nesting level befo re * Set active readers count for outermost nesting level befo re
* accessing the pointer. See smp_mb_master(). * accessing the pointer. See smp_mb_master().
*/ */
smp_mb_slave(RCU_MB_GROUP); smp_mb_slave(RCU_MB_GROUP);
} else { } else {
_CAA_STORE_SHARED(rcu_reader.ctr, tmp + RCU_GP_COUNT); _CMM_STORE_SHARED(rcu_reader.ctr, tmp + RCU_GP_COUNT);
} }
} }
static inline void _rcu_read_unlock(void) static inline void _rcu_read_unlock(void)
{ {
unsigned long tmp; unsigned long tmp;
tmp = rcu_reader.ctr; tmp = rcu_reader.ctr;
/* /*
* Finish using rcu before decrementing the pointer. * Finish using rcu before decrementing the pointer.
* See smp_mb_master(). * See smp_mb_master().
*/ */
if (likely((tmp & RCU_GP_CTR_NEST_MASK) == RCU_GP_COUNT)) { if (likely((tmp & RCU_GP_CTR_NEST_MASK) == RCU_GP_COUNT)) {
smp_mb_slave(RCU_MB_GROUP); smp_mb_slave(RCU_MB_GROUP);
_CAA_STORE_SHARED(rcu_reader.ctr, rcu_reader.ctr - RCU_GP_CO UNT); _CMM_STORE_SHARED(rcu_reader.ctr, rcu_reader.ctr - RCU_GP_CO UNT);
/* write rcu_reader.ctr before read futex */ /* write rcu_reader.ctr before read futex */
smp_mb_slave(RCU_MB_GROUP); smp_mb_slave(RCU_MB_GROUP);
wake_up_gp(); wake_up_gp();
} else { } else {
_CAA_STORE_SHARED(rcu_reader.ctr, rcu_reader.ctr - RCU_GP_CO UNT); _CMM_STORE_SHARED(rcu_reader.ctr, rcu_reader.ctr - RCU_GP_CO UNT);
} }
cmm_barrier(); /* Ensure the compiler does not reorder us with mute x */ cmm_barrier(); /* Ensure the compiler does not reorder us with mute x */
} }
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#endif /* _URCU_STATIC_H */ #endif /* _URCU_STATIC_H */
 End of changes. 5 change blocks. 
5 lines changed or deleted 5 lines changed or added


 wfqueue-static.h   wfqueue-static.h 
skipping to change at line 82 skipping to change at line 82
* uatomic_xchg() implicit memory barrier orders earlier stores to d ata * uatomic_xchg() implicit memory barrier orders earlier stores to d ata
* structure containing node and setting node->next to NULL before * structure containing node and setting node->next to NULL before
* publication. * publication.
*/ */
old_tail = uatomic_xchg(&q->tail, node); old_tail = uatomic_xchg(&q->tail, node);
/* /*
* At this point, dequeuers see a NULL old_tail->next, which indicat es * At this point, dequeuers see a NULL old_tail->next, which indicat es
* that the queue is being appended to. The following store will app end * that the queue is being appended to. The following store will app end
* "node" to the queue from a dequeuer perspective. * "node" to the queue from a dequeuer perspective.
*/ */
CAA_STORE_SHARED(*old_tail, node); CMM_STORE_SHARED(*old_tail, node);
} }
/* /*
* It is valid to reuse and free a dequeued node immediately. * It is valid to reuse and free a dequeued node immediately.
* *
* No need to go on a waitqueue here, as there is no possible state in whic h the * No need to go on a waitqueue here, as there is no possible state in whic h the
* list could cause dequeue to busy-loop needlessly while waiting for anoth er * list could cause dequeue to busy-loop needlessly while waiting for anoth er
* thread to be scheduled. The queue appears empty until tail->next is set by * thread to be scheduled. The queue appears empty until tail->next is set by
* enqueue. * enqueue.
*/ */
struct cds_wfq_node * struct cds_wfq_node *
___cds_wfq_dequeue_blocking(struct cds_wfq_queue *q) ___cds_wfq_dequeue_blocking(struct cds_wfq_queue *q)
{ {
struct cds_wfq_node *node, *next; struct cds_wfq_node *node, *next;
int attempt = 0; int attempt = 0;
/* /*
* Queue is empty if it only contains the dummy node. * Queue is empty if it only contains the dummy node.
*/ */
if (q->head == &q->dummy && CAA_LOAD_SHARED(q->tail) == &q->dummy.ne xt) if (q->head == &q->dummy && CMM_LOAD_SHARED(q->tail) == &q->dummy.ne xt)
return NULL; return NULL;
node = q->head; node = q->head;
/* /*
* Adaptative busy-looping waiting for enqueuer to complete enqueue. * Adaptative busy-looping waiting for enqueuer to complete enqueue.
*/ */
while ((next = CAA_LOAD_SHARED(node->next)) == NULL) { while ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
if (++attempt >= WFQ_ADAPT_ATTEMPTS) { if (++attempt >= WFQ_ADAPT_ATTEMPTS) {
poll(NULL, 0, WFQ_WAIT); /* Wait for 10ms */ poll(NULL, 0, WFQ_WAIT); /* Wait for 10ms */
attempt = 0; attempt = 0;
} else } else
caa_cpu_relax(); caa_cpu_relax();
} }
/* /*
* Move queue head forward. * Move queue head forward.
*/ */
q->head = next; q->head = next;
 End of changes. 3 change blocks. 
3 lines changed or deleted 3 lines changed or added


 wfstack-static.h   wfstack-static.h 
skipping to change at line 70 skipping to change at line 70
assert(node->next == NULL); assert(node->next == NULL);
/* /*
* uatomic_xchg() implicit memory barrier orders earlier stores to n ode * uatomic_xchg() implicit memory barrier orders earlier stores to n ode
* (setting it to NULL) before publication. * (setting it to NULL) before publication.
*/ */
old_head = uatomic_xchg(&s->head, node); old_head = uatomic_xchg(&s->head, node);
/* /*
* At this point, dequeuers see a NULL node->next, they should busy- wait * At this point, dequeuers see a NULL node->next, they should busy- wait
* until node->next is set to old_head. * until node->next is set to old_head.
*/ */
CAA_STORE_SHARED(node->next, old_head); CMM_STORE_SHARED(node->next, old_head);
} }
/* /*
* Returns NULL if stack is empty. * Returns NULL if stack is empty.
*/ */
struct cds_wfs_node * struct cds_wfs_node *
___cds_wfs_pop_blocking(struct cds_wfs_stack *s) ___cds_wfs_pop_blocking(struct cds_wfs_stack *s)
{ {
struct cds_wfs_node *head, *next; struct cds_wfs_node *head, *next;
int attempt = 0; int attempt = 0;
retry: retry:
head = CAA_LOAD_SHARED(s->head); head = CMM_LOAD_SHARED(s->head);
if (head == CDS_WF_STACK_END) if (head == CDS_WF_STACK_END)
return NULL; return NULL;
/* /*
* Adaptative busy-looping waiting for push to complete. * Adaptative busy-looping waiting for push to complete.
*/ */
while ((next = CAA_LOAD_SHARED(head->next)) == NULL) { while ((next = CMM_LOAD_SHARED(head->next)) == NULL) {
if (++attempt >= CDS_WFS_ADAPT_ATTEMPTS) { if (++attempt >= CDS_WFS_ADAPT_ATTEMPTS) {
poll(NULL, 0, CDS_WFS_WAIT); /* Wait for 10ms */ poll(NULL, 0, CDS_WFS_WAIT); /* Wait for 10ms */
attempt = 0; attempt = 0;
} else } else
caa_cpu_relax(); caa_cpu_relax();
} }
if (uatomic_cmpxchg(&s->head, head, next) == head) if (uatomic_cmpxchg(&s->head, head, next) == head)
return head; return head;
else else
goto retry; /* Concurrent modification. Retry. * / goto retry; /* Concurrent modification. Retry. * /
 End of changes. 3 change blocks. 
3 lines changed or deleted 3 lines changed or added

This html diff was produced by rfcdiff 1.41. The latest version is available from http://tools.ietf.org/tools/rfcdiff/