oculus1
diff libovr/Src/Kernel/OVR_Atomic.h @ 3:b069a5c27388
added a couple more stuff, fixed all the LibOVR line endings
| author | John Tsiombikas <nuclear@member.fsf.org> |
|---|---|
| date | Sun, 15 Sep 2013 04:10:05 +0300 |
| parents | e2f9e4603129 |
| children |
line diff
1.1 --- a/libovr/Src/Kernel/OVR_Atomic.h Sat Sep 14 17:51:03 2013 +0300 1.2 +++ b/libovr/Src/Kernel/OVR_Atomic.h Sun Sep 15 04:10:05 2013 +0300 1.3 @@ -1,1 +1,859 @@ 1.4 -/************************************************************************************ 1.5 1.6 PublicHeader: OVR.h 1.7 Filename : OVR_Atomic.h 1.8 Content : Contains atomic operations and inline fastest locking 1.9 functionality. Will contain #ifdefs for OS efficiency. 1.10 Have non-thread-safe implementaion if not available. 1.11 Created : September 19, 2012 1.12 Notes : 1.13 1.14 Copyright : Copyright 2012 Oculus VR, Inc. All Rights reserved. 1.15 1.16 Use of this software is subject to the terms of the Oculus license 1.17 agreement provided at the time of installation or download, or which 1.18 otherwise accompanies this software in either electronic or hard copy form. 1.19 1.20 ************************************************************************************/ 1.21 #ifndef OVR_Atomic_h 1.22 #define OVR_Atomic_h 1.23 1.24 #include "OVR_Types.h" 1.25 1.26 // Include System thread functionality. 1.27 #if defined(OVR_OS_WIN32) 1.28 #include <windows.h> 1.29 #else 1.30 #include <pthread.h> 1.31 #endif 1.32 1.33 1.34 namespace OVR { 1.35 1.36 1.37 // ****** Declared classes 1.38 1.39 // If there is NO thread support we implement AtomicOps and 1.40 // Lock objects as no-ops. The other classes are not defined. 1.41 template<class C> class AtomicOps; 1.42 template<class T> class AtomicInt; 1.43 template<class T> class AtomicPtr; 1.44 1.45 class Lock; 1.46 1.47 1.48 //----------------------------------------------------------------------------------- 1.49 // ***** AtomicOps 1.50 1.51 // Atomic operations are provided by the AtomicOps templates class, 1.52 // implemented through system-specific AtomicOpsRaw specializations. 1.53 // It provides several fundamental operations such as Exchange, ExchangeAdd 1.54 // CompareAndSet, and Store_Release. Each function includes several memory 1.55 // synchronization versions, important for multiprocessing CPUs with weak 1.56 // memory consistency. The following memory fencing strategies are supported: 1.57 // 1.58 // - NoSync. No memory synchronization is done for atomic op. 1.59 // - Release. All other memory writes are completed before atomic op 1.60 // writes its results. 1.61 // - Acquire. Further memory reads are forced to wait until atomic op 1.62 // executes, guaranteeing that the right values will be seen. 1.63 // - Sync. A combination of Release and Acquire. 1.64 1.65 1.66 // *** AtomicOpsRaw 1.67 1.68 // AtomicOpsRaw is a specialized template that provides atomic operations 1.69 // used by AtomicOps. This class has two fundamental qualities: (1) it 1.70 // defines a type T of correct size, and (2) provides operations that work 1.71 // atomically, such as Exchange_Sync and CompareAndSet_Release. 1.72 1.73 // AtomicOpsRawBase class contains shared constants/classes for AtomicOpsRaw. 1.74 // The primary thing is does is define sync class objects, whose destructor and 1.75 // constructor provide places to insert appropriate synchronization calls, on 1.76 // systems where such calls are necessary. So far, the breakdown is as follows: 1.77 // 1.78 // - X86 systems don't need custom syncs, since their exchange/atomic 1.79 // instructions are implicitly synchronized. 1.80 // - PowerPC requires lwsync/isync instructions that can use this mechanism. 1.81 // - If some other systems require a mechanism where syncing type is associated 1.82 // with a particular instruction, the default implementation (which implements 1.83 // all Sync, Acquire, and Release modes in terms of NoSync and fence) may not 1.84 // work. Ii that case it will need to be #ifdef-ed conditionally. 1.85 1.86 struct AtomicOpsRawBase 1.87 { 1.88 #if !defined(OVR_ENABLE_THREADS) || defined(OVR_CPU_X86) || defined(OVR_OS_WIN32) || defined(OVR_OS_IPHONE) 1.89 // Need to have empty constructor to avoid class 'unused' variable warning. 1.90 struct FullSync { inline FullSync() { } }; 1.91 struct AcquireSync { inline AcquireSync() { } }; 1.92 struct ReleaseSync { inline ReleaseSync() { } }; 1.93 1.94 #elif defined(OVR_CPU_PPC64) || defined(OVR_CPU_PPC) 1.95 struct FullSync { inline FullSync() { asm volatile("sync\n"); } ~FullSync() { asm volatile("isync\n"); } }; 1.96 struct AcquireSync { inline AcquireSync() { } ~AcquireSync() { asm volatile("isync\n"); } }; 1.97 struct ReleaseSync { inline ReleaseSync() { asm volatile("sync\n"); } }; 1.98 1.99 #elif defined(OVR_CPU_MIPS) 1.100 struct FullSync { inline FullSync() { asm volatile("sync\n"); } ~FullSync() { asm volatile("sync\n"); } }; 1.101 struct AcquireSync { inline AcquireSync() { } ~AcquireSync() { asm volatile("sync\n"); } }; 1.102 struct ReleaseSync { inline ReleaseSync() { asm volatile("sync\n"); } }; 1.103 1.104 #elif defined(OVR_CPU_ARM) 1.105 struct FullSync { inline FullSync() { asm volatile("dmb\n"); } ~FullSync() { asm volatile("dmb\n"); } }; 1.106 struct AcquireSync { inline AcquireSync() { } ~AcquireSync() { asm volatile("dmb\n"); } }; 1.107 struct ReleaseSync { inline ReleaseSync() { asm volatile("dmb\n"); } }; 1.108 1.109 1.110 #elif defined(OVR_CC_GNU) && (__GNUC__ >= 4) 1.111 // __sync functions are already full sync 1.112 struct FullSync { inline FullSync() { } }; 1.113 struct AcquireSync { inline AcquireSync() { } }; 1.114 struct ReleaseSync { inline ReleaseSync() { } }; 1.115 #endif 1.116 }; 1.117 1.118 1.119 // 4-Byte raw data atomic op implementation class. 1.120 struct AtomicOpsRaw_4ByteImpl : public AtomicOpsRawBase 1.121 { 1.122 #if !defined(OVR_ENABLE_THREADS) 1.123 1.124 // Provide a type for no-thread-support cases. Used by AtomicOpsRaw_DefImpl. 1.125 typedef UInt32 T; 1.126 1.127 // *** Thread - Safe Atomic Versions. 1.128 1.129 #elif defined(OVR_OS_WIN32) 1.130 1.131 // Use special defined for VC6, where volatile is not used and 1.132 // InterlockedCompareExchange is declared incorrectly. 1.133 typedef LONG T; 1.134 #if defined(OVR_CC_MSVC) && (OVR_CC_MSVC < 1300) 1.135 typedef T* InterlockTPtr; 1.136 typedef LPVOID ET; 1.137 typedef ET* InterlockETPtr; 1.138 #else 1.139 typedef volatile T* InterlockTPtr; 1.140 typedef T ET; 1.141 typedef InterlockTPtr InterlockETPtr; 1.142 #endif 1.143 inline static T Exchange_NoSync(volatile T* p, T val) { return InterlockedExchange((InterlockTPtr)p, val); } 1.144 inline static T ExchangeAdd_NoSync(volatile T* p, T val) { return InterlockedExchangeAdd((InterlockTPtr)p, val); } 1.145 inline static bool CompareAndSet_NoSync(volatile T* p, T c, T val) { return InterlockedCompareExchange((InterlockETPtr)p, (ET)val, (ET)c) == (ET)c; } 1.146 1.147 #elif defined(OVR_CPU_PPC64) || defined(OVR_CPU_PPC) 1.148 typedef UInt32 T; 1.149 static inline UInt32 Exchange_NoSync(volatile UInt32 *i, UInt32 j) 1.150 { 1.151 UInt32 ret; 1.152 1.153 asm volatile("1:\n\t" 1.154 "lwarx %[r],0,%[i]\n\t" 1.155 "stwcx. %[j],0,%[i]\n\t" 1.156 "bne- 1b\n" 1.157 : "+m" (*i), [r] "=&b" (ret) : [i] "b" (i), [j] "b" (j) : "cc", "memory"); 1.158 1.159 return ret; 1.160 } 1.161 1.162 static inline UInt32 ExchangeAdd_NoSync(volatile UInt32 *i, UInt32 j) 1.163 { 1.164 UInt32 dummy, ret; 1.165 1.166 asm volatile("1:\n\t" 1.167 "lwarx %[r],0,%[i]\n\t" 1.168 "add %[o],%[r],%[j]\n\t" 1.169 "stwcx. %[o],0,%[i]\n\t" 1.170 "bne- 1b\n" 1.171 : "+m" (*i), [r] "=&b" (ret), [o] "=&r" (dummy) : [i] "b" (i), [j] "b" (j) : "cc", "memory"); 1.172 1.173 return ret; 1.174 } 1.175 1.176 static inline bool CompareAndSet_NoSync(volatile UInt32 *i, UInt32 c, UInt32 value) 1.177 { 1.178 UInt32 ret; 1.179 1.180 asm volatile("1:\n\t" 1.181 "lwarx %[r],0,%[i]\n\t" 1.182 "cmpw 0,%[r],%[cmp]\n\t" 1.183 "mfcr %[r]\n\t" 1.184 "bne- 2f\n\t" 1.185 "stwcx. %[val],0,%[i]\n\t" 1.186 "bne- 1b\n\t" 1.187 "2:\n" 1.188 : "+m" (*i), [r] "=&b" (ret) : [i] "b" (i), [cmp] "b" (c), [val] "b" (value) : "cc", "memory"); 1.189 1.190 return (ret & 0x20000000) ? 1 : 0; 1.191 } 1.192 1.193 #elif defined(OVR_CPU_MIPS) 1.194 typedef UInt32 T; 1.195 1.196 static inline UInt32 Exchange_NoSync(volatile UInt32 *i, UInt32 j) 1.197 { 1.198 UInt32 ret; 1.199 1.200 asm volatile("1:\n\t" 1.201 "ll %[r],0(%[i])\n\t" 1.202 "sc %[j],0(%[i])\n\t" 1.203 "beq %[j],$0,1b\n\t" 1.204 "nop \n" 1.205 : "+m" (*i), [r] "=&d" (ret) : [i] "d" (i), [j] "d" (j) : "cc", "memory"); 1.206 1.207 return ret; 1.208 } 1.209 1.210 static inline UInt32 ExchangeAdd_NoSync(volatile UInt32 *i, UInt32 j) 1.211 { 1.212 UInt32 ret; 1.213 1.214 asm volatile("1:\n\t" 1.215 "ll %[r],0(%[i])\n\t" 1.216 "addu %[j],%[r],%[j]\n\t" 1.217 "sc %[j],0(%[i])\n\t" 1.218 "beq %[j],$0,1b\n\t" 1.219 "nop \n" 1.220 : "+m" (*i), [r] "=&d" (ret) : [i] "d" (i), [j] "d" (j) : "cc", "memory"); 1.221 1.222 return ret; 1.223 } 1.224 1.225 static inline bool CompareAndSet_NoSync(volatile UInt32 *i, UInt32 c, UInt32 value) 1.226 { 1.227 UInt32 ret, dummy; 1.228 1.229 asm volatile("1:\n\t" 1.230 "move %[r],$0\n\t" 1.231 "ll %[o],0(%[i])\n\t" 1.232 "bne %[o],%[c],2f\n\t" 1.233 "move %[r],%[v]\n\t" 1.234 "sc %[r],0(%[i])\n\t" 1.235 "beq %[r],$0,1b\n\t" 1.236 "nop \n\t" 1.237 "2:\n" 1.238 : "+m" (*i),[r] "=&d" (ret), [o] "=&d" (dummy) : [i] "d" (i), [c] "d" (c), [v] "d" (value) 1.239 : "cc", "memory"); 1.240 1.241 return ret; 1.242 } 1.243 1.244 #elif defined(OVR_CPU_ARM) && defined(OVR_CC_ARM) 1.245 typedef UInt32 T; 1.246 1.247 static inline UInt32 Exchange_NoSync(volatile UInt32 *i, UInt32 j) 1.248 { 1.249 for(;;) 1.250 { 1.251 T r = __ldrex(i); 1.252 if (__strex(j, i) == 0) 1.253 return r; 1.254 } 1.255 } 1.256 static inline UInt32 ExchangeAdd_NoSync(volatile UInt32 *i, UInt32 j) 1.257 { 1.258 for(;;) 1.259 { 1.260 T r = __ldrex(i); 1.261 if (__strex(r + j, i) == 0) 1.262 return r; 1.263 } 1.264 } 1.265 1.266 static inline bool CompareAndSet_NoSync(volatile UInt32 *i, UInt32 c, UInt32 value) 1.267 { 1.268 for(;;) 1.269 { 1.270 T r = __ldrex(i); 1.271 if (r != c) 1.272 return 0; 1.273 if (__strex(value, i) == 0) 1.274 return 1; 1.275 } 1.276 } 1.277 1.278 #elif defined(OVR_CPU_ARM) 1.279 typedef UInt32 T; 1.280 1.281 static inline UInt32 Exchange_NoSync(volatile UInt32 *i, UInt32 j) 1.282 { 1.283 UInt32 ret, dummy; 1.284 1.285 asm volatile("1:\n\t" 1.286 "ldrex %[r],[%[i]]\n\t" 1.287 "strex %[t],%[j],[%[i]]\n\t" 1.288 "cmp %[t],#0\n\t" 1.289 "bne 1b\n\t" 1.290 : "+m" (*i), [r] "=&r" (ret), [t] "=&r" (dummy) : [i] "r" (i), [j] "r" (j) : "cc", "memory"); 1.291 1.292 return ret; 1.293 } 1.294 1.295 static inline UInt32 ExchangeAdd_NoSync(volatile UInt32 *i, UInt32 j) 1.296 { 1.297 UInt32 ret, dummy, test; 1.298 1.299 asm volatile("1:\n\t" 1.300 "ldrex %[r],[%[i]]\n\t" 1.301 "add %[o],%[r],%[j]\n\t" 1.302 "strex %[t],%[o],[%[i]]\n\t" 1.303 "cmp %[t],#0\n\t" 1.304 "bne 1b\n\t" 1.305 : "+m" (*i), [r] "=&r" (ret), [o] "=&r" (dummy), [t] "=&r" (test) : [i] "r" (i), [j] "r" (j) : "cc", "memory"); 1.306 1.307 return ret; 1.308 } 1.309 1.310 static inline bool CompareAndSet_NoSync(volatile UInt32 *i, UInt32 c, UInt32 value) 1.311 { 1.312 UInt32 ret = 1, dummy, test; 1.313 1.314 asm volatile("1:\n\t" 1.315 "ldrex %[o],[%[i]]\n\t" 1.316 "cmp %[o],%[c]\n\t" 1.317 "bne 2f\n\t" 1.318 "strex %[r],%[v],[%[i]]\n\t" 1.319 "cmp %[r],#0\n\t" 1.320 "bne 1b\n\t" 1.321 "2:\n" 1.322 : "+m" (*i),[r] "=&r" (ret), [o] "=&r" (dummy), [t] "=&r" (test) : [i] "r" (i), [c] "r" (c), [v] "r" (value) 1.323 : "cc", "memory"); 1.324 1.325 return !ret; 1.326 } 1.327 1.328 #elif defined(OVR_CPU_X86) 1.329 typedef UInt32 T; 1.330 1.331 static inline UInt32 Exchange_NoSync(volatile UInt32 *i, UInt32 j) 1.332 { 1.333 asm volatile("xchgl %1,%[i]\n" 1.334 : "+m" (*i), "=q" (j) : [i] "m" (*i), "1" (j) : "cc", "memory"); 1.335 1.336 return j; 1.337 } 1.338 1.339 static inline UInt32 ExchangeAdd_NoSync(volatile UInt32 *i, UInt32 j) 1.340 { 1.341 asm volatile("lock; xaddl %1,%[i]\n" 1.342 : "+m" (*i), "+q" (j) : [i] "m" (*i) : "cc", "memory"); 1.343 1.344 return j; 1.345 } 1.346 1.347 static inline bool CompareAndSet_NoSync(volatile UInt32 *i, UInt32 c, UInt32 value) 1.348 { 1.349 UInt32 ret; 1.350 1.351 asm volatile("lock; cmpxchgl %[v],%[i]\n" 1.352 : "+m" (*i), "=a" (ret) : [i] "m" (*i), "1" (c), [v] "q" (value) : "cc", "memory"); 1.353 1.354 return (ret == c); 1.355 } 1.356 1.357 #elif defined(OVR_CC_GNU) && (__GNUC__ >= 4 && __GNUC_MINOR__ >= 1) 1.358 1.359 typedef UInt32 T; 1.360 1.361 static inline T Exchange_NoSync(volatile T *i, T j) 1.362 { 1.363 T v; 1.364 do { 1.365 v = *i; 1.366 } while (!__sync_bool_compare_and_swap(i, v, j)); 1.367 return v; 1.368 } 1.369 1.370 static inline T ExchangeAdd_NoSync(volatile T *i, T j) 1.371 { 1.372 return __sync_fetch_and_add(i, j); 1.373 } 1.374 1.375 static inline bool CompareAndSet_NoSync(volatile T *i, T c, T value) 1.376 { 1.377 return __sync_bool_compare_and_swap(i, c, value); 1.378 } 1.379 1.380 #endif // OS 1.381 }; 1.382 1.383 1.384 // 8-Byte raw data data atomic op implementation class. 1.385 // Currently implementation is provided only on systems with 64-bit pointers. 1.386 struct AtomicOpsRaw_8ByteImpl : public AtomicOpsRawBase 1.387 { 1.388 #if !defined(OVR_64BIT_POINTERS) || !defined(OVR_ENABLE_THREADS) 1.389 1.390 // Provide a type for no-thread-support cases. Used by AtomicOpsRaw_DefImpl. 1.391 typedef UInt64 T; 1.392 1.393 // *** Thread - Safe OS specific versions. 1.394 #elif defined(OVR_OS_WIN32) 1.395 1.396 // This is only for 64-bit systems. 1.397 typedef LONG64 T; 1.398 typedef volatile T* InterlockTPtr; 1.399 inline static T Exchange_NoSync(volatile T* p, T val) { return InterlockedExchange64((InterlockTPtr)p, val); } 1.400 inline static T ExchangeAdd_NoSync(volatile T* p, T val) { return InterlockedExchangeAdd64((InterlockTPtr)p, val); } 1.401 inline static bool CompareAndSet_NoSync(volatile T* p, T c, T val) { return InterlockedCompareExchange64((InterlockTPtr)p, val, c) == c; } 1.402 1.403 #elif defined(OVR_CPU_PPC64) 1.404 1.405 typedef UInt64 T; 1.406 1.407 static inline UInt64 Exchange_NoSync(volatile UInt64 *i, UInt64 j) 1.408 { 1.409 UInt64 dummy, ret; 1.410 1.411 asm volatile("1:\n\t" 1.412 "ldarx %[r],0,%[i]\n\t" 1.413 "mr %[o],%[j]\n\t" 1.414 "stdcx. %[o],0,%[i]\n\t" 1.415 "bne- 1b\n" 1.416 : "+m" (*i), [r] "=&b" (ret), [o] "=&r" (dummy) : [i] "b" (i), [j] "b" (j) : "cc"); 1.417 1.418 return ret; 1.419 } 1.420 1.421 static inline UInt64 ExchangeAdd_NoSync(volatile UInt64 *i, UInt64 j) 1.422 { 1.423 UInt64 dummy, ret; 1.424 1.425 asm volatile("1:\n\t" 1.426 "ldarx %[r],0,%[i]\n\t" 1.427 "add %[o],%[r],%[j]\n\t" 1.428 "stdcx. %[o],0,%[i]\n\t" 1.429 "bne- 1b\n" 1.430 : "+m" (*i), [r] "=&b" (ret), [o] "=&r" (dummy) : [i] "b" (i), [j] "b" (j) : "cc"); 1.431 1.432 return ret; 1.433 } 1.434 1.435 static inline bool CompareAndSet_NoSync(volatile UInt64 *i, UInt64 c, UInt64 value) 1.436 { 1.437 UInt64 ret, dummy; 1.438 1.439 asm volatile("1:\n\t" 1.440 "ldarx %[r],0,%[i]\n\t" 1.441 "cmpw 0,%[r],%[cmp]\n\t" 1.442 "mfcr %[r]\n\t" 1.443 "bne- 2f\n\t" 1.444 "stdcx. %[val],0,%[i]\n\t" 1.445 "bne- 1b\n\t" 1.446 "2:\n" 1.447 : "+m" (*i), [r] "=&b" (ret), [o] "=&r" (dummy) : [i] "b" (i), [cmp] "b" (c), [val] "b" (value) : "cc"); 1.448 1.449 return (ret & 0x20000000) ? 1 : 0; 1.450 } 1.451 1.452 #elif defined(OVR_CC_GNU) && (__GNUC__ >= 4 && __GNUC_MINOR__ >= 1) 1.453 1.454 typedef UInt64 T; 1.455 1.456 static inline T Exchange_NoSync(volatile T *i, T j) 1.457 { 1.458 T v; 1.459 do { 1.460 v = *i; 1.461 } while (!__sync_bool_compare_and_swap(i, v, j)); 1.462 return v; 1.463 } 1.464 1.465 static inline T ExchangeAdd_NoSync(volatile T *i, T j) 1.466 { 1.467 return __sync_fetch_and_add(i, j); 1.468 } 1.469 1.470 static inline bool CompareAndSet_NoSync(volatile T *i, T c, T value) 1.471 { 1.472 return __sync_bool_compare_and_swap(i, c, value); 1.473 } 1.474 1.475 #endif // OS 1.476 }; 1.477 1.478 1.479 // Default implementation for AtomicOpsRaw; provides implementation of mem-fenced 1.480 // atomic operations where fencing is done with a sync object wrapped around a NoSync 1.481 // operation implemented in the base class. If such implementation is not possible 1.482 // on a given platform, #ifdefs can be used to disable it and then op functions can be 1.483 // implemented individually in the appropriate AtomicOpsRaw<size> class. 1.484 1.485 template<class O> 1.486 struct AtomicOpsRaw_DefImpl : public O 1.487 { 1.488 typedef typename O::T O_T; 1.489 typedef typename O::FullSync O_FullSync; 1.490 typedef typename O::AcquireSync O_AcquireSync; 1.491 typedef typename O::ReleaseSync O_ReleaseSync; 1.492 1.493 // If there is no thread support, provide the default implementation. In this case, 1.494 // the base class (0) must still provide the T declaration. 1.495 #ifndef OVR_ENABLE_THREADS 1.496 1.497 // Atomic exchange of val with argument. Returns old val. 1.498 inline static O_T Exchange_NoSync(volatile O_T* p, O_T val) { O_T old = *p; *p = val; return old; } 1.499 // Adds a new val to argument; returns its old val. 1.500 inline static O_T ExchangeAdd_NoSync(volatile O_T* p, O_T val) { O_T old = *p; *p += val; return old; } 1.501 // Compares the argument data with 'c' val. 1.502 // If succeeded, stores val int '*p' and returns true; otherwise returns false. 1.503 inline static bool CompareAndSet_NoSync(volatile O_T* p, O_T c, O_T val) { if (*p==c) { *p = val; return 1; } return 0; } 1.504 1.505 #endif 1.506 1.507 // If NoSync wrapped implementation may not be possible, it this block should be 1.508 // replaced with per-function implementation in O. 1.509 // "AtomicOpsRaw_DefImpl<O>::" prefix in calls below. 1.510 inline static O_T Exchange_Sync(volatile O_T* p, O_T val) { O_FullSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::Exchange_NoSync(p, val); } 1.511 inline static O_T Exchange_Release(volatile O_T* p, O_T val) { O_ReleaseSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::Exchange_NoSync(p, val); } 1.512 inline static O_T Exchange_Acquire(volatile O_T* p, O_T val) { O_AcquireSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::Exchange_NoSync(p, val); } 1.513 inline static O_T ExchangeAdd_Sync(volatile O_T* p, O_T val) { O_FullSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::ExchangeAdd_NoSync(p, val); } 1.514 inline static O_T ExchangeAdd_Release(volatile O_T* p, O_T val) { O_ReleaseSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::ExchangeAdd_NoSync(p, val); } 1.515 inline static O_T ExchangeAdd_Acquire(volatile O_T* p, O_T val) { O_AcquireSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::ExchangeAdd_NoSync(p, val); } 1.516 inline static bool CompareAndSet_Sync(volatile O_T* p, O_T c, O_T val) { O_FullSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::CompareAndSet_NoSync(p,c,val); } 1.517 inline static bool CompareAndSet_Release(volatile O_T* p, O_T c, O_T val) { O_ReleaseSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::CompareAndSet_NoSync(p,c,val); } 1.518 inline static bool CompareAndSet_Acquire(volatile O_T* p, O_T c, O_T val) { O_AcquireSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::CompareAndSet_NoSync(p,c,val); } 1.519 1.520 // Loads and stores with memory fence. These have only the relevant versions. 1.521 #ifdef OVR_CPU_X86 1.522 // On X86, Store_Release is implemented as exchange. Note that we can also 1.523 // consider 'sfence' in the future, although it is not as compatible with older CPUs. 1.524 inline static void Store_Release(volatile O_T* p, O_T val) { Exchange_Release(p, val); } 1.525 #else 1.526 inline static void Store_Release(volatile O_T* p, O_T val) { O_ReleaseSync sync; OVR_UNUSED(sync); *p = val; } 1.527 #endif 1.528 inline static O_T Load_Acquire(const volatile O_T* p) { O_AcquireSync sync; OVR_UNUSED(sync); return *p; } 1.529 }; 1.530 1.531 1.532 template<int size> 1.533 struct AtomicOpsRaw : public AtomicOpsRawBase { }; 1.534 1.535 template<> 1.536 struct AtomicOpsRaw<4> : public AtomicOpsRaw_DefImpl<AtomicOpsRaw_4ByteImpl> 1.537 { 1.538 // Ensure that assigned type size is correct. 1.539 AtomicOpsRaw() 1.540 { OVR_COMPILER_ASSERT(sizeof(AtomicOpsRaw_DefImpl<AtomicOpsRaw_4ByteImpl>::T) == 4); } 1.541 }; 1.542 template<> 1.543 struct AtomicOpsRaw<8> : public AtomicOpsRaw_DefImpl<AtomicOpsRaw_8ByteImpl> 1.544 { 1.545 AtomicOpsRaw() 1.546 { OVR_COMPILER_ASSERT(sizeof(AtomicOpsRaw_DefImpl<AtomicOpsRaw_8ByteImpl>::T) == 8); } 1.547 }; 1.548 1.549 1.550 // *** AtomicOps - implementation of atomic Ops for specified class 1.551 1.552 // Implements atomic ops on a class, provided that the object is either 1.553 // 4 or 8 bytes in size (depending on the AtomicOpsRaw specializations 1.554 // available). Relies on AtomicOpsRaw for much of implementation. 1.555 1.556 template<class C> 1.557 class AtomicOps 1.558 { 1.559 typedef AtomicOpsRaw<sizeof(C)> Ops; 1.560 typedef typename Ops::T T; 1.561 typedef volatile typename Ops::T* PT; 1.562 // We cast through unions to (1) avoid pointer size compiler warnings 1.563 // and (2) ensure that there are no problems with strict pointer aliasing. 1.564 union C2T_union { C c; T t; }; 1.565 1.566 public: 1.567 // General purpose implementation for standard syncs. 1.568 inline static C Exchange_Sync(volatile C* p, C val) { C2T_union u; u.c = val; u.t = Ops::Exchange_Sync((PT)p, u.t); return u.c; } 1.569 inline static C Exchange_Release(volatile C* p, C val) { C2T_union u; u.c = val; u.t = Ops::Exchange_Release((PT)p, u.t); return u.c; } 1.570 inline static C Exchange_Acquire(volatile C* p, C val) { C2T_union u; u.c = val; u.t = Ops::Exchange_Acquire((PT)p, u.t); return u.c; } 1.571 inline static C Exchange_NoSync(volatile C* p, C val) { C2T_union u; u.c = val; u.t = Ops::Exchange_NoSync((PT)p, u.t); return u.c; } 1.572 inline static C ExchangeAdd_Sync(volatile C* p, C val) { C2T_union u; u.c = val; u.t = Ops::ExchangeAdd_Sync((PT)p, u.t); return u.c; } 1.573 inline static C ExchangeAdd_Release(volatile C* p, C val) { C2T_union u; u.c = val; u.t = Ops::ExchangeAdd_Release((PT)p, u.t); return u.c; } 1.574 inline static C ExchangeAdd_Acquire(volatile C* p, C val) { C2T_union u; u.c = val; u.t = Ops::ExchangeAdd_Acquire((PT)p, u.t); return u.c; } 1.575 inline static C ExchangeAdd_NoSync(volatile C* p, C val) { C2T_union u; u.c = val; u.t = Ops::ExchangeAdd_NoSync((PT)p, u.t); return u.c; } 1.576 inline static bool CompareAndSet_Sync(volatile C* p, C c, C val) { C2T_union u,cu; u.c = val; cu.c = c; return Ops::CompareAndSet_Sync((PT)p, cu.t, u.t); } 1.577 inline static bool CompareAndSet_Release(volatile C* p, C c, C val){ C2T_union u,cu; u.c = val; cu.c = c; return Ops::CompareAndSet_Release((PT)p, cu.t, u.t); } 1.578 inline static bool CompareAndSet_Relse(volatile C* p, C c, C val){ C2T_union u,cu; u.c = val; cu.c = c; return Ops::CompareAndSet_Acquire((PT)p, cu.t, u.t); } 1.579 inline static bool CompareAndSet_NoSync(volatile C* p, C c, C val) { C2T_union u,cu; u.c = val; cu.c = c; return Ops::CompareAndSet_NoSync((PT)p, cu.t, u.t); } 1.580 // Loads and stores with memory fence. These have only the relevant versions. 1.581 inline static void Store_Release(volatile C* p, C val) { C2T_union u; u.c = val; Ops::Store_Release((PT)p, u.t); } 1.582 inline static C Load_Acquire(const volatile C* p) { C2T_union u; u.t = Ops::Load_Acquire((PT)p); return u.c; } 1.583 }; 1.584 1.585 1.586 1.587 // Atomic value base class - implements operations shared for integers and pointers. 1.588 template<class T> 1.589 class AtomicValueBase 1.590 { 1.591 protected: 1.592 typedef AtomicOps<T> Ops; 1.593 public: 1.594 1.595 volatile T Value; 1.596 1.597 inline AtomicValueBase() { } 1.598 explicit inline AtomicValueBase(T val) { Ops::Store_Release(&Value, val); } 1.599 1.600 // Most libraries (TBB and Joshua Scholar's) library do not do Load_Acquire 1.601 // here, since most algorithms do not require atomic loads. Needs some research. 1.602 inline operator T() const { return Value; } 1.603 1.604 // *** Standard Atomic inlines 1.605 inline T Exchange_Sync(T val) { return Ops::Exchange_Sync(&Value, val); } 1.606 inline T Exchange_Release(T val) { return Ops::Exchange_Release(&Value, val); } 1.607 inline T Exchange_Acquire(T val) { return Ops::Exchange_Acquire(&Value, val); } 1.608 inline T Exchange_NoSync(T val) { return Ops::Exchange_NoSync(&Value, val); } 1.609 inline bool CompareAndSet_Sync(T c, T val) { return Ops::CompareAndSet_Sync(&Value, c, val); } 1.610 inline bool CompareAndSet_Release(T c, T val) { return Ops::CompareAndSet_Release(&Value, c, val); } 1.611 inline bool CompareAndSet_Acquire(T c, T val) { return Ops::CompareAndSet_Relse(&Value, c, val); } 1.612 inline bool CompareAndSet_NoSync(T c, T val) { return Ops::CompareAndSet_NoSync(&Value, c, val); } 1.613 // Load & Store. 1.614 inline void Store_Release(T val) { Ops::Store_Release(&Value, val); } 1.615 inline T Load_Acquire() const { return Ops::Load_Acquire(&Value); } 1.616 }; 1.617 1.618 1.619 // ***** AtomicPtr - Atomic pointer template 1.620 1.621 // This pointer class supports atomic assignments with release, 1.622 // increment / decrement operations, and conditional compare + set. 1.623 1.624 template<class T> 1.625 class AtomicPtr : public AtomicValueBase<T*> 1.626 { 1.627 typedef typename AtomicValueBase<T*>::Ops Ops; 1.628 1.629 public: 1.630 // Initialize pointer value to 0 by default; use Store_Release only with explicit constructor. 1.631 inline AtomicPtr() : AtomicValueBase<T*>() { this->Value = 0; } 1.632 explicit inline AtomicPtr(T* val) : AtomicValueBase<T*>(val) { } 1.633 1.634 // Pointer access. 1.635 inline T* operator -> () const { return this->Load_Acquire(); } 1.636 1.637 // It looks like it is convenient to have Load_Acquire characteristics 1.638 // for this, since that is convenient for algorithms such as linked 1.639 // list traversals that can be added to bu another thread. 1.640 inline operator T* () const { return this->Load_Acquire(); } 1.641 1.642 1.643 // *** Standard Atomic inlines (applicable to pointers) 1.644 1.645 // ExhangeAdd considers pointer size for pointers. 1.646 template<class I> 1.647 inline T* ExchangeAdd_Sync(I incr) { return Ops::ExchangeAdd_Sync(&this->Value, ((T*)0) + incr); } 1.648 template<class I> 1.649 inline T* ExchangeAdd_Release(I incr) { return Ops::ExchangeAdd_Release(&this->Value, ((T*)0) + incr); } 1.650 template<class I> 1.651 inline T* ExchangeAdd_Acquire(I incr) { return Ops::ExchangeAdd_Acquire(&this->Value, ((T*)0) + incr); } 1.652 template<class I> 1.653 inline T* ExchangeAdd_NoSync(I incr) { return Ops::ExchangeAdd_NoSync(&this->Value, ((T*)0) + incr); } 1.654 1.655 // *** Atomic Operators 1.656 1.657 inline T* operator = (T* val) { this->Store_Release(val); return val; } 1.658 1.659 template<class I> 1.660 inline T* operator += (I val) { return ExchangeAdd_Sync(val) + val; } 1.661 template<class I> 1.662 inline T* operator -= (I val) { return operator += (-val); } 1.663 1.664 inline T* operator ++ () { return ExchangeAdd_Sync(1) + 1; } 1.665 inline T* operator -- () { return ExchangeAdd_Sync(-1) - 1; } 1.666 inline T* operator ++ (int) { return ExchangeAdd_Sync(1); } 1.667 inline T* operator -- (int) { return ExchangeAdd_Sync(-1); } 1.668 }; 1.669 1.670 1.671 // ***** AtomicInt - Atomic integer template 1.672 1.673 // Implements an atomic integer type; the exact type to use is provided 1.674 // as an argument. Supports atomic Acquire / Release semantics, atomic 1.675 // arithmetic operations, and atomic conditional compare + set. 1.676 1.677 template<class T> 1.678 class AtomicInt : public AtomicValueBase<T> 1.679 { 1.680 typedef typename AtomicValueBase<T>::Ops Ops; 1.681 1.682 public: 1.683 inline AtomicInt() : AtomicValueBase<T>() { } 1.684 explicit inline AtomicInt(T val) : AtomicValueBase<T>(val) { } 1.685 1.686 1.687 // *** Standard Atomic inlines (applicable to int) 1.688 inline T ExchangeAdd_Sync(T val) { return Ops::ExchangeAdd_Sync(&this->Value, val); } 1.689 inline T ExchangeAdd_Release(T val) { return Ops::ExchangeAdd_Release(&this->Value, val); } 1.690 inline T ExchangeAdd_Acquire(T val) { return Ops::ExchangeAdd_Acquire(&this->Value, val); } 1.691 inline T ExchangeAdd_NoSync(T val) { return Ops::ExchangeAdd_NoSync(&this->Value, val); } 1.692 // These increments could be more efficient because they don't return a value. 1.693 inline void Increment_Sync() { ExchangeAdd_Sync((T)1); } 1.694 inline void Increment_Release() { ExchangeAdd_Release((T)1); } 1.695 inline void Increment_Acquire() { ExchangeAdd_Acquire((T)1); } 1.696 inline void Increment_NoSync() { ExchangeAdd_NoSync((T)1); } 1.697 1.698 // *** Atomic Operators 1.699 1.700 inline T operator = (T val) { this->Store_Release(val); return val; } 1.701 inline T operator += (T val) { return ExchangeAdd_Sync(val) + val; } 1.702 inline T operator -= (T val) { return ExchangeAdd_Sync(0 - val) - val; } 1.703 1.704 inline T operator ++ () { return ExchangeAdd_Sync((T)1) + 1; } 1.705 inline T operator -- () { return ExchangeAdd_Sync(((T)0)-1) - 1; } 1.706 inline T operator ++ (int) { return ExchangeAdd_Sync((T)1); } 1.707 inline T operator -- (int) { return ExchangeAdd_Sync(((T)0)-1); } 1.708 1.709 // More complex atomic operations. Leave it to compiler whether to optimize them or not. 1.710 T operator &= (T arg) 1.711 { 1.712 T comp, newVal; 1.713 do { 1.714 comp = this->Value; 1.715 newVal = comp & arg; 1.716 } while(!this->CompareAndSet_Sync(comp, newVal)); 1.717 return newVal; 1.718 } 1.719 1.720 T operator |= (T arg) 1.721 { 1.722 T comp, newVal; 1.723 do { 1.724 comp = this->Value; 1.725 newVal = comp | arg; 1.726 } while(!this->CompareAndSet_Sync(comp, newVal)); 1.727 return newVal; 1.728 } 1.729 1.730 T operator ^= (T arg) 1.731 { 1.732 T comp, newVal; 1.733 do { 1.734 comp = this->Value; 1.735 newVal = comp ^ arg; 1.736 } while(!this->CompareAndSet_Sync(comp, newVal)); 1.737 return newVal; 1.738 } 1.739 1.740 T operator *= (T arg) 1.741 { 1.742 T comp, newVal; 1.743 do { 1.744 comp = this->Value; 1.745 newVal = comp * arg; 1.746 } while(!this->CompareAndSet_Sync(comp, newVal)); 1.747 return newVal; 1.748 } 1.749 1.750 T operator /= (T arg) 1.751 { 1.752 T comp, newVal; 1.753 do { 1.754 comp = this->Value; 1.755 newVal = comp / arg; 1.756 } while(!CompareAndSet_Sync(comp, newVal)); 1.757 return newVal; 1.758 } 1.759 1.760 T operator >>= (unsigned bits) 1.761 { 1.762 T comp, newVal; 1.763 do { 1.764 comp = this->Value; 1.765 newVal = comp >> bits; 1.766 } while(!CompareAndSet_Sync(comp, newVal)); 1.767 return newVal; 1.768 } 1.769 1.770 T operator <<= (unsigned bits) 1.771 { 1.772 T comp, newVal; 1.773 do { 1.774 comp = this->Value; 1.775 newVal = comp << bits; 1.776 } while(!this->CompareAndSet_Sync(comp, newVal)); 1.777 return newVal; 1.778 } 1.779 }; 1.780 1.781 1.782 1.783 //----------------------------------------------------------------------------------- 1.784 // ***** Lock 1.785 1.786 // Lock is a simplest and most efficient mutual-exclusion lock class. 1.787 // Unlike Mutex, it cannot be waited on. 1.788 1.789 class Lock 1.790 { 1.791 // NOTE: Locks are not allocatable and they themselves should not allocate 1.792 // memory by standard means. This is the case because StandardAllocator 1.793 // relies on this class. 1.794 // Make 'delete' private. Don't do this for 'new' since it can be redefined. 1.795 void operator delete(void*) {} 1.796 1.797 1.798 // *** Lock implementation for various platforms. 1.799 1.800 #if !defined(OVR_ENABLE_THREADS) 1.801 1.802 public: 1.803 // With no thread support, lock does nothing. 1.804 inline Lock() { } 1.805 inline Lock(unsigned) { } 1.806 inline ~Lock() { } 1.807 inline void DoLock() { } 1.808 inline void Unlock() { } 1.809 1.810 // Windows. 1.811 #elif defined(OVR_OS_WIN32) 1.812 1.813 CRITICAL_SECTION cs; 1.814 public: 1.815 Lock(unsigned spinCount = 0); 1.816 ~Lock(); 1.817 // Locking functions. 1.818 inline void DoLock() { ::EnterCriticalSection(&cs); } 1.819 inline void Unlock() { ::LeaveCriticalSection(&cs); } 1.820 1.821 #else 1.822 pthread_mutex_t mutex; 1.823 1.824 public: 1.825 static pthread_mutexattr_t RecursiveAttr; 1.826 static bool RecursiveAttrInit; 1.827 1.828 Lock (unsigned dummy = 0) 1.829 { 1.830 if (!RecursiveAttrInit) 1.831 { 1.832 pthread_mutexattr_init(&RecursiveAttr); 1.833 pthread_mutexattr_settype(&RecursiveAttr, PTHREAD_MUTEX_RECURSIVE); 1.834 RecursiveAttrInit = 1; 1.835 } 1.836 pthread_mutex_init(&mutex,&RecursiveAttr); 1.837 } 1.838 ~Lock () { pthread_mutex_destroy(&mutex); } 1.839 inline void DoLock() { pthread_mutex_lock(&mutex); } 1.840 inline void Unlock() { pthread_mutex_unlock(&mutex); } 1.841 1.842 #endif // OVR_ENABLE_THREDS 1.843 1.844 1.845 public: 1.846 // Locker class, used for automatic locking 1.847 class Locker 1.848 { 1.849 public: 1.850 Lock *pLock; 1.851 inline Locker(Lock *plock) 1.852 { pLock = plock; pLock->DoLock(); } 1.853 inline ~Locker() 1.854 { pLock->Unlock(); } 1.855 }; 1.856 }; 1.857 1.858 1.859 1.860 } // OVR 1.861 1.862 #endif 1.863 \ No newline at end of file 1.864 +/************************************************************************************ 1.865 + 1.866 +PublicHeader: OVR.h 1.867 +Filename : OVR_Atomic.h 1.868 +Content : Contains atomic operations and inline fastest locking 1.869 + functionality. Will contain #ifdefs for OS efficiency. 1.870 + Have non-thread-safe implementaion if not available. 1.871 +Created : September 19, 2012 1.872 +Notes : 1.873 + 1.874 +Copyright : Copyright 2012 Oculus VR, Inc. All Rights reserved. 1.875 + 1.876 +Use of this software is subject to the terms of the Oculus license 1.877 +agreement provided at the time of installation or download, or which 1.878 +otherwise accompanies this software in either electronic or hard copy form. 1.879 + 1.880 +************************************************************************************/ 1.881 +#ifndef OVR_Atomic_h 1.882 +#define OVR_Atomic_h 1.883 + 1.884 +#include "OVR_Types.h" 1.885 + 1.886 +// Include System thread functionality. 1.887 +#if defined(OVR_OS_WIN32) 1.888 +#include <windows.h> 1.889 +#else 1.890 +#include <pthread.h> 1.891 +#endif 1.892 + 1.893 + 1.894 +namespace OVR { 1.895 + 1.896 + 1.897 +// ****** Declared classes 1.898 + 1.899 +// If there is NO thread support we implement AtomicOps and 1.900 +// Lock objects as no-ops. The other classes are not defined. 1.901 +template<class C> class AtomicOps; 1.902 +template<class T> class AtomicInt; 1.903 +template<class T> class AtomicPtr; 1.904 + 1.905 +class Lock; 1.906 + 1.907 + 1.908 +//----------------------------------------------------------------------------------- 1.909 +// ***** AtomicOps 1.910 + 1.911 +// Atomic operations are provided by the AtomicOps templates class, 1.912 +// implemented through system-specific AtomicOpsRaw specializations. 1.913 +// It provides several fundamental operations such as Exchange, ExchangeAdd 1.914 +// CompareAndSet, and Store_Release. Each function includes several memory 1.915 +// synchronization versions, important for multiprocessing CPUs with weak 1.916 +// memory consistency. The following memory fencing strategies are supported: 1.917 +// 1.918 +// - NoSync. No memory synchronization is done for atomic op. 1.919 +// - Release. All other memory writes are completed before atomic op 1.920 +// writes its results. 1.921 +// - Acquire. Further memory reads are forced to wait until atomic op 1.922 +// executes, guaranteeing that the right values will be seen. 1.923 +// - Sync. A combination of Release and Acquire. 1.924 + 1.925 + 1.926 +// *** AtomicOpsRaw 1.927 + 1.928 +// AtomicOpsRaw is a specialized template that provides atomic operations 1.929 +// used by AtomicOps. This class has two fundamental qualities: (1) it 1.930 +// defines a type T of correct size, and (2) provides operations that work 1.931 +// atomically, such as Exchange_Sync and CompareAndSet_Release. 1.932 + 1.933 +// AtomicOpsRawBase class contains shared constants/classes for AtomicOpsRaw. 1.934 +// The primary thing is does is define sync class objects, whose destructor and 1.935 +// constructor provide places to insert appropriate synchronization calls, on 1.936 +// systems where such calls are necessary. So far, the breakdown is as follows: 1.937 +// 1.938 +// - X86 systems don't need custom syncs, since their exchange/atomic 1.939 +// instructions are implicitly synchronized. 1.940 +// - PowerPC requires lwsync/isync instructions that can use this mechanism. 1.941 +// - If some other systems require a mechanism where syncing type is associated 1.942 +// with a particular instruction, the default implementation (which implements 1.943 +// all Sync, Acquire, and Release modes in terms of NoSync and fence) may not 1.944 +// work. Ii that case it will need to be #ifdef-ed conditionally. 1.945 + 1.946 +struct AtomicOpsRawBase 1.947 +{ 1.948 +#if !defined(OVR_ENABLE_THREADS) || defined(OVR_CPU_X86) || defined(OVR_OS_WIN32) || defined(OVR_OS_IPHONE) 1.949 + // Need to have empty constructor to avoid class 'unused' variable warning. 1.950 + struct FullSync { inline FullSync() { } }; 1.951 + struct AcquireSync { inline AcquireSync() { } }; 1.952 + struct ReleaseSync { inline ReleaseSync() { } }; 1.953 + 1.954 +#elif defined(OVR_CPU_PPC64) || defined(OVR_CPU_PPC) 1.955 + struct FullSync { inline FullSync() { asm volatile("sync\n"); } ~FullSync() { asm volatile("isync\n"); } }; 1.956 + struct AcquireSync { inline AcquireSync() { } ~AcquireSync() { asm volatile("isync\n"); } }; 1.957 + struct ReleaseSync { inline ReleaseSync() { asm volatile("sync\n"); } }; 1.958 + 1.959 +#elif defined(OVR_CPU_MIPS) 1.960 + struct FullSync { inline FullSync() { asm volatile("sync\n"); } ~FullSync() { asm volatile("sync\n"); } }; 1.961 + struct AcquireSync { inline AcquireSync() { } ~AcquireSync() { asm volatile("sync\n"); } }; 1.962 + struct ReleaseSync { inline ReleaseSync() { asm volatile("sync\n"); } }; 1.963 + 1.964 +#elif defined(OVR_CPU_ARM) 1.965 + struct FullSync { inline FullSync() { asm volatile("dmb\n"); } ~FullSync() { asm volatile("dmb\n"); } }; 1.966 + struct AcquireSync { inline AcquireSync() { } ~AcquireSync() { asm volatile("dmb\n"); } }; 1.967 + struct ReleaseSync { inline ReleaseSync() { asm volatile("dmb\n"); } }; 1.968 + 1.969 + 1.970 +#elif defined(OVR_CC_GNU) && (__GNUC__ >= 4) 1.971 + // __sync functions are already full sync 1.972 + struct FullSync { inline FullSync() { } }; 1.973 + struct AcquireSync { inline AcquireSync() { } }; 1.974 + struct ReleaseSync { inline ReleaseSync() { } }; 1.975 +#endif 1.976 +}; 1.977 + 1.978 + 1.979 +// 4-Byte raw data atomic op implementation class. 1.980 +struct AtomicOpsRaw_4ByteImpl : public AtomicOpsRawBase 1.981 +{ 1.982 +#if !defined(OVR_ENABLE_THREADS) 1.983 + 1.984 + // Provide a type for no-thread-support cases. Used by AtomicOpsRaw_DefImpl. 1.985 + typedef UInt32 T; 1.986 + 1.987 + // *** Thread - Safe Atomic Versions. 1.988 + 1.989 +#elif defined(OVR_OS_WIN32) 1.990 + 1.991 + // Use special defined for VC6, where volatile is not used and 1.992 + // InterlockedCompareExchange is declared incorrectly. 1.993 + typedef LONG T; 1.994 +#if defined(OVR_CC_MSVC) && (OVR_CC_MSVC < 1300) 1.995 + typedef T* InterlockTPtr; 1.996 + typedef LPVOID ET; 1.997 + typedef ET* InterlockETPtr; 1.998 +#else 1.999 + typedef volatile T* InterlockTPtr; 1.1000 + typedef T ET; 1.1001 + typedef InterlockTPtr InterlockETPtr; 1.1002 +#endif 1.1003 + inline static T Exchange_NoSync(volatile T* p, T val) { return InterlockedExchange((InterlockTPtr)p, val); } 1.1004 + inline static T ExchangeAdd_NoSync(volatile T* p, T val) { return InterlockedExchangeAdd((InterlockTPtr)p, val); } 1.1005 + inline static bool CompareAndSet_NoSync(volatile T* p, T c, T val) { return InterlockedCompareExchange((InterlockETPtr)p, (ET)val, (ET)c) == (ET)c; } 1.1006 + 1.1007 +#elif defined(OVR_CPU_PPC64) || defined(OVR_CPU_PPC) 1.1008 + typedef UInt32 T; 1.1009 + static inline UInt32 Exchange_NoSync(volatile UInt32 *i, UInt32 j) 1.1010 + { 1.1011 + UInt32 ret; 1.1012 + 1.1013 + asm volatile("1:\n\t" 1.1014 + "lwarx %[r],0,%[i]\n\t" 1.1015 + "stwcx. %[j],0,%[i]\n\t" 1.1016 + "bne- 1b\n" 1.1017 + : "+m" (*i), [r] "=&b" (ret) : [i] "b" (i), [j] "b" (j) : "cc", "memory"); 1.1018 + 1.1019 + return ret; 1.1020 + } 1.1021 + 1.1022 + static inline UInt32 ExchangeAdd_NoSync(volatile UInt32 *i, UInt32 j) 1.1023 + { 1.1024 + UInt32 dummy, ret; 1.1025 + 1.1026 + asm volatile("1:\n\t" 1.1027 + "lwarx %[r],0,%[i]\n\t" 1.1028 + "add %[o],%[r],%[j]\n\t" 1.1029 + "stwcx. %[o],0,%[i]\n\t" 1.1030 + "bne- 1b\n" 1.1031 + : "+m" (*i), [r] "=&b" (ret), [o] "=&r" (dummy) : [i] "b" (i), [j] "b" (j) : "cc", "memory"); 1.1032 + 1.1033 + return ret; 1.1034 + } 1.1035 + 1.1036 + static inline bool CompareAndSet_NoSync(volatile UInt32 *i, UInt32 c, UInt32 value) 1.1037 + { 1.1038 + UInt32 ret; 1.1039 + 1.1040 + asm volatile("1:\n\t" 1.1041 + "lwarx %[r],0,%[i]\n\t" 1.1042 + "cmpw 0,%[r],%[cmp]\n\t" 1.1043 + "mfcr %[r]\n\t" 1.1044 + "bne- 2f\n\t" 1.1045 + "stwcx. %[val],0,%[i]\n\t" 1.1046 + "bne- 1b\n\t" 1.1047 + "2:\n" 1.1048 + : "+m" (*i), [r] "=&b" (ret) : [i] "b" (i), [cmp] "b" (c), [val] "b" (value) : "cc", "memory"); 1.1049 + 1.1050 + return (ret & 0x20000000) ? 1 : 0; 1.1051 + } 1.1052 + 1.1053 +#elif defined(OVR_CPU_MIPS) 1.1054 + typedef UInt32 T; 1.1055 + 1.1056 + static inline UInt32 Exchange_NoSync(volatile UInt32 *i, UInt32 j) 1.1057 + { 1.1058 + UInt32 ret; 1.1059 + 1.1060 + asm volatile("1:\n\t" 1.1061 + "ll %[r],0(%[i])\n\t" 1.1062 + "sc %[j],0(%[i])\n\t" 1.1063 + "beq %[j],$0,1b\n\t" 1.1064 + "nop \n" 1.1065 + : "+m" (*i), [r] "=&d" (ret) : [i] "d" (i), [j] "d" (j) : "cc", "memory"); 1.1066 + 1.1067 + return ret; 1.1068 + } 1.1069 + 1.1070 + static inline UInt32 ExchangeAdd_NoSync(volatile UInt32 *i, UInt32 j) 1.1071 + { 1.1072 + UInt32 ret; 1.1073 + 1.1074 + asm volatile("1:\n\t" 1.1075 + "ll %[r],0(%[i])\n\t" 1.1076 + "addu %[j],%[r],%[j]\n\t" 1.1077 + "sc %[j],0(%[i])\n\t" 1.1078 + "beq %[j],$0,1b\n\t" 1.1079 + "nop \n" 1.1080 + : "+m" (*i), [r] "=&d" (ret) : [i] "d" (i), [j] "d" (j) : "cc", "memory"); 1.1081 + 1.1082 + return ret; 1.1083 + } 1.1084 + 1.1085 + static inline bool CompareAndSet_NoSync(volatile UInt32 *i, UInt32 c, UInt32 value) 1.1086 + { 1.1087 + UInt32 ret, dummy; 1.1088 + 1.1089 + asm volatile("1:\n\t" 1.1090 + "move %[r],$0\n\t" 1.1091 + "ll %[o],0(%[i])\n\t" 1.1092 + "bne %[o],%[c],2f\n\t" 1.1093 + "move %[r],%[v]\n\t" 1.1094 + "sc %[r],0(%[i])\n\t" 1.1095 + "beq %[r],$0,1b\n\t" 1.1096 + "nop \n\t" 1.1097 + "2:\n" 1.1098 + : "+m" (*i),[r] "=&d" (ret), [o] "=&d" (dummy) : [i] "d" (i), [c] "d" (c), [v] "d" (value) 1.1099 + : "cc", "memory"); 1.1100 + 1.1101 + return ret; 1.1102 + } 1.1103 + 1.1104 +#elif defined(OVR_CPU_ARM) && defined(OVR_CC_ARM) 1.1105 + typedef UInt32 T; 1.1106 + 1.1107 + static inline UInt32 Exchange_NoSync(volatile UInt32 *i, UInt32 j) 1.1108 + { 1.1109 + for(;;) 1.1110 + { 1.1111 + T r = __ldrex(i); 1.1112 + if (__strex(j, i) == 0) 1.1113 + return r; 1.1114 + } 1.1115 + } 1.1116 + static inline UInt32 ExchangeAdd_NoSync(volatile UInt32 *i, UInt32 j) 1.1117 + { 1.1118 + for(;;) 1.1119 + { 1.1120 + T r = __ldrex(i); 1.1121 + if (__strex(r + j, i) == 0) 1.1122 + return r; 1.1123 + } 1.1124 + } 1.1125 + 1.1126 + static inline bool CompareAndSet_NoSync(volatile UInt32 *i, UInt32 c, UInt32 value) 1.1127 + { 1.1128 + for(;;) 1.1129 + { 1.1130 + T r = __ldrex(i); 1.1131 + if (r != c) 1.1132 + return 0; 1.1133 + if (__strex(value, i) == 0) 1.1134 + return 1; 1.1135 + } 1.1136 + } 1.1137 + 1.1138 +#elif defined(OVR_CPU_ARM) 1.1139 + typedef UInt32 T; 1.1140 + 1.1141 + static inline UInt32 Exchange_NoSync(volatile UInt32 *i, UInt32 j) 1.1142 + { 1.1143 + UInt32 ret, dummy; 1.1144 + 1.1145 + asm volatile("1:\n\t" 1.1146 + "ldrex %[r],[%[i]]\n\t" 1.1147 + "strex %[t],%[j],[%[i]]\n\t" 1.1148 + "cmp %[t],#0\n\t" 1.1149 + "bne 1b\n\t" 1.1150 + : "+m" (*i), [r] "=&r" (ret), [t] "=&r" (dummy) : [i] "r" (i), [j] "r" (j) : "cc", "memory"); 1.1151 + 1.1152 + return ret; 1.1153 + } 1.1154 + 1.1155 + static inline UInt32 ExchangeAdd_NoSync(volatile UInt32 *i, UInt32 j) 1.1156 + { 1.1157 + UInt32 ret, dummy, test; 1.1158 + 1.1159 + asm volatile("1:\n\t" 1.1160 + "ldrex %[r],[%[i]]\n\t" 1.1161 + "add %[o],%[r],%[j]\n\t" 1.1162 + "strex %[t],%[o],[%[i]]\n\t" 1.1163 + "cmp %[t],#0\n\t" 1.1164 + "bne 1b\n\t" 1.1165 + : "+m" (*i), [r] "=&r" (ret), [o] "=&r" (dummy), [t] "=&r" (test) : [i] "r" (i), [j] "r" (j) : "cc", "memory"); 1.1166 + 1.1167 + return ret; 1.1168 + } 1.1169 + 1.1170 + static inline bool CompareAndSet_NoSync(volatile UInt32 *i, UInt32 c, UInt32 value) 1.1171 + { 1.1172 + UInt32 ret = 1, dummy, test; 1.1173 + 1.1174 + asm volatile("1:\n\t" 1.1175 + "ldrex %[o],[%[i]]\n\t" 1.1176 + "cmp %[o],%[c]\n\t" 1.1177 + "bne 2f\n\t" 1.1178 + "strex %[r],%[v],[%[i]]\n\t" 1.1179 + "cmp %[r],#0\n\t" 1.1180 + "bne 1b\n\t" 1.1181 + "2:\n" 1.1182 + : "+m" (*i),[r] "=&r" (ret), [o] "=&r" (dummy), [t] "=&r" (test) : [i] "r" (i), [c] "r" (c), [v] "r" (value) 1.1183 + : "cc", "memory"); 1.1184 + 1.1185 + return !ret; 1.1186 + } 1.1187 + 1.1188 +#elif defined(OVR_CPU_X86) 1.1189 + typedef UInt32 T; 1.1190 + 1.1191 + static inline UInt32 Exchange_NoSync(volatile UInt32 *i, UInt32 j) 1.1192 + { 1.1193 + asm volatile("xchgl %1,%[i]\n" 1.1194 + : "+m" (*i), "=q" (j) : [i] "m" (*i), "1" (j) : "cc", "memory"); 1.1195 + 1.1196 + return j; 1.1197 + } 1.1198 + 1.1199 + static inline UInt32 ExchangeAdd_NoSync(volatile UInt32 *i, UInt32 j) 1.1200 + { 1.1201 + asm volatile("lock; xaddl %1,%[i]\n" 1.1202 + : "+m" (*i), "+q" (j) : [i] "m" (*i) : "cc", "memory"); 1.1203 + 1.1204 + return j; 1.1205 + } 1.1206 + 1.1207 + static inline bool CompareAndSet_NoSync(volatile UInt32 *i, UInt32 c, UInt32 value) 1.1208 + { 1.1209 + UInt32 ret; 1.1210 + 1.1211 + asm volatile("lock; cmpxchgl %[v],%[i]\n" 1.1212 + : "+m" (*i), "=a" (ret) : [i] "m" (*i), "1" (c), [v] "q" (value) : "cc", "memory"); 1.1213 + 1.1214 + return (ret == c); 1.1215 + } 1.1216 + 1.1217 +#elif defined(OVR_CC_GNU) && (__GNUC__ >= 4 && __GNUC_MINOR__ >= 1) 1.1218 + 1.1219 + typedef UInt32 T; 1.1220 + 1.1221 + static inline T Exchange_NoSync(volatile T *i, T j) 1.1222 + { 1.1223 + T v; 1.1224 + do { 1.1225 + v = *i; 1.1226 + } while (!__sync_bool_compare_and_swap(i, v, j)); 1.1227 + return v; 1.1228 + } 1.1229 + 1.1230 + static inline T ExchangeAdd_NoSync(volatile T *i, T j) 1.1231 + { 1.1232 + return __sync_fetch_and_add(i, j); 1.1233 + } 1.1234 + 1.1235 + static inline bool CompareAndSet_NoSync(volatile T *i, T c, T value) 1.1236 + { 1.1237 + return __sync_bool_compare_and_swap(i, c, value); 1.1238 + } 1.1239 + 1.1240 +#endif // OS 1.1241 +}; 1.1242 + 1.1243 + 1.1244 +// 8-Byte raw data data atomic op implementation class. 1.1245 +// Currently implementation is provided only on systems with 64-bit pointers. 1.1246 +struct AtomicOpsRaw_8ByteImpl : public AtomicOpsRawBase 1.1247 +{ 1.1248 +#if !defined(OVR_64BIT_POINTERS) || !defined(OVR_ENABLE_THREADS) 1.1249 + 1.1250 + // Provide a type for no-thread-support cases. Used by AtomicOpsRaw_DefImpl. 1.1251 + typedef UInt64 T; 1.1252 + 1.1253 + // *** Thread - Safe OS specific versions. 1.1254 +#elif defined(OVR_OS_WIN32) 1.1255 + 1.1256 + // This is only for 64-bit systems. 1.1257 + typedef LONG64 T; 1.1258 + typedef volatile T* InterlockTPtr; 1.1259 + inline static T Exchange_NoSync(volatile T* p, T val) { return InterlockedExchange64((InterlockTPtr)p, val); } 1.1260 + inline static T ExchangeAdd_NoSync(volatile T* p, T val) { return InterlockedExchangeAdd64((InterlockTPtr)p, val); } 1.1261 + inline static bool CompareAndSet_NoSync(volatile T* p, T c, T val) { return InterlockedCompareExchange64((InterlockTPtr)p, val, c) == c; } 1.1262 + 1.1263 +#elif defined(OVR_CPU_PPC64) 1.1264 + 1.1265 + typedef UInt64 T; 1.1266 + 1.1267 + static inline UInt64 Exchange_NoSync(volatile UInt64 *i, UInt64 j) 1.1268 + { 1.1269 + UInt64 dummy, ret; 1.1270 + 1.1271 + asm volatile("1:\n\t" 1.1272 + "ldarx %[r],0,%[i]\n\t" 1.1273 + "mr %[o],%[j]\n\t" 1.1274 + "stdcx. %[o],0,%[i]\n\t" 1.1275 + "bne- 1b\n" 1.1276 + : "+m" (*i), [r] "=&b" (ret), [o] "=&r" (dummy) : [i] "b" (i), [j] "b" (j) : "cc"); 1.1277 + 1.1278 + return ret; 1.1279 + } 1.1280 + 1.1281 + static inline UInt64 ExchangeAdd_NoSync(volatile UInt64 *i, UInt64 j) 1.1282 + { 1.1283 + UInt64 dummy, ret; 1.1284 + 1.1285 + asm volatile("1:\n\t" 1.1286 + "ldarx %[r],0,%[i]\n\t" 1.1287 + "add %[o],%[r],%[j]\n\t" 1.1288 + "stdcx. %[o],0,%[i]\n\t" 1.1289 + "bne- 1b\n" 1.1290 + : "+m" (*i), [r] "=&b" (ret), [o] "=&r" (dummy) : [i] "b" (i), [j] "b" (j) : "cc"); 1.1291 + 1.1292 + return ret; 1.1293 + } 1.1294 + 1.1295 + static inline bool CompareAndSet_NoSync(volatile UInt64 *i, UInt64 c, UInt64 value) 1.1296 + { 1.1297 + UInt64 ret, dummy; 1.1298 + 1.1299 + asm volatile("1:\n\t" 1.1300 + "ldarx %[r],0,%[i]\n\t" 1.1301 + "cmpw 0,%[r],%[cmp]\n\t" 1.1302 + "mfcr %[r]\n\t" 1.1303 + "bne- 2f\n\t" 1.1304 + "stdcx. %[val],0,%[i]\n\t" 1.1305 + "bne- 1b\n\t" 1.1306 + "2:\n" 1.1307 + : "+m" (*i), [r] "=&b" (ret), [o] "=&r" (dummy) : [i] "b" (i), [cmp] "b" (c), [val] "b" (value) : "cc"); 1.1308 + 1.1309 + return (ret & 0x20000000) ? 1 : 0; 1.1310 + } 1.1311 + 1.1312 +#elif defined(OVR_CC_GNU) && (__GNUC__ >= 4 && __GNUC_MINOR__ >= 1) 1.1313 + 1.1314 + typedef UInt64 T; 1.1315 + 1.1316 + static inline T Exchange_NoSync(volatile T *i, T j) 1.1317 + { 1.1318 + T v; 1.1319 + do { 1.1320 + v = *i; 1.1321 + } while (!__sync_bool_compare_and_swap(i, v, j)); 1.1322 + return v; 1.1323 + } 1.1324 + 1.1325 + static inline T ExchangeAdd_NoSync(volatile T *i, T j) 1.1326 + { 1.1327 + return __sync_fetch_and_add(i, j); 1.1328 + } 1.1329 + 1.1330 + static inline bool CompareAndSet_NoSync(volatile T *i, T c, T value) 1.1331 + { 1.1332 + return __sync_bool_compare_and_swap(i, c, value); 1.1333 + } 1.1334 + 1.1335 +#endif // OS 1.1336 +}; 1.1337 + 1.1338 + 1.1339 +// Default implementation for AtomicOpsRaw; provides implementation of mem-fenced 1.1340 +// atomic operations where fencing is done with a sync object wrapped around a NoSync 1.1341 +// operation implemented in the base class. If such implementation is not possible 1.1342 +// on a given platform, #ifdefs can be used to disable it and then op functions can be 1.1343 +// implemented individually in the appropriate AtomicOpsRaw<size> class. 1.1344 + 1.1345 +template<class O> 1.1346 +struct AtomicOpsRaw_DefImpl : public O 1.1347 +{ 1.1348 + typedef typename O::T O_T; 1.1349 + typedef typename O::FullSync O_FullSync; 1.1350 + typedef typename O::AcquireSync O_AcquireSync; 1.1351 + typedef typename O::ReleaseSync O_ReleaseSync; 1.1352 + 1.1353 + // If there is no thread support, provide the default implementation. In this case, 1.1354 + // the base class (0) must still provide the T declaration. 1.1355 +#ifndef OVR_ENABLE_THREADS 1.1356 + 1.1357 + // Atomic exchange of val with argument. Returns old val. 1.1358 + inline static O_T Exchange_NoSync(volatile O_T* p, O_T val) { O_T old = *p; *p = val; return old; } 1.1359 + // Adds a new val to argument; returns its old val. 1.1360 + inline static O_T ExchangeAdd_NoSync(volatile O_T* p, O_T val) { O_T old = *p; *p += val; return old; } 1.1361 + // Compares the argument data with 'c' val. 1.1362 + // If succeeded, stores val int '*p' and returns true; otherwise returns false. 1.1363 + inline static bool CompareAndSet_NoSync(volatile O_T* p, O_T c, O_T val) { if (*p==c) { *p = val; return 1; } return 0; } 1.1364 + 1.1365 +#endif 1.1366 + 1.1367 + // If NoSync wrapped implementation may not be possible, it this block should be 1.1368 + // replaced with per-function implementation in O. 1.1369 + // "AtomicOpsRaw_DefImpl<O>::" prefix in calls below. 1.1370 + inline static O_T Exchange_Sync(volatile O_T* p, O_T val) { O_FullSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::Exchange_NoSync(p, val); } 1.1371 + inline static O_T Exchange_Release(volatile O_T* p, O_T val) { O_ReleaseSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::Exchange_NoSync(p, val); } 1.1372 + inline static O_T Exchange_Acquire(volatile O_T* p, O_T val) { O_AcquireSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::Exchange_NoSync(p, val); } 1.1373 + inline static O_T ExchangeAdd_Sync(volatile O_T* p, O_T val) { O_FullSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::ExchangeAdd_NoSync(p, val); } 1.1374 + inline static O_T ExchangeAdd_Release(volatile O_T* p, O_T val) { O_ReleaseSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::ExchangeAdd_NoSync(p, val); } 1.1375 + inline static O_T ExchangeAdd_Acquire(volatile O_T* p, O_T val) { O_AcquireSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::ExchangeAdd_NoSync(p, val); } 1.1376 + inline static bool CompareAndSet_Sync(volatile O_T* p, O_T c, O_T val) { O_FullSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::CompareAndSet_NoSync(p,c,val); } 1.1377 + inline static bool CompareAndSet_Release(volatile O_T* p, O_T c, O_T val) { O_ReleaseSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::CompareAndSet_NoSync(p,c,val); } 1.1378 + inline static bool CompareAndSet_Acquire(volatile O_T* p, O_T c, O_T val) { O_AcquireSync sync; OVR_UNUSED(sync); return AtomicOpsRaw_DefImpl<O>::CompareAndSet_NoSync(p,c,val); } 1.1379 + 1.1380 + // Loads and stores with memory fence. These have only the relevant versions. 1.1381 +#ifdef OVR_CPU_X86 1.1382 + // On X86, Store_Release is implemented as exchange. Note that we can also 1.1383 + // consider 'sfence' in the future, although it is not as compatible with older CPUs. 1.1384 + inline static void Store_Release(volatile O_T* p, O_T val) { Exchange_Release(p, val); } 1.1385 +#else 1.1386 + inline static void Store_Release(volatile O_T* p, O_T val) { O_ReleaseSync sync; OVR_UNUSED(sync); *p = val; } 1.1387 +#endif 1.1388 + inline static O_T Load_Acquire(const volatile O_T* p) { O_AcquireSync sync; OVR_UNUSED(sync); return *p; } 1.1389 +}; 1.1390 + 1.1391 + 1.1392 +template<int size> 1.1393 +struct AtomicOpsRaw : public AtomicOpsRawBase { }; 1.1394 + 1.1395 +template<> 1.1396 +struct AtomicOpsRaw<4> : public AtomicOpsRaw_DefImpl<AtomicOpsRaw_4ByteImpl> 1.1397 +{ 1.1398 + // Ensure that assigned type size is correct. 1.1399 + AtomicOpsRaw() 1.1400 + { OVR_COMPILER_ASSERT(sizeof(AtomicOpsRaw_DefImpl<AtomicOpsRaw_4ByteImpl>::T) == 4); } 1.1401 +}; 1.1402 +template<> 1.1403 +struct AtomicOpsRaw<8> : public AtomicOpsRaw_DefImpl<AtomicOpsRaw_8ByteImpl> 1.1404 +{ 1.1405 + AtomicOpsRaw() 1.1406 + { OVR_COMPILER_ASSERT(sizeof(AtomicOpsRaw_DefImpl<AtomicOpsRaw_8ByteImpl>::T) == 8); } 1.1407 +}; 1.1408 + 1.1409 + 1.1410 +// *** AtomicOps - implementation of atomic Ops for specified class 1.1411 + 1.1412 +// Implements atomic ops on a class, provided that the object is either 1.1413 +// 4 or 8 bytes in size (depending on the AtomicOpsRaw specializations 1.1414 +// available). Relies on AtomicOpsRaw for much of implementation. 1.1415 + 1.1416 +template<class C> 1.1417 +class AtomicOps 1.1418 +{ 1.1419 + typedef AtomicOpsRaw<sizeof(C)> Ops; 1.1420 + typedef typename Ops::T T; 1.1421 + typedef volatile typename Ops::T* PT; 1.1422 + // We cast through unions to (1) avoid pointer size compiler warnings 1.1423 + // and (2) ensure that there are no problems with strict pointer aliasing. 1.1424 + union C2T_union { C c; T t; }; 1.1425 + 1.1426 +public: 1.1427 + // General purpose implementation for standard syncs. 1.1428 + inline static C Exchange_Sync(volatile C* p, C val) { C2T_union u; u.c = val; u.t = Ops::Exchange_Sync((PT)p, u.t); return u.c; } 1.1429 + inline static C Exchange_Release(volatile C* p, C val) { C2T_union u; u.c = val; u.t = Ops::Exchange_Release((PT)p, u.t); return u.c; } 1.1430 + inline static C Exchange_Acquire(volatile C* p, C val) { C2T_union u; u.c = val; u.t = Ops::Exchange_Acquire((PT)p, u.t); return u.c; } 1.1431 + inline static C Exchange_NoSync(volatile C* p, C val) { C2T_union u; u.c = val; u.t = Ops::Exchange_NoSync((PT)p, u.t); return u.c; } 1.1432 + inline static C ExchangeAdd_Sync(volatile C* p, C val) { C2T_union u; u.c = val; u.t = Ops::ExchangeAdd_Sync((PT)p, u.t); return u.c; } 1.1433 + inline static C ExchangeAdd_Release(volatile C* p, C val) { C2T_union u; u.c = val; u.t = Ops::ExchangeAdd_Release((PT)p, u.t); return u.c; } 1.1434 + inline static C ExchangeAdd_Acquire(volatile C* p, C val) { C2T_union u; u.c = val; u.t = Ops::ExchangeAdd_Acquire((PT)p, u.t); return u.c; } 1.1435 + inline static C ExchangeAdd_NoSync(volatile C* p, C val) { C2T_union u; u.c = val; u.t = Ops::ExchangeAdd_NoSync((PT)p, u.t); return u.c; } 1.1436 + inline static bool CompareAndSet_Sync(volatile C* p, C c, C val) { C2T_union u,cu; u.c = val; cu.c = c; return Ops::CompareAndSet_Sync((PT)p, cu.t, u.t); } 1.1437 + inline static bool CompareAndSet_Release(volatile C* p, C c, C val){ C2T_union u,cu; u.c = val; cu.c = c; return Ops::CompareAndSet_Release((PT)p, cu.t, u.t); } 1.1438 + inline static bool CompareAndSet_Relse(volatile C* p, C c, C val){ C2T_union u,cu; u.c = val; cu.c = c; return Ops::CompareAndSet_Acquire((PT)p, cu.t, u.t); } 1.1439 + inline static bool CompareAndSet_NoSync(volatile C* p, C c, C val) { C2T_union u,cu; u.c = val; cu.c = c; return Ops::CompareAndSet_NoSync((PT)p, cu.t, u.t); } 1.1440 + // Loads and stores with memory fence. These have only the relevant versions. 1.1441 + inline static void Store_Release(volatile C* p, C val) { C2T_union u; u.c = val; Ops::Store_Release((PT)p, u.t); } 1.1442 + inline static C Load_Acquire(const volatile C* p) { C2T_union u; u.t = Ops::Load_Acquire((PT)p); return u.c; } 1.1443 +}; 1.1444 + 1.1445 + 1.1446 + 1.1447 +// Atomic value base class - implements operations shared for integers and pointers. 1.1448 +template<class T> 1.1449 +class AtomicValueBase 1.1450 +{ 1.1451 +protected: 1.1452 + typedef AtomicOps<T> Ops; 1.1453 +public: 1.1454 + 1.1455 + volatile T Value; 1.1456 + 1.1457 + inline AtomicValueBase() { } 1.1458 + explicit inline AtomicValueBase(T val) { Ops::Store_Release(&Value, val); } 1.1459 + 1.1460 + // Most libraries (TBB and Joshua Scholar's) library do not do Load_Acquire 1.1461 + // here, since most algorithms do not require atomic loads. Needs some research. 1.1462 + inline operator T() const { return Value; } 1.1463 + 1.1464 + // *** Standard Atomic inlines 1.1465 + inline T Exchange_Sync(T val) { return Ops::Exchange_Sync(&Value, val); } 1.1466 + inline T Exchange_Release(T val) { return Ops::Exchange_Release(&Value, val); } 1.1467 + inline T Exchange_Acquire(T val) { return Ops::Exchange_Acquire(&Value, val); } 1.1468 + inline T Exchange_NoSync(T val) { return Ops::Exchange_NoSync(&Value, val); } 1.1469 + inline bool CompareAndSet_Sync(T c, T val) { return Ops::CompareAndSet_Sync(&Value, c, val); } 1.1470 + inline bool CompareAndSet_Release(T c, T val) { return Ops::CompareAndSet_Release(&Value, c, val); } 1.1471 + inline bool CompareAndSet_Acquire(T c, T val) { return Ops::CompareAndSet_Relse(&Value, c, val); } 1.1472 + inline bool CompareAndSet_NoSync(T c, T val) { return Ops::CompareAndSet_NoSync(&Value, c, val); } 1.1473 + // Load & Store. 1.1474 + inline void Store_Release(T val) { Ops::Store_Release(&Value, val); } 1.1475 + inline T Load_Acquire() const { return Ops::Load_Acquire(&Value); } 1.1476 +}; 1.1477 + 1.1478 + 1.1479 +// ***** AtomicPtr - Atomic pointer template 1.1480 + 1.1481 +// This pointer class supports atomic assignments with release, 1.1482 +// increment / decrement operations, and conditional compare + set. 1.1483 + 1.1484 +template<class T> 1.1485 +class AtomicPtr : public AtomicValueBase<T*> 1.1486 +{ 1.1487 + typedef typename AtomicValueBase<T*>::Ops Ops; 1.1488 + 1.1489 +public: 1.1490 + // Initialize pointer value to 0 by default; use Store_Release only with explicit constructor. 1.1491 + inline AtomicPtr() : AtomicValueBase<T*>() { this->Value = 0; } 1.1492 + explicit inline AtomicPtr(T* val) : AtomicValueBase<T*>(val) { } 1.1493 + 1.1494 + // Pointer access. 1.1495 + inline T* operator -> () const { return this->Load_Acquire(); } 1.1496 + 1.1497 + // It looks like it is convenient to have Load_Acquire characteristics 1.1498 + // for this, since that is convenient for algorithms such as linked 1.1499 + // list traversals that can be added to bu another thread. 1.1500 + inline operator T* () const { return this->Load_Acquire(); } 1.1501 + 1.1502 + 1.1503 + // *** Standard Atomic inlines (applicable to pointers) 1.1504 + 1.1505 + // ExhangeAdd considers pointer size for pointers. 1.1506 + template<class I> 1.1507 + inline T* ExchangeAdd_Sync(I incr) { return Ops::ExchangeAdd_Sync(&this->Value, ((T*)0) + incr); } 1.1508 + template<class I> 1.1509 + inline T* ExchangeAdd_Release(I incr) { return Ops::ExchangeAdd_Release(&this->Value, ((T*)0) + incr); } 1.1510 + template<class I> 1.1511 + inline T* ExchangeAdd_Acquire(I incr) { return Ops::ExchangeAdd_Acquire(&this->Value, ((T*)0) + incr); } 1.1512 + template<class I> 1.1513 + inline T* ExchangeAdd_NoSync(I incr) { return Ops::ExchangeAdd_NoSync(&this->Value, ((T*)0) + incr); } 1.1514 + 1.1515 + // *** Atomic Operators 1.1516 + 1.1517 + inline T* operator = (T* val) { this->Store_Release(val); return val; } 1.1518 + 1.1519 + template<class I> 1.1520 + inline T* operator += (I val) { return ExchangeAdd_Sync(val) + val; } 1.1521 + template<class I> 1.1522 + inline T* operator -= (I val) { return operator += (-val); } 1.1523 + 1.1524 + inline T* operator ++ () { return ExchangeAdd_Sync(1) + 1; } 1.1525 + inline T* operator -- () { return ExchangeAdd_Sync(-1) - 1; } 1.1526 + inline T* operator ++ (int) { return ExchangeAdd_Sync(1); } 1.1527 + inline T* operator -- (int) { return ExchangeAdd_Sync(-1); } 1.1528 +}; 1.1529 + 1.1530 + 1.1531 +// ***** AtomicInt - Atomic integer template 1.1532 + 1.1533 +// Implements an atomic integer type; the exact type to use is provided 1.1534 +// as an argument. Supports atomic Acquire / Release semantics, atomic 1.1535 +// arithmetic operations, and atomic conditional compare + set. 1.1536 + 1.1537 +template<class T> 1.1538 +class AtomicInt : public AtomicValueBase<T> 1.1539 +{ 1.1540 + typedef typename AtomicValueBase<T>::Ops Ops; 1.1541 + 1.1542 +public: 1.1543 + inline AtomicInt() : AtomicValueBase<T>() { } 1.1544 + explicit inline AtomicInt(T val) : AtomicValueBase<T>(val) { } 1.1545 + 1.1546 + 1.1547 + // *** Standard Atomic inlines (applicable to int) 1.1548 + inline T ExchangeAdd_Sync(T val) { return Ops::ExchangeAdd_Sync(&this->Value, val); } 1.1549 + inline T ExchangeAdd_Release(T val) { return Ops::ExchangeAdd_Release(&this->Value, val); } 1.1550 + inline T ExchangeAdd_Acquire(T val) { return Ops::ExchangeAdd_Acquire(&this->Value, val); } 1.1551 + inline T ExchangeAdd_NoSync(T val) { return Ops::ExchangeAdd_NoSync(&this->Value, val); } 1.1552 + // These increments could be more efficient because they don't return a value. 1.1553 + inline void Increment_Sync() { ExchangeAdd_Sync((T)1); } 1.1554 + inline void Increment_Release() { ExchangeAdd_Release((T)1); } 1.1555 + inline void Increment_Acquire() { ExchangeAdd_Acquire((T)1); } 1.1556 + inline void Increment_NoSync() { ExchangeAdd_NoSync((T)1); } 1.1557 + 1.1558 + // *** Atomic Operators 1.1559 + 1.1560 + inline T operator = (T val) { this->Store_Release(val); return val; } 1.1561 + inline T operator += (T val) { return ExchangeAdd_Sync(val) + val; } 1.1562 + inline T operator -= (T val) { return ExchangeAdd_Sync(0 - val) - val; } 1.1563 + 1.1564 + inline T operator ++ () { return ExchangeAdd_Sync((T)1) + 1; } 1.1565 + inline T operator -- () { return ExchangeAdd_Sync(((T)0)-1) - 1; } 1.1566 + inline T operator ++ (int) { return ExchangeAdd_Sync((T)1); } 1.1567 + inline T operator -- (int) { return ExchangeAdd_Sync(((T)0)-1); } 1.1568 + 1.1569 + // More complex atomic operations. Leave it to compiler whether to optimize them or not. 1.1570 + T operator &= (T arg) 1.1571 + { 1.1572 + T comp, newVal; 1.1573 + do { 1.1574 + comp = this->Value; 1.1575 + newVal = comp & arg; 1.1576 + } while(!this->CompareAndSet_Sync(comp, newVal)); 1.1577 + return newVal; 1.1578 + } 1.1579 + 1.1580 + T operator |= (T arg) 1.1581 + { 1.1582 + T comp, newVal; 1.1583 + do { 1.1584 + comp = this->Value; 1.1585 + newVal = comp | arg; 1.1586 + } while(!this->CompareAndSet_Sync(comp, newVal)); 1.1587 + return newVal; 1.1588 + } 1.1589 + 1.1590 + T operator ^= (T arg) 1.1591 + { 1.1592 + T comp, newVal; 1.1593 + do { 1.1594 + comp = this->Value; 1.1595 + newVal = comp ^ arg; 1.1596 + } while(!this->CompareAndSet_Sync(comp, newVal)); 1.1597 + return newVal; 1.1598 + } 1.1599 + 1.1600 + T operator *= (T arg) 1.1601 + { 1.1602 + T comp, newVal; 1.1603 + do { 1.1604 + comp = this->Value; 1.1605 + newVal = comp * arg; 1.1606 + } while(!this->CompareAndSet_Sync(comp, newVal)); 1.1607 + return newVal; 1.1608 + } 1.1609 + 1.1610 + T operator /= (T arg) 1.1611 + { 1.1612 + T comp, newVal; 1.1613 + do { 1.1614 + comp = this->Value; 1.1615 + newVal = comp / arg; 1.1616 + } while(!CompareAndSet_Sync(comp, newVal)); 1.1617 + return newVal; 1.1618 + } 1.1619 + 1.1620 + T operator >>= (unsigned bits) 1.1621 + { 1.1622 + T comp, newVal; 1.1623 + do { 1.1624 + comp = this->Value; 1.1625 + newVal = comp >> bits; 1.1626 + } while(!CompareAndSet_Sync(comp, newVal)); 1.1627 + return newVal; 1.1628 + } 1.1629 + 1.1630 + T operator <<= (unsigned bits) 1.1631 + { 1.1632 + T comp, newVal; 1.1633 + do { 1.1634 + comp = this->Value; 1.1635 + newVal = comp << bits; 1.1636 + } while(!this->CompareAndSet_Sync(comp, newVal)); 1.1637 + return newVal; 1.1638 + } 1.1639 +}; 1.1640 + 1.1641 + 1.1642 + 1.1643 +//----------------------------------------------------------------------------------- 1.1644 +// ***** Lock 1.1645 + 1.1646 +// Lock is a simplest and most efficient mutual-exclusion lock class. 1.1647 +// Unlike Mutex, it cannot be waited on. 1.1648 + 1.1649 +class Lock 1.1650 +{ 1.1651 + // NOTE: Locks are not allocatable and they themselves should not allocate 1.1652 + // memory by standard means. This is the case because StandardAllocator 1.1653 + // relies on this class. 1.1654 + // Make 'delete' private. Don't do this for 'new' since it can be redefined. 1.1655 + void operator delete(void*) {} 1.1656 + 1.1657 + 1.1658 + // *** Lock implementation for various platforms. 1.1659 + 1.1660 +#if !defined(OVR_ENABLE_THREADS) 1.1661 + 1.1662 +public: 1.1663 + // With no thread support, lock does nothing. 1.1664 + inline Lock() { } 1.1665 + inline Lock(unsigned) { } 1.1666 + inline ~Lock() { } 1.1667 + inline void DoLock() { } 1.1668 + inline void Unlock() { } 1.1669 + 1.1670 + // Windows. 1.1671 +#elif defined(OVR_OS_WIN32) 1.1672 + 1.1673 + CRITICAL_SECTION cs; 1.1674 +public: 1.1675 + Lock(unsigned spinCount = 0); 1.1676 + ~Lock(); 1.1677 + // Locking functions. 1.1678 + inline void DoLock() { ::EnterCriticalSection(&cs); } 1.1679 + inline void Unlock() { ::LeaveCriticalSection(&cs); } 1.1680 + 1.1681 +#else 1.1682 + pthread_mutex_t mutex; 1.1683 + 1.1684 +public: 1.1685 + static pthread_mutexattr_t RecursiveAttr; 1.1686 + static bool RecursiveAttrInit; 1.1687 + 1.1688 + Lock (unsigned dummy = 0) 1.1689 + { 1.1690 + if (!RecursiveAttrInit) 1.1691 + { 1.1692 + pthread_mutexattr_init(&RecursiveAttr); 1.1693 + pthread_mutexattr_settype(&RecursiveAttr, PTHREAD_MUTEX_RECURSIVE); 1.1694 + RecursiveAttrInit = 1; 1.1695 + } 1.1696 + pthread_mutex_init(&mutex,&RecursiveAttr); 1.1697 + } 1.1698 + ~Lock () { pthread_mutex_destroy(&mutex); } 1.1699 + inline void DoLock() { pthread_mutex_lock(&mutex); } 1.1700 + inline void Unlock() { pthread_mutex_unlock(&mutex); } 1.1701 + 1.1702 +#endif // OVR_ENABLE_THREDS 1.1703 + 1.1704 + 1.1705 +public: 1.1706 + // Locker class, used for automatic locking 1.1707 + class Locker 1.1708 + { 1.1709 + public: 1.1710 + Lock *pLock; 1.1711 + inline Locker(Lock *plock) 1.1712 + { pLock = plock; pLock->DoLock(); } 1.1713 + inline ~Locker() 1.1714 + { pLock->Unlock(); } 1.1715 + }; 1.1716 +}; 1.1717 + 1.1718 + 1.1719 + 1.1720 +} // OVR 1.1721 + 1.1722 +#endif