- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

# tbb_machine.h

    00001 /*
    00002     Copyright 2005-2012 Intel Corporation.  All Rights Reserved.
    00003
    00004     This file is part of Threading Building Blocks.
    00005
    00006     Threading Building Blocks is free software; you can redistribute it
    00007     and/or modify it under the terms of the GNU General Public License
    00008     version 2 as published by the Free Software Foundation.
    00009
    00010     Threading Building Blocks is distributed in the hope that it will be
    00011     useful, but WITHOUT ANY WARRANTY; without even the implied warranty
    00012     of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    00013     GNU General Public License for more details.
    00014
    00015     You should have received a copy of the GNU General Public License
    00016     along with Threading Building Blocks; if not, write to the Free Software
    00017     Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
    00018
    00019     As a special exception, you may use this file as part of a free software
    00020     library without restriction.  Specifically, if other files instantiate
    00021     templates or use macros or inline functions from this file, or you compile
    00022     this file and link it with other files to produce an executable, this
    00023     file does not by itself cause the resulting executable to be covered by
    00024     the GNU General Public License.  This exception does not however
    00025     invalidate any other reasons why the executable file might be covered by
    00026     the GNU General Public License.
    00027 */
    00028
    00029 #ifndef __TBB_machine_H
    00030 #define __TBB_machine_H
    00031
    00124 #include "tbb_stddef.h"
    00125
    00126 namespace tbb {
    00127 namespace internal {
    00128
    00130 // Overridable helpers declarations
    00131 //
    00132 // A machine/*.h file may choose to define these templates, otherwise it must
    00133 // request default implementation by setting appropriate __TBB_USE_GENERIC_XXX macro(s).
    00134 //
    00135 template <typename T, std::size_t S>
    00136 struct machine_load_store;
    00137
    00138 template <typename T, std::size_t S>
    00139 struct machine_load_store_relaxed;
    00140
    00141 template <typename T, std::size_t S>
    00142 struct machine_load_store_seq_cst;
    00143 //
    00144 // End of overridable helpers declarations
    00146
    00147 template<size_t S> struct atomic_selector;
    00148
    00149 template<> struct atomic_selector<1> {
    00150     typedef int8_t word;
    00151     inline static word fetch_store ( volatile void* location, word value );
    00152 };
    00153
    00154 template<> struct atomic_selector<2> {
    00155     typedef int16_t word;
    00156     inline static word fetch_store ( volatile void* location, word value );
    00157 };
    00158
    00159 template<> struct atomic_selector<4> {
    00160 #if _MSC_VER && !_WIN64
    00161     // Work-around that avoids spurious /Wp64 warnings
    00162     typedef intptr_t word;
    00163 #else
    00164     typedef int32_t word;
    00165 #endif
    00166     inline static word fetch_store ( volatile void* location, word value );
    00167 };
    00168
    00169 template<> struct atomic_selector<8> {
    00170     typedef int64_t word;
    00171     inline static word fetch_store ( volatile void* location, word value );
    00172 };
    00173
    00174 }} // namespaces internal, tbb
    00175
    00176 #if _WIN32||_WIN64
    00177
    00178 #ifdef _MANAGED
    00179 #pragma managed(push, off)
    00180 #endif
    00181
    00182     #if __MINGW64__ || __MINGW32__
    00183         extern "C" __declspec(dllimport) int __stdcall SwitchToThread( void );
    00184         #define __TBB_Yield()  SwitchToThread()
    00185         #if (TBB_USE_GCC_BUILTINS && __TBB_GCC_BUILTIN_ATOMICS_PRESENT)
    00186             #include "machine/gcc_generic.h"
    00187         #elif __MINGW64__
    00188             #include "machine/linux_intel64.h"
    00189         #elif __MINGW32__
    00190             #include "machine/linux_ia32.h"
    00191         #endif
    00192     #elif (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT)
    00193         #include "machine/icc_generic.h"
    00194     #elif defined(_M_IX86)
    00195         #include "machine/windows_ia32.h"
    00196     #elif defined(_M_X64)
    00197         #include "machine/windows_intel64.h"
    00198     #elif _XBOX
    00199         #include "machine/xbox360_ppc.h"
    00200     #endif
    00201
    00202 #ifdef _MANAGED
    00203 #pragma managed(pop)
    00204 #endif
    00205
    00206 #elif __TBB_DEFINE_MIC
    00207
    00208     #include "machine/mic_common.h"
    00209     //TODO: check if ICC atomic intrinsics are available for MIC
    00210     #include "machine/linux_intel64.h"
    00211
    00212 #elif __linux__ || __FreeBSD__ || __NetBSD__
    00213
    00214     #if (TBB_USE_GCC_BUILTINS && __TBB_GCC_BUILTIN_ATOMICS_PRESENT)
    00215         #include "machine/gcc_generic.h"
    00216     #elif (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT)
    00217         #include "machine/icc_generic.h"
    00218     #elif __i386__
    00219         #include "machine/linux_ia32.h"
    00220     #elif __x86_64__
    00221         #include "machine/linux_intel64.h"
    00222     #elif __ia64__
    00223         #include "machine/linux_ia64.h"
    00224     #elif __powerpc__
    00225         #include "machine/mac_ppc.h"
    00226     #elif __TBB_GCC_BUILTIN_ATOMICS_PRESENT
    00227         #include "machine/gcc_generic.h"
    00228     #endif
    00229     #include "machine/linux_common.h"
    00230
    00231 #elif __APPLE__
    00232     //TODO:  TBB_USE_GCC_BUILTINS is not used for Mac, Sun, Aix
    00233     #if (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT)
    00234         #include "machine/icc_generic.h"
    00235     #elif __i386__
    00236         #include "machine/linux_ia32.h"
    00237     #elif __x86_64__
    00238         #include "machine/linux_intel64.h"
    00239     #elif __POWERPC__
    00240         #include "machine/mac_ppc.h"
    00241     #endif
    00242     #include "machine/macos_common.h"
    00243
    00244 #elif _AIX
    00245
    00246     #include "machine/ibm_aix51.h"
    00247
    00248 #elif __sun || __SUNPRO_CC
    00249
    00250     #define __asm__ asm
    00251     #define __volatile__ volatile
    00252
    00253     #if __i386  || __i386__
    00254         #include "machine/linux_ia32.h"
    00255     #elif __x86_64__
    00256         #include "machine/linux_intel64.h"
    00257     #elif __sparc
    00258         #include "machine/sunos_sparc.h"
    00259     #endif
    00260     #include <sched.h>
    00261
    00262     #define __TBB_Yield() sched_yield()
    00263
    00264 #endif /* OS selection */
    00265
    00266 #ifndef __TBB_64BIT_ATOMICS
    00267     #define __TBB_64BIT_ATOMICS 1
    00268 #endif
    00269
    00270 //TODO: replace usage of these functions with usage of tbb::atomic, and then remove them
    00271 //TODO: map functions with W suffix to use cast to tbb::atomic and according op, i.e. as_atomic().op()
    00272 // Special atomic functions
    00273 #if __TBB_USE_FENCED_ATOMICS
    00274     #define __TBB_machine_cmpswp1   __TBB_machine_cmpswp1full_fence
    00275     #define __TBB_machine_cmpswp2   __TBB_machine_cmpswp2full_fence
    00276     #define __TBB_machine_cmpswp4   __TBB_machine_cmpswp4full_fence
    00277     #define __TBB_machine_cmpswp8   __TBB_machine_cmpswp8full_fence
    00278
    00279     #if __TBB_WORDSIZE==8
    00280         #define __TBB_machine_fetchadd8             __TBB_machine_fetchadd8full_fence
    00281         #define __TBB_machine_fetchstore8           __TBB_machine_fetchstore8full_fence
    00282         #define __TBB_FetchAndAddWrelease(P,V)      __TBB_machine_fetchadd8release(P,V)
    00283         #define __TBB_FetchAndIncrementWacquire(P)  __TBB_machine_fetchadd8acquire(P,1)
    00284         #define __TBB_FetchAndDecrementWrelease(P)  __TBB_machine_fetchadd8release(P,(-1))
    00285     #else
    00286         #define __TBB_machine_fetchadd4             __TBB_machine_fetchadd4full_fence
    00287         #define __TBB_machine_fetchstore4           __TBB_machine_fetchstore4full_fence
    00288         #define __TBB_FetchAndAddWrelease(P,V)      __TBB_machine_fetchadd4release(P,V)
    00289         #define __TBB_FetchAndIncrementWacquire(P)  __TBB_machine_fetchadd4acquire(P,1)
    00290         #define __TBB_FetchAndDecrementWrelease(P)  __TBB_machine_fetchadd4release(P,(-1))
    00291     #endif /* __TBB_WORDSIZE==4 */
    00292 #else /* !__TBB_USE_FENCED_ATOMICS */
    00293     #define __TBB_FetchAndAddWrelease(P,V)      __TBB_FetchAndAddW(P,V)
    00294     #define __TBB_FetchAndIncrementWacquire(P)  __TBB_FetchAndAddW(P,1)
    00295     #define __TBB_FetchAndDecrementWrelease(P)  __TBB_FetchAndAddW(P,(-1))
    00296 #endif /* !__TBB_USE_FENCED_ATOMICS */
    00297
    00298 #if __TBB_WORDSIZE==4
    00299     #define __TBB_CompareAndSwapW(P,V,C)    __TBB_machine_cmpswp4(P,V,C)
    00300     #define __TBB_FetchAndAddW(P,V)         __TBB_machine_fetchadd4(P,V)
    00301     #define __TBB_FetchAndStoreW(P,V)       __TBB_machine_fetchstore4(P,V)
    00302 #elif  __TBB_WORDSIZE==8
    00303     #if __TBB_USE_GENERIC_DWORD_LOAD_STORE || __TBB_USE_GENERIC_DWORD_FETCH_ADD || __TBB_USE_GENERIC_DWORD_FETCH_STORE
    00304         #error These macros should only be used on 32-bit platforms.
    00305     #endif
    00306
    00307     #define __TBB_CompareAndSwapW(P,V,C)    __TBB_machine_cmpswp8(P,V,C)
    00308     #define __TBB_FetchAndAddW(P,V)         __TBB_machine_fetchadd8(P,V)
    00309     #define __TBB_FetchAndStoreW(P,V)       __TBB_machine_fetchstore8(P,V)
    00310 #else /* __TBB_WORDSIZE != 8 */
    00311     #error Unsupported machine word size.
    00312 #endif /* __TBB_WORDSIZE */
    00313
    00314 #ifndef __TBB_Pause
    00315     inline void __TBB_Pause(int32_t) {
    00316         __TBB_Yield();
    00317     }
    00318 #endif
    00319
    00320 namespace tbb {
    00321
    00323 inline void atomic_fence () { __TBB_full_memory_fence(); }
    00324
    00325 namespace internal {
    00326
    00328
    00329 class atomic_backoff : no_copy {
    00331
    00333     static const int32_t LOOPS_BEFORE_YIELD = 16;
    00334     int32_t count;
    00335 public:
    00336     atomic_backoff() : count(1) {}
    00337
    00339     void pause() {
    00340         if( count<=LOOPS_BEFORE_YIELD ) {
    00341             __TBB_Pause(count);
    00342             // Pause twice as long the next time.
    00343             count*=2;
    00344         } else {
    00345             // Pause is so long that we might as well yield CPU to scheduler.
    00346             __TBB_Yield();
    00347         }
    00348     }
    00349
    00350     // pause for a few times and then return false immediately.
    00351     bool bounded_pause() {
    00352         if( count<=LOOPS_BEFORE_YIELD ) {
    00353             __TBB_Pause(count);
    00354             // Pause twice as long the next time.
    00355             count*=2;
    00356             return true;
    00357         } else {
    00358             return false;
    00359         }
    00360     }
    00361
    00362     void reset() {
    00363         count = 1;
    00364     }
    00365 };
    00366
    00368
    00369 template<typename T, typename U>
    00370 void spin_wait_while_eq( const volatile T& location, U value ) {
    00371     atomic_backoff backoff;
    00372     while( location==value ) backoff.pause();
    00373 }
    00374
    00376
    00377 template<typename T, typename U>
    00378 void spin_wait_until_eq( const volatile T& location, const U value ) {
    00379     atomic_backoff backoff;
    00380     while( location!=value ) backoff.pause();
    00381 }
    00382
    00383 //TODO: add static_assert for the requirements stated below
    00384 //TODO: check if it works with signed types
    00385
    00386 // there are following restrictions/limitations for this operation:
    00387 //  - T should be unsigned, otherwise sign propagation will break correctness of bit manipulations.
    00388 //  - T should be integer type of at most 4 bytes, for the casts and calculations to work.
    00389 //      (Together, these rules limit applicability of Masked CAS to uint8_t and uint16_t only,
    00390 //      as it does nothing useful for 4 bytes).
    00391 //  - The operation assumes that the architecture consistently uses either little-endian or big-endian:
    00392 //      it does not support mixed-endian or page-specific bi-endian architectures.
    00393 // This function is the only use of __TBB_BIG_ENDIAN.
    00394 #if (__TBB_BIG_ENDIAN!=-1)
    00395     #if ( __TBB_USE_GENERIC_PART_WORD_CAS)
    00396         #error generic implementation of part-word CAS was explicitly disabled for this configuration
    00397     #endif
    00398 template<typename T>
    00399 inline T __TBB_MaskedCompareAndSwap (volatile T * const ptr, const T value, const T comparand ) {
    00400     struct endianness{ static bool is_big_endian(){
    00401         #ifndef __TBB_BIG_ENDIAN
    00402             const uint32_t probe = 0x03020100;
    00403             return (((const char*)(&probe))[0]==0x03);
    00404         #elif (__TBB_BIG_ENDIAN==0) || (__TBB_BIG_ENDIAN==1)
    00405             return __TBB_BIG_ENDIAN;
    00406         #else
    00407             #error unexpected value of __TBB_BIG_ENDIAN
    00408         #endif
    00409     }};
    00410
    00411     const uint32_t byte_offset            = (uint32_t) ((uintptr_t)ptr & 0x3);
    00412     volatile uint32_t * const aligned_ptr = (uint32_t*)((uintptr_t)ptr - byte_offset );
    00413
    00414     // location of T within uint32_t for a C++ shift operation
    00415     const uint32_t bits_to_shift     = 8*(endianness::is_big_endian() ? (4 - sizeof(T) - (byte_offset)) : byte_offset);
    00416     const uint32_t mask              = (((uint32_t)1<<(sizeof(T)*8)) - 1 )<<bits_to_shift;
    00417     const uint32_t shifted_comparand = ((uint32_t)comparand << bits_to_shift)&mask;
    00418     const uint32_t shifted_value     = ((uint32_t)value     << bits_to_shift)&mask;
    00419
    00420     for(atomic_backoff b;;b.pause()) {
    00421         const uint32_t surroundings  = *aligned_ptr & ~mask ; // reload the aligned_ptr value which might change during the pause
    00422         const uint32_t big_comparand = surroundings | shifted_comparand ;
    00423         const uint32_t big_value     = surroundings | shifted_value     ;
    00424         // __TBB_machine_cmpswp4 presumed to have full fence.
    00425         // Cast shuts up /Wp64 warning
    00426         const uint32_t big_result = (uint32_t)__TBB_machine_cmpswp4( aligned_ptr, big_value, big_comparand );
    00427         if( big_result == big_comparand                    // CAS succeeded
    00428           || ((big_result ^ big_comparand) & mask) != 0)   // CAS failed and the bits of interest have changed
    00429         {
    00430             return T((big_result & mask) >> bits_to_shift);
    00431         }
    00432         else continue;                                     // CAS failed but the bits of interest left unchanged
    00433     }
    00434 }
    00435 #endif
    00436 template<size_t S, typename T>
    00437 inline T __TBB_CompareAndSwapGeneric (volatile void *ptr, T value, T comparand );
    00438
    00439 template<>
    00440 inline uint8_t __TBB_CompareAndSwapGeneric <1,uint8_t> (volatile void *ptr, uint8_t value, uint8_t comparand ) {
    00441 #if __TBB_USE_GENERIC_PART_WORD_CAS
    00442     return __TBB_MaskedCompareAndSwap<uint8_t>((volatile uint8_t *)ptr,value,comparand);
    00443 #else
    00444     return __TBB_machine_cmpswp1(ptr,value,comparand);
    00445 #endif
    00446 }
    00447
    00448 template<>
    00449 inline uint16_t __TBB_CompareAndSwapGeneric <2,uint16_t> (volatile void *ptr, uint16_t value, uint16_t comparand ) {
    00450 #if __TBB_USE_GENERIC_PART_WORD_CAS
    00451     return __TBB_MaskedCompareAndSwap<uint16_t>((volatile uint16_t *)ptr,value,comparand);
    00452 #else
    00453     return __TBB_machine_cmpswp2(ptr,value,comparand);
    00454 #endif
    00455 }
    00456
    00457 template<>
    00458 inline uint32_t __TBB_CompareAndSwapGeneric <4,uint32_t> (volatile void *ptr, uint32_t value, uint32_t comparand ) {
    00459     // Cast shuts up /Wp64 warning
    00460     return (uint32_t)__TBB_machine_cmpswp4(ptr,value,comparand);
    00461 }
    00462
    00463 #if __TBB_64BIT_ATOMICS
    00464 template<>
    00465 inline uint64_t __TBB_CompareAndSwapGeneric <8,uint64_t> (volatile void *ptr, uint64_t value, uint64_t comparand ) {
    00466     return __TBB_machine_cmpswp8(ptr,value,comparand);
    00467 }
    00468 #endif
    00469
    00470 template<size_t S, typename T>
    00471 inline T __TBB_FetchAndAddGeneric (volatile void *ptr, T addend) {
    00472     atomic_backoff b;
    00473     T result;
    00474     for(;;) {
    00475         result = *reinterpret_cast<volatile T *>(ptr);
    00476         // __TBB_CompareAndSwapGeneric presumed to have full fence.
    00477         if( __TBB_CompareAndSwapGeneric<S,T> ( ptr, result+addend, result )==result )
    00478             break;
    00479         b.pause();
    00480     }
    00481     return result;
    00482 }
    00483
    00484 template<size_t S, typename T>
    00485 inline T __TBB_FetchAndStoreGeneric (volatile void *ptr, T value) {
    00486     atomic_backoff b;
    00487     T result;
    00488     for(;;) {
    00489         result = *reinterpret_cast<volatile T *>(ptr);
    00490         // __TBB_CompareAndSwapGeneric presumed to have full fence.
    00491         if( __TBB_CompareAndSwapGeneric<S,T> ( ptr, value, result )==result )
    00492             break;
    00493         b.pause();
    00494     }
    00495     return result;
    00496 }
    00497
    00498 #if __TBB_USE_GENERIC_PART_WORD_CAS
    00499 #define __TBB_machine_cmpswp1 tbb::internal::__TBB_CompareAndSwapGeneric<1,uint8_t>
    00500 #define __TBB_machine_cmpswp2 tbb::internal::__TBB_CompareAndSwapGeneric<2,uint16_t>
    00501 #endif
    00502
    00503 #if __TBB_USE_GENERIC_FETCH_ADD || __TBB_USE_GENERIC_PART_WORD_FETCH_ADD
    00504 #define __TBB_machine_fetchadd1 tbb::internal::__TBB_FetchAndAddGeneric<1,uint8_t>
    00505 #define __TBB_machine_fetchadd2 tbb::internal::__TBB_FetchAndAddGeneric<2,uint16_t>
    00506 #endif
    00507
    00508 #if __TBB_USE_GENERIC_FETCH_ADD
    00509 #define __TBB_machine_fetchadd4 tbb::internal::__TBB_FetchAndAddGeneric<4,uint32_t>
    00510 #endif
    00511
    00512 #if __TBB_USE_GENERIC_FETCH_ADD || __TBB_USE_GENERIC_DWORD_FETCH_ADD
    00513 #define __TBB_machine_fetchadd8 tbb::internal::__TBB_FetchAndAddGeneric<8,uint64_t>
    00514 #endif
    00515
    00516 #if __TBB_USE_GENERIC_FETCH_STORE || __TBB_USE_GENERIC_PART_WORD_FETCH_STORE
    00517 #define __TBB_machine_fetchstore1 tbb::internal::__TBB_FetchAndStoreGeneric<1,uint8_t>
    00518 #define __TBB_machine_fetchstore2 tbb::internal::__TBB_FetchAndStoreGeneric<2,uint16_t>
    00519 #endif
    00520
    00521 #if __TBB_USE_GENERIC_FETCH_STORE
    00522 #define __TBB_machine_fetchstore4 tbb::internal::__TBB_FetchAndStoreGeneric<4,uint32_t>
    00523 #endif
    00524
    00525 #if __TBB_USE_GENERIC_FETCH_STORE || __TBB_USE_GENERIC_DWORD_FETCH_STORE
    00526 #define __TBB_machine_fetchstore8 tbb::internal::__TBB_FetchAndStoreGeneric<8,uint64_t>
    00527 #endif
    00528
    00529 #if __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE
    00530 #define __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(S)                                             \
    00531     atomic_selector<S>::word atomic_selector<S>::fetch_store ( volatile void* location, word value ) {  \
    00532         return __TBB_machine_fetchstore##S( location, value );                                          \
    00533     }
    00534
    00535 __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(1)
    00536 __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(2)
    00537 __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(4)
    00538 __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(8)
    00539
    00540 #undef __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE
    00541 #endif /* __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE */
    00542
    00543 #if __TBB_USE_GENERIC_DWORD_LOAD_STORE
    00544 inline void __TBB_machine_store8 (volatile void *ptr, int64_t value) {
    00545     for(;;) {
    00546         int64_t result = *(int64_t *)ptr;
    00547         if( __TBB_machine_cmpswp8(ptr,value,result)==result ) break;
    00548     }
    00549 }
    00550
    00551 inline int64_t __TBB_machine_load8 (const volatile void *ptr) {
    00552     // Comparand and new value may be anything, they only must be equal, and
    00553     // the value should have a low probability to be actually found in 'location'.
    00554     const int64_t anyvalue = 2305843009213693951LL;
    00555     return __TBB_machine_cmpswp8(const_cast<volatile void *>(ptr),anyvalue,anyvalue);
    00556 }
    00557 #endif /* __TBB_USE_GENERIC_DWORD_LOAD_STORE */
    00558
    00559 #if __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE
    00560
    00566 template <typename T, size_t S>
    00567 struct machine_load_store {
    00568     static T load_with_acquire ( const volatile T& location ) {
    00569         T to_return = location;
    00570         __TBB_acquire_consistency_helper();
    00571         return to_return;
    00572     }
    00573     static void store_with_release ( volatile T &location, T value ) {
    00574         __TBB_release_consistency_helper();
    00575         location = value;
    00576     }
    00577 };
    00578
    00579 //in general, plain load and store of 32bit compiler is not atomic for 64bit types
    00580 #if __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS
    00581 template <typename T>
    00582 struct machine_load_store<T,8> {
    00583     static T load_with_acquire ( const volatile T& location ) {
    00584         return (T)__TBB_machine_load8( (const volatile void*)&location );
    00585     }
    00586     static void store_with_release ( volatile T& location, T value ) {
    00587         __TBB_machine_store8( (volatile void*)&location, (int64_t)value );
    00588     }
    00589 };
    00590 #endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */
    00591 #endif /* __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE */
    00592
    00593 #if __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE
    00594 template <typename T, size_t S>
    00595 struct machine_load_store_seq_cst {
    00596     static T load ( const volatile T& location ) {
    00597         __TBB_full_memory_fence();
    00598         return machine_load_store<T,S>::load_with_acquire( location );
    00599     }
    00600 #if __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE
    00601     static void store ( volatile T &location, T value ) {
    00602         atomic_selector<S>::fetch_store( (volatile void*)&location, (typename atomic_selector<S>::word)value );
    00603     }
    00604 #else /* !__TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE */
    00605     static void store ( volatile T &location, T value ) {
    00606         machine_load_store<T,S>::store_with_release( location, value );
    00607         __TBB_full_memory_fence();
    00608     }
    00609 #endif /* !__TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE */
    00610 };
    00611
    00612 #if __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS
    00613
    00615 template <typename T>
    00616 struct machine_load_store_seq_cst<T,8> {
    00617     static T load ( const volatile T& location ) {
    00618         // Comparand and new value may be anything, they only must be equal, and
    00619         // the value should have a low probability to be actually found in 'location'.
    00620         const int64_t anyvalue = 2305843009213693951LL;
    00621         return __TBB_machine_cmpswp8( (volatile void*)const_cast<volatile T*>(&location), anyvalue, anyvalue );
    00622     }
    00623     static void store ( volatile T &location, T value ) {
    00624         int64_t result = (volatile int64_t&)location;
    00625         while ( __TBB_machine_cmpswp8((volatile void*)&location, (int64_t)value, result) != result )
    00626             result = (volatile int64_t&)location;
    00627     }
    00628 };
    00629 #endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */
    00630 #endif /*__TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE */
    00631
    00632 #if __TBB_USE_GENERIC_RELAXED_LOAD_STORE
    00633 // Relaxed operations add volatile qualifier to prevent compiler from optimizing them out.
    00637 template <typename T, size_t S>
    00638 struct machine_load_store_relaxed {
    00639     static inline T load ( const volatile T& location ) {
    00640         return location;
    00641     }
    00642     static inline void store ( volatile T& location, T value ) {
    00643         location = value;
    00644     }
    00645 };
    00646
    00647 #if __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS
    00648 template <typename T>
    00649 struct machine_load_store_relaxed<T,8> {
    00650     static inline T load ( const volatile T& location ) {
    00651         return (T)__TBB_machine_load8( (const volatile void*)&location );
    00652     }
    00653     static inline void store ( volatile T& location, T value ) {
    00654         __TBB_machine_store8( (volatile void*)&location, (int64_t)value );
    00655     }
    00656 };
    00657 #endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */
    00658 #endif /* __TBB_USE_GENERIC_RELAXED_LOAD_STORE */
    00659
    00660 #undef __TBB_WORDSIZE //this macro is forbidden to use outside of atomic machinery
    00661
    00662 template<typename T>
    00663 inline T __TBB_load_with_acquire(const volatile T &location) {
    00664     return machine_load_store<T,sizeof(T)>::load_with_acquire( location );
    00665 }
    00666 template<typename T, typename V>
    00667 inline void __TBB_store_with_release(volatile T& location, V value) {
    00668     machine_load_store<T,sizeof(T)>::store_with_release( location, T(value) );
    00669 }
    00671 inline void __TBB_store_with_release(volatile size_t& location, size_t value) {
    00672     machine_load_store<size_t,sizeof(size_t)>::store_with_release( location, value );
    00673 }
    00674
    00675 template<typename T>
    00676 inline T __TBB_load_full_fence(const volatile T &location) {
    00677     return machine_load_store_seq_cst<T,sizeof(T)>::load( location );
    00678 }
    00679 template<typename T, typename V>
    00680 inline void __TBB_store_full_fence(volatile T& location, V value) {
    00681     machine_load_store_seq_cst<T,sizeof(T)>::store( location, T(value) );
    00682 }
    00684 inline void __TBB_store_full_fence(volatile size_t& location, size_t value) {
    00685     machine_load_store_seq_cst<size_t,sizeof(size_t)>::store( location, value );
    00686 }
    00687
    00688 template<typename T>
    00689 inline T __TBB_load_relaxed (const volatile T& location) {
    00690     return machine_load_store_relaxed<T,sizeof(T)>::load( const_cast<T&>(location) );
    00691 }
    00692 template<typename T, typename V>
    00693 inline void __TBB_store_relaxed ( volatile T& location, V value ) {
    00694     machine_load_store_relaxed<T,sizeof(T)>::store( const_cast<T&>(location), T(value) );
    00695 }
    00697 inline void __TBB_store_relaxed ( volatile size_t& location, size_t value ) {
    00698     machine_load_store_relaxed<size_t,sizeof(size_t)>::store( const_cast<size_t&>(location), value );
    00699 }
    00700
    00701 // Macro __TBB_TypeWithAlignmentAtLeastAsStrict(T) should be a type with alignment at least as
    00702 // strict as type T.  The type should have a trivial default constructor and destructor, so that
    00703 // arrays of that type can be declared without initializers.
    00704 // It is correct (but perhaps a waste of space) if __TBB_TypeWithAlignmentAtLeastAsStrict(T) expands
    00705 // to a type bigger than T.
    00706 // The default definition here works on machines where integers are naturally aligned and the
    00707 // strictest alignment is 64.
    00708 #ifndef __TBB_TypeWithAlignmentAtLeastAsStrict
    00709
    00710 #if __TBB_ATTRIBUTE_ALIGNED_PRESENT
    00711
    00712 #define __TBB_DefineTypeWithAlignment(PowerOf2)       \
    00713 struct __TBB_machine_type_with_alignment_##PowerOf2 { \
    00714     uint32_t member[PowerOf2/sizeof(uint32_t)];       \
    00715 } __attribute__((aligned(PowerOf2)));
    00716 #define __TBB_alignof(T) __alignof__(T)
    00717
    00718 #elif __TBB_DECLSPEC_ALIGN_PRESENT
    00719
    00720 #define __TBB_DefineTypeWithAlignment(PowerOf2)       \
    00721 __declspec(align(PowerOf2))                           \
    00722 struct __TBB_machine_type_with_alignment_##PowerOf2 { \
    00723     uint32_t member[PowerOf2/sizeof(uint32_t)];       \
    00724 };
    00725 #define __TBB_alignof(T) __alignof(T)
    00726
    00727 #else /* A compiler with unknown syntax for data alignment */
    00728 #error Must define __TBB_TypeWithAlignmentAtLeastAsStrict(T)
    00729 #endif
    00730
    00731 /* Now declare types aligned to useful powers of two */
    00732 // TODO: Is __TBB_DefineTypeWithAlignment(8) needed on 32 bit platforms?
    00733 __TBB_DefineTypeWithAlignment(16)
    00734 __TBB_DefineTypeWithAlignment(32)
    00735 __TBB_DefineTypeWithAlignment(64)
    00736
    00737 typedef __TBB_machine_type_with_alignment_64 __TBB_machine_type_with_strictest_alignment;
    00738
    00739 // Primary template is a declaration of incomplete type so that it fails with unknown alignments
    00740 template<size_t N> struct type_with_alignment;
    00741
    00742 // Specializations for allowed alignments
    00743 template<> struct type_with_alignment<1> { char member; };
    00744 template<> struct type_with_alignment<2> { uint16_t member; };
    00745 template<> struct type_with_alignment<4> { uint32_t member; };
    00746 template<> struct type_with_alignment<8> { uint64_t member; };
    00747 template<> struct type_with_alignment<16> {__TBB_machine_type_with_alignment_16 member; };
    00748 template<> struct type_with_alignment<32> {__TBB_machine_type_with_alignment_32 member; };
    00749 template<> struct type_with_alignment<64> {__TBB_machine_type_with_alignment_64 member; };
    00750
    00751 #if __TBB_ALIGNOF_NOT_INSTANTIATED_TYPES_BROKEN
    00753
    00755 template<size_t Size, typename T>
    00756 struct work_around_alignment_bug {
    00757     static const size_t alignment = __TBB_alignof(T);
    00758 };
    00759 #define __TBB_TypeWithAlignmentAtLeastAsStrict(T) tbb::internal::type_with_alignment<tbb::internal::work_around_alignment_bug<sizeof(T),T>::alignment>
    00760 #else
    00761 #define __TBB_TypeWithAlignmentAtLeastAsStrict(T) tbb::internal::type_with_alignment<__TBB_alignof(T)>
    00762 #endif  /* __TBB_ALIGNOF_NOT_INSTANTIATED_TYPES_BROKEN */
    00763
    00764 #endif  /* __TBB_TypeWithAlignmentAtLeastAsStrict */
    00765
    00766 // Template class here is to avoid instantiation of the static data for modules that don't use it
    00767 template<typename T>
    00768 struct reverse {
    00769     static const T byte_table[256];
    00770 };
    00771 // An efficient implementation of the reverse function utilizes a 2^8 lookup table holding the bit-reversed
    00772 // values of [0..2^8 - 1]. Those values can also be computed on the fly at a slightly higher cost.
    00773 template<typename T>
    00774 const T reverse<T>::byte_table[256] = {
    00775     0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0, 0x10, 0x90, 0x50, 0xD0, 0x30, 0xB0, 0x70, 0xF0,
    00776     0x08, 0x88, 0x48, 0xC8, 0x28, 0xA8, 0x68, 0xE8, 0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, 0x78, 0xF8,
    00777     0x04, 0x84, 0x44, 0xC4, 0x24, 0xA4, 0x64, 0xE4, 0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4,
    00778     0x0C, 0x8C, 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC, 0x1C, 0x9C, 0x5C, 0xDC, 0x3C, 0xBC, 0x7C, 0xFC,
    00779     0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, 0x62, 0xE2, 0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2,
    00780     0x0A, 0x8A, 0x4A, 0xCA, 0x2A, 0xAA, 0x6A, 0xEA, 0x1A, 0x9A, 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA,
    00781     0x06, 0x86, 0x46, 0xC6, 0x26, 0xA6, 0x66, 0xE6, 0x16, 0x96, 0x56, 0xD6, 0x36, 0xB6, 0x76, 0xF6,
    00782     0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE, 0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE,
    00783     0x01, 0x81, 0x41, 0xC1, 0x21, 0xA1, 0x61, 0xE1, 0x11, 0x91, 0x51, 0xD1, 0x31, 0xB1, 0x71, 0xF1,
    00784     0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, 0x69, 0xE9, 0x19, 0x99, 0x59, 0xD9, 0x39, 0xB9, 0x79, 0xF9,
    00785     0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5, 0x15, 0x95, 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5,
    00786     0x0D, 0x8D, 0x4D, 0xCD, 0x2D, 0xAD, 0x6D, 0xED, 0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, 0x7D, 0xFD,
    00787     0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3, 0x13, 0x93, 0x53, 0xD3, 0x33, 0xB3, 0x73, 0xF3,
    00788     0x0B, 0x8B, 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB, 0x1B, 0x9B, 0x5B, 0xDB, 0x3B, 0xBB, 0x7B, 0xFB,
    00789     0x07, 0x87, 0x47, 0xC7, 0x27, 0xA7, 0x67, 0xE7, 0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7,
    00790     0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF, 0x1F, 0x9F, 0x5F, 0xDF, 0x3F, 0xBF, 0x7F, 0xFF
    00791 };
    00792
    00793 } // namespace internal
    00794 } // namespace tbb
    00795
    00796 // Preserving access to legacy APIs
    00797 using tbb::internal::__TBB_load_with_acquire;
    00798 using tbb::internal::__TBB_store_with_release;
    00799
    00800 // Mapping historically used names to the ones expected by atomic_load_store_traits
    00801 #define __TBB_load_acquire  __TBB_load_with_acquire
    00802 #define __TBB_store_release __TBB_store_with_release
    00803
    00804 #ifndef __TBB_Log2
    00805 inline intptr_t __TBB_Log2( uintptr_t x ) {
    00806     if( x==0 ) return -1;
    00807     intptr_t result = 0;
    00808     uintptr_t tmp;
    00809
    00810     if( sizeof(x)>4 && (tmp = ((uint64_t)x)>>32)) { x=tmp; result += 32; }
    00811     if( (tmp = x>>16) ) { x=tmp; result += 16; }
    00812     if( (tmp = x>>8) )  { x=tmp; result += 8; }
    00813     if( (tmp = x>>4) )  { x=tmp; result += 4; }
    00814     if( (tmp = x>>2) )  { x=tmp; result += 2; }
    00815     return (x&2)? result+1: result;
    00816 }
    00817 #endif
    00818
    00819 #ifndef __TBB_AtomicOR
    00820 inline void __TBB_AtomicOR( volatile void *operand, uintptr_t addend ) {
    00821     tbb::internal::atomic_backoff b;
    00822     for(;;) {
    00823         uintptr_t tmp = *(volatile uintptr_t *)operand;
    00824         uintptr_t result = __TBB_CompareAndSwapW(operand, tmp|addend, tmp);
    00825         if( result==tmp ) break;
    00826         b.pause();
    00827     }
    00828 }
    00829 #endif
    00830
    00831 #ifndef __TBB_AtomicAND
    00832 inline void __TBB_AtomicAND( volatile void *operand, uintptr_t addend ) {
    00833     tbb::internal::atomic_backoff b;
    00834     for(;;) {
    00835         uintptr_t tmp = *(volatile uintptr_t *)operand;
    00836         uintptr_t result = __TBB_CompareAndSwapW(operand, tmp&addend, tmp);
    00837         if( result==tmp ) break;
    00838         b.pause();
    00839     }
    00840 }
    00841 #endif
    00842
    00843 #ifndef __TBB_Flag
    00844 typedef unsigned char __TBB_Flag;
    00845 #endif
    00846 typedef __TBB_atomic __TBB_Flag __TBB_atomic_flag;
    00847
    00848 #ifndef __TBB_TryLockByte
    00849 inline bool __TBB_TryLockByte( __TBB_atomic_flag &flag ) {
    00850     return __TBB_machine_cmpswp1(&flag,1,0)==0;
    00851 }
    00852 #endif
    00853
    00854 #ifndef __TBB_LockByte
    00855 inline __TBB_Flag __TBB_LockByte( __TBB_atomic_flag& flag ) {
    00856     if ( !__TBB_TryLockByte(flag) ) {
    00857         tbb::internal::atomic_backoff b;
    00858         do {
    00859             b.pause();
    00860         } while ( !__TBB_TryLockByte(flag) );
    00861     }
    00862     return 0;
    00863 }
    00864 #endif
    00865
    00866 #ifndef  __TBB_UnlockByte
    00867 #define __TBB_UnlockByte __TBB_store_with_release
    00868 #endif
    00869
    00870 #ifndef __TBB_ReverseByte
    00871 inline unsigned char __TBB_ReverseByte(unsigned char src) {
    00872     return tbb::internal::reverse<unsigned char>::byte_table[src];
    00873 }
    00874 #endif
    00875
    00876 template<typename T>
    00877 T __TBB_ReverseBits(T src) {
    00878     T dst;
    00879     unsigned char *original = (unsigned char *) &src;
    00880     unsigned char *reversed = (unsigned char *) &dst;
    00881
    00882     for( int i = sizeof(T)-1; i >= 0; i-- )
    00883         reversed[i] = __TBB_ReverseByte( original[sizeof(T)-i-1] );
    00884
    00885     return dst;
    00886 }
    00887
    00888 #endif /* __TBB_machine_H */

Copyright © 2005-2012 Intel Corporation. All Rights Reserved.

Intel, Pentium, Intel Xeon, Itanium, Intel XScale and VTune are registered trademarks or trademarks of Intel Corporation or its subsidiaries in the United States and other countries.

* Other names and brands may be claimed as the property of others.