- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

# concurrent_hash_map.h

    00001 /*
    00002     Copyright 2005-2012 Intel Corporation.  All Rights Reserved.
    00003
    00004     This file is part of Threading Building Blocks.
    00005
    00006     Threading Building Blocks is free software; you can redistribute it
    00007     and/or modify it under the terms of the GNU General Public License
    00008     version 2 as published by the Free Software Foundation.
    00009
    00010     Threading Building Blocks is distributed in the hope that it will be
    00011     useful, but WITHOUT ANY WARRANTY; without even the implied warranty
    00012     of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    00013     GNU General Public License for more details.
    00014
    00015     You should have received a copy of the GNU General Public License
    00016     along with Threading Building Blocks; if not, write to the Free Software
    00017     Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
    00018
    00019     As a special exception, you may use this file as part of a free software
    00020     library without restriction.  Specifically, if other files instantiate
    00021     templates or use macros or inline functions from this file, or you compile
    00022     this file and link it with other files to produce an executable, this
    00023     file does not by itself cause the resulting executable to be covered by
    00024     the GNU General Public License.  This exception does not however
    00025     invalidate any other reasons why the executable file might be covered by
    00026     the GNU General Public License.
    00027 */
    00028
    00029 #ifndef __TBB_concurrent_hash_map_H
    00030 #define __TBB_concurrent_hash_map_H
    00031
    00032 #include "tbb_stddef.h"
    00033
    00034 #if !TBB_USE_EXCEPTIONS && _MSC_VER
    00035     // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers
    00036     #pragma warning (push)
    00037     #pragma warning (disable: 4530)
    00038 #endif
    00039
    00040 #include <iterator>
    00041 #include <utility>      // Need std::pair
    00042 #include <cstring>      // Need std::memset
    00043
    00044 #if !TBB_USE_EXCEPTIONS && _MSC_VER
    00045     #pragma warning (pop)
    00046 #endif
    00047
    00048 #include "cache_aligned_allocator.h"
    00049 #include "tbb_allocator.h"
    00050 #include "spin_rw_mutex.h"
    00051 #include "atomic.h"
    00052 #include "aligned_space.h"
    00053 #include "tbb_exception.h"
    00054 #include "tbb_profiling.h"
    00055 #include "internal/_concurrent_unordered_impl.h" // Need tbb_hasher
    00056 #if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS
    00057 #include <typeinfo>
    00058 #endif
    00059 #if __TBB_STATISTICS
    00060 #include <stdio.h>
    00061 #endif
    00062
    00063 namespace tbb {
    00064
    00066 template<typename Key>
    00067 struct tbb_hash_compare {
    00068     static size_t hash( const Key& a ) { return tbb_hasher(a); }
    00069     static bool equal( const Key& a, const Key& b ) { return a == b; }
    00070 };
    00071
    00072 namespace interface5 {
    00073
    00074     template<typename Key, typename T, typename HashCompare = tbb_hash_compare<Key>, typename A = tbb_allocator<std::pair<Key, T> > >
    00075     class concurrent_hash_map;
    00076
    00078     namespace internal {
    00079
    00080
    00082     typedef size_t hashcode_t;
    00084     struct hash_map_node_base : tbb::internal::no_copy {
    00086         typedef spin_rw_mutex mutex_t;
    00088         typedef mutex_t::scoped_lock scoped_t;
    00090         hash_map_node_base *next;
    00091         mutex_t mutex;
    00092     };
    00094     static hash_map_node_base *const rehash_req = reinterpret_cast<hash_map_node_base*>(size_t(3));
    00096     static hash_map_node_base *const empty_rehashed = reinterpret_cast<hash_map_node_base*>(size_t(0));
    00098     class hash_map_base {
    00099     public:
    00101         typedef size_t size_type;
    00103         typedef size_t hashcode_t;
    00105         typedef size_t segment_index_t;
    00107         typedef hash_map_node_base node_base;
    00109         struct bucket : tbb::internal::no_copy {
    00111             typedef spin_rw_mutex mutex_t;
    00113             typedef mutex_t::scoped_lock scoped_t;
    00114             mutex_t mutex;
    00115             node_base *node_list;
    00116         };
    00118         static size_type const embedded_block = 1;
    00120         static size_type const embedded_buckets = 1<<embedded_block;
    00122         static size_type const first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096
    00124         static size_type const pointers_per_table = sizeof(segment_index_t) * 8; // one segment per bit
    00126         typedef bucket *segment_ptr_t;
    00128         typedef segment_ptr_t segments_table_t[pointers_per_table];
    00130         atomic<hashcode_t> my_mask;
    00132         segments_table_t my_table;
    00134         atomic<size_type> my_size; // It must be in separate cache line from my_mask due to performance effects
    00136         bucket my_embedded_segment[embedded_buckets];
    00137 #if __TBB_STATISTICS
    00138         atomic<unsigned> my_info_resizes; // concurrent ones
    00139         mutable atomic<unsigned> my_info_restarts; // race collisions
    00140         atomic<unsigned> my_info_rehashes;  // invocations of rehash_bucket
    00141 #endif
    00143         hash_map_base() {
    00144             std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128   or 64*8=512
    00145                 + sizeof(my_size) + sizeof(my_mask)  // 4+4 or 8+8
    00146                 + embedded_buckets*sizeof(bucket) ); // n*8 or n*16
    00147             for( size_type i = 0; i < embedded_block; i++ ) // fill the table
    00148                 my_table[i] = my_embedded_segment + segment_base(i);
    00149             my_mask = embedded_buckets - 1;
    00150             __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks");
    00151 #if __TBB_STATISTICS
    00152             my_info_resizes = 0; // concurrent ones
    00153             my_info_restarts = 0; // race collisions
    00154             my_info_rehashes = 0;  // invocations of rehash_bucket
    00155 #endif
    00156         }
    00157
    00159         static segment_index_t segment_index_of( size_type index ) {
    00160             return segment_index_t( __TBB_Log2( index|1 ) );
    00161         }
    00162
    00164         static segment_index_t segment_base( segment_index_t k ) {
    00165             return (segment_index_t(1)<<k & ~segment_index_t(1));
    00166         }
    00167
    00169         static size_type segment_size( segment_index_t k ) {
    00170             return size_type(1)<<k; // fake value for k==0
    00171         }
    00172
    00174         static bool is_valid( void *ptr ) {
    00175             return reinterpret_cast<uintptr_t>(ptr) > uintptr_t(63);
    00176         }
    00177
    00179         static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) {
    00180             if( is_initial ) std::memset(ptr, 0, sz*sizeof(bucket) );
    00181             else for(size_type i = 0; i < sz; i++, ptr++) {
    00182                 *reinterpret_cast<intptr_t*>(&ptr->mutex) = 0;
    00183                 ptr->node_list = rehash_req;
    00184             }
    00185         }
    00186
    00188         static void add_to_bucket( bucket *b, node_base *n ) {
    00189             __TBB_ASSERT(b->node_list != rehash_req, NULL);
    00190             n->next = b->node_list;
    00191             b->node_list = n; // its under lock and flag is set
    00192         }
    00193
    00195         struct enable_segment_failsafe : tbb::internal::no_copy {
    00196             segment_ptr_t *my_segment_ptr;
    00197             enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {}
    00198             ~enable_segment_failsafe() {
    00199                 if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress
    00200             }
    00201         };
    00202
    00204         void enable_segment( segment_index_t k, bool is_initial = false ) {
    00205             __TBB_ASSERT( k, "Zero segment must be embedded" );
    00206             enable_segment_failsafe watchdog( my_table, k );
    00207             cache_aligned_allocator<bucket> alloc;
    00208             size_type sz;
    00209             __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment");
    00210             if( k >= first_block ) {
    00211                 sz = segment_size( k );
    00212                 segment_ptr_t ptr = alloc.allocate( sz );
    00213                 init_buckets( ptr, sz, is_initial );
    00214                 itt_hide_store_word( my_table[k], ptr );
    00215                 sz <<= 1;// double it to get entire capacity of the container
    00216             } else { // the first block
    00217                 __TBB_ASSERT( k == embedded_block, "Wrong segment index" );
    00218                 sz = segment_size( first_block );
    00219                 segment_ptr_t ptr = alloc.allocate( sz - embedded_buckets );
    00220                 init_buckets( ptr, sz - embedded_buckets, is_initial );
    00221                 ptr -= segment_base(embedded_block);
    00222                 for(segment_index_t i = embedded_block; i < first_block; i++) // calc the offsets
    00223                     itt_hide_store_word( my_table[i], ptr + segment_base(i) );
    00224             }
    00225             itt_store_word_with_release( my_mask, sz-1 );
    00226             watchdog.my_segment_ptr = 0;
    00227         }
    00228
    00230         bucket *get_bucket( hashcode_t h ) const throw() { // TODO: add throw() everywhere?
    00231             segment_index_t s = segment_index_of( h );
    00232             h -= segment_base(s);
    00233             segment_ptr_t seg = my_table[s];
    00234             __TBB_ASSERT( is_valid(seg), "hashcode must be cut by valid mask for allocated segments" );
    00235             return &seg[h];
    00236         }
    00237
    00238         // internal serial rehashing helper
    00239         void mark_rehashed_levels( hashcode_t h ) throw () {
    00240             segment_index_t s = segment_index_of( h );
    00241             while( segment_ptr_t seg = my_table[++s] )
    00242                 if( seg[h].node_list == rehash_req ) {
    00243                     seg[h].node_list = empty_rehashed;
    00244                     mark_rehashed_levels( h + ((hashcode_t)1<<s) ); // optimized segment_base(s)
    00245                 }
    00246         }
    00247
    00249         // Splitting into two functions should help inlining
    00250         inline bool check_mask_race( const hashcode_t h, hashcode_t &m ) const {
    00251             hashcode_t m_now, m_old = m;
    00252             m_now = (hashcode_t) itt_load_word_with_acquire( my_mask );
    00253             if( m_old != m_now )
    00254                 return check_rehashing_collision( h, m_old, m = m_now );
    00255             return false;
    00256         }
    00257
    00259         bool check_rehashing_collision( const hashcode_t h, hashcode_t m_old, hashcode_t m ) const {
    00260             __TBB_ASSERT(m_old != m, NULL); // TODO?: m arg could be optimized out by passing h = h&m
    00261             if( (h & m_old) != (h & m) ) { // mask changed for this hashcode, rare event
    00262                 // condition above proves that 'h' has some other bits set beside 'm_old'
    00263                 // find next applicable mask after m_old    //TODO: look at bsl instruction
    00264                 for( ++m_old; !(h & m_old); m_old <<= 1 ) // at maximum few rounds depending on the first block size
    00265                     ;
    00266                 m_old = (m_old<<1) - 1; // get full mask from a bit
    00267                 __TBB_ASSERT((m_old&(m_old+1))==0 && m_old <= m, NULL);
    00268                 // check whether it is rehashing/ed
    00269                 if( itt_load_word_with_acquire(get_bucket(h & m_old)->node_list) != rehash_req )
    00270                 {
    00271 #if __TBB_STATISTICS
    00272                     my_info_restarts++; // race collisions
    00273 #endif
    00274                     return true;
    00275                 }
    00276             }
    00277             return false;
    00278         }
    00279
    00281         segment_index_t insert_new_node( bucket *b, node_base *n, hashcode_t mask ) {
    00282             size_type sz = ++my_size; // prefix form is to enforce allocation after the first item inserted
    00283             add_to_bucket( b, n );
    00284             // check load factor
    00285             if( sz >= mask ) { // TODO: add custom load_factor
    00286                 segment_index_t new_seg = __TBB_Log2( mask+1 ); //optimized segment_index_of
    00287                 __TBB_ASSERT( is_valid(my_table[new_seg-1]), "new allocations must not publish new mask until segment has allocated");
    00288                 if( !itt_hide_load_word(my_table[new_seg])
    00289                   && __TBB_CompareAndSwapW(&my_table[new_seg], 2, 0) == 0 )
    00290                     return new_seg; // The value must be processed
    00291             }
    00292             return 0;
    00293         }
    00294
    00296         void reserve(size_type buckets) {
    00297             if( !buckets-- ) return;
    00298             bool is_initial = !my_size;
    00299             for( size_type m = my_mask; buckets > m; m = my_mask )
    00300                 enable_segment( segment_index_of( m+1 ), is_initial );
    00301         }
    00303         void internal_swap(hash_map_base &table) {
    00304             std::swap(this->my_mask, table.my_mask);
    00305             std::swap(this->my_size, table.my_size);
    00306             for(size_type i = 0; i < embedded_buckets; i++)
    00307                 std::swap(this->my_embedded_segment[i].node_list, table.my_embedded_segment[i].node_list);
    00308             for(size_type i = embedded_block; i < pointers_per_table; i++)
    00309                 std::swap(this->my_table[i], table.my_table[i]);
    00310         }
    00311     };
    00312
    00313     template<typename Iterator>
    00314     class hash_map_range;
    00315
    00317
    00319     template<typename Container, typename Value>
    00320     class hash_map_iterator
    00321         : public std::iterator<std::forward_iterator_tag,Value>
    00322     {
    00323         typedef Container map_type;
    00324         typedef typename Container::node node;
    00325         typedef hash_map_base::node_base node_base;
    00326         typedef hash_map_base::bucket bucket;
    00327
    00328         template<typename C, typename T, typename U>
    00329         friend bool operator==( const hash_map_iterator<C,T>& i, const hash_map_iterator<C,U>& j );
    00330
    00331         template<typename C, typename T, typename U>
    00332         friend bool operator!=( const hash_map_iterator<C,T>& i, const hash_map_iterator<C,U>& j );
    00333
    00334         template<typename C, typename T, typename U>
    00335         friend ptrdiff_t operator-( const hash_map_iterator<C,T>& i, const hash_map_iterator<C,U>& j );
    00336
    00337         template<typename C, typename U>
    00338         friend class hash_map_iterator;
    00339
    00340         template<typename I>
    00341         friend class hash_map_range;
    00342
    00343         void advance_to_next_bucket() { // TODO?: refactor to iterator_base class
    00344             size_t k = my_index+1;
    00345             while( my_bucket && k <= my_map->my_mask ) {
    00346                 // Following test uses 2's-complement wizardry
    00347                 if( k& (k-2) ) // not the beginning of a segment
    00348                     ++my_bucket;
    00349                 else my_bucket = my_map->get_bucket( k );
    00350                 my_node = static_cast<node*>( my_bucket->node_list );
    00351                 if( hash_map_base::is_valid(my_node) ) {
    00352                     my_index = k; return;
    00353                 }
    00354                 ++k;
    00355             }
    00356             my_bucket = 0; my_node = 0; my_index = k; // the end
    00357         }
    00358 #if !defined(_MSC_VER) || defined(__INTEL_COMPILER)
    00359         template<typename Key, typename T, typename HashCompare, typename A>
    00360         friend class interface5::concurrent_hash_map;
    00361 #else
    00362     public: // workaround
    00363 #endif
    00365         const Container *my_map;
    00366
    00368         size_t my_index;
    00369
    00371         const bucket *my_bucket;
    00372
    00374         node *my_node;
    00375
    00376         hash_map_iterator( const Container &map, size_t index, const bucket *b, node_base *n );
    00377
    00378     public:
    00380         hash_map_iterator() {}
    00381         hash_map_iterator( const hash_map_iterator<Container,typename Container::value_type> &other ) :
    00382             my_map(other.my_map),
    00383             my_index(other.my_index),
    00384             my_bucket(other.my_bucket),
    00385             my_node(other.my_node)
    00386         {}
    00387         Value& operator*() const {
    00388             __TBB_ASSERT( hash_map_base::is_valid(my_node), "iterator uninitialized or at end of container?" );
    00389             return my_node->item;
    00390         }
    00391         Value* operator->() const {return &operator*();}
    00392         hash_map_iterator& operator++();
    00393
    00395         hash_map_iterator operator++(int) {
    00396             hash_map_iterator old(*this);
    00397             operator++();
    00398             return old;
    00399         }
    00400     };
    00401
    00402     template<typename Container, typename Value>
    00403     hash_map_iterator<Container,Value>::hash_map_iterator( const Container &map, size_t index, const bucket *b, node_base *n ) :
    00404         my_map(&map),
    00405         my_index(index),
    00406         my_bucket(b),
    00407         my_node( static_cast<node*>(n) )
    00408     {
    00409         if( b && !hash_map_base::is_valid(n) )
    00410             advance_to_next_bucket();
    00411     }
    00412
    00413     template<typename Container, typename Value>
    00414     hash_map_iterator<Container,Value>& hash_map_iterator<Container,Value>::operator++() {
    00415         my_node = static_cast<node*>( my_node->next );
    00416         if( !my_node ) advance_to_next_bucket();
    00417         return *this;
    00418     }
    00419
    00420     template<typename Container, typename T, typename U>
    00421     bool operator==( const hash_map_iterator<Container,T>& i, const hash_map_iterator<Container,U>& j ) {
    00422         return i.my_node == j.my_node && i.my_map == j.my_map;
    00423     }
    00424
    00425     template<typename Container, typename T, typename U>
    00426     bool operator!=( const hash_map_iterator<Container,T>& i, const hash_map_iterator<Container,U>& j ) {
    00427         return i.my_node != j.my_node || i.my_map != j.my_map;
    00428     }
    00429
    00431
    00432     template<typename Iterator>
    00433     class hash_map_range {
    00434         typedef typename Iterator::map_type map_type;
    00435         Iterator my_begin;
    00436         Iterator my_end;
    00437         mutable Iterator my_midpoint;
    00438         size_t my_grainsize;
    00440         void set_midpoint() const;
    00441         template<typename U> friend class hash_map_range;
    00442     public:
    00444         typedef std::size_t size_type;
    00445         typedef typename Iterator::value_type value_type;
    00446         typedef typename Iterator::reference reference;
    00447         typedef typename Iterator::difference_type difference_type;
    00448         typedef Iterator iterator;
    00449
    00451         bool empty() const {return my_begin==my_end;}
    00452
    00454         bool is_divisible() const {
    00455             return my_midpoint!=my_end;
    00456         }
    00458         hash_map_range( hash_map_range& r, split ) :
    00459             my_end(r.my_end),
    00460             my_grainsize(r.my_grainsize)
    00461         {
    00462             r.my_end = my_begin = r.my_midpoint;
    00463             __TBB_ASSERT( !empty(), "Splitting despite the range is not divisible" );
    00464             __TBB_ASSERT( !r.empty(), "Splitting despite the range is not divisible" );
    00465             set_midpoint();
    00466             r.set_midpoint();
    00467         }
    00469         template<typename U>
    00470         hash_map_range( hash_map_range<U>& r) :
    00471             my_begin(r.my_begin),
    00472             my_end(r.my_end),
    00473             my_midpoint(r.my_midpoint),
    00474             my_grainsize(r.my_grainsize)
    00475         {}
    00476 #if TBB_DEPRECATED
    00478         hash_map_range( const Iterator& begin_, const Iterator& end_, size_type grainsize_ = 1 ) :
    00479             my_begin(begin_),
    00480             my_end(end_),
    00481             my_grainsize(grainsize_)
    00482         {
    00483             if(!my_end.my_index && !my_end.my_bucket) // end
    00484                 my_end.my_index = my_end.my_map->my_mask + 1;
    00485             set_midpoint();
    00486             __TBB_ASSERT( grainsize_>0, "grainsize must be positive" );
    00487         }
    00488 #endif
    00490         hash_map_range( const map_type &map, size_type grainsize_ = 1 ) :
    00491             my_begin( Iterator( map, 0, map.my_embedded_segment, map.my_embedded_segment->node_list ) ),
    00492             my_end( Iterator( map, map.my_mask + 1, 0, 0 ) ),
    00493             my_grainsize( grainsize_ )
    00494         {
    00495             __TBB_ASSERT( grainsize_>0, "grainsize must be positive" );
    00496             set_midpoint();
    00497         }
    00498         const Iterator& begin() const {return my_begin;}
    00499         const Iterator& end() const {return my_end;}
    00501         size_type grainsize() const {return my_grainsize;}
    00502     };
    00503
    00504     template<typename Iterator>
    00505     void hash_map_range<Iterator>::set_midpoint() const {
    00506         // Split by groups of nodes
    00507         size_t m = my_end.my_index-my_begin.my_index;
    00508         if( m > my_grainsize ) {
    00509             m = my_begin.my_index + m/2u;
    00510             hash_map_base::bucket *b = my_begin.my_map->get_bucket(m);
    00511             my_midpoint = Iterator(*my_begin.my_map,m,b,b->node_list);
    00512         } else {
    00513             my_midpoint = my_end;
    00514         }
    00515         __TBB_ASSERT( my_begin.my_index <= my_midpoint.my_index,
    00516             "my_begin is after my_midpoint" );
    00517         __TBB_ASSERT( my_midpoint.my_index <= my_end.my_index,
    00518             "my_midpoint is after my_end" );
    00519         __TBB_ASSERT( my_begin != my_midpoint || my_begin == my_end,
    00520             "[my_begin, my_midpoint) range should not be empty" );
    00521     }
    00522
    00523     } // internal
    00525
    00527
    00556 template<typename Key, typename T, typename HashCompare, typename Allocator>
    00557 class concurrent_hash_map : protected internal::hash_map_base {
    00558     template<typename Container, typename Value>
    00559     friend class internal::hash_map_iterator;
    00560
    00561     template<typename I>
    00562     friend class internal::hash_map_range;
    00563
    00564 public:
    00565     typedef Key key_type;
    00566     typedef T mapped_type;
    00567     typedef std::pair<const Key,T> value_type;
    00568     typedef hash_map_base::size_type size_type;
    00569     typedef ptrdiff_t difference_type;
    00570     typedef value_type *pointer;
    00571     typedef const value_type *const_pointer;
    00572     typedef value_type &reference;
    00573     typedef const value_type &const_reference;
    00574     typedef internal::hash_map_iterator<concurrent_hash_map,value_type> iterator;
    00575     typedef internal::hash_map_iterator<concurrent_hash_map,const value_type> const_iterator;
    00576     typedef internal::hash_map_range<iterator> range_type;
    00577     typedef internal::hash_map_range<const_iterator> const_range_type;
    00578     typedef Allocator allocator_type;
    00579
    00580 protected:
    00581     friend class const_accessor;
    00582     struct node;
    00583     typedef typename Allocator::template rebind<node>::other node_allocator_type;
    00584     node_allocator_type my_allocator;
    00585     HashCompare my_hash_compare;
    00586
    00587     struct node : public node_base {
    00588         value_type item;
    00589         node( const Key &key ) : item(key, T()) {}
    00590         node( const Key &key, const T &t ) : item(key, t) {}
    00591         // exception-safe allocation, see C++ Standard 2003, clause 5.3.4p17
    00592         void *operator new( size_t /*size*/, node_allocator_type &a ) {
    00593             void *ptr = a.allocate(1);
    00594             if(!ptr)
    00595                 tbb::internal::throw_exception(tbb::internal::eid_bad_alloc);
    00596             return ptr;
    00597         }
    00598         // match placement-new form above to be called if exception thrown in constructor
    00599         void operator delete( void *ptr, node_allocator_type &a ) { a.deallocate(static_cast<node*>(ptr),1); }
    00600     };
    00601
    00602     void delete_node( node_base *n ) {
    00603         my_allocator.destroy( static_cast<node*>(n) );
    00604         my_allocator.deallocate( static_cast<node*>(n), 1);
    00605     }
    00606
    00607     node *search_bucket( const key_type &key, bucket *b ) const {
    00608         node *n = static_cast<node*>( b->node_list );
    00609         while( is_valid(n) && !my_hash_compare.equal(key, n->item.first) )
    00610             n = static_cast<node*>( n->next );
    00611         __TBB_ASSERT(n != internal::rehash_req, "Search can be executed only for rehashed bucket");
    00612         return n;
    00613     }
    00614
    00616     class bucket_accessor : public bucket::scoped_t {
    00617         bucket *my_b;
    00618     public:
    00619         bucket_accessor( concurrent_hash_map *base, const hashcode_t h, bool writer = false ) { acquire( base, h, writer ); }
    00621         inline void acquire( concurrent_hash_map *base, const hashcode_t h, bool writer = false ) {
    00622             my_b = base->get_bucket( h );
    00623             // TODO: actually, notification is unnecessary here, just hiding double-check
    00624             if( itt_load_word_with_acquire(my_b->node_list) == internal::rehash_req
    00625                 && try_acquire( my_b->mutex, /*write=*/true ) )
    00626             {
    00627                 if( my_b->node_list == internal::rehash_req ) base->rehash_bucket( my_b, h ); //recursive rehashing
    00628             }
    00629             else bucket::scoped_t::acquire( my_b->mutex, writer );
    00630             __TBB_ASSERT( my_b->node_list != internal::rehash_req, NULL);
    00631         }
    00633         bool is_writer() { return bucket::scoped_t::is_writer; }
    00635         bucket *operator() () { return my_b; }
    00636     };
    00637
    00638     // TODO refactor to hash_base
    00639     void rehash_bucket( bucket *b_new, const hashcode_t h ) {
    00640         __TBB_ASSERT( *(intptr_t*)(&b_new->mutex), "b_new must be locked (for write)");
    00641         __TBB_ASSERT( h > 1, "The lowermost buckets can't be rehashed" );
    00642         __TBB_store_with_release(b_new->node_list, internal::empty_rehashed); // mark rehashed
    00643         hashcode_t mask = ( 1u<<__TBB_Log2( h ) ) - 1; // get parent mask from the topmost bit
    00644 #if __TBB_STATISTICS
    00645         my_info_rehashes++; // invocations of rehash_bucket
    00646 #endif
    00647
    00648         bucket_accessor b_old( this, h & mask );
    00649
    00650         mask = (mask<<1) | 1; // get full mask for new bucket
    00651         __TBB_ASSERT( (mask&(mask+1))==0 && (h & mask) == h, NULL );
    00652     restart:
    00653         for( node_base **p = &b_old()->node_list, *n = __TBB_load_with_acquire(*p); is_valid(n); n = *p ) {
    00654             hashcode_t c = my_hash_compare.hash( static_cast<node*>(n)->item.first );
    00655 #if TBB_USE_ASSERT
    00656             hashcode_t bmask = h & (mask>>1);
    00657             bmask = bmask==0? 1 : ( 1u<<(__TBB_Log2( bmask )+1 ) ) - 1; // minimal mask of parent bucket
    00658             __TBB_ASSERT( (c & bmask) == (h & bmask), "hash() function changed for key in table" );
    00659 #endif
    00660             if( (c & mask) == h ) {
    00661                 if( !b_old.is_writer() )
    00662                     if( !b_old.upgrade_to_writer() ) {
    00663                         goto restart; // node ptr can be invalid due to concurrent erase
    00664                     }
    00665                 *p = n->next; // exclude from b_old
    00666                 add_to_bucket( b_new, n );
    00667             } else p = &n->next; // iterate to next item
    00668         }
    00669     }
    00670
    00671 public:
    00672
    00673     class accessor;
    00675     class const_accessor : private node::scoped_t /*which derived from no_copy*/ {
    00676         friend class concurrent_hash_map<Key,T,HashCompare,Allocator>;
    00677         friend class accessor;
    00678     public:
    00680         typedef const typename concurrent_hash_map::value_type value_type;
    00681
    00683         bool empty() const {return !my_node;}
    00684
    00686         void release() {
    00687             if( my_node ) {
    00688                 node::scoped_t::release();
    00689                 my_node = 0;
    00690             }
    00691         }
    00692
    00694         const_reference operator*() const {
    00695             __TBB_ASSERT( my_node, "attempt to dereference empty accessor" );
    00696             return my_node->item;
    00697         }
    00698
    00700         const_pointer operator->() const {
    00701             return &operator*();
    00702         }
    00703
    00705         const_accessor() : my_node(NULL) {}
    00706
    00708         ~const_accessor() {
    00709             my_node = NULL; // scoped lock's release() is called in its destructor
    00710         }
    00711     protected:
    00712         bool is_writer() { return node::scoped_t::is_writer; }
    00713         node *my_node;
    00714         hashcode_t my_hash;
    00715     };
    00716
    00718     class accessor: public const_accessor {
    00719     public:
    00721         typedef typename concurrent_hash_map::value_type value_type;
    00722
    00724         reference operator*() const {
    00725             __TBB_ASSERT( this->my_node, "attempt to dereference empty accessor" );
    00726             return this->my_node->item;
    00727         }
    00728
    00730         pointer operator->() const {
    00731             return &operator*();
    00732         }
    00733     };
    00734
    00736     concurrent_hash_map(const allocator_type &a = allocator_type())
    00737         : internal::hash_map_base(), my_allocator(a)
    00738     {}
    00739
    00741     concurrent_hash_map(size_type n, const allocator_type &a = allocator_type())
    00742         : my_allocator(a)
    00743     {
    00744         reserve( n );
    00745     }
    00746
    00748     concurrent_hash_map( const concurrent_hash_map& table, const allocator_type &a = allocator_type())
    00749         : internal::hash_map_base(), my_allocator(a)
    00750     {
    00751         internal_copy(table);
    00752     }
    00753
    00755     template<typename I>
    00756     concurrent_hash_map(I first, I last, const allocator_type &a = allocator_type())
    00757         : my_allocator(a)
    00758     {
    00759         reserve( std::distance(first, last) ); // TODO: load_factor?
    00760         internal_copy(first, last);
    00761     }
    00762
    00764     concurrent_hash_map& operator=( const concurrent_hash_map& table ) {
    00765         if( this!=&table ) {
    00766             clear();
    00767             internal_copy(table);
    00768         }
    00769         return *this;
    00770     }
    00771
    00772
    00774
    00776     void rehash(size_type n = 0);
    00777
    00779     void clear();
    00780
    00782     ~concurrent_hash_map() { clear(); }
    00783
    00784     //------------------------------------------------------------------------
    00785     // Parallel algorithm support
    00786     //------------------------------------------------------------------------
    00787     range_type range( size_type grainsize=1 ) {
    00788         return range_type( *this, grainsize );
    00789     }
    00790     const_range_type range( size_type grainsize=1 ) const {
    00791         return const_range_type( *this, grainsize );
    00792     }
    00793
    00794     //------------------------------------------------------------------------
    00795     // STL support - not thread-safe methods
    00796     //------------------------------------------------------------------------
    00797     iterator begin() {return iterator(*this,0,my_embedded_segment,my_embedded_segment->node_list);}
    00798     iterator end() {return iterator(*this,0,0,0);}
    00799     const_iterator begin() const {return const_iterator(*this,0,my_embedded_segment,my_embedded_segment->node_list);}
    00800     const_iterator end() const {return const_iterator(*this,0,0,0);}
    00801     std::pair<iterator, iterator> equal_range( const Key& key ) { return internal_equal_range(key, end()); }
    00802     std::pair<const_iterator, const_iterator> equal_range( const Key& key ) const { return internal_equal_range(key, end()); }
    00803
    00805     size_type size() const { return my_size; }
    00806
    00808     bool empty() const { return my_size == 0; }
    00809
    00811     size_type max_size() const {return (~size_type(0))/sizeof(node);}
    00812
    00814     size_type bucket_count() const { return my_mask+1; }
    00815
    00817     allocator_type get_allocator() const { return this->my_allocator; }
    00818
    00820     void swap(concurrent_hash_map &table);
    00821
    00822     //------------------------------------------------------------------------
    00823     // concurrent map operations
    00824     //------------------------------------------------------------------------
    00825
    00827     size_type count( const Key &key ) const {
    00828         return const_cast<concurrent_hash_map*>(this)->lookup(/*insert*/false, key, NULL, NULL, /*write=*/false );
    00829     }
    00830
    00832
    00833     bool find( const_accessor &result, const Key &key ) const {
    00834         result.release();
    00835         return const_cast<concurrent_hash_map*>(this)->lookup(/*insert*/false, key, NULL, &result, /*write=*/false );
    00836     }
    00837
    00839
    00840     bool find( accessor &result, const Key &key ) {
    00841         result.release();
    00842         return lookup(/*insert*/false, key, NULL, &result, /*write=*/true );
    00843     }
    00844
    00846
    00847     bool insert( const_accessor &result, const Key &key ) {
    00848         result.release();
    00849         return lookup(/*insert*/true, key, NULL, &result, /*write=*/false );
    00850     }
    00851
    00853
    00854     bool insert( accessor &result, const Key &key ) {
    00855         result.release();
    00856         return lookup(/*insert*/true, key, NULL, &result, /*write=*/true );
    00857     }
    00858
    00860
    00861     bool insert( const_accessor &result, const value_type &value ) {
    00862         result.release();
    00863         return lookup(/*insert*/true, value.first, &value.second, &result, /*write=*/false );
    00864     }
    00865
    00867
    00868     bool insert( accessor &result, const value_type &value ) {
    00869         result.release();
    00870         return lookup(/*insert*/true, value.first, &value.second, &result, /*write=*/true );
    00871     }
    00872
    00874
    00875     bool insert( const value_type &value ) {
    00876         return lookup(/*insert*/true, value.first, &value.second, NULL, /*write=*/false );
    00877     }
    00878
    00880     template<typename I>
    00881     void insert(I first, I last) {
    00882         for(; first != last; ++first)
    00883             insert( *first );
    00884     }
    00885
    00887
    00888     bool erase( const Key& key );
    00889
    00891
    00892     bool erase( const_accessor& item_accessor ) {
    00893         return exclude( item_accessor );
    00894     }
    00895
    00897
    00898     bool erase( accessor& item_accessor ) {
    00899         return exclude( item_accessor );
    00900     }
    00901
    00902 protected:
    00904     bool lookup( bool op_insert, const Key &key, const T *t, const_accessor *result, bool write );
    00905
    00907     bool exclude( const_accessor &item_accessor );
    00908
    00910     template<typename I>
    00911     std::pair<I, I> internal_equal_range( const Key& key, I end ) const;
    00912
    00914     void internal_copy( const concurrent_hash_map& source );
    00915
    00916     template<typename I>
    00917     void internal_copy(I first, I last);
    00918
    00920
    00922     const_pointer internal_fast_find( const Key& key ) const {
    00923         hashcode_t h = my_hash_compare.hash( key );
    00924         hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask );
    00925         node *n;
    00926     restart:
    00927         __TBB_ASSERT((m&(m+1))==0, NULL);
    00928         bucket *b = get_bucket( h & m );
    00929         // TODO: actually, notification is unnecessary here, just hiding double-check
    00930         if( itt_load_word_with_acquire(b->node_list) == internal::rehash_req )
    00931         {
    00932             bucket::scoped_t lock;
    00933             if( lock.try_acquire( b->mutex, /*write=*/true ) ) {
    00934                 if( b->node_list == internal::rehash_req)
    00935                     const_cast<concurrent_hash_map*>(this)->rehash_bucket( b, h & m ); //recursive rehashing
    00936             }
    00937             else lock.acquire( b->mutex, /*write=*/false );
    00938             __TBB_ASSERT(b->node_list!=internal::rehash_req,NULL);
    00939         }
    00940         n = search_bucket( key, b );
    00941         if( n )
    00942             return &n->item;
    00943         else if( check_mask_race( h, m ) )
    00944             goto restart;
    00945         return 0;
    00946     }
    00947 };
    00948
    00949 #if _MSC_VER && !defined(__INTEL_COMPILER)
    00950     // Suppress "conditional expression is constant" warning.
    00951     #pragma warning( push )
    00952     #pragma warning( disable: 4127 )
    00953 #endif
    00954
    00955 template<typename Key, typename T, typename HashCompare, typename A>
    00956 bool concurrent_hash_map<Key,T,HashCompare,A>::lookup( bool op_insert, const Key &key, const T *t, const_accessor *result, bool write ) {
    00957     __TBB_ASSERT( !result || !result->my_node, NULL );
    00958     bool return_value;
    00959     hashcode_t const h = my_hash_compare.hash( key );
    00960     hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask );
    00961     segment_index_t grow_segment = 0;
    00962     node *n, *tmp_n = 0;
    00963     restart:
    00964     {//lock scope
    00965         __TBB_ASSERT((m&(m+1))==0, NULL);
    00966         return_value = false;
    00967         // get bucket
    00968         bucket_accessor b( this, h & m );
    00969
    00970         // find a node
    00971         n = search_bucket( key, b() );
    00972         if( op_insert ) {
    00973             // [opt] insert a key
    00974             if( !n ) {
    00975                 if( !tmp_n ) {
    00976                     if(t) tmp_n = new( my_allocator ) node(key, *t);
    00977                     else  tmp_n = new( my_allocator ) node(key);
    00978                 }
    00979                 if( !b.is_writer() && !b.upgrade_to_writer() ) { // TODO: improved insertion
    00980                     // Rerun search_list, in case another thread inserted the item during the upgrade.
    00981                     n = search_bucket( key, b() );
    00982                     if( is_valid(n) ) { // unfortunately, it did
    00983                         b.downgrade_to_reader();
    00984                         goto exists;
    00985                     }
    00986                 }
    00987                 if( check_mask_race(h, m) )
    00988                     goto restart; // b.release() is done in ~b().
    00989                 // insert and set flag to grow the container
    00990                 grow_segment = insert_new_node( b(), n = tmp_n, m );
    00991                 tmp_n = 0;
    00992                 return_value = true;
    00993             }
    00994         } else { // find or count
    00995             if( !n ) {
    00996                 if( check_mask_race( h, m ) )
    00997                     goto restart; // b.release() is done in ~b(). TODO: replace by continue
    00998                 return false;
    00999             }
    01000             return_value = true;
    01001         }
    01002     exists:
    01003         if( !result ) goto check_growth;
    01004         // TODO: the following seems as generic/regular operation
    01005         // acquire the item
    01006         if( !result->try_acquire( n->mutex, write ) ) {
    01007             // we are unlucky, prepare for longer wait
    01008             tbb::internal::atomic_backoff trials;
    01009             do {
    01010                 if( !trials.bounded_pause() ) {
    01011                     // the wait takes really long, restart the operation
    01012                     b.release();
    01013                     __TBB_ASSERT( !op_insert || !return_value, "Can't acquire new item in locked bucket?" );
    01014                     __TBB_Yield();
    01015                     m = (hashcode_t) itt_load_word_with_acquire( my_mask );
    01016                     goto restart;
    01017                 }
    01018             } while( !result->try_acquire( n->mutex, write ) );
    01019         }
    01020     }//lock scope
    01021     result->my_node = n;
    01022     result->my_hash = h;
    01023 check_growth:
    01024     // [opt] grow the container
    01025     if( grow_segment ) {
    01026 #if __TBB_STATISTICS
    01027         my_info_resizes++; // concurrent ones
    01028 #endif
    01029         enable_segment( grow_segment );
    01030     }
    01031     if( tmp_n ) // if op_insert only
    01032         delete_node( tmp_n );
    01033     return return_value;
    01034 }
    01035
    01036 template<typename Key, typename T, typename HashCompare, typename A>
    01037 template<typename I>
    01038 std::pair<I, I> concurrent_hash_map<Key,T,HashCompare,A>::internal_equal_range( const Key& key, I end_ ) const {
    01039     hashcode_t h = my_hash_compare.hash( key );
    01040     hashcode_t m = my_mask;
    01041     __TBB_ASSERT((m&(m+1))==0, NULL);
    01042     h &= m;
    01043     bucket *b = get_bucket( h );
    01044     while( b->node_list == internal::rehash_req ) {
    01045         m = ( 1u<<__TBB_Log2( h ) ) - 1; // get parent mask from the topmost bit
    01046         b = get_bucket( h &= m );
    01047     }
    01048     node *n = search_bucket( key, b );
    01049     if( !n )
    01050         return std::make_pair(end_, end_);
    01051     iterator lower(*this, h, b, n), upper(lower);
    01052     return std::make_pair(lower, ++upper);
    01053 }
    01054
    01055 template<typename Key, typename T, typename HashCompare, typename A>
    01056 bool concurrent_hash_map<Key,T,HashCompare,A>::exclude( const_accessor &item_accessor ) {
    01057     __TBB_ASSERT( item_accessor.my_node, NULL );
    01058     node_base *const n = item_accessor.my_node;
    01059     hashcode_t const h = item_accessor.my_hash;
    01060     hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask );
    01061     do {
    01062         // get bucket
    01063         bucket_accessor b( this, h & m, /*writer=*/true );
    01064         node_base **p = &b()->node_list;
    01065         while( *p && *p != n )
    01066             p = &(*p)->next;
    01067         if( !*p ) { // someone else was first
    01068             if( check_mask_race( h, m ) )
    01069                 continue;
    01070             item_accessor.release();
    01071             return false;
    01072         }
    01073         __TBB_ASSERT( *p == n, NULL );
    01074         *p = n->next; // remove from container
    01075         my_size--;
    01076         break;
    01077     } while(true);
    01078     if( !item_accessor.is_writer() ) // need to get exclusive lock
    01079         item_accessor.upgrade_to_writer(); // return value means nothing here
    01080     item_accessor.release();
    01081     delete_node( n ); // Only one thread can delete it
    01082     return true;
    01083 }
    01084
    01085 template<typename Key, typename T, typename HashCompare, typename A>
    01086 bool concurrent_hash_map<Key,T,HashCompare,A>::erase( const Key &key ) {
    01087     node_base *n;
    01088     hashcode_t const h = my_hash_compare.hash( key );
    01089     hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask );
    01090 restart:
    01091     {//lock scope
    01092         // get bucket
    01093         bucket_accessor b( this, h & m );
    01094     search:
    01095         node_base **p = &b()->node_list;
    01096         n = *p;
    01097         while( is_valid(n) && !my_hash_compare.equal(key, static_cast<node*>(n)->item.first ) ) {
    01098             p = &n->next;
    01099             n = *p;
    01100         }
    01101         if( !n ) { // not found, but mask could be changed
    01102             if( check_mask_race( h, m ) )
    01103                 goto restart;
    01104             return false;
    01105         }
    01106         else if( !b.is_writer() && !b.upgrade_to_writer() ) {
    01107             if( check_mask_race( h, m ) ) // contended upgrade, check mask
    01108                 goto restart;
    01109             goto search;
    01110         }
    01111         *p = n->next;
    01112         my_size--;
    01113     }
    01114     {
    01115         typename node::scoped_t item_locker( n->mutex, /*write=*/true );
    01116     }
    01117     // note: there should be no threads pretending to acquire this mutex again, do not try to upgrade const_accessor!
    01118     delete_node( n ); // Only one thread can delete it due to write lock on the bucket
    01119     return true;
    01120 }
    01121
    01122 template<typename Key, typename T, typename HashCompare, typename A>
    01123 void concurrent_hash_map<Key,T,HashCompare,A>::swap(concurrent_hash_map<Key,T,HashCompare,A> &table) {
    01124     std::swap(this->my_allocator, table.my_allocator);
    01125     std::swap(this->my_hash_compare, table.my_hash_compare);
    01126     internal_swap(table);
    01127 }
    01128
    01129 template<typename Key, typename T, typename HashCompare, typename A>
    01130 void concurrent_hash_map<Key,T,HashCompare,A>::rehash(size_type sz) {
    01131     reserve( sz ); // TODO: add reduction of number of buckets as well
    01132     hashcode_t mask = my_mask;
    01133     hashcode_t b = (mask+1)>>1; // size or first index of the last segment
    01134     __TBB_ASSERT((b&(b-1))==0, NULL); // zero or power of 2
    01135     bucket *bp = get_bucket( b ); // only the last segment should be scanned for rehashing
    01136     for(; b <= mask; b++, bp++ ) {
    01137         node_base *n = bp->node_list;
    01138         __TBB_ASSERT( is_valid(n) || n == internal::empty_rehashed || n == internal::rehash_req, "Broken internal structure" );
    01139         __TBB_ASSERT( *reinterpret_cast<intptr_t*>(&bp->mutex) == 0, "concurrent or unexpectedly terminated operation during rehash() execution" );
    01140         if( n == internal::rehash_req ) { // rehash bucket, conditional because rehashing of a previous bucket may affect this one
    01141             hashcode_t h = b; bucket *b_old = bp;
    01142             do {
    01143                 __TBB_ASSERT( h > 1, "The lowermost buckets can't be rehashed" );
    01144                 hashcode_t m = ( 1u<<__TBB_Log2( h ) ) - 1; // get parent mask from the topmost bit
    01145                 b_old = get_bucket( h &= m );
    01146             } while( b_old->node_list == internal::rehash_req );
    01147             // now h - is index of the root rehashed bucket b_old
    01148             mark_rehashed_levels( h ); // mark all non-rehashed children recursively across all segments
    01149             for( node_base **p = &b_old->node_list, *q = *p; is_valid(q); q = *p ) {
    01150                 hashcode_t c = my_hash_compare.hash( static_cast<node*>(q)->item.first );
    01151                 if( (c & mask) != h ) { // should be rehashed
    01152                     *p = q->next; // exclude from b_old
    01153                     bucket *b_new = get_bucket( c & mask );
    01154                     __TBB_ASSERT( b_new->node_list != internal::rehash_req, "hash() function changed for key in table or internal error" );
    01155                     add_to_bucket( b_new, q );
    01156                 } else p = &q->next; // iterate to next item
    01157             }
    01158         }
    01159     }
    01160 #if TBB_USE_PERFORMANCE_WARNINGS
    01161     int current_size = int(my_size), buckets = int(mask)+1, empty_buckets = 0, overpopulated_buckets = 0; // usage statistics
    01162     static bool reported = false;
    01163 #endif
    01164 #if TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS
    01165     for( b = 0; b <= mask; b++ ) {// only last segment should be scanned for rehashing
    01166         if( b & (b-2) ) ++bp; // not the beginning of a segment
    01167         else bp = get_bucket( b );
    01168         node_base *n = bp->node_list;
    01169         __TBB_ASSERT( *reinterpret_cast<intptr_t*>(&bp->mutex) == 0, "concurrent or unexpectedly terminated operation during rehash() execution" );
    01170         __TBB_ASSERT( is_valid(n) || n == internal::empty_rehashed, "Broken internal structure" );
    01171 #if TBB_USE_PERFORMANCE_WARNINGS
    01172         if( n == internal::empty_rehashed ) empty_buckets++;
    01173         else if( n->next ) overpopulated_buckets++;
    01174 #endif
    01175 #if TBB_USE_ASSERT
    01176         for( ; is_valid(n); n = n->next ) {
    01177             hashcode_t h = my_hash_compare.hash( static_cast<node*>(n)->item.first ) & mask;
    01178             __TBB_ASSERT( h == b, "hash() function changed for key in table or internal error" );
    01179         }
    01180 #endif
    01181     }
    01182 #endif // TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS
    01183 #if TBB_USE_PERFORMANCE_WARNINGS
    01184     if( buckets > current_size) empty_buckets -= buckets - current_size;
    01185     else overpopulated_buckets -= current_size - buckets; // TODO: load_factor?
    01186     if( !reported && buckets >= 512 && ( 2*empty_buckets > current_size || 2*overpopulated_buckets > current_size ) ) {
    01187         tbb::internal::runtime_warning(
    01188             "Performance is not optimal because the hash function produces bad randomness in lower bits in %s.\nSize: %d  Empties: %d  Overlaps: %d",
    01189             typeid(*this).name(), current_size, empty_buckets, overpopulated_buckets );
    01190         reported = true;
    01191     }
    01192 #endif
    01193 }
    01194
    01195 template<typename Key, typename T, typename HashCompare, typename A>
    01196 void concurrent_hash_map<Key,T,HashCompare,A>::clear() {
    01197     hashcode_t m = my_mask;
    01198     __TBB_ASSERT((m&(m+1))==0, NULL);
    01199 #if TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS
    01200 #if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS
    01201     int current_size = int(my_size), buckets = int(m)+1, empty_buckets = 0, overpopulated_buckets = 0; // usage statistics
    01202     static bool reported = false;
    01203 #endif
    01204     bucket *bp = 0;
    01205     // check consistency
    01206     for( segment_index_t b = 0; b <= m; b++ ) {
    01207         if( b & (b-2) ) ++bp; // not the beginning of a segment
    01208         else bp = get_bucket( b );
    01209         node_base *n = bp->node_list;
    01210         __TBB_ASSERT( is_valid(n) || n == internal::empty_rehashed || n == internal::rehash_req, "Broken internal structure" );
    01211         __TBB_ASSERT( *reinterpret_cast<intptr_t*>(&bp->mutex) == 0, "concurrent or unexpectedly terminated operation during clear() execution" );
    01212 #if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS
    01213         if( n == internal::empty_rehashed ) empty_buckets++;
    01214         else if( n == internal::rehash_req ) buckets--;
    01215         else if( n->next ) overpopulated_buckets++;
    01216 #endif
    01217 #if __TBB_EXTRA_DEBUG
    01218         for(; is_valid(n); n = n->next ) {
    01219             hashcode_t h = my_hash_compare.hash( static_cast<node*>(n)->item.first );
    01220             h &= m;
    01221             __TBB_ASSERT( h == b || get_bucket(h)->node_list == internal::rehash_req, "hash() function changed for key in table or internal error" );
    01222         }
    01223 #endif
    01224     }
    01225 #if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS
    01226 #if __TBB_STATISTICS
    01227     printf( "items=%d buckets: capacity=%d rehashed=%d empty=%d overpopulated=%d"
    01228         " concurrent: resizes=%u rehashes=%u restarts=%u\n",
    01229         current_size, int(m+1), buckets, empty_buckets, overpopulated_buckets,
    01230         unsigned(my_info_resizes), unsigned(my_info_rehashes), unsigned(my_info_restarts) );
    01231     my_info_resizes = 0; // concurrent ones
    01232     my_info_restarts = 0; // race collisions
    01233     my_info_rehashes = 0;  // invocations of rehash_bucket
    01234 #endif
    01235     if( buckets > current_size) empty_buckets -= buckets - current_size;
    01236     else overpopulated_buckets -= current_size - buckets; // TODO: load_factor?
    01237     if( !reported && buckets >= 512 && ( 2*empty_buckets > current_size || 2*overpopulated_buckets > current_size ) ) {
    01238         tbb::internal::runtime_warning(
    01239             "Performance is not optimal because the hash function produces bad randomness in lower bits in %s.\nSize: %d  Empties: %d  Overlaps: %d",
    01240             typeid(*this).name(), current_size, empty_buckets, overpopulated_buckets );
    01241         reported = true;
    01242     }
    01243 #endif
    01244 #endif//TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS
    01245     my_size = 0;
    01246     segment_index_t s = segment_index_of( m );
    01247     __TBB_ASSERT( s+1 == pointers_per_table || !my_table[s+1], "wrong mask or concurrent grow" );
    01248     cache_aligned_allocator<bucket> alloc;
    01249     do {
    01250         __TBB_ASSERT( is_valid( my_table[s] ), "wrong mask or concurrent grow" );
    01251         segment_ptr_t buckets_ptr = my_table[s];
    01252         size_type sz = segment_size( s ? s : 1 );
    01253         for( segment_index_t i = 0; i < sz; i++ )
    01254             for( node_base *n = buckets_ptr[i].node_list; is_valid(n); n = buckets_ptr[i].node_list ) {
    01255                 buckets_ptr[i].node_list = n->next;
    01256                 delete_node( n );
    01257             }
    01258         if( s >= first_block) // the first segment or the next
    01259             alloc.deallocate( buckets_ptr, sz );
    01260         else if( s == embedded_block && embedded_block != first_block )
    01261             alloc.deallocate( buckets_ptr, segment_size(first_block)-embedded_buckets );
    01262         if( s >= embedded_block ) my_table[s] = 0;
    01263     } while(s-- > 0);
    01264     my_mask = embedded_buckets - 1;
    01265 }
    01266
    01267 template<typename Key, typename T, typename HashCompare, typename A>
    01268 void concurrent_hash_map<Key,T,HashCompare,A>::internal_copy( const concurrent_hash_map& source ) {
    01269     reserve( source.my_size ); // TODO: load_factor?
    01270     hashcode_t mask = source.my_mask;
    01271     if( my_mask == mask ) { // optimized version
    01272         bucket *dst = 0, *src = 0;
    01273         bool rehash_required = false;
    01274         for( hashcode_t k = 0; k <= mask; k++ ) {
    01275             if( k & (k-2) ) ++dst,src++; // not the beginning of a segment
    01276             else { dst = get_bucket( k ); src = source.get_bucket( k ); }
    01277             __TBB_ASSERT( dst->node_list != internal::rehash_req, "Invalid bucket in destination table");
    01278             node *n = static_cast<node*>( src->node_list );
    01279             if( n == internal::rehash_req ) { // source is not rehashed, items are in previous buckets
    01280                 rehash_required = true;
    01281                 dst->node_list = internal::rehash_req;
    01282             } else for(; n; n = static_cast<node*>( n->next ) ) {
    01283                 add_to_bucket( dst, new( my_allocator ) node(n->item.first, n->item.second) );
    01284                 ++my_size; // TODO: replace by non-atomic op
    01285             }
    01286         }
    01287         if( rehash_required ) rehash();
    01288     } else internal_copy( source.begin(), source.end() );
    01289 }
    01290
    01291 template<typename Key, typename T, typename HashCompare, typename A>
    01292 template<typename I>
    01293 void concurrent_hash_map<Key,T,HashCompare,A>::internal_copy(I first, I last) {
    01294     hashcode_t m = my_mask;
    01295     for(; first != last; ++first) {
    01296         hashcode_t h = my_hash_compare.hash( first->first );
    01297         bucket *b = get_bucket( h & m );
    01298         __TBB_ASSERT( b->node_list != internal::rehash_req, "Invalid bucket in destination table");
    01299         node *n = new( my_allocator ) node(first->first, first->second);
    01300         add_to_bucket( b, n );
    01301         ++my_size; // TODO: replace by non-atomic op
    01302     }
    01303 }
    01304
    01305 } // namespace interface5
    01306
    01307 using interface5::concurrent_hash_map;
    01308
    01309
    01310 template<typename Key, typename T, typename HashCompare, typename A1, typename A2>
    01311 inline bool operator==(const concurrent_hash_map<Key, T, HashCompare, A1> &a, const concurrent_hash_map<Key, T, HashCompare, A2> &b) {
    01312     if(a.size() != b.size()) return false;
    01313     typename concurrent_hash_map<Key, T, HashCompare, A1>::const_iterator i(a.begin()), i_end(a.end());
    01314     typename concurrent_hash_map<Key, T, HashCompare, A2>::const_iterator j, j_end(b.end());
    01315     for(; i != i_end; ++i) {
    01316         j = b.equal_range(i->first).first;
    01317         if( j == j_end || !(i->second == j->second) ) return false;
    01318     }
    01319     return true;
    01320 }
    01321
    01322 template<typename Key, typename T, typename HashCompare, typename A1, typename A2>
    01323 inline bool operator!=(const concurrent_hash_map<Key, T, HashCompare, A1> &a, const concurrent_hash_map<Key, T, HashCompare, A2> &b)
    01324 {    return !(a == b); }
    01325
    01326 template<typename Key, typename T, typename HashCompare, typename A>
    01327 inline void swap(concurrent_hash_map<Key, T, HashCompare, A> &a, concurrent_hash_map<Key, T, HashCompare, A> &b)
    01328 {    a.swap( b ); }
    01329
    01330 #if _MSC_VER && !defined(__INTEL_COMPILER)
    01331     #pragma warning( pop )
    01332 #endif // warning 4127 is back
    01333
    01334 } // namespace tbb
    01335
    01336 #endif /* __TBB_concurrent_hash_map_H */

Copyright © 2005-2012 Intel Corporation. All Rights Reserved.

Intel, Pentium, Intel Xeon, Itanium, Intel XScale and VTune are registered trademarks or trademarks of Intel Corporation or its subsidiaries in the United States and other countries.

* Other names and brands may be claimed as the property of others.