You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
268 lines
11 KiB
C++
268 lines
11 KiB
C++
/*
|
|
Copyright 2005-2014 Intel Corporation. All Rights Reserved.
|
|
|
|
This file is part of Threading Building Blocks. Threading Building Blocks is free software;
|
|
you can redistribute it and/or modify it under the terms of the GNU General Public License
|
|
version 2 as published by the Free Software Foundation. Threading Building Blocks is
|
|
distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
|
|
implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
See the GNU General Public License for more details. You should have received a copy of
|
|
the GNU General Public License along with Threading Building Blocks; if not, write to the
|
|
Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
|
As a special exception, you may use this file as part of a free software library without
|
|
restriction. Specifically, if other files instantiate templates or use macros or inline
|
|
functions from this file, or you compile this file and link it with other files to produce
|
|
an executable, this file does not by itself cause the resulting executable to be covered
|
|
by the GNU General Public License. This exception does not however invalidate any other
|
|
reasons why the executable file might be covered by the GNU General Public License.
|
|
*/
|
|
|
|
#include "tbb/tbb_config.h"
|
|
#if __TBB_TSX_AVAILABLE
|
|
#include "tbb/spin_rw_mutex.h"
|
|
#include "tbb/tbb_machine.h"
|
|
#include "itt_notify.h"
|
|
#include "governor.h"
|
|
#include "tbb/atomic.h"
|
|
|
|
// __TBB_RW_MUTEX_DELAY_TEST shifts the point where flags aborting speculation are
|
|
// added to the read-set of the operation. If 1, will add the test just before
|
|
// the transaction is ended.
|
|
#ifndef __TBB_RW_MUTEX_DELAY_TEST
|
|
#define __TBB_RW_MUTEX_DELAY_TEST 1
|
|
#endif
|
|
|
|
#if defined(_MSC_VER) && defined(_Wp64)
|
|
// Workaround for overzealous compiler warnings in /Wp64 mode
|
|
#pragma warning (disable: 4244)
|
|
#endif
|
|
|
|
namespace tbb {
|
|
|
|
namespace interface8 {
|
|
namespace internal {
|
|
|
|
// abort code for mutexes that detect a conflict with another thread.
|
|
// value is hexadecimal
|
|
enum {
|
|
speculation_transaction_aborted = 0x01,
|
|
speculation_can_retry = 0x02,
|
|
speculation_memadd_conflict = 0x04,
|
|
speculation_buffer_overflow = 0x08,
|
|
speculation_breakpoint_hit = 0x10,
|
|
speculation_nested_abort = 0x20,
|
|
speculation_xabort_mask = 0xFF000000,
|
|
speculation_xabort_shift = 24,
|
|
speculation_retry = speculation_transaction_aborted
|
|
| speculation_can_retry
|
|
| speculation_memadd_conflict
|
|
};
|
|
|
|
// maximum number of times to retry
|
|
static const int retry_threshold_read = 10;
|
|
static const int retry_threshold_write = 10;
|
|
|
|
//! Release speculative mutex
|
|
void x86_rtm_rw_mutex::internal_release(x86_rtm_rw_mutex::scoped_lock& s) {
|
|
switch(s.transaction_state) {
|
|
case RTM_transacting_writer:
|
|
case RTM_transacting_reader:
|
|
{
|
|
__TBB_ASSERT(__TBB_machine_is_in_transaction(), "transaction_state && not speculating");
|
|
#if __TBB_RW_MUTEX_DELAY_TEST
|
|
if(s.transaction_state == RTM_transacting_reader) {
|
|
if(this->w_flag) __TBB_machine_transaction_conflict_abort();
|
|
} else {
|
|
if(this->state) __TBB_machine_transaction_conflict_abort();
|
|
}
|
|
#endif
|
|
__TBB_machine_end_transaction();
|
|
s.my_scoped_lock.internal_set_mutex(NULL);
|
|
}
|
|
break;
|
|
case RTM_real_reader:
|
|
__TBB_ASSERT(!this->w_flag, "w_flag set but read lock acquired");
|
|
s.my_scoped_lock.release();
|
|
break;
|
|
case RTM_real_writer:
|
|
__TBB_ASSERT(this->w_flag, "w_flag unset but write lock acquired");
|
|
this->w_flag = false;
|
|
s.my_scoped_lock.release();
|
|
break;
|
|
case RTM_not_in_mutex:
|
|
__TBB_ASSERT(false, "RTM_not_in_mutex, but in release");
|
|
default:
|
|
__TBB_ASSERT(false, "invalid transaction_state");
|
|
}
|
|
s.transaction_state = RTM_not_in_mutex;
|
|
}
|
|
|
|
//! Acquire write lock on the given mutex.
|
|
void x86_rtm_rw_mutex::internal_acquire_writer(x86_rtm_rw_mutex::scoped_lock& s, bool only_speculate)
|
|
{
|
|
__TBB_ASSERT(s.transaction_state == RTM_not_in_mutex, "scoped_lock already in transaction");
|
|
if(tbb::internal::governor::speculation_enabled()) {
|
|
int num_retries = 0;
|
|
unsigned int abort_code;
|
|
do {
|
|
tbb::internal::atomic_backoff backoff;
|
|
if(this->state) {
|
|
if(only_speculate) return;
|
|
do {
|
|
backoff.pause(); // test the spin_rw_mutex (real readers or writers)
|
|
} while(this->state);
|
|
}
|
|
// _xbegin returns -1 on success or the abort code, so capture it
|
|
if(( abort_code = __TBB_machine_begin_transaction()) == ~(unsigned int)(0) )
|
|
{
|
|
// started speculation
|
|
#if !__TBB_RW_MUTEX_DELAY_TEST
|
|
if(this->state) { // add spin_rw_mutex to read-set.
|
|
// reader or writer grabbed the lock, so abort.
|
|
__TBB_machine_transaction_conflict_abort();
|
|
}
|
|
#endif
|
|
s.transaction_state = RTM_transacting_writer;
|
|
s.my_scoped_lock.internal_set_mutex(this); // need mutex for release()
|
|
return; // successfully started speculation
|
|
}
|
|
++num_retries;
|
|
} while( (abort_code & speculation_retry) != 0 && (num_retries < retry_threshold_write) );
|
|
}
|
|
|
|
if(only_speculate) return; // should apply a real try_lock...
|
|
s.my_scoped_lock.acquire(*this, true); // kill transactional writers
|
|
__TBB_ASSERT(!w_flag, "After acquire for write, w_flag already true");
|
|
w_flag = true; // kill transactional readers
|
|
s.transaction_state = RTM_real_writer;
|
|
return;
|
|
}
|
|
|
|
//! Acquire read lock on given mutex.
|
|
// only_speculate : true if we are doing a try_acquire. If true and we fail to speculate, don't
|
|
// really acquire the lock, return and do a try_acquire on the contained spin_rw_mutex. If
|
|
// the lock is already held by a writer, just return.
|
|
void x86_rtm_rw_mutex::internal_acquire_reader(x86_rtm_rw_mutex::scoped_lock& s, bool only_speculate) {
|
|
__TBB_ASSERT(s.transaction_state == RTM_not_in_mutex, "scoped_lock already in transaction");
|
|
if(tbb::internal::governor::speculation_enabled()) {
|
|
int num_retries = 0;
|
|
unsigned int abort_code;
|
|
do {
|
|
tbb::internal::atomic_backoff backoff;
|
|
// if in try_acquire, and lock is held as writer, don't attempt to speculate.
|
|
if(w_flag) {
|
|
if(only_speculate) return;
|
|
do {
|
|
backoff.pause(); // test the spin_rw_mutex (real readers or writers)
|
|
} while(w_flag);
|
|
}
|
|
// _xbegin returns -1 on success or the abort code, so capture it
|
|
if((abort_code = __TBB_machine_begin_transaction()) == ~(unsigned int)(0) )
|
|
{
|
|
// started speculation
|
|
#if !__TBB_RW_MUTEX_DELAY_TEST
|
|
if(w_flag) { // add w_flag to read-set.
|
|
__TBB_machine_transaction_conflict_abort(); // writer grabbed the lock, so abort.
|
|
}
|
|
#endif
|
|
s.transaction_state = RTM_transacting_reader;
|
|
s.my_scoped_lock.internal_set_mutex(this); // need mutex for release()
|
|
return; // successfully started speculation
|
|
}
|
|
// fallback path
|
|
// retry only if there is any hope of getting into a transaction soon
|
|
// Retry in the following cases (from Section 8.3.5 of Intel(R)
|
|
// Architecture Instruction Set Extensions Programming Reference):
|
|
// 1. abort caused by XABORT instruction (bit 0 of EAX register is set)
|
|
// 2. the transaction may succeed on a retry (bit 1 of EAX register is set)
|
|
// 3. if another logical processor conflicted with a memory address
|
|
// that was part of the transaction that aborted (bit 2 of EAX register is set)
|
|
// That is, retry if (abort_code & 0x7) is non-zero
|
|
++num_retries;
|
|
} while( (abort_code & speculation_retry) != 0 && (num_retries < retry_threshold_read) );
|
|
}
|
|
|
|
if(only_speculate) return;
|
|
s.my_scoped_lock.acquire( *this, false );
|
|
s.transaction_state = RTM_real_reader;
|
|
}
|
|
|
|
//! Upgrade reader to become a writer.
|
|
/** Returns whether the upgrade happened without releasing and re-acquiring the lock */
|
|
bool x86_rtm_rw_mutex::internal_upgrade(x86_rtm_rw_mutex::scoped_lock& s)
|
|
{
|
|
switch(s.transaction_state) {
|
|
case RTM_real_reader: {
|
|
s.transaction_state = RTM_real_writer;
|
|
bool no_release = s.my_scoped_lock.upgrade_to_writer();
|
|
__TBB_ASSERT(!w_flag, "After upgrade_to_writer, w_flag already true");
|
|
w_flag = true;
|
|
return no_release;
|
|
}
|
|
case RTM_transacting_reader:
|
|
s.transaction_state = RTM_transacting_writer;
|
|
// don't need to add w_flag to read_set even if __TBB_RW_MUTEX_DELAY_TEST
|
|
// because the this pointer (the spin_rw_mutex) will be sufficient on release.
|
|
return true;
|
|
default:
|
|
__TBB_ASSERT(false, "Invalid state for upgrade");
|
|
return false;
|
|
}
|
|
}
|
|
|
|
//! Downgrade writer to a reader.
|
|
bool x86_rtm_rw_mutex::internal_downgrade(x86_rtm_rw_mutex::scoped_lock& s) {
|
|
switch(s.transaction_state) {
|
|
case RTM_real_writer:
|
|
s.transaction_state = RTM_real_reader;
|
|
__TBB_ASSERT(w_flag, "Before downgrade_to_reader w_flag not true");
|
|
w_flag = false;
|
|
return s.my_scoped_lock.downgrade_to_reader();
|
|
case RTM_transacting_writer:
|
|
#if __TBB_RW_MUTEX_DELAY_TEST
|
|
if(this->state) { // a reader or writer has acquired mutex for real.
|
|
__TBB_machine_transaction_conflict_abort();
|
|
}
|
|
#endif
|
|
s.transaction_state = RTM_transacting_reader;
|
|
return true;
|
|
default:
|
|
__TBB_ASSERT(false, "Invalid state for downgrade");
|
|
return false;
|
|
}
|
|
}
|
|
|
|
//! Try to acquire write lock on the given mutex.
|
|
// There may be reader(s) which acquired the spin_rw_mutex, as well as possibly
|
|
// transactional reader(s). If this is the case, the acquire will fail, and assigning
|
|
// w_flag will kill the transactors. So we only assign w_flag if we have successfully
|
|
// acquired the lock.
|
|
bool x86_rtm_rw_mutex::internal_try_acquire_writer(x86_rtm_rw_mutex::scoped_lock& s)
|
|
{
|
|
internal_acquire_writer(s, /*only_speculate=*/true);
|
|
if(s.transaction_state == RTM_transacting_writer) {
|
|
return true;
|
|
}
|
|
__TBB_ASSERT(s.transaction_state == RTM_not_in_mutex, "Trying to acquire writer which is already allocated");
|
|
// transacting write acquire failed. try_acquire the real mutex
|
|
bool result = s.my_scoped_lock.try_acquire(*this, true);
|
|
if(result) {
|
|
// only shoot down readers if we're not transacting ourselves
|
|
__TBB_ASSERT(!w_flag, "After try_acquire_writer, w_flag already true");
|
|
w_flag = true;
|
|
s.transaction_state = RTM_real_writer;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
void x86_rtm_rw_mutex::internal_construct() {
|
|
ITT_SYNC_CREATE(this, _T("tbb::x86_rtm_rw_mutex"), _T(""));
|
|
}
|
|
|
|
} // namespace internal
|
|
} // namespace interface8
|
|
} // namespace tbb
|
|
|
|
#endif /* __TBB_TSX_AVAILABLE */
|