root/threadsync.cpp

DEFINITIONS

1. dmtcp_libdlLockLock
2. dmtcp_libdlLockUnlock
3. initThread
4. initMotherOfAll
5. acquireLocks
6. releaseLocks
7. resetLocks
8. isThisThreadHoldingAnyLocks
9. isOkToGrabLock
10. setOkToGrabLock
11. unsetOkToGrabLock
12. setSendCkptSignalOnFinalUnlock
13. sendCkptSignalOnFinalUnlock
14. dmtcp_setThreadPerformingDlopenDlsym
15. dmtcp_unsetThreadPerformingDlopenDlsym
16. isThreadPerformingDlopenDlsym
17. setThreadPerformingDlopenDlsym
18. unsetThreadPerformingDlopenDlsym
19. destroyDmtcpWorkerLockLock
20. destroyDmtcpWorkerLockTryLock
21. destroyDmtcpWorkerLockUnlock
22. delayCheckpointsLock
23. delayCheckpointsUnlock
24. incrementWrapperExecutionLockLockCount
25. decrementWrapperExecutionLockLockCount
26. incrementThreadCreationLockLockCount
27. decrementThreadCreationLockLockCount
28. libdlLockLock
29. libdlLockUnlock
30. wrapperExecutionLockLock
31. wrapperExecutionLockLockExcl
32. wrapperExecutionLockUnlock
33. threadCreationLockLock
34. threadCreationLockUnlock
35. dmtcp_plugin_disable_ckpt
36. dmtcp_plugin_enable_ckpt
37. waitForThreadsToFinishInitialization
38. incrementUninitializedThreadCount
39. decrementUninitializedThreadCount
40. threadFinishedInitialization
41. incrNumUserThreads
42. processPreResumeCB
43. waitForUserThreadsToFinishPreResumeCB

   1 /****************************************************************************
   2  *   Copyright (C) 2006-2013 by Jason Ansel, Kapil Arya, and Gene Cooperman *
   3  *   jansel@csail.mit.edu, kapil@ccs.neu.edu, gene@ccs.neu.edu              *
   4  *                                                                          *
   5  *  This file is part of DMTCP.                                             *
   6  *                                                                          *
   7  *  DMTCP is free software: you can redistribute it and/or                  *
   8  *  modify it under the terms of the GNU Lesser General Public License as   *
   9  *  published by the Free Software Foundation, either version 3 of the      *
  10  *  License, or (at your option) any later version.                         *
  11  *                                                                          *
  12  *  DMTCP is distributed in the hope that it will be useful,                *
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of          *
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
  15  *  GNU Lesser General Public License for more details.                     *
  16  *                                                                          *
  17  *  You should have received a copy of the GNU Lesser General Public        *
  18  *  License along with DMTCP:dmtcp/src.  If not, see                        *
  19  *  <http://www.gnu.org/licenses/>.                                         *
  20  ****************************************************************************/
  22 #include <stdlib.h>
  23 #include <unistd.h>
  24 #include <pthread.h>
  25 #include <sys/types.h>
  27 #include "threadsync.h"
  28 #include "dmtcpworker.h"
  29 #include "syscallwrappers.h"
  31 using namespace dmtcp;
  33 /*
  34  * _wrapperExecutionLock is used to make the checkpoint safe by making sure
  35  *   that no user-thread is executing any DMTCP wrapper code when it receives
  36  *   the checkpoint signal.
  37  * Working:
  38  *   On entering the wrapper in DMTCP, the user-thread acquires the read lock,
  39  *     and releases it before leaving the wrapper.
  40  *   When the Checkpoint-thread wants to send the SUSPEND signal to user
  41  *     threads, it must acquire the write lock. It is blocked until all the
  42  *     existing read-locks by user threads have been released. NOTE that this
  43  *     is a WRITER-PREFERRED lock.
  44  *
  45  * There is a corner case too -- the newly created thread that has not been
  46  *   initialized yet; we need to take some extra efforts for that.
  47  * Here are the steps to handle the newly created uninitialized thread:
  48  *   A counter (_uninitializedThreadCount) for the number of newly
  49  *     created uninitialized threads is kept.  The counter is made
  50  *     thread-safe by using a mutex.
  51  *   The calling thread (parent) increments the counter before calling clone.
  52  *   The newly created child thread decrements the counter at the end of
  53  *     initialization in MTCP/DMTCP.
  54  *   After acquiring the Write lock, the checkpoint thread waits until the
  55  *     number of uninitialized threads is zero. At that point, no thread is
  56  *     executing in the clone wrapper and it is safe to do a checkpoint.
  57  *
  58  * XXX: Currently this security is provided only for the clone wrapper; this
  59  * should be extended to other calls as well.           -- KAPIL
  60  */
  61 // NOTE: PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP is not POSIX.
  62 static pthread_rwlock_t
_wrapperExecutionLock = PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP;
  64 static pthread_rwlock_t
_threadCreationLock = PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP;
  66 static bool _wrapperExecutionLockAcquiredByCkptThread = false;
  67 static bool _threadCreationLockAcquiredByCkptThread = false;
  69 static pthread_mutex_t destroyDmtcpWorkerLock = PTHREAD_MUTEX_INITIALIZER;
  70 static pthread_mutex_t theCkptCanStart = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
  72 static pthread_mutex_t libdlLock = PTHREAD_MUTEX_INITIALIZER;
  73 static pid_t libdlLockOwner = 0;
  75 static pthread_mutex_t uninitializedThreadCountLock = PTHREAD_MUTEX_INITIALIZER;
  76 static int _uninitializedThreadCount = 0;
  77 static bool _checkpointThreadInitialized = false;
  79 #define INVALID_USER_THREAD_COUNT 0
  80 static int preResumeThreadCount = INVALID_USER_THREAD_COUNT;
  81 static pthread_mutex_t preResumeThreadCountLock = PTHREAD_MUTEX_INITIALIZER;
  83 static __thread int _wrapperExecutionLockLockCount = 0;
  84 static __thread int _threadCreationLockLockCount = 0;
  85 #if TRACK_DLOPEN_DLSYM_FOR_LOCKS
  86 static __thread bool _threadPerformingDlopenDlsym = false;
  87 #endif
  88 static __thread bool _sendCkptSignalOnFinalUnlock = false;
  89 static __thread bool _isOkToGrabWrapperExecutionLock = true;
  90 static __thread bool _hasThreadFinishedInitialization = false;
  93 /* The following two functions dmtcp_libdlLock{Lock,Unlock} are used by dlopen
  94  * plugin.
  95  */
  96 extern "C" int dmtcp_libdlLockLock() {
  97   return ThreadSync::libdlLockLock();
  98 }
 100 extern "C" void dmtcp_libdlLockUnlock() {
ThreadSync::libdlLockUnlock();
 102 }
 104 void ThreadSync::initThread()
 105 {
 106   // If we don't initialize these thread local variables here. If not done
 107   // here, there can be a race between checkpoint processing and this
 108   // thread trying to initialize some thread-local variable. Here is a possible
 109   // calltrace:
 110   // pthread_start -> threadFinishedInitialization -> stopthisthread ->
 111   // callbackHoldsAnyLocks -> JASSERT().
_wrapperExecutionLockLockCount = 0;
_threadCreationLockLockCount = 0;
 114 #if TRACK_DLOPEN_DLSYM_FOR_LOCKS
_threadPerformingDlopenDlsym = false;
 116 #endif
_sendCkptSignalOnFinalUnlock = false;
_isOkToGrabWrapperExecutionLock = true;
_hasThreadFinishedInitialization = false;
 120 }
 122 void ThreadSync::initMotherOfAll()
 123 {
initThread();
_hasThreadFinishedInitialization = true;
 126 }
 128 void ThreadSync::acquireLocks()
 129 {
JASSERT(WorkerState::currentState() == WorkerState::RUNNING);
 131   /* TODO: We should introduce the notion of lock ranks/priorities for all
 132    * these locks to prevent future deadlocks due to rank violation.
 133    */
JTRACE("waiting for dmtcp_lock():"
 136          " to get synchronized with _runCoordinatorCmd if we use DMTCP API");
_dmtcp_lock();
JTRACE("Waiting for lock(&theCkptCanStart)");
JASSERT(_real_pthread_mutex_lock(&theCkptCanStart) == 0)(JASSERT_ERRNO);
JTRACE("Waiting for libdlLock");
JASSERT(_real_pthread_mutex_lock(&libdlLock) == 0) (JASSERT_ERRNO);
JTRACE("Waiting for threads creation lock");
JASSERT(_real_pthread_rwlock_wrlock(&_threadCreationLock) == 0)
 147     (JASSERT_ERRNO);
_threadCreationLockAcquiredByCkptThread = true;
JTRACE("Waiting for other threads to exit DMTCP-Wrappers");
JASSERT(_real_pthread_rwlock_wrlock(&_wrapperExecutionLock) == 0)
 152     (JASSERT_ERRNO);
_wrapperExecutionLockAcquiredByCkptThread = true;
JTRACE("Waiting for newly created threads to finish initialization")
 156     (_uninitializedThreadCount);
waitForThreadsToFinishInitialization();
unsetOkToGrabLock();
JTRACE("Done acquiring all locks");
 161 }
 163 void ThreadSync::releaseLocks()
 164 {
JASSERT(WorkerState::currentState() == WorkerState::SUSPENDED);
JTRACE("Releasing ThreadSync locks");
JASSERT(_real_pthread_rwlock_unlock(&_wrapperExecutionLock) == 0)
 169     (JASSERT_ERRNO);
_wrapperExecutionLockAcquiredByCkptThread = false;
JASSERT(_real_pthread_rwlock_unlock(&_threadCreationLock) == 0)
 172     (JASSERT_ERRNO);
_threadCreationLockAcquiredByCkptThread = false;
JASSERT(_real_pthread_mutex_unlock(&libdlLock) == 0) (JASSERT_ERRNO);
JASSERT(_real_pthread_mutex_unlock(&theCkptCanStart) == 0) (JASSERT_ERRNO);
_dmtcp_unlock();
setOkToGrabLock();
 179 }
 181 void ThreadSync::resetLocks()
 182 {
pthread_rwlock_t newLock = PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP;
_wrapperExecutionLock = newLock;
_threadCreationLock = newLock;
_wrapperExecutionLockLockCount = 0;
_threadCreationLockLockCount = 0;
 189 #if TRACK_DLOPEN_DLSYM_FOR_LOCKS
_threadPerformingDlopenDlsym = false;
 191 #endif
_sendCkptSignalOnFinalUnlock = false;
_isOkToGrabWrapperExecutionLock = true;
_hasThreadFinishedInitialization = true;
pthread_mutex_t newCountLock = PTHREAD_MUTEX_INITIALIZER;
uninitializedThreadCountLock = newCountLock;
pthread_mutex_t newPreResumeThreadCountLock = PTHREAD_MUTEX_INITIALIZER;
preResumeThreadCountLock = newPreResumeThreadCountLock;
pthread_mutex_t newDestroyDmtcpWorker = PTHREAD_MUTEX_INITIALIZER;
destroyDmtcpWorkerLock = newDestroyDmtcpWorker;
pthread_mutex_t newLibdlLock = PTHREAD_MUTEX_INITIALIZER;
libdlLock = newLibdlLock;
libdlLockOwner = 0;
_checkpointThreadInitialized = false;
_wrapperExecutionLockAcquiredByCkptThread = false;
_threadCreationLockAcquiredByCkptThread = false;
 211 }
 213 bool ThreadSync::isThisThreadHoldingAnyLocks()
 214 {
 215   // If the wrapperExec lock has been acquired by the ckpt thread, then we are
 216   // certainly not holding it :). It's possible for the count to be still '1',
 217   // as it may happen that the thread got suspended after releasing the lock
 218   // and before decrementing the lock-count.
 219   if (_hasThreadFinishedInitialization == false) {
 220     return true;
 221   }
 222   return (_wrapperExecutionLockAcquiredByCkptThread == false ||
_threadCreationLockAcquiredByCkptThread == false) &&
 224          (_threadCreationLockLockCount > 0 ||
_wrapperExecutionLockLockCount > 0);
 226 }
 228 bool ThreadSync::isOkToGrabLock()
 229 {
 230   return _isOkToGrabWrapperExecutionLock;
 231 }
 233 void ThreadSync::setOkToGrabLock()
 234 {
_isOkToGrabWrapperExecutionLock = true;
 236 }
 238 void ThreadSync::unsetOkToGrabLock()
 239 {
_isOkToGrabWrapperExecutionLock = false;
 241 }
 243 void ThreadSync::setSendCkptSignalOnFinalUnlock()
 244 {
JASSERT(_sendCkptSignalOnFinalUnlock == false);
_sendCkptSignalOnFinalUnlock = true;
 247 }
 249 void ThreadSync::sendCkptSignalOnFinalUnlock()
 250 {
 251   if (_sendCkptSignalOnFinalUnlock && isThisThreadHoldingAnyLocks() == false) {
_sendCkptSignalOnFinalUnlock = false;
JASSERT(raise(DmtcpWorker::determineCkptSignal()) == 0)
 254       (getpid()) (dmtcp_gettid()) (JASSERT_ERRNO);
 255   }
 256 }
 258 #if TRACK_DLOPEN_DLSYM_FOR_LOCKS
 259 extern "C" LIB_PRIVATE
 260 void dmtcp_setThreadPerformingDlopenDlsym()
 261 {
ThreadSync::setThreadPerformingDlopenDlsym();
 263 }
 265 extern "C" LIB_PRIVATE
 266 void dmtcp_unsetThreadPerformingDlopenDlsym()
 267 {
ThreadSync::unsetThreadPerformingDlopenDlsym();
 269 }
 271 bool ThreadSync::isThreadPerformingDlopenDlsym()
 272 {
 273   return _threadPerformingDlopenDlsym;
 274 }
 276 void ThreadSync::setThreadPerformingDlopenDlsym()
 277 {
_threadPerformingDlopenDlsym = true;
 279 }
 281 void ThreadSync::unsetThreadPerformingDlopenDlsym()
 282 {
_threadPerformingDlopenDlsym = false;
 284 }
 285 #endif
 287 void ThreadSync::destroyDmtcpWorkerLockLock()
 288 {
JASSERT(_real_pthread_mutex_lock(&destroyDmtcpWorkerLock) == 0)
 290     (JASSERT_ERRNO);
 291 }
 293 int ThreadSync::destroyDmtcpWorkerLockTryLock()
 294 {
 295   return _real_pthread_mutex_trylock(&destroyDmtcpWorkerLock);
 296 }
 298 void ThreadSync::destroyDmtcpWorkerLockUnlock()
 299 {
JASSERT(_real_pthread_mutex_unlock(&destroyDmtcpWorkerLock) == 0)
 301     (JASSERT_ERRNO);
 302 }
 304 void ThreadSync::delayCheckpointsLock()
 305 {
JASSERT(_real_pthread_mutex_lock(&theCkptCanStart)==0)(JASSERT_ERRNO);
 307 }
 309 void ThreadSync::delayCheckpointsUnlock() {
JASSERT(_real_pthread_mutex_unlock(&theCkptCanStart)==0)(JASSERT_ERRNO);
 311 }
 313 static void incrementWrapperExecutionLockLockCount()
 314 {
_wrapperExecutionLockLockCount++;
 316 }
 318 static void decrementWrapperExecutionLockLockCount()
 319 {
 320   if (_wrapperExecutionLockLockCount <= 0) {
JASSERT(false) (_wrapperExecutionLockLockCount)
 322       .Text("wrapper-execution lock count can't be negative");
 323   }
_wrapperExecutionLockLockCount--;
ThreadSync::sendCkptSignalOnFinalUnlock();
 326 }
 328 static void incrementThreadCreationLockLockCount()
 329 {
_threadCreationLockLockCount++;
 331 }
 333 static void decrementThreadCreationLockLockCount()
 334 {
_threadCreationLockLockCount--;
ThreadSync::sendCkptSignalOnFinalUnlock();
 337 }
 339 bool ThreadSync::libdlLockLock()
 340 {
 341   int saved_errno = errno;
 342   bool lockAcquired = false;
 343   if (WorkerState::currentState() == WorkerState::RUNNING &&
libdlLockOwner !=  dmtcp_gettid()) {
JASSERT(_real_pthread_mutex_lock(&libdlLock) == 0);
libdlLockOwner = dmtcp_gettid();
lockAcquired = true;
 348   }
errno = saved_errno;
 350   return lockAcquired;
 351 }
 353 void ThreadSync::libdlLockUnlock()
 354 {
 355   int saved_errno = errno;
JASSERT(libdlLockOwner == 0 || libdlLockOwner == dmtcp_gettid())
 357     (libdlLockOwner) (dmtcp_gettid());
JASSERT (WorkerState::currentState() == WorkerState::RUNNING);
libdlLockOwner = 0;
JASSERT(_real_pthread_mutex_unlock(&libdlLock) == 0);
errno = saved_errno;
 362 }
 364 // XXX: Handle deadlock error code
 365 // NOTE: Don't do any fancy stuff in this wrapper which can cause the process
 366 //       to go into DEADLOCK
 367 bool ThreadSync::wrapperExecutionLockLock()
 368 {
 369   int saved_errno = errno;
 370   bool lockAcquired = false;
 371   // Ignore locks if we are about to exit
 372   if (DmtcpWorker::exitInProgress()) {
 373     return false;
 374   }
 375   while (1) {
 376     if (WorkerState::currentState() == WorkerState::RUNNING &&
 377 #if TRACK_DLOPEN_DLSYM_FOR_LOCKS
isThreadPerformingDlopenDlsym() == false &&
 379 #endif
isOkToGrabLock() == true &&
_wrapperExecutionLockLockCount == 0) {
incrementWrapperExecutionLockLockCount();
 383       int retVal = _real_pthread_rwlock_tryrdlock(&_wrapperExecutionLock);
 384       if (retVal != 0 && retVal == EBUSY) {
decrementWrapperExecutionLockLockCount();
 386         struct timespec sleepTime = {0, 100*1000*1000};
nanosleep(&sleepTime, NULL);
 388         continue;
 389       }
 390       if (retVal != 0 && retVal != EDEADLK) {
fprintf(stderr, "ERROR %d at %s:%d %s: Failed to acquire lock\n",
errno, __FILE__, __LINE__, __PRETTY_FUNCTION__);
_exit(1);
 394       }
 395       // retVal should always be 0 (success) here.
lockAcquired = retVal == 0 ? true : false;
 397       if (!lockAcquired) {
decrementWrapperExecutionLockLockCount();
 399       }
 400     }
 401     break;
 402   }
errno = saved_errno;
 404   return lockAcquired;
 405 }
 407 /*
 408  * Execute fork() and exec() wrappers in exclusive mode
 409  *
 410  * fork() and exec() wrappers pass on the state/information about the current
 411  * process/program to the to-be-created process/program.
 412  *
 413  * There can be a potential race in the wrappers if this information gets
 414  * changed between the point where it was acquired and the point where the
 415  * process/program is created. An example of this situation would be a
 416  * different thread executing an open() call in parallel creating a
 417  * file-descriptor, which is not a part of the information/state gathered
 418  * earlier. This can result in unexpected behavior and can cause the
 419  * program/process to fail.
 420  *
 421  * This patch fixes this by acquiring the Wrapper-protection-lock in exclusive
 422  * mode (write-lock) when executing these wrappers. This guarantees that no
 423  * other thread would be executing inside a wrapper that can change the process
 424  * state/information.
 425  *
 426  * NOTE:
 427  * 1. Currently, we do not have WRAPPER_EXECUTION_LOCK/UNLOCK for socket()
 428  *    family of wrapper. That would be fixed in a later commit.
 429  * 2. We need to come up with a strategy for certain blocking system calls
 430  *    that can change the state of the process (e.g. accept).
 431  * 3. Using trywrlock() can result in starvation if multiple other threads are
 432  *    rapidly acquiring releasing the lock. For example thread A acquires the
 433  *    rdlock for 100 ms. Thread B executes and trywrlock and fails. Thread B
 434  *    sleeps goes to sleep for some time. While thread B is sleeping, thread A
 435  *    releases the rdlock and reacquires it or some other thread acquires the
 436  *    rdlock. This would cause the thread B to starve. This scenario can be
 437  *    easily observed if thread A calls
 438  *      epoll_wait(fd, events, max_events, -1).
 439  *    It is wrapped by the epoll_wait wrapper in IPC plugin, which then makes
 440  *    repeated calls to _real_epoll_wait with smaller timeout.
 441  */
 442 bool ThreadSync::wrapperExecutionLockLockExcl()
 443 {
 444   int saved_errno = errno;
 445   bool lockAcquired = false;
 446   // Ignore locks if we are about to exit
 447   if (DmtcpWorker::exitInProgress()) {
 448     return false;
 449   }
 450   if (WorkerState::currentState() == WorkerState::RUNNING) {
incrementWrapperExecutionLockLockCount();
 452     int retVal = _real_pthread_rwlock_wrlock(&_wrapperExecutionLock);
 453     if (retVal != 0 && retVal != EDEADLK) {
fprintf(stderr, "ERROR %s:%d %s: Failed to acquire lock\n",
__FILE__, __LINE__, __PRETTY_FUNCTION__);
_exit(1);
 457     }
lockAcquired = retVal == 0 ? true : false;
 459     if (!lockAcquired) {
decrementWrapperExecutionLockLockCount();
 461     }
 462   }
errno = saved_errno;
 464   return lockAcquired;
 465 }
 467 // NOTE: Don't do any fancy stuff in this wrapper which can cause the process
 468 // to go into DEADLOCK
 469 void ThreadSync::wrapperExecutionLockUnlock()
 470 {
 471   int saved_errno = errno;
 472   // Ignore locks if we are about to exit
 473   /*
 474    * NOTE: Ideally, this function should never be called from a wrapper if
 475    *       exitInProgress is set, but there are two cases when it gets
 476    *       called despite the flag being set:
 477    *       1) The exitInProgress flag is set, and then an incorrectly
 478    *          implemented wrapper calls this function even though the
 479    *          lock function returned false. See commit:
 480    *          f2d2a7c6feba38ab0b0cb8e09a4ad6cc37d9f330 for an example.
 481    *       2) A correctly implemented wrapper calls this but the exitInProgress
 482    *          is set after having acquired the wrapperExecution lock. This can
 483    *          occur if a user thread called exit while another was
 484    *          in the middle of the wrapper.
 485    *       The following if clause guards against both the scenarios.
 486    * TODO: Add a message that warns the user if any of the above
 487    *       two cases are seen.
 488    *
 489    */
 491   if (DmtcpWorker::exitInProgress()) {
 492     return;
 493   }
 494   if (_real_pthread_rwlock_unlock(&_wrapperExecutionLock) != 0) {
fprintf(stderr, "ERROR %s:%d %s: Failed to release lock\n",
__FILE__, __LINE__, __PRETTY_FUNCTION__);
_exit(1);
 498   } else {
decrementWrapperExecutionLockLockCount();
 500   }
errno = saved_errno;
 502 }
 504 bool ThreadSync::threadCreationLockLock()
 505 {
 506   int saved_errno = errno;
 507   bool lockAcquired = false;
 508   while (1) {
 509     if (WorkerState::currentState() == WorkerState::RUNNING) {
incrementThreadCreationLockLockCount();
 511       int retVal = _real_pthread_rwlock_tryrdlock(&_threadCreationLock);
 512       if (retVal != 1 && retVal == EBUSY) {
decrementThreadCreationLockLockCount();
 514         struct timespec sleepTime = {0, 100*1000*1000};
nanosleep(&sleepTime, NULL);
 516         continue;
 517       }
 518       if (retVal != 0 && retVal != EDEADLK) {
fprintf(stderr, "ERROR %s:%d %s: Failed to acquire lock\n",
__FILE__, __LINE__, __PRETTY_FUNCTION__);
_exit(1);
 522       }
 523       // retVal should always be 0 (success) here.
lockAcquired = retVal == 0 ? true : false;
 526       // If for some reason, the lock was not acquired, decrement the count
 527       // that we incremented at the start of this block.
 528       if (!lockAcquired) {
decrementThreadCreationLockLockCount();
 530       }
 531     }
 532     break;
 533   }
errno = saved_errno;
 535   return lockAcquired;
 536 }
 538 void ThreadSync::threadCreationLockUnlock()
 539 {
 540   int saved_errno = errno;
 541   if (WorkerState::currentState() != WorkerState::RUNNING) {
fprintf(stderr, "DMTCP INTERNAL ERROR: %s:%d %s:\n"
 543             "       This process is not in RUNNING state and yet this thread\n"
 544             "       managed to acquire the threadCreationLock.\n"
 545             "       This should not be happening, something is wrong.",
__FILE__, __LINE__, __PRETTY_FUNCTION__);
_exit(1);
 548   }
 549   if (_real_pthread_rwlock_unlock(&_threadCreationLock) != 0) {
fprintf(stderr, "ERROR %s:%d %s: Failed to release lock\n",
__FILE__, __LINE__, __PRETTY_FUNCTION__);
_exit(1);
 553   } else {
decrementThreadCreationLockLockCount();
 555   }
errno = saved_errno;
 557 }
 559 // GNU g++ uses __thread.  But the C++0x standard says to use thread_local.
 560 //   If your compiler fails here, you can: change "__thread" to "thread_local";
 561 //   or delete "__thread" (but if user code calls these routines from multiple
 562 //   threads, it will not be thread-safe).
 563 //   In GCC 4.3 and later, g++ supports -std=c++0x and -std=g++0x.
 564 extern "C"
 565 int dmtcp_plugin_disable_ckpt()
 566 {
 567   return ThreadSync::wrapperExecutionLockLock();
 568 }
 570 extern "C"
 571 void dmtcp_plugin_enable_ckpt()
 572 {
ThreadSync::wrapperExecutionLockUnlock();
 574 }
 577 void ThreadSync::waitForThreadsToFinishInitialization()
 578 {
 579   while (_uninitializedThreadCount != 0) {
 580     struct timespec sleepTime = {0, 10*1000*1000};
JTRACE("sleeping")(sleepTime.tv_nsec);
nanosleep(&sleepTime, NULL);
 583   }
 584 }
 586 void ThreadSync::incrementUninitializedThreadCount()
 587 {
 588   int saved_errno = errno;
 589   if (WorkerState::currentState() == WorkerState::RUNNING) {
JASSERT(_real_pthread_mutex_lock(&uninitializedThreadCountLock) == 0)
 591       (JASSERT_ERRNO);
_uninitializedThreadCount++;
 593     //JTRACE(":") (_uninitializedThreadCount);
JASSERT(_real_pthread_mutex_unlock(&uninitializedThreadCountLock) == 0)
 595       (JASSERT_ERRNO);
 596   }
errno = saved_errno;
 598 }
 600 void ThreadSync::decrementUninitializedThreadCount()
 601 {
 602   int saved_errno = errno;
 603   if (WorkerState::currentState() == WorkerState::RUNNING) {
JASSERT(_real_pthread_mutex_lock(&uninitializedThreadCountLock) == 0)
 605       (JASSERT_ERRNO);
JASSERT(_uninitializedThreadCount > 0) (_uninitializedThreadCount);
_uninitializedThreadCount--;
 608     //JTRACE(":") (_uninitializedThreadCount);
JASSERT(_real_pthread_mutex_unlock(&uninitializedThreadCountLock) == 0)
 610       (JASSERT_ERRNO);
 611   }
errno = saved_errno;
 613 }
 615 void ThreadSync::threadFinishedInitialization()
 616 {
 617   // The following line is to make sure the thread-local data is initialized
 618   // before any wrapper call is made.
_hasThreadFinishedInitialization = false;
decrementUninitializedThreadCount();
_hasThreadFinishedInitialization = true;
ThreadSync::sendCkptSignalOnFinalUnlock();
 623 }
 625 void ThreadSync::incrNumUserThreads()
 626 {
 627   // This routine is called from within stopthisthread so it is not safe to
 628   // call JNOTE/JTRACE etc.
 629   if (_real_pthread_mutex_lock(&preResumeThreadCountLock) != 0) {
JASSERT(false) .Text("Failed to acquire preResumeThreadCountLock");
 631   }
preResumeThreadCount++;
 633   if (_real_pthread_mutex_unlock(&preResumeThreadCountLock) != 0) {
JASSERT(false) .Text("Failed to release preResumeThreadCountLock");
 635   }
 636 }
 638 void ThreadSync::processPreResumeCB()
 639 {
 640   if (_real_pthread_mutex_lock(&preResumeThreadCountLock) != 0) {
JASSERT(false) .Text("Failed to acquire preResumeThreadCountLock");
 642   }
JASSERT(preResumeThreadCount > 0) (dmtcp_gettid()) (preResumeThreadCount);
preResumeThreadCount--;
 645   if (_real_pthread_mutex_unlock(&preResumeThreadCountLock) != 0) {
JASSERT(false) .Text("Failed to release preResumeThreadCountLock");
 647   }
 648 }
 650 void ThreadSync::waitForUserThreadsToFinishPreResumeCB()
 651 {
 652   if (preResumeThreadCount != INVALID_USER_THREAD_COUNT) {
 653     while (preResumeThreadCount != 0) {
 654       struct timespec sleepTime = {0, 10*1000*1000};
nanosleep(&sleepTime, NULL);
 656     }
 657   }
 658   // Now we wait for the lock to make sure that the user threads have released
 659   // it.
 660   if (_real_pthread_mutex_lock(&preResumeThreadCountLock) != 0) {
JASSERT(false) .Text("Failed to acquire preResumeThreadCountLock");
 662   }
 663   if (_real_pthread_mutex_unlock(&preResumeThreadCountLock) != 0) {
JASSERT(false) .Text("Failed to release preResumeThreadCountLock");
 665   }
 666 }