root/threadlist.cpp

DEFINITIONS

1. lock_threads
2. unlk_threads
3. save_sp
4. resetOnFork
5. init
6. initThread
7. threadExit
8. updateTid
9. killCkpthread
10. prepareMtcpHeader
11. checkpointhread
12. suspendThreads
13. stopthisthread
14. waitForAllRestored
15. postRestart
16. restarthread
17. Thread_UpdateState
18. Thread_SaveSigState
19. Thread_RestoreSigState
20. addToActiveList
21. threadIsDead
22. getNewThread
23. emptyFreeList

   1 #include <pthread.h>
   2 #include <signal.h>
   3 #include <sys/types.h>
   4 #include <sys/syscall.h>
   5 #include <unistd.h>
   6 #include <semaphore.h>
   7 #include <sys/resource.h>
   8 #include <linux/version.h>
   9 #include "config.h"
  10 #ifdef HAS_PR_SET_PTRACER
  11 # include <sys/prctl.h>
  12 #endif
  13 #include "threadlist.h"
  14 #include "siginfo.h"
  15 #include "dmtcpalloc.h"
  16 #include "syscallwrappers.h"
  17 #include "mtcpinterface.h"
  18 #include "ckptserializer.h"
  19 #include "uniquepid.h"
  20 #include "jalloc.h"
  21 #include "jassert.h"
  22 #include "util.h"
  23 #include "mtcp/mtcp_header.h"
  25 // For i386 and x86_64, SETJMP currently has bugs.  Don't turn this
  26 //   on for them until they are debugged.
  27 // Default is to use  setcontext/getcontext.
  28 #if defined(__arm__) || defined(__aarch64__)
  29 # define SETJMP /* setcontext/getcontext not defined for ARM glibc */
  30 #endif
  32 #ifdef SETJMP
  33 # include <setjmp.h>
  34 #else
  35 # include <ucontext.h>
  36 #endif
  39 using namespace dmtcp;
  41 //Globals
  42 volatile int restoreInProgress = 0;
Thread *motherofall = NULL;
  44 void **motherofall_saved_sp = NULL;
ThreadTLSInfo *motherofall_tlsInfo = NULL;
pid_t motherpid = 0;
sigset_t sigpending_global;
Thread *activeThreads = NULL;
  49 void *saved_sysinfo;
MYINFO_GS_T myinfo_gs __attribute__ ((visibility ("hidden")));
  53 static Thread *threads_freelist = NULL;
  54 static pthread_mutex_t threadlistLock = PTHREAD_MUTEX_INITIALIZER;
  55 static pthread_mutex_t threadStateLock = PTHREAD_MUTEX_INITIALIZER;
  57 static pthread_rwlock_t *threadResumeLock = NULL;
  59 static __thread Thread *curThread = NULL;
  60 static Thread *ckptThread = NULL;
  61 static int numUserThreads = 0;
  62 static int originalstartup;
  64 static sem_t sem_start;
  65 static sem_t semNotifyCkptThread;
  66 static sem_t semWaitForCkptThreadSignal;
  68 static void *checkpointhread (void *dummy);
  69 static void suspendThreads();
  70 static void stopthisthread(int sig);
  71 static int restarthread(void *threadv);
  72 static int Thread_UpdateState(Thread *th,
ThreadState newval,
ThreadState oldval);
  75 static void Thread_SaveSigState(Thread *th);
  76 static void Thread_RestoreSigState(Thread *th);
  78 /*****************************************************************************
  79  *
  80  * Lock and unlock the 'activeThreads' list
  81  *
  82  *****************************************************************************/
  83 static void lock_threads (void) {
JASSERT(_real_pthread_mutex_lock(&threadlistLock) == 0) (JASSERT_ERRNO);
  85 }
  86 static void unlk_threads (void) {
JASSERT(_real_pthread_mutex_unlock(&threadlistLock) == 0) (JASSERT_ERRNO);
  88 }
  90 /*****************************************************************************
  91  *
  92  * We will use the region beyond the end of stack for our temporary stack.
  93  * glibc sigsetjmp will mangle pointers;  We need the unmangled pointer.
  94  * So, we can't rely on parsing the jmpbuf for the saved sp.
  95  *
  96  *****************************************************************************/
  97 static void save_sp(void **sp)
  98 {
  99 #if defined(__i386__) || defined(__x86_64__)
 100   asm volatile (CLEAN_FOR_64_BIT(mov %%esp,%0)
 101                 : "=g" (*sp)
 102                 : : "memory");
 103 #elif defined(__arm__) || defined(__aarch64__)
 104   asm volatile ("mov %0,sp"
 105                 : "=r" (*sp)
 106                 : : "memory");
 107 #else
 108 # error "assembly instruction not translated"
 109 #endif
 110 }
 112 /*****************************************************************************
 113  *
 114  * Get _real_ tid/pid
 115  *
 116  *****************************************************************************/
 118 /*****************************************************************************
 119  *
 120  * New process. Empty the activeThreads list
 121  *
 122  *****************************************************************************/
 123 void ThreadList::resetOnFork()
 124 {
lock_threads();
 126   while (activeThreads != NULL) {
ThreadList::threadIsDead(activeThreads); // takes care of updating "activeThreads" ptr.
 128   }
unlk_threads();
 130 }
 132 /*****************************************************************************
 133  *
 134  *  This routine must be called at startup time to initiate checkpointing
 135  *
 136  *****************************************************************************/
 137 void ThreadList::init()
 138 {
 139   /* Save this process's pid.  Then verify that the TLS has it where it should
 140    * be. When we do a restore, we will have to modify each thread's TLS with the
 141    * new motherpid. We also assume that GS uses the first GDT entry for its
 142    * descriptor.
 143    */
 145   /* libc/getpid can lie if we had used kernel fork() instead of libc fork(). */
motherpid = THREAD_REAL_TID();
TLSInfo_VerifyPidTid(motherpid, motherpid);
SigInfo::setupCkptSigHandler(&stopthisthread);
 151   /* Set up caller as one of our threads so we can work on it */
motherofall = ThreadList::getNewThread();
motherofall_saved_sp = &motherofall->saved_sp;
motherofall_tlsInfo = &motherofall->tlsInfo;
updateTid(motherofall);
sem_init(&sem_start, 0, 0);
sem_init(&semNotifyCkptThread, 0, 0);
sem_init(&semWaitForCkptThreadSignal, 0, 0);
originalstartup = 1;
pthread_t checkpointhreadid;
 163   /* Spawn off a thread that will perform the checkpoints from time to time */
JASSERT(pthread_create(&checkpointhreadid, NULL, checkpointhread, NULL) == 0);
 166   /* Stop until checkpoint thread has finished initializing.
 167    * Some programs (like gcl) implement their own glibc functions in
 168    * a non-thread-safe manner.  In case we're using non-thread-safe glibc,
 169    * don't run the checkpoint thread and user thread at the same time.
 170    */
errno = 0;
 172   while (-1 == sem_wait(&sem_start) && errno == EINTR)
errno = 0;
sem_destroy(&sem_start);
 175 }
 177 /*****************************************************************************
 178  *
 179  *****************************************************************************/
 180 void ThreadList::initThread(Thread* th, int (*fn)(void*), void *arg, int flags,
 181                             int *ptid, int *ctid)
 182 {
 183   /* Save exactly what the caller is supplying */
th->fn    = fn;
th->arg   = arg;
th->flags = flags;
th->ptid  = ptid;
th->ctid  = ctid;
th->next  = NULL;
th->state = ST_RUNNING;
 192   /* libpthread may recycle the thread stacks after the thread exits (due to
 193    * return, pthread_exit, or pthread_cancel) by reusing them for a different
 194    * thread created by a subsequent call to pthread_create().
 195    *
 196    * Part of thread-stack also contains the "struct pthread" with pid and tid
 197    * as member fields. While reusing the stack for the new thread, the tid
 198    * field is reset but the pid field is left unchanged (under the assumption
 199    * that pid never changes). This causes a problem if the thread exited before
 200    * checkpoint and the new thread is created after restart and hence the pid
 201    * field contains the wrong value (pre-ckpt pid as opposed to current-pid).
 202    *
 203    * The solution is to put the motherpid in the pid slot every time a new
 204    * thread is created to make sure that struct pthread has the correct value.
 205    */
TLSInfo_UpdatePid();
 207 }
 209 /*****************************************************************************
 210  *
 211  * Thread exited/exiting.
 212  *
 213  *****************************************************************************/
 214 void ThreadList::threadExit()
 215 {
curThread->state = ST_ZOMBIE;
 217 }
 219 /*****************************************************************************
 220  *
 221  *****************************************************************************/
 222 void ThreadList::updateTid(Thread *th)
 223 {
 224   if (curThread == NULL)
curThread = th;
th->tid = THREAD_REAL_TID();
th->virtual_tid = dmtcp_gettid();
JTRACE("starting thread") (th->tid) (th->virtual_tid);
 229   // Check and remove any thread descriptor which has the same tid as ours.
 230   // Also, remove any dead threads from the list.
ThreadList::addToActiveList();
 232 }
 234 /*************************************************************************
 235  *
 236  *  Send a signal to ckpt-thread to wake it up from select call and exit.
 237  *
 238  *************************************************************************/
 239 void ThreadList::killCkpthread()
 240 {
JTRACE("Kill checkpinthread") (ckptThread->tid);
THREAD_TGKILL(motherpid, ckptThread->tid, SigInfo::ckptSignal());
 243 }
 245 /*************************************************************************
 246  *
 247  *  Prepare MTCP Header
 248  *
 249  *************************************************************************/
 250 static void prepareMtcpHeader(MtcpHeader *mtcpHdr)
 251 {
memset(mtcpHdr, 0, sizeof(*mtcpHdr));
strncpy(mtcpHdr->signature, MTCP_SIGNATURE, strlen(MTCP_SIGNATURE) + 1);
mtcpHdr->saved_brk = sbrk(0);
 255   // TODO: Now that we have a separate mtcp dir, the code dealing with
 256   // restoreBuf should go in there.
mtcpHdr->restore_addr = (void*) ProcessInfo::instance().restoreBufAddr();
mtcpHdr->restore_size = ProcessInfo::instance().restoreBufLen();
mtcpHdr->vdsoStart = (void*) ProcessInfo::instance().vdsoStart();
mtcpHdr->vdsoEnd = (void*) ProcessInfo::instance().vdsoEnd();
mtcpHdr->vvarStart = (void*) ProcessInfo::instance().vvarStart();
mtcpHdr->vvarEnd = (void*) ProcessInfo::instance().vvarEnd();
mtcpHdr->post_restart = &ThreadList::postRestart;
memcpy(&mtcpHdr->motherofall_tls_info,
 267          &motherofall->tlsInfo,
 268          sizeof(motherofall->tlsInfo));
mtcpHdr->tls_pid_offset = TLSInfo_GetPidOffset();
mtcpHdr->tls_tid_offset = TLSInfo_GetTidOffset();
mtcpHdr->myinfo_gs = myinfo_gs;
 272 }
 274 /*************************************************************************
 275  *
 276  *  This executes as a thread.  It sleeps for the checkpoint interval
 277  *    seconds, then wakes to write the checkpoint file.
 278  *
 279  *************************************************************************/
 280 static void *checkpointhread (void *dummy)
 281 {
 282   /* This is the start function of the checkpoint thread.
 283    * We also call sigsetjmp/getcontext to get a snapshot of this call frame,
 284    * since we will never exit this call frame.  We always return
 285    * to this call frame at time of startup, on restart.  Hence, restart
 286    * will forget any modifications to our local variables since restart.
 287    */
ckptThread = curThread;
ckptThread->state = ST_CKPNTHREAD;
 292   /* For checkpoint thread, we want to block delivery of all but some special
 293    * signals
 294    */
 295   {
 296     /*
 297      * For the checkpoint thread, we should not block SIGSETXID which is used
 298      * by the setsid family of system calls to change the session leader. Glibc
 299      * uses this signal to notify the process threads of the change in session
 300      * leader information. This signal is not documented and is used internally
 301      * by glibc. It is defined in <glibc-src-root>/nptl/pthreadP.h
 302      * screen was getting affected by this since it used setsid to change the
 303      * session leaders.
 304      * Similarly, SIGCANCEL/SIGTIMER is undocumented, but used by glibc.
 305      */
 306 #define SIGSETXID (__SIGRTMIN + 1)
 307 #define SIGCANCEL (__SIGRTMIN) /* aka SIGTIMER */
sigset_t set;
sigfillset(&set);
sigdelset(&set, SIGSETXID);
sigdelset(&set, SIGCANCEL);
JASSERT(pthread_sigmask(SIG_SETMASK, &set, NULL) == 0);
 315   }
Thread_SaveSigState(ckptThread);
TLSInfo_SaveTLSState(&ckptThread->tlsInfo);
 319   /* Release user thread after we've initialized. */
sem_post(&sem_start);
 322   /* Set up our restart point.  I.e., we get jumped to here after a restore. */
 323 #ifdef SETJMP
JASSERT(sigsetjmp(ckptThread->jmpbuf, 1) >= 0) (JASSERT_ERRNO);
 325 #else
JASSERT(getcontext(&ckptThread->savctx) == 0) (JASSERT_ERRNO);
 327 #endif
save_sp(&ckptThread->saved_sp);
JTRACE("after sigsetjmp/getcontext")
 330     (curThread->tid) (curThread->virtual_tid) (curThread->saved_sp);
 332   if (originalstartup) {
originalstartup = 0;
 334   } else {
 335     /* We are being restored.  Wait for all other threads to finish being
 336      * restored before resuming checkpointing.
 337      */
JTRACE("waiting for other threads after restore");
ThreadList::waitForAllRestored(ckptThread);
JTRACE("resuming after restore");
 341   }
 343   /* This is a sleep-checkpoint-resume loop by the checkpoint thread.
 344    * On restart, we arrive back at getcontext, above, and then re-enter the loop.
 345    */
 346   while (1) {
 347     /* Wait a while between writing checkpoint files */
JTRACE("before callbackSleepBetweenCheckpoint(0)");
callbackSleepBetweenCheckpoint(0);
restoreInProgress = 0;
 353     // We need to reinitialize the lock.
pthread_rwlock_t rwLock = PTHREAD_RWLOCK_INITIALIZER;
threadResumeLock = &rwLock;
JASSERT(_real_pthread_rwlock_wrlock(threadResumeLock) == 0) (JASSERT_ERRNO);
suspendThreads();
SigInfo::saveSigHandlers();
 360     /* Do this once, same for all threads.  But restore for each thread. */
 361     if (TLSInfo_HaveThreadSysinfoOffset())
saved_sysinfo = TLSInfo_GetThreadSysinfo();
 364     /* All other threads halted in 'stopthisthread' routine (they are all
 365      * in state ST_SUSPENDED).  It's safe to write checkpoint file now.
 366      */
 368     // Update generation, in case user callback calls dmtcp_get_generation().
uint32_t computation_generation =
SharedData::getCompId()._computation_generation;
ProcessInfo::instance().set_generation(computation_generation);
JTRACE("before callbackSleepBetweenCheckpoint(0)");
callbackPreCheckpoint();
 376     // Remove stale threads from activeThreads list.
ThreadList::emptyFreeList();
MtcpHeader mtcpHdr;
prepareMtcpHeader(&mtcpHdr);
 381     /* That's it, folks.  We just did the checkpoint.  After this, we will meet
 382      *   on the flip side of checkpoint.
 383      */
CkptSerializer::writeCkptImage(&mtcpHdr, sizeof(mtcpHdr));
 386     /* NOTE: This code is only for the checkpoint thread.  If you're looking for
 387      *      what the user threads do at checkpoint time, see:  stopthisthread()
 388      *
 389      * There are two ways for the checkpoint thread to return from a checkpoint:
 390      *                 resume and restart
 391      * If we're here, we just resume'd after checkpoint.  It's the same process.
 392      * If we chose checkpoint, 'bin/mtcp_restart' created a new process.  The
 393      *   source code is in 'src/mtcp'.  The program 'bin/mtcp_restart' will map
 394      *   our memory into the new process, and then meet us back here by calling
 395      *   the function specified by 'mtcpHdr->post_restart':
 396      *                                        ThreadList::postRestart().
 397      *   Actually, postRestart() will start the user threads and then call
 398      *   restarthread() for the 'motherofall' thread.  Then, restarthread()
 399      *   will call setcontext(), in order to arrive back at getcontext() here
 400      *   in this function, just before the 'while(1)' loop.
 401      * FIXME:  The 'motherofall' thread is the primary thread of the process.
 402      *   On launch, 'motherofall' was the user thread exeicuting main().
 403      *   and the checkpoint thread was the second thread.  But now,
 404      *   motherofall will be the checkpoint thread.  Why do we switch at the
 405      *   time of restart?  Should we fix this?
 406      */
JTRACE("before callbackPostCheckpoint(0, NULL)");
callbackPostCheckpoint(0, NULL);
 410     /* Resume all threads. */
JTRACE("resuming everything");
JASSERT(_real_pthread_rwlock_unlock(threadResumeLock) == 0) (JASSERT_ERRNO);
JTRACE("everything resumed");
 414   }
 415   return NULL;
 416 }
 418 static void suspendThreads()
 419 {
 420   int needrescan;
Thread *thread;
Thread *next;
 424   /* Halt all other threads - force them to call stopthisthread
 425    * If any have blocked checkpointing, wait for them to unblock before
 426    * signalling
 427    */
lock_threads();
 429   do {
needrescan = 0;
numUserThreads = 0;
 432     for (thread = activeThreads; thread != NULL; thread = next) {
next = thread->next;
 434       int ret;
 435       /* Do various things based on thread's state */
 436       switch (thread->state) {
 438         case ST_RUNNING:
 439           /* Thread is running. Send it a signal so it will call stopthisthread.
 440            * We will need to rescan (hopefully it will be suspended by then)
 441            */
 442           if (Thread_UpdateState(thread, ST_SIGNALED, ST_RUNNING)) {
 443             if (THREAD_TGKILL(motherpid, thread->tid, SigInfo::ckptSignal()) < 0) {
JASSERT(errno == ESRCH) (JASSERT_ERRNO) (thread->tid)
 445                 .Text("error signalling thread");
ThreadList::threadIsDead(thread);
 447             } else {
needrescan = 1;
 449             }
 450           }
 451           break;
 453         case ST_ZOMBIE:
ret = THREAD_TGKILL(motherpid, thread->tid, 0);
JASSERT(ret == 0 || errno == ESRCH);
 456           if (ret == -1 && errno == ESRCH) {
ThreadList::threadIsDead(thread);
 458           }
 459           break;
 461         case ST_SIGNALED:
 462           if (THREAD_TGKILL(motherpid, thread->tid, 0) == -1 && errno == ESRCH) {
ThreadList::threadIsDead(thread);
 464           } else {
needrescan = 1;
 466           }
 467           break;
 469         case ST_SUSPINPROG:
numUserThreads++;
 471           break;
 473         case ST_SUSPENDED:
numUserThreads++;
 475           break;
 477         case ST_CKPNTHREAD:
 478           break;
 480         default:
JASSERT(false);
 482       }
 483     }
 484     if (needrescan) usleep(10);
 485   } while (needrescan);
unlk_threads();
 488   for (int i = 0; i < numUserThreads; i++) {
sem_wait(&semNotifyCkptThread);
 490   }
JASSERT(activeThreads != NULL);
JTRACE("everything suspended") (numUserThreads);
 494 }
 496 /*************************************************************************
 497  *
 498  *  Signal handler for user threads.
 499  *
 500  *************************************************************************/
 501 void stopthisthread (int signum)
 502 {
 503   // If this is checkpoint thread, exit immediately
 504   if (curThread == ckptThread) return;
 506   /* Possible state change scenarios:
 507    * 1. STOPSIGNAL received from ckpt-thread. In this case, the ckpt-thread
 508    * already changed the state to ST_SIGNALED. No need to check for locks.
 509    * Proceed normally.
 510    *
 511    * 2. STOPSIGNAL received from Superior thread. In this case we change the
 512    * state to ST_SIGNALED, if currently in ST_RUNNING. If we are holding
 513    * any locks (callback_holds_any_locks), we return from the signal handler.
 514    *
 515    * 3. STOPSIGNAL raised by this thread itself, after releasing all the locks.
 516    * In this case, we had already changed the state to ST_SIGNALED as a
 517    * result of step (2), so the ckpt-thread will never send us a signal.
 518    *
 519    * 4. STOPSIGNAL received from Superior thread. Ckpt-threads sends a signal
 520    * before we had a chance to change state from ST_RUNNING ->
 521    * ST_SIGNALED. This puts the STOPSIGNAL in the queue. The ckpt-thread will
 522    * later call sigaction(STOPSIGNAL, SIG_IGN) followed by
 523    * sigaction(STOPSIGNAL, stopthisthread) to discard all pending signals.
 524    */
 525   if (Thread_UpdateState(curThread, ST_SIGNALED, ST_RUNNING)) {
 526     int retval;
callbackHoldsAnyLocks(&retval);
 528     if (retval) return;
 529   }
 531   // make sure we don't get called twice for same thread
 532   if (Thread_UpdateState(curThread, ST_SUSPINPROG, ST_SIGNALED)) {
Thread_SaveSigState(curThread); // save sig state (and block sig delivery)
TLSInfo_SaveTLSState(&curThread->tlsInfo); // save thread local storage state
 536     /* Set up our restart point, ie, we get jumped to here after a restore */
 537 #ifdef SETJMP
JASSERT(sigsetjmp(curThread->jmpbuf, 1) >= 0);
 539 #else
JASSERT(getcontext(&curThread->savctx) == 0);
 541 #endif
save_sp(&curThread->saved_sp);
JTRACE("Thread after sigsetjmp/getcontext")
 545       (curThread->tid) (curThread->virtual_tid)
 546       (curThread->saved_sp) (__builtin_return_address(0));
 548     if (!restoreInProgress) {
 549       /* We are a user thread and all context is saved.
 550        * Wait for ckpt thread to write ckpt, and resume.
 551        */
 553       /* This sets a static variable in dmtcp.  It must be passed
 554        * from this user thread to ckpt thread before writing ckpt image
 555        */
 556       if (dmtcp_ptrace_enabled == NULL) {
callbackPreSuspendUserThread();
 558       }
 560       /* Tell the checkpoint thread that we're all saved away */
JASSERT(Thread_UpdateState(curThread, ST_SUSPENDED, ST_SUSPINPROG));
sem_post(&semNotifyCkptThread);
 564       /* This sets a static variable in dmtcp.  It must be passed
 565        * from this user thread to ckpt thread before writing ckpt image
 566        */
 567       if (dmtcp_ptrace_enabled != NULL && dmtcp_ptrace_enabled()) {
callbackPreSuspendUserThread();
 569       }
 571       /* Then wait for the ckpt thread to write the ckpt file then wake us up */
JTRACE("User thread suspended") (curThread->tid);
 574       // We can't use sem_wait here because sem_wait registers a cleanup
 575       // handler before going into blocking wait. The handler is popped before
 576       // returning from it. However, on restart, the thread will do a longjump
 577       // and thus will never come out of the sem_wait, thus the handler is
 578       // never popped. This causes a problem later on during pthread_exit. The
 579       // pthread_exit routine executes all registered cleanup handlers.
 580       // However, the sem_wait cleanup handler is now invalid and thus we get a
 581       // segfault.
 582       // The change in sem_wait behavior was first introduce in glibc 2.21.
JASSERT(_real_pthread_rwlock_rdlock(threadResumeLock) == 0)
 584         (JASSERT_ERRNO);
JASSERT(_real_pthread_rwlock_unlock(threadResumeLock) == 0)
 586         (JASSERT_ERRNO);
JTRACE("User thread resuming") (curThread->tid);
 589     } else {
 590       /* Else restoreinprog >= 1;  This stuff executes to do a restart */
ThreadList::waitForAllRestored(curThread);
JTRACE("User thread restored") (curThread->tid);
 593     }
JASSERT(Thread_UpdateState(curThread, ST_RUNNING, ST_SUSPENDED));
callbackPreResumeUserThread(restoreInProgress);
JTRACE("User thread returning to user code")
 600       (curThread->tid) (__builtin_return_address(0));
 601   }
 602 }
 604 /*****************************************************************************
 605  *
 606  *  Wait for all threads to finish restoring their context, then release them
 607  *  all to continue on their way.
 608  *
 609  *****************************************************************************/
 610 void ThreadList::waitForAllRestored(Thread *thread)
 611 {
 612   if (thread == ckptThread) {
 613     int i;
 614     for (i = 0; i < numUserThreads; i++) {
sem_wait(&semNotifyCkptThread);
 616     }
JTRACE("before callback_post_ckpt(1=restarting)");
callbackPostCheckpoint(1, NULL); //mtcp_restoreargv_start_addr);
JTRACE("after callback_post_ckpt(1=restarting)");
SigInfo::restoreSigHandlers();
 624     /* raise the signals which were pending for the entire process at the time
 625      * of checkpoint. It is assumed that if a signal is pending for all threads
 626      * including the ckpt-thread, then it was sent to the process as opposed to
 627      * sent to individual threads.
 628      */
 629     for (i = SIGRTMAX; i > 0; --i) {
 630       if (sigismember(&sigpending_global, i) == 1) {
kill(getpid(), i);
 632       }
 633     }
 635     // if this was last of all, wake everyone up
 636     for (i = 0; i < numUserThreads; i++) {
sem_post(&semWaitForCkptThreadSignal);
 638     }
 639   } else {
sem_post(&semNotifyCkptThread);
sem_wait(&semWaitForCkptThreadSignal);
Thread_RestoreSigState(thread);
 643   }
 644 }
 646 /*****************************************************************************
 647  *
 648  *****************************************************************************/
 649 void ThreadList::postRestart(void)
 650 {
Thread *thread;
sigset_t tmp;
 654   /* If DMTCP_RESTART_PAUSE set, sleep 15 seconds and allow gdb attach. */
 655   if (getenv("MTCP_RESTART_PAUSE") || getenv("DMTCP_RESTART_PAUSE")) {
 656 #ifdef HAS_PR_SET_PTRACER
prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0); // Allow 'gdb attach'
 658 #endif
 659     struct timespec delay = {15, 0}; /* 15 seconds */
printf("Pausing 15 seconds. Do:  gdb <PROGNAME> %ld\n",
 661            (long)THREAD_REAL_TID());
nanosleep(&delay, NULL);
 663 #ifdef HAS_PR_SET_PTRACER
prctl(PR_SET_PTRACER, 0, 0, 0, 0); ; // Revert permission to default.
 665 #endif
 666   }
 668   /* Fill in the new mother process id */
motherpid = THREAD_REAL_TID();
motherofall->tid = motherpid;
restoreInProgress = 1;
sigfillset(&tmp);
 675   for (thread = activeThreads; thread != NULL; thread = thread->next) {
 676     struct MtcpRestartThreadArg mtcpRestartThreadArg;
 677     sigandset(&sigpending_global, &tmp, &(thread->sigpending));
tmp = sigpending_global;
 680     if (thread == motherofall) continue;
 682     /* DMTCP needs to know virtual_tid of the thread being recreated by the
 683      *  following clone() call.
 684      *
 685      * Threads are created by using syscall which is intercepted by DMTCP and
 686      *  the virtual_tid is sent to DMTCP as a field of MtcpRestartThreadArg
 687      *  structure. DMTCP will automatically extract the actual argument
 688      *  (clonearg->arg) from clone_arg and will pass it on to the real
 689      *  clone call.
 690      */
 691     void *clonearg = thread;
 692     if (dmtcp_real_to_virtual_pid != NULL) {
mtcpRestartThreadArg.arg = thread;
mtcpRestartThreadArg.virtualTid = thread->virtual_tid;
clonearg = &mtcpRestartThreadArg;
 696     }
 698     /* Create the thread so it can finish restoring itself. */
pid_t tid = _real_clone(restarthread,
 700                             // -128 for red zone
 701                             (void*)((char*)thread->saved_sp - 128),
 702                             /* Don't do CLONE_SETTLS (it'll puke).  We do it
 703                              * later via restoreTLSState. */
thread->flags & ~CLONE_SETTLS,
clonearg, thread->ptid, NULL, thread->ctid);
JASSERT (tid > 0); // (JASSERT_ERRNO) .Text("Error recreating thread");
JTRACE("Thread recreated") (thread->tid) (tid);
 709   }
restarthread (motherofall);
 711 }
 713 /*****************************************************************************
 714  *
 715  *****************************************************************************/
 716 static int restarthread (void *threadv)
 717 {
Thread *thread = (Thread*) threadv;
thread->tid = THREAD_REAL_TID();
 720   // This function and related ones are defined in src/mtcp/restore_libc.c
TLSInfo_RestoreTLSState(&thread->tlsInfo);
 723   if (TLSInfo_HaveThreadSysinfoOffset())
TLSInfo_SetThreadSysinfo(saved_sysinfo);
 726   /* Jump to the stopthisthread routine just after sigsetjmp/getcontext call.
 727    * Note that if this is the restored checkpointhread, it jumps to the
 728    * checkpointhread routine
 729    */
JTRACE("calling siglongjmp/setcontext") (thread->tid) (thread->virtual_tid);
 731 #ifdef SETJMP
 732   siglongjmp(thread->jmpbuf, 1); /* Shouldn't return */
 733 #else
 734   setcontext(&thread->savctx); /* Shouldn't return */
 735 #endif
JASSERT(false);
 737   return (0); /* NOTREACHED : stop compiler warning */
 738 }
 740 /*****************************************************************************
 741  *
 742  *****************************************************************************/
 743 int Thread_UpdateState(Thread *th, ThreadState newval, ThreadState oldval)
 744 {
 745   int res = 0;
JASSERT(_real_pthread_mutex_lock(&threadStateLock) == 0);
 747   if (oldval == th->state) {;
th->state = newval;
res = 1;
 750   }
JASSERT(_real_pthread_mutex_unlock(&threadStateLock) == 0);
 752   return res;
 753 }
 755 /*****************************************************************************
 756  *
 757  *  Save signal mask and list of pending signals delivery
 758  *
 759  *****************************************************************************/
 760 void Thread_SaveSigState(Thread *th)
 761 {
 762   // Save signal block mask
JASSERT(pthread_sigmask (SIG_SETMASK, NULL, &th->sigblockmask) == 0);
 765   // Save pending signals
sigpending(&th->sigpending);
 767 }
 769 /*****************************************************************************
 770  *
 771  *  Restore signal mask and all pending signals
 772  *
 773  *****************************************************************************/
 774 void Thread_RestoreSigState (Thread *th)
 775 {
 776   int i;
JTRACE("restoring signal mask for thread") (th->virtual_tid);
JASSERT(pthread_sigmask (SIG_SETMASK, &th->sigblockmask, NULL) == 0);
 780   // Raise the signals which were pending for only this thread at the time of
 781   // checkpoint.
 782   for (i = SIGRTMAX; i > 0; --i) {
 783     if (sigismember(&th->sigpending, i)  == 1  &&
sigismember(&th->sigblockmask, i) == 1 &&
sigismember(&sigpending_global, i) == 0 &&
i != dmtcp_get_ckpt_signal()) {
 787       if (i != SIGCHLD) {
JNOTE("\n*** WARNING:  SIGCHLD was delivered prior to ckpt.\n"
 789                "*** Will raise it on restart.  If not desired, change\n"
 790                "*** this line raising SIGCHLD.");
 791       }
raise(i);
 793     }
 794   }
 795 }
 798 /*****************************************************************************
 799  *
 800  * If there is a thread descriptor with the same tid, it must be from a dead
 801  * thread. Remove it now.
 802  *
 803  *****************************************************************************/
 804 void ThreadList::addToActiveList()
 805 {
 806   int tid;
Thread *thread;
Thread *next_thread;
lock_threads();
tid = curThread->tid;
JASSERT (tid != 0);
 815   // First remove duplicate descriptors.
 816   for (thread = activeThreads; thread != NULL; thread = next_thread) {
next_thread = thread->next;
 818     if (thread != curThread && thread->tid == tid) {
JTRACE("Removing duplicate thread descriptor")
 820         (thread->tid) (thread->virtual_tid);
 821       // There will be at most one duplicate descriptor.
threadIsDead(thread);
 823       continue;
 824     }
 825     /* NOTE:  ST_ZOMBIE is used only for the sake of efficiency.  We
 826      *   test threads in state ST_ZOMBIE using tgkill to remove them
 827      *   early (before reaching a checkpoint) so that the
 828      *   threadrdescriptor list does not grow too long.
 829      */
 830     if (thread->state == ST_ZOMBIE) {
 831       /* if no thread with this tid, then we can remove zombie descriptor */
 832       if (-1 == THREAD_TGKILL(motherpid, thread->tid, 0)) {
JTRACE("Killing zombie thread") (thread->tid);
threadIsDead(thread);
 835       }
 836     }
 837   }
curThread->next = activeThreads;
curThread->prev = NULL;
 841   if (activeThreads != NULL) {
activeThreads->prev = curThread;
 843   }
activeThreads = curThread;
unlk_threads();
 847   return;
 848 }
 850 /*****************************************************************************
 851  *
 852  *  Thread has exited - move it from activeThreads list to freelist.
 853  *
 854  *  threadisdead() used to free() the Thread struct before returning. However,
 855  *  if we do that while in the middle of a checkpoint, the call to free() might
 856  *  deadlock in JAllocator. For this reason, we put the to-be-removed threads
 857  *  on this threads_freelist and call free() only when it is safe to do so.
 858  *
 859  *  This has an added benefit of reduced number of calls to malloc() as the
 860  *  Thread structs in the freelist can be recycled.
 861  *
 862  *****************************************************************************/
 863 void ThreadList::threadIsDead (Thread *thread)
 864 {
JASSERT(thread != NULL);
JTRACE("Putting thread on freelist") (thread->tid);
 868   /* Remove thread block from 'threads' list */
 869   if (thread->prev != NULL) {
thread->prev->next = thread->next;
 871   }
 872   if (thread->next != NULL) {
thread->next->prev = thread->prev;
 874   }
 875   if (thread == activeThreads) {
activeThreads = activeThreads->next;
 877   }
thread->next = threads_freelist;
threads_freelist = thread;
 881 }
 883 /*****************************************************************************
 884  *
 885  * Return thread from freelist.
 886  *
 887  *****************************************************************************/
Thread *ThreadList::getNewThread()
 889 {
Thread *thread;
lock_threads();
 893   if (threads_freelist == NULL) {
thread = (Thread*) JALLOC_HELPER_MALLOC(sizeof(Thread));
JASSERT(thread != NULL);
 896   } else {
thread = threads_freelist;
threads_freelist = threads_freelist->next;
 899   }
unlk_threads();
memset(thread, 0, sizeof (*thread));
 902   return thread;
 903 }
 905 /*****************************************************************************
 906  *
 907  * Call free() on all threads_freelist items
 908  *
 909  *****************************************************************************/
 910 void ThreadList::emptyFreeList()
 911 {
lock_threads();
 914   while (threads_freelist != NULL) {
Thread *thread = threads_freelist;
threads_freelist = threads_freelist->next;
JALLOC_HELPER_FREE(thread);
 918   }
unlk_threads();
 921 }