root/mtcp/restore_libc.c

DEFINITIONS

1. STATIC_TLS_TID_OFFSET
2. TLSInfo_GetTidOffset
3. TLSInfo_GetPidOffset
4. memsubarray
5. get_tls_segreg
6. get_tls_base_addr
7. get_at_sysinfo
8. TLSInfo_HaveThreadSysinfoOffset
9. TLSInfo_GetThreadSysinfo
10. TLSInfo_SetThreadSysinfo
11. TLSInfo_VerifyPidTid
12. TLSInfo_UpdatePid
13. TLSInfo_SaveTLSState
14. TLSInfo_RestoreTLSState
15. TLSInfo_PostRestart

   1 /*****************************************************************************
   2  * Copyright (C) 2010-2014 Kapil Arya <kapil@ccs.neu.edu>                    *
   3  * Copyright (C) 2010-2014 Gene Cooperman <gene@ccs.neu.edu>                 *
   4  *                                                                           *
   5  * DMTCP is free software: you can redistribute it and/or                    *
   6  * modify it under the terms of the GNU Lesser General Public License as     *
   7  * published by the Free Software Foundation, either version 3 of the        *
   8  * License, or (at your option) any later version.                           *
   9  *                                                                           *
  10  * DMTCP is distributed in the hope that it will be useful,                  *
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of            *
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             *
  13  * GNU Lesser General Public License for more details.                       *
  14  *                                                                           *
  15  * You should have received a copy of the GNU Lesser General Public          *
  16  * License along with DMTCP.  If not, see <http://www.gnu.org/licenses/>.    *
  17  *****************************************************************************/
  19 #include <elf.h>
  20 #include <stdlib.h>
  21 #include <unistd.h>
  22 #include <string.h>
  23 #include <errno.h>
  24 #include <sys/syscall.h>
  25 #include <sys/resource.h>
  26 #include <sys/personality.h>
  27 #include <linux/version.h>
  28 #include <gnu/libc-version.h>
  29 #include "mtcp_sys.h"
  30 #include "restore_libc.h"
  31 #include "tlsutil.h"
  33 int mtcp_sys_errno;
  35 #if defined(__x86_64__) || defined(__aarch64__)
  36 # define ELF_AUXV_T Elf64_auxv_t
  37 # define UINT_T uint64_t
  38 #else
  39   // else __i386__ and __arm__
  40 # define ELF_AUXV_T Elf32_auxv_t
  41 # define UINT_T uint32_t
  42 #endif
  44 extern MYINFO_GS_T myinfo_gs;
  47 /* Offset computed (&x.pid - &x) for
  48  *   struct pthread x;
  49  * as found in:  glibc-2.5/nptl/descr.h
  50  * It was 0x4c and 0x48 for pid and tid for i386.
  51  * Roughly, the definition is:
  52  *glibc-2.5/nptl/descr.h:
  53  * struct pthread
  54  * {
  55  *  union {
  56  *   tcbheader_t tcbheader;
  57  *   void *__padding[16];
  58  *  };
  59  *  list_t list;
  60  *  pid_t tid;
  61  *  pid_t pid;
  62  *  ...
  63  * } __attribute ((aligned (TCB_ALIGNMENT)));
  64  *
  65  *glibc-2.5/nptl/sysdeps/pthread/list.h:
  66  * typedef struct list_head
  67  * {
  68  *  struct list_head *next;
  69  *  struct list_head *prev;
  70  * } list_t;
  71  *
  72  * NOTE: glibc-2.10 changes the size of __padding from 16 to 24.  --KAPIL
  73  *
  74  * NOTE: glibc-2.11 further changes the size tcphead_t without updating the
  75  *       size of __padding in struct pthread. We need to add an extra 512 bytes
  76  *       to accommodate this.                                    -- KAPIL
  77  */
  79 #if !__GLIBC_PREREQ (2,1)
  80 # error "glibc version too old"
  81 #endif
  83 // NOTE: tls_tid_offset, tls_pid_offset determine offset independently of
  84 //     glibc version.  These STATIC_... versions serve as a double check.
  85 // Calculate offsets of pid/tid in pthread 'struct user_desc'
  86 // The offsets are needed for two reasons:
  87 //  1. glibc pthread functions cache the pid; must update this after restart
  88 //  2. glibc pthread functions cache the tid; pthread functions pass address
  89 //     of cached tid to clone, and MTCP grabs it; But MTCP is still missing
  90 //     the address where pthread cached the tid of motherofall.  So, it can't
  91 //     update.
  92 static int STATIC_TLS_TID_OFFSET()
  93 {
  94   static int offset = -1;
  95   if (offset != -1)
  96     return offset;
  98   char *ptr;
  99   long major = strtol(gnu_get_libc_version(), &ptr, 10);
 100   long minor = strtol(ptr+1, NULL, 10);
ASSERT (major == 2);
 103   if (minor >= 11) {
 104 #ifdef __x86_64__
offset = 26*sizeof(void *) + 512;
 106 #else
offset = 26*sizeof(void *);
 108 #endif
 109   } else if (minor == 10) {
offset = 26*sizeof(void *);
 111   } else {
offset = 18*sizeof(void *);
 113   }
 115   return offset;
 116 }
 118 #if 0
 119 # if __GLIBC_PREREQ (2,11)
 120 #  ifdef __x86_64__
 121 #   define STATIC_TLS_TID_OFFSET() (26*sizeof(void *) + 512)
 122 #  else
 123 #   define STATIC_TLS_TID_OFFSET() (26*sizeof(void *))
 124 #  endif
 126 # elif __GLIBC_PREREQ (2,10)
 127 #   define STATIC_TLS_TID_OFFSET() (26*sizeof(void *))
 129 # else
 130 #   define STATIC_TLS_TID_OFFSET() (18*sizeof(void *))
 131 # endif
 132 #endif
 134 # define STATIC_TLS_PID_OFFSET() (STATIC_TLS_TID_OFFSET() + sizeof(pid_t))
 136 /* WHEN WE HAVE CONFIDENCE IN THIS VERSION, REMOVE ALL OTHER __GLIBC_PREREQ
 137  * AND MAKE THIS THE ONLY VERSION.  IT SHOULD BE BACKWARDS COMPATIBLE.
 138  */
 139 /* These function definitions should succeed independently of the glibc version.
 140  * They use get_thread_area() to match (tid, pid) and find offset.
 141  * In other code, on restart, that offset is used to set (tid,pid) to
 142  *   the latest tid and pid of the new thread, instead of the (tid,pid)
 143  *   of the original thread.
 144  * SEE: "struct pthread" in glibc-2.XX/nptl/descr.h for 'struct pthread'.
 145  */
 147 /* Can remove the unused attribute when this __GLIBC_PREREQ is the only one. */
 148 static char *memsubarray (char *array, char *subarray, size_t len)
 149                                          __attribute__ ((unused));
 150 static uint64_t get_tls_segreg(void);
 151 static void *get_tls_base_addr(void);
 152 extern void **motherofall_saved_sp;
 153 extern ThreadTLSInfo *motherofall_tlsInfo;
 155 /*****************************************************************************
 156  *
 157  *****************************************************************************/
 158 int TLSInfo_GetTidOffset(void)
 160 {
 161   static int tid_offset = -1;
 162   if (tid_offset == -1) {
 163     struct {pid_t tid; pid_t pid;} tid_pid;
 164     /* struct pthread has adjacent fields, tid and pid, in that order.
 165      * Try to find at what offset that bit patttern occurs in struct pthread.
 166      */
 167     char * tmp;
tid_pid.tid = mtcp_sys_getpid();
tid_pid.pid = mtcp_sys_getpid();
 170     /* Get entry number of current thread descriptor from its segment register:
 171      * Segment register / 8 is the entry_number for the "thread area", which
 172      * is of type 'struct user_desc'.   The base_addr field of that struct
 173      * points to the struct pthread for the thread with that entry_number.
 174      * The tid and pid are contained in the 'struct pthread'.
 175      *   So, to access the tid/pid fields, first find the entry number.
 176      * Then fill in the entry_number field of an empty 'struct user_desc', and
 177      * get_thread_area(struct user_desc *uinfo) will fill in the rest.
 178      * Then use the filled in base_address field to get the 'struct pthread'.
 179      * The function tcp_get_tls_base_addr() returns this 'struct pthread' addr.
 180      */
 181     void * pthread_desc = get_tls_base_addr();
 182     /* A false hit for tid_offset probably can't happen since a new
 183      * 'struct pthread' is zeroed out before adding tid and pid.
 184      * pthread_desc below is defined as 'struct pthread' in glibc:nptl/descr.h
 185      */
tmp = memsubarray((char *)pthread_desc, (char *)&tid_pid, sizeof(tid_pid));
 187     if (tmp == NULL) {
PRINTF("WARNING: Couldn't find offsets of tid/pid in thread_area.\n"
 189              "  Now relying on the value determined using the\n"
 190              "  glibc version with which DMTCP was compiled.");
 191       return STATIC_TLS_TID_OFFSET();
 192       //mtcp_abort();
 193     }
tid_offset = tmp - (char *)pthread_desc;
 196     if (tid_offset != STATIC_TLS_TID_OFFSET()) {
PRINTF("WARNING: tid_offset (%d) different from expected.\n"
 198              "  It is possible that DMTCP was compiled with a different\n"
 199              "  glibc version than the one it's dynamically linking to.\n"
 200              "  Continuing anyway.  If this fails, please try again.",
tid_offset);
 202     }
DPRINTF("tid_offset: %d\n", tid_offset);
 204     if (tid_offset % sizeof(int) != 0) {
PRINTF("WARNING: tid_offset is not divisible by sizeof(int).\n"
 206              "  Now relying on the value determined using the\n"
 207              "  glibc version with which DMTCP was compiled.");
 208       return STATIC_TLS_TID_OFFSET();
 209       //mtcp_abort();
 210     }
 211     /* Should we do a double-check, and spawn a new thread and see
 212      *  if its TID matches at this tid_offset?  This would give greater
 213      *  confidence, but for the reasons above, it's probably not necessary.
 214      */
 215   }
 216   return tid_offset;
 217 }
 219 /*****************************************************************************
 220  *
 221  *****************************************************************************/
 222 int TLSInfo_GetPidOffset(void)
 223 {
 224   static int pid_offset = -1;
 225   struct {pid_t tid; pid_t pid;} tid_pid;
 226   if (pid_offset == -1) {
 227     int tid_offset = TLSInfo_GetTidOffset();
pid_offset = tid_offset + (char *)&(tid_pid.pid) - (char *)&tid_pid;
DPRINTF("pid_offset: %d\n", pid_offset);
 230   }
 231   return pid_offset;
 232 }
 234 static char *memsubarray (char *array, char *subarray, size_t len)
 235 {
 236    char *i_ptr;
size_t j;
 238    int word1 = *(int *)subarray;
 239    // Assume subarray length is at least sizeof(int) and < 2048.
ASSERT (len >= sizeof(int));
 241    for (i_ptr = array; i_ptr < array+2048; i_ptr++) {
 242      if (*(int *)i_ptr == word1) {
 243        for (j = 0; j < len; j++)
 244          if (i_ptr[j] != subarray[j])
 245            break;
 246         if (j == len)
 247           return i_ptr;
 248      }
 249    }
 250    return NULL;
 251 }
 253 static uint64_t get_tls_segreg(void)
 254 {
segreg_t tlssegreg;
 256 #ifdef __i386__
 257   asm volatile ("movw %%gs,%0" : "=g" (tlssegreg)); /* any general register */
 258 #elif __x86_64__
 259   /* q = a,b,c,d for i386; 8 low bits of r class reg for x86_64 */
 260   asm volatile ("movw %%fs,%0" : "=g" (tlssegreg));
 261 #elif __arm__
 262   asm volatile ("mrc     p15, 0, %0, c13, c0, 3  @ load_tp_hard\n\t"
 263                 : "=r" (tlssegreg));
 264 #elif defined(__aarch64__)
 265   asm volatile ("mrs  %0, tpidr_el0" : "=r" (tlssegreg) );
 266 #endif
 267   return (uint64_t)tlssegreg;
 268 }
 270 /*
 271 static int
 272 tls_get_thread_area(void *uinfo, MYINFO_GS_T dummy)
 273 {
 274   asm volatile ("mrs   %0, tpidr_el0" : "=r" (myinfo_gs) );
 275   myinfo_gs = myinfo_gs - 1776;
 276   *(unsigned long int *)&(((struct user_desc *)uinfo)->base_addr) = myinfo_gs;
 277    return myinfo_gs;
 278 }
 279 
 280 static int
 281 tls_set_thread_area(void *uinfo, MYINFO_GS_T dummy)
 282 {
 283   myinfo_gs = *(unsigned long int *)&(((struct user_desc *)uinfo)->base_addr);
 284   myinfo_gs = myinfo_gs + 1776;
 285   asm volatile ("msr     tpidr_el0, %[gs]" : : [gs] "r" (myinfo_gs) );
 286 }
 287 */
 289 static void* get_tls_base_addr()
 290 {
 291   struct user_desc gdtentrytls;
gdtentrytls.entry_number = get_tls_segreg() / 8;
 294   if (tls_get_thread_area(&gdtentrytls, myinfo_gs) == -1) {
PRINTF("Error getting GDT TLS entry: %d\n", errno);
_exit(0);
 297   }
 298   return (void *)(*(unsigned long *)&(gdtentrytls.base_addr));
 299 }
 301 // Returns value for AT_SYSINFO in kernel's auxv
 302 // Ideally:  mtcp_at_sysinfo() == *mtcp_addr_sysinfo()
 303 // Best if we call this early, before the user makes problems
 304 // by moving environment variables, putting in a weird stack, etc.
 305 extern char **environ;
 306 static void * get_at_sysinfo()
 307 {
 308   void **stack;
 309   int i;
ELF_AUXV_T *auxv;
 311   static char **my_environ = NULL;
 313   if (my_environ == NULL)
my_environ = environ;
 315 #if 0
 316   // Walk the stack.
 317 #if defined(__i386__) || defined(__x86_64__)
 318   asm volatile (CLEAN_FOR_64_BIT(mov %%ebp, %0\n\t)
 319                 : "=g" (stack) );
 320 #else
 321 # error "current architecture not supported"
 322 #endif
MTCP_PRINTF("stack 2: %p\n", stack);
 325   // When popping stack/%ebp yields zero, that's the ELF loader telling us that
 326   // this is "_start", the first call frame, which was created by ELF.
 327   for ( ; *stack != NULL; stack = *stack )
 328     ;
 330   // Go beyond first call frame:
 331   // Next look for &(argv[argc]) on stack;  (argv[argc] == NULL)
 332   for (i = 1; stack[i] != NULL; i++)
 333     ;
 334   // Do some error checks
 335   if ( &(stack[i]) - stack > 100000 ) {
MTCP_PRINTF("Error:  overshot stack\n");
exit(1);
 338   }
stack = &stack[i];
 340 #else
stack = (void **)&my_environ[-1];
 343   if (*stack != NULL) {
PRINTF("Error: This should be argv[argc] == NULL and it's not. NO &argv[argc]");
_exit(0);
 346   }
 347 #endif
 348   // stack[-1] should be argv[argc-1]
 349   if ( (void **)stack[-1] < stack || (void **)stack[-1] > stack + 100000 ) {
PRINTF("Error: candidate argv[argc-1] failed consistency check");
_exit(0);
 352   }
 353   for (i = 1; stack[i] != NULL; i++)
 354     if ( (void **)stack[i] < stack || (void **)stack[i] > stack + 10000 ) {
PRINTF("Error: candidate argv[i] failed consistency check");
_exit(0);
 357     }
stack = &stack[i+1];
 359   // Now stack is beginning of auxiliary vector (auxv)
 360   // auxv->a_type = AT_NULL marks the end of auxv
 361   for (auxv = (ELF_AUXV_T *)stack; auxv->a_type != AT_NULL; auxv++) {
 362     // mtcp_printf("0x%x 0x%x\n", auxv->a_type, auxv->a_un.a_val);
 363     if ( auxv->a_type == (UINT_T)AT_SYSINFO ) {
 364       //JNOTE("AT_SYSINFO") (&auxv->a_un.a_val) (auxv->a_un.a_val);
 365       return (void *)auxv->a_un.a_val;
 366     }
 367   }
 368   return NULL;  /* Couldn't find AT_SYSINFO */
 369 }
 371 // From glibc-2.7: glibc-2.7/nptl/sysdeps/i386/tls.h
 372 // SYSINFO_OFFSET given by:
 373 //  #include "glibc-2.7/nptl/sysdeps/i386/tls.h"
 374 //  tcbhead_t dummy;
 375 //  #define SYSINFO_OFFSET &(dummy.sysinfo) - &dummy
 377 // Some reports say it was 0x18 in past.  Should we also check that?
 378 #define DEFAULT_SYSINFO_OFFSET "0x10"
 380 int TLSInfo_HaveThreadSysinfoOffset()
 381 {
 382 #ifdef RESET_THREAD_SYSINFO
 383   static int result = -1; // Reset to 0 or 1 on first call.
 384 #else
 385   static int result = 0;
 386 #endif
 387   if (result == -1) {
 388     void * sysinfo;
 389 #if defined(__i386__) || defined(__x86_64__)
 390   asm volatile (CLEAN_FOR_64_BIT(mov %%gs:) DEFAULT_SYSINFO_OFFSET ", %0\n\t"
 391                 : "=r" (sysinfo) );
 392 #elif defined(__arm__)
 393   asm volatile ("mrc     p15, 0, %0, c13, c0, 3  @ load_tp_hard\n\t"
 394                 : "=r" (sysinfo) );
 395 #elif defined(__aarch64__)
 396   asm volatile ("mrs     %0, tpidr_el0" : "=r" (sysinfo) );
 397 #else
 398 # error "current architecture not supported"
 399 #endif
result = (sysinfo == get_at_sysinfo());
 401   }
 402   return result;
 403 }
 405 // AT_SYSINFO is what kernel calls sysenter address in vdso segment.
 406 // Kernel saves it for each thread in %gs:SYSINFO_OFFSET ??
 407 //  as part of kernel TCB (thread control block) at beginning of TLS ??
 408 void *TLSInfo_GetThreadSysinfo()
 409 {
 410   void *sysinfo;
 411 #if defined(__i386__) || defined(__x86_64__)
 412   asm volatile (CLEAN_FOR_64_BIT(mov %%gs:) DEFAULT_SYSINFO_OFFSET ", %0\n\t"
 413                 : "=r" (sysinfo) );
 414 #elif defined(__arm__)
 415   asm volatile ("mrc     p15, 0, %0, c13, c0, 3  @ load_tp_hard\n\t"
 416                 : "=r" (sysinfo) );
 417 #elif defined(__aarch64__)
 418   asm volatile ("mrs     %0, tpidr_el0" : "=r" (sysinfo) );
 419 #else
 420 # error "current architecture not supported"
 421 #endif
 422   return sysinfo;
 423 }
 425 void TLSInfo_SetThreadSysinfo(void *sysinfo) {
 426 #if defined(__i386__) || defined(__x86_64__)
 427   asm volatile (CLEAN_FOR_64_BIT(mov %0, %%gs:) DEFAULT_SYSINFO_OFFSET "\n\t"
 428                 : : "r" (sysinfo) );
 429 #elif defined(__arm__)
mtcp_sys_kernel_set_tls(sysinfo);
 431 #elif defined(__aarch64__)
 432   asm volatile ("msr     tpidr_el0, %[gs]" : : [gs] "r" (sysinfo) );
 433 #else
 434 # error "current architecture not supported"
 435 #endif
 436 }
 438 /*****************************************************************************
 439  *
 440  *
 441  *****************************************************************************/
 442 void TLSInfo_VerifyPidTid(pid_t pid, pid_t tid)
 443 {
pid_t tls_pid, tls_tid;
 445   char *addr = (char*)get_tls_base_addr();
tls_pid = *(pid_t *) (addr + TLSInfo_GetPidOffset());
tls_tid = *(pid_t *) (addr + TLSInfo_GetTidOffset());
 449   if ((tls_pid != pid) || (tls_tid != tid)) {
PRINTF("ERROR: getpid(%d), tls pid(%d), and tls tid(%d) must all match\n",
 451            (int)mtcp_sys_getpid(), tls_pid, tls_tid);
_exit(0);
 453   }
 454 }
 456 void TLSInfo_UpdatePid()
 457 {
pid_t  *tls_pid = (pid_t *) ((char*)get_tls_base_addr() +
TLSInfo_GetPidOffset());
 460   *tls_pid = mtcp_sys_getpid();
 461 }
 463 /*****************************************************************************
 464  *
 465  *  Save state necessary for TLS restore
 466  *  Linux saves stuff in the GDT, switching it on a per-thread basis
 467  *
 468  *****************************************************************************/
 469 void TLSInfo_SaveTLSState (ThreadTLSInfo *tlsInfo)
 470 {
 471   int i;
 473 #ifdef __i386__
 474   asm volatile ("movw %%fs,%0" : "=m" (tlsInfo->fs));
 475   asm volatile ("movw %%gs,%0" : "=m" (tlsInfo->gs));
 476 #elif __x86_64__
 477   //asm volatile ("movl %%fs,%0" : "=m" (tlsInfo->fs));
 478   //asm volatile ("movl %%gs,%0" : "=m" (tlsInfo->gs));
 479 #elif __arm__ || __aarch64__
 480   // Follow x86_64 for arm.
 481 #endif
memset (tlsInfo->gdtentrytls, 0, sizeof tlsInfo->gdtentrytls);
 485 /* FIXME:  IF %fs IS NOT READ into tlsInfo->fs AT BEGINNING OF THIS
 486  *   FUNCTION, HOW CAN WE REFER TO IT AS  tlsInfo->TLSSEGREG?
 487  *   WHAT IS THIS CODE DOING?
 488  */
i = tlsInfo->TLSSEGREG / 8;
tlsInfo->gdtentrytls[0].entry_number = i;
 491   if (tls_get_thread_area (&(tlsInfo->gdtentrytls[0]), myinfo_gs) == -1) {
PRINTF("Error saving GDT TLS entry: %d\n", errno);
_exit(0);
 494   }
 495   //PRINTF("TLSINFO base_addr: %p \n\n", tlsInfo->gdtentrytls[0].base_addr);
 496 }
 498 /*****************************************************************************
 499  *
 500  *  Restore the GDT entries that are part of a thread's state
 501  *
 502  *  The kernel provides set_thread_area system call for a thread to alter a
 503  *  particular range of GDT entries, and it switches those entries on a
 504  *  per-thread basis.  So from our perspective, this is per-thread state that is
 505  *  saved outside user addressable memory that must be manually saved.
 506  *
 507  *****************************************************************************/
 508 void TLSInfo_RestoreTLSState(ThreadTLSInfo *tlsInfo)
 509 {
 510   /* Every architecture needs a register to point to the current
 511    * TLS (thread-local storage).  This is where we set it up.
 512    */
 514   /* Patch 'struct user_desc' (gdtentrytls) of glibc to contain the
 515    * the new pid and tid.
 516    */
 517   *(pid_t *)(*(unsigned long *)&(tlsInfo->gdtentrytls[0].base_addr)
 518              + TLSInfo_GetPidOffset()) = mtcp_sys_getpid();
 519   if (mtcp_sys_kernel_gettid() == mtcp_sys_getpid()) {
 520     *(pid_t *)(*(unsigned long *)&(tlsInfo->gdtentrytls[0].base_addr)
 521                + TLSInfo_GetTidOffset()) = mtcp_sys_getpid();
 522   }
 524   /* Now pass this to the kernel, so it can adjust the segment descriptor.
 525    * This will make different kernel calls according to the CPU architecture. */
 526   if (tls_set_thread_area (&(tlsInfo->gdtentrytls[0]), myinfo_gs) != 0) {
PRINTF("Error restoring GDT TLS entry: %d\n", errno);
mtcp_abort();
 529   }
 531   /* Finally, if this is i386, we need to set %gs to refer to the segment
 532    * descriptor that we're using above.  We restore the original pointer.
 533    * For the other architectures (not i386), the kernel call above
 534    * already did the equivalent work of setting up thread registers.
 535    */
 536 #ifdef __i386__
 537   asm volatile ("movw %0,%%fs" : : "m" (tlsInfo->fs));
 538   asm volatile ("movw %0,%%gs" : : "m" (tlsInfo->gs));
 539 #elif __x86_64__
 540 /* Don't directly set fs.  It would only set 32 bits, and we just
 541  *  set the full 64-bit base of fs, using sys_set_thread_area,
 542  *  which called arch_prctl.
 543  *asm volatile ("movl %0,%%fs" : : "m" (tlsInfo->fs));
 544  *asm volatile ("movl %0,%%gs" : : "m" (tlsInfo->gs));
 545  */
 546 #elif __arm__
 547 /* ARM treats this same as x86_64 above. */
 548 #elif defined(__aarch64__)
 549 # warning "TODO: Implementation for ARM64?"
 550 #endif
 551 }
 553 #if 0
 554 /*****************************************************************************
 555  *
 556  *  The original program's memory and files have been restored
 557  *
 558  *****************************************************************************/
 559 void TLSInfo_PostRestart()
 560 {
 562   /* Now we can access all of the memory from the time of the checkpoint.
 563    * We will continue to use the temporary stack from mtcp_restart.c, for now.
 564    * When we return from the signal handler, this primary thread will
 565    *   return to its original stack.
 566    */
 568   /* However, we can only make direct kernel calls, and not libc calls.
 569    * TLSInfo_RestoreTLSState will do the following:
 570    * Step 1:  Patch 'struct user_desc' (gdtentrytls) of glibc to have
 571    *     the new pid and tid.
 572    * Step 2:  Make kernel call to set up the corresponding
 573    *     segment descriptor and/or set any TLS per-thread register.
 574    * Step 3 (for Intel only):  Restore %fs and %gs to refer to the the
 575    *     segment descriptor for this primary thread (only %gs needed for i386).
 576    */
 577   //TLSInfo_RestoreTLSState(motherofall_tlsInfo);
 579 #if 0
 580   /* FOR DEBUGGING:  Now test if we can make a kernel call through libc. */
mtcp_sys_write(1, "shell ../../util/gdb-add-libdmtcp-symbol-file.py PID PC\n",
 582            sizeof("shell ../../util/gdb-add-libdmtcp-symbol-file.py PID PC\n"));
 583 #if defined(__i386__) || defined(__x86_64__)
 584       asm volatile ("int3"); // Do breakpoint; send SIGTRAP, caught by gdb
 585 #elif defined(__aarch64__)
 586       asm("brk 0");
 587 #else
DPRINTF("IN GDB: interrupt (^C); add-symbol-file ...; (gdb) print x=0\n");
 589       { int x = 1; while (x); } // Stop execution for user to type command.
 590 #endif
 591   int rc = nice(0);
 592 #endif
 594   // We have now verified that libc functionality is restored.
Thread_RestoreAllThreads();
 596 }
 597 #endif