// See comments above for open_ckpt_to_read()
int dmtcp::CkptSerializer::openDmtcpCheckpointFile(const dmtcp::string& path,
                                                   int *offset,
                                                   int skipBytes)
{
  char buf[1024];
  // Function also sets dmtcp::ext_decomp_pid::ConnectionToFds
  int fd = open_ckpt_to_read(path.c_str());
  // The rest of this function is for compatibility with original definition.
  JASSERT(fd >= 0) (path) .Text("Failed to open file.");
  const int len = strlen(DMTCP_FILE_HEADER);
  JASSERT(_real_read(fd, buf, len) == len)(path) .Text("_real_read() failed");
  if (strncmp(buf, DMTCP_FILE_HEADER, len) == 0) {
    JTRACE("opened checkpoint file [uncompressed]")(path);
  } else {
    close_ckpt_to_read(fd);
    fd = open_ckpt_to_read(path.c_str()); /* Re-open from beginning */
    JASSERT(fd >= 0) (path) .Text("Failed to open file.");
  }

  if (offset != NULL) {
    *offset = strlen(DMTCP_FILE_HEADER);
  }

  skipBytes -= strlen(DMTCP_FILE_HEADER);
  if (skipBytes > 0) {
    JASSERT(dmtcp::Util::skipBytes(fd, skipBytes) == skipBytes) (skipBytes);
  }
  return fd;
}

bool isTorqueFile(dmtcp::string relpath, dmtcp::string &path)
{
  JTRACE("Start");
  switch( rmgr_type ){
  case Empty:
    probeTorque();
    if( rmgr_type != torque )
      return false;
    break;
  case torque:
    break;
  default:
    return false;
  }

  if( torque_home().size() == 0 )
    return false;

  dmtcp::string abspath = torque_home() + "/" + relpath;
  JTRACE("Compare path with")(path)(abspath);
  if( path.size() < abspath.size() )
    return false;

  if( path.substr(0,abspath.size()) == abspath )
    return true;

  return false;
}

static void rem_trailing_slash(dmtcp::string &path)
{
    size_t i = path.size() - 1;
    while( (path[i] == ' ' || path[i] == '/' || path == "\\" ) && i>0 )
      i--;
    if( i+1 < path.size() )
      path = path.substr(0,i+1);
}

static bool isImportantEnv ( dmtcp::string str )
{
  str = str.substr(0, str.find("="));

  for ( size_t i=0; i<ourImportantEnvsCnt; ++i ) {
    if ( str == ourImportantEnvs[i] )
      return true;
  }
  return false;
}

bool isTorqueStderr(dmtcp::string &path)
{
  if( !isTorqueIOFile(path) )
    return false;

  dmtcp::string suffix = ".ER";

  if( (path.substr(path.size() - suffix.size()) == suffix) ){
    return true;
  }

  return false;
}

static void clear_path(dmtcp::string &path)
{
  size_t i;
  for(i=0;i<path.size();i++){
    if( path[i] == '/' || path[i] == '\\' ){
      size_t j = i+1;
      while( (path[j] == '/' || path[j] == '\\') && j < path.size() ){
        j++;
      }
      if( j != i+1 ){
        path.erase(i+1,j-(i+1));
      }
    }
  }
}

int openSharedFile(dmtcp::string name, int flags)
{
  int fd;
  // try to create, truncate & open file
  if ((fd = open(name.c_str(), O_EXCL|O_CREAT|O_TRUNC | flags, 0600)) >= 0) {
    return fd;
  }
  if (fd < 0 && errno == EEXIST) {
    if ((fd = open(name.c_str(), flags, 0600)) > 0) {
      return fd;
    }
  }
  // unable to create & open OR open
  JASSERT(false)(name)(strerror(errno)).Text("Cannot open file");
  return -1;
}

const DmtcpLocalStatus* __real_dmtcpGetLocalStatus(){
  //these must be static so their memory is not deleted.
  static dmtcp::string ckpt;
  static dmtcp::string pid;
  static DmtcpLocalStatus status;
  ckpt.reserve(1024);

  //get filenames
  pid=dmtcp::UniquePid::ThisProcess().toString();
  ckpt=dmtcp::UniquePid::getCkptFilename();

  status.numCheckpoints          = numCheckpoints;
  status.numRestarts             = numRestarts;
  status.checkpointFilename      = ckpt.c_str();
  status.uniquePidStr            = pid.c_str();
  return &status;
}

static int queryPbsConfig(dmtcp::string option, dmtcp::string &pbs_config)
{
  int fds[2];
  const char *pbs_config_path = "pbs-config";
  static const char *pbs_config_args[] = { "pbs-config", option.c_str(), NULL };
  int cpid;

  if( pipe(fds) == -1){
    // just go away - we cannot serve this request
    JTRACE("Cannot create pipe to execute pbs-config to find Torque/PBS library!");
    return -1;
  }

  cpid = _real_fork();

  if( cpid < 0 ){
    JTRACE( "ERROR: cannot execute pbs-config. Will not run tm_spawn!");
    return -1;
  }
  if( cpid == 0 ){
    JTRACE ( "child process, will exec into external de-compressor");
    fds[1] = dup(dup(dup(fds[1])));
    close(fds[0]);
    JASSERT(dup2(fds[1], STDOUT_FILENO) == STDOUT_FILENO);
    close(fds[1]);
    _real_execvp(pbs_config_path, (char **)pbs_config_args);
    /* should not get here */
    JASSERT(false)("ERROR: Failed to exec pbs-config. tm_spawn will fail with TM_BADINIT")(strerror(errno));
    exit(0);
  }

  /* parent process */
  JTRACE ( "created child process for pbs-config")(cpid);
  int status;
  if( waitpid(cpid,&status,0) < 0 ){
    return -1;
  }
  if( !( WIFEXITED(status) && WEXITSTATUS(status) == 0 ) ){
    return -1;
  }

  // set descriptor as non-blocking
  // JTRACE ( "Set pipe fds[0] as non-blocking");
  int flags = fcntl(fds[0], F_GETFL);
  fcntl(fds[0], F_SETFL, flags | O_NONBLOCK);

  //JTRACE ( "Read pbs-config output from pipe");
  pbs_config = "";
  char buf[256];
  int count = 0;
  while( (count = read(fds[0], buf, 255)) > 0 ){
    buf[count] = '\0';
    pbs_config += dmtcp::string() + buf;
  }

  JTRACE ( "pbs-config output:")(pbs_config);
  return 0;
}

bool isTorqueHomeFile(dmtcp::string &path)
{
  // check if file is in home directory
  char *ptr;
  dmtcp::string hpath = "";

  if ((ptr = getenv("HOME"))) {
    hpath = dmtcp::string() + ptr;
    JTRACE("Home directory:")(hpath)(path);
  }else{
    JTRACE("Cannot determine user HOME directory!");
    return false;
  }

  if( hpath.size() >= path.size() ){
    JTRACE("Length of path is less than home dir");
    return false;
  }

  if( path.substr(0,hpath.size()) != hpath ){
    JTRACE("prefix of path is not home directory")(path)(hpath);
    return false;
  }

  dmtcp::string suffix1 = ".OU", suffix2 = ".ER";

  if( !( (path.substr(path.size() - suffix1.size()) == suffix1) ||
        (path.substr(path.size() - suffix2.size()) == suffix2) ) ){
    JTRACE("path has no .OU or .ER suffix")(path);
    return false;
  }

  char jobid[256];
  sprintf(jobid,"%lu",torque_jobid);
  dmtcp::string spool_path = hpath + "/.pbs_spool/" + jobid;
  dmtcp::string home_path = hpath + jobid;

  if( path.substr(0,spool_path.size()) == spool_path ){
    JTRACE("File is located in $HOME/.pbs_spool/. It is Torque/PBS stdio file")(path);
    return true;
  }

  if( path.substr(0,home_path.size()) == home_path ){
    JTRACE("File is located in $HOME/. It is Torque/PBS stdio file")(path);
    return true;
  }

  return false;
}

int findLibTorque_maps(dmtcp::string &libpath)
{
  // /proc/self/maps looks like: "<start addr>-<end addr> <mode> <offset> <device> <inode> <libpath>
  // we need to extract libpath
  dmtcp::Util::ProcMapsArea area;
  int ret = -1;

  // we will search for first libpath and first libname
  int fd = _real_open ( "/proc/self/maps", O_RDONLY);

  if( fd < 0 ){
    JTRACE("Cannot open /proc/self/maps file");
    return -1;
  }

  while( dmtcp::Util::readProcMapsLine(fd, &area) ){
    libpath = area.name;
    JNOTE("Inspect new /proc/seft/maps line")(libpath);
    if( libpath.size() == 0 ){
      JNOTE("anonymous region, skip");
      continue;
    }

    if( libpath.find("libtorque") != dmtcp::string::npos ){
      // this is library path that contains libtorque. This is what we need
      JTRACE("Torque PBS libpath")(libpath);
      ret = 0;
      break;
    }else{
      JNOTE("Not a libtorque region")(libpath);
    }
  }

  _real_close(fd);
  return ret;
}

static void writeCurrentLogFileNameToPrevLogFile(dmtcp::string& path)
{
#ifdef DEBUG
  dmtcp::ostringstream o;
  o << "========================================\n"
    << "This process exec()'d into a new program\n"
    << "Program Name: " << jalib::Filesystem::GetProgramName() << "\n"
    << "New JAssertLog Path: " << getLogFilePath() << "\n"
    << "========================================\n";

  int fd = open(path.c_str(), O_WRONLY | O_APPEND, 0);
  if (fd != -1) {
    Util::writeAll(fd, o.str().c_str(), o.str().length());
  }
  _real_close(fd);
#endif
}

static dmtcp::string _resolveSymlink(dmtcp::string path)
{
  dmtcp::string device = jalib::Filesystem::ResolveSymlink(path);
  if (dmtcp_real_to_virtual_pid && path.length() > 0 &&
      dmtcp::Util::strStartsWith(device, "/proc/")) {
    int index = 6;
    char *rest;
    char newpath[128];
    JASSERT(device.length() < sizeof newpath);
    pid_t realPid = strtol(&path[index], &rest, 0);
    if (realPid > 0 && *rest == '/') {
      pid_t virtualPid = dmtcp_real_to_virtual_pid(realPid);
      sprintf(newpath, "/proc/%d%s", virtualPid, rest);
      device = newpath;
    }
  }
  return device;
}

int dmtcp::CkptSerializer::openDmtcpCheckpointFile(const dmtcp::string& path){
  int fd = open( path.c_str(), O_RDONLY);
  JASSERT(fd>=0)(path).Text("Failed to open file.");
  char buf[512];
  const int len = strlen(DMTCP_FILE_HEADER);
  JASSERT(_real_read(fd, buf, len)==len)(path).Text("_real_read() failed");
  if(strncmp(buf, DMTCP_FILE_HEADER, len)==0){
    JTRACE("opened checkpoint file [uncompressed]")(path);
    return fd;
  }else{
    close(fd);
    dmtcp::string cmd = dmtcp::string()+"exec gzip -d - < '"+path+"'";
    FILE* t = _real_popen(cmd.c_str(),"r");
    JASSERT(t!=NULL)(path)(cmd).Text("Failed to launch gzip.");
    JTRACE ( "created gzip child process to uncompress checkpoint file");
    fd = fileno(t);
    JASSERT(_real_read(fd, buf, len)==len)(cmd)(path).Text("Invalid checkpoint file");
    JASSERT(strncmp(buf, DMTCP_FILE_HEADER, len)==0)(path).Text("Invalid checkpoint file");
    JTRACE("opened checkpoint file [compressed]")(path);
    return fd;
  }
}

const char* dmtcp_get_uniquepid_str()
{
  static dmtcp::string uniquepid_str;
  uniquepid_str = dmtcp::UniquePid::ThisProcess(true).toString();
  return uniquepid_str.c_str();
}

int main ( int argc, char** argv )
{
  initializeJalib();

  if (! getenv(ENV_VAR_QUIET))
    setenv(ENV_VAR_QUIET, "0", 0);

  processArgs(&argc, &argv);

  // If --ssh-slave and --prefix both are present, verify that the prefix-dir
  // of this binary (dmtcp_checkpoint) is same as the one provided with
  // --prefix
  if (isSSHSlave && getenv(ENV_VAR_PREFIX_PATH) != NULL) {
    const char *str = getenv(ENV_VAR_PREFIX_PATH);
    dmtcp::string prefixDir = jalib::Filesystem::ResolveSymlink(str);
    dmtcp::string programPrefixDir =
      jalib::Filesystem::DirName(jalib::Filesystem::GetProgramDir());
    JASSERT(prefixDir == programPrefixDir)
      (prefixDir) (programPrefixDir);
  }

  dmtcp::UniquePid::setTmpDir(getenv(ENV_VAR_TMPDIR));
  dmtcp::UniquePid::ThisProcess(true);
  dmtcp::Util::initializeLogFile();

#ifdef FORKED_CHECKPOINTING
  /* When this is robust, add --forked-checkpointing option on command-line,
   * with #ifdef FORKED_CHECKPOINTING around the option, change default of
   * configure.ac, dmtcp/configure.ac, to enable, and change them
   * from enable-forked... to disable-...
   */
  setenv(ENV_VAR_FORKED_CKPT, "1", 1);
#endif

  if (jassert_quiet == 0)
    JASSERT_STDERR << DMTCP_BANNER;

  // This code will go away when zero-mapped pages are implemented in MTCP.
  struct rlimit rlim;
  getrlimit(RLIMIT_STACK, &rlim);
  if (rlim.rlim_cur > 256*1024*1024 && rlim.rlim_cur != RLIM_INFINITY)
    JASSERT_STDERR <<
      "*** WARNING:  RLIMIT_STACK > 1/4 GB.  This causes each thread to"
      "\n***  receive a 1/4 GB stack segment.  Checkpoint/restart will be slow,"
      "\n***  and will potentially break if many threads are created."
      "\n*** Suggest setting (sh/bash):  ulimit -s 10000"
      "\n***                (csh/tcsh):  limit stacksize 10000"
      "\n*** prior to using DMTCP.  (This will be fixed in the future, when"
      "\n*** DMTCP supports restoring zero-mapped pages.)\n\n\n" ;
  // Remove this when zero-mapped pages are supported.  For segments with
  // no file backing:  Start with 4096 (page) offset and keep doubling offset
  // until finding region of memory segment with many zeroes.
  // Then mark as CS_ZERO_PAGES in MTCP instead of CS_RESTORE (or mark
  // entire segment as CS_ZERO_PAGES and then overwrite with CS_RESTORE
  // region for portion to be read back from checkpoint image.
  // For CS_ZERO_PAGES region, mmap // on restart, but don't write in zeroes.
  // Also, after checkpointing segment, munmap zero pages, and mmap them again.
  // Don't try to find all pages.  The above strategy may increase
  // the non-zero-mapped mapped pages to no more than double the actual
  // non-zero region (assuming that the zero-mapped pages are contiguous).
  // - Gene

  testMatlab(argv[0]);
  testJava(argv);  // Warn that -Xmx flag needed to limit virtual memory size

  // If dmtcphijack.so is in standard search path and _also_ has setgid access,
  //   then LD_PRELOAD will work.
  // Otherwise, it will only work if the application does not use setuid and
  //   setgid access.  So, we test //   if the application does not use
  //   setuid/setgid.  (See 'man ld.so')
  // FIXME:  ALSO DO THIS FOR execwrappers.cpp:dmtcpPrepareForExec()
  //   Should pass dmtcphijack.so path, and let testSetuid determine
  //     if setgid is set for it.  If so, no problem:  continue.
  //   If not, call testScreen() and adapt 'screen' to run using
  //     Util::patchArgvIfSetuid(argv[0], argv, &newArgv) (which shouldn't
  //     will just modify argv[0] to point to /tmp/[email protected]/screen
  //     and other modifications:  doesn't need newArgv).
  //   If it's not 'screen' and if no setgid for dmtcphijack.so, then testSetuid
  //    should issue the warning, unset our LD_PRELOAD, and hope for the best.
  //    A program like /usr/libexec/utempter/utempter (Fedora path)
  //    is short-lived and can be safely run.  Ideally, we should
  //    disable checkpoints while utempter is running, and enable checkpoints
  //    when utempter finishes.  See possible model at
  //    execwrappers.cpp:execLibProcessAndExit(), since the same applies
  //    to running /lib/libXXX.so for running libraries as executables.
  if (testSetuid(argv[0])) {
    char **newArgv;
    // THIS NEXT LINE IS DANGEROUS.  MOST setuid PROGRAMS CAN'T RUN UNPRIVILEGED
    dmtcp::Util::patchArgvIfSetuid(argv[0], argv, &newArgv);
    argv = newArgv;
  };

  if (argc > 0) {
    JTRACE("dmtcp_checkpoint starting new program:")(argv[0]);
  }

  //set up CHECKPOINT_DIR
  if(getenv(ENV_VAR_CHECKPOINT_DIR) == NULL){
    const char* ckptDir = get_current_dir_name();
    if(ckptDir != NULL ){
//.........这里部分代码省略.........

void runMtcpRestore(const char* path, int offset, size_t argvSize,
                    size_t envSize)
{
  static dmtcp::string mtcprestart =
    jalib::Filesystem::FindHelperUtility ("mtcp_restart");

  // Tell mtcp_restart process to write its debugging information to
  // PROTECTED_STDERR_FD. This way we prevent it from spitting out garbage onto
  // FD_STDERR if it is being used by the user process in a special way.
  char protected_stderr_fd_str[16];
  sprintf(protected_stderr_fd_str, "%d", PROTECTED_STDERR_FD);

#ifdef USE_MTCP_FD_CALLING
  int fd = ConnectionToFds::openMtcpCheckpointFile(path);
  char buf[64];
  char buf2[64];

  sprintf(buf, "%d", fd);
  // gzip_child_pid set by openMtcpCheckpointFile() above.
  sprintf(buf2, "%d", dmtcp::ConnectionToFds::gzip_child_pid);
  char* newArgs[] = {
    (char*) mtcprestart.c_str(),
    (char*) "--stderr-fd",
    protected_stderr_fd_str,
    (char*) "--fd",
    buf,
    (char*) "--gzip-child-pid",
    buf2,
    NULL
  };
  if (dmtcp::ConnectionToFds::gzip_child_pid == -1) { // If no gzip compression
    newArgs[3] = NULL;
  }
  JTRACE ("launching mtcp_restart --fd")(fd)(path);
#else
  char buf[64];

  sprintf(buf, "%d", offset);
  char* newArgs[] = {
    (char*) mtcprestart.c_str(),
    (char*) "--stderr-fd",
    protected_stderr_fd_str,
    (char*) "--offset",
    buf,
    (char*) path,
    NULL
  };
  JTRACE ("launching mtcp_restart --offset")(path)(offset);
#endif

  // Create the placeholder for "MTCP_OLDPERS" environment.
  // setenv("MTCP_OLDPERS_DUMMY", "XXXXXXXXXXXXXXXX", 1);
  // FIXME: Put an explanation of the logic below.   -- Kapil
#define ENV_PTR(x) ((char*) (getenv(x) - strlen(x) - 1))
  char* dummyEnviron = NULL;
  const int dummyEnvironIndex = 0; // index in newEnv[]
  const int pathIndex = 1; // index in newEnv[]
  // Eventually, newEnv = {ENV_PTR("MTCP_OLDPERS"), ENV_PTR("PATH"), NULL}
  char* newEnv[3] = {NULL, NULL, NULL};
  // Will put ENV_PTR("MTCP_OLDPERS") here.
  newEnv[dummyEnvironIndex] = (char*) dummyEnviron;
  newEnv[pathIndex] = (getenv("PATH") ? ENV_PTR("PATH") : NULL);

  size_t newArgsSize = 0;
  for (int i = 0; newArgs[i] != 0; i++) {
    newArgsSize += strlen(newArgs[i]) + 1;
  }
  size_t newEnvSize = 0;
  for (int i = 0; newEnv[i] != 0; i++) {
    newEnvSize += strlen(newEnv[i]) + 1;
  }
  size_t originalArgvEnvSize = argvSize + envSize;
  size_t newArgvEnvSize = newArgsSize + newEnvSize + strlen(newArgs[0]);
  size_t argvSizeDiff = originalArgvEnvSize - newArgvEnvSize;
  dummyEnviron = (char*) malloc(argvSizeDiff);
  memset(dummyEnviron, '0', (argvSizeDiff >= 1 ? argvSizeDiff - 1 : 0));
  strncpy(dummyEnviron,
          ENV_VAR_DMTCP_DUMMY "=0",
          strlen(ENV_VAR_DMTCP_DUMMY "="));
  dummyEnviron[argvSizeDiff - 1] = '\0';

  newEnv[dummyEnvironIndex] = dummyEnviron;
  JTRACE("Args/Env Sizes")
    (newArgsSize) (newEnvSize) (argvSize) (envSize) (argvSizeDiff);

  execve (newArgs[0], newArgs, newEnv);
  JASSERT (false) (newArgs[0]) (newArgs[1]) (JASSERT_ERRNO)
          .Text ("exec() failed");
}

EXTERNC const char* dmtcp_get_ckpt_dir()
{
  static dmtcp::string tmpdir;
  tmpdir = dmtcp::ProcessInfo::instance().getCkptDir();
  return tmpdir.c_str();
}

EXTERNC const char* dmtcp_get_coord_ckpt_dir(void)
{
  static dmtcp::string dir;
  dir = CoordinatorAPI::instance().getCoordCkptDir();
  return dir.c_str();
}

EXTERNC const char* dmtcp_get_ckpt_filename(void)
{
  static dmtcp::string filename;
  filename = dmtcp::ProcessInfo::instance().getCkptFilename();
  return filename.c_str();
}