def testRecoverSession(self):
        # Create a checkpoint.
        checkpoint_dir = os.path.join(self.get_temp_dir(), "recover_session")
        try:
            gfile.DeleteRecursively(checkpoint_dir)
        except OSError:
            pass  # Ignore
        gfile.MakeDirs(checkpoint_dir)

        with tf.Graph().as_default():
            v = tf.Variable(1, name="v")
            sm = tf.train.SessionManager(ready_op=tf.assert_variables_initialized())
            saver = tf.train.Saver({"v": v})
            sess, initialized = sm.recover_session("", saver=saver, checkpoint_dir=checkpoint_dir)
            self.assertFalse(initialized)
            sess.run(v.initializer)
            self.assertEquals(1, sess.run(v))
            saver.save(sess, os.path.join(checkpoint_dir, "recover_session_checkpoint"))
        # Create a new Graph and SessionManager and recover.
        with tf.Graph().as_default():
            v = tf.Variable(2, name="v")
            with self.test_session():
                self.assertEqual(False, tf.is_variable_initialized(v).eval())
            sm2 = tf.train.SessionManager(ready_op=tf.assert_variables_initialized())
            saver = tf.train.Saver({"v": v})
            sess, initialized = sm2.recover_session("", saver=saver, checkpoint_dir=checkpoint_dir)
            self.assertTrue(initialized)
            self.assertEqual(True, tf.is_variable_initialized(sess.graph.get_tensor_by_name("v:0")).eval(session=sess))
            self.assertEquals(1, sess.run(v))

  def testWaitForSessionLocalInit(self):
    server = tf.train.Server.create_local_server()
    with tf.Graph().as_default() as graph:
      v = tf.Variable(1, name="v")
      w = tf.Variable(
          v,
          trainable=False,
          collections=[tf.GraphKeys.LOCAL_VARIABLES],
          name="w")
      sm = tf.train.SessionManager(
          graph=graph,
          ready_op=tf.report_uninitialized_variables(),
          ready_for_local_init_op=tf.report_uninitialized_variables(
              tf.all_variables()),
          local_init_op=w.initializer)

      # Initialize v but not w
      s = tf.Session(server.target, graph=graph)
      s.run(v.initializer)

      sess = sm.wait_for_session(server.target, max_wait_secs=3)
      self.assertEqual(
          True,
          tf.is_variable_initialized(sess.graph.get_tensor_by_name("v:0")).eval(
              session=sess))
      self.assertEqual(
          True,
          tf.is_variable_initialized(sess.graph.get_tensor_by_name("w:0")).eval(
              session=sess))
      self.assertEquals(1, sess.run(v))
      self.assertEquals(1, sess.run(w))

 def testPrepareSessionWithReadyForLocalInitOp(self):
   with tf.Graph().as_default():
     v = tf.Variable(1, name="v")
     w = tf.Variable(
         v,
         trainable=False,
         collections=[tf.GraphKeys.LOCAL_VARIABLES],
         name="w")
     with self.test_session():
       self.assertEqual(False, tf.is_variable_initialized(v).eval())
       self.assertEqual(False, tf.is_variable_initialized(w).eval())
     sm2 = tf.train.SessionManager(
         ready_op=tf.report_uninitialized_variables(),
         ready_for_local_init_op=tf.report_uninitialized_variables(
             tf.all_variables()),
         local_init_op=w.initializer)
     sess = sm2.prepare_session("", init_op=v.initializer)
     self.assertEqual(
         True,
         tf.is_variable_initialized(sess.graph.get_tensor_by_name("v:0")).eval(
             session=sess))
     self.assertEqual(
         True,
         tf.is_variable_initialized(sess.graph.get_tensor_by_name("w:0")).eval(
             session=sess))
     self.assertEquals(1, sess.run(v))
     self.assertEquals(1, sess.run(w))

 def testRecoverSessionNoChkptStillRunsLocalInitOp(self):
   # This test checks for backwards compatibility.
   # In particular, we continue to ensure that recover_session will execute
   # local_init_op exactly once, regardless of whether the session was
   # successfully recovered.
   with tf.Graph().as_default():
     w = tf.Variable(
         1,
         trainable=False,
         collections=[tf.GraphKeys.LOCAL_VARIABLES],
         name="w")
     with self.test_session():
       self.assertEqual(False, tf.is_variable_initialized(w).eval())
     sm2 = tf.train.SessionManager(
         ready_op=tf.report_uninitialized_variables(),
         ready_for_local_init_op=None,
         local_init_op=w.initializer)
     # Try to recover session from None
     sess, initialized = sm2.recover_session(
         "", saver=None, checkpoint_dir=None)
     # Succeeds because recover_session still run local_init_op
     self.assertFalse(initialized)
     self.assertEqual(
         True,
         tf.is_variable_initialized(sess.graph.get_tensor_by_name("w:0")).eval(
             session=sess))
     self.assertEquals(1, sess.run(w))

 def testIsVariableInitialized(self):
   for use_gpu in [True, False]:
     with self.test_session(use_gpu=use_gpu):
       v0 = state_ops.variable_op([1, 2], tf.float32)
       self.assertEqual(False, tf.is_variable_initialized(v0).eval())
       tf.assign(v0, [[2.0, 3.0]]).eval()
       self.assertEqual(True, tf.is_variable_initialized(v0).eval())

  def testRecoverSessionWithReadyForLocalInitOpFailsToReadyLocal(self):
    # We use ready_for_local_init_op=tf.report_uninitialized_variables(),
    # which causes recover_session to not run local_init_op, and to return
    # initialized=False

    # Create a checkpoint.
    checkpoint_dir = os.path.join(
        self.get_temp_dir(),
        "recover_session_ready_for_local_init_fails_to_ready_local")
    try:
      gfile.DeleteRecursively(checkpoint_dir)
    except errors.OpError:
      pass  # Ignore
    gfile.MakeDirs(checkpoint_dir)

    with tf.Graph().as_default():
      v = tf.Variable(1, name="v")
      sm = tf.train.SessionManager(ready_op=tf.report_uninitialized_variables())
      saver = tf.train.Saver({"v": v})
      sess, initialized = sm.recover_session(
          "", saver=saver, checkpoint_dir=checkpoint_dir)
      self.assertFalse(initialized)
      sess.run(v.initializer)
      self.assertEquals(1, sess.run(v))
      saver.save(sess, os.path.join(checkpoint_dir,
                                    "recover_session_checkpoint"))
    # Create a new Graph and SessionManager and recover.
    with tf.Graph().as_default():
      v = tf.Variable(2, name="v")
      w = tf.Variable(
          v,
          trainable=False,
          collections=[tf.GraphKeys.LOCAL_VARIABLES],
          name="w")
      with self.test_session():
        self.assertEqual(False, tf.is_variable_initialized(v).eval())
        self.assertEqual(False, tf.is_variable_initialized(w).eval())
      sm2 = tf.train.SessionManager(
          ready_op=tf.report_uninitialized_variables(),
          ready_for_local_init_op=tf.report_uninitialized_variables(),
          local_init_op=w.initializer)
      saver = tf.train.Saver({"v": v})
      sess, initialized = sm2.recover_session(
          "", saver=saver, checkpoint_dir=checkpoint_dir)
      self.assertFalse(initialized)
      self.assertEqual(
          True,
          tf.is_variable_initialized(sess.graph.get_tensor_by_name("v:0")).eval(
              session=sess))
      self.assertEqual(
          False,
          tf.is_variable_initialized(sess.graph.get_tensor_by_name("w:0")).eval(
              session=sess))
      self.assertEquals(1, sess.run(v))

    def testPrepareSessionFails(self):
        checkpoint_dir = os.path.join(self.get_temp_dir(), "prepare_session")
        checkpoint_dir2 = os.path.join(self.get_temp_dir(), "prepare_session2")
        try:
            gfile.DeleteRecursively(checkpoint_dir)
            gfile.DeleteRecursively(checkpoint_dir2)
        except OSError:
            pass  # Ignore
        gfile.MakeDirs(checkpoint_dir)

        with tf.Graph().as_default():
            v = tf.Variable([1.0, 2.0, 3.0], name="v")
            sm = tf.train.SessionManager(ready_op=tf.assert_variables_initialized())
            saver = tf.train.Saver({"v": v})
            sess = sm.prepare_session(
                "", init_op=tf.initialize_all_variables(), saver=saver, checkpoint_dir=checkpoint_dir
            )
            self.assertAllClose([1.0, 2.0, 3.0], sess.run(v))
            checkpoint_filename = os.path.join(checkpoint_dir, "prepare_session_checkpoint")
            saver.save(sess, checkpoint_filename)
        # Create a new Graph and SessionManager and recover.
        with tf.Graph().as_default():
            # Renames the checkpoint directory.
            os.rename(checkpoint_dir, checkpoint_dir2)
            gfile.MakeDirs(checkpoint_dir)
            v = tf.Variable([6.0, 7.0, 8.0], name="v")
            with self.test_session():
                self.assertEqual(False, tf.is_variable_initialized(v).eval())
            tf.train.SessionManager(ready_op=tf.assert_variables_initialized())
            saver = tf.train.Saver({"v": v})
            # This should fail as there's no checkpoint within 2 seconds.
            with self.assertRaisesRegexp(RuntimeError, "no init_op or init_fn was given"):
                sess = sm.prepare_session(
                    "",
                    init_op=None,
                    saver=saver,
                    checkpoint_dir=checkpoint_dir,
                    wait_for_checkpoint=True,
                    max_wait_secs=2,
                )
            # Rename the checkpoint directory back.
            gfile.DeleteRecursively(checkpoint_dir)
            os.rename(checkpoint_dir2, checkpoint_dir)
            # This should succeed as there's checkpoint.
            sess = sm.prepare_session(
                "", init_op=None, saver=saver, checkpoint_dir=checkpoint_dir, wait_for_checkpoint=True, max_wait_secs=2
            )
            self.assertEqual(True, tf.is_variable_initialized(sess.graph.get_tensor_by_name("v:0")).eval(session=sess))

 def testPrepareSessionDidNotInitLocalVariable(self):
   with tf.Graph().as_default():
     v = tf.Variable(1, name="v")
     w = tf.Variable(
         v,
         trainable=False,
         collections=[tf.GraphKeys.LOCAL_VARIABLES],
         name="w")
     with self.test_session():
       self.assertEqual(False, tf.is_variable_initialized(v).eval())
       self.assertEqual(False, tf.is_variable_initialized(w).eval())
     sm2 = tf.train.SessionManager(
         ready_op=tf.report_uninitialized_variables())
     with self.assertRaisesRegexp(RuntimeError,
                                  "Init operations did not make model ready"):
       sm2.prepare_session("", init_op=v.initializer)

    def initialize_uninitialized(self, sess):
        global_vars          = tf.global_variables()
        is_not_initialized   = sess.run([tf.is_variable_initialized(var) for var in global_vars])
        not_initialized_vars = [v for (v, f) in zip(global_vars, is_not_initialized) if not f]

        if len(not_initialized_vars):
            sess.run(tf.variables_initializer(not_initialized_vars))

def initialize_uninitialized(sess):
    global_vars          = tf.global_variables()
    is_not_initialized   = sess.run([tf.is_variable_initialized(var) for var in global_vars])
    not_initialized_vars = [v for (v, f) in zip(global_vars, is_not_initialized) if not f]

    print([str(i.name) for i in not_initialized_vars]) # only for testing
    if len(not_initialized_vars):
        sess.run(tf.variables_initializer(not_initialized_vars))

    def _build(self):
        unconstrained = self._build_parameter()
        constrained = self._build_constrained(unconstrained)
        prior = self._build_prior(unconstrained, constrained)

        self._is_initialized_tensor = tf.is_variable_initialized(unconstrained)
        self._unconstrained_tensor = unconstrained
        self._constrained_tensor = constrained
        self._prior_tensor = prior

 def testPrepareSessionWithInsufficientReadyForLocalInitCheck(self):
   with tf.Graph().as_default():
     v = tf.Variable(1, name="v")
     w = tf.Variable(
         v,
         trainable=False,
         collections=[tf.GraphKeys.LOCAL_VARIABLES],
         name="w")
     with self.test_session():
       self.assertEqual(False, tf.is_variable_initialized(v).eval())
       self.assertEqual(False, tf.is_variable_initialized(w).eval())
     sm2 = tf.train.SessionManager(
         ready_op=tf.report_uninitialized_variables(),
         ready_for_local_init_op=None,
         local_init_op=w.initializer)
   with self.assertRaisesRegexp(tf.errors.FailedPreconditionError,
                                "Attempting to use uninitialized value v"):
     sm2.prepare_session("", init_op=None)

  def testRecoverSessionFailsStillRunsLocalInitOp(self):
    # Create a checkpoint.
    checkpoint_dir = os.path.join(
        self.get_temp_dir(),
        "recover_session_ready_for_local_init_fails_stil_run")
    try:
      gfile.DeleteRecursively(checkpoint_dir)
    except errors.OpError:
      pass  # Ignore
    gfile.MakeDirs(checkpoint_dir)

    # Create a new Graph and SessionManager and recover.
    with tf.Graph().as_default():
      v = tf.Variable(2, name="v")
      w = tf.Variable(
          1,
          trainable=False,
          collections=[tf.GraphKeys.LOCAL_VARIABLES],
          name="w")
      with self.test_session():
        self.assertEqual(False, tf.is_variable_initialized(v).eval())
        self.assertEqual(False, tf.is_variable_initialized(w).eval())
      sm2 = tf.train.SessionManager(
          ready_op=tf.report_uninitialized_variables(),
          ready_for_local_init_op=None,
          local_init_op=w.initializer)
      saver = tf.train.Saver({"v": v})
      sess, initialized = sm2.recover_session(
          "",
          saver=saver,
          checkpoint_dir=checkpoint_dir,
          wait_for_checkpoint=False)
      self.assertFalse(initialized)
      self.assertEqual(
          False,
          tf.is_variable_initialized(sess.graph.get_tensor_by_name("v:0")).eval(
              session=sess))
      self.assertEqual(
          True,
          tf.is_variable_initialized(sess.graph.get_tensor_by_name("w:0")).eval(
              session=sess))
      self.assertEquals(1, sess.run(w))

def initialize_uninitialized(sess = None):
    """
    Initialize unitialized variables, doesn't affect those already initialized
    :param sess: in which session to initialize stuff. Defaults to tf.get_default_session()
    """
    sess = sess or tf.get_default_session()
    global_vars          = tf.global_variables()
    is_not_initialized   = sess.run([tf.is_variable_initialized(var) for var in global_vars])
    not_initialized_vars = [v for (v, f) in zip(global_vars, is_not_initialized) if not f]

    if len(not_initialized_vars):
        sess.run(tf.variables_initializer(not_initialized_vars))

 def save(self, save_file_name = None):
     variables = []
     for i in self._vars:
         if tf.is_variable_initialized(i).eval():
             try:
                 variables.append((self.removeUUIDandColon(i.name),i.value().eval()))
             except:
                 #TODO: Don't do this. Limit exceptions to known expected ones.
                 pass
     if save_file_name is None:
         save_file_name = self._file_name
     with open(self._file_name, "wb") as file:
         pkl.dump(variables, file)

  def testPrepareSessionReadyWithInit(self):
    with tf.Graph().as_default():
      v = tf.Variable(1, name="v")
      is_v_initialized = tf.is_variable_initialized(v)
      with self.test_session():
        self.assertEqual(False, is_v_initialized.eval())
      sm = tf.train.SessionManager()

      # Prepare session returns a session even though v is not initialized
      # because no ready_op was provided, so model is trivially ready
      sess = sm.prepare_session("")
      self.assertEqual(False, sess.run(is_v_initialized))
      sess.run(v.initializer)
      self.assertEqual(True, sess.run(is_v_initialized))

def initialize_uninitialized_variables(sess):
    """
    Only initialize the weights that have not yet been initialized by other
    means, such as importing a metagraph and a checkpoint. It's useful when
    extending an existing model.
    """
    uninit_vars    = []
    uninit_tensors = []
    for var in tf.global_variables():
        uninit_vars.append(var)
        uninit_tensors.append(tf.is_variable_initialized(var))
    uninit_bools = sess.run(uninit_tensors)
    uninit = zip(uninit_bools, uninit_vars)
    uninit = [var for init, var in uninit if not init]
    sess.run(tf.variables_initializer(uninit))

def init_uninited_vars(vars=None):
    if vars is None: vars = tf.global_variables()
    test_vars = []; test_ops = []
    with tf.control_dependencies(None): # ignore surrounding control_dependencies
        for var in vars:
            assert is_tf_expression(var)
            try:
                tf.get_default_graph().get_tensor_by_name(var.name.replace(':0', '/IsVariableInitialized:0'))
            except KeyError:
                # Op does not exist => variable may be uninitialized.
                test_vars.append(var)
                with absolute_name_scope(var.name.split(':')[0]):
                    test_ops.append(tf.is_variable_initialized(var))
    init_vars = [var for var, inited in zip(test_vars, run(test_ops)) if not inited]
    run([var.initializer for var in init_vars])

def _create_autosummary_var(name, value_expr):
    assert not _autosummary_finalized
    v = tf.cast(value_expr, tf.float32)
    if v.shape.ndims is 0:
        v = [v, np.float32(1.0)]
    elif v.shape.ndims is 1:
        v = [tf.reduce_sum(v), tf.cast(tf.shape(v)[0], tf.float32)]
    else:
        v = [tf.reduce_sum(v), tf.reduce_prod(tf.cast(tf.shape(v), tf.float32))]
    v = tf.cond(tf.is_finite(v[0]), lambda: tf.stack(v), lambda: tf.zeros(2))
    with tf.control_dependencies(None):
        var = tf.Variable(tf.zeros(2)) # [numerator, denominator]
    update_op = tf.cond(tf.is_variable_initialized(var), lambda: tf.assign_add(var, v), lambda: tf.assign(var, v))
    if name in _autosummary_vars:
        _autosummary_vars[name].append(var)
    else:
        _autosummary_vars[name] = [var]
    return update_op

def initialize_uninitialized_global_variables(sess):
    """
    Only initializes the variables of a TensorFlow session that were not
    already initialized.
    :param sess: the TensorFlow session
    :return:
    """
    # List all global variables
    global_vars = tf.global_variables()

    # Find initialized status for all variables
    is_var_init = [tf.is_variable_initialized(var) for var in global_vars]
    is_initialized = sess.run(is_var_init)

    # List all variables that were not initialized previously
    not_initialized_vars = [var for (var, init) in
                            zip(global_vars, is_initialized) if not init]

    # Initialize all uninitialized variables found, if any
    if len(not_initialized_vars):
        sess.run(tf.variables_initializer(not_initialized_vars))