def testRecoverSession(self):
        # Create a checkpoint.
        checkpoint_dir = os.path.join(self.get_temp_dir(), "recover_session")
        try:
            gfile.DeleteRecursively(checkpoint_dir)
        except OSError:
            pass  # Ignore
        gfile.MakeDirs(checkpoint_dir)

        with tf.Graph().as_default():
            v = tf.Variable(1, name="v")
            sm = tf.train.SessionManager(ready_op=tf.assert_variables_initialized())
            saver = tf.train.Saver({"v": v})
            sess, initialized = sm.recover_session("", saver=saver, checkpoint_dir=checkpoint_dir)
            self.assertFalse(initialized)
            sess.run(v.initializer)
            self.assertEquals(1, sess.run(v))
            saver.save(sess, os.path.join(checkpoint_dir, "recover_session_checkpoint"))
        # Create a new Graph and SessionManager and recover.
        with tf.Graph().as_default():
            v = tf.Variable(2, name="v")
            with self.test_session():
                self.assertEqual(False, tf.is_variable_initialized(v).eval())
            sm2 = tf.train.SessionManager(ready_op=tf.assert_variables_initialized())
            saver = tf.train.Saver({"v": v})
            sess, initialized = sm2.recover_session("", saver=saver, checkpoint_dir=checkpoint_dir)
            self.assertTrue(initialized)
            self.assertEqual(True, tf.is_variable_initialized(sess.graph.get_tensor_by_name("v:0")).eval(session=sess))
            self.assertEquals(1, sess.run(v))

    def testPrepareSessionFails(self):
        checkpoint_dir = os.path.join(self.get_temp_dir(), "prepare_session")
        checkpoint_dir2 = os.path.join(self.get_temp_dir(), "prepare_session2")
        try:
            gfile.DeleteRecursively(checkpoint_dir)
            gfile.DeleteRecursively(checkpoint_dir2)
        except OSError:
            pass  # Ignore
        gfile.MakeDirs(checkpoint_dir)

        with tf.Graph().as_default():
            v = tf.Variable([1.0, 2.0, 3.0], name="v")
            sm = tf.train.SessionManager(ready_op=tf.assert_variables_initialized())
            saver = tf.train.Saver({"v": v})
            sess = sm.prepare_session(
                "", init_op=tf.initialize_all_variables(), saver=saver, checkpoint_dir=checkpoint_dir
            )
            self.assertAllClose([1.0, 2.0, 3.0], sess.run(v))
            checkpoint_filename = os.path.join(checkpoint_dir, "prepare_session_checkpoint")
            saver.save(sess, checkpoint_filename)
        # Create a new Graph and SessionManager and recover.
        with tf.Graph().as_default():
            # Renames the checkpoint directory.
            os.rename(checkpoint_dir, checkpoint_dir2)
            gfile.MakeDirs(checkpoint_dir)
            v = tf.Variable([6.0, 7.0, 8.0], name="v")
            with self.test_session():
                self.assertEqual(False, tf.is_variable_initialized(v).eval())
            tf.train.SessionManager(ready_op=tf.assert_variables_initialized())
            saver = tf.train.Saver({"v": v})
            # This should fail as there's no checkpoint within 2 seconds.
            with self.assertRaisesRegexp(RuntimeError, "no init_op or init_fn was given"):
                sess = sm.prepare_session(
                    "",
                    init_op=None,
                    saver=saver,
                    checkpoint_dir=checkpoint_dir,
                    wait_for_checkpoint=True,
                    max_wait_secs=2,
                )
            # Rename the checkpoint directory back.
            gfile.DeleteRecursively(checkpoint_dir)
            os.rename(checkpoint_dir2, checkpoint_dir)
            # This should succeed as there's checkpoint.
            sess = sm.prepare_session(
                "", init_op=None, saver=saver, checkpoint_dir=checkpoint_dir, wait_for_checkpoint=True, max_wait_secs=2
            )
            self.assertEqual(True, tf.is_variable_initialized(sess.graph.get_tensor_by_name("v:0")).eval(session=sess))

    def __init__(self, session, optimizer_critic, optimizer_actor, critic_network, actor_network, gamma_lmbda,
                 state_dim, num_actions, summary_writer=None, summary_every=5):

        self.session = session
        self.summary_writer = summary_writer
        self.optimizer_critic = optimizer_critic
        self.optimizer_actor = optimizer_actor

        self.actor_network = actor_network
        self.critic_network = critic_network

        self.state_dim = state_dim
        self.num_actions = num_actions
        self.gamma_lmbda = tf.constant(gamma_lmbda)

        # initialize the graph on tensorflow
        self.create_variables()
        var_lists = tf.get_collection(tf.GraphKeys.VARIABLES)
        self.session.run(tf.initialize_variables(var_lists))

        # make sure the variables in graph are initialized
        self.session.run(tf.assert_variables_initialized())

        if self.summary_writer is not None:
            self.summary_writer.add_graph(self.session.graph)
            self.summary_every = summary_every

 def testPrepareSessionSucceedsWithInitFeedDict(self):
     with tf.Graph().as_default():
         p = tf.placeholder(tf.float32, shape=(3,))
         v = tf.Variable(p, name="v")
         sm = tf.train.SessionManager(ready_op=tf.assert_variables_initialized())
         sess = sm.prepare_session("", init_op=tf.initialize_all_variables(), init_feed_dict={p: [1.0, 2.0, 3.0]})
         self.assertAllClose([1.0, 2.0, 3.0], sess.run(v))

 def testPrepareSessionSucceedsWithInitFn(self):
   with tf.Graph().as_default():
     v = tf.Variable([125], name="v")
     sm = tf.train.SessionManager(ready_op=tf.assert_variables_initialized())
     sess = sm.prepare_session("",
                               init_fn=lambda sess: sess.run(v.initializer))
     self.assertAllClose([125], sess.run(v))

def deconv2d(x, num_filters, filter_size=[3, 3], stride=[1, 1], pad='SAME', nonlinearity=None, init_scale=1., counters={}, init=False, ema=None, **kwargs):
    ''' transposed convolutional layer '''
    name = get_name('deconv2d', counters)
    xs = int_shape(x)
    if pad == 'SAME':
        target_shape = [xs[0], xs[1] * stride[0],
                        xs[2] * stride[1], num_filters]
    else:
        target_shape = [xs[0], xs[1] * stride[0] + filter_size[0] -
                        1, xs[2] * stride[1] + filter_size[1] - 1, num_filters]
    with tf.variable_scope(name):
        if init:
            # data based initialization of parameters
            V = tf.get_variable('V', filter_size + [num_filters, int(x.get_shape(
            )[-1])], tf.float32, tf.random_normal_initializer(0, 0.05), trainable=True)
            V_norm = tf.nn.l2_normalize(V.initialized_value(), [0, 1, 3])
            x_init = tf.nn.conv2d_transpose(x, V_norm, target_shape, [
                                            1] + stride + [1], padding=pad)
            m_init, v_init = tf.nn.moments(x_init, [0, 1, 2])
            scale_init = init_scale / tf.sqrt(v_init + 1e-8)
            g = tf.get_variable('g', dtype=tf.float32,
                                initializer=scale_init, trainable=True)
            b = tf.get_variable('b', dtype=tf.float32,
                                initializer=-m_init * scale_init, trainable=True)
            x_init = tf.reshape(scale_init, [
                                1, 1, 1, num_filters]) * (x_init - tf.reshape(m_init, [1, 1, 1, num_filters]))
            if nonlinearity is not None:
                x_init = nonlinearity(x_init)
            return x_init

        else:
            V, g, b = get_vars_maybe_avg(['V', 'g', 'b'], ema)
            tf.assert_variables_initialized([V, g, b])

            # use weight normalization (Salimans & Kingma, 2016)
            W = tf.reshape(g, [1, 1, num_filters, 1]) * \
                tf.nn.l2_normalize(V, [0, 1, 3])

            # calculate convolutional layer output
            x = tf.nn.conv2d_transpose(
                x, W, target_shape, [1] + stride + [1], padding=pad)
            x = tf.nn.bias_add(x, b)

            # apply nonlinearity
            if nonlinearity is not None:
                x = nonlinearity(x)
            return x

  def __init__(self, session,
                     optimizer,
                     actor_network,
                     critic_network,
                     state_dim,
                     action_dim,
                     batch_size=32,
                     replay_buffer_size=1000000, # size of replay buffer
                     store_replay_every=1,       # how frequent to store experience
                     discount_factor=0.99,       # discount future rewards
                     target_update_rate=0.01,
                     reg_param=0.01,             # regularization constants
                     max_gradient=5,             # max gradient norms
                     noise_sigma=0.20,
                     noise_theta=0.15,
                     summary_writer=None,
                     summary_every=100):

    # tensorflow machinery
    self.session        = session
    self.optimizer      = optimizer
    self.summary_writer = summary_writer

    # model components
    self.actor_network  = actor_network
    self.critic_network = critic_network
    self.replay_buffer  = ReplayBuffer(buffer_size=replay_buffer_size)

    # training parameters
    self.batch_size         = batch_size
    self.state_dim          = state_dim
    self.action_dim         = action_dim
    self.discount_factor    = discount_factor
    self.target_update_rate = target_update_rate
    self.max_gradient       = max_gradient
    self.reg_param          = reg_param

    # Ornstein-Uhlenbeck noise for exploration
    self.noise_var = tf.Variable(tf.zeros([1, action_dim]))
    noise_random = tf.random_normal([1, action_dim], stddev=noise_sigma)
    self.noise = self.noise_var.assign_sub((noise_theta) * self.noise_var - noise_random)

    # counters
    self.store_replay_every   = store_replay_every
    self.store_experience_cnt = 0
    self.train_iteration      = 0

    # create and initialize variables
    self.create_variables()
    var_lists = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
    self.session.run(tf.variables_initializer(var_lists))

    # make sure all variables are initialized
    self.session.run(tf.assert_variables_initialized())

    if self.summary_writer is not None:
      # graph was not available when journalist was created
      self.summary_writer.add_graph(self.session.graph)
      self.summary_every = summary_every

  def __init__(self, session,
                     optimizer,
                     policy_network,
                     state_dim,
                     num_actions,
                     init_exp=0.5,         # initial exploration prob
                     final_exp=0.0,        # final exploration prob
                     anneal_steps=10000,   # N steps for annealing exploration
                     discount_factor=0.99, # discount future rewards
                     reg_param=0.001,      # regularization constants
                     max_gradient=5,       # max gradient norms
                     summary_writer=None,
                     summary_every=100):

    # tensorflow machinery
    self.session        = session
    self.optimizer      = optimizer
    self.summary_writer = summary_writer

    # model components
    self.policy_network = policy_network

    # training parameters
    self.state_dim       = state_dim
    self.num_actions     = num_actions
    self.discount_factor = discount_factor
    self.max_gradient    = max_gradient
    self.reg_param       = reg_param

    # exploration parameters
    self.exploration  = init_exp
    self.init_exp     = init_exp
    self.final_exp    = final_exp
    self.anneal_steps = anneal_steps

    # counters
    self.train_iteration = 0

    # rollout buffer
    self.state_buffer  = []
    self.reward_buffer = []
    self.action_buffer = []

    # record reward history for normalization
    self.all_rewards = []
    self.max_reward_length = 1000000

    # create and initialize variables
    self.create_variables()
    var_lists = tf.get_collection(tf.GraphKeys.VARIABLES)
    self.session.run(tf.initialize_variables(var_lists))

    # make sure all variables are initialized
    self.session.run(tf.assert_variables_initialized())

    if self.summary_writer is not None:
      # graph was not available when journalist was created
      self.summary_writer.add_graph(self.session.graph)
      self.summary_every = summary_every

def initializeRemainingVars(sess,feed_dict):
    varlist = tf.global_variables()
    for var in varlist:
        try:
            sess.run(tf.assert_variables_initialized([var]))
        except tf.errors.FailedPreconditionError:
            sess.run(tf.variables_initializer([var]))
            print('Initializing variable:%s'%var.name)

    def testWaitForSessionReturnsNoneAfterTimeout(self):
        with tf.Graph().as_default():
            tf.Variable(1, name="v")
            sm = tf.train.SessionManager(ready_op=tf.assert_variables_initialized(), recovery_wait_secs=1)

            # Set max_wait_secs to allow us to try a few times.
            with self.assertRaises(errors.DeadlineExceededError):
                sm.wait_for_session(master="", max_wait_secs=3)

 def start(self):
   with self._sess.graph.as_default():
     self.run(tf.assert_variables_initialized())
     # create and launch threads for all queue_runners
     # it is like start_queue_runners, but manually
     for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
       self._threads.extend(qr.create_threads(
         self._sess, coord=self._coord, daemon=True, start=True
       ))

 def _create_initializers(self):
   if self._var_count != len(tf.all_variables()):
     self._saver = tf.train.Saver(tf.all_variables(), max_to_keep=5)
     self._init = tf.initialize_all_variables()
     self._check_inited = tf.assert_variables_initialized()
     self._var_count = len(tf.all_variables())
     if self._summary_writer:
       self._summaries = tf.merge_all_summaries()
       self._summary_writer.add_graph(tf.get_default_graph().as_graph_def())

  def __init__(self, session,
                     optimizer,
                     policy_network,
                     observation_dim,
                     num_actions,
                     gru_unit_size,
                     num_step,
                     num_layers,
                     save_path,
                     global_step,
                     max_gradient=5,
                     entropy_bonus=0.001,
                     summary_writer=None,
                     loss_function="l2",
                     summary_every=100):

    # tensorflow machinery
    self.session        = session
    self.optimizer      = optimizer
    self.summary_writer = summary_writer
    self.summary_every  = summary_every
    self.gru_unit_size  = gru_unit_size
    self.num_step       = num_step
    self.num_layers     = num_layers
    self.no_op          = tf.no_op()

    # model components
    self.policy_network  = policy_network
    self.observation_dim = observation_dim
    self.num_actions     = num_actions
    self.loss_function   = loss_function

    # training parameters
    self.max_gradient    = max_gradient
    self.entropy_bonus   = entropy_bonus

    #counter
    self.global_step = global_step

    # create and initialize variables
    self.create_variables()
    var_lists = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
    self.session.run(tf.variables_initializer(var_lists))

    # make sure all variables are initialized
    self.session.run(tf.assert_variables_initialized())

    # try load saved model
    self.saver = tf.train.Saver(tf.global_variables())
    self.save_path = save_path
    self.load_model()

    if self.summary_writer is not None:
      # graph was not available when journalist was created
      self.summary_writer.add_graph(self.session.graph)
      self.summary_every = summary_every

 def testVariables(self):
   with tf.Graph().as_default(), self.test_session() as sess:
     v = tf.Variable([1, 2])
     w = tf.Variable([3, 4])
     _ = v, w
     inited = tf.assert_variables_initialized()
     with self.assertRaisesOpError("Attempting to use uninitialized value"):
       sess.run(inited)
     tf.initialize_all_variables().run()
     sess.run(inited)

 def testVariableList(self):
   with tf.Graph().as_default(), self.test_session() as sess:
     v = tf.Variable([1, 2])
     w = tf.Variable([3, 4])
     inited = tf.assert_variables_initialized([v])
     with self.assertRaisesOpError("Attempting to use uninitialized value"):
       inited.op.run()
     sess.run(w.initializer)
     with self.assertRaisesOpError("Attempting to use uninitialized value"):
       inited.op.run()
     v.initializer.run()
     inited.op.run()

 def _create_initializers(self):
   if self._var_count != len(tf.all_variables()):
     save_dir = os.path.dirname(self._save_path) if self._save_path else None
     if save_dir and not tf.gfile.IsDirectory(save_dir):
       tf.gfile.MakeDirs(save_dir)
     self._saver = tf.train.Saver(tf.all_variables(), max_to_keep=5)
     self._init = tf.initialize_all_variables()
     self._check_inited = tf.assert_variables_initialized()
     self._var_count = len(tf.all_variables())
     if self._summary_writer:
       self._summaries = tf.merge_all_summaries()
       self._summary_writer.add_graph(tf.get_default_graph())

 def is_initialized_in(self, session):
     """Check if the TensorFlow variables are initialized"""
     if self._initialized:
         return True
     tf_var_list = self.get_tensorflow_variables()
     if len(tf_var_list) == 0:
         return True
     try:
         session.run(tf.assert_variables_initialized(self.get_tensorflow_variables()))
         self._initialized = True
         return True
     except tf.errors.FailedPreconditionError:
         return False

def dense(x, num_units, nonlinearity=None, init_scale=1., counters={}, init=False, ema=None, **kwargs):
    ''' fully connected layer '''
    name = get_name('dense', counters)
    with tf.variable_scope(name):
        if init:
            # data based initialization of parameters
            V = tf.get_variable('V', [int(x.get_shape()[
                                1]), num_units], tf.float32, tf.random_normal_initializer(0, 0.05), trainable=True)
            V_norm = tf.nn.l2_normalize(V.initialized_value(), [0])
            x_init = tf.matmul(x, V_norm)
            m_init, v_init = tf.nn.moments(x_init, [0])
            scale_init = init_scale / tf.sqrt(v_init + 1e-10)
            g = tf.get_variable('g', dtype=tf.float32,
                                initializer=scale_init, trainable=True)
            b = tf.get_variable('b', dtype=tf.float32,
                                initializer=-m_init * scale_init, trainable=True)
            x_init = tf.reshape(
                scale_init, [1, num_units]) * (x_init - tf.reshape(m_init, [1, num_units]))
            if nonlinearity is not None:
                x_init = nonlinearity(x_init)
            return x_init

        else:
            V, g, b = get_vars_maybe_avg(['V', 'g', 'b'], ema)
            tf.assert_variables_initialized([V, g, b])

            # use weight normalization (Salimans & Kingma, 2016)
            x = tf.matmul(x, V)
            scaler = g / tf.sqrt(tf.reduce_sum(tf.square(V), [0]))
            x = tf.reshape(scaler, [1, num_units]) * \
                x + tf.reshape(b, [1, num_units])

            # apply nonlinearity
            if nonlinearity is not None:
                x = nonlinearity(x)
            return x

 def testPrepareSessionSucceeds(self):
     with tf.Graph().as_default():
         v = tf.Variable([1.0, 2.0, 3.0], name="v")
         sm = tf.train.SessionManager(ready_op=tf.assert_variables_initialized())
         sess = sm.prepare_session("", init_op=tf.initialize_all_variables())
         self.assertAllClose([1.0, 2.0, 3.0], sess.run(v))

 def testNoVars(self):
   with tf.Graph().as_default():
     self.assertEqual(None, tf.assert_variables_initialized())