def forward_prop_nodes(i_start, size, acts, offset):
  # Note: In the corpus that we've seen, parse trees are always ordered such that
  # iteration forward through the list will be in bottom-up order.
  # Conversely, iteration in reverse is always top-down.
  # This enables a simple iterative algorithm. If this were not the case,
  # putting the nodes in order by a postorder traversal would fix it.
  def fwd_continue(*parms):
    (_, sz, cur, _) = parms
    return tf.less(cur, sz, name='cur_le_size')

  def forward_prop(*parms):
    (i0, sz, cur, act) = parms
    with tf.device('/gpu:0'):
      gact = act
      gcur = cur
      next_idx = i0 + gcur
    node_out = tf.reshape(forward_node(next_idx, act, offset), [1, FLAGS.wvs, 1], name='node_out')
    tf.scatter_add(gact, tf.pack([gcur]), node_out, name='act_update')
    act = gact
    return [i0, sz, cur + iONE, act]

  with tf.device('/cpu:0'):
    i_start = tf.convert_to_tensor(i_start, dtype=tf.int32, name='i_start')
    size = tf.convert_to_tensor(size, dtype=tf.int32, name='size')
    iZ = tf.convert_to_tensor(0, dtype=tf.int32, name='ZERO')

  while_parms = [i_start, size, iZ, acts]
  wresult = tf.while_loop(fwd_continue, forward_prop, while_parms, parallel_iterations=1,
                          name='forward_prop_while')
  (_, _, _, result) = wresult
  return tf.slice(result, [0, 0, 0], tf.pack([size, -1, -1]), name='fwd_prop_nodes')

  def testLSTMBasicToBlockPeeping(self):
    with self.test_session(use_gpu=self._use_gpu) as sess:
      batch_size = 2
      input_size = 3
      cell_size = 4
      sequence_length = 5

      inputs = []
      for _ in range(sequence_length):
        inp = tf.convert_to_tensor(
            np.random.randn(batch_size, input_size),
            dtype=tf.float32)
        inputs.append(inp)

      initializer = tf.random_uniform_initializer(-0.01, 0.01, seed=19890212)
      with tf.variable_scope("basic", initializer=initializer):
        cell = tf.nn.rnn_cell.LSTMCell(cell_size,
                                       use_peepholes=True,
                                       state_is_tuple=True)
        outputs, _ = tf.nn.rnn(cell, inputs, dtype=tf.float32)

        sess.run([tf.initialize_all_variables()])
        basic_outputs = sess.run(outputs)
        basic_grads = sess.run(tf.gradients(outputs, inputs))
        basic_wgrads = sess.run(tf.gradients(outputs, tf.trainable_variables()))

      with tf.variable_scope("block", initializer=initializer):
        w = tf.get_variable("w",
                            shape=[input_size + cell_size, cell_size * 4],
                            dtype=tf.float32)
        b = tf.get_variable("b",
                            shape=[cell_size * 4],
                            dtype=tf.float32,
                            initializer=tf.zeros_initializer)

        wci = tf.get_variable("wci", shape=[cell_size], dtype=tf.float32)
        wcf = tf.get_variable("wcf", shape=[cell_size], dtype=tf.float32)
        wco = tf.get_variable("wco", shape=[cell_size], dtype=tf.float32)

        _, _, _, _, _, _, outputs = fused_lstm(
            tf.convert_to_tensor(sequence_length,
                                 dtype=tf.int64),
            inputs,
            w,
            b,
            wci=wci,
            wcf=wcf,
            wco=wco,
            cell_clip=0,
            use_peephole=True)

        sess.run([tf.initialize_all_variables()])
        block_outputs = sess.run(outputs)
        block_grads = sess.run(tf.gradients(outputs, inputs))
        block_wgrads = sess.run(tf.gradients(outputs, [w, b, wci, wcf, wco]))

      self.assertAllClose(basic_outputs, block_outputs)
      self.assertAllClose(basic_grads, block_grads)
      for basic, block in zip(basic_wgrads, block_wgrads):
        self.assertAllClose(basic, block, rtol=1e-2, atol=1e-2)

def one_minus_pseudo_unitcell_transfer_op(direction, mps, left_dominant,
                                          right_dominant, vector):
    """
    calculates action of 11-Transfer-Operator +|r)(l|
    Parameters:
    ---------------------------
    direction:  int or str 
                if (1,'l','left'): do left multiplication
                if (-1,'r','right'): do right multiplication
    mps:        InfiniteMPSCentralGauge object
                an infinite mps
    left_dominant:  tf.tensor of shape (mps.D[0],mps.D[0])
                    left dominant eigenvector of the unit-cell transfer operator of mps
    right_dominant: tf.tensor of shape (mps.D[-1],mps.D[-1])
                    right dominant eigenvector of the unit-cell transfer operator of mps
    vector:         tf.tensor of shape (mps.D[0]*mps.D[0]) or (mps.D[-1]*mps.D[-1])
                    the input vector
    Returns
    ---------------------------
    np.ndarray of shape (mps.D[0]*mps.D[0]) or (mps.D[-1]*mps.D[-1])

    """

    if direction in (1, 'l', 'left'):
        x = tf.reshape(tf.convert_to_tensor(vector), (mps.D[0], mps.D[0]))
        temp = x - mps.unitcell_transfer_op('left', x) + ncon(
            [x, right_dominant], [[1, 2], [1, 2]]) * left_dominant
        return tf.reshape(temp, [mps.D[-1] * mps.D[-1]]).numpy()

    if direction in (-1, 'r', 'right'):
        x = tf.reshape(tf.convert_to_tensor(vector), [mps.D[-1], mps.D[-1]])
        temp = x - mps.unitcell_transfer_op('right', x) + ncon(
            [left_dominant, x], [[1, 2], [1, 2]]) * right_dominant
        return tf.reshape(temp, [mps.D[0] * mps.D[0]]).numpy()

  def testOneOpCond(self):
    with self.test_session():
      v = tf.Variable(0)
      c = tf.convert_to_tensor(0)
      one = tf.convert_to_tensor(1)
      two = tf.convert_to_tensor(2)
      p = tf.greater_equal(c, 1)

      def a():
        return tf.assign(v, one)

      def b():
        return tf.assign(v, two)

      i = tf.cond(p, a, b)
      self.assertTrue(isinstance(i, tf.Tensor))
      tf.initialize_all_variables().run()

      self.assertEqual(0, v.eval())

      # True case: c = 2 is >= 1, v is set to 1.
      self.assertEqual(1, i.eval(feed_dict={c.name: 2}))
      self.assertEqual(1, v.eval())

      # False case: c = 0 is not >= 1, v is set to 2.
      self.assertEqual(2, i.eval(feed_dict={c.name: 0}))
      self.assertEqual(2, v.eval())

def _prepare_args_with_initial_simplex(objective_function,
                                       initial_simplex,
                                       objective_at_initial_simplex,
                                       batch_evaluate_objective):
  """Evaluates the objective function at the specified initial simplex."""
  initial_simplex = tf.convert_to_tensor(initial_simplex)

  # If d is the dimension of the problem, the number of vertices in the
  # simplex should be d+1. From this, we can infer the number of dimensions
  # as n - 1 where n is the number of vertices specified.
  num_vertices = tf.shape(initial_simplex)[0]
  dim = num_vertices - 1
  num_evaluations = 0

  if objective_at_initial_simplex is None:
    objective_at_initial_simplex, n_evals = _evaluate_objective_multiple(
        objective_function, initial_simplex, batch_evaluate_objective)
    num_evaluations += n_evals
  objective_at_initial_simplex = tf.convert_to_tensor(
      objective_at_initial_simplex)
  return (dim,
          num_vertices,
          initial_simplex,
          objective_at_initial_simplex,
          num_evaluations)

def test_CE_loss(sess, CE_arrays):
    y, y_hat = CE_arrays
    y = tf.convert_to_tensor(y, dtype=tf.float64)
    y_hat = tf.convert_to_tensor(y_hat, dtype=tf.float64)
    sess.run(cross_entropy_loss(y,y_hat))
    assert 1
    print("CE_loss ran to completion")

 def testSumGradArgs(self):
   with self.test_session(use_gpu=False):
     indices = [tf.convert_to_tensor([0, 1, 2, 3]),
                tf.convert_to_tensor([2, 3])]
     values = [tf.convert_to_tensor([2, 3, 5, 7]), tf.convert_to_tensor([1, 1])]
     self.assertAllEqual(
         tf.dynamic_stitch(indices, values).eval(), [2, 3, 1, 1])

def file_to_dataset(filepath, N, vectorizer):
    #convert file to dataset, returns X and Y tensors of integer indexes describing the N words (X) leading up to (Y)
    f = open(filepath)
    lines = f.readlines()
    X = []
    Y = []
    line_index = 0
    for line in lines:
        if line_index % 1000 == 0:
            print(str(line_index) +'/' len(lines) + ' lines read...')
        line_index +=1
        words =  line.split()
        padding = (N-1)*['<pre>']

        #remove /n at the end, add padding to front
        words = padding + words[:-1]
        for i in range(0, len(words) - N):
            x = []
            y = vectorizer.transform(words[i+N]).toarray()
            x_words = words[i:i+N]
            for word in x_words:
                x.append(vectorizer.transform(word).toarray())
            X.append(x)
            Y.append(y)

    return tf.convert_to_tensor(X), tf.convert_to_tensor(Y)

 def _compare(self, func, x, y, dtype):
     with self.test_session(use_gpu=False):
         out = func(
             tf.convert_to_tensor(np.array([x]).astype(dtype)), tf.convert_to_tensor(np.array([y]).astype(dtype))
         )
         ret = out.eval()
     return ret[0]

    def log_prob(self, xs, zs):
        """
        Parameters
        ----------
        xs : dict of str to tf.Tensor
            Data dictionary. Each key is a data structure used in the
            model (Theano shared variable), and its value is the
            corresponding realization (tf.Tensor).
        zs : dict of str to tf.Tensor
            Latent variable dictionary. Each key names a latent variable
            used in the model (str), and its value is the corresponding
            realization (tf.Tensor).

        Returns
        -------
        tf.Tensor
            Scalar, the log joint density log p(xs, zs).

        Notes
        -----
        It wraps around a Python function. The Python function takes
        inputs of type np.ndarray and outputs a np.ndarray.
        """
        # Store keys so that ``_py_log_prob_args`` knows how each
        # value corresponds to a key.
        self.xs_keys = list(six.iterkeys(xs))
        self.zs_keys = list(six.iterkeys(zs))

        # Pass in all tensors as a flattened list for tf.py_func().
        inputs = [tf.convert_to_tensor(x) for x in six.itervalues(xs)]
        inputs += [tf.convert_to_tensor(z) for z in six.itervalues(zs)]

        return tf.py_func(self._py_log_prob_args, inputs, [tf.float32])[0]

 def __init__(self, data_dir, data_list, input_size, 
              random_scale, random_mirror, ignore_label, img_mean, coord):
     '''Initialise an ImageReader.
     
     Args:
       data_dir: path to the directory with images and masks.
       data_list: path to the file with lines of the form '/path/to/image /path/to/mask'.
       input_size: a tuple with (height, width) values, to which all the images will be resized.
       random_scale: whether to randomly scale the images prior to random crop.
       random_mirror: whether to randomly mirror the images prior to random crop.
       ignore_label: index of label to ignore during the training.
       img_mean: vector of mean colour values.
       coord: TensorFlow queue coordinator.
     '''
     self.data_dir = data_dir
     self.data_list = data_list
     self.input_size = input_size
     self.coord = coord
     
     self.image_list, self.label_list = read_labeled_image_list(self.data_dir, self.data_list)
     self.images = tf.convert_to_tensor(self.image_list, dtype=tf.string)
     self.labels = tf.convert_to_tensor(self.label_list, dtype=tf.string)
     self.queue = tf.train.slice_input_producer([self.images, self.labels],
                                                shuffle=input_size is not None) # not shuffling if it is val
     self.image, self.label = read_images_from_disk(self.queue, self.input_size, random_scale, random_mirror, ignore_label, img_mean) 

  def testPlaceholder(self):
    with self.test_session(use_gpu=True):
      # Test using placeholder with a defined shape.
      ph_0 = tf.placeholder(tf.int32, shape=[])
      result_0 = tf.convert_to_tensor([[0, 0, 0],
                                       [0, ph_0, 0],
                                       [0, 0, 0]])
      self.assertAllEqual([[0, 0, 0],
                           [0, 1, 0],
                           [0, 0, 0]],
                          result_0.eval(feed_dict={ph_0: 1}))
      self.assertAllEqual([[0, 0, 0],
                           [0, 2, 0],
                           [0, 0, 0]],
                          result_0.eval(feed_dict={ph_0: 2}))

      # Test using placeholder with an undefined shape.
      ph_1 = tf.placeholder(tf.int32)
      result_1 = tf.convert_to_tensor([[0, 0, 0],
                                       [0, ph_1, 0],
                                       [0, 0, 0]])
      self.assertAllEqual([[0, 0, 0], [0, 1, 0], [0, 0, 0]],
                          result_1.eval(feed_dict={ph_1: 1}))
      self.assertAllEqual([[0, 0, 0], [0, 2, 0], [0, 0, 0]],
                          result_1.eval(feed_dict={ph_1: 2}))

 def test_regression_metrics(self):
     with self.test_session():
         y_a = tf.convert_to_tensor(np.random.random((6, 7)))
         y_b = tf.convert_to_tensor(np.random.random((6, 7)))
         for metric in all_regression_metrics:
             output = metric(y_a, y_b)
             assert output.eval().shape == ()

    def testLoop_1(self):
        with self.test_session():
            zero = tf.convert_to_tensor(0)
            one = tf.convert_to_tensor(1)
            n = tf.constant(10)

            enter_zero = control_flow_ops.enter(zero, "foo_1", False)
            enter_one = control_flow_ops.enter(one, "foo_1", False)
            enter_n = control_flow_ops.enter(n, "foo_1", False)
            merge_zero = control_flow_ops.merge([enter_zero, enter_zero], name="merge_zero")[0]
            merge_one = control_flow_ops.merge([enter_one, enter_one], name="merge_one")[0]
            merge_n = control_flow_ops.merge([enter_n, enter_n], name="merge_n")[0]
            less_op = tf.less(merge_n, merge_n)
            cond_op = control_flow_ops.loop_cond(less_op)
            switch_zero = control_flow_ops.switch(merge_zero, cond_op)
            switch_one = control_flow_ops.switch(merge_one, cond_op)
            switch_n = control_flow_ops.switch(merge_n, cond_op)
            next_zero = control_flow_ops.next_iteration(switch_zero[1])
            next_one = control_flow_ops.next_iteration(switch_one[1])
            next_n = control_flow_ops.next_iteration(switch_n[1])
            merge_zero.op._update_input(1, next_zero)
            merge_one.op._update_input(1, next_one)
            merge_n.op._update_input(1, next_n)
            exit_n = control_flow_ops.exit(switch_n[0])

            result = exit_n.eval()
        self.assertAllEqual(10, result)

def batched_index(values, indices):
  """Equivalent to `values[:, indices]`.

  Performs indexing on batches and sequence-batches by reducing over
  zero-masked values. Compared to indexing with `tf.gather` this approach is
  more general and TPU-friendly, but may be less efficient if `num_values`
  is large. It works with tensors whose shapes are unspecified or
  partially-specified, but this op will only do shape checking on shape
  information available at graph construction time. When complete shape
  information is absent, certain shape incompatibilities may not be detected at
  runtime! See `indexing_ops_test` for detailed examples.

  Args:
    values: tensor of shape `[B, num_values]` or `[T, B, num_values]`
    indices: tensor of shape `[B]` or `[T, B]` containing indices.

  Returns:
    Tensor of shape `[B]` or `[T, B]` containing values for the given indices.

  Raises: ValueError if values and indices have sizes that are known
    statically (i.e. during graph construction), and those sizes are not
    compatible (see shape descriptions in Args list above).
  """
  with tf.name_scope("batch_indexing", values=[values, indices]):
    values = tf.convert_to_tensor(values)
    indices = tf.convert_to_tensor(indices)
    assert_compatible_shapes(values.shape, indices.shape)

    one_hot_indices = tf.one_hot(
        indices, tf.shape(values)[-1], dtype=values.dtype)
    return tf.reduce_sum(values * one_hot_indices, axis=-1)

  def __init__(self,
               loc=0.,
               scale=1.,
               validate_args=False,
               name="gumbel"):
    """Instantiates the `Gumbel` bijector.

    Args:
      loc: Float-like `Tensor` that is the same dtype and is
        broadcastable with `scale`.
        This is `loc` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`.
      scale: Positive Float-like `Tensor` that is the same dtype and is
        broadcastable with `loc`.
        This is `scale` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`.
      validate_args: Python `bool` indicating whether arguments should be
        checked for correctness.
      name: Python `str` name given to ops managed by this object.
    """
    self._graph_parents = []
    self._name = name
    self._validate_args = validate_args
    with self._name_scope("init", values=[loc, scale]):
      self._loc = tf.convert_to_tensor(loc, name="loc")
      self._scale = tf.convert_to_tensor(scale, name="scale")
      tf.assert_same_float_dtype([self._loc, self._scale])
      if validate_args:
        self._scale = control_flow_ops.with_dependencies([
            tf.assert_positive(
                self._scale, message="Argument scale was not positive")
        ], self._scale)

    super(Gumbel, self).__init__(
        validate_args=validate_args,
        forward_min_event_ndims=0,
        name=name)

def test_multivariate_normal(session_tf, x, mu, cov_sqrt):
    cov = np.dot(cov_sqrt, cov_sqrt.T)
    L = np.linalg.cholesky(cov)

    if len(x.shape) != 2 or len(mu.shape) != 2:
        with pytest.raises(Exception) as e_info:
            gp_result = logdensities.multivariate_normal(
                tf.convert_to_tensor(x),
                tf.convert_to_tensor(mu),
                tf.convert_to_tensor(L))
    else:
        x_tf = tf.placeholder(settings.float_type)
        mu_tf = tf.placeholder(settings.float_type)
        gp_result = logdensities.multivariate_normal(
            x_tf, mu_tf, tf.convert_to_tensor(L))

        gp_result = session_tf.run(gp_result, feed_dict={x_tf: x, mu_tf: mu})

        if mu.shape[1] > 1:
            if x.shape[1] > 1:
                sp_result = [mvn.logpdf(x[:,i], mu[:,i], cov) for i in range(mu.shape[1])]
            else:
                sp_result = [mvn.logpdf(x.ravel(), mu[:, i], cov) for i in range(mu.shape[1])]
        else:
            sp_result = mvn.logpdf(x.T, mu.ravel(), cov)
        assert_allclose(gp_result, sp_result)

  def check_split_latent_conditioning(self, merge_std):
    with tf.Graph().as_default():
      rng = np.random.RandomState(0)
      x_rand = rng.randn(12, 32, 32, 32).astype(np.float32)
      latent_rand = rng.randn(12, 32, 32, 16).astype(np.float32)
      x_t = tf.convert_to_tensor(x_rand)
      latent_t = tf.convert_to_tensor(latent_rand)
      hparams = glow.glow_hparams()
      hparams.level_scale = merge_std
      hparams.add_hparam("latent_dist_encoder", "pointwise")

      # Test initalization.
      # x2 ~ N(scale * latent, 1.0) where initial scale is 1.0
      exp_x2 = x_rand[:, :, :, 16:]
      exp_eps = x_rand[:, :, :, 16:] - latent_rand
      x_inv, _, eps, x2_t, _ = glow_ops.split(
          merge_std, x_t, cond_latents=latent_t, hparams=hparams,
          condition=True)
      # Test reversibility.
      x_inv_inv, _, _ = glow_ops.split(
          merge_std, x_inv, cond_latents=latent_t, eps=eps, reverse=True,
          hparams=hparams, condition=True)
      with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        actual_eps, actual_x2, diff_np = sess.run([eps, x2_t, x_inv_inv - x_t])
        self.assertTrue(np.allclose(diff_np, 0.0, atol=1e-5))
        self.assertTrue(np.allclose(actual_eps, exp_eps))
        self.assertTrue(np.allclose(exp_x2, actual_x2))

def _teacher_forcing_ratio_decay(init_tfr, global_step, hparams):
		#################################################################
		# Narrow Cosine Decay:

		# Phase 1: tfr = 1
		# We only start learning rate decay after 10k steps

		# Phase 2: tfr in ]0, 1[
		# decay reach minimal value at step ~280k

		# Phase 3: tfr = 0
		# clip by minimal teacher forcing ratio value (step >~ 280k)
		#################################################################
		#Compute natural cosine decay
		tfr = tf.train.cosine_decay(init_tfr,
			global_step=global_step - hparams.tacotron_teacher_forcing_start_decay, #tfr = 1 at step 10k
			decay_steps=hparams.tacotron_teacher_forcing_decay_steps, #tfr = 0 at step ~280k
			alpha=hparams.tacotron_teacher_forcing_decay_alpha, #tfr = 0% of init_tfr as final value
			name='tfr_cosine_decay')

		#force teacher forcing ratio to take initial value when global step < start decay step.
		narrow_tfr = tf.cond(
			tf.less(global_step, tf.convert_to_tensor(hparams.tacotron_teacher_forcing_start_decay)),
			lambda: tf.convert_to_tensor(init_tfr),
			lambda: tfr)

		return narrow_tfr

  def testSingleExampleWithSparseAndDense(self):
    original = example(features=features(
        {"c": float_feature([3, 4]),
         "st_a": float_feature([3.0, 4.0])}))

    serialized = original.SerializeToString()

    expected_st_a = (
        np.array([[0], [1]], dtype=np.int64),  # indices
        np.array([3.0, 4.0], dtype=np.float32),  # values
        np.array([2], dtype=np.int64))  # shape: max_values = 2

    a_default = [1, 2, 3]
    b_default = np.random.rand(3, 3).astype(bytes)
    expected_output = {
        "st_a": expected_st_a,
        "a": [a_default],
        "b": b_default,
        "c": np.array([3, 4], dtype=np.float32),
    }

    self._test({
        "example_names": tf.convert_to_tensor("in1"),
        "serialized": tf.convert_to_tensor(serialized),
        "features": {
            "st_a": tf.VarLenFeature(tf.float32),
            "a": tf.FixedLenFeature((1, 3), tf.int64, default_value=a_default),
            "b": tf.FixedLenFeature((3, 3), tf.string, default_value=b_default),
            # Feature "c" must be provided, since it has no default_value.
            "c": tf.FixedLenFeature((2,), tf.float32),
        }
    }, expected_output)