def enc_step(step):
   """Encoder step."""
   if autoenc_decay < 1.0:
     quant_step = autoenc_quantize(step, 16, nmaps, self.do_training)
     if backward:
       exp_glob = tf.train.exponential_decay(1.0, self.global_step - 10000,
                                             1000, autoenc_decay)
       dec_factor = 1.0 - exp_glob  # * self.do_training
       dec_factor = tf.cond(tf.less(self.global_step, 10500),
                            lambda: tf.constant(0.05), lambda: dec_factor)
     else:
       dec_factor = 1.0
     cur = tf.cond(tf.less(tf.random_uniform([]), dec_factor),
                   lambda: quant_step, lambda: step)
   else:
     cur = step
   if dropout > 0.0001:
     cur = tf.nn.dropout(cur, keep_prob)
   if act_noise > 0.00001:
     cur += tf.truncated_normal(tf.shape(cur)) * act_noise_scale
   # Do nconvs-many CGRU steps.
   if do_jit and tf.get_variable_scope().reuse:
     with jit_scope():
       for layer in xrange(nconvs):
         cur = conv_gru([], cur, kw, kh, nmaps, conv_rate(layer),
                        cutoff, "ecgru_%d" % layer, do_layer_norm)
   else:
     for layer in xrange(nconvs):
       cur = conv_gru([], cur, kw, kh, nmaps, conv_rate(layer),
                      cutoff, "ecgru_%d" % layer, do_layer_norm)
   return cur

def set_logp_to_neg_inf(X, logp, bounds):
    """Set `logp` to negative infinity when `X` is outside the allowed bounds.

    # Arguments
        X: tensorflow.Tensor
            The variable to apply the bounds to
        logp: tensorflow.Tensor
            The log probability corrosponding to `X`
        bounds: list of `Region` objects
            The regions corrosponding to allowed regions of `X`

    # Returns
        logp: tensorflow.Tensor
            The newly bounded log probability
    """
    conditions = []
    for l, u in bounds:
        lower_is_neg_inf = not isinstance(l, tf.Tensor) and np.isneginf(l)
        upper_is_pos_inf = not isinstance(u, tf.Tensor) and np.isposinf(u)

        if not lower_is_neg_inf and upper_is_pos_inf:
            conditions.append(tf.greater(X, l))
        elif lower_is_neg_inf and not upper_is_pos_inf:
            conditions.append(tf.less(X, u))
        elif not (lower_is_neg_inf or upper_is_pos_inf):
            conditions.append(tf.logical_and(tf.greater(X, l), tf.less(X, u)))

    if len(conditions) > 0:
        is_inside_bounds = conditions[0]
        for condition in conditions[1:]:
            is_inside_bounds = tf.logical_or(is_inside_bounds, condition)

        logp = tf.select(is_inside_bounds, logp, tf.fill(tf.shape(X), config.dtype(-np.inf)))

    return logp

def prune_outside_window(boxlist, window, scope=None):
  """Prunes bounding boxes that fall outside a given window.

  This function prunes bounding boxes that even partially fall outside the given
  window. See also clip_to_window which only prunes bounding boxes that fall
  completely outside the window, and clips any bounding boxes that partially
  overflow.

  Args:
    boxlist: a BoxList holding M_in boxes.
    window: a float tensor of shape [4] representing [ymin, xmin, ymax, xmax]
      of the window
    scope: name scope.

  Returns:
    pruned_corners: a tensor with shape [M_out, 4] where M_out <= M_in
    valid_indices: a tensor with shape [M_out] indexing the valid bounding boxes
     in the input tensor.
  """
  with tf.name_scope(scope, 'PruneOutsideWindow'):
    y_min, x_min, y_max, x_max = tf.split(
        value=boxlist.get(), num_or_size_splits=4, axis=1)
    win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window)
    coordinate_violations = tf.concat([
        tf.less(y_min, win_y_min), tf.less(x_min, win_x_min),
        tf.greater(y_max, win_y_max), tf.greater(x_max, win_x_max)
    ], 1)
    valid_indices = tf.reshape(
        tf.where(tf.logical_not(tf.reduce_any(coordinate_violations, 1))), [-1])
    return gather(boxlist, valid_indices), valid_indices

def compute_IOU(bboxA, bboxB):
    """Compute the Intersection Over Union.
    Args:
        bboxA: [N X 4 tensor] format = [left, top, right, bottom]
        bboxB: [N X 4 tensor] 

    Return:
        IOU: [N X 1 tensor]
    """

    x1A, y1A, x2A, y2A = tf.split(1, 4, bboxA)
    x1B, y1B, x2B, y2B = tf.split(1, 4, bboxB)

    # compute intersection
    x1_max = tf.maximum(x1A, x1B)
    y1_max = tf.maximum(y1A, y1B)
    x2_min = tf.minimum(x2A, x2B)
    y2_min = tf.minimum(y2A, y2B)

    # overlap_flag = tf.logical_and( tf.less(x1_max, x2_min), tf.less(y1_max, y2_min))

    overlap_flag = tf.to_float(tf.less(x1_max, x2_min)) * \
        tf.to_float(tf.less(y1_max, y2_min))

    overlap_area = tf.mul(overlap_flag, tf.mul(
        x2_min - x1_max, y2_min - y1_max))

    # compute union
    areaA = tf.mul(x2A - x1A, y2A - y1A)
    areaB = tf.mul(x2B - x1B, y2B - y1B)
    union_area = areaA + areaB - overlap_area

    return tf.div(overlap_area, union_area)

        def loop(step_, beams_, beam_value_, golden_value_, golden_inside_, step_valid_, g_id_, golden_record, beam_record):
            cur_feat_x_ = tf.gather(x, step_)
            cur_golden_path_ = tf.gather(golden_path, tf.range(step_))
            cur_golden_feat_ = self._add_tag_dynamic(cur_feat_x_, cur_golden_path_)
            # cur_golden_output_ = self._build_cnn(cur_golden_feat_)
            cur_golden_output_ = build(cur_golden_feat_)
            cur_golden_node_ = tf.gather(golden_path, tf.reshape(step_, [1]))
            golden_value_ = tf.add(golden_value_,
                                  tf.slice(cur_golden_output_, tf.concat(0, [[0], cur_golden_node_]), [1, 1]))

            cur_beam_ = tf.unpack(beams_, num=self.beam_size)
            cur_beam_feat_ = tf.concat(0, [self._add_tag_dynamic(cur_feat_x_, tf.reshape(e, [-1])) for e in cur_beam_])
            # cur_beam_output_ = self._build_cnn(cur_beam_feat_)
            cur_beam_output_ = build(cur_beam_feat_)

            golden_record = golden_record.write(step_, cur_golden_output_)
            beam_record = beam_record.write(step_, cur_beam_output_)

            beam_value_, beams_ = self._top_beams_new(cur_beam_output_, beam_value_, beams_)
            new_golden_path_ = tf.gather(golden_path, tf.range(step_ + 1))
            # golden_beam_id_ = index_of_tensor(new_golden_path_, beams_)
            g_id_ = index_of_tensor(new_golden_path_, beams_)
            golden_inside_ = tf.select(tf.less(tf.shape(g_id_)[0], 1),
                                       tf.constant(False, tf.bool), tf.constant(True, tf.bool))

            step_valid_ = tf.logical_and(tf.less(step_+1, length), tf.less(step_+1, self.max_step_tracked))
            return [step_ + 1, beams_, beam_value_, golden_value_, golden_inside_, step_valid_, g_id_, golden_record, beam_record]

def l1_smooth_losses(predict_boxes, gtboxes, object_weights, classes_weights=None):
    '''

    :param predict_boxes: [minibatch_size, -1]
    :param gtboxes: [minibatch_size, -1]
    :param object_weights: [minibatch_size, ]. 1.0 represent object, 0.0 represent others(ignored or background)
    :return:
    '''

    diff = predict_boxes - gtboxes
    abs_diff = tf.cast(tf.abs(diff), tf.float32)

    if classes_weights is None:
        '''
        first_stage:
        predict_boxes :[minibatch_size, 5]
        gtboxes: [minibatchs_size, 5]
        '''
        anchorwise_smooth_l1norm = tf.reduce_sum(
            tf.where(tf.less(abs_diff, 1), 0.5 * tf.square(abs_diff), abs_diff - 0.5),
            axis=1) * object_weights
    else:
        '''
        fast_rcnn stage:
        predict_boxes: [minibatch_size, 5*num_classes]
        gtboxes: [minibatch_size, 5*num_classes]
        classes_weights : [minibatch_size, 5*num_classes]
        '''
        anchorwise_smooth_l1norm = tf.reduce_sum(
            tf.where(tf.less(abs_diff, 1), 0.5*tf.square(abs_diff)*classes_weights,
                     (abs_diff - 0.5)*classes_weights),
            axis=1)*object_weights
    return tf.reduce_mean(anchorwise_smooth_l1norm, axis=0)  # reduce mean

def image_distortions(image, distortions):
    distort_left_right_random = distortions[0]
    mirror = tf.less(tf.pack([1.0, distort_left_right_random, 1.0]), 0.5)
    image = tf.reverse(image, mirror)
    distort_up_down_random = distortions[1]
    mirror = tf.less(tf.pack([distort_up_down_random, 1.0, 1.0]), 0.5)
    image = tf.reverse(image, mirror)
    return image

def learning_rate_schedule():  # Function which controls learning rate during training
  import tensorflow as tf
  step = tf.train.get_or_create_global_step()
  lr = tf.case([(tf.less(step,  1000), lambda: tf.constant(0.0004)),
                (tf.less(step, 10000), lambda: tf.constant(0.01)),
                (tf.less(step, 40000), lambda: tf.constant(0.005)),
                (tf.less(step, 55000), lambda: tf.constant(0.0005)),
                (tf.less(step, 65000), lambda: tf.constant(0.00005))])
  return lr

    def testWhileCond_3(self):
        with self.test_session():
            n = tf.convert_to_tensor(0)
            c = lambda x: tf.less(x, 10)
            b = lambda x: control_flow_ops.cond(tf.less(0, 1), lambda: tf.add(x, 1), lambda: tf.sub(x, 1))
            r = control_flow_ops.While(c, b, [n])

            result = r.eval()
        self.assertTrue(check_op_order(n.graph))
        self.assertAllEqual(10, result)

 def bottleneck(self, x):
   hparams = self.hparams
   x = tf.tanh(tf.layers.dense(x, hparams.bottleneck_bits, name="bottleneck"))
   d = x + tf.stop_gradient(2.0 * tf.to_float(tf.less(0.0, x)) - 1.0 - x)
   if hparams.mode == tf.estimator.ModeKeys.TRAIN:
     noise = tf.random_uniform(common_layers.shape_list(x))
     noise = 2.0 * tf.to_float(tf.less(hparams.bottleneck_noise, noise)) - 1.0
     d *= noise
   x = common_layers.mix(d, x, hparams.discretize_warmup_steps,
                         hparams.mode == tf.estimator.ModeKeys.TRAIN)
   return x, 0.0

def augment_2d(inputs, rotation=0, horizontal_flip=False, vertical_flip=False):
    """Apply additive augmentation on 2D data.

    # Arguments
      rotation: A float, the degree range for rotation (0 <= rotation < 180),
          e.g. 3 for random image rotation between (-3.0, 3.0).
      horizontal_flip: A boolean, whether to allow random horizontal flip,
          e.g. true for 50% possibility to flip image horizontally.
      vertical_flip: A boolean, whether to allow random vertical flip,
          e.g. true for 50% possibility to flip image vertically.

    # Returns
      input data after augmentation, whose shape is the same as its original.
    """
    if inputs.dtype != tf.float32:
        inputs = tf.image.convert_image_dtype(inputs, dtype=tf.float32)

    with tf.name_scope('augmentation'):
        shp = tf.shape(inputs)
        batch_size, height, width = shp[0], shp[1], shp[2]
        width = tf.cast(width, tf.float32)
        height = tf.cast(height, tf.float32)

        transforms = []
        identity = tf.constant([1, 0, 0, 0, 1, 0, 0, 0], dtype=tf.float32)

        if rotation > 0:
            angle_rad = rotation * 3.141592653589793 / 180.0
            angles = tf.random_uniform([batch_size], -angle_rad, angle_rad)
            f = tf.contrib.image.angles_to_projective_transforms(angles,
                                                                 height, width)
            transforms.append(f)

        if horizontal_flip:
            coin = tf.less(tf.random_uniform([batch_size], 0, 1.0), 0.5)
            shape = [-1., 0., width, 0., 1., 0., 0., 0.]
            flip_transform = tf.convert_to_tensor(shape, dtype=tf.float32)
            flip = tf.tile(tf.expand_dims(flip_transform, 0), [batch_size, 1])
            noflip = tf.tile(tf.expand_dims(identity, 0), [batch_size, 1])
            transforms.append(tf.where(coin, flip, noflip))

        if vertical_flip:
            coin = tf.less(tf.random_uniform([batch_size], 0, 1.0), 0.5)
            shape = [1., 0., 0., 0., -1., height, 0., 0.]
            flip_transform = tf.convert_to_tensor(shape, dtype=tf.float32)
            flip = tf.tile(tf.expand_dims(flip_transform, 0), [batch_size, 1])
            noflip = tf.tile(tf.expand_dims(identity, 0), [batch_size, 1])
            transforms.append(tf.where(coin, flip, noflip))

    if transforms:
        f = tf.contrib.image.compose_transforms(*transforms)
        inputs = tf.contrib.image.transform(inputs, f, interpolation='BILINEAR')
    return inputs

  def testCondWhile_1(self):
    with self.test_session():
      n = tf.convert_to_tensor(0, name="n")
      c = lambda x: tf.less(x, 10)
      b = lambda x: tf.add(x, 1)
      r = tf.cond(tf.less(0, 1),
                  lambda: control_flow_ops.While(c, b, [n]),
                  lambda: n)

      result = r.eval()
    self.assertTrue(check_op_order(n.graph))
    self.assertAllEqual(10, result)

def tanh_discrete_bottleneck(x, bottleneck_bits, bottleneck_noise,
                             discretize_warmup_steps, mode):
  """Simple discretization through tanh, flip bottleneck_noise many bits."""
  x = tf.tanh(tf.layers.dense(x, bottleneck_bits,
                              name="tanh_discrete_bottleneck"))
  d = x + tf.stop_gradient(2.0 * tf.to_float(tf.less(0.0, x)) - 1.0 - x)
  if mode == tf.estimator.ModeKeys.TRAIN:
    noise = tf.random_uniform(common_layers.shape_list(x))
    noise = 2.0 * tf.to_float(tf.less(bottleneck_noise, noise)) - 1.0
    d *= noise
  d = common_layers.mix(d, x, discretize_warmup_steps,
                        mode == tf.estimator.ModeKeys.TRAIN)
  return d, 0.0

def process_features(image, label, do_augment = False):
    # Do any image preprocessing/augmentation here...
    with tf.name_scope('process_features'):

        # Crop driving view (Start from row 126, and slice 100 rows)
        image = tf.slice(image, [126, 0, 0], [100, 256, 3])

        # Resize image
        image = tf.image.resize_images(image, [66, 200])

        if do_augment == True:
            # Change or not change colors
            def do_color_changes():
                distorted_image = tf.image.random_brightness(image, max_delta=32. / 255.)
                distorted_image = tf.image.random_saturation(distorted_image, lower=0.5, upper=1.5)
                distorted_image = tf.image.random_hue(distorted_image, max_delta=0.2)
                distorted_image = tf.image.random_contrast(distorted_image, lower=0.5, upper=1.5)
                return distorted_image

            def no_color_change():
                distorted_image = image
                return distorted_image
            # Uniform variable in [0,1)
            flip_coin_color = tf.random_uniform(shape=[], minval=0., maxval=1., dtype=tf.float32)
            pred_color = tf.less(flip_coin_color, 0.5)
            # Randomically select doing color augmentation
            image = tf.cond(pred_color, do_color_changes, no_color_change, name='if_color')

            # Change or not change colors
            def flip_image_steering():
                distorted_image = tf.image.flip_left_right(image)
                distorted_label = -label
                return distorted_image, distorted_label

            def no_flip_image_steering():
                distorted_image = image
                distorted_label = label
                return distorted_image, distorted_label
            # Uniform variable in [0,1)
            flip_coin_flip = tf.random_uniform(shape=[], minval=0., maxval=1., dtype=tf.float32)
            pred_flip = tf.less(flip_coin_flip, 0.5)
            # Randomically select doing color augmentation
            image, label = tf.cond(pred_flip, flip_image_steering, no_flip_image_steering, name='if_steering')

        # Convert from [0, 255] -> [-0.5, 0.5] floats.
        image = tf.cast(image, tf.float32) * (1. / 255.0)
        #image = tf.image.per_image_standardization(image)

    return image, label

def unwrap(p, discont=np.pi, axis=-1):
  """Unwrap a cyclical phase tensor.

  Args:
    p: Phase tensor.
    discont: Float, size of the cyclic discontinuity.
    axis: Axis of which to unwrap.

  Returns:
    unwrapped: Unwrapped tensor of same size as input.
  """
  dd = diff(p, axis=axis)
  ddmod = tf.mod(dd + np.pi, 2.0 * np.pi) - np.pi
  idx = tf.logical_and(tf.equal(ddmod, -np.pi), tf.greater(dd, 0))
  ddmod = tf.where(idx, tf.ones_like(ddmod) * np.pi, ddmod)
  ph_correct = ddmod - dd
  idx = tf.less(tf.abs(dd), discont)
  ddmod = tf.where(idx, tf.zeros_like(ddmod), dd)
  ph_cumsum = tf.cumsum(ph_correct, axis=axis)

  shape = p.get_shape().as_list()
  shape[axis] = 1
  ph_cumsum = tf.concat([tf.zeros(shape, dtype=p.dtype), ph_cumsum], axis=axis)
  unwrapped = p + ph_cumsum
  return unwrapped

 def testCond_2(self):
   with self.test_session():
     x = tf.constant(10)
     r = tf.cond(tf.less(1, 0), lambda: tf.add(x, 1), lambda: tf.sub(x, 1))
     result = r.eval()
   self.assertTrue(check_op_order(x.graph))
   self.assertAllEqual(9, result)

  def testLoop_2(self):
    with self.test_session():
      zero = tf.constant(0)
      one = tf.constant(1)
      n = tf.constant(10)

      enter_i = control_flow_ops.enter(zero, "foo", False)
      enter_one = control_flow_ops.enter(one, "foo", True)
      enter_n = control_flow_ops.enter(n, "foo", True)

      merge_i = control_flow_ops.merge([enter_i, enter_i])[0]

      less_op = tf.less(merge_i, enter_n)
      cond_op = control_flow_ops.loop_cond(less_op)
      switch_i = control_flow_ops.switch(merge_i, cond_op)

      add_i = tf.add(switch_i[1], enter_one)

      with tf.device("/gpu:0"):
        next_i = control_flow_ops.next_iteration(add_i)
      merge_i.op._update_input(1, next_i)

      exit_i = control_flow_ops.exit(switch_i[0])
      result = exit_i.eval()
    self.assertAllEqual(10, result)

  def gather(self, indices, keys=None):
    """Returns elements at the specified indices from the buffer.

    Args:
      indices: (list of integers or rank 1 int Tensor) indices in the buffer to
        retrieve elements from.
      keys: List of keys of tensors to retrieve. If None retrieve all.
    Returns:
      A list of tensors, where each element in the list is obtained by indexing
        one of the tensors in the buffer.
    Raises:
      ValueError: If gather is called before calling the add function.
      tf.errors.InvalidArgumentError: If indices are bigger than the number of
        items in the buffer.
    """
    if not self._tensors:
      raise ValueError('The add function must be called before calling gather.')
    if keys is None:
      keys = self._tensors.keys()
    with tf.name_scope('Gather'):
      index_bound_assert = tf.Assert(
          tf.less(
              tf.to_int64(tf.reduce_max(indices)),
              tf.minimum(self.get_num_adds(), self._buffer_size)),
          ['Index out of bounds.'])
      with tf.control_dependencies([index_bound_assert]):
        indices = tf.convert_to_tensor(indices)

      batch = []
      for key in keys:
        batch.append(tf.gather(self._tensors[key], indices, name=key))
      return batch

    def rpn_proposals(self):
        with tf.variable_scope('rpn_proposals'):
            rpn_decode_boxes = encode_and_decode.decode_boxes(encode_boxes=self.rpn_encode_boxes,
                                                              reference_boxes=self.anchors,
                                                              scale_factors=self.scale_factors)

            rpn_softmax_scores = slim.softmax(self.rpn_scores)
            rpn_object_score = rpn_softmax_scores[:, 1]  # second column represent object

            if self.top_k_nms:
                rpn_object_score, top_k_indices = tf.nn.top_k(rpn_object_score, k=self.top_k_nms)
                rpn_decode_boxes = tf.gather(rpn_decode_boxes, top_k_indices)

            # NMS
            valid_indices = tf_wrapper.nms_rotate_tf(boxes_list=rpn_decode_boxes,
                                                     scores=rpn_object_score,
                                                     iou_threshold=self.rpn_nms_iou_threshold,
                                                     max_output_size=self.max_proposals_num,
                                                     use_gpu=cfgs.NMS_USE_GPU)

            valid_boxes = tf.gather(rpn_decode_boxes, valid_indices)
            valid_scores = tf.gather(rpn_object_score, valid_indices)
            # print_tensors(valid_scores, 'rpn_score')
            rpn_proposals_boxes, rpn_proposals_scores = tf.cond(
                tf.less(tf.shape(valid_boxes)[0], self.max_proposals_num),
                lambda: boxes_utils.padd_boxes_with_zeros(valid_boxes, valid_scores,
                                                          self.max_proposals_num),
                lambda: (valid_boxes, valid_scores))

            return rpn_proposals_boxes, rpn_proposals_scores

 def testControlFlow(self):
   with self.test_session() as sess:
     v0 = tf.get_variable("v0", [], initializer=tf.constant_initializer(0))
     var_dict = {}
     # Call get_variable in each of the cond clauses.
     def var_in_then_clause():
       v1 = tf.get_variable("v1", [1], initializer=tf.constant_initializer(1))
       var_dict["v1"] = v1
       return v1 + v0
     def var_in_else_clause():
       v2 = tf.get_variable("v2", [1], initializer=tf.constant_initializer(2))
       var_dict["v2"] = v2
       return v2 + v0
     add = control_flow_ops.cond(tf.less(v0, 10),
                                 var_in_then_clause,
                                 var_in_else_clause)
     v1 = var_dict["v1"]
     v2 = var_dict["v2"]
     # We should be able to initialize and run v1 and v2 without initializing
     # v0, even if the variable was created with a control dep on v0.
     sess.run(v1.initializer)
     self.assertEqual([1], sess.run(v1))
     sess.run(v2.initializer)
     self.assertEqual([2], sess.run(v2))
     # v0 should still be uninitialized.
     with self.assertRaisesRegexp(tf.OpError, "uninitialized"):
       sess.run(v0)
     # We should not be able to run 'add' yet.
     with self.assertRaisesRegexp(tf.OpError, "uninitialized"):
       sess.run(add)
     # If we initialize v0 we should be able to run 'add'.
     sess.run(v0.initializer)
     sess.run(add)