def test_export_landmark_ljson(mock_open, exists, json_dump):
    exists.return_value = False
    fake_path = '/fake/fake.ljson'
    with open(fake_path) as f:
        type(f).name = PropertyMock(return_value=fake_path)
        mio.export_landmark_file(test_lg, f, extension='ljson')
    json_dump.assert_called_once()

def test_export_landmark_pts(mock_open, exists, save_txt):
    exists.return_value = False
    fake_path = '/fake/fake.pts'
    with open(fake_path) as f:
        type(f).name = PropertyMock(return_value=fake_path)
        mio.export_landmark_file(test_lg, f, extension='pts')
    save_txt.assert_called_once()

def test_export_filepath_no_overwrite(mock_open, exists, landmark_types):
    exists.return_value = False
    mio.export_landmark_file(test_lg, fake_path)
    mock_open.assert_called_once_with("wb")
    landmark_types.__getitem__.assert_called_once_with(".fake")
    export_function = landmark_types.__getitem__.return_value
    export_function.assert_called_once()

def test_export_filepath_no_overwrite(mock_open, exists, landmark_types):
    exists.return_value = False
    mio.export_landmark_file(test_lg, fake_path)
    mock_open.assert_called_with('wb')
    landmark_types.__getitem__.assert_called_with('.fake')
    export_function = landmark_types.__getitem__.return_value
    assert export_function.call_count == 1

def test_export_filepath_overwrite_exists(mock_open, exists, landmark_types):
    exists.return_value = True
    mio.export_landmark_file(test_lg, fake_path, overwrite=True)
    mock_open.assert_called_once_with('wb')
    landmark_types.__getitem__.assert_called_once_with('.fake')
    export_function = landmark_types.__getitem__.return_value
    export_function.assert_called_once()

def test_export_filepath_explicit_ext_dot(mock_open, exists, landmark_types):
    exists.return_value = False
    mio.export_landmark_file(test_lg, fake_path, extension='.fake')
    mock_open.assert_called_once_with('wb')
    landmark_types.__getitem__.assert_called_once_with('.fake')
    export_function = landmark_types.__getitem__.return_value
    export_function.assert_called_once()

def test_export_file_handle_file_non_file_buffer(mock_open, exists, landmark_types):
    exists.return_value = False
    with open(fake_path) as f:
        del f.name  # Equivalent to raising an AttributeError side effect
        mio.export_landmark_file(test_lg, f, extension="fake")
    landmark_types.__getitem__.assert_called_once_with(".fake")
    export_function = landmark_types.__getitem__.return_value
    export_function.assert_called_once()

def test_export_file_handle_file_exists_overwrite(mock_open, exists, landmark_types):
    exists.return_value = True
    with open(fake_path) as f:
        type(f).name = PropertyMock(return_value=fake_path)
        mio.export_landmark_file(test_lg, f, overwrite=True, extension="fake")
    landmark_types.__getitem__.assert_called_once_with(".fake")
    export_function = landmark_types.__getitem__.return_value
    export_function.assert_called_once()

def test_export_filepath_explicit_ext_no_dot(mock_open, exists, landmark_types):
    exists.return_value = False
    landmark_types.__contains__.return_value = True
    mio.export_landmark_file(test_lg, fake_path, extension='fake')
    mock_open.assert_called_with('wb')
    landmark_types.__getitem__.assert_called_with('.fake')
    export_function = landmark_types.__getitem__.return_value
    assert export_function.call_count == 1

def test_export_file_handle_file_non_file_buffer(mock_open, exists,
                                                 landmark_types):
    exists.return_value = False
    landmark_types.__contains__.return_value = True
    with open(fake_path) as f:
        del f.name  # Equivalent to raising an AttributeError side effect
        mio.export_landmark_file(test_lg, f, extension='fake')
    landmark_types.__getitem__.assert_called_with('.fake')
    export_function = landmark_types.__getitem__.return_value
    assert export_function.call_count == 1

def test_export_file_handle_file_exists_overwrite(mock_open, exists,
                                                  landmark_types):
    exists.return_value = True
    landmark_types.__contains__.return_value = True
    with open(fake_path) as f:
        type(f).name = PropertyMock(return_value=fake_path)
        mio.export_landmark_file(test_lg, f, overwrite=True, extension='fake')
    landmark_types.__getitem__.assert_called_with('.fake')
    export_function = landmark_types.__getitem__.return_value
    assert export_function.call_count == 1

def test_export_landmark_ljson_nan_values(mock_open, exists):
    exists.return_value = False
    mock_writer = MagicMock()
    mock_open.return_value.__enter__.return_value = mock_writer
    fake_path = '/fake/fake.ljson'
    with open(fake_path) as f:
        type(f).name = PropertyMock(return_value=fake_path)
        mio.export_landmark_file(nan_lg, f, extension='ljson')

    # yeah this is grim, but it should work.
    assert 'null' in '{}'.format(mock_open.mock_calls)

def test_export_landmark_ljson_nan_values(mock_open, exists):
    exists.return_value = False
    fake_path = '/fake/fake.ljson'
    with open(fake_path) as f:
        type(f).name = PropertyMock(return_value=fake_path)
        mio.export_landmark_file(nan_lg, f, extension='ljson')

    # This is a bit ugly, but we parse the write calls to check that json
    # wrote null values
    first_null = mock_open.mock_calls[97][1][0][1:].strip()
    second_null = mock_open.mock_calls[98][1][0][1:].strip()
    assert first_null == 'null'
    assert second_null == 'null'

def process_frame(frame_name, clip, img_type, svm_p, loop=False):
    """
    Applies the AAM fitter (global var) in a frame. Additionally, it might apply an
    SVM to verify it's a face if required.
    :param frame_name: str: Name of the frame along with extension, e.g. '000001.png'.
    :param clip:       str: Name of the clip.
    :param img_type:   str: Suffix (extension) of the frames, e.g. '.png'.
    :param svm_p:      dict: Required params for SVM classification.
    :param loop:       bool: (optional) Declares whether this is a 2nd fit for AAM (loop).
    :return:
    """
    global fitter
    name = frame_name[:frame_name.rfind('.')]
    p0 = clip.path_read_ln[0] + name + '_0.pts'
    # find if this is 2nd fit or 1st.
    if loop:  # if 2nd fit, then if landmark is 'approved', return. Otherwise proceed.
        try:
            ln = import_landmark_file(p0)
            copy2(p0, clip.path_write_ln[0] + name + '_0.pts')
            return      # if the landmark already exists, return (for performance improvement)
        except ValueError:
            pass
        try:
            ln = import_landmark_file(clip.path_read_ln[1] + name + '_0.pts')
        except ValueError:  # either not found or no suitable importer
            return
    else:
        try:
            ln = import_landmark_file(p0)
        except ValueError:  # either not found or no suitable importer
            return
    im = im_read_greyscale(frame_name, clip.path_frames, img_type)
    if not im:
        return
    im.landmarks['PTS2'] = ln
    fr = fitter.fit_from_shape(im, im.landmarks['PTS2'].lms, crop_image=0.3)
    p_wr = clip.path_write_ln[0] + im.path.stem + '_0.pts'
    export_landmark_file(fr.fitted_image.landmarks['final'], p_wr, overwrite=True)

    # apply SVM classifier by extracting patches (is face or not).
    if not svm_p['apply']:
        return
    im.landmarks.clear()  # temp solution
    im.landmarks['ps_pbaam'] = fr.fitted_image.landmarks['final']
    im_cp = im.crop_to_landmarks_proportion(0.2, group='ps_pbaam')
    im_cp = svm_p['feat'](im_cp)
    im2 = warp_image_to_reference_shape(im_cp, svm_p['refFrame'], 'ps_pbaam')
    _p_nd = im2.extract_patches_around_landmarks(group='source', as_single_array=True,
                                                 patch_shape=svm_p['patch_s']).flatten()
    if svm_p['clf'].decision_function(_p_nd) > 0:
        copy2(p_wr, clip.path_write_ln[1] + im.path.stem + '_0.pts')

def detect_in_frame(frame_name, clip, img_type):
    # if normalise=True in im_read_greyscale: before calling dlib detector, image should be converted to uint8
    im = im_read_greyscale(frame_name, clip.path_frames, img_type, normalise=False)
    if not im:
        print(frame_name, clip.path_frames)
        return
    res_dlib = dlib_init_detector(im, group_prefix='dlib')  # call dlib detector
    im_pili = np.array(im.as_PILImage())
    for kk, g in enumerate(im.landmarks.group_labels):
        pts_end = im.path.stem + '_' + str(kk) + pts_type_out  # define the ending of each pts that will be exported
        export_landmark_file(im.landmarks[g], clip.path_write_ln[0] + pts_end, overwrite=True)
        # from bounding box to points (dlib predictor)
        init_pc = detection_to_pointgraph(predictor_dlib(im_pili, pointgraph_to_rect(im.landmarks[g].lms)))
        export_landmark_file(LandmarkGroup.init_with_all_label(init_pc), clip.path_write_ln[1] + pts_end, overwrite=True)

def test_export_landmark_ljson_3d(mock_open, exists, json_dump):
    exists.return_value = False
    fake_path = "/fake/fake3d.ljson"
    test3d_lg = test_lg.copy()
    fake_z_points = np.random.random(test3d_lg.lms.points.shape[0])
    test3d_lg.lms.points = np.concatenate([test3d_lg.lms.points, fake_z_points[..., None]], axis=-1)

    with open(fake_path) as f:
        type(f).name = PropertyMock(return_value=fake_path)
        mio.export_landmark_file(test3d_lg, f, extension="ljson")

    json_dump.assert_called_once()
    json_points = np.array(json_dump.call_args[0][0]["landmarks"]["points"])
    assert_allclose(json_points[:, -1], fake_z_points)

def test_export_landmark_ljson_3d(mock_open, exists, json_dump):
    exists.return_value = False
    fake_path = '/fake/fake3d.ljson'
    test3d_lg = test_lg.copy()
    fake_z_points = np.random.random(test3d_lg.points.shape[0])
    test3d_lg.points = np.concatenate([
        test3d_lg.points, fake_z_points[..., None]], axis=-1)

    with open(fake_path) as f:
        type(f).name = PropertyMock(return_value=fake_path)
        mio.export_landmark_file(test3d_lg, f, extension='ljson')

    assert json_dump.call_count == 1
    json_points = np.array(json_dump.call_args[0][0]['landmarks']['points'])
    assert_allclose(json_points[:, -1], fake_z_points)

def save_bounding_boxes(pattern, detector_type, group=None,
                        sythesize_problematic=False, overwrite=False):
    import menpo.io as mio
    from menpo.landmark import LandmarkGroup
    from menpo.model import PCAModel
    try:
        detector = _DETECTORS[detector_type]()
    except KeyError:
        detector_list = ', '.join(list(_DETECTORS.keys()))
        raise ValueError('Valid detector types are: {}'.format(detector_list))
    print('Running {} detector on {}'.format(detector_type, pattern))
    bboxes = {img.path: detect_and_check(img, detector, group=group)
              for img in mio.import_images(pattern, normalise=False,
                                           verbose=True)}

    # find all the detections that failed
    problematic = filter(lambda x: x[1]['d'] is None, bboxes.items())
    print('Failed to detect {} objects'.format(len(problematic)))
    if len(problematic) > 0 and sythesize_problematic:
        print('Learning detector traits and sythesizing fits for {} '
              'images'.format(len(problematic)))
        # get the good detections
        detections = filter(lambda x: x['d'] is not None, bboxes.values())
        # normalize these to size [1, 1], centred on origin
        normed_detections = [normalize(r['gt']).apply(r['d'])
                             for r in detections]
        # build a PCA model from good detections
        pca = PCAModel(normed_detections)

        for p, r in problematic:
            # generate a new bbox offset in the normalized space by using
            # our learnt PCA basis
            d = random_instance(pca)
            # apply an inverse transform to place it on the image
            bboxes[p]['d'] = normalize(r['gt']).pseudoinverse().apply(d)
    to_save = len(bboxes)
    if not sythesize_problematic:
        to_save = to_save - len(problematic)
    print('Saving out {} {} detections'.format(to_save, detector_type))
    # All done, save out results
    for p, r in bboxes.items():
        if r['d'] is not None:
            lg = LandmarkGroup.init_with_all_label(r['d'])
            mio.export_landmark_file(lg, p.parent /
                                     (p.stem + '_{}.ljson'.format(detector_type)),
                                     overwrite=overwrite)

def process_lns_path(process, shapes=None, p_in=None, p_out=None, overwrite=None):
    """
    Processes a list of landmark files. The processing is performed per shape (file)
    and depends on the process function defined.
    Can be provided either the shapes directly or an import path.
    If an exporting path is provided, the bounding boxes will be
    exported there.

    :param process: (function) Process function that accepts a landmark (menpo.landmark)
                    and returns the same type processed.
    :param shapes:  (list, optional) List of shapes.
    :param p_in:    (string, optional) Input path for shapes if shapes is not provided.
    :param p_out:   (string, optional) Output path for the processed landmarks.
    :param overwrite: (bool, optional) Whether to overwrite existing files in p_out.
    :return:
    """
    if p_out is not None:
        assert(isdir(p_out))

    if shapes is None:
        # import the shapes from p_in.
        assert(isdir(p_in))
        shapes = list(mio.import_landmark_files(p_in))

    ln_out = []
    # dummy image
    im = mio.import_builtin_asset.lenna_png()
    # loop over the shapes to convert to bounding boxes.
    for ln in shapes:
        # process each shape by utilising the process function.
        im.landmarks['g'] = process(ln)

        if p_out is not None:
            # if path is provided, export it.
            mio.export_landmark_file(im.landmarks['g'], p_out + ln.path.name, 
                                     overwrite=overwrite)
        ln_out.append(im.landmarks['g'])
        
    return ln_out

def predict_in_frame(frame_name, clip, img_type):
    global detector
    im = im_read_greyscale(frame_name, clip.path_frames, img_type, normalise=False)

    res_dlib = detector(im)
    num_res = len(res_dlib)
    if num_res == 0:
        return
    num1 = 1                # num1 and s1: Values if there are more than 10 detections in the image
    if num_res > 9:
        num1 = 2
    s1 = '%0' + str(num1)
    im_pili = np.array(im.as_PILImage())
    for kk in range(0, 1):   # num_res to keep all, here keeping ONLY the most confident one
        pts_end = im.path.stem + '_' + str(kk) + pts_type_out
        ln = im.landmarks['ffld2_' + (s1 + 'd') % kk]
        mio.export_landmark_file(ln, clip.path_write_ln[0] + pts_end, overwrite=True)
        # convert to landmarks
        det_frame = predictor_dlib(im_pili, pointgraph_to_rect(ln.lms))
        init_pc = detection_to_pointgraph(det_frame)
        mio.export_landmark_file(LandmarkGroup.init_with_all_label(init_pc),
                                 clip.path_write_ln[1] + pts_end, overwrite=True)