def run_evoked(subject_id):
    subject = "sub%03d" % subject_id
    print("processing subject: %s" % subject)
    data_path = op.join(meg_dir, subject)
    for run in range(1, 7):
        run_fname = op.join(data_path, 'run_%02d_filt_sss_raw.fif' % run)
        if not os.path.exists(run_fname):
            continue

        raw = mne.io.Raw(run_fname)

        events = mne.find_events(raw, stim_channel='STI101', consecutive='increasing',
                                 min_duration=0.003, verbose=True)
        for key in all_events_id:
            events = mne.merge_events(events, all_events_id[key], events_id[key])

        mask = (events[:, 2] == 1) | (events[:, 2] == 2) | (events[:, 2] == 3)
        events = events[mask]

        # df = pd.read_csv(data_path + '/Trials/run_%02d_trldef.txt' % run, sep='\t', header=None)
        # ev = np.c_[df[1], np.zeros_like(df[0]), le.transform(df[3])]
        # ev[:, 0] = np.round(ev[:, 0] / 3.)  # decimation by 3
        # ev[:, 0] += raw.first_samp
        # ev[:, 0] -= 452
        # ev[ev[:, 2] == 3, 2] = 4
        # ev[ev[:, 2] == 2, 2] = 3
        # ev[ev[:, 2] == 4, 2] = 2

        # print events - ev
        print "S %s - R %s" % (subject, run)
        # print (events - ev)[:, 2]
        # assert not np.any((events - ev)[:, 1:])
        # assert np.max(np.abs((events - ev)[:, 0])) == 1

        mne.write_events(op.join(data_path, 'run_%02d_filt_sss-eve.fif' % run), events)

def test_io_events():
    """Test IO for events
    """
    events = mne.read_events(fname)
    mne.write_events('events.fif', events)
    events2 = mne.read_events('events.fif')
    assert_array_almost_equal(events, events2)

def score(p, subjects):
    """Scoring function"""
    for subj in subjects:
        print("  Running subject %s... " % subj, end="")

        # Figure out what our filenames should be
        out_dir = op.join(p.work_dir, subj, p.list_dir)
        if not op.isdir(out_dir):
            os.mkdir(out_dir)

        for run_name in p.run_names:
            fname = op.join(p.work_dir, subj, p.raw_dir, (run_name % subj) + p.raw_fif_tag)
            events, presses = extract_expyfun_events(fname)[:2]
            for ii in range(len(events)):
                events[ii, 2] = _expyfun_dict[events[ii, 2]]
            fname_out = op.join(out_dir, "ALL_" + (run_name % subj) + "-eve.lst")
            mne.write_events(fname_out, events)

            # get subject performance
            devs = events[:, 2] % 2 == 1
            has_presses = np.array([len(pr) > 0 for pr in presses], bool)
            n_devs = np.sum(devs)
            hits = np.sum(has_presses[devs])
            fas = np.sum(has_presses[~devs])
            misses = n_devs - hits
            crs = (len(devs) - n_devs) - fas
            print("HMFC: %s, %s, %s, %s" % (hits, misses, fas, crs))

def test_io_cov():
    """Test IO for noise covariance matrices
    """
    events = mne.read_events(fname)
    mne.write_events('events.fif', events)
    events2 = mne.read_events(fname)
    assert_array_almost_equal(events, events2)

def test_io_set():
    """Test importing EEGLAB .set files"""
    from scipy import io

    _test_raw_reader(read_raw_eeglab, input_fname=raw_fname, montage=montage)
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')
        _test_raw_reader(read_raw_eeglab, input_fname=raw_fname_onefile,
                         montage=montage)
        raw = read_raw_eeglab(input_fname=raw_fname_onefile, montage=montage)
        raw2 = read_raw_eeglab(input_fname=raw_fname, montage=montage)
        assert_array_equal(raw[:][0], raw2[:][0])
    # one warning per each preload=False or str with raw_fname_onefile
    assert_equal(len(w), 3)

    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')
        epochs = read_epochs_eeglab(epochs_fname)
        epochs2 = read_epochs_eeglab(epochs_fname_onefile)
    # 3 warnings for each read_epochs_eeglab because there are 3 epochs
    # associated with multiple events
    assert_equal(len(w), 6)
    assert_array_equal(epochs.get_data(), epochs2.get_data())

    # test different combinations of events and event_ids
    temp_dir = _TempDir()
    out_fname = op.join(temp_dir, 'test-eve.fif')
    write_events(out_fname, epochs.events)
    event_id = {'S255/S8': 1, 'S8': 2, 'S255/S9': 3}

    epochs = read_epochs_eeglab(epochs_fname, epochs.events, event_id)
    epochs = read_epochs_eeglab(epochs_fname, out_fname, event_id)
    assert_raises(ValueError, read_epochs_eeglab, epochs_fname,
                  None, event_id)
    assert_raises(ValueError, read_epochs_eeglab, epochs_fname,
                  epochs.events, None)

    # test if .dat file raises an error
    eeg = io.loadmat(epochs_fname, struct_as_record=False,
                     squeeze_me=True)['EEG']
    eeg.data = 'epochs_fname.dat'
    bad_epochs_fname = op.join(temp_dir, 'test_epochs.set')
    io.savemat(bad_epochs_fname, {'EEG':
               {'trials': eeg.trials, 'srate': eeg.srate,
                'nbchan': eeg.nbchan, 'data': eeg.data,
                'epoch': eeg.epoch, 'event': eeg.event,
                'chanlocs': eeg.chanlocs}})
    shutil.copyfile(op.join(base_dir, 'test_epochs.fdt'),
                    op.join(temp_dir, 'test_epochs.dat'))
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')
        assert_raises(NotImplementedError, read_epochs_eeglab,
                      bad_epochs_fname)
    assert_equal(len(w), 3)

def save_event_file_ctl(subject, data_folder):
    """
    Parameters
    ----------
    subject : subject name
    data_folder : string

    Returns : None
        Save the events file to the data_folder.
    -------

    """
    test_trigger = {
        "P21": 251,
        "P22": 251,
        "P23": 251,
        "P24": 251,
        "P25": 251,
        "P26": 251,
        "P26": 251,
        "P27": 254,
        "P28": 252,
        "P29": 252,
        "P30": 254,
        "P31": 254,
        "P32": 254,
        "P33": 254,
        "P34": 254,
        "P35": 254,
        "P36": 254,
        "P37": 254,
        "P38": 254
    }

    # extract only the test trails, remember python is 0 index!
    idx = np.concatenate(
        [np.arange(30, 89, 2), np.arange(120, 179, 2), np.arange(210, 269, 2)])

    raw = mne.io.Raw(data_folder + "%s_bp_ica-raw.fif" % subject,
                     preload=False)
    events = mne.find_events(raw)

    # find the events that match the individual triggers
    test_events = events[:, 2] == test_trigger[subject]

    events = events[test_events]  # select the events
    events = events[idx]  # select the test trials

    # save events file
    mne.write_events(data_folder + "%s-eve.fif" % subject, events)

def extract_events_from_file(data_root, subject, verbose=True):
    bdf_filepath = os.path.join(data_root, 'eeg', 'biosemi', '{}.bdf'.format(subject))
    markers_filepath = os.path.join(data_root, 'eeg', 'biosemi', '{}_EEG_Data.mat'.format(subject))

    raw = read_raw_edf(bdf_filepath, preload=True, verbose=verbose)

    log.debug(raw)
    log.debug(raw.info)

    events = extract_events_from_raw(raw, markers_filepath, subject, verbose)

    # Writing events
    output_filepath = os.path.join(data_root, 'eeg', 'imported', subject, '{}-eve.fif.gz'.format(subject))
    ensure_parent_dir_exists(output_filepath)
    mne.write_events(output_filepath, events)

def test_io_events():
    """Test IO for events
    """
    events = read_events(fname)
    write_events("events.fif", events)
    events2 = read_events("events.fif")
    assert_array_almost_equal(events, events2)

    a = read_events("events.fif", include=1)
    b = read_events("events.fif", include=[1])
    c = read_events("events.fif", exclude=[2, 3, 4, 5, 32])
    d = read_events("events.fif", include=1, exclude=[2, 3])
    assert_array_equal(a, b)
    assert_array_equal(a, c)
    assert_array_equal(a, d)

def save_event_file(subject, data_folder):
    """

    Parameters
    ----------
    subject : subject name
    data_folder : string

    Returns
    -------

    """
    raw = mne.io.Raw(data_folder + "%s_bp_ica-raw.fif" % subject, preload=True)
    events = mne.find_events(raw)
    mne.write_events(data_folder + "%s-eve.fif" % subject, events)

def default_score(p, subjects, run_indices):
    """Default scoring function that just passes event numbers through"""
    for si, subj in enumerate(subjects):
        print('  Scoring subject %s... ' % subj)

        # Figure out what our filenames should be
        raw_fnames = get_raw_fnames(p, subj, 'raw', False, False,
                                    run_indices[si])
        eve_fnames = get_event_fnames(p, subj, run_indices[si])

        for raw_fname, eve_fname in zip(raw_fnames, eve_fnames):
            with warnings.catch_warnings(record=True):
                raw = mne.io.read_raw_fif(raw_fname, allow_maxshield=True)
            events = mne.find_events(raw, stim_channel='STI101',
                                     shortest_event=1)
            mne.write_events(eve_fname, events)

def score(p, subjects, run_indices):
    # run_indices = [r for ri, r in enumerate(run_indices) if ri in p.subject_indices]
    for subj, ri in zip(subjects, run_indices):
        if ri is None:
            runs = p.run_names
        else:
            runs = ['%s_%d' % (subj, r+1) for r in ri]
        print('Running subject %s...' % subj)
        # Figure out what our filenames should be
        out_dir = op.join(p.work_dir, subj, p.list_dir)
        if not op.isdir(out_dir):
            os.mkdir(out_dir)
        raw_names = get_raw_fnames(p, subj, 'raw', True, False, ri)
        # assert len(raw_names) == 8
        out_dir = op.join(p.work_dir, subj, p.list_dir)
        if not op.isdir(out_dir):
            os.mkdir(out_dir)
        for rii, fname in zip(runs, raw_names):
            print('  Run %s...' % op.basename(fname))
            
            # sjjoo_20160810: To incorporate session that has incomplete data
#            if not op.isfile(fname):
#                raise RuntimeError(' File %s not found' % fname)
            if op.isfile(fname):
            # read events, make sure we got the correct number of trials
                raw = mne.io.Raw(fname, allow_maxshield=True)
                ev = mne.find_events(raw, stim_channel='STI101', shortest_event=1)
                if op.basename(fname).__contains__('_1_raw'):
                    ev[:, 2] += 100
                elif op.basename(fname).__contains__('_2_raw'):
                    ev[:, 2] += 200
                elif op.basename(fname).__contains__('_3_raw'):
                    ev[:, 2] += 100
                elif op.basename(fname).__contains__('_4_raw'):
                    ev[:, 2] += 200
                elif op.basename(fname).__contains__('_5_raw'):
                    ev[:, 2] += 100
                elif op.basename(fname).__contains__('_6_raw'):
                    ev[:, 2] += 200
                elif op.basename(fname).__contains__('_7_raw'):
                    ev[:, 2] += 100
                elif op.basename(fname).__contains__('_8_raw'):
                    ev[:, 2] += 200
                fname_out = 'ALL_' + safe_inserter(rii,
                                                   subj) + '-eve.lst'
                fname_out = op.join(out_dir, fname_out)
                mne.write_events(fname_out, ev)

def test_io_set_epochs_events(tmpdir):
    """Test different combinations of events and event_ids."""
    tmpdir = str(tmpdir)
    out_fname = op.join(tmpdir, 'test-eve.fif')
    events = np.array([[4, 0, 1], [12, 0, 2], [20, 0, 3], [26, 0,  3]])
    write_events(out_fname, events)
    event_id = {'S255/S8': 1, 'S8': 2, 'S255/S9': 3}
    out_fname = op.join(tmpdir, 'test-eve.fif')
    epochs = read_epochs_eeglab(epochs_fname_mat, events, event_id)
    assert_equal(len(epochs.events), 4)
    assert epochs.preload
    assert epochs._bad_dropped
    epochs = read_epochs_eeglab(epochs_fname_mat, out_fname, event_id)
    pytest.raises(ValueError, read_epochs_eeglab, epochs_fname_mat,
                  None, event_id)
    pytest.raises(ValueError, read_epochs_eeglab, epochs_fname_mat,
                  epochs.events, None)

def score(p, subjects, run_indices):
    # run_indices = [r for ri, r in enumerate(run_indices) if ri in p.subject_indices]
    for subj, ri in zip(subjects, run_indices):
        if ri is None:
            runs = p.run_names
        else:
            runs = ["%s_%d" % (subj, r + 1) for r in ri]
        print("Running subject %s..." % subj)
        # Figure out what our filenames should be
        out_dir = op.join(p.work_dir, subj, p.list_dir)
        if not op.isdir(out_dir):
            os.mkdir(out_dir)
        raw_names = get_raw_fnames(p, subj, "raw", True, False, ri)
        # assert len(raw_names) == 8
        out_dir = op.join(p.work_dir, subj, p.list_dir)
        if not op.isdir(out_dir):
            os.mkdir(out_dir)
        for rii, fname in zip(runs, raw_names):
            print("  Run %s..." % op.basename(fname))
            if not op.isfile(fname):
                raise RuntimeError(" File %s not found" % fname)
            # read events, make sure we got the correct number of trials
            raw = mne.io.Raw(fname, allow_maxshield=True)
            ev = mne.find_events(raw, stim_channel="STI101", shortest_event=1)
            if op.basename(fname).__contains__("_1_raw"):
                ev[:, 2] += 100
            elif op.basename(fname).__contains__("_2_raw"):
                ev[:, 2] += 200
            elif op.basename(fname).__contains__("_3_raw"):
                ev[:, 2] += 100
            elif op.basename(fname).__contains__("_4_raw"):
                ev[:, 2] += 200
            elif op.basename(fname).__contains__("_5_raw"):
                ev[:, 2] += 100
            elif op.basename(fname).__contains__("_6_raw"):
                ev[:, 2] += 200
            elif op.basename(fname).__contains__("_7_raw"):
                ev[:, 2] += 100
            elif op.basename(fname).__contains__("_8_raw"):
                ev[:, 2] += 200
            fname_out = "ALL_" + safe_inserter(rii, subj) + "-eve.lst"
            fname_out = op.join(out_dir, fname_out)
            mne.write_events(fname_out, ev)

def run_events(subject_id):
    subject = "sub%03d" % subject_id
    print("processing subject: %s" % subject)
    data_path = op.join(meg_dir, subject)
    for run in range(1, 7):
        run_fname = op.join(data_path, 'run_%02d_filt_sss_raw.fif' % run)
        if not os.path.exists(run_fname):
            continue

        raw = mne.io.Raw(run_fname, add_eeg_ref=False)
        mask = 4096 + 256  # mask for excluding high order bits
        events = mne.find_events(raw, stim_channel='STI101',
                                 consecutive='increasing', mask=mask,
                                 mask_type='not_and', min_duration=0.003,
                                 verbose=True)

        print("S %s - R %s" % (subject, run))

        fname_events = op.join(data_path, 'run_%02d_filt_sss-eve.fif' % run)
        mne.write_events(fname_events, events)

def default_score(p, subjects):
    """Default scoring function that just passes event numbers through"""
    for subj in subjects:
        print('  Scoring subject %s... ' % subj)

        # Figure out what our filenames should be
        out_dir = op.join(p.work_dir, subj, p.list_dir)
        if not op.isdir(out_dir):
            os.mkdir(out_dir)

        for run_name in p.run_names:
            fname = op.join(p.work_dir, subj, p.raw_dir,
                            (run_name % subj) + p.raw_fif_tag)
            with warnings.catch_warnings(record=True):
                raw = mne.io.read_raw_fif(fname, allow_maxshield=True)
            events = mne.find_events(raw, stim_channel='STI101',
                                     shortest_event=1)
            fname_out = op.join(out_dir,
                                'ALL_' + (run_name % subj) + '-eve.lst')
            mne.write_events(fname_out, events)

def score(p, subjects):
    """Scoring function"""
    for subj in subjects:
        print('  Running subject %s... ' % subj, end='')

        # Figure out what our filenames should be
        out_dir = op.join(p.work_dir, subj, p.list_dir)
        if not op.isdir(out_dir):
            os.mkdir(out_dir)

        for run_name in p.run_names:
            print(subj)
            fname = op.join(p.work_dir, subj, p.raw_dir,
                            (run_name % subj) + p.raw_fif_tag)
            # events = mne.find_events(mne.io.Raw(fname, allow_maxshield='True'), stim_channel='STI101')
            #events = events[events[:, 2] == 1]
            events, _ = extract_expyfun_events(fname)[:2]
            events[:, 2] += 10
            fname_out = op.join(out_dir,
                                'ALL_' + (run_name % subj) + '-eve.lst')
            mne.write_events(fname_out, events)

def make_events_files(sub_id, session):
    """ This function read in a fiff file and write out a event file
    """

    # SETUP AND LOAD FILES ####
    # name with subject id & session
    fname = "sub_%d_%s" % (sub_id, session)

    # load the raw fif
    print '\nLoading raw file'
    raw = fiff.Raw(fname + "_tsss_mc_autobad.fif", preload=False)

    # EPOCHS ####
    events = mne.find_events(raw, stim_channel="STI101")
    events_classic = []
    events_interupt = []
    for i in range(len(events)):
        if i > 0:
            if events[i, 2] == 1 and events[i - 1, 2] == 1:
                events_classic.append(i)
            elif events[i, 2] == 1 and events[i - 1, 2] == 2:
                events_interupt.append(i)
   
    if len(events_classic) > 0:
        outname = "sub_%d_%s.eve" % (sub_id, session)
        mne.write_events(outname, events[events_classic])

    if len(events_interupt) is not 0:
        outname_classic = "sub_%d_%s_%s.eve" % (sub_id, session, "classic")
        outname_interrupt = "sub_%d_%s_%s.eve" % (sub_id, session, "interrupt")

        mne.write_events(outname_classic, events[events_classic])
        mne.write_events(outname_interrupt, events[events_interupt])

def compute_proj_ecg(in_path, in_fif_fname, h_tmin, h_tmax, n_grad, n_mag, n_eeg, l_freq, h_freq, filter_length, n_jobs, ch_name, reject, avg_ref):

    # Reading fif File
    raw = io.Raw(in_fif_fname)
    prefix = in_fif_fname[:-8]
#    prefix = 'sc3_BaleenLPRun4'
    print prefix
    in_fif_fname = in_path + in_fif_fname
    print in_fif_fname
    out_path = os.path.join(in_path + 'ssp/mne/')

    out_fif_fname = in_path + 'ssp/mne/' + prefix + '_clean_ecg_raw.fif'
    ecg_proj_fname = in_path  + prefix + '_ecg_proj.fif'
    ecg_event_fname = in_path + 'ssp/mne/' + prefix + '_ecg-eve.fif'        
    flag = 0
    event_id = 999
    print 'Implementing ECG artifact rejection on data'
    ecg_events, _, _ = mne.preprocessing.find_ecg_events(raw,event_id, ch_name=None)
    print ecg_events
    if not len(ecg_events) < 30:
           # print ecg_event_fname
           print "Writing ECG events in %s" % ecg_event_fname
           mne.write_events(ecg_event_fname, ecg_events)
        
    # Making projector
    print 'Computing ECG projector'
    print out_path
    #os.chdir('/cluster/kuperberg/SemPrMM/MEG/data/ac1/ssp')
    #os.getcwd()
    command = ('mne_process_raw --cd %s --raw %s --events %s --makeproj '
                       '--projtmin %s --projtmax %s --saveprojtag _ecg_proj '
                       '--projnmag %s --projngrad %s --projevent 999 --highpass 5 '
                       '--lowpass 35 --projmagrej 8000 --projgradrej 5000'
                       % (in_path, in_fif_fname, ecg_event_fname, h_tmin, h_tmax,  n_mag, n_grad)) ##6/1/13 CU:projectors fixed 1 1 0 
    print command
    st = os.system(command)
    if st != 0:
            print "Error while running : %s" % command
                   
    return in_fif_fname, ecg_proj_fname, len(ecg_event_fname), out_fif_fname

def compute_proj_eog(in_path, in_fif_fname, e_tmin, e_tmax, n_grad, n_mag, n_eeg, l_freq, h_freq, filter_length, n_jobs, ch_name, reject, avg_ref):


    # Reading fif File
    raw = io.Raw(in_fif_fname)
    prefix = in_fif_fname[:-8]
#    prefix = 'sc3_BaleenLPRun4'
    print prefix
    in_fif_fname = in_path + in_fif_fname
    print in_fif_fname
    out_path = os.path.join(in_path + 'ssp/mne/')

    out_fif_fname = in_path + 'ssp/mne/' + prefix + '_clean_eog_raw.fif'
    eog_proj_fname = in_path + prefix + '_eog_proj.fif' 
    eog_event_fname = in_path + 'ssp/mne/' + prefix + '_eog-eve.fif'
    flag=0
    event_id = 998
    print 'Finding EOG events from the data...'
    eog_events= mne.preprocessing.find_eog_events(raw,event_id, ch_name = 'EOG061')

    if not len(eog_events)<20:
        print eog_event_fname
        print "Writing EOG events in %s" % eog_event_fname
        mne.write_events(eog_event_fname, eog_events)
        print eog_proj_fname
        
    print "Computing the EOG projector"
    command = ('mne_process_raw --cd %s --raw %s --events %s --makeproj '
                               '--projtmin %s --projtmax %s --saveprojtag _eog_proj '
                               '--projnmag %s --projngrad %s --projevent 998 --highpass 0.3 '
                               '--lowpass 35 --filtersize 8192 --projmagrej 8000 --projgradrej 7000'  ###filtersize 8192, projeegrej was 500 by default!!!
                               % (in_path, in_fif_fname, eog_event_fname, e_tmin, e_tmax, n_mag, n_grad))  #e_tmin, e_tmax, #1/15/13 CU: refer the speadsheet(projectors custom made for each subject)  --projmagrej 5500 --projgradrej 3000 --projeegrej 900
    print command                               
    st = os.system(command)
    if st != 0:
            print "Error while running : %s" % command
                    
    return in_fif_fname, eog_proj_fname, len(eog_event_fname), out_fif_fname

def compute_proj_ecg(in_path, in_fif_fname, tmin, tmax, n_grad, n_mag, n_eeg, l_freq, h_freq, filter_length, n_jobs, ch_name, reject, avg_ref):

    # Reading fif File
    raw_in = fiff.Raw(in_fif_fname)
    prefix = in_fif_fname[:-8]
    print prefix
    in_fif_fname = in_path + in_fif_fname
    print in_fif_fname
    out_path = os.path.join(in_path + 'ssp/')

    out_fif_fname = in_path + 'ssp/' + prefix + '_clean_ecg_raw.fif'
    ecg_proj_fname = in_path  + prefix + '_ecg_proj.fif'
    ecg_event_fname = in_path + 'ssp/' + prefix + '_ecg-eve.fif'        
    flag = 0
    
    print 'Implementing ECG artifact rejection on data'
    ecg_events, _, _ = mne.artifacts.find_ecg_events(raw_in)
    if not len(ecg_events) < 30:
		print ecg_event_fname
		print "Writing ECG events in %s" % ecg_event_fname
		mne.write_events(ecg_event_fname, ecg_events)
	
		# Making projector
		print 'Computing ECG projector'
                print out_path
	        #os.chdir('/cluster/kuperberg/SemPrMM/MEG/data/ac1/ssp')
                #os.getcwd()
		command = ('mne_process_raw --cd %s --raw %s --events %s --makeproj '
				   '--projtmin %s --projtmax %s --saveprojtag _ecg_proj '
				   '--projnmag %s --projngrad %s --projneeg 0 --projevent 999 --highpass 5 '
				   '--lowpass 35 --projmagrej 4000 --projgradrej 3000 --projeegrej 250 '
		         	   % (in_path, in_fif_fname, ecg_event_fname, tmin, tmax, n_mag, n_grad)) ##10/1/12 CU after changing the number of projectors for ECG(mag1, grad1, eeg0) 
		
		st = os.system(command)
		if st != 0:
			print "Error while running : %s" % command
##                    
    return in_fif_fname, ecg_proj_fname, len(ecg_event_fname), out_fif_fname