def moveFilesIntoFolders(in_dir,mat_file,out_dir,out_file_commands,pad_zeros_in=8,pad_zeros_out=4):
	arr=scipy.io.loadmat(mat_file)['ranges'];
	# videos=np.unique(arr);
	commands=[];
	for shot_no in range(arr.shape[1]):
		print shot_no,arr.shape[1];
		start_idx=arr[0,shot_no];
		end_idx=arr[1,shot_no];
		video_idx=arr[2,shot_no];
		out_dir_video=os.path.join(out_dir,str(video_idx));
		util.mkdir(out_dir_video);
		# print 
		# raw_input();
		shot_idx=np.where(shot_no==np.where(video_idx==arr[2,:])[0])[0][0]+1;
		out_dir_shot=os.path.join(out_dir_video,str(shot_idx));
		util.mkdir(out_dir_shot);

		# print start_idx,end_idx
		for idx,frame_no in enumerate(range(start_idx,end_idx+1)):
			in_file=os.path.join(in_dir,padZeros(frame_no,pad_zeros_in)+'.jpg');
			out_file=os.path.join(out_dir_shot,'frame'+padZeros(idx+1,pad_zeros_out)+'.jpg');
			command='mv '+in_file+' '+out_file;
			commands.append(command);
	print len(commands);
	util.writeFile(out_file_commands,commands);

def writeTrainTxt(out_file_train,video_dirs,im_dir,tif_dir,subsample=5):
    print len(video_dirs);

    # video_dirs=video_dirs[:10];
    pairs=[];
    for idx_vid_dir,vid_dir in enumerate(video_dirs):
        print idx_vid_dir,vid_dir
        tif_dir_curr=os.path.join(vid_dir,tif_dir);
        im_dir_curr=os.path.join(vid_dir,im_dir);
        tif_names=[file_curr for file_curr in os.listdir(tif_dir_curr) if file_curr.endswith('.tif')];
        tif_names=sortTifNames(tif_names);

        for tif_name in tif_names[::subsample]:
            # print tif_name
            jpg_file=os.path.join(im_dir_curr,tif_name.replace('.tif','.jpg'));

            # print jpg_file,os.path.exists(jpg_file)
            if os.path.exists(jpg_file):
                # print jpg_file
                tif_file=os.path.join(tif_dir_curr,tif_name);
                pairs.append(jpg_file+' '+tif_file);

        # raw_input();

    print len(pairs);
    # print pairs[:10];
    # pair_one=[p[:p.index(' ')] for p in pairs];
    # vid_dirs=[p[:p[:p.rindex('/')].rindex('/')] for p in pair_one];
    # print len(set(vid_dirs));
    random.shuffle(pairs);
    util.writeFile(out_file_train,pairs);

def shortenTrainingData(train_txt,train_txt_new,ratio_txt,val_txt_new=None):
    # pos_human='/disk3/maheen_data/headC_160/noFlow_gaussian_human/pos_flos/positives_onlyHuman_withFlow.txt';
    # neg_human='/disk3/maheen_data/headC_160/neg_flos/negatives_onlyHuman_withFlow.txt';

    # pos_human_small='/disk3/maheen_data/headC_160/noFlow_gaussian_human/pos_flos/positives_onlyHuman_withFlow_oneHundreth.txt';
    # neg_human_small='/disk3/maheen_data/headC_160/neg_flos/negatives_onlyHuman_withFlow_oneHundreth.txt';

    # ratio_txt=100;
    # shortenTrainingData(pos_human,pos_human_small,ratio_txt);
    # shortenTrainingData(neg_human,neg_human_small,ratio_txt);

    train_data=util.readLinesFromFile(train_txt);
    # print ratio_txt
    if ratio_txt<1:
        ratio_txt=int(len(train_data)*ratio_txt);
        # print ratio_txt;

    random.shuffle(train_data);
    train_data_new=train_data[:ratio_txt];
    print len(train_data),len(train_data_new);
    util.writeFile(train_txt_new,train_data_new);

    if val_txt_new is not None:
        val_data=train_data[ratio_txt:];
        print len(val_data);
        util.writeFile(val_txt_new,val_data);

def main():
	text_list='/disk2/aprilExperiments/dual_flow/list_of_dats_to_move.txt';
	text_mv='/disk2/aprilExperiments/dual_flow/list_of_dats_to_move_commands.sh';
	models=util.readLinesFromFile(text_list);
	path_to_storage='/media/maheenrashid/Seagate\ Backup\ Plus\ Drive/maheen_data';
	path_to_replace='/disk2';
	
	mv_commands=[];

	for model in models:
		if not os.path.exists(model):
			continue;
		dir_curr=model[:model.rindex('/')];
		dir_new=dir_curr.replace(path_to_replace,path_to_storage);

		# print dir_new;
		
		command='mkdir -p '+dir_new;
		# print command;
		mv_command='mv -v '+model+' '+dir_new+'/';
		# print mv_command
		mv_commands.append(mv_command);
		subprocess.call(command,shell=True);
		# raw_input();

	util.writeFile(text_mv,mv_commands);
	print text_mv

def script_writeCommandsForPreprocessing(all_dirs_file,command_file_pre,num_proc,check_file=None):
    all_dirs=util.readLinesFromFile(all_dirs_file);
    all_dirs=[dir_curr[:-1] for dir_curr in all_dirs];
    
    if check_file is not None:
        all_dirs=getRemainingDirs(all_dirs,check_file);

    command_pre='echo '
    command_middle_1=';cd ~/Downloads/opticalflow; matlab -nojvm -nodisplay -nosplash -r "out_folder=\''
    command_middle='\';saveTrainingData" > '
    command_end=' 2>&1';

    commands=[];
    for dir_curr in all_dirs:
        dir_curr=util.escapeString(dir_curr);
        log_file=os.path.join(dir_curr,'log.txt');
        command=command_pre+dir_curr+command_middle_1+dir_curr+command_middle+log_file+command_end;
        commands.append(command);
    
    idx_range=util.getIdxRange(len(commands),len(commands)/num_proc)
    command_files=[];
    for i,start_idx in enumerate(idx_range[:-1]):
        command_file_curr=command_file_pre+str(i)+'.txt'
        end_idx=idx_range[i+1]
        commands_rel=commands[start_idx:end_idx];
        util.writeFile(command_file_curr,commands_rel);
        command_files.append(command_file_curr);
    return command_files;

def experiment1(datasets, numClusters):

	###############---VECTOR CONFIGURATION---################

	# Configure data, resulting in a list of dictionaries (labels-->vectors)
	# There is a dictionary for each dataset, stored in the same order as in the datasets list
	# dataDictionaries = randomlyConfigureActiveColumns(datasets, 5, True)
	# OR:
	dataDictionaries = util.explicitlyConfigureActiveColumns(datasets, [0,1,2,3], True) 

	###############---VECTOR NORMALIZATION---################

	# At this point, have list of dictionaries. Each dictionary contains labels mapping to vectors.
	# All of the vectors are the same dimensionality, build in the way that we specified for configuration.
	normalizedDictionaries = []
	for d in dataDictionaries:
		# print d, "\n"
		normalizedDictionaries.append(normalize.normalize(d)) # THERE ARE ALSO OTHER WAYS TO NORMALIZE

	###################---CLUSTERING---#####################
	clusterResults = cluster.gonzalez(util.crunchDictionaryList(normalizedDictionaries), numClusters, distance.euclidean);

	##################---STORE RESULTS---####################

	# Prepare to write experiment file
	clusteringAlgorithmInfo = "gonzalez"
	distanceMeasurementInfo = "euclidean"
	vectorConfigurationInfo = "explicitly configured, same columns used across datasets, Indices used: [0,1,2,3]"

	util.writeFile(1, numClusters, clusteringAlgorithmInfo, distanceMeasurementInfo,vectorConfigurationInfo,"", clusterResults[1])

def fpcCompile (text, encodedText, encoding, fileName):
	assert type(text) is unicode
	assert type(encodedText) is str
	assert encoding != None

	r = _pPas.match(text)
	if r != None:
		modName = r.group(1).encode('ascii')
		baseName = modName + '.$$$'

		if fileName == None:
			fName = baseName
			# inCurDir = True
		else:
			d = os.path.dirname(fileName)
			if (d == '') or sameFile(os.getcwd(), d):
				fName = baseName
				# inCurDir = True
			else:
				fName = os.path.join(d, baseName)
				# inCurDir = False

		if not os.path.exists(fName):
			try:
				try:
					util.writeFile( fName, encodedText.replace('\t', ' '), sync=False )
				except Exception, e:
					msg = tr('#File write error') + ': ' + exMsg(e)
					return (msg, None, None)

				try:
					e, o = cmd(["fpc", fName])
				except Exception, e:
					msg = 'fpc: ' + exMsg(e)
					return (msg, None, None)

				msg = e + o.decode( encoding )

				eLines = e.count('\n')
				errs = []
				warns = []
				i = eLines
				for l in o.split('\n'):
					r = _pfpcLine.match(l + '\n')
					if r and (r.group(1) == baseName):
						line = int(r.group(2)) - 1
						col = int(r.group(3)) - 1
						pos = (line, col)
						link = (i, pos)
						m = r.group(4)
						if m.startswith('Error:') or m.startswith('Fatal:'):
							errs.append(link)
						else:
							warns.append(link)
					i = i + 1
				return (msg, errs, warns)
			finally:

def convertFileToFloOnly(neg_flo,out_file_neg):
    neg_flo=util.readLinesFromFile(neg_flo);
    neg_only_flo=[];
    for neg_flo_curr in neg_flo:
        neg_flo_curr=neg_flo_curr.split(' ');
        neg_only_flo.append(neg_flo_curr[-1]+' '+neg_flo_curr[1]);

    assert len(neg_only_flo)==len(neg_flo);
    util.writeFile(out_file_neg,neg_only_flo);

def writeScriptToGetFloViz(input_files,output_files,out_file_sh,path_to_binary=None):
    if path_to_binary is None:
        path_to_binary='/home/maheenrashid/Downloads/flow-code/color_flow';

    lines=[];
    for input_file,output_file in zip(input_files,output_files):
        line=path_to_binary+' '+input_file+' '+output_file;
        lines.append(line);
    util.writeFile(out_file_sh,lines);

def writeTrainFilesWithFlow(old_train_file,dir_flo_im,new_train_file,ext='.png'):
    lines=util.readLinesFromFile(old_train_file);
    img_files=[line[:line.index(' ')] for line in lines];
    file_names=util.getFileNames(img_files,ext=False);
    flo_im_files=[os.path.join(dir_flo_im,file_name+ext) for file_name in file_names];
    for flo_im_file in flo_im_files:
        assert os.path.exists(flo_im_file);

    lines_new=[line+' '+flo_im_curr for line,flo_im_curr in zip(lines,flo_im_files)];
    util.writeFile(new_train_file,lines_new);

def script_writeTrainFile():
	dir_val='/disk2/ms_coco/train2014';
	out_dir='/disk2/mayExperiments/train_data';
	util.mkdir(out_dir);

	imgs=util.getEndingFiles(dir_val,'.jpg');
	imgs=[os.path.join(dir_val,file_curr) for file_curr in imgs];
	imgs.sort();
	out_file=os.path.join(out_dir,'train.txt');
	util.writeFile(out_file,imgs)

def script_writeCommandsForExperiment():
    # out_dir='/disk3/maheen_data/debug_networks/noFixCopyByLayer';
    # model_file='/home/maheenrashid/Downloads/debugging_jacob/optical_flow_prediction/examples/opticalflow/final.caffemodel';

    out_dir='/disk3/maheen_data/debug_networks/noFixCopyByLayerAlexNet';
    model_file='/home/maheenrashid/Downloads/debugging_jacob/optical_flow_prediction/models/bvlc_alexnet/bvlc_alexnet.caffemodel';

    util.mkdir(out_dir);
    train_txt_orig_path='/disk3/maheen_data/debug_networks/noFix/train.txt';

    template_deploy_file='deploy_debug_noFix.prototxt';
    template_solver_file='solver_debug.prototxt';

    train_file=os.path.join(out_dir,'train.txt');
    
    shutil.copyfile(train_txt_orig_path,train_file);

    base_lr=0.0001;
    snapshot=100;
    layers=['conv1','conv2','conv3','conv4','conv5','fc6','fc7'];

    command_pre=os.path.join(out_dir,'debug_');
    commands=[];

    for idx in range(len(layers)):
        # if idx==0:
        #     fix_layers=layers[0];
        #     layer_str=str(fix_layers);
        #     model_file_curr=None;
        # else:
        fix_layers=layers[:idx+1];
    
        layer_str='_'.join(fix_layers);
        model_file_curr=model_file
        # print fix_layers

        if idx<len(layers)/2:
            gpu=0;
        else:
            gpu=1;


        snapshot_prefix=os.path.join(out_dir,'opt_noFix_'+layer_str+'_');
        out_deploy_file=os.path.join(out_dir,'deploy_'+layer_str+'.prototxt');
        out_solver_file=os.path.join(out_dir,'solver_'+layer_str+'.prototxt');
        log_file=os.path.join(out_dir,'log_'+layer_str+'.log');
        replaceSolverFile(out_solver_file,template_solver_file,out_deploy_file,base_lr,snapshot,snapshot_prefix,gpu);
        replaceDeployFile(out_deploy_file,template_deploy_file,train_file,fix_layers);
        command=printTrainingCommand(out_solver_file,log_file,model_file_curr);
        commands.append(command);
    
    command_file_1=command_pre+'0.sh';
    util.writeFile(command_file_1,commands[:len(commands)/2]);
    command_file_2=command_pre+'1.sh';
    util.writeFile(command_file_2,commands[len(commands)/2:]);

def script_writeValFile():
	dir_val='/disk2/ms_coco/val2014';
	out_dir='/disk2/mayExperiments/validation';
	util.mkdir(out_dir);

	imgs=util.getEndingFiles(dir_val,'.jpg');
	imgs=[os.path.join(dir_val,file_curr) for file_curr in imgs];
	imgs.sort();
	imgs=imgs[:5000];
	out_file=os.path.join(out_dir,'val.txt');
	util.writeFile(out_file,imgs)

def script_saveSegSavingInfoFiles():

    dir_overlaps = '/disk3/maheen_data/headC_160_noFlow_bbox/mat_overlaps_no_neg_1000';
    out_dir='/disk3/maheen_data/debugging_score_and_scale';
    img_dir_meta='/disk2/mayExperiments/validation/rescaled_images';
    out_dir_npy=os.path.join(out_dir,'npy_for_idx');
    out_file_test_pre=os.path.join(out_dir,'test_with_seg');
    # out_file_test_big=os.path.join(out_dir,'test_with_seg_big.txt');
    util.mkdir(out_dir_npy);
    num_to_pick=10;


    mat_overlaps = util.getFilesInFolder(dir_overlaps,'.npz');
    # mat_overlaps = mat_overlaps[:10];

    args=[];
    for idx_mat_overlap_file,mat_overlap_file in enumerate(mat_overlaps):
        args.append((mat_overlap_file,num_to_pick,idx_mat_overlap_file));

    
    p = multiprocessing.Pool(multiprocessing.cpu_count());
    pred_scores_all = p.map(loadAndPickN,args);
    print len(args);


    lines_to_write={};
    # lines_to_write_big=[];
    img_names=util.getFileNames(mat_overlaps,ext=False);
    for img_name,pred_scores in zip(img_names,pred_scores_all):
        img_num_uni=np.unique(pred_scores[:,1]);
        for img_num in img_num_uni:
            img_num=int(img_num);
            curr_im=os.path.join(img_dir_meta,str(img_num),img_name+'.jpg');
            # print curr_im
            assert os.path.exists(curr_im);
            out_dir_npy_curr = os.path.join(out_dir_npy,str(img_num));
            util.mkdir(out_dir_npy_curr);
            out_file = os.path.join(out_dir_npy_curr,img_name+'.npy');
            pred_scores_rel = pred_scores[pred_scores[:,1]==img_num,:];

            np.save(out_file,pred_scores_rel);

            if img_num in lines_to_write:
                lines_to_write[img_num].append(curr_im+' '+out_file);
            else:
                lines_to_write[img_num]=[curr_im+' '+out_file];
            

    
    for img_num in lines_to_write.keys():
        out_file_test=out_file_test_pre+'_'+str(img_num)+'.txt';
        print out_file_test,len(lines_to_write[img_num]);
        util.writeFile(out_file_test,lines_to_write[img_num]);

def recordContainingFiles(dirs,num_to_evaluate,out_file_hmdb,post_dir='images',ext='.flo'):
	random.shuffle(dirs);
	print len(dirs);
	dirs=dirs[:num_to_evaluate];
	print dirs[0]
	tifs=[];
	for idx_dir_curr,dir_curr in enumerate(dirs):
		print idx_dir_curr
		tif_files=[os.path.join(dir_curr,file_curr) for file_curr in util.getFilesInFolder(os.path.join(dir_curr,post_dir),ext)];
		tifs.extend(tif_files);
	print len(tifs)
	util.writeFile(out_file_hmdb,tifs);

def main():
	path_meta='/disk2/res11/tubePatches';
	out_commands='/disk2/res11/commands_deleteAllImages.txt';
	dirs=[os.path.join(path_meta,dir_curr) for dir_curr in os.listdir(path_meta) if os.path.isdir(os.path.join(path_meta,dir_curr))];
	print len(dirs);
	commands=[];
	for dir_curr in dirs:
		dirs_in=[os.path.join(dir_curr,dir_in) for dir_in in os.listdir(dir_curr) if os.path.isdir(os.path.join(dir_curr,dir_in))];
		commands.extend(['rm -v '+dir_in+'/*.jpg' for dir_in in dirs_in]);
	print len(commands);
	print commands[:10];
	util.writeFile(out_commands,commands);

def writeCommands_hacky(out_file_commands,dirs,caffe_bin,deploy_name,path_to_model,gpu):
    commands=[];
    for dir_curr in dirs:
        out_deploy=os.path.join(dir_curr,deploy_name);
        args = [caffe_bin, 'test', '-model', util.escapeString(out_deploy),
            '-weights', path_to_model,
            '-iterations', '1',
            '-gpu', str(gpu)]

        cmd = str.join(' ', args)
        commands.append(cmd)
    util.writeFile(out_file_commands,commands);

def script_writeNegFile():


    dir_flow='/disk2/aprilExperiments/deep_proposals/flow/results_neg'
    out_text='/disk2/aprilExperiments/deep_proposals/flow/test_neg.txt';
    # util.mkdir(dir_flow);

    neg_text='/disk2/marchExperiments/deep_proposals/negatives.txt';
    lines=util.readLinesFromFile(neg_text);
    neg_images=[line_curr[:line_curr.index(' ')] for line_curr in lines];
    neg_images=neg_images[:100];
    to_write=[neg_image+' 1' for neg_image in neg_images]
    util.writeFile(out_text,to_write);

def writeh5ImgFile(dir_neg,out_file_match):

    lines=[];
    h5_files=[os.path.join(dir_neg,file_curr) for file_curr in os.listdir(dir_neg) if file_curr.endswith('.h5')];
    print len(h5_files)
    for idx_file_curr,file_curr in enumerate(h5_files):
        if idx_file_curr%100==0:
            print idx_file_curr
        img_file=util.readLinesFromFile(file_curr.replace('.h5','.txt'))[0].strip();
        # print file_curr,img_file
        lines.append(file_curr+' '+img_file);

    util.writeFile(out_file_match,lines);

def gpcpCompile (text, encodedText, encoding, fileName):
	assert type(text) is unicode
	assert type(encodedText) is str
	assert encoding != None

	r = _pMod.match(text)
	if r != None:
		modName = r.group(1).encode('ascii')
		baseName = modName + '.$$$'

		if not os.path.exists(baseName):
			try:
				try:
					util.writeFile( baseName, encodedText.replace('\t', ' '), sync=False )
				except Exception, e:
					msg = tr('#File write error') + ': ' + exMsg(e)
					return (msg, None, None)

				try:
					e, o = cmd(["gpcp", "/nodebug", "/hsize=32000", "/unsafe", baseName])
				except Exception, e:
					msg = 'gpcp: ' + exMsg(e)
					return (msg, None, None)

				msg = e + o.decode( encoding )

				eLines = e.count('\n')
				errs = []
				warns = []
				i = eLines
				state = 0 # 0 - outside, 1 - line pos matched
				for l in o.split('\n'):
					if state == 0:
						r = _pgpcpLine.match(l)
						if r:
							line = int(r.group(1)) - 1
							state = 1 # line pos matched
					elif state == 1:
						if ' Warning: ' in l:
							x = warns
						else:
							x = errs
						col = l.split('^')[0].count('-')
						pos = (line, col)
						x.append( (i - 1, pos) )
						x.append( (i, pos) )
						state = 0 # outside
					i = i + 1
				return (msg, errs, warns)

			finally: