def __init__(self,  data_fold=None, full_dataset=False, out_path=None, in_path=None, output_patches=False, scale=1.5, minSize=(200,200), windowSize=(40,40), stepSize=15):
        self.scale = scale
        self.minSize = minSize
        self.windowSize = windowSize
        self.output_patches = output_patches 
 
        self.stepSize = stepSize if self.output_patches == False else 30
        self.total_window_num = 0
        if data_fold is None:
            self.data_fold = utils.TRAINING if self.output_patches or full_dataset else utils.VALIDATION
        else:
            self.data_fold = data_fold

        self.in_path = in_path if in_path is not None else utils.get_path(in_or_out=utils.IN, data_fold=self.data_fold, full_dataset=full_dataset)
        self.img_names = [img_name for img_name in os.listdir(self.in_path) if img_name.endswith('.jpg')]
        self.img_names = self.img_names[:20] if DEBUG else self.img_names

        self.detections = Detections()
        folder_name = 'scale{}_minSize{}-{}_windowSize{}-{}_stepSize{}_dividedSizes/'.format(self.scale, self.minSize[0], self.minSize[1], 
                                                self.windowSize[0], self.windowSize[1], self.stepSize) 

        self.out_path = out_path if out_path is not None else '{}'.format(utils.get_path(full_dataset=True, in_or_out=utils.OUT, slide=True, data_fold=self.data_fold, out_folder_name=folder_name))

        self.evaluation = Evaluation(full_dataset=full_dataset, method='slide',folder_name=folder_name, save_imgs=False, out_path=self.out_path, 
                            detections=self.detections, in_path=self.in_path)

def _export_datasets(dataset, features, classes, origin, sufix):
    from itertools import combinations
    from tasks.linker import params
    from multiprocessing.pool import Pool
    
    nfolds = 3
    folds = [i for i in range(nfolds)]
    partitions = [list(c) + list((set(folds) - set(c))) for c in combinations(folds, 2)]
    datasets = _fold(dataset, nfolds)
    
    for pt in partitions:
        training = []
        for i in pt[:-1]: 
            training.extend(datasets[i])
        test = datasets[pt[-1]]
         
        name_ = 'all{}{}{}'.format(origin, sufix + '_tr', pt[-1])
        filename = get_path('datasets', '{}.arff'.format(name_))
        classes_ = [next((v['short_name'] for k, v in params.items() if v['metadata_uri'] == c), None) for c in classes]    
        dataset_ = ([d, classes_] for d in _chunks(training, os.cpu_count()))
        with Pool(os.cpu_count()) as p: sets_ = p.starmap(_expand, dataset_); dataset_ = []
        for s in sets_: dataset_.extend(s)
        with Pool(os.cpu_count()) as p: dataset_ = p.map(_flatten, dataset_)
        _save(dataset_, features, 'class', name_, filename)
        
        
        name_ = 'all{}{}{}'.format(origin, sufix + '_tt', pt[-1])
        filename = get_path('datasets', '{}.arff'.format(name_))
        dataset_ = ([l, classes_] for l in test)
        with Pool(os.cpu_count()) as p: dataset_ = p.starmap(_concat, dataset_)
        with Pool(os.cpu_count()) as p: dataset_ = p.map(_flatten, dataset_)
        _save_test(dataset_, features, 'class', name_, filename)

def main(pickled_evaluation=False, combo_f_name=None, output_patches=True, 
                detector_params=None, original_dataset=True, save_imgs=True, data_fold=utils.VALIDATION):
    combo_f_name = None
    try:
        opts, args = getopt.getopt(sys.argv[1:], "f:")
    except getopt.GetoptError:
        sys.exit(2)
        print 'Command line failed'
    for opt, arg in opts:
        if opt == '-f':
            combo_f_name = arg

    assert combo_f_name is not None
    detector = viola_detector_helpers.get_detectors(combo_f_name)

    viola = False if data_fold == utils.TRAINING else True
    in_path = utils.get_path(viola=viola, in_or_out=utils.IN, data_fold=data_fold)

    #name the output_folder
    folder_name = ['combo'+combo_f_name]
    for k, v in detector_params.iteritems():
        folder_name.append('{0}{1}'.format(k,v))
    folder_name = '_'.join(folder_name)

    out_path = utils.get_path(out_folder_name=folder_name, viola=True, in_or_out=utils.OUT, data_fold=data_fold)
    out_path = 'output_viola_uninhabited/'
    viola = ViolaDetector(pickled_evaluation=pickled_evaluation, output_patches=output_patches,  
                            out_path=out_path, in_path=in_path, folder_name = folder_name, 
                            save_imgs=save_imgs, detector_names=detector,  **detector_params)
    return viola

def predict_target_with_query(
        sparql, query, source, timeout=TIMEOUT, limit=LIMIT):
    """Predicts target with given query.

    For example for pagerank_bidi:
    SELECT distinct(?target) ?score {
     { dbr:Circle ?p ?target .}
     UNION
     { ?target ?q dbr:Circle . }
     ?target dbo:wikiPageRank ?score
    }
    ORDER BY DESC(?score)
    LIMIT 100
    """
    q = query % {'source': source.n3()}
    q += '\nLIMIT %d' % limit
    t, q_res = gp_query._query(sparql, timeout, q)
    res_rows_path = ['results', 'bindings']
    bindings = sparql_json_result_bindings_to_rdflib(
        get_path(q_res, res_rows_path, default=[])
    )
    target_scores = [
        (get_path(row, [TARGET_VAR]), get_path(row, [Variable('score')]))
        for row in bindings]
    # print(target_scores)
    return target_scores

def main(): 
    script_dirname = os.path.abspath(os.path.dirname(__file__))
    output_patches = False 
    fold = utils.TRAINING if output_patches else utils.VALIDATION
    
    #only use this path to get the names of the files you want to use
    in_path = utils.get_path(in_or_out=utils.IN, data_fold=fold)
    in_path_selective = script_dirname+'/' #this is where the files actually live
    img_names = [img for img in os.listdir(in_path) if img.endswith('jpg')]
    image_filenames = [in_path_selective+img for img in os.listdir(in_path) if img.endswith('jpg')]


    #get the proposals
    k, scale = get_parameters()
    sim = 'all'
    color = 'hsv'
    cmd = 'selective_search'
    if cmd == 'selective_search':
        folder_name = 'k{}_scale{}_sim{}_color{}_FIXING/'.format(k, scale, sim, color)
    else:
        folder_name = 'selectiveRCNN/'
    print 'Folder name is: {}'.format(folder_name)

    with Timer() as t:
        boxes = get_windows(image_filenames, script_dirname, cmd=cmd, k=k, scale=scale)
    print 'Time to process {}'.format(t.secs)

    detections = Detections()
    detections.total_time = t.secs
    out_path = utils.get_path(selective=True, in_or_out=utils.OUT, data_fold=fold, out_folder_name=folder_name)

    evaluation = Evaluation(#use_corrected_roofs=True,
                        report_name='report.txt', method='windows', 
                        folder_name=folder_name,  out_path=out_path, 
                        detections=detections, in_path=in_path)
    
    #score the proposals
    for img, proposals in zip(img_names, boxes):
        print 'Evaluating {}'.format(img)
        print("Found {} windows".format(len(proposals)))

        proposals = selectionboxes2polygons(proposals)
        detections.set_detections(detection_list=proposals,roof_type='metal', img_name=img)  
        detections.set_detections(detection_list=proposals,roof_type='thatch', img_name=img)  
        print 'Evaluating...'
        evaluation.score_img(img, (1200,2000)) 

        evaluation.save_images(img)

    save_training_TP_FP_using_voc(evaluation, img_names, in_path_selective, out_folder_name=folder_name, neg_thresh=0.3)
    evaluation.print_report() 

    with open(out_path+'evaluation.pickle', 'wb') as f:
        pickle.dump(evaluation, f)

def _ask_chunk_result_extractor(q_res, _vars, _ret_val_mapping):
    chunk_res = {}
    res_rows_path = ['results', 'bindings']
    bindings = sparql_json_result_bindings_to_rdflib(
        get_path(q_res, res_rows_path, default=[])
    )
    for row in bindings:
        row_res = tuple([get_path(row, [v]) for v in _vars])
        stps = _ret_val_mapping[row_res]
        chunk_res.update({stp: True for stp in stps})
    return chunk_res

def _var_subst_chunk_result_ext(q_res, _sel_var_and_vars, _, **kwds):
    var, _vars = _sel_var_and_vars
    chunk_res = Counter()
    res_rows_path = ['results', 'bindings']
    bindings = sparql_json_result_bindings_to_rdflib(
        get_path(q_res, res_rows_path, default=[])
    )

    for row in bindings:
        row_res = get_path(row, [var])
        count_res = int(get_path(row, [COUNT_VAR], '0'))
        chunk_res[row_res] += count_res
    return chunk_res

def _combined_chunk_res(q_res, _vars, _ret_val_mapping):
    chunk_res = {}
    res_rows_path = ['results', 'bindings']
    bindings = sparql_json_result_bindings_to_rdflib(
        get_path(q_res, res_rows_path, default=[])
    )
    for row in bindings:
        row_res = tuple([get_path(row, [v]) for v in _vars])
        stps = _ret_val_mapping[row_res]
        ask_res = int(get_path(row, [ASK_VAR], '0'))
        count_res = int(get_path(row, [COUNT_VAR], '0'))
        chunk_res.update({stp: (ask_res, count_res) for stp in stps})
    return chunk_res

def _load_statistics():
    global _statistics
    filename = get_path('datasets' , 'statistics.json')
    _statistics = load_data(filename, verbose=False)
    if not _statistics or not isinstance(_statistics, dict) : 
        _statistics = {}
    print ('')

async def fetch_all():
    """Fetch pdf files."""

    with get_path("answers").open() as answers_file:
        nsolved = sum(1 for line in answers_file)

    cookies = {
        'DYNSRV': 'lin-10-170-0-31',
        'PHPSESSID': 'a4fb01c0de27e200683b4d556461b5aa',
        'keep_alive': '1119831347%23333574%233PpV0T6RtnqnCB6GNF4PvEH1TiEX1nlc'
    }

    headers = {
        'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:58.0) Gecko/20100101 Firefox/58.0',
    }

    async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
        coros = [fetch_one(session, num) for num in range(1, nsolved + 1)]

        results = await asyncio.gather(*coros)
        files = 0
        for num in results:
            if num is not None:
                print(f'Saved a file for problem {num}')
                files += 1

        return files

    def __init__(self, config, test=False, target=None):
        self._cwd = os.getcwd()
        self._root = get_path()
        self._config = config
        self._test = test
        self._target = target

        try:
            self._skeleton_path = resource_filename(__name__, os.path.join('skeleton', 'default'))
        except NotImplementedError:
            self._skeleton_path = os.path.join(sys.prefix, 'skeleton', 'default')

        try:
            self._assetPath = os.path.join(resource_filename(__name__, os.path.join('assets', 'manage.py')))
        except NotImplementedError:
            self._assetPath = os.path.join(sys.prefix, 'assets', 'manage.py')

        self._projectName = self._config['name']
        if 'deployment_path' in self._config:
            self._deployment_path = self._config['deployment_path']
        else:
            self._deployment_path = ''
        if 'zip_path' in self._config:
            self._zip_path = self._config['zip_path']
        else:
            self._zip_path = ''
        if 'doc_path' in self._config:
            self._doc_path = self._config['doc_path']
        else:
            self._doc_path = ''

def main():
    size = 40
    total = spanning = 0

    graph = Graph(size)

    with get_path("data", "network.txt").open() as data_file:
        for u, line in enumerate(data_file):
            for v, w in enumerate(line.rstrip().split(",")):
                if v > u and w != "-":
                    w = int(w)
                    total += w
                    graph.insert_edge(u, v, w)

    # Implement the Prim–Jarník algorithm
    D = [float("inf")] * size
    root = 0  # can be any vertex of the graph

    heap = [(0, root)]
    seen = [False] * size

    while heap:
        w, u = heappop(heap)
        if not seen[u]:
            spanning += w
            seen[u] = True

            for e in graph.incident_edges(u):
                v = e.opposite(u)
                if e.weight < D[v]:
                    D[v] = e.weight
                    heappush(heap, (e.weight, v))

    return total - spanning

def plot_loss():
    path = utils.get_path(neural=True, in_or_out=utils.OUT, data_fold=utils.TRAINING)
    path_slide = path+'slide/'
    path_viola = path+'viola/'
    for path in [path_slide, path_viola]:
        for file in os.listdir(path):
            if file.endswith('_history'):
                training_loss = list()
                validation_loss = list()
                with open(path+file, 'rb') as csv_file:
                    csv_reader = csv.reader(csv_file, delimiter='\t')
                    for i, row in enumerate(csv_reader):
                        if i==0:
                            continue
                        training_loss.append(float(row[1]))
                        validation_loss.append(float(row[2]))

                plt.plot(training_loss, linewidth=3, label='train loss')
                plt.plot(validation_loss, linewidth=3, label='valid loss')

                #plt.title('History of {0}'.format(file[:-(len('_history'))]))
                plt.legend(loc='best')
                plt.grid()
                plt.xlabel("epoch")
                plt.ylabel("loss")

                plot_name = path+file+'.jpg' 
                plt.savefig(plot_name)
                plt.close()

    def __init__(self, config):
        self._cwd = os.getcwd()
        self._root = get_path()
        self._config = config
        self._verify_ssl = True

        if 'urls' not in self._config:
            raise MissingKeyError('Could not find url settings in either global or local configuration file.')

        if 'upload' not in self._config['urls']:
            raise MissingKeyError('Could not find an upload url in either the global or local configuration file.')
        else:
            self._upload_url = self._config['upload_url']

        if 'login_url' not in self._config['urls']:
            self._login_url = self._upload_url
        else:
            self._login_url = self._config['login_url']

        if 'username' not in self._config['urls']:
            self._username = raw_input('Please provide the username for your upload server (or leave blank if none is required): ')
        else:
            self._username = self._config['urls']['username'].encode()

        if 'password' not in self._config['urls']:
            self._password = getpass.getpass('Please provide the password for your upload server (or leave blank if none is required): ')
        else:
            self._password = self._config['urls']['password'].encode()

        self._zip_name = self._config['name'] + '_v' + self._config['version'] + '.zip'
        self._zip_path = os.path.join(self._cwd, 'build', self._zip_name)

    def __init__(self, handle):
        ''' Initialize the toolbars and the work surface '''
        super(BBoardActivity, self).__init__(handle)

        self.datapath = get_path(activity, 'instance')

        self._hw = get_hardware()

        self._playback_buttons = {}
        self._audio_recordings = {}
        self.colors = profile.get_color().to_string().split(',')

        self._setup_toolbars()
        self._setup_canvas()

        self.slides = []
        self._setup_workspace()

        self._buddies = [profile.get_nick_name()]
        self._setup_presence_service()

        self._thumbs = []
        self._thumbnail_mode = False

        self._recording = False
        self._grecord = None
        self._alert = None

        self._dirty = False

def save_training_TP_FP_using_voc(evaluation, img_names, in_path, out_folder_name=None, neg_thresh=0.3):
    '''use the voc scores to decide if a patch should be saved as a TP or FP or not
    '''
    assert out_folder_name is not None
    general_path = utils.get_path(neural=True, data_fold=utils.TRAINING, in_or_out=utils.IN, out_folder_name=out_folder_name)
    path_true = general_path+'truepos_from_selective_search/'
    utils.mkdir(path_true)

    path_false = general_path+'falsepos_from_selective_search/'
    utils.mkdir(path_false)

    for img_name in img_names:
        good_detections = defaultdict(list)
        bad_detections = defaultdict(list)
        try:
            img = cv2.imread(in_path+img_name, flags=cv2.IMREAD_COLOR)
        except:
            print 'Cannot open image'
            sys.exit(-1)

        for roof_type in utils.ROOF_TYPES:
            detection_scores = evaluation.detections.best_score_per_detection[img_name][roof_type]
            for detection, score in detection_scores:
                if score > 0.5:
                    #true positive
                    good_detections[roof_type].append(detection)
                if score < neg_thresh:
                    #false positive
                    bad_detections[roof_type].append(detection)
                
        for roof_type in utils.ROOF_TYPES:
            extraction_type = 'good'
            save_training_FP_and_TP_helper(img_name, evaluation, good_detections[roof_type], path_true, general_path, img, roof_type, extraction_type, (0,255,0))               
            extraction_type = 'background'
            save_training_FP_and_TP_helper(img_name, evaluation, bad_detections[roof_type], path_false, general_path, img, roof_type, extraction_type, (0,0,255))               

	def add(self, cate):
		url = cate['url']

		domain = get_domain(url)
		subdomains = get_subdomains(url)
		paths = get_path(url).split('/')
		query = urlparse.urlparse(url).query

		if domain not in self.root:
			self.root[domain] = {'sub':{}, 'path':{}}

		node = self.root[domain]
		if len(subdomains) > 1 or len(subdomains) == 1 and subdomains[0] != 'www':
			for sub in subdomains:
				if sub not in node['sub']:
					node['sub'][sub] = {'sub':{}, 'path':{}}
				node = node['sub'][sub]

		for path in paths:
			if path not in node['path']:
				node['path'][path] = {'path':{}}
			node = node['path'][path]

		if query:
			node['path']['query___' + query] = {'path':{}}
			node = node['path']['query___' + query]

		node['cate'] = cate

def main():
    chars_saved = 0

    # From the problem definition:
    # You can assume that all the Roman numerals in the file contain no more
    # than four consecutive identical units.

    with get_path("data", "roman.txt").open() as data_file:
        for line in data_file:
            if "VIIII" in line:
                chars_saved += 3  # VIIII => IX
            elif "IIII" in line:
                chars_saved += 2  # IIII => IV

            if "LXXXX" in line:
                chars_saved += 3  # LXXXX => XC
            elif "XXXX" in line:
                chars_saved += 2  # XXXX => XL

            if "DCCCC" in line:
                chars_saved += 3  # DCCCC => CM
            elif "CCCC" in line:
                chars_saved += 2  # CCCC => CD

    return chars_saved

 def __init__(self):
     import os
     from utils import get_path
     self.categories_by_id = None
     self.categories_by_name = None
     self._filename = get_path('fs-categories.json')
     self._api_venues = Foursquare(client_id=_fsc.CLIENT_ID, client_secret=_fsc.CLIENT_SECRET)
     self.load_categories()

def main():
    args = parse_args()

    if args.path:
        paths = []
        for path in args.path:
            path_obj = get_path("python", path)
            if not path_obj.exists():
                print(f"File not found: {path}")
                return 1
            paths.append(path_obj)
    else:
        paths = sorted(get_path("python").glob("problem???.py"))

    loop = asyncio.get_event_loop()
    loop.run_until_complete(add_docstrings(paths))
    return 0