def _print_progress(self, iteration, n_iter,
                     cost=None, time_interval=10):
     if self.print_progress > 0:
         s = '\rIteration: %d/%d' % (iteration, n_iter)
         if cost:
             s += ' | Cost %.2f' % cost
         if self.print_progress > 1:
             if not hasattr(self, 'ela_str_'):
                 self.ela_str_ = '00:00:00'
             if not iteration % time_interval:
                 ela_sec = time() - self._init_time
                 self.ela_str_ = self._to_hhmmss(ela_sec)
             s += ' | Elapsed: %s' % self.ela_str_
             if self.print_progress > 2:
                 if not hasattr(self, 'eta_str_'):
                     self.eta_str_ = '00:00:00'
                 if not iteration % time_interval:
                     eta_sec = ((ela_sec / float(iteration)) *
                                n_iter - ela_sec)
                     if eta_sec < 0.0:
                         eta_sec = 0.0
                     self.eta_str_ = self._to_hhmmss(eta_sec)
                 s += ' | ETA: %s' % self.eta_str_
         stderr.write(s)
         stderr.flush()

 def _validate_schema(self, corpus):
     if self.schema_file is None:
         if self.options.schema is None:
             return True
         self.schema = Schema(self.options.schema)
     else:
         self.schema = Schema('<internal>', self.schema_file)
     print 'Schema validation:'
     ignore = set()
     if self.options.ignore is not None:
         f = open(self.options.ignore)
         for l in f:
             ignore.add(l.strip())
         f.close()
     nerr = 0
     nsil = 0
     for msg in self.schema.check_corpus(corpus):
         nerr += 1
         if msg in ignore:
             nsil += 1
         else:
             stderr.write(msg + '\n')
     if nerr == 0:
         print '    ok'
         return True
     if self.options.ignore is not None:
         if nerr == nsil:
             print '   ', str(nerr),'errors (all silenced)'
             return True
         print '   ', str(nerr), 'errors (', str(nsil), 'silenced)'
         return False
     print '   ', str(nerr), 'errors'
     return False

def plotSequences(seq,filename):
    from sys import stderr, argv
    from os import popen
    from os.path import basename
    from re import sub;
    import FWCore.ParameterSet.Config as cms
    stderr.write("Writing plot to %s\n" % (filename,))
    dot = popen("dot -Tpng > %s" % (filename,), "w")
    dot.write("digraph G { \n rankdir=\"LR\" \n")
    class visitor(object):
        def __init__(self,seq,dot):
            self._dot = dot
            self._stack = []
            self._seq = seq.label()
            self._dot.write( "%s [  shape=rect style=filled fillcolor=%s label=\"%s\" ]" % (self._seq,'orange',self._seq) + "\n" )
        def seq(self, seq):
            self._stack.append(self._seq)
            self._seq = seq.label()
        def enter(self,v):
            if isinstance(v, cms.Sequence):
                self._dot.write( "%s [  shape=rect style=filled fillcolor=%s label=\"%s\" ]" % (v.label(),'orange',v.label()) + "\n" )
                self.dep(v)
                self.seq(v)
            if isinstance(v, (cms.EDProducer, cms.EDFilter, cms.EDAnalyzer)):
                self._dot.write( "%s [  shape=rect style=filled fillcolor=%s label=\"%s\" ]" % (v.label(),'green',v.label()) + "\n" ) 
                self.dep(v)
        def leave(self,v):
            if isinstance(v, cms.Sequence):
                self._seq = self._stack.pop()
        def dep(self,v):
            self._dot.write("%s -> %s" %(v.label(), self._seq) +"\n")
    seq.visit(visitor(seq,dot))
    dot.write("}\n")
    dot.close()

def main():
    DCT_H = 30
    DCT_W = 40
    a = 1
    while a < len(argv):
        if argv[a] == '-h': 
            DCT_H = int(argv[a+1])
            i += 2
        elif argv[a] == '-w': 
            DCT_W = int(argv[a+1])
            i += 2
        else:
            stderr.write('Unknown option: %s\n' % argv[a])
            return 1
            
    for fname in stdin:
        fname=fname.strip()
        if len(fname) == 0: continue
        f = basename(fname)[:3] # frame
        us = basename(dirname(fname)).split('_')
        u, s = us[0], us[1]
        dct = reduce(lambda x,y: x+y,
                     GetDCT(fname)[:DCT_H,:DCT_W].tolist(), [])
        odir = 'data/features/video/%s/%s' % (u, s)
        if not exists(odir):
            system('mkdir -p %s' % odir)
        fdct = open('%s/%s.dct' % (odir, f), 'w')
        for i in dct:
            fdct.write('%f\n' % i)
        fdct.close()
    return 0

def main(args):
    napalm = {"name": "Napalm", "func": Napalm, "flag_set": args.napalm}
    mamont = {"name": "Mamont", "func": Mamont, "flag_set": args.mamont}
    filewatcher = {"name": "FileWatcher", "func": Filewatcher, "flag_set": args.filewatcher}
    filemare = {"name": "FileMare", "func": Filemare, "flag_set": args.filemare}

    custom_functions = []
    for routine in (napalm, mamont, filewatcher, filemare):
        if(routine["flag_set"]):
            custom_functions.append(routine)

    # Process -fw, -fm, -ma, -na flags if they are set
    if(len(custom_functions) > 0):
        functions = custom_functions
    else:
        functions = (napalm, mamont, filewatcher, filemare)

    # Start the scraping process
    for function in functions:
        try:
            stderr.write("\t-=[ {0} ]=-\n".format(function["name"]))
            stderr.flush()
            function["func"](args).search()
        except(KeyboardInterrupt, EOFError):
            continue

    stderr.write("\n")
    stderr.flush()

def result():
    global indent
    indent -= 1
    if midline:
        stderr.write(" ")
    else:
        margin()

def whichEC2Service(env):
	stderr.write("\x1b[2J\x1b[H")
	if env == 'None':
		import awstools as tool
		e = tool.whichEnvironment()		# optimize these two lines:
		whichEC2Service(e)				# whichEC2Service(tool.whichEnvironment())
	else:
		print 'what kind of thing might you be thinking about?:\n'
		print '   0)  let me out, please'
		print '   1)  query existing vpcs'
		print '   2)  create a vpc'
		print '   3)  modify a vpc'
		print '   4)  delete a vpc'
		print ''
		m = raw_input('what\'s your pleasure?: ')
		print ''
		
		if m == '0':
			exit()
		elif m == '1':
			queryEC2(env)
		elif m == '2':
			createEC2(env)
		elif m == '3':
			modifyEC2(env)
		elif m == '4':
			deleteEC2(env)
		else:
			print '\nuhm, no. let\'s try this again.... '
			whichEC2Service(env)

 def _load_links(self):
     """ Sorts supported and not supported links in two lists and returns them. """
     try: # Process the arguments, either read links from file or directly
          # from the command line with the -u/--url flag.
         if self.args.file:
             with open(self.args.file) as file_:
                 links = [url.decode("utf8").strip() for url in file_ if url.strip()]
         else:
             links = [url.decode("utf8").strip() for url in self.args.url if url.strip()]
         # Remove duplicates
         links = list(set(links))
     except IOError:
         stderr.write("Couldn't open input file, are you sure the path is correct?\n")
         stderr.flush()
         exit(1)
     not_supported = []
     supported = []
     for link in links:
         parse = urlparse.urlparse(link)
         if(not parse.scheme):
             link = "http://{0}".format(link)
         if(self.is_supported(link) and parse.path):
             supported.append(link)
         else:
             not_supported.append(link)
     return supported, not_supported

def unpacktype(binstr, member, mtype):
    offset = member[1]
    size = member[2]
    fmt = ''

    if mtype == STR:
        fmt = str(size) + 's'
    elif mtype == INT:
        fmt = 'I' if size == 4 else 'Q'
    elif mtype == SHT:
        fmt = 'H'
    else:
        calling_fxn = sys._getframe(1)
        stderr.write("ERROR %s.%s tried to unpack the unknown type %d.\n" % (
        callingclass(calling_fxn), calling_fxn.f_code.co_name, mtype))
        return None

    if struct.calcsize(fmt) != len(binstr[offset:size + offset]):
        calling_fxn = sys._getframe(1)
        stderr.write("ERROR %s.%s tried to unpack '%s' (fmt size: %d) from %d bytes.\n" % (
        callingclass(calling_fxn), calling_fxn.f_code.co_name, fmt, struct.calcsize(fmt),
        len(binstr[offset:size + offset])))
        return None

    return struct.unpack(fmt, binstr[offset:size + offset])[0]

 def reducer(self, station, counts):
     try:
         if station[0] == 'error':
             yield(station,sum(data))
             return
         self.increment_counter('MrJob Counters','reducer',1)
         dic_TMIN={}
         dic_TMAX={}
         dic_PRCP={}
         for vec in counts:
             if vec[1]=='TMIN':
                 dic_TMIN[vec[0]]=vec[2]
             elif vec[1]=='TMAX':
                 dic_TMAX[vec[0]]=vec[2]
             else:
                 dic_PRCP[vec[0]]=vec[2]
         validyear=[]
         mean_MEAN=[]
         mean_PRCP=[]
         diff_MEAN=[]
         for key in dic_TMIN:
             if dic_TMAX.has_key(key) and dic_PRCP.has_key(key):
                 validyear.append(key)
                 mm=np.mean(np.array(dic_TMAX[key]))-np.mean(np.array(dic_TMIN[key]))
                 #mean_MEAN.append(mm/2.0)
                 #mean_PRCP.append(np.mean(np.array(dic_PRCP[key])))
                 diff_MEAN.append(mm)
         if len(validyear)>0:
             yield (station, np.mean(np.array(diff_MEAN)))
         #else:
         #    yield (station, _)
     except Exception, e:
         #yield (('error','reducer', str(e)), 1)
         stderr.write('Error in reducer')

 def doCombination(self):
     ## Contrary to Number-counting models, here each channel PDF already contains the nuisances
     ## So we just have to build the combined pdf
     if len(self.DC.bins) > 1 or not self.options.forceNonSimPdf:
         for (postfixIn,postfixOut) in [ ("","_s"), ("_bonly","_b") ]:
             simPdf = ROOT.RooSimultaneous("model"+postfixOut, "model"+postfixOut, self.out.binCat) if self.options.noOptimizePdf else ROOT.RooSimultaneousOpt("model"+postfixOut, "model"+postfixOut, self.out.binCat)
             for b in self.DC.bins:
                 pdfi = self.out.pdf("pdf_bin%s%s" % (b,postfixIn))
                 simPdf.addPdf(pdfi, b)
             if len(self.DC.systs) and (not self.options.noOptimizePdf) and self.options.moreOptimizeSimPdf:
                 simPdf.addExtraConstraints(self.out.nuisPdfs)
             if self.options.verbose:
                 stderr.write("Importing combined pdf %s\n" % simPdf.GetName()); stderr.flush()
             self.out._import(simPdf)
             if self.options.noBOnly: break
     else:
         self.out._import(self.out.pdf("pdf_bin%s"       % self.DC.bins[0]).clone("model_s"), ROOT.RooFit.Silence())
         if not self.options.noBOnly: 
             self.out._import(self.out.pdf("pdf_bin%s_bonly" % self.DC.bins[0]).clone("model_b"), ROOT.RooFit.Silence())
     if self.options.fixpars:
         pars = self.out.pdf("model_s").getParameters(self.out.obs)
         iter = pars.createIterator()
         while True:
             arg = iter.Next()
             if arg == None: break;
             if arg.InheritsFrom("RooRealVar") and arg.GetName() != "r": 
                 arg.setConstant(True);

def grabber():
    while(True):
        shuffle(BOARDS)
        for b in BOARDS:
            yield gen.Task(proc_board, b)
        stderr.write('Grabber went to sleep\n')
        yield gen.Task(IOLoop.instance().add_timeout, time.time() + 24 * 60 * 60)

 def featurize_and_uniq_triangles_stdin(self):
     tri_group = set()
     tri_group_head = None
     cnt = 0
     for l in stdin:
         cnt += 1
         if cnt % 10000 == 0:
             stderr.write('{0}\n'.format(cnt))
         try:
             l_ = l.decode('utf8').strip().split('\t')
             this_tri = '\t'.join(l_[0:4])
             if not tri_group_head:
                 tri_group_head = this_tri
                 tri_group.add(tuple(l_))
             elif tri_group_head == this_tri:
                 tri_group.add(tuple(l_))
             else:
                 if tri_group:
                     feat, pair = self.featurize_group(tri_group)
                     self.print_pair_with_features(pair, feat)
                 tri_group = set()
                 tri_group.add(tuple(l_))
                 tri_group_head = this_tri
         except:
             logging.exception('Exception at line: {0}'.format(l.strip()))
     feat, pair = self.featurize_group(tri_group)
     self.print_pair_with_features(pair, feat)

def test_ewens (kind, abd, test_failed):
    '''
    test estimation of theta by max likelihood with ewens formula
    '''
    t0 = time ()
    test_ok=True
    print ' Testing Ewens algorithm with BCI %s dataset' % kind
    if kind == 'full':
        wanted_theta = 34.962
        wanted_lnl   = 318.849
        time_max = 0.05
    elif kind == 'short':
        wanted_theta = 33.302
        wanted_lnl   = 162.742
        time_max = 0.05
    abd.ewens_optimal_params()
    abd.set_current_model ('ewens')
    model = abd.get_model ('ewens')
    print '  -> Optimal value of theta: %.3f' % abd.theta
    if round (abd.theta, 3) != wanted_theta:
        stderr.write ('\n test failed in ewens test (theta should have been %s)\n' %\
              wanted_theta)
        test_ok=False

    print '  -> likelihood of theta: %.3f' % model.lnL
    if round (model.lnL, 3) != wanted_lnl:
        stderr.write ('\n test failed in ewens test (theta should have been %s)\n' %\
              wanted_lnl)
        test_ok=False
   
    print '\n  Elapsed time (should be < %s): %s sec\n' % (time_max, time() - t0)
    if not test_ok:
        test_failed.append ('etienne optimization')
    return test_failed

 def residue( self, pos ):
     ''' return frag_idx of that pos '''
     assert self._initialized
     if pos in self.positions(): 
         return self.__dict[ pos ]
     else: 
         stderr.write("ERROR: %s is not in this pose\n" % pos )

	def __init__(self, message="", default_text='', modal=True):
		gtk.Dialog.__init__(self)
		self.add_buttons(gtk.STOCK_CANCEL, gtk.RESPONSE_CLOSE,
		      gtk.STOCK_OK, gtk.RESPONSE_ACCEPT)
		#self.set_title("Icon search")
		if modal:
			self.set_modal(True)
		self.set_border_width(5)
		self.set_size_request(400, 300)
		self.combobox=gtk.combo_box_new_text()
		self.combobox.set_size_request(200, 20)
		hbox=gtk.HBox(False,2)
		
		#format: actual icon, name, context
		self.model=gtk.ListStore(gtk.gdk.Pixbuf, gobject.TYPE_STRING, gobject.TYPE_STRING)
		self.modelfilter=self.model.filter_new()
		self.modelfilter.set_visible_func(self.search_icons)
		self.iconview=gtk.IconView()
		self.iconview.set_model(self.modelfilter)
		self.iconview.set_pixbuf_column(0)
		self.iconview.set_text_column(1)
		self.iconview.set_selection_mode(gtk.SELECTION_SINGLE)
		self.iconview.set_item_width(72)
		self.iconview.set_size_request(200, 220)
		defaulttheme=gtk.icon_theme_get_default()
		self.combobox.connect('changed', self.category_changed)
		self.refine=gtk.Entry()
		self.refine.connect('changed', self.category_changed)
		self.refine.set_icon_from_stock(gtk.ENTRY_ICON_SECONDARY, gtk.STOCK_FIND)
		self.refine.set_size_request(200, 30)
		list2=[]
		for c in defaulttheme.list_contexts():
			current=defaulttheme.list_icons(context=c)
			list2+=set(current)
			self.combobox.append_text(c)
			for i in current:
				try:
					self.model.append([defaulttheme.load_icon(i, 32,
									  gtk.ICON_LOOKUP_USE_BUILTIN),
									  i,c])
				except GError as err: stderr.write('Error loading "%s": %s\n' % (i, err.args[0]))
		other=list(set(defaulttheme.list_icons())-(set(list2)))
		for i in other:
			self.model.append([defaulttheme.load_icon(i, 32,
									  gtk.ICON_LOOKUP_USE_BUILTIN),
									  i,"Other"])
		self.combobox.prepend_text("Other")
		scrolled = gtk.ScrolledWindow()
		scrolled.add(self.iconview)
		scrolled.props.hscrollbar_policy = gtk.POLICY_NEVER
		scrolled.props.vscrollbar_policy = gtk.POLICY_AUTOMATIC
		hbox.add(self.combobox)
		hbox.add(self.refine)
		self.vbox.add(hbox)
		self.vbox.add(scrolled)
		self.combobox.set_active(0)
		
		self.iconview.connect('selection-changed', self.get_icon_name)
		
		self.vbox.show_all()

def main():
    """
    main function
    """

    all_members = get_all(pytadbit)
    get_all(pytadbit.utils, all_members)
    get_all(tad_cmo, all_members)
    get_all(mapper, all_members)
    get_all(restriction_enzymes, all_members)
    get_all(analyze, all_members)
    get_all(filter, all_members)
    modules = set([all_members[m]['son'].__module__ for m in all_members])

    global LINKS
    LINKS = parse_doc_index()

    numclasses = nfunctions = 0
    nummodules = len(modules)

    # title
    print '======================================='
    print 'Summary of TADbit classes and functions'
    print '=======================================\n'

    # body
    print ''
    for module in sorted(modules):
        print print_doc(module, header=1)
        
        submodules = [m for m in all_members
                      if all_members[m]['son'].__module__ == module]
        dadies = set([all_members[m]['dady'] for m in submodules
                      if all_members[m]['dady'][0] in uppercase])
        for member in sorted(submodules,
                             key=lambda x:all_members[x]['son'].__name__[0] in uppercase):
            if all_members[member]['dady'] in dadies or member in dadies:
                continue
            if all_members[member]['son'].__name__[0] in uppercase:
                print print_doc(all_members[member]['son'], header=2, indent=3)
                numclasses += 1
            else:
                nfunctions += 1
                print print_doc(all_members[member]['son'], header=3, indent=3)
        for dady in sorted(dadies):
            numclasses += 1
            print print_doc(all_members[dady]['son'], offset=9, header=2)
            for member in sorted(submodules):
                if all_members[member]['dady'] != dady:
                    continue
                nfunctions += 1
                print print_doc(all_members[member]['son'], header=3,
                                indent=6)

    # footnotes
    print ''
    print '.. [#first] functions generating plots\n'
    print '.. [#second] functions writing text files\n'
    stderr.write('Reporting %s modules, %s classes and %s functions\n' %(
        nummodules, numclasses, nfunctions))

def establish_connection(hostname):
    '''
    Prompt for the hostname to contact and the username and password, if
    necessary.
    '''

    if not hostname:
        # Prompt for the hostname
        hostname = raw_input('Hostname: ')

    host_uri = "https://%s" % hostname

    # Run this script as root and this will use the
    # local UNIX socket. Otherwise, modify this line
    # to match the system you are connecting to

    if getuid() == 0 and hostname == 'localhost':
        c = connect(hostname)
    else:
        # Prompt for user and password
        user = raw_input('User: ')
        passwd = getpass("Password: ")

        c = connect(host_uri, user, passwd)

    if not c:
        stderr.write("Couldn't authenticate\n")
        exit(1)
    return c

def runopf(casedata=None, ppopt=None, fname='', solvedcase=''):
    """Runs an optimal power flow.

    @see: L{rundcopf}, L{runuopf}

    @author: Ray Zimmerman (PSERC Cornell)
    """
    ## default arguments
    if casedata is None:
        casedata = join(dirname(__file__), 'case9')
    ppopt = ppoption(ppopt)

    ##-----  run the optimal power flow  -----
    r = opf(casedata, ppopt)

    ##-----  output results  -----
    if fname:
        fd = None
        try:
            fd = open(fname, "a")
        except IOError as detail:
            stderr.write("Error opening %s: %s.\n" % (fname, detail))
        finally:
            if fd is not None:
                printpf(r, fd, ppopt)
                fd.close()

    else:
        printpf(r, stdout, ppopt)

    ## save solved case
    if solvedcase:
        savecase(solvedcase, r)

    return r

def queryEC2(e):
	stderr.write("\x1b[2J\x1b[H")
	print 'how are we gonna slice and dice?:\n'
	print '   0)  let me out, please'
	print '   1)  by instance name'
	print '   2)  by instance id'
	print '   3)  by instance tags'
	print '   4)  by instance values'
	print '   5)  by security groups'
	print ''
	m = raw_input('slice or dice?: ')
	print ''
	
	if m == '0':
		exit()
	elif m == '1':
		getEC2ByName()
	elif m == '2':
		getEC2ByID
	elif m == '3':
		getEC2ByTags()
	elif m == '4':
		getEC2ByValues()
	elif m == '5':
		getEC2BySGs()
	else:
		print '\nuhm, no. let\'s try this again.... '
		queryEC2(e)