def report_worker(sid):
    try:
        job = get_job(sid) 

        log.info("=============  STARTING WORKER  ==============")
        log.debug(job)
        from ast import literal_eval
        job['series'] = literal_eval(job['series'])  # From string
        # Expand paths to full location on filesystem 
        output_filename = os.path.join(
            app.config['UPLOAD_FOLDER'], 
            next(tempfile._get_candidate_names()) + '.pdf')

        # Make list of input datafiles
        input_datafiles = [
            os.path.join(app.config['UPLOAD_FOLDER'], f['temporary_name'])
            for f in get_files(sid)
        ]

        report.report(input_datafiles, output_filename, 
                      **{**job, 'pdf': True, 'htm': False})

        log.info("=============  WORKER FINISHED  ==============")

        # Update finished job 
        upd_job(sid, 'generated_pdf', output_filename)
        upd_job(sid, 'status', 'done')

    except Exception as e:
        log.error("Exception occurred in worker thread")
        log.error(sys.exc_info()[0])

        upd_job(sid, 'status', 'error')
        upd_job(sid, 'generated_pdf', None)
        raise e

def send(trans):
    """Sends an email to the list."""
    me = '[email protected]'
    you = '[email protected]'
    today = datetime.date.today()

    # Create message container - the correct MIME type is multipart/alternative.
    msg = MIMEMultipart('alternative')
    msg['Subject'] = 'Magic Cones Report {0}'.format(today)
    msg['From'] = me
    msg['To'] = you

    # create content
    text = TEXT.format(report=report(trans), rules=rules())
    html = HTML.format(report=report(trans, html=True), rules=rules(html=True))
    part1 = MIMEText(text, 'plain')
    part2 = MIMEText(html, 'html')
    msg.attach(part1)
    msg.attach(part2)

    # Send the message
    pw = passwd(me)
    s = smtplib.SMTP('smtp.gmail.com', 587)
    #s.ehlo()
    s.starttls()
    #s.ehlo()
    s.login(me, pw)
    s.sendmail(me, you, msg.as_string())
    s.quit()

 def delete_all(self,really=False):
     if not really:
         report('seriously?  well ok, but pass `really=True`')
     else:
         for x in self:
             report('deleting: '+str(x))
             del self[x]

 def mapData(self):
   # pracujeme s: http://sigma.nkp.cz/F/?func=file&file_name=find-b&local_base=ADR
   report("INFO: SiglaMapper.mapData")
   siglaURI = False
   sigla = self.doc.getXPath("//varfield[@id='040']/subfield[@label='a']")
   if not sigla == []:
     sigla = sigla[0]
     # Check whether the sigla is already cached.
     if sigla in cache.siglaCache.keys():
       siglaURI = cache.siglaCache[sigla]
     else:
       doc = self.searchAlephBase("http://sigma.nkp.cz", "ADR", "SIG", sigla)
       if doc:
         xpath = "present/record/doc_number"
         docNum = doc.getXPath(xpath)
         if not docNum == []:
           docNum = docNum[0].lstrip("0")
           siglaURI = "http://sigma.nkp.cz/X?op=doc-num&base=ADR&doc-num=" + docNum
           cache.siglaCache[sigla] = siglaURI
   if siglaURI:
     return [(
       self.representationURI,
       rdflibWrapper.namespaces["dc"]["creator"], 
       rdflib.URIRef(siglaURI)
     )]
   else:
     return False

def report_test():
    os.makedirs(TEST_PATH, exist_ok=True)
    print_memory('pre')

    for i, line in enumerate(DATAFILES):

        objgraph.show_growth(limit=3)

        input_files, location = line[0], line[1]
        report_name = TEST_PATH + '/test_' + str(i) + '.pdf'

        log.info(input_files)
        log.info(location)
        log.info(report_name)

        try:
            report.report(input_files, report_name, location=location)

            log.info("Report saved to {}".format(report_name))

        except Exception as e:
            log.exception("Exception occurred when reporting (test failed):")

        print_memory(i)

    plot_object_counts()

 def delete_file(self, key=None, item=None):
     report('deleting file')
     if key and item:
         self.record['errors'] += 1
         self.record['error'] = 'cant pass key and item to delete_file.'
         return
     if not (key or item):
         self.record['errors'] += 1
         self.record['error'] = 'need either key or item'
         return
     if not key:
         if item is None:
             self.record['errors'] += 1
             self.record['error'] = 'item is none'
             return
         key = item and item.id
     if not key:
         self.record['errors'] += 1
         self.record['error'] = 'item is none'
     report('deleting file',key)
     if not ope(key):
         self.record['errors'] += 1
         self.record['error']='file does not exist.'
     os.remove(key) # TODO: use unipath
     self.record['files_deleted'] += 1
     self.delete_record(key)

    def callback(self, item=None, **kargs):
        report(item._id)

        if item._id not in self.collisions['fname']:
            self.seek_fname_collision(item)
        if item._id not in self.collisions['md5']:
            self.seek_md5_collision(item)

def update_db(db, _id, dct, schema=None):
    """  stupid.. have to delete and restore instead of update? """

    if not schema:
        report('SCHEMA NOT PROVIDED!!!!!!!')
        report('updating db',[db, _id, dct])
        doc = db[_id]
        report('before',doc.items())

        for x in dct:
            doc[x] = dct[x]

        # TODO: use db.update(doc) ?
        #db[doc.id] = doc

        report('after', doc)
        report('updated "{id}" with new values for keys'.format(id=_id), dct.keys())
    else:
        doc = schema.load(db, _id)
        for x in dct:
            val = dct[x]
            fieldtype = getattr(schema, x).__class__
            if fieldtype==ListField:
                val = demjson.decode(val)
            elif fieldtype==StringField:
                pass
            else:
                raise Exception, 'NIY:'+str(fieldtype)
            setattr(doc, x, val)
        doc.store(db)

 def __init__(self, path=None, settings=None,
              items=[], fill=None,
              force=False, wrap_exit=True,
              **kargs):
     """ fill+path determine self.query """
     if wrap_exit:
         self.__call__ = wrap_kbi(self.__call__)
     settings._engine.get_server() # HACK: ensure we're initialized
     self.record = defaultdict(lambda: 0)
     if self.requires_path:
        if not path or not ope(path):
            raise FileDoesntExist(str(path))
     self.path = path and abspath(path)
     self.conf = settings
     self.force = force
     if fill:
         if path is not None:
             raise SystemExit('if you use --fill you cant '
                              'use a path (and vice versa)')
     self.fill = fill
     if items:
         report('instantiated {0} with size {1} item-list'.format(
             self,len(items)))
         self.__iter__ = lambda himself: ([i.id, i] for i in items)
     self.record_invocation()

 def __iter__(self):
     t1 = now()
     report('starting query: ')
     q = self.query
     t2 = now()
     report('finished query ({0}s)'.format(t2-t1))
     return q

 def validateURI(self, uri):
   """Zjišťuje, zdali je zadané URI dostupné."""
   report("INFO: validating URI %s" % (uri))
   try:
     urllib2.urlopen(urllib2.Request(uri, None, {"Accept" : "application/rdf+xml"})) # Validation on marccodes.heroku.com doesn't work without the Accept HTTP header
     return True
   except urllib2.HTTPError:
     return False

 def getParsedDoc(self, url):
   """Na zadané URL nebo urllib2.Request vrátí naparsovaný XML dokument.""" 
   report("INFO: Mapper.getParsedDoc on URL %s" % (url))
   result = urllib2.urlopen(url)
   doc = result.read()
   result.close()
   doc = Record(libxml2.parseDoc(doc))
   return doc

 def record_collision(self, reason, item_ids, item=None):
     self.collisions[reason] += item_ids
     item_ids = sorted(item_ids)
     event = Event(reason=reason, item_ids=item_ids,
                   details=dict(md5=item.md5))
     event.store(self.events_db)
     report(' - by {0}: found {1} events'.format(
         reason, len(item_ids)))

def show(project):
    build(project)
    proot = opj(src_root, project)
    report("serving "+proot)
    def f():
        webbrowser.open(URL)
    threading.Thread(target=f).start()
    local("cd {0} && poole --serve".format(proot))

 def run_and_collect(self, cmd):
     """ for gathering the output from file(1) and md5(1) etc """
     cmd = cmd.replace('`','\`')
     try:
         return os.popen(cmd).read().strip()
     except IOError,e:
         report("IOError: " + str(e))
         return None

def get_or_create(DB_NAME):
    from ixle.settings import Settings
    server = Settings().server
    if DB_NAME not in server:
        report("creating database {0} on {1} ".format(DB_NAME, server))
        server.create(DB_NAME)
    db = server[DB_NAME]
    return db

def call_agent_on_dir(agent_nick, dirname):
    kls = get_agent_by_name(agent_nick)
    agent_obj = kls(path=dirname, settings=conf())
    result = agent_obj()
    if result is None:
        report('WARNING: got None-result from agent, '
               'should have been self.record.')
        result = {}
    return agent_obj, result

 def seek_md5_collision(self, item):
     if not item.md5:
         report(' - md5 not set, calling subagent');
         self.md5er.callback(item)
     reason = 'md5'
     results = self.find_matches(item, 'md5')
     if not len(results): return
     item_ids = [row.value['_id'] for row in results] + [item._id]
     self.record_collision(reason, item_ids, item)

 def collapse(self):
     assert self.unipath.isdir()
     parent = self.unipath.parent
     for path in self.unipath.listdir():
         subitem = Item.objects.get(path=path)
         fname = subitem.fname
         new_path = opj(parent, fname)
         report("moving: ", path, new_path)
         subitem._move(new_path)

 def main(self):
     abspath = self['_']
     if abspath and ope(abspath):
         return '<font style="color:green;">exists ✓</font>'
     msg = '(this file is not available.  is the drive mounted?)'
     report(msg.replace('this file','"{0}"'.format(abspath)))
     return ''.join([
         '<small><font style="color:red;margin-left:15px;">',
         "<img src=/static/img/icons/red-x.png width=15px>"+msg,
         '</font></small>'])