def load_data(self):
        X, Y = [], []
        for file in os.listdir(self.path):
            if file == 'truth.txt' or file == '.DS_Store':
                continue
            print "loading file -->" + file
            tree = ET.parse(os.path.join(self.path, file))
            root = tree.getroot()
            document = Document(language=root.get('lang'), name=root.get('id'))
            for d in root.findall('document'):
                document.add_document(d.text)
            user, gender, age_group, extroverted, stable, agreeable, conscientious, open = self.truth[
                root.get('id')].split(":::")

            traits = PersonalityTraits(extroverted=float(extroverted), stable=float(stable), agreeable=float(agreeable),
                                       conscientious=float(conscientious), open=float(open))
            usr = Author(gender=gender, age_group=age_group, traits=traits)
            document.author = usr
            X.append(document)
            Y.append(self.truth[root.get('id')])
        print "done loading files"

        self.X = X
        self.Y = Y
        return self

  def store_digest(self, digest):
    docproof = Document.get_doc(digest)
    if docproof:
      return {"success" : False, "reason": "existing", "digest": digest, "args": [export_timestamp(docproof.timestamp)]}

    d = Document.new(digest)
    self.doc = d
    return {"success": True, "digest": d.digest}

  def get(self):
    # Provide login/logout URLs.
    user_info = get_user_info()
    if user_info is None:
      login_url = users.create_login_url('/')
    else:
      login_url = users.create_logout_url('/')

    # Collect list of error messages which gets shown to the user.
    error_messages = self.request.params.getall('error_message')
    view_user = user_info  # for now
    did_search = False

    # Fetch media for view user.
    media = MediaObject.all().filter('owner', user_info)
    media = media.filter('lacks_document', True)
    media = media.order('creation')
    limit = 50
    if self.request.get("limit"):
      limit = long(self.request.get("limit"))
    media = media.fetch(limit)

    docs = Document.all().filter('owner', user_info)
    tags = self.request.get("tags")
    if tags:
      did_search = True
      for tag in re.split('\s*,\s*', tags):
        docs = docs.filter("tags", tag)
    docs = docs.fetch(limit)

    untagged_docs = Document.all().filter('owner', user_info).filter("no_tags", True).fetch(limit)

    upcoming_due = Document.all().filter('owner', user_info)
    upcoming_due = upcoming_due.filter("due_date !=", None)
    upcoming_due = upcoming_due.order("due_date")
    upcoming_due = upcoming_due.fetch(30)

    top_message = ""
    if self.request.get("saved_doc"):
      docid = long(self.request.get("saved_doc"))
      top_message = "Saved <a href='/doc/%d'>doc %d</a>" % (docid, docid)

    # Render view.
    self.response.out.write(template.render('main.html', {
        "did_search": did_search,
        "media": media,
        "docs": docs,
        "untagged_docs": untagged_docs,
        "upcoming_due_docs": upcoming_due,
        "view_user": view_user,
        "login_url": login_url,
        "user_info": user_info,
        "top_message": top_message,
        }, debug=True))

 def handle(self):
     n = 0
     for old in DocumentProof.all():
         doc = Document.get_doc(old.digest)
         if not doc:
             doc = Document.new(old.digest)
             doc.pending = old.tx == None
             doc.tx = old.tx
             doc.timestamp = old.timestamp
             doc.blockstamp = old.blockstamp
             n += 1
             if n == 10:
                 break 
             
     return {"success" : True, "processed": n}

  def get(self):
    actionable = Document.get_actionable()
    for d in actionable:
      ret = d.blockchain_certify()

      subject = "Document certified: %s %s" % (ret['success'], d.digest)
      body = subject + "\n\nmesage: %s" % (ret['message'])

 def get(self, docid):
     user_info = get_user_info()
     if user_info is None:
         self.redirect("/?error_message=%s" % "login required to view docs")
     docid = long(docid)
     doc = Document.get_by_id(docid, parent=user_info)
     if doc is None:
         self.response.out.write("Docid %d not found." % (docid))
         return
     pages = MediaObject.get(doc.pages)
     size = self.request.get("size")
     if not size:
         size = 1200
     show_single_list = long(size) > 600
     self.response.out.write(
         template.render(
             "doc.html",
             {
                 "doc": doc,
                 "pages": pages,
                 "user_info": user_info,
                 "size": size,
                 "show_single_list": show_single_list,
             },
             debug=True,
         )
     )

    def run(self):
        result = {}
        x, y, y_actual = [], [], []
        for file in os.listdir(self.path):
            if file == 'truth.txt' or file == '.DS_Store':
                continue
            tree = ET.parse(os.path.join(self.path, file))
            root = tree.getroot()
            document = Document(language=root.get('lang'), name=root.get('id'))

            for d in root.findall('document'):
                document.add_document(d.text)
            x_test = [document]  # vector

            temp_result = {}
            for predictor in self.model:
                # print predictor

                if predictor.name == 'age_gender':
                    prediction = predictor.clf.predict(x_test)  # predict
                    temp_result.update(
                        predictor.label_extractor(list(predictor.label_encoder.inverse_transform(prediction))[0]))
                    document.author.gender = temp_result['gender']
                    document.author.age_group = temp_result['age_group']
                if predictor.name == 'personality':
                    target = predictor.label_encoder.classes_
                    prediction = list(predictor.clf.predict_proba(x_test))[0]
                    prediction = [change_range(p, 1.0, 0.0, 0.5, -0.5) for p in prediction]
                    temp_result.update(predictor.label_extractor(target, prediction))



            document.author.personality_traits.extroverted = temp_result['extroverted']
            document.author.personality_traits.agreeable = temp_result['agreeable']
            document.author.personality_traits.conscientious = temp_result['conscientious']
            document.author.personality_traits.stable = temp_result['stable']
            document.author.personality_traits.open = temp_result['open']

            result[os.path.splitext(file)[0]] = document
            # y.extend(prediction)
            # print y
            x.append(os.path.splitext(file)[0])
            # y_actual.append(predictor.label_extractor(self.truth[root.get('id')]))
        self.x_test = x_test
        # self.y_prediction = y
        # self.y_actual = self.label_encoder.transform(y_actual)
        self.result = result

 def get(self):
   actionable = Document.get_actionable()
   for d in actionable:
     ret = d.blockchain_certify()
     sender_address = "[email protected]"
     subject = "Document certified: %s %s" % (ret['success'], d.digest)
     body = subject + "\n\nmesage: %s" % (ret['message'])
     mail.send_mail(sender_address, ADMIN_EMAIL, subject, body)

        def store_media():
            """Store media object info in datastore.

      Also updates the user-info record to keep count of media objects.

      This function is run as a transaction.
      """
            user_info = UserInfo.get_by_key_name("user:%s" % user_email)
            if user_info is None:
                error_messages.append("User record has been deleted.  " "Try uploading again")
                return

            media = MediaObject(
                parent=user_info,
                owner=user_info,
                blob=blob_info.key(),
                creation=blob_info.creation,
                content_type=blob_info.content_type,
                filename=blob_info.filename,
                size=int(blob_info.size),
                lacks_document=True,
            )

            user_info.media_objects += 1
            db.put(user_info)
            db.put(media)

            if bool(is_doc) and is_doc != "0":
                tag_list = []
                if tags is not None:
                    tag_list = [x for x in re.split("\s*,\s*", tags) if x]

                doc = Document(
                    parent=user_info,
                    owner=user_info,
                    pages=[media.key()],
                    title=title,
                    description=description,
                    no_tags=(len(tag_list) == 0),
                    tags=tag_list,
                )
                db.put(doc)
                media.document = doc.key()
                media.lacks_document = False
                db.put(media)

    def process_payment(self, satoshis, digest):
        secret = self.request.get("secret")
        if len(digest) != 64 or secret != CALLBACK_SECRET or satoshis < MIN_SATOSHIS_PAYMENT:
            return {"success" : False, "reason" : "format or payment below " + str(MIN_SATOSHIS_PAYMENT)}

        doc = Document.get_doc(digest)
        if not doc:
            return {"success" : False, "reason" : "Couldn't find document"}
        doc.pending = False
        doc.put()

        return {"success" : True}

 def handle(self):
   digest = self.request.get("d")
   doc = Document.get_doc(digest)
   if not doc or doc.tx:
     return {"success" : False, "error": "format"}
   # TODO: add check to prevent double timestamping
   txid, message = publish_data(doc.digest.decode('hex'))
   if txid:
     doc.tx = txid
     doc.txstamp = datetime.datetime.now()
     LatestBlockchainDocuments.get_inst().add_document(digest)
     doc.put()
   return {"success" : txid is not None, "tx" : txid, "message" : message}

  def handle(self):
    digest = self.request.get("d")
    doc = Document.get_doc(digest)
    if not digest:
      return {"success": False, "reason": "format"}
    if not doc:
      return {"success": False, "reason": "nonexistent"}
    if doc.tx:
      return {"success": True, "status": "confirmed", "transaction": doc.tx}
    if doc.is_actionable():
      return {"success": True, "status": "pending"}

    return {"success": True, "status": "registered"}

  def handle(self):
    digest = self.request.get("d")
    doc = Document.get_doc(digest)
    if not doc or not doc.payment_address:
      return {"success" : False, "error": "format"}
    
    txs = get_txs_for_addr(doc.payment_address)
    if not txs or len(txs) == 0:
      return {"success" : False, "error": "no transactions"}
    tx_hash, tx_timestamp = txs[0]
    doc.confirmed(tx_hash, tx_timestamp)

    LatestBlockchainDocuments.get_inst().add_document(doc)
    return {"success" : True, "tx" : doc.tx}

    def post(self):
        user_info = get_user_info()
        if user_info is None:
            self.redirect("/?error_message=%s" % "login required to view docs")
        docid = long(self.request.get("docid"))
        doc = Document.get_by_id(docid, parent=user_info)
        if doc is None:
            self.response.out.write("Docid %d not found." % (docid))
            return

        mode = self.request.get("mode")
        if mode == "break":
            break_and_delete_doc(user_info, doc)
            self.response.out.write(
                "[&lt;&lt; <a href='/'>Back</a>] Docid %d deleted and images broken out as un-annotated." % docid
            )
            return
        if mode == "delete":
            delete_doc_and_images(user_info, doc)
            self.response.out.write("[&lt;&lt; <a href='/'>Back</a>] Docid %d and its images deleted." % docid)
            return

        # Simple properties:
        doc.physical_location = self.request.get("physical_location")
        doc.title = self.request.get("title")

        # Tags
        doc.tags = [x for x in re.split("\s*,\s*", self.request.get("tags")) if x]
        doc.no_tags = len(doc.tags) == 0

        # Document Date
        date = self.request.get("date")
        if date:
            doc.doc_date = datetime.datetime.strptime(date, "%Y-%m-%d")
            doc.no_date = False
        else:
            doc.doc_date = None
            doc.no_date = True

        # Due date
        due_date_str = self.request.get("due_date")
        doc.due_date = None
        if due_date_str:
            doc.due_date = datetime.datetime.strptime(due_date_str, "%Y-%m-%d")

        def store():
            db.put(doc)

        db.run_in_transaction(store)
        self.redirect("/?saved_doc=" + str(docid))

    def handle(self):
        digest = self.request.get("d")
        doc = Document.get_doc(digest)
        if not digest:
            return {"success": False, "reason": "format"}
        if not doc:
            return {"success": False, "reason": "nonexistent"}
        if doc.tx:
            return {"success": True, "status": "confirmed", "transaction": doc.tx, "txstamp": doc.txstamp}
        if doc.is_actionable():
            return {"success": True, "status": "pending"}

        return {
            "success": True,
            "status": "registered",
            "pay_address": doc.payment_address,
            "price": MIN_SATOSHIS_PAYMENT,
        }

  def get(self):
    self.response.headers['Cache-Control'] = "private"
    self.response.headers['Content-Type'] = "text/plain; charset=utf-8"

    user = UserInfo.get_by_key_name('user:[email protected]')

    docs = Document.all().filter('owner', user)
    docs = docs.fetch(10000)
    self.response.out.write("# got %d docs\n" % len(docs))
    for doc in docs:
      self.response.out.write("%s tags[%s] date[%s] title[%s] \n" % (doc.display_url, doc.tag_comma_separated, doc.date_yyyy_mm_dd, doc.title_or_empty_string))
      for page in doc.pages:
        self.response.out.write(" has_page: %d\n" % (page.id_or_name()))
    meds = MediaObject.all().filter('owner', user)
    meds = meds.fetch(10000)
    self.response.out.write("# got %d mediaobjects\n" % len(meds))
    for mo in meds:
      self.response.out.write("%s creation[%s] size[%d]\n" % (mo.url_path, str(mo.creation), mo.size))

class DocumentTestCase(unittest.TestCase):
    def setUp(self):
        self.doc = Document()

    def test_insert(self):
        self.doc.insert(0, "here")
        self.assertEqual(self.doc.visible_text, "here")

        self.doc.insert(0, "again ")
        self.assertEqual(self.doc.visible_text, "again here")

        self.doc.insert(8, "1")
        self.assertEqual(self.doc.visible_text, "again he1re")

        self.doc.insert(8, "23")
        self.assertEqual(self.doc.visible_text, "again he231re")

        self.doc.insert(len(self.doc.visible_text), "ing")
        self.assertEqual(self.doc.visible_text, "again he231reing")

    def test_remove(self):
        self.doc.insert(0, "here is some stuff\nover a few lines")
        self.assertEqual(self.doc.visible_text, "here is some stuff\nover a few lines")

        self.doc.remove(0, 5)
        self.assertEqual(self.doc.visible_text, "is some stuff\nover a few lines")

        self.doc.remove(1, 1)
        self.assertEqual(self.doc.visible_text, "i some stuff\nover a few lines")

        self.doc.remove(len(self.doc.visible_text) - 4, 4)
        self.assertEqual(self.doc.visible_text, "i some stuff\nover a few l")

    def test_search(self):
        self.doc.insert(0, "hello there\nthis is a test\nof search")
        self.doc.search("hello")
        self.assertEqual(self.doc.current_search, "hello")

        self.assertEqual(self.doc.visible_text, "hello there")
        self.doc.search("o")
        self.assertEqual(self.doc.visible_text, "hello there\nof search")
        self.doc.search("th")
        self.assertEqual(self.doc.visible_text, "hello there\nthis is a test")
        self.doc.search("")
        self.assertEqual(self.doc.visible_text, "hello there\nthis is a test\nof search")

    def test_search_multiple_words(self):
        self.doc.insert(0, "hello there\nthis is a test\nof search")
        self.doc.search("this test")
        self.assertEqual(self.doc.current_search, "this test")

        self.assertEqual(self.doc.visible_text, "this is a test")

    def test_search_and_insert(self):
        self.doc.search("test")
        self.assertEqual(self.doc.visible_text, "")

        self.doc.search("")
        self.doc.insert(0, "hello there\nthis is a test\nof search")

        self.doc.search("test")
        self.assertEqual(self.doc.visible_text, "this is a test")
        self.doc.insert(0, "again ")
        self.assertEqual(self.doc.visible_text, "again this is a test")
        self.assertEqual(self.doc.text, "hello there\nagain this is a test\nof search")

        self.doc.search("")
        self.assertEqual(self.doc.visible_text, "hello there\nagain this is a test\nof search")

        self.doc.search("search")
        self.assertEqual(self.doc.visible_text, "of search")
        self.doc.insert(0, "hello ")
        self.assertEqual(self.doc.visible_text, "hello of search")

        self.doc.search("hello")
        self.assertEqual(self.doc.visible_text, "hello there\nhello of search")
        self.doc.insert(11, "go")
        self.assertEqual(self.doc.visible_text, "hello therego\nhello of search")
        self.assertEqual(self.doc.text, "hello therego\nagain this is a test\nhello of search")
        self.doc.insert(15, "23")
        self.assertEqual(self.doc.visible_text, "hello therego\nh23ello of search")
        self.assertEqual(self.doc.text, "hello therego\nagain this is a test\nh23ello of search")

    def test_search_and_remove(self):
        self.doc.search("")
        self.doc.insert(0, "hello there\nthis is a test\nof search\nhello there")

        self.doc.search("hello")
        self.assertEqual(self.doc.visible_text, "hello there\nhello there")

        self.doc.remove(0, 3)
        self.assertEqual(self.doc.visible_text, "lo there\nhello there")

        self.doc.remove(len(self.doc.visible_text) - 3, 3)
        self.assertEqual(self.doc.visible_text, "lo there\nhello th")

        # now remove accross multiple visible regions
        self.doc.remove(7, 3)
        self.assertEqual(self.doc.visible_text, "lo therello th")

#.........这里部分代码省略.........

 def setUp(self):
     self.doc = Document()

 def get(self):
   actionable = Document.get_actionable()
   url = SECRET_ADMIN_PATH + '/autopay'
   for d in actionable:
     self.response.write('<a href="%s?d=%s">%s</a><br /><br />' % (url, d.digest, d.digest))

 def handle(self):
   digest = self.request.get("d")
   doc = Document.get_doc(digest)
   if not doc:
     return {"success" : False, "error": "format"}
   return doc.blockchain_certify()