def treinar():

    print """>>> Carregando categorias..."""
    CATEGORIAS = os.listdir('./data')
    if '.svn' in CATEGORIAS:
        CATEGORIAS.remove('.svn')


    print ">>> Instanciando treinador\n"    
    guesser = Bayes()
    try:
        for categoria in CATEGORIAS:
            print ">>> Treinando categoria %s" % categoria
            arquivos = os.listdir("%s/%s" % (CAMINHO_CATEGORIAS, categoria))
            if '.svn' in arquivos:
                arquivos.remove('.svn')
            for arquivo in arquivos:
                arquivo = open('%s/%s/%s' % (CAMINHO_CATEGORIAS, categoria, arquivo), 'r')               
                texto = arquivo.read()
                guesser.train(categoria, texto)
        print "\n>>> Salvando base de conhecimento...\n"
        guesser.save("conhecimento.bay")
        print "Voil?!\n"
    except:
        print "N?o foi poss?vel treinar a base"

class Guesser(object):
    
    def __init__(self, project):
        self.project = project
        self.bayes = Bayes()
        self._train()
        self.data = []
        self.best = []
    
    def _train(self):
        for sentence in self.project.classified():
            self.bayes.train(sentence.get_classification(), sentence.sentence)
    
    def guess(self):
        for sentence in self.project.to_classify():
            data = {'sentence_id': sentence.id}
            data['guesses'] = self.bayes.guess(sentence.sentence)
            self.data.append(data)
        return self.data
    
    def best_matches(self):
        if not self.data: return []
        for matches in self.data:
            try:
                matches['guesses'] = sorted(matches['guesses'], key=lambda x:x[1], reverse=True)[0]
            except:
                matches['guesses'] = (None, None)
            match = {}
            match['id'] = matches['sentence_id']
            match['guess'] = matches['guesses'][0]
            match['certainty'] = matches['guesses'][1]
            self.best.append(match)
        return self.best

 def test_untrainedGuess(self):
     """
     The C{guess} method of a L{Bayes} instance with no training data returns
     an empty list.
     """
     bayes = Bayes()
     self.assertEquals(bayes.guess("hello, world"), [])

 def _load_guesser(self):
     if Bayes is None:
         return None
     guesser = Bayes()
     print guesser
     print dir(guesser)
     guesser.load('commands.bays')
     return guesser

def get_bayes(id=GLOBAL):
    if not id in guessers.keys():
        bayes = Bayes(tokenizer=statustok)
        fn = filename(id=id)
        if os.path.exists(fn):
            bayes.load(fn)
        log.debug("Created classifier for '%s' at '%s'" % (id, fn))
        guessers[id] = bayes
    return guessers[id]

 def train(self,bucket,words):
   """
   Nominate a bucket to which the words apply, and train accordingly
   """
   if bucket != "" and words != "":
     try:
       Bayes.train(self,bucket,words)
       Bayes.save(self,self.brain)
     except:
       print "Failed to learn"
   else:
     return None

def check_junk(phrase):
    try:
        from reverend.thomas import Bayes
        g = Bayes()
        g.load("config/kikoo.bot")
        result = g.guess(phrase)
        print result
        if result:
            return int(result[0][0])
        else:
            return -1
    except:
        return -1

 def __init__(self, parent, guesser=None, itemClass=None):
     self.status = StatusBar(parent)
     self.status.pack(side=BOTTOM, fill=X)
     Frame.__init__(self, parent)
     self.pack(side=TOP, fill=BOTH)
     self.itemsPerPage = 20
     self.rows = []
     for i in range(self.itemsPerPage):
         self.rows.append(ItemRow())
     self.items = []
     self.files = []
     self.cursor = 0
     self.dirty = False
     if guesser is None:
         from reverend.thomas import Bayes
         self.guesser = Bayes()
     else:
         self.guesser = guesser
     if itemClass is None:
         self.itemClass = TextItem
     else:
         self.itemClass = itemClass
     for row in self.rows:
         row.summary.set('foo')
     self.initViews()

 def __init__(self, non_spam_train_dir, spam_train_dir):
     self.non_spam_train_dir = non_spam_train_dir
     self.spam_train_dir = spam_train_dir
     self.naive_bayes_classifier = Bayes()
     self.total_num_train_files = 0
     self.total_num_test_files = 0
     self.num_misclass = 0

def get_db(private_path, username):
  path = os.path.join(os.path.join(private_path, username), 'spam.bayes')
  guesser = Bayes()

  # load the spam DB
  try:
    guesser.load(path)
  except IOError:
    print "Creating a new spam filter database"

    parent_directory = os.path.dirname(path)
    if not os.path.isdir(parent_directory):
      os.makedirs(parent_directory)

    guesser.save(path)

  return guesser, path

 def untrained(self, cr, uid, ids, context=None):
     for id in ids:
         record = self.read(cr, uid, id, ['category_id','description'])
         if record['description']:
             group_obj = self.pool.get('crm.bayes.group')
             cat_obj = self.pool.get('crm.bayes.categories')
             cat_rec = cat_obj.read(cr, uid, record['category_id'][0],[])
             guesser = Bayes()
             data = ""
             for rec in group_obj.browse(cr, uid, [cat_rec['group_id'][0]]):
                 if rec['train_data']:
                     data += rec['train_data']
             if data :
                 myfile = file(file_path+"crm_bayes.bay", 'w')
                 myfile.write(data)
                 myfile.close()
                 guesser.load(file_path+"crm_bayes.bay")
             guesser.untrain(cat_rec['name'],record['description'])
             guesser.save(file_path+"crm_bayes.bay")
             myfile = file(file_path+"crm_bayes.bay", 'r')
             data= ""
             for fi in myfile.readlines():
                 data += fi
             group_obj.write(cr, uid, cat_rec['group_id'][0], {'train_data': data})
             cat_obj.write(cr, uid, record['category_id'][0], {'train_messages':int(cat_rec['train_messages']) - 1 })
             cr.execute("select sum(train_messages) as tot_train,sum(guess_messages) as tot_guess from crm_bayes_categories where group_id=%d"% cat_rec['group_id'][0])
             rec = cr.dictfetchall()
             if rec[0]['tot_guess']:
                 percantage = float(rec[0]['tot_guess'] *100)  / float(rec[0]['tot_guess'] + rec[0]['tot_train'])
             else :
                 percantage = 0.0
             group_obj.write(cr, uid, cat_rec['group_id'][0], {'train_data': data,'automate_test':percantage})            
             self.write(cr, uid, id, {'state_bayes':'untrained'})
     return True    

  def __init__(self,name):
    Bayes.__init__(self)

    self.brain = name + '.bay'

    try:
      Bayes.load(self,self.brain)
      print "[Bayes] Brain loaded ok"
    except:
      print "[Alert] Failed to load bayesian brain - %s, creating it now" % self.brain
      Bayes.save(self,self.brain)
      Bayes.load(self,self.brain)

    def getLanguageGuesses(self, stopWords, corpus, languages):
        from reverend.thomas import Bayes

        # charset
        charset = 'us-ascii'

        # instantiate guesser
        guesser = Bayes()

        # go through language in order to train guesser
        for selectLanguage in languages:
            if selectLanguage != 'automatic':
                stopWordString = stopWords.getStopWordString(selectLanguage)
                guesser.train(selectLanguage, stopWordString.encode(charset, 'replace'))
        
        # get list of possible languages
        languageGuesses = guesser.guess(corpus.encode(charset, 'replace'))
        
        return languageGuesses

 def guess_message(self,cr,uid,ids,context={}):
     cases = self.browse(cr, uid, ids)
     result_lang=[]
     if cases.description :
         guesser = Bayes()
         group_obj = self.pool.get('crm.bayes.group')
         data = ""
         for rec in group_obj.browse(cr, uid, group_obj.search(cr,uid,[('active','=',True)])):
             if rec['train_data']:
                 data += rec['train_data']
         if data :
             myfile = file("/tmp/crm_bayes.bay", 'w')
             myfile.write(data)
             myfile.close()
             guesser.load('/tmp/crm_bayes.bay')
             result_lang = guesser.guess(cases.description)
     guess_re = []
     for le in result_lang:
         guess_re.append((le[0],le[1]*100))
     return guess_re

 def action_guess(self, cr, uid, ids, context=None):
     guesser = Bayes()
     group_obj = self.pool.get('crm.bayes.group')
     if result:
         for res in range(0, len(result)):
             result.pop(0)
     data = ""
     for rec in group_obj.browse(cr, uid, context['active_ids']):
         if rec['train_data']:
             data += rec['train_data']
     result_lang=[]
     if data:
         myfile = file("/tmp/crm_bayes.bay", 'w')
         myfile.write(data)
         myfile.close()
         guesser.load('/tmp/crm_bayes.bay')
         message = self.read(cr, uid, ids, ['name'])
         result_lang = guesser.guess(message[0]['name'])
         
     cat_obj = self.pool.get('crm.bayes.categories')
     cat_id = cat_obj.search(cr, uid, [])
     for re in cat_obj.read(cr, uid, cat_id, ['name']):
         flag = False
         for r in result_lang:
             if r[0] == re['name']:
                 result.append(r)
                 flag = True
                 break
         if not flag:
             result.append((re['name'],0))
     context_new = {}
     context_new.update({'from_wiz':True})
     context_new.update({'group_id':context.get('active_id',False)})
     return {
         'context': context_new,
         'view_type': 'form', 
         "view_mode": 'form', 
         'res_model': 'crm.bayes.test.train', 
         'type': 'ir.actions.act_window', 
         'target':'new', 
      }

def retrain(request):
    # Retrain your brain
    user = User.objects.get(user=request.user)
    posts = Post.objects.filter(user=user)
    bayes = Brain.objects.get(user=user)
    brain = Bayes()
    #brain.loads(base64.decodestring(bayes.data))

    tagcount = 0
    # retrain the brain based on existing tags
    for post in posts:
        print post.title, "::",
        for tag in post.tags.all():
            text = "%s %s %s" % (post.title, post.author, post.summary)
            brain.train(tag, text)
            tagcount += 1
            print tag,
        print
    brain.save('%s.db' % user)
    bayes.data = base64.encodestring(brain.saves())
    bayes.save()

    message = 'Found %s tags' % tagcount
    params = {'Messages': [message,]}
    return response(request, 'mainapp/index.html', params)

def classificar():

    print ">>> Instanciando classificador"    
    guesser = Bayes()
    print ">>> Carregando base de conhecimento"    
    try:
        guesser.load(CAMINHO_CONHECIMENTO)
    except IOError: 
        print "Erro. Não foi possível carregar a base. Certifique-se de que existe o arquivo %s." % CAMINHO_CONHECIMENTO
        sys.exit(1)

    try:
        arquivos = os.listdir("%s/" % CAMINHO_TOCLASSIFY)
        if '.svn' in arquivos:
            arquivos.remove('.svn')
        for nome_arquivo in arquivos:
            arquivo = open('%s/%s' % (CAMINHO_TOCLASSIFY, nome_arquivo), 'r')               
            texto = arquivo.read()
            guess = guesser.guess(texto)
            print "\n>>> Arquivo %s:\n %s\n" % (nome_arquivo, guess)
    except:
        print "Erro. Não foi possível classificar."

 def action_train(self, cr, uid, ids, context=None):
     cat_obj = self.pool.get('crm.bayes.categories')
     group_obj = self.pool.get('crm.bayes.group')
     message_obj = self.pool.get('crm.bayes.test.guess')
     
     for id in ids:
         cat_id = self.read(cr, uid, id, ['category_id','name'])         
         cat_id = cat_id[0]['category_id']
         if  result :
             max_list = max(result, key=lambda k: k[1])
             if cat_id:
                 cat_guess_msg = cat_obj.read(cr, uid, cat_id, ['train_messages'])
                 cat_obj.write(cr, uid, cat_id, {'train_messages' :cat_guess_msg['train_messages'] + 1})
             if max_list[1] > 0 and not cat_id:
                 cat_id = cat_obj.search(cr, uid, [('name','=',max_list[0])])[0]
                 cat_guess_msg = cat_obj.read(cr, uid, cat_id, ['guess_messages'])
                 cat_obj.write(cr, uid, cat_id, {'guess_messages' :cat_guess_msg['guess_messages'] + 1})
                 self.write(cr, uid, ids, {'category_id':cat_id})
         if cat_id :
             cat_rec = cat_obj.read(cr, uid, cat_id, [])
             guesser = Bayes()
             data = ""
             for rec in group_obj.browse(cr, uid, [cat_rec['group_id'][0]]):
                 if rec['train_data']:
                     data += rec['train_data']
             if data :
                 myfile = file(file_path+"crm_bayes.bay", 'w')
                 myfile.write(data)
                 myfile.close()
                 guesser.load(file_path+"crm_bayes.bay")
                 
             guesser.train(cat_rec['name'], message_obj.read(cr, uid, id)[0]['name'])
             guesser.save(file_path+"crm_bayes.bay")
             myfile = file(file_path+"crm_bayes.bay", 'r')
             data=""
             for fi in myfile.readlines():
                 data += fi 
             cr.execute("select sum(train_messages) as tot_train,sum(guess_messages) as tot_guess from crm_bayes_categories where group_id=%d"% cat_rec['group_id'][0])
             rec = cr.dictfetchall()
             if not rec[0]['tot_guess']:
                 rec[0]['tot_guess'] =0
             percantage = float(rec[0]['tot_guess'] *100)  / float(rec[0]['tot_guess'] + rec[0]['tot_train'])
             group_obj.write(cr, uid, cat_rec['group_id'][0], {'train_data': data,'automate_test':percantage})            
         else :
             raise osv.except_osv(_('Error !'),_('Please Select Category! '))
     return {
         'view_type': 'form', 
         "view_mode": 'form', 
         'res_model': 'crm.bayes.train.message', 
         'type': 'ir.actions.act_window', 
         'target':'new', 
      }

def main():
    """
    Perform a test run of the FeedFilter using defaults.
    """
    # Create a new Bayes guesser, attempt to load data
    guesser = Bayes()
    guesser.load(BAYES_DATA_FN)
    
    # Open up the databases, load the subscriptions, get new entries.
    feed_db, entry_db = openDBs(FEED_DB_FN, ENTRY_DB_FN)
    feeds   = [ x.strip() for x in open(FEEDS_FN, "r").readlines() ]
    entries = getNewFeedEntries(feeds, feed_db, entry_db)
    
    # Build the feed filter.
    f = BayesFilter(guesser, entries)
    f.FEED_META['feed.title']   = FEED_TITLE
    f.FEED_META['feed.tagline'] = FEED_TAGLINE
    
    # Output the feed as both RSS and Atom.
    open(FEED_NAME_FN % 'rss', 'w').write(f.scrape_rss())
    open(FEED_NAME_FN % 'atom', 'w').write(f.scrape_atom())
    
    # Close the databases and save the current guesser's state to disk.
    closeDBs(feed_db, entry_db)

def mark(request, flag):
    
    id = request.GET.get('post', None)
    feed = request.GET.get('feed', None)
    category = request.GET.get('category') 
    tag = request.GET.get('tag') or None
    
    try:
        if feed:
           posts = Post.objects.filter(feed=feed)
        else:
           posts = Post.objects.filter(id=id)
    except Post.DoesNotExist:
        return HttpResponseRedirect('/')
    
    bayes = Brain.objects.get(user=request.user) #login required
    brain = Bayes()
    brain.loads(base64.decodestring(bayes.data))
    
    if flag in ('read', 'unread'):
        flag = flag == 'read'
        posts.update(read=flag) 
    else:
        for post in posts:
            text = "%s %s %s" % (post.title, post.author, post.summary)
            t1 = Tag.objects.get(id=flag)
            if t1 in post.tags.all() and not feed:
                post.tags.remove(t1) 
                brain.untrain(t1.name, text)
            else:
                post.tags.add(t1)
                brain.train(t1.name, text)
            post.save()    
        
    bayes.data = base64.encodestring(brain.saves())
    bayes.save()
        
    if category:
       return HttpResponseRedirect('/?category=%s' % category)
    elif feed:
       return HttpResponseRedirect('/?feed=%s' % feed)
    elif tag:
       return HttpResponseRedirect('/?tag=%s' % tag)
    else:
       return HttpResponseRedirect('/')