def _calc_score(self, lt_new):
     try:
         import ssdeep
     except ImportError:
         raise ImportError(
                 "ltgroup algorithm <ssdeep> needs python package ssdeep")
     ret = []
     h1 = ssdeep.hash(str(lt_new))
     if self._mem_hash:
         if len(self._d_hash) == 0:
             # initialize d_hash
             for lt in self._lttable:
                 h = ssdeep.hash(str(lt))
                 self._d_hash[lt.ltid] = h
         for ltid, lt_temp in enumerate(self._lttable):
             h2 = self._d_hash[lt_temp.ltid]
             score = ssdeep.compare(h1, h2)
             ret.append((ltid, score))
         self._d_hash[lt_new.ltid] = h1
     else:
         for lt_temp in self._lttable:
             ltid = lt_temp.ltid
             score = hash_score(str(lt_new), str(lt_temp))
             ret.append((ltid, score))
     return ret

    def testComputeHash(self):
        self.assertEqual(
            ssdeep.hash("Also called fuzzy hashes, Ctph can match inputs that have homologies."),
            "3:AXGBicFlgVNhBGcL6wCrFQEv:AXGHsNhxLsr2C"
        )

        self.assertEqual(
            ssdeep.hash("Also called fuzzy hashes, CTPH can match inputs that have homologies."),
            "3:AXGBicFlIHBGcL6wCrFQEv:AXGH6xLsr2C"
        )

    def getFileProperties(self, filename,fc):
        self.logger.info("Getting file ID")
        fp={'filename':filename}
        try:
            #File size
            fp['size']=len(fc)
            #MD5
            m=hashlib.md5()
            m.update(fc)
            fp['md5']=m.hexdigest()
            #SHA1
            m=hashlib.sha1()
            m.update(fc)
            fp['sha1']=m.hexdigest()
            #SHA256
            m=hashlib.sha256()
            m.update(fc)
            fp['sha256']=m.hexdigest()
            #SSDEEP
            fp['ssdeep']=ssdeep.hash(fc)
            #Magic
            fp['magic']=magic.from_buffer(fc)
            #Exiftool
            #NOTE: exiftool shits itself on certian formats, wipe it's ass someday
            fp['filetype']=self.et.get_tag('FileType',self.incoming+filename)
            #Tag
            if fp['magic'] is not '':
                fp['tags']=[fp['magic'].split()[0].lower()]
            else:
                fp['tags']=[]

        except IOError as e:
            self.logger.error("IO Error", exc_info=True)

        return {'_id':fp['sha1'],'id':fp}

def getPerFunctionHash():
	"""
		Iterates on program function and, for each, computes
	 	- MD5 sum
	 	- SSDEEP
	"""
	functions = Functions()
	hashes = {}
	for function in functions:
		funcCode = getFunctionCode(function)
		funcCode = cleanUpCode(function, funcCode)
		ssdeepstr = ssdeep.hash(funcCode)
		md5str = md5.new(funcCode).hexdigest()
		#lsh = LSHash(512, len(funcCode))
		#lsh.index(funcCode)
		# TODO ADD OTHER TYPE OF HASHES
		hashes[function] = {
			"md5" : md5str,
			"ssdeep" : ssdeepstr,
		}
		if debug:
			print "sub_%08x %s %s" % (function, md5str, ssdeepstr) # DEBUG

	if dump: # save hash table in dump mode
		fd = open("./%s/%s.pickle" % (dumpdir, "hashes"), "w")
		pickle.dump(hashes, fd)
		fd.close()
	return hashes

    def build_sample(self, data, url=None):
        if not data:
            return None

        p = dict()
        p["type"] = self.get_sample_type(data)
        if p["type"] is None:
            return None

        p["md5"] = hashlib.md5(data).hexdigest()
        p["sha1"] = hashlib.sha1(data).hexdigest()

        if SSDEEP:
            p["ssdeep"] = ssdeep.hash(data)

        if p["type"] in ("PE",):
            imphash = self.get_imphash(data)
            if imphash:
                p["imphash"] = imphash

        if url:
            p["url"] = url

        p["data"] = base64.b64encode(data)

        return p

    def build_sample(self, data, url = None, sampletype = None):
        if not data:
            return None

        p = dict()

        if sampletype:
            p['type'] = sampletype
        else:
            p['type'] = self.get_sample_type(data)

        if p['type'] is None:
            return None

        p['md5']    = hashlib.md5(data).hexdigest()
        p['sha1']   = hashlib.sha1(data).hexdigest()
        p['sha256'] = hashlib.sha256(data).hexdigest()

        if SSDEEP:
            p['ssdeep'] = ssdeep.hash(data)

        if p['type'] in ('PE', ):
            imphash = self.get_imphash(data)
            if imphash:
                p['imphash'] = imphash

        if url:
            p['url'] = url

        p['data'] = base64.b64encode(data)

        return p

def compute_ssdeep(fp):
    try:
        buff = fp.read()
    except AttributeError:
        pass
    else:
        return ssdeep.hash(buff)

def getfuzzyhash():
    """Returns fuzzy hash of spam.
    This function returns hash generated using the ssdeep library.
    Hash is generated using the combination of mail's body + subject.
    Msg length is being checked because SSDEEP has some issues with comparing hashes
    of small spams. If spam's body is very less or non existent, we add our randomText to body.
    There would be certain cases when there wouldn't be any html or text portion i.e. email body would be empty. Hence forth len = html/text + subject
    In shivamaindb.py if len < 10 then keeping comparision ratio higher
    """
    if mailFields['html']:
        if len(mailFields['html']) < 150:
            data = mailFields['html'] + " " + mailFields['subject'] + randomText
        else:
            data = mailFields['html'] + " " + mailFields['subject']
        mailFields['len'] = len(mailFields['html']) + len(mailFields['subject'])
    
    elif mailFields['text']:
        if len(mailFields['text']) < 150:
            data = mailFields['text'] + " " + mailFields['subject'] + randomText
        else:
            data = mailFields['text'] + " " + mailFields['subject']
        mailFields['len'] = len(mailFields['text']) + len(mailFields['subject'])
    else:
        # Test mails without body and limited chars in subject
        data = mailFields['subject'] + mailFields['from'] + randomText
        mailFields['len'] = len(mailFields['subject'])
    
    return ssdeep.hash(data)

    def compute_hashes(self):
        """ Compute the file hashes """
        filename = self.get_file_path(self.sha256)

        # Make sure the file exists and is readable
        if not os.access(filename, os.R_OK):
            flash('There was an error while trying to analyse the file.', 'danger')
            return False

        with open(filename, 'rb') as f:
            buf = f.read()

        if self.sha256 is None:
            self.sha256 = hashlib.sha256(buf).hexdigest()
        if self.sha1 is None:
            self.sha1 = hashlib.sha1(buf).hexdigest()
        if self.md5 is None:
            self.md5 = hashlib.md5(buf).hexdigest()
        if self.ssdeep is None:
            self.ssdeep = ssdeep.hash(buf)
        if self.mime is None:
            try:
                self.mime = magic.from_buffer(buf, mime=True).decode('utf-8')
            except:
                self.mime = None
        if self.entropy is None:
            self.entropy = self.compute_entropy(buf)

def hash_data(file):
    md5 = hashlib.md5(file.content).hexdigest()
    sha128 = hashlib.sha1(file.content).hexdigest()
    sha256 = hashlib.sha256(file.content).hexdigest()
    sha512 = hashlib.sha512(file.content).hexdigest()
    ssdeep_hash = ssdeep.hash(file.content)
    hash = {'evil_finder_md5':md5, 'evil_finder_sha128':sha128, 'evil_finder_sha256':sha256, 'evil_finder_sha512':sha512, 'evil_finder_ssdeep':ssdeep_hash}
    return hash

 def execute(self, input_data):
     raw_bytes = input_data['sample']['raw_bytes']
     self.meta['sha1'] = hashlib.sha1(raw_bytes).hexdigest()
     self.meta['sha256'] = hashlib.sha256(raw_bytes).hexdigest()
     self.meta['ssdeep'] = ssd.hash(raw_bytes)
     self.meta['entropy'] = self._entropy(raw_bytes)
     self.meta.update(input_data['meta'])
     return self.meta

def main():
    """Entry function."""
    parser = argparse.ArgumentParser(
        description='Process Fuzzy hashing comparison between project url and \
        return project urls')

    parser.add_argument("input_csv_file", help="Specify the csv file to read")
    parser.add_argument(
        "number_urls", help="Number of found urls to process fuzzy hashing \
        (max = 10)")

    args = parser.parse_args()

    input_csv_file = args.input_csv_file
    number_urls = int(args.number_urls)

    header_names = ['acronym', 'title', 'projectUrl', 'foundProjectUrl1',
                    'foundProjectUrl2', 'foundProjectUrl3',
                    'foundProjectUrl4', 'foundProjectUrl5',
                    'foundProjectUrl6', 'foundProjectUrl7',
                    'foundProjectUrl8', 'foundProjectUrl9',
                    'foundProjectUrl10']
    df = pd.read_csv(input_csv_file, sep=',', quotechar='"',
                     names=header_names, index_col=False)

    df['projectUrlHash'] = np.nan
    df['foundProjectUrl1Hash'] = np.nan
    df['foundProjectUrl2Hash'] = np.nan
    df['foundProjectUrl3Hash'] = np.nan
    df['foundProjectUrl4Hash'] = np.nan
    df['foundProjectUrl5Hash'] = np.nan
    df['foundProjectUrl6Hash'] = np.nan
    df['foundProjectUrl7Hash'] = np.nan
    df['foundProjectUrl8Hash'] = np.nan
    df['foundProjectUrl9Hash'] = np.nan
    df['foundProjectUrl10Hash'] = np.nan
    df['MatchScore1'] = np.nan
    df['MatchScore2'] = np.nan
    df['MatchScore3'] = np.nan
    df['MatchScore4'] = np.nan
    df['MatchScore5'] = np.nan
    df['MatchScore6'] = np.nan
    df['MatchScore7'] = np.nan
    df['MatchScore8'] = np.nan
    df['MatchScore9'] = np.nan
    df['MatchScore10'] = np.nan

    for index, row in df.iterrows():
        print "computing fuzzy hash for project %s" % row['acronym']
        try:
            df.ix[index, 'projectUrlHash'] = ssdeep.hash(
                urllib2.urlopen(row['projectUrl'], timeout=10).read())
        except urllib2.HTTPError, e:
            print e.code
        except urllib2.URLError, e:
            print e.reason

    def _run(self, scanObject, result, depth, args):
        '''
        Assumes:
            there is a string like object in scanObject.buffer
        Ensures:
            hash values added using scanObject.addMetadata

        Laika Config File Options:
            hashmd5:    "1" = md5.hexdigest,    "0" = omit
            hashSHA1:   "1" = sha1.hexdigest,   "0" = omit
            hashSHA256: "1" = sha256.hexdigest, "0" = omit
            hashSHA512: "1" = sha256.hexdigest, "0" = omit
            hashSHA1:   "1" = sha1.hexdigest,   "0" = omit
            ssdeep:     "1" = ssdeep.hash,      "0" = omit

        Function Arguments:
        :param scanObject:<laikaboss.objectmodel.ScanObject>
        :param result:<laikaboss.objectmodel.ScanResult>
        :param depth:<int>
        :param args:<dict> --execution flow controls--
                    Valid args names <str> (value must be 1, 0, "1", or "0")
                        1/"1": Generate the hash of named type
                        0/"0": Omit the hash of named type
                        default args:
                        {"md5":1,
                         "SHA1":0,
                         "SHA256":1,
                         "SHA512":1,
                         "ssdeep":0}

        :return: Always returns a empty list (no child objects)
        '''
        moduleResult = []
        metaDict = {}
        if int(get_option(args, 'md5', 'hashmd5', "md5" in self.module_defaults)):
            metaDict['md5'] = hashlib.md5(scanObject.buffer).hexdigest()
        if int(get_option(args, 'SHA1', 'hashSHA1', "SHA1" in self.module_defaults)):
            metaDict['SHA1'] = hashlib.sha1(scanObject.buffer).hexdigest()
        if int(get_option(args, 'SHA256', 'hashSHA256', "SHA256" in self.module_defaults)):
            metaDict['SHA256'] = hashlib.sha256(scanObject.buffer).hexdigest()
        if int(get_option(args, 'SHA512', 'hashSHA512', "SHA512" in self.module_defaults)):
            metaDict['SHA512'] = hashlib.sha512(scanObject.buffer).hexdigest()
        if int(get_option(args, 'ssdeep', 'hashssdeep', "ssdeep" in self.module_defaults)):
            #only import ssdeep if dispatched.
            #Prevents import error if you don't have/want the package
            #python should keep handing you the original, minimal/no overhead
            try:
                import ssdeep
                metaDict['ssdeep'] = ssdeep.hash(scanObject.buffer)
            except ImportError:
                metaDict['ssdeep'] = "" #indicate ssdeep was configured but failed


        scanObject.addMetadata(self.module_name, "HASHES", metaDict)
        
        return moduleResult

def META_BASIC_INFO(s, buff):

   BASIC_INFO = OrderedDict([('MD5', hashlib.md5(buff).hexdigest()),
                           ('SHA1', hashlib.sha1(buff).hexdigest()),
                           ('SHA256', hashlib.sha256(buff).hexdigest()),
                           ('SHA512', hashlib.sha512(buff).hexdigest()),
                           ('ssdeep' , ssdeep.hash(buff)),
                           ('Size', '%s bytes' % len(buff))])

   return BASIC_INFO

def processMeta(pe,fc, profile):
    profile[PROFILE.STATIC][META.fileSize]=len(fc)
    profile[PROFILE.STATIC][META.timeStamp]=pe.FILE_HEADER.TimeDateStamp
    profile[PROFILE.STATIC][META.dll]=pe.FILE_HEADER.IMAGE_FILE_DLL
    profile[PROFILE.STATIC][META.numberSec]=pe.FILE_HEADER.NumberOfSections
    profile[PROFILE.STATIC][META.importHash]=pe.get_imphash()
    profile[PROFILE.STATIC][META.md5]=hashlib.md5(fc).hexdigest()
    profile[PROFILE.STATIC][META.sha1]=hashlib.sha1(fc).hexdigest()
    profile[PROFILE.STATIC][META.ssdeep]=ssdeep.hash(fc)
    return profile

 def execute(self, input_data):
     raw_bytes = input_data['sample']['raw_bytes']
     sha1 = hashlib.sha1(raw_bytes).hexdigest()
     sha256 = hashlib.sha256(raw_bytes).hexdigest()
     ssdeep = ssd.hash(raw_bytes)
     entropy = self._entropy(raw_bytes)
     output = {name:value for name,value in locals().iteritems()
             if name not in ['self', 'input_data','raw_bytes']}
     output.update(input_data['meta'])
     return output

def META_BASIC_INFO(s, buff):

   BASIC_INFO = { 'MD5' : hashlib.md5(buff).hexdigest(),
                  'SHA1' : hashlib.sha1(buff).hexdigest(),
                  'SHA256' : hashlib.sha256(buff).hexdigest(),
                  'SHA512' : hashlib.sha512(buff).hexdigest(),
                  'ssdeep' : ssdeep.hash(buff),
                  'Size' : '%s bytes' % len(buff) }

   return BASIC_INFO

def get_hash_tuple(functions, filename):
    """ Creates the binary tuple for use in Malfunction and Mallearn

    Results in the form: (Binary Hash, [**ssdeep hashes])"""

    function_hashes = []
    binary_hash = get_binary_hash(filename)
    for function in functions:
        function_hashes.append(ssdeep.hash(function))
    return (binary_hash, function_hashes)

 def _run(self, scanObject, result, depth, args):
     moduleResult = [] 
     metaDict = {}
     #metaDict['SHA224'] = hashlib.sha224(scanObject.buffer).hexdigest()
     metaDict['SHA256'] = hashlib.sha256(scanObject.buffer).hexdigest()
     #metaDict['SHA384'] = hashlib.sha384(scanObject.buffer).hexdigest()
     metaDict['SHA512'] = hashlib.sha512(scanObject.buffer).hexdigest()[0:32]
     metaDict['ssdeep'] = ssdeep.hash(scanObject.buffer)
     scanObject.addMetadata(self.module_name, "HASHES", metaDict)
     return moduleResult

    def build_apk_sample(self, data, url = None):
        sample = {
            "md5"   : hashlib.md5(data).hexdigest(),
            "sha1"  : hashlib.sha1(data).hexdigest(),
            "raw"   : data,
            "data"  : base64.b64encode(data),
            "type"  : "APK",
        }

        if SSDEEP:
            sample['ssdeep'] = ssdeep.hash(data)

        return sample