def extractCorpus(streamfile, dict, corpus):
  from gensim.corpora.dictionary import Dictionary
  from gensim.corpora import MmCorpus
  print('extractCorpus '+str(streamfile))
  packets=rdpcap(streamfile)
  buff=b''

  maxSize=4

  words=[]

  # Limit to the first 100 packets
  for x in range(100):
    if x<len(packets):
      if 'Raw' in packets[x]:
        buff=buff+bytes(packets[x]['Raw'])

  for size in range(1, maxSize):
    for x in range(len(buff)):
      sub=buff[x:x+size]
      if len(sub)==size:
        words.append(sub)

  corpus.append(dict.doc2bow(words))

  return corpus

    def testBasic(self):
        iface = self.tap.name
        record = PacketRecord()

        # start capture
        process = Popen([APP, iface, CAPTURE_FILE], stdout=DEV_NULL, stderr=DEV_NULL)

        # send packets
        for i in range(PACKET_COUNT):
            packet = IP(dst="www.google.com")/ICMP()
            sendp(packet, iface=iface, verbose=False)
            record.add_sent(packet)

        # wait for stragglers
        time.sleep(1)

        # stop capture
        process.terminate()
        # hack: send one more packet to make sure capture closes immediately
        sendp(IP(), iface=iface, verbose=False)
        process.poll()

        # verify capture file
        for packet in rdpcap(CAPTURE_FILE):
            record.add_received(packet)
        self.assertTrue(record.verify())

def analyse_pcap(pcap_file):

    really_old_seq = quite_old_seq = old_seq = current_seq = 0

    print "x, y"

    packets = scapy.rdpcap(pcap_file)
    for packet in packets:

        # Weed out SYN retransmissions.

        if is_retransmission(packet):
            continue

        really_old_seq = quite_old_seq
        quite_old_seq = old_seq
        old_seq = current_seq
        current_seq = packet[scapy.TCP].seq

        x = current_seq - old_seq
        y = old_seq - quite_old_seq
        z = quite_old_seq - really_old_seq

        if x < 0:
            x = SEQ_MAX - abs(x)
        if y < 0:
            y = SEQ_MAX - abs(y)
        if z < 0:
            z = SEQ_MAX - abs(z)

        print "%d, %d" % (x, y)

def mymain():
	packets=scapy.rdpcap('login.pcap')
	mylist=[]
	for p in packets:
		#p.show()
		#sys.stdout=open("my", 'w')
		Get_http(p)

def read_capture(filename):
    packets = scapy.rdpcap(filename)
    # Hold the TCP packet fields
    tcp = []
    # Hold the UDP packet fields
    udp = []
    for p in packets:
        if p.haslayer(scapy.TCP) == 1:
            p[scapy.TCP].fields.update({"src": p[scapy.IP].fields["src"]})
            p[scapy.TCP].fields.update({"dst": p[scapy.IP].fields["dst"]})
            if p.haslayer(scapy.Raw) == 1:
                p[scapy.TCP].fields.update({"Raw": (p[scapy.Raw].load)})
            else:
                p[scapy.TCP].fields.update({"Raw": ""})
            tcp.append(p[scapy.TCP].fields)
        elif p.haslayer(scapy.UDP) == 1:
            p[scapy.UDP].fields.update({"src": p[scapy.IP].fields["src"]})
            p[scapy.UDP].fields.update({"dst": p[scapy.IP].fields["dst"]})
            if p.haslayer(scapy.Raw) == 1:
                p[scapy.UDP].fields.update({"Raw": (p[scapy.Raw].load)})
            else:
                p[scapy.UDP].fields.update({"Raw": ""})
            udp.append(p[scapy.UDP].fields)
    # Order the packets by sequence in case the stream needs to be reconstructed
    tcp = sorted(tcp, key=lambda paq: paq["seq"])
    packets = {}
    packets["tcp"] = tcp
    packets["udp"] = udp

    return packets

def read_pcap(filename):
    """
    @param filename: Filesystem path to the pcap.

    Returns:
      [{"client": "\x17\x52\x15"}, {"server": "\x17\x15\x13"}]
    """
    from scapy.all import IP, Raw, rdpcap

    packets = rdpcap(filename)

    checking_first_packet = True
    client_ip_addr = None
    server_ip_addr = None

    ssl_packets = []
    messages = []

    """
    pcap assumptions:

    pcap only contains packets exchanged between a Tor client and a Tor
    server.  (This assumption makes sure that there are only two IP addresses
    in the pcap file)

    The first packet of the pcap is sent from the client to the server. (This
    assumption is used to get the IP address of the client.)

    All captured packets are TLS packets: that is TCP session
    establishment/teardown packets should be filtered out (no SYN/SYN+ACK)
    """

    """
    Minimally validate the pcap and also find out what's the client
    and server IP addresses.
    """
    for packet in packets:
        if checking_first_packet:
            client_ip_addr = packet[IP].src
            checking_first_packet = False
        else:
            if packet[IP].src != client_ip_addr:
                server_ip_addr = packet[IP].src

        try:
            if (packet[Raw]):
                ssl_packets.append(packet)
        except IndexError:
            pass

    """Form our list."""
    for packet in ssl_packets:
        if packet[IP].src == client_ip_addr:
            messages.append({"client": str(packet[Raw])})
        elif packet[IP].src == server_ip_addr:
            messages.append({"server": str(packet[Raw])})
        else:
            raise("Detected third IP address! pcap is corrupted.")

    return messages

def test_recv_many_packets_out_of_order():
    """
    We should be able to put multiple packets together, if they come
    out of order and are repeated.
    """
    packet_log = rdpcap("test/inputs/wget-36000-nums.pcap")
    listener, conn = create_session(packet_log)
    _, syn_ack, _, push_ack = packet_log[:4]

    listener.dispatch(syn_ack)
    payload = get_payload(push_ack)
    conn.send(payload)

    p1, p2, p3  = packet_log[5], packet_log[7], packet_log[8]

    # Send the packets out of order and repeated
    listener.dispatch(p2)
    listener.dispatch(p3)
    listener.dispatch(p2)
    listener.dispatch(p1) # Right
    listener.dispatch(p3)
    listener.dispatch(p2) # Right
    listener.dispatch(p3) # Right

    # Check that the contents of the packet is right
    # This is a good test because one of the packets starts with a 6 or
    # something
    conn.state = "CLOSED"
    recv = conn.recv(38000)
    assert recv[-36001:-1]  == "1234567890" * 3600

def main(count, input_cap, output_txt, mac_filter):
	mac_counts = {}
	pkt_count = 0
	seq['last'] = 0
	seq['init'] = False
	capture = s.rdpcap(input_cap)
	foutput = open(output_txt, 'wt')
	for pkt in capture:
		n = 0;
		pkt_count += 1
		try:
			if pkt[s.Ether].dst in mac_counts:
				mac_counts[pkt[s.Ether].dst] = mac_counts[pkt[s.Ether].dst] + 1
			else:
				mac_counts[pkt[s.Ether].dst] = 1
				
			# Deal with AVTP packets
			if pkt[s.Ether].type == 0x22f0:
				# look for the requested MAC
				if pkt[s.Ether].dst == mac_filter:
					n = pkt_avtp(pkt, foutput, pkt_count)
		except IndexError:
			print "Unknown ethernet type"
		count = count - n
		if count == 0:
			break
	foutput.close();
	if count != 0:
		print "Could not find the specified MAC, or MAC count"
		print "Mac counts"
		print mac_counts
	print "Complete"

def upload():
    filepath = app.config['UPLOAD_FOLDER']
    upload = Upload()
    if request.method == 'GET':
        return render_template('./upload/upload.html')
    elif request.method == 'POST':
        pcap = upload.pcap.data
        if upload.validate_on_submit():
            pcapname = pcap.filename
            if allowed_file(pcapname):
                name1 = random_name()
                name2 = get_filetype(pcapname)
                global PCAP_NAME, PCAPS
                PCAP_NAME = name1 + name2
                try:
                    pcap.save(os.path.join(filepath, PCAP_NAME))
                    PCAPS = rdpcap(os.path.join(filepath, PCAP_NAME))
                    os.system('rm -rf ' + filepath + '*')
                    flash('恭喜你,上传成功！')
                    return render_template('./upload/upload.html')
                except Exception as e:
                    flash('上传错误,错误信息:' +str(e))
                    return render_template('./upload/upload.html')
            else:
                flash('上传失败,请上传允许的数据包格式!')
                return render_template('./upload/upload.html')
        else:
            return render_template('./upload/upload.html')

    def en_code(self,d_file):
        '''主操作，加载参数过滤并展示信息'''
        global coun_num
        #增加数据库计数器
        regx=r'[A-Z]{3,4}.*?\ HTTP'
        d_packet=scapy.rdpcap(d_file)
        one_regx=regx_raw()
        for i in xrange(len(d_packet)):
            #获取每一个数据包信息
            try:
                if d_packet[i]['Raw'].load.startswith('GET'):

                    vlue_s,snffer_url=one_regx.attack_url(d_packet[i]['Raw'].load)
                    shost=one_regx.regx_host(d_packet[i]['Raw'].load)
                    sscrip=d_packet[i]['IP'].src
                    if vlue_s=='100' or vlue_s=='200':
                        print '警告',sscrip,snffer_url
                        coun_num=coun_num+1
                        dd.insurl(coun_num,snffer_url,'GET',sscrip,one_regx.regx_host(d_packet[i]['Raw'].load),'1','A')
                    else:
                        print '正常：',sscrip,snffer_url
                        pass
                '''
                if d_packet[i]['Raw'].load.startswith('GET') or d_packet[i]['Raw'].load.startswith('POST'):
                    print d_packet[i]['IP'].src,'==>',d_packet[i]['IP'].dst,re.findall(regx,d_packet[i]['Raw'].load)[0]
                    print d_packet[i]['Raw'].load
                else:
                    pass
                '''
            except:
                pass

def read_dump(pcap_file):
    """
    Read PCAP file
    Return dict of packets with serialized flat packet as key
    """
    dump = defaultdict(list)
    packs = []
    count = 0

    if not be_quite:
        sys.stdout.write("Reading file " + pcap_file + "\n")
        sys.stdout.flush()

    for packet in rdpcap(pcap_file):
        if not be_quite:
            sys.stdout.write(":")
            sys.stdout.flush()

        count += 1
        ser = serialize(packet)
        dump[ser].append(packet)
        packs.append(packet)

    if not be_quite:
        sys.stdout.write("\nFound " + str(count) + " packets\n\n")

    return (dump, packs)

    def seq_analysis(self, pcapfile):
        """
        this method act as an interface for the dissect() method.
        and to represents the data in the required format.
        @param pcapfile: path to a pcap/cap library
        """
        packetslist = rdpcap(pcapfile)
        pktsfields = []
        protocols = []
        entry = {}
        recognized = False
        for pkt in packetslist:
            firstlayer = True
            if pkt:
                if firstlayer:
                    firstlayer = False
                    self.packet = pkt
                    fields = self.dissect(self.packet)

                load = pkt
                while load.payload:
                    load = load.payload
                    self.packet = load

                    fields = self.dissect(self.packet)

                    if fields[0]:
                        if fields[0] == "NoPayload":
                            break

def packet_count(pcap):
    conf.verb = 0
    try:
        pkts = rdpcap(pcap)
        return len(pkts)
    except Exception as e:
        return str(e)

def extractLengths(streamfile, lengthfile):
  packets=rdpcap(streamfile)
  lengths=[]

  for packet in packets:
    if 'IP' in packet:
      try:
        l=packet['IP'].fields['len']
      except:
        print('IP packet has no length')
        continue
    elif 'IPv6' in packet:
      try:
        l=packet['IPv6'].fields['len']
      except:
        print('IPv6 packet has no length')
        continue
    else:
      print('Non-IP packet: '+str(packet))
      continue
    lengths.append(l)

  maxlen=max(lengths)
  lengthCount=[0]*(maxlen+1)
  for l in lengths:
    lengthCount[l]=lengthCount[l]+1

  f=open(lengthfile, 'wb')
  for count in lengthCount:
    f.write(str(count)+"\n")
  f.close()

def main(count, input_cap):
	mac_data = {}
	n_avtp_streams = 0
	foutput = []
	pkt_count = 0
	ts_good = 0
	capture = s.rdpcap(input_cap)
	for pkt in capture:
		n = 0;
		pkt_count += 1
		try:
			# Deal with AVTP packets
			if (pkt[s.Ether].type == 0x8100 and pkt[s.Dot1Q].type == 0x22f0):
				avtp = pkt[AVTP]
				if avtp.controlData == 0x0:
					if pkt[s.Ether].dst in mac_data:
						mac_data[pkt[s.Ether].dst]['avtp_count'] = mac_data[pkt[s.Ether].dst]['avtp_count'] + 1
					else:
						print "Packet %d, found AVTP stream with destination MAC %s" % (pkt_count, pkt[s.Ether].dst)
						mac_data[pkt[s.Ether].dst] = {}
						mac_data[pkt[s.Ether].dst]['this_mac'] = pkt[s.Ether].dst
						mac_data[pkt[s.Ether].dst]['avtp_count'] = 1
						mac_data[pkt[s.Ether].dst]['fname'] = 'seq%d.csv' % n_avtp_streams
						mac_data[pkt[s.Ether].dst]['fout'] = open(mac_data[pkt[s.Ether].dst]['fname'], 'wt')
						mac_data[pkt[s.Ether].dst]['seq'] = {}
						mac_data[pkt[s.Ether].dst]['seq']['last'] = 0
						mac_data[pkt[s.Ether].dst]['seq']['init'] = False
						mac_data[pkt[s.Ether].dst]['wraps'] = 0
						mac_data[pkt[s.Ether].dst]['prev_pkt_ts'] = 0
						mac_data[pkt[s.Ether].dst]['ts_count'] = 0
						mac_data[pkt[s.Ether].dst]['ts_accum'] = 0
						mac_data[pkt[s.Ether].dst]['ts_uncertain_count'] = 0
						n_avtp_streams = n_avtp_streams + 1
					
					if avtp.timestampUncertain:
						ts_good = 0
						mac_data[pkt[s.Ether].dst]['ts_uncertain_count'] = mac_data[pkt[s.Ether].dst]['ts_uncertain_count'] + 1
					else:
						ts_good = ts_good + 1

					# when we have 2 MACs, process the packet
					if n_avtp_streams == 2 and ts_good > 2:
						if mac_data[pkt[s.Ether].dst]['ts_count'] == 0:
							print "At packet %d start unpacking AVTP dest MAC %s" % (pkt_count, pkt[s.Ether].dst)
						n = pkt_avtp(pkt, mac_data[pkt[s.Ether].dst]['fout'], pkt_count, mac_data[pkt[s.Ether].dst])
		except IndexError:
			print "Unknown ethernet type packet %d" % pkt_count
		count = count - n
		if count == 0:
			break
	for k, v in mac_data.items():
		v['fout'].close();
		print "MAC %s %d AVTP timestamps stored to %s" % (k, v['ts_count'], v['fname'])
		print "         Timestamp uncertain count: %d" % v['ts_uncertain_count']

	if count != 0:
		print "Could not find the specified packets counts"
	
	print "Complete"

def sniff_responses():
    packets = rdpcap("./74db9d6b62579fea4525d40e6848433f-net03.pcap")
    for packet in packets:
        if DNSRR in packet:
            data = packet[DNSRR].rdata
            # first byte is a length byte as per TXT record RFC
            data = b64decode(str(correct_base64_padding(data[1:])))
            print(" ".join("{0:02x}".format(c) for c in data))

def get_MPTCP_syn(i):
    try:
        tf = tempfile.NamedTemporaryFile()
        execCommand("sudo tcpdump -c 1 -w " + tf.name + ".cap -i " + i + " \"tcp[tcpflags] & tcp-syn != 0\" 2>/dev/null", shell = True)
        scan = rdpcap("" + tf.name + ".cap")
    finally:
        execCommand("rm -f " + tf.name + ".cap", shell = True)
    return scan[0]

def sniff_ackseq(i, srcIP):
    try:
        tf = tempfile.NamedTemporaryFile()
        execCommand("sudo tcpdump -c 1 -w " + tf.name + ".cap -i " + i + " \"src net " + srcIP + "\" 2>/dev/null", shell = True)
        scan = rdpcap("" + tf.name + ".cap")
    finally:
        execCommand("rm -f " + tf.name + ".cap", shell = True)
    return scan[0]

def get_MPTCP_ack(i, dstIP):
    try:
        tf = tempfile.NamedTemporaryFile()
        execCommand("sudo tcpdump -c 1 -w " + tf.name + ".cap -i " + i + " \"tcp[tcpflags] & (tcp-ack) != 0 and tcp[tcpflags] & (tcp-syn) == 0 and dst net " + dstIP + "\" 2>/dev/null", shell = True)
        scan = rdpcap("" + tf.name + ".cap")
    finally:
        execCommand("rm -f " + tf.name + ".cap", shell = True)
    return scan[0]

	def doTest(self):
		#redirect stdout to file
		sys.stdout = open(self.outputFile, 'w')
		pkts = rdpcap(self.inputFile)
		pkts.show()
		print '\n', '-' * 20, 'Packets Details:', '\n'
		for p in pkts:
			p.show()
			print '=' * 70