def add_link( self, url ):

        """
            Adds a link to the link widget.

            Only adds if its not already present.
        """

        if url not in self.links:

            self.links.add( url )


            rowCounts = self.links_ui.rowCount()
            nextRow = rowCounts + 1
            nextPosition = rowCounts    # row count is the length, but position is zero-based

            self.links_ui.setRowCount( nextRow )

            urlEntry = QTableWidgetItem( url )
            statusEntry = QTableWidgetItem( '' )

            statusEntry.setTextAlignment( Qt.AlignCenter )

            urlEntry.setFlags( urlEntry.flags() & ~Qt.ItemIsEditable ) # not editable
            statusEntry.setFlags( statusEntry.flags() & ~Qt.ItemIsEditable ) # not editable

            self.links_ui.setItem( nextPosition, 0, urlEntry )
            self.links_ui.setItem( nextPosition, 1, statusEntry )


                # check if online
            stream = Stream( url.split() )

            stream.is_online( statusEntry )

 def _on_accept(self):
     _logger.debug('_on_accept')
     while True:
         try:
             sock, addr = self._fd.accept()
             _logger.debug('fd: %d accept fd: %d',
                           self._fd.fileno(), sock.fileno())
         except socket.error as msg:
             if msg.errno == errno.ECONNABORTED:
                 continue
             if msg.errno != errno.EAGAIN and msg.errno != errno.EINPROGRESS:
                 _logger.error('fd: %d, accept: %s',
                               self._fd.fileno(), os.strerror(msg.errno))
                 self._fd.close()
                 if self._onClosed is not None:
                     try:
                         self._onClosed(self)
                     except Exception as ex:
                         _logger.error('_onClosed: %s', str(ex))
                         _logger.exception(traceback.format_exc())
             return
         else:
             new_stream = Stream(sock, prefix=self._prefix)
             new_stream._connected = True
             try:
                 self._onAccepted(new_stream, addr)
             except Exception as e:
                 _logger.error('_onAccepted: %s', e)
                 _logger.exception(traceback.format_exc())
                 new_stream.close()

 def main(self):
     #twitter authorization
     auth = OAuthHandler(AuthDetails.consumer_key, AuthDetails.consumer_secret)
     auth.set_access_token(AuthDetails.access_token, AuthDetails.access_token_secret)
     language = 'en'
     pt = ProcessTweet()
     searchTerm = pt.unicodetostring(self.searchTerm)
     stopAt = pt.unicodetostring(self.stopAt)
     #calls method to train the classfier
     tr = Training()
     (priors, likelihood) = tr.starttraining()
     #stream tweets from twitter
     twitterStream = Stream(auth, Listener(searchTerm, stopAt))
     twitterStream.filter(track=[searchTerm], languages=[language])
     sen = Sentiment()
     sentiment_tally = Counter()
     (sentiment_tally, tweet_list) = sen.gettweetstoanalyse(priors, likelihood, searchTerm)
     tr = Training()
     sen = Sentiment()
     (neutral, positive, negative) = sen.analyse(sentiment_tally)
     tweet_list = self.edittweetlists(tweet_list)
     #truncate streamtweets table
     self.removetweetsfromdatabase()
     #save training data
     tr.savetrainingdatatodb(priors, likelihood)
     return (neutral, positive, negative, tweet_list)

    def get(self):
        search_item = self.request.get("search_item")
        print(search_item)
        streams = Stream.query().fetch()
        nameList = list()
        cover_list = []
        stream_list = []
        for stream in streams:
            nameList.append(stream.name)

        index = list()
        for i in xrange(len(nameList)):
            index.append(LCS(nameList[i], search_item))
        tmp = zip(index, nameList)
        tmp.sort(reverse=True)
        # we only show five most relation streams
        for i in xrange(len(tmp)):
            stream = Stream.query(Stream.name == tmp[i][1]).fetch()[0]
            stream_list.append(stream.name)
            # self.response.write(stream.numberofpictures)
            # if stream.numberofpictures > 0:
            #   pictures=db.GqlQuery("SELECT * FROM Picture " +"WHERE ANCESTOR IS :1 "+"ORDER BY uploaddate DESC",db.Key.from_path('Stream',stream.name))
            cover_list.append(stream.coverurl)

        dictPassed = {"streamList": stream_list, "displayCovers": cover_list}
        jsonObj = json.dumps(dictPassed, sort_keys=True, indent=4, separators=(",", ": "))
        print("write json")
        self.response.write(jsonObj)

    def get(self):
        url = self.request.url
        stream_name = re.findall('searchStream=(\S+)',url)
        if len(stream_name) == 0:
            self.response.write(url)
        else:
            stream_name = re.findall('searchStream=(\S+)',url)[0]
            streams = Stream.query().fetch()
            nameList = list()
            for stream in streams:
                nameList.append(stream.name)

            index = list()
            for i in xrange(len(nameList)):
                index.append(LCS(nameList[i], stream_name))
            tmp = zip(index, nameList)
            tmp.sort(reverse = True)
            #we only show five most relation streams
            if len(tmp) < 5:
                showNum = len(tmp)
            else:
                showNum = 5
            self.response.write(SEARCH_PAGE_TEMPLATE)
            self.response.write('<p>%d results for <b>%s</b>,<br>  click on image to view stream</p>' % (showNum,stream_name))
            for i in xrange(showNum):
                stream = Stream.query(Stream.name==tmp[i][1]).fetch()[0]
                #self.response.write(stream.numberofpictures)
                if stream.numberofpictures > 0:
                    pictures=db.GqlQuery("SELECT * FROM Picture " +"WHERE ANCESTOR IS :1 "+"ORDER BY uploaddate DESC",db.Key.from_path('Stream',stream.name))
                    self.response.write('<table border="1" style="width:100%"><table style = "width:10%">')
                    self.response.out.write('<td><div style = "position:relative;"><a href = "%s"><img src="img?img_id=%s" ></img><div style = "position: absolute; left:150px; top:20px"></a>%s</div></div></td>' % (stream.url, pictures[0].key(),stream.name))
                    self.response.write('</table>')
                else:
                    self.response.out.write('<td><div style = "position:relative;"><a href = "%s"><img src="http://www.estatesale.com/img/no_image.gif" ></img><div style = "position: absolute; left:150px; top:20px"></a>%s</div></div></td>' % (stream.url, stream.name))

    def __pipe__(self, inpipe):
        accept_branch, reject_branch = itertools.tee(iter(inpipe))
	accept = lambda x: abs(x) <  self.thresh
	reject = lambda x: abs(x) >= self.thresh
	self.iterator = itertools.ifilter(accept, accept_branch)
	reject_branch = itertools.ifilter(reject, reject_branch)
	Stream.pipe(reject_branch, self.named_stream)
	return self

def main(argv):
    if len(argv) < 2:
        print("usage: python server.py [port]")
        return 1
    port = int(argv[1])
    stream = Stream(port)
    stream.accept_incoming_connections()
    stream.start()

    def __init__(self, **kw):
        """ Default constructor
        Creates and opens a serial port

        **kw - keyword arguments to pass into a pySerial serial port
        """
        Stream.__init__(self)
        self.port = serial.Serial(**kw)

 def handle(self):
     self.data = self.request.recv(1024).strip()
     if self.data == "getStreamUrl":
         stream_url = Stream.getStream()
         self.request.sendall(stream_url)
     elif self.data == "stopStream":
         Stream.stopStream()
         self.request.sendall("StreamClosed")

    def test_produce(self):
        test = []

        def test_produce_cb(stream, data):
            test.append(data)

        stream = Stream()
        stream.produce_cb = test_produce_cb
        stream.produce("test")
        self.assertEqual(test, ["test"])

    def _on_decode_error(self, received):
        self._disable_heartbeat()
        self._stream._encoders = []
        backend = Stream(prefix="SIMPLE")

        def tunnel_ready_to_send(_):
            backend.start_receiving()

        def tunnel_send_buffer_full(_):
            backend.stop_receiving()

        def tunnel_received(_, data, _addr):
            backend.send(data)
            return backend.is_ready_to_send()

        def tunnel_closed(_):
            backend.close()

        def backend_received(_, data, _addr):
            self._stream.send(data)
            return self._stream.is_ready_to_send()

        def backend_closed(_self):
            self._stream.close()

        self._stream.set_on_ready_to_send(tunnel_ready_to_send)
        self._stream.set_on_send_buffer_full(tunnel_send_buffer_full)
        self._stream.set_on_received(tunnel_received)
        self._stream.set_on_closed(tunnel_closed)
        backend.set_on_received(backend_received)
        backend.set_on_closed(backend_closed)
        if received is not None and len(received) > 0:
            backend.send(received)
        backend.connect(UNKNOWN_CONN_ADDR, UNKNOWN_CONN_PORT)

    def select_stream_from_entry( self ):

        """
            Gets the values from the ui elements, and executes the program in json mode, to determine if the values are valid
        """
        url = self.get_complete_url()
        split_url = url.split()

        self.messages_ui.append( 'Trying to open stream: {}'.format( url ) )

        stream = Stream( split_url )

        stream.start( self.messages_ui )

class StreamTestCase(AsyncHTTPTestCase):
    def get_app(self):
        return Application([('/',HelloHandler)])

    def test_read(self):
        s = socket.socket(socket.AF_INET,socket.SOCK_STREAM,0)
        s.connect(("localhost",self.get_http_port()))
        self.stream = Stream(s,io_loop=self.io_loop)
        self.stream.write(b"GET / HTTP/1.0\r\n\r\n")
        self.stream.on("data",self.on_data1)
        self.wait()

    def on_data1(self,data):
        self.assertEquals(data.split("\r\n\r\n")[-1],"Hello")
        self.stop()

    def post(self):
        original_url0 = self.request.headers['Referer']
        original_url = original_url0
        if "%3D" not in original_url:
            original_url += '%3D%3D'
            original_url += users.get_current_user().nickname()

        stream_name=re.findall('=(.*)%3D%3D',original_url)
        if(len(stream_name)<1):
            stream_name=re.findall('%3D(.*)%3D%3D',original_url)[0]
        else:
            stream_name=stream_name[0]

        user_name=re.findall('%3D%3D(.*)\?',original_url)
        if(len(user_name)<1):
            user_name=re.findall('%3D%3D(.*)',original_url)[0]
        else:
            user_name=user_name[0]

        user_name=user_name.split('%40')
        if(len(user_name)>1):
            user_name=user_name[0]+'@'+user_name[1]
        else:
            user_name=user_name[0]

        #Change!# stream=Stream.query(Stream.name==stream_name, Stream.author_name==user_name).fetch()[0]
        stream=Stream.query(Stream.name==stream_name).fetch()[0]

        if users.get_current_user():
            #stream.subscribers.append(users.get_current_user().nickname())
            stream.subscribers.append(users.get_current_user().email())
        stream.put()

        self.redirect(original_url0)

    def initialize(self):
        if self._stream is None:
            self._stream = Stream(prefix='TUNNEL')

        # self._stream.set_buffer_size(BUFF_SIZE)
        self._stream.set_tcp_no_delay()

        self._stream.append_send_handler(obscure.pack_data)
        self._stream.append_send_handler(obscure.random_padding)
        # self._stream.append_send_handler(obscure.gen_aes_encrypt())
        self._stream.append_send_handler(obscure.gen_xor_encrypt())
        # self._stream.append_send_handler(obscure.base64_encode)
        self._stream.append_send_handler(obscure.gen_http_encode(self._connect_to is not None))
        self._stream.append_receive_handler(obscure.gen_http_decode(self._connect_to is not None))
        # self._stream.append_receive_handler(obscure.base64_decode)
        self._stream.append_receive_handler(obscure.gen_xor_decrypt())
        # self._stream.append_receive_handler(obscure.gen_aes_decrypt())
        self._stream.append_receive_handler(obscure.unpad_random)
        self._stream.append_receive_handler(obscure.unpack_data)

        self._stream.set_on_ready_to_send(lambda _: self._on_tunnel_ready_to_send())
        self._stream.set_on_send_buffer_full(lambda _: self._on_tunnel_send_buffer_full())
        self._stream.set_on_received(lambda _, data, addr: self._on_received(data, addr))
        self._stream.set_on_fin_received(lambda _: self._on_fin_received())
        self._stream.set_on_closed(lambda _: self._on_closed())

        if self._connect_to is not None:
            self._stream.connect(*self._connect_to)
        else:
            self._stream.set_on_decode_error(lambda _, received: self._on_decode_error(received))
            self._stream.start_receiving()
        self._enable_heartbeat()

    def post(self):
        original_url0 = self.request.headers["Referer"]
        original_url = original_url0
        if "%3D" not in original_url:
            original_url += "%3D%3D"
            original_url += users.get_current_user().nickname()

        stream_name = re.findall("=(.*)%3D%3D", original_url)
        if len(stream_name) < 1:
            stream_name = re.findall("%3D(.*)%3D%3D", original_url)[0]
        else:
            stream_name = stream_name[0]

        user_name = re.findall("%3D%3D(.*)\?", original_url)
        if len(user_name) < 1:
            user_name = re.findall("%3D%3D(.*)", original_url)[0]
        else:
            user_name = user_name[0]

        user_name = user_name.split("%40")
        if len(user_name) > 1:
            user_name = user_name[0] + "@" + user_name[1]
        else:
            user_name = user_name[0]

        # Change!# stream=Stream.query(Stream.name==stream_name, Stream.author_name==user_name).fetch()[0]
        stream = Stream.query(Stream.name == stream_name).fetch()[0]

        if users.get_current_user():
            # stream.subscribers.append(users.get_current_user().nickname())
            # users.get_current_user().email() = users.get_current_user().email().lower()
            stream.subscribers.append(users.get_current_user().email().lower())
        stream.put()

        self.redirect(original_url0)

    def get(self):
        status = (0, 0)

        stream_name = re.findall("%3D(.*)", self.request.url)[0]

        # Change!# stream=Stream.query(Stream.name==stream_name, Stream.author==users.get_current_user()).fetch()[0]
        stream = Stream.query(Stream.name == stream_name).fetch()[0]

        if stream.author == users.get_current_user():
            status = (1, 1)
        elif users.get_current_user():
            status = (1, 0)
        else:
            self.redirect(users.create_login_url(self.request.url))

        pictures = db.GqlQuery(
            "SELECT *FROM Picture " + "WHERE ANCESTOR IS :1 " + "ORDER BY uploaddate DESC LIMIT 3",
            db.Key.from_path("Stream", stream_name),
        )

        uploadurl = blobstore.create_upload_url("/upload")
        showmoreurl = urllib.urlencode({"showmore": stream.name + "==" + users.get_current_user().nickname()})
        geoviewurl = urllib.urlencode({"geoview": stream.name + "==" + users.get_current_user().nickname()})
        template_values = {
            "user_name": users.get_current_user().nickname(),
            "showmoreurl": showmoreurl,
            "stream_name": stream_name,
            "pictures": pictures,
            "status": status,
            "uploadurl": uploadurl,
            "geoviewurl": geoviewurl,
        }
        template = JINJA_ENVIRONMENT.get_template("viewsinglestream_index.html")
        self.response.write(template.render(template_values))

    def get(self, photoIndexes, currentLocation):
        passedInCoord = currentLocation.split('_')
        lat = float(passedInCoord[0])
        lon = float(passedInCoord[1])

        displayImageObjs = []
        # displayPhotoList = []

        stream_query = Stream.query()
        for stream in stream_query:
            photos = db.GqlQuery("SELECT * FROM Picture " + "WHERE ANCESTOR IS :1 " +"ORDER BY uploaddate DESC" , db.Key.from_path('Stream', stream.name))
            for photo in photos:
                photoUrl = images.get_serving_url(photo.imgkey)
                photoUrl = str(photoUrl) + "=s500"

                photoDict = {}
                photoDict["photoServingURL"] = photoUrl
                photoDict["date"] = str(photo.uploaddate)
                photoDict["loc"] = str(photo.loc)
                photoDict["streamName"] = str(stream.name)
                # photoDict["streamID"] = str(stream.key.id())
                photoCoord = str(photo.loc).split(',')
                plat = float(photoCoord[0])
                plon = float(photoCoord[1])
                R = 6373.0

                lat1 = radians(lat)
                lon1 = radians(lon)
                lat2 = radians(plat)
                lon2 = radians(plon)

                dlon = lon2 - lon1
                dlat = lat2 - lat1
                a = (sin(dlat / 2)) ** 2 + cos(lat1) * cos(lat2) * (sin(dlon / 2)) ** 2
                c = 2 * atan2(sqrt(a), sqrt(1-a))
                distance = R * c
                photoDict["actualDistance"] = distance
                if distance > 10:
                    photoDict["strDistance"] = str(distance).split('.', 1)[0] + 'km'
                else:
                    photoDict["strDistance"] = str(distance * 1000).split('.', 1)[0] + 'm'
                displayImageObjs.append(photoDict)

        displayImageObjs = sorted(displayImageObjs, key = lambda k: k['actualDistance'])
        passedPhotos = []
        morePhotos = "False"
        indexURL = photoIndexes
        indexList = str(photoIndexes).split('_')
        if len(displayImageObjs) - 1 > int(indexList[1]):
            for i in range(int(indexList[0]), int(indexList[1]) + 1):
                passedPhotos.append(displayImageObjs[i])
            indexURL = str(int(indexList[0]) + 16) + '_' + str(int(indexList[1]) + 16)
            morePhotos = "True"
        else:
            for i in range(int(indexList[0]), len(displayImageObjs)):
                passedPhotos.append(displayImageObjs[i])

        dictPassed = {'user': None, 'morePhotos': morePhotos, 'indexURL': indexURL,'displayImageObjs': passedPhotos}#'displayPhotoList' : displayStreamList
        jsonObj = json.dumps(dictPassed, sort_keys=True, indent=4, separators=(',', ': '))
        self.response.write(jsonObj)

    def setUp(self):
        print("Starting test...")
        self.freq = 10
        self._server = SocketWriter(freq = self.freq) # First tests: producer at 10 Hz

        self.com = ComChannel("localhost")
        self.stream = Stream(self.com, 61000, maxlength = 3)

class Scanner(object):
    def initialize_scan(self, string):
        self.tokens = []
        self.stream = Stream(string)

    def produce(self, token):
        self.tokens.append(token)

    def tokenize(self):
        while self.tokenize_one():
            pass
        return self.tokens

    def tokenize_one(self):
        i = self.stream.index

        if self.stream.eof():
            self.eof()
            return False

        for tokenizer_pair in self.state:
            tokenizer = tokenizer_pair[0]
            action = tokenizer_pair[1]
            if tokenizer.applies(self.stream):
                j = self.stream.index
                action(self, tokenizer.token(self.stream, i, j))
                return True

        raise StandardError("No tokenizer applied.")