def test_copytruncate_on_smaller(self):
     self.test_readlines()
     self.copytruncate()
     new_lines = "4\n5\n"
     self.append(new_lines)
     pygtail = Pygtail(self.logfile.name, copytruncate=True)
     self.assertEqual(pygtail.read(), new_lines)

    def __init__(self, filepath, groupby, groupname=None):
        self.groupmatch = re.compile(groupby)
        # write an offset file so that we start somewhat at the end of the file

        # either filepath is a path or a syslogd url
        (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(filepath)
        if scheme == 'syslog':
            host, port = netloc.split(':')
            self.fin = QueueFile()
            self.server = SocketServer.UDPServer((host, int(port)), SyslogUDPHandler)
            self.server.queue = self.fin

            th = threading.Thread(target=lambda: self.server.serve_forever(poll_interval=0.5))
            th.setDaemon(True)
            th.start()
        else:
            # Create a temporal file with offset info
            self.offsetpath = "/tmp/" + str(uuid.uuid4())
            try:
                inode = os.stat(filepath).st_ino
                offset = os.path.getsize(filepath) - 1024
            except OSError:
                pass
            else:
                if offset > 0:
                    foffset = open(self.offsetpath, "w")
                    foffset.write("%s\n%s" % (inode, offset))
                    foffset.close()

            self.fin = Pygtail(filepath, offset_file=self.offsetpath, copytruncate=True)

        # List of matchings
        self.match_definitions = []
        # Regex group name for grouping
        self.groupbygroup = groupname

 def test_subsequent_read_with_new_data(self):
     pygtail = Pygtail(self.logfile.name)
     self.assertEqual(pygtail.read(), self.test_str)
     new_lines = "4\n5\n"
     self.append(new_lines)
     new_pygtail = Pygtail(self.logfile.name)
     self.assertEqual(new_pygtail.read(), new_lines)

 def _test_copytruncate_larger(self, onoff):
     self.test_readlines()
     self.copytruncate()
     self.append(self.test_str)
     new_lines = "4\n5\n"
     self.append(new_lines)
     pygtail = Pygtail(self.logfile.name, copytruncate=onoff)
     self.assertEqual(pygtail.read(), new_lines)

 def test_copytruncate_off_smaller(self):
     self.test_readlines()
     self.copytruncate()
     new_lines = "4\n5\n"
     self.append(new_lines)
     pygtail = Pygtail(self.logfile.name, copytruncate=False)
     self.assertEqual(pygtail.read(), None)
     self.assertRegexpMatches(sys.stderr.getvalue(), r".*?\bWARN\b.*?\bshrank\b.*")

 def test_logrotate_without_close(self):
     new_lines = ["4\n5\n", "6\n7\n"]
     pygtail = Pygtail(self.logfile.name)
     pygtail.read()
     self.append(new_lines[0])
     # note it doesn't matter what filename the file gets rotated to
     os.rename(self.logfile.name, "%s.somethingodd" % self.logfile.name)
     self.append(new_lines[1])
     self.assertEqual(pygtail.read(), ''.join(new_lines))

 def test_logrotate_with_delay_compress(self):
     new_lines = ["4\n5\n", "6\n7\n"]
     pygtail = Pygtail(self.logfile.name)
     pygtail.read()
     self.append(new_lines[0])
     os.rename(self.logfile.name, "%s.1" % self.logfile.name)
     self.append(new_lines[1])
     pygtail = Pygtail(self.logfile.name)
     self.assertEqual(pygtail.read(), ''.join(new_lines))

 def test_logrotate(self):
     new_lines = ["4\n5\n", "6\n7\n"]
     pygtail = Pygtail(self.logfile.name)
     pygtail.read()
     self.append(new_lines[0])
     os.rename(self.logfile.name, "%s.1" % self.logfile.name)
     self.append(new_lines[1])
     pygtail = Pygtail(self.logfile.name)
     self.assertEquals(pygtail.read(), "".join(new_lines))

 def test_copytruncate_off_smaller_without_close(self):
     new_lines = ["4\n5\n", "6\n7\n"]
     pygtail = Pygtail(self.logfile.name, copytruncate=True)
     pygtail.read()
     self.append(new_lines[0])
     read1 = pygtail.read()
     self.copytruncate()
     self.append(new_lines[1])
     read2 = pygtail.read()
     self.assertEqual([read1,read2], new_lines)

    def test_copytruncate_off_smaller(self):
        self.test_readlines()
        self.copytruncate()
        new_lines = "4\n5\n"
        self.append(new_lines)

        sys.stderr = captured = io.BytesIO() if PY2 else io.StringIO()
        pygtail = Pygtail(self.logfile.name, copytruncate=False)
        captured_value = captured.getvalue()
        sys.stderr = sys.__stderr__

        assert_class = self.assertRegex if sys.version_info >= (3, 1) else self.assertRegexpMatches
        assert_class(captured_value, r".*?\bWARN\b.*?\bshrank\b.*")
        self.assertEqual(pygtail.read(), None)

    def __init__(self, filepath, groupby):

        self.groupmatch = re.compile(groupby)

        # write an offset file so that we start somewhat at the end of the file
       
        self.offsetpath = "/tmp/" + str(uuid.uuid4())
        #print self.offsetpath
        try:
            inode = os.stat(filepath).st_ino
            offset = os.path.getsize(filepath) - 1024
            #print inode
            #print offset
        except OSError:
            pass
        else:
            if offset > 0:
                #print 'write offset'
                foffset = open(self.offsetpath, "w")
                foffset.write ("%s\n%s" % (inode, offset))
                foffset.close()

        self.fin = Pygtail(filepath, offset_file=self.offsetpath, copytruncate=True)
        #self.fin.readlines()

        self.match_definitions = []

class FileFollower():
    '''
    Use pygtail to keep track of EOF and rotated files, catch exceptions to
    make things more seamless
    '''
    def __init__(self, path):
        self.path = path
        self.pygtail = None
        self.last_inode = 0

    def next(self):
        line = ''
        curr_inode = 0
        if self.pygtail is None:
            try:
                # remove last offset file if the log file is different
                # PygTail's inode detection doesn't work in certain cases
                curr_inode = os.stat(self.path).st_ino
                if self.last_inode != curr_inode:
                    os.unlink(self.path + '.offset')
                    self.last_inode = curr_inode
                    log.debug('deleted offset file, inode difference')
            except Exception as e:
                log.info('inode checking failed (not terminal): %s' % e)
            self.pygtail = Pygtail(self.path)
        try:
            line = self.pygtail.next()
        except StopIteration as si:
            # Need to get a new instance of pygtail after this incase the inode
            # has changed
            self.pygtail = None
            return False
        return line

    def test_logrotate_without_delay_compress(self):
        new_lines = ["4\n5\n", "6\n7\n"]
        pygtail = Pygtail(self.logfile.name)
        pygtail.read()
        self.append(new_lines[0])

        # put content to gzip file
        gzip_handle = gzip.open("%s.1.gz" % self.logfile.name, 'wb')
        with open(self.logfile.name, 'rb') as logfile:
            gzip_handle.write(logfile.read())
        gzip_handle.close()

        with open(self.logfile.name, 'w'):
            # truncate file
            pass

        self.append(new_lines[1])
        pygtail = Pygtail(self.logfile.name)
        self.assertEqual(pygtail.read(), ''.join(new_lines))

class GroupingTail (object):

    def __init__(self, filepath, groupby):

        self.groupmatch = re.compile(groupby)

        # write an offset file so that we start somewhat at the end of the file
       
        self.offsetpath = "/tmp/" + str(uuid.uuid4())
        #print self.offsetpath
        try:
            inode = os.stat(filepath).st_ino
            offset = os.path.getsize(filepath) - 1024
            #print inode
            #print offset
        except OSError:
            pass
        else:
            if offset > 0:
                #print 'write offset'
                foffset = open(self.offsetpath, "w")
                foffset.write ("%s\n%s" % (inode, offset))
                foffset.close()

        self.fin = Pygtail(filepath, offset_file=self.offsetpath, copytruncate=True)
        #self.fin.readlines()

        self.match_definitions = []

    def update(self):
        for line in self.fin.readlines():
            #print 'line: %s' % line
            mo = self.groupmatch.match(line)
            if mo is not None and mo.groups():
                groupname = mo.groups()[0].replace(".", "_").replace("-", "_")
                for match in self.match_definitions:
                    instrument = match["instrument"]
                    instrument.write(groupname, line)

    def add_match(self,  instance_name, valuetype, instrument):
        self.match_definitions.append(dict(
            instance_name=instance_name,
            valuetype=valuetype,
            instrument=instrument
        ))

    def read_metrics(self):
        for match in self.match_definitions:
            instance_name = match["instance_name"]
            instrument = match["instrument"]
            valuetype = match["valuetype"]

            for groupname, value in instrument.read():
                metric_name = "%s.%s" % (groupname, instance_name)
                yield (metric_name, valuetype, value)

    def download_file(self, url):
        try:
            # Check that the log file exists.
            assert os.path.isfile(LOG_FILE)

            # Check that the offset directory exists.
            assert os.path.isdir(OFFSET_DIR)

            # Check that the offset directory is writeable.
            assert os.access(OFFSET_DIR, os.W_OK)

            # Check that the log file is writeable.
            assert os.access(LOG_FILE, os.R_OK)
        except AssertionError:
            sys.stderr.write('Error: One or more preconditions failed.\n')
            # Exit 13, don't restart tcollector.
            sys.exit(13)

        # If the offset file exists, it had better not be empty.
        if os.path.isfile(OFFSET_FILE):
            try:
                assert os.path.getsize(OFFSET_FILE) > 0
            except AssertionError:
                os.remove(OFFSET_FILE)

        # We're not using paranoid mode in Pygtail for performance.
        maillog = Pygtail(LOG_FILE, offset_file=OFFSET_FILE)

        for line in maillog:
            try:
                process_line(line)
            except LineProcessingError:
                pass

            if STOP:
               # Force the offset file to update before we shutdown.
               # If pygtail is not paranoid=True, this is necessary to ensure
               # that the offset gets written after SIGTERM.
               maillog._update_offset_file()
               break

    def on_modified(self, event):
        super(TailContentCollector, self).on_modified(event)

        # what = 'directory' if event.is_directory else 'file'
        # logging.info("Modified %s: %s", what, event.src_path)
        # split_path = None
        log_file = event.src_path.split("/")

        # prepare offset file directory
        if not os.path.exists(self.offset_dir):
            os.makedirs(self.offset_dir)
        offset_file = "%s/%s.os" % (self.offset_dir, log_file[-1])
        # offset file must separate with monitor directory, and is local variable...
        tailor = Pygtail(event.src_path, offset_file, paranoid=True)
        appended = tailor.read()
        if appended:
            # must use gbk decoding...
            decodelines = appended.decode("gbk")
            # execute callback function...
            self.onchange(event.src_path, decodelines)
        else:
            logging.info("empty content: %s", event.src_path)

 def next(self):
     line = ''
     curr_inode = 0
     if self.pygtail is None:
         try:
             # remove last offset file if the log file is different
             # PygTail's inode detection doesn't work in certain cases
             curr_inode = os.stat(self.path).st_ino
             if self.last_inode != curr_inode:
                 os.unlink(self.path + '.offset')
                 self.last_inode = curr_inode
                 log.debug('deleted offset file, inode difference')
         except Exception as e:
             log.info('inode checking failed (not terminal): %s' % e)
         self.pygtail = Pygtail(self.path)
     try:
         line = self.pygtail.next()
     except StopIteration as si:
         # Need to get a new instance of pygtail after this incase the inode
         # has changed
         self.pygtail = None
         return False
     return line

class GroupingTail (object):

    def __init__(self, filepath, groupby, groupname=None):

        self.groupmatch = re.compile(groupby)

        # write an offset file so that we start somewhat at the end of the file

        # either filepath is a path or a syslogd url
        (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(filepath)
        if scheme == 'syslog':
            host, port = netloc.split(':')
            self.fin = QueueFile()
            self.server = SocketServer.UDPServer((host, int(port)), SyslogUDPHandler)
            self.server.queue = self.fin

            th = threading.Thread(target=lambda: self.server.serve_forever(poll_interval=0.5))
            th.daemon = True
            th.start()

        else:

            self.offsetpath = "/tmp/" + str(uuid.uuid4())
            #print self.offsetpath
            try:
                inode = os.stat(filepath).st_ino
                offset = os.path.getsize(filepath) - 1024
                #print inode
                #print offset
            except OSError:
                pass
            else:
                if offset > 0:
                    #print 'write offset'
                    foffset = open(self.offsetpath, "w")
                    foffset.write ("%s\n%s" % (inode, offset))
                    foffset.close()

            self.fin = Pygtail(filepath, offset_file=self.offsetpath, copytruncate=True)
            #self.fin.readlines()

        self.match_definitions = []
        self.groupbygroup = groupname

    def __del__(self):
        if hasattr(self, 'server'):
            self.server.socket.close()

    def update(self):
        for line in self.fin.readlines():
            #print 'line: %s' % line
            groupname = None
            mo = self.groupmatch.match(line)
            if mo is not None:
                if self.groupbygroup is None and mo.groups():
                    groupname = mo.groups()[0]
                elif self.groupbygroup is not None:
                    groupname = mo.groupdict().get(self.groupbygroup)
            if groupname is not None:
                groupname = groupname.replace(".", "_").replace("-", "_")
                for match in self.match_definitions:
                    instrument = match["instrument"]
                    instrument.write(groupname, line)

    def add_match(self,  instance_name, valuetype, instrument):
        self.match_definitions.append(dict(
            instance_name=instance_name,
            valuetype=valuetype,
            instrument=instrument
        ))

    def read_metrics(self):
        for match in self.match_definitions:
            instance_name = match["instance_name"]
            instrument = match["instrument"]
            valuetype = match["valuetype"]

            for groupname, value in instrument.read():
                metric_name = "%s.%s" % (groupname, instance_name)
                yield (metric_name, valuetype, value)

class GroupingTail(object):
    def __init__(self, filepath, groupby, groupname=None):
        self.groupmatch = re.compile(groupby)
        # write an offset file so that we start somewhat at the end of the file

        # either filepath is a path or a syslogd url
        (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(filepath)
        if scheme == 'syslog':
            host, port = netloc.split(':')
            self.fin = QueueFile()
            self.server = SocketServer.UDPServer((host, int(port)), SyslogUDPHandler)
            self.server.queue = self.fin

            th = threading.Thread(target=lambda: self.server.serve_forever(poll_interval=0.5))
            th.setDaemon(True)
            th.start()
        else:
            # Create a temporal file with offset info
            self.offsetpath = "/tmp/" + str(uuid.uuid4())
            try:
                inode = os.stat(filepath).st_ino
                offset = os.path.getsize(filepath) - 1024
            except OSError:
                pass
            else:
                if offset > 0:
                    foffset = open(self.offsetpath, "w")
                    foffset.write("%s\n%s" % (inode, offset))
                    foffset.close()

            self.fin = Pygtail(filepath, offset_file=self.offsetpath, copytruncate=True)

        # List of matchings
        self.match_definitions = []
        # Regex group name for grouping
        self.groupbygroup = groupname

    def __del__(self):
        if hasattr(self, 'server'):
            self.server.socket.close()

    # Update method processing last lines
    def update(self):
        for line in self.fin.readlines():
            groupname = None
            mo = self.groupmatch.match(line)
            if mo is not None:
                if self.groupbygroup is None and mo.groups():
                    # No groupbygroup get first group name
                    groupname = mo.groups()[0]
                elif self.groupbygroup is not None:
                    # Get groupname from line
                    groupname = mo.groupdict().get(self.groupbygroup)

            if groupname is not None:
                # Normalize groupname
                groupname = groupname.replace(".", "_").replace("-", "_")
                # Check all possible matchings
                for match in self.match_definitions:
                    instrument = match["instrument"]
                    instrument.write(groupname, line)

    # Attatch match to groupingtail class
    def add_match(self, instance_name, valuetype, instrument):
        self.match_definitions.append(dict(
            instance_name=instance_name,
            valuetype=valuetype,
            instrument=instrument
        ))

    # Get stored values from instrument
    def read_metrics(self):
        # For all matchings
        for match in self.match_definitions:
            instance_name = match["instance_name"]
            instrument = match["instrument"]
            valuetype = match["valuetype"]

            # Get metric info
            for groupname, value in instrument.read():
                # Construct grouping name for this metric value
                metric_name = "%s*%s" % (groupname, instance_name)
                # Send metric info
                yield (metric_name, valuetype, value)

    if not os.path.exists(".offset"):
        os.makedirs(".offset")

    try:
        if is_erased(file_name):
            # forget about saved offset
            # delete offset file
            # print "remove file"  # test point
            os.remove(OFFSET_FILE)
    except IOError:
        # Error occur when first time run
        pass

    try:
        pyg = Pygtail(file_name, OFFSET_FILE)
        first_line = pyg.next()
        # get log format and log type
        log_type, log_format = parser.detect_log_type(first_line)

        for line in Pygtail(file_name, OFFSET_FILE):
            # print line  # test point
            error_info = parser.parse_log(line, log_type, log_format)
            status_code = error_info['status_code']

            if status_code == 502 or status_code == 503:
                client.capture(
                    "raven.events.Message",
                    message=log_type + " " + str(status_code),
                    extra=error_info,
                    date=error_info['time']