def test_module_with_different_metric_logs(self):
        monitor_one = ScalyrLoggingTest.FakeMonitor('testing one')
        monitor_two = ScalyrLoggingTest.FakeMonitor('testing two')

        metric_file_one = tempfile.mktemp('.log')
        metric_file_two = tempfile.mktemp('.log')

        logger_one = scalyr_logging.getLogger('scalyr_agent.builtin_monitors.foo(1)')
        logger_two = scalyr_logging.getLogger('scalyr_agent.builtin_monitors.foo(2)')

        logger_one.openMetricLogForMonitor(metric_file_one, monitor_one)
        logger_two.openMetricLogForMonitor(metric_file_two, monitor_two)

        logger_one.report_values({'foo': 5})
        logger_two.report_values({'bar': 4})

        # The value should only appear in the metric log file and not the main one.
        self.assertTrue(self.__log_contains('foo=5', file_path=metric_file_one))
        self.assertTrue(self.__log_contains('bar=4', file_path=metric_file_two))

        self.assertFalse(self.__log_contains('foo=5', file_path=metric_file_two))
        self.assertFalse(self.__log_contains('bar=4', file_path=metric_file_one))

        self.assertFalse(self.__log_contains('foo=5'))
        self.assertFalse(self.__log_contains('bar=4'))

        logger_one.closeMetricLog()
        logger_two.closeMetricLog()

 def test_component_name(self):
     self.assertEquals(self.__logger.component, 'core')
     self.assertEquals(scalyr_logging.getLogger('scalyr_agent').component, 'core')
     self.assertEquals(scalyr_logging.getLogger('scalyr_agent.foo').component, 'core')
     self.assertEquals(scalyr_logging.getLogger('scalyr_agent.foo.bar').component, 'core')
     self.assertEquals(scalyr_logging.getLogger('scalyr_agent.builtin_monitors.foo').component, 'monitor:foo')
     self.assertEquals(scalyr_logging.getLogger('scalyr_agent.builtin_monitors.foo(ok)').component,
                       'monitor:foo(ok)')

    def test_pass_in_module_with_metric(self):
        monitor_instance = ScalyrLoggingTest.FakeMonitor('testing')
        metric_file_path = tempfile.mktemp('.log')

        monitor_logger = scalyr_logging.getLogger('scalyr_agent.builtin_monitors.foo(1)')
        monitor_logger.openMetricLogForMonitor(metric_file_path, monitor_instance)
        scalyr_logging.getLogger('scalyr_agent.builtin_monitors.foo').report_values({'foo': 5},
                                                                                    monitor=monitor_instance)

        # The value should only appear in the metric log file and not the main one.
        self.assertTrue(self.__log_contains('foo=5', file_path=metric_file_path))
        self.assertFalse(self.__log_contains('foo=5'))

        monitor_logger.closeMetricLog()

    def test_run_udp_server(self):
        config = {
            'module': 'scalyr_agent.builtin_monitors.syslog_monitor',
            'protocols': 'udp:5514',
        }
        self.monitor = SyslogMonitor(config, scalyr_logging.getLogger("syslog_monitor[test]"))
        self.monitor.open_metric_log()
        self.monitor.start()

        time.sleep(0.1)

        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.sockets.append(s)

        expected = "UDP Test %s" % (uuid.uuid4())
        s.sendto(expected, ('localhost', 5514))
        time.sleep(1)
        self.monitor.stop(wait_on_join=False)
        self.monitor = None
        f = open('agent_syslog.log')
        actual = f.read().strip()
        self.assertTrue(
            expected in actual,
            "Unable to find '%s' in output:\n\t %s" % (expected, actual)
        )

    def build_monitor(monitor_config, additional_python_paths, default_sample_interval_secs, global_config ):
        """Builds an instance of a ScalyrMonitor for the specified monitor configuration.

        @param monitor_config: The monitor configuration object for the monitor that should be created.  It will
            have keys such as 'module' that specifies the module containing the monitor, as well as others.
        @param additional_python_paths: A list of paths (separate by os.pathsep) to add to the PYTHONPATH when
            instantiating the module in case it needs to packages in other directories.
        @param global_config: The global configuration object

        @type monitor_config: dict
        @type additional_python_paths: str

        @return:  The appropriate ScalyrMonitor instance as controlled by the configuration.
        @rtype: scalyr_monitor.ScalyrMonitor
        """
        # Set up the logs to do the right thing.
        module_name = monitor_config['module']
        monitor_id = monitor_config['id']

        # We have to update this variable before we create monitor instances so that it is used.
        ScalyrMonitor.DEFAULT_SAMPLE_INTERVAL_SECS = default_sample_interval_secs

        # Load monitor.
        monitor_class = MonitorsManager.load_monitor(module_name, additional_python_paths)

        # Instantiate and initialize it.
        return monitor_class(monitor_config, scalyr_logging.getLogger("%s(%s)" % (module_name, monitor_id)), global_config=global_config)

 def setUp(self):
     self.config_commandline = {
         "module": "scalyr_agent.builtin_monitors.linux_process_metrics",
         "id": "myapp",
         "commandline": ".foo.*",
     }
     self.monitor = ProcessMonitor(self.config_commandline, scalyr_logging.getLogger("syslog_monitor[test]"))

    def test_errors_for_monitor(self):
        monitor_instance = ScalyrLoggingTest.FakeMonitor('testing')
        metric_file_path = tempfile.mktemp('.log')

        monitor_logger = scalyr_logging.getLogger('scalyr_agent.builtin_monitors.foo(1)')
        monitor_logger.openMetricLogForMonitor(metric_file_path, monitor_instance)
        monitor_logger.error('Foo')

        self.assertEquals(monitor_instance.errors, 1)

        monitor_logger.closeMetricLog()

    def test_metric_logging_with_bad_name(self):
        monitor_instance = ScalyrLoggingTest.FakeMonitor('testing')
        metric_file_path = tempfile.mktemp('.log')

        monitor_logger = scalyr_logging.getLogger('scalyr_agent.builtin_monitors.foo(1)')
        monitor_logger.openMetricLogForMonitor(metric_file_path, monitor_instance)

        self.assertRaises(scalyr_logging.BadMetricOrFieldName, monitor_logger.emit_value, '1name', 5)
        self.assertRaises(scalyr_logging.BadMetricOrFieldName, monitor_logger.emit_value, 'name+hi', 5)
        self.assertRaises(scalyr_logging.BadMetricOrFieldName, monitor_logger.emit_value, 'name', 5, {'hi+': 6})

        monitor_logger.closeMetricLog()

    def test_run_multiple_servers( self ):
        config = {
            'module': 'scalyr_agent.builtin_monitors.syslog_monitor',
            'protocols': 'udp:8000, tcp:8001, udp:8002, tcp:8003',
        }
        self.monitor = SyslogMonitor( config, scalyr_logging.getLogger( "syslog_monitor[test]" ) )
        self.monitor.open_metric_log()

        self.monitor.start()

        time.sleep( 0.01 )

        udp = socket.socket( socket.AF_INET, socket.SOCK_DGRAM )
        self.sockets.append( udp )

        tcp1 = socket.socket()
        self.sockets.append( tcp1 )

        tcp2 = socket.socket()
        self.sockets.append( tcp2 )

        self.connect( tcp1, ('localhost', 8001) )
        self.connect( tcp2, ('localhost', 8003) )


        expected_udp1 = "UDP Test"
        udp.sendto( expected_udp1, ('localhost', 8000) )

        expected_udp2 = "UDP2 Test"
        udp.sendto( expected_udp2, ('localhost', 8002) )

        expected_tcp1 = "TCP Test\n"
        tcp1.sendall( expected_tcp1 )

        expected_tcp2 = "TCP2 Test\n"
        tcp2.sendall( expected_tcp2 )

        time.sleep( 1 )

        self.monitor.stop( wait_on_join=False )
        self.monitor = None

        self.handler.flush()
        actual = self.stream.getvalue().strip()

        expected_tcp1 = expected_tcp1.strip()
        expected_tcp2 = expected_tcp2.strip()

        self.assertTrue( expected_udp1 in actual, "Unable to find '%s' in output:\n\t %s" % (expected_udp1, actual)  )
        self.assertTrue( expected_udp2 in actual, "Unable to find '%s' in output:\n\t %s" % (expected_udp2, actual)  )
        self.assertTrue( expected_tcp1 in actual, "Unable to find '%s' in output:\n\t %s" % (expected_tcp1, actual)  )
        self.assertTrue( expected_tcp2 in actual, "Unable to find '%s' in output:\n\t %s" % (expected_tcp2, actual)  )

    def test_logging_to_metric_log(self):
        monitor_instance = ScalyrLoggingTest.FakeMonitor('testing')
        metric_file_path = tempfile.mktemp('.log')

        monitor_logger = scalyr_logging.getLogger('scalyr_agent.builtin_monitors.foo(1)')
        monitor_logger.openMetricLogForMonitor(metric_file_path, monitor_instance)
        monitor_logger.info('foobaz is fine', emit_to_metric_log=True)

        self.assertEquals(monitor_instance.reported_lines, 1)

        # The value should only appear in the metric log file and not the main one.
        self.assertTrue(self.__log_contains('foobaz is fine', file_path=metric_file_path))
        self.assertFalse(self.__log_contains('foobaz is fine'))

        monitor_logger.closeMetricLog()

    def test_rate_limit(self):
        self.__log_path = tempfile.mktemp('.log')
        scalyr_logging.set_log_destination(use_disk=True, logs_directory=os.path.dirname(self.__log_path),
                                           agent_log_file_path=self.__log_path, max_write_burst=250, log_write_rate=0)
        self.__logger = scalyr_logging.getLogger('scalyr_agent.agent_main')

        string_300 = ''

        for i in range(300):
            string_300 = '%sa' % string_300

        self.__logger.info('First message')
        self.assertTrue(self.__log_contains('First message'))

        self.__logger.info('Dropped message %s', string_300)
        self.assertFalse(self.__log_contains('Dropped message'))

        self.__logger.info('Second message')
        self.assertTrue(self.__log_contains('Second message'))
        self.assertTrue(self.__log_contains('Warning, skipped writing 1 log lines'))

    def test_limit_once_per_x_secs(self):
        log = scalyr_logging.getLogger('scalyr_agent.foo')
        log.info('First record', limit_once_per_x_secs=60.0, limit_key='foo', current_time=0.0)
        log.info('Second record', limit_once_per_x_secs=60.0, limit_key='foo', current_time=1.0)
        log.info('Third record', limit_once_per_x_secs=60.0, limit_key='foo', current_time=61.0)

        self.assertTrue(self.__log_contains('First record'))
        self.assertFalse(self.__log_contains('Second record'))
        self.assertTrue(self.__log_contains('Third record'))

        # Now test with different keys.
        log.info('First record', limit_once_per_x_secs=30.0, limit_key='foo', current_time=0.0)
        log.info('Second record', limit_once_per_x_secs=60.0, limit_key='bar', current_time=1.0)
        log.info('Third record', limit_once_per_x_secs=30.0, limit_key='foo', current_time=31.0)
        log.info('Fourth record', limit_once_per_x_secs=60.0, limit_key='bar', current_time=31.0)

        self.assertTrue(self.__log_contains('First record'))
        self.assertTrue(self.__log_contains('Second record'))
        self.assertTrue(self.__log_contains('Third record'))
        self.assertFalse(self.__log_contains('Fourth record'))

    def build_monitor(monitor_config, additional_python_paths):
        """Builds an instance of a ScalyrMonitor for the specified monitor configuration.

        @param monitor_config: The monitor configuration object for the monitor that should be created.  It will
            have keys such as 'module' that specifies the module containing the monitor, as well as others.
        @param additional_python_paths: A list of paths (separate by os.pathsep) to add to the PYTHONPATH when
            instantiating the module in case it needs to packages in other directories.

        @type monitor_config: dict
        @type additional_python_paths: str

        @return:  The appropriate ScalyrMonitor instance as controlled by the configuration.
        @rtype: scalyr_monitor.ScalyrMonitor
        """
        # Set up the logs to do the right thing.
        module_name = monitor_config['module']
        monitor_id = monitor_config['id']

        # Augment the PYTHONPATH if requested to locate the module.
        original_path = list(sys.path)

        # Also add in scalyr_agent/../monitors/local and scalyr_agent/../monitors/contrib to the Python path to search
        # for monitors.  (They are always in the parent directory of the scalyr_agent package.
        path_to_package_parent = os.path.dirname(get_package_root())
        sys.path.append(os.path.join(path_to_package_parent, 'monitors', 'local'))
        sys.path.append(os.path.join(path_to_package_parent, 'monitors', 'contrib'))

        # Add in the additional paths.
        if additional_python_paths is not None and len(additional_python_paths) > 0:
            for x in additional_python_paths.split(os.pathsep):
                sys.path.append(x)

        # Load monitor.
        try:
            monitor_class = MonitorsManager.__load_class_from_module(module_name)
        finally:
            # Be sure to reset the PYTHONPATH
            sys.path = original_path

        # Instantiate and initialize it.
        return monitor_class(monitor_config, scalyr_logging.getLogger("%s(%s)" % (module_name, monitor_id)))

    def build_monitor(monitor_config, additional_python_paths):
        """Builds an instance of a ScalyrMonitor for the specified monitor configuration.

        @param monitor_config: The monitor configuration object for the monitor that should be created.  It will
            have keys such as 'module' that specifies the module containing the monitor, as well as others.
        @param additional_python_paths: A list of paths (separate by os.pathsep) to add to the PYTHONPATH when
            instantiating the module in case it needs to packages in other directories.

        @type monitor_config: dict
        @type additional_python_paths: str

        @return:  The appropriate ScalyrMonitor instance as controlled by the configuration.
        @rtype: scalyr_monitor.ScalyrMonitor
        """
        # Set up the logs to do the right thing.
        module_name = monitor_config['module']
        monitor_id = monitor_config['id']

        # Load monitor.
        monitor_class = MonitorsManager.load_monitor(module_name, additional_python_paths)

        # Instantiate and initialize it.
        return monitor_class(monitor_config, scalyr_logging.getLogger("%s(%s)" % (module_name, monitor_id)))

 def test_config_port_too_high(self):
     config = {
         'module': 'scalyr_agent.builtin_monitors.syslog_monitor',
         'protocols': 'udp:70000'
     }
     self.assertRaises(Exception, lambda: SyslogMonitor(config, scalyr_logging.getLogger("syslog_monitor[test]")))

 def test_config_protocol_invalid(self):
     config = {
         'module': 'scalyr_agent.builtin_monitors.syslog_monitor',
         'protocols': 'XXX'
     }
     self.assertRaises(Exception, lambda: SyslogMonitor(config, scalyr_logging.getLogger("syslog_monitor[test]")))

 def test_config_protocol_multiple_two(self):
     config = {
         'module': 'scalyr_agent.builtin_monitors.syslog_monitor',
         'protocols': 'tcp, udp'
     }
     self.assertNoException(lambda: SyslogMonitor(config, scalyr_logging.getLogger("syslog_monitor[test]")))

__author__ = '[email protected]'

import os
import threading
import time
import sys

import scalyr_agent.scalyr_logging as scalyr_logging
import scalyr_agent.util as scalyr_util

from scalyr_agent import json_lib
from scalyr_agent.util import StoppableThread
from scalyr_agent.log_processing import LogMatcher, LogFileProcessor
from scalyr_agent.agent_status import CopyingManagerStatus

log = scalyr_logging.getLogger(__name__)


class CopyingParameters(object):
    """Tracks the copying parameters that should be used for sending requests to Scalyr and adjusts them over time
    according to success and failures of requests.

    The copying parameters boil down to two parameters:  the maximum number of bytes that can be sent to Scalyr
    in a request (self.current_bytes_allowed_to_send), and the minimum amount of time to wait between requests
    (self.current_sleep_interval).

    This implements a truncated binary backoff algorithm.
    """
    def __init__(self, configuration):
        """Initialize the parameters based on the thresholds defined in the configuration file.

 def test_initialize_monitor(self):
     monitor = ProcessMonitor(self.config_commandline, scalyr_logging.getLogger("syslog_monitor[test]"))
     self.assertEqual(monitor._ProcessMonitor__metrics_history, defaultdict(dict))
     self.assertEqual(monitor._ProcessMonitor__aggregated_metrics, {})

 def test_sibling_modules(self):
     child = scalyr_logging.getLogger('external_package.my_monitor')
     child.info('Sibling statement')
     self.assertTrue(self.__log_contains('Sibling statement'))