def generate_traffic(self, activity, legend_keys, report_type):
        """ Generate traffic data during the time the user was logged-in.
        """
        cache = {}
        combined_activity = []
        for event in activity:
            # handle dns names in host along with IP address
            host = event[0].split('|', 1)[0]

            timefilter = TimeFilter(string_to_datetime(event[1]),
                                    string_to_datetime(event[2]))
            # if event occurs in less than a minute, add extra minute to report
            while len(timefilter.profiler_minutes()) == 1:
                timefilter.end += datetime.timedelta(minutes=1)

            # normalize times to minute increments
            mins = timefilter.profiler_minutes()
            tf = TimeFilter(mins[0], mins[-1])

            if self.options.usecache and report_type == 'timeseries':
                # only consider a hit when whole time period is covered
                minutes = tf.profiler_minutes(astimestamp=True)

                if host in cache and all(t in cache[host] for t in minutes):
                    data = [cache[host][t] for t in minutes]
                else:
                    legend, data = self.traffic_report(host, tf, report_type)
                    # store results in cache by host->times->data
                    cache.setdefault(host, {}).update((int(x[0]), x) for x in data)
            else:
                legend, data = self.traffic_report(host, tf, report_type)

            if data:
                if self.options.aggregate and report_type == 'timeseries':
                    # generate running averages over data samples received
                    # first convert empty strings to zeros, then run averages
                    columns = map(lambda c: [0 if x == '' else x for x in c],
                                                                itertools.izip(*data))
                    aggmap = [x[1] for x in TCOLUMNS]
                    aggregates = [aggmap[i](x) for i, x in enumerate(columns)]
                    combined_activity.append(list(event) + aggregates)
                elif report_type == 'timeseries' or report_type == 'summary':
                    # create entry for each element in report
                    for row in data:
                        r = ['--' if x == '' else x for x in row]
                        combined_activity.append(list(event) + r)
                else:
                    raise RuntimeError('unknown report type: %s' % report_type)

            else:
                # populate result with blanks
                combined_activity.append(list(event) + ['--'] * len(legend))

        traffic_legend = [c.key for c in legend]

        legend = legend_keys + traffic_legend
        return legend, combined_activity

 def test_resolution(self):
     groupby = self.profiler.groupbys.host
     columns = [self.profiler.columns.key.host_ip,
                self.profiler.columns.value.avg_bytes,
                self.profiler.columns.value.avg_pkts]
     sort_col = self.profiler.columns.value.avg_bytes
     trafficexpr = TrafficFilter("host 10/8")
     resolutions = [["1min", "last 5 min"],
                    ["15min", "last 1 hour"],
                    ["hour", "last 4 hours"],
                    ["6hour", "last 1 day"],
                    ["day", "last 1 week"],
                    ["3600", "last 4 hours"],  # hour resolution
                    ["60", "last 5 min"],      # minute resolution
                    #["week", "last 4 weeks"],
                    #["month", "last 12 months"],
                    #Commented values blow up with a
                    #E       RvbdHTTPException: 400 Unknown time resolution.
                    ]
     for (resolution, duration) in resolutions:
         timerange = TimeFilter.parse_range(duration)
         with TrafficSummaryReport(self.profiler) as rep:
             rep.run(groupby, columns,
                     sort_col, timerange,
                     trafficexpr, resolution=resolution)

    def main(self):
        if self.options.listinterfacegroups:
            self.list_interface_groups(self.options.host, self.options.sshusername,
                                       self.options.sshpassword)
            return

        if self.options.listhostgroups:
            self.list_host_groups(NetProfiler(self.options.host, auth=self.auth))
            return

        try:
            timefilter = TimeFilter.parse_range(self.options.timefilter)
        except ValueError:
            print("Could not parse time filter expression.")
            return

        profiler = NetProfiler(self.options.host, auth=self.auth)

        if not self.options.clean:
            print("Reporting on the period: {0}\n"
                  "Using the traffic filter: {1}\n"
                  "Calculating data at percentile {2}\n"
                  "Averaging based on buckets of {3} minutes"
                  "".format(self.options.timefilter,
                            self.options.trafficfilter,
                            self.options.percentile,
                            self.options.buckettime))
            if self.options.graph:
                print("Saving a graph to {}".format(self.options.graph))
            print()

        trafficfilter = TrafficFilter(self.options.trafficfilter)

        self.report_item(profiler, timefilter, trafficfilter,
                         self.options.buckettime, self.options.percentile)

    def main(self):

        self.timefilter = TimeFilter.parse_range(self.options.timefilter)
        if self.options.trafficexpr:
            self.trafficexpr = TrafficFilter(self.options.trafficexpr)
        else:
            self.trafficexpr = None

        with MultiQueryReport(self.netprofiler) as report:
            report.run(template_id=int(self.options.template_id),
                       timefilter=self.timefilter,
                       trafficexpr=self.trafficexpr)
            print('Report Template {id} successfully run.'
                  .format(id=self.options.template_id))

            self.netprofiler.conn.download(
                '/api/profiler/1.6/reporting/reports/{id}/view.{fmt}'
                .format(id=report.id, fmt=self.options.fmt),
                path=self.options.pathname,
                overwrite=True
            )

            print('Completed Report {id} downloaded to {path}.'
                  .format(id=report.id,
                          path=self.options.pathname))

    def main(self):
        """ Setup query and run report with default column set
        """
        if self.options.timerange:
            timefilter = TimeFilter.parse_range(self.options.timerange)
        else:
            timefilter = TimeFilter(self.options.time0, self.options.time1)
        trafficexpr = TrafficFilter(self.options.trafficexpr)

        columns = [self.netprofiler.columns.key.srv_host_ip,
                   self.netprofiler.columns.key.app_info,
                   self.netprofiler.columns.key.start_time,
                   self.netprofiler.columns.key.end_time,
                   self.netprofiler.columns.value.s2c_total_bytes,
                   self.netprofiler.columns.value.s2c_total_pkts,
                   self.netprofiler.columns.value.response_time,
                   self.netprofiler.columns.value.server_delay]

        report = TrafficFlowListReport(self.netprofiler)
        report.run(columns, timefilter=timefilter, trafficexpr=trafficexpr)
        data = report.get_data()
        report.delete()

        headers = [c.key for c in columns]

        Formatter.print_table(data, headers)

    def test_traffic_summary_report(self):
        groupby = self.profiler.groupbys.host
        columns = [self.profiler.columns.key.host_ip,
                   self.profiler.columns.value.avg_bytes,
                   self.profiler.columns.value.avg_pkts]
        sort_col = self.profiler.columns.value.avg_bytes
        timerange = TimeFilter.parse_range("last 1 h")
        trafficexpr = TrafficFilter("host 10/8")

        with TrafficSummaryReport(self.profiler) as rep:
            rep.run(groupby, columns,
                    sort_col, timerange,
                    trafficexpr)
            legend = rep.get_legend()
            self.assertEqual(len(legend), 3)
            legend = rep.get_legend(columns=[self.profiler.columns.key.host_ip,
                                             self.profiler.columns.value.avg_bytes])
            self.assertEqual(len(legend), 2)
            self.assertEqual(legend[0].key, 'host_ip')
            self.assertEqual(legend[1].key, 'avg_bytes')

            data = rep.get_data()
            if data:
                self.assertEqual(len(data[0]), 3)

            #check that data is refetched from cache
            data = rep.get_data()

            data = rep.get_data(columns=[self.profiler.columns.key.host_ip,
                                         self.profiler.columns.value.avg_bytes])
            if data:
                self.assertEqual(len(data[0]), 2)

    def main(self):
        # groupby validation should be part of validate_args, but there
        # is no NetProfiler initialized at that part of the initialization
        try:
            self.groupby = self.netprofiler.groupbys[self.options.groupby]
        except KeyError:
            if self.options.groupby not in self.netprofiler.groupbys.values():
                self.parser.error('Invalid groupby chosen.')
            else:
                self.groupby = self.options.groupby

        self.timefilter = TimeFilter.parse_range(self.options.timefilter)
        if self.options.trafficexpr:
            self.trafficexpr = TrafficFilter(self.options.trafficexpr)
        else:
            self.trafficexpr = None

        with TrafficSummaryReport(self.netprofiler) as report:
            report.run(columns=self.options.columns.split(','),
                       groupby=self.groupby,
                       sort_col=self.options.sortby,
                       timefilter=self.timefilter,
                       trafficexpr=self.trafficexpr,
                       centricity=self.centricity,
                       resolution='auto')
            data = report.get_data()
            legend = [c.label for c in report.get_legend()]

        self.print_data(data, legend)

    def main(self):
        """ Setup query and run report with default column set
        """
        if self.options.timerange:
            timefilter = TimeFilter.parse_range(self.options.timerange)
        else:
            timefilter = TimeFilter(self.options.time0, self.options.time1)

        if self.options.trafficexpr:
            trafficexpr = TrafficFilter(self.options.trafficexpr)
        else:
            trafficexpr = None

        legend_columns, all_data = self.identity_report(timefilter=timefilter,
                                                        trafficexpr=trafficexpr,
                                                        testfile=self.options.testfile)

        legend, activity = self.analyze_login_data(all_data, legend_columns)

        if activity and self.options.timeseries_report:
            headers, tbl_data = self.generate_traffic(activity, legend, 'timeseries')
        elif activity and self.options.summary_report:
            headers, tbl_data = self.generate_traffic(activity, legend, 'summary')
        else:
            headers = ('Host IP', 'Login Time', 'Logout Time', 'Duration')
            tbl_data = [(x[0], format_time(x[1]), format_time(x[2]), x[3])
                                                                for x in activity]

        if self.options.csv:
            Formatter.print_csv(tbl_data, headers)
        elif self.options.tsv:
            Formatter.print_csv(tbl_data, headers, delim='\t')
        else:
            Formatter.print_table(tbl_data, headers)

    def main(self):
        # Create and run a traffic summary report of all hosts in use
        # and then take that data and send it to a specified host group
        report = TrafficSummaryReport(self.netprofiler)

        # Run the report
        report.run(
            groupby=self.netprofiler.groupbys.host,
            columns=[self.netprofiler.columns.key.host_ip,
                     self.netprofiler.columns.key.group_name],
            sort_col=self.netprofiler.columns.key.group_name,
            timefilter=TimeFilter.parse_range(self.options.timefilter),
            trafficexpr=TrafficFilter(self.options.trafficexpr)
        )

        # Store the report's data
        data = report.get_data()
        # Grab the type_name and group_name from options.group
        (type_name, group_name) = self.options.group.split(':', 1)
        # Create an array to store the new config data
        new_config_entries = []

        # Using data from the report, put it in config-readable format.
        for i in range(len(data)):
            new_config_entries.append({'cidr': data[i][0] + '/32',
                                       'name': group_name})
        # Make sure that if there were no entries returned,
        # we don't overwrite the old data
        if len(new_config_entries) == 0:
            print('ERROR: Report returned zero hosts for supplied parameters')
            return

        # Get the ID of the host type specified by name
        host_types = self.netprofiler.api.host_group_types.get_all()
        target_type_id = -1
        for i, host_type in enumerate(host_types):
            if type_name == host_type['name']:
                target_type_id = host_type['id']
                break
        # If target_type_id is still -1, then we didn't find that host
        if target_type_id == -1:
            print('ERROR: Host Group Type: "' + type_name + '" was not found.')
            return

        # Get the current config from the target host group
        config = self.netprofiler.api.host_group_types.get_config(target_type_id)
        old_config_size = len(config)
        # If the append flag is not true,
        # remove all entries in config matching group_name
        if self.options.append is False:
            config = filter(lambda a: a['name'] != group_name, config)

        config.extend(new_config_entries)
        new_config_size = len(config)
        self.netprofiler.api.host_group_types.set_config(target_type_id, config)
        print("Successfully updated type: " + type_name +
              ", group: " + group_name)
        print("The old config had " + str(old_config_size) +
              " elements. It now has " + str(new_config_size) + " elements.\n")

    def test_timefilter(self):
        tfilter = TimeFilter.parse_range('9:01:36 to 10:04:39')

        testtime = tfilter.start.replace(minute=33, second=59)
        self.assertTrue(tfilter.compare_time(testtime))
        testtime = tfilter.end.replace(minute=44)
        self.assertFalse(tfilter.compare_time(testtime))

        minutes = tfilter.profiler_minutes()
        self.assertEqual(len(minutes), 64)
        minutes = tfilter.profiler_minutes(astimestamp=True)
        self.assertEqual(len(minutes), 64)
        minutes = tfilter.profiler_minutes(aslocal=True)
        self.assertEqual(len(minutes), 64)

        tfilter = TimeFilter.parse_range('9:01:36 to 9:02:33')
        minutes = tfilter.profiler_minutes()
        self.assertEqual(len(minutes), 1)

    def main(self):
        netprof = self.netprofiler

        timefilter = TimeFilter.parse_range(self.options.timefilter)

        # Create and run a traffic summary report of all server ports in use
        report = TrafficSummaryReport(netprof)

        # Run the report
        report.run(
            groupby=netprof.groupbys.port,
            columns=[netprof.columns.key.protoport,
                     netprof.columns.key.protocol,
                     netprof.columns.key.port,
                     netprof.columns.value.avg_bytes],
            sort_col=netprof.columns.value.avg_bytes,
            timefilter=timefilter)

        # Retrieve and print data
        ports_data = report.get_data()[:int(self.options.N)]
        report.delete()

        # Now create a new report using the ports_data
        report = TrafficTimeSeriesReport(netprof)

        # The format the query_columns for 'ports' is:
        #    'ports' = [{'name': 'tcp/80'},
        #               {'name': 'tcp/443'},
        #               {'name': 'icmp/0'}]
        # For most protocols, this works just fine from the report data,
        # but for icmp the result from data is 'icmp/0/0' -- where the two
        # zeros are type and code.  This doesn't work for input to
        # netprofiler, it expects type and code to be smushed into a single
        # 16-bit number (type << 8 | code).
        query_columns = []
        for (protoport, protocol, port, avgbytes) in ports_data:
            if protoport.startswith('icmp'):
                protoport = 'icmp/%s' % (port)

            query_columns.append({'name': protoport})

        # Run the report
        report.run(columns=[netprof.columns.key.time,
                            netprof.columns.value.avg_bytes],
                   resolution='1 min',
                   query_columns_groupby='ports',
                   query_columns=query_columns,
                   timefilter=timefilter)

        # Get the data!
        data = report.get_data()
        Formatter.print_table(
            data, padding=1,
            headers=(['time'] + [q['name'] for q in query_columns]))

    def test_identity_report(self):
        timerange = TimeFilter.parse_range('last 30 m')

        with IdentityReport(self.profiler) as report:
            report.run(timefilter=timerange)
            legend = report.get_legend()
            data = report.get_data()
            keys = [c.key for c in legend]
            self.assertTrue('time' in keys)
            self.assertTrue('username' in keys)
            if data:
                self.assertEqual(len(data[0]), 9)

 def test_report_with_area(self):
     groupby = self.profiler.groupbys.host
     columns = [self.profiler.columns.key.host_ip,
                self.profiler.columns.value.avg_bytes,
                self.profiler.columns.value.avg_pkts]
     sort_col = self.profiler.columns.value.avg_bytes
     timerange = TimeFilter.parse_range("last 1 h")
     trafficexpr = TrafficFilter("host 10/8")
     area = self.profiler.areas.vxlan_tenant
     with TrafficSummaryReport(self.profiler) as rep:
         rep.run(groupby, columns,
                 sort_col, timerange,
                 trafficexpr, area=area)

    def main(self):
        self.timefilter = TimeFilter.parse_range(self.options.timefilter)
        if self.options.trafficexpr:
            self.trafficexpr = TrafficFilter(self.options.trafficexpr)
        else:
            self.trafficexpr = None

        with HostTimeSeriesReport(self.netprofiler) as report:
            report.run(timefilter=self.timefilter,
                       trafficexpr=self.trafficexpr)
            data = report.get_data()
            legend = [c.label for c in report.get_legend()]

        self.print_data(data, legend)

    def main(self):
        self.timefilter = TimeFilter.parse_range(self.options.timefilter)
        if self.options.trafficexpr:
            self.trafficexpr = TrafficFilter(self.options.trafficexpr)
        else:
            self.trafficexpr = None

        with TrafficOverallTimeSeriesReport(self.netprofiler) as report:
            report.run(columns=self.options.columns.split(','),
                       timefilter=self.timefilter,
                       trafficexpr=self.trafficexpr,
                       centricity=self.centricity)
            data = report.get_data()
            legend = [c.label for c in report.get_legend()]

        self.print_data(data, legend)

    def main(self):
        self.timefilter = TimeFilter.parse_range(self.options.timefilter)
        if self.options.trafficexpr:
            self.trafficexpr = TrafficFilter(self.options.trafficexpr)
        else:
            self.trafficexpr = None

        with TrafficFlowListReport(self.netprofiler) as report:
            report.run(columns=self.options.columns.split(','),
                       sort_col=self.options.sortby,
                       timefilter=self.timefilter,
                       trafficexpr=self.trafficexpr)
            data = report.get_data()
            legend = [c.label for c in report.get_legend()]

        self.print_data(data, legend)

    def main(self):
        # Create and run a traffic summary report of all server ports in use
        # by hosts in 10/8
        report = TrafficOverallTimeSeriesReport(self.netprofiler)

        # Run the report
        report.run(
            columns=[self.netprofiler.columns.key.time,
                     self.netprofiler.columns.value.avg_bytes,
                     self.netprofiler.columns.value.network_rtt],
            timefilter=TimeFilter.parse_range("last 15 m"),
            trafficexpr=TrafficFilter("host 10/8")
        )

        # Retrieve and print data
        data = report.get_data()
        printer = pprint.PrettyPrinter(2)
        printer.pprint(data)

    def test_unsupported_column(self):
        groupby = self.profiler.groupbys.port

        # host_ip shouldn't be included as part of 'port' groupby
        columns = [self.profiler.columns.key.host_ip,
                   self.profiler.columns.value.avg_bytes,
                   self.profiler.columns.value.avg_pkts]
        sort_col = self.profiler.columns.value.avg_bytes

        timerange = TimeFilter.parse_range("last 1 h")
        trafficexpr = TrafficFilter("host 10/8")

        report = TrafficSummaryReport(self.profiler)
        kwds = dict(groupby=groupby,
                    columns=columns,
                    sort_col=sort_col,
                    timefilter=timerange,
                    trafficexpr=trafficexpr)
        self.assertRaises(RvbdException, report.run, None, kwds)

    def main(self):
        # Create and run a traffic summary report of all server ports in use
        # by hosts in 10/8
        report = TrafficSummaryReport(self.netprofiler)

        # Run the report
        report.run(
            groupby=self.netprofiler.groupbys.port,
            columns=[self.netprofiler.columns.key.protoport,
                     self.netprofiler.columns.key.protoport_name,
                     self.netprofiler.columns.value.avg_bytes,
                     self.netprofiler.columns.value.network_rtt],
            sort_col=self.netprofiler.columns.value.avg_bytes,
            timefilter=TimeFilter.parse_range("last 15 m"),
            trafficexpr=TrafficFilter("host 10/8")
        )

        # Retrieve and print data
        data = report.get_data()
        printer = pprint.PrettyPrinter(2)
        printer.pprint(data[:20])

    def test_traffic_flow_list_report(self):
        columns = [self.profiler.columns.key.srv_host_ip,
                   self.profiler.columns.key.app_info,
                   self.profiler.columns.key.start_time,
                   self.profiler.columns.key.end_time,
                   self.profiler.columns.value.s2c_total_bytes,
                   self.profiler.columns.value.s2c_total_pkts,
                   self.profiler.columns.value.response_time,
                   self.profiler.columns.value.server_delay]
        timerange = TimeFilter.parse_range("last 1 h")
        trafficexpr = TrafficFilter("host 10/8")

        with TrafficFlowListReport(self.profiler) as report:
            report.run(columns, timefilter=timerange, trafficexpr=trafficexpr)

            legend = report.get_legend()
            keys = [c.key for c in legend]
            self.assertTrue('app_info' in keys)

            data = report.get_data()
            if data:
                self.assertEqual(len(data[0]), 8)