def slurp(self):
        """
        :returns: item_list - list of IAM SSH Keypairs.
        :returns: exception_map - A dict where the keys are a tuple containing the
            location of the exception and the value is the actual exception

        """
        self.prep_for_slurp()

        item_list = []
        exception_map = {}
        from security_monkey.common.sts_connect import connect
        for account in self.accounts:
            try:
                account_db = Account.query.filter(Account.name == account).first()
                account_number = account_db.number
                ec2 = connect(account, 'ec2')
                regions = ec2.get_all_regions()
            except Exception as e:  # EC2ResponseError
                # Some Accounts don't subscribe to EC2 and will throw an exception here.
                exc = BotoConnectionIssue(str(e), 'keypair', account, None)
                self.slurp_exception((self.index, account), exc, exception_map, source="{}-watcher".format(self.index))
                continue

            for region in regions:
                app.logger.debug("Checking {}/{}/{}".format(Keypair.index, account, region.name))

                try:
                    rec2 = connect(account, 'ec2', region=region)
                    kps = self.wrap_aws_rate_limited_call(
                        rec2.get_all_key_pairs
                    )
                except Exception as e:
                    if region.name not in TROUBLE_REGIONS:
                        exc = BotoConnectionIssue(str(e), 'keypair', account, region.name)
                        self.slurp_exception((self.index, account, region.name), exc, exception_map,
                                             source="{}-watcher".format(self.index))
                    continue

                app.logger.debug("Found {} {}".format(len(kps), Keypair.i_am_plural))
                for kp in kps:

                    if self.check_ignore_list(kp.name):
                        continue

                    arn = 'arn:aws:ec2:{region}:{account_number}:key-pair/{name}'.format(
                        region=region.name,
                        account_number=account_number,
                        name=kp.name)

                    item_list.append(KeypairItem(region=region.name, account=account, name=kp.name, arn=arn,
                                                 config={
                                                     'fingerprint': kp.fingerprint,
                                                     'arn': arn,
                                                     'name': kp.name
                                                 }))
        return item_list, exception_map

    def slurp(self):
        """
        :returns: item_list - list of S3 Buckets.
        :returns: exception_map - A dict where the keys are a tuple containing the
            location of the exception and the value is the actual exception
        """
        item_list = []
        exception_map = {}

        from security_monkey.common.sts_connect import connect
        for account in self.accounts:

            try:
                s3conn = connect(account, 's3', calling_format=OrdinaryCallingFormat())
                all_buckets = s3conn.get_all_buckets()
            except Exception as e:
                exc = BotoConnectionIssue(str(e), 's3', account, None)
                self.slurp_exception((self.index, account), exc, exception_map)
                continue

            for bucket in all_buckets:
                app.logger.debug("Slurping %s (%s) from %s" % (self.i_am_singular, bucket.name, account))
                
                ### Check if this bucket is on the Ignore List ###
                ignore_item = False
                for ignore_item_name in IGNORE_PREFIX[self.index]:
                    if bucket.name.lower().startswith(ignore_item_name.lower()):
                        ignore_item = True
                        break
      
                if ignore_item:
                    continue                

                try:
                    region = self.translate_location_to_region(bucket.get_location())
                    if region == '':
                        s3regionconn = connect(account, 's3', calling_format=OrdinaryCallingFormat())
                        region = 'us-east-1'
                    else:
                        s3regionconn = connect(account, 's3', region=region, calling_format=OrdinaryCallingFormat())
                    bhandle = s3regionconn.get_bucket(bucket)
                    s3regionconn.close()
                except Exception as e:
                    exc = S3PermissionsIssue(bucket.name)
                    # Unfortunately, we can't get the region, so the entire account
                    # will be skipped in find_changes, not just the bad bucket.
                    self.slurp_exception((self.index, account), exc, exception_map)
                    continue
    
                app.logger.debug("Slurping %s (%s) from %s/%s" % (self.i_am_singular, bucket.name, account, region))
                bucket_dict = self.conv_bucket_to_dict(bhandle, account, region, bucket.name, exception_map)

                item = S3Item(account=account, region=region, name=bucket.name, config=bucket_dict)
                item_list.append(item)

        return item_list, exception_map

  def slurp(self):
    """
    :returns: item_list - list of IAM SSH Keypairs.
    :returns: exception_map - A dict where the keys are a tuple containing the
        location of the exception and the value is the actual exception

    """
    item_list = []
    exception_map = {}
    from security_monkey.common.sts_connect import connect
    for account in self.accounts:
      try:
        ec2 = connect(account, 'ec2')
        regions = ec2.get_all_regions()
      except Exception as e:  # EC2ResponseError
        # Some Accounts don't subscribe to EC2 and will throw an exception here.
        exc = BotoConnectionIssue(str(e), 'keypair', account, None)
        self.slurp_exception((self.index, account), exc, exception_map)
        continue

      for region in regions:
        app.logger.debug("Checking {}/{}/{}".format(Keypair.index, account, region.name))

        try:
          rec2 = connect(account, 'ec2', region=region)
          kps = self.wrap_aws_rate_limited_call(
            rec2.get_all_key_pairs
          )
        except Exception as e:
          if region.name not in TROUBLE_REGIONS:
            exc = BotoConnectionIssue(str(e), 'keypair', account, region.name)
            self.slurp_exception((self.index, account, region.name), exc, exception_map)
          continue

        app.logger.debug("Found {} {}".format(len(kps), Keypair.i_am_plural))
        for kp in kps:

          ### Check if this Keypair is on the Ignore List ###
          ignore_item = False
          for ignore_item_name in IGNORE_PREFIX[self.index]:
            if kp.name.lower().startswith(ignore_item_name.lower()):
              ignore_item = True
              break

          if ignore_item:
            continue

          item_list.append(KeypairItem(region=region.name, account=account, name=kp.name,
                                       config={
                                           'fingerprint': kp.fingerprint
                                       }))
    return item_list, exception_map

 def get_all_certs_in_region(self, account, region, exception_map):
     from security_monkey.common.sts_connect import connect
     import traceback
     all_certs = []
     app.logger.debug("Checking {}/{}/{}".format(self.index, account, region))
     try:
         iamconn = connect(account, 'iam', region=region)
         marker = None
         while True:
             certs = self.wrap_aws_rate_limited_call(
                 iamconn.list_server_certs,
                 marker=marker
             )
             all_certs.extend(certs.server_certificate_metadata_list)
             if certs.is_truncated == u'true':
                 marker = certs.marker
             else:
                 break
     except Exception as e:
         app.logger.warn(traceback.format_exc())
         if region not in TROUBLE_REGIONS:
             exc = BotoConnectionIssue(str(e), self.index, account, region)
             self.slurp_exception((self.index, account, region), exc, exception_map)
     app.logger.info("Found {} {} from {}/{}".format(len(all_certs), self.i_am_plural, account, region))
     return all_certs

    def slurp(self):
        """
        :returns: item_list - list of Redshift Policies.
        :returns: exception_map - A dict where the keys are a tuple containing the
            location of the exception and the value is the actual exception

        """
        self.prep_for_slurp()
        from security_monkey.common.sts_connect import connect
        item_list = []
        exception_map = {}
        for account in self.accounts:
            for region in regions():
                app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name))
                try:
                    redshift = connect(account, 'redshift', region=region)
                    response = self.wrap_aws_rate_limited_call(
                        redshift.describe_clusters
                    )
                    all_clusters = response['DescribeClustersResponse']['DescribeClustersResult']['Clusters']
                except Exception as e:
                    if region.name not in TROUBLE_REGIONS:
                        exc = BotoConnectionIssue(str(e), 'redshift', account, region.name)
                        self.slurp_exception((self.index, account, region.name), exc, exception_map)
                    continue
                app.logger.debug("Found {} {}".format(len(all_clusters), Redshift.i_am_plural))
                for cluster in all_clusters:
                    cluster_id = cluster['ClusterIdentifier']
                    if self.check_ignore_list(cluster_id):
                        continue

                    item = RedshiftCluster(region=region.name, account=account, name=cluster_id, config=dict(cluster))
                    item_list.append(item)

        return item_list, exception_map

    def get_all_subnets(self, **kwargs):
        from security_monkey.common.sts_connect import connect
        conn = connect(kwargs['account_name'], 'vpc', region=kwargs['region'],
                       assumed_role=kwargs['assumed_role'])

        all_subnets = self.wrap_aws_rate_limited_call(conn.get_all_subnets)
        return all_subnets

    def slurp(self):
        """
        :returns: item_list - list of endpoints.
        :returns: exception_map - A dict where the keys are a tuple containing the
            location of the exception and the value is the actual exception

        """
        self.prep_for_slurp()

        item_list = []
        exception_map = {}
        from security_monkey.common.sts_connect import connect
        for account in self.accounts:
            for region in regions():
                app.logger.debug(
                    "Checking {}/{}/{}".format(self.index, account, region.name))
                try:
                    conn = connect(account, 'boto3.ec2.client', region=region)
                    all_vpc_endpoints_resp = self.wrap_aws_rate_limited_call(
                        conn.describe_vpc_endpoints
                    )

                    all_vpc_endpoints = all_vpc_endpoints_resp.get(
                        'VpcEndpoints', [])
                except Exception as e:
                    if region.name not in TROUBLE_REGIONS:
                        exc = BotoConnectionIssue(
                            str(e), self.index, account, region.name)
                        self.slurp_exception(
                            (self.index, account, region.name), exc, exception_map)
                    continue
                app.logger.debug("Found {} {}".format(
                    len(all_vpc_endpoints), self.i_am_plural))

                for endpoint in all_vpc_endpoints:

                    endpoint_name = endpoint.get(u'VpcEndpointId')

                    if self.check_ignore_list(endpoint_name):
                        continue

                    service = endpoint.get('ServiceName', '').split('.')[-1]

                    config = {
                        "id": endpoint.get('VpcEndpointId'),
                        "policy_document": endpoint.get('PolicyDocument', {}),
                        "service_name": endpoint.get('ServiceName'),
                        "service": service,
                        "route_table_ids": endpoint.get('RouteTableIds', []),
                        "creation_time_stamp": str(endpoint.get('CreationTimestamp')),
                        "state": endpoint.get('State'),
                        "vpc_id": endpoint.get('VpcId'),
                    }

                    item = EndpointItem(
                        region=region.name, account=account, name=endpoint_name, config=config, source_watcher=self)
                    item_list.append(item)

        return item_list, exception_map

    def slurp(self):
        """
        :returns: item_list - list of configs.
        :returns: exception_map - A dict where the keys are a tuple containing the
            location of the exception and the value is the actual exception
        """
        self.prep_for_slurp()

        item_list = []
        exception_map = {}
        from security_monkey.common.sts_connect import connect
        for account in self.accounts:
            for region in regions():
                app.logger.debug(
                    "Checking {}/{}/{}".format(self.index, account, region.name))
                if region.name not in AVAILABLE_REGIONS:
                    continue

                try:
                    configService = connect(
                        account, 'boto3.config.client', region=region)
                    response = self.wrap_aws_rate_limited_call(
                        configService.describe_config_rules
                    )
                    config_rules = response.get('ConfigRules', [])
                except Exception as e:
                    app.logger.debug("Exception found: {}".format(e))
                    if region.name not in TROUBLE_REGIONS:
                        exc = BotoConnectionIssue(
                            str(e), self.index, account, region.name)
                        self.slurp_exception(
                            (self.index, account, region.name), exc, exception_map)
                    continue
                app.logger.debug("Found {} {}.".format(
                    len(config_rules), self.i_am_plural))

                for config_rule in config_rules:
                    name = config_rule.get('ConfigRuleName')

                    if self.check_ignore_list(name):
                        continue

                    item_config = {
                        'config_rule': name,
                        'config_rule_arn': config_rule.get('ConfigRuleArn'),
                        'config_rule_id': config_rule.get('ConfigRuleId'),
                        'scope': config_rule.get('Scope', {}),
                        'source': config_rule.get('Source', {}),
                        'imput_parameters': config_rule.get('InputParameters'),
                        'maximum_execution_frequency': config_rule.get('MaximumExecutionFrequency'),
                        'config_rule_state': config_rule.get('ConfigRuleState'),
                    }

                    item = ConfigItem(
                        region=region.name, account=account, name=name,
                        arn=config_rule.get('ConfigRuleArn'), config=item_config)
                    item_list.append(item)

        return item_list, exception_map

    def describe_configuration_recorders(self, **kwargs):
        from security_monkey.common.sts_connect import connect

        config_service = connect(kwargs["account_name"], "boto3.config.client", region=kwargs["region"])
        response = self.wrap_aws_rate_limited_call(config_service.describe_configuration_recorders)
        config_recorders = response.get("ConfigurationRecorders", [])

        return [recorder for recorder in config_recorders if not self.check_ignore_list(recorder.get("name"))]

    def get_all_es_domains_in_region(self, account, region):
        from security_monkey.common.sts_connect import connect
        client = connect(account, "boto3.es.client", region=region)
        app.logger.debug("Checking {}/{}/{}".format(ElasticSearchService.index, account, region.name))
        # No need to paginate according to: client.can_paginate("list_domain_names")
        domains = self.wrap_aws_rate_limited_call(client.list_domain_names)["DomainNames"]

        return client, domains

    def describe_db_instances(self, **kwargs):
        from security_monkey.common.sts_connect import connect
        rds = connect(kwargs['account_name'], 'boto3.rds.client', region=kwargs['region'],
                      assumed_role=kwargs['assumed_role'])

        response = self.wrap_aws_rate_limited_call(rds.describe_db_instances)
        rds_db_instances = response.get('DBInstances')
        return rds_db_instances

    def describe_vpc_peering_connections(self, **kwargs):
        from security_monkey.common.sts_connect import connect
        conn = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'],
                       assumed_role=kwargs['assumed_role'])

        peering_info = self.wrap_aws_rate_limited_call(
            conn.describe_vpc_peering_connections)
        return peering_info

    def describe_route_tables(self, **kwargs):
        from security_monkey.common.sts_connect import connect
        conn = connect(kwargs['account_name'], 'boto3.ec2.client',
                       region=kwargs['region'], assumed_role=kwargs['assumed_role'])

        response = self.wrap_aws_rate_limited_call(conn.describe_route_tables)
        all_route_tables = response.get('RouteTables', [])
        return all_route_tables

    def slurp(self):
        """
        :returns: item_list - list of subnets.
        :returns: exception_map - A dict where the keys are a tuple containing the
            location of the exception and the value is the actual exception

        """
        self.prep_for_slurp()

        item_list = []
        exception_map = {}
        from security_monkey.common.sts_connect import connect
        for account in self.accounts:
            for region in regions():
                app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name))
                try:
                    conn = connect(account, 'vpc', region=region)
                    all_subnets = self.wrap_aws_rate_limited_call(
                        conn.get_all_subnets
                    )
                except Exception as e:
                    if region.name not in TROUBLE_REGIONS:
                        exc = BotoConnectionIssue(str(e), self.index, account, region.name)
                        self.slurp_exception((self.index, account, region.name), exc, exception_map)
                    continue
                app.logger.debug("Found {} {}".format(len(all_subnets), self.i_am_plural))

                for subnet in all_subnets:

                    subnet_name = subnet.tags.get(u'Name', None)
                    if subnet_name:
                        subnet_name = "{0} ({1})".format(subnet_name, subnet.id)
                    else:
                        subnet_name = subnet.id

                    if self.check_ignore_list(subnet_name):
                        continue

                    config = {
                        "name": subnet.tags.get(u'Name', None),
                        "id": subnet.id,
                        "cidr_block": subnet.cidr_block,
                        "availability_zone": subnet.availability_zone,
                        # TODO:
                        # available_ip_address_count is likely to change often
                        # and should be in the upcoming ephemeral section.
                        # "available_ip_address_count": subnet.available_ip_address_count,
                        "defaultForAz": subnet.defaultForAz,
                        "mapPublicIpOnLaunch": subnet.mapPublicIpOnLaunch,
                        "state": subnet.state,
                        "tags": dict(subnet.tags),
                        "vpc_id": subnet.vpc_id
                    }

                    item = SubnetItem(region=region.name, account=account, name=subnet_name, config=config)
                    item_list.append(item)

        return item_list, exception_map

    def describe_network_acls(self, **kwargs):
        from security_monkey.common.sts_connect import connect
        conn = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'],
                       assumed_role=kwargs['assumed_role'])

        networkacls_resp = self.wrap_aws_rate_limited_call(
            conn.describe_network_acls)
        networkacls = networkacls_resp.get('NetworkAcls', [])
        return networkacls

    def slurp(self):
        """
        :returns: item_list - list of Redshift Policies.
        :returns: exception_map - A dict where the keys are a tuple containing the
            location of the exception and the value is the actual exception

        """
        self.prep_for_slurp()
        from security_monkey.common.sts_connect import connect
        item_list = []
        exception_map = {}
        for account in self.accounts:
            account_db = Account.query.filter(Account.name == account).first()
            account_number = account_db.identifier

            for region in regions():
                app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name))
                try:
                    redshift = connect(account, 'redshift', region=region)

                    all_clusters = []
                    marker = None
                    while True:
                        response = self.wrap_aws_rate_limited_call(
                            redshift.describe_clusters,
                            marker=marker
                        )
                        all_clusters.extend(response['DescribeClustersResponse']['DescribeClustersResult']['Clusters'])
                        if response['DescribeClustersResponse']['DescribeClustersResult']['Marker'] is not None:
                            marker = response['DescribeClustersResponse']['DescribeClustersResult']['Marker']
                        else:
                            break

                except Exception as e:
                    if region.name not in TROUBLE_REGIONS:
                        exc = BotoConnectionIssue(str(e), 'redshift', account, region.name)
                        self.slurp_exception((self.index, account, region.name), exc, exception_map,
                                             source="{}-watcher".format(self.index))
                    continue
                app.logger.debug("Found {} {}".format(len(all_clusters), Redshift.i_am_plural))
                for cluster in all_clusters:
                    cluster_id = cluster['ClusterIdentifier']
                    if self.check_ignore_list(cluster_id):
                        continue

                    arn = ARN_PREFIX + ':redshift:{region}:{account_number}:cluster:{name}'.format(
                        region=region.name,
                        account_number=account_number,
                        name=cluster_id)

                    cluster['arn'] = arn

                    item = RedshiftCluster(region=region.name, account=account, name=cluster_id, arn=arn,
                                           config=dict(cluster), source_watcher=self)
                    item_list.append(item)

        return item_list, exception_map

    def slurp(self):
        """
        :returns: item_list - list of IAM Groups.
        :returns: exception_map - A dict where the keys are a tuple containing the
            location of the exception and the value is the actual exception
        """
        self.prep_for_slurp()
        item_list = []
        exception_map = {}

        from security_monkey.common.sts_connect import connect
        for account in self.accounts:

            try:
                iam = connect(account, 'iam')
                groups = self.get_all_groups(iam)
            except Exception as e:
                exc = BotoConnectionIssue(str(e), 'iamgroup', account, None)
                self.slurp_exception((self.index, account, 'universal'), exc, exception_map)
                continue

            for group in groups:
                app.logger.debug("Slurping %s (%s) from %s" % (self.i_am_singular, group.group_name, account))

                if self.check_ignore_list(group.group_name):
                    continue

                item_config = {
                    'group': dict(group),
                    'grouppolicies': {},
                    'users': {}
                }

                ### GROUP POLICIES ###
                group_policies = self.get_all_group_policies(iam, group.group_name)

                for policy_name in group_policies:
                    policy = self.wrap_aws_rate_limited_call(iam.get_group_policy, group.group_name, policy_name)
                    policy = policy.policy_document
                    policy = urllib.unquote(policy)
                    try:
                        policydict = json.loads(policy)
                    except:
                        exc = InvalidAWSJSON(policy)
                        self.slurp_exception((self.index, account, 'universal', group.group_name), exc, exception_map)

                    item_config['grouppolicies'][policy_name] = dict(policydict)

                ### GROUP USERS ###
                group_users = self.get_all_group_users(iam, group['group_name'])
                for user in group_users:
                    item_config['users'][user.arn] = user.user_name

                item = IAMGroupItem(account=account, name=group.group_name, config=item_config)
                item_list.append(item)

        return item_list, exception_map

    def describe_dhcp_options(self, **kwargs):
        from security_monkey.common.sts_connect import connect
        conn = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'],
                       assumed_role=kwargs['assumed_role'])

        dhcp_option_sets_resp = self.wrap_aws_rate_limited_call(
            conn.describe_dhcp_options)
        dhcp_option_sets = dhcp_option_sets_resp.get('DhcpOptions', [])
        return dhcp_option_sets

    def describe_images(self, **kwargs):
        from security_monkey.common.sts_connect import connect
        ec2 = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'],
                      assumed_role=kwargs['assumed_role'])

        response = self.wrap_aws_rate_limited_call(ec2.describe_images,
                                                   Owners=['self'])
        images = response.get('Images')
        return images

    def describe_instances(self, **kwargs):
        from security_monkey.common.sts_connect import connect
        ec2 = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'],
                      assumed_role=kwargs['assumed_role'])

        response = self.wrap_aws_rate_limited_call(
            ec2.describe_instances
        )
        reservations = response.get('Reservations')
        return reservations