def __init__(self, server=None, port=None):

        self.server = "localhost"
        self.port = 28015
        self.r = None

        try:
            import rethinkdb as r
        except ImportError:
            from sys import exit
            print "The rethinkdb client driver is required for this object"
            exit()

        if server:
            self.server = server

        if port:
            self.port = port

        try:
            # Top level objects
            r.connect(self.server, self.port).repl()
            self.r = r

        except r.errors.RqlDriverError:
            from sys import exit
            print "WARNING.  Could not connect to %s port %s" % (self.server, self.port)
            exit()

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-r", "--rethinkdb-host", default="localhost:28015")
    parser.add_argument("-m", "--machine-id", default=socket.gethostname())
    args = parser.parse_args()
    host, port = args.rethinkdb_host.split(":")

    r.connect(host, port).repl()
    try:
        r.db("logcentral")
    except r.ReqlOpFailedError:
        r.db_create("logcentral").run()

    db = r.db("logcentral")

    if 'cursor_state' not in db.table_list().run():
        r.db("logcentral").table_create("cursor_state").run()

    if 'log' not in db.table_list().run():
        r.db("logcentral").table_create("log").run()

    cursor_table = r.db("logcentral").table('cursor_state')
    log_table = r.db("logcentral").table('log')

    c = cursor_table.get(args.machine_id).run()
    c = None if c is None else c['cursor']

    for line in yield_log_lines(c):
        cursor, data = prepare_for_table(line, args.machine_id)
        log_table.insert(data).run()
        cursor_table.insert({"id": args.machine_id, "cursor": cursor}, durability="soft", conflict="replace").run()

def connect_db():
    """connecting to rethinkDB"""
    r.connect('localhost', 28015).repl()
    connection = r.connect(host='localhost',
                           port=28015,
                           db='fbscrap')
    return connection

def setup_rethinkdb():
    import rethinkdb as r
    r.connect( "localhost", 28015).repl()
    try:
        r.db_create("nonpublic").run()
    except:
        pass
    try:
        r.db_create("public").run()
    except:
        pass
    db = r.db("public")
    dbs_and_tables = {'nonpublic': ['third_party_creds', 'subscribers', 'users', 'sessions'], 'public': ['crawling_instructions', 'apps', 'police_internal_affairs_cases', 'police_internal_affairs_allegations', 'organizations', 'tables', 'queries']}
    
    for database in dbs_and_tables.keys():
        try:
            r.db_create(database).run()
        except:
            pass
        db = r.db(database)
        tables_needed = dbs_and_tables[database]
        existing_tables = db.table_list().run()
        tables_to_create = set(tables_needed) - set(existing_tables) # remove existing tables from what we need
        for table in tables_to_create:
            db.table_create(table).run()
    for table in dbs_and_tables['public']:
        #tables_ids = [item['id'] for item in r.db('public').table('tables').run()]
        #if not table in tables_ids:
        if 'police' in table:
            category = "policing"
        else:
            category = "People's NSA"
        r.db('public').table('tables').insert({'id': table, 'name': table.replace('_', ' ').capitalize(), 'categories': [category]}, conflict='update').run()

 def __init__(self):
     r.connect('builder', 28015).repl()
     self.db = r.db('leevalley')
     #if 'sessions' not in self.db.tableList().run():
     #    self.sessions_table = self.db.table_create('sessions').run()
     #else:
     self.sessions_table = self.db.table('sessions')

    def test_setup_db(self):
        """ Test creation of a db and tables """
        # test that the 'TEST' database doesn't exist
        with rethinkdb.connect(host='localhost', port=28015) as conn:
            db_list = rethinkdb.db_list().run(conn)
            self.assertTrue('TEST' not in db_list)

        creations = self.run_setup_db()

        # confirm the correct tables were created
        self.assertSetEqual(creations,
            set(template.test_dataset.keys()+template.test_tables))

        with rethinkdb.connect(host='localhost', port=28015) as conn:
            # test that the 'TEST' database was created
            db_list = rethinkdb.db_list().run(conn)
            self.assertTrue('TEST' in db_list)
            conn.use('TEST')

            # test that the 'test' table was created
            table_list = rethinkdb.table_list().run(conn)
            self.assertEqual(len(table_list),
                len(template.test_dataset.keys()+template.test_tables))
            self.assertTrue(template.test_dataset.keys()[0] in table_list)

            # test that the data is correct by checking columns
            data = [row for row in rethinkdb.table(
                template.test_dataset.keys()[0]).run(conn)]
            with open(template.test_json) as f:
              self.assertSetEqual(
                  set(data[0].keys())-set([u'id']),
                  set(json.loads(f.read())[0].keys()))

        self.run_clear_test_db()

def main():
    import rethinkdb as r
    from rethinkdb.errors import RqlRuntimeError

    # Lib para auxilio na insercao de dados de teste
    from faker import Factory
    fake = Factory.create('pt_BR')

    # Conecta ao banco local
    r.connect(HOST, PORT).repl()

    try:
        r.db_drop(DBNAME).run()
    except RqlRuntimeError:
        pass

    # Cria o banco de dados
    r.db_create(DBNAME).run()

    # Cria a tabela
    r.db(DBNAME).table_create(TABLENAME).run()

    # Insere os registros na tabela
    for frase in range(TOTAL_FRASES):
        reg = {
            'id': frase,
            'frase': fake.text(),
            'autor': fake.name()
        }
        r.db(DBNAME).table(TABLENAME).insert(reg).run()

def index():

    import rethinkdb as r
    r.connect('localhost', 28015).repl()

    try:
        r.db_create('wlps').run()
    except RqlRuntimeError:
        pass

    try:
        r.db('wlps').table_create('episode').run()
    except RqlRuntimeError:
        pass

    try:
        r.db('wlps').table_create('show').run()
    except RqlRuntimeError:
        pass

    try:
        r.db('wlps').table_create('notifications').run()
    except RqlRuntimeError:
        pass

    try:
        r.db('wlps').table_create('queue').run()
    except RqlRuntimeError:
        pass

def load_data_dirs(user, dirs, state_id):
    try:
        r.connect('localhost', 30815, db='materialscommons').repl()
        for directory in dirs:
            load_directory(user, directory, state_id)
    except Exception as exc:
        raise load_data_dirs.retry(exc=exc)

 def test_start(self):
     assert not dbc.alive()
     dbc.start()
     try:
         r.connect()
     except r.errors.ReqlDriverError:
         assert False

def main():
    # connect rethinkdb
    rethinkdb.connect("localhost", 28015, "mysql")
    try:
        rethinkdb.db_drop("mysql").run()
    except:
        pass
    rethinkdb.db_create("mysql").run()

    tables = ["dept_emp", "dept_manager", "titles",
              "salaries", "employees", "departments"]
    for table in tables:
        rethinkdb.db("mysql").table_create(table).run()

    stream = BinLogStreamReader(
        connection_settings=MYSQL_SETTINGS,
        blocking=True,
        only_events=[DeleteRowsEvent, WriteRowsEvent, UpdateRowsEvent],
    )

    # process Feed
    for binlogevent in stream:
        if not isinstance(binlogevent, WriteRowsEvent):
            continue

        for row in binlogevent.rows:
            if not binlogevent.schema == "employees":
                continue

            vals = {}
            vals = {str(k): str(v) for k, v in row["values"].iteritems()}
            rethinkdb.table(binlogevent.table).insert(vals).run()

    stream.close()

    def parse_message(self, message):
        message = json.loads(message['data'])
        rethinkdb.connect('localhost', 28015).repl()
        log_db = rethinkdb.db('siri').table('logs')

        data = {
            'channel': message['channel'],
            'timestamp': rethinkdb.now(),
            'user': message['user'],
            'content': message['content'],
            'server': message['server'],
            'bot': message['bot']
        }

        log_db.insert(data).run()

        urls = re.findall(url_re, message['content'])
        if urls:
            for url in urls:
                urldata = {
                    'url': url,
                    'user': message['user'],
                    'channel': message['channel'],
                    'server': message['server'],
                    'bot': message['bot'],
                    'timestamp': rethinkdb.now()
                }
                gevent.spawn(self.parse_url, urldata)

        data['timestamp'] = datetime.datetime.utcnow()
        self.red.publish('irc_chat', json.dumps(data, default=json_datetime))

    def parse_url(self, urldata):
        print 'GOT URL: %s' % urldata
        allowed_types = ['text', 'audio', 'image']
        video_hosts = ['www.youtube.com', 'youtube.com', 'vimeo.com', 'www.vimeo.com', 'youtu.be']
        
        r = requests.get(urldata['url'], timeout=5)
        if r.status_code == 200:
            content_type = r.headers['content-type'].split('/')[0]
            
            if content_type not in allowed_types:
                return None
            
            if content_type == 'text':
                parse = urlparse.urlparse(urldata['url'])
                if parse.hostname in video_hosts:
                    urldata['type'] = 'video'
                else:
                    urldata['type'] = 'website'

                try:
                    urldata['title'] = lxml.html.parse(urldata['url']).find(".//title").text
                except:
                    urldata['title'] = "No Title"
                
            else:
                urldata['title'] = content_type.title()
                urldata['type'] = content_type

            rethinkdb.connect('localhost', 28015).repl()
            url_db = rethinkdb.db('siri').table('urls')
            url_db.insert(urldata).run()

            urldata['timestamp'] = datetime.datetime.utcnow()
            self.red.publish('irc_urls', json.dumps(urldata, default=json_datetime))

def go():
    with except_printer():
        r.connect(host="localhost", port="123abc")
    with except_printer():
        r.expr({'err': r.error('bob')}).run(c)
    with except_printer():
        r.expr([1,2,3, r.error('bob')]).run(c)
    with except_printer():
        (((r.expr(1) + 1) - 8) * r.error('bob')).run(c)
    with except_printer():
        r.expr([1,2,3]).append(r.error('bob')).run(c)
    with except_printer():
        r.expr([1,2,3, r.error('bob')])[1:].run(c)
    with except_printer():
        r.expr({'a':r.error('bob')})['a'].run(c)
    with except_printer():
        r.db('test').table('test').filter(lambda a: a.contains(r.error('bob'))).run(c)
    with except_printer():
        r.expr(1).do(lambda x: r.error('bob')).run(c)
    with except_printer():
        r.expr(1).do(lambda x: x + r.error('bob')).run(c)
    with except_printer():
        r.branch(r.db('test').table('test').get(0)['a'].contains(r.error('bob')), r.expr(1), r.expr(2)).run(c)
    with except_printer():
        r.expr([1,2]).reduce(lambda a,b: a + r.error("bob")).run(c)

def conn_rethinkdb(host, port, auth_key):
    """connect to rethinkdb"""
    try:
        r.connect(host, port, auth_key=auth_key).repl()
    except ReqlDriverError as error:
        print "Error connecting to RethinkDB:", error
        exit(1)

def rethinkdb():
    """Prepare database and table in RethinkDB"""
    from rethinkdb.errors import ReqlOpFailedError, ReqlRuntimeError
    conn = r.connect(host=conf.RethinkDBConf.HOST)

    # Create database
    try:
        r.db_create(conf.RethinkDBConf.DB).run(conn)
        click.secho('Created database {}'.format(conf.RethinkDBConf.DB),
                    fg='yellow')
    except ReqlOpFailedError:
        click.secho('Database {} already exists'.format(conf.RethinkDBConf.DB),
                    fg='green')

    # Create table 'domains'
    conn = r.connect(host=conf.RethinkDBConf.HOST,
                     db=conf.RethinkDBConf.DB)
    try:
        r.table_create('domains', durability=conf.RethinkDBConf.DURABILITY).\
            run(conn)
        click.secho('Created table domains', fg='yellow')
    except ReqlOpFailedError:
        click.secho('Table domains already exists', fg='green')
    
    # Create index on domains.name
    try:
        r.table('domains').index_create('name').run(conn)
        click.secho('Created index domains.name', fg='yellow')
    except ReqlRuntimeError:
        click.secho('Index domains.name already exists', fg='green')

def check_db():
    r.connect(properties.get('RETHINK_HOST'), properties.get('RETHINK_PORT')).repl()
    
    if 'relayr' in r.db_list().run():
        return True

    return False

def update():
    print "update() begin"

    # Connect to ReThinkDB
    r.connect("localhost", 28015).repl()

    # Get XML data from remote API and parse it
    url = "http://api.ezaxess.com/v2/pd/longbeach/crimes/all"
    root = ElementTree.fromstring(requests.get(url).content)

    for item in root.findall("item"):
        # Construct Python dictionary from XML nodes
        incident = {
            "id": int(item.find("id").text),
            "item_id": int(item.find("id").text),
            "case_id": int(item.find("case_number").text),
            "incident_id": int(item.find("incident_id").text),
            "title": item.find("title").text.strip(),
            "description": item.find("description").text.strip(),
            "time": dateutil.parser.parse(item.find("date_occured").text),
            "address": item.find("block_address").text.strip(),
            "city": item.find("city").text.strip(),
            "state": item.find("state").text.strip(),
            "latitude": item.find("latitude").text.strip(),
            "longitude": item.find("longitude").text.strip(),
        }

        response = r.db("lbpd").table("incidents").insert(incident, conflict="update").run()

        print (incident["id"], response["inserted"])

    print "update() completed"

def main():

    argparser = argparse.ArgumentParser()
    subparsers = argparser.add_subparsers(help='Firmware MapReduce Controls', dest='command')

    parser_guid_group = subparsers.add_parser("guid_group", help= "Groupby UEFI file GUIDs.")
    parser_object_group = subparsers.add_parser("object_group", help= "Groupby Object hashs.")
    
    parser_vendor_object_sum = subparsers.add_parser("vendor_object_sum", help= "Sum objects by vendor.")
    parser_vendor_content_sum = subparsers.add_parser("vendor_content_sum", help= "Sum content by vendor.")

    parser_vendor_object_count = subparsers.add_parser("vendor_object_count", help= "Count objects by vendor.")
    parser_vendor_content_count = subparsers.add_parser("vendor_content_count", help= "Count content by vendor.")

    parser_vendor_update_count = subparsers.add_parser("vendor_update_count", help= "Count updates by vendor.")
    parser_vendor_products_count = subparsers.add_parser("vendor_product_count", help= "Count products by vendor.")

    args = argparser.parse_args()
    controller = Controller()
    command = "command_%s" % args.command

    r.connect("localhost", 28015).repl()
    db = r.db("uefi")

    command_ptr = getattr(controller, command, None)
    if command_ptr is not None:
        print "Running command (%s)..." % args.command
        begin = time.time()
        db.table("stats").insert(command_ptr(db, args).limit(99999)).run()
        end = time.time()
        print "...finished (%d) seconds." % (end-begin)
    else:
        print "Cannot find command: %s" % command

def test_basic():
    """
    run a db temporarily
    """
    from testre.runner import run

    with run() as the_port:
        port = the_port

        connection = rethinkdb.connect(port=port)

        result = rethinkdb.db('test').table_create('testre').run(connection)
        assert_equals(result['tables_created'], 1)

    with assert_raises(rethinkdb.ReqlDriverError):
        connection = rethinkdb.connect(port=port)

    # a new connection should happen on a new db
    with run(port=port) as the_port:
        assert_equals(port, the_port)

        connection = rethinkdb.connect(port=port)

        assert_equals(rethinkdb.db('test').table_list().run(connection), [])

    with assert_raises(rethinkdb.ReqlDriverError):
        connection = rethinkdb.connect(port=port)