def manage_initial_broks_done_brok(self, b):
        if self.con is None:
            return
        logger.info("[Active Directory UI] AD/LDAP: manage_initial_broks_done_brok, go for pictures")

        searchScope = ldap.SCOPE_SUBTREE
        ## retrieve all attributes - again adjust to your needs - see documentation for more options
        #retrieveAttributes = ["userPrincipalName", "thumbnailPhoto", "samaccountname", "email"]

        logger.info("[Active Directory UI] Contacts? %d" % len(self.app.datamgr.get_contacts()))

        for c in self.app.datamgr.get_contacts():
            logger.debug("[Active Directory UI] Doing photo lookup for contact: %s" % c.get_name())
            elts = self.find_contact_entry(c)

            if elts is None:
                logger.warning("[Active Directory UI] No ldap entry for %s" % c.get_name())
                continue

            # Ok, try to get photo from the entry
            try:
                photo = elts[self.photo_attr][0]
                try:
                    p = os.path.join(self.app.photo_dir, c.get_name()+'.jpg')
                    f = open(p, 'wb')
                    f.write(photo)
                    f.close()
                    logger.info("[Active Directory UI] Photo wrote for %s" % c.get_name())
                except Exception, exp:
                    logger.error("[Active Directory UI] Cannot write %s : %s" % (p, str(exp)))
            except KeyError:
                logger.warning("[Active Directory UI] No photo for %s" % c.get_name())

def get_page(name, type):
    global params

    # user = app.check_user_authentication()

    logger.debug("[WebUI-cvhost], get_page for %s, type: '%s'", name, type)

    try:
        currentdir = os.path.dirname(os.path.realpath(__file__))
        configuration_file = "%s/%s.cfg" % (currentdir, type)
        logger.debug("Plugin configuration file: %s", configuration_file)
        scp = config_parser('#', '=')
        z = params.copy()
        z.update(scp.parse_config(configuration_file))
        params = z

        logger.debug("[WebUI-cvhost] configuration loaded.")
        logger.debug("[WebUI-cvhost] configuration, load: %s (%s)", params['svc_load_name'], params['svc_load_used'])
        logger.debug("[WebUI-cvhost] configuration, cpu: %s (%s)", params['svc_cpu_name'], params['svc_cpu_used'])
        logger.debug("[WebUI-cvhost] configuration, disk: %s (%s)", params['svc_dsk_name'], params['svc_dsk_used'])
        logger.debug("[WebUI-cvhost] configuration, memory: %s (%s)", params['svc_mem_name'], params['svc_mem_used'])
        logger.debug("[WebUI-cvhost] configuration, network: %s (%s)", params['svc_net_name'], params['svc_net_used'])
        # logger.info("[WebUI-cvhost] configuration, printer: %s (%s)", params['svc_prn_name'], params['svc_prn_used'])
    except Exception, exp:
        logger.warning("[WebUI-cvhost] configuration file (%s) not available or bad formed: %s", configuration_file, str(exp))
        app.redirect404()
        all_perfs = {}
        all_states = {}
        return {'app': app, 'config': type, 'all_perfs':all_perfs, 'all_states':all_states}

    def main(self):
        global app

        # Change process name (seen in ps or top)
        self.set_proctitle(self.name)

        # It's an external module, so we need to be sure that we manage
        # the signals
        self.set_exit_handler()

        # Go for Http open :)
        self.init_http()

        # We fill the global variable with our Queue() link
        # with the arbiter, because the page should be a non-class
        # one function
        app = self

        # We will loop forever on the http socket
        input = [self.srv.socket]

        # Main blocking loop
        while not self.interrupted:
            input = [self.srv.socket]
            try:
                inputready, _, _ = select.select(input, [], [], 1)
            except select.error, e:
                logger.warning("[WS_Arbiter] Exception: %s", str(e))
                continue
            for s in inputready:
                # If it's a web request, ask the webserver to do it
                if s == self.srv.socket:
                    self.srv.handle_request()

 def create_pack(self, buf, name):
     if not json:
         logger.warning("[Pack] cannot load the pack file '%s': missing json lib", name)
         return
     # Ok, go compile the code
     try:
         d = json.loads(buf)
         if "name" not in d:
             logger.error("[Pack] no name in the pack '%s'", name)
             return
         p = Pack({})
         p.pack_name = d["name"]
         p.description = d.get("description", "")
         p.macros = d.get("macros", {})
         p.templates = d.get("templates", [p.pack_name])
         p.path = d.get("path", "various/")
         p.doc_link = d.get("doc_link", "")
         p.services = d.get("services", {})
         p.commands = d.get("commands", [])
         if not p.path.endswith("/"):
             p.path += "/"
         # Ok, add it
         self[p.id] = p
     except ValueError, exp:
         logger.error("[Pack] error in loading pack file '%s': '%s'", name, exp)

    def load(self):
        now = int(time.time())
        # We get all modules file with .py
        modules_files = [fname[:-3] for fname in os.listdir(self.modules_path)
                         if fname.endswith(".py")]

        # And directories
        modules_files.extend([fname for fname in os.listdir(self.modules_path)
                               if os.path.isdir(os.path.join(self.modules_path, fname))])

        # Now we try to load them
        # So first we add their dir into the sys.path
        if not self.modules_path in sys.path:
            sys.path.append(self.modules_path)

        # We try to import them, but we keep only the one of
        # our type
        del self.imported_modules[:]
        for fname in modules_files:
            #print "Try to load", fname
            try:
                m = __import__(fname)
                if not hasattr(m, 'properties'):
                    continue

                # We want to keep only the modules of our type
                if self.modules_type in m.properties['daemons']:
                    self.imported_modules.append(m)
            except Exception, exp:
                logger.warning("Importing module %s: %s" % (fname, exp))

    def get_instances(self):
        self.clear_instances()
        for (mod_conf, module) in self.modules_assoc:
            mod_conf.properties = module.properties.copy()
            try:
                inst = module.get_instance(mod_conf)
                if not isinstance(inst, BaseModule):
                    raise TypeError('Returned instance is not of type BaseModule (%s) !'
                                    % type(inst))
            except Exception as err:
                logger.error("The module %s raised an exception %s, I remove it! traceback=%s",
                             mod_conf.get_name(), err, traceback.format_exc())
            else:
                # Give the module the data to which module it is load from
                inst.set_loaded_into(self.modules_type)
                self.instances.append(inst)


        for inst in self.instances:
            # External are not init now, but only when they are started
            if not inst.is_external and not self.try_instance_init(inst):
                # If the init failed, we put in in the restart queue
                logger.warning("The module '%s' failed to init, I will try to restart it later",
                               inst.get_name())
                self.to_restart.append(inst)

        return self.instances

def decode_values(pktype, plen, buf):
    nvalues = short.unpack_from(buf, header.size)[0]
    off = header.size + short.size + nvalues
    valskip = double.size

    # check the packet head
    if ((valskip + 1) * nvalues + short.size + header.size) != plen:
        return []
    if double.size != number.size:
        return []

    result = []
    for dstype in map(ord, buf[header.size + short.size:off]):
        if (dstype == DS_TYPE_COUNTER or dstype == DS_TYPE_DERIVE or dstype == DS_TYPE_ABSOLUTE):
            v = (dstype, number.unpack_from(buf, off)[0])
            result.append(v)
            off += valskip
        elif dstype == DS_TYPE_GAUGE:
            v = (dstype, double.unpack_from(buf, off)[0])
            result.append(v)
            off += valskip
        else:
            logger.warning("[Collectd] DS type %i unsupported" % dstype)

    return result

def show_logs():
    user = checkauth()    

    message,db = getdb(params['db_name'])
    if not db:
        return {
            'app': app,
            'user': user, 
            'message': message,
            'params': params,
            'records': []
        }

    records=[]

    try:
        logger.warning("[Logs] Fetching records from database: %s (max %d)" % (params['logs_type'], params['logs_limit']))
        for log in db.logs.find({ "$and" : [ { "type" : { "$in": params['logs_type'] }}, { "host_name" : { "$in": params['logs_hosts'] }}, { "service_description" : { "$in": params['logs_services'] }}  ]}).sort("time", -1).limit(params['logs_limit']):
            records.append({
                "date" : int(log["time"]),
                "host" : log['host_name'],
                "service" : log['service_description'],
                "message" : log['message']
            })
        message = "%d records fetched from database." % len(records)
        logger.debug("[Logs] %d records fetched from database." % len(records))
    except Exception, exp:
        logger.error("[Logs] Exception when querying database: %s" % (str(exp)))

    def set_ui_user_preference(self, user, key, value):
        if not self.is_connected:
            if not self.open():
                logger.error("[WebUI-MongoDBPreferences] error during initialization, no database connection!")
                return None

        if not user:
            logger.warning("[WebUI-MongoDBPreferences] error set_ui_user_preference, no user!")
            return None

        try:
            # check a collection exist for this user
            u = self.db.ui_user_preferences.find_one({'_id': user.get_name()})
            if not u:
                # no collection for this user? create a new one
                self.db.ui_user_preferences.save({'_id': user.get_name(), key: value})

            r = self.db.ui_user_preferences.update({'_id': user.get_name()}, {'$set': {key: value}})
            # Maybe there was no doc there, if so, create an empty one
            if not r:
                # Maybe the user exist, if so, get the whole user entry
                u = self.db.ui_user_preferences.find_one({'_id': user.get_name()})
                if not u:
                    logger.debug ("[WebUI-MongoDBPreferences] No user entry for %s, I create a new one", user.get_name())
                    self.db.ui_user_preferences.save({'_id': user.get_name(), key: value})
                else:  # ok, it was just the key that was missing, just update it and save it
                    u[key] = value
                    logger.debug ("[WebUI-MongoDBPreferences] Just saving the new key in the user pref")
                    self.db.ui_user_preferences.save(u)
        except Exception, e:
            logger.warning("[WebUI-MongoDBPreferences] Exception: %s", str(e))
            self.is_connected = False
            return None

    def get_new_broks(self, type='scheduler'):
        # Get the good links tab for looping..
        links = self.get_links_from_type(type)
        if links is None:
            logger.debug('Type unknown for connection! %s' % type)
            return

        # We check for new check in each schedulers and put
        # the result in new_checks
        for sched_id in links:
            try:
                con = links[sched_id]['con']
                if con is not None:  # None = not initialized
                    t0 = time.time()
                    tmp_broks = con.get_broks(self.name)
                    logger.debug("%s Broks get in %s" % (len(tmp_broks), time.time() - t0))
                    for b in tmp_broks.values():
                        b.instance_id = links[sched_id]['instance_id']

                    # Ok, we can add theses broks to our queues
                    self.add_broks_to_queue(tmp_broks.values())

                else:  # no con? make the connection
                    self.pynag_con_init(sched_id, type=type)
            # Ok, con is not known, so we create it
            except KeyError, exp:
                logger.debug("Key error for get_broks : %s" % str(exp))
                try:
                    logger.debug(''.join(Pyro.util.getPyroTraceback(exp)))
                except:
                    pass
                self.pynag_con_init(sched_id, type=type)
            except Pyro.errors.ProtocolError, exp:
                logger.warning("Connection problem to the %s %s: %s" % (type, links[sched_id]['name'], str(exp)))
                links[sched_id]['con'] = None

 def create_pack(self, buf, name):
     if not json:
         logger.warning("[Pack] cannot load the pack file '%s': missing json lib", name)
         return
     # Ok, go compile the code
     try:
         d = json.loads(buf)
         if not 'name' in d:
             logger.error("[Pack] no name in the pack '%s'", name)
             return
         p = Pack({})
         p.pack_name = d['name']
         p.description = d.get('description', '')
         p.macros = d.get('macros', {})
         p.templates = d.get('templates', [p.pack_name])
         p.path = d.get('path', 'various/')
         p.doc_link = d.get('doc_link', '')
         p.services = d.get('services', {})
         p.commands = d.get('commands', [])
         if not p.path.endswith('/'):
             p.path += '/'
         # Ok, add it
         self[p.id] = p
     except ValueError, exp:
         logger.error("[Pack] error in loading pack file '%s': '%s'", name, exp)

    def get_live_data_log(self):
        """Like get_live_data, but for log objects"""
        # finalize the filter stacks
        self.mongo_time_filter_stack.and_elements(self.mongo_time_filter_stack.qsize())
        self.mongo_filter_stack.and_elements(self.mongo_filter_stack.qsize())
        if self.use_aggressive_sql:
            # Be aggressive, get preselected data from sqlite and do less
            # filtering in python. But: only a subset of Filter:-attributes
            # can be mapped to columns in the logs-table, for the others
            # we must use "always-true"-clauses. This can result in
            # funny and potentially ineffective sql-statements
            mongo_filter_func = self.mongo_filter_stack.get_stack()
        else:
            # Be conservative, get everything from the database between
            # two dates and apply the Filter:-clauses in python
            mongo_filter_func = self.mongo_time_filter_stack.get_stack()
        dbresult = []
        mongo_filter = mongo_filter_func()
        logger.debug("[Logstore MongoDB] Mongo filter is %s" % str(mongo_filter))
        # We can apply the filterstack here as well. we have columns and filtercolumns.
        # the only additional step is to enrich log lines with host/service-attributes
        # A timerange can be useful for a faster preselection of lines

        filter_element = eval('{ ' + mongo_filter + ' }')
        logger.debug("[LogstoreMongoDB] Mongo filter is %s" % str(filter_element))
        columns = ['logobject', 'attempt', 'logclass', 'command_name', 'comment', 'contact_name', 'host_name', 'lineno', 'message', 'plugin_output', 'service_description', 'state', 'state_type', 'time', 'type']
        if not self.is_connected == CONNECTED:
            logger.warning("[LogStoreMongoDB] sorry, not connected")
        else:
            dbresult = [Logline([(c,) for c in columns], [x[col] for col in columns]) for x in self.db[self.collection].find(filter_element).sort([(u'time', pymongo.ASCENDING), (u'lineno', pymongo.ASCENDING)])]
        return dbresult

 def get_module(self, mod_name):
     if self.modules_dir and self.modules_dir not in sys.path:
         sys.path.append(self.modules_dir)
     try:
         return importlib.import_module('.module', mod_name)
     except ImportError as err:
         logger.warning('Cannot import %s as a package (%s) ; trying as bare module..',
                        mod_name, err)
     mod_dir = abspath(join(self.modules_dir, mod_name))
     mod_file = join(mod_dir, 'module.py')
     if os.path.exists(mod_file):
         # important, equivalent to import fname from module.py:
         load_it = lambda: imp.load_source(mod_name, mod_file)
     else:
         load_it = lambda: imp.load_compiled(mod_name, mod_file+'c')
     # We add this dir to sys.path so the module can load local files too
     if mod_dir not in sys.path:
         sys.path.append(mod_dir)
     try:
         return load_it()
     except Exception as err:
         logger.warning("Importing module %s failed: %s ; backtrace=%s",
                        mod_name, err, traceback.format_exc())
         sys.path.remove(mod_dir)
         raise

    def manage_log_brok(self, brok):
        """
        Parse a Shinken log brok to enqueue a log line for Index insertion
        """
        d = date.today()
        index_name = self.index_prefix + "-" + d.strftime("%Y.%m.%d")

        line = brok.data["log"]
        if re.match("^\[[0-9]*\] [A-Z][a-z]*.:", line):
            # Match log which NOT have to be stored
            logger.warning("[elastic-logs] do not store: %s", line)
            return

        logline = Logline(line=line)
        logline_dict = logline.as_dict()
        logline_dict.update({"@timestamp": datetime.utcfromtimestamp(int(logline_dict["time"])).isoformat() + "Z"})
        values = {"_index": index_name, "_type": "shinken-logs", "_source": logline_dict}

        # values = logline.as_dict()
        if logline.logclass != LOGCLASS_INVALID:
            logger.debug("[elastic-logs] store log line values: %s", values)
            self.logs_cache.append(values)
        else:
            logger.info("[elastic-logs] This line is invalid: %s", line)

        return

    def linkify_hg_by_realms(self, realms):
        # Now we explode the realm value if we've got one
        # The group realm must not override a host one (warning?)
        for hg in self:
            if not hasattr(hg, 'realm'):
                continue

            # Maybe the value is void?
            if not hg.realm.strip():
                continue

            r = realms.find_by_name(hg.realm.strip())
            if r is not None:
                hg.realm = r
                logger.debug("[hostgroups] %s is in %s realm", hg.get_name(), r.get_name())
            else:
                err = "the hostgroup %s got an unknown realm '%s'" % (hg.get_name(), hg.realm)
                hg.configuration_errors.append(err)
                hg.realm = None
                continue

            for h in hg:
                if h is None:
                    continue
                if h.realm is None or h.got_default_realm:  # default value not hasattr(h, 'realm'):
                    logger.debug("[hostgroups] apply a realm %s to host %s from a hostgroup rule (%s)",  \
                        hg.realm.get_name(), h.get_name(), hg.get_name())
                    h.realm = hg.realm
                else:
                    if h.realm != hg.realm:
                        logger.warning("[hostgroups] host %s it not in the same realm than it's hostgroup %s",  \
                            h.get_name(), hg.get_name())

    def commit_logs(self):
        """
        Peridically called (commit_period), this method prepares a bunch of queued logs (commit_colume) to insert them in the DB
        """
        if not self.logs_cache:
            return

        if not self.is_connected == CONNECTED:
            if not self.open():
                logger.warning("[mongo-logs] log commiting failed")
                logger.warning("[mongo-logs] %d lines to insert in database", len(self.logs_cache))
                return

        logger.debug("[mongo-logs] commiting ...")

        logger.debug("[mongo-logs] %d lines to insert in database (max insertion is %d lines)", len(self.logs_cache), self.commit_volume)

        # Flush all the stored log lines
        logs_to_commit = 1
        now = time.time()
        some_logs = []
        while True:
            try:
                # result = self.db[self.logs_collection].insert_one(self.logs_cache.popleft())
                some_logs.append(self.logs_cache.popleft())
                logs_to_commit = logs_to_commit + 1
                if logs_to_commit >= self.commit_volume:
                    break
            except IndexError:
                logger.debug("[mongo-logs] prepared all available logs for commit")
                break
            except Exception, exp:
                logger.error("[mongo-logs] exception: %s", str(exp))

    def load(self):
        if self.modules_path not in sys.path:
            sys.path.append(self.modules_path)

        modules_files = [fname
                         for fname in listdir(self.modules_path)
                         if isdir(join(self.modules_path, fname))]

        del self.imported_modules[:]
        for mod_name in modules_files:
            mod_file = abspath(join(self.modules_path, mod_name, 'module.py'))
            mod_dir = os.path.normpath(os.path.dirname(mod_file))
            mod = self.try_load(mod_name, mod_dir)
            if not mod:
                continue
            try:
                is_our_type = self.modules_type in mod.properties['daemons']
            except Exception as err:
                logger.warning("Bad module file for %s : cannot check its properties['daemons']"
                               "attribute : %s", mod_file, err)
            else:  # We want to keep only the modules of our type
                if is_our_type:
                    self.imported_modules.append(mod)

        # Now we want to find in theses modules the ones we are looking for
        del self.modules_assoc[:]
        for mod_conf in self.modules:
            module_type = uniform_module_type(mod_conf.module_type)
            for module in self.imported_modules:
                if uniform_module_type(module.properties['type']) == module_type:
                    self.modules_assoc.append((mod_conf, module))
                    break
            else:  # No module is suitable, we emit a Warning
                logger.warning("The module type %s for %s was not found in modules!",
                               module_type, mod_conf.get_name())

    def get_live_data(self, cs):
        """Find the objects which match the request.

        This function scans a list of objects (hosts, services, etc.) and
        applies the filter functions first. The remaining objects are
        converted to simple dicts which have only the keys that were
        requested through Column: attributes. """
        # We will use prefiltercolumns here for some serious speedup.
        # For example, if nagvis wants Filter: host_groups >= hgxy
        # we don't have to use the while list of hostgroups in
        # the innermost loop
        # Filter: host_groups >= linux-servers
        # host_groups is a service attribute
        # We can get all services of all hosts of all hostgroups and filter at the end
        # But it would save a lot of time to already filter the hostgroups. This means host_groups must be hard-coded
        # Also host_name, but then we must filter the second step.
        # And a mixture host_groups/host_name with FilterAnd/Or? Must have several filter functions

        handler = self.objects_get_handlers.get(self.table, None)
        if not handler:
            logger.warning("[Livestatus Query] Got unhandled table: %s" % (self.table))
            return []

        result = handler(self, cs)
        # Now we have a list of full objects (Host, Service, ....)

        #if self.limit:
        #    if isinstance(res, list):
        #        res = res[:self.limit]
        #    else:
        #        res = list(res)[:self.limit]

        return result

    def hook_load_retention(self, daemon):

        # Now the new redis way :)
        logger.debug("[RedisRetention] asking me to load the retention objects")

        # We got list of loaded data from retention server
        ret_hosts = {}
        ret_services = {}

        # We must load the data and format as the scheduler want :)
        for h in daemon.hosts:
            key = "HOST-%s" % h.host_name
            if self.sentinel_servers:
                for attempt in range(self.wait_for_failover):
                    try:
                        val = self.mc.get(key)
                        break
                    except:
                        logger.warning("[RedisRetention] Couldn't connect to redis server, trying to reconnect...")
                        time.sleep(1)
                else:
                    logger.error("[RedisRetention] Timeout connecting to redis server!")
            else:
                val = self.mc.get(key)
            if val is not None:
                # redis get unicode, but we send string, so we are ok
                #val = str(unicode(val))
                val = cPickle.loads(val)
                ret_hosts[h.host_name] = val

        for s in daemon.services:
            key = "SERVICE-%s,%s" % (s.host.host_name, s.service_description)
            # space are not allowed in memcache key.. so change it by SPACE token
            key = key.replace(' ', 'SPACE')
            #print "Using key", key
            if self.sentinel_servers:
                for attempt in range(self.wait_for_failover):
                    try:
                        val = self.mc.get(key)
                        break
                    except:
                        logger.warning("[RedisRetention] Couldn't connect to redis server, trying to reconnect...")
                        time.sleep(1)
                else:
                    logger.error("[RedisRetention] Timeout connecting to redis server!")
            else:
                val = self.mc.get(key)
            if val is not None:
                #val = str(unicode(val))
                val = cPickle.loads(val)
                ret_services[(s.host.host_name, s.service_description)] = val

        all_data = {'hosts': ret_hosts, 'services': ret_services}

        # Ok, now comme load them scheduler :)
        daemon.restore_retention_data(all_data)

        logger.info("[RedisRetention] Retention objects loaded successfully.")

        return True

def load_config(app):
    global params
    
    import os
    from webui2.config_parser import config_parser
    try:
        currentdir = os.path.dirname(os.path.realpath(__file__))
        configuration_file = "%s/%s" % (currentdir, 'plugin.cfg')
        logger.info("[WebUI-logs] Plugin configuration file: %s", configuration_file)
        scp = config_parser('#', '=')
        z = params.copy()
        z.update(scp.parse_config(configuration_file))
        params = z

        params['logs_type'] = [item.strip() for item in params['logs_type'].split(',')]
        if len(params['logs_hosts']) > 0:
            params['logs_hosts'] = [item.strip() for item in params['logs_hosts'].split(',')]
        if len(params['logs_services']) > 0:
            params['logs_services'] = [item.strip() for item in params['logs_services'].split(',')]
        
        logger.info("[WebUI-logs] configuration loaded.")
        logger.info("[WebUI-logs] configuration, fetching types: %s", params['logs_type'])
        logger.info("[WebUI-logs] configuration, hosts: %s", params['logs_hosts'])
        logger.info("[WebUI-logs] configuration, services: %s", params['logs_services'])
        return True
    except Exception, exp:
        logger.warning("[WebUI-logs] configuration file (%s) not available: %s", configuration_file, str(exp))
        return False