def get_all_host_facter_message():
    # 由于puppet的yaml文件是ruby格式的，需要进行转换
    yaml.add_multi_constructor(u"!ruby/object:", construct_ruby_object)
    yaml.add_constructor(u"!ruby/sym", construct_ruby_sym)
    # 获取所有有facters信息的主机文件名称
    for dirpath, dirnames, filenames in os.walk(yaml_dir):
        # 只需要处理yaml目录下得yaml结尾的文件
        if dirpath == yaml_dir:
            for file in filenames:
                file_name, file_ext = os.path.splitext(file)
                if file_ext == '.yaml':
                    host_yaml_path = yaml_dir + '/' + file
                    # 得到yaml文件内容, 字典形式
                    host_yaml_result_dict = yaml_file_handle(host_yaml_path)
                    # 对单个agent的数据进行处理
                    if host_yaml_result_dict:
                        # 有key为facts,所以可以直接查找facts key值
                        if host_yaml_result_dict.has_key('facts'):
                            data_dict = host_yaml_result_dict['facts']['values']
                        # 没有的直接取
                        else:
                            data_dict = host_yaml_result_dict['values']

                    # 现在就可以data数据进行处理，取得我们所需要得数据
                    result_dict = handle_facter_message(data_dict)
                    all_host_facter_message[file_name] = result_dict
    # return我们最终的数据结果集
    return all_host_facter_message

def load(contents):
    #example
    yaml.add_multi_constructor(u"!ruby/object:", construct_ruby_object)
    yaml.add_constructor(u"!ruby/sym", construct_ruby_sym)
    stream = contents
    mydata = yaml.load(stream)
    return mydata

def modules_list():

	if 'X-Auth-Token' not in request.headers:
		syslog.syslog("modules request failed because X-Auth-Token was missing from the request")
		abort(401)
	if request.headers['X-Auth-Token'] != app.config['AUTH_TOKEN']:
		app.logger.warn('modules request failed because the X-Auth-Token was incorrect')
		abort(401)

	## ask the puppet server for a list of modules
	(rcode, stdout, stderr) = sysexec([app.config['PUPPET_BINARY'], "module", "--confdir", app.config['PUPPET_CONFDIR'], "list", "--render-as", "yaml"])	

	if rcode != 0:
		syslog.syslog("puppet module list failed")
		syslog.syslog("stdout: " + str(stdout))
		syslog.syslog("stderr: " + str(stderr))
		abort(500)
	else:
		## Try to validate the YAML
		try:
			# work around the stupid ruby object shit in puppet yaml
			yaml.add_multi_constructor(u"!ruby/object:", construct_ruby_object)
			modules = yaml.load(stdout)
		except yaml.YAMLError as ex:
			syslog.syslog("puppet module list returned invalid YAML")
			syslog.syslog("invalid YAML: " + stdout)
			abort(500)

		r = make_response(yaml.dump(modules))
		r.headers['Content-Type'] = "application/x-yaml"
		return r

def configure():
    """
    Configures YAML parser for Step serialization and deserialization
    Called in drain/__init__.py
    """
    yaml.add_multi_representer(Step, step_multi_representer)
    yaml.add_multi_constructor('!step', step_multi_constructor)

def read_callback():
  yaml.add_multi_constructor("!", identity)
  logger('verb', "starting run")
  for report_dir in os.listdir(PuppetReportsConfig.reports_dir):
    logger('verb', "parsing: %s" % report_dir)
    reports_dir = os.listdir(PuppetReportsConfig.reports_dir + '/' + report_dir)
    reports_dir.sort
    last_report = reports_dir[-1]
    last_report_file = PuppetReportsConfig.reports_dir + '/' + report_dir + '/' + last_report
    with open(last_report_file, "r") as stream:
      data = yaml.load(stream)
      data = map_value(data)
      results = compute_metrics(data)
      logger('verb', "ready to send")
      for k in results:
        logger('verb', ("pushing value for %s => %s = %s" % (report_dir, k, results[k])))
        val = collectd.Values(plugin=NAME, type='gauge')
        val.plugin_instance = report_dir
        val.type_instance = k
        try:
          val.values = [ float(results[k]) ]
        except:
          logger('warn', ("value %s => %s for %s cannot be parsed to float" % (k, results[k], report_dir)))
          val.values = [ 0.0 ]
        val.dispatch()

def doactualizalogpuppet(filetext):

    ahora=datetime.datetime.today()
    yaml.add_multi_constructor(u"!ruby/object:", construct_ruby_object)
    yaml.add_constructor(u"!ruby/sym", construct_ruby_sym)
    mydata = yaml.load(filetext)
    host=mydata["host"].upper()
    host=host[:host.index('.')]
    estadoglobal="OK"
    output = StringIO.StringIO()
    logs=mydata["logs"]
    output.write("<center><br><b>Logs de puppet</b><br><br>")   
    for item in logs:
       output.write("<table style='border: solid 1px #000000;width:95%;'>")
       if 'file' in item:
            output.write("<tr style='border: solid 1px #000000;'><td width='10%'>File</td><td>"+item['file']+":"+str(item['line'])+"</td></tr>")
                        
       output.write("<tr style='border: solid 1px #000000;'>")
       if item['level'] == 'err' :	   
           estadoglobal="ERROR"
           output.write("<td width='10%'><font color='red'>Level</font></td>")       
           output.write("<td><font color='red'>"+item['level']+"</font></td>")
       else:
           output.write("<td width='10%'>Level</td>")       
           output.write("<td>"+item['level']+"</td>")
       output.write("</tr>")           
       output.write("<tr><td width='10%'>Message</td><td>"+item['message']+"</td></tr>")
       output.write("</table><br>")
	   
    output.write("<br><b>Clases y recursos aplicados</b><br><br>")      
    recursos=mydata["resource_statuses"]
    output.write("<table style='border: solid 1px #000000;width:95%'>")
    for item in recursos:
        eventos=recursos[item]['events']
        descripcion=recursos[item]['source_description']
        estado="OK"
        for evento in eventos:
            valor=evento['status']
            if valor == "failure" :
                 estado="ERROR"
                 estadoglobal="ERROR"
                 break	 
        if estado=="OK":                         			 
            output.write("<tr style='border: solid 1px #000000;'><td width='90%'>"+ descripcion +"</td><td>"+estado+"</td></tr>")
        else:
            output.write("<tr style='border: solid 1px #000000;'><td width='90%'><font color='red'>"+ descripcion +"</font></td><td><font color='red'>"+estado+"</font></td></tr>")
    output.write("</table><br><br></center>")
    
    fila=cdb((cdb.maquinas.host.upper()==host) & (cdb.maquinas.tipohost!='WINDOWS')).select().last()    
    if fila==None:
        pass
    else:		
        #Movemos el puntero del fichero al comienzo del mismo, ya que si no no se vuelca a la tabla.    
        output.seek(0)   
        fila.update_record(ultimopuppet=ahora,estadopuppet=estadoglobal,logpuppet=cdb.maquinas.logpuppet.store(output,filename=host))
    output.close()
        
    return "OK"

def configure_callback(conf):
  yaml.add_multi_constructor("!", identity)
  logger('verb', "configuring")

  for node in conf.children:
    if node.key == 'ReportsDir':
      PuppetReportsConfig.reports_dir = node.values[0]
    else:
      logger('verb', "unknown config key in puppet module: %s" % node.key)

def init_yaml_ruby_parsers():
    def construct_ruby_object(loader, suffix, node):
        return loader.construct_yaml_map(node)

    def construct_ruby_sym(loader, node):
        return loader.construct_yaml_str(node)

    yaml.add_multi_constructor(u"!ruby/object:", construct_ruby_object)
    yaml.add_constructor(u"!ruby/sym", construct_ruby_sym)

def parase_yaml():

    yaml.add_multi_constructor(u"!ruby/object:", construct_ruby_object)
    yaml.add_constructor(u"!ruby/sym", construct_ruby_sym)
    
    stream = file('/home/201602250326.yaml','r')
    mydata = yaml.load(stream)

    return mydata

    def _init_parser(self):
        #Custom YAML constructs for ruby objects for puppet files parsing
        def _construct_ruby_object(loader, suffix, node):
                return loader.construct_yaml_map(node)

        def _construct_ruby_sym(loader, suffix, node):
            return loader.construct_yaml_str(node)

        yaml.add_multi_constructor(u"!ruby/object:", _construct_ruby_object)
        yaml.add_multi_constructor(u"!ruby/sym", _construct_ruby_sym)

def initialize():
    """
    Initialize the configuration system by installing YAML handlers.
    Automatically done on first call to load() specified in this file.
    """
    global is_initialized
    # Add the custom multi-constructor
    yaml.add_multi_constructor('!obj:', multi_constructor)
    yaml.add_multi_constructor('!pkl:', multi_constructor_pkl)
    is_initialized = True

def initialize():
    """Add constructors to yaml parser.
    """
    from eemeter.meter.base import MeterBase
    from eemeter.models.temperature_sensitivity import Model

    yaml.add_multi_constructor('!obj:', multi_constructor_obj)
    yaml.add_constructor('!setting', constructor_setting)
    yaml.add_multi_representer(MeterBase, multi_representer_obj)
    yaml.add_multi_representer(Model, multi_representer_obj)

    is_initialized = True

def create(cont):
    def construct_ruby_object(loader, suffix, node):
        return loader.construct_yaml_map(node)

    def construct_ruby_sym(loader, node):
        return loader.construct_yaml_str(node)
    
    def timestamp_constructor(loader, node):
        return dateutil.parser.parse(node.value)
    
    yaml.add_multi_constructor(u"!ruby/object:", construct_ruby_object)
    yaml.add_constructor(u"!ruby/sym", construct_ruby_sym)
    yaml.add_constructor(u'tag:yaml.org,2002:timestamp', timestamp_constructor)

    d=yaml.load(cont)
    
    certname=d['host']
    hostname=certname[0:-5]
    
    hosts=Host.objects.filter(name__iexact=hostname)
    if hosts:
        host=hosts.first()
        
        ppreport=PPREPORT()
        ppreport.status=d['status']
        ppreport.time=d['time']
        ppreport.version=d['puppet_version']
        ppreport.save()
        
        for m1 in d['metrics']:
            for m2 in d['metrics'][m1]['values']:
                ppmetrics=PPMETRICS()
                ppmetrics.report=ppreport
                ppmetrics.category=d['metrics'][m1]['name']
                ppmetrics.name=m2[0]
                ppmetrics.value=m2[2]
                ppmetrics.save()
        
        for rel in d['logs']:
            ppreportlog=PPREPORTLOG()
            ppreportlog.report=ppreport
            ppreportlog.level=rel['level']
            ppreportlog.message=rel['message']
            ppreportlog.time=rel['time']
            ppreportlog.save()
        
        res=Resource()
        res.host=host
        res.name=""
        res.type="ppreport"
        res.resource_id=ppreport.id
        res.save()

    def _get_all_facts_by_hostnames(self, hostnames):
        def construct_ruby_object(loader, suffix, node):
            return loader.construct_yaml_map(node)

        def construct_ruby_sym(loader, node):
            return loader.construct_yaml_str(node)

        yaml.add_multi_constructor(u"!ruby/object:", construct_ruby_object)
        yaml.add_constructor(u"!ruby/sym", construct_ruby_sym)
        for hostname in hostnames:
            data = self._get_data_for_hostname(hostname)
            if data:
                return yaml.load(data)['values']

def add_constructors():
    yaml.add_constructor(u"!assert", assert_constructor)
    yaml.add_constructor(u"!record", record_constructor)
    yaml.add_constructor(u"!python", python_constructor)
    yaml.add_constructor(u"!menuitem", menuitem_constructor)
    yaml.add_constructor(u"!act_window", act_window_constructor)
    yaml.add_constructor(u"!function", function_constructor)
    yaml.add_constructor(u"!report", report_constructor)
    yaml.add_constructor(u"!context", context_constructor)
    yaml.add_constructor(u"!delete", delete_constructor)
    yaml.add_constructor(u"!url", url_constructor)
    yaml.add_constructor(u"!eval", eval_constructor)
    yaml.add_multi_constructor(u"!ref", ref_constructor)
    yaml.add_constructor(u"!ir_set", ir_set_constructor)

def configure_callback(conf):
    yaml.add_multi_constructor("!", identity)
    logger('verb', "configuring")

    for node in conf.children:
        if node.key == 'ReportsDir':
            PuppetReportsConfig.reports_dir = node.values[0]
        elif node.key == 'Verbose':
            PuppetReportsConfig.verbose = True
        elif node.key == 'Interval':
            PuppetReportsConfig.interval = float(node.values[0])
            logger('verb', "set interval to: %s" %
                   PuppetReportsConfig.interval)
        else:
            logger('verb', "unknown config key in puppet module: %s" %
                   node.key)

def load_yaml(stream):
    """Parse Puppet YAML into Python objects"""
    # Puppet YAML contains Ruby objects which need to be defined
    def construct_ruby_object(loader, suffix, node):
        return loader.construct_yaml_map(node)
    def construct_ruby_sym(loader, node):
        return loader.construct_yaml_str(node)
    yaml.add_multi_constructor('!ruby/object:', construct_ruby_object)
    yaml.add_constructor('!ruby/sym', construct_ruby_sym)
    try:
        document = yaml.load(stream)
    except yaml.loader.ConstructorError as e:
        if "unhashable type: 'list'" in str(e):
            raise NotImplementedError("http://pyyaml.org/ticket/169")
        else:
            raise
    return document

def readMonitoringConfig():
  # Use Puppet's parser to convert monitoringserver.pp into YAML
  manifest = os.path.join(os.path.dirname(__file__), 'manifests', 'monitoringserver.pp')
  parseScript = '''
    require 'puppet/parser'
    parser = Puppet::Parser::Parser.new(Puppet[:environment])
    parser.file = ARGV[0]
    print ZAML.dump(parser.parse)
  '''
  data, dummy = subprocess.Popen(['ruby', '', manifest],
                  stdin=subprocess.PIPE,
                  stdout=subprocess.PIPE,
                  stderr=subprocess.PIPE).communicate(parseScript)

  # See http://stackoverflow.com/q/8357650/785541 on parsing Puppet's YAML
  yaml.add_multi_constructor(u"!ruby/object:", lambda loader, suffix, node: loader.construct_yaml_map(node))
  yaml.add_constructor(u"!ruby/sym", lambda loader, node: loader.construct_yaml_str(node))
  return yaml.load(data)

    def buildMetadata(self):
        TARFile.buildMetadata(self)

        try: 
            import yaml            
        except:
            print("Skipping GEM analysis of %s because YAML is not supported by this python" % self.filename)
            return
        
        try:
            import gzip
        except:
            print("Skipping GEM analysis of %s because GZip is not supported by this python" % self.filename)
            return        

        print "Doing GEM analysis..."
        try:
            tf = tarfile.open(self.filename, "r")
            
            for tarEntry in tf.getmembers():
                if not tarEntry.isfile(): continue
                if not tarEntry.name == 'metadata.gz': continue
                
                handle = tf.extractfile(tarEntry)
                gzipHandle = gzip.GzipFile(fileobj=handle)

                yaml.add_multi_constructor(u"!ruby/object:", construct_ruby_object)
                yaml.add_constructor(u"!ruby/sym", construct_ruby_sym)

                gemSpec = yaml.load(gzipHandle)
                # print "FOUND GEMSPEC %s" % str(gemSpec)
                
                # For debugging....
                # import pprint
                # pp = pprint.PrettyPrinter(indent=4)
                # pp.pprint(gemSpec)
                
                self.makeMetadataFromGemspec(gemSpec)
                
                files.filetype.qClose(gzipHandle)
                files.filetype.qClose(handle)
        finally:
            files.filetype.qClose(tf)

def test_short_form_func_in_yaml_teamplate():
    template = """---
    KeyB64: !Base64 valueToEncode
    KeyRef: !Ref foo
    KeyAnd: !And
      - A
      - B
    KeyEquals: !Equals [A, B]
    KeyIf: !If [A, B, C]
    KeyNot: !Not [A]
    KeyOr: !Or [A, B]
    KeyFindInMap: !FindInMap [A, B, C]
    KeyGetAtt: !GetAtt A.B
    KeyGetAZs: !GetAZs A
    KeyImportValue: !ImportValue A
    KeyJoin: !Join [ ":", [A, B, C] ]
    KeySelect: !Select [A, B]
    KeySplit: !Split [A, B]
    KeySub: !Sub A
    """
    yaml.add_multi_constructor('', yaml_tag_constructor)
    template_dict = yaml.load(template)
    key_and_expects = [
        ['KeyRef', {'Ref': 'foo'}],
        ['KeyB64', {'Fn::Base64': 'valueToEncode'}],
        ['KeyAnd', {'Fn::And': ['A', 'B']}],
        ['KeyEquals', {'Fn::Equals': ['A', 'B']}],
        ['KeyIf', {'Fn::If': ['A', 'B', 'C']}],
        ['KeyNot', {'Fn::Not': ['A']}],
        ['KeyOr', {'Fn::Or': ['A', 'B']}],
        ['KeyFindInMap', {'Fn::FindInMap': ['A', 'B', 'C']}],
        ['KeyGetAtt', {'Fn::GetAtt': ['A', 'B']}],
        ['KeyGetAZs', {'Fn::GetAZs': 'A'}],
        ['KeyImportValue', {'Fn::ImportValue': 'A'}],
        ['KeyJoin', {'Fn::Join': [ ":", [ 'A', 'B', 'C' ] ]}],
        ['KeySelect', {'Fn::Select': ['A', 'B']}],
        ['KeySplit', {'Fn::Split': ['A', 'B']}],
        ['KeySub', {'Fn::Sub': 'A'}],
    ]
    for k, v in key_and_expects:
        template_dict.should.have.key(k).which.should.be.equal(v)