logstash过滤器使用ruby把时间格式转UNIX_MS

原作者: [db:作者] 来自: [db:来源] 收藏邀请

　　日志里一般都会有时间，格式如“2020-09-04 10:08:08”，怎么转成毫秒呢，格式如“1598609188959”？

　　假如我们的日志里仅有一个时间字段，因为我们这里转换前是没有毫秒的，所以可以直接转成秒后补3个0。直接看配置：

input{
 beats {
    port => "5044"
  }
}

filter{
    mutate {
        add_field => {
            "requestTimestamp" => "%{message}"            
        }
    }

    date{
         match => ["requestTimestamp","YYYY-MM-dd HH:mm:ss"]    
         target =>"requestTimestamp"
    }
    
    ruby{
        code => "event.set('requestTimestamp',event.get('requestTimestamp').to_i*1000)"
    }
          
}
output {
  stdout { codec => rubydebug }
}

　　假设filebeat整不断往logstash输送时间日志，我们启动后看到的将是这样的：

{
    "requestTimestamp" => 1599211641000,
          "@timestamp" => 2020-09-04T09:27:25.754Z,
             "message" => "2020-09-04 17:27:21",
                 "log" => {
        "offset" => 483,
          "file" => {
            "path" => "D:\\wlf\\logs\\cdr-200200006022-08971-2020090417.0.log"
        }
    },
            "@version" => "1",
                "host" => {
        "name" => "wulf00"
    },
               "agent" => {
                  "id" => "ae375dc0-d6e2-488c-be87-2544c05b1242",
                "name" => "wulf00",
            "hostname" => "wulf00",
        "ephemeral_id" => "8f29c3c9-08ea-4f6b-9508-ae187ec22f0b",
             "version" => "7.9.0",
                "type" => "filebeat"
    },
                 "ecs" => {
        "version" => "1.5.0"
    },
               "input" => {
        "type" => "log"
    },
                "tags" => [
        [0] "beats_input_codec_plain_applied"
    ]
}
{
    "requestTimestamp" => 1599211656000,
          "@timestamp" => 2020-09-04T09:27:40.758Z,
             "message" => "2020-09-04 17:27:36",
                 "log" => {
        "offset" => 504,
          "file" => {
            "path" => "D:\\wlf\\logs\\cdr-200200006022-08971-2020090417.0.log"
        }
    },
            "@version" => "1",
                "host" => {
        "name" => "wulf00"
    },
               "agent" => {
                  "id" => "ae375dc0-d6e2-488c-be87-2544c05b1242",
                "name" => "wulf00",
            "hostname" => "wulf00",
        "ephemeral_id" => "8f29c3c9-08ea-4f6b-9508-ae187ec22f0b",
             "version" => "7.9.0",
                "type" => "filebeat"
    },
                 "ecs" => {
        "version" => "1.5.0"
    },
               "input" => {
        "type" => "log"
    },
                "tags" => [
        [0] "beats_input_codec_plain_applied"
    ]

　　以上是时间格式为“YYYY-MM-dd HH:mm:ss”的情况，那么“YYYY-MM-dd HH:mm:ss SSS"的情况又如何呢？改下logstash配置文件：

input{
 beats {
    port => "5044"
  }
}

filter{    
    mutate {
        add_field => {
            "requestTimestamp" => "%{message}"            
        }
    }

    date{
         match => ["requestTimestamp","YYYY-MM-dd HH:mm:ss.SSS"]    
         target =>"requestTimestamp"
    }
    
    ruby{
        code => "event.set('requestTimestamp',(event.get('requestTimestamp').to_f.round(3)*1000).to_i)"
    }
          
}
output {
  stdout { codec => rubydebug }
}

　　重启logstash，得到如下结果：

{
               "input" => {
        "type" => "log"
    },
          "@timestamp" => 2020-09-04T09:45:39.448Z,
                "host" => {
        "name" => "wulf00"
    },
                 "log" => {
        "offset" => 75,
          "file" => {
            "path" => "D:\\wlf\\logs\\cdr-200200006022-08971-2020090417.0.log"
        }
    },
                 "ecs" => {
        "version" => "1.5.0"
    },
    "requestTimestamp" => 1599212733416,
             "message" => "2020-09-04 17:45:33.416",
               "agent" => {
                "type" => "filebeat",
             "version" => "7.9.0",
        "ephemeral_id" => "8f29c3c9-08ea-4f6b-9508-ae187ec22f0b",
            "hostname" => "wulf00",
                  "id" => "ae375dc0-d6e2-488c-be87-2544c05b1242",
                "name" => "wulf00"
    },
            "@version" => "1",
                "tags" => [
        [0] "beats_input_codec_plain_applied"
    ]
}
{
               "input" => {
        "type" => "log"
    },
          "@timestamp" => 2020-09-04T09:44:54.172Z,
                "host" => {
        "name" => "wulf00"
    },
                 "log" => {
        "offset" => 0,
          "file" => {
            "path" => "D:\\wlf\\logs\\cdr-200200006022-08971-2020090417.0.log"
        }
    },
                 "ecs" => {
        "version" => "1.5.0"
    },
    "requestTimestamp" => 1599212688414,
             "message" => "2020-09-04 17:44:48.414",
               "agent" => {
                "type" => "filebeat",
             "version" => "7.9.0",
        "ephemeral_id" => "8f29c3c9-08ea-4f6b-9508-ae187ec22f0b",
            "hostname" => "wulf00",
                  "id" => "ae375dc0-d6e2-488c-be87-2544c05b1242",
                "name" => "wulf00"
    },
            "@version" => "1",
                "tags" => [
        [0] "beats_input_codec_plain_applied"
    ]
}