两份简单的logstash配置

input{http{port=>7474}}
filter{

    grok{
        match =>{
            #"message" => "%{COMBINEDAPACHELOG}"
            "message" => '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:[@metadata][timestamp]}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent}'
        }
    }
   # mutate{
   #     copy => { "@timestamp" => "read_timestamp"}
   # }

    ruby {
        code => "event.set('@read_timestamp',event.get('@timestamp'))"
    }

    # 20/May/2015:21:05:15 +0000
    #date{
    #    match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
    #}

    date{
        match => ["[@metadata][timestamp]","dd/MMM/yyyy:HH:mm:ss Z"]
    }

    geoip{
        source => "clientip"
        fields => ["latitude","longitude","city_name","country_name","region_name"]
    }

    useragent{
        source => "agent"
        target => "useragent"
    }

    mutate{
        convert => { "bytes" => "integer" }
    }

    mutate{
        remove_field =>["headers","message"]
    }

}

output{stdout{codec=>rubydebug}}

apache_logstash.conf

input {  
  stdin { } 
}

filter {
  grok {
    match => {
      "message" => '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}'
    }
  }

  date {
    match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ]
    locale => en
  }

  geoip {
    source => "clientip"
  }

  useragent {
    source => "agent"
    target => "useragent"
  }
}

output {
  stdout {
    codec => dots {}
  }

  elasticsearch {
    index => "apache_elastic_example"
    template => "./apache_template.json"
    template_name => "apache_elastic_example"
    template_overwrite => true
  }
}

input{
    #http{
    #    port => 7474
    #}

    stdin{}
   # file{
   #     path => "/Users/rockybean/Downloads/es/6.1/logstash-6.1.1/demo_data/apache_logs/apache_logs"
   #     start_position => "beginning"
   # }
}

filter{
    #mutate{add_field => {"[@metadata][debug]"=>true}}

    grok{
        match => {
            "message" => '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:[@metadata][timestamp]}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent}'
        }
    }

    ruby{
        code => "event.set('@read_timestamp',event.get('@timestamp'))"
    }

    # 20/May/2015:21:05:56 +0000
    date{
        match => ["[@metadata][timestamp]","dd/MMM/yyyy:HH:mm:ss Z"]
    }

    mutate{
        convert => {"bytes" => "integer"}
    }

    geoip{
        source => "clientip"
        fields => ["location","country_name","city_name","region_name"]
    }

    useragent{
        source => "agent"
        target => "useragent"
    }

    mutate{remove_field=>["headers"]}

    mutate{
        add_field=>{
            "[@metadata][index]" => "apache_logs_%{+YYYY.MM}"
        }
    }

    if "_grokparsefailure" in [tags] {
        mutate{
            replace=>{
                "[@metadata][index]" => "apache_logs_failure_%{+YYYY.MM}"
            }
        }
    }else{
        mutate{remove_field=>["message"]}

    }

}

output{
    if [@metadata][debug]{
        stdout{codec=>rubydebug{metadata=>true}}
    }else{
        stdout{codec=>dots}

        elasticsearch{
            index => "%{[@metadata][index]}"
            document_type => "doc"
        }
    }
}

文章永久链接：https://tech.souyunku.com/33221