input {
    beats {
        port => 5044
    }
}
filter {
    if [message] =~ "^#|\.(css|js|ico|png|xml|jpg|JPG|gif|jpeg|eot|svg|woff|webp\?) " {
        drop {}
    }
    if [message] =~ "robots.txt" {
        drop {}
    }
    if [message] =~ "PIE.htc" {
        drop {}
    }

    grok {
        match => { "message" => "%{COMBINEDAPACHELOG}"}
    }
    date {
        match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
    }
    geoip {
        source => "clientip"
    }
    useragent {
        source => "agent"
    }
    mutate {
        convert => [ "bytes", "integer" ]
    }
    if [device] == "Spider" {
        drop {}
    }
# ip filter
    if [clientip] == "52.78.127.176" {
        drop {}
    }
    if [clientip] == "72.14.199.151" {
        drop {}
    }
    if [clientip] == "203.133.169.75" {
        drop {}
    }

# field added
    if [request] {
        mutate {
            add_field => {
                "reqs" => "%{request}"
            }
        }
    }
    mutate {
        split => ["reqs", "?"]
        add_field => { "uri" => "%{[reqs][0]}" }
        add_field => { "req_uri" => "%{[reqs][0]}" }
    }
    if [reqs][1] {
        mutate {
            add_field => { "querystring" => "%{[reqs][1]}" }
        }
    }
    if ![querystring] {
        mutate {
            add_field => { "querystring" => "-" }
        }
    }
    if [uri] =~ "articles" {
        mutate {
            split => ["uri", "/"]
            add_field => { "p_category" => "%{uri[2]}" }
        }
    }
    urldecode {
        field => "querystring"
    }


    # params
    if [request] =~ "\?" {
        kv {
            field_split => "&"
            source => "querystring"
            include_keys => [ "query", "redirectUrl" ]
            prefix => "param_"
        }
    }


    mutate {
        remove_field => [
            "reqs",
            "uri"
        ]
    }

}
output {
   elasticsearch {}
}