收集Nginx访问日志

收集Nginx访问日志

安装nginx

vim /etc/yum.repos.d/nginx.repo 
[nginx]
name=nginx repo
baseurl=http://nginx.org/packages/centos/7/$basearch/
gpgcheck=0
enabled=1

yum -y install nginx
systemctl start nginx

filebeat配置

vim /etc/filebeat/filebeat.yml

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  tags: ["nginx"]
  fields_under_root: true
  fields:
    project: microservice
    app: nginx

output.logstash:
  hosts: ["192.168.0.11:5044"]

重启服务

systemctl restart filebeat.service

1.写Grok正则匹配nginx访问日志：

logstash配置

vim /opt/elk/logstash/conf.d/test.conf

input {
  beats {
   host => "0.0.0.0"
   port => 5044
  }
}
filter {
  grok {
  match => {
  "message" => "%{IPV4:remote_addr} - (%{USERNAME:remote_user}|-) \[%{HTTPDATE:time_local}\] \"%{WORD:request_method} %{URIPATHPARAM:request_uri} HTTP/%{NUMBER:http_protocol}\" %{NUMBER:http_status} %{NUMBER:body_bytes_sent} \"%{GREEDYDATA:http_referer}\" \"%{GREEDYDATA:http_user_agent}\" \"(%{IPV4:http_x_forwarded_for}|-)\""
  }
}


  if [app] == "product" and [project] == "microservice" {
  mutate {
  add_field => {
  "[@metadata][target_index]" => "microservice-product-%{+YYYY.MM}"
	}
  }
} else if [app] == "gateway" and [project] == "microservice" {
  mutate {
  add_field => {
  "[@metadata][target_index]" => "microservice-gateway-%{+YYYY.MM.dd}"
	}
  }
} else if [app] == "nginx" and [project] == "microservice" {
  mutate {
  add_field => {
  "[@metadata][target_index]" => "microservice-nginx-%{+YYYY.MM.dd}"
	}
  }
} else {
  mutate {
  add_field => {
  "[@metadata][target_index]" => "unknown-%{+YYYY}"
	  }
	}
     }
}
output {
  elasticsearch {
  hosts => "192.168.0.11:9200"
  index => "%{[@metadata][target_index]}"
  }
}

热加载配置

kill -HUP <logstash pid>

验证

访问nginx，查看kibana页面

2.将Nginx访问日志格式改为JSON收集：

修改nginx配置文件(Nginx访问日志格式改为JSON)

vim /etc/nginx/nginx.conf

log_format json  '{ "@timestamp": "$time_iso8601", '
    '"remote_addr": "$remote_addr", '
    '"remote_user": "$remote_user", '
    '"body_bytes_sent": "$body_bytes_sent", '
    '"request_time": "$request_time", '
    '"status": "$status", '
    '"request_uri": "$request_uri", '
    '"request_method": "$request_method", '
    '"http_referrer": "$http_referer", '
    '"http_x_forwarded_for": "$http_x_forwarded_for", '
    '"http_user_agent": "$http_user_agent"}';

access_log  /var/log/nginx/access.log  json;

重新启动服务

systemctl restart nginx

查看访问日志

修改logstash配置文件(Logstash再使用JSON过滤插件解析)

vim /opt/elk/logstash/conf.d/test.conf

input {
  beats {
   host => "0.0.0.0"
   port => 5044
  }
}
filter {
  json {
    source => "message"
  }

  if [app] == "product" and [project] == "microservice" {
  mutate {
  add_field => {
  "[@metadata][target_index]" => "microservice-product-%{+YYYY.MM}"
	}
  }
} else if [app] == "gateway" and [project] == "microservice" {
  mutate {
  add_field => {
  "[@metadata][target_index]" => "microservice-gateway-%{+YYYY.MM.dd}"
	}
  }
} else if [app] == "nginx" and [project] == "microservice" {
  mutate {
  add_field => {
  "[@metadata][target_index]" => "microservice-nginx-%{+YYYY.MM.dd}"
	}
  }
} else {
  mutate {
  add_field => {
  "[@metadata][target_index]" => "unknown-%{+YYYY}"
	  }
	}
     }
}
output {
  elasticsearch {
  hosts => "192.168.0.11:9200"
  index => "%{[@metadata][target_index]}"
  }
}

热加载配置

kill -HUP <logstash pid>

在kibana页面上查看