The document outlines the setup and configuration of a logging system using Logstash, Elasticsearch, and Kibana to analyze Apache access log files. It provides detailed input configurations for Logstash, including data filtering and geographic IP resolution, and describes dashboard elements in Kibana for visualizing the logged data. Additionally, examples of queries and visualizations for effective data representation and analysis are included.

1. FINDOLOGIC GmbH
Boring Logfiles → Sexy Dashboards

2. Boring Logfiles
178.236.7.219 - - [27/Oct/2014:06:43:13 +0100] "GET
/ps/amazon.de/index.php?shopkey=DEADBEEF&shopurl=https
%3A%2F%2Fwww.amazon.de
%2F&multishop_id=0&userip=173.194.112.2&referer=https
%3A%2F%2Fwww.amazon.de%2FTechnik
%2FFernseher&count=0&group[]=2&service=search&query=le
d HTTP/1.1" 200 12363 "-" "-"

3. Sexy Dashboards

4. Stack
● Elasticsearch
● Logstash
● Kibana

5. Infrastruktur
App 1 App 2 App 3
Logstash 1 Logstash 2 Logstash 3
ElasticSearch
Kibana

6. Logstash Architektur
Input Filter Output
● Grok
● Search / Replace
● Split
● GeoIP
● DNS
● URL-Decode
● ...
● ElasticSearch
● Redis
● CouchDB
● ...
● Pipe
● Socket
● File
● ...

7. Logstash stand-alone
● Logs zentralisieren
● In DB speichern
● Durchsuchbar machen

8. Setup
● Logstash herunterladen, extrahieren
● Elasticsearch installieren
● fertig

9. Georg?
● CTO bei FINDOLOGIC GmbH
● http://www.findologic.com
● g.sorst@findologic.com
● @piefke_schorsch

10. Hands on
● Grok
● kv
● Query
● Endpoint
● GeoIP

11. Logstash.conf
input {
file {
path => "/home/georg/coding/logstash-apache/access.log"
start_position => beginning
}
#pipe {
# command => "ssh frontend1.findologic.com tail -f /var/log/apache2/access.log"
#}
}
filter {
grok {
match => { "message" => "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} [%{HTTPDATE:timestamp}] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent}" }
}
# split the query string: query=test&count=0&first=10 → request_query: test, request_count: 0, request_first: 10
kv {
field_split => "&?"
source => "request"
prefix => "request_"
}
# urldecode does not decode the + into whitespace so do this manually
mutate {
gsub => [
"request_query", "+", " "
]
}
# URL-decode the query so it is easier to read. We don't really about the other request_* parameters
urldecode {
field => "request_query"
}
# extract the script file, eg. index.php
mutate {
gsub => [
# Remove the query string including the ? : /path/index.php?key=value -> /path/index.php
"request", "?.*$", "",
# Remove everything up to the last / : /path/index.php -> index.php
"request", "^.*/", ""
]
}
# Default to index.php if no script file is given
if [request] == "" {
mutate {
replace => [ "request", "index.php" ]
}
}
# If the IP is syntactically valid resolve it geographically
if [request_userip] =~ "^d+.d+.d+.d+$" {
geoip {
source => "request_userip"
target => "client_geoip"
}
}
}
output {
elasticsearch {
host => localhost
}
#stdout { codec => rubydebug }
}

12. Kibana Dashboard
{
"title": "Logstash Search",
"services": {
"query": {
"list": {
"0": {
"query": "*",
"alias": "",
"color": "#7EB26D",
"id": 0,
"pin": false,
"type": "lucene",
"enable": true
},
"1": {
"id": 1,
"color": "#EAB839",
"alias": "Suche",
"pin": false,
"type": "lucene",
"enable": true,
"query": "request:index.php"
},
"2": {
"id": 2,
"color": "#6ED0E0",
"alias": "Autocomplete",
"pin": false,
"type": "lucene",
"enable": true,
"query": "request:autocomplete.php"
},
"3": {
"id": 3,
"color": "#EF843C",
"alias": "",
"pin": false,
"type": "lucene",
"enable": true,
"query": "client_geoip.country_code2:DE OR client_geoip.country_code2:AT OR client_geoip.country_code2:CH"
}
},
"ids": [
0,
1,
2,
3
]
},
"filter": {
"list": {
"0": {
"type": "time",
"field": "@timestamp",
"from": "now-24h",
"to": "now",
"mandate": "must",
"active": true,
"alias": "",
"id": 0
},
"1": {
"type": "time",
"from": "2014-10-27T18:04:33.357Z",
"to": "2014-10-27T19:18:47.459Z",
"field": "@timestamp",
"mandate": "must",
"active": true,
"alias": "",
"id": 1
}
},
"ids": [
0,
1
]
}
},
"rows": [
{
"title": "Graph",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "histogram",
"mode": "count",
"time_field": "@timestamp",
"value_field": null,
"auto_int": true,
"resolution": 100,
"interval": "30s",
"fill": 3,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": true,
"stack": true,
"points": false,
"lines": false,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"queries": {
"mode": "selected",
"ids": [
1,
2
]
},
"title": "Events over time",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": true
},
"scale": 1,
"y_format": "none",
"grid": {
"max": null,
"min": 0
},
"annotate": {
"enable": false,
"query": "*",
"size": 20,
"field": "_type",
"sort": [
"_score",
"desc"
]
},
"pointradius": 5,
"show_query": true,
"legend_counts": true,
"zerofill": true,
"derivative": false
}
],
"notice": false
},
{
"title": "",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 6,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "request_query",
"exclude": [],
"missing": false,
"other": false,
"size": 10,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "pie",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": ""
},
{
"error": false,
"span": 6,
"editable": true,
"type": "bettermap",
"loadingEditor": false,
"field": "client_geoip.location",
"size": 1000,
"spyable": true,
"tooltip": "_id",
"queries": {
"mode": "selected",
"ids": [
3
]
}
}
],
"notice": false
},
{
"title": "Events",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"title": "All events",
"error": false,
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "table",
"size": 100,
"pages": 5,
"offset": 0,
"sort": [
"@timestamp",
"desc"
],
"style": {
"font-size": "9pt"
},
"overflow": "min-height",
"fields": [],
"localTime": true,
"timeField": "@timestamp",
"highlight": [],
"sortable": true,
"header": true,
"paging": true,
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3
]
},
"field_list": true,
"status": "Stable",
"trimFactor": 300,
"normTimes": true,
"all_fields": false
}
],
"notice": false
}
],
"editable": true,
"failover": false,
"index": {
"interval": "day",
"pattern": "[logstash-]YYYY.MM.DD",
"default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED",
"warm_fields": true
},
"style": "light",
"panel_hints": true,
"pulldowns": [
{
"type": "query",
"collapse": false,
"notice": false,
"query": "*",
"pinned": true,
"history": [
"client_geoip.country_code2:DE OR client_geoip.country_code2:AT OR client_geoip.country_code2:CH",
"request:autocomplete.php",
"request:index.php",
"*"
],
"remember": 10,
"enable": true
},
{
"type": "filtering",
"collapse": false,
"notice": true,
"enable": true
}
],
"nav": [
{
"type": "timepicker",
"collapse": false,
"notice": false,
"status": "Stable",
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
],
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"timefield": "@timestamp",
"now": false,
"filter_id": 0,
"enable": true
}
],
"loader": {
"save_gist": false,
"save_elasticsearch": true,
"save_local": true,
"save_default": true,
"save_temp": true,
"save_temp_ttl_enable": true,
"save_temp_ttl": "30d",
"load_gist": true,
"load_elasticsearch": true,
"load_elasticsearch_size": 20,
"load_local": true,
"hide": false
},
"refresh": false
}
Vergrößern zum Anzeigen