Add ELK

source/resources/apps/elasticsearch/README

@@ -2,26 +2,41 @@
 apt update
 apt -y dist-upgrade
 
-# Install elasticsearch
+##############################################################################
+# Install Java dependency
+apt install openjdk-8-jre-headless
+
+
+# Install Elasticsearch version 6 (latest)
 # Get key
 wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
 # Install deps (should be installed already):
 apt-get -y install apt-transport-https
-# Set up repo for previous release 5.x (current is 6.x).
+# Set up repo for release 6.x
-# Graylog 2 only works with 5.x. Graylog 3, due in 2018, works with elasticsearch 6.x
+echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-6.x.list
-#echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-6.x.list
-
-echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | tee -a /etc/apt/sources.list.d/elastic-5.x.list
 
 # Disable apt-cache in /etc/apt/apt.conf, it doesn't work with https
 apt update
-apt install openjdk-8-jre-headless
+
-apt install elasticsearch
+# It doesn't appear the open source version is in the repo, needs manual install. XXX
-# Note for Version 6.x, use
 #apt install elasticsearch-oss
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-6.3.2.deb
+dpkg -i elasticsearch-oss-6.3.2.deb
 
 # Configure a cluster name and answer on IP.
 # Open firewall
+# Allow elasticsearch
+-A INPUT -p tcp --dport 9200 -j ACCEPT
+-A INPUT -p tcp --dport 9300 -j ACCEPT
+
+# Set up configuration:
+vim /etc/elasticsearch/elasticsearch.yml
+# Set:
+cluster.name: elasticsearch
+network.host: 10.22.22.124
+
+# Start:
+systemctl start elasticsearch.service
 
 # Start on boot:
 systemctl enable elasticsearch.service
@@ -29,3 +44,4 @@ systemctl enable elasticsearch.service
 ### XXX Backups
 ### XXX Prometheus :)
 
+##############################################################################

source/resources/apps/elasticsearch/README-v5

@@ -0,0 +1,34 @@
+# Clone Debian Stretch template, set up IPs, hostname, ssh keys
+apt update
+apt -y dist-upgrade
+
+##############################################################################
+# Install Elasticsearch version 5
+# Graylog 2 can only use Elasticsearch 5, not latest version 6
+# Get key
+wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
+# Install deps (should be installed already):
+apt-get -y install apt-transport-https
+# Set up repo for previous release 5.x (current is 6.x).
+# Graylog 2 only works with 5.x. Graylog 3, due in 2018, works with elasticsearch 6.x
+
+echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | tee -a /etc/apt/sources.list.d/elastic-5.x.list
+
+# Disable apt-cache in /etc/apt/apt.conf, it doesn't work with https
+apt update
+apt install openjdk-8-jre-headless
+apt install elasticsearch
+
+# Configure a cluster name and answer on IP.
+# Open firewall
+# Allow elasticsearch
+-A INPUT -p tcp --dport 9200 -j ACCEPT
+-A INPUT -p tcp --dport 9300 -j ACCEPT
+
+# Start on boot:
+systemctl enable elasticsearch.service
+
+### XXX Backups
+### XXX Prometheus :)
+
+##############################################################################

source/resources/apps/kibana/README

@@ -14,7 +14,8 @@ wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add
 echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-6.x.list
 # Disable apt cache in /etc/apt/apt.conf
 apt update
-apt-get install kibana
+# XXX Note the "kibana" package is non-free. Use the -oss package!
+apt-get install kibana-oss
 
 # Start on boot:
 systemctl enable kibana.service

source/resources/apps/logstash/README

@@ -16,8 +16,46 @@ echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" | sudo tee
 apt update
 apt-get install logstash
 
-# Start on boot:
+# Configure
-systemctl enable logstash.service
+vim /etc/logstash/logstash.yml
+http.host: "10.22.22.108"
+http.port: 9600
+
+
+cat > /etc/logstash/conf.d/logstash-syslog.conf <<EOF
+input {
+  tcp {
+    port => 5000
+    type => syslog
+  }
+  udp {
+    port => 5000
+    type => syslog
+  }
+}
+
+filter {
+  if [type] == "syslog" {
+    grok {
+      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
+      add_field => [ "received_at", "%{@timestamp}" ]
+      add_field => [ "received_from", "%{host}" ]
+    }
+    date {
+      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
+    }
+  }
+}
+output {
+  elasticsearch { hosts => ["10.22.22.124:9200"] }
+  stdout { codec => rubydebug }
+}
+EOF
+
+
+
+# Start:
+systemctl start logstash.service
 
 # Open firewall