forksand-it-manual/source/resources/apps/prometheus/README

apt update
# Stretch:
apt install -t stretch-backports prometheus
# Buster:
apt install prometheus

# On main prom1 server, running buster:
apt install prometheus-alertmanager prometheus prometheus-node-exporter

# Open up ports on firewall:
# Prometheus
-A INPUT -p tcp --dport 9090 -j ACCEPT
-A INPUT -p tcp --dport 9093 -j ACCEPT
-A INPUT -p tcp --dport 9094 -j ACCEPT
-A INPUT -p tcp --dport 9100 -j ACCEPT

# Install exporter on all nodes
# Stretch:
apt install -t stretch-backports prometheus-node-exporter
# Buster:
apt install prometheus-node-exporter

# To enable various collectors on a node, edit:
vim /etc/default/prometheus-node-exporter

# Ones of note:
-collector.netstat.fields=(.*) --collector.vmstat.fields=(.*)  --collector.interrupts

# Add to all: XXX not done
################
--collector.netstat.fields="^(.*_(InErrors|InErrs)|Ip_Forwarding|Ip(6|Ext)_(InOctets|OutOctets)|Icmp6?_(InMsgs|OutMsgs)|TcpExt_(Listen.*|Syncookies.*)|Tcp_(ActiveOpens|PassiveOpens|RetransSegs|CurrEstab)|Udp6?_(InDatagrams|OutDatagrams|NoPorts))$"
--collector.vmstat.fields="^(oom_kill|pgpg|pswp|pg.*fault).*"
--collector.interrupts
--collector.ntp
--collector.systemd
################

##############################################################################
# For MySQL servers, on all servers:
apt install prometheus-mysqld-exporter
# On sql1, log into database:
CREATE USER 'prometheus-mysqld-exporter'@'10.42.1.0/255.255.255.0' IDENTIFIED BY 'password' WITH MAX_USER_CONNECTIONS 3;
GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO 'prometheus-mysqld-exporter'@'10.42.1.0/255.255.255.0';
FLUSH PRIVILEGES;

# On sql1 sql2 sql3 sql4 servers:
# Edit
vim /etc/default/prometheus-mysqld-exporter

# Set this line (setting correct local IP):
export DATA_SOURCE_NAME="prometheus-mysqld-exporter:password@(10.42.1.171:3306)/"

# on prom1 /etc/prometheus/prometheus.yml
  - job_name: 'mysql-monitor'# Job name
    static_configs:
      - targets: ['10.42.1.171:9104']
        labels:
            alias: db1# alias name given to this instance

# Add firewall rule to sql1 sql2 sql3 sql4
#-A INPUT -s 10.22.22.0/24 -p tcp --dport 9104 -j ACCEPT
-A INPUT -s 10.42.1.0/24 -p tcp --dport 9104 -j ACCEPT

##############################################################################
# Borg exporter:
https://github.com/teemow/prometheus-borg-exporter
https://grafana.com/dashboards/1573
##############################################################################
# Email
# graylog:
##############################################################################
# Apache:
https://grafana.com/dashboards/4859
https://grafana.com/dashboards/4865
##############################################################################
# nginx
https://grafana.com/dashboards/6482
https://grafana.com/dashboards/4368
https://grafana.com/dashboards/6787
##############################################################################
# postgres
https://grafana.com/dashboards/3300
https://grafana.com/dashboards/455
https://grafana.com/dashboards/3742
https://grafana.com/dashboards/6742

# on pg1 pg2 pg3 pg4
apt install prometheus-postgres-exporter daemon
# See /usr/share/doc/prometheus-postgres-exporter/README.Debian
vim /etc/default/prometheus-postgres-exporter
DATA_SOURCE_NAME='user=prometheus host=/run/postgresql dbname=postgres'

# The run:
sudo -u postgres psql
# In database add:
------------------------------------------------------
  CREATE USER prometheus;
  ALTER USER prometheus SET SEARCH_PATH TO prometheus,pg_catalog;
  
  CREATE SCHEMA prometheus AUTHORIZATION prometheus;
  
  CREATE FUNCTION prometheus.f_select_pg_stat_activity()
  RETURNS setof pg_catalog.pg_stat_activity
  LANGUAGE sql
  SECURITY DEFINER
  AS $$
    SELECT * from pg_catalog.pg_stat_activity;
  $$;
  
  CREATE FUNCTION prometheus.f_select_pg_stat_replication()
  RETURNS setof pg_catalog.pg_stat_replication
  LANGUAGE sql
  SECURITY DEFINER
  AS $$
    SELECT * from pg_catalog.pg_stat_replication;
  $$;
  
  CREATE VIEW prometheus.pg_stat_replication
  AS
    SELECT * FROM prometheus.f_select_pg_stat_replication();
  
  CREATE VIEW prometheus.pg_stat_activity
  AS
    SELECT * FROM prometheus.f_select_pg_stat_activity();
  
  GRANT SELECT ON prometheus.pg_stat_replication TO prometheus;
  GRANT SELECT ON prometheus.pg_stat_activity TO prometheus;
------------------------------------------------------
------------------------------------------------------

# Restart

# Add firewall rule
-A INPUT -s 10.52.1.0/24 -p tcp --dport 9187 -j ACCEPT

# prometheus.yml
# Some dashboards work with this
  - job_name: 'postgres'
    honor_labels: true
    static_configs:
      - targets: ['10.52.1.91:9187', '10.52.1.92:9187', '10.52.1.93:9187', '10.52.1.94:9187']

# Other dashboards work with this
  - job_name: 'postgresql'
    honor_labels: true
    static_configs:
      - targets: ['10.52.1.91:9187', '10.52.1.92:9187', '10.52.1.93:9187', '10.52.1.94:9187']

##############################################################################
# Unbound DNS
https://grafana.com/dashboards/3930
##############################################################################
# libvirt_exporter
# FAIL
go get github.com/kumina/libvirt_exporter
go build github.com/kumina/libvirt_exporter
# Then copy this binary to targets sf-001 sf-002 sf-003 sf-004
~/go/bin/libvirt_exporter

apt install libvirt-daemon

##############################################################################