forksand-it-manual/source/resources/apps/prometheus

apt update

# Stretch:

apt install -t stretch-backports prometheus

# Buster:

apt install prometheus



# On main prom1 server, running buster:

apt install prometheus-alertmanager prometheus prometheus-node-exporter



# Open up ports on firewall:

# Prometheus

-A INPUT -p tcp --dport 9090 -j ACCEPT

-A INPUT -p tcp --dport 9093 -j ACCEPT

-A INPUT -p tcp --dport 9094 -j ACCEPT

-A INPUT -p tcp --dport 9100 -j ACCEPT



# Install exporter on all nodes

# Stretch:

apt install -t stretch-backports prometheus-node-exporter

# Buster:

apt install prometheus-node-exporter



# To enable various collectors on a node, edit:

vim /etc/default/prometheus-node-exporter



# Ones of note:

-collector.netstat.fields=(.*) --collector.vmstat.fields=(.*)  --collector.interrupts



# Add to all: XXX not done

################

--collector.netstat.fields="^(.*_(InErrors|InErrs)|Ip_Forwarding|Ip(6|Ext)_(InOctets|OutOctets)|Icmp6?_(InMsgs|OutMsgs)|TcpExt_(Listen.*|Syncookies.*)|Tcp_(ActiveOpens|PassiveOpens|RetransSegs|CurrEstab)|Udp6?_(InDatagrams|OutDatagrams|NoPorts))$"

--collector.vmstat.fields="^(oom_kill|pgpg|pswp|pg.*fault).*"

--collector.interrupts

--collector.ntp

--collector.systemd

################



##############################################################################

# For MySQL servers, on all servers:

apt install prometheus-mysqld-exporter

# On sql1, log into database:

CREATE USER 'prometheus-mysqld-exporter'@'10.42.1.0/255.255.255.0' IDENTIFIED BY 'password' WITH MAX_USER_CONNECTIONS 3;

GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO 'prometheus-mysqld-exporter'@'10.42.1.0/255.255.255.0';

FLUSH PRIVILEGES;



# On sql1 sql2 sql3 sql4 servers:

# Edit

vim /etc/default/prometheus-mysqld-exporter



# Set this line (setting correct local IP):

export DATA_SOURCE_NAME="prometheus-mysqld-exporter:password@(10.42.1.171:3306)/"



# on prom1 /etc/prometheus/prometheus.yml

  - job_name: 'mysql-monitor'# Job name

    static_configs:

      - targets: ['10.42.1.171:9104']

        labels:

            alias: db1# alias name given to this instance



# Add firewall rule to sql1 sql2 sql3 sql4

#-A INPUT -s 10.22.22.0/24 -p tcp --dport 9104 -j ACCEPT

-A INPUT -s 10.42.1.0/24 -p tcp --dport 9104 -j ACCEPT



##############################################################################

# Borg exporter:

https://github.com/teemow/prometheus-borg-exporter

https://grafana.com/dashboards/1573

##############################################################################

# Email

# graylog:

##############################################################################

# Apache:

https://grafana.com/dashboards/4859

https://grafana.com/dashboards/4865

##############################################################################

# nginx

https://grafana.com/dashboards/6482

https://grafana.com/dashboards/4368

https://grafana.com/dashboards/6787

##############################################################################

# postgres

https://grafana.com/dashboards/3300

https://grafana.com/dashboards/455

https://grafana.com/dashboards/3742

https://grafana.com/dashboards/6742



# on pg1 pg2 pg3 pg4

apt install prometheus-postgres-exporter daemon

# See /usr/share/doc/prometheus-postgres-exporter/README.Debian

vim /etc/default/prometheus-postgres-exporter

DATA_SOURCE_NAME='user=prometheus host=/run/postgresql dbname=postgres'



# The run:

sudo -u postgres psql

# In database add:

------------------------------------------------------

  CREATE USER prometheus;

  ALTER USER prometheus SET SEARCH_PATH TO prometheus,pg_catalog;

  

  CREATE SCHEMA prometheus AUTHORIZATION prometheus;

  

  CREATE FUNCTION prometheus.f_select_pg_stat_activity()

  RETURNS setof pg_catalog.pg_stat_activity

  LANGUAGE sql

  SECURITY DEFINER

  AS $$

    SELECT * from pg_catalog.pg_stat_activity;

  $$;

  

  CREATE FUNCTION prometheus.f_select_pg_stat_replication()

  RETURNS setof pg_catalog.pg_stat_replication

  LANGUAGE sql

  SECURITY DEFINER

  AS $$

    SELECT * from pg_catalog.pg_stat_replication;

  $$;

  

  CREATE VIEW prometheus.pg_stat_replication

  AS

    SELECT * FROM prometheus.f_select_pg_stat_replication();

  

  CREATE VIEW prometheus.pg_stat_activity

  AS

    SELECT * FROM prometheus.f_select_pg_stat_activity();

  

  GRANT SELECT ON prometheus.pg_stat_replication TO prometheus;

  GRANT SELECT ON prometheus.pg_stat_activity TO prometheus;

------------------------------------------------------

------------------------------------------------------



# Restart



# Add firewall rule

-A INPUT -s 10.52.1.0/24 -p tcp --dport 9187 -j ACCEPT



# prometheus.yml

# Some dashboards work with this

  - job_name: 'postgres'

    honor_labels: true

    static_configs:

      - targets: ['10.52.1.91:9187', '10.52.1.92:9187', '10.52.1.93:9187', '10.52.1.94:9187']



# Other dashboards work with this

  - job_name: 'postgresql'

    honor_labels: true

    static_configs:

      - targets: ['10.52.1.91:9187', '10.52.1.92:9187', '10.52.1.93:9187', '10.52.1.94:9187']



##############################################################################

# Unbound DNS

https://grafana.com/dashboards/3930

##############################################################################

# libvirt_exporter

# FAIL

go get github.com/kumina/libvirt_exporter

go build github.com/kumina/libvirt_exporter

# Then copy this binary to targets sf-001 sf-002 sf-003 sf-004

~/go/bin/libvirt_exporter



apt install libvirt-daemon



##############################################################################