apt update # Stretch: apt install -t stretch-backports prometheus # Buster: apt install prometheus # On main prom1 server, running buster: apt install prometheus-alertmanager prometheus prometheus-node-exporter # Open up ports on firewall: # Prometheus -A INPUT -p tcp --dport 9090 -j ACCEPT -A INPUT -p tcp --dport 9093 -j ACCEPT -A INPUT -p tcp --dport 9094 -j ACCEPT -A INPUT -p tcp --dport 9100 -j ACCEPT # Install exporter on all nodes # Stretch: apt install -t stretch-backports prometheus-node-exporter # Buster: apt install prometheus-node-exporter # To enable various collectors on a node, edit: vim /etc/default/prometheus-node-exporter # Ones of note: -collector.netstat.fields=(.*) --collector.vmstat.fields=(.*) --collector.interrupts # Add to all: XXX not done ################ --collector.netstat.fields="^(.*_(InErrors|InErrs)|Ip_Forwarding|Ip(6|Ext)_(InOctets|OutOctets)|Icmp6?_(InMsgs|OutMsgs)|TcpExt_(Listen.*|Syncookies.*)|Tcp_(ActiveOpens|PassiveOpens|RetransSegs|CurrEstab)|Udp6?_(InDatagrams|OutDatagrams|NoPorts))$" --collector.vmstat.fields="^(oom_kill|pgpg|pswp|pg.*fault).*" --collector.interrupts --collector.ntp --collector.systemd ################ # Just copy over an existing one: # XXX not really. #scp -p testo4:/etc/default/prometheus-node-exporter /etc/default/prometheus-node-exporter # Start on boot: systemctl enable prometheus-node-exporter ############################################################################## # For MySQL servers, on all servers: apt install prometheus-mysqld-exporter # On sql1, log into database: CREATE USER 'prometheus-mysqld-exporter'@'10.42.1.0/255.255.255.0' IDENTIFIED BY 'password' WITH MAX_USER_CONNECTIONS 3; GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO 'prometheus-mysqld-exporter'@'10.42.1.0/255.255.255.0'; FLUSH PRIVILEGES; # On sql1 sql2 sql3 sql4 servers: # Edit vim /etc/default/prometheus-mysqld-exporter # Set this line (setting correct local IP): export DATA_SOURCE_NAME="prometheus-mysqld-exporter:password@(10.42.1.171:3306)/" # on prom1 /etc/prometheus/prometheus.yml - job_name: 'mysql-monitor'# Job name static_configs: - targets: ['10.42.1.171:9104'] labels: alias: db1# alias name given to this instance # Add firewall rule to sql1 sql2 sql3 sql4 #-A INPUT -s 10.22.22.0/24 -p tcp --dport 9104 -j ACCEPT -A INPUT -s 10.42.1.0/24 -p tcp --dport 9104 -j ACCEPT ############################################################################## # Borg exporter: https://github.com/teemow/prometheus-borg-exporter https://grafana.com/dashboards/1573 ############################################################################## # Email # graylog: ############################################################################## # Apache: https://grafana.com/dashboards/4859 https://grafana.com/dashboards/4865 ############################################################################## # nginx https://grafana.com/dashboards/6482 https://grafana.com/dashboards/4368 https://grafana.com/dashboards/6787 ############################################################################## # postgres https://grafana.com/dashboards/3300 https://grafana.com/dashboards/455 https://grafana.com/dashboards/3742 https://grafana.com/dashboards/6742 # on pg1 pg2 pg3 pg4 apt install prometheus-postgres-exporter daemon # See /usr/share/doc/prometheus-postgres-exporter/README.Debian vim /etc/default/prometheus-postgres-exporter DATA_SOURCE_NAME='user=prometheus host=/run/postgresql dbname=postgres' # The run: sudo -u postgres psql # In database add: ------------------------------------------------------ CREATE USER prometheus; ALTER USER prometheus SET SEARCH_PATH TO prometheus,pg_catalog; CREATE SCHEMA prometheus AUTHORIZATION prometheus; CREATE FUNCTION prometheus.f_select_pg_stat_activity() RETURNS setof pg_catalog.pg_stat_activity LANGUAGE sql SECURITY DEFINER AS $$ SELECT * from pg_catalog.pg_stat_activity; $$; CREATE FUNCTION prometheus.f_select_pg_stat_replication() RETURNS setof pg_catalog.pg_stat_replication LANGUAGE sql SECURITY DEFINER AS $$ SELECT * from pg_catalog.pg_stat_replication; $$; CREATE VIEW prometheus.pg_stat_replication AS SELECT * FROM prometheus.f_select_pg_stat_replication(); CREATE VIEW prometheus.pg_stat_activity AS SELECT * FROM prometheus.f_select_pg_stat_activity(); GRANT SELECT ON prometheus.pg_stat_replication TO prometheus; GRANT SELECT ON prometheus.pg_stat_activity TO prometheus; ------------------------------------------------------ ------------------------------------------------------ # Restart # Add firewall rule -A INPUT -s 10.52.1.0/24 -p tcp --dport 9187 -j ACCEPT # prometheus.yml # Some dashboards work with this - job_name: 'postgres' honor_labels: true static_configs: - targets: ['10.52.1.91:9187', '10.52.1.92:9187', '10.52.1.93:9187', '10.52.1.94:9187'] # Other dashboards work with this - job_name: 'postgresql' honor_labels: true static_configs: - targets: ['10.52.1.91:9187', '10.52.1.92:9187', '10.52.1.93:9187', '10.52.1.94:9187'] ############################################################################## # Unbound DNS https://grafana.com/dashboards/3930 ############################################################################## # libvirt_exporter # FAIL go get github.com/kumina/libvirt_exporter go build github.com/kumina/libvirt_exporter # Then copy this binary to targets sf-001 sf-002 sf-003 sf-004 ~/go/bin/libvirt_exporter apt install libvirt-daemon ############################################################################## # Dovecot Exporter # https://github.com/kumina/dovecot_exporter.git # # On a go build server: XXX go get github.com/kumina/dovecot_exporter go build github.com/kumina/dovecot_exporter scp -p go/bin/dovecot_exporter mx1: # On mail server mx1 # Set up user to run as. XXX # Open up firewall port 9166 # Dovecot Prometheus -A INPUT -s 10.22.22.0/24 -p tcp --dport 9166 -j ACCEPT # Run thusly, XXX set up user, chown path ./dovecot_exporter --dovecot.socket-path="/var/run/dovecot/stats-reader" # Or? ./dovecot_exporter --dovecot.socket-path="/var/run/dovecot/stats-reader" --dovecot.scopes="jebba@forksand.com" ##############################################################################