# repmgr Postgres cluster

https://repmgr.org/docs/repmgr.html

Postgres 11 with streaming replication and backups.

# Servers

pg-oca             --- Master Postgres Server
pg-oca-rep1        --- Standby Postgres Server
pg-oca-hdd         --- Standby Postgres Server
pg-oca-barman      --- Postgres Backup Server
pg-oca-witness-1   --- Postgres Witness Server

Each server has two ethernet interfaces. One for the VPN for ssh,
and the second for database backend communication.

# Install
Add Postgres community repository to each server.

wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -

echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list

Install Postgres 11 client and repmgr on all servers.

apt update
apt install postgresql-client-11 postgresql-11-repmgr

Install Postgres 11 server on pg-oca and install repmgr module.
apt install postgresql-11 postgresql-11-repmgr

# System Configuration

Configure firewall.

#=============== Postgres Cluster ===================
-A INPUT --dest 10.104.11.0/24 -p icmp -m icmp --icmp-type 8 -j ACCEPT
-A INPUT --dest 10.53.1.0/24 -p icmp -m icmp --icmp-type 8 -j ACCEPT
-A INPUT -p tcp --dest 10.53.1.0/24 --dport 61 -j ACCEPT
-A INPUT -p tcp --dest 10.53.1.0/24 --dport 5432 -j ACCEPT
#=============== End of Postgres Cluster =============



Set up /etc/hosts on all nodes:
cat >> /etc/hosts << EOF
10.104.11.44    oca
10.104.11.116   pg-oca
10.104.11.131   pg-oca-rep1
10.104.11.132   pg-oca-hdd
10.104.11.133   pg-oca-barman
10.104.11.134   pg-oca-witness-1

10.53.1.44      db-oca
10.53.1.116     db-pg-oca
10.53.1.131     db-pg-oca-rep1
10.53.1.132     db-pg-oca-hdd
10.53.1.133     db-pg-oca-barman
10.53.1.134     db-pg-oca-witness-1
EOF


Configure base Postgres 11 server.
pg_hba.conf and tuning


On master server pg-oca and pg-oca-rep1

cat > /etc/postgresql/11/main/conf.d/replication.conf << EOF
# Enable replication connections; set this figure to at least one more
# than the number of standbys which will connect to this server
# (note that repmgr will execute `pg_basebackup` in WAL streaming mode,
# which requires two free WAL senders)

max_wal_senders = 10

# Ensure WAL files contain enough information to enable read-only queries
# on the standby.
#
#  PostgreSQL 9.5 and earlier: one of 'hot_standby' or 'logical'
#  PostgreSQL 9.6 and later: one of 'replica' or 'logical'
#    ('hot_standby' will still be accepted as an alias for 'replica')
#
# See: https://www.postgresql.org/docs/current/static/runtime-config-wal.html#GUC-WAL-LEVEL

wal_level = 'replica'

# Enable read-only queries on a standby
# (Note: this will be ignored on a primary but we recommend including
# it anyway)

hot_standby = on

# Enable WAL file archiving
archive_mode = on

# Set archive command to a script or application that will safely store
# you WALs in a secure place. /bin/true is an example of a command that
# ignores archiving. Use something more sensible.
# XXX
archive_command = '/bin/true'

# If you have configured "pg_basebackup_options"
# in "repmgr.conf" to include the setting "--xlog-method=fetch" (from
# PostgreSQL 10 "--wal-method=fetch"), *and* you have not set
# "restore_command" in "repmgr.conf"to fetch WAL files from another
# source such as Barman, you'll need to set "wal_keep_segments" to a
# high enough value to ensure that all WAL files generated while
# the standby is being cloned are retained until the standby starts up.
#
# wal_keep_segments = 5000
   
EOF


Set up pg_hba.conf on hosts pg-oca and pg-oca-rep1

cat > /etc/postgresql/11/main/pg_hba.conf << EOF
# Local
local   all           all                                 peer
local   all           postgres                            peer
host    all           all         127.0.0.1/32            md5

# Replication
local   replication   repmgr                              trust
host    replication   repmgr      127.0.0.1/32            trust
host    replication   repmgr      10.53.1.116/32          trust
host    replication   repmgr      10.53.1.131/32          trust
host    replication   repmgr      10.53.1.132/32          trust
host    replication   repmgr      10.53.1.133/32          trust
host    replication   repmgr      10.53.1.134/32          trust

# repmgr
local   repmgr        repmgr                              trust
host    repmgr        repmgr      127.0.0.1/32            trust
host    repmgr        repmgr      10.53.1.116/32          trust
host    repmgr        repmgr      10.53.1.131/32          trust
host    repmgr        repmgr      10.53.1.132/32          trust
host    repmgr        repmgr      10.53.1.133/32          trust
host    repmgr        repmgr      10.53.1.134/32          trust

# OCA
host    all           oca         10.53.1.44/32           trust
host    all           postgres    10.53.1.44/32           trust
EOF


Set up configuration. Note to change listen_addresses below:

cat > /etc/postgresql/11/main/conf.d/pgconfig.conf << EOF
# Memory Configuration
shared_buffers = 8GB
effective_cache_size = 24GB
work_mem = 128MB
maintenance_work_mem = 2GB

# Checkpoint Related Configuration
min_wal_size = 1GB
max_wal_size = 3GB
checkpoint_completion_target = 0.9
wal_buffers = 16MB

# Network Related Configuration
# XXX Set
#listen_addresses = '10.53.1.130'
max_connections = 256

# Hard Drive Configuration
random_page_cost = 1.1
effective_io_concurrency = 300

# Adjust the minimum time to collect data
log_min_duration_statement = '10s'
log_autovacuum_min_duration = 0
EOF

Configure ssh between nodes. Which users?

# repmgr Cluster Master Configuration
Set up nodes to communicate.

Node pg-oca is master.
Node pg-oca-rep1 does streaming replication of pg-oca.
Node pg-oca-hdd does streaming replication of pg-oca-rep1 and stores to hard disk platters instead of NVMe drives.
Node pg-oca-barman runs the Postgres barman backup system, and does backups of pg-oca-rep1.
Node pg-oca-witness-1 is a Postgres witness server. It isn't needed at present.

Run on pg-oca:
sudo su - postgres
createuser -s repmgr
createdb repmgr -O repmgr

# repmgr Cluster Standby Configuration
On the standby, do not create a PostgreSQL instance.

From pg-oca:
 Check the primary database is reachable from the standby using psql:

psql 'host=db-pg-oca user=repmgr dbname=repmgr connect_timeout=2'


# Set up Master repmgr

cat > /etc/repmgr.conf << EOF
node_id=1
node_name=pg-oca
conninfo='host=db-pg-oca user=repmgr dbname=repmgr connect_timeout=2'
data_directory='/var/lib/postgresql/11/main'
pg_bindir='/usr/lib/postgresql/11/bin'
repmgr_bindir='/usr/lib/postgresql/11/bin'
EOF

Register primary server:

sudo su - postgres
repmgr -f /etc/repmgr.conf primary register

# Clone Standby Server
Log into pg-oca-rep1

cat > /etc/repmgr.conf << EOF
node_id=2
node_name=pg-oca-rep1
conninfo='host=db-pg-oca-rep1 user=repmgr dbname=repmgr connect_timeout=2'
data_directory='/var/lib/postgresql/11/main'
pg_bindir='/usr/lib/postgresql/11/bin'
repmgr_bindir='/usr/lib/postgresql/11/bin'
EOF

systemctl stop postgresql

sudo su - postgres
Test it, dry run
repmgr -h db-pg-oca -U repmgr -d repmgr -f /etc/repmgr.conf standby clone --force --dry-run

If good, then run it:
repmgr -h db-pg-oca -U repmgr -d repmgr -f /etc/repmgr.conf standby clone --force

Then as root, start server:
systemctl start postgresql

As user postgres:
sudo su - postgres
repmgr standby register

Check it out:
repmgr cluster show

# Create Database

Log into main server pg-oca.
sudo su - postgres
createuser --createdb oca
createdb --owner=oca oca
