#!/bin/bash

exit 0

# https://www.citusdata.com/download/

# Run on all nodes:
curl https://install.citusdata.com/community/deb.sh | sudo bash
#sudo apt-get -y install postgresql-10-citus-7.4
sudo apt-get -y install postgresql-10-citus

# Outputs (dont run):
##############################################################################
Success. You can now start the database server using:

    /usr/lib/postgresql/10/bin/pg_ctl -D /var/lib/postgresql/10/main -l logfile start

Ver Cluster Port Status Owner    Data directory              Log file
10  main    5432 down   postgres /var/lib/postgresql/10/main /var/log/postgresql/postgresql-10-main.log
##############################################################################


# Run on all nodes
sudo pg_conftool 10 main set shared_preload_libraries citus
# set up IPs, for each node
# XXX THIS FAILS because the entry doesn't get quoted, so postgres doesn't start.
# pg1
sudo pg_conftool 10 main set listen_addresses '10.52.1.91'
# pg2
sudo pg_conftool 10 main set listen_addresses '10.52.1.92'
# pg3
sudo pg_conftool 10 main set listen_addresses '10.52.1.93'
# pg4
sudo pg_conftool 10 main set listen_addresses '10.52.1.94'
# XXX to fix, edit /etc/postgresql/10/main/postgresql.conf and change to be
# like this (note single quotes ')
# pg1
listen_addresses = '10.52.1.91'         # what IP address(es) to listen on;
# pg2
listen_addresses = '10.52.1.92'         # what IP address(es) to listen on;
# pg3
listen_addresses = '10.52.1.93'         # what IP address(es) to listen on;
# pg4
listen_addresses = '10.52.1.94'         # what IP address(es) to listen on;


# Set up access permissions file as root:
cat > /etc/postgresql/10/main/pg_hba.conf <<EOF
# IPv4 local connections
host    all             all             127.0.0.1/32            md5
# Database administrative login by Unix domain socket
local   all             postgres                                peer
# "local" is for Unix domain socket connections only
local   all             all                                     peer

# Allow Postgres Citus Nodes XXX use password hash
host    all             all             10.52.1.91/32            trust
host    all             all             10.52.1.92/32            trust
host    all             all             10.52.1.93/32            trust
host    all             all             10.52.1.94/32            trust

# Allow Postgres Clients XXX use password hash
# ocadev2
host    all             all             10.52.1.195/32            trust
# ocadev3
host    all             all             10.52.1.196/32            trust
# ocadev4
host    all             all             10.52.1.197/32            trust
EOF

# make it start automatically when computer does
update-rc.d postgresql enable

# Set up firewall
vim /etc/iptables.test.rules
# add rule:
-A INPUT -s 10.52.1.0/24 -d 10.52.1.0/24 -p tcp -m tcp --dport 5432 -j ACCEPT

# Run as root:
iptables-restore < /etc/iptables.test.rules
iptables-save > /etc/iptables.up.rules

cd /etc ; git add . ; git commit -a -m 'Setup Postgres Citus Cluster'

# start the db server
sudo service postgresql restart

# Run on all nodes:
sudo -i -u postgres psql -c "CREATE EXTENSION citus;"

sudo -i -u postgres psql -c "ALTER SYSTEM SET citus.enable_statistics_collection = 'off';"

# Run on just ONE node (pg1):
# Don't add the main host node or not? XXX
#sudo -i -u postgres psql -c "SELECT * from master_add_node('10.52.1.91', 5432);"
sudo -i -u postgres psql -c "SELECT * from master_add_node('10.52.1.92', 5432);"
sudo -i -u postgres psql -c "SELECT * from master_add_node('10.52.1.93', 5432);"
sudo -i -u postgres psql -c "SELECT * from master_add_node('10.52.1.94', 5432);"

# Verify it is all good
sudo -i -u postgres psql -c "SELECT * FROM master_get_active_worker_nodes();"

# Use postgres:
sudo -i -u postgres psql


# Misc notes:
postgres=# create database foo;
NOTICE:  Citus partially supports CREATE DATABASE for distributed databases
DETAIL:  Citus does not propagate CREATE DATABASE command to workers
HINT:  You can manually create a database and its extensions on workers.
CREATE DATABASE

#######
# OCA #
#######
# On ocadev2:
sudo -i -u postgres
# Backup database, run on ocadev2:
#pg_dump forksand > forksand-ocadev2-pg_dump.sql
pg_dump --format=custom --clean --create --no-owner forksand > forksand-ocadev2.sql

# Add "ocadev2" user to database. Run on ocadev2:
createuser --createdb --host=10.52.1.91 --username=postgres ocadev2
# NOTICE:  not propagating CREATE ROLE/USER commands to worker nodes
# HINT:  Connect to worker nodes directly to manually create all necessary users and roles.
# XXX
createuser --createdb --host=10.52.1.92 --username=postgres ocadev2
createuser --createdb --host=10.52.1.93 --username=postgres ocadev2
createuser --createdb --host=10.52.1.94 --username=postgres ocadev2

# Create database:
createdb --owner=ocadev2 --host=10.52.1.91 --username=postgres ocadev2_forksand
#NOTICE:  Citus partially supports CREATE DATABASE for distributed databases
#DETAIL:  Citus does not propagate CREATE DATABASE command to workers
#HINT:  You can manually create a database and its extensions on workers.
# XXX
createdb --owner=ocadev2 --host=10.52.1.92 --username=postgres ocadev2_forksand
createdb --owner=ocadev2 --host=10.52.1.93 --username=postgres ocadev2_forksand
createdb --owner=ocadev2 --host=10.52.1.94 --username=postgres ocadev2_forksand

# Run this logged in to master pg1
# (XXX connect to other database ?)
\connect ocadev2_forksand
SET citus.shard_max_size TO '64MB';
# How many copies to distribute (just run on master node):
SET citus.shard_replication_factor TO '2';


# Upload database dump to Postgres server
# XXX not the best way, \copy ?
#cat forksand-ocadev2.sql | psql --host 10.52.1.91 --user ocadev2 ocadev2_forksand
pg_restore --format=custom --no-owner --host=10.52.1.91 --username=ocadev2 --dbname=ocadev2_forksand forksand-ocadev2.sql

# On ocadev2, using SQL dump from above:
#psql --host 10.52.1.91 --user ocadev2 ocadev2_forksand



# To connect to the master node from client node:
psql --host 10.52.1.91 --user postgres

# To connect to ocadev2 forksand database:
psql --host 10.52.1.91 --user ocadev2 ocadev2_forksand


