From 296944becb6ad403fbf884fd019fdb2a924e39f9 Mon Sep 17 00:00:00 2001 From: Jeff Moe Date: Sat, 7 Jul 2018 23:20:30 -0600 Subject: [PATCH] Set up Postgres Citus cluster --- .../forksand-postgres-citus-setup | 166 ++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100755 source/resources/apps/postgres-citus/forksand-postgres-citus-setup diff --git a/source/resources/apps/postgres-citus/forksand-postgres-citus-setup b/source/resources/apps/postgres-citus/forksand-postgres-citus-setup new file mode 100755 index 0000000..fa08e83 --- /dev/null +++ b/source/resources/apps/postgres-citus/forksand-postgres-citus-setup @@ -0,0 +1,166 @@ +#!/bin/bash + +exit 0 + +# https://www.citusdata.com/download/ + +# Run on all nodes: +curl https://install.citusdata.com/community/deb.sh | sudo bash +#sudo apt-get -y install postgresql-10-citus-7.4 +sudo apt-get -y install postgresql-10-citus + +# Outputs (dont run): +############################################################################## +Success. You can now start the database server using: + + /usr/lib/postgresql/10/bin/pg_ctl -D /var/lib/postgresql/10/main -l logfile start + +Ver Cluster Port Status Owner Data directory Log file +10 main 5432 down postgres /var/lib/postgresql/10/main /var/log/postgresql/postgresql-10-main.log +############################################################################## + + +# Run on all nodes +sudo pg_conftool 10 main set shared_preload_libraries citus +# set up IPs, for each node +# XXX THIS FAILS because the entry doesn't get quoted, so postgres doesn't start. +# pg1 +sudo pg_conftool 10 main set listen_addresses '10.52.1.91' +# pg2 +sudo pg_conftool 10 main set listen_addresses '10.52.1.92' +# pg3 +sudo pg_conftool 10 main set listen_addresses '10.52.1.93' +# pg4 +sudo pg_conftool 10 main set listen_addresses '10.52.1.94' +# XXX to fix, edit /etc/postgresql/10/main/postgresql.conf and change to be +# like this (note single quotes ') +# pg1 +listen_addresses = '10.52.1.91' # what IP address(es) to listen on; +# pg2 +listen_addresses = '10.52.1.92' # what IP address(es) to listen on; +# pg3 +listen_addresses = '10.52.1.93' # what IP address(es) to listen on; +# pg4 +listen_addresses = '10.52.1.94' # what IP address(es) to listen on; + + +# Set up access permissions file as root: +cat > /etc/postgresql/10/main/pg_hba.conf < /etc/iptables.up.rules + +cd /etc ; git add . ; git commit -a -m 'Setup Postgres Citus Cluster' + +# start the db server +sudo service postgresql restart + +# Run on all nodes: +sudo -i -u postgres psql -c "CREATE EXTENSION citus;" + +sudo -i -u postgres psql -c "ALTER SYSTEM SET citus.enable_statistics_collection = 'off';" + +# Run on just ONE node (pg1): +# Don't add the main host node or not? XXX +#sudo -i -u postgres psql -c "SELECT * from master_add_node('10.52.1.91', 5432);" +sudo -i -u postgres psql -c "SELECT * from master_add_node('10.52.1.92', 5432);" +sudo -i -u postgres psql -c "SELECT * from master_add_node('10.52.1.93', 5432);" +sudo -i -u postgres psql -c "SELECT * from master_add_node('10.52.1.94', 5432);" + +# Verify it is all good +sudo -i -u postgres psql -c "SELECT * FROM master_get_active_worker_nodes();" + +# Use postgres: +sudo -i -u postgres psql + + +# Misc notes: +postgres=# create database foo; +NOTICE: Citus partially supports CREATE DATABASE for distributed databases +DETAIL: Citus does not propagate CREATE DATABASE command to workers +HINT: You can manually create a database and its extensions on workers. +CREATE DATABASE + +####### +# OCA # +####### +# On ocadev2: +sudo -i -u postgres +# Backup database, run on ocadev2: +#pg_dump forksand > forksand-ocadev2-pg_dump.sql +pg_dump --format=custom --clean --create --no-owner forksand > forksand-ocadev2.sql + +# Add "ocadev2" user to database. Run on ocadev2: +createuser --createdb --host=10.52.1.91 --username=postgres ocadev2 +# NOTICE: not propagating CREATE ROLE/USER commands to worker nodes +# HINT: Connect to worker nodes directly to manually create all necessary users and roles. +# XXX +createuser --createdb --host=10.52.1.92 --username=postgres ocadev2 +createuser --createdb --host=10.52.1.93 --username=postgres ocadev2 +createuser --createdb --host=10.52.1.94 --username=postgres ocadev2 + +# Create database: +createdb --owner=ocadev2 --host=10.52.1.91 --username=postgres ocadev2_forksand +#NOTICE: Citus partially supports CREATE DATABASE for distributed databases +#DETAIL: Citus does not propagate CREATE DATABASE command to workers +#HINT: You can manually create a database and its extensions on workers. +# XXX +createdb --owner=ocadev2 --host=10.52.1.92 --username=postgres ocadev2_forksand +createdb --owner=ocadev2 --host=10.52.1.93 --username=postgres ocadev2_forksand +createdb --owner=ocadev2 --host=10.52.1.94 --username=postgres ocadev2_forksand + +# Run this logged in to master pg1 +# (XXX connect to other database ?) +\connect ocadev2_forksand +SET citus.shard_max_size TO '64MB'; +# How many copies to distribute (just run on master node): +SET citus.shard_replication_factor TO '2'; + + +# Upload database dump to Postgres server +# XXX not the best way, \copy ? +#cat forksand-ocadev2.sql | psql --host 10.52.1.91 --user ocadev2 ocadev2_forksand +pg_restore --format=custom --no-owner --host=10.52.1.91 --username=ocadev2 --dbname=ocadev2_forksand forksand-ocadev2.sql + +# On ocadev2, using SQL dump from above: +#psql --host 10.52.1.91 --user ocadev2 ocadev2_forksand + + + +# To connect to the master node from client node: +psql --host 10.52.1.91 --user postgres + +# To connect to ocadev2 forksand database: +psql --host 10.52.1.91 --user ocadev2 ocadev2_forksand + +