forksand-it-manual/source/resources/apps/forksand-nodes-bootstrap/forksand-the-bootstrap

#!/bin/bash
# forksand-bootstrap-the
# GPLv3+
# This script does some initial setup and config
# Sets up Proxmox.
# IPv6 is left enabled.
# Firewalling is done through Proxmox.
# Edit below to add Proxmox Enterprise Key. XXX broken, use community repo.

# XXX set up hostname

# XXX set network to auto not hotplug XXX

# Log script
exec > >(tee /root/bootstrap-the.log) 2>/root/bootstrap-the.err

set -x

# Set locale
echo "en_US.UTF-8 UTF-8" > /etc/locale.gen
locale-gen
update-locale

# XXX Set timezone
ln -sf /usr/share/zoneinfo/America/Denver /etc/localtime

# Set up git for tracking. XXX Ansible... XXX
echo 'Acquire::http::Proxy "http://192.168.110.72:3142";' > /etc/apt/apt.conf
apt-get -y install git sudo
cd /etc
git init
chmod og-rwx /etc/.git

cat > /etc/.gitignore <<EOF
prelink.cache
*.swp
ld.so.cache
adjtime
blkid.tab
blkid.tab.old
mtab
resolv.conf
asound.state
mtab.fuselock
aliases.db
EOF

git config --global user.name "debian"
git config --global user.email git@localhost
cd /etc ; git add . ; git commit -a -m 'Set up new Debian Stretch the server.'

# SET UP APT
#
cat > /etc/apt/sources.list <<EOF
deb http://mirrors.kernel.org/debian/ stretch-backports main
deb http://mirrors.kernel.org/debian/ stretch main
deb http://mirrors.kernel.org/debian/ stretch-updates main
deb http://security.debian.org/ stretch/updates main
EOF

# Make apt use IPv4:
echo 'Acquire::ForceIPv4 "true";' | tee /etc/apt/apt.conf.d/99force-ipv4

git add /etc/apt/apt.conf.d/99force-ipv4
git commit -m "Force APT to use IPv4, not IPv6." /etc/apt/apt.conf.d/99force-ipv4

cd /etc ; git add . ; git commit -a -m 'Set up apt.'

# UPGRADE SERVER
apt-get update
apt-get -y dist-upgrade --download-only
DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confnew" dist-upgrade

cd /etc ; git add . ; git commit -a -m 'Update base install'

# ZFS tools
modprobe zfs

apt-get -y --download-only install					\
	--no-install-recommends						\
	apt-transport-https						\
	bzip2								\
	ca-certificates							\
	colordiff							\
	cpufrequtils							\
	curl								\
	debian-archive-keyring						\
	exuberant-ctags							\
	git								\
	host								\
	less								\
	locales								\
	lsb-release							\
	man-db								\
	manpages							\
	molly-guard							\
	net-tools							\
	ntp								\
	openssh-server							\
	python3								\
	rsync								\
	tcpdump								\
	telnet								\
	traceroute							\
	vim								\
	vim-scripts							\
	zfsutils-linux

DEBIAN_FRONTEND=noninteractive apt-get -y 				\
	-o Dpkg::Options::="--force-confdef"				\
	-o Dpkg::Options::="--force-confnew"				\
	install								\
	--no-install-recommends						\
	apt-transport-https						\
	bzip2								\
	ca-certificates							\
	colordiff							\
	cpufrequtils							\
	curl								\
	debian-archive-keyring						\
	exuberant-ctags							\
	git								\
	host								\
	less								\
	locales								\
	lsb-release							\
	man-db								\
	manpages							\
	molly-guard							\
	net-tools							\
	ntp								\
	openssh-server							\
	python3								\
	rsync								\
	tcpdump								\
	telnet								\
	traceroute							\
	vim								\
	vim-scripts							\
	zfsutils-linux

cd /etc ; git add . ; git commit -a -m 'Install base packages'

# Speed up
echo 'GOVERNOR="performance"' > /etc/default/cpufrequtils
/etc/init.d/cpufrequtils restart
cd /etc ; git add . ; git commit -a -m 'Set up cpufrequtils'

# Small user tweaks
echo :syntax on > ~/.vimrc
echo :syntax on > /home/jebba/.vimrc
chown jebba:jebba /home/jebba/.vimrc
echo export EDITOR=vi >> /root/.bashrc

# XXX Passwordless sudo XXX Ya, probably remove
sed -i -e 's/%sudo\tALL=(ALL:ALL) ALL/%sudo ALL=(ALL) NOPASSWD: ALL/g' /etc/sudoers

adduser jebba sudo
cd /etc ; git add . ; git commit -a -m 'Set up passwordless sudo'

# SSH config XXX sed cruft
sed -i  \
 -e 's/PermitRootLogin yes/PermitRootLogin prohibit-password/g' \
 -e 's/\#PermitRootLogin prohibit-password/PermitRootLogin prohibit-password/g' \
 -e 's/\#PasswordAuthentication yes/PasswordAuthentication no/g' \
 -e 's/\#X11Forwarding yes/X11Forwarding no/g' \
 /etc/ssh/sshd_config

echo 'KexAlgorithms curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256' >> /etc/ssh/sshd_config

echo 'Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr' >> /etc/ssh/sshd_config

# Need to update/fix for Debian Buster (testing/10). This line breaks Buster:
#echo 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-ripemd160-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,hmac-ripemd160,umac-128@openssh.com' >> /etc/ssh/sshd_config

# XXX Add admins as only allowed ssh users
# XXX add user for ansbile
echo "AllowUsers jebba root" >> /etc/ssh/sshd_config

cd /etc ; git add . ; git commit -a -m 'Set up sshd'
systemctl restart sshd

# Startup XXX disable unneeded.
for i in rsync exim4 saned
do echo $i 
  /usr/sbin/update-rc.d $i disable
done
# XXX KILL THIS, listening on public port (firewalled, but still):
# tcp        0      0 0.0.0.0:5355            0.0.0.0:*               LISTEN      296/systemd-resolve
cd /etc ; git add . ; git commit -a -m 'Turn off junk on boot'

# GRUB
sed -i -e 's/^GRUB_TIMEOUT=5/GRUB_TIMEOUT=4/g' /etc/default/grub
sed -i -e 's/^#GRUB_TERMINAL=console/GRUB_TERMINAL=console/g' /etc/default/grub
echo 'GRUB_DISABLE_OS_PROBER=true' >> /etc/default/grub

update-grub

cd /etc ; git add . ; git commit -a -m 'GRUB tweaks'

# Fix network to come up on boot
sed -i -e 's/allow-hotplug/auto/g' /etc/network/interfaces
cd /etc ; git add . ; git commit -a -m 'Auto start network'

# XXX not sure why this is getting installed:
apt-get -y autoremove

# Proxmox
#cat > /etc/apt/sources.list.d/pve-enterprise.list<<EOF
##deb https://enterprise.proxmox.com/debian/pve stretch pve-enterprise
#EOF
cat > /etc/apt/sources.list.d/pve-no-subscription.list<<EOF
deb http://download.proxmox.com/debian/pve stretch pve-no-subscription
EOF

# Add Proxmox enterprise key XXX Add key 
#cat > /etc/apt/auth.conf<<EOF
#machine enterprise.proxmox.com
# login pve2s-0000000000
# password 00000000000000000000000000000000
#EOF

# XXX crufty add proxmox apt key
wget http://download.proxmox.com/debian/proxmox-ve-release-5.x.gpg -O /etc/apt/trusted.gpg.d/proxmox-ve-release-5.x.gpg

apt-get update
apt-get -y dist-upgrade --download-only
DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confnew" dist-upgrade

apt-get -y				 				\
	install								\
	ksm-control-daemon						\
	omping								\
	proxmox-ve

cd /etc ; git add . ; git commit -a -m 'Install Proxmox'
apt clean

exit 0

cd /etc ; git add . ; git commit -a -m 'Initial Proxmox configuration'
#
# XXX use postfix not exim4
#
# Create --> Linux Bridge:
#   vmbr0

# rebootz
#
# Set up templates

# Cluster Corosync
exit 0
echo "10.8.8.88 truck-coro" >> /etc/hosts
echo "10.8.8.90 swutch-coro" >> /etc/hosts
echo "10.8.8.87 wall-coro" >> /etc/hosts
echo "10.8.8.66 the-coro" >> /etc/hosts
echo "10.99.99.88 truck-fs" >> /etc/hosts
echo "10.99.99.90 swutch-fs" >> /etc/hosts
echo "10.99.99.87 wall-fs" >> /etc/hosts
echo "10.99.99.66 the-fs" >> /etc/hosts

# Test cluster ping
for i in truck-coro swutch-coro wall-coro the-coro 
do ping -q -c1 $i
done

# more stuff
apt remove os-prober

# Disable enp3s0 (Autostart no)
#
# set up vmbr0 to the main IP, gateway, etc.
# Create Linux Bridge in web interface
# vmbr0
# 192.168.110.66
# 255.255.255.0
# Gateway 192.168.110.252
# Autostart
# VLAN Aware
# Bridge:  enp3s0f1
# Comment Main bridge

# Set up corosync ethernet interfaces
# 10.8.8.66
# 255.255.255.0
# Autostart
# VLAN Aware
# Bridge enx000acd31ac3d
# Comment the-coro

# Set up ceph ethernet interfaces
# 10.99.99.66
# 255.255.255.0
# Autostart
# VLAN Aware
# Bridge enx000acd31ac3e
# Comment fs-coro

# rebooootz

# Add the to /etc/hosts on other servers:
10.8.8.66 the-coro
10.99.99.66 the-fs

# Add the the ssh key to ONE node

# Add truck, wall, swutch ssh keys to the


# Test flood multicast on private interface
omping -c 10000 -i 0.001 -F -q swutch-coro truck-coro the-coro wall-coro
# Ten minute test:
omping -c 600 -i 1 -q swutch-coro truck-coro wall-coro the-coro

# Set up ssh as root to/from all nodes
# Best way to do this ... XXX
echo "fookey" >> /root/.ssh/authorized_keys
# test SSH
/etc/init.d/ssh restart

for i in the wall truck swutch ;do ssh $i hostname ;done
for i in the-coro wall-coro truck-coro swutch-coro ;do ssh $i hostname ;done
for i in the-fs wall-fs truck-fs swutch-fs ;do ssh $i hostname ;done


# Run on the:
pvecm add 10.8.8.88 --ring0_addr the-coro

# If `tcpdump -vvv -i enp10s0` show bad udp checksums, run this:
# XXX ok on the, wall, swutch, truck
ethtool -K enp10s0 gso off
ethtool --offload enp10s0 rx off tx off

# Run on all nodes:
pveceph install --version luminous

# Then run on remaining nodes, the:
pveceph createmon

# On all nodes:
pveceph createmgr

# internal drives
# Create a GPT disklabel with fdisk
fdisk /dev/nvme0n1
# g
# w
pveceph createosd /dev/nvme0n1
# Create a GPT disklabel with fdisk
fdisk /dev/sda
# g
# w
pveceph createosd /dev/sda


#===================== XXX best way?  XXX ====================
# XXX maybe not needed ?
# XXX actually, remove this and do no auth since it is private network.
mkdir /etc/pve/priv/ceph
cp -p /etc/pve/priv/ceph.client.admin.keyring /etc/pve/priv/ceph/my-ceph-storage.keyring
# Edit on just one node (shared on all)
vim /etc/pve/storage.cfg

# Do this instead of my-ceph-storage.keyring
# Edit on one node:
vim /etc/pve/ceph.conf
auth cluster required = none
auth service required = none
auth client required = none
# restart stuff
systemctl stop ceph\*.service ceph\*.target
mkdir /etc/pve/priv/ceph/old
mv /etc/pve/priv/ceph/*keyring  /etc/pve/priv/ceph/old/
#===================== XXX best way?  XXX ====================