#!/bin/bash # forksand-bootstrap-the # GPLv3+ # This script does some initial setup and config # Sets up Proxmox. # IPv6 is left enabled. # Firewalling is done through Proxmox. # Edit below to add Proxmox Enterprise Key. XXX broken, use community repo. # XXX set up hostname # XXX set network to auto not hotplug XXX # Log script exec > >(tee /root/bootstrap-the.log) 2>/root/bootstrap-the.err set -x # Set locale echo "en_US.UTF-8 UTF-8" > /etc/locale.gen locale-gen update-locale # XXX Set timezone ln -sf /usr/share/zoneinfo/America/Denver /etc/localtime # Set up git for tracking. XXX Ansible... XXX echo 'Acquire::http::Proxy "http://192.168.110.72:3142";' > /etc/apt/apt.conf apt-get -y install git sudo cd /etc git init chmod og-rwx /etc/.git cat > /etc/.gitignore < /etc/apt/sources.list < /etc/default/cpufrequtils /etc/init.d/cpufrequtils restart cd /etc ; git add . ; git commit -a -m 'Set up cpufrequtils' # Small user tweaks echo :syntax on > ~/.vimrc echo :syntax on > /home/jebba/.vimrc chown jebba:jebba /home/jebba/.vimrc echo export EDITOR=vi >> /root/.bashrc # XXX Passwordless sudo XXX Ya, probably remove sed -i -e 's/%sudo\tALL=(ALL:ALL) ALL/%sudo ALL=(ALL) NOPASSWD: ALL/g' /etc/sudoers adduser jebba sudo cd /etc ; git add . ; git commit -a -m 'Set up passwordless sudo' # SSH config XXX sed cruft sed -i \ -e 's/PermitRootLogin yes/PermitRootLogin prohibit-password/g' \ -e 's/\#PermitRootLogin prohibit-password/PermitRootLogin prohibit-password/g' \ -e 's/\#PasswordAuthentication yes/PasswordAuthentication no/g' \ -e 's/\#X11Forwarding yes/X11Forwarding no/g' \ /etc/ssh/sshd_config echo 'KexAlgorithms curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256' >> /etc/ssh/sshd_config echo 'Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr' >> /etc/ssh/sshd_config # Need to update/fix for Debian Buster (testing/10). This line breaks Buster: #echo 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-ripemd160-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,hmac-ripemd160,umac-128@openssh.com' >> /etc/ssh/sshd_config # XXX Add admins as only allowed ssh users # XXX add user for ansbile echo "AllowUsers jebba root" >> /etc/ssh/sshd_config cd /etc ; git add . ; git commit -a -m 'Set up sshd' systemctl restart sshd # Startup XXX disable unneeded. for i in rsync exim4 saned do echo $i /usr/sbin/update-rc.d $i disable done # XXX KILL THIS, listening on public port (firewalled, but still): # tcp 0 0 0.0.0.0:5355 0.0.0.0:* LISTEN 296/systemd-resolve cd /etc ; git add . ; git commit -a -m 'Turn off junk on boot' # GRUB sed -i -e 's/^GRUB_TIMEOUT=5/GRUB_TIMEOUT=4/g' /etc/default/grub sed -i -e 's/^#GRUB_TERMINAL=console/GRUB_TERMINAL=console/g' /etc/default/grub echo 'GRUB_DISABLE_OS_PROBER=true' >> /etc/default/grub update-grub cd /etc ; git add . ; git commit -a -m 'GRUB tweaks' # Fix network to come up on boot sed -i -e 's/allow-hotplug/auto/g' /etc/network/interfaces cd /etc ; git add . ; git commit -a -m 'Auto start network' # XXX not sure why this is getting installed: apt-get -y autoremove # Proxmox #cat > /etc/apt/sources.list.d/pve-enterprise.list< /etc/apt/sources.list.d/pve-no-subscription.list< /etc/apt/auth.conf< Linux Bridge: # vmbr0 # rebootz # # Set up templates # Cluster Corosync exit 0 echo "10.8.8.88 truck-coro" >> /etc/hosts echo "10.8.8.90 swutch-coro" >> /etc/hosts echo "10.8.8.87 wall-coro" >> /etc/hosts echo "10.8.8.66 the-coro" >> /etc/hosts echo "10.99.99.88 truck-fs" >> /etc/hosts echo "10.99.99.90 swutch-fs" >> /etc/hosts echo "10.99.99.87 wall-fs" >> /etc/hosts echo "10.99.99.66 the-fs" >> /etc/hosts # Test cluster ping for i in truck-coro swutch-coro wall-coro the-coro do ping -q -c1 $i done # more stuff apt remove os-prober # Disable enp3s0 (Autostart no) # # set up vmbr0 to the main IP, gateway, etc. # Create Linux Bridge in web interface # vmbr0 # 192.168.110.66 # 255.255.255.0 # Gateway 192.168.110.252 # Autostart # VLAN Aware # Bridge: enp3s0f1 # Comment Main bridge # Set up corosync ethernet interfaces # 10.8.8.66 # 255.255.255.0 # Autostart # VLAN Aware # Bridge enx000acd31ac3d # Comment the-coro # Set up ceph ethernet interfaces # 10.99.99.66 # 255.255.255.0 # Autostart # VLAN Aware # Bridge enx000acd31ac3e # Comment fs-coro # rebooootz # Add the to /etc/hosts on other servers: 10.8.8.66 the-coro 10.99.99.66 the-fs # Add the the ssh key to ONE node # Add truck, wall, swutch ssh keys to the # Test flood multicast on private interface omping -c 10000 -i 0.001 -F -q swutch-coro truck-coro the-coro wall-coro # Ten minute test: omping -c 600 -i 1 -q swutch-coro truck-coro wall-coro the-coro # Set up ssh as root to/from all nodes # Best way to do this ... XXX echo "fookey" >> /root/.ssh/authorized_keys # test SSH /etc/init.d/ssh restart for i in the wall truck swutch ;do ssh $i hostname ;done for i in the-coro wall-coro truck-coro swutch-coro ;do ssh $i hostname ;done for i in the-fs wall-fs truck-fs swutch-fs ;do ssh $i hostname ;done # Run on the: pvecm add 10.8.8.88 --ring0_addr the-coro # If `tcpdump -vvv -i enp10s0` show bad udp checksums, run this: # XXX ok on the, wall, swutch, truck ethtool -K enp10s0 gso off ethtool --offload enp10s0 rx off tx off # Run on all nodes: pveceph install --version luminous # Then run on remaining nodes, the: pveceph createmon # On all nodes: pveceph createmgr # internal drives # Create a GPT disklabel with fdisk fdisk /dev/nvme0n1 # g # w pveceph createosd /dev/nvme0n1 # Create a GPT disklabel with fdisk fdisk /dev/sda # g # w pveceph createosd /dev/sda #===================== XXX best way? XXX ==================== # XXX maybe not needed ? # XXX actually, remove this and do no auth since it is private network. mkdir /etc/pve/priv/ceph cp -p /etc/pve/priv/ceph.client.admin.keyring /etc/pve/priv/ceph/my-ceph-storage.keyring # Edit on just one node (shared on all) vim /etc/pve/storage.cfg # Do this instead of my-ceph-storage.keyring # Edit on one node: vim /etc/pve/ceph.conf auth cluster required = none auth service required = none auth client required = none # restart stuff systemctl stop ceph\*.service ceph\*.target mkdir /etc/pve/priv/ceph/old mv /etc/pve/priv/ceph/*keyring /etc/pve/priv/ceph/old/ #===================== XXX best way? XXX ====================