#!/bin/bash # forksand-bootstrap-truck # GPLv3+ # This script does some initial setup and config # Sets up Proxmox. # IPv6 is left enabled. # Firewalling is done through Proxmox. # Edit below to add Proxmox Enterprise Key. XXX broken, use community repo. # XXX set up hostname # XXX set network to auto not hotplug XXX # Log script exec > >(tee /root/bootstrap-truck.log) 2>/root/bootstrap-truck.err set -x # Set locale echo "en_US.UTF-8 UTF-8" > /etc/locale.gen locale-gen update-locale # XXX Set timezone ln -sf /usr/share/zoneinfo/America/Denver /etc/localtime # Set up git for tracking. XXX Ansible... XXX echo 'Acquire::http::Proxy "http://192.168.110.72:3142";' > /etc/apt/apt.conf apt-get -y install git sudo cd /etc git init chmod og-rwx /etc/.git cat > /etc/.gitignore < /etc/apt/sources.list < /etc/default/cpufrequtils /etc/init.d/cpufrequtils restart cd /etc ; git add . ; git commit -a -m 'Set up cpufrequtils' # Small user tweaks echo :syntax on > ~/.vimrc echo :syntax on > /home/jebba/.vimrc chown jebba:jebba /home/jebba/.vimrc echo export EDITOR=vi >> /root/.bashrc # XXX Passwordless sudo XXX Ya, probably remove sed -i -e 's/%sudo\tALL=(ALL:ALL) ALL/%sudo ALL=(ALL) NOPASSWD: ALL/g' /etc/sudoers adduser jebba sudo cd /etc ; git add . ; git commit -a -m 'Set up passwordless sudo' # SSH config XXX sed cruft sed -i \ -e 's/PermitRootLogin yes/PermitRootLogin prohibit-password/g' \ -e 's/\#PermitRootLogin prohibit-password/PermitRootLogin prohibit-password/g' \ -e 's/\#PasswordAuthentication yes/PasswordAuthentication no/g' \ -e 's/\#X11Forwarding yes/X11Forwarding no/g' \ /etc/ssh/sshd_config echo 'KexAlgorithms curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256' >> /etc/ssh/sshd_config echo 'Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr' >> /etc/ssh/sshd_config # Need to update/fix for Debian Buster (testing/10). This line breaks Buster: #echo 'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-ripemd160-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,hmac-ripemd160,umac-128@openssh.com' >> /etc/ssh/sshd_config # XXX Add admins as only allowed ssh users # XXX add user for ansbile echo "AllowUsers jebba root" >> /etc/ssh/sshd_config cd /etc ; git add . ; git commit -a -m 'Set up sshd' systemctl restart sshd # Startup XXX disable unneeded. for i in rsync exim4 saned do echo $i /usr/sbin/update-rc.d $i disable done # XXX KILL THIS, listening on public port (firewalled, but still): # tcp 0 0 0.0.0.0:5355 0.0.0.0:* LISTEN 296/systemd-resolve cd /etc ; git add . ; git commit -a -m 'Turn off junk on boot' # GRUB sed -i -e 's/^GRUB_TIMEOUT=5/GRUB_TIMEOUT=4/g' /etc/default/grub sed -i -e 's/^#GRUB_TERMINAL=console/GRUB_TERMINAL=console/g' /etc/default/grub echo 'GRUB_DISABLE_OS_PROBER=true' >> /etc/default/grub update-grub cd /etc ; git add . ; git commit -a -m 'GRUB tweaks' # Fix network to come up on boot sed -i -e 's/allow-hotplug/auto/g' /etc/network/interfaces cd /etc ; git add . ; git commit -a -m 'Auto start network' # XXX not sure why this is getting installed: apt-get -y autoremove # Proxmox #cat > /etc/apt/sources.list.d/pve-enterprise.list< /etc/apt/sources.list.d/pve-no-subscription.list< /etc/apt/auth.conf< Permissions --> Users # Add user with Realm Proxmox VE authentication server. # Give user root permissions: Datacenter --> Permissions --> Add --> User permission. # Path: / User: j Role: Administrator # XXX Or create admin group, add perms to that... # Permissions --> Authentication. Set Proxmox VE authentication server to default. # # Enable firewall. # Datacenter --> truck (host) --> Firewall --> Add. # Open up for SSH and SSH alt port. # Enable firewall for datacenter: # Datacenter --> Firewall --> Options --> Firewall --> Yes # Enable firewall for truck: # Open up for SSH and SSH alt port. # REJECT everything coming in. (then DROP) # Reorder to ACCEPT SSH at top # cd /etc ; git add . ; git commit -a -m 'Initial Proxmox configuration' # # Reboot! truck (host) --> Restart # # XXX # Datacenter --> Firewall --> Add. # REJECT any in # # Storage # Datacenter --> Storage --> Edit local. Enable all content (add VZDump) # # XXX postfix # # DNS # truck (host) --> System --> DNS # Add servers: # 208.67.222.222 208.67.220.220 37.235.1.174 # # Netwok # truck (host) --> System --> Network # Fix subnet mask, IP in web gui. # Create --> Linux Bridge: # vmbr0 # XXX best way for this server? No subnet. # # rebootz # # Set up templates # XXX TOTAL MEH XXX # add this to the workstation: # 127.0.0.1 localhost truck-tun # Run: # ssh -N -C -L 8020:localhost:8006 truck # Then use URLs # https://truck-tun:8020 # Or you can only be logged into one at a time. # XXX find better workaround # Cluster Corosync exit 0 echo "10.8.8.88 truck-coro" >> /etc/hosts echo "10.8.8.90 swutch-coro" >> /etc/hosts echo "10.8.8.87 wall-coro" >> /etc/hosts echo "10.111.111.88 truck-fs" >> /etc/hosts echo "10.111.111.90 swutch-fs" >> /etc/hosts echo "10.111.111.87 wall-fs" >> /etc/hosts # Test cluster ping for i in truck-coro swutch-coro wall-coro do ping -q -c1 $i done # more stuff apt install postfix apt remove os-prober # Disable enp3s0 (Autostart no) # set up vmbr0 to the main IP, gateway, etc. # Create Linux Bridge in web interface # vmbr0 # 192.168.55.88 # 255.255.255.0 # Autostart # VLAN Aware # Bridge: enp3s0 # Comment Main bridge # Test flood multicast on private interface omping -c 10000 -i 0.001 -F -q swutch-coro truck-coro wall-coro # Ten minute test: omping -c 600 -i 1 -q swutch-coro truck-coro wall-coro # Set up ssh as root to/from all nodes # Best way to do this ... XXX echo "fookey" >> /root/.ssh/authorized_keys for i in swutch-coro truck-coro wall-coro do ssh $i hostname done # Run just on truck: pvecm create red --bindnet0_addr 10.8.8.88 --ring0_addr truck-coro # Run on wall: pvecm add 10.8.8.88 --ring0_addr wall-coro # Run on swutch: pvecm add 10.8.8.88 --ring0_addr swutch-coro # If `tcpdump -vvv -i enp10s0` show bad udp checksums, run this: # XXX ok on truck, wall, swutch ethtool -K enp10s0 gso off ethtool --offload enp10s0 rx off tx off # Setup 10.99.99.0/24 addresses for Ceph on enp16s0 # Run on all nodes: pveceph install --version luminous # Run just on one node (truck): pveceph init --network 10.99.99.0/24 pveceph createmon # Then run on remaining nodes (or via GUI) pveceph createmon # On all nodes: pveceph createmr # XXX missing ZFS tools apt install zfsutils-linux modprobe zfs # Add USB drive to swutch and run on it: # Create a GPT disklabel with fdisk fdisk /dev/sdb # g # w pveceph createosd /dev/sdb # XXX actually, remove this and do no auth since it is private network. mkdir /etc/pve/priv/ceph cp -p /etc/pve/priv/ceph.client.admin.keyring /etc/pve/priv/ceph/my-ceph-storage.keyring # Edit on just one node (shared on all) vim /etc/pve/storage.cfg # Do this instead of my-ceph-storage.keyring # Edit on one node: vim /etc/pve/ceph.conf auth cluster required = none auth service required = none auth client required = none # restart stuff systemctl stop ceph\*.service ceph\*.target mkdir /etc/pve/priv/ceph/old mv /etc/pve/priv/ceph/*keyring /etc/pve/priv/ceph/old/