#!/bin/bash # forksand-sf-proxmox # GPLv3+ # This script does some initial setup and config # Sets up Proxmox. # XXX DRAFT XXX exit 0 # Run this on workstation: # ssh -N -C -L 18021:localhost:8006 sf-001 # ssh -N -C -L 18022:localhost:8006 sf-002 # ssh -N -C -L 18023:localhost:8006 sf-003 # ssh -N -C -L 18024:localhost:8006 sf-004 # ssh -N -C -L 18025:localhost:8006 sf-005 # Only logging into one at a time is possible. Session management :| XXX # firefox https://localhost:18021 # firefox https://localhost:18022 # firefox https://localhost:18023 # firefox https://localhost:18024 # firefox https://localhost:18025 # # Login as root user via PAM # Set up Enterprise Key, if used # # cd /etc ; git add . ; git commit -a -m 'Initial Proxmox configuration' # # # Network # Set up static IPs on each interface in Proxmox web gui # See this screenshot for an example from sf-004: # forksand-it-manual/source/resources/images/proxmox-network.png # cd /etc ; git add . ; git commit -a -m 'Configure ethernet addresses' # Reboot! # Configure Corosync # Set up hosts # XXX MAKE SURE NEW NODES GET ADDED TO EXISTING SERVER /etc/hosts # XXX Update existing hosts with new nodes! cat > /etc/hosts <> /etc/modules modprobe rbd cd /etc ; git commit -m 'Use rbd kernel module' /etc/modules # Run this on just sf-001 # Doesn't need to be run again: # pveceph init --network 10.99.99.0/24 # cd /etc ; git add . ; git commit -m 'Ceph init' ceph/ pve/ceph.conf # With new nodes, I think rest can be done via web gui XXX # XXX It doesn't appear to set up Ceph symbolic links when adding # nodes added after initial setup (e.g. sf-005). # So run this, on nodes sf-005 and subsequent nodes: cd /etc/ceph/ ln -s /etc/pve/ceph.conf ceph.conf # It also needs a key file, copy onto new nodes, sf-005 and newer: scp -p sf-001:/etc/ceph/ceph.client.admin.keyring . cd /etc ; git commit -a -m 'Add ceph config' # Reboot... # on sf-002 sf-003 sf-004 sf-005 sf-006 sf-007 cd /etc ; git add . ; git commit -a -m 'Ceph setup' # DONT RUN SIMULTANEOUSLY XXX # Run on sf-001 # Does not need to be run again: # pveceph createmon cd /etc ; git add . ; git commit -m 'Ceph createmon' # Do the other ones via web gui # cd /etc ; git add . ; git commit -m 'Add Ceph mons via web gui' # Looks like this auto creates. Not needed #pveceph createmgr #cd /etc ; git add . ; git commit -m 'Ceph createmgr' # Create GPT partition table on each Ceph drive, with correct device name: # gdisk /dev/sd[X] # This can be done via web interface now too. :) # sf-001 # Then 'w' to write new GPT table gdisk /dev/sdb gdisk /dev/sdc gdisk /dev/sdd # sf-002 gdisk /dev/sdb gdisk /dev/sdc gdisk /dev/sdd # sf-003 # Create blank GPT ? gdisk /dev/nvme0n1 gdisk /dev/nvme1n1 gdisk /dev/nvme2n1 gdisk /dev/nvme3n1 # sf-004 gdisk /dev/nvme0n1 gdisk /dev/nvme1n1 gdisk /dev/nvme2n1 gdisk /dev/nvme3n1 gdisk /dev/nvme4n1 gdisk /dev/nvme5n1 # sf-005 # Note /dev/sda is /boot and encrypted / gdisk /dev/sdb gdisk /dev/sdc gdisk /dev/sdd gdisk /dev/sde gdisk /dev/sdf # sf-006, sf-007 # Added via web interface # Add OSDs # Do this via web, not command line. cd /etc ; git add . ; git commit -m 'Add Ceph OSDs via web gui' # Via Web GUI, add pools: # XXX Pools have changed since this doc. # Name: sharkpool # Size: 3 (default) # Min Size: 2 (default) # Crush Rule: replicated rule (default) # pg_num: 64 (default) XXX try 256 # Add storages: unchecked (default) cd /etc ; git add . ; git commit -m 'Add Ceph pools via web gui' ## Add keyring so it can be PVE ceph client # Not needed ??? ## Just on sf-001 #mkdir /etc/pve/priv/ceph #cp /etc/pve/priv/ceph.client.admin.keyring /etc/pve/priv/ceph/my-ceph-storage.keyring # In web GUI, go to: # Datacenter -> storage # and add and RDB (PVE) container/images # # XXX # Reinstall if things go bad: # /etc/init.d/ceph stop ; apt -y purge ceph ceph-base ceph-mgr ceph-mon ceph-osd ; apt -y autoremove --purge ; mv /var/lib/ceph/ /var/lib/foobar-ceph ; rm -rf /etc/pve/ceph* /etc/ceph/ceph.conf /etc/pve/priv/ceph* /etc/systemd/system/ceph-mon.target.wants /etc/systemd/system/ceph-mon.target.wants/ceph-mon@sf-001.service /etc/ceph/ceph.client.admin.keyring apt -y --reinstall install ceph-common # rm stuff that isn't in ceph-common find /etc -name "*ceph*" pveceph install # XXX # MISC # Data Center --> Permissions --> Users # Add user with Realm Proxmox VE authentication server. # Give user root permissions: Datacenter --> Permissions --> Add --> User permission. # Path: / User: j Role: Administrator # XXX Or create admin group, add perms to that... # Permissions --> Authentication. Set Proxmox VE authentication server to default. # Storage # Datacenter --> Storage --> Edit local. Enable all content (add VZDump) # # DNS # sf-003 (host) --> System --> DNS # Add servers: # 208.67.222.222 208.67.220.220 37.235.1.174 # # XXX Add remote syslogging, monitoring, backups, etc. #