On sf-001 and sf-002 with disk platters:

# Does not survive reboot XXX
# sf-001 sf-002
echo 65536 > /sys/block/sdb/queue/nr_requests
echo 65536 > /sys/block/sdc/queue/nr_requests
echo 65536 > /sys/block/sdd/queue/nr_requests

# sf-001 sf-002
echo 8192 > /sys/block/sdb/queue/read_ahead_kb
echo 8192 > /sys/block/sdc/queue/read_ahead_kb
echo 8192 > /sys/block/sdd/queue/read_ahead_kb

# MTU 9000 for Ceph
# sf-001 sf-002
ip link set dev eno2 mtu 9000
ip link set dev enp134s0f1 mtu 9000
# sf-003
ip link set dev eno2 mtu 9000
ip link set dev enp216s0f0 mtu 9000
# sf-004
ip link set dev eno2 mtu 9000
ip link set dev enp216s0f1 mtu 9000
# On all sf-001 sf-002 sf-003 sf-004
# XXX fail
#ip link set dev bond0 mtu 9000
ifconfig bond0 mtu 9000


# Only use 3 mon for Ceph. So remove it from sf-001

# Create 2 OSD per NVMe ?

##############################################################################
# sysctl.conf possibilities from:
# http://tracker.ceph.com/projects/ceph/wiki/Tuning_for_All_Flash_Deployments

# disable TIME_WAIT.. wait ..
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1

# Controls the use of TCP syncookies
net.ipv4.tcp_syncookies = 0

# double amount of allowed conntrack
net.netfilter.nf_conntrack_max = 2621440
net.netfilter.nf_conntrack_tcp_timeout_established = 1800

# Disable netfilter on bridges.
net.bridge.bridge-nf-call-ip6tables = 0
net.bridge.bridge-nf-call-iptables = 0
net.bridge.bridge-nf-call-arptables = 0

# Controls the maximum size of a message, in bytes
kernel.msgmnb = 65536

# Controls the default maxmimum size of a mesage queue
kernel.msgmax = 65536

# Controls the maximum shared segment size, in bytes
kernel.shmmax = 68719476736

# Controls the maximum number of shared memory segments, in pages
kernel.shmall = 4294967296

##############################################################################

# Increase PG ? Currently at 256.

# Is scrubbing enabled?

##########################
# sample:
ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok config show | grep thread

ceph tell osd.* injectargs '--osd_disk_threads 4'
# Resulted in this line for each OSD (16 total):
osd.0: osd_disk_threads = '4' rocksdb_separate_wal_dir = 'false' (not observed, change may require restart) 

# http://xiaoquqi.github.io/blog/2015/06/28/ceph-performance-optimization-summary/

# Turn off NUMA. Add to grub kernel boot line:
numa=off
# Run
update-grub
# With NUMA enabled (from dmesg):
[Wed Jul 11 15:52:46 2018] NUMA: Initialized distance table, cnt=2
[Wed Jul 11 15:52:46 2018] NUMA: Node 0 [mem 0x00000000-0x7fffffff] + [mem 0x100000000-0x87fffffff] -> [mem 0x00000000-0x87fffffff]
[Wed Jul 11 15:52:46 2018] mempolicy: Enabling automatic NUMA balancing. Configure with numa_balancing= or the kernel.numa_balancing sysctl
[Wed Jul 11 15:52:46 2018] pci_bus 0000:00: on NUMA node 0
etc.

# CGroup
# Pin each OSD to a CPU, somehow


# Increase default values in Ceph config


# Create fast pool from command line:
ceph osd crush rule create-replicated <rule-name> <root> <failure-domain> <class>
ceph osd crush rule create-replicated fast default host nvme
# and? Didnt appear in proxmox interface
# Then create a pool in the web gui.


# Then do this to have pool use new rule:
#ceph osd pool set nvmepool crush_rule fast
# nope


# Create disk platter crush rule for slow pool from command line:
ceph osd crush rule create-replicated slow default host hdd
# Then add pool with "slow" crush rule in web GUI

# Then add under Data Center --> Storage, the new pool. Use rbd, and krbd


##############################
# Change disk encryption password, check disk with crypto, then:
#cryptsetup -y luksAddKey /dev/sdb1
#cryptsetup luksRemoveKey /dev/sdb1

#cryptsetup -y luksAddKey /dev/sda2
#cryptsetup luksRemoveKey /dev/sda2