Skip to content

Instantly share code, notes, and snippets.

@napobear

napobear/gist:0fa4d5b0399b2ae30f43b3b9c1a2c726

Forked from mahescho/gist:dd892e07ffdcf50b76aa66b484f4169a
Created April 5, 2025 15:37
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save napobear/0fa4d5b0399b2ae30f43b3b9c1a2c726 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save napobear/0fa4d5b0399b2ae30f43b3b9c1a2c726 to your computer and use it in GitHub Desktop.
Download ZIP
Proxmox Cluster with Tunderbolt Mesh Network and Ceph on Minisforum MS-01
Raw
gistfile1.txt
First raw Draft
echo "deb http://download.proxmox.com/debian/pve bookworm pve-no-subscription" > /etc/apt/sources.list.d/pve-ns.list
echo "deb http://download.proxmox.com/debian/ceph-squid bookworm no-subscription" > /etc/apt/sources.list.d/ceph-ns.list
echo "thunderbolt" >> /etc/modules
echo "thunderbolt-net" >> /etc/modules
apt install vim lldpd thunderbolt-tools frr iperf3
cat << EOF >> /etc/network/interfaces
auto lo:0
iface lo:0 inet static
address 10.0.0.22X/32
auto en05
iface en05 inet manual
mtu 65520
#do not edit it GUI
auto en06
iface en06 inet manual
mtu 65520
#do not edit in GUI
EOF
N=$(echo -n $(hostname) | tail -c 1)
sed -i "s/X/$N/g" /etc/network/interfaces
cat << EOF > /etc/systemd/network/00-thunderbolt0.link
[Match]
Path=pci-0000:00:0d.2
Driver=thunderbolt-net
[Link]
MACAddressPolicy=none
Name=en05
EOF
cat << EOF > /etc/systemd/network/00-thunderbolt1.link
[Match]
Path=pci-0000:00:0d.3
Driver=thunderbolt-net
[Link]
MACAddressPolicy=none
Name=en06
EOF
cat << EOF > /etc/udev/rules.d/10-tb-en.rules
ACTION=="move", SUBSYSTEM=="net", KERNEL=="en05", RUN+="/usr/local/bin/pve-en05.sh"
ACTION=="move", SUBSYSTEM=="net", KERNEL=="en06", RUN+="/usr/local/bin/pve-en06.sh"
EOF
cat << EOF > /usr/local/bin/pve-en05.sh
#!/bin/bash
# this brings the renamed interface up and reprocesses any settings in /etc/network/interfaces for the renamed interface
/usr/sbin/ifup en05
EOF
cat << EOF > /usr/local/bin/pve-en06.sh
#!/bin/bash
# this brings the renamed interface up and reprocesses any settings in /etc/network/interfaces for the renamed interface
/usr/sbin/ifup en06
EOF
chmod +x /usr/local/bin/*.sh
update-initramfs -u -k all
sed -i 's/fabricd=no/fabricd=yes/' /etc/frr/daemons
cat << EOF >> /etc/frr/frr.conf
hostname pve0X
ip forwarding
no ipv6 forwarding
!
interface en05
ip router openfabric 1
ipv6 router openfabric 1
exit
!
interface en06
ip router openfabric 1
ipv6 router openfabric 1
exit
!
interface lo
ip router openfabric 1
ipv6 router openfabric 1
openfabric passive
exit
!
router openfabric 1
net 49.0000.0000.000X.00
fabric-tier 0
exit
!
EOF
N=$(echo -n $(hostname) | tail -c 1)
sed -i "s/X/$N/g" /etc/frr/frr.conf
cat << EOF > /usr/local/bin/restart-frr.sh
#!/bin/sh
# Delayed start script to tell frr to reload ensuring that it sees thunderbolt links towards other nodes.
# condition: is there any en05 and en06 network interface and frr service up
COUNTER=0
while [ \$COUNTER -lt 5 ]; do
ifup en05
ifup en06
sleep 2;
TEST=\$(ip a | egrep "en05|en06" | grep UP | wc -l)
if [ \$TEST -eq 2 ]; then
TEST_SVC=\$(service frr status | grep "active (running)" | wc -l)
if [ \$TEST_SVC -eq 1 ]; then
service frr reload;
echo "frr service reload request sent"
exit 0;
fi
fi
COUNTER=\$((COUNTER+1));
done
echo "Failed to request frr service reload: request NOT sent"
exit 1;
EOF
chmod +x /usr/local/bin/restart-frr.sh
cat <<EOF >> /etc/systemd/system/restart-frr.service
[Unit]
After=network.target
[Service]
ExecStart=/usr/local/bin/restart-frr.sh
[Install]
WantedBy=default.target
EOF
systemctl daemon-reload
systemctl enable restart-frr.service
vtysh -c "show openfabric topology"
vtysh -c "show running-config"
pveceph init --network 10.0.0.0/24
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment