You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

vxlan-and-evpn.adoc 43KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107
  1. ////
  2. This is currently not included, because
  3. - it requires ifupdown2
  4. - routing needs more documentation
  5. ////
  6. VXLAN layer2 with vlan unware linux bridges
  7. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  8. VXLAN is an overlay network to carry Ethernet traffic over an existing IP network
  9. while accommodating a very large number of tenants. It is defined in RFC 7348.
  10. Each overlay network is known as a VXLAN Segment and identified by a unique
  11. 24-bit segment ID called a VXLAN Network Identifier (VNI).
  12. VXLAN encapsulation add 50bytes overhead, so you need to increase mtu on your host
  13. physical interfaces to 1550 at minimum. (or decrease mtu inside your vms to 1450)
  14. For BUM traffic (broadcast / unknown unicast traffic, multicast),
  15. we have 3 differents vxlan setup modes : multicast, unicast, bgp-evpn
  16. image::images/vxlan-l2-vlanunaware.svg["vxlan l2 bridge vlan unaware",align="center"]
  17. multicast mode
  18. ^^^^^^^^^^^^^^
  19. This scenario relies in head end replication, meaning that end host in case
  20. of not having any entry for the destination MAC address will send out an ARP
  21. to other devices / VTEPs in the VXLAN network.
  22. This is done by sending the request to the VXLAN multicast group,
  23. remote VTEPs will get the packet and answer accordingly direct to the originating VTEP.
  24. * node1
  25. ----
  26. auto eno1
  27. iface eno1 inet manual
  28. mtu 1550
  29. auto vmbr0
  30. iface vmbr0 inet static
  31. address 192.168.0.1
  32. netmask 255.255.255.0
  33. bridge_ports eno1
  34. bridge_stp off
  35. bridge_fd 0
  36. auto vxlan2
  37. iface vxlan2 inet manual
  38. vxlan-id 2
  39. vxlan-svcnodeip 225.20.1.1
  40. vxlan-physdev eno1
  41. auto vmbr2
  42. iface vmbr2 inet manual
  43. bridge_ports vxlan2
  44. bridge_stp off
  45. bridge_fd 0
  46. auto vxlan3
  47. iface vxlan3 inet manual
  48. vxlan-id 3
  49. vxlan-svcnodeip 225.20.1.1
  50. vxlan-physdev eno1
  51. auto vmbr3
  52. iface vmbr3 inet manual
  53. bridge_ports vxlan3
  54. bridge_stp off
  55. bridge_fd 0
  56. ----
  57. * node2
  58. ----
  59. auto eno1
  60. iface eno1 inet manual
  61. mtu 1550
  62. auto vmbr0
  63. iface vmbr0 inet static
  64. address 192.168.0.2
  65. netmask 255.255.255.0
  66. bridge_ports eno1
  67. bridge_stp off
  68. bridge_fd 0
  69. auto vxlan2
  70. iface vxlan2 inet manual
  71. vxlan-id 2
  72. vxlan-svcnodeip 225.20.1.1
  73. vxlan-physdev eno1
  74. auto vmbr2
  75. iface vmbr2 inet manual
  76. bridge_ports vxlan2
  77. bridge_stp off
  78. bridge_fd 0
  79. auto vxlan3
  80. iface vxlan3 inet manual
  81. vxlan-id 3
  82. vxlan-svcnodeip 225.20.1.1
  83. vxlan-physdev eno1
  84. auto vmbr3
  85. iface vmbr3 inet manual
  86. bridge_ports vxlan3
  87. bridge_stp off
  88. bridge_fd 0
  89. ----
  90. * node3
  91. ----
  92. auto eno1
  93. iface eno1 inet manual
  94. mtu 1550
  95. auto vmbr0
  96. iface vmbr0 inet static
  97. address 192.168.0.3
  98. netmask 255.255.255.0
  99. bridge_ports eno1
  100. bridge_stp off
  101. bridge_fd 0
  102. auto vxlan2
  103. iface vxlan2 inet manual
  104. vxlan-id 2
  105. vxlan-svcnodeip 225.20.1.1
  106. vxlan-physdev eno1
  107. auto vmbr2
  108. iface vmbr2 inet manual
  109. bridge_ports vxlan2
  110. bridge_stp off
  111. bridge_fd 0
  112. auto vxlan3
  113. iface vxlan3 inet manual
  114. vxlan-id 3
  115. vxlan-svcnodeip 225.20.1.1
  116. vxlan-physdev eno1
  117. auto vmbr3
  118. iface vmbr3 inet manual
  119. bridge_ports vxlan3
  120. bridge_stp off
  121. bridge_fd 0
  122. ----
  123. unicast mode
  124. ^^^^^^^^^^^^
  125. We can replace multicast by head-end replication of BUM frames to a statically configured lists of remote VTEPs.
  126. The VXLAN is defined without a remote multicast group.
  127. Instead, all the remote VTEPs are associated with the all-zero address:
  128. a BUM frame will be duplicated to all these destinations.
  129. The VXLAN device will still learn remote addresses automatically using source-address learning.
  130. * node1
  131. ----
  132. auto eno1
  133. iface eno1 inet manual
  134. mtu 1550
  135. auto vmbr0
  136. iface vmbr0 inet static
  137. address 192.168.0.1
  138. netmask 255.255.255.0
  139. bridge_ports eno1
  140. bridge_stp off
  141. bridge_fd 0
  142. auto vxlan2
  143. iface vxlan2 inet manual
  144. vxlan-id 2
  145. vxlan_remoteip 192.168.0.2
  146. vxlan_remoteip 192.168.0.3
  147. auto vmbr2
  148. iface vmbr2 inet manual
  149. bridge_ports vxlan2
  150. bridge_stp off
  151. bridge_fd 0
  152. auto vxlan3
  153. iface vxlan2 inet manual
  154. vxlan-id 3
  155. vxlan_remoteip 192.168.0.2
  156. vxlan_remoteip 192.168.0.3
  157. auto vmbr3
  158. iface vmbr3 inet manual
  159. bridge_ports vxlan3
  160. bridge_stp off
  161. bridge_fd 0
  162. ----
  163. * node2
  164. ----
  165. auto eno1
  166. iface eno1 inet manual
  167. mtu 1550
  168. auto vmbr0
  169. iface vmbr0 inet static
  170. address 192.168.0.2
  171. netmask 255.255.255.0
  172. bridge_ports eno1
  173. bridge_stp off
  174. bridge_fd 0
  175. auto vxlan2
  176. iface vxlan2 inet manual
  177. vxlan-id 2
  178. vxlan_remoteip 192.168.0.1
  179. vxlan_remoteip 192.168.0.3
  180. auto vmbr2
  181. iface vmbr2 inet manual
  182. bridge_ports vxlan2
  183. bridge_stp off
  184. bridge_fd 0
  185. auto vxlan3
  186. iface vxlan2 inet manual
  187. vxlan-id 3
  188. vxlan_remoteip 192.168.0.1
  189. vxlan_remoteip 192.168.0.3
  190. auto vmbr3
  191. iface vmbr3 inet manual
  192. bridge_ports vxlan3
  193. bridge_stp off
  194. bridge_fd 0
  195. ----
  196. * node3
  197. ----
  198. auto eno1
  199. iface eno1 inet manual
  200. mtu 1550
  201. auto vmbr0
  202. iface vmbr0 inet static
  203. address 192.168.0.3
  204. netmask 255.255.255.0
  205. bridge_ports eno1
  206. bridge_stp off
  207. bridge_fd 0
  208. auto vxlan2
  209. iface vxlan2 inet manual
  210. vxlan-id 2
  211. vxlan_remoteip 192.168.0.2
  212. vxlan_remoteip 192.168.0.3
  213. auto vmbr2
  214. iface vmbr2 inet manual
  215. bridge_ports vxlan2
  216. bridge_stp off
  217. bridge_fd 0
  218. auto vxlan3
  219. iface vxlan2 inet manual
  220. vxlan-id 3
  221. vxlan_remoteip 192.168.0.2
  222. vxlan_remoteip 192.168.0.3
  223. auto vmbr3
  224. iface vmbr3 inet manual
  225. bridge_ports vxlan3
  226. bridge_stp off
  227. bridge_fd 0
  228. ----
  229. bgp-evpn
  230. ^^^^^^^^
  231. VTEPs use control plane learning/distribution via BGP for remote MAC addresses instead of data plane learning.
  232. VTEPs have the ability to suppress ARP flooding over VXLAN tunnels.
  233. The control plane used here is FRR, a bgp routing software.
  234. Each node in the proxmox cluster peer with each others nodes.
  235. For bigger networks, or multiple proxmox clusters,
  236. it's possible to use external bgp route reflector servers.
  237. * node1
  238. ----
  239. auto eno1
  240. iface eno1 inet manual
  241. mtu 1550
  242. auto vmbr0
  243. iface vmbr0 inet static
  244. address 192.168.0.1
  245. netmask 255.255.255.0
  246. bridge_ports eno1
  247. bridge_stp off
  248. bridge_fd 0
  249. auto vxlan2
  250. iface vxlan2 inet manual
  251. vxlan-id 2
  252. vxlan-local-tunnelip 192.168.0.1
  253. bridge-learning off
  254. bridge-arp-nd-suppress on
  255. bridge-unicast-flood off
  256. bridge-multicast-flood off
  257. auto vmbr2
  258. iface vmbr2 inet manual
  259. bridge_ports vxlan2
  260. bridge_stp off
  261. bridge_fd 0
  262. auto vxlan3
  263. iface vxlan3 inet manual
  264. vxlan-id 3
  265. vxlan-local-tunnelip 192.168.0.1
  266. bridge-learning off
  267. bridge-arp-nd-suppress on
  268. bridge-unicast-flood off
  269. bridge-multicast-flood off
  270. auto vmbr3
  271. iface vmbr3 inet manual
  272. bridge_ports vxlan3
  273. bridge_stp off
  274. bridge_fd 0
  275. ----
  276. /etc/frr/frr.conf
  277. ----
  278. router bgp 1234
  279. no bgp default ipv4-unicast
  280. coalesce-time 1000
  281. neighbor 192.168.0.2 remote-as 1234
  282. neighbor 192.168.0.3 remote-as 1234
  283. !
  284. address-family l2vpn evpn
  285. neighbor 192.168.0.2 activate
  286. neighbor 192.168.0.3 activate
  287. advertise-all-vni
  288. exit-address-family
  289. !
  290. line vty
  291. !
  292. ----
  293. * node2
  294. ----
  295. auto eno1
  296. iface eno1 inet manual
  297. mtu 1550
  298. auto vmbr0
  299. iface vmbr0 inet static
  300. address 192.168.0.2
  301. netmask 255.255.255.0
  302. bridge_ports eno1
  303. bridge_stp off
  304. bridge_fd 0
  305. auto vxlan2
  306. iface vxlan2 inet manual
  307. vxlan-id 2
  308. vxlan-local-tunnelip 192.168.0.2
  309. bridge-learning off
  310. bridge-arp-nd-suppress on
  311. bridge-unicast-flood off
  312. bridge-multicast-flood off
  313. auto vmbr2
  314. iface vmbr2 inet manual
  315. bridge_ports vxlan2
  316. bridge_stp off
  317. bridge_fd 0
  318. auto vxlan3
  319. iface vxlan3 inet manual
  320. vxlan-id 3
  321. vxlan-local-tunnelip 192.168.0.2
  322. bridge-learning off
  323. bridge-arp-nd-suppress on
  324. bridge-unicast-flood off
  325. bridge-multicast-flood off
  326. auto vmbr3
  327. iface vmbr3 inet manual
  328. bridge_ports vxlan3
  329. bridge_stp off
  330. bridge_fd 0
  331. ----
  332. /etc/frr/frr.conf
  333. ----
  334. router bgp 1234
  335. no bgp default ipv4-unicast
  336. coalesce-time 1000
  337. neighbor 192.168.0.1 remote-as 1234
  338. neighbor 192.168.0.3 remote-as 1234
  339. !
  340. address-family l2vpn evpn
  341. neighbor 192.168.0.1 activate
  342. neighbor 192.168.0.3 activate
  343. advertise-all-vni
  344. exit-address-family
  345. !
  346. line vty
  347. !
  348. ----
  349. * node3
  350. ----
  351. auto eno1
  352. iface eno1 inet manual
  353. mtu 1550
  354. auto vmbr0
  355. iface vmbr0 inet static
  356. address 192.168.0.2
  357. netmask 255.255.255.0
  358. bridge_ports eno1
  359. bridge_stp off
  360. bridge_fd 0
  361. auto vxlan2
  362. iface vxlan2 inet manual
  363. vxlan-id 2
  364. vxlan-local-tunnelip 192.168.0.3
  365. bridge-learning off
  366. bridge-arp-nd-suppress on
  367. bridge-unicast-flood off
  368. bridge-multicast-flood off
  369. auto vmbr2
  370. iface vmbr2 inet manual
  371. bridge_ports vxlan2
  372. bridge_stp off
  373. bridge_fd 0
  374. auto vxlan3
  375. iface vxlan3 inet manual
  376. vxlan-id 3
  377. vxlan-local-tunnelip 192.168.0.3
  378. bridge-learning off
  379. bridge-arp-nd-suppress on
  380. bridge-unicast-flood off
  381. bridge-multicast-flood off
  382. auto vmbr3
  383. iface vmbr3 inet manual
  384. bridge_ports vxlan3
  385. bridge_stp off
  386. bridge_fd 0
  387. ----
  388. /etc/frr/frr.conf
  389. ----
  390. router bgp 1234
  391. no bgp default ipv4-unicast
  392. coalesce-time 1000
  393. neighbor 192.168.0.1 remote-as 1234
  394. neighbor 192.168.0.2 remote-as 1234
  395. !
  396. address-family l2vpn evpn
  397. neighbor 192.168.0.1 activate
  398. neighbor 192.168.0.2 activate
  399. advertise-all-vni
  400. exit-address-family
  401. !
  402. line vty
  403. !
  404. ----
  405. VXLAN layer3 routing with anycast gateway
  406. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  407. With this need, each vmbr bridge will be the gateway for the vm.
  408. Same vmbr on different node, will have same ip address and same mac address,
  409. to have working vm live migration and no network disruption.
  410. VXLAN layer3 routing only work with FRR and non-aware bridge.
  411. (vlan aware bridge support is buggy currently).
  412. asymmetric model
  413. ^^^^^^^^^^^^^^^^
  414. This is the simplest mode. To get it work, all vxlan need to be defined on all nodes.
  415. The asymmetric model allows routing and bridging on the VXLAN tunnel ingress,
  416. but only bridging on the egress.
  417. This results in bi-directional VXLAN traffic traveling on different VNIs
  418. in each direction (always the destination VNI) across the routed infrastructure.
  419. image::images/vxlan-l3-asymmetric.svg["vxlan l3 asymmetric",align="center"]
  420. * node1
  421. ----
  422. auto eno1
  423. iface eno1 inet manual
  424. mtu 1550
  425. auto vmbr0
  426. iface vmbr0 inet static
  427. address 192.168.0.1
  428. netmask 255.255.255.0
  429. bridge_ports eno1
  430. bridge_stp off
  431. bridge_fd 0
  432. auto vxlan2
  433. iface vxlan2 inet manual
  434. vxlan-id 2
  435. vxlan-local-tunnelip 192.168.0.1
  436. bridge-learning off
  437. bridge-arp-nd-suppress on
  438. bridge-unicast-flood off
  439. bridge-multicast-flood off
  440. auto vmbr2
  441. iface vmbr2 inet static
  442. address 10.0.2.254
  443. netmask 255.255.255.0
  444. hwaddress 44:39:39:FF:40:94
  445. bridge_ports vxlan2
  446. bridge_stp off
  447. bridge_fd 0
  448. ip-forward on
  449. ip6-forward on
  450. arp-accept on
  451. auto vxlan3
  452. iface vxlan3 inet manual
  453. vxlan-id 3
  454. vxlan-local-tunnelip 192.168.0.1
  455. bridge-learning off
  456. bridge-arp-nd-suppress on
  457. bridge-unicast-flood off
  458. bridge-multicast-flood off
  459. auto vmbr3
  460. iface vmbr3 inet static
  461. address 10.0.3.254
  462. netmask 255.255.255.0
  463. hwaddress 44:39:39:FF:40:94
  464. bridge_ports vxlan3
  465. bridge_stp off
  466. bridge_fd 0
  467. ip-forward on
  468. ip6-forward on
  469. arp-accept on
  470. ----
  471. frr.conf
  472. ----
  473. router bgp 1234
  474. bgp router-id 192.168.0.1
  475. no bgp default ipv4-unicast
  476. coalesce-time 1000
  477. neighbor 192.168.0.2 remote-as 1234
  478. neighbor 192.168.0.3 remote-as 1234
  479. !
  480. address-family l2vpn evpn
  481. neighbor 192.168.0.2 activate
  482. neighbor 192.168.0.3 activate
  483. advertise-all-vni
  484. exit-address-family
  485. !
  486. line vty
  487. !
  488. ----
  489. * node2
  490. ----
  491. auto eno1
  492. iface eno1 inet manual
  493. mtu 1550
  494. auto vmbr0
  495. iface vmbr0 inet static
  496. address 192.168.0.2
  497. netmask 255.255.255.0
  498. bridge_ports eno1
  499. bridge_stp off
  500. bridge_fd 0
  501. auto vxlan2
  502. iface vxlan2 inet manual
  503. vxlan-id 2
  504. vxlan-local-tunnelip 192.168.0.2
  505. bridge-learning off
  506. bridge-arp-nd-suppress on
  507. bridge-unicast-flood off
  508. bridge-multicast-flood off
  509. auto vmbr2
  510. iface vmbr2 inet static
  511. address 10.0.2.254
  512. netmask 255.255.255.0
  513. hwaddress 44:39:39:FF:40:94
  514. bridge_ports vxlan2
  515. bridge_stp off
  516. bridge_fd 0
  517. ip-forward on
  518. ip6-forward on
  519. arp-accept on
  520. auto vxlan3
  521. iface vxlan3 inet manual
  522. vxlan-id 3
  523. vxlan-local-tunnelip 192.168.0.2
  524. bridge-learning off
  525. bridge-arp-nd-suppress on
  526. bridge-unicast-flood off
  527. bridge-multicast-flood off
  528. auto vmbr3
  529. iface vmbr3 inet static
  530. address 10.0.3.254
  531. netmask 255.255.255.0
  532. hwaddress 44:39:39:FF:40:94
  533. bridge_ports vxlan3
  534. bridge_stp off
  535. bridge_fd 0
  536. ip-forward on
  537. ip6-forward on
  538. arp-accept on
  539. ----
  540. frr.conf
  541. ----
  542. router bgp 1234
  543. bgp router-id 192.168.0.2
  544. no bgp default ipv4-unicast
  545. coalesce-time 1000
  546. neighbor 192.168.0.1 remote-as 1234
  547. neighbor 192.168.0.3 remote-as 1234
  548. !
  549. address-family l2vpn evpn
  550. neighbor 192.168.0.1 activate
  551. neighbor 192.168.0.3 activate
  552. advertise-all-vni
  553. exit-address-family
  554. !
  555. line vty
  556. !
  557. ----
  558. * node3
  559. ----
  560. auto eno1
  561. iface eno1 inet manual
  562. mtu 1550
  563. auto vmbr0
  564. iface vmbr0 inet static
  565. address 192.168.0.3
  566. netmask 255.255.255.0
  567. bridge_ports eno1
  568. bridge_stp off
  569. bridge_fd 0
  570. auto vxlan2
  571. iface vxlan2 inet manual
  572. vxlan-id 2
  573. vxlan-local-tunnelip 192.168.0.3
  574. bridge-learning off
  575. bridge-arp-nd-suppress on
  576. bridge-unicast-flood off
  577. bridge-multicast-flood off
  578. auto vmbr2
  579. iface vmbr2 inet static
  580. address 10.0.2.254
  581. netmask 255.255.255.0
  582. hwaddress 44:39:39:FF:40:94
  583. bridge_ports vxlan2
  584. bridge_stp off
  585. bridge_fd 0
  586. ip-forward on
  587. ip6-forward on
  588. arp-accept on
  589. auto vxlan3
  590. iface vxlan3 inet manual
  591. vxlan-id 3
  592. vxlan-local-tunnelip 192.168.0.3
  593. bridge-learning off
  594. bridge-arp-nd-suppress on
  595. bridge-unicast-flood off
  596. bridge-multicast-flood off
  597. auto vmbr3
  598. iface vmbr3 inet static
  599. address 10.0.3.254
  600. netmask 255.255.255.0
  601. hwaddress 44:39:39:FF:40:94
  602. bridge_ports vxlan3
  603. bridge_stp off
  604. bridge_fd 0
  605. ip-forward on
  606. ip6-forward on
  607. arp-accept on
  608. ----
  609. frr.conf
  610. ----
  611. router bgp 1234
  612. bgp router-id 192.168.0.3
  613. no bgp default ipv4-unicast
  614. coalesce-time 1000
  615. neighbor 192.168.0.1 remote-as 1234
  616. neighbor 192.168.0.2 remote-as 1234
  617. !
  618. address-family l2vpn evpn
  619. neighbor 192.168.0.1 activate
  620. neighbor 192.168.0.2 activate
  621. advertise-all-vni
  622. exit-address-family
  623. !
  624. line vty
  625. !
  626. ----
  627. symmetric model
  628. ^^^^^^^^^^^^^^^
  629. With this model, you don't need to have all vxlan on all nodes.
  630. This model will also be needed to route traffic to an external router.
  631. The symmetric model routes and bridges on both the ingress and the egress leafs.
  632. This results in bi-directional traffic being able to travel on the same VNI, hence the symmetric name.
  633. However, a new specialty transit VNI is used for all routed VXLAN traffic, called the L3VNI.
  634. All traffic that needs to be routed will be routed onto the L3VNI, tunneled across the layer 3 Infrastructure,
  635. routed off the L3VNI to the appropriate VLAN and ultimately bridged to the destination.
  636. A vrf is needed for the L3VNI, so all vmbr bridge need to be in the vrf if they want to be able to reach each others.
  637. image::images/vxlan-l3-symmetric.svg["vxlan l3 symmetric",align="center"]
  638. * node1
  639. ----
  640. auto vrf1
  641. iface vrf1
  642. vrf-table auto
  643. auto eno1
  644. iface eno1 inet manual
  645. mtu 1550
  646. auto vmbr0
  647. iface vmbr0 inet static
  648. address 192.168.0.1
  649. netmask 255.255.255.0
  650. bridge_ports eno1
  651. bridge_stp off
  652. bridge_fd 0
  653. auto vxlan2
  654. iface vxlan2 inet manual
  655. vxlan-id 2
  656. vxlan-local-tunnelip 192.168.0.1
  657. bridge-learning off
  658. bridge-arp-nd-suppress on
  659. bridge-unicast-flood off
  660. bridge-multicast-flood off
  661. auto vmbr2
  662. iface vmbr2 inet static
  663. bridge_ports vxlan2
  664. bridge_stp off
  665. bridge_fd 0
  666. address 10.0.2.254
  667. netmask 255.255.255.0
  668. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2
  669. vrf vrf1
  670. ip-forward on
  671. ip6-forward on
  672. arp-accept on
  673. auto vxlan3
  674. iface vxlan3 inet manual
  675. vxlan-id 3
  676. vxlan-local-tunnelip 192.168.0.1
  677. bridge-learning off
  678. bridge-arp-nd-suppress on
  679. bridge-unicast-flood off
  680. bridge-multicast-flood off
  681. auto vmbr3
  682. iface vmbr3 inet static
  683. bridge_ports vxlan3
  684. bridge_stp off
  685. bridge_fd 0
  686. address 10.0.3.254
  687. netmask 255.255.255.0
  688. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3
  689. vrf vrf1
  690. ip-forward on
  691. ip6-forward on
  692. arp-accept on
  693. #interconnect vxlan-vfr l3vni
  694. auto vxlan4000
  695. iface vxlan4000 inet manual
  696. vxlan-id 4000
  697. vxlan-local-tunnelip 192.168.0.1
  698. bridge-learning off
  699. bridge-arp-nd-suppress on
  700. bridge-unicast-flood off
  701. bridge-multicast-flood off
  702. auto vmbr4000
  703. iface vmbr4000 inet manual
  704. bridge_ports vxlan4000
  705. bridge_stp off
  706. bridge_fd 0
  707. vrf vrf1
  708. ----
  709. frr.conf
  710. ----
  711. vrf vrf1
  712. vni 4000
  713. exit-vrf
  714. !
  715. router bgp 1234
  716. bgp router-id 192.168.0.1
  717. no bgp default ipv4-unicast
  718. coalesce-time 1000
  719. neighbor 192.168.0.2 remote-as 1234
  720. neighbor 192.168.0.3 remote-as 1234
  721. !
  722. address-family l2vpn evpn
  723. neighbor 192.168.0.2 activate
  724. neighbor 192.168.0.3 activate
  725. advertise-all-vni
  726. exit-address-family
  727. !
  728. line vty
  729. !
  730. ----
  731. * node2
  732. ----
  733. auto vrf1
  734. iface vrf1
  735. vrf-table auto
  736. auto eno1
  737. iface eno1 inet manual
  738. mtu 1550
  739. auto vmbr0
  740. iface vmbr0 inet static
  741. address 192.168.0.2
  742. netmask 255.255.255.0
  743. bridge_ports eno1
  744. bridge_stp off
  745. bridge_fd 0
  746. auto vxlan2
  747. iface vxlan2 inet manual
  748. vxlan-id 2
  749. vxlan-local-tunnelip 192.168.0.2
  750. bridge-learning off
  751. bridge-arp-nd-suppress on
  752. bridge-unicast-flood off
  753. bridge-multicast-flood off
  754. auto vmbr2
  755. iface vmbr2 inet static
  756. bridge_ports vxlan2
  757. bridge_stp off
  758. bridge_fd 0
  759. address 10.0.2.254
  760. netmask 255.255.255.0
  761. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2
  762. vrf vrf1
  763. ip-forward on
  764. ip6-forward on
  765. arp-accept on
  766. auto vxlan3
  767. iface vxlan3 inet manual
  768. vxlan-id 3
  769. vxlan-local-tunnelip 192.168.0.2
  770. bridge-learning off
  771. bridge-arp-nd-suppress on
  772. bridge-unicast-flood off
  773. bridge-multicast-flood off
  774. auto vmbr3
  775. iface vmbr3 inet static
  776. bridge_ports vxlan3
  777. bridge_stp off
  778. bridge_fd 0
  779. address 10.0.3.254
  780. netmask 255.255.255.0
  781. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3
  782. vrf vrf1
  783. ip-forward on
  784. ip6-forward on
  785. arp-accept on
  786. #interconnect vxlan-vfr l3vni
  787. auto vxlan4000
  788. iface vxlan4000 inet manual
  789. vxlan-id 4000
  790. vxlan-local-tunnelip 192.168.0.2
  791. bridge-learning off
  792. bridge-arp-nd-suppress on
  793. bridge-unicast-flood off
  794. bridge-multicast-flood off
  795. auto vmbr4000
  796. iface vmbr4000 inet manual
  797. bridge_ports vxlan4000
  798. bridge_stp off
  799. bridge_fd 0
  800. vrf vrf1
  801. ----
  802. frr.conf
  803. ----
  804. vrf vrf1
  805. vni 4000
  806. exit-vrf
  807. !
  808. router bgp 1234
  809. bgp router-id 192.168.0.2
  810. no bgp default ipv4-unicast
  811. coalesce-time 1000
  812. neighbor 192.168.0.1 remote-as 1234
  813. neighbor 192.168.0.3 remote-as 1234
  814. !
  815. address-family l2vpn evpn
  816. neighbor 192.168.0.1 activate
  817. neighbor 192.168.0.3 activate
  818. advertise-all-vni
  819. exit-address-family
  820. !
  821. line vty
  822. !
  823. ----
  824. * node3
  825. ----
  826. auto vrf1
  827. iface vrf1
  828. vrf-table auto
  829. auto eno1
  830. iface eno1 inet manual
  831. mtu 1550
  832. auto vmbr0
  833. iface vmbr0 inet static
  834. address 192.168.0.3
  835. netmask 255.255.255.0
  836. bridge_ports eno1
  837. bridge_stp off
  838. bridge_fd 0
  839. auto vxlan2
  840. iface vxlan2 inet manual
  841. vxlan-id 2
  842. vxlan-local-tunnelip 192.168.0.3
  843. bridge-learning off
  844. bridge-arp-nd-suppress on
  845. bridge-unicast-flood off
  846. bridge-multicast-flood off
  847. auto vmbr2
  848. iface vmbr2 inet static
  849. bridge_ports vxlan2
  850. bridge_stp off
  851. bridge_fd 0
  852. address 10.0.2.254
  853. netmask 255.255.255.0
  854. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2
  855. vrf vrf1
  856. ip-forward on
  857. ip6-forward on
  858. arp-accept on
  859. auto vxlan3
  860. iface vxlan3 inet manual
  861. vxlan-id 3
  862. vxlan-local-tunnelip 192.168.0.3
  863. bridge-learning off
  864. bridge-arp-nd-suppress on
  865. bridge-unicast-flood off
  866. bridge-multicast-flood off
  867. auto vmbr3
  868. iface vmbr3 inet static
  869. bridge_ports vxlan3
  870. bridge_stp off
  871. bridge_fd 0
  872. address 10.0.3.254
  873. netmask 255.255.255.0
  874. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3
  875. vrf vrf1
  876. ip-forward on
  877. ip6-forward on
  878. arp-accept on
  879. #interconnect vxlan-vfr l3vni
  880. auto vxlan4000
  881. iface vxlan4000 inet manual
  882. vxlan-id 4000
  883. vxlan-local-tunnelip 192.168.0.3
  884. bridge-learning off
  885. bridge-arp-nd-suppress on
  886. bridge-unicast-flood off
  887. bridge-multicast-flood off
  888. auto vmbr4000
  889. iface vmbr4000 inet manual
  890. bridge_ports vxlan4000
  891. bridge_stp off
  892. bridge_fd 0
  893. vrf vrf1
  894. ----
  895. frr.conf
  896. ----
  897. vrf vrf1
  898. vni 4000
  899. exit-vrf
  900. !
  901. router bgp 1234
  902. bgp router-id 192.168.0.3
  903. no bgp default ipv4-unicast
  904. coalesce-time 1000
  905. neighbor 192.168.0.1 remote-as 1234
  906. neighbor 192.168.0.2 remote-as 1234
  907. !
  908. address-family l2vpn evpn
  909. neighbor 192.168.0.1 activate
  910. neighbor 192.168.0.2 activate
  911. advertise-all-vni
  912. exit-address-family
  913. !
  914. line vty
  915. !
  916. ----
  917. VXLAN layer3 routing with anycast gateway + routing to outside with external router with static default gw
  918. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  919. Routing to outside need the symmetric model.
  920. 1 gateway node
  921. ^^^^^^^^^^^^^^
  922. In this example, we'll use only 1 proxmox node as exit gateway. (node1)
  923. This node announce the default gw in vrf1 (default originate) and forward to his own default gateway (192.168.0.254) (no bgp between router and node1)
  924. *node1
  925. ----
  926. auto vrf1
  927. iface vrf1
  928. vrf-table auto
  929. auto eno1
  930. iface eno1 inet manual
  931. mtu 1550
  932. auto vmbr0
  933. iface vmbr0 inet static
  934. address 192.168.0.1
  935. netmask 255.255.255.0
  936. gateway 192.168.0.254
  937. bridge_ports eno1
  938. bridge_stp off
  939. bridge_fd 0
  940. ip-forward on
  941. ip6-forward on
  942. auto vxlan2
  943. iface vxlan2 inet manual
  944. vxlan-id 2
  945. vxlan-local-tunnelip 192.168.0.1
  946. bridge-learning off
  947. bridge-arp-nd-suppress on
  948. bridge-unicast-flood off
  949. bridge-multicast-flood off
  950. auto vmbr2
  951. iface vmbr2 inet static
  952. bridge_ports vxlan2
  953. bridge_stp off
  954. bridge_fd 0
  955. address 10.0.2.254
  956. netmask 255.255.255.0
  957. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2
  958. vrf vrf1
  959. ip-forward on
  960. ip6-forward on
  961. arp-accept on
  962. auto vxlan3
  963. iface vxlan3 inet manual
  964. vxlan-id 3
  965. vxlan-local-tunnelip 192.168.0.1
  966. bridge-learning off
  967. bridge-arp-nd-suppress on
  968. bridge-unicast-flood off
  969. bridge-multicast-flood off
  970. auto vmbr3
  971. iface vmbr3 inet static
  972. bridge_ports vxlan3
  973. bridge_stp off
  974. bridge_fd 0
  975. address 10.0.3.254
  976. netmask 255.255.255.0
  977. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3
  978. vrf vrf1
  979. ip-forward on
  980. ip6-forward on
  981. arp-accept on
  982. #interconnect vxlan-vfr l3vni
  983. auto vxlan4000
  984. iface vxlan4000 inet manual
  985. vxlan-id 4000
  986. vxlan-local-tunnelip 192.168.0.1
  987. bridge-learning off
  988. bridge-arp-nd-suppress on
  989. bridge-unicast-flood off
  990. bridge-multicast-flood off
  991. auto vmbr4000
  992. iface vmbr4000 inet manual
  993. bridge_ports vxlan4000
  994. bridge_stp off
  995. bridge_fd 0
  996. vrf vrf1
  997. ----
  998. frr.conf
  999. ----
  1000. vrf vrf1
  1001. vni 4000
  1002. exit-vrf
  1003. !
  1004. router bgp 1234
  1005. bgp router-id 192.168.0.1
  1006. no bgp default ipv4-unicast
  1007. coalesce-time 1000
  1008. neighbor 192.168.0.2 remote-as 1234
  1009. neighbor 192.168.0.3 remote-as 1234
  1010. !
  1011. address-family ipv4 unicast
  1012. import vrf vrf1
  1013. exit-address-family
  1014. !
  1015. address-family ipv6 unicast
  1016. import vrf vrf1
  1017. exit-address-family
  1018. !
  1019. address-family l2vpn evpn
  1020. neighbor 192.168.0.2 activate
  1021. neighbor 192.168.0.3 activate
  1022. advertise-all-vni
  1023. exit-address-family
  1024. !
  1025. router bgp 1234 vrf vrf1
  1026. !
  1027. address-family ipv4 unicast
  1028. redistribute connected
  1029. exit-address-family
  1030. !
  1031. address-family ipv6 unicast
  1032. redistribute connected
  1033. exit-address-family
  1034. !
  1035. address-family l2vpn evpn
  1036. default-originate ipv4
  1037. default-originate ipv6
  1038. exit-address-family
  1039. !
  1040. line vty
  1041. !
  1042. ----
  1043. * node2
  1044. ----
  1045. auto vrf1
  1046. iface vrf1
  1047. vrf-table auto
  1048. auto eno1
  1049. iface eno1 inet manual
  1050. mtu 1550
  1051. auto vmbr0
  1052. iface vmbr0 inet static
  1053. address 192.168.0.2
  1054. netmask 255.255.255.0
  1055. bridge_ports eno1
  1056. bridge_stp off
  1057. bridge_fd 0
  1058. auto vxlan2
  1059. iface vxlan2 inet manual
  1060. vxlan-id 2
  1061. vxlan-local-tunnelip 192.168.0.2
  1062. bridge-learning off
  1063. bridge-arp-nd-suppress on
  1064. bridge-unicast-flood off
  1065. bridge-multicast-flood off
  1066. auto vmbr2
  1067. iface vmbr2 inet static
  1068. bridge_ports vxlan2
  1069. bridge_stp off
  1070. bridge_fd 0
  1071. address 10.0.2.254
  1072. netmask 255.255.255.0
  1073. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2
  1074. vrf vrf1
  1075. ip-forward on
  1076. ip6-forward on
  1077. arp-accept on
  1078. auto vxlan3
  1079. iface vxlan3 inet manual
  1080. vxlan-id 3
  1081. vxlan-local-tunnelip 192.168.0.2
  1082. bridge-learning off
  1083. bridge-arp-nd-suppress on
  1084. bridge-unicast-flood off
  1085. bridge-multicast-flood off
  1086. auto vmbr3
  1087. iface vmbr3 inet static
  1088. bridge_ports vxlan3
  1089. bridge_stp off
  1090. bridge_fd 0
  1091. address 10.0.3.254
  1092. netmask 255.255.255.0
  1093. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3
  1094. vrf vrf1
  1095. ip-forward on
  1096. ip6-forward on
  1097. arp-accept on
  1098. #interconnect vxlan-vfr l3vni
  1099. auto vxlan4000
  1100. iface vxlan4000 inet manual
  1101. vxlan-id 4000
  1102. vxlan-local-tunnelip 192.168.0.2
  1103. bridge-learning off
  1104. bridge-arp-nd-suppress on
  1105. bridge-unicast-flood off
  1106. bridge-multicast-flood off
  1107. auto vmbr4000
  1108. iface vmbr4000 inet manual
  1109. bridge_ports vxlan4000
  1110. bridge_stp off
  1111. bridge_fd 0
  1112. vrf vrf1
  1113. ----
  1114. frr.conf
  1115. ----
  1116. vrf vrf1
  1117. vni 4000
  1118. exit-vrf
  1119. !
  1120. router bgp 1234
  1121. bgp router-id 192.168.0.2
  1122. no bgp default ipv4-unicast
  1123. coalesce-time 1000
  1124. neighbor 192.168.0.1 remote-as 1234
  1125. neighbor 192.168.0.3 remote-as 1234
  1126. !
  1127. address-family l2vpn evpn
  1128. neighbor 192.168.0.1 activate
  1129. neighbor 192.168.0.3 activate
  1130. advertise-all-vni
  1131. exit-address-family
  1132. !
  1133. line vty
  1134. !
  1135. ----
  1136. * node3
  1137. ----
  1138. auto vrf1
  1139. iface vrf1
  1140. vrf-table auto
  1141. auto eno1
  1142. iface eno1 inet manual
  1143. mtu 1550
  1144. auto vmbr0
  1145. iface vmbr0 inet static
  1146. address 192.168.0.3
  1147. netmask 255.255.255.0
  1148. bridge_ports eno1
  1149. bridge_stp off
  1150. bridge_fd 0
  1151. auto vxlan2
  1152. iface vxlan2 inet manual
  1153. vxlan-id 2
  1154. vxlan-local-tunnelip 192.168.0.3
  1155. bridge-learning off
  1156. bridge-arp-nd-suppress on
  1157. bridge-unicast-flood off
  1158. bridge-multicast-flood off
  1159. auto vmbr2
  1160. iface vmbr2 inet static
  1161. bridge_ports vxlan2
  1162. bridge_stp off
  1163. bridge_fd 0
  1164. address 10.0.2.254
  1165. netmask 255.255.255.0
  1166. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2
  1167. vrf vrf1
  1168. ip-forward on
  1169. ip6-forward on
  1170. arp-accept on
  1171. auto vxlan3
  1172. iface vxlan3 inet manual
  1173. vxlan-id 3
  1174. vxlan-local-tunnelip 192.168.0.3
  1175. bridge-learning off
  1176. bridge-arp-nd-suppress on
  1177. bridge-unicast-flood off
  1178. bridge-multicast-flood off
  1179. auto vmbr3
  1180. iface vmbr3 inet static
  1181. bridge_ports vxlan3
  1182. bridge_stp off
  1183. bridge_fd 0
  1184. address 10.0.3.254
  1185. netmask 255.255.255.0
  1186. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3
  1187. vrf vrf1
  1188. ip-forward on
  1189. ip6-forward on
  1190. arp-accept on
  1191. #interconnect vxlan-vfr l3vni
  1192. auto vxlan4000
  1193. iface vxlan4000 inet manual
  1194. vxlan-id 4000
  1195. vxlan-local-tunnelip 192.168.0.3
  1196. bridge-learning off
  1197. bridge-arp-nd-suppress on
  1198. bridge-unicast-flood off
  1199. bridge-multicast-flood off
  1200. auto vmbr4000
  1201. iface vmbr4000 inet manual
  1202. bridge_ports vxlan4000
  1203. bridge_stp off
  1204. bridge_fd 0
  1205. vrf vrf1
  1206. ----
  1207. frr.conf
  1208. ----
  1209. vrf vrf1
  1210. vni 4000
  1211. exit-vrf
  1212. !
  1213. router bgp 1234
  1214. bgp router-id 192.168.0.3
  1215. no bgp default ipv4-unicast
  1216. coalesce-time 1000
  1217. neighbor 192.168.0.1 remote-as 1234
  1218. neighbor 192.168.0.2 remote-as 1234
  1219. !
  1220. address-family l2vpn evpn
  1221. neighbor 192.168.0.1 activate
  1222. neighbor 192.168.0.2 activate
  1223. advertise-all-vni
  1224. exit-address-family
  1225. !
  1226. line vty
  1227. !
  1228. ----
  1229. multiple gateway nodes
  1230. ^^^^^^^^^^^^^^^^^^^^^^
  1231. In this example, all nodes will be used as exit gateway. (But you can use only 2 nodes if you want)
  1232. All nodes have a a default gw to the external router (192.168.0.254) (no bgp between router and node1)
  1233. and announce this default gw in the vrf (default originate)
  1234. The external router have ecmp routes to all proxmox nodes.(balancing).
  1235. If the router send the packet to a wrong node (vm is not on this node), this node will route through
  1236. vxlan the packet to final destination.
  1237. If you have multiple gateway nodes, disable rp_filter as packet could incoming in a 1 node, and outgoing
  1238. to another node.
  1239. sysctl.conf tuning
  1240. -----
  1241. net.ipv4.conf.default.rp_filter=0
  1242. net.ipv4.conf.all.rp_filter=0
  1243. -----
  1244. *node1
  1245. ----
  1246. auto vrf1
  1247. iface vrf1
  1248. vrf-table auto
  1249. auto eno1
  1250. iface eno1 inet manual
  1251. mtu 1550
  1252. auto vmbr0
  1253. iface vmbr0 inet static
  1254. address 192.168.0.1
  1255. netmask 255.255.255.0
  1256. gateway 192.168.0.254
  1257. bridge_ports eno1
  1258. bridge_stp off
  1259. bridge_fd 0
  1260. ip-forward on
  1261. ip6-forward on
  1262. auto vxlan2
  1263. iface vxlan2 inet manual
  1264. vxlan-id 2
  1265. vxlan-local-tunnelip 192.168.0.1
  1266. bridge-learning off
  1267. bridge-arp-nd-suppress on
  1268. bridge-unicast-flood off
  1269. bridge-multicast-flood off
  1270. auto vmbr2
  1271. iface vmbr2 inet static
  1272. bridge_ports vxlan2
  1273. bridge_stp off
  1274. bridge_fd 0
  1275. address 10.0.2.254
  1276. netmask 255.255.255.0
  1277. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2
  1278. vrf vrf1
  1279. ip-forward on
  1280. ip6-forward on
  1281. arp-accept on
  1282. auto vxlan3
  1283. iface vxlan3 inet manual
  1284. vxlan-id 3
  1285. vxlan-local-tunnelip 192.168.0.1
  1286. bridge-learning off
  1287. bridge-arp-nd-suppress on
  1288. bridge-unicast-flood off
  1289. bridge-multicast-flood off
  1290. auto vmbr3
  1291. iface vmbr3 inet static
  1292. bridge_ports vxlan3
  1293. bridge_stp off
  1294. bridge_fd 0
  1295. address 10.0.3.254
  1296. netmask 255.255.255.0
  1297. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3
  1298. vrf vrf1
  1299. ip-forward on
  1300. ip6-forward on
  1301. arp-accept on
  1302. #interconnect vxlan-vfr l3vni
  1303. auto vxlan4000
  1304. iface vxlan4000 inet manual
  1305. vxlan-id 4000
  1306. vxlan-local-tunnelip 192.168.0.1
  1307. bridge-learning off
  1308. bridge-arp-nd-suppress on
  1309. bridge-unicast-flood off
  1310. bridge-multicast-flood off
  1311. auto vmbr4000
  1312. iface vmbr4000 inet manual
  1313. bridge_ports vxlan4000
  1314. bridge_stp off
  1315. bridge_fd 0
  1316. vrf vrf1
  1317. ----
  1318. frr.conf
  1319. ----
  1320. vrf vrf1
  1321. vni 4000
  1322. exit-vrf
  1323. !
  1324. router bgp 1234
  1325. bgp router-id 192.168.0.1
  1326. no bgp default ipv4-unicast
  1327. coalesce-time 1000
  1328. neighbor 192.168.0.2 remote-as 1234
  1329. neighbor 192.168.0.3 remote-as 1234
  1330. !
  1331. address-family ipv4 unicast
  1332. import vrf vrf1
  1333. exit-address-family
  1334. !
  1335. address-family ipv6 unicast
  1336. import vrf vrf1
  1337. exit-address-family
  1338. !
  1339. address-family l2vpn evpn
  1340. neighbor 192.168.0.2 activate
  1341. neighbor 192.168.0.3 activate
  1342. advertise-all-vni
  1343. exit-address-family
  1344. !
  1345. router bgp 1234 vrf vrf1
  1346. !
  1347. address-family ipv4 unicast
  1348. redistribute connected
  1349. exit-address-family
  1350. !
  1351. address-family ipv6 unicast
  1352. redistribute connected
  1353. exit-address-family
  1354. !
  1355. address-family l2vpn evpn
  1356. default-originate ipv4
  1357. default-originate ipv6
  1358. exit-address-family
  1359. !
  1360. line vty
  1361. !
  1362. ----
  1363. * node2
  1364. ----
  1365. auto vrf1
  1366. iface vrf1
  1367. vrf-table auto
  1368. auto eno1
  1369. iface eno1 inet manual
  1370. mtu 1550
  1371. auto vmbr0
  1372. iface vmbr0 inet static
  1373. address 192.168.0.2
  1374. netmask 255.255.255.0
  1375. gateway 192.168.0.254
  1376. bridge_ports eno1
  1377. bridge_stp off
  1378. bridge_fd 0
  1379. ip-forward on
  1380. ip6-forward on
  1381. auto vxlan2
  1382. iface vxlan2 inet manual
  1383. vxlan-id 2
  1384. vxlan-local-tunnelip 192.168.0.2
  1385. bridge-learning off
  1386. bridge-arp-nd-suppress on
  1387. bridge-unicast-flood off
  1388. bridge-multicast-flood off
  1389. auto vmbr2
  1390. iface vmbr2 inet static
  1391. bridge_ports vxlan2
  1392. bridge_stp off
  1393. bridge_fd 0
  1394. address 10.0.2.254
  1395. netmask 255.255.255.0
  1396. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2
  1397. vrf vrf1
  1398. ip-forward on
  1399. ip6-forward on
  1400. arp-accept on
  1401. auto vxlan3
  1402. iface vxlan3 inet manual
  1403. vxlan-id 3
  1404. vxlan-local-tunnelip 192.168.0.2
  1405. bridge-learning off
  1406. bridge-arp-nd-suppress on
  1407. bridge-unicast-flood off
  1408. bridge-multicast-flood off
  1409. auto vmbr3
  1410. iface vmbr3 inet static
  1411. bridge_ports vxlan3
  1412. bridge_stp off
  1413. bridge_fd 0
  1414. address 10.0.3.254
  1415. netmask 255.255.255.0
  1416. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3
  1417. vrf vrf1
  1418. ip-forward on
  1419. ip6-forward on
  1420. arp-accept on
  1421. #interconnect vxlan-vfr l3vni
  1422. auto vxlan4000
  1423. iface vxlan4000 inet manual
  1424. vxlan-id 4000
  1425. vxlan-local-tunnelip 192.168.0.2
  1426. bridge-learning off
  1427. bridge-arp-nd-suppress on
  1428. bridge-unicast-flood off
  1429. bridge-multicast-flood off
  1430. auto vmbr4000
  1431. iface vmbr4000 inet manual
  1432. bridge_ports vxlan4000
  1433. bridge_stp off
  1434. bridge_fd 0
  1435. vrf vrf1
  1436. ----
  1437. frr.conf
  1438. ----
  1439. vrf vrf1
  1440. vni 4000
  1441. exit-vrf
  1442. !
  1443. router bgp 1234
  1444. bgp router-id 192.168.0.2
  1445. no bgp default ipv4-unicast
  1446. coalesce-time 1000
  1447. neighbor 192.168.0.1 remote-as 1234
  1448. neighbor 192.168.0.3 remote-as 1234
  1449. !
  1450. address-family ipv4 unicast
  1451. import vrf vrf1
  1452. exit-address-family
  1453. !
  1454. address-family ipv6 unicast
  1455. import vrf vrf1
  1456. exit-address-family
  1457. !
  1458. address-family l2vpn evpn
  1459. neighbor 192.168.0.1 activate
  1460. neighbor 192.168.0.3 activate
  1461. advertise-all-vni
  1462. exit-address-family
  1463. !
  1464. address-family ipv4 unicast
  1465. redistribute connected
  1466. exit-address-family
  1467. !
  1468. address-family ipv6 unicast
  1469. redistribute connected
  1470. exit-address-family
  1471. !
  1472. address-family l2vpn evpn
  1473. default-originate ipv4
  1474. default-originate ipv6
  1475. exit-address-family
  1476. !
  1477. line vty
  1478. !
  1479. ----
  1480. * node3
  1481. ----
  1482. auto vrf1
  1483. iface vrf1
  1484. vrf-table auto
  1485. auto eno1
  1486. iface eno1 inet manual
  1487. mtu 1550
  1488. auto vmbr0
  1489. iface vmbr0 inet static
  1490. address 192.168.0.3
  1491. netmask 255.255.255.0
  1492. gateway 192.168.0.254
  1493. bridge_ports eno1
  1494. bridge_stp off
  1495. bridge_fd 0
  1496. ip-forward on
  1497. ip6-forward on
  1498. auto vxlan2
  1499. iface vxlan2 inet manual
  1500. vxlan-id 2
  1501. vxlan-local-tunnelip 192.168.0.3
  1502. bridge-learning off
  1503. bridge-arp-nd-suppress on
  1504. bridge-unicast-flood off
  1505. bridge-multicast-flood off
  1506. auto vmbr2
  1507. iface vmbr2 inet static
  1508. bridge_ports vxlan2
  1509. bridge_stp off
  1510. bridge_fd 0
  1511. address 10.0.2.254
  1512. netmask 255.255.255.0
  1513. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2
  1514. vrf vrf1
  1515. ip-forward on
  1516. ip6-forward on
  1517. arp-accept on
  1518. auto vxlan3
  1519. iface vxlan3 inet manual
  1520. vxlan-id 3
  1521. vxlan-local-tunnelip 192.168.0.3
  1522. bridge-learning off
  1523. bridge-arp-nd-suppress on
  1524. bridge-unicast-flood off
  1525. bridge-multicast-flood off
  1526. auto vmbr3
  1527. iface vmbr3 inet static
  1528. bridge_ports vxlan3
  1529. bridge_stp off
  1530. bridge_fd 0
  1531. address 10.0.3.254
  1532. netmask 255.255.255.0
  1533. hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3
  1534. vrf vrf1
  1535. ip-forward on
  1536. ip6-forward on
  1537. arp-accept on
  1538. #interconnect vxlan-vfr l3vni
  1539. auto vxlan4000
  1540. iface vxlan4000 inet manual
  1541. vxlan-id 4000
  1542. vxlan-local-tunnelip 192.168.0.3
  1543. bridge-learning off
  1544. bridge-arp-nd-suppress on
  1545. bridge-unicast-flood off
  1546. bridge-multicast-flood off
  1547. auto vmbr4000
  1548. iface vmbr4000 inet manual
  1549. bridge_ports vxlan4000
  1550. bridge_stp off
  1551. bridge_fd 0
  1552. vrf vrf1
  1553. ----
  1554. frr.conf
  1555. ----
  1556. vrf vrf1
  1557. vni 4000
  1558. exit-vrf
  1559. !
  1560. router bgp 1234
  1561. bgp router-id 192.168.0.3
  1562. no bgp default ipv4-unicast
  1563. coalesce-time 1000
  1564. neighbor 192.168.0.1 remote-as 1234
  1565. neighbor 192.168.0.2 remote-as 1234
  1566. !
  1567. address-family ipv4 unicast
  1568. import vrf vrf1
  1569. exit-address-family
  1570. !
  1571. address-family ipv6 unicast
  1572. import vrf vrf1
  1573. exit-address-family
  1574. !
  1575. address-family l2vpn evpn
  1576. neighbor 192.168.0.1 activate
  1577. neighbor 192.168.0.2 activate
  1578. advertise-all-vni
  1579. exit-address-family
  1580. !
  1581. router bgp 1234 vrf vrf1
  1582. !
  1583. address-family ipv4 unicast
  1584. redistribute connected
  1585. exit-address-family
  1586. !
  1587. address-family ipv6 unicast
  1588. redistribute connected
  1589. exit-address-family
  1590. !
  1591. address-family l2vpn evpn
  1592. default-originate ipv4
  1593. default-originate ipv6
  1594. exit-address-family
  1595. !
  1596. line vty
  1597. !
  1598. ----
  1599. Note
  1600. ^^^^
  1601. If your external router doesn't support 'ECMP static routes' to reach multiple
  1602. {pve} nodes, you can setup an HA floating vip on proxmox nodes by using the
  1603. Virtual Router Redundancy Protocol (VRRP).
  1604. In this example, we will setup an floating 192.168.0.10 IP on node1 and node2.
  1605. Node1 is the primary with failover to node2 in case of outage.
  1606. This setup currently needs 'vrrpd' package (`apt install vrrpd`).
  1607. #TODO : It should be possible to do it with frr directly with last version.
  1608. * node1
  1609. ----
  1610. auto vmbr0
  1611. iface vmbr0 inet static
  1612. address 192.168.0.1
  1613. netmask 255.255.255.0
  1614. gateway 192.168.0.254
  1615. bridge_ports eno1
  1616. bridge_stp off
  1617. bridge_fd 0
  1618. vrrp-id 1
  1619. vrrp-priority 1
  1620. vrrp-virtual-ip 192.168.0.10
  1621. ----
  1622. * node2
  1623. ----
  1624. auto vmbr0
  1625. iface vmbr0 inet static
  1626. address 192.168.0.2
  1627. netmask 255.255.255.0
  1628. gateway 192.168.0.254
  1629. bridge_ports eno1
  1630. bridge_stp off
  1631. bridge_fd 0
  1632. vrrp-id 1
  1633. vrrp-priority 2
  1634. vrrp-virtual-ip 192.168.0.10
  1635. ----
  1636. gateway node(s) with a upstream bgp router
  1637. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  1638. Setup is almost the same than with a static gateway, but we'll connect to an upstream bgp router.
  1639. example with node1 as gateway (192.168.0.1) for evpn-bgp, and an upstream bgp router (running frr too) 192.168.0.254.
  1640. * node1
  1641. frr.conf
  1642. ----
  1643. vrf vrf1
  1644. vni 4000
  1645. exit-vrf
  1646. !
  1647. router bgp 1234
  1648. bgp router-id 192.168.0.1
  1649. no bgp default ipv4-unicast
  1650. coalesce-time 1000
  1651. neighbor 192.168.0.2 remote-as 1234
  1652. neighbor 192.168.0.3 remote-as 1234
  1653. neighbor 192.168.0.254 remote-as external
  1654. !
  1655. address-family ipv4 unicast
  1656. import vrf vrf1
  1657. neighbor 192.168.0.254 activate
  1658. exit-address-family
  1659. !
  1660. address-family ipv6 unicast
  1661. import vrf vrf1
  1662. neighbor 192.168.0.254 activate
  1663. exit-address-family
  1664. !
  1665. address-family l2vpn evpn
  1666. neighbor 192.168.0.1 activate
  1667. neighbor 192.168.0.2 activate
  1668. neighbor 192.168.0.254 activate
  1669. advertise-all-vni
  1670. exit-address-family
  1671. !
  1672. router bgp 1234 vrf vrf1
  1673. !
  1674. address-family ipv4 unicast
  1675. redistribute connected
  1676. exit-address-family
  1677. !
  1678. address-family ipv6 unicast
  1679. redistribute connected
  1680. exit-address-family
  1681. !
  1682. address-family l2vpn evpn
  1683. default-originate ipv4
  1684. default-originate ipv6
  1685. exit-address-family
  1686. !
  1687. line vty
  1688. !
  1689. ----
  1690. * bgp router
  1691. frr.conf
  1692. ----
  1693. ip prefix-list NO32 seq 10 permit 0.0.0.0/0 ge 8 le 24
  1694. ip prefix-list NO32 seq 20 deny any
  1695. !
  1696. router bgp 25253
  1697. bgp router-id 192.168.0.254
  1698. bgp bestpath as-path multipath-relax
  1699. neighbor 192.168.0.1 remote-as external
  1700. neighbor 192.168.0.1 capability extended-nexthop
  1701. !
  1702. address-family ipv4 unicast
  1703. neighbor 192.168.0.1 default-originate
  1704. neighbor 192.168.0.1 prefix-list NO32 in #don't import /32 route from evpn
  1705. exit-address-family
  1706. !
  1707. address-family ipv6 unicast
  1708. neighbor 192.168.0.1 default-originate
  1709. neighbor 192.168.0.1 prefix-list NO32 in #don't import /32 route from evpn
  1710. exit-address-family
  1711. !
  1712. !
  1713. ---
  1714. Route Reflectors
  1715. ^^^^^^^^^^^^^^^^
  1716. If you have a lot of proxmox nodes, or multiple proxmox clusters, you may want
  1717. to avoid that all node peers with each others nodes.
  1718. For this, you can create dedicated route reflectors (RR) servers. As a RR is a
  1719. single point of failure, a minimum of two servers acting as an RR is highly
  1720. recommended for redundancy.
  1721. Below is an example of configuration with 'frr', with `rrserver1
  1722. (192.168.0.200)' and `rrserver2 (192.168.0.201)`.
  1723. rrserver1
  1724. ----
  1725. router bgp 1234
  1726. bgp router-id 192.168.0.200
  1727. bgp cluster-id 1.1.1.1 #cluster-id must be the same on each route reflector
  1728. bgp log-neighbor-changes
  1729. no bgp default ipv4-unicast
  1730. neighbor fabric peer-group
  1731. neighbor fabric remote-as 1234
  1732. neighbor fabric capability extended-nexthop
  1733. neighbor fabric update-source 192.168.0.200
  1734. bgp listen range 192.168.0.0/24 peer-group fabric #allow any proxmoxnode client in the network range
  1735. !
  1736. address-family l2vpn evpn
  1737. neighbor fabric activate
  1738. neighbor fabric route-reflector-client
  1739. neighbor fabric allowas-in
  1740. exit-address-family
  1741. !
  1742. exit
  1743. !
  1744. ---
  1745. rrserver2
  1746. ----
  1747. router bgp 1234
  1748. bgp router-id 192.168.0.201
  1749. bgp cluster-id 1.1.1.1
  1750. bgp log-neighbor-changes
  1751. no bgp default ipv4-unicast
  1752. neighbor fabric peer-group
  1753. neighbor fabric remote-as 1234
  1754. neighbor fabric capability extended-nexthop
  1755. neighbor fabric update-source 192.168.0.201
  1756. bgp listen range 192.168.0.0/24 peer-group fabric
  1757. !
  1758. address-family l2vpn evpn
  1759. neighbor fabric activate
  1760. neighbor fabric route-reflector-client
  1761. neighbor fabric allowas-in
  1762. exit-address-family
  1763. !
  1764. exit
  1765. !
  1766. ---
  1767. proxmoxnode(s)
  1768. ----
  1769. router bgp 1234
  1770. bgp router-id 192.168.0.x
  1771. no bgp default ipv4-unicast
  1772. coalesce-time 1000
  1773. neighbor 192.168.0.200 remote-as 1234
  1774. neighbor 192.168.0.201 remote-as 1234
  1775. !
  1776. address-family l2vpn evpn
  1777. neighbor 192.168.0.200 activate
  1778. neighbor 192.168.0.201 activate
  1779. advertise-all-vni
  1780. exit-address-family
  1781. !
  1782. ----