From 25c6e57ff2c1c9eb04d14e98ce24e0cfe856c0e5 Mon Sep 17 00:00:00 2001 From: Kaihang Zhang Date: Thu, 12 Jan 2023 17:18:16 +0800 Subject: [PATCH] feat: Add supports for dual-stack --- build/virt-prerunner/Dockerfile | 8 +- build/virt-prerunner/ip6tables-wrapper | 17 + cmd/route-advertisement-daemon/main.go | 305 ++++++++++++++++++ cmd/virt-prerunner/dhcpv6.json.conf | 56 ++++ cmd/virt-prerunner/main.go | 288 +++++++++++++---- ...rt.virtink.smartx.com_virtualmachines.yaml | 8 +- go.mod | 9 +- go.sum | 12 +- pkg/apis/virt/v1alpha1/types.go | 5 +- pkg/controller/vm_controller.go | 22 +- pkg/controller/vm_webhook.go | 10 +- pkg/controller/vm_webhook_test.go | 13 +- pkg/ipv6util/ipv6util.go | 16 + 13 files changed, 674 insertions(+), 95 deletions(-) create mode 100755 build/virt-prerunner/ip6tables-wrapper create mode 100644 cmd/route-advertisement-daemon/main.go create mode 100644 cmd/virt-prerunner/dhcpv6.json.conf create mode 100644 pkg/ipv6util/ipv6util.go diff --git a/build/virt-prerunner/Dockerfile b/build/virt-prerunner/Dockerfile index ad6dbb5..058650a 100644 --- a/build/virt-prerunner/Dockerfile +++ b/build/virt-prerunner/Dockerfile @@ -11,10 +11,11 @@ RUN go mod download COPY cmd/ cmd/ COPY pkg/ pkg/ RUN --mount=type=cache,target=/root/.cache/go-build go build -a cmd/virt-prerunner/main.go +RUN --mount=type=cache,target=/root/.cache/go-build go build -o rad -a cmd/route-advertisement-daemon/main.go -FROM alpine +FROM alpine:3.17 -RUN apk add --no-cache curl screen dnsmasq cdrkit iptables iproute2 qemu-virtiofsd dpkg util-linux s6-overlay nmap-ncat +RUN apk add --no-cache curl screen dnsmasq kea-dhcp6 cdrkit iptables ip6tables iproute2 qemu-virtiofsd dpkg util-linux s6-overlay nmap-ncat RUN set -eux; \ mkdir /var/lib/cloud-hypervisor; \ @@ -40,6 +41,7 @@ COPY build/virt-prerunner/cloud-hypervisor-finish.sh /etc/s6-overlay/s6-rc.d/clo RUN touch /etc/s6-overlay/s6-rc.d/user/contents.d/cloud-hypervisor COPY --from=builder /workspace/main /usr/bin/virt-prerunner +COPY --from=builder /workspace/rad /usr/bin/rad COPY build/virt-prerunner/virt-prerunner-type /etc/s6-overlay/s6-rc.d/virt-prerunner/type COPY build/virt-prerunner/virt-prerunner-up /etc/s6-overlay/s6-rc.d/virt-prerunner/up COPY build/virt-prerunner/virt-prerunner-run.sh /etc/s6-overlay/scripts/virt-prerunner-run.sh @@ -50,5 +52,7 @@ ENTRYPOINT ["/init"] COPY build/virt-prerunner/iptables-wrapper /sbin/iptables-wrapper RUN update-alternatives --install /sbin/iptables iptables /sbin/iptables-wrapper 100 +COPY build/virt-prerunner/ip6tables-wrapper /sbin/ip6tables-wrapper +RUN update-alternatives --install /sbin/ip6tables ip6tables /sbin/ip6tables-wrapper 100 ADD build/virt-prerunner/virt-init-volume.sh /usr/bin/virt-init-volume diff --git a/build/virt-prerunner/ip6tables-wrapper b/build/virt-prerunner/ip6tables-wrapper new file mode 100755 index 0000000..89d9cf3 --- /dev/null +++ b/build/virt-prerunner/ip6tables-wrapper @@ -0,0 +1,17 @@ +#!/bin/sh + +set +e + +ip6tables-legacy -nvL + +if [ $? -eq 0 ] +then + mode=legacy +else + mode=nft +fi + +update-alternatives --install /sbin/ip6tables ip6tables "/sbin/ip6tables-${mode}" 100 +update-alternatives --set ip6tables "/sbin/ip6tables-${mode}" > /dev/null + +exec "$0" "$@" diff --git a/cmd/route-advertisement-daemon/main.go b/cmd/route-advertisement-daemon/main.go new file mode 100644 index 0000000..6df1018 --- /dev/null +++ b/cmd/route-advertisement-daemon/main.go @@ -0,0 +1,305 @@ +package main + +import ( + "flag" + "fmt" + "log" + "net" + "net/netip" + "os/exec" + "time" + + "github.com/mdlayher/ndp" + "golang.org/x/net/ipv6" + + "github.com/smartxworks/virtink/pkg/ipv6util" +) + +var ( + iface string + router string + isRemoteRoute bool + client string + clientHWAddr string + prefix string + + linkLocalAllRouters = netip.MustParseAddr("ff02::2") +) + +func main() { + log.SetPrefix("route-advertisement-daemon: ") + src, dst, cidr, err := validateVars() + if err != nil { + log.Fatalf("ERROR: validate vars: %s", err) + } + if src != nil && isRemoteRoute { + if _, err := executeCommand("ip", "-6", "neigh", "add", client, "lladdr", clientHWAddr, "dev", iface); err != nil { + log.Fatalf("ERROR: add neighbor entry for client: %s", err) + } + + ipv6LLA := src + if !src.IsLinkLocalUnicast() { + mac, err := tryDiscoverNeighborMAC(iface, src, 5) + if err != nil { + log.Fatalf("ERROR: discover router MAC: %s", err) + } + ipv6LLA = ipv6util.GenerateEUI64Address(net.ParseIP("fe80::0"), mac) + mac2, err := tryDiscoverNeighborMAC(iface, ipv6LLA, 5) + if err != nil { + log.Fatalf("ERROR: discover router MAC: %s", err) + } + if mac.String() != mac2.String() { + log.Fatalf("ERROR: failed to get router link-local address") + } + } + + if _, err := executeCommand("ip6tables", "-A", "OUTPUT", "-o", iface, "--src", ipv6LLA.String(), "-p", "icmpv6", "--icmpv6-type", "neighbor-solicitation", "-j", "DROP"); err != nil { + log.Fatalf("ERROR: drop neighbor solicitation on interface: %s", err) + } + if _, err := executeCommand("ip6tables", "-A", "OUTPUT", "-o", iface, "--src", ipv6LLA.String(), "-p", "icmpv6", "--icmpv6-type", "neighbor-advertisement", "-j", "DROP"); err != nil { + log.Fatalf("ERROR: drop neighbor advertisement on interface: %s", err) + } + + // As described in RFC 4861 section-4.2, the srouce address of RA must be the link-local + // address assigned to the interface from which the message is sent, so the LLA of default + // router have to be added to the interface. And the followings need to be done. + // 1.Disable DAD of the interface + // 2.Add static neighbor entry for the client, otherwise the interface will send a NS + // message with it's MAC in options to client + // 3.Drop NA message from interface responsed to NS message learning default router LLA + // 4.Drop NS message from interface with default router LLA in options + if executeCommand("ip", "addr", "add", fmt.Sprintf("%s/64", ipv6LLA.String()), "dev", iface); err != nil { + log.Fatalf("ERROR: add IPv6 addr to the interface: %s", err) + } + + src = ipv6LLA + } + + if err := startRouteAdvertisement(iface, src, dst, cidr); err != nil { + log.Fatalf("ERROR: start route advertisement: %s", err) + } +} + +func validateVars() (net.IP, net.IP, *net.IPNet, error) { + if iface == "" { + return nil, nil, nil, fmt.Errorf("the interface may not be empty") + } + + var src net.IP + if router != "" { + src = net.ParseIP(router) + if src == nil { + return nil, nil, nil, fmt.Errorf("the router IPv6 address (%s) is illegal", router) + } + if isRemoteRoute { + if clientHWAddr == "" { + return nil, nil, nil, fmt.Errorf("the client-hardware-addr may not be empty when router is remote") + } + } + } + + if client == "" { + if clientHWAddr == "" { + return nil, nil, nil, fmt.Errorf("the client and client-hardware-addr may not both be empty") + } + clientMAC, err := net.ParseMAC(clientHWAddr) + if err != nil { + return nil, nil, nil, fmt.Errorf("parse MAC: %s", err) + } + client = ipv6util.GenerateEUI64Address(net.ParseIP("fe80::0"), clientMAC).String() + } + dst := net.ParseIP(client) + if dst == nil { + return nil, nil, nil, fmt.Errorf("the client IPv6 address (%s) is illegal", client) + } + if !dst.IsLinkLocalUnicast() { + return nil, nil, nil, fmt.Errorf("the client IPv6 address should be a link-local address") + } + + if prefix == "" { + return nil, nil, nil, fmt.Errorf("the prefix may not be empty") + } + _, cidr, err := net.ParseCIDR(prefix) + if err != nil { + return nil, nil, nil, fmt.Errorf("the prefix (%s) is illegal", prefix) + } + + return src, dst, cidr, nil +} + +func tryDiscoverNeighborMAC(ifaceName string, ip net.IP, retry int) (net.HardwareAddr, error) { + for i := retry; i > 0; i-- { + mac, err := discoverNeighborMAC(ifaceName, ip) + if err != nil { + return nil, err + } + if mac == nil { + log.Println("INFO: retry in 5s") + continue + } + return mac, nil + } + + return nil, fmt.Errorf("failed to discover neighbor MAC. Try %d times", retry) +} + +func discoverNeighborMAC(ifaceName string, ip net.IP) (net.HardwareAddr, error) { + iface, err := net.InterfaceByName(ifaceName) + if err != nil { + return nil, fmt.Errorf("get interface by name: %s", err) + } + conn, err := tryCreateNDPConn(iface, 5) + if err != nil { + return nil, fmt.Errorf("create NDP connection: %s", err) + } + defer conn.Close() + + target := netip.MustParseAddr(ip.String()) + solicitationAddr, err := ndp.SolicitedNodeMulticast(target) + if err != nil { + return nil, fmt.Errorf("determine solicited-node multicast address: %s", err) + } + solicitation := &ndp.NeighborSolicitation{ + TargetAddress: target, + Options: []ndp.Option{ + &ndp.LinkLayerAddress{ + Direction: ndp.Source, + Addr: iface.HardwareAddr, + }, + }, + } + if err := conn.WriteTo(solicitation, nil, solicitationAddr); err != nil { + return nil, fmt.Errorf("write neighbor solicitation: %s", err) + } + + var f ipv6.ICMPFilter + f.SetAll(true) + f.Accept(ipv6.ICMPTypeNeighborAdvertisement) + if err := conn.SetICMPFilter(&f); err != nil { + return nil, fmt.Errorf("set ICMPv6 filter: %s", err) + } + + msg, _, from, err := conn.ReadFrom() + if err != nil { + return nil, fmt.Errorf("read NDP message: %s", err) + } + if target.WithZone(ifaceName).Compare(from) != 0 && target.Compare(from) != 0 { + log.Println("INFO: the NDP message is not from solicitation target") + return nil, nil + } + advertisement := msg.(*ndp.NeighborAdvertisement) + if len(advertisement.Options) != 1 { + return nil, fmt.Errorf("get %d option(s) in neighbor advertisement, but expect one", len(advertisement.Options)) + } + linkLayerAddr, ok := advertisement.Options[0].(*ndp.LinkLayerAddress) + if !ok { + return nil, fmt.Errorf("advertisement option is not a link-layer address") + } + return linkLayerAddr.Addr, nil +} + +func tryCreateNDPConn(iface *net.Interface, retry int) (*ndp.Conn, error) { + var err error + for i := retry; i > 0; i-- { + conn, _, err := ndp.Listen(iface, ndp.LinkLocal) + if err != nil { + // caused by tap device state down? + log.Printf("Warnning: listen interface link-local address: %s. Retry in 5s\n", err) + time.Sleep(5 * time.Second) + } + if err == nil { + return conn, nil + } + } + + return nil, fmt.Errorf("listen interface link-local address: %s. Retry %d times", err, retry) +} + +func startRouteAdvertisement(ifaceName string, src net.IP, dst net.IP, cidr *net.IPNet) error { + iface, err := net.InterfaceByName(ifaceName) + if err != nil { + return fmt.Errorf("get interface by name: %s", err) + } + conn, err := tryCreateNDPConn(iface, 5) + if err != nil { + return fmt.Errorf("create NDP connection: %s", err) + } + defer conn.Close() + + var filter ipv6.ICMPFilter + filter.SetAll(true) + filter.Accept(ipv6.ICMPTypeRouterSolicitation) + if err := conn.SetICMPFilter(&filter); err != nil { + return fmt.Errorf("apply ICMPv6 filter: %s", err) + } + if err := conn.JoinGroup(linkLocalAllRouters); err != nil { + return fmt.Errorf("join IPv6 link-local all routers multicast group: %s", err) + } + + prefixLen, _ := cidr.Mask.Size() + advertisement := &ndp.RouterAdvertisement{ + CurrentHopLimit: 255, + RouterLifetime: 65535 * time.Second, + ManagedConfiguration: true, + OtherConfiguration: true, + Options: []ndp.Option{ + &ndp.PrefixInformation{ + PrefixLength: uint8(prefixLen), + Prefix: netip.MustParseAddr(cidr.IP.String()), + OnLink: true, + ValidLifetime: 4294967295 * time.Second, + }, + }, + } + + controlMsg := &ipv6.ControlMessage{ + HopLimit: 255, + Src: src, + } + + if src == nil { + advertisement.RouterLifetime = 0 + controlMsg = nil + } + + for { + _, _, from, err := conn.ReadFrom() + if err != nil { + log.Printf("Warnning: read NDP message: %s. Retry in 5s\n", err) + time.Sleep(5 * time.Second) + continue + } + target := netip.MustParseAddr(dst.String()) + if target.WithZone(ifaceName).Compare(from) != 0 && target.Compare(from) != 0 { + continue + } + + if err := conn.WriteTo(advertisement, controlMsg, netip.MustParseAddr(dst.String())); err != nil { + return fmt.Errorf("send route advertisement: %s", err) + } + log.Printf("INFO: reply RS from %s\n", dst.String()) + } +} + +func executeCommand(name string, arg ...string) (string, error) { + cmd := exec.Command(name, arg...) + output, err := cmd.CombinedOutput() + if err != nil { + return string(output), fmt.Errorf("%q: %s: %s", cmd.String(), err, output) + } + return string(output), nil +} + +func init() { + flag.StringVar(&iface, "interface", "", "The interface to listen to.") + flag.StringVar(&router, "router", "", "The IPv6 address of the default router. "+ + "It's recommanded to use the link-local address of the router, "+ + "otherwise the SLAAC link-local address formed by router hardware address will be used.") + flag.BoolVar(&isRemoteRoute, "is-remote-route", false, "") + flag.StringVar(&client, "client", "", "The IPv6 link-local address of the client to advertise to. "+ + "The SLAAC link-local address formed by client hardware address will be used when empty.") + flag.StringVar(&clientHWAddr, "client-hardware-addr", "", "The hardware address of the client.") + flag.StringVar(&prefix, "prefix", "", "The prefix of the subnet.") + + flag.Parse() +} diff --git a/cmd/virt-prerunner/dhcpv6.json.conf b/cmd/virt-prerunner/dhcpv6.json.conf new file mode 100644 index 0000000..f6a4cdd --- /dev/null +++ b/cmd/virt-prerunner/dhcpv6.json.conf @@ -0,0 +1,56 @@ +{ +"Dhcp6": { + "valid-lifetime": 4294967295, + "interfaces-config": { + "interfaces": [ "{{ .iface }}" ], + "service-sockets-max-retries": 10 + }, + "lease-database": { + "type": "memfile", + "persist": true, + "name": "/var/run/virtink/kea/{{ .iface }}/dhcpv6/dhcp6.leases" + }, + "subnet6": [{ + "subnet": "{{ .prefix }}", + "pools": [{ "pool": "{{ .ip }}/128" }], + "interface": "{{ .iface }}" + }], + "loggers": [{ + "name": "kea-dhcp6", + "output_options": [{ + "output": "/var/run/virtink/kea/{{ .iface }}/dhcpv6/kea-dhcp6.log" + }], + "severity": "INFO" +{{- if and .dnsServer .domainSearch }} + }], + "option-data": [ + { + "name": "dns-servers", + "data": "{{ .dnsServer }}" + }, + { + "name": "domain-search", + "data": "{{ .domainSearch }}" + } + ] +{{- else if .dnsServer }} + }], + "option-data": [ + { + "name": "dns-servers", + "data": "{{ .dnsServer }}" + } + ] +{{- else if .domainSearch }} + }], + "option-data": [ + { + "name": "domain-search", + "data": "{{ .domainSearch }}" + } + ] +{{- else}} + }] +{{- end}} +} +} diff --git a/cmd/virt-prerunner/main.go b/cmd/virt-prerunner/main.go index 18c41d1..a2ebfb2 100644 --- a/cmd/virt-prerunner/main.go +++ b/cmd/virt-prerunner/main.go @@ -21,10 +21,12 @@ import ( "github.com/namsral/flag" "github.com/subgraph/libmacouflage" "github.com/vishvananda/netlink" + netutils "k8s.io/utils/net" virtv1alpha1 "github.com/smartxworks/virtink/pkg/apis/virt/v1alpha1" "github.com/smartxworks/virtink/pkg/cloudhypervisor" "github.com/smartxworks/virtink/pkg/cpuset" + "github.com/smartxworks/virtink/pkg/ipv6util" ) func main() { @@ -237,7 +239,7 @@ func buildVMConfig(ctx context.Context, vm *virtv1alpha1.VirtualMachine) (*cloud Id: iface.Name, Mac: iface.MAC, } - if err := setupMasqueradeNetwork(linkName, iface.Masquerade.CIDR, &netConfig); err != nil { + if err := setupMasqueradeNetwork(linkName, iface.Masquerade.IPv4CIDR, iface.Masquerade.IPv6CIDR, &netConfig); err != nil { return nil, fmt.Errorf("setup masquerade network: %s", err) } vmConfig.Net = append(vmConfig.Net, &netConfig) @@ -277,19 +279,10 @@ func buildVMConfig(ctx context.Context, vm *virtv1alpha1.VirtualMachine) (*cloud return &vmConfig, nil } -func setupBridgeNetwork(linkName string, cidr string, netConfig *cloudhypervisor.NetConfig) error { - _, subnet, err := net.ParseCIDR(cidr) +func setupBridgeNetwork(linkName string, ipv4CIDR string, netConfig *cloudhypervisor.NetConfig) error { + bridgeIPv4Net, err := generateBridgeIPNet(ipv4CIDR) if err != nil { - return fmt.Errorf("parse CIDR: %s", err) - } - - bridgeIP, err := nextIP(subnet.IP, subnet) - if err != nil { - return fmt.Errorf("generate bridge IP: %s", err) - } - bridgeIPNet := net.IPNet{ - IP: bridgeIP, - Mask: subnet.Mask, + return fmt.Errorf("generate bridge IPv4 net: %s", err) } link, err := netlink.LinkByName(linkName) @@ -299,7 +292,7 @@ func setupBridgeNetwork(linkName string, cidr string, netConfig *cloudhypervisor netConfig.Mtu = link.Attrs().MTU bridgeName := fmt.Sprintf("br-%s", linkName) - bridge, err := createBridge(bridgeName, &bridgeIPNet, link.Attrs().MTU) + bridge, err := createBridge(bridgeName, bridgeIPv4Net, nil, link.Attrs().MTU) if err != nil { return fmt.Errorf("create bridge: %s", err) } @@ -307,18 +300,28 @@ func setupBridgeNetwork(linkName string, cidr string, netConfig *cloudhypervisor linkMAC := link.Attrs().HardwareAddr netConfig.Mac = linkMAC.String() - var linkAddr *net.IPNet - linkAddrs, err := netlink.AddrList(link, netlink.FAMILY_V4) + var linkIPv4Addr *netlink.Addr + var linkIPv6Addr *netlink.Addr + linkAddrs, err := netlink.AddrList(link, netlink.FAMILY_ALL) if err != nil { return fmt.Errorf("list link addrs: %s", err) } - if len(linkAddrs) > 0 { - linkAddr = linkAddrs[0].IPNet + for idx, addr := range linkAddrs { + if netutils.IsIPv6(addr.IP) && !addr.IP.IsLinkLocalUnicast() { + linkIPv6Addr = &linkAddrs[idx] + } + if netutils.IsIPv4(addr.IP) { + linkIPv4Addr = &linkAddrs[idx] + } } - linkRoutes, err := netlink.RouteList(link, netlink.FAMILY_V4) + linkIPv4Routes, err := netlink.RouteList(link, netlink.FAMILY_V4) + if err != nil { + return fmt.Errorf("list link IPv4 routes: %s", err) + } + linkIPv6Routes, err := netlink.RouteList(link, netlink.FAMILY_V6) if err != nil { - return fmt.Errorf("list link routes: %s", err) + return fmt.Errorf("list link IPv6 routes: %s", err) } if err := netlink.LinkSetDown(link); err != nil { @@ -330,9 +333,11 @@ func setupBridgeNetwork(linkName string, cidr string, netConfig *cloudhypervisor } newLinkName := link.Attrs().Name - if linkAddr != nil { - if err := netlink.AddrDel(link, &linkAddrs[0]); err != nil { - return fmt.Errorf("delete link address: %s", err) + if linkIPv4Addr != nil || linkIPv6Addr != nil { + if linkIPv4Addr != nil { + if err := netlink.AddrDel(link, linkIPv4Addr); err != nil { + return fmt.Errorf("delete link IPv4 address: %s", err) + } } originalLinkName := link.Attrs().Name @@ -350,8 +355,15 @@ func setupBridgeNetwork(linkName string, cidr string, netConfig *cloudhypervisor if err := netlink.LinkAdd(dummy); err != nil { return fmt.Errorf("add dummy interface: %s", err) } - if err := netlink.AddrReplace(dummy, &linkAddrs[0]); err != nil { - return fmt.Errorf("replace dummy interface address: %s", err) + if linkIPv4Addr != nil { + if err := netlink.AddrReplace(dummy, linkIPv4Addr); err != nil { + return fmt.Errorf("replace dummy interface IPv4 address: %s", err) + } + } + if linkIPv6Addr != nil { + if err := netlink.AddrReplace(dummy, linkIPv6Addr); err != nil { + return fmt.Errorf("replace dummy interface IPv6 address: %s", err) + } } } @@ -373,66 +385,109 @@ func setupBridgeNetwork(linkName string, cidr string, netConfig *cloudhypervisor } netConfig.Tap = tapName - if linkAddr != nil { - var linkGateway net.IP - var routes []netlink.Route - for _, route := range linkRoutes { + if linkIPv4Addr != nil { + var ipv4Gateway net.IP + var ipv4Routes []netlink.Route + for _, route := range linkIPv4Routes { if route.Dst == nil && len(route.Src) == 0 && len(route.Gw) == 0 { continue } - if len(linkGateway) == 0 && route.Dst == nil { - linkGateway = route.Gw + if len(ipv4Gateway) == 0 && route.Dst == nil { + ipv4Gateway = route.Gw + } + ipv4Routes = append(ipv4Routes, route) + } + if err := startDHCPv4Server(bridgeName, linkMAC, linkIPv4Addr.IPNet, ipv4Gateway, ipv4Routes); err != nil { + return fmt.Errorf("start DHCPv4 server: %s", err) + } + } + if linkIPv6Addr != nil { + clientLLA := ipv6util.GenerateEUI64Address(net.ParseIP("fe80::0"), linkMAC) + if _, err := executeCommand("ip6tables", "-A", "INPUT", "-i", bridgeName, "!", "-s", clientLLA.String(), "-p", "udp", "-m", "multiport", "--sports", "546", "-j", "DROP"); err != nil { + return fmt.Errorf("allow DHCPv6 request only from client: %s", err) + } + if err := startDHCPv6Server(bridgeName, linkMAC, linkIPv6Addr.IPNet); err != nil { + return fmt.Errorf("start DHCPv6 server: %s", err) + } + + var route *netlink.Route + for idx, r := range linkIPv6Routes { + if r.Gw != nil { + route = &linkIPv6Routes[idx] + break } - routes = append(routes, route) } - if err := startDHCPServer(bridgeName, linkMAC, linkAddr, linkGateway, routes); err != nil { - return fmt.Errorf("start DHCP server: %s", err) + + command := exec.Command("rad", "-interface", bridgeName, "-client-hardware-addr", linkMAC.String(), "-prefix", linkIPv6Addr.IPNet.String()) + command.Stdout = os.Stdout + command.Stderr = os.Stderr + if route != nil { + command.Args = append(command.Args, "-router", route.Gw.String(), "-is-remote-route", "true") + } + if err := command.Start(); err != nil { + return fmt.Errorf("start RAD: %s", err) } } return nil } -func setupMasqueradeNetwork(linkName string, cidr string, netConfig *cloudhypervisor.NetConfig) error { - _, subnet, err := net.ParseCIDR(cidr) +func setupMasqueradeNetwork(linkName string, ipv4CIDR string, ipv6CIDR string, netConfig *cloudhypervisor.NetConfig) error { + link, err := netlink.LinkByName(linkName) if err != nil { - return fmt.Errorf("parse CIDR: %s", err) + return fmt.Errorf("get link: %s", err) } + netConfig.Mtu = link.Attrs().MTU - bridgeIP, err := nextIP(subnet.IP, subnet) + var linkIPv4Addr *netlink.Addr + var linkIPv6Addr *netlink.Addr + linkAddrs, err := netlink.AddrList(link, netlink.FAMILY_ALL) if err != nil { - return fmt.Errorf("generate bridge IP: %s", err) + return fmt.Errorf("list link addrs: %s", err) } - bridgeIPNet := net.IPNet{ - IP: bridgeIP, - Mask: subnet.Mask, + for idx, addr := range linkAddrs { + if netutils.IsIPv6(addr.IP) && !addr.IP.IsLinkLocalUnicast() { + linkIPv6Addr = &linkAddrs[idx] + } + if netutils.IsIPv4(addr.IP) { + linkIPv4Addr = &linkAddrs[idx] + } } - link, err := netlink.LinkByName(linkName) + bridgeIPv4Net, err := generateBridgeIPNet(ipv4CIDR) if err != nil { - return fmt.Errorf("get link: %s", err) + return fmt.Errorf("generate bridge IPv4 Net: %s", err) + } + bridgeIPv6Net, err := generateBridgeIPNet(ipv6CIDR) + if err != nil { + return fmt.Errorf("generate bridge IPv6 Net: %s", err) } - netConfig.Mtu = link.Attrs().MTU bridgeName := fmt.Sprintf("br-%s", linkName) - bridge, err := createBridge(bridgeName, &bridgeIPNet, link.Attrs().MTU) + bridge, err := createBridge(bridgeName, bridgeIPv4Net, bridgeIPv6Net, link.Attrs().MTU) if err != nil { return fmt.Errorf("create bridge: %s", err) } - vmIP, err := nextIP(bridgeIP, subnet) + vmIPv4Net, err := generateNextIPNet(bridgeIPv4Net) if err != nil { - return fmt.Errorf("generate vm IP: %s", err) + return fmt.Errorf("generate VM IPv4 net: %s", err) } - vmIPNet := &net.IPNet{ - IP: vmIP, - Mask: subnet.Mask, + vmIPv6Net, err := generateNextIPNet(bridgeIPv6Net) + if err != nil { + return fmt.Errorf("generate VM IPv6 net: %s", err) } if _, err := executeCommand("iptables", "-t", "nat", "-A", "POSTROUTING", "-o", linkName, "-j", "MASQUERADE"); err != nil { - return fmt.Errorf("add masquerade rule: %s", err) + return fmt.Errorf("add IPv4 masquerade rule: %s", err) + } + if _, err := executeCommand("iptables", "-t", "nat", "-A", "PREROUTING", "-i", linkName, "-j", "DNAT", "--to-destination", vmIPv4Net.IP.String()); err != nil { + return fmt.Errorf("add IPv4 prerouting rule: %s", err) + } + if _, err := executeCommand("ip6tables", "-t", "nat", "-A", "POSTROUTING", "-o", linkName, "-j", "MASQUERADE"); err != nil { + return fmt.Errorf("add IPv6 masquerade rule: %s", err) } - if _, err := executeCommand("iptables", "-t", "nat", "-A", "PREROUTING", "-i", linkName, "-j", "DNAT", "--to-destination", vmIP.String()); err != nil { - return fmt.Errorf("add prerouting rule: %s", err) + if _, err := executeCommand("ip6tables", "-t", "nat", "-A", "PREROUTING", "-i", linkName, "-j", "DNAT", "--to-destination", vmIPv6Net.IP.String()); err != nil { + return fmt.Errorf("add IPv6 prerouting rule: %s", err) } tapName := fmt.Sprintf("tap-%s", linkName) @@ -445,16 +500,59 @@ func setupMasqueradeNetwork(linkName string, cidr string, netConfig *cloudhyperv if err != nil { return fmt.Errorf("parse VM MAC: %s", err) } + if linkIPv4Addr != nil { + if err := startDHCPv4Server(bridgeName, vmMAC, vmIPv4Net, bridgeIPv4Net.IP, nil); err != nil { + return fmt.Errorf("start DHCPv4 server: %s", err) + } + } + if linkIPv6Addr != nil { + clientLLA := ipv6util.GenerateEUI64Address(net.ParseIP("fe80::0"), vmMAC) + if _, err := executeCommand("ip6tables", "-A", "INPUT", "-i", bridgeName, "!", "-s", clientLLA.String(), "-p", "udp", "-m", "multiport", "--sports", "546", "-j", "DROP"); err != nil { + return fmt.Errorf("allow DHCPv6 request only from client: %s", err) + } + if err := startDHCPv6Server(bridgeName, vmMAC, vmIPv6Net); err != nil { + return fmt.Errorf("start DHCPv6 server: %s", err) + } - if err := startDHCPServer(bridgeName, vmMAC, vmIPNet, bridgeIP, nil); err != nil { - return fmt.Errorf("start DHCP server: %s", err) + var bridgeIPv6LLA *netlink.Addr + bridgeAddrs, err := netlink.AddrList(bridge, netlink.FAMILY_V6) + if err != nil { + return fmt.Errorf("list bridge addrs: %s", err) + } + for idx, addr := range bridgeAddrs { + if addr.IP.IsLinkLocalUnicast() { + bridgeIPv6LLA = &bridgeAddrs[idx] + break + } + } + + command := exec.Command("rad", "-interface", bridgeName, "-router", bridgeIPv6LLA.IP.String(), "-client", clientLLA.String(), "-prefix", vmIPv6Net.String()) + command.Stdout = os.Stdout + command.Stderr = os.Stderr + if err := command.Start(); err != nil { + return fmt.Errorf("start RAD: %s", err) + } } + return nil } -func nextIP(ip net.IP, subnet *net.IPNet) (net.IP, error) { - nextIP := make(net.IP, len(ip)) - copy(nextIP, ip) +func generateBridgeIPNet(cidr string) (*net.IPNet, error) { + _, subnet, err := net.ParseCIDR(cidr) + if err != nil { + return nil, fmt.Errorf("parse CIDR: %s", err) + } + + ipNet, err := generateNextIPNet(subnet) + if err != nil { + return nil, fmt.Errorf("generate next IP net: %s", err) + } + return ipNet, nil +} + +func generateNextIPNet(subnet *net.IPNet) (*net.IPNet, error) { + nextIP := make(net.IP, len(subnet.IP)) + copy(nextIP, subnet.IP) for j := len(nextIP) - 1; j >= 0; j-- { nextIP[j]++ if nextIP[j] > 0 { @@ -464,10 +562,14 @@ func nextIP(ip net.IP, subnet *net.IPNet) (net.IP, error) { if subnet != nil && !subnet.Contains(nextIP) { return nil, fmt.Errorf("no more available IP in subnet %q", subnet.String()) } - return nextIP, nil + ipNet := net.IPNet{ + IP: nextIP, + Mask: subnet.Mask, + } + return &ipNet, nil } -func createBridge(bridgeName string, bridgeIPNet *net.IPNet, mtu int) (netlink.Link, error) { +func createBridge(bridgeName string, bridgeIPv4Net *net.IPNet, bridgeIPv6Net *net.IPNet, mtu int) (netlink.Link, error) { bridge := &netlink.Bridge{ LinkAttrs: netlink.LinkAttrs{ Name: bridgeName, @@ -478,8 +580,16 @@ func createBridge(bridgeName string, bridgeIPNet *net.IPNet, mtu int) (netlink.L return nil, err } - if err := netlink.AddrAdd(bridge, &netlink.Addr{IPNet: bridgeIPNet}); err != nil { - return nil, fmt.Errorf("set bridge addr: %s", err) + if bridgeIPv4Net != nil { + if err := netlink.AddrAdd(bridge, &netlink.Addr{IPNet: bridgeIPv4Net}); err != nil { + return nil, fmt.Errorf("set bridge IPv4 addr: %s", err) + } + } + + if bridgeIPv6Net != nil { + if err := netlink.AddrAdd(bridge, &netlink.Addr{IPNet: bridgeIPv6Net}); err != nil { + return nil, fmt.Errorf("set bridge IPv6 addr: %s", err) + } } if err := netlink.LinkSetUp(bridge); err != nil { @@ -519,7 +629,7 @@ func createTap(bridge netlink.Link, tapName string, mtu int) (netlink.Link, erro //go:embed dnsmasq.conf var dnsmasqConf string -func startDHCPServer(ifaceName string, mac net.HardwareAddr, ipNet *net.IPNet, gateway net.IP, routes []netlink.Route) error { +func startDHCPv4Server(ifaceName string, mac net.HardwareAddr, ipNet *net.IPNet, gateway net.IP, routes []netlink.Route) error { rc, err := resolvconf.Get() if err != nil { return fmt.Errorf("get resolvconf: %s", err) @@ -593,6 +703,56 @@ func sortAndFormatRoutes(routes []netlink.Route) string { return strings.Join(items, ",") } +//go:embed dhcpv6.json.conf +var dhcpv6Conf string + +func startDHCPv6Server(ifaceName string, mac net.HardwareAddr, ipNet *net.IPNet) error { + keaRunStateDir := fmt.Sprintf("/var/run/virtink/kea/%s/dhcpv6", ifaceName) + if err := os.MkdirAll(keaRunStateDir, 0755); err != nil { + return fmt.Errorf("create kea DHCPv6 run state dir: %s", err) + } + + dhcpv6ConfPath := fmt.Sprintf("/var/run/virtink/kea/%s/dhcpv6/dhcpv6.json", ifaceName) + dhcpv6ConfFile, err := os.Create(dhcpv6ConfPath) + if err != nil { + return fmt.Errorf("create kea DHCPv6 config file: %s", err) + } + defer dhcpv6ConfFile.Close() + + rc, err := resolvconf.Get() + if err != nil { + return fmt.Errorf("get resolvconf: %s", err) + } + + _, prefix, err := net.ParseCIDR(ipNet.String()) + if err != nil { + return fmt.Errorf("parse CIDR: %s", err) + } + + data := map[string]string{ + "iface": ifaceName, + "ip": ipNet.IP.String(), + "prefix": prefix.String(), + "mac": mac.String(), + "dnsServer": strings.Join(resolvconf.GetNameservers(rc.Content, types.IPv6), ","), + "domainSearch": strings.Join(resolvconf.GetSearchDomains(rc.Content), ","), + } + + if err := template.Must(template.New("dhcpv6.json.conf").Parse(dhcpv6Conf)).Execute(dhcpv6ConfFile, data); err != nil { + return fmt.Errorf("write kea DHCPv6 config file: %s", err) + } + + command := exec.Command("kea-dhcp6", "-c", fmt.Sprintf("/var/run/virtink/kea/%s/dhcpv6/dhcpv6.json", ifaceName)) + command.Env = os.Environ() + command.Env = append(command.Env, + fmt.Sprintf("KEA_PIDFILE_DIR=/var/run/virtink/kea/%s/dhcpv6", ifaceName), + fmt.Sprintf("KEA_LOCKFILE_DIR=/var/run/virtink/kea/%s/dhcpv6", ifaceName)) + if err := command.Start(); err != nil { + return fmt.Errorf("start kea DHCPv6 server: %s", err) + } + return nil +} + func executeCommand(name string, arg ...string) (string, error) { cmd := exec.Command(name, arg...) output, err := cmd.CombinedOutput() diff --git a/deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml b/deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml index 9d1117f..0b30a85 100644 --- a/deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml +++ b/deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml @@ -903,7 +903,13 @@ spec: type: string masquerade: properties: - cidr: + ipv4CIDR: + description: CIDR for IPv4 network. Default to 10.0.2.0/30 + if not specified + type: string + ipv6CIDR: + description: CIDR for IPv6 network. Default to fd10:0:2::/120 + if not specified type: string type: object name: diff --git a/go.mod b/go.mod index bc90157..3fa83da 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/hoisie/mustache v0.0.0-20160804235033-6375acf62c69 github.com/iancoleman/strcase v0.2.0 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.3.0 + github.com/mdlayher/ndp v1.0.0 github.com/moby/sys/mountinfo v0.6.2 github.com/namsral/flag v1.7.4-pre github.com/nasa9084/go-openapi v0.0.0-20210722142352-4a81d737faf6 @@ -19,6 +20,8 @@ require ( github.com/stretchr/testify v1.7.0 github.com/subgraph/libmacouflage v0.0.1 github.com/vishvananda/netlink v1.1.0 + golang.org/x/net v0.0.0-20220923203811-8be639271d50 + golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 google.golang.org/grpc v1.47.0 gopkg.in/fsnotify.v1 v1.4.7 inet.af/tcpproxy v0.0.0-20220326234310-be3ee21c9fa0 @@ -27,6 +30,7 @@ require ( k8s.io/apiserver v0.24.1 k8s.io/client-go v0.24.1 k8s.io/kubelet v0.24.1 + k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 kubevirt.io/containerized-data-importer-api v1.50.0 sigs.k8s.io/controller-runtime v0.12.1 sigs.k8s.io/controller-tools v0.9.0 @@ -57,7 +61,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.6 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect @@ -89,9 +93,7 @@ require ( go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect - golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/sys v0.0.0-20220908164124-27713097b956 // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect @@ -109,7 +111,6 @@ require ( k8s.io/component-base v0.24.1 // indirect k8s.io/klog/v2 v2.60.1 // indirect k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect - k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect diff --git a/go.sum b/go.sum index eef02d1..7ef8b74 100644 --- a/go.sum +++ b/go.sum @@ -256,8 +256,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -375,6 +376,8 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mdlayher/ndp v1.0.0 h1:rcFaJVj04Rj47ZlV/t3iZcuKzlpwBuBsD3gR9AHDzcI= +github.com/mdlayher/ndp v1.0.0/go.mod h1:+3vkk6YnlL8ZTRTjmQanCNQFqDKOpP2zNyHl2HqyoZs= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -689,8 +692,8 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220923203811-8be639271d50 h1:vKyz8L3zkd+xrMeIaBsQ/MNVPVFSffdaU3ZyYlBGFnI= +golang.org/x/net v0.0.0-20220923203811-8be639271d50/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -790,8 +793,9 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956 h1:XeJjHH1KiLpKGb6lvMiksZ9l0fVUh+AmGcm0nOMEBOY= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc= +golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/pkg/apis/virt/v1alpha1/types.go b/pkg/apis/virt/v1alpha1/types.go index 30765a4..be45d0a 100644 --- a/pkg/apis/virt/v1alpha1/types.go +++ b/pkg/apis/virt/v1alpha1/types.go @@ -109,7 +109,10 @@ type InterfaceBridge struct { } type InterfaceMasquerade struct { - CIDR string `json:"cidr,omitempty"` + // CIDR for IPv4 network. Default to 10.0.2.0/30 if not specified + IPv4CIDR string `json:"ipv4CIDR,omitempty"` + // CIDR for IPv6 network. Default to fd10:0:2::/120 if not specified + IPv6CIDR string `json:"ipv6CIDR,omitempty"` } type InterfaceSRIOV struct { diff --git a/pkg/controller/vm_controller.go b/pkg/controller/vm_controller.go index 5b918b1..1487bec 100644 --- a/pkg/controller/vm_controller.go +++ b/pkg/controller/vm_controller.go @@ -640,17 +640,6 @@ func (r *VMReconciler) buildVMPod(ctx context.Context, vm *virtv1alpha1.VirtualM return nil, fmt.Errorf("interface not found for network: %s", network.Name) } - if iface.Masquerade != nil { - vmPod.Spec.InitContainers = append(vmPod.Spec.InitContainers, corev1.Container{ - Name: "enable-ip-forward", - Image: r.PrerunnerImageName, - SecurityContext: &corev1.SecurityContext{ - Privileged: &[]bool{true}[0], - }, - Command: []string{"sysctl", "-w", "net.ipv4.ip_forward=1"}, - }) - } - switch { case network.Multus != nil: networks = append(networks, netv1.NetworkSelectionElement{ @@ -726,6 +715,17 @@ func (r *VMReconciler) buildVMPod(ctx context.Context, vm *virtv1alpha1.VirtualM } } + if len(vm.Spec.Instance.Interfaces) > 0 { + vmPod.Spec.InitContainers = append(vmPod.Spec.InitContainers, corev1.Container{ + Name: "configure-sysctl-for-ip", + Image: r.PrerunnerImageName, + SecurityContext: &corev1.SecurityContext{ + Privileged: &[]bool{true}[0], + }, + Command: []string{"sysctl", "-w", "net.ipv4.ip_forward=1", "net.ipv6.conf.all.forwarding=1", "net.ipv6.conf.default.accept_dad=0"}, + }) + } + if len(networks) > 0 { networksJSON, err := json.Marshal(networks) if err != nil { diff --git a/pkg/controller/vm_webhook.go b/pkg/controller/vm_webhook.go index 16fe869..a9b9457 100644 --- a/pkg/controller/vm_webhook.go +++ b/pkg/controller/vm_webhook.go @@ -172,8 +172,11 @@ func MutateVM(ctx context.Context, vm *virtv1alpha1.VirtualMachine, oldVM *virtv } if vm.Spec.Instance.Interfaces[i].Masquerade != nil { - if vm.Spec.Instance.Interfaces[i].Masquerade.CIDR == "" { - vm.Spec.Instance.Interfaces[i].Masquerade.CIDR = "10.0.2.0/30" + if vm.Spec.Instance.Interfaces[i].Masquerade.IPv4CIDR == "" { + vm.Spec.Instance.Interfaces[i].Masquerade.IPv4CIDR = "10.0.2.0/30" + } + if vm.Spec.Instance.Interfaces[i].Masquerade.IPv6CIDR == "" { + vm.Spec.Instance.Interfaces[i].Masquerade.IPv6CIDR = "fd10:0:2::/120" } } } @@ -495,7 +498,8 @@ func ValidateInterfaceBindingMethod(ctx context.Context, bindingMethod *virtv1al if cnt > 1 { errs = append(errs, field.Forbidden(fieldPath.Child("masquerade"), "may not specify more than 1 binding method")) } else { - errs = append(errs, ValidateCIDR(bindingMethod.Masquerade.CIDR, 4, fieldPath.Child("masquerade").Child("cidr"))...) + errs = append(errs, ValidateCIDR(bindingMethod.Masquerade.IPv4CIDR, 4, fieldPath.Child("masquerade").Child("ipv4CIDR"))...) + errs = append(errs, ValidateCIDR(bindingMethod.Masquerade.IPv6CIDR, 4, fieldPath.Child("masquerade").Child("ipv6CIDR"))...) } } if bindingMethod.SRIOV != nil { diff --git a/pkg/controller/vm_webhook_test.go b/pkg/controller/vm_webhook_test.go index 963765a..579d37b 100644 --- a/pkg/controller/vm_webhook_test.go +++ b/pkg/controller/vm_webhook_test.go @@ -343,21 +343,23 @@ func TestValidateVM(t *testing.T) { vm := validVM.DeepCopy() vm.Spec.Instance.Interfaces[0].InterfaceBindingMethod.Bridge = nil vm.Spec.Instance.Interfaces[0].InterfaceBindingMethod.Masquerade = &virtv1alpha1.InterfaceMasquerade{ - CIDR: "", + IPv4CIDR: "", + IPv6CIDR: "", } return vm }(), - invalidFields: []string{"spec.instance.interfaces[0].masquerade.cidr"}, + invalidFields: []string{"spec.instance.interfaces[0].masquerade.ipv4CIDR", "spec.instance.interfaces[0].masquerade.ipv6CIDR"}, }, { vm: func() *virtv1alpha1.VirtualMachine { vm := validVM.DeepCopy() vm.Spec.Instance.Interfaces[0].InterfaceBindingMethod.Bridge = nil vm.Spec.Instance.Interfaces[0].InterfaceBindingMethod.Masquerade = &virtv1alpha1.InterfaceMasquerade{ - CIDR: "10.0.2.0/31", + IPv4CIDR: "10.0.2.0/31", + IPv6CIDR: "fd10:0:2::/127", } return vm }(), - invalidFields: []string{"spec.instance.interfaces[0].masquerade.cidr"}, + invalidFields: []string{"spec.instance.interfaces[0].masquerade.ipv4CIDR", "spec.instance.interfaces[0].masquerade.ipv6CIDR"}, }, { vm: func() *virtv1alpha1.VirtualMachine { vm := validVM.DeepCopy() @@ -595,7 +597,8 @@ func TestMutateVM(t *testing.T) { return vm }(), assert: func(vm *virtv1alpha1.VirtualMachine) { - assert.Equal(t, vm.Spec.Instance.Interfaces[0].Masquerade.CIDR, "10.0.2.0/30") + assert.Equal(t, vm.Spec.Instance.Interfaces[0].Masquerade.IPv4CIDR, "10.0.2.0/30") + assert.Equal(t, vm.Spec.Instance.Interfaces[0].Masquerade.IPv6CIDR, "fd10:0:2::/120") }, }} for _, tc := range tests { diff --git a/pkg/ipv6util/ipv6util.go b/pkg/ipv6util/ipv6util.go new file mode 100644 index 0000000..21c173e --- /dev/null +++ b/pkg/ipv6util/ipv6util.go @@ -0,0 +1,16 @@ +package ipv6util + +import "net" + +func GenerateEUI64Address(prefix net.IP, mac net.HardwareAddr) net.IP { + ip := make([]byte, 16) + copy(ip[0:8], prefix[0:8]) + + copy(ip[8:11], mac[0:3]) + ip[8] ^= 0x02 + ip[11] = 0xff + ip[12] = 0xfe + copy(ip[13:16], mac[3:6]) + + return ip +}