From 7e73541af1dd5f432a0a538e9ce036a73d0b6e72 Mon Sep 17 00:00:00 2001 From: Ali Mohamed Date: Sun, 28 Jun 2026 10:43:53 +0300 Subject: [PATCH] feat(network): forward guest DNS to Docker's embedded resolver On Docker user custom networks the DNS resolver (127.0.0.11) is loopback only and unreachable by the unikernel guest. Detect this case per container, expose the resolver via a virtual resolver IP using tc redirects between the tap and lo + a PREROUTING DNAT, and rewrite the guest's resolv.conf to point at that IP. Signed-off-by: Ali Mohamed --- .github/contributors.yaml | 3 + Makefile | 1 + internal/constants/network_constants.go | 5 +- pkg/network/localhost/docker.go | 109 ++++++++ pkg/network/localhost/local_dns.go | 314 ++++++++++++++++++++++++ pkg/network/localhost/local_dns_test.go | 123 ++++++++++ pkg/network/network.go | 9 +- pkg/network/network_dynamic.go | 26 +- pkg/network/network_test.go | 2 +- pkg/unikontainers/types/types.go | 13 +- pkg/unikontainers/unikontainers.go | 15 +- pkg/unikontainers/urunc_config.go | 20 ++ pkg/unikontainers/urunc_config_test.go | 17 ++ pkg/unikontainers/utils.go | 11 + tests/e2e/common.go | 17 ++ tests/e2e/crictl.go | 10 + tests/e2e/ctr.go | 10 + tests/e2e/docker.go | 8 + tests/e2e/docker_test.go | 30 ++- tests/e2e/nerdctl.go | 7 + tests/e2e/suite_test.go | 18 ++ tests/e2e/test_cases.go | 18 ++ tests/e2e/test_functions.go | 4 + 23 files changed, 766 insertions(+), 24 deletions(-) create mode 100644 pkg/network/localhost/docker.go create mode 100644 pkg/network/localhost/local_dns.go create mode 100644 pkg/network/localhost/local_dns_test.go diff --git a/.github/contributors.yaml b/.github/contributors.yaml index 411f9f42a..400b3b8de 100644 --- a/.github/contributors.yaml +++ b/.github/contributors.yaml @@ -107,3 +107,6 @@ users: Chennamma-Hotkar: name: Chennamma Hotkar email: channuhotkar@gmail.com + alimx07: + name: Ali Mohamed + email: amx746@gmail.com diff --git a/Makefile b/Makefile index 0f87fa84c..3ae74e551 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,7 @@ URUNC_SRC += $(wildcard $(CURDIR)/pkg/unikontainers/unikernels/*.go) URUNC_SRC += $(wildcard $(CURDIR)/pkg/unikontainers/types/*.go) URUNC_SRC += $(wildcard $(CURDIR)/pkg/unikontainers/initrd/*.go) URUNC_SRC += $(wildcard $(CURDIR)/pkg/network/*.go) +URUNC_SRC += $(wildcard $(CURDIR)/pkg/network/localhost/*.go) SHIM_SRC := $(wildcard $(CURDIR)/cmd/containerd-shim-urunc-v2/*.go) SHIM_SRC += $(wildcard $(CURDIR)/pkg/containerd-shim/*.go) SHIM_SRC += $(wildcard $(CURDIR)/pkg/containerd-shim/containerd/*.go) diff --git a/internal/constants/network_constants.go b/internal/constants/network_constants.go index de8a30ccb..640fe8574 100644 --- a/internal/constants/network_constants.go +++ b/internal/constants/network_constants.go @@ -18,6 +18,7 @@ const ( StaticNetworkTapIP = "172.16.1.1" StaticNetworkUnikernelIP = "172.16.1.2" // TODO: Experiment with DynamicNetworkTapIP starting from 172.16.X.1 - DynamicNetworkTapIP = "172.16.X.2" - QueueProxyRedirectIP = "172.16.1.2" + DynamicNetworkTapIP = "172.16.X.2" + QueueProxyRedirectIP = "172.16.1.2" + LocalhostDNSResolverIP = "192.168.100.100" ) diff --git a/pkg/network/localhost/docker.go b/pkg/network/localhost/docker.go new file mode 100644 index 000000000..520fe0e0b --- /dev/null +++ b/pkg/network/localhost/docker.go @@ -0,0 +1,109 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package localhost + +import ( + "errors" + "fmt" + "net" + "os/exec" + "strings" + "syscall" + + "github.com/vishvananda/netlink" +) + +func dockerRules(f *Forwarder) error { + + tcpPort, udpPort, err := dockerDNSPorts(f.LoIP) + if err != nil { + return err + } + lhlog.Debugf("Docker DNS resolver: tcp/%s udp/%s", tcpPort, udpPort) + + if err := dnat(f.VirtIP, "udp", net.JoinHostPort(f.LoIP.String(), udpPort)); err != nil { + return err + } + if err := dnat(f.VirtIP, "tcp", net.JoinHostPort(f.LoIP.String(), tcpPort)); err != nil { + return err + } + lhlog.Debug("Applied Docker DNAT rules") + + return nil +} + +func dockerDNSPorts(loIP net.IP) (string, string, error) { + var tcpPort, udpPort string + + // syscall.AF_INET : IPv4 + tcp, err := netlink.SocketDiagTCPInfo(syscall.AF_INET) + if err != nil { + return "", "", fmt.Errorf("could not list TCP sockets in namespace: %w", err) + } + for _, s := range tcp { + if s.InetDiagMsg.ID.Source.String() == loIP.String() { + tcpPort = fmt.Sprintf("%d", s.InetDiagMsg.ID.SourcePort) + break + } + } + udp, err := netlink.SocketDiagUDPInfo(syscall.AF_INET) + if err != nil { + return "", "", fmt.Errorf("could not list UDP sockets in namespace: %w", err) + } + for _, s := range udp { + if s.InetDiagMsg.ID.Source.String() == loIP.String() { + udpPort = fmt.Sprintf("%d", s.InetDiagMsg.ID.SourcePort) + break + } + } + if tcpPort == "" || udpPort == "" { + return "", "", fmt.Errorf("could not find docker embedded resolver ports for %s (tcp=%q udp=%q)", loIP, tcpPort, udpPort) + } + return tcpPort, udpPort, nil +} + +// ClearRules removes DNAT rules dockerRules could have added. +func clearRules() error { + + // This will be run from kill() so no memory state could be used. + // we list all IPtable rules from scratch. + + // iptables-save output example: + // -A PREROUTING -d 192.168.100.100/32 -i lo -p udp -j DNAT --to-destination 127.0.0.11:60269 + ipt, err := exec.LookPath("iptables-save") + if err != nil { + return err + } + out, err := exec.Command(ipt, "-t", "nat").Output() //nolint:gosec + if err != nil { + return fmt.Errorf("iptables-save failed: %w", err) + } + + var retErr error + for line := range strings.SplitSeq(string(out), "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || fields[0] != "-A" || fields[1] != "PREROUTING" { + continue + } + if !strings.Contains(line, "-i lo") || !strings.Contains(line, "-j DNAT") { + continue + } + args := append([]string{"-t", "nat", "-D"}, fields[1:]...) + if err := ipTablesExec(args); err != nil { + retErr = errors.Join(retErr, err) + } + } + return retErr +} diff --git a/pkg/network/localhost/local_dns.go b/pkg/network/localhost/local_dns.go new file mode 100644 index 000000000..55d7cecc0 --- /dev/null +++ b/pkg/network/localhost/local_dns.go @@ -0,0 +1,314 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package localhost + +import ( + "bytes" + "errors" + "fmt" + "net" + "os" + "os/exec" + "strings" + + "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +var lhlog = logrus.WithField("subsystem", "network") + +// rules installs the custom part of the forwarding for some scenario. +type rules func(*Forwarder) error + +type Forwarder struct { + VirtIP net.IP // virtual resolver IP the guest dials + LoIP net.IP // loopback IP the real resolver listens on + ResolvConf string // host path of the container's resolv.conf + custom rules // scenario-specific extra rules, picked by Detect +} + +func Detect(resolvConf string, virtIP string) (*Forwarder, error) { + if resolvConf == "" { + return nil, nil + } + data, err := os.ReadFile(resolvConf) + if err != nil { + return nil, err + } + loIP := loNameserver(string(data)) + if loIP == nil { + return nil, nil + } + + ip := net.ParseIP(virtIP) + if ip == nil { + return nil, fmt.Errorf("invalid virtual resolver IP %s", virtIP) + } + + f := &Forwarder{ + VirtIP: ip, + LoIP: loIP, + ResolvConf: resolvConf, + + // now by default, if we detect localhost nameserver we apply dockerRules + // which is the only case we cover. + custom: dockerRules, + } + return f, nil +} + +// RewriteResolvConf points the loopback nameservers in the container's +// resolv.conf at the virtIP. +func (f *Forwarder) RewriteResolvConf() error { + data, err := os.ReadFile(f.ResolvConf) + if err != nil { + return fmt.Errorf("failed to read %s: %w", f.ResolvConf, err) + } + + lines := strings.Split(string(data), "\n") + out := make([]string, 0, len(lines)) + + for _, line := range lines { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == "nameserver" { + if ip := net.ParseIP(fields[1]); ip != nil && ip.IsLoopback() { + out = append(out, "nameserver "+f.VirtIP.String()) + continue + } + } + out = append(out, line) + } + + if err := os.WriteFile(f.ResolvConf, []byte(strings.Join(out, "\n")), 0o644); err != nil { //nolint: gosec + return fmt.Errorf("failed to write %s: %w", f.ResolvConf, err) + } + return nil +} + +// loNameserver returns the first loopback nameserver in resolv.conf data, if any. +func loNameserver(data string) net.IP { + for line := range strings.SplitSeq(data, "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == "nameserver" { + if ip := net.ParseIP(fields[1]); ip != nil && ip.IsLoopback() { + return ip + } + } + } + return nil +} + +// Apply installs the host side plumbing in the current netns: +// the general tc +// Kernel flags required +// the scenario's custom rules +func (f *Forwarder) Apply(tap netlink.Link, eth netlink.Link) error { + lo, err := netlink.LinkByName("lo") + if err != nil { + return err + } + + if err := addClsact(lo); err != nil { + return fmt.Errorf("addClsact(lo) failed: %w", err) + } + if err := tapToLo(tap, lo, f.VirtIP); err != nil { + return fmt.Errorf("tapToLo(%s) failed: %w", tap.Attrs().Name, err) + } + + // The guest reuses the container's MAC, so replies must be addressed to it. + if err := loToTap(lo, tap, f.VirtIP, eth.Attrs().HardwareAddr); err != nil { + return fmt.Errorf("loToTap(%s) failed: %w", tap.Attrs().Name, err) + } + lhlog.Debug("Applied tc redirects between tap and lo") + + err = setSysctls(map[string]string{ + "/proc/sys/net/ipv4/conf/lo/route_localnet": "1", + fmt.Sprintf("/proc/sys/net/ipv4/conf/%s/accept_local", tap.Attrs().Name): "1", + "/proc/sys/net/ipv4/conf/lo/accept_local": "1", + }) + if err != nil { + return fmt.Errorf("failed to set required kernel flags for forwarding: %w", err) + } + lhlog.Debug("Applied kernel flags required for packet forwarding") + + return f.custom(f) +} + +func dnat(virtIP net.IP, proto string, dst string) error { + args := []string{ + "-t", "nat", "-A", "PREROUTING", + "-i", "lo", + "-d", virtIP.String(), + "-p", proto, + "-j", "DNAT", + "--to-destination", dst, + "--wait", "1", + } + if err := ipTablesExec(args); err != nil { + return err + } + return nil +} + +func addClsact(link netlink.Link) error { + clsact := &netlink.Clsact{ + QdiscAttrs: netlink.QdiscAttrs{ + LinkIndex: link.Attrs().Index, + Parent: netlink.HANDLE_CLSACT, + }, + } + return netlink.QdiscAdd(clsact) +} + +func tapToLo(tap netlink.Link, lo netlink.Link, virtIP net.IP) error { + + // tc filter add dev ingress protocol ip pref x \ + // flower dst_ip \ + // action skbedit ptype host pipe \ + // action mirred ingress redirect dev lo + return netlink.FilterAdd(&netlink.Flower{ + FilterAttrs: netlink.FilterAttrs{ + LinkIndex: tap.Attrs().Index, + Parent: netlink.MakeHandle(0xffff, 0), + Priority: 100, + Protocol: unix.ETH_P_IP, + }, + DestIP: virtIP, + Actions: []netlink.Action{ + &netlink.SkbEditAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_PIPE, + }, + PType: uint16Ptr(unix.PACKET_HOST), + }, &netlink.MirredAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_STOLEN, + }, + MirredAction: netlink.TCA_INGRESS_REDIR, + Ifindex: lo.Attrs().Index, + }, + }, + }) +} + +func loToTap(lo netlink.Link, tap netlink.Link, virtIP net.IP, guestMAC net.HardwareAddr) error { + + // tc filter add dev lo egress protocol ip pref x \ + // flower src_ip \ + // action pedit ex munge eth dst set pipe \ + // action mirred egress redirect dev + return netlink.FilterAdd(&netlink.Flower{ + FilterAttrs: netlink.FilterAttrs{ + LinkIndex: lo.Attrs().Index, + Parent: netlink.HANDLE_MIN_EGRESS, + Priority: 100, + Protocol: unix.ETH_P_IP, + }, + SrcIP: virtIP, + Actions: []netlink.Action{ + &netlink.PeditAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_PIPE, + }, + DstMacAddr: guestMAC, + }, + &netlink.MirredAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_STOLEN, + }, + MirredAction: netlink.TCA_EGRESS_REDIR, + Ifindex: tap.Attrs().Index, + }, + }, + }) +} + +func setSysctls(flags map[string]string) error { + for flag, value := range flags { + file, err := os.OpenFile(flag, os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("failed to open %s: %w", flag, err) + } + defer file.Close() + + _, err = file.WriteString(value) + if err != nil { + return fmt.Errorf("failed to set flag %s to %s: %w", flag, value, err) + } + } + return nil +} + +// Cleanup removes the host side plumbing installed by Apply: +// - Del TC link +// - Del rules applied (e.g. iptable rules) +func Cleanup() error { + var retErr error + + lo, err := netlink.LinkByName("lo") + if err != nil { + return fmt.Errorf("LinkByName(lo) failed: %w", err) + } + if err := deleteClsact(lo); err != nil { + retErr = errors.Join(retErr, fmt.Errorf("failed to delete clsact qdisc on lo: %w", err)) + } + if err := clearRules(); err != nil { + retErr = errors.Join(retErr, fmt.Errorf("failed to flush DNAT rules: %w", err)) + } + return retErr +} + +func deleteClsact(link netlink.Link) error { + qdiscs, err := netlink.QdiscList(link) + if err != nil { + return fmt.Errorf("QdiscList(%s) failed: %w", link.Attrs().Name, err) + } + for _, q := range qdiscs { + if _, ok := q.(*netlink.Clsact); ok { + if err := netlink.QdiscDel(q); err != nil { + return fmt.Errorf("QdiscDel(clsact on %s) failed: %w", link.Attrs().Name, err) + } + } + } + return nil +} + +func uint16Ptr(v uint16) *uint16 { + return &v +} + +func ipTablesExec(args []string) error { + var stdout, stderr bytes.Buffer + + ipt, err := exec.LookPath("iptables") + if err != nil { + return err + } + args = append([]string{ipt}, args...) + cmd := exec.Cmd{ + Path: ipt, + Args: args, + Stdout: &stdout, + Stderr: &stderr, + } + if err := cmd.Run(); err != nil { + if _, ok := err.(*exec.ExitError); ok { + return fmt.Errorf("iptables command %s failed: %s", cmd.String(), stderr.String()) + } + return err + } + return nil +} diff --git a/pkg/network/localhost/local_dns_test.go b/pkg/network/localhost/local_dns_test.go new file mode 100644 index 000000000..204ad67de --- /dev/null +++ b/pkg/network/localhost/local_dns_test.go @@ -0,0 +1,123 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package localhost + +import ( + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeTemp(t *testing.T, dir, content string) string { + t.Helper() + require.NoError(t, os.MkdirAll(dir, 0o755)) + path := filepath.Join(dir, "resolv.conf") + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) //nolint: gosec + return path +} + +func TestDetect(t *testing.T) { + tmp := t.TempDir() + + tests := []struct { + name string + dir string // subdirectory holding resolv.conf + content string + virtIP string + wantNil bool + wantLoIP string + wantVirtIP string + }{ + { + name: "docker embedded resolver", + dir: "var/lib/docker/containers/abc", + content: "nameserver 127.0.0.11\noptions ndots:0\n", + virtIP: "192.168.100.100", + wantLoIP: "127.0.0.11", + wantVirtIP: "192.168.100.100", + }, + { + name: "loopback after public nameserver", + dir: "mixed", + content: "nameserver 8.8.8.8\nnameserver 127.0.0.11\n", + virtIP: "192.168.100.100", + wantLoIP: "127.0.0.11", + wantVirtIP: "192.168.100.100", + }, + { + name: "no loopback nameserver", + dir: "plain", + content: "search example.com\nnameserver 8.8.8.8\n", + virtIP: "192.168.100.100", + wantNil: true, + }, + { + name: "garbage lines are ignored", + dir: "garbage", + content: "# a comment\nnameserver\nnot-an-ip\n", + virtIP: "192.168.100.100", + wantNil: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := writeTemp(t, filepath.Join(tmp, tt.dir), tt.content) + fwd, err := Detect(path, tt.virtIP) + require.NoError(t, err) + if tt.wantNil { + assert.Nil(t, fwd, "Detect() should not return a Forwarder") + return + } + require.NotNil(t, fwd, "Detect() should return a Forwarder") + assert.Equal(t, tt.wantLoIP, fwd.LoIP.String()) + assert.Equal(t, tt.wantVirtIP, fwd.VirtIP.String()) + assert.Equal(t, path, fwd.ResolvConf) + assert.Equal(t, reflect.ValueOf(dockerRules).Pointer(), reflect.ValueOf(fwd.custom).Pointer(), "custom rules should be dockerRules") + }) + } + + t.Run("empty path", func(t *testing.T) { + fwd, err := Detect("", "192.168.100.100") + assert.NoError(t, err) + assert.Nil(t, fwd) + }) + + t.Run("missing file", func(t *testing.T) { + fwd, err := Detect(filepath.Join(tmp, "does/not/exist"), "192.168.100.100") + assert.Error(t, err) + assert.Nil(t, fwd) + }) +} + +func TestRewriteResolvConf(t *testing.T) { + tmp := t.TempDir() + content := "search example.com\nnameserver 127.0.0.11\nnameserver 8.8.8.8\noptions ndots:0\n" + path := writeTemp(t, filepath.Join(tmp, "var/lib/docker/containers/abc"), content) + + fwd, err := Detect(path, "192.168.100.100") + require.NoError(t, err) + require.NotNil(t, fwd) + require.NoError(t, fwd.RewriteResolvConf()) + + got, err := os.ReadFile(path) + require.NoError(t, err) + want := "search example.com\nnameserver 192.168.100.100\nnameserver 8.8.8.8\noptions ndots:0\n" + assert.Equal(t, want, string(got), "only loopback nameservers should be rewritten") +} diff --git a/pkg/network/network.go b/pkg/network/network.go index 6fb18c6c6..a718ba9fc 100644 --- a/pkg/network/network.go +++ b/pkg/network/network.go @@ -25,6 +25,8 @@ import ( "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" + + "github.com/urunc-dev/urunc/pkg/network/localhost" ) const ( @@ -36,6 +38,9 @@ var netlog = logrus.WithField("subsystem", "network") type UnikernelNetworkInfo struct { TapDevice string EthDevice Interface + // DNSServer is the virtual resolver IP the guest should be configured + // with, set only when localhost DNS forwarding was applied. + DNSServer string } type Manager interface { NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) @@ -50,12 +55,12 @@ type Interface struct { MTU int } -func NewNetworkManager(networkType string) (Manager, error) { +func NewNetworkManager(networkType string, fwd *localhost.Forwarder) (Manager, error) { switch networkType { case "static": return &StaticNetwork{}, nil case "dynamic": - return &DynamicNetwork{}, nil + return &DynamicNetwork{DNSForwarder: fwd}, nil default: return nil, fmt.Errorf("network manager %s not supported", networkType) diff --git a/pkg/network/network_dynamic.go b/pkg/network/network_dynamic.go index 1d6a2237b..f32e77004 100644 --- a/pkg/network/network_dynamic.go +++ b/pkg/network/network_dynamic.go @@ -18,9 +18,14 @@ import ( "fmt" "strconv" "strings" + + "github.com/urunc-dev/urunc/pkg/network/localhost" ) type DynamicNetwork struct { + // DNSForwader when set, is applied on the tap/veth the dynamic NetworkSetup creates + // so the guest DNS queries get redirected to the host loopback resolver. + DNSForwarder *localhost.Forwarder } // NetworkSetup checks if any tap device is available in the current netns. If it is, it assumes a running unikernel @@ -62,8 +67,25 @@ func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkI return nil, fmt.Errorf("getInterfaceInfo(%s) failed: %w", redirectLink.Attrs().Name, err) } - return &UnikernelNetworkInfo{ + info := &UnikernelNetworkInfo{ TapDevice: newTapDevice.Attrs().Name, EthDevice: ifInfo, - }, nil + } + + // DNS forwarding is a good feature, but the unikernel can still run without it. + // so we fail open, failure here is only logged and DNSServer stays empty. + if n.DNSForwarder != nil { + if err := n.DNSForwarder.Apply(newTapDevice, redirectLink); err != nil { + netlog.Warnf("failed to apply localhost forwarding rules: %v", err) + return info, nil + } + if err := n.DNSForwarder.RewriteResolvConf(); err != nil { + netlog.Warnf("failed to rewrite container's resolv.conf: %v", err) + return info, nil + } + info.DNSServer = n.DNSForwarder.VirtIP.String() + netlog.Debugf("localhost forwarding applied: virtual resolvIP %s", info.DNSServer) + } + + return info, nil } diff --git a/pkg/network/network_test.go b/pkg/network/network_test.go index da8c1d3d3..409a3a7cd 100644 --- a/pkg/network/network_test.go +++ b/pkg/network/network_test.go @@ -46,7 +46,7 @@ func TestNewNetworkManager(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - got, err := NewNetworkManager(tt.networkType) + got, err := NewNetworkManager(tt.networkType, nil) if tt.expectedErr { assert.Error(t, err, "NewNetworkManager() should return an error") } else { diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index c6388e2cc..a4806a363 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -45,12 +45,13 @@ type VMM interface { } type NetDevParams struct { - IP string // The veth device IP - Mask string // The veth device mask - Gateway string // The veth device gateway - MAC string // The MAC address of the guest network device - TapDev string // The tap device name - MTU int // The MTU value of the tap device + IP string // The veth device IP + Mask string // The veth device mask + Gateway string // The veth device gateway + MAC string // The MAC address of the guest network device + TapDev string // The tap device name + MTU int // The MTU value of the tap device + DNSServer string // The resolver IP when localhost forwarding is active, empty otherwise } type BlockDevParams struct { diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index fc297f937..c7ec170ef 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -30,6 +30,7 @@ import ( "syscall" "github.com/urunc-dev/urunc/pkg/network" + "github.com/urunc-dev/urunc/pkg/network/localhost" "github.com/urunc-dev/urunc/pkg/unikontainers/hypervisors" "github.com/urunc-dev/urunc/pkg/unikontainers/types" "github.com/urunc-dev/urunc/pkg/unikontainers/unikernels" @@ -203,7 +204,13 @@ func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { networkType := u.getNetworkType() uniklog.WithField("network type", networkType).Debug("Retrieved network type") netArgs := types.NetDevParams{} - netManager, err := network.NewNetworkManager(networkType) + + fwd, err := localhost.Detect(resolvConfSource(u.Spec.Mounts), u.UruncCfg.Network.DNSResolverIP) + if err != nil { + uniklog.Warnf("failed to detect loopback resolver: %v", err) + } + + netManager, err := network.NewNetworkManager(networkType, fwd) if err != nil { return netArgs, fmt.Errorf("failed to create network manager for %s type: %v", networkType, err) } @@ -226,8 +233,9 @@ func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { // virtual ethernet interface inside the namespace netArgs.MAC = networkInfo.EthDevice.MAC netArgs.MTU = networkInfo.EthDevice.MTU + // Empty unless dynamic networking applied localhost DNS forwarding. + netArgs.DNSServer = networkInfo.DNSServer } - return netArgs, nil } @@ -730,6 +738,9 @@ func (u *Unikontainer) Kill() error { return err } + if err := localhost.Cleanup(); err != nil { + uniklog.Errorf("failed to cleanup localhost forwarding rules: %v", err) + } err = network.CleanupAllUruncTaps() if err != nil { uniklog.Errorf("failed to cleanup tap devices: %v", err) diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 5f21d106e..0d89dd0ce 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -19,6 +19,7 @@ import ( "strings" "github.com/BurntSushi/toml" + "github.com/urunc-dev/urunc/internal/constants" "github.com/urunc-dev/urunc/pkg/unikontainers/types" ) @@ -34,9 +35,14 @@ type UruncTimestamps struct { Destination string `toml:"destination"` // Used to specify a file for timestamps } +type UruncNetwork struct { + DNSResolverIP string `toml:"dns_resolver_ip"` +} + type UruncConfig struct { Log UruncLog `toml:"log"` Timestamps UruncTimestamps `toml:"timestamps"` + Network UruncNetwork `toml:"network"` Monitors map[string]types.MonitorConfig `toml:"monitors"` ExtraBins map[string]types.ExtraBinConfig `toml:"extra_binaries"` } @@ -78,6 +84,12 @@ func defaultTimestampsConfig() UruncTimestamps { } } +func defaultNetworkConfig() UruncNetwork { + return UruncNetwork{ + DNSResolverIP: constants.LocalhostDNSResolverIP, + } +} + func defaultMonitorsConfig() map[string]types.MonitorConfig { return map[string]types.MonitorConfig{ "qemu": {DefaultMemoryMB: 256, DefaultVCPUs: 1}, @@ -98,6 +110,7 @@ func defaultUruncConfig() *UruncConfig { return &UruncConfig{ Log: defaultLogConfig(), Timestamps: defaultTimestampsConfig(), + Network: defaultNetworkConfig(), Monitors: defaultMonitorsConfig(), ExtraBins: defaultExtraBinConfig(), } @@ -120,6 +133,9 @@ func (p *UruncConfig) Map() map[string]string { // them to this map. this map will be used to save the rest of the urunc config to state.json cfgMap := make(map[string]string) + if p.Network.DNSResolverIP != "" { + cfgMap["urunc_config.network.dns_resolver_ip"] = p.Network.DNSResolverIP + } for hv, hvCfg := range p.Monitors { prefix := "urunc_config.monitors." + hv + "." cfgMap[prefix+"default_memory_mb"] = strconv.FormatUint(uint64(hvCfg.DefaultMemoryMB), 10) @@ -140,10 +156,14 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { // since log and timestamps are loaded at the start of urunc, we will not be reading // them from this map. this map will be used to parse the rest of the urunc config from state.json cfg := &UruncConfig{ + Network: defaultNetworkConfig(), Monitors: defaultMonitorsConfig(), ExtraBins: defaultExtraBinConfig(), } + if val, ok := cfgMap["urunc_config.network.dns_resolver_ip"]; ok && val != "" { + cfg.Network.DNSResolverIP = val + } for key, val := range cfgMap { if !strings.HasPrefix(key, "urunc_config.monitors.") { continue diff --git a/pkg/unikontainers/urunc_config_test.go b/pkg/unikontainers/urunc_config_test.go index 89328847e..9be63add0 100644 --- a/pkg/unikontainers/urunc_config_test.go +++ b/pkg/unikontainers/urunc_config_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/urunc-dev/urunc/internal/constants" "github.com/urunc-dev/urunc/pkg/unikontainers/types" ) @@ -31,6 +32,7 @@ const ( testHvtMemoryKey = "urunc_config.monitors.hvt.default_memory_mb" testVirtiofsdPathKey = "urunc_config.extra_binaries.virtiofsd.path" testVirtiofsdOptsKey = "urunc_config.extra_binaries.virtiofsd.options" + testDNSResolverIPKey = "urunc_config.network.dns_resolver_ip" testVirtiofsdDefOpts = "--cache always --sandbox none" testBinOpts = "opt1 opt2" testQemuBinaryPath = "/usr/bin/qemu" @@ -48,6 +50,19 @@ func TestUruncConfigFromMap(t *testing.T) { assert.NotNil(t, config) assert.Equal(t, defaultMonitorsConfig(), config.Monitors) assert.Equal(t, defaultExtraBinConfig(), config.ExtraBins) + assert.Equal(t, defaultNetworkConfig(), config.Network) + }) + + t.Run("network dns resolver ip is picked up", func(t *testing.T) { + t.Parallel() + cfgMap := map[string]string{ + testDNSResolverIPKey: "192.168.150.150", + } + + config := UruncConfigFromMap(cfgMap) + + assert.NotNil(t, config) + assert.Equal(t, "192.168.150.150", config.Network.DNSResolverIP) }) t.Run("single monitor with all fields", func(t *testing.T) { @@ -381,6 +396,7 @@ func TestUruncConfigMap(t *testing.T) { "urunc_config.monitors.firecracker.binary_path", testVirtiofsdPathKey, testVirtiofsdOptsKey, + testDNSResolverIPKey, } for _, key := range expectedKeys { @@ -393,6 +409,7 @@ func TestUruncConfigMap(t *testing.T) { assert.Equal(t, "", cfgMap[testQemuBinaryKey]) assert.Equal(t, "/usr/libexec/virtiofsd", cfgMap[testVirtiofsdPathKey]) assert.Equal(t, testVirtiofsdDefOpts, cfgMap[testVirtiofsdOptsKey]) + assert.Equal(t, constants.LocalhostDNSResolverIP, cfgMap[testDNSResolverIPKey]) }) t.Run("custom config produces expected map", func(t *testing.T) { diff --git a/pkg/unikontainers/utils.go b/pkg/unikontainers/utils.go index dd9fd0b58..12a1e220c 100644 --- a/pkg/unikontainers/utils.go +++ b/pkg/unikontainers/utils.go @@ -337,3 +337,14 @@ func executeHook(hook specs.Hook, state []byte) error { return nil } + +// resolvConfSource returns the host file bind-mounted over the container's +// /etc/resolv.conf, or "" when there is no such mount. +func resolvConfSource(mounts []specs.Mount) string { + for _, m := range mounts { + if m.Destination == "/etc/resolv.conf" { + return m.Source + } + } + return "" +} diff --git a/tests/e2e/common.go b/tests/e2e/common.go index e232c636f..7c43163ff 100644 --- a/tests/e2e/common.go +++ b/tests/e2e/common.go @@ -32,12 +32,14 @@ type testTool interface { setContainerID(string) createPod() (string, error) createContainer() (string, error) + createNetwork() (string, error) startContainer(bool) (string, error) runContainer(bool) (string, error) stopContainer() error stopPod() error rmContainer() error rmPod() error + rmNetwork() error logContainer() (string, error) searchContainer(string) (bool, error) searchPod(string) (bool, error) @@ -63,6 +65,7 @@ type containerTestArgs struct { Memory string Cli string Volumes []containerVolume + Network string StaticNet bool SideContainers []string Skippable bool @@ -90,6 +93,9 @@ func commonNewContainerCmd(a containerTestArgs) string { if a.Memory != "" { cmdBase += fmt.Sprintf("-m %s ", a.Memory) } + if a.Network != "" { + cmdBase += fmt.Sprintf("--network %s ", a.Network) + } if a.UID != 0 && a.GID != 0 { cmdBase += fmt.Sprintf("-u %d:%d ", a.UID, a.GID) } @@ -144,6 +150,17 @@ func commonRmImage(tool string, image string) error { return nil } +func commonNetworkCreate(tool string, network string) (string, error) { + cmdBase := tool + " network create " + network + return commonCmdExec(cmdBase) +} + +func commonNetworkRm(tool string, network string) error { + cmdBase := tool + " network rm " + network + _, err := commonCmdExec(cmdBase) + return err +} + func commonCreate(tool string, cntrArgs containerTestArgs) (output string, err error) { cmdBase := tool + " create " cmdBase += commonNewContainerCmd(cntrArgs) diff --git a/tests/e2e/crictl.go b/tests/e2e/crictl.go index 05ed0f05c..9b5226275 100644 --- a/tests/e2e/crictl.go +++ b/tests/e2e/crictl.go @@ -201,6 +201,11 @@ func (i *crictlInfo) createContainer() (string, error) { return commonCmdExec(cmdBase) } +func (i *crictlInfo) createNetwork() (string, error) { + // Not supported by crictl + return "", errToolDoesNotSupport +} + func (i *crictlInfo) startContainer(bool) (string, error) { cmdBase := crictlName cmdBase += " start " @@ -298,6 +303,11 @@ func (i *crictlInfo) rmPod() error { return nil } +func (i *crictlInfo) rmNetwork() error { + // Not supported by crictl + return errToolDoesNotSupport +} + func (i *crictlInfo) logContainer() (string, error) { return commonLogs(crictlName, i.containerID) } diff --git a/tests/e2e/ctr.go b/tests/e2e/ctr.go index 627a5c498..11e1c5fe6 100644 --- a/tests/e2e/ctr.go +++ b/tests/e2e/ctr.go @@ -96,6 +96,11 @@ func (i *ctrInfo) createContainer() (string, error) { return commonCmdExec(cmdBase) } +func (i *ctrInfo) createNetwork() (string, error) { + // Not supported by ctr + return "", errToolDoesNotSupport +} + // nolint:unused func (i *ctrInfo) startPod() (string, error) { // Not supported by ctr @@ -159,6 +164,11 @@ func (i *ctrInfo) rmPod() error { return errToolDoesNotSupport } +func (i *ctrInfo) rmNetwork() error { + // Not supported by ctr + return errToolDoesNotSupport +} + func (i *ctrInfo) logContainer() (string, error) { // Not supported by ctr // TODO: We need to fix this diff --git a/tests/e2e/docker.go b/tests/e2e/docker.go index 9c0982a9c..0496d9d19 100644 --- a/tests/e2e/docker.go +++ b/tests/e2e/docker.go @@ -66,6 +66,10 @@ func (i *dockerInfo) createContainer() (string, error) { return commonCreate(dockerName, i.testArgs) } +func (i *dockerInfo) createNetwork() (string, error) { + return commonNetworkCreate(dockerName, i.testArgs.Network) +} + // nolint:unused func (i *dockerInfo) startPod() (string, error) { // Not supported by docker @@ -108,6 +112,10 @@ func (i *dockerInfo) rmPod() error { return errToolDoesNotSupport } +func (i *dockerInfo) rmNetwork() error { + return commonNetworkRm(dockerName, i.testArgs.Network) +} + func (i *dockerInfo) logContainer() (string, error) { return commonLogs(dockerName, i.containerID) } diff --git a/tests/e2e/docker_test.go b/tests/e2e/docker_test.go index 4d61ba07f..20e94ac15 100644 --- a/tests/e2e/docker_test.go +++ b/tests/e2e/docker_test.go @@ -27,7 +27,6 @@ var _ = Describe("Docker", Ordered, ContinueOnFailure, func() { images := getTestImages(cases) err := pullAllImages(testDocker, images) Expect(err).NotTo(HaveOccurred(), "Failed to pull docker images") - DeferCleanup(func() { removeAllImages(testDocker, images) }) @@ -43,12 +42,25 @@ var _ = Describe("Docker", Ordered, ContinueOnFailure, func() { } }) - DescribeTable("unikernel containers", - func(tc containerTestArgs) { - skipMissingVolumes(tc) - tool = newDockerTool(tc) - runDetachedTest(tool, tc) - }, - toTableEntries(dockerTestCases()), - ) + Context("foreground containers", func() { + DescribeTable("unikernel containers", + func(tc containerTestArgs) { + skipMissingVolumes(tc) + tool = newDockerTool(tc) + runForegroundTest(tool, tc) + }, + toTableEntries(selectTestCases(dockerTestCases(), false)), + ) + }) + + Context("detached containers", func() { + DescribeTable("unikernel containers", + func(tc containerTestArgs) { + skipMissingVolumes(tc) + tool = newDockerTool(tc) + runDetachedTest(tool, tc) + }, + toTableEntries(selectTestCases(dockerTestCases(), true)), + ) + }) }) diff --git a/tests/e2e/nerdctl.go b/tests/e2e/nerdctl.go index 024e83791..5ebd5704e 100644 --- a/tests/e2e/nerdctl.go +++ b/tests/e2e/nerdctl.go @@ -66,6 +66,10 @@ func (i *nerdctlInfo) createContainer() (string, error) { return commonCreate(nerdctlName, i.testArgs) } +func (i *nerdctlInfo) createNetwork() (string, error) { + return commonNetworkCreate(nerdctlName, i.testArgs.Network) +} + // nolint:unused func (i *nerdctlInfo) startPod() (string, error) { // Not supported by nerdctl @@ -108,6 +112,9 @@ func (i *nerdctlInfo) rmPod() error { return errToolDoesNotSupport } +func (i *nerdctlInfo) rmNetwork() error { + return commonNetworkRm(nerdctlName, i.testArgs.Network) +} func (i *nerdctlInfo) logContainer() (string, error) { return commonLogs(nerdctlName, i.containerID) } diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go index 03d7db82e..c58cc1c9e 100644 --- a/tests/e2e/suite_test.go +++ b/tests/e2e/suite_test.go @@ -97,6 +97,12 @@ func captureContainerLogs(tool testTool) { // runDetachedTest runs a container in detached mode: create, start, and // verify via TestFunc. func runDetachedTest(tool testTool, tc containerTestArgs) { + if tc.Network != "" { + By("Creating user-defined network") + output, err := tool.createNetwork() + Expect(err).NotTo(HaveOccurred(), "Failed to create network %s: %s", tc.Network, output) + } + By("Creating container") cID, err := tool.createContainer() Expect(err).NotTo(HaveOccurred(), "Failed to create container: %s", cID) @@ -120,6 +126,12 @@ func runDetachedTest(tool testTool, tc containerTestArgs) { GinkgoLogr.Error(err, "Failed to verify removal") } } + if tc.Network != "" { + By("Removing user-defined network") + if err := tool.rmNetwork(); err != nil { + GinkgoLogr.Error(err, "Failed to remove network "+tc.Network) + } + } }) By("Starting container") @@ -137,6 +149,12 @@ func runDetachedTest(tool testTool, tc containerTestArgs) { func runForegroundTest(tool testTool, tc containerTestArgs) { tool.setContainerID(tc.Name) + if tc.Network != "" { + By("Creating user-defined network") + output, err := tool.createNetwork() + Expect(err).NotTo(HaveOccurred(), "Failed to create network %s: %s", tc.Network, output) + } + DeferCleanup(func() { if tool.getContainerID() != "" { By("Cleaning up container") diff --git a/tests/e2e/test_cases.go b/tests/e2e/test_cases.go index 31bec3d88..2b52f5962 100644 --- a/tests/e2e/test_cases.go +++ b/tests/e2e/test_cases.go @@ -984,6 +984,24 @@ func dockerTestCases() []containerTestArgs { Skippable: false, TestFunc: pingTest, }, + { + Image: "harbor.nbfc.io/nubificus/urunc/busybox-qemu-linux-raw:latest", + Name: "Qemu-linux-docker-user-defined-network-external", + Devmapper: false, + Seccomp: true, + UID: 0, + GID: 0, + Groups: []int64{}, + Memory: "", + Cli: "/bin/nslookup google-public-dns-a.google.com", + Volumes: []containerVolume{}, + Network: "urunc-dns-test", + StaticNet: false, + SideContainers: []string{}, + Skippable: false, + ExpectOut: "Address: 8.8.8.8", + TestFunc: matchTest, + }, { Image: "harbor.nbfc.io/nubificus/urunc/net-spt-mirage:latest", Name: "Spt-mirage-UserGroup", diff --git a/tests/e2e/test_functions.go b/tests/e2e/test_functions.go index 9e205c49e..fb68c5707 100644 --- a/tests/e2e/test_functions.go +++ b/tests/e2e/test_functions.go @@ -59,6 +59,10 @@ func testCleanup(tool testTool) error { return fmt.Errorf("Failed to remove container: %v", err) } + err = tool.rmNetwork() + if err != nil { + return fmt.Errorf("Failed to remove network: %v", err) + } return testVerifyRm(tool) }