diff --git a/.github/workflows/build-kubectl-image.yaml b/.github/workflows/build-kubectl-image.yaml index f685d552b5a4..b36ed2834d77 100644 --- a/.github/workflows/build-kubectl-image.yaml +++ b/.github/workflows/build-kubectl-image.yaml @@ -8,7 +8,7 @@ on: # Allow manual triggering push: branches: - - main + - msft-preview paths: - 'tools/packaging/kubectl/Dockerfile' - '.github/workflows/build-kubectl-image.yaml' diff --git a/.github/workflows/ci-on-push.yaml b/.github/workflows/ci-on-push.yaml index e9ca81a0b482..c4073f8f3a63 100644 --- a/.github/workflows/ci-on-push.yaml +++ b/.github/workflows/ci-on-push.yaml @@ -1,13 +1,19 @@ name: Kata Containers CI on: - pull_request_target: # zizmor: ignore[dangerous-triggers] See #11332. + # Upstream uses `pull_request_target` to have access to secrets for + # PRs from forks but: + # (1) `pull_request_target` only runs on the default branch and we + # have multiple leading branches, so we need to use `pull_request`. + # (2) We can use `pull_request` practically since we don't expect PRs + # from external contributors. + pull_request: branches: - - 'main' + - 'msft-preview' types: # Adding 'labeled' to the list of activity types that trigger this event # (default: opened, synchronize, reopened) so that we can run this # workflow when the 'ok-to-test' label is added. - # Reference: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target + # Reference: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request - opened - synchronize - reopened diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1c8d849d625b..deb9d6457e0e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -108,6 +108,7 @@ jobs: QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }} build-kata-static-tarball-s390x: + if: false # msft-preview doesn't have these runners permissions: contents: read packages: write @@ -123,6 +124,7 @@ jobs: QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }} build-kata-static-tarball-ppc64le: + if: false # msft-preview doesn't have these runners permissions: contents: read packages: write @@ -172,6 +174,7 @@ jobs: build-and-publish-tee-confidential-unencrypted-image: name: build-and-publish-tee-confidential-unencrypted-image + if: false # msft-preview can't push to GHCR permissions: contents: read packages: write @@ -277,7 +280,7 @@ jobs: target-branch: ${{ inputs.target-branch }} run-k8s-tests-on-aks: - if: ${{ inputs.skip-test != 'yes' }} + if: false # msft-preview doesn't test on AKS yet needs: publish-kata-deploy-payload-amd64 uses: ./.github/workflows/run-k8s-tests-on-aks.yaml @@ -298,7 +301,7 @@ jobs: AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }} run-k8s-tests-on-arm64: - if: ${{ inputs.skip-test != 'yes' }} + if: false # msft-preview doesn't have these runners needs: publish-kata-deploy-payload-arm64 uses: ./.github/workflows/run-k8s-tests-on-arm64.yaml with: @@ -310,7 +313,7 @@ jobs: target-branch: ${{ inputs.target-branch }} run-k8s-tests-on-nvidia-gpu: - if: ${{ inputs.skip-test != 'yes' }} + if: false # msft-preview doesn't have these runners needs: publish-kata-deploy-payload-amd64 uses: ./.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml with: @@ -365,7 +368,7 @@ jobs: AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }} run-k8s-tests-on-ppc64le: - if: ${{ inputs.skip-test != 'yes' }} + if: false # msft-preview doesn't have these runners needs: publish-kata-deploy-payload-ppc64le uses: ./.github/workflows/run-k8s-tests-on-ppc64le.yaml with: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 4730cc74d0c8..e3209b1b1ae8 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -13,9 +13,9 @@ name: "CodeQL Advanced" on: push: - branches: [ "main" ] + branches: [ "msft-preview", "release/*" ] pull_request: - branches: [ "main" ] + branches: [ "msft-preview", "release/*" ] schedule: - cron: '45 0 * * 1' diff --git a/.github/workflows/commit-message-check.yaml b/.github/workflows/commit-message-check.yaml index 469dd91bad4c..0f4695d3a6bf 100644 --- a/.github/workflows/commit-message-check.yaml +++ b/.github/workflows/commit-message-check.yaml @@ -16,7 +16,7 @@ env: error_msg: |+ See the document below for help on formatting commits for the project. - https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md#patch-format + https://github.com/kata-containers/community/blob/msft-preview/CONTRIBUTING.md#patch-format jobs: commit-message-check: diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index e101d1ee144e..9ede371b4fa1 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -2,7 +2,7 @@ name: Documentation on: push: branches: - - main + - msft-preview permissions: {} jobs: deploy-docs: diff --git a/.github/workflows/gatekeeper.yaml b/.github/workflows/gatekeeper.yaml index 9854f3b5ea1c..b930e3592580 100644 --- a/.github/workflows/gatekeeper.yaml +++ b/.github/workflows/gatekeeper.yaml @@ -5,7 +5,13 @@ name: Gatekeeper # reporting the status. on: - pull_request_target: # zizmor: ignore[dangerous-triggers] See #11332. + # Upstream uses `pull_request_target` to have access to secrets for + # PRs from forks but: + # (1) `pull_request_target` only runs on the default branch and we + # have multiple leading branches, so we need to use `pull_request`. + # (2) We can use `pull_request` practically since we don't expect PRs + # from external contributors. + pull_request: types: - opened - synchronize diff --git a/.github/workflows/osv-scanner.yaml b/.github/workflows/osv-scanner.yaml index 7c3c730e118b..4413d74739c4 100644 --- a/.github/workflows/osv-scanner.yaml +++ b/.github/workflows/osv-scanner.yaml @@ -9,11 +9,11 @@ name: OSV-Scanner on: workflow_dispatch: pull_request: - branches: [ "main" ] + branches: [ "msft-preview" ] schedule: - cron: '0 1 * * 0' push: - branches: [ "main" ] + branches: [ "msft-preview" ] permissions: {} diff --git a/.github/workflows/payload-after-push.yaml b/.github/workflows/payload-after-push.yaml index 891ccea029fc..4a88dc139107 100644 --- a/.github/workflows/payload-after-push.yaml +++ b/.github/workflows/payload-after-push.yaml @@ -2,7 +2,7 @@ name: CI | Publish Kata Containers payload on: push: branches: - - main + - msft-preview workflow_dispatch: permissions: {} diff --git a/.github/workflows/push-oras-tarball-cache.yaml b/.github/workflows/push-oras-tarball-cache.yaml index 1243f32959d8..a52fb3ce3894 100644 --- a/.github/workflows/push-oras-tarball-cache.yaml +++ b/.github/workflows/push-oras-tarball-cache.yaml @@ -1,11 +1,11 @@ # Push gperf and busybox tarballs to the ORAS cache (ghcr.io) so that # download-with-oras-cache.sh can pull them instead of hitting upstream. -# Runs when versions.yaml changes on main (e.g. after a PR merge) or manually. +# Runs when versions.yaml changes on msft-preview (e.g. after a PR merge) or manually. name: CI | Push ORAS tarball cache on: push: branches: - - main + - msft-preview paths: - 'versions.yaml' workflow_dispatch: diff --git a/.github/workflows/release-amd64.yaml b/.github/workflows/release-amd64.yaml index 9707260d7dba..fb1a47c8836b 100644 --- a/.github/workflows/release-amd64.yaml +++ b/.github/workflows/release-amd64.yaml @@ -66,7 +66,7 @@ jobs: # We need to do such trick here as the format of the $GITHUB_REF # is "refs/tags/" tag=$(echo "$GITHUB_REF" | cut -d/ -f3-) - if [ "${tag}" = "main" ]; then + if [ "${tag}" = "msft-preview" ]; then tag=$(./tools/packaging/release/release.sh release-version) tags=("${tag}" "latest") else diff --git a/.github/workflows/release-arm64.yaml b/.github/workflows/release-arm64.yaml index e81d83d54f74..874d22e2f063 100644 --- a/.github/workflows/release-arm64.yaml +++ b/.github/workflows/release-arm64.yaml @@ -66,7 +66,7 @@ jobs: # We need to do such trick here as the format of the $GITHUB_REF # is "refs/tags/" tag=$(echo "$GITHUB_REF" | cut -d/ -f3-) - if [ "${tag}" = "main" ]; then + if [ "${tag}" = "msft-preview" ]; then tag=$(./tools/packaging/release/release.sh release-version) tags=("${tag}" "latest") else diff --git a/.github/workflows/release-ppc64le.yaml b/.github/workflows/release-ppc64le.yaml index 2a63579ff9ee..4260554e87a2 100644 --- a/.github/workflows/release-ppc64le.yaml +++ b/.github/workflows/release-ppc64le.yaml @@ -63,7 +63,7 @@ jobs: # We need to do such trick here as the format of the $GITHUB_REF # is "refs/tags/" tag=$(echo "$GITHUB_REF" | cut -d/ -f3-) - if [ "${tag}" = "main" ]; then + if [ "${tag}" = "msft-preview" ]; then tag=$(./tools/packaging/release/release.sh release-version) tags=("${tag}" "latest") else diff --git a/.github/workflows/release-s390x.yaml b/.github/workflows/release-s390x.yaml index 8f86d63e70fc..585add2e113b 100644 --- a/.github/workflows/release-s390x.yaml +++ b/.github/workflows/release-s390x.yaml @@ -67,7 +67,7 @@ jobs: # We need to do such trick here as the format of the $GITHUB_REF # is "refs/tags/" tag=$(echo "$GITHUB_REF" | cut -d/ -f3-) - if [ "${tag}" = "main" ]; then + if [ "${tag}" = "msft-preview" ]; then tag=$(./tools/packaging/release/release.sh release-version) tags=("${tag}" "latest") else diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml index bbe22d5a826f..93e0994d62ab 100644 --- a/.github/workflows/scorecard.yaml +++ b/.github/workflows/scorecard.yaml @@ -5,10 +5,10 @@ name: Scorecard supply-chain security on: # For Branch-Protection check. Only the default branch is supported. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + # https://github.com/ossf/scorecard/blob/msft-preview/docs/checks.md#branch-protection branch_protection_rule: push: - branches: [ "main" ] + branches: [ "msft-preview" ] workflow_dispatch: permissions: {} diff --git a/.github/workflows/static-checks-self-hosted.yaml b/.github/workflows/static-checks-self-hosted.yaml index b654764c737e..a4db44251de1 100644 --- a/.github/workflows/static-checks-self-hosted.yaml +++ b/.github/workflows/static-checks-self-hosted.yaml @@ -23,7 +23,7 @@ jobs: build-checks: needs: skipper - if: ${{ needs.skipper.outputs.skip_static != 'yes' }} + if: false # msft-preview doesn't have these runners strategy: fail-fast: false matrix: diff --git a/.github/workflows/static-checks.yaml b/.github/workflows/static-checks.yaml index fe605e632d8e..aabe047e2a34 100644 --- a/.github/workflows/static-checks.yaml +++ b/.github/workflows/static-checks.yaml @@ -110,6 +110,7 @@ jobs: - "make static-checks" env: GOPATH: ${{ github.workspace }} + target_branch: msft-preview permissions: contents: read # for checkout packages: write # for push to ghcr.io diff --git a/.gitignore b/.gitignore index 252e072f2ee4..d7b2f49a6af1 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,23 @@ tools/packaging/static-build/agent/install_libseccomp.sh .direnv **/.DS_Store site/ + +# Microsoft-specific +.cargo/ +src/agent/samples/policy/test-input/ +src/tarfs/**/*.cmd +src/tarfs/**/*.ko +src/tarfs/**/*.mod +src/tarfs/**/*.mod.c +src/tarfs/**/*.o +src/tarfs/**/modules.order +src/tarfs/**/Module.symvers +src/tarfs-cvm/ +tools/osbuilder/kata-containers-igvm.img +tools/osbuilder/kata-containers-igvm-debug.img +tools/osbuilder/igvm-debug-measurement.cose +tools/osbuilder/igvm-measurement.cose +tools/osbuilder/root_hash.txt +tools/osbuilder/igvm.log +tools/osbuilder/kata-opa.service +tools/osbuilder/rootfs-builder/opa/ diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000000..b3c89efc852e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,41 @@ + + +## Security + +Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin). + +If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. + +## Reporting Security Issues + +**Please do not report security vulnerabilities through public GitHub issues.** + +Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). + +If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). + +You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). + +Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: + + * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) + * Full paths of source file(s) related to the manifestation of the issue + * The location of the affected source code (tag/branch/commit or direct URL) + * Any special configuration required to reproduce the issue + * Step-by-step instructions to reproduce the issue + * Proof-of-concept or exploit code (if possible) + * Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. + +If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. + +## Preferred Languages + +We prefer all communications to be in English. + +## Policy + +Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). + + diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/endpoint_persist.rs b/src/runtime-rs/crates/resource/src/network/endpoint/endpoint_persist.rs index 31507c209f9a..dbd32b6d69d4 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/endpoint_persist.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/endpoint_persist.rs @@ -13,6 +13,10 @@ pub struct PhysicalEndpointState { pub vendor_id: String, pub device_id: String, pub hard_addr: String, + pub is_vf: bool, + pub iface_name: String, + /// "pci" or "vmbus" + pub bus_type: String, } #[derive(Serialize, Deserialize, Clone, Default)] diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs index 9bb1dbbcf9b6..bd2c40134379 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs @@ -4,21 +4,23 @@ // SPDX-License-Identifier: Apache-2.0 // +use std::io::{self, Error}; use std::path::Path; use std::sync::Arc; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use hypervisor::device::device_manager::{do_handle_device, DeviceManager}; -use hypervisor::device::DeviceConfig; -use hypervisor::{device::driver, Hypervisor}; +use hypervisor::device::driver::NetworkConfig; +use hypervisor::device::{DeviceConfig, DeviceType}; +use hypervisor::{device::driver, Hypervisor, NetworkDevice}; use hypervisor::{get_vfio_device, VfioConfig}; use tokio::sync::RwLock; use super::endpoint_persist::{EndpointState, PhysicalEndpointState}; use super::Endpoint; +use crate::network::network_pair::NetworkPair; use crate::network::utils::{self, link}; -pub const SYS_PCI_DEVICES_PATH: &str = "/sys/bus/pci/devices"; #[derive(Debug)] pub struct VendorDevice { @@ -53,29 +55,68 @@ pub struct PhysicalEndpoint { bdf: String, driver: String, vendor_device_id: VendorDevice, + is_vf: bool, + bus_type: link::BusType, + net_pair: NetworkPair, d: Arc>, } impl PhysicalEndpoint { - pub fn new(name: &str, hardware_addr: &[u8], d: Arc>) -> Result { - let driver_info = link::get_driver_info(name).context("get driver info")?; - let bdf = driver_info.bus_info; - let sys_pci_devices_path = Path::new(SYS_PCI_DEVICES_PATH); - // get driver by following symlink /sys/bus/pci/devices/$bdf/driver - let driver_path = sys_pci_devices_path.join(&bdf).join("driver"); - let link = driver_path.read_link().context("read link")?; - let driver = link + /// Create a new PhysicalEndpoint. + /// + /// For VF (SR-IOV Virtual Function) devices, the NIC will be passed through + /// to the VM via VFIO — no tap/bridge pair is needed, so we create a minimal + /// stub NetworkPair. + /// + /// For non-VF physical NICs, we need a real tap+bridge pair (like veth + /// endpoints use) so that traffic can flow between the host NIC and the VM + /// through the hypervisor's TAP backend. The `handle`, `idx`, `model`, and + /// `queues` parameters are required to create this pair. + pub async fn new( + handle: &rtnetlink::Handle, + name: &str, + hardware_addr: &[u8], + idx: u32, + model: &str, + queues: usize, + d: Arc>, + ) -> Result { + // Determine bus type (PCI or VMBus) and resolve the sysfs device path. + // For PCI: uses ethtool to get BDF, path = /sys/bus/pci/devices/ + // For VMBus: resolves device symlink, path = /sys/bus/vmbus/devices/ + let (sys_iface_device_path, bdf, bus_type) = + link::get_iface_device_path(name).context("get iface device path")?; + let sys_device_path = Path::new(&sys_iface_device_path); + + // Get driver by following symlink /driver + let driver_path = sys_device_path.join("driver"); + let link_target = driver_path.read_link().context("read link")?; + let driver = link_target .file_name() .map_or(String::new(), |v| v.to_str().unwrap().to_owned()); - // get vendor and device id from pci space (sys/bus/pci/devices/$bdf) - let iface_device_path = sys_pci_devices_path.join(&bdf).join("device"); + // Get vendor and device id from sysfs device path + let iface_device_path = sys_device_path.join("device"); let device_id = std::fs::read_to_string(&iface_device_path) .with_context(|| format!("read device path {:?}", &iface_device_path))?; - let iface_vendor_path = sys_pci_devices_path.join(&bdf).join("vendor"); + let iface_vendor_path = sys_device_path.join("vendor"); let vendor_id = std::fs::read_to_string(&iface_vendor_path) .with_context(|| format!("read vendor path {:?}", &iface_vendor_path))?; + let is_vf = link::is_vf(name).context("check if is vf")?; + + // VF devices use VFIO passthrough — no real tap/bridge needed. + // Non-VF devices need a real tap+bridge pair for the hypervisor to + // connect the host NIC traffic into the VM, matching Go's + // createNetworkInterfacePair() call in createPhysicalEndpoint(). + let net_pair = if is_vf { + NetworkPair::new_for_physical(name, hardware_addr, is_vf) + .context("new network pair for physical vf endpoint")? + } else { + NetworkPair::new(handle, idx, name, model, queues) + .await + .context("new network pair for physical non-vf endpoint")? + }; Ok(Self { iface_name: name.to_string(), @@ -84,9 +125,34 @@ impl PhysicalEndpoint { .context("new vendor device")?, driver, bdf, + is_vf, + bus_type, + net_pair, d, }) } + + #[allow(dead_code)] + pub fn network_pair(&self) -> &NetworkPair { + &self.net_pair + } + + fn get_network_config(&self) -> Result { + let iface = &self.net_pair.tap.tap_iface; + let guest_mac = utils::parse_mac(&iface.hard_addr).ok_or_else(|| { + Error::new( + io::ErrorKind::InvalidData, + format!("hard_addr {}", &iface.hard_addr), + ) + })?; + + Ok(NetworkConfig { + host_dev_name: iface.name.clone(), + virt_iface_name: self.net_pair.virt_iface.name.clone(), + guest_mac: Some(guest_mac), + ..Default::default() + }) + } } #[async_trait] @@ -100,51 +166,74 @@ impl Endpoint for PhysicalEndpoint { } async fn attach(&self) -> Result<()> { - // bind physical interface from host driver and bind to vfio - driver::bind_device_to_vfio( - &self.bdf, - &self.driver, - &self.vendor_device_id.vendor_device_id(), - ) - .with_context(|| format!("bind physical endpoint from {} to vfio", &self.driver))?; - - let vfio_device = get_vfio_device(self.bdf.clone()).context("get vfio device failed.")?; - let vfio_dev_config = &mut VfioConfig { - host_path: vfio_device.clone(), - dev_type: "pci".to_string(), - hostdev_prefix: "physical_nic_".to_owned(), - ..Default::default() - }; + if self.is_vf { + // bind physical interface from host driver and bind to vfio + driver::bind_device_to_vfio( + &self.bdf, + &self.driver, + &self.vendor_device_id.vendor_device_id(), + ) + .with_context(|| format!("bind physical endpoint from {} to vfio", &self.driver))?; - // create and insert VFIO device into Kata VM - do_handle_device(&self.d, &DeviceConfig::VfioCfg(vfio_dev_config.clone())) - .await - .context("do handle device failed.")?; + let vfio_device = + get_vfio_device(self.bdf.clone()).context("get vfio device failed.")?; + let vfio_dev_config = &mut VfioConfig { + host_path: vfio_device.clone(), + dev_type: "pci".to_string(), + hostdev_prefix: "physical_nic_".to_owned(), + ..Default::default() + }; - Ok(()) + // create and insert VFIO device into Kata VM + do_handle_device(&self.d, &DeviceConfig::VfioCfg(vfio_dev_config.clone())) + .await + .context("do handle device failed.")?; + + Ok(()) + } else { + self.net_pair + .add_network_model() + .await + .context("add network model")?; + let config = self.get_network_config().context("get network config")?; + do_handle_device(&self.d, &DeviceConfig::NetworkCfg(config)) + .await + .context("do handle network Physical endpoint device failed.")?; + Ok(()) + } } // detach for physical endpoint unbinds the physical network interface from vfio-pci // and binds it back to the saved host driver. async fn detach(&self, _hypervisor: &dyn Hypervisor) -> Result<()> { - // bind back the physical network interface to host. - // we need to do this even if a new network namespace has not - // been created by virt-containers. - - // we do not need to enter the network namespace to bind back the - // physical interface to host driver. - driver::bind_device_to_host( - &self.bdf, - &self.driver, - &self.vendor_device_id.vendor_device_id(), - ) - .with_context(|| { - format!( - "bind physical endpoint device from vfio to {}", - &self.driver + if self.is_vf { + driver::bind_device_to_host( + &self.bdf, + &self.driver, + &self.vendor_device_id.vendor_device_id(), ) - })?; - Ok(()) + .with_context(|| { + format!( + "bind physical endpoint device from vfio to {}", + &self.driver + ) + })?; + Ok(()) + } else { + self.net_pair + .del_network_model() + .await + .context("del network model")?; + let config = self.get_network_config().context("get network config")?; + _hypervisor + .remove_device(DeviceType::Network(NetworkDevice { + config, + ..Default::default() + })) + .await + .context("remove Physical endpoint device by hypervisor failed.")?; + Ok(()) + } } async fn save(&self) -> Option { @@ -155,6 +244,12 @@ impl Endpoint for PhysicalEndpoint { vendor_id: self.vendor_device_id.vendor_id.clone(), device_id: self.vendor_device_id.device_id.clone(), hard_addr: self.hard_addr.clone(), + is_vf: self.is_vf, + iface_name: self.iface_name.clone(), + bus_type: match self.bus_type { + link::BusType::Pci => "pci".to_string(), + link::BusType::Vmbus => "vmbus".to_string(), + }, }), ..Default::default() }) diff --git a/src/runtime-rs/crates/resource/src/network/network_pair.rs b/src/runtime-rs/crates/resource/src/network/network_pair.rs index 8be2d392b722..d7f5e57ad444 100644 --- a/src/runtime-rs/crates/resource/src/network/network_pair.rs +++ b/src/runtime-rs/crates/resource/src/network/network_pair.rs @@ -39,6 +39,35 @@ pub struct NetworkPair { } impl NetworkPair { + pub(crate) fn new_for_physical(name: &str, hardware_addr: &[u8], is_vf: bool) -> Result { + let unique_id = kata_sys_util::rand::UUID::new(); + let model = network_model::new("none").context("new none model")?; + let hard_addr = utils::get_mac_addr(hardware_addr).context("get mac addr")?; + + Ok(Self { + tap: TapInterface { + id: String::from(&unique_id), + name: String::new(), + tap_iface: NetworkInterface { + name: if is_vf { + String::new() + } else { + name.to_string() + }, + hard_addr: hard_addr.clone(), + ..Default::default() + }, + }, + virt_iface: NetworkInterface { + name: name.to_string(), + hard_addr, + ..Default::default() + }, + model, + network_qos: false, + }) + } + pub(crate) async fn new( handle: &rtnetlink::Handle, idx: u32, @@ -266,4 +295,33 @@ mod tests { } } } + + #[test] + fn test_new_for_physical_vf() { + let name = "enp3s0f1"; + let hardware_addr = &[0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff]; + let pair = NetworkPair::new_for_physical(name, hardware_addr, true).unwrap(); + + // VF: tap_iface.name should be empty (VFIO passthrough, no tap needed) + assert!(pair.tap.tap_iface.name.is_empty()); + // virt_iface should still carry the interface name + assert_eq!(pair.virt_iface.name, name); + // MAC addresses should be set on both sides + assert!(!pair.tap.tap_iface.hard_addr.is_empty()); + assert_eq!(pair.tap.tap_iface.hard_addr, pair.virt_iface.hard_addr); + assert!(!pair.network_qos); + } + + #[test] + fn test_new_for_physical_non_vf() { + let name = "enp4s0"; + let hardware_addr = &[0x11, 0x22, 0x33, 0x44, 0x55, 0x66]; + let pair = NetworkPair::new_for_physical(name, hardware_addr, false).unwrap(); + + // Non-VF: tap_iface.name should be the interface name (used by hypervisor) + assert_eq!(pair.tap.tap_iface.name, name); + assert_eq!(pair.virt_iface.name, name); + assert!(!pair.tap.tap_iface.hard_addr.is_empty()); + assert_eq!(pair.tap.tap_iface.hard_addr, pair.virt_iface.hard_addr); + } } diff --git a/src/runtime-rs/crates/resource/src/network/network_with_netns.rs b/src/runtime-rs/crates/resource/src/network/network_with_netns.rs index a872cf97f280..9aee40bcf1c9 100644 --- a/src/runtime-rs/crates/resource/src/network/network_with_netns.rs +++ b/src/runtime-rs/crates/resource/src/network/network_with_netns.rs @@ -231,8 +231,17 @@ async fn create_endpoint( &attrs.name, nix::unistd::gettid() ); - let t = PhysicalEndpoint::new(&attrs.name, &attrs.hardware_addr, d) - .context("new physical endpoint")?; + let t = PhysicalEndpoint::new( + handle, + &attrs.name, + &attrs.hardware_addr, + idx, + &config.network_model, + config.queues, + d, + ) + .await + .context("new physical endpoint")?; Arc::new(t) } else { info!( @@ -297,9 +306,7 @@ fn is_physical_iface(name: &str) -> Result { if name == "lo" { return Ok(false); } - let driver_info = link::get_driver_info(name)?; - if driver_info.bus_info.split(':').count() != 3 { - return Ok(false); - } - Ok(true) + // A physical interface is one backed by a PCI or VMBus device. + // This mirrors Go's isPhysicalIface which checks ParentDevBus == "pci" || "vmbus". + Ok(link::get_bus_type(name)?.is_some()) } diff --git a/src/runtime-rs/crates/resource/src/network/utils/link/mod.rs b/src/runtime-rs/crates/resource/src/network/utils/link/mod.rs index 86027f2fee2f..367e07068222 100644 --- a/src/runtime-rs/crates/resource/src/network/utils/link/mod.rs +++ b/src/runtime-rs/crates/resource/src/network/utils/link/mod.rs @@ -12,6 +12,97 @@ mod macros; mod manager; pub use manager::get_link_from_message; +use std::path::Path; + +use anyhow::{anyhow, Context, Result}; + +/// The bus type backing a physical network interface. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BusType { + Pci, + Vmbus, +} + +/// Determines the bus type of a network interface by reading the +/// `/sys/class/net//device/subsystem` symlink. +/// +/// Returns `Some(BusType)` for PCI or VMBus devices, `None` for virtual +/// interfaces that have no backing hardware device (veth, bridge, etc.). +pub fn get_bus_type(name: &str) -> Result> { + let subsystem_path = Path::new("/sys/class/net") + .join(name) + .join("device/subsystem"); + match std::fs::read_link(&subsystem_path) { + Ok(target) => { + let bus_name = target.file_name().and_then(|f| f.to_str()).unwrap_or(""); + match bus_name { + "pci" => Ok(Some(BusType::Pci)), + "vmbus" => Ok(Some(BusType::Vmbus)), + _ => Ok(None), + } + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(anyhow!(e).context(format!( + "failed to read subsystem symlink for interface {}", + name + ))), + } +} + +/// Returns the sysfs device path and device identifier for a network interface. +/// +/// For PCI devices, uses ethtool to get the BDF (e.g. "0000:01:00.0") and +/// returns `/sys/bus/pci/devices/`. +/// +/// For VMBus devices, resolves `/sys/class/net//device` to get the +/// device GUID and returns `/sys/bus/vmbus/devices/`. +/// +/// This mirrors Go's `getIfaceDevicePath()` which branches on +/// `link.Attrs().ParentDevBus`. +pub fn get_iface_device_path(name: &str) -> Result<(String, String, BusType)> { + let bus_type = get_bus_type(name)? + .ok_or_else(|| anyhow!("unsupported or no bus for interface {}", name))?; + + match bus_type { + BusType::Pci => { + let driver_info = get_driver_info(name).context("get driver info")?; + let bdf = driver_info.bus_info; + let path = Path::new("/sys/bus/pci/devices").join(&bdf); + Ok((path.to_string_lossy().into_owned(), bdf, BusType::Pci)) + } + BusType::Vmbus => { + let device_link = Path::new("/sys/class/net").join(name).join("device"); + let resolved = + std::fs::canonicalize(&device_link).context("resolve vmbus device path")?; + let guid = resolved + .file_name() + .and_then(|f| f.to_str()) + .ok_or_else(|| anyhow!("failed to get vmbus device guid"))? + .to_string(); + let path = Path::new("/sys/bus/vmbus/devices").join(&guid); + Ok((path.to_string_lossy().into_owned(), guid, BusType::Vmbus)) + } + } +} + +/// Checks whether a network interface is an SR-IOV Virtual Function (VF). +/// +/// Only PCI devices can be VFs — VMBus devices are always non-VF. +/// For PCI, checks for the `physfn` symlink under the device's sysfs directory. +pub fn is_vf(name: &str) -> Result { + match get_bus_type(name)? { + Some(BusType::Pci) => { + let driver_info = get_driver_info(name).context("get driver info")?; + let bdf = &driver_info.bus_info; + let physfn_path = Path::new("/sys/bus/pci/devices").join(bdf).join("physfn"); + Ok(std::fs::read_link(&physfn_path).is_ok()) + } + // VMBus devices do not support SR-IOV. + Some(BusType::Vmbus) => Ok(false), + None => Ok(false), + } +} + use std::os::unix::io::RawFd; use netlink_packet_route::link::State; diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 31c65627d3eb..094bdb722974 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -201,11 +201,11 @@ STRATOVIRTPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTCMD) STRATOVIRTVALIDHYPERVISORPATHS := [\"$(STRATOVIRTPATH)\"] # Default number of vCPUs -DEFVCPUS := 1 +DEFVCPUS ?= 1 # Default maximum number of vCPUs DEFMAXVCPUS := 0 # Default memory size in MiB -DEFMEMSZ := 2048 +DEFMEMSZ ?= 2048 # Default memory slots # Cases to consider : # - nvdimm rootfs image @@ -286,6 +286,15 @@ DEFSTATICRESOURCEMGMT ?= false DEFSTATICRESOURCEMGMT_TEE = true DEFSTATICRESOURCEMGMT_NV = true +# Default memory and vcpus for use for workloads within the sandbox if no specific workload values are requested +DEFSTATICSANDBOXWORKLOADMEM ?= 2048 +DEFSTATICSANDBOXWORKLOADVCPUS ?= 1 + +# If set, the runtime will enforce that pods deployed in a sandbox +# explicitly setting memory limits using resources.limits.memory +# allow at least this amount of memory in MiB so that the sandbox can properly start. +DEFSANDBOXWORKLOADMEMMIN ?= 128 + DEFDISABLEIMAGENVDIMM ?= false DEFDISABLEIMAGENVDIMM_NV = true DEFDISABLEIMAGENVDIMM_CLH ?= true @@ -780,6 +789,9 @@ USER_VARS += DEFSTATICRESOURCEMGMT_FC USER_VARS += DEFSTATICRESOURCEMGMT_STRATOVIRT USER_VARS += DEFSTATICRESOURCEMGMT_TEE USER_VARS += DEFSTATICRESOURCEMGMT_NV +USER_VARS += DEFSTATICSANDBOXWORKLOADMEM +USER_VARS += DEFSANDBOXWORKLOADMEMMIN +USER_VARS += DEFSTATICSANDBOXWORKLOADVCPUS USER_VARS += DEFBINDMOUNTS USER_VARS += DEFCREATECONTAINERTIMEOUT USER_VARS += DEFDANCONF diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index 8718b56f3988..1e5ca9aa5b00 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -76,7 +76,7 @@ kernel_params = "@KERNELPARAMS@" # < 0 --> will be set to the actual number of physical cores # > 0 <= number of physical cores --> will be set to the specified number # > number of physical cores --> will be set to the actual number of physical cores -default_vcpus = 1 +default_vcpus = @DEFVCPUS@ # Default maximum number of vCPUs per SB/VM: # unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number @@ -229,6 +229,12 @@ disable_image_nvdimm = @DEFDISABLEIMAGENVDIMM_CLH@ # The default setting is "no-port" hot_plug_vfio = "no-port" +# In a confidential compute environment hot-plugging can compromise +# security. +# Enable cold-plugging of VFIO devices to a root-port. +# The default setting is "no-port", which means disabled. +cold_plug_vfio = "no-port" + # Path to OCI hook binaries in the *guest rootfs*. # This does not affect host-side hooks which must instead be added to # the OCI spec passed to the runtime. @@ -432,6 +438,21 @@ sandbox_cgroup_only = @DEFSANDBOXCGROUPONLY@ # - When running single containers using a tool like ctr, container sizing information will be available. static_sandbox_resource_mgmt = @DEFSTATICRESOURCEMGMT_CLH@ +# If set, the runtime will use the value as the default workload memory in MB for the sandbox when no workload memory request is passed +# down to the shim via the OCI when static sandbox resource management is enabled. With this, we ensure that workloads have a proper +# default amount of memory available within the sandbox. +static_sandbox_default_workload_mem=@DEFSTATICSANDBOXWORKLOADMEM@ + +# If set, the runtime will use the value as the default number of vcpus for the sandbox when no workload vcpu request is passed +# down to the shim via the OCI when static sandbox resource management is enabled. With this, we ensure that workloads have a proper +# default amount of vcpus available within the sandbox. +static_sandbox_default_workload_vcpus=@DEFSTATICSANDBOXWORKLOADVCPUS@ + +# The runtime will enforce that pods deployed in a sandbox +# explicitly setting memory limits using resources.limits.memory +# allow at least this amount of memory in MiB so that the sandbox can properly start. +sandbox_workload_mem_min=@DEFSANDBOXWORKLOADMEMMIN@ + # If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path. # This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory. # If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts` diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in index 387671d79a90..88a40931deff 100644 --- a/src/runtime/pkg/katautils/config-settings.go.in +++ b/src/runtime/pkg/katautils/config-settings.go.in @@ -58,7 +58,7 @@ var systemdUnitName = "kata-containers.target" const defaultKernelParams = "" const defaultMachineType = "q35" -const defaultVCPUCount uint32 = 1 +const defaultVCPUCount uint32 = 0 const defaultMaxVCPUCount uint32 = 0 const defaultMemSize uint32 = 2048 // MiB const defaultMemSlots uint32 = 10 diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 29549c237852..91421f297695 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -181,27 +181,30 @@ type hypervisor struct { } type runtime struct { - InterNetworkModel string `toml:"internetworking_model"` - JaegerEndpoint string `toml:"jaeger_endpoint"` - JaegerUser string `toml:"jaeger_user"` - JaegerPassword string `toml:"jaeger_password"` - VfioMode string `toml:"vfio_mode"` - GuestSeLinuxLabel string `toml:"guest_selinux_label"` - SandboxBindMounts []string `toml:"sandbox_bind_mounts"` - Experimental []string `toml:"experimental"` - Tracing bool `toml:"enable_tracing"` - DisableNewNetNs bool `toml:"disable_new_netns"` - DisableGuestSeccomp bool `toml:"disable_guest_seccomp"` - EnableVCPUsPinning bool `toml:"enable_vcpus_pinning"` - Debug bool `toml:"enable_debug"` - SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` - StaticSandboxResourceMgmt bool `toml:"static_sandbox_resource_mgmt"` - EnablePprof bool `toml:"enable_pprof"` - DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"` - CreateContainerTimeout uint64 `toml:"create_container_timeout"` - DanConf string `toml:"dan_conf"` - ForceGuestPull bool `toml:"experimental_force_guest_pull"` - PodResourceAPISock string `toml:"pod_resource_api_sock"` + InterNetworkModel string `toml:"internetworking_model"` + JaegerEndpoint string `toml:"jaeger_endpoint"` + JaegerUser string `toml:"jaeger_user"` + JaegerPassword string `toml:"jaeger_password"` + VfioMode string `toml:"vfio_mode"` + GuestSeLinuxLabel string `toml:"guest_selinux_label"` + SandboxBindMounts []string `toml:"sandbox_bind_mounts"` + Experimental []string `toml:"experimental"` + Tracing bool `toml:"enable_tracing"` + DisableNewNetNs bool `toml:"disable_new_netns"` + DisableGuestSeccomp bool `toml:"disable_guest_seccomp"` + EnableVCPUsPinning bool `toml:"enable_vcpus_pinning"` + Debug bool `toml:"enable_debug"` + SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` + StaticSandboxResourceMgmt bool `toml:"static_sandbox_resource_mgmt"` + StaticSandboxWorkloadDefaultMem uint32 `toml:"static_sandbox_default_workload_mem"` + StaticSandboxWorkloadDefaultVcpus float32 `toml:"static_sandbox_default_workload_vcpus"` + SandboxWorkloadMemMin uint32 `toml:"sandbox_workload_mem_min"` + EnablePprof bool `toml:"enable_pprof"` + DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"` + CreateContainerTimeout uint64 `toml:"create_container_timeout"` + DanConf string `toml:"dan_conf"` + ForceGuestPull bool `toml:"experimental_force_guest_pull"` + PodResourceAPISock string `toml:"pod_resource_api_sock"` } type agent struct { @@ -1614,6 +1617,9 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat config.EnableVCPUsPinning = tomlConf.Runtime.EnableVCPUsPinning config.GuestSeLinuxLabel = tomlConf.Runtime.GuestSeLinuxLabel config.StaticSandboxResourceMgmt = tomlConf.Runtime.StaticSandboxResourceMgmt + config.StaticSandboxWorkloadDefaultMem = tomlConf.Runtime.StaticSandboxWorkloadDefaultMem + config.SandboxWorkloadMemMin = tomlConf.Runtime.SandboxWorkloadMemMin + config.StaticSandboxWorkloadDefaultVcpus = tomlConf.Runtime.StaticSandboxWorkloadDefaultVcpus config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs config.EnablePprof = tomlConf.Runtime.EnablePprof @@ -1918,11 +1924,11 @@ func checkPCIeConfig(coldPlug config.PCIePort, hotPlug config.PCIePort, machineT return nil } if hypervisorType == virtcontainers.ClhHypervisor { - if coldPlug != config.NoPort { - return fmt.Errorf("cold-plug not supported on CLH") + if coldPlug != config.NoPort && coldPlug != config.RootPort { + return fmt.Errorf("only cold-plug=%s or %s supported on CLH", config.NoPort, config.RootPort) } - if hotPlug != config.RootPort { - return fmt.Errorf("only hot-plug=%s supported on CLH", config.RootPort) + if hotPlug != config.NoPort && hotPlug != config.RootPort { + return fmt.Errorf("only hot-plug=%s or %s supported on CLH", config.NoPort, config.RootPort) } } @@ -1994,11 +2000,6 @@ func checkHypervisorConfig(config vc.HypervisorConfig) error { } memSizeMB := int64(config.MemorySize) - - if memSizeMB == 0 { - return errors.New("VM memory cannot be zero") - } - mb := int64(1024 * 1024) for _, image := range images { diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index 54a7127d7060..1415168976ce 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -943,7 +943,7 @@ func TestHypervisorDefaults(t *testing.T) { assert.Equal(h.machineType(), defaultMachineType, "default hypervisor machine type wrong") assert.Equal(h.defaultVCPUs(), float32(defaultVCPUCount), "default vCPU number is wrong") assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong") - assert.Equal(h.defaultMemSz(), defaultMemSize, "default memory size is wrong") + assert.Equal(h.defaultMemSz(), uint32(0), "default memory size is wrong") machineType := "foo" h.MachineType = machineType @@ -1526,7 +1526,7 @@ func TestCheckHypervisorConfig(t *testing.T) { // function, hence no test for it here. data := []testData{ - {"", "", 0, true, false}, + {"", "", 0, false, false}, {imageENOENT, "", 2, true, false}, {"", initrdENOENT, 2, true, false}, diff --git a/src/runtime/pkg/katautils/create_test.go b/src/runtime/pkg/katautils/create_test.go index 903e68d95dea..a0ec8ae07579 100644 --- a/src/runtime/pkg/katautils/create_test.go +++ b/src/runtime/pkg/katautils/create_test.go @@ -431,9 +431,11 @@ func TestVfioChecksClh(t *testing.T) { } assert.NoError(f(config.NoPort, config.NoPort)) assert.NoError(f(config.NoPort, config.RootPort)) + assert.NoError(f(config.RootPort, config.NoPort)) assert.Error(f(config.RootPort, config.RootPort)) - assert.Error(f(config.RootPort, config.NoPort)) assert.Error(f(config.NoPort, config.SwitchPort)) + assert.Error(f(config.SwitchPort, config.NoPort)) + assert.Error(f(config.BridgePort, config.NoPort)) } func TestVfioCheckQemu(t *testing.T) { diff --git a/src/runtime/pkg/oci/utils.go b/src/runtime/pkg/oci/utils.go index 99d86515a9bc..f5e74169aed0 100644 --- a/src/runtime/pkg/oci/utils.go +++ b/src/runtime/pkg/oci/utils.go @@ -153,6 +153,15 @@ type RuntimeConfig struct { // any later resource updates. StaticSandboxResourceMgmt bool + // Memory to allocate for workloads within the sandbox when workload memory is unspecified + StaticSandboxWorkloadDefaultMem uint32 + + // vcpus to allocate for workloads within the sandbox when workload vcpus is unspecified + StaticSandboxWorkloadDefaultVcpus float32 + + // Minimum memory (in MiB) to enforce is allocated for workloads within the sandbox when workload memory is specified + SandboxWorkloadMemMin uint32 + // Determines if create a netns for hypervisor process DisableNewNetNs bool @@ -1196,6 +1205,10 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st StaticResourceMgmt: runtime.StaticSandboxResourceMgmt, + StaticWorkloadDefaultMem: runtime.StaticSandboxWorkloadDefaultMem, + + StaticWorkloadDefaultVcpus: runtime.StaticSandboxWorkloadDefaultVcpus, + ShmSize: shmSize, VfioMode: runtime.VfioMode, @@ -1226,6 +1239,15 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st // with the base number of CPU/memory (which is equal to the default CPU/memory specified for the runtime // configuration or annotations) as well as any specified workload resources. if sandboxConfig.StaticResourceMgmt { + // If no Limits are set in pod config, use StaticWorkloadDefaultMem/Vcpus to ensure the containers generally + // have a reasonable amount of resources available + if sandboxConfig.SandboxResources.WorkloadMemMB == 0 { + sandboxConfig.SandboxResources.WorkloadMemMB = sandboxConfig.StaticWorkloadDefaultMem + } + if sandboxConfig.SandboxResources.WorkloadCPUs == 0 { + sandboxConfig.SandboxResources.WorkloadCPUs = sandboxConfig.StaticWorkloadDefaultVcpus + } + sandboxConfig.SandboxResources.BaseCPUs = sandboxConfig.HypervisorConfig.NumVCPUsF sandboxConfig.SandboxResources.BaseMemMB = sandboxConfig.HypervisorConfig.MemorySize @@ -1243,6 +1265,10 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st } + if sandboxConfig.SandboxResources.WorkloadMemMB < runtime.SandboxWorkloadMemMin { + return vc.SandboxConfig{}, fmt.Errorf("pod memory limit too low: minimum %dMiB, got %dMiB", runtime.SandboxWorkloadMemMin, sandboxConfig.SandboxResources.WorkloadMemMB) + } + return sandboxConfig, nil } diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 844004928ba0..5ad4a6bb0e2f 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -968,6 +968,44 @@ func (clh *cloudHypervisor) hotplugAddBlockDevice(drive *config.BlockDrive) erro return err } +// coldPlugVFIODevice appends a VFIO device to the VM configuration so that it +// is present when the VM is created (before boot). Cloud Hypervisor's CreateVM +// API accepts a list of devices that are attached at VM creation time, which +// effectively provides cold-plug semantics — the guest sees the device on its +// PCI bus from the very first enumeration. +func (clh *cloudHypervisor) coldPlugVFIODevice(device *config.VFIODev) error { + switch device.Type { + case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType: + // Supported PCI VFIO device types for Cloud Hypervisor. + default: + return fmt.Errorf("VFIO device %+v has unsupported type %v; only PCI VFIO devices are supported in Cloud Hypervisor", device, device.Type) + } + if strings.TrimSpace(device.SysfsDev) == "" { + return fmt.Errorf("VFIO device %q has empty or invalid SysfsDev path", device.ID) + } + + clh.Logger().WithFields(log.Fields{ + "device": device.ID, + "sysfs": device.SysfsDev, + "bdf": device.BDF, + }).Info("Cold-plugging VFIO device into VM config") + + clhDevice := *chclient.NewDeviceConfig(device.SysfsDev) + clhDevice.SetIommu(clh.config.IOMMU) + clhDevice.SetId(device.ID) + + if clh.vmconfig.Devices != nil { + *clh.vmconfig.Devices = append(*clh.vmconfig.Devices, clhDevice) + } else { + clh.vmconfig.Devices = &[]chclient.DeviceConfig{clhDevice} + } + + // Track the device ID so that it can be referenced later (e.g. for removal). + clh.devicesIds[device.ID] = device.ID + + return nil +} + func (clh *cloudHypervisor) hotPlugVFIODevice(device *config.VFIODev) error { cl := clh.client() ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second) @@ -1334,6 +1372,8 @@ func (clh *cloudHypervisor) AddDevice(ctx context.Context, devInfo interface{}, clh.addVSock(defaultGuestVSockCID, v.UdsPath) case types.Volume: err = clh.addVolume(v) + case config.VFIODev: + err = clh.coldPlugVFIODevice(&v) default: clh.Logger().WithField("function", "AddDevice").Warnf("Add device of type %v is not supported.", v) return fmt.Errorf("Not implemented support for %s", v) diff --git a/src/runtime/virtcontainers/clh_test.go b/src/runtime/virtcontainers/clh_test.go index 0a90982d32ed..5849ed76fbb2 100644 --- a/src/runtime/virtcontainers/clh_test.go +++ b/src/runtime/virtcontainers/clh_test.go @@ -682,6 +682,94 @@ func TestCloudHypervisorHotplugRemoveDevice(t *testing.T) { assert.Error(err, "Hotplug remove pmem block device expected error") } +func TestCloudHypervisorColdPlugVFIODevice(t *testing.T) { + assert := assert.New(t) + + clhConfig, err := newClhConfig() + assert.NoError(err) + + clh := &cloudHypervisor{} + clh.config = clhConfig + clh.devicesIds = make(map[string]string) + clh.vmconfig = *chclient.NewVmConfig(*chclient.NewPayloadConfig()) + + // Cold-plug a PCI VFIO device + dev := &config.VFIODev{ + ID: "gpu0", + SysfsDev: "/sys/bus/pci/devices/0000:41:00.0", + BDF: "0000:41:00.0", + Type: config.VFIOPCIDeviceNormalType, + } + err = clh.coldPlugVFIODevice(dev) + assert.NoError(err, "Cold-plug PCI VFIO device expected no error") + + // Verify the device was added to vmconfig.Devices + assert.NotNil(clh.vmconfig.Devices) + assert.Len(*clh.vmconfig.Devices, 1) + assert.Equal("/sys/bus/pci/devices/0000:41:00.0", (*clh.vmconfig.Devices)[0].Path) + assert.Equal("gpu0", clh.devicesIds["gpu0"]) + + // Cold-plug a second device + dev2 := &config.VFIODev{ + ID: "gpu1", + SysfsDev: "/sys/bus/pci/devices/0000:42:00.0", + BDF: "0000:42:00.0", + Type: config.VFIOPCIDeviceNormalType, + } + err = clh.coldPlugVFIODevice(dev2) + assert.NoError(err, "Cold-plug second VFIO device expected no error") + assert.Len(*clh.vmconfig.Devices, 2) + + // AP mediated device should fail + apDev := &config.VFIODev{ + ID: "ap0", + Type: config.VFIOAPDeviceMediatedType, + } + err = clh.coldPlugVFIODevice(apDev) + assert.Error(err, "Cold-plug AP mediated device expected error") + + // Error type (0) should fail + errDev := &config.VFIODev{ + ID: "bad0", + SysfsDev: "/sys/bus/pci/devices/0000:43:00.0", + Type: config.VFIODeviceErrorType, + } + err = clh.coldPlugVFIODevice(errDev) + assert.Error(err, "Cold-plug error-type device expected error") + + // Empty SysfsDev should fail + emptySysfsDev := &config.VFIODev{ + ID: "bad1", + Type: config.VFIOPCIDeviceNormalType, + } + err = clh.coldPlugVFIODevice(emptySysfsDev) + assert.Error(err, "Cold-plug with empty SysfsDev expected error") +} + +func TestCloudHypervisorAddDeviceVFIO(t *testing.T) { + assert := assert.New(t) + + clhConfig, err := newClhConfig() + assert.NoError(err) + + clh := &cloudHypervisor{} + clh.config = clhConfig + clh.devicesIds = make(map[string]string) + clh.vmconfig = *chclient.NewVmConfig(*chclient.NewPayloadConfig()) + + // AddDevice with VFIODev type should cold-plug + dev := config.VFIODev{ + ID: "nic0", + SysfsDev: "/sys/bus/pci/devices/0000:05:00.0", + BDF: "0000:05:00.0", + Type: config.VFIOPCIDeviceNormalType, + } + err = clh.AddDevice(context.Background(), dev, VfioDev) + assert.NoError(err, "AddDevice VFIO expected no error") + assert.NotNil(clh.vmconfig.Devices) + assert.Len(*clh.vmconfig.Devices, 1) +} + func TestClhGenerateSocket(t *testing.T) { assert := assert.New(t) diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 5d0522351c48..5eb95b74412c 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -67,7 +67,7 @@ const ( procCPUInfo = "/proc/cpuinfo" - defaultVCPUs = float32(1) + defaultVCPUs = float32(0) // 2 GiB defaultMemSzMiB = 2048 @@ -84,7 +84,7 @@ const ( vSockLogsPort = 1025 // MinHypervisorMemory is the minimum memory required for a VM. - MinHypervisorMemory = 256 + MinHypervisorMemory = 0 defaultMsize9p = 8192 diff --git a/src/runtime/virtcontainers/network.go b/src/runtime/virtcontainers/network.go index 0960d47d3706..9d6a8faedeb9 100644 --- a/src/runtime/virtcontainers/network.go +++ b/src/runtime/virtcontainers/network.go @@ -315,13 +315,16 @@ func generateVCNetworkStructures(ctx context.Context, endpoints []Endpoint) ([]* routes = append(routes, &r) } + gatewaySet := gatewaySetFromRoutes(endpoint.Properties().Routes) + for _, neigh := range endpoint.Properties().Neighbors { - var n pbTypes.ARPNeighbor - if !validGuestNeighbor(neigh) { + if !validGuestNeighbor(neigh, gatewaySet) { continue } + var n pbTypes.ARPNeighbor + n.Device = endpoint.Name() n.State = int32(neigh.State) n.Flags = int32(neigh.Flags) diff --git a/src/runtime/virtcontainers/network_linux.go b/src/runtime/virtcontainers/network_linux.go index 3f4b419d22c1..4c3f2e3407da 100644 --- a/src/runtime/virtcontainers/network_linux.go +++ b/src/runtime/virtcontainers/network_linux.go @@ -123,39 +123,45 @@ func (n *LinuxNetwork) addSingleEndpoint(ctx context.Context, s *Sandbox, netInf // Check if interface is a physical interface. Do not create // tap interface/bridge if it is. - isPhysical, err := isPhysicalIface(netInfo.Iface.Name) - if err != nil { - return nil, err + // netInfo.Link may be nil when coming from the hotplug path + // (Sandbox.generateNetInfo does not populate it). Resolve the link + // now so isPhysicalIface() and createPhysicalEndpoint() have access + // to its attributes (e.g. ParentDevBus). + if netInfo.Link == nil { + l, err := netlink.LinkByName(netInfo.Iface.Name) + if err != nil { + return nil, fmt.Errorf("failed to resolve link %q: %w", netInfo.Iface.Name, err) + } + netInfo.Link = l } + isPhysical := isPhysicalIface(netInfo.Link) + var err error + idx := len(n.eps) - if isPhysical { - if s.config.HypervisorConfig.ColdPlugVFIO == config.NoPort { - // When `cold_plug_vfio` is set to "no-port", the PhysicalEndpoint's VFIO device cannot be attached to the guest VM. - // Fail early to prevent the VF interface from being unbound and rebound to the VFIO driver. - return nil, fmt.Errorf("unable to add PhysicalEndpoint %s because cold_plug_vfio is disabled", netInfo.Iface.Name) + // Avoid endpoint naming conflicts + // When creating a new endpoint, we check existing endpoint names and automatically adjust the naming of the new endpoint to ensure uniqueness. + lastIdx := -1 + if len(n.eps) > 0 { + lastEndpoint := n.eps[len(n.eps)-1] + re := regexp.MustCompile("[0-9]+") + matchStr := re.FindString(lastEndpoint.Name()) + parsedIdx, err := strconv.ParseInt(matchStr, 10, strconv.IntSize) + if err != nil { + return nil, err } + lastIdx = int(parsedIdx) + } + if idx <= lastIdx { + idx = lastIdx + 1 + } + + if isPhysical { networkLogger().WithField("interface", netInfo.Iface.Name).Info("Physical network interface found") - endpoint, err = createPhysicalEndpoint(netInfo) + isFVIODisabled := (s.config.HypervisorConfig.ColdPlugVFIO == config.NoPort) + endpoint, err = createPhysicalEndpoint(idx, netInfo, isFVIODisabled, n.interworkingModel) } else { var socketPath string - idx := len(n.eps) - - // Avoid endpoint naming conflicts - // When creating a new endpoint, we check existing endpoint names and automatically adjust the naming of the new endpoint to ensure uniqueness. - lastIdx := -1 - if len(n.eps) > 0 { - lastEndpoint := n.eps[len(n.eps)-1] - re := regexp.MustCompile("[0-9]+") - matchStr := re.FindString(lastEndpoint.Name()) - n, err := strconv.ParseInt(matchStr, 10, 64) - if err != nil { - return nil, err - } - lastIdx = int(n) - } - if idx <= lastIdx { - idx = lastIdx + 1 - } + // Check if this is a dummy interface which has a vhost-user socket associated with it socketPath, err = vhostUserSocketPath(netInfo) if err != nil { @@ -654,6 +660,8 @@ func getLinkForEndpoint(endpoint Endpoint, netHandle *netlink.Handle) (netlink.L var link netlink.Link switch ep := endpoint.(type) { + case *PhysicalEndpoint: + link = &netlink.Device{} case *VethEndpoint: link = &netlink.Veth{} case *MacvlanEndpoint: @@ -676,6 +684,10 @@ func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink. } switch expectedLink.Type() { + case (&netlink.Device{}).Type(): + if l, ok := link.(*netlink.Device); ok { + return l, nil + } case (&netlink.Tuntap{}).Type(): if l, ok := link.(*netlink.Tuntap); ok { return l, nil @@ -1302,6 +1314,15 @@ func addRxRateLimiter(endpoint Endpoint, maxRate uint64) error { linkName = netPair.TapInterface.TAPIface.Name case *MacvtapEndpoint, *TapEndpoint: linkName = endpoint.Name() + case *PhysicalEndpoint: + if ep.IsVF { + // VF endpoints use VFIO passthrough; rate limiting is handled by + // the hypervisor / physical function and does not apply here. + return nil + } + // Non-VF physical NICs are tap-backed; apply rx limiting on the tap. + netPair := endpoint.NetworkPair() + linkName = netPair.TapInterface.TAPIface.Name default: return fmt.Errorf("Unsupported endpointType %s for adding rx rate limiter", ep.Type()) } @@ -1474,6 +1495,15 @@ func addTxRateLimiter(endpoint Endpoint, maxRate uint64) error { case *MacvtapEndpoint, *TapEndpoint: linkName = endpoint.Name() + case *PhysicalEndpoint: + if ep.IsVF { + // VF endpoints use VFIO passthrough; rate limiting is handled by + // the hypervisor / physical function and does not apply here. + return nil + } + // Non-VF physical NICs are tap-backed; apply IFB-based tx limiting via tap. + netPair = endpoint.NetworkPair() + linkName = netPair.TapInterface.TAPIface.Name default: return fmt.Errorf("Unsupported endpointType %s for adding tx rate limiter", ep.Type()) } @@ -1532,6 +1562,12 @@ func removeRxRateLimiter(endpoint Endpoint, networkNSPath string) error { linkName = netPair.TapInterface.TAPIface.Name case *MacvtapEndpoint, *TapEndpoint: linkName = endpoint.Name() + case *PhysicalEndpoint: + if ep.IsVF { + return nil + } + netPair := endpoint.NetworkPair() + linkName = netPair.TapInterface.TAPIface.Name default: return fmt.Errorf("Unsupported endpointType %s for removing rx rate limiter", ep.Type()) } @@ -1564,8 +1600,14 @@ func removeTxRateLimiter(endpoint Endpoint, networkNSPath string) error { } case *MacvtapEndpoint, *TapEndpoint: linkName = endpoint.Name() + case *PhysicalEndpoint: + if ep.IsVF { + return nil + } + netPair := endpoint.NetworkPair() + linkName = netPair.TapInterface.TAPIface.Name default: - return fmt.Errorf("Unsupported endpointType %s for adding tx rate limiter", ep.Type()) + return fmt.Errorf("Unsupported endpointType %s for removing tx rate limiter", ep.Type()) } if err := doNetNS(networkNSPath, func(_ ns.NetNS) error { @@ -1614,7 +1656,39 @@ func validGuestRoute(route netlink.Route) bool { return route.Protocol != unix.RTPROT_KERNEL } -func validGuestNeighbor(neigh netlink.Neigh) bool { - // We add only static ARP entries - return neigh.State == netlink.NUD_PERMANENT +// neighbor is valid if it is static or a default-gateway +func validGuestNeighbor(neigh netlink.Neigh, gatewaySet map[string]struct{}) bool { + // need a MAC for the guest + if neigh.HardwareAddr == nil { + return false + } + // Keep all static entries + if neigh.State == netlink.NUD_PERMANENT { + return true + } + // Gateway-only exception: allow the default-gateway IP: + // On some setups, the pod subnet gateway does not appear in the host ARP cache as a static entry. + // On these setups an ARP request storm happens when many Kata PODs are started at the same time and they all look for the gateway MAC address. + // This forces the gateway to churn a lot of ARP requests and render the ARP request full, hence dropping some ARP requests. + // Manually pre-populating the ARP entry in the UVM guest ARP cache for that gateway solves that problem. + _, isGw := gatewaySet[neigh.IP.String()] + return isGw +} + +// helper: default routes => set of gateway IP strings +func gatewaySetFromRoutes(routes []netlink.Route) map[string]struct{} { + gatewaySet := make(map[string]struct{}) + for _, route := range routes { + if route.Gw == nil { + continue + } + if route.Dst == nil { + gatewaySet[route.Gw.String()] = struct{}{} + continue + } + if ones, _ := route.Dst.Mask.Size(); ones == 0 { // 0.0.0.0/0 or ::/0 + gatewaySet[route.Gw.String()] = struct{}{} + } + } + return gatewaySet } diff --git a/src/runtime/virtcontainers/persist/api/network.go b/src/runtime/virtcontainers/persist/api/network.go index 256ce807b5df..d62495a61b93 100644 --- a/src/runtime/virtcontainers/persist/api/network.go +++ b/src/runtime/virtcontainers/persist/api/network.go @@ -44,6 +44,9 @@ type PhysicalEndpoint struct { BDF string Driver string VendorDeviceID string + NetPair NetworkInterfacePair + BusType string + IsVF bool } type MacvtapEndpoint struct { diff --git a/src/runtime/virtcontainers/physical_endpoint.go b/src/runtime/virtcontainers/physical_endpoint.go index 7e88c4a874e1..9ac58078b081 100644 --- a/src/runtime/virtcontainers/physical_endpoint.go +++ b/src/runtime/virtcontainers/physical_endpoint.go @@ -14,12 +14,14 @@ import ( "path/filepath" "strings" + "github.com/containernetworking/plugins/pkg/ns" "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers" resCtrl "github.com/kata-containers/kata-containers/src/runtime/pkg/resourcecontrol" persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/safchain/ethtool" + "github.com/vishvananda/netlink" ) var physicalTrace = getNetworkTrace(PhysicalEndpointType) @@ -27,6 +29,7 @@ var physicalTrace = getNetworkTrace(PhysicalEndpointType) // PhysicalEndpoint gathers a physical network interface and its properties type PhysicalEndpoint struct { IfaceName string + IsVF bool HardAddr string EndpointProperties NetworkInfo EndpointType EndpointType @@ -35,6 +38,10 @@ type PhysicalEndpoint struct { VendorDeviceID string PCIPath vcTypes.PciPath CCWDevice *vcTypes.CcwDevice + NetPair NetworkInterfacePair + BusType string + RxRateLimiter bool + TxRateLimiter bool } // Properties returns the properties of the physical interface. @@ -84,7 +91,7 @@ func (endpoint *PhysicalEndpoint) SetProperties(properties NetworkInfo) { // NetworkPair returns the network pair of the endpoint. func (endpoint *PhysicalEndpoint) NetworkPair() *NetworkInterfacePair { - return nil + return &endpoint.NetPair } // Attach for physical endpoint binds the physical network interface to @@ -93,29 +100,38 @@ func (endpoint *PhysicalEndpoint) Attach(ctx context.Context, s *Sandbox) error span, ctx := physicalTrace(ctx, "Attach", endpoint) defer span.End() - // Unbind physical interface from host driver and bind to vfio - // so that it can be passed to qemu. - vfioPath, err := bindNICToVFIO(endpoint) - if err != nil { - return err - } - - c, err := resCtrl.DeviceToCgroupDeviceRule(vfioPath) - if err != nil { + if endpoint.IsVF { + // Unbind physical interface from host driver and bind to vfio + // so that it can be passed to qemu. + vfioPath, err := bindNICToVFIO(endpoint) + if err != nil { + return err + } + + c, err := resCtrl.DeviceToCgroupDeviceRule(vfioPath) + if err != nil { + return err + } + + d := config.DeviceInfo{ + ContainerPath: vfioPath, + DevType: string(c.Type), + Major: c.Major, + Minor: c.Minor, + ColdPlug: true, + Port: s.config.HypervisorConfig.ColdPlugVFIO, + } + + _, err = s.AddDevice(ctx, d) return err - } + } else { + h := s.hypervisor + if err := xConnectVMNetwork(ctx, endpoint, h); err != nil { + return err + } - d := config.DeviceInfo{ - ContainerPath: vfioPath, - DevType: string(c.Type), - Major: c.Major, - Minor: c.Minor, - ColdPlug: true, - Port: s.config.HypervisorConfig.ColdPlugVFIO, + return h.AddDevice(ctx, endpoint, NetDev) } - - _, err = s.AddDevice(ctx, d) - return err } // Detach for physical endpoint unbinds the physical network interface from vfio-pci @@ -124,13 +140,26 @@ func (endpoint *PhysicalEndpoint) Detach(ctx context.Context, netNsCreated bool, span, _ := physicalTrace(ctx, "Detach", endpoint) defer span.End() - // Bind back the physical network interface to host. - // We need to do this even if a new network namespace has not - // been created by virtcontainers. - - // We do not need to enter the network namespace to bind back the - // physical interface to host driver. - return bindNICToHost(endpoint) + if endpoint.IsVF { + + // Bind back the physical network interface to host. + // We need to do this even if a new network namespace has not + // been created by virtcontainers. + + // We do not need to enter the network namespace to bind back the + // physical interface to host driver. + return bindNICToHost(endpoint) + } else { + // The network namespace would have been deleted at this point + // if it has not been created by virtcontainers. + if !netNsCreated { + return nil + } + + return doNetNS(netNsPath, func(_ ns.NetNS) error { + return xDisconnectVMNetwork(ctx, endpoint) + }) + } } // HotAttach for physical endpoint not supported yet @@ -138,28 +167,40 @@ func (endpoint *PhysicalEndpoint) HotAttach(ctx context.Context, s *Sandbox) err span, ctx := physicalTrace(ctx, "HotAttach", endpoint) defer span.End() - // Unbind physical interface from host driver and bind to vfio - // so that it can be passed to the hypervisor. - vfioPath, err := bindNICToVFIO(endpoint) - if err != nil { + if endpoint.IsVF { + // Unbind physical interface from host driver and bind to vfio + // so that it can be passed to the hypervisor. + vfioPath, err := bindNICToVFIO(endpoint) + if err != nil { + return err + } + + c, err := resCtrl.DeviceToCgroupDeviceRule(vfioPath) + if err != nil { + return err + } + + d := config.DeviceInfo{ + ContainerPath: vfioPath, + DevType: string(c.Type), + Major: c.Major, + Minor: c.Minor, + ColdPlug: false, + } + + _, err = s.AddDevice(ctx, d) return err + } else { + h := s.hypervisor + if err := xConnectVMNetwork(ctx, endpoint, h); err != nil { + return err + } + + if _, err := h.HotplugAddDevice(ctx, endpoint, NetDev); err != nil { + return err + } + return nil } - - c, err := resCtrl.DeviceToCgroupDeviceRule(vfioPath) - if err != nil { - return err - } - - d := config.DeviceInfo{ - ContainerPath: vfioPath, - DevType: string(c.Type), - Major: c.Major, - Minor: c.Minor, - ColdPlug: false, - } - - _, err = s.AddDevice(ctx, d) - return err } // HotDetach for physical endpoint not supported yet @@ -170,76 +211,101 @@ func (endpoint *PhysicalEndpoint) HotDetach(ctx context.Context, s *Sandbox, net var vfioPath string var err error - if vfioPath, err = drivers.GetVFIODevPath(endpoint.BDF); err != nil { - return err - } - - c, err := resCtrl.DeviceToCgroupDeviceRule(vfioPath) - if err != nil { - return err - } - - d := config.DeviceInfo{ - ContainerPath: vfioPath, - DevType: string(c.Type), - Major: c.Major, - Minor: c.Minor, - ColdPlug: false, + if endpoint.IsVF { + if vfioPath, err = drivers.GetVFIODevPath(endpoint.BDF); err != nil { + return err + } + + c, err := resCtrl.DeviceToCgroupDeviceRule(vfioPath) + if err != nil { + return err + } + + d := config.DeviceInfo{ + ContainerPath: vfioPath, + DevType: string(c.Type), + Major: c.Major, + Minor: c.Minor, + ColdPlug: false, + } + + device := s.devManager.FindDevice(&d) + s.devManager.RemoveDevice(device.DeviceID()) + + // We do not need to enter the network namespace to bind back the + // physical interface to host driver. + return bindNICToHost(endpoint) + } else { + if !netNsCreated { + return nil + } + + if err := doNetNS(netNsPath, func(_ ns.NetNS) error { + return xDisconnectVMNetwork(ctx, endpoint) + }); err != nil { + networkLogger().WithError(err).Warn("Error un-bridging virtual ep") + } + + h := s.hypervisor + if _, err := h.HotplugRemoveDevice(ctx, endpoint, NetDev); err != nil { + return err + } + return nil } +} - device := s.devManager.FindDevice(&d) - s.devManager.RemoveDevice(device.DeviceID()) +// isPhysicalIface checks if an interface is a physical device by inspecting +// the link's ParentDevBus attribute. Returns true when the bus is "pci" or +// "vmbus", which covers both SR-IOV VFs and non-VF physical NICs (e.g. +// VMBus-backed devices). ParentDevBus is populated by the kernel via netlink +// and does not require sysfs access inside the network namespace. +func isPhysicalIface(link netlink.Link) bool { - // We do not need to enter the network namespace to bind back the - // physical interface to host driver. - return bindNICToHost(endpoint) + isParent := (link.Attrs().ParentDevBus == "pci" || link.Attrs().ParentDevBus == "vmbus") + return isParent } -// isPhysicalIface checks if an interface is a physical device. -// We use ethtool here to not rely on device sysfs inside the network namespace. -func isPhysicalIface(ifaceName string) (bool, error) { - if ifaceName == "lo" { - return false, nil - } +var sysBusPath = "/sys/bus/" - ethHandle, err := ethtool.NewEthtool() - if err != nil { - return false, err - } - defer ethHandle.Close() - - bus, err := ethHandle.BusInfo(ifaceName) - if err != nil { - return false, nil - } +// Get vendor and device id from pci space (sys/bus/pci/devices, or sys/bus/vmbus/devices, ...) +func getDevicesPath(link netlink.Link) string { + return filepath.Join(sysBusPath, link.Attrs().ParentDevBus, "devices") +} - // Check for a pci bus format - tokens := strings.Split(bus, ":") - if len(tokens) != 3 { - return false, nil +// Get vendor and device id from pci space (sys/bus/pci/devices/$BusDeviceInfo) +func getIfaceDevicePath(link netlink.Link, deviceInterfaceName string) (string, string, error) { + + if link.Attrs().ParentDevBus == "pci" { + // Get ethtool handle to derive driver and bus + ethHandle, err := ethtool.NewEthtool() + if err != nil { + return "", "", err + } + defer ethHandle.Close() + + // Get Bus info + bdf, err := ethHandle.BusInfo(deviceInterfaceName) + if err != nil { + return "", "", err + } + // Get device by following symlink /sys/bus/pci/devices/$bdf + return filepath.Join(getDevicesPath(link), bdf), bdf, nil + } else if link.Attrs().ParentDevBus == "vmbus" { + return filepath.Join(getDevicesPath(link), link.Attrs().ParentDev), link.Attrs().ParentDev, nil + } else { + return "", "", fmt.Errorf("unsupported ParentDevBus: %s", link.Attrs().ParentDevBus) } - - return true, nil } -var sysPCIDevicesPath = "/sys/bus/pci/devices" +func createPhysicalEndpoint(idx int, netInfo NetworkInfo, isFVIODisabled bool, interworkingModel NetInterworkingModel) (*PhysicalEndpoint, error) { -func createPhysicalEndpoint(netInfo NetworkInfo) (*PhysicalEndpoint, error) { - // Get ethtool handle to derive driver and bus - ethHandle, err := ethtool.NewEthtool() + sysIfaceDevicePath, bdf, err := getIfaceDevicePath(netInfo.Link, netInfo.Iface.Name) if err != nil { return nil, err } - defer ethHandle.Close() - // Get BDF - bdf, err := ethHandle.BusInfo(netInfo.Iface.Name) - if err != nil { - return nil, err - } - - // Get driver by following symlink /sys/bus/pci/devices/$bdf/driver - driverPath := filepath.Join(sysPCIDevicesPath, bdf, "driver") + // Get driver by following symlink /sys/bus/pci/devices/$bdf/driver or /sys/bus/vmbus/devices/$guid/driver + driverPath := filepath.Join(sysIfaceDevicePath, "driver") link, err := os.Readlink(driverPath) if err != nil { return nil, err @@ -247,9 +313,8 @@ func createPhysicalEndpoint(netInfo NetworkInfo) (*PhysicalEndpoint, error) { driver := filepath.Base(link) - // Get vendor and device id from pci space (sys/bus/pci/devices/$bdf) - - ifaceDevicePath := filepath.Join(sysPCIDevicesPath, bdf, "device") + // Get device by following symlink /sys/bus/pci/devices/$bdf/device or /sys/bus/vmbus/devices/$guid/device + ifaceDevicePath := filepath.Join(sysIfaceDevicePath, "device") contents, err := os.ReadFile(ifaceDevicePath) if err != nil { return nil, err @@ -257,24 +322,55 @@ func createPhysicalEndpoint(netInfo NetworkInfo) (*PhysicalEndpoint, error) { deviceID := strings.TrimSpace(string(contents)) - // Vendor id - ifaceVendorPath := filepath.Join(sysPCIDevicesPath, bdf, "vendor") + // Vendor id (/sys/bus/pci/devices/$bdf/device or /sys/bus/vmbus/devices/$guid/driver) + ifaceVendorPath := filepath.Join(sysIfaceDevicePath, "vendor") contents, err = os.ReadFile(ifaceVendorPath) if err != nil { return nil, err } + // A common way to identify VFs is by checking for the 'physfn' symlink + // pointing to the Physical Function (PF) + isVF := true + netPair := NetworkInterfacePair{} + _, err = filepath.EvalSymlinks(filepath.Join(sysIfaceDevicePath, "physfn")) + if err != nil { + isVF = false + + if idx < 0 { + return nil, fmt.Errorf("invalid network endpoint index: %d", idx) + } + + netPair, err = createNetworkInterfacePair(idx, netInfo.Iface.Name, interworkingModel) + if err != nil { + return nil, err + } + + if netInfo.Iface.Name != "" { + netPair.VirtIface.Name = netInfo.Iface.Name + } + } else { + if isFVIODisabled { + // When `cold_plug_vfio` is set to "no-port", the PhysicalEndpoint's VFIO device cannot be attached to the guest VM. + // Fail early to prevent the VF interface from being unbound and rebound to the VFIO driver. + return nil, fmt.Errorf("Unable to add PhysicalEndpoint %s because cold_plug_vfio is disabled", netInfo.Iface.Name) + } + } + vendorID := strings.TrimSpace(string(contents)) vendorDeviceID := fmt.Sprintf("%s %s", vendorID, deviceID) vendorDeviceID = strings.TrimSpace(vendorDeviceID) physicalEndpoint := &PhysicalEndpoint{ IfaceName: netInfo.Iface.Name, + IsVF: isVF, HardAddr: netInfo.Iface.HardwareAddr.String(), VendorDeviceID: vendorDeviceID, EndpointType: PhysicalEndpointType, Driver: driver, BDF: bdf, + NetPair: netPair, + BusType: netInfo.Link.Attrs().ParentDevBus, } return physicalEndpoint, nil @@ -289,6 +385,8 @@ func bindNICToHost(endpoint *PhysicalEndpoint) error { } func (endpoint *PhysicalEndpoint) save() persistapi.NetworkEndpoint { + netpair := saveNetIfPair(&endpoint.NetPair) + return persistapi.NetworkEndpoint{ Type: string(endpoint.Type()), @@ -296,6 +394,9 @@ func (endpoint *PhysicalEndpoint) save() persistapi.NetworkEndpoint { BDF: endpoint.BDF, Driver: endpoint.Driver, VendorDeviceID: endpoint.VendorDeviceID, + NetPair: *netpair, + BusType: endpoint.BusType, + IsVF: endpoint.IsVF, }, } } @@ -304,26 +405,30 @@ func (endpoint *PhysicalEndpoint) load(s persistapi.NetworkEndpoint) { endpoint.EndpointType = PhysicalEndpointType if s.Physical != nil { + netpair := loadNetIfPair(&s.Physical.NetPair) + endpoint.NetPair = *netpair endpoint.BDF = s.Physical.BDF endpoint.Driver = s.Physical.Driver endpoint.VendorDeviceID = s.Physical.VendorDeviceID + endpoint.BusType = s.Physical.BusType + endpoint.IsVF = s.Physical.IsVF } } -// unsupported func (endpoint *PhysicalEndpoint) GetRxRateLimiter() bool { - return false + return endpoint.RxRateLimiter } func (endpoint *PhysicalEndpoint) SetRxRateLimiter() error { - return fmt.Errorf("rx rate limiter is unsupported for physical endpoint") + endpoint.RxRateLimiter = true + return nil } -// unsupported func (endpoint *PhysicalEndpoint) GetTxRateLimiter() bool { - return false + return endpoint.TxRateLimiter } func (endpoint *PhysicalEndpoint) SetTxRateLimiter() error { - return fmt.Errorf("tx rate limiter is unsupported for physical endpoint") + endpoint.TxRateLimiter = true + return nil } diff --git a/src/runtime/virtcontainers/physical_endpoint_test.go b/src/runtime/virtcontainers/physical_endpoint_test.go index d3a096607953..10b496db6938 100644 --- a/src/runtime/virtcontainers/physical_endpoint_test.go +++ b/src/runtime/virtcontainers/physical_endpoint_test.go @@ -10,6 +10,8 @@ package virtcontainers import ( "context" "net" + "os" + "path/filepath" "testing" "github.com/containernetworking/plugins/pkg/ns" @@ -20,36 +22,125 @@ import ( "github.com/vishvananda/netns" ) -func TestPhysicalEndpoint_HotAttach(t *testing.T) { +func TestPhysicalEndpoint_HotAttach_VF(t *testing.T) { assert := assert.New(t) v := &PhysicalEndpoint{ IfaceName: "eth0", HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), + IsVF: true, } s := &Sandbox{ hypervisor: &mockHypervisor{}, } + // VF path tries to bind to VFIO which fails without real hardware err := v.HotAttach(context.Background(), s) assert.Error(err) } -func TestPhysicalEndpoint_HotDetach(t *testing.T) { +func TestPhysicalEndpoint_HotAttach_NonVF(t *testing.T) { assert := assert.New(t) v := &PhysicalEndpoint{ IfaceName: "eth0", HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), + IsVF: false, } s := &Sandbox{ hypervisor: &mockHypervisor{}, } + // Non-VF path tries xConnectVMNetwork which fails without tap/bridge setup + err := v.HotAttach(context.Background(), s) + assert.Error(err) +} + +func TestPhysicalEndpoint_HotDetach_VF(t *testing.T) { + assert := assert.New(t) + v := &PhysicalEndpoint{ + IfaceName: "eth0", + HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), + IsVF: true, + } + + s := &Sandbox{ + hypervisor: &mockHypervisor{}, + } + + // VF path tries to get VFIO dev path which fails without real hardware err := v.HotDetach(context.Background(), s, true, "") assert.Error(err) } +func TestPhysicalEndpoint_HotDetach_NonVF_NetNsNotCreated(t *testing.T) { + assert := assert.New(t) + v := &PhysicalEndpoint{ + IfaceName: "eth0", + HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), + IsVF: false, + } + + s := &Sandbox{ + hypervisor: &mockHypervisor{}, + } + + // Non-VF with netNsCreated=false should return nil immediately + err := v.HotDetach(context.Background(), s, false, "") + assert.NoError(err) +} + +func TestPhysicalEndpoint_HotDetach_NonVF_NetNsCreated(t *testing.T) { + assert := assert.New(t) + v := &PhysicalEndpoint{ + IfaceName: "eth0", + HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), + IsVF: false, + } + + s := &Sandbox{ + hypervisor: &mockHypervisor{}, + } + + // Non-VF with netNsCreated=true but empty path calls doNetNS("", ...), + // xDisconnectVMNetwork error is only logged as a warning (not returned). + // mockHypervisor.HotplugRemoveDevice succeeds, so overall returns nil. + err := v.HotDetach(context.Background(), s, true, "") + assert.NoError(err) +} + +func TestPhysicalEndpoint_NetworkPair(t *testing.T) { + assert := assert.New(t) + + netPair := NetworkInterfacePair{ + VirtIface: NetworkInterface{ + Name: "eth0", + }, + } + + v := &PhysicalEndpoint{ + IfaceName: "eth0", + NetPair: netPair, + } + + result := v.NetworkPair() + assert.NotNil(result) + assert.Equal("eth0", result.VirtIface.Name) +} + +func TestPhysicalEndpoint_Detach_NonVF_NetNsNotCreated(t *testing.T) { + assert := assert.New(t) + v := &PhysicalEndpoint{ + IfaceName: "eth0", + HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), + IsVF: false, + } + + // Non-VF with netNsCreated=false should return nil immediately + err := v.Detach(context.Background(), false, "") + assert.NoError(err) +} + func TestIsPhysicalIface(t *testing.T) { assert := assert.New(t) @@ -88,12 +179,252 @@ func TestIsPhysicalIface(t *testing.T) { err = netlinkHandle.LinkAdd(link) assert.NoError(err) + // Fetch the link back from the kernel so its attributes (e.g. ParentDevBus) + // reflect reality rather than whatever was set on the local struct. + kernelLink, err := netlinkHandle.LinkByName(testNetIface) + assert.NoError(err) + var isPhysical bool err = doNetNS(n.Path(), func(_ ns.NetNS) error { - var err error - isPhysical, err = isPhysicalIface(testNetIface) - return err + isPhysical = isPhysicalIface(kernelLink) + return nil }) assert.NoError(err) assert.False(isPhysical) } + +func TestIsPhysicalIface_PCI(t *testing.T) { + assert := assert.New(t) + link := &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: "eth0", + ParentDevBus: "pci", + }, + } + assert.True(isPhysicalIface(link)) +} + +func TestIsPhysicalIface_VMBus(t *testing.T) { + assert := assert.New(t) + link := &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: "eth0", + ParentDevBus: "vmbus", + }, + } + assert.True(isPhysicalIface(link)) +} + +func TestIsPhysicalIface_NoBus(t *testing.T) { + assert := assert.New(t) + link := &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: "veth0", + ParentDevBus: "", + }, + } + assert.False(isPhysicalIface(link)) +} + +func TestGetDevicesPath(t *testing.T) { + assert := assert.New(t) + + pciLink := &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + ParentDevBus: "pci", + }, + } + assert.Equal("/sys/bus/pci/devices", getDevicesPath(pciLink)) + + vmbusLink := &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + ParentDevBus: "vmbus", + }, + } + assert.Equal("/sys/bus/vmbus/devices", getDevicesPath(vmbusLink)) +} + +func TestGetIfaceDevicePath_UnsupportedBus(t *testing.T) { + assert := assert.New(t) + + link := &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + ParentDevBus: "usb", + }, + } + _, _, err := getIfaceDevicePath(link, "eth0") + assert.Error(err) + assert.Contains(err.Error(), "unsupported ParentDevBus") +} + +func TestGetIfaceDevicePath_VMBus(t *testing.T) { + assert := assert.New(t) + + guid := "00000000-0000-0000-0000-000000000001" + link := &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + ParentDevBus: "vmbus", + ParentDev: guid, + }, + } + path, bdf, err := getIfaceDevicePath(link, "eth0") + assert.NoError(err) + assert.Equal(guid, bdf) + assert.Equal(filepath.Join("/sys/bus/vmbus/devices", guid), path) +} + +func TestCreatePhysicalEndpoint_NegativeIdx(t *testing.T) { + assert := assert.New(t) + + // Create a temp directory to mock sysfs + tmpDir := t.TempDir() + origSysBusPath := sysBusPath + sysBusPath = tmpDir + defer func() { sysBusPath = origSysBusPath }() + + guid := "00000000-0000-0000-0000-000000000001" + vmbusDevDir := filepath.Join(tmpDir, "vmbus", "devices", guid) + assert.NoError(os.MkdirAll(vmbusDevDir, 0755)) + + // Create driver symlink + driverTarget := filepath.Join(tmpDir, "drivers", "hv_netvsc") + assert.NoError(os.MkdirAll(driverTarget, 0755)) + assert.NoError(os.Symlink(driverTarget, filepath.Join(vmbusDevDir, "driver"))) + + // Create device and vendor files + assert.NoError(os.WriteFile(filepath.Join(vmbusDevDir, "device"), []byte("0x1572\n"), 0644)) + assert.NoError(os.WriteFile(filepath.Join(vmbusDevDir, "vendor"), []byte("0x8086\n"), 0644)) + + // No physfn symlink → not a VF → negative idx should fail + hwAddr, _ := net.ParseMAC("aa:bb:cc:dd:ee:ff") + netInfo := NetworkInfo{ + Iface: NetlinkIface{ + LinkAttrs: netlink.LinkAttrs{ + Name: "eth0", + ParentDevBus: "vmbus", + ParentDev: guid, + HardwareAddr: hwAddr, + }, + Type: "", + }, + Link: &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: "eth0", + ParentDevBus: "vmbus", + ParentDev: guid, + HardwareAddr: hwAddr, + }, + }, + } + + _, err := createPhysicalEndpoint(-1, netInfo, false, DefaultNetInterworkingModel) + assert.Error(err) + assert.Contains(err.Error(), "invalid network endpoint index") +} + +func TestPhysicalEndpoint_SaveLoad_NonVF(t *testing.T) { + assert := assert.New(t) + + netPair := NetworkInterfacePair{ + TapInterface: TapInterface{ + ID: "tap-id-1", + Name: "br-tap1", + TAPIface: NetworkInterface{ + Name: "tap1", + HardAddr: "aa:bb:cc:dd:ee:ff", + }, + }, + VirtIface: NetworkInterface{ + Name: "eth0", + HardAddr: "aa:bb:cc:dd:ee:ff", + }, + NetInterworkingModel: DefaultNetInterworkingModel, + } + + endpoint := &PhysicalEndpoint{ + IfaceName: "eth0", + HardAddr: "aa:bb:cc:dd:ee:ff", + EndpointType: PhysicalEndpointType, + BDF: "0000:01:00.0", + Driver: "mlx5_core", + VendorDeviceID: "0x8086 0x1572", + IsVF: false, + NetPair: netPair, + BusType: "pci", + } + + saved := endpoint.save() + assert.NotNil(saved.Physical) + assert.Equal("0000:01:00.0", saved.Physical.BDF) + assert.Equal("mlx5_core", saved.Physical.Driver) + assert.Equal("pci", saved.Physical.BusType) + assert.Equal("tap1", saved.Physical.NetPair.TAPIface.Name) + assert.Equal("eth0", saved.Physical.NetPair.VirtIface.Name) + assert.False(saved.Physical.IsVF) + + loaded := &PhysicalEndpoint{} + loaded.load(saved) + assert.Equal(PhysicalEndpointType, loaded.EndpointType) + assert.Equal("0000:01:00.0", loaded.BDF) + assert.Equal("mlx5_core", loaded.Driver) + assert.Equal("0x8086 0x1572", loaded.VendorDeviceID) + assert.Equal("pci", loaded.BusType) + assert.Equal("tap1", loaded.NetPair.TapInterface.TAPIface.Name) + assert.Equal("eth0", loaded.NetPair.VirtIface.Name) + assert.Equal(DefaultNetInterworkingModel, loaded.NetPair.NetInterworkingModel) + assert.False(loaded.IsVF) +} + +func TestCreatePhysicalEndpoint_NonVF_HappyPath(t *testing.T) { + assert := assert.New(t) + + tmpDir := t.TempDir() + origSysBusPath := sysBusPath + sysBusPath = tmpDir + defer func() { sysBusPath = origSysBusPath }() + + guid := "00000000-0000-0000-0000-000000000002" + vmbusDevDir := filepath.Join(tmpDir, "vmbus", "devices", guid) + assert.NoError(os.MkdirAll(vmbusDevDir, 0755)) + + // Create driver symlink + driverTarget := filepath.Join(tmpDir, "drivers", "hv_netvsc") + assert.NoError(os.MkdirAll(driverTarget, 0755)) + assert.NoError(os.Symlink(driverTarget, filepath.Join(vmbusDevDir, "driver"))) + + // Create device and vendor files + assert.NoError(os.WriteFile(filepath.Join(vmbusDevDir, "device"), []byte("0x1572\n"), 0644)) + assert.NoError(os.WriteFile(filepath.Join(vmbusDevDir, "vendor"), []byte("0x8086\n"), 0644)) + + // No physfn symlink → not a VF + hwAddr, _ := net.ParseMAC("aa:bb:cc:dd:ee:ff") + netInfo := NetworkInfo{ + Iface: NetlinkIface{ + LinkAttrs: netlink.LinkAttrs{ + Name: "eth1", + ParentDevBus: "vmbus", + ParentDev: guid, + HardwareAddr: hwAddr, + }, + Type: "", + }, + Link: &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: "eth1", + ParentDevBus: "vmbus", + ParentDev: guid, + HardwareAddr: hwAddr, + }, + }, + } + + ep, err := createPhysicalEndpoint(0, netInfo, false, DefaultNetInterworkingModel) + assert.NoError(err) + assert.NotNil(ep) + assert.False(ep.IsVF) + assert.Equal("eth1", ep.IfaceName) + assert.Equal("hv_netvsc", ep.Driver) + assert.Equal(guid, ep.BDF) + assert.Equal("vmbus", ep.BusType) + assert.Equal("eth1", ep.NetPair.VirtIface.Name) +} diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/README.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/README.md index e2b529582305..20bdbf5c1558 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/README.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/README.md @@ -16,6 +16,7 @@ Install the following dependencies: ```shell go get github.com/stretchr/testify/assert go get golang.org/x/oauth2 +go get golang.org/x/net/context ``` Put the package under your project folder and add the following in import: @@ -176,3 +177,6 @@ Each of these functions takes a value of the given basic type and returns a poin * `PtrTime` ## Author + + + diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 2b1d05b8f5f3..cc0ebc2f54ea 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -161,6 +161,11 @@ type SandboxConfig struct { HypervisorConfig HypervisorConfig + StaticWorkloadDefaultMem uint32 + + StaticWorkloadDefaultVcpus float32 + + // Memory to allocate for workloads within the sandbox when workload memory is unspecified ShmSize uint64 SandboxResources SandboxResourceSizing diff --git a/src/tools/csi-kata-directvolume/go.mod b/src/tools/csi-kata-directvolume/go.mod index e0b7579ba536..ce56c7bdba2a 100644 --- a/src/tools/csi-kata-directvolume/go.mod +++ b/src/tools/csi-kata-directvolume/go.mod @@ -14,7 +14,6 @@ require ( github.com/kubernetes-csi/csi-lib-utils v0.16.0 github.com/pborman/uuid v1.2.1 github.com/stretchr/testify v1.8.4 - golang.org/x/net v0.50.0 golang.org/x/sys v0.41.0 google.golang.org/grpc v1.63.2 k8s.io/apimachinery v0.28.2 @@ -36,6 +35,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/ulikunitz/xz v0.5.15 // indirect + golang.org/x/net v0.50.0 // indirect golang.org/x/text v0.34.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect google.golang.org/protobuf v1.33.0 // indirect diff --git a/src/tools/csi-kata-directvolume/go.sum b/src/tools/csi-kata-directvolume/go.sum index d208a65546f4..892c1353eb71 100644 --- a/src/tools/csi-kata-directvolume/go.sum +++ b/src/tools/csi-kata-directvolume/go.sum @@ -66,8 +66,6 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -79,14 +77,10 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/tests/integration/cri-containerd/integration-tests.sh b/tests/integration/cri-containerd/integration-tests.sh index 7c4bf9253311..0280dda2f6af 100755 --- a/tests/integration/cri-containerd/integration-tests.sh +++ b/tests/integration/cri-containerd/integration-tests.sh @@ -29,6 +29,9 @@ RUNTIME=${RUNTIME:-containerd-shim-kata-${KATA_HYPERVISOR}-v2} FACTORY_TEST=${FACTORY_TEST:-""} ARCH=$(uname -m) SANDBOXER=${SANDBOXER:-"podsandbox"} +# This is to avoid the following error in msft-preview: +# pod memory limit too low: minimum 128MiB, got 0Mib +POD_MEMORY_LIMIT_IN_BYTES=${POD_MEMORY_LIMIT_IN_BYTES:-268435456} # 256Mi containerd_runtime_type="io.containerd.kata-${KATA_HYPERVISOR}.v2" @@ -191,6 +194,9 @@ metadata: name: busybox-sandbox1 namespace: default uid: busybox-sandbox1-uid +linux: + resources: + memory_limit_in_bytes: ${POD_MEMORY_LIMIT_IN_BYTES} EOF #TestContainerSwap has created its own container_yaml. @@ -502,6 +508,9 @@ metadata: name: busybox-device-cgroup-sandbox namespace: default uid: busybox-device-cgroup-sandbox-uid +linux: + resources: + memory_limit_in_bytes: ${POD_MEMORY_LIMIT_IN_BYTES} EOF cat > "$container1_yaml" </dev/null || echo "unknown") @@ -86,7 +94,7 @@ endif ################################################################################ .PHONY: all -all: image initrd +all: image initrd igvm rootfs-%: $(ROOTFS_BUILD_DEST)/.%$(ROOTFS_MARKER_SUFFIX) @ # DONT remove. This is not cancellation rule. @@ -156,6 +164,10 @@ $(DRACUT_OVERLAY_DIR): mkdir -p $@/etc/modules-load.d echo $(DRACUT_KMODULES) | tr " " "\n" > $@/etc/modules-load.d/kata-modules.conf +.PHONY: igvm +igvm: $(TARGET_IMAGE) + $(IGVM_BUILDER) -o $(IGVM_BUILD_DEST) -s $(IGVM_SVN) + .PHONY: test test: $(TEST_RUNNER) "$(DISTRO)" @@ -208,7 +220,8 @@ install-scripts: .PHONY: clean clean: - rm -rf $(TARGET_ROOTFS_MARKER) $(TARGET_ROOTFS) $(TARGET_IMAGE) $(TARGET_INITRD) $(DRACUT_OVERLAY_DIR) + rm -rf $(TARGET_ROOTFS_MARKER) $(TARGET_ROOTFS) $(TARGET_IMAGE) $(TARGET_INITRD) $(DRACUT_OVERLAY_DIR) $(TARGET_IGVM) $(TARGET_IGVM_DEBUG) $(TARGET_IGVM_MSMT) $(TARGET_IGVM_DEBUG_MSMT) $(TARGET_IGVM_LOG) + rm -rf $(IGVM_TOOL_SRC) # Prints the name of the variable passed as suffix to the print- target, # E.g., if Makefile contains: diff --git a/tools/osbuilder/README.md b/tools/osbuilder/README.md index ea05162c9f29..f8fe104508a1 100644 --- a/tools/osbuilder/README.md +++ b/tools/osbuilder/README.md @@ -83,6 +83,10 @@ the image. Ubuntu is the default distro for building the rootfs, to use a different one, you can set `DISTRO=alpine|debian|ubuntu|cbl-mariner`. For example `make USE_DOCKER=true DISTRO=alpine rootfs` will make an Alpine rootfs using Docker. +### Azure Linux host node deployment + +See [the node builder documentation](node-builder/azure-linux/README.md). + ### Rootfs creation This section shows how to build a basic rootfs using the default distribution. diff --git a/tools/osbuilder/igvm-builder/azure-linux/config.sh b/tools/osbuilder/igvm-builder/azure-linux/config.sh new file mode 100644 index 000000000000..ade604dd6046 --- /dev/null +++ b/tools/osbuilder/igvm-builder/azure-linux/config.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2024 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +# this is where the kernel-uvm package installation places bzImage, see SPEC file +BZIMAGE_BIN="/usr/share/cloud-hypervisor/bzImage" + +IGVM_EXTRACT_FOLDER="${SCRIPT_DIR}/igvm-tooling" +CLH_ACPI_TABLES_DIR="${IGVM_EXTRACT_FOLDER}/src/igvm/acpi/acpi-clh/" +IGVM_PY_FILE="${IGVM_EXTRACT_FOLDER}/src/igvm/igvmgen.py" + +IGVM_BUILD_VARS="-kernel ${BZIMAGE_BIN} -boot_mode x64 -vtl 0 -svme 1 -encrypted_page 1 -pvalidate_opt 1 -acpi ${CLH_ACPI_TABLES_DIR}" + +IGVM_KERNEL_PARAMS_COMMON="dm-mod.create=\"dm-verity,,,ro,0 ${IMAGE_DATA_SECTORS} verity 1 /dev/vda1 /dev/vda2 ${IMAGE_DATA_BLOCK_SIZE} ${IMAGE_HASH_BLOCK_SIZE} ${IMAGE_DATA_BLOCKS} 0 sha256 ${IMAGE_ROOT_HASH} ${IMAGE_SALT}\" \ + root=/dev/dm-0 rootflags=data=ordered,errors=remount-ro ro rootfstype=ext4 panic=1 no_timer_check noreplace-smp systemd.unit=kata-containers.target systemd.mask=systemd-networkd.service \ + systemd.mask=systemd-networkd.socket agent.enable_signature_verification=false" +IGVM_KERNEL_PROD_PARAMS="${IGVM_KERNEL_PARAMS_COMMON} quiet" +IGVM_KERNEL_DEBUG_PARAMS="${IGVM_KERNEL_PARAMS_COMMON} console=hvc0 systemd.log_target=console agent.log=debug agent.debug_console agent.debug_console_vport=1026" + +IGVM_FILE_NAME="kata-containers-igvm.img" +IGVM_DBG_FILE_NAME="kata-containers-igvm-debug.img" +IGVM_MEASUREMENT_FILE_NAME="igvm-measurement.cose" +IGVM_DBG_MEASUREMENT_FILE_NAME="igvm-debug-measurement.cose" diff --git a/tools/osbuilder/igvm-builder/azure-linux/igvm_lib.sh b/tools/osbuilder/igvm-builder/azure-linux/igvm_lib.sh new file mode 100644 index 000000000000..e5b13307445f --- /dev/null +++ b/tools/osbuilder/igvm-builder/azure-linux/igvm_lib.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2024 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +install_igvm_tool() +{ + echo "Installing IGVM tool" + if [ -d ${IGVM_EXTRACT_FOLDER} ]; then + echo "${IGVM_EXTRACT_FOLDER} folder already exists, assuming tool is already installed" + return + fi + + # the igvm tool on Azure Linux will soon be properly installed through dnf via kata-packages-uvm-build + # as of now, even when installing with pip3, we cannot delete the source folder as the ACPI tables are not being installed anywhere, hence relying on this folder + echo "Determining and downloading latest IGVM tooling release, and extracting including ACPI tables" + IGVM_VER=$(curl -sL "https://api.github.com/repos/microsoft/igvm-tooling/releases/latest" | jq -r .tag_name | sed 's/^v//') + curl -sL "https://github.com/microsoft/igvm-tooling/archive/refs/tags/${IGVM_VER}.tar.gz" | tar --no-same-owner -xz + mv igvm-tooling-${IGVM_VER} ${IGVM_EXTRACT_FOLDER} + + echo "Installing IGVM module msigvm (${IGVM_VER}) via pip3" + pushd ${IGVM_EXTRACT_FOLDER}/src + pip3 install --no-deps ./ + popd +} + +uninstall_igvm_tool() +{ + echo "Uninstalling IGVM tool" + + rm -rf ${IGVM_EXTRACT_FOLDER} + pip3 uninstall -y msigvm +} + +build_igvm_files() +{ + echo "Reading Kata image dm_verity root hash information from root_hash file" + ROOT_HASH_FILE="${SCRIPT_DIR}/../root_hash.txt" + + if [ ! -f "${ROOT_HASH_FILE}" ]; then + echo "Could no find image root hash file '${ROOT_HASH_FILE}', aborting" + exit 1 + fi + + IMAGE_ROOT_HASH=$(sed -e 's/Root hash:\s*//g;t;d' "${ROOT_HASH_FILE}") + IMAGE_SALT=$(sed -e 's/Salt:\s*//g;t;d' "${ROOT_HASH_FILE}") + IMAGE_DATA_BLOCKS=$(sed -e 's/Data blocks:\s*//g;t;d' "${ROOT_HASH_FILE}") + IMAGE_DATA_BLOCK_SIZE=$(sed -e 's/Data block size:\s*//g;t;d' "${ROOT_HASH_FILE}") + IMAGE_DATA_SECTORS_PER_BLOCK=$((IMAGE_DATA_BLOCK_SIZE / 512)) + IMAGE_DATA_SECTORS=$((IMAGE_DATA_BLOCKS * IMAGE_DATA_SECTORS_PER_BLOCK)) + IMAGE_HASH_BLOCK_SIZE=$(sed -e 's/Hash block size:\s*//g;t;d' "${ROOT_HASH_FILE}") + + # reloading the config file as various variables depend on above values + load_config_distro + + echo "Building (debug) IGVM files and creating their reference measurement files" + # we could call into the installed binary '~/.local/bin/igvmgen' when adding to PATH or, better, into 'python3 -m msigvm' + # however, as we still need the installation directory for the ACPI tables, we leave things as is for now + # at the same time we seem to need to call pip3 install for invoking the tool at all + python3 ${IGVM_PY_FILE} $IGVM_BUILD_VARS -o $IGVM_FILE_NAME -measurement_file $IGVM_MEASUREMENT_FILE_NAME -append "$IGVM_KERNEL_PROD_PARAMS" -svn $SVN + python3 ${IGVM_PY_FILE} $IGVM_BUILD_VARS -o $IGVM_DBG_FILE_NAME -measurement_file $IGVM_DBG_MEASUREMENT_FILE_NAME -append "$IGVM_KERNEL_DEBUG_PARAMS" -svn $SVN + + if [ "${PWD}" -ef "$(readlink -f $OUT_DIR)" ]; then + echo "OUT_DIR matches with current dir, not moving build artifacts" + else + echo "Moving build artifacts to ${OUT_DIR}" + mv $IGVM_FILE_NAME $IGVM_DBG_FILE_NAME $IGVM_MEASUREMENT_FILE_NAME $IGVM_DBG_MEASUREMENT_FILE_NAME $OUT_DIR + fi +} diff --git a/tools/osbuilder/igvm-builder/igvm_builder.sh b/tools/osbuilder/igvm-builder/igvm_builder.sh new file mode 100755 index 000000000000..8e539f69d941 --- /dev/null +++ b/tools/osbuilder/igvm-builder/igvm_builder.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2024 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o pipefail +set -o errtrace + +[ -n "$DEBUG" ] && set -x + +SCRIPT_DIR="$(dirname $(readlink -f $0))" + +# distro-specific config file +typeset -r CONFIG_SH="config.sh" + +# Name of an optional distro-specific file which, if it exists, must implement the +# install_igvm_tool, build_igvm_files, and uninstall_igvm_tool functions. +typeset -r LIB_SH="igvm_lib.sh" + +load_config_distro() +{ + distro_config_dir="${SCRIPT_DIR}/${DISTRO}" + + [ -d "${distro_config_dir}" ] || die "Could not find configuration directory '${distro_config_dir}'" + + if [ -e "${distro_config_dir}/${LIB_SH}" ]; then + igvm_lib="${distro_config_dir}/${LIB_SH}" + echo "igvm_lib.sh file found. Loading content" + source "${igvm_lib}" + fi + + # Source config.sh from distro, depends on root_hash based variables here + igvm_config="${distro_config_dir}/${CONFIG_SH}" + source "${igvm_config}" +} + +DISTRO="azure-linux" +MODE="build" + +while getopts ":o:s:iu" OPTIONS; do + case "${OPTIONS}" in + o ) OUT_DIR=$OPTARG ;; + s ) SVN=$OPTARG ;; + i ) MODE="install" ;; + u ) MODE="uninstall" ;; + \? ) + echo "Error - Invalid Option: -$OPTARG" 1>&2 + exit 1 + ;; + : ) + echo "Error - Invalid Option: -$OPTARG requires an argument" 1>&2 + exit 1 + ;; + esac +done + +echo "IGVM builder script" +echo "-- OUT_DIR -> $OUT_DIR" +echo "-- SVN -> $SVN" +echo "-- DISTRO -> $DISTRO" +echo "-- MODE -> $MODE" + +if [ -n "$DISTRO" ]; then + load_config_distro +else + echo "DISTRO must be specified" + exit 1 +fi + +case "$MODE" in + "install") + install_igvm_tool + ;; + "uninstall") + uninstall_igvm_tool + ;; + "build") + build_igvm_files + ;; +esac diff --git a/tools/osbuilder/node-builder/azure-linux/Makefile b/tools/osbuilder/node-builder/azure-linux/Makefile new file mode 100644 index 000000000000..85ebf59e2114 --- /dev/null +++ b/tools/osbuilder/node-builder/azure-linux/Makefile @@ -0,0 +1,77 @@ +# Copyright (c) 2024 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +BUILD_TYPE := release + +export SHIM_REDEPLOY_CONFIG := yes + +ifeq ($(BUILD_TYPE),debug) + export AGENT_BUILD_TYPE := debug + export AGENT_POLICY_FILE := allow-all.rego + export SHIM_USE_DEBUG_CONFIG := yes +else + export AGENT_BUILD_TYPE := release + export AGENT_POLICY_FILE := allow-set-policy.rego + export SHIM_USE_DEBUG_CONFIG := no +endif + +.PHONY: all +all: package uvm + +.PHONY: all-confpods +all-confpods: package-confpods uvm-confpods + +.PHONY: package +package: + ./package_build.sh + +.PHONY: package-confpods +package-confpods: + CONF_PODS=yes ./package_build.sh + +.PHONY: uvm +uvm: + ./uvm_build.sh + +.PHONY: uvm-confpods +uvm-confpods: + CONF_PODS=yes ./uvm_build.sh + +.PHONY: clean +clean: + ./clean.sh + +.PHONY: clean-confpods +clean-confpods: + CONF_PODS=yes ./clean.sh + +.PHONY: deploy +deploy: deploy-package deploy-uvm + +.PHONY: deploy-package +deploy-package: + ./package_install.sh + +.PHONY: deploy-package-tools +deploy-package-tools: + ./package_tools_install.sh + +.PHONY: deploy-uvm +deploy-uvm: + ./uvm_install.sh + +.PHONY: deploy-confpods +deploy-confpods: deploy-confpods-package deploy-confpods-uvm + +.PHONY: deploy-confpods-package +deploy-confpods-package: + CONF_PODS=yes ./package_install.sh + +.PHONY: deploy-confpods-package-tools +deploy-confpods-package-tools: + CONF_PODS=yes ./package_tools_install.sh + +.PHONY: deploy-confpods-uvm +deploy-confpods-uvm: + CONF_PODS=yes ./uvm_install.sh diff --git a/tools/osbuilder/node-builder/azure-linux/README.md b/tools/osbuilder/node-builder/azure-linux/README.md new file mode 100644 index 000000000000..9d143d207791 --- /dev/null +++ b/tools/osbuilder/node-builder/azure-linux/README.md @@ -0,0 +1,294 @@ +# Overview + +This guide serves as a reference on how to build and install the underlying software stack for *Pod Sandboxing with AKS* and for *Confidential Containers on AKS* using Azure Linux. +This enables running Kata (Confidential) Containers via the OCI interface, or via a local kubelet, or leveraging AKS' Kubernetes solution. + +In the following, the terms *Kata* and *Kata-CC* refer to *Pod Sandboxing with AKS* and *Confidential Containers on AKS*, respectively. +The term *building* refers to build the components from source, whereas the term *installing* refers to utilizing components released by the Azure Linux team for straightforward evaluation. + +The guide provides the steps for two different environments: +- Azure Linux 3 based systems, such as Azure VMs + - Variant I: Utilize released components + - Variant II: Build components from source +- AKS nodes (based on Azure Linux 2 as of today) + +# Steps for Azure Linux 3 based environments + +## Set up AzL3 environment + +While build can happen in any Azure Linux 3 based environment, the stack can only be evaluated on environments with proper virtualization support and, for Kata-CC, on top of AMD SEV-SNP. An example of such environment are Azure Linux 3 based Azure VMs using a proper SKU: +- Deploy an Azure Linux 3 VM via `az vm create` using a [CC vm size SKU](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasccv5-dcadsccv5-series) + - Example: `az vm create --resource-group --name --os-disk-size-gb --public-ip-sku Standard --size --admin-username azureuser --ssh-key-values --image ` +- SSH onto the VM + +Not validated for evaluation: Install [Azure Linux 3](https://github.com/microsoft/azurelinux) on a bare metal machine supporting AMD SEV-SNP. + +To merely build the stack, we refer to the official [Azure Linux GitHub page](https://github.com/microsoft/azurelinux) to set up an Azure Linux 3 environment. + +## Deploy required host packages (incl. VMM, SEV-SNP capable kernel and Microsoft Hypervisor) and extend containerd configuration + +Install relevant packages, append a configuration snippet to `/etc/containerd/config.toml` to register the Kata(-CC) handlers, then reboot the system: +``` +sudo dnf -y makecache +sudo dnf -y install kata-packages-host + +sudo tee -a /etc/containerd/config.toml 2&>1 <` + - For build and deployment of both Kata and Kata-CC artifacts, first run the `make all` and `make deploy` commands to build and install the Kata Containers for AKS components followed by `make clean`, and then run `make all-confpods` and `make deploy-confpods` to build and install the Confidential Containers for AKS components - or vice versa (using `make clean-confpods`). + +## Debug builds + +This section describes how to build and deploy in debug mode. + +`make all-confpods` takes the following variables: + + * `AGENT_BUILD_TYPE`: Specify `release` (default) to build the agent in + release mode, or `debug` to build it in debug mode. + * `AGENT_POLICY_FILE`: Specify `allow-set-policy.rego` (default) to use + a restrictive policy, or `allow-all.rego` to use a permissive policy. + +`make deploy-confpods` takes the following variable: + + * `SHIM_USE_DEBUG_CONFIG`: Specify `no` (default) to use the production + configuration, or `yes` to use the debug configuration (all debug + logging enabled). In this case you'll want to enable debug logging + in containerd as well. Note that this variable has no effect if + `SHIM_REDEPLOY_CONFIG=no`. + +In general, you can specify the debug configuration for all the above +variables by using `BUILD_TYPE=debug` as such: + +```shell +sudo make BUILD_TYPE=debug all-confpods deploy-confpods +``` + +Also note that make still lets you override the other variables even +after setting `BUILD_TYPE`. For example, you can use the production shim +config with `BUILD_TYPE=debug`: + +```shell +sudo make BUILD_TYPE=debug SHIM_USE_DEBUG_CONFIG=no all-confpods deploy-confpods +``` + +### Prevent redeploying the shim configuration + +If you're manually modifying the shim configuration directly on the host +during development and you don't want to redeploy and overwrite that +file each time you redeploy binaries, you can separately specify the +`SHIM_REDEPLOY_CONFIG` (default `yes`): + +```shell +sudo make SHIM_REDEPLOY_CONFIG=no all-confpods deploy-confpods +``` + +Note that this variable is independent from the other variables +mentioned above. So if you want to avoid redeploying the shim +configuration AND build in debug mode, you have to use the following +command: + +```shell +sudo make BUILD_TYPE=debug SHIM_REDEPLOY_CONFIG=no all-confpods deploy-confpods +``` + +# Run Kata (Confidential) Containers + +## Run via CRI or via containerd API + +Use e.g. `crictl` (or `ctr`) to schedule Kata (Confidential) containers, referencing either the Kata or Kata-CC handlers. + +Note: On Kubernetes nodes, pods created via `crictl` will be deleted by the control plane. + +The following instructions serve as a general reference: +- Install `crictl`, `cni` binaries, and set runtime endpoint in `crictl` configuration: + + ``` + sudo dnf -y install cri-tools cni + sudo crictl config --set runtime-endpoint=unix:///run/containerd/containerd.sock + ``` + +- Set a proper CNI configuration and create a sample pod manifest: This step is omitted as it depends on the individual needs. + +- Run pods with `crictl`, for example: + + `sudo crictl runp -T 30s -r ` + +- Run containers with `ctr`, for example a confidential container: + + `sudo ctr -n=k8s.io image pull --snapshotter=tardev docker.io/library/busybox:latest` + + `sudo ctr -n=k8s.io run --cni --runtime io.containerd.run.kata-cc.v2 --runtime-config-path /opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml --snapshotter tardev -t --rm docker.io/library/busybox:latest hello sh` + +For further usage we refer to the upstream `crictl` (or `ctr`) and CNI documentation. + +## Run via Kubernetes + +If your environment was set up through `az aks create` the respective node is ready to run Kata (Confidential) Containers as AKS Kubernetes pods. +Other types of Kubernetes clusters should work as well. While this document doesn't cover how to set-up those clusters, you can +apply the kata and kata-cc runtime classes to your cluster from the machine that holds your kubeconfig file, for example: +``` +cat << EOF > runtimeClass-kata-cc.yaml +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: kata-cc +handler: kata-cc +overhead: + podFixed: + memory: "600Mi" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" +EOF + +cat << EOF > runtimeClass-kata.yaml +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: kata +handler: kata +overhead: + podFixed: + memory: "600Mi" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" +EOF + +kubectl apply -f runtimeClass-kata-cc.yaml -f runtimeClass-kata.yaml +``` + +And label your node appropriately: +``` +kubectl label node katacontainers.io/kata-runtime=true +``` + +# Build attestation scenarios +The build artifacts for the UVM ConfPods target include an IGVM file and a so-called reference measurement file (unsigned). The IGVM file is being loaded into memory measured by the AMD SEV-SNP PSP (when a Confidental Container is started). With this and with the Kata security policy feature, attestation scenarios can be built: the reference measurement (often referred to as 'endorsement') can, for example, be signed by a trusted party (such as Microsoft in Confidential Containers on AKS) and be compared with the actual measurement part of the attestation report. The latter can be retrieved through respective system calls inside the Kata Confidential Containers Guest VM. + +An example for an attestation scenario through Microsoft Azure Attestation is presented in [Attestation in Confidential containers on Azure Container Instances](https://learn.microsoft.com/en-us/azure/container-instances/confidential-containers-attestation-concepts). +Documentation for leveraging the Kata security policy feature can be found in [Security policy for Confidential Containers on Azure Kubernetes Service](https://learn.microsoft.com/en-us/azure/confidential-computing/confidential-containers-aks-security-policy). diff --git a/tools/osbuilder/node-builder/azure-linux/clean.sh b/tools/osbuilder/node-builder/azure-linux/clean.sh new file mode 100755 index 000000000000..11cf6fb03bd5 --- /dev/null +++ b/tools/osbuilder/node-builder/azure-linux/clean.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2024 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o pipefail +set -o errtrace + +[ -n "$DEBUG" ] && set -x + +script_dir="$(dirname $(readlink -f $0))" +repo_dir="${script_dir}/../../../../" + +common_file="common.sh" +source "${common_file}" + +pushd "${repo_dir}" + +echo "Clean debug shim config" +pushd src/runtime/config/ +rm -f "${SHIM_DBG_CONFIG_FILE_NAME}" +popd + +echo "Clean runtime build" +pushd src/runtime/ +make clean SKIP_GO_VERSION_CHECK=1 +popd + +echo "Clean agent build" +pushd src/agent/ +make clean +popd + +rm -rf ${AGENT_INSTALL_DIR} + +echo "Clean UVM build" +pushd tools/osbuilder/ +sudo -E PATH=$PATH make DISTRO=cbl-mariner clean +popd + +echo "Clean IGVM tool installation" + + +if [ "${CONF_PODS}" == "yes" ]; then + + echo "Clean tardev-snapshotter tarfs driver build" + pushd src/tarfs + set_uvm_kernel_vars + if [ -n "${UVM_KERNEL_HEADER_DIR}" ]; then + make clean KDIR=${UVM_KERNEL_HEADER_DIR} + fi + popd + + echo "Clean utarfs binary build" + pushd src/utarfs/ + make clean + popd + + echo "Clean tardev-snapshotter overlay binary build" + pushd src/overlay/ + make clean + popd + + echo "Clean tardev-snapshotter service build" + pushd src/tardev-snapshotter/ + make clean + popd +fi + +popd diff --git a/tools/osbuilder/node-builder/azure-linux/common.sh b/tools/osbuilder/node-builder/azure-linux/common.sh new file mode 100755 index 000000000000..8b0665c47aa2 --- /dev/null +++ b/tools/osbuilder/node-builder/azure-linux/common.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2024 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +script_dir="$(dirname $(readlink -f $0))" +lib_file="${script_dir}/../../scripts/lib.sh" +source "${lib_file}" + +OS_VERSION=$(sort -r /etc/*-release | gawk 'match($0, /^(VERSION_ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }' | tr -d '"') + +([[ "${OS_VERSION}" == "2.0" ]] || [[ "${OS_VERSION}" == "3.0" ]]) || die "OS_VERSION: value '${OS_VERSION}' must equal 3.0 (default) or 2.0" + +if [ "${CONF_PODS}" == "yes" ]; then + INSTALL_PATH_PREFIX="/opt/confidential-containers" + UVM_TOOLS_PATH_OSB="${INSTALL_PATH_PREFIX}/uvm/tools/osbuilder" + UVM_TOOLS_PATH_SRC="${INSTALL_PATH_PREFIX}/uvm/src" + UVM_PATH_DEFAULT="${INSTALL_PATH_PREFIX}/share/kata-containers" + IMG_FILE_NAME="kata-containers.img" + IGVM_FILE_NAME="kata-containers-igvm.img" + IGVM_DBG_FILE_NAME="kata-containers-igvm-debug.img" + UVM_MEASUREMENT_FILE_NAME="igvm-measurement.cose" + UVM_DBG_MEASUREMENT_FILE_NAME="igvm-debug-measurement.cose" + SHIM_CONFIG_PATH="${INSTALL_PATH_PREFIX}/share/defaults/kata-containers" + SHIM_CONFIG_FILE_NAME="configuration-clh-snp.toml" + SHIM_CONFIG_INST_FILE_NAME="${SHIM_CONFIG_FILE_NAME}" + SHIM_DBG_CONFIG_FILE_NAME="configuration-clh-snp-debug.toml" + SHIM_DBG_CONFIG_INST_FILE_NAME="${SHIM_DBG_CONFIG_FILE_NAME}" + DEBUGGING_BINARIES_PATH="${INSTALL_PATH_PREFIX}/bin" + SHIM_BINARIES_PATH="/usr/local/bin" + SHIM_BINARY_NAME="containerd-shim-kata-cc-v2" +else + INSTALL_PATH_PREFIX="/usr" + UVM_TOOLS_PATH_OSB="/opt/kata-containers/uvm/tools/osbuilder" + UVM_TOOLS_PATH_SRC="/opt/kata-containers/uvm/src" + UVM_PATH_DEFAULT="${INSTALL_PATH_PREFIX}/share/kata-containers" + IMG_FILE_NAME="kata-containers.img" + SHIM_CONFIG_PATH="${INSTALL_PATH_PREFIX}/share/defaults/kata-containers" + SHIM_CONFIG_FILE_NAME="configuration-clh.toml" + SHIM_CONFIG_INST_FILE_NAME="configuration.toml" + SHIM_DBG_CONFIG_FILE_NAME="configuration-clh-debug.toml" + SHIM_DBG_CONFIG_INST_FILE_NAME="${SHIM_DBG_CONFIG_FILE_NAME}" + DEBUGGING_BINARIES_PATH="${INSTALL_PATH_PREFIX}/local/bin" + SHIM_BINARIES_PATH="${INSTALL_PATH_PREFIX}/local/bin" + SHIM_BINARY_NAME="containerd-shim-kata-v2" +fi + +# this is where cloud-hypervisor-cvm gets installed (see package SPEC) +CLOUD_HYPERVISOR_LOCATION="/usr/bin/cloud-hypervisor" +# this is where kernel-uvm gets installed (see package SPEC) +KERNEL_BINARY_LOCATION="/usr/share/cloud-hypervisor/vmlinux.bin" +# Mariner 3: different binary name +if [ "${OS_VERSION}" == "2.0" ]; then + VIRTIOFSD_BINARY_LOCATION="/usr/libexec/virtiofsd-rs" +else + VIRTIOFSD_BINARY_LOCATION="/usr/libexec/virtiofsd" +fi + +AGENT_INSTALL_DIR="${script_dir}/agent-install" + +set_uvm_kernel_vars() { + UVM_KERNEL_VERSION=$(rpm -q --queryformat '%{VERSION}' kernel-uvm-devel) + UVM_KERNEL_RELEASE=$(rpm -q --queryformat '%{RELEASE}' kernel-uvm-devel) + UVM_KERNEL_HEADER_DIR="/usr/src/linux-headers-${UVM_KERNEL_VERSION}-${UVM_KERNEL_RELEASE}" +} diff --git a/tools/osbuilder/node-builder/azure-linux/package_build.sh b/tools/osbuilder/node-builder/azure-linux/package_build.sh new file mode 100755 index 000000000000..346ba5a9f092 --- /dev/null +++ b/tools/osbuilder/node-builder/azure-linux/package_build.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2024 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o pipefail +set -o errtrace + +[ -n "$DEBUG" ] && set -x + +AGENT_BUILD_TYPE=${AGENT_BUILD_TYPE:-release} +CONF_PODS=${CONF_PODS:-no} + +script_dir="$(dirname $(readlink -f $0))" +repo_dir="${script_dir}/../../../../" + +common_file="common.sh" +source "${common_file}" + +# these options ensure we produce the proper CLH config file +runtime_make_flags="SKIP_GO_VERSION_CHECK=1 QEMUCMD= FCCMD= ACRNCMD= STRATOVIRTCMD= DEFAULT_HYPERVISOR=cloud-hypervisor + DEFMEMSZ=0 DEFSTATICSANDBOXWORKLOADMEM=512 DEFVCPUS=0 DEFSTATICSANDBOXWORKLOADVCPUS=1 DEFVIRTIOFSDAEMON=${VIRTIOFSD_BINARY_LOCATION} PREFIX=${INSTALL_PATH_PREFIX}" + +# - for vanilla Kata we use the kernel binary. For ConfPods we use IGVM, so no need to provide kernel path. +# - for vanilla Kata we explicitly set DEFSTATICRESOURCEMGMT_CLH. For ConfPods, +# the variable DEFSTATICRESOURCEMGMT_TEE is used which defaults to false +# - for ConfPods we explicitly set the cloud-hypervisor path. The path is independent of the PREFIX variable +# as we have a single CLH binary for both vanilla Kata and ConfPods +if [ "${CONF_PODS}" == "no" ]; then + runtime_make_flags+=" DEFSTATICRESOURCEMGMT_CLH=true KERNELPATH_CLH=${KERNEL_BINARY_LOCATION} DEFSANDBOXWORKLOADMEMMIN=128" +else + runtime_make_flags+=" CLHPATH=${CLOUD_HYPERVISOR_LOCATION} DEFSANDBOXWORKLOADMEMMIN=192" +fi + +# On Mariner 3.0 we use cgroupsv2 with a single sandbox cgroup +if [ "${OS_VERSION}" == "3.0" ]; then + runtime_make_flags+=" DEFSANDBOXCGROUPONLY=true" +fi + +agent_make_flags="LIBC=gnu OPENSSL_NO_VENDOR=Y DESTDIR=${AGENT_INSTALL_DIR} BUILD_TYPE=${AGENT_BUILD_TYPE}" + +if [ "${CONF_PODS}" == "yes" ]; then + agent_make_flags+=" AGENT_POLICY=yes" +fi + +pushd "${repo_dir}" + +if [ "${CONF_PODS}" == "yes" ]; then + + echo "Building utarfs binary" + pushd src/utarfs/ + make all + popd + + echo "Building kata-overlay binary" + pushd src/overlay/ + make all + popd + + echo "Building tardev-snapshotter service binary" + pushd src/tardev-snapshotter/ + make all + popd +fi + +echo "Building shim binary and configuration" +pushd src/runtime/ +if [ "${CONF_PODS}" == "yes" ] || [ "${OS_VERSION}" == "3.0" ]; then + make ${runtime_make_flags} +else + # Mariner 2 pod sandboxing uses cgroupsv1 - note: cannot add the kernelparams in above assignments, + # leads to quotation issue. Hence, implementing the conditional check right here at the time of the make command + make ${runtime_make_flags} KERNELPARAMS="systemd.legacy_systemd_cgroup_controller=yes systemd.unified_cgroup_hierarchy=0" +fi +popd + +pushd src/runtime/config/ +echo "Creating shim debug configuration" +cp "${SHIM_CONFIG_FILE_NAME}" "${SHIM_DBG_CONFIG_FILE_NAME}" +sed -i '/^#enable_debug =/s|^#||g' "${SHIM_DBG_CONFIG_FILE_NAME}" +sed -i '/^#debug_console_enabled =/s|^#||g' "${SHIM_DBG_CONFIG_FILE_NAME}" + +if [ "${CONF_PODS}" == "yes" ]; then + echo "Adding debug igvm to SNP shim debug configuration" + sed -i "s|${IGVM_FILE_NAME}|${IGVM_DBG_FILE_NAME}|g" "${SHIM_DBG_CONFIG_FILE_NAME}" +fi +popd + +echo "Building agent binary and generating service files" +pushd src/agent/ +make ${agent_make_flags} +make install ${agent_make_flags} +popd + +popd diff --git a/tools/osbuilder/node-builder/azure-linux/package_install.sh b/tools/osbuilder/node-builder/azure-linux/package_install.sh new file mode 100755 index 000000000000..791cff5d92d2 --- /dev/null +++ b/tools/osbuilder/node-builder/azure-linux/package_install.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2024 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o pipefail +set -o errtrace + +[ -n "$DEBUG" ] && set -x + +CONF_PODS=${CONF_PODS:-no} +PREFIX=${PREFIX:-} +SHIM_REDEPLOY_CONFIG=${SHIM_REDEPLOY_CONFIG:-yes} +SHIM_USE_DEBUG_CONFIG=${SHIM_USE_DEBUG_CONFIG:-no} +START_SERVICES=${START_SERVICES:-yes} + +script_dir="$(dirname $(readlink -f $0))" +repo_dir="${script_dir}/../../../../" + +common_file="common.sh" +source "${common_file}" + +pushd "${repo_dir}" + +echo "Creating target directories" +mkdir -p "${PREFIX}/${SHIM_CONFIG_PATH}" +mkdir -p "${PREFIX}/${DEBUGGING_BINARIES_PATH}" +mkdir -p "${PREFIX}/${SHIM_BINARIES_PATH}" + +if [ "${CONF_PODS}" == "yes" ]; then + echo "Installing tardev-snapshotter binaries and service file" + mkdir -p ${PREFIX}/usr/sbin + cp -a --backup=numbered src/utarfs/target/release/utarfs ${PREFIX}/usr/sbin/mount.tar + mkdir -p ${PREFIX}/usr/bin + cp -a --backup=numbered src/overlay/target/release/kata-overlay ${PREFIX}/usr/bin/ + cp -a --backup=numbered src/tardev-snapshotter/target/release/tardev-snapshotter ${PREFIX}/usr/bin/ + mkdir -p ${PREFIX}/usr/lib/systemd/system/ + cp -a --backup=numbered src/tardev-snapshotter/tardev-snapshotter.service ${PREFIX}/usr/lib/systemd/system/ + + echo "Enabling and starting snapshotter service" + if [ "${START_SERVICES}" == "yes" ]; then + systemctl enable tardev-snapshotter && systemctl daemon-reload && systemctl restart tardev-snapshotter + fi +fi + +echo "Installing diagnosability binaries (monitor, runtime, collect-data script)" +cp -a --backup=numbered src/runtime/kata-monitor "${PREFIX}/${DEBUGGING_BINARIES_PATH}" +cp -a --backup=numbered src/runtime/kata-runtime "${PREFIX}/${DEBUGGING_BINARIES_PATH}" +chmod +x src/runtime/data/kata-collect-data.sh +cp -a --backup=numbered src/runtime/data/kata-collect-data.sh "${PREFIX}/${DEBUGGING_BINARIES_PATH}" + +echo "Installing shim binary" +cp -a --backup=numbered src/runtime/containerd-shim-kata-v2 "${PREFIX}/${SHIM_BINARIES_PATH}"/"${SHIM_BINARY_NAME}" + +if [ "${SHIM_REDEPLOY_CONFIG}" == "yes" ]; then + echo "Installing shim configuration" + cp -a --backup=numbered src/runtime/config/"${SHIM_CONFIG_FILE_NAME}" "${PREFIX}/${SHIM_CONFIG_PATH}/${SHIM_CONFIG_INST_FILE_NAME}" + cp -a --backup=numbered src/runtime/config/"${SHIM_DBG_CONFIG_FILE_NAME}" "${PREFIX}/${SHIM_CONFIG_PATH}/${SHIM_DBG_CONFIG_INST_FILE_NAME}" + + if [ "${SHIM_USE_DEBUG_CONFIG}" == "yes" ]; then + # We simply override the release config with the debug config, + # which is probably fine when debugging. Not symlinking as that + # would create cycles the next time this script is called. + echo "Overriding shim configuration with debug configuration" + cp -a --backup=numbered src/runtime/config/"${SHIM_DBG_CONFIG_FILE_NAME}" "${PREFIX}/${SHIM_CONFIG_PATH}/${SHIM_CONFIG_INST_FILE_NAME}" + fi +else + echo "Skipping installation of shim configuration" +fi + +popd diff --git a/tools/osbuilder/node-builder/azure-linux/package_tools_install.sh b/tools/osbuilder/node-builder/azure-linux/package_tools_install.sh new file mode 100755 index 000000000000..a1f32ca88576 --- /dev/null +++ b/tools/osbuilder/node-builder/azure-linux/package_tools_install.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2024 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o pipefail +set -o errtrace + +[ -n "$DEBUG" ] && set -x + +CONF_PODS=${CONF_PODS:-no} +PREFIX=${PREFIX:-} + +script_dir="$(dirname $(readlink -f $0))" +repo_dir="${script_dir}/../../../../" + +common_file="common.sh" +source "${common_file}" + +pushd "${repo_dir}" + +echo "Creating target directories" +mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/scripts" +mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/rootfs-builder/cbl-mariner" +mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/image-builder" +mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/bin" +mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/lib/systemd/system" + +if [ "${CONF_PODS}" == "yes" ]; then + mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_SRC}/kata-opa" + mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs" + mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/igvm-builder/azure-linux" +fi + +echo "Installing UVM build scripting" +cp -a --backup=numbered tools/osbuilder/Makefile "${PREFIX}/${UVM_TOOLS_PATH_OSB}/" +cp -a --backup=numbered tools/osbuilder/scripts/lib.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/scripts/" +cp -a --backup=numbered tools/osbuilder/rootfs-builder/rootfs.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/rootfs-builder/" +cp -a --backup=numbered tools/osbuilder/rootfs-builder/cbl-mariner/config.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/rootfs-builder/cbl-mariner/" +cp -a --backup=numbered tools/osbuilder/rootfs-builder/cbl-mariner/rootfs_lib.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/rootfs-builder/cbl-mariner/" +cp -a --backup=numbered tools/osbuilder/image-builder/image_builder.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/image-builder/" +cp -a --backup=numbered tools/osbuilder/image-builder/nsdax.gpl.c "${PREFIX}/${UVM_TOOLS_PATH_OSB}/image-builder/" +cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/Makefile "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/" +cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/clean.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/" +cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/common.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/" +cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/uvm_build.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/" +cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/uvm_install.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/" + +echo "Installing agent binary and service files" +cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/agent-install/usr/bin/kata-agent "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/bin/" +cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/agent-install/usr/lib/systemd/system/kata-containers.target "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/lib/systemd/system/" +cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/agent-install/usr/lib/systemd/system/kata-agent.service "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/lib/systemd/system/" + +if [ "${CONF_PODS}" == "yes" ]; then + cp -a --backup=numbered src/kata-opa/allow-all.rego "${PREFIX}/${UVM_TOOLS_PATH_SRC}/kata-opa/" + cp -a --backup=numbered src/kata-opa/allow-set-policy.rego "${PREFIX}/${UVM_TOOLS_PATH_SRC}/kata-opa/" + cp -a --backup=numbered src/tarfs/Makefile "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs/" + cp -a --backup=numbered src/tarfs/tarfs.c "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs/" + cp -a --backup=numbered tools/osbuilder/igvm-builder/igvm_builder.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/igvm-builder/" + cp -a --backup=numbered tools/osbuilder/igvm-builder/azure-linux/config.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/igvm-builder/azure-linux/" + cp -a --backup=numbered tools/osbuilder/igvm-builder/azure-linux/igvm_lib.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/igvm-builder/azure-linux/" +fi + +popd diff --git a/tools/osbuilder/node-builder/azure-linux/uvm_build.sh b/tools/osbuilder/node-builder/azure-linux/uvm_build.sh new file mode 100755 index 000000000000..c24b71e2e7d5 --- /dev/null +++ b/tools/osbuilder/node-builder/azure-linux/uvm_build.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2024 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o pipefail +set -o errtrace + +[ -n "$DEBUG" ] && set -x + +AGENT_POLICY_FILE="${AGENT_POLICY_FILE:-allow-set-policy.rego}" +CONF_PODS=${CONF_PODS:-no} +IGVM_SVN=${IGVM_SVN:-0} + +script_dir="$(dirname $(readlink -f $0))" +repo_dir="${script_dir}/../../../../" + +agent_policy_file_abs="${repo_dir}/src/kata-opa/${AGENT_POLICY_FILE}" + +common_file="common.sh" +source "${common_file}" + +# This ensures that a pre-built agent binary is being injected into the rootfs +rootfs_make_flags="AGENT_SOURCE_BIN=${AGENT_INSTALL_DIR}/usr/bin/kata-agent OS_VERSION=${OS_VERSION}" + +if [ "${CONF_PODS}" == "yes" ]; then + rootfs_make_flags+=" AGENT_POLICY=yes CONF_GUEST=yes AGENT_POLICY_FILE=${agent_policy_file_abs}" +fi + +if [ "${CONF_PODS}" == "yes" ]; then + set_uvm_kernel_vars + if [ -z "${UVM_KERNEL_HEADER_DIR}" ]; then + exit 1 + fi +fi + +pushd "${repo_dir}" + +echo "Building rootfs and including pre-built agent binary" +pushd tools/osbuilder +# This command requires sudo because of dnf-installing packages into rootfs. As a suite, following commands require sudo as well as make clean +sudo -E PATH=$PATH make ${rootfs_make_flags} -B DISTRO=cbl-mariner rootfs +ROOTFS_PATH="$(readlink -f ./cbl-mariner_rootfs)" +popd + +echo "Installing agent service files into rootfs" +sudo cp ${AGENT_INSTALL_DIR}/usr/lib/systemd/system/kata-containers.target ${ROOTFS_PATH}/usr/lib/systemd/system/kata-containers.target +sudo cp ${AGENT_INSTALL_DIR}/usr/lib/systemd/system/kata-agent.service ${ROOTFS_PATH}/usr/lib/systemd/system/kata-agent.service + +if [ "${CONF_PODS}" == "yes" ]; then + echo "Building tarfs kernel driver and installing into rootfs" + pushd src/tarfs + make KDIR=${UVM_KERNEL_HEADER_DIR} + sudo make KDIR=${UVM_KERNEL_HEADER_DIR} KVER=${UVM_KERNEL_VERSION} INSTALL_MOD_PATH=${ROOTFS_PATH} install + popd + + echo "Building dm-verity protected image based on rootfs" + pushd tools/osbuilder + sudo -E PATH=$PATH make DISTRO=cbl-mariner MEASURED_ROOTFS=yes DM_VERITY_FORMAT=kernelinit IMAGE_SIZE_ALIGNMENT_MB=2 image + popd + + echo "Building IGVM and UVM measurement files" + pushd tools/osbuilder + sudo chmod o+r root_hash.txt + sudo make igvm DISTRO=cbl-mariner IGVM_SVN=${IGVM_SVN} + popd +else + echo "Building image based on rootfs" + pushd tools/osbuilder + sudo -E PATH=$PATH make DISTRO=cbl-mariner IMAGE_SIZE_ALIGNMENT_MB=2 image + popd +fi + +popd diff --git a/tools/osbuilder/node-builder/azure-linux/uvm_install.sh b/tools/osbuilder/node-builder/azure-linux/uvm_install.sh new file mode 100755 index 000000000000..09e2cfa386eb --- /dev/null +++ b/tools/osbuilder/node-builder/azure-linux/uvm_install.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2024 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o pipefail +set -o errtrace + +[ -n "$DEBUG" ] && set -x + +CONF_PODS=${CONF_PODS:-no} + +script_dir="$(dirname $(readlink -f $0))" +repo_dir="${script_dir}/../../../../" + +common_file="common.sh" +source "${common_file}" + +UVM_PATH=${UVM_PATH:-${UVM_PATH_DEFAULT}} + +pushd "${repo_dir}" + +pushd tools/osbuilder + +echo "Creating target directory" +mkdir -p "${UVM_PATH}" + +echo "Installing UVM files to target directory" +if [ "${CONF_PODS}" == "yes" ]; then + cp -a --backup=numbered "${IGVM_FILE_NAME}" "${UVM_PATH}" + cp -a --backup=numbered "${IGVM_DBG_FILE_NAME}" "${UVM_PATH}" + cp -a --backup=numbered "${UVM_MEASUREMENT_FILE_NAME}" "${UVM_PATH}" + cp -a --backup=numbered "${UVM_DBG_MEASUREMENT_FILE_NAME}" "${UVM_PATH}" +fi + +cp -a --backup=numbered "${IMG_FILE_NAME}" "${UVM_PATH}" + +popd + +popd diff --git a/tools/testing/gatekeeper/required-tests.yaml b/tools/testing/gatekeeper/required-tests.yaml index 11d4066593e7..1a25d1f9b8b4 100644 --- a/tools/testing/gatekeeper/required-tests.yaml +++ b/tools/testing/gatekeeper/required-tests.yaml @@ -64,49 +64,61 @@ mapping: - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nydus (lts, dragonball) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nydus (lts, qemu) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-amd64 (active, cloud-hypervisor) / run-cri-containerd-amd64 (active, cloud-hypervisor) - - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-amd64 (active, clh) / run-cri-containerd-amd64 (active, clh) - - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-amd64 (active, dragonball) / run-cri-containerd-amd64 (active, dragonball) + # msft-preview: pod memory limit too low: minimum 128MiB, got 0MiB + # - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-amd64 (active, clh) / run-cri-containerd-amd64 (active, clh) + # msft-preview: load podSandboxConfig: error converting YAML to JSON: yaml: line 6: found character that cannot start any token + # - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-amd64 (active, dragonball) / run-cri-containerd-amd64 (active, dragonball) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-amd64 (active, qemu-runtime-rs) / run-cri-containerd-amd64 (active, qemu-runtime-rs) - - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-amd64 (active, qemu) / run-cri-containerd-amd64 (active, qemu) - - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-amd64 (lts, clh) / run-cri-containerd-amd64 (lts, clh) + # msft-preview: load podSandboxConfig: error converting YAML to JSON: yaml: line 6: found character that cannot start any token + # - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-amd64 (active, qemu) / run-cri-containerd-amd64 (active, qemu) + # msft-preview: pod memory limit too low: minimum 128MiB, got 0MiB + # - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-amd64 (lts, clh) / run-cri-containerd-amd64 (lts, clh) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-amd64 (lts, cloud-hypervisor) / run-cri-containerd-amd64 (lts, cloud-hypervisor) - - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-amd64 (lts, dragonball) / run-cri-containerd-amd64 (lts, dragonball) + # msft-preview: load podSandboxConfig: error converting YAML to JSON: yaml: line 6: found character that cannot start any token + # - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-amd64 (lts, dragonball) / run-cri-containerd-amd64 (lts, dragonball) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-amd64 (lts, qemu-runtime-rs) / run-cri-containerd-amd64 (lts, qemu-runtime-rs) - - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-amd64 (lts, qemu) / run-cri-containerd-amd64 (lts, qemu) - - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-s390x (active, qemu) / run-cri-containerd-s390x (active, qemu) - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-arm64 / run-k8s-tests-on-arm64 (qemu, kubeadm) - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (cbl-mariner, clh, normal) - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (cbl-mariner, clh, small, containerd) - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (cbl-mariner, clh, small, oci-distribution) - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, clh, normal) - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, clh, small) - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, cloud-hypervisor, normal) - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, cloud-hypervisor, small) - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, dragonball, normal) - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, dragonball, small) - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, qemu, normal) - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, qemu, small) - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, qemu-runtime-rs, small) - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, qemu-runtime-rs, normal) - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-zvsi / run-k8s-tests (devmapper, qemu, kubeadm) - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-zvsi / run-k8s-tests (nydus, qemu-coco-dev, kubeadm) + # msft-preview: load podSandboxConfig: error converting YAML to JSON: yaml: line 6: found character that cannot start any token + # - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-amd64 (lts, qemu) / run-cri-containerd-amd64 (lts, qemu) + # msft-preview: We don't have s390x runners. + # - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-s390x (active, qemu) / run-cri-containerd-s390x (active, qemu) + # msft-preview: We don't run k8s tests yet. + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-arm64 / run-k8s-tests-on-arm64 (qemu, kubeadm) + # msft-preview: Can't run these tests because the GHCR push isn't set up. + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (cbl-mariner, clh, normal) + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (cbl-mariner, clh, small, containerd) + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (cbl-mariner, clh, small, oci-distribution) + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, clh, normal) + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, clh, small) + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, cloud-hypervisor, normal) + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, cloud-hypervisor, small) + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, dragonball, normal) + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, dragonball, small) + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, qemu, normal) + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, qemu, small) + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, qemu-runtime-rs, small) + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (ubuntu, qemu-runtime-rs, normal) + # msft-preview: We don't run k8s tests yet. + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-zvsi / run-k8s-tests (devmapper, qemu, kubeadm) + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-zvsi / run-k8s-tests (nydus, qemu-coco-dev, kubeadm) # - Kata Containers CI / kata-containers-ci-on-push / run-kata-coco-tests / run-k8s-tests-on-tee (sev-snp, qemu-snp) - - Kata Containers CI / kata-containers-ci-on-push / run-kata-coco-tests / run-k8s-tests-coco-nontee (qemu-coco-dev, nydus, guest-pull) - - Kata Containers CI / kata-containers-ci-on-push / run-kata-coco-tests / run-k8s-tests-coco-nontee (qemu-coco-dev-runtime-rs, nydus, guest-pull) + # - Kata Containers CI / kata-containers-ci-on-push / run-kata-coco-tests / run-k8s-tests-coco-nontee (qemu-coco-dev, nydus, guest-pull) + # - Kata Containers CI / kata-containers-ci-on-push / run-kata-coco-tests / run-k8s-tests-coco-nontee (qemu-coco-dev-runtime-rs, nydus, guest-pull) - Kata Containers CI / kata-containers-ci-on-push / run-kata-deploy-tests / run-kata-deploy-tests (qemu, k0s) - Kata Containers CI / kata-containers-ci-on-push / run-kata-deploy-tests / run-kata-deploy-tests (qemu, k3s) - Kata Containers CI / kata-containers-ci-on-push / run-kata-deploy-tests / run-kata-deploy-tests (qemu, microk8s) - Kata Containers CI / kata-containers-ci-on-push / run-kata-deploy-tests / run-kata-deploy-tests (qemu, rke2) - Kata Containers CI / kata-containers-ci-on-push / run-kata-monitor-tests / run-monitor (qemu, crio) - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-nvidia-gpu / run-nvidia-gpu-tests-on-amd64 - - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-nvidia-gpu / run-nvidia-gpu-snp-tests-on-amd64 + # msft-preview: We don't run k8s tests yet. + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-nvidia-gpu / run-nvidia-gpu-tests-on-amd64 + # - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-nvidia-gpu / run-nvidia-gpu-snp-tests-on-amd64 required-labels: - ok-to-test build: # Checks that the kata-containers static tarball is created names: # ci-on-push.yaml (ci.yaml) - - Kata Containers CI / kata-containers-ci-on-push / build-and-publish-tee-confidential-unencrypted-image + # msft-preview: Can't run this test because the GHCR push isn't set up. + # - Kata Containers CI / kata-containers-ci-on-push / build-and-publish-tee-confidential-unencrypted-image - Kata Containers CI / kata-containers-ci-on-push / build-kata-static-tarball-amd64 / build-asset (cloud-hypervisor-glibc, test) - Kata Containers CI / kata-containers-ci-on-push / build-kata-static-tarball-amd64 / build-asset (cloud-hypervisor, test) - Kata Containers CI / kata-containers-ci-on-push / build-kata-static-tarball-amd64 / build-asset (firecracker, test) @@ -173,35 +185,36 @@ mapping: - Static checks / check-kernel-config-version - Static checks / static-checks (make static-checks) # static-checks-self-hosted.yaml - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make check, agent, src/agent, rust, libdevmapper, libseccomp, protobuf-compiler, clang, ub... - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make check, dragonball, src/dragonball, rust, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make check, genpolicy, src/tools/genpolicy, rust, protobuf-compiler, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make check, kata-ctl, src/tools/kata-ctl, rust, protobuf-compiler, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make check, runtime-rs, src/runtime-rs, rust, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make check, runtime, src/runtime, golang, XDG_RUNTIME_DIR, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make check, trace-forwarder, src/tools/trace-forwarder, rust, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make check, agent-ctl, src/tools/agent-ctl, rust, protobuf-compiler, clang, ubuntu-24.04-s... - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make test, agent, src/agent, rust, libdevmapper, libseccomp, protobuf-compiler, clang, ubu... - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make test, dragonball, src/dragonball, rust, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make test, genpolicy, src/tools/genpolicy, rust, protobuf-compiler, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make test, kata-ctl, src/tools/kata-ctl, rust, protobuf-compiler, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make test, runtime-rs, src/runtime-rs, rust, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make test, runtime, src/runtime, golang, XDG_RUNTIME_DIR, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make test, trace-forwarder, src/tools/trace-forwarder, rust, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make vendor, agent-ctl, src/tools/agent-ctl, rust, protobuf-compiler, clang, ubuntu-24.04-... - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make vendor, agent, src/agent, rust, libdevmapper, libseccomp, protobuf-compiler, clang, u... - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make vendor, dragonball, src/dragonball, rust, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make vendor, genpolicy, src/tools/genpolicy, rust, protobuf-compiler, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make vendor, kata-ctl, src/tools/kata-ctl, rust, protobuf-compiler, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make vendor, runtime-rs, src/runtime-rs, rust, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make vendor, runtime, src/runtime, golang, XDG_RUNTIME_DIR, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make vendor, trace-forwarder, src/tools/trace-forwarder, rust, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (sudo -E PATH="$PATH" make test, agent-ctl, src/tools/agent-ctl, rust, protobuf-compiler, c... - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (sudo -E PATH="$PATH" make test, dragonball, src/dragonball, rust, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (sudo -E PATH="$PATH" make test, genpolicy, src/tools/genpolicy, rust, protobuf-compiler, u... - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (sudo -E PATH="$PATH" make test, kata-ctl, src/tools/kata-ctl, rust, protobuf-compiler, ubu... - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (sudo -E PATH="$PATH" make test, runtime-rs, src/runtime-rs, rust, ubuntu-24.04-s390x) - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (sudo -E PATH="$PATH" make test, runtime, src/runtime, golang, XDG_RUNTIME_DIR, ubuntu-24.0... - - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (sudo -E PATH="$PATH" make test, trace-forwarder, src/tools/trace-forwarder, rust, ubuntu-2... + # msft-preview: We don't have s390x runners. + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make check, agent, src/agent, rust, libdevmapper, libseccomp, protobuf-compiler, clang, ub... + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make check, dragonball, src/dragonball, rust, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make check, genpolicy, src/tools/genpolicy, rust, protobuf-compiler, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make check, kata-ctl, src/tools/kata-ctl, rust, protobuf-compiler, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make check, runtime-rs, src/runtime-rs, rust, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make check, runtime, src/runtime, golang, XDG_RUNTIME_DIR, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make check, trace-forwarder, src/tools/trace-forwarder, rust, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make check, agent-ctl, src/tools/agent-ctl, rust, protobuf-compiler, clang, ubuntu-24.04-s... + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make test, agent, src/agent, rust, libdevmapper, libseccomp, protobuf-compiler, clang, ubu... + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make test, dragonball, src/dragonball, rust, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make test, genpolicy, src/tools/genpolicy, rust, protobuf-compiler, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make test, kata-ctl, src/tools/kata-ctl, rust, protobuf-compiler, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make test, runtime-rs, src/runtime-rs, rust, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make test, runtime, src/runtime, golang, XDG_RUNTIME_DIR, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make test, trace-forwarder, src/tools/trace-forwarder, rust, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make vendor, agent-ctl, src/tools/agent-ctl, rust, protobuf-compiler, clang, ubuntu-24.04-... + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make vendor, agent, src/agent, rust, libdevmapper, libseccomp, protobuf-compiler, clang, u... + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make vendor, dragonball, src/dragonball, rust, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make vendor, genpolicy, src/tools/genpolicy, rust, protobuf-compiler, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make vendor, kata-ctl, src/tools/kata-ctl, rust, protobuf-compiler, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make vendor, runtime-rs, src/runtime-rs, rust, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make vendor, runtime, src/runtime, golang, XDG_RUNTIME_DIR, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (make vendor, trace-forwarder, src/tools/trace-forwarder, rust, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (sudo -E PATH="$PATH" make test, agent-ctl, src/tools/agent-ctl, rust, protobuf-compiler, c... + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (sudo -E PATH="$PATH" make test, dragonball, src/dragonball, rust, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (sudo -E PATH="$PATH" make test, genpolicy, src/tools/genpolicy, rust, protobuf-compiler, u... + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (sudo -E PATH="$PATH" make test, kata-ctl, src/tools/kata-ctl, rust, protobuf-compiler, ubu... + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (sudo -E PATH="$PATH" make test, runtime-rs, src/runtime-rs, rust, ubuntu-24.04-s390x) + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (sudo -E PATH="$PATH" make test, runtime, src/runtime, golang, XDG_RUNTIME_DIR, ubuntu-24.0... + # - Static checks self-hosted / build-checks (ubuntu-24.04-s390x) / check (sudo -E PATH="$PATH" make test, trace-forwarder, src/tools/trace-forwarder, rust, ubuntu-2... required-labels: - ok-to-test diff --git a/tools/testing/gatekeeper/skips.py b/tools/testing/gatekeeper/skips.py index d639f9be8a20..0d1058d1b2e4 100644 --- a/tools/testing/gatekeeper/skips.py +++ b/tools/testing/gatekeeper/skips.py @@ -106,4 +106,4 @@ def get_features(self, target_branch): _TESTS = sys.argv[1] == '-t' else: _TESTS = False - sys.exit(Checks().run(_TESTS, os.getenv("TARGET_BRANCH", "main"))) + sys.exit(Checks().run(_TESTS, os.getenv("TARGET_BRANCH", "msft-preview"))) diff --git a/tools/testing/kata-webhook/deploy/webhook.yaml b/tools/testing/kata-webhook/deploy/webhook.yaml index efd7d8225a8c..cb4283e787fd 100644 --- a/tools/testing/kata-webhook/deploy/webhook.yaml +++ b/tools/testing/kata-webhook/deploy/webhook.yaml @@ -20,7 +20,7 @@ spec: spec: containers: - name: pod-annotate-webhook - image: quay.io/kata-containers/kata-webhook-example:latest + image: marineraks.azurecr.io/kata-containers/kata-webhook:min_memory_limit imagePullPolicy: Always env: - name: RUNTIME_CLASS @@ -29,6 +29,12 @@ spec: name: kata-webhook key: runtime_class optional: true + - name: MIN_MEMORY_LIMIT + valueFrom: + configMapKeyRef: + name: kata-webhook + key: min_memory_limit + optional: true args: - -tls-cert-file=/etc/webhook/certs/cert.pem - -tls-key-file=/etc/webhook/certs/key.pem @@ -74,3 +80,4 @@ metadata: name: kata-webhook data: runtime_class: kata + min_memory_limit: "128Mi" diff --git a/tools/testing/kata-webhook/main.go b/tools/testing/kata-webhook/main.go index 6f3f8bd1c56d..63d0e477fea4 100644 --- a/tools/testing/kata-webhook/main.go +++ b/tools/testing/kata-webhook/main.go @@ -14,6 +14,7 @@ import ( "strings" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/sirupsen/logrus" @@ -75,6 +76,23 @@ func annotatePodMutator(_ context.Context, ar *kwhmodel.AdmissionReview, obj met kataRuntimeClassName := getRuntimeClass(runtimeClassEnvKey, "kata") pod.Spec.RuntimeClassName = &kataRuntimeClassName + minMemoryLimit, foundMinMemoryLimit := os.LookupEnv("MIN_MEMORY_LIMIT") + + if foundMinMemoryLimit { + minMemoryLimitVal := resource.MustParse(minMemoryLimit) + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Resources.Limits == nil { + continue + } else { + currentMemoryLimit := pod.Spec.Containers[i].Resources.Limits.Memory().Value() + if currentMemoryLimit < minMemoryLimitVal.Value() { + pod.Spec.Containers[i].Resources.Limits["memory"] = resource.MustParse(minMemoryLimit) + fmt.Println("memory limit too low. Updating to : ", pod.Spec.Containers[i].Resources.Limits) + } + } + } + } + return &kwhmutating.MutatorResult{ MutatedObject: pod, }, nil