bindMountContainerRootfs(
func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, err error) { span, _ := k.trace("createContainer") defer span.Finish() var ctrStorages []*grpc.Storage var ctrDevices []*grpc.Device var rootfs *grpc.Storage // This is the guest absolute root path for that container. rootPathParent := filepath.Join(kataGuestSharedDir(), c.id) rootPath := filepath.Join(rootPathParent, c.rootfsSuffix) // In case the container creation fails, the following defer statement // takes care of rolling back actions previously performed. defer func() { if err != nil { k.Logger().WithError(err).Error("createContainer failed") k.rollbackFailingContainerCreation(c) } }() if rootfs, err = k.buildContainerRootfs(sandbox, c, rootPathParent); err != nil { return nil, err } else if rootfs != nil { // Add rootfs to the list of container storage. // We only need to do this for block based rootfs, as we // want the agent to mount it into the right location // (kataGuestSharedDir/ctrID/ ctrStorages = append(ctrStorages, rootfs) } ociSpec := c.GetPatchedOCISpec() if ociSpec == nil { return nil, errorMissingOCISpec } // Handle container mounts newMounts, ignoredMounts, err := c.mountSharedDirMounts(getMountPath(sandbox.id), kataGuestSharedDir()) if err != nil { return nil, err } k.handleShm(ociSpec.Mounts, sandbox) epheStorages := k.handleEphemeralStorage(ociSpec.Mounts) ctrStorages = append(ctrStorages, epheStorages...) localStorages := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix) ctrStorages = append(ctrStorages, localStorages...)
ls /var/lib/docker/volumes/data1/ _data root@p:/home/pcl# ls /var/lib/docker/volumes/data1/_data/
find /var/lib/docker/overlay2/0302ff6f07ca65ebbf82884a33e0dcec795e0de1d5af7006d5473402028099eb/ -name data1 /var/lib/docker/overlay2/0302ff6f07ca65ebbf82884a33e0dcec795e0de1d5af7006d5473402028099eb/merged/data1 /var/lib/docker/overlay2/0302ff6f07ca65ebbf82884a33e0dcec795e0de1d5af7006d5473402028099eb/diff/data
func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPathParent string) (*grpc.Storage, error) { if c.state.Fstype != "" && c.state.BlockDeviceID != "" { // The rootfs storage volume represents the container rootfs // mount point inside the guest. // It can be a block based device (when using block based container // overlay on the host) mount or a 9pfs one (for all other overlay // implementations). rootfs := &grpc.Storage{} // This is a block based device rootfs. device := sandbox.devManager.GetDeviceByID(c.state.BlockDeviceID) if device == nil { k.Logger().WithField("device", c.state.BlockDeviceID).Error("failed to find device by id") return nil, fmt.Errorf("failed to find device by id %q", c.state.BlockDeviceID) } blockDrive, ok := device.GetDeviceInfo().(*config.BlockDrive) if !ok || blockDrive == nil { k.Logger().Error("malformed block drive") return nil, fmt.Errorf("malformed block drive") } switch { case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio: rootfs.Driver = kataMmioBlkDevType rootfs.Source = blockDrive.VirtPath case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlockCCW: rootfs.Driver = kataBlkCCWDevType rootfs.Source = blockDrive.DevNo case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock: rootfs.Driver = kataBlkDevType if blockDrive.PCIAddr == "" { rootfs.Source = blockDrive.VirtPath } else { rootfs.Source = blockDrive.PCIAddr } case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioSCSI: rootfs.Driver = kataSCSIDevType rootfs.Source = blockDrive.SCSIAddr default: return nil, fmt.Errorf("Unknown block device driver: %s", sandbox.config.HypervisorConfig.BlockDeviceDriver) } rootfs.MountPoint = rootPathParent rootfs.Fstype = c.state.Fstype if c.state.Fstype == "xfs" { rootfs.Options = []string{"nouuid"} } // Ensure container mount destination exists // TODO: remove dependency on shared fs path. shared fs is just one kind of storage sources. // we should not always use shared fs path for all kinds of storage. Stead, all storage // should be bind mounted to a tmpfs path for containers to use. if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil { return nil, err } return rootfs, nil } // This is not a block based device rootfs. // We are going to bind mount it into the 9pfs // shared drive between the host and the guest. // With 9pfs we don't need to ask the agent to // mount the rootfs as the shared directory // (kataGuestSharedDir) is already mounted in the // guest. We only need to mount the rootfs from // the host and it will show up in the guest. if err := bindMountContainerRootfs(k.ctx, getMountPath(sandbox.id), c.id, c.rootFs.Target, false); err != nil { return nil, err } return nil, nil }
// bindMountContainerRootfs bind mounts a container rootfs into a 9pfs shared // directory between the guest and the host. func bindMountContainerRootfs(ctx context.Context, shareDir, cid, cRootFs string, readonly bool) error { span, _ := trace(ctx, "bindMountContainerRootfs") defer span.Finish() rootfsDest := filepath.Join(shareDir, cid, rootfsDir) return bindMount(ctx, cRootFs, rootfsDest, readonly, "private") }
// bindMount bind mounts a source in to a destination. This will // do some bookkeeping: // * evaluate all symlinks // * ensure the source exists // * recursively create the destination // pgtypes stands for propagation types, which are shared, private, slave, and ubind. func bindMount(ctx context.Context, source, destination string, readonly bool, pgtypes string) error { span, _ := trace(ctx, "bindMount") defer span.Finish() if source == "" { return fmt.Errorf("source must be specified") } if destination == "" { return fmt.Errorf("destination must be specified") } absSource, err := filepath.EvalSymlinks(source) if err != nil { return fmt.Errorf("Could not resolve symlink for source %v", source) } if err := ensureDestinationExists(absSource, destination); err != nil { return fmt.Errorf("Could not create destination mount point %v: %v", destination, err) } if err := syscall.Mount(absSource, destination, "bind", syscall.MS_BIND, ""); err != nil { return fmt.Errorf("Could not bind mount %v to %v: %v", absSource, destination, err) } if pgtype, exist := propagationTypes[pgtypes]; exist { if err := syscall.Mount("none", destination, "", pgtype, ""); err != nil { return fmt.Errorf("Could not make mount point %v %s: %v", destination, pgtypes, err) } } else { return fmt.Errorf("Wrong propagation type %s", pgtypes) } // For readonly bind mounts, we need to remount with the readonly flag. // This is needed as only very recent versions of libmount/util-linux support "bind,ro" if readonly { return syscall.Mount(absSource, destination, "bind", uintptr(syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY), "") } return nil }
func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, rootfsSuffix string) []*grpc.Storage { var localStorages []*grpc.Storage for idx, mnt := range mounts { if mnt.Type == KataLocalDevType { // Set the mount source path to a the desired directory point in the VM. // In this case it is located in the sandbox directory. // We rely on the fact that the first container in the VM has the same ID as the sandbox ID. // In Kubernetes, this is usually the pause container and we depend on it existing for // local directories to work. mounts[idx].Source = filepath.Join(kataGuestSharedDir(), sandboxID, rootfsSuffix, KataLocalDevType, filepath.Base(mnt.Source)) // Create a storage struct so that the kata agent is able to create the // directory inside the VM. localStorage := &grpc.Storage{ Driver: KataLocalDevType, Source: KataLocalDevType, Fstype: KataLocalDevType, MountPoint: mounts[idx].Source, Options: localDirOptions, } localStorages = append(localStorages, localStorage) } } return localStorages }
docker run -it --runtime=kata-runtime -v data1:/data1 --rm debian /bin/bash
root@0e13f5a22fca:/#
"Mounts": [ { "Type": "volume", "Name": "data1", "Source": "/var/lib/docker/volumes/data1/_data", "Destination": "/data1", "Driver": "local", "Mode": "z", "RW": true, "Propagation": "" } ],
docker inspect 0e13f5a22fca [ { "Id": "0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e", "Created": "2020-12-04T00:57:07.930022529Z", "Path": "/bin/bash", "Args": [], "State": { "Status": "running", "Running": true, "Paused": false, "Restarting": false, "OOMKilled": false, "Dead": false, "Pid": 3105787, "ExitCode": 0, "Error": "", "StartedAt": "2020-12-04T00:57:10.936012588Z", "FinishedAt": "0001-01-01T00:00:00Z" }, "Image": "sha256:2aa621c7db398cbad23b51b63348fedf0a779542d52f71acd4f24072cb872170", "ResolvConfPath": "/var/lib/docker/containers/0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e/resolv.conf", "HostnamePath": "/var/lib/docker/containers/0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e/hostname", "HostsPath": "/var/lib/docker/containers/0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e/hosts", "LogPath": "/var/lib/docker/containers/0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e/0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e-json.log", "Name": "/distracted_haibt", "RestartCount": 0, "Driver": "overlay2", "Platform": "linux", "MountLabel": "", "ProcessLabel": "", "AppArmorProfile": "docker-default", "ExecIDs": null, "HostConfig": { "Binds": [ "data1:/data1" ], "ContainerIDFile": "", "LogConfig": { "Type": "json-file", "Config": { "max-size": "100m" } }, "NetworkMode": "default", "PortBindings": {}, "RestartPolicy": { "Name": "no", "MaximumRetryCount": 0 }, "AutoRemove": true, "VolumeDriver": "", "VolumesFrom": null, "CapAdd": null, "CapDrop": null, "Dns": [], "DnsOptions": [], "DnsSearch": [], "ExtraHosts": null, "GroupAdd": null, "IpcMode": "shareable", "Cgroup": "", "Links": null, "OomScoreAdj": 0, "PidMode": "", "Privileged": false, "PublishAllPorts": false, "ReadonlyRootfs": false, "SecurityOpt": null, "UTSMode": "", "UsernsMode": "", "ShmSize": 67108864, "Runtime": "kata-runtime", "ConsoleSize": [ 0, 0 ], "Isolation": "", "CpuShares": 0, "Memory": 0, "NanoCpus": 0, "CgroupParent": "", "BlkioWeight": 0, "BlkioWeightDevice": [], "BlkioDeviceReadBps": null, "BlkioDeviceWriteBps": null, "BlkioDeviceReadIOps": null, "BlkioDeviceWriteIOps": null, "CpuPeriod": 0, "CpuQuota": 0, "CpuRealtimePeriod": 0, "CpuRealtimeRuntime": 0, "CpusetCpus": "", "CpusetMems": "", "Devices": [], "DeviceCgroupRules": null, "DiskQuota": 0, "KernelMemory": 0, "MemoryReservation": 0, "MemorySwap": 0, "MemorySwappiness": null, "OomKillDisable": false, "PidsLimit": 0, "Ulimits": null, "CpuCount": 0, "CpuPercent": 0, "IOMaximumIOps": 0, "IOMaximumBandwidth": 0, "MaskedPaths": [ "/proc/asound", "/proc/acpi", "/proc/kcore", "/proc/keys", "/proc/latency_stats", "/proc/timer_list", "/proc/timer_stats", "/proc/sched_debug", "/proc/scsi", "/sys/firmware" ], "ReadonlyPaths": [ "/proc/bus", "/proc/fs", "/proc/irq", "/proc/sys", "/proc/sysrq-trigger" ] }, "GraphDriver": { "Data": { "LowerDir": "/var/lib/docker/overlay2/e17f8f08ca2d3e8dd4dc29d9167fca9856beeb25cac37104bd43573d628c849d-init/diff:/var/lib/docker/overlay2/53cfc5f6a35592815e1df5bcdb226c096e7bdbd59fb0c58253d82c055c2b8435/diff", "MergedDir": "/var/lib/docker/overlay2/e17f8f08ca2d3e8dd4dc29d9167fca9856beeb25cac37104bd43573d628c849d/merged", "UpperDir": "/var/lib/docker/overlay2/e17f8f08ca2d3e8dd4dc29d9167fca9856beeb25cac37104bd43573d628c849d/diff", "WorkDir": "/var/lib/docker/overlay2/e17f8f08ca2d3e8dd4dc29d9167fca9856beeb25cac37104bd43573d628c849d/work" }, "Name": "overlay2" }, "Mounts": [ { "Type": "volume", "Name": "data1", "Source": "/var/lib/docker/volumes/data1/_data", "Destination": "/data1", "Driver": "local", "Mode": "z", "RW": true, "Propagation": "" } ], "Config": { "Hostname": "0e13f5a22fca", "Domainname": "", "User": "", "AttachStdin": true, "AttachStdout": true, "AttachStderr": true, "Tty": true, "OpenStdin": true, "StdinOnce": true, "Env": [ "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ], "Cmd": [ "/bin/bash" ], "Image": "debian", "Volumes": null, "WorkingDir": "", "Entrypoint": null, "OnBuild": null, "Labels": {} }, "NetworkSettings": { "Bridge": "", "SandboxID": "b7dab902309c0390682b0f572832ecb996f8cb1fe3fb942ebe4a1a2cd8301fe6", "HairpinMode": false, "LinkLocalIPv6Address": "", "LinkLocalIPv6PrefixLen": 0, "Ports": {}, "SandboxKey": "/var/run/docker/netns/b7dab902309c", "SecondaryIPAddresses": null, "SecondaryIPv6Addresses": null, "EndpointID": "e15b8cdb018fac7f63bf18cbafd3904cfbdf7c5102d3542ad5d0f6c7f51cbe27", "Gateway": "172.17.0.1", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "IPAddress": "172.17.0.3", "IPPrefixLen": 16, "IPv6Gateway": "", "MacAddress": "02:42:ac:11:00:03", "Networks": { "bridge": { "IPAMConfig": null, "Links": null, "Aliases": null, "NetworkID": "46f664be20d1ea20abc7661d495d3f7f5283588f940faa72b9c0ab5d077b076a", "EndpointID": "e15b8cdb018fac7f63bf18cbafd3904cfbdf7c5102d3542ad5d0f6c7f51cbe27", "Gateway": "172.17.0.1", "IPAddress": "172.17.0.3", "IPPrefixLen": 16, "IPv6Gateway": "", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "MacAddress": "02:42:ac:11:00:03", "DriverOpts": null } } } } ]
docker run -it --runtime=kata-runtime -v data1:/data1 --rm debian /bin/bash root@0e13f5a22fca:/# ls data1/ root@0e13f5a22fca:/#
time="2020-12-04T01:01:45.94077978Z" level=debug msg="new request" debug_console=true name=kata-agent pid=56 req="container_id:\"0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e\" exec_id:\"0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e\" len:32768 " request=/grpc.AgentService/ReadStdout sandbox=0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e source=agent ls: cannot access 'data': No such file or directory root@ubuntu:/# ls data ls: cannot access 'data': No such file or directory root@ubuntu:/#
kata虚拟机看不到
root@ubuntu:/# ls data ls: cannot access 'data': No such file or directory root@ubuntu:/# ls bin dev home lost+found mnt proc run srv tmp var boot etc lib media opt root sbin sys usr vmi.sh root@ubuntu:/#
浙公网安备 33010602011771号