// ListAndWatch lists devices and update that list according to the health status func(plugin *NvidiaDevicePlugin) ListAndWatch(e *pluginapi.Empty, s pluginapi.DevicePlugin_ListAndWatchServer) error { if err := s.Send(&pluginapi.ListAndWatchResponse{Devices: plugin.apiDevices()}); err != nil { return err } for { select { case <-plugin.stop: returnnil case d := <-plugin.health: d.Health = pluginapi.Unhealthy klog.Infof("'%s' device marked unhealthy: %s", plugin.rm.Resource(), d.ID) if err := s.Send(&pluginapi.ListAndWatchResponse{Devices: plugin.apiDevices()}); err != nil { returnnil } } } }
// Allocate which return list of devices. func(plugin *NvidiaDevicePlugin) Allocate(ctx context.Context, reqs *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) { responses := pluginapi.AllocateResponse{} for _, req := range reqs.ContainerRequests { if err := plugin.rm.ValidateRequest(req.DevicesIDs); err != nil { returnnil, fmt.Errorf("invalid allocation request for %q: %w", plugin.rm.Resource(), err) } response, err := plugin.getAllocateResponse(req.DevicesIDs) if err != nil { returnnil, fmt.Errorf("failed to get allocate response: %v", err) } responses.ContainerResponses = append(responses.ContainerResponses, response) } return &responses, nil }
Allocate 函数接收 AllocateRequest ,其内容为一个 DeviceID 的列表。例如,当我们申请一个 nvidia.com/gpu: 2 的 Pod 时,Kubelet 发送的 AllocateRequest 中的 DeviceID 列表就应包含两个 Device ID
Allocate 返回的 AllocateResponse 基本结构如下:
1 2 3 4 5 6 7 8 9 10 11 12 13 14
type ContainerAllocateResponse struct { // List of environment variable to be set in the container to access one of more devices. Envs map[string]string`protobuf:"bytes,1,rep,name=envs,proto3" json:"envs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // Mounts for the container. Mounts []*Mount `protobuf:"bytes,2,rep,name=mounts,proto3" json:"mounts,omitempty"` // Devices for the container. Devices []*DeviceSpec `protobuf:"bytes,3,rep,name=devices,proto3" json:"devices,omitempty"` // Container annotations to pass to the container runtime Annotations map[string]string`protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // CDI devices for the container. CDIDevices []*CDIDevice `protobuf:"bytes,5,rep,name=cdi_devices,json=cdiDevices,proto3" json:"cdi_devices,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_sizecache int32`json:"-"` }
Usage: nvidia-container-cli configure [OPTION...] ROOTFS Configure a container with GPU support by exposing device drivers to it.
This command enters the namespace of the container process referred by PID (or the current parent process if none specified) and performs the necessary steps to ensure that the given capabilities are available inside the container. It is assumed that the container has been created but not yet started, and the host filesystem is accessible (i.e. chroot/pivot_root hasn't been called).
Options: -?, --help Give this help list -c, --compute Enable compute capability --compat32 Enable 32bits compatibility -d, --device=ID Device UUID(s) or index(es) to isolate -D, --display Enable display capability -g, --graphics Enable graphics capability -l, --ldconfig=PATH Path to the ldconfig binary --mig-config=ID Enable configuration of MIG devices --mig-monitor=ID Enable monitoring of MIG devices -n, --ngx Enable ngx capability --no-cgroups Don't use cgroup enforcement --no-devbind Don't bind mount devices -p, --pid=PID Container PID -r, --require=EXPR Check container requirements -u, --utility Enable utility capability --usage Give a short usage message -v, --video Enable video capability -V, --version Print program version
Kubelet Device Manager 核心机制
Device Manager 通过 Socket 与 Device Plugin 对接,负责发现、上报及分配设备资源,确保 Pod 能够正常使用设备资源
if utilfeature.DefaultFeatureGate.Enabled(features.ResourceHealthStatus) { // compare with old device's health and send update to the channel if needed updatePodUIDFn := func(deviceID string) { podUID, _ := m.podDevices.getPodAndContainerForDevice(deviceID) if podUID != "" { podsToUpdate.Insert(podUID) } } if oldDev, ok := oldDevices[dev.ID]; ok { if oldDev.Health != dev.Health { updatePodUIDFn(dev.ID) } } else { updatePodUIDFn(dev.ID) } }
m.allDevices[resourceName][dev.ID] = dev if dev.Health == pluginapi.Healthy { m.healthyDevices[resourceName].Insert(dev.ID) healthyCount++ } else { m.unhealthyDevices[resourceName].Insert(dev.ID) } } m.mutex.Unlock()
if utilfeature.DefaultFeatureGate.Enabled(features.ResourceHealthStatus) { iflen(podsToUpdate) > 0 { select { case m.update <- resourceupdates.Update{PodUIDs: podsToUpdate.UnsortedList()}: default: klog.ErrorS(goerrors.New("device update channel is full"), "discard pods info", "podsToUpdate", podsToUpdate.UnsortedList()) } } }
func(m *ManagerImpl) GetCapacity() (v1.ResourceList, v1.ResourceList, []string) { needsUpdateCheckpoint := false var capacity = v1.ResourceList{} var allocatable = v1.ResourceList{} deletedResources := sets.New[string]() m.mutex.Lock() for resourceName, devices := range m.healthyDevices { eI, ok := m.endpoints[resourceName] if (ok && eI.e.stopGracePeriodExpired()) || !ok { // The resources contained in endpoints and (un)healthyDevices // should always be consistent. Otherwise, we run with the risk // of failing to garbage collect non-existing resources or devices. if !ok { klog.InfoS("Unexpected: healthyDevices and endpoints are out of sync") } delete(m.endpoints, resourceName) delete(m.healthyDevices, resourceName) deletedResources.Insert(resourceName) needsUpdateCheckpoint = true } else { capacity[v1.ResourceName(resourceName)] = *resource.NewQuantity(int64(devices.Len()), resource.DecimalSI) allocatable[v1.ResourceName(resourceName)] = *resource.NewQuantity(int64(devices.Len()), resource.DecimalSI) } } for resourceName, devices := range m.unhealthyDevices { eI, ok := m.endpoints[resourceName] if (ok && eI.e.stopGracePeriodExpired()) || !ok { if !ok { klog.InfoS("Unexpected: unhealthyDevices and endpoints became out of sync") } delete(m.endpoints, resourceName) delete(m.unhealthyDevices, resourceName) deletedResources.Insert(resourceName) needsUpdateCheckpoint = true } else { capacityCount := capacity[v1.ResourceName(resourceName)] unhealthyCount := *resource.NewQuantity(int64(devices.Len()), resource.DecimalSI) capacityCount.Add(unhealthyCount) capacity[v1.ResourceName(resourceName)] = capacityCount } } m.mutex.Unlock() if needsUpdateCheckpoint { if err := m.writeCheckpoint(); err != nil { klog.ErrorS(err, "Failed to write checkpoint file") } } return capacity, allocatable, deletedResources.UnsortedList() }
缺乏有效的调度反馈机制:设备插件和 kube-scheduler 之间缺乏有效的反馈回路。这意味着,即使 kube-scheduler 为一个待调度的 Pod 找到了看似合适的节点,设备插件也无法根据节点的就绪状态等关键因素进行进一步的“校验”,这可能导致 Pod 被错误地调度到尚未完全准备好接收新资源的“未就绪”节点上
Kubernetes 只允许通过 Kubelet 来分配设备,这样就导致无法获得全局最优的资源编排,也就是从根本上无法发挥最大的资源效能。比如一个集群内有两台节点,都有相同的设备,剩余的可分配设备数量相等,但是实际上两个节点上的设备所在的硬件拓扑会导致 Pod 的运行时效果差异较大,没有调度器介入的情况下,很有可能无法抵消这种差异
当前 Kubelet 所能处理的最大 NUMA 节点个数是 8。若 NUMA 节点数超过 8, 枚举可能的 NUMA 亲和性并为之生成提示时会发生状态爆炸