cmd\kube-controller-manager\app\apps.go
startDeploymentController
func startDeploymentController(ctx context.Context, controllerContext ControllerContext) (controller.Interface, bool, error) {
dc, err := deployment.NewDeploymentController(
controllerContext.InformerFactory.Apps().V1().Deployments(),
controllerContext.InformerFactory.Apps().V1().ReplicaSets(), // deploy底层本质是rs控制器
controllerContext.InformerFactory.Core().V1().Pods(),
controllerContext.ClientBuilder.ClientOrDie("deployment-controller"),
)
if err != nil {
return nil, true, fmt.Errorf("error creating Deployment controller: %v", err)
}
go dc.Run(ctx, int(controllerContext.ComponentConfig.DeploymentController.ConcurrentDeploymentSyncs))
return nil, true, nil
}
pkg\controller\deployment\deployment_controller.go
NewDeploymentController
func NewDeploymentController(dInformer appsinformers.DeploymentInformer, rsInformer appsinformers.ReplicaSetInformer, podInformer coreinformers.PodInformer, client clientset.Interface) (*DeploymentController, error) {
...
dInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: dc.addDeployment,
UpdateFunc: dc.updateDeployment,
DeleteFunc: dc.deleteDeployment,
})
rsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: dc.addReplicaSet,
UpdateFunc: dc.updateReplicaSet,
DeleteFunc: dc.deleteReplicaSet,
})
podInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
DeleteFunc: dc.deletePod,
})
dc.syncHandler = dc.syncDeployment
dc.enqueueDeployment = dc.enqueue
return dc, nil
}
addDeployment
func (dc *DeploymentController) addDeployment(obj interface{}) {
d := obj.(*apps.Deployment)
dc.enqueueDeployment(d)
}
updateDeployment
func (dc *DeploymentController) updateDeployment(old, cur interface{}) {
oldD := old.(*apps.Deployment)
curD := cur.(*apps.Deployment)
dc.enqueueDeployment(curD)
}
deleteDeployment
func (dc *DeploymentController) deleteDeployment(obj interface{}) {
d, ok := obj.(*apps.Deployment)
if !ok {
tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
d, ok = tombstone.Obj.(*apps.Deployment)
}
dc.enqueueDeployment(d)
}
addReplicaSet
func (dc *DeploymentController) addReplicaSet(obj interface{}) {
rs := obj.(*apps.ReplicaSet)
if controllerRef := metav1.GetControllerOf(rs); controllerRef != nil {
// 解析rs的deploy
d := dc.resolveControllerRef(rs.Namespace, controllerRef)
dc.enqueueDeployment(d)
return
}
// 匹配不到则获取所有deploy进行适配比对
ds := dc.getDeploymentsForReplicaSet(rs)
if len(ds) == 0 {
return
}
for _, d := range ds {
dc.enqueueDeployment(d)
}
}
updateReplicaSet
func (dc *DeploymentController) updateReplicaSet(old, cur interface{}) {
curRS := cur.(*apps.ReplicaSet)
oldRS := old.(*apps.ReplicaSet)
if curRS.ResourceVersion == oldRS.ResourceVersion {
// ResourceVersion一致则是相同对象不处理
return
}
curControllerRef := metav1.GetControllerOf(curRS)
oldControllerRef := metav1.GetControllerOf(oldRS)
controllerRefChanged := !reflect.DeepEqual(curControllerRef, oldControllerRef)
if controllerRefChanged && oldControllerRef != nil {
// The ControllerRef was changed. Sync the old controller, if any.
// 新老rs上层控制deploy不一致则更新更新老版本rs
if d := dc.resolveControllerRef(oldRS.Namespace, oldControllerRef); d != nil {
dc.enqueueDeployment(d)
}
}
// 找的到上层deploy，同addReplicaSet
if curControllerRef != nil {
d := dc.resolveControllerRef(curRS.Namespace, curControllerRef)
dc.enqueueDeployment(d)
return
}
// 找不到上层deploy，同addReplicaSet
labelChanged := !reflect.DeepEqual(curRS.Labels, oldRS.Labels)
if labelChanged || controllerRefChanged {
ds := dc.getDeploymentsForReplicaSet(curRS)
for _, d := range ds {
dc.enqueueDeployment(d)
}
}
}
deleteReplicaSet
func (dc *DeploymentController) deleteReplicaSet(obj interface{}) {
rs, ok := obj.(*apps.ReplicaSet)
if !ok {
tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
rs, ok = tombstone.Obj.(*apps.ReplicaSet)
}
controllerRef := metav1.GetControllerOf(rs)
if controllerRef == nil {
// 没有上层deploy说明已经被删了 return
}
d := dc.resolveControllerRef(rs.Namespace, controllerRef)
dc.enqueueDeployment(d)
}
deletePod
func (dc *DeploymentController) deletePod(obj interface{}) {
pod, ok := obj.(*v1.Pod)
if !ok {
tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
pod, ok = tombstone.Obj.(*v1.Pod)
}
if d := dc.getDeploymentForPod(pod); d != nil && d.Spec.Strategy.Type == apps.RecreateDeploymentStrategyType {
rsList, err := util.ListReplicaSets(d, util.RsListFromClient(dc.client.AppsV1()))
podMap, err := dc.getPodMapForDeployment(d, rsList)
numPods := 0
for _, podList := range podMap {
numPods += len(podList)
}
if numPods == 0 {
dc.enqueueDeployment(d)
}
}
}
resolveControllerRef
func (dc *DeploymentController) resolveControllerRef(namespace string, controllerRef *metav1.OwnerReference) *apps.Deployment {
if controllerRef.Kind != controllerKind.Kind {
// controllerKind.Kind is "Deployment"
return nil
}
d, err := dc.dLister.Deployments(namespace).Get(controllerRef.Name)
if err != nil {
return nil
}
if d.UID != controllerRef.UID {
// uid变化说明deploy已经变更了，不再是该rs的上层
return nil
}
return d
}
syncDeployment
func (dc *DeploymentController) syncDeployment(ctx context.Context, key string) error {
// 获取deployment信息并深copy
namespace, name, err := cache.SplitMetaNamespaceKey(key)
deployment, err := dc.dLister.Deployments(namespace).Get(name)
d := deployment.DeepCopy()
// 当d没有设置LabelSelector时代表他选择这个namespace下所有pod，会有一条警告事件
everything := metav1.LabelSelector{}
if reflect.DeepEqual(d.Spec.Selector, &everything) {
dc.eventRecorder.Eventf(d, v1.EventTypeWarning, "SelectingAll", "This deployment is selecting all pods. A non-empty selector is required.")
if d.Status.ObservedGeneration < d.Generation {
// 更新deplyment的观测版本号到最新
d.Status.ObservedGeneration = d.Generation
dc.client.AppsV1().Deployments(d.Namespace).UpdateStatus(ctx, d, metav1.UpdateOptions{})
}
return nil
}
// 获取这个deployment对应的replicaSet集合
rsList, err := dc.getReplicaSetsForDeployment(ctx, d)
// 获取所有replicaSet集合下的pod集合，这里使用了字典进行保存
// 检测pod是否以pod-template-hash生成正确。在不同版本中，这个hash算法可能有所不同，会导致升级后deploy检测不到以前的pod会导致服务不可用；然后按新hash算法生成全新的pod，而之前的pod没有被清理会导致浪费2份资源
// 检测在重新部署时没有老pod
podMap, err := dc.getPodMapForDeployment(d, rsList)
// 如果已经被删除了，则只更新状态信息
if d.DeletionTimestamp != nil {
return dc.syncStatusOnly(ctx, d, rsList)
}
// 验证一些其他情况
if err = dc.checkPausedConditions(ctx, d); err != nil {
return err
}
if d.Spec.Paused {
return dc.sync(ctx, d, rsList)
}
// 回滚情况
if getRollbackTo(d) != nil { --->pkg\controller\deployment\rollback.go
return dc.rollback(ctx, d, rsList) --->pkg\controller\deployment\rollback.go
}
// 滚动升级情况
scalingEvent, err := dc.isScalingEvent(ctx, d, rsList)
if err != nil {
return err
}
if scalingEvent {
return dc.sync(ctx, d, rsList)
}
switch d.Spec.Strategy.Type {
case apps.RecreateDeploymentStrategyType:
// 重建情况
return dc.rolloutRecreate(ctx, d, rsList, podMap)
case apps.RollingUpdateDeploymentStrategyType:
// 更新情况
return dc.rolloutRolling(ctx, d, rsList)
}
return fmt.Errorf("unexpected deployment strategy type: %s", d.Spec.Strategy.Type)
}
getReplicaSetsForDeployment
func (dc *DeploymentController) getReplicaSetsForDeployment(ctx context.Context, d *apps.Deployment) ([]*apps.ReplicaSet, error) {
// 获取该namespace下所有replicatset列表
rsList, err := dc.rsLister.ReplicaSets(d.Namespace).List(labels.Everything())
// 获取这个deplotment要选择匹配的label
deploymentSelector, err := metav1.LabelSelectorAsSelector(d.Spec.Selector)
...
// 基于ReplicaSetController进行比对计算
cm := controller.NewReplicaSetControllerRefManager(dc.rsControl, d, deploymentSelector, controllerKind, canAdoptFunc)
return cm.ClaimReplicaSets(ctx, rsList)
}
getPodMapForDeployment
func (dc *DeploymentController) getPodMapForDeployment(d *apps.Deployment, rsList []*apps.ReplicaSet) (map[types.UID][]*v1.Pod, error) {
// 根据label找到目标pod
selector, err := metav1.LabelSelectorAsSelector(d.Spec.Selector)
pods, err := dc.podLister.Pods(d.Namespace).List(selector)
// 初始化号一个容量为rsList数量的map
podMap := make(map[types.UID][]*v1.Pod, len(rsList))
for _, rs := range rsList {
podMap[rs.UID] = []*v1.Pod{}
}
// 循环目标pod放置到自己的rs里
for _, pod := range pods {
controllerRef := metav1.GetControllerOf(pod)
if controllerRef == nil {
continue
}
if _, ok := podMap[controllerRef.UID]; ok {
podMap[controllerRef.UID] = append(podMap[controllerRef.UID], pod)
}
}
}
pkg\controller\deployment\sync.go
syncStatusOnly
func (dc *DeploymentController) syncStatusOnly(ctx context.Context, d *apps.Deployment, rsList []*apps.ReplicaSet) error {
newRS, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(ctx, d, rsList, false)
allRSs := append(oldRSs, newRS)
return dc.syncDeploymentStatus(ctx, allRSs, newRS, d)
}
syncDeploymentStatus
func (dc *DeploymentController) syncDeploymentStatus(ctx context.Context, allRSs []*apps.ReplicaSet, newRS *apps.ReplicaSet, d *apps.Deployment) error {
newStatus := calculateStatus(allRSs, newRS, d)
// 更新状态
newDeployment := d
newDeployment.Status = newStatus
_, err := dc.client.AppsV1().Deployments(newDeployment.Namespace).UpdateStatus(ctx, newDeployment, metav1.UpdateOptions{})
return err
}
pkg\controller\deployment\sync.go
sync
func (dc *DeploymentController) sync(ctx context.Context, d *apps.Deployment, rsList []*apps.ReplicaSet) error {
newRS, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(ctx, d, rsList, false)
if err := dc.scale(ctx, d, newRS, oldRSs); err != nil {
return err
}
if d.Spec.Paused && getRollbackTo(d) == nil {
if err := dc.cleanupDeployment(ctx, oldRSs, d); err != nil {
return err
}
}
allRSs := append(oldRSs, newRS)
return dc.syncDeploymentStatus(ctx, allRSs, newRS, d)
}
scale
sync
sync
sync
pkg\controller\deployment\rollback.go
rollback
func (dc *DeploymentController) rollback(ctx context.Context, d *apps.Deployment, rsList []*apps.ReplicaSet) error {
newRS, allOldRSs, err := dc.getAllReplicaSetsAndSyncRevision(ctx, d, rsList, true) --->pkg\controller\deployment\sync.go
// 组合全部rs列表
allRSs := append(allOldRSs, newRS)
if rollbackTo.Revision == 0 {
if rollbackTo.Revision = deploymentutil.LastRevision(allRSs); rollbackTo.Revision == 0 {
// 如果回滚版本为0，则验证能拿到的最老版本号是否为0，不是的话则放弃回滚操作
dc.emitRollbackWarningEvent(d, deploymentutil.RollbackRevisionNotFound, "Unable to find last revision.")
return dc.updateDeploymentAndClearRollbackTo(ctx, d)
}
}
// 循环所有rs获取其版本号
for _, rs := range allRSs {
v, err := deploymentutil.Revision(rs)
if err != nil {
continue
}
if v == rollbackTo.Revision {
klog.V(4).Infof("Found replica set %q with desired revision %d", rs.Name, v)
// 找到了则进行回滚
performedRollback, err := dc.rollbackToTemplate(ctx, d, rs)
if performedRollback && err == nil {
dc.emitRollbackNormalEvent(d, fmt.Sprintf("Rolled back deployment %q to revision %d", d.Name, rollbackTo.Revision))
}
return err
}
}
// 否则放弃回滚
dc.emitRollbackWarningEvent(d, deploymentutil.RollbackRevisionNotFound, "Unable to find the revision to rollback to.")
return dc.updateDeploymentAndClearRollbackTo(ctx, d)
}
getRollbackTo
func getRollbackTo(d *apps.Deployment) *extensions.RollbackConfig {
// 获取历史版本号
revision := d.Annotations[apps.DeprecatedRollbackTo]
if revision == "" {
return nil
}
revision64, err := strconv.ParseInt(revision, 10, 64)
// 返回当时的配置情况
return &extensions.RollbackConfig{
Revision: revision64,
}
}
pkg\controller\deployment\recreate.go
rolloutRecreate
pkg\controller\deployment\rolling.go
rolloutRolling
pkg\controller\deployment\sync.go
func (dc *DeploymentController) getAllReplicaSetsAndSyncRevision(ctx context.Context, d *apps.Deployment, rsList []*apps.ReplicaSet, createIfNotExisted bool) (*apps.ReplicaSet, []*apps.ReplicaSet, error) {
_, allOldRSs := deploymentutil.FindOldReplicaSets(d, rsList) --->pkg\controller\deployment\util\deployment_util.go
// Get new replica set with the updated revision number
newRS, err := dc.getNewReplicaSet(ctx, d, rsList, allOldRSs, createIfNotExisted)
if err != nil {
return nil, nil, err
}
return newRS, allOldRSs, nil
}
**func (dc *DeploymentController) getNewReplicaSet(ctx context.Context, d *apps.Deployment, rsList, oldRSs []*apps.ReplicaSet, createIfNotExisted bool) (*apps.ReplicaSet, error) {**
// 获取最新的rs
existingNewRS := deploymentutil.FindNewReplicaSet(d, rsList)
// 获取(maxOldRevision
maxOldRevision := deploymentutil.MaxRevision(oldRSs)
// 更新最新rs的maxOldRevision为老(maxOldRevision加1
newRevision := strconv.FormatInt(maxOldRevision+1, 10)
// 最新rs存在我们需要同步其信息并打补丁
if existingNewRS != nil {
rsCopy := existingNewRS.DeepCopy()
// annotation是否需要更新
annotationsUpdated := deploymentutil.SetNewReplicaSetAnnotations(d, rsCopy, newRevision, true, maxRevHistoryLengthInChars)
// 是否在更新周期内
minReadySecondsNeedsUpdate := rsCopy.Spec.MinReadySeconds != d.Spec.MinReadySeconds
if annotationsUpdated || minReadySecondsNeedsUpdate {
// 两者有任意不满足则进行rs信息更新，并修改更新时间
rsCopy.Spec.MinReadySeconds = d.Spec.MinReadySeconds
return dc.client.AppsV1().ReplicaSets(rsCopy.ObjectMeta.Namespace).Update(ctx, rsCopy, metav1.UpdateOptions{})
}
// 验证是否需要更新depoyment的状态
needsUpdate := deploymentutil.SetDeploymentRevision(d, rsCopy.Annotations[deploymentutil.RevisionAnnotation])
cond := deploymentutil.GetDeploymentCondition(d.Status, apps.DeploymentProgressing)
if deploymentutil.HasProgressDeadline(d) && cond == nil {
msg := fmt.Sprintf("Found new replica set %q", rsCopy.Name)
condition := deploymentutil.NewDeploymentCondition(apps.DeploymentProgressing, v1.ConditionTrue, deploymentutil.FoundNewRSReason, msg)
deploymentutil.SetDeploymentCondition(&d.Status, *condition)
needsUpdate = true
}
// 更新depoyment的状态
if needsUpdate {
var err error
if _, err = dc.client.AppsV1().Deployments(d.Namespace).UpdateStatus(ctx, d, metav1.UpdateOptions{}); err != nil {
return nil, err
}
}
return rsCopy, nil
}
// 如果不存在时不需要创建，则什么都不做
if !createIfNotExisted {
return nil, nil
}
// 创建一个新的rs
newRSTemplate := *d.Spec.Template.DeepCopy()
podTemplateSpecHash := controller.ComputeHash(&newRSTemplate, d.Status.CollisionCount)
newRSTemplate.Labels = labelsutil.CloneAndAddLabel(d.Spec.Template.Labels, apps.DefaultDeploymentUniqueLabelKey, podTemplateSpecHash)
// Add podTemplateHash label to selector.
newRSSelector := labelsutil.CloneSelectorAndAddLabel(d.Spec.Selector, apps.DefaultDeploymentUniqueLabelKey, podTemplateSpecHash)
// Create new ReplicaSet
newRS := apps.ReplicaSet{
ObjectMeta: metav1.ObjectMeta{
// Make the name deterministic, to ensure idempotence
Name: d.Name + "-" + podTemplateSpecHash,
Namespace: d.Namespace,
OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(d, controllerKind)},
Labels: newRSTemplate.Labels,
},
Spec: apps.ReplicaSetSpec{
Replicas: new(int32),
MinReadySeconds: d.Spec.MinReadySeconds,
Selector: newRSSelector,
Template: newRSTemplate,
},
}
// 更新全部rs列表
allRSs := append(oldRSs, &newRS)
newReplicasCount, err := deploymentutil.NewRSNewReplicas(d, allRSs, &newRS)
if err != nil {
return nil, err
}
*(newRS.Spec.Replicas) = newReplicasCount
// Set new replica set's annotation
deploymentutil.SetNewReplicaSetAnnotations(d, &newRS, newRevision, false, maxRevHistoryLengthInChars)
// 检测哈希冲突情况
alreadyExists := false
createdRS, err := dc.client.AppsV1().ReplicaSets(d.Namespace).Create(ctx, &newRS, metav1.CreateOptions{})
switch {
// 在缓存未更新及快速部署时可能会出现该情况
case errors.IsAlreadyExists(err):
alreadyExists = true
// Fetch a copy of the ReplicaSet.
rs, rsErr := dc.rsLister.ReplicaSets(newRS.Namespace).Get(newRS.Name)
if rsErr != nil {
return nil, rsErr
}
// If the Deployment owns the ReplicaSet and the ReplicaSet's PodTemplateSpec is semantically
// deep equal to the PodTemplateSpec of the Deployment, it's the Deployment's new ReplicaSet.
// Otherwise, this is a hash collision and we need to increment the collisionCount field in
// the status of the Deployment and requeue to try the creation in the next sync.
controllerRef := metav1.GetControllerOf(rs)
if controllerRef != nil && controllerRef.UID == d.UID && deploymentutil.EqualIgnoreHash(&d.Spec.Template, &rs.Spec.Template) {
createdRS = rs
err = nil
break
}
// Matching ReplicaSet is not equal - increment the collisionCount in the DeploymentStatus
// and requeue the Deployment.
if d.Status.CollisionCount == nil {
d.Status.CollisionCount = new(int32)
}
preCollisionCount := *d.Status.CollisionCount
*d.Status.CollisionCount++
// Update the collisionCount for the Deployment and let it requeue by returning the original
// error.
_, dErr := dc.client.AppsV1().Deployments(d.Namespace).UpdateStatus(ctx, d, metav1.UpdateOptions{})
if dErr == nil {
klog.V(2).Infof("Found a hash collision for deployment %q - bumping collisionCount (%d->%d) to resolve it", d.Name, preCollisionCount, *d.Status.CollisionCount)
}
return nil, err
case errors.HasStatusCause(err, v1.NamespaceTerminatingCause):
// 如果namespace正在被terminating，那么后续所有资源都不创建
return nil, err
case err != nil:
// 剩下其他error情况则直接更新deployment状态
msg := fmt.Sprintf("Failed to create new replica set %q: %v", newRS.Name, err)
if deploymentutil.HasProgressDeadline(d) {
cond := deploymentutil.NewDeploymentCondition(apps.DeploymentProgressing, v1.ConditionFalse, deploymentutil.FailedRSCreateReason, msg)
deploymentutil.SetDeploymentCondition(&d.Status, *cond)
_, _ = dc.client.AppsV1().Deployments(d.Namespace).UpdateStatus(ctx, d, metav1.UpdateOptions{})
}
dc.eventRecorder.Eventf(d, v1.EventTypeWarning, deploymentutil.FailedRSCreateReason, msg)
return nil, err
}
if !alreadyExists && newReplicasCount > 0 {
dc.eventRecorder.Eventf(d, v1.EventTypeNormal, "ScalingReplicaSet", "Scaled up replica set %s to %d", createdRS.Name, newReplicasCount)
}
// 正常创建并更新状态
needsUpdate := deploymentutil.SetDeploymentRevision(d, newRevision)
if !alreadyExists && deploymentutil.HasProgressDeadline(d) {
msg := fmt.Sprintf("Created new replica set %q", createdRS.Name)
condition := deploymentutil.NewDeploymentCondition(apps.DeploymentProgressing, v1.ConditionTrue, deploymentutil.NewReplicaSetReason, msg)
deploymentutil.SetDeploymentCondition(&d.Status, *condition)
needsUpdate = true
}
if needsUpdate {
_, err = dc.client.AppsV1().Deployments(d.Namespace).UpdateStatus(ctx, d, metav1.UpdateOptions{})
}
return createdRS, err
}
pkg\controller\deployment\util\deployment_util.go
func FindOldReplicaSets(deployment *apps.Deployment, rsList []*apps.ReplicaSet) ([]*apps.ReplicaSet, []*apps.ReplicaSet) {
var requiredRSs []*apps.ReplicaSet
var allRSs []*apps.ReplicaSet
newRS := FindNewReplicaSet(deployment, rsList)
for _, rs := range rsList {
// 过滤掉已经更新好的rs
if newRS != nil && rs.UID == newRS.UID {
continue
}
allRSs = append(allRSs, rs)
if *(rs.Spec.Replicas) != 0 {
// 处理副本数为0情况
requiredRSs = append(requiredRSs, rs)
}
}
return requiredRSs, allRSs
}
func FindNewReplicaSet(deployment *apps.Deployment, rsList []*apps.ReplicaSet) *apps.ReplicaSet {
// 基于创建时间排序
sort.Sort(controller.ReplicaSetsByCreationTimestamp(rsList))
for i := range rsList {
if EqualIgnoreHash(&rsList[i].Spec.Template, &deployment.Spec.Template) {
// 基于Template模板对比
return rsList[i]
}
}
return nil
}
