关于annotation_prometheus_io_scrape及kubernetes_service_annotation_prometheus_io_port：
在k8s中，如果deployment的yaml文件指定了annotation_prometheus_io_scrape及kubernetes_service_annotation_prometheus_io_port，那么基于prometheus的发现规则，需要在被发现
的目的target定义注解匹配annotation_prometheus_io_scrape的值为true和kubernetes_service_annotation_prometheus_io_port对应的端口号如9153，且必须匹配成功该注解才会
- job_name: 'kubernetes-nginx-pods'
kubernetes_sd_configs:
- role: pod
namespaces: #可选指定namepace，如果不指定就是发现所有的namespace中的pod
names:
- myserver
- magedu
relabel_configs:
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
action: replace
target_label: __scheme__
regex: (https?)
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
action: replace
target_label: __address__
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: kubernetes_pod_name
root@k8s-master2:~/1.prometheus-case-files# cat case3-4-nginx.yaml
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
app: magedu-nginx-deployment-label
name: magedu-nginx-deployment
namespace: magedu
spec:
replicas: 1
selector:
matchLabels:
app: magedu-nginx-selector
template:
metadata:
labels:
app: magedu-nginx-selector
annotations:
prometheus.io/port: "9913"
prometheus.io/scrape: "true"
spec:
containers:
- name: magedu-nginx-container
image: gaciaga/nginx-vts:1.11.12-alpine-vts-0.1.14
#imagePullPolicy: IfNotPresent
imagePullPolicy: Always
ports:
- containerPort: 80
protocol: TCP
name: http
- containerPort: 443
protocol: TCP
name: https
env:
- name: "password"
value: "123456"
- name: "age"
value: "20"
#resources:
# limits:
# cpu: 500m
# memory: 512Mi
# requests:
# cpu: 500m
# memory: 256Mi
- name: magedu-nginx-exporter-container
image: sophos/nginx-vts-exporter
args:
- '-nginx.scrape-uri=http://127.0.0.1/status/format/json'
ports:
- containerPort: 9113
---
kind: Service
apiVersion: v1
metadata:
labels:
app: magedu-nginx-service-label
name: magedu-nginx-service
namespace: magedu
spec:
type: NodePort
ports:
- name: http
port: 80
protocol: TCP
targetPort: 80
nodePort: 30092
- name: https
port: 443
protocol: TCP
targetPort: 443
nodePort: 30093
selector:
app: magedu-nginx-selector
root@k8s-master2:~/1.prometheus-case-files# kubectl apply -f case3-4-nginx.yaml
![]()
查看nginx指标数据
![]()
![]()
prometheus 部署在k8s集群以外并实现服务发现：
创建prometheus 用于发现的账号
root@k8s-master2:~/1.prometheus-case-files# cat case4-prom-rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus
namespace: monitoring
---
apiVersion: v1
kind: Secret
type: kubernetes.io/service-account-token
metadata:
name: monitoring-token
namespace: monitoring
annotations:
kubernetes.io/service-account.name: "prometheus"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: prometheus
rules:
- apiGroups:
- ""
resources:
- nodes
- services
- endpoints
- pods
- nodes/proxy
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- configmaps
- nodes/metrics
verbs:
- get
- nonResourceURLs:
- /metrics
verbs:
- get
---
#apiVersion: rbac.authorization.k8s.io/v1beta1
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: prometheus
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: prometheus
subjects:
- kind: ServiceAccount
name: prometheus
namespace: monitoring
root@k8s-master2:~/1.prometheus-case-files# kubectl apply -f case4-prom-rbac.yaml
serviceaccount/prometheus created
secret/monitoring-token created
clusterrole.rbac.authorization.k8s.io/prometheus created
clusterrolebinding.rbac.authorization.k8s.io/prometheus created
root@k8s-master2:~/1.prometheus-case-files# kubectl get secrets -n monitoring
NAME TYPE DATA AGE
default-token-x4vf9 kubernetes.io/service-account-token 3 3d21h
monitor-token-qvn4x kubernetes.io/service-account-token 3 3d20h
monitoring-token kubernetes.io/service-account-token 3 28s
prometheus-token-xvg8t kubernetes.io/service-account-token 3 28s
root@k8s-ansible-harbor:~# vim /apps/prometheus/k8s.token
eyJhbGciOiJSUzI1NiIsImtpZCI6IjFteE1PM1dkRTVCdFNLQnBOSGU4WWdyQ1B6eEY1RHBIMkZBODV4empYTlkifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5p
by9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJtb25pdG9yaW5nIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZWNyZXQubmFtZSI6InByb21ldGhldXMtdG9rZW4iLCJrdWJlcm5ldGVzLml
vL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoicHJvbWV0aGV1cyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjE2NjY1M2EyLTJiNz
ktNGE3Zi05YzY5LTFlYTQyMWQ3ZWRkNCIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDptb25pdG9yaW5nOnByb21ldGhldXMifQ.WxmhAuM2yt90tCDle3Algb-b3OYY1nmAmBlqcqFiVoSgrwVWH-
FnrdB7rYdgtkSULyM2FNBubY2gZUHE3mQ22OGlUbKcVrI5O6ipyJwwQyAgijV-Tz1d0tlMtNm_4fX1ZGSyOZbd2Qyk59z-Qsxq80crASNYQAFYetIaeyxMSPPZqVUfwKJJJ0EaAVAKbiaS9QLrj3lA9RwiB
_OSErIK1wTtDxmgyz2LvzP-UEXrW2Dku-x1k5OtCJv0TDL370k2aVaxN7zkD8BqwrgCvy9M_XoMFUml_41iOFl4WmFI9m5oPfHCJQyF1D6Djn-oS15lCi7LxFoRHgdDg1RX7PchZw
root@k8s-ansible-harbor:/apps/prometheus# cat prometheus.yml
#node节点发现
- job_name: 'kubernetes-nodes-monitor'
scheme: http
tls_config:
insecure_skip_verify: true
bearer_token_file: /apps/prometheus/k8s.token
kubernetes_sd_configs:
- role: node
api_server: https://10.4.7.134:6443
tls_config:
insecure_skip_verify: true
bearer_token_file: /apps/prometheus/k8s.token
relabel_configs:
- source_labels: [__address__]
regex: '(.*):10250'
replacement: '${1}:9100'
target_label: __address__
action: replace
- source_labels: [__meta_kubernetes_node_label_failure_domain_beta_kubernetes_io_region]
regex: '(.*)'
replacement: '${1}'
action: replace
target_label: LOC
- source_labels: [__meta_kubernetes_node_label_failure_domain_beta_kubernetes_io_region]
regex: '(.*)'
replacement: 'NODE'
action: replace
target_label: Type
- source_labels: [__meta_kubernetes_node_label_failure_domain_beta_kubernetes_io_region]
regex: '(.*)'
replacement: 'K8S-test'
action: replace
target_label: Env
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
![]()
联邦集群
root@k8s-ansible-harbor:/apps/prometheus# vi prometheus.yml
- job_name: 'prometheus-federate-k8s-cluster'
scrape_interval: 10s
honor_labels: true
metrics_path: '/federate'
params:
'match[]':
- '{job="prometheus"}'
- '{__name__=~"job:.*"}'
- '{__name__=~"node.*"}'
static_configs:
- targets:
- '10.4.7.134:30090'
root@k8s-ansible-harbor:/apps/prometheus# systemctl restart prometheus
![]()
Pod伸缩简介：
根据当前pod的负载，动态调整 pod副本数量，业务高峰期自动扩容pod的副本数以尽快响应pod的请求。
在业务低峰期对pod进行缩容，实现降本增效的目的。
公有云支持node级别的弹性伸缩。
默认根据pod的cpu内存的利用率进行弹性伸缩 假如定义一个最低是3个（业务低峰期）最多是20个 这个范围内 hpa控制器对deployment中的资源利用率进行统计，
统计发现超出规定值（hpa设置阈值）以后 就扩容
也可以基于prometheus指标做扩容
手动扩容
root@k8s-master1:~# kubectl scale deployment magedu-tomcat-app1-deployment --replicas=2 -n magedu
#命令使用帮助
root@k8s-master1:~# kubectl --help | grep scale
scale Set a new size for a deployment, replica set, or replication controller
autoscale Auto-scale a deployment, replica set, stateful set, or replication controller
动态伸缩控制器类型：
水平pod自动缩放器(HPA)：
基于pod 资源利用率横向调整
pod副本数量。
垂直pod自动缩放器(VPA)：
基于pod资源利用率，调整对单个pod的最大资源限制，不能与HPA同时使用。
集群伸缩(Cluster Autoscaler,CA)
基于集群中node 资源使用情况，动态伸缩node节点，从而保证有CPU和内存资源用于创建pod。
![]()
![]()
hpa控制器，每隔一段时间像向api查询数据，如果没有触发hpa的条件就不会有操作。如果有触发hpa的操作就会对应的扩缩容，
HPA控制器简介：
Horizontal Pod Autoscaling (HPA)控制器，根据预定义好的阈值及pod当前的资源利用率，自动控制在k8s集群中运行的pod数量(自动弹性水平自动伸
缩).
--horizontal-pod-autoscaler-sync-period #默认每隔15s(可以通过–horizontal-pod-autoscaler-sync-period修改)查询metrics的资源使用情况。
--horizontal-pod-autoscaler-downscale-stabilization #缩容间隔周期，默认5分钟。
--horizontal-pod-autoscaler-sync-period #HPA控制器同步pod副本数的间隔周期
--horizontal-pod-autoscaler-cpu-initialization-period #初始化延迟时间，在此时间内 pod的CPU 资源指标将不会生效，默认为5分钟。
--horizontal-pod-autoscaler-initial-readiness-delay #用于设置 pod 准备时间， 在此时间内的 pod 统统被认为未就绪及不采集数据,默认为30秒。
--horizontal-pod-autoscaler-tolerance #HPA控制器能容忍的数据差异(浮点数，默认为0.1)，即新的指标要与当前的阈值差异在0.1或以上,即要大于
1+0.1=1.1，或小于1-0.1=0.9，比如阈值为CPU利用率50%,当前为80%,那么80/50=1.6 > 1.1则会触发扩容，分之会缩容。
即触发条件：avg(CurrentPodsConsumption) / Target >1.1 或 <0.9=把N个pod的数据相加后根据pod的数量计算出平均数除以阈值,大于1.1就扩容，小
于0.9就缩容。
计算公式：TargetNumOfPods = ceil(sum(CurrentPodsCPUUtilization) / Target) #ceil是一个向上取整的目的pod整数。 pod扩容数量和当前pod数量有关，相乘不足1向上取整。
指标数据需要部署metrics-server，即HPA使用metrics-server作为数据源。
https://github.com/kubernetes-sigs/metrics-server
在k8s 1.1引入HPA控制器，早期使用Heapster组件采集pod指标数据，在k8s 1.11版本开始使用Metrices Server完成数据采集，然后将采集到的数据通过
API（Aggregated API，汇总API），例如metrics.k8s.io、custom.metrics.k8s.io、external.metrics.k8s.io，然后再把数据提供给HPA控制器进行查询，
以实现基于某个资源利用率对pod进行扩缩容的目的。
![]()
root@k8s-master2:~/2.hpa-metrics-server-0.6.1-case# cat hpa-autoscaling-v2.yaml
apiVersion: autoscaling/v2 #定义API版本
kind: HorizontalPodAutoscaler #定义资源对象类型为HorizontalPodAutoscaler
metadata: #定义元数据
namespace: magedu #创建到指定的namespace
name: magedu-tomcat-app1-podautoscaler #HPA控制器名称
labels: #自定义标签
app: magedu-tomcat-app1 #标签1
version: v2 #标签2
spec: #定义对象具体信息
scaleTargetRef: #定义水平伸缩的目标对象，Deployment、ReplicationController/ReplicaSet
kind: Deployment #指定伸缩目标类型为Deployment控制器
apiVersion: apps/v1 #Deployment API版本
name: magedu-tomcat-app1-deployment #目标Deployment名称
minReplicas: 3 #最小pod副本数
maxReplicas: 10 #最大pod副本数
metrics: #基于指定的指标数据进行pod副本自动伸缩
- type: Resource #定义指标资源
resource: #定义指标资源具体信息
name: memory #资源名称为memory
target: #目标阈值
type: Utilization #触发类型为利用率
averageUtilization: 50 #平均利用率50%
- type: Resource #定义指标资源
resource: #定义指标资源具体信息
name: cpu #资源名称为cpu
target: ##目标阈值
type: Utilization #触发类型为利用率
averageUtilization: 40 #平均利用率50%
root@k8s-master2:~/2.hpa-metrics-server-0.6.1-case# kubectl apply -f hpa-autoscaling-v2.yaml
horizontalpodautoscaler.autoscaling/magedu-tomcat-app1-podautoscaler created
![]()
这里必须有数据，一个yaml文件资源限制，一个安装metrics-server组件，这里前边是内存后边是cpu
浙公网安备 33010602011771号