kubernetes第七次课（未完还有ceph）

重建重启pod

root@k8s-master2:~# kubectl rollout restart deployment magedu-tomcat-app1-deployment -n magedu

业务访问日志，错误日志，系统日志

root@k8s-node1:~# apt install openjdk-8-jdk -y

root@k8s-node1:~# tar xf kafka_2.13-2.4.0.tgz

root@k8s-node1:/apps# ln -sv /apps/kafka_2.13-2.4.0/ /apps/kafka

配置文件

root@k8s-node1:/apps/kafka# grep -v "^#" /apps/kafka/config/server.properties

broker.id=1

advertised.listeners=PLAINTEXT://172.31.150.37:9092

num.network.threads=3

num.io.threads=8

socket.send.buffer.bytes=102400

socket.receive.buffer.bytes=102400

socket.request.max.bytes=104857600

log.dirs=/tmp/kafka-logs

num.partitions=1

num.recovery.threads.per.data.dir=1

offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1

log.retention.hours=168

log.segment.bytes=1073741824

log.retention.check.interval.ms=300000

zookeeper.connect=172.31.150.37:32181,172.31.150.39:32182,172.31.150.36:32183

zookeeper.connection.timeout.ms=60000

group.initial.rebalance.delay.ms=0

root@k8s-node1:/apps/kafka# mkdir -p /tmp/kafka-logs

root@k8s-node1:/apps/kafka# /apps/kafka/bin/kafka-server-start.sh -daemon /apps/kafka/config/server.properties

构建镜像

root@k8s-master1:~/ELK-cases/1.daemonset-logstash/1.logstash-image-Dockerfile# cat Dockerfile
FROM logstash:7.12.1

USER root
WORKDIR /usr/share/logstash
#RUN rm -rf config/logstash-sample.conf
ADD logstash.yml /usr/share/logstash/config/logstash.yml
ADD logstash.conf /usr/share/logstash/pipeline/logstash.conf

root@k8s-master1:~/ELK-cases/1.daemonset-logstash/1.logstash-image-Dockerfile# vi build-commond.sh
#!/bin/bash

docker build -t www.awen.com/baseimages/logstash:v7.12.1-json-file-log-v1 .

docker push www.awen.com/baseimages/logstash:v7.12.1-json-file-log-v1

root@k8s-master1:~/ELK-cases/1.daemonset-logstash/1.logstash-image-Dockerfile# bash build-commond.sh

root@k8s-master1:~/ELK-cases/1.daemonset-logstash# kubectl apply -f 2.DaemonSet-logstash.yaml

root@k8s-master1:~/ELK-cases/1.daemonset-logstash# cat 2.DaemonSet-logstash.yaml

apiVersion: apps/v1

kind: DaemonSet

metadata:

namespace: kube-system

labels:

k8s-app: logstash-logging

spec:

selector:

matchLabels:

template:

metadata:

labels:

spec:

tolerations:

# this toleration is to have the daemonset runnable on master nodes

# remove it if your masters can't run pods

- key: node-role.kubernetes.io/master

operator: Exists

effect: NoSchedule

containers:

- name: logstash-elasticsearch

image: www.awen.com/baseimages/logstash:v7.12.1-json-file-log-v1

env:

- name: "KAFKA_SERVER"

value: "10.4.7.134:9092,10.4.7.135:9092,10.4.7.136:9092"

- name: "TOPIC_ID"

value: "jsonfile-log-topic"

- name: "CODEC"

value: "json"

# resources:

# limits:

# cpu: 1000m

# memory: 1024Mi

# requests:

# cpu: 500m

# memory: 1024Mi

volumeMounts:

- name: varlog #定义宿主机系统日志挂载路径

mountPath: /var/log #宿主机系统日志挂载点

- name: varlibdockercontainers #定义容器日志挂载路径,和logstash配置文件中的收集路径保持一直

mountPath: /var/lib/docker/containers #docker挂载路径

#mountPath: /var/log/pods #containerd挂载路径,此路径与logstash的日志收集路径必须一致

readOnly: false

terminationGracePeriodSeconds: 30

volumes:

- name: varlog

hostPath:

path: /var/log #宿主机系统日志

- name: varlibdockercontainers

hostPath:

path: /var/lib/docker/containers #docker的宿主机日志路径

#path: /var/log/pods #containerd的宿主机日志路径

logstash配置文件

root@k8s-harbor:/etc/logstash/conf.d# vi kafka-to-es.conf

input {

kafka {

bootstrap_servers => "10.4.7.134:9092,10.4.7.135:9092,10.4.7.136:9092"

topics => ["jsonfile-log-topic"]

codec => "json"

}

output {

#if [fields][type] == "app1-access-log" {

if [type] == "jsonfile-daemonset-applog" {

elasticsearch {

hosts => ["10.4.7.137:9200","10.4.7.138:9200"]

index => "jsonfile-daemonset-applog-%{+YYYY.MM.dd}"

user => "magedu"

password => "123456"

}}

if [type] == "jsonfile-daemonset-syslog" {

elasticsearch {

hosts => ["10.4.7.137:9200","10.4.7.138:9200"]

index => "jsonfile-daemonset-syslog-%{+YYYY.MM.dd}"

user => "magedu"

password => "123456"

}}

}

root@k8s-harbor:/etc/logstash/conf.d# systemctl start logstash.service

root@k8s-harbor:/etc/logstash/conf.d# tail -f /var/log/logstash/logstash-plain.log

日志收集成功

kibana 创建视图

日志示例之二：sidecar模式实现日志收集：

使用sidcar容器(一个pod多容器)收集当前pod内一个或者多个业务容器的日志(通常基于

emptyDir实现业务容器与sidcar之间的日志共享)。

root@k8s-master1:~/ELK-cases/2.sidecar-logstash/1.logstash-image-Dockerfile# vi build-commond.sh
#!/bin/bash

docker build -t www.awen.com/baseimages/logstash:v7.12.1-sidecar .
docker push www.awen.com/baseimages/logstash:v7.12.1-sidecar

root@k8s-master1:~/ELK-cases/2.sidecar-logstash/1.logstash-image-Dockerfile# bash build-commond.sh

sidecar容器日志收集

root@k8s-master1:~/ELK-cases/2.sidecar-logstash# cat 2.tomcat-app1.yaml

kind: Deployment

#apiVersion: extensions/v1beta1

apiVersion: apps/v1

metadata:

labels:

app: magedu-tomcat-app1-deployment-label

namespace: magedu

spec:

replicas: 3

selector:

matchLabels:

app: magedu-tomcat-app1-selector

template:

metadata:

labels:

app: magedu-tomcat-app1-selector

spec:

containers:

- name: sidecar-container

image: www.awen.com/baseimages/logstash:v7.12.1-sidecar

imagePullPolicy: IfNotPresent

#imagePullPolicy: Always

env:

- name: "KAFKA_SERVER"

value: "10.4.7.134:9092,10.4.7.135:9092,10.4.7.136:9092"

- name: "TOPIC_ID"

value: "tomcat-app1-topic"

- name: "CODEC"

value: "json"

volumeMounts:

- name: applogs

mountPath: /var/log/applog

- name: magedu-tomcat-app1-container

image: registry.cn-hangzhou.aliyuncs.com/zhangshijie/tomcat-app1:v1

imagePullPolicy: IfNotPresent

#imagePullPolicy: Always

ports:

- containerPort: 8080

protocol: TCP

env:

- name: "password"

value: "123456"

- name: "age"

value: "18"

resources:

limits:

cpu: 1

memory: "512Mi"

requests:

cpu: 500m

memory: "512Mi"

volumeMounts:

- name: applogs

mountPath: /apps/tomcat/logs

startupProbe:

httpGet:

path: /myapp/index.html

port: 8080

initialDelaySeconds: 5 #首次检测延迟5s

failureThreshold: 3 #从成功转为失败的次数

periodSeconds: 3 #探测间隔周期

readinessProbe:

httpGet:

#path: /monitor/monitor.html

path: /myapp/index.html

port: 8080

initialDelaySeconds: 5

periodSeconds: 3

timeoutSeconds: 5

successThreshold: 1

failureThreshold: 3

livenessProbe:

httpGet:

#path: /monitor/monitor.html

path: /myapp/index.html

port: 8080

initialDelaySeconds: 5

periodSeconds: 3

timeoutSeconds: 5

successThreshold: 1

failureThreshold: 3

volumes:

- name: applogs #定义通过emptyDir实现业务容器与sidecar容器的日志共享，以让sidecar收集业务容器中的日志

emptyDir: {}

创建tomcat服务service

root@k8s-master1:~/ELK-cases/2.sidecar-logstash# cat 3.tomcat-service.yaml

---

kind: Service

apiVersion: v1

metadata:

labels:

app: magedu-tomcat-app1-service-label

namespace: magedu

spec:

type: NodePort

ports:

- name: http

port: 80

protocol: TCP

targetPort: 8080

nodePort: 40080

selector:

app: magedu-tomcat-app1-selector

访问测试

创建tomcat服务service

root@k8s-master1:~/ELK-cases/2.sidecar-logstash# cat 3.tomcat-service.yaml

---

kind: Service

apiVersion: v1

metadata:

labels:

app: magedu-tomcat-app1-service-label

namespace: magedu

spec:

type: NodePort

ports:

- name: http

port: 80

protocol: TCP

targetPort: 8080

nodePort: 40080

selector:

app: magedu-tomcat-app1-selector

容器中logstash配置文件

root@k8s-master1:~/ELK-cases/2.sidecar-logstash/1.logstash-image-Dockerfile# cat logstash.conf

input {

file {

path => "/var/log/applog/catalina.out"

start_position => "beginning"

type => "app1-sidecar-catalina-log"

}

file {

path => "/var/log/applog/localhost_access_log.*.txt"

start_position => "beginning"

type => "app1-sidecar-access-log"

}

output {

if [type] == "app1-sidecar-catalina-log" {

kafka {

bootstrap_servers => "${KAFKA_SERVER}"

topic_id => "${TOPIC_ID}"

batch_size => 16384 #logstash每次向ES传输的数据量大小,单位为字节

codec => "${CODEC}"

} }

if [type] == "app1-sidecar-access-log" {

kafka {

bootstrap_servers => "${KAFKA_SERVER}"

topic_id => "${TOPIC_ID}"

batch_size => 16384

codec => "${CODEC}"

}}

}

root@k8s-harbor:/etc/logstash/conf.d# cat kafka-to-es.conf

input {

kafka {

bootstrap_servers => "10.4.7.134:9092,10.4.7.135:9092,10.4.7.136:9092"

topics => ["tomcat-app1-topic"]

codec => "json"

}

output {

#if [fields][type] == "app1-access-log" {

if [type] == "app1-sidecar-access-log" {

elasticsearch {

hosts => ["10.4.7.137:9200","10.4.7.138:9200"]

index => "sidecar-app1-accesslog-%{+YYYY.MM.dd}"

user => "magedu"

password => "123456"

}

#if [fields][type] == "app1-catalina-log" {

if [type] == "app1-sidecar-catalina-log" {

elasticsearch {

hosts => ["10.4.7.137:9200","10.4.7.138:9200"]

index => "sidecar-app1-catalinalog-%{+YYYY.MM.dd}"

user => "magedu"

password => "123456"

}

# stdout {

# codec => rubydebug

# }

}

root@k8s-harbor:/etc/logstash/conf.d# systemctl restart logstash.service

循环访问

root@k8s-node3:~# while true ;do curl http://10.4.7.136:40080/myapp/index.html;sleep 0.5; sleep 0.5;done

catalina.out日志

容器内置日志收集服务进程简介：

在容器内置日志收集服务进程，收集当前容器的业务容器日志等。

构建镜像

root@k8s-master1:~/ELK-cases/3.container-filebeat-process/1.webapp-filebeat-image-Dockerfile# bash build-command.sh v1

root@k8s-master1:~/ELK-cases/3.container-filebeat-process# cat 3.tomcat-app1.yaml

kind: Deployment

#apiVersion: extensions/v1beta1

apiVersion: apps/v1

metadata:

labels:

app: magedu-tomcat-app1-filebeat-deployment-label

namespace: magedu

spec:

replicas: 2

selector:

matchLabels:

app: magedu-tomcat-app1-filebeat-selector

template:

metadata:

labels:

app: magedu-tomcat-app1-filebeat-selector

spec:

containers:

- name: magedu-tomcat-app1-filebeat-container

image: www.awen.com/magedu/tomcat-app1:v1

imagePullPolicy: IfNotPresent

#imagePullPolicy: Always

ports:

- containerPort: 8080

protocol: TCP

env:

- name: "password"

value: "123456"

- name: "age"

value: "18"

resources:

limits:

cpu: 1

memory: "512Mi"

requests:

cpu: 500m

memory: "512Mi"

root@k8s-master1:~/ELK-cases/3.container-filebeat-process# cat 4.tomcat-service.yaml

---

kind: Service

apiVersion: v1

metadata:

labels:

app: magedu-tomcat-app1-filebeat-service-label

namespace: magedu

spec:

type: NodePort

ports:

- name: http

port: 80

protocol: TCP

targetPort: 8080

nodePort: 30092

selector:

app: magedu-tomcat-app1-filebeat-selector

测试访问

root@k8s-node3:~# while true ; do curl http://10.4.7.137:30092/myapp/index.html;sleep 0.5 ; sleep 0.5;done
<h1>tomcat app1 test page</h1>

root@k8s-harbor:/etc/logstash/conf.d# vi kafka-to-es.conf

input {

kafka {

bootstrap_servers => "10.4.7.134:9092,10.4.7.135:9092,10.4.7.136:9092"

topics => ["filebeat-magedu-app1"]

codec => "json"

}

output {

if [fields][type] == "filebeat-tomcat-catalina" {

elasticsearch {

hosts => ["10.4.7.138:9200","10.4.7.137:9200"]

index => "filebeat-tomcat-catalina-%{+YYYY.MM.dd}"

user => "magedu"

password => "123456"

}}

if [fields][type] == "filebeat-tomcat-accesslog" {

elasticsearch {

hosts => ["10.4.7.137:9200","10.4.7.138:9200"]

index => "filebeat-tomcat-accesslog-%{+YYYY.MM.dd}"

user => "magedu"

password => "123456"

}}

}

kubernetes升级

业务低峰期升级

1.二进制部署二进制文件替换。service、pod可能会被重启，（kubelet升级）

root@k8s-ansible:/etc/kubeasz/bin# cp kube-apiserver kube-controller-manager kube-scheduler kube-proxy kubectl /opt

root@k8s-ansible:/usr/local/src/kubernetes/server/bin# cp kube-apiserver kube-controller-manager kube-scheduler kube-proxy kubectl /etc/kubeasz/bin

root@k8s-ansible:/usr/local/src/kubernetes/server/bin# ./kube-apiserver --version
Kubernetes v1.23.17

升级（先驱逐）

root@k8s-ansible:/etc/kubeasz# ./ezctl upgrade k8s-01

root@k8s-node1-awen:~# docker load -i /opt/kube/images/k8s-dns-node-cache_1.21.1.tar

root@k8s-ansible:/etc/kubeasz# kubectl get node
NAME STATUS ROLES AGE VERSION
10.4.7.134 Ready,SchedulingDisabled node 12h v1.23.17
10.4.7.135 Ready,SchedulingDisabled node 12h v1.23.17
10.4.7.136 Ready,SchedulingDisabled node 12h v1.23.17
10.4.7.137 Ready node 3h13m v1.23.17
10.4.7.138 Ready node 12h v1.23.17
10.4.7.139 Ready node 12h v1.23.17

手动升级

0.暂停node调度kubectl cordon 10.4.7.139

1.pod驱逐 kubectl drain 10.4.7.139 --ignore-daemonsets --delete-emptydir-date

2.关闭服务 kubectl stop kubelet.service kube-proxy.service

3 分发二进制文件service文件等配置文件 cp kubelet kube-proxy 10.4.7.139:/usr/local/bin/

4 启动服务 kubectl start kubelet.service kube-proxy.service

5 取消cordon kubectl uncordon 10.4.7.139

只有kubelet 版本改变 get node 才会版本改变

coredns 足够的资源

cordon：警戒线，标记node不被调度新的pod

uncordon：取消警戒标记为cordon的node

node停止接受新pod调度预期下线

root@k8s-ansible-harbor:/etc/kubeasz# kubectl cordon 10.4.7.137

drain：主动驱逐node上的pod,用于node下线等场景

用于主机出现紧急下线

root@k8s-ansible-harbor:/etc/kubeasz# kubectl drain 10.4.7.138 --ignore-daemonsets --delete-emptydir-data

查看集群信息

root@k8s-ansible-harbor:/etc/kubeasz# kubectl config view

apiVersion: v1

clusters:

- cluster:

certificate-authority-data: DATA+OMITTED

server: https://10.4.7.134:6443

contexts:

- context:

cluster: cluster1

user: admin

current-context: context-cluster1

kind: Config

preferences: {}

users:

- name: admin

user:

client-certificate-data: REDACTED

client-key-data: REDACTED

集群外prometheus监控coredns

root@k8s-ansible-harbor:/apps/prometheus# vi prometheus.yml

- job_name: "coredns"
static_configs:
- targets: ["10.4.7.138:30009"]

root@k8s-ansible-harbor:/apps/prometheus# systemctl restart prometheus
root@k8s-ansible-harbor:/apps/prometheus# systemctl status prometheus

root@k8s-ansible-harbor:/apps/prometheus# dpkg -i grafana-enterprise_9.3.1_amd64.deb

root@k8s-ansible-harbor:/apps/prometheus# apt install -f libfontconfig1

apt --fix-broken install

root@k8s-ansible-harbor:/apps/prometheus# apt install -f libfontconfig1

posted on 2023-03-27 09:05 倔强不屈的阿文阅读(0) 评论(0) 收藏举报

刷新页面返回顶部

tshxawen

kubernetes第七次课（未完还有ceph）

导航

公告