八、实战案例之基于ELK实现容器日志收集与展示

kubernetes实战案例之基于ELK实现容器日志收集与展示实现pod中日志收集之至ELK，自定义字段数据格式转换、排序、基于日志实现pod自愈、自动扩容等 

收集日志的方法：
    1、通过DaemonSet在宿主机启动日志收集客户端Fluentd、logstash、filebeat
        -v 选项把宿主机的日志挂载容器里，在容器把日志收集到ES
        配置较多
        
    2、手动安装日志收集客户端
        手动配置
        
    3、在pod内置一个filebeat
        配置简单
            在容器里面启动一个filebeat进程
            在pod再启动一个filebeat容器
    

实战:存取取数据库filebeat  -- redis -- logstat -- ELK
 
 
 
1.安装es服务器和插件： https://www.cnblogs.com/Yuanbangchen/p/16984995.html

2.安装redis服务器：https://www.cnblogs.com/Yuanbangchen/p/17275460.html

3-7步骤参考:https://www.cnblogs.com/Yuanbangchen/p/17264215.html

3.镜像构建，对初始镜像进行基础软件安装,此镜像为nginx tomcat jdk的基本镜像  安装日志收集工具filebeat
[root@localhost7C centos]# ll
-rw-r--r-- 1 root root      174 4月   7 2020 build-command.sh
-rw-r--r-- 1 root root      503 4月   9 2020 Dockerfile
-rw-r--r-- 1 root root 24694569 4月   7 2020 filebeat-7.6.1-x86_64.rpm

[root@localhost7C centos]# cat Dockerfile 
#自定义Centos 基础镜像
from from harbor.zzhz.com/baseimages/centos:7.6.1810 
LABEL MAINTAINER="2973707860@qq.com"
ADD filebeat-7.6.1-x86_64.rpm /tmp
RUN yum install -y /tmp/filebeat-7.6.1-x86_64.rpm vim wget tree  lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel \ 
openssl openssl-devel iproute net-tools iotop &&  rm -rf /etc/localtime /tmp/filebeat-7.6.1-x86_64.rpm \
&& ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && useradd  nginx -u 2019 && useradd www -u 2020

#harbor先创建/baseimages项目
[root@localhost7C centos]# cat build-command.sh 
#!/bin/bash
docker build -t  harbor.zzhz.com/baseimages/magedu-centos-base:7.6.1810  .

docker push harbor.zzhz.com/baseimages/magedu-centos-base:7.6.1810

#执行构造
[root@localhost7C centos]# chmod  +x build-command.sh 
[root@localhost7C centos]# ./build-command.sh




4.基于基础的centos镜像 harbor.zzhz.com/baseimages/magedu-centos-base:7.6.1810，制作公司内部基础镜像--jdk镜像

[root@localhost7C jdk-1.8.212]# ll
-rw-r--r-- 1 root root       138 3月  28 15:38 build-command.sh
-rw-r--r-- 1 root root       388 3月  28 15:36 Dockerfile
-rw-r--r-- 1 root root 195013152 4月   7 2020 jdk-8u212-linux-x64.tar.gz
-rw-r--r-- 1 root root      2128 3月  28 15:39 profile

[root@localhost7C jdk-1.8.212]# cat Dockerfile 
#JDK Base Image
FROM harbor.zzhz.com/baseimages/magedu-centos-base:7.6.1810
MAINTAINER zhangshijie "zhangshijie@magedu.net"

ADD jdk-8u212-linux-x64.tar.gz /usr/local/src/
RUN ln -sv /usr/local/src/jdk1.8.0_212 /usr/local/jdk 
ADD profile /etc/profile

ENV JAVA_HOME /usr/local/jdk
ENV JRE_HOME $JAVA_HOME/jre
ENV CLASSPATH $JAVA_HOME/lib/:$JRE_HOME/lib/
ENV PATH $PATH:$JAVA_HOME/bin



[root@localhost7C jdk-1.8.212]# cat build-command.sh 
#!/bin/bash
docker build -t harbor.zzhz.com/pub-images/jdk-base:v8.212  .
sleep 1
docker push  harbor.zzhz.com/pub-images/jdk-base:v8.212


#profile文件的来源：使用docker 运行一个临时容器复制出来。
[root@localhost7C jdk-1.8.212]# cat profile 
 ....
 ....
 ....
unset i
unset -f pathmunge
export LANG=en_US.UTF-8
export HISTTIMEFORMAT="%F %T `whoami` "

#主要是这几行。
export JAVA_HOME=/usr/local/jdk
export TOMCAT_HOME=/apps/tomcat
export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$TOMCAT_HOME/bin:$PATH
export CLASSPATH=.$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib:$JAVA_HOME/lib/tools.jar


#执行构造
[root@localhost7C jdk-1.8.212]# chmod  +x build-command.sh 
[root@localhost7C jdk-1.8.212]# ./build-command.sh

#测试
[root@localhost7C jdk-1.8.212]# docker run  -it --rm  harbor.zzhz.com/pub-images/jdk-base:v8.212 bash
[root@5cd43e965b82 /]# java  -version
java version "1.8.0_212"
Java(TM) SE Runtime Environment (build 1.8.0_212-b10)
Java HotSpot(TM) 64-Bit Server VM (build 25.212-b10, mixed mode)




5.基于jdk镜像制作tomcat基础镜像
[root@localhost7C tomcat-base-8.5.43]# ll
-rw-r--r-- 1 root root 9717059 4月   7 2020 apache-tomcat-8.5.43.tar.gz
-rw-r--r-- 1 root root     146 3月  28 15:48 build-command.sh
-rw-r--r-- 1 root root     341 3月  28 15:48 Dockerfile


[root@localhost7C tomcat-base-8.5.43]# cat Dockerfile 
#Tomcat 8.5.43基础镜像
FROM harbor.zzhz.com/pub-images/jdk-base:v8.212 
MAINTAINER zhangshijie "zhangshijie@magedu.net"
RUN mkdir /apps /data/tomcat/webapps /data/tomcat/logs -pv 
ADD apache-tomcat-8.5.43.tar.gz  /apps
RUN useradd tomcat -u 2021 && ln -sv /apps/apache-tomcat-8.5.43 /apps/tomcat && chown -R nginx.nginx /apps /data -R


[root@localhost7C tomcat-base-8.5.43]# cat build-command.sh 
#!/bin/bash
docker build -t harbor.zzhz.com/pub-images/tomcat-base:v8.5.43  .
sleep 3
docker push  harbor.zzhz.com/pub-images/tomcat-base:v8.5.43


[root@localhost7C tomcat-base-8.5.43]# chmod +x build-command.sh 
[root@localhost7C tomcat-base-8.5.43]# ./build-command.sh 


#测试访问tomcat基础镜像启动为容器：
[root@localhost7C tomcat-base-8.5.43]# docker run -it --rm -p 8801:8080  harbor.zzhz.com/pub-images/tomcat-base:v8.5.43  bash
[root@cfb5972b39ac /]# /apps/tomcat/bin/catalina.sh  start
Using CATALINA_BASE:   /apps/tomcat
Using CATALINA_HOME:   /apps/tomcat
Using CATALINA_TMPDIR: /apps/tomcat/temp
Using JRE_HOME:        /usr/local/jdk/jre
Using CLASSPATH:       /apps/tomcat/bin/bootstrap.jar:/apps/tomcat/bin/tomcat-juli.jar
Tomcat started.

[root@cfb5972b39ac /]# ps aux
USER        PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
root          1  0.0  0.1  11824  1924 pts/0    Ss   15:52   0:00 bash
root         23 63.4  4.9 3999372 91544 pts/0   Sl   15:53   0:03 /usr/local/jdk/jre/bin/java -Djava.util.logging.config.file=/apps/tomcat/conf/logging.properties -Djava.util.logging.manage
root         73  0.0  0.0  51748  1732 pts/0    R+   15:53   0:00 ps aux


#测试
[root@localhost7K ~]# curl 192.168.80.120:8801




6.tomcat业务镜像：tomcat业务镜像app1制作,后期按此步骤制作app2、appN镜像
[root@localhost7C linux39]# cd tomcat-app1/
[root@localhost7C tomcat-app1]# ll
-rwxr-xr-x 1 root root   165 4月   7 2020 build-command.sh
-rwxr-xr-x 1 root root 23611 4月   7 2020 catalina.sh  
-rw-r--r-- 1 root root   548 4月   9 2020 Dockerfile
-rw-r--r-- 1 root root   416 4月   9 2020 filebeat.yml
-rw-r--r-- 1 root root    12 4月   7 2020 index.html
-rw-r--r-- 1 root root   266 4月   9 2020 myapp.tar.gz
-rwxr-xr-x 1 root root   556 4月   9 2020 run_tomcat.sh
-rw-r--r-- 1 root root  6460 4月   7 2020 server.xml    



[root@localhost7C tomcat-app1]# cat Dockerfile 
#tomcat web1
FROM harbor.zzhz.com/pub-images/tomcat-base:v8.5.43 
ADD catalina.sh /apps/tomcat/bin/catalina.sh
ADD server.xml /apps/tomcat/conf/server.xml
#ADD myapp/* /data/tomcat/webapps/myapp/
ADD myapp.tar.gz /data/tomcat/webapps/myapp/
ADD run_tomcat.sh /apps/tomcat/bin/run_tomcat.sh
ADD filebeat.yml /etc/filebeat/filebeat.yml 
RUN mkdir /data/tomcat/webapps/myapp/images  /data/tomcat/webapps/myapp/static -p
RUN chown  -R nginx.nginx /data/ /apps/ 
EXPOSE 8080 8443
CMD ["/apps/tomcat/bin/run_tomcat.sh"]


[root@localhost7C tomcat-app1]# cat index.html 
tomcat app1



#收集工具配置文件
[root@localhost7C tomcat-app1]# cat filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /apps/tomcat/logs/catalina.out
  fields:
    type: k8s-tomcat-catalina

filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 1

output.redis:
  hosts: ["srv-devops-redis.magedu.svc.zzhz.local:6379"]  #redis地址
  key: "k8s-linux39-myapp"
  db: 1
  timeout: 5
  password: "123456"



#启动文件
[root@localhost7C tomcat-app1]# cat run_tomcat.sh 
#!/bin/bash
/usr/share/filebeat/bin/filebeat -e -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat &
su - nginx -c "/apps/tomcat/bin/catalina.sh start"
tail -f /etc/hosts

[root@localhost7C tomcat-app1]# cat server.xml 
其它省略
<Host name="localhost"  appBase="/data/tomcat/webapps"  unpackWARs="true" autoDeploy="true">





[root@localhost7C tomcat-app1]# cat build-command.sh 
#!/bin/bash
TAG=$1
docker build -t  harbor.zzhz.com/linux39/tomcat-app1:${TAG} .
sleep 3
docker push  harbor.zzhz.com/linux39/tomcat-app1:${TAG}




#重点，所有脚本要执行权限。
[root@localhost7C tomcat-app1]# chmod +x *.sh
[root@localhost7C tomcat-app1]# ./build-command.sh  2022-02-22


#测试访问tomcat基础镜像启动为容器：
[root@localhost7C tomcat-app1]#docker run -it --rm -p 8801:8080 harbor.zzhz.com/linux39/tomcat-app1:2022-02-22

[root@localhost7K ~]# curl  192.168.80.120:8801/myapp/index.html
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>马哥教育</title>
</head>
<body>
    <h1>linux39 myapp v1</h1>
    <h1>linux39 myapp v2</h1>
</body>
</html>




7.在k8s环境创建tomcat业务pod
[root@localhost7C tomcat-app1]# cat tomcat-app1.yaml 
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  labels:
    app: linux39-tomcat-app1-deployment-label
  name: linux39-tomcat-app1-deployment
  namespace: linux39
spec:
  replicas: 1
  selector:
    matchLabels:
      app: linux39-tomcat-app1-selector
  template:
    metadata:
      labels:
        app: linux39-tomcat-app1-selector
    spec:
      containers:
      - name: linux39-tomcat-app1-container
        image: harbor.zzhz.com/linux39/tomcat-app1:2022-04-14
        #command: ["/apps/tomcat/bin/run_tomcat.sh"]
        #imagePullPolicy: IfNotPresent
        imagePullPolicy: Always
        ports:
        - containerPort: 8080
          protocol: TCP
          name: http
        env:
        - name: "password"
          value: "123456"
        - name: "age"
          value: "18"
        resources:
          limits:
            cpu: 1
            memory: "512Mi"
          requests:
            cpu: 500m
            memory: "512Mi"
        volumeMounts:
        - name: linux39-images
          mountPath: /data/tomcat/webapps/myapp/images
          readOnly: false
        - name: linux39-static
          mountPath: /data/tomcat/webapps/myapp/static
          readOnly: false
      volumes:
      - name: linux39-images
        nfs:
          server: 192.168.80.110
          path: /data/linux39/images
      - name: linux39-static
        nfs:
          server: 192.168.80.110
          path: /data/linux39/static
      #nodeSelector:
      #  project: linux39
---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: linux39-tomcat-app1-service-label
  name: linux39-tomcat-app1-service
  namespace: linux39
spec:
  type: NodePort
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 8080
    nodePort: 30003
  selector:
    app: linux39-tomcat-app1-selector
[root@localhost7C tomcat-app1]# kubectl apply  -f tomcat-app1.yaml 

访问测试：tomcat产生数据库

8.logstash服务器配置文件
[root@localhost7I ]# cat /etc/logstash/conf.d/log.conf 
input {
    redis {
      host => "192.168.80.150"
      port => "36379"
      db => 1
      password => "123456"
      data_type => "list"
      key => "k8s-linux39-myapp"
    }
}

output {
   if [fields][type] == "k8s-tomcat-catalina" {
    elasticsearch {
      hosts => ["192.168.80.190:9200"]
      index => "k8s-linux39-tomcat-myapp-catalina-outlog-%{+YYYY.MM.dd}"
    }      
   }
}


9测试: 使用 head cerebro kibana等工具查看。