k8s helm 搭建EFK

k8 helm 搭建EFK

ELK已经被阿里巴巴收购了，目前是国际上的开源软件

例如收集全国各地的日志，以上海为例
上海地区的每台机器都部署一个logstash，然后日志转发给一个总的logstash，logstash日志转发给es的两个客户端es-C，交由es的Master端es-M,再存储在es-D的存储中。对外提供一个接口，kibana可视化对外提供数据展示

但是有个弊端，logstash和es都是java写的，java写的东西很重，起来后内存可能占到300M，而go写的占40-50M左右，如果仅仅为了收集日志，1000台机器上都安装logstash，占用资源太多了，所以我们用go写的flant，目前只有es是java写的，还可以接受

1.1 添加 Google incubator 仓库

同样，这里是外网，国内无法直接访问，我把需要安装的包存放到百度网盘上了
链接：https://pan.baidu.com/s/1jOeXtjpIxfc4obNHhDec-Q
提取码：dfbu

helm repo add incubator http://storage.googleapis.com/kubernetes-charts-incubator

下载后，在各个机器节点上导入镜像

//注意，如果自己下载的efk包，一定确保es和kibana版本一致，精确到小版本，否则无法使用，一定要同一个版本
docker load -i elasticsearch-oss.tar
docker load -i fluentd-elasticsearch.tar
docker load -i kibanaoss.tar

2.1 安装es

kubectl create ns efk
cd charts/;tar zxvf elasticsearch-1.10.2.tgz
cd elasticsearch;vim values.yaml

appVersion: "6.4.2"
image:
  repository: "docker.elastic.co/elasticsearch/elasticsearch-oss"
  tag: "6.4.2"
  pullPolicy: "IfNotPresent"
initImage:
  repository: "busybox"
  tag: "latest"
  pullPolicy: "Always"
cluster:
  name: "elasticsearch"
  #这是监控efk各个组件的功能，是收费的，如果没交费需要把这里关闭
  xpackEnable: false
  config: {}
  additionalJavaOpts: ""
  env:
  #master节点不能低于2个，否则无法对外工作
    MINIMUM_MASTER_NODES: "2"
client:
  name: client
  replicas: 2
  serviceType: ClusterIP
  loadBalancerIP: {}
  loadBalancerSourceRanges: {}
  heapSize: "512m"
  antiAffinity: "soft"
  nodeAffinity: {}
  nodeSelector: {}
  tolerations: []
  resources:
    limits:
      cpu: "1"
    requests:
      cpu: "25m"
      memory: "512Mi"
  priorityClassName: ""
  podDisruptionBudget:
    enabled: false
    minAvailable: 1
master:
  name: master
  exposeHttp: false
  replicas: 3
  heapSize: "512m"
  persistence:
  #是否开启持久卷，我们这里不开了，生产中是要打开的
    enabled: false
    accessMode: ReadWriteOnce
    name: data
    size: "4Gi"
  antiAffinity: "soft"
  nodeAffinity: {}
  nodeSelector: {}
  tolerations: []
  resources:
    limits:
      cpu: "1"
    requests:
      cpu: "25m"
      memory: "512Mi"
  priorityClassName: ""
  podDisruptionBudget:
    enabled: false
  updateStrategy:
    type: OnDelete
data:
  name: data
  exposeHttp: false
  replicas: 2
  heapSize: "1536m"
  persistence:
    enabled: false
    accessMode: ReadWriteOnce
    name: data
    size: "30Gi"
  terminationGracePeriodSeconds: 3600
  antiAffinity: "soft"
  nodeAffinity: {}
  nodeSelector: {}
  tolerations: []
  resources:
    limits:
      cpu: "1"
    requests:
      cpu: "25m"
      memory: "1536Mi"
  priorityClassName: ""
  podDisruptionBudget:
    enabled: false
    maxUnavailable: 1
  updateStrategy:
    type: OnDelete
extraInitContainers: |


helm fetch incubator/elasticsearch
helm  install --name els1 --namespace=efk -f values.yaml incubator/elasticsearch
kubectl  run cirror-$RANDOM --rm -it --image=cirros -- /bin/sh
	curl Elasticsearch:Port/_cat/nodes

2.1 部署 Fluentd

helm fetch stable/fluentd-elasticsearch
vim  values.yaml
	# 更改其中 Elasticsearch 访问地址
helm install --name flu1 --namespace=efk -f values.yaml stable/fluentd-elasticsearch

3.1 部署 kibana

helm fetch stable/kibana --version 0.14.8
helm install --name kib1 --namespace=efk -f values.yaml stable/kibana --version 0.14.8