K8S 安装 dolphinscheduler

参考文档

https://dolphinscheduler.apache.org/zh-cn/docs/3.1.7/guide/installation/kubernetes

前期工作

# 重新打包镜像（加入 mysql-connector jar 包）
# 准备工作，在开发机上操作(192.168.1.81 )
mkdir -p /data/images/apache && cd /data/images/apache

wget https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar
mkdir dolphinscheduler-api  dolphinscheduler-master  dolphinscheduler-worker

echo dolphinscheduler-api  dolphinscheduler-master  dolphinscheduler-worker |xargs -n 1 cp mysql-connector-java-8.0.16.jar


# 打包 dolphinscheduler-master 镜像
cd /data/images/apache/dolphinscheduler-master

cat >> Dockerfile << EOF 
FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-master:3.1.7
COPY mysql-connector-java-8.0.16.jar /opt/dolphinscheduler/libs
EOF

docker build -t harbor.junengcloud.com/apache/dolphinscheduler-master-mysql:3.1.7 .
docker push harbor.junengcloud.com/apache/dolphinscheduler-master-mysql:3.1.7


# 打包 dolphinscheduler-worker 镜像
cd /data/images/apache/dolphinscheduler-worker

cat >> Dockerfile << EOF 
FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-worker:3.1.7
COPY mysql-connector-java-8.0.16.jar /opt/dolphinscheduler/libs
EOF

docker build -t harbor.junengcloud.com/apache/dolphinscheduler-worker-mysql:3.1.7 .
docker push harbor.junengcloud.com/apache/dolphinscheduler-worker-mysql:3.1.7 

# 打包 dolphinscheduler-api 镜像
cd /data/images/apache/dolphinscheduler-api

cat >> Dockerfile << EOF 
FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-api:3.1.7
COPY mysql-connector-java-8.0.16.jar /opt/dolphinscheduler/libs
EOF

docker build -t harbor.junengcloud.com/apache/dolphinscheduler-api-mysql:3.1.7 .
docker push harbor.junengcloud.com/apache/dolphinscheduler-api-mysql:3.1.7 


# 上传 dolphinscheduler-tools 镜像到公司内部 harbor 镜像仓库
docker pull  dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-tools:3.1.7
docker tag  dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler-tools:3.1.7 harbor.junengcloud.com/apache/dolphinscheduler-tools:3.1.7
docker push harbor.junengcloud.com/apache/dolphinscheduler-tools:3.1.7

# 下载源码包
https://dolphinscheduler.apache.org/zh-cn/download/3.1.7

tar -zxvf apache-dolphinscheduler-<version>-src.tar.gz
cd apache-dolphinscheduler-<version>-src/deploy/kubernetes/
mv dolphinscheduler /tmp/

安装

mkdir -p /data/yaml/basic-service/dolphinscheduler && cd /data/yaml/basic-service/dolphinscheduler
mv /tmp/dolphinscheduler . && cd dolphinscheduler

# 安装依赖
helm repo add bitnami https://charts.bitnami.com/bitnami
helm dependency update .

cd /data/yaml/basic-service/dolphinscheduler

# 根据自身需求进行修改
cat >> my-values.yaml << EOF 
image:
  registry: "harbor.junengcloud.com/apache"
  tag: "3.1.7"
  master: dolphinscheduler-master-mysql
  worker: dolphinscheduler-worker-mysql
  api: dolphinscheduler-api-mysql

postgresql:
  postgresqlPassword: "HdYl#20230704"    # postgresql
  persistence:
    enabled: true
    storageClass: "nfs-client"           # 设置 storageClass
  resources:
    limits:
      cpu: "500m"
      memory: "1Gi"
    requests:
      cpu: "250m"
      memory: "512Mi"

zookeeper:
  persistence:
    enabled: true
    size: "5Gi"
    storageClass: "nfs-client"
  resources:
    limits:
      cpu: "500m"
      memory: "1Gi"
    requests:
      cpu: "250m"
      memory: "512Mi"

common:
  configmap:
    RESOURCE_UPLOAD_PATH: "/dolphinscheduler"
  fsFileResourcePersistence:
    enabled: true
    accessModes:
    - "ReadWriteMany"
    storageClassName: "nfs-client"
    storage: "5Gi"

conf:
  common:
    resource.hdfs.fs.defaultFS: "file:///"

master:
  replicas: "1"
  resources:
    limits:
      memory: "2Gi"
      cpu: "4"
    requests:
      memory: "1Gi"
      cpu: "500m"
  persistentVolumeClaim:
    enabled: true
    accessModes:
    - "ReadWriteOnce"
    storageClassName: "nfs-client"
    storage: "5Gi"

worker:
  replicas: "1"
  resources:
    limits:
      memory: "2Gi"
      cpu: "4"
    requests:
      memory: "1Gi"
      cpu: "500m"
  persistentVolumeClaim:
    enabled: true
    ## dolphinscheduler data volume
    dataPersistentVolume:
      enabled: true
      accessModes:
      - "ReadWriteOnce"
      storageClassName: "nfs-client"
      storage: "2Gi"
    ## dolphinscheduler logs volume
    logsPersistentVolume:
      enabled: true
      accessModes:
      - "ReadWriteOnce"
      storageClassName: "nfs-client"
      storage: "2Gi"

alert:
  resources:
    limits:
      memory: "2Gi"
      cpu: "1"
    requests:
      memory: "512Mi"
      cpu: "500m"
  persistentVolumeClaim:
    enabled: true
    accessModes:
    - "ReadWriteOnce"
    storageClassName: "nfs-client"
    storage: "2Gi"

api:
  service:
    type: "NodePort"
    nodePort: "30002"
  resources:
    limits:
      memory: "1Gi"
      cpu: "1"
    requests:
      memory: "512Mi"
      cpu: "500m"
  persistentVolumeClaim:
    enabled: true
    accessModes:
    - "ReadWriteOnce"
    storageClassName: "nfs-client"
    storage: "2Gi"

EOF

# 创建命名空间
kubectl create ns basic-service

helm -n basic-service install dolphinscheduler -f my-values.yaml  ./dolphinscheduler