ZhangZhihui's Blog  
Dockerfile - base, spark-base, spark-master, spark-worker, jupyterlab (Spark 3.4.1)

build.sh:

#!/bin/bash
#
# -- Build Apache Spark Standalone Cluster Docker Images

# ----------------------------------------------------------------------------------------------------------------------
# -- Variables ---------------------------------------------------------------------------------------------------------
# ----------------------------------------------------------------------------------------------------------------------

BUILD_DATE="$(date -u +'%Y-%m-%d')"
SPARK_VERSION="3.4.1"
HADOOP_VERSION="3"
DELTA_SPARK_VERSION="2.4.0"
DELTALAKE_VERSION="0.10.0"
JUPYTERLAB_VERSION="4.0.2"
PANDAS_VERSION="2.0.1"
DELTA_PACKAGE_VERSION="delta-core_2.12:2.4.0"
SPARK_VERSION_MAJOR=${SPARK_VERSION:0:1}
SPARK_XML_PACKAGE_VERSION="spark-xml_2.12:0.16.0"
SPARKSQL_MAGIC_VERSION="0.0.3"
KAFKA_PYTHON_VERSION="2.0.2"

# ----------------------------------------------------------------------------------------------------------------------
# -- Functions----------------------------------------------------------------------------------------------------------
# ----------------------------------------------------------------------------------------------------------------------

function cleanContainers() {

    container="$(docker ps -a | grep 'jupyterlab' | awk '{print $1}')"
    docker stop "${container}"
    docker rm "${container}"

    container="$(docker ps -a | grep 'spark-worker' -m 1 | awk '{print $1}')"
    while [ -n "${container}" ];
    do
      docker stop "${container}"
      docker rm "${container}"
      container="$(docker ps -a | grep 'spark-worker' -m 1 | awk '{print $1}')"
    done

    container="$(docker ps -a | grep 'spark-master' | awk '{print $1}')"
    docker stop "${container}"
    docker rm "${container}"

    container="$(docker ps -a | grep 'spark-base' | awk '{print $1}')"
    docker stop "${container}"
    docker rm "${container}"

    container="$(docker ps -a | grep 'base' | awk '{print $1}')"
    docker stop "${container}"
    docker rm "${container}"

}

function cleanImages() {

      docker rmi -f "$(docker images | grep -m 1 'jupyterlab' | awk '{print $3}')"

      docker rmi -f "$(docker images | grep -m 1 'spark-worker' | awk '{print $3}')"
      docker rmi -f "$(docker images | grep -m 1 'spark-master' | awk '{print $3}')"
      docker rmi -f "$(docker images | grep -m 1 'spark-base' | awk '{print $3}')"

      docker rmi -f "$(docker images | grep -m 1 'base' | awk '{print $3}')"

}

function cleanVolume() {
  docker volume rm "distributed-file-system"
}

function buildImages() {


    docker build \
      --build-arg build_date="${BUILD_DATE}" \
      --build-arg scala_version="${SCALA_VERSION}" \
      --build-arg delta_spark_version="${DELTA_SPARK_VERSION}" \
      --build-arg deltalake_version="${DELTALAKE_VERSION}" \
      --build-arg pandas_version="${PANDAS_VERSION}" \
      -f docker/base/Dockerfile \
      -t base:latest .

    docker build \
      --build-arg build_date="${BUILD_DATE}" \
      --build-arg scala_version="${SCALA_VERSION}" \
      --build-arg delta_spark_version="${DELTA_SPARK_VERSION}" \
      --build-arg deltalake_version="${DELTALAKE_VERSION}" \
      --build-arg pandas_version="${PANDAS_VERSION}" \
      --build-arg spark_version="${SPARK_VERSION}" \
      --build-arg hadoop_version="${HADOOP_VERSION}" \
      --build-arg delta_package_version="${DELTA_PACKAGE_VERSION}" \
      --build-arg spark_xml_package_version="${SPARK_XML_PACKAGE_VERSION}" \
      -f docker/spark-base/Dockerfile \
      -t spark-base:${SPARK_VERSION} .

    docker build \
      --build-arg build_date="${BUILD_DATE}" \
      --build-arg spark_version="${SPARK_VERSION}" \
      -f docker/spark-master/Dockerfile \
      -t spark-master:${SPARK_VERSION} .

    docker build \
      --build-arg build_date="${BUILD_DATE}" \
      --build-arg spark_version="${SPARK_VERSION}" \
      -f docker/spark-worker/Dockerfile \
      -t spark-worker:${SPARK_VERSION} .

    docker build \
      --build-arg build_date="${BUILD_DATE}" \
      --build-arg scala_version="${SCALA_VERSION}" \
      --build-arg delta_spark_version="${DELTA_SPARK_VERSION}" \
      --build-arg deltalake_version="${DELTALAKE_VERSION}" \
      --build-arg pandas_version="${PANDAS_VERSION}" \
      --build-arg spark_version="${SPARK_VERSION}" \
      --build-arg jupyterlab_version="${JUPYTERLAB_VERSION}" \
      --build-arg sparksql_magic_version="${SPARKSQL_MAGIC_VERSION}" \
      --build-arg kafka_python_version="${KAFKA_PYTHON_VERSION}" \
      -f docker/jupyterlab/Dockerfile \
      -t jupyterlab:${JUPYTERLAB_VERSION}-spark-${SPARK_VERSION} .

}

# ----------------------------------------------------------------------------------------------------------------------
# -- Main --------------------------------------------------------------------------------------------------------------
# ----------------------------------------------------------------------------------------------------------------------

cleanContainers;
cleanImages;
cleanVolume;
buildImages;

 

base/Dockerfile:

ARG java_image_tag=17-jre
FROM eclipse-temurin:${java_image_tag}

# -- Layer: Image Metadata

ARG build_date
ARG delta_spark_version
ARG deltalake_version
ARG pandas_version

LABEL org.label-schema.build-date=${build_date}
LABEL org.label-schema.description="Data Engineering wih Apache Spark and Delta Lake Cookbook  - Cluster base image"
LABEL org.label-schema.schema-version="1.0"

# -- Layer: OS + Python + Scala

ARG shared_workspace=/opt/workspace

RUN mkdir -p ${shared_workspace}/data
RUN mkdir -p /usr/share/man/man1
RUN apt-get update -y
RUN apt-get install -y --no-install-recommends curl python3 r-base netcat-traditional
RUN ln -s /usr/bin/python3 /usr/bin/python
RUN apt-get clean
RUN rm -rf /var/lib/apt/lists/*

RUN apt-get update -y
RUN apt-get install -y --no-install-recommends build-essential manpages-dev python3-pip python3-dev python3-venv
# RUN pip3 install  --no-cache-dir --upgrade pip
RUN apt-get clean
RUN rm -rf /var/lib/apt/lists/*

RUN python3 -m venv /opt/myenv
RUN /opt/myenv/bin/pip install --no-cache-dir --upgrade pip

# We are explicitly pinning the versions of various libraries which this Docker image runs on.
RUN /opt/myenv/bin/pip install --quiet --no-cache-dir delta-spark==${delta_spark_version}
RUN /opt/myenv/bin/pip install --quiet --no-cache-dir deltalake==${deltalake_version}
RUN /opt/myenv/bin/pip install --quiet --no-cache-dir pandas==${pandas_version}

ENV SCALA_HOME="/usr/bin/scala"
ENV PATH=/opt/myenv/bin:${PATH}:${SCALA_HOME}/bin
ENV SHARED_WORKSPACE=${shared_workspace}

# -- Runtime

VOLUME ${shared_workspace}
CMD ["bash"]

Successfully built.

 

spark-base/Dockerfile:

FROM base

# -- Layer: Image Metadata
ARG build_date
ARG delta_package_version
ARG spark_xml_package_version

LABEL org.label-schema.build-date=${build_date}
LABEL org.label-schema.description="Data Engineering wih Apache Spark and Delta Lake Cookbook - Spark base image"
LABEL org.label-schema.schema-version="1.0"

# -- Layer: Apache Spark
ARG spark_version
ARG hadoop_version

RUN curl https://archive.apache.org/dist/spark/spark-${spark_version}/spark-${spark_version}-bin-hadoop${hadoop_version}.tgz -o spark.tgz
RUN tar -xf spark.tgz
RUN mv spark-${spark_version}-bin-hadoop${hadoop_version} /usr/bin/
RUN echo "alias pyspark=/usr/bin/spark-${spark_version}-bin-hadoop${hadoop_version}/bin/pyspark" >> ~/.bashrc
RUN echo "alias spark-shell=/usr/bin/spark-${spark_version}-bin-hadoop${hadoop_version}/bin/spark-shell" >> ~/.bashrc
RUN mkdir /usr/bin/spark-${spark_version}-bin-hadoop${hadoop_version}/logs
RUN rm spark.tgz

ENV SPARK_HOME /usr/bin/spark-${spark_version}-bin-hadoop${hadoop_version}
ENV SPARK_MASTER_HOST spark-master
ENV SPARK_MASTER_PORT 7077
ENV PYSPARK_PYTHON python3

# -- Runtime
WORKDIR ${SPARK_HOME}

USER root

ARG NBuser=NBuser
ARG GROUP=NBuser

RUN groupadd -r ${GROUP} && useradd -r -m -g ${GROUP} ${NBuser}

RUN chown -R "${NBuser}":"${GROUP}" /home/"${NBuser}"/
RUN chown -R "${NBuser}":"${GROUP}" "${SPARK_HOME}"
RUN chown -R "${NBuser}":"${GROUP}" "${SHARED_WORKSPACE}"

USER ${NBuser}


RUN ${SPARK_HOME}/bin/spark-shell --packages io.delta:${delta_package_version} \
    --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" \
    --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog"
RUN ${SPARK_HOME}/bin/spark-shell --packages com.databricks:${spark_xml_package_version}
RUN ${SPARK_HOME}/bin/spark-shell --packages org.apache.spark:spark-sql-kafka-0-10_2.12:3.4.1

Successfully built.

 

spark-master/Dockerfile:

ARG spark_version
FROM spark-base:${spark_version}

# -- Layer: Image Metadata

ARG build_date

LABEL org.label-schema.build-date=${build_date}
LABEL org.label-schema.description="Spark master image"
LABEL org.label-schema.schema-version="1.0"

# -- Runtime

EXPOSE 8080 7077

CMD bin/spark-class org.apache.spark.deploy.master.Master >> logs/spark-master.out

Successfully built.

 

spark-worker/Dockerfile:

ARG spark_version
FROM spark-base:${spark_version}

# -- Layer: Image Metadata

ARG build_date

LABEL org.label-schema.build-date=${build_date}
LABEL org.label-schema.description="Spark worker image"
LABEL org.label-schema.schema-version="1.0"

# -- Runtime

EXPOSE 8081

CMD bin/spark-class org.apache.spark.deploy.worker.Worker spark://${SPARK_MASTER_HOST}:${SPARK_MASTER_PORT} >> logs/spark-worker.out

Successfully built.

 

jupyterlab/Dockerfile:

FROM base

# -- Layer: Image Metadata

ARG build_date

LABEL org.label-schema.build-date=${build_date}
LABEL org.label-schema.name="Data Engineering wih Apache Spark and Delta Lake Cookbook - JupyterLab Image"
LABEL org.label-schema.description="JupyterLab image"


# -- Layer: Notebooks and data

# ADD docker/jupyterlab/kafka-producer.py /

# -- Layer: JupyterLab + Python kernel for PySpark

ARG spark_version
ARG jupyterlab_version
ARG sparksql_magic_version
ARG kafka_python_version

RUN pip install --no-cache-dir wget==3.2
RUN pip install --no-cache-dir pyspark==${spark_version}
RUN pip install --no-cache-dir jupyterlab==${jupyterlab_version}
RUN pip install --no-cache-dir sparksql-magic==${sparksql_magic_version}
RUN pip install --no-cache-dir kafka-python==${kafka_python_version}

EXPOSE 8888

WORKDIR ${SHARED_WORKSPACE}
CMD jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --allow-root --NotebookApp.token=;python

Successfully built.

 

docker-compose.yml:

version: "3.6"
volumes:
  shared-workspace:
    name: "distributed-file-system"
    driver: local
    driver_opts:
      o: bind
      type: none
      device: ./
services:
  zookeeper:
    image: docker.io/bitnami/zookeeper:3.8.2
    container_name: zookeeper
    ports:
      - "2181:2181"
    volumes:
      - shared-workspace:/opt/workspace
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes
  kafka:
    image: docker.io/bitnami/kafka:3.5.1
    container_name: kafka
    ports:
      - "9092:9092"
    environment:
      - BITNAMI_DEBUG=yes
      - KAFKA_BROKER_ID=1
      - KAFKA_ENABLE_KRAFT=false
      - KAFKA_CFG_LISTENERS=PLAINTEXT://kafka:9092
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT
      - ALLOW_PLAINTEXT_LISTENER=yes
    depends_on:
      - zookeeper
  jupyterlab:
    image: jupyterlab:4.0.2-spark-3.4.1
    container_name: jupyterlab
    ports:
      - 8888:8888
      - 4040:4040
    volumes:
      - shared-workspace:/opt/workspace
  spark-master:
    image: spark-master:3.4.1
    container_name: spark-master
    ports:
      - 8080:8080
      - 7077:7077
    volumes:
      - shared-workspace:/opt/workspace
  spark-worker-1:
    image: spark-worker:3.4.1
    container_name: spark-worker-1
    environment:
      - SPARK_WORKER_CORES=1
      - SPARK_WORKER_MEMORY=512m
    ports:
      - 8081:8081
    volumes:
      - shared-workspace:/opt/workspace
    depends_on:
      - spark-master
  spark-worker-2:
    image: spark-worker:3.4.1
    container_name: spark-worker-2
    environment:
      - SPARK_WORKER_CORES=1
      - SPARK_WORKER_MEMORY=512m
    ports:
      - 8082:8081
    volumes:
      - shared-workspace:/opt/workspace
    depends_on:
      - spark-master

 

zzh@ZZHPC:~/zd/Github/Data-Engineering-with-Databricks-Cookbook-main$ docker compose up
WARN[0000] /home/zzh/zd/Github/Data-Engineering-with-Databricks-Cookbook-main/docker-compose.yml: `version` is obsolete 
[+] Running 8/5
 ✔ Network data-engineering-with-databricks-cookbook-main_default  Created                                                                                  0.1s 
 ✔ Volume "distributed-file-system"                                Created                                                                                  0.0s 
 ✔ Container spark-master                                          Created                                                                                  0.0s 
 ✔ Container zookeeper                                             Created                                                                                  0.0s 
 ✔ Container jupyterlab                                            Created                                                                                  0.0s 
 ✔ Container kafka                                                 Created                                                                                  0.0s 
 ✔ Container spark-worker-1                                        Created                                                                                  0.0s 
 ✔ Container spark-worker-2                                        Created                                                                                  0.0s 
Attaching to jupyterlab, kafka, spark-master, spark-worker-1, spark-worker-2, zookeeper
zookeeper       | zookeeper 13:22:18.55 
zookeeper       | zookeeper 13:22:18.55 Welcome to the Bitnami zookeeper container
zookeeper       | zookeeper 13:22:18.56 Subscribe to project updates by watching https://github.com/bitnami/containers
zookeeper       | zookeeper 13:22:18.56 Submit issues and feature requests at https://github.com/bitnami/containers/issues
zookeeper       | zookeeper 13:22:18.56 
zookeeper       | zookeeper 13:22:18.57 INFO  ==> ** Starting ZooKeeper setup **
zookeeper       | zookeeper 13:22:18.60 WARN  ==> You have set the environment variable ALLOW_ANONYMOUS_LOGIN=yes. For safety reasons, do not use this flag in a production environment.
zookeeper       | zookeeper 13:22:18.62 INFO  ==> Initializing ZooKeeper...
zookeeper       | zookeeper 13:22:18.62 INFO  ==> No injected configuration file found, creating default config files...
zookeeper       | zookeeper 13:22:18.75 INFO  ==> No additional servers were specified. ZooKeeper will run in standalone mode...
zookeeper       | 
zookeeper       | zookeeper 13:22:18.76 INFO  ==> Deploying ZooKeeper from scratch...
zookeeper       | zookeeper 13:22:18.76 INFO  ==> ** ZooKeeper setup finished! **
zookeeper       | zookeeper 13:22:18.78 INFO  ==> ** Starting ZooKeeper **
zookeeper       | /opt/bitnami/java/bin/java
zookeeper       | ZooKeeper JMX enabled by default
zookeeper       | Using config: /opt/bitnami/zookeeper/bin/../conf/zoo.cfg
kafka           | kafka 13:22:18.96 INFO  ==> 
kafka           | kafka 13:22:18.96 INFO  ==> Welcome to the Bitnami kafka container
kafka           | kafka 13:22:18.96 INFO  ==> Subscribe to project updates by watching https://github.com/bitnami/containers
kafka           | kafka 13:22:18.97 INFO  ==> Submit issues and feature requests at https://github.com/bitnami/containers/issues
kafka           | kafka 13:22:18.97 INFO  ==> 
kafka           | kafka 13:22:18.97 INFO  ==> ** Starting Kafka setup **
kafka           | kafka 13:22:19.10 DEBUG ==> Validating settings in KAFKA_* env vars...
kafka           | kafka 13:22:19.11 WARN  ==> The KAFKA_ZOOKEEPER_PROTOCOL environment variable does not configure SASL and/or SSL, this setting is not recommended for production environments.
kafka           | kafka 13:22:19.15 WARN  ==> Kafka has been configured with a PLAINTEXT listener, this setting is not recommended for production environments.
kafka           | kafka 13:22:19.18 INFO  ==> Initializing Kafka...
kafka           | kafka 13:22:19.20 INFO  ==> No injected configuration files found, creating default config files
kafka           | kafka 13:22:19.45 INFO  ==> ** Kafka setup finished! **
kafka           | 
kafka           | kafka 13:22:19.47 INFO  ==> ** Starting Kafka **
jupyterlab      | [I 2025-02-01 13:22:19.649 ServerApp] jupyter_lsp | extension was successfully linked.
jupyterlab      | [I 2025-02-01 13:22:19.655 ServerApp] jupyter_server_terminals | extension was successfully linked.
......
jupyterlab      | [I 2025-02-01 13:22:20.111 ServerApp] Serving notebooks from local directory: /opt/workspace
jupyterlab      | [I 2025-02-01 13:22:20.112 ServerApp] Jupyter Server 2.15.0 is running at:
jupyterlab      | [I 2025-02-01 13:22:20.112 ServerApp] http://023938fcaaee:8888/lab
jupyterlab      | [I 2025-02-01 13:22:20.113 ServerApp]     http://127.0.0.1:8888/lab
jupyterlab      | [I 2025-02-01 13:22:20.113 ServerApp] Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).
jupyterlab      | [I 2025-02-01 13:22:20.572 ServerApp] Skipped non-installed server(s): bash-language-server, dockerfile-language-server-nodejs, javascript-typescript-langserver, jedi-language-server, julia-language-server, pyright, python-language-server, python-lsp-server, r-languageserver, sql-language-server, texlab, typescript-language-server, unified-language-server, vscode-css-languageserver-bin, vscode-html-languageserver-bin, vscode-json-languageserver-bin, yaml-language-server
zookeeper       | 2025-02-01 13:22:20,694 [myid:] - INFO  [main:o.a.z.s.q.QuorumPeerConfig@177] - Reading configuration from: /opt/bitnami/zookeeper/bin/../conf/zoo.cfg
zookeeper       | 2025-02-01 13:22:20,717 [myid:] - INFO  [main:o.a.z.s.q.QuorumPeerConfig@440] - clientPortAddress is 0.0.0.0:2181
......
zookeeper       | 2025-02-01 13:22:20,728 [myid:1] - INFO  [main:o.a.z.s.DatadirCleanupManager@78] - autopurge.snapRetainCount set to 3
zookeeper       | 2025-02-01 13:22:20,729 [myid:1] - INFO  [main:o.a.z.s.DatadirCleanupManager@79] - autopurge.purgeInterval set to 0
......
zookeeper       | 2025-02-01 13:22:20,735 [myid:1] - INFO  [main:o.a.z.j.ManagedUtil@46] - Log4j 1.2 jmx support not found; jmx disabled.
......
zookeeper       | 2025-02-01 13:22:20,757 [myid:1] - INFO  [main:o.a.z.s.ZooKeeperServerMain@123] - Starting server
zookeeper       | 2025-02-01 13:22:20,851 [myid:1] - INFO  [main:o.a.z.s.ServerMetrics@64] - ServerMetrics initialized with provider org.apache.zookeeper.metrics.impl.DefaultMetricsProvider@302c971f
zookeeper       | 2025-02-01 13:22:20,867 [myid:1] - INFO  [main:o.a.z.s.a.DigestAuthenticationProvider@47] - ACL digest algorithm is: SHA1
zookeeper       | 2025-02-01 13:22:20,867 [myid:1] - INFO  [main:o.a.z.s.a.DigestAuthenticationProvider@61] - zookeeper.DigestAuthenticationProvider.enabled = true
zookeeper       | 2025-02-01 13:22:20,892 [myid:1] - INFO  [main:o.a.z.s.p.FileTxnSnapLog@124] - zookeeper.snapshot.trust.empty : false
zookeeper       | 2025-02-01 13:22:20,991 [myid:1] - INFO  [main:o.a.z.ZookeeperBanner@42] - 
zookeeper       | 2025-02-01 13:22:20,992 [myid:1] - INFO  [main:o.a.z.ZookeeperBanner@42] -   ______                  _                                          
zookeeper       | 2025-02-01 13:22:20,994 [myid:1] - INFO  [main:o.a.z.ZookeeperBanner@42] -  |___  /                 | |                                         
zookeeper       | 2025-02-01 13:22:20,994 [myid:1] - INFO  [main:o.a.z.ZookeeperBanner@42] -     / /    ___     ___   | | __   ___    ___   _ __     ___   _ __   
zookeeper       | 2025-02-01 13:22:20,994 [myid:1] - INFO  [main:o.a.z.ZookeeperBanner@42] -    / /    / _ \   / _ \  | |/ /  / _ \  / _ \ | '_ \   / _ \ | '__|
zookeeper       | 2025-02-01 13:22:20,994 [myid:1] - INFO  [main:o.a.z.ZookeeperBanner@42] -   / /__  | (_) | | (_) | |   <  |  __/ |  __/ | |_) | |  __/ | |    
zookeeper       | 2025-02-01 13:22:20,994 [myid:1] - INFO  [main:o.a.z.ZookeeperBanner@42] -  /_____|  \___/   \___/  |_|\_\  \___|  \___| | .__/   \___| |_|
zookeeper       | 2025-02-01 13:22:20,994 [myid:1] - INFO  [main:o.a.z.ZookeeperBanner@42] -                                               | |                     
zookeeper       | 2025-02-01 13:22:20,994 [myid:1] - INFO  [main:o.a.z.ZookeeperBanner@42] -                                               |_|                     
zookeeper       | 2025-02-01 13:22:20,995 [myid:1] - INFO  [main:o.a.z.ZookeeperBanner@42] - 
zookeeper       | 2025-02-01 13:22:21,007 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:zookeeper.version=3.8.2-139d619b58292d7734b4fc83a0f44be4e7b0c986, built on 2023-07-05 19:24 UTC
zookeeper       | 2025-02-01 13:22:21,011 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:host.name=c945301d3b4c
zookeeper       | 2025-02-01 13:22:21,012 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:java.version=11.0.20
zookeeper       | 2025-02-01 13:22:21,012 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:java.vendor=BellSoft
zookeeper       | 2025-02-01 13:22:21,012 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:java.home=/opt/bitnami/java
zookeeper       | 2025-02-01 13:22:21,012 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:java.class.path=/opt/bitnami/zookeeper/bin/../zookeeper-server/target/classes:/opt/bitnami/zookeeper/bin/../build/classes:/opt/bitnami/zookeeper/bin/../zookeeper-server/target/lib/*.jar:/opt/bitnami/zookeeper/bin/../build/lib/*.jar:/opt/bitnami/zookeeper/bin/../lib/zookeeper-prometheus-metrics-3.8.2.jar:/opt/bitnami/zookeeper/bin/../lib/zookeeper-jute-3.8.2.jar:/opt/bitnami/zookeeper/bin/../lib/zookeeper-3.8.2.jar:/opt/bitnami/zookeeper/bin/../lib/snappy-java-1.1.10.1.jar:/opt/bitnami/zookeeper/bin/../lib/slf4j-api-1.7.30.jar:/opt/bitnami/zookeeper/bin/../lib/simpleclient_servlet-0.9.0.jar:/opt/bitnami/zookeeper/bin/../lib/simpleclient_hotspot-0.9.0.jar:/opt/bitnami/zookeeper/bin/../lib/simpleclient_common-0.9.0.jar:/opt/bitnami/zookeeper/bin/../lib/simpleclient-0.9.0.jar:/opt/bitnami/zookeeper/bin/../lib/netty-transport-native-unix-common-4.1.94.Final.jar:/opt/bitnami/zookeeper/bin/../lib/netty-transport-native-epoll-4.1.94.Final.jar:/opt/bitnami/zookeeper/bin/../lib/netty-transport-classes-epoll-4.1.94.Final.jar:/opt/bitnami/zookeeper/bin/../lib/netty-transport-4.1.94.Final.jar:/opt/bitnami/zookeeper/bin/../lib/netty-resolver-4.1.94.Final.jar:/opt/bitnami/zookeeper/bin/../lib/netty-handler-4.1.94.Final.jar:/opt/bitnami/zookeeper/bin/../lib/netty-common-4.1.94.Final.jar:/opt/bitnami/zookeeper/bin/../lib/netty-codec-4.1.94.Final.jar:/opt/bitnami/zookeeper/bin/../lib/netty-buffer-4.1.94.Final.jar:/opt/bitnami/zookeeper/bin/../lib/metrics-core-4.1.12.1.jar:/opt/bitnami/zookeeper/bin/../lib/logback-core-1.2.10.jar:/opt/bitnami/zookeeper/bin/../lib/logback-classic-1.2.10.jar:/opt/bitnami/zookeeper/bin/../lib/jline-2.14.6.jar:/opt/bitnami/zookeeper/bin/../lib/jetty-util-ajax-9.4.51.v20230217.jar:/opt/bitnami/zookeeper/bin/../lib/jetty-util-9.4.51.v20230217.jar:/opt/bitnami/zookeeper/bin/../lib/jetty-servlet-9.4.51.v20230217.jar:/opt/bitnami/zookeeper/bin/../lib/jetty-server-9.4.51.v20230217.jar:/opt/bitnami/zookeeper/bin/../lib/jetty-security-9.4.51.v20230217.jar:/opt/bitnami/zookeeper/bin/../lib/jetty-io-9.4.51.v20230217.jar:/opt/bitnami/zookeeper/bin/../lib/jetty-http-9.4.51.v20230217.jar:/opt/bitnami/zookeeper/bin/../lib/javax.servlet-api-3.1.0.jar:/opt/bitnami/zookeeper/bin/../lib/jackson-databind-2.15.2.jar:/opt/bitnami/zookeeper/bin/../lib/jackson-core-2.15.2.jar:/opt/bitnami/zookeeper/bin/../lib/jackson-annotations-2.15.2.jar:/opt/bitnami/zookeeper/bin/../lib/commons-io-2.11.0.jar:/opt/bitnami/zookeeper/bin/../lib/commons-cli-1.5.0.jar:/opt/bitnami/zookeeper/bin/../lib/audience-annotations-0.12.0.jar:/opt/bitnami/zookeeper/bin/../zookeeper-*.jar:/opt/bitnami/zookeeper/bin/../zookeeper-server/src/main/resources/lib/*.jar:/opt/bitnami/zookeeper/bin/../conf:
zookeeper       | 2025-02-01 13:22:21,012 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:java.library.path=/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib
zookeeper       | 2025-02-01 13:22:21,012 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:java.io.tmpdir=/tmp
zookeeper       | 2025-02-01 13:22:21,012 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:java.compiler=<NA>
zookeeper       | 2025-02-01 13:22:21,012 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:os.name=Linux
zookeeper       | 2025-02-01 13:22:21,012 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:os.arch=amd64
zookeeper       | 2025-02-01 13:22:21,012 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:os.version=6.8.0-52-generic
zookeeper       | 2025-02-01 13:22:21,012 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:user.name=?
zookeeper       | 2025-02-01 13:22:21,015 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:user.home=?
zookeeper       | 2025-02-01 13:22:21,016 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:user.dir=/
zookeeper       | 2025-02-01 13:22:21,017 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:os.memory.free=1005MB
zookeeper       | 2025-02-01 13:22:21,019 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:os.memory.max=1024MB
zookeeper       | 2025-02-01 13:22:21,023 [myid:1] - INFO  [main:o.a.z.Environment@98] - Server environment:os.memory.total=1024MB
zookeeper       | 2025-02-01 13:22:21,023 [myid:1] - INFO  [main:o.a.z.s.ZooKeeperServer@140] - zookeeper.enableEagerACLCheck = false
zookeeper       | 2025-02-01 13:22:21,023 [myid:1] - INFO  [main:o.a.z.s.ZooKeeperServer@153] - zookeeper.digest.enabled = true
......
zookeeper       | 2025-02-01 13:22:21,047 [myid:1] - INFO  [main:o.a.z.s.BlueThrottle@141] - Weighed connection throttling is disabled
zookeeper       | 2025-02-01 13:22:21,057 [myid:1] - INFO  [main:o.a.z.s.ZooKeeperServer@1311] - minSessionTimeout set to 4000 ms
zookeeper       | 2025-02-01 13:22:21,061 [myid:1] - INFO  [main:o.a.z.s.ZooKeeperServer@1320] - maxSessionTimeout set to 40000 ms
zookeeper       | 2025-02-01 13:22:21,074 [myid:1] - INFO  [main:o.a.z.s.ResponseCache@45] - getData response cache size is initialized with value 400.
zookeeper       | 2025-02-01 13:22:21,074 [myid:1] - INFO  [main:o.a.z.s.ResponseCache@45] - getChildren response cache size is initialized with value 400.
zookeeper       | 2025-02-01 13:22:21,076 [myid:1] - INFO  [main:o.a.z.s.u.RequestPathMetricsCollector@109] - zookeeper.pathStats.slotCapacity = 60
zookeeper       | 2025-02-01 13:22:21,079 [myid:1] - INFO  [main:o.a.z.s.u.RequestPathMetricsCollector@110] - zookeeper.pathStats.slotDuration = 15
......
zookeeper       | 2025-02-01 13:22:21,097 [myid:1] - INFO  [main:o.a.z.s.AuthenticationHelper@66] - zookeeper.enforce.auth.enabled = false
zookeeper       | 2025-02-01 13:22:21,097 [myid:1] - INFO  [main:o.a.z.s.AuthenticationHelper@67] - zookeeper.enforce.auth.schemes = []
zookeeper       | 2025-02-01 13:22:21,097 [myid:1] - INFO  [main:o.a.z.s.ZooKeeperServer@376] - Created server with tickTime 2000 ms minSessionTimeout 4000 ms maxSessionTimeout 40000 ms clientPortListenBacklog -1 datadir /bitnami/zookeeper/data/version-2 snapdir /bitnami/zookeeper/data/version-2
zookeeper       | 2025-02-01 13:22:21,200 [myid:1] - INFO  [main:o.e.j.u.l.Log@170] - Logging initialized @2322ms to org.eclipse.jetty.util.log.Slf4jLog
zookeeper       | 2025-02-01 13:22:21,464 [myid:1] - WARN  [main:o.e.j.s.h.ContextHandler@1662] - o.e.j.s.ServletContextHandler@4ddbbdf8{/,null,STOPPED} contextPath ends with /*
zookeeper       | 2025-02-01 13:22:21,466 [myid:1] - WARN  [main:o.e.j.s.h.ContextHandler@1673] - Empty contextPath
zookeeper       | 2025-02-01 13:22:21,522 [myid:1] - INFO  [main:o.e.j.s.Server@375] - jetty-9.4.51.v20230217; built: 2023-02-17T08:19:37.309Z; git: b45c405e4544384de066f814ed42ae3dceacdd49; jvm 11.0.20+8-LTS
zookeeper       | 2025-02-01 13:22:21,612 [myid:1] - INFO  [main:o.e.j.s.s.DefaultSessionIdManager@334] - DefaultSessionIdManager workerName=node0
zookeeper       | 2025-02-01 13:22:21,614 [myid:1] - INFO  [main:o.e.j.s.s.DefaultSessionIdManager@339] - No SessionScavenger set, using defaults
zookeeper       | 2025-02-01 13:22:21,623 [myid:1] - INFO  [main:o.e.j.s.s.HouseKeeper@132] - node0 Scavenging every 660000ms
zookeeper       | 2025-02-01 13:22:21,653 [myid:1] - WARN  [main:o.e.j.s.ConstraintSecurityHandler@759] - ServletContext@o.e.j.s.ServletContextHandler@4ddbbdf8{/,null,STARTING} has uncovered http methods for path: /*
zookeeper       | 2025-02-01 13:22:21,703 [myid:1] - INFO  [main:o.e.j.s.h.ContextHandler@921] - Started o.e.j.s.ServletContextHandler@4ddbbdf8{/,null,AVAILABLE}
zookeeper       | 2025-02-01 13:22:21,758 [myid:1] - INFO  [main:o.e.j.s.AbstractConnector@333] - Started ServerConnector@3e2059ae{HTTP/1.1, (http/1.1)}{0.0.0.0:8080}
zookeeper       | 2025-02-01 13:22:21,761 [myid:1] - INFO  [main:o.e.j.s.Server@415] - Started @2882ms
zookeeper       | 2025-02-01 13:22:21,762 [myid:1] - INFO  [main:o.a.z.s.a.JettyAdminServer@196] - Started AdminServer on address 0.0.0.0, port 8080 and command URL /commands
zookeeper       | 2025-02-01 13:22:21,777 [myid:1] - INFO  [main:o.a.z.s.ServerCnxnFactory@169] - Using org.apache.zookeeper.server.NIOServerCnxnFactory as server connection factory
zookeeper       | 2025-02-01 13:22:21,781 [myid:1] - WARN  [main:o.a.z.s.ServerCnxnFactory@309] - maxCnxns is not configured, using default value 0.
zookeeper       | 2025-02-01 13:22:21,787 [myid:1] - INFO  [main:o.a.z.s.NIOServerCnxnFactory@652] - Configuring NIO connection handler with 10s sessionless connection timeout, 1 selector thread(s), 12 worker threads, and 64 kB direct buffers.
zookeeper       | 2025-02-01 13:22:21,792 [myid:1] - INFO  [main:o.a.z.s.NIOServerCnxnFactory@660] - binding to port 0.0.0.0/0.0.0.0:2181
zookeeper       | 2025-02-01 13:22:21,852 [myid:1] - INFO  [main:o.a.z.s.w.WatchManagerFactory@42] - Using org.apache.zookeeper.server.watch.WatchManager as watch manager
zookeeper       | 2025-02-01 13:22:21,861 [myid:1] - INFO  [main:o.a.z.s.w.WatchManagerFactory@42] - Using org.apache.zookeeper.server.watch.WatchManager as watch manager
zookeeper       | 2025-02-01 13:22:21,862 [myid:1] - INFO  [main:o.a.z.s.ZKDatabase@132] - zookeeper.snapshotSizeFactor = 0.33
zookeeper       | 2025-02-01 13:22:21,863 [myid:1] - INFO  [main:o.a.z.s.ZKDatabase@152] - zookeeper.commitLogCount=500
zookeeper       | 2025-02-01 13:22:21,896 [myid:1] - INFO  [main:o.a.z.s.p.SnapStream@61] - zookeeper.snapshot.compression.method = CHECKED
zookeeper       | 2025-02-01 13:22:21,896 [myid:1] - INFO  [main:o.a.z.s.p.FileTxnSnapLog@479] - Snapshotting: 0x0 to /bitnami/zookeeper/data/version-2/snapshot.0
zookeeper       | 2025-02-01 13:22:21,927 [myid:1] - INFO  [main:o.a.z.s.ZKDatabase@289] - Snapshot loaded in 64 ms, highest zxid is 0x0, digest is 1371985504
zookeeper       | 2025-02-01 13:22:21,930 [myid:1] - INFO  [main:o.a.z.s.p.FileTxnSnapLog@479] - Snapshotting: 0x0 to /bitnami/zookeeper/data/version-2/snapshot.0
zookeeper       | 2025-02-01 13:22:21,931 [myid:1] - INFO  [main:o.a.z.s.ZooKeeperServer@558] - Snapshot taken in 1 ms
zookeeper       | 2025-02-01 13:22:21,977 [myid:] - INFO  [ProcessThread(sid:0 cport:2181)::o.a.z.s.PrepRequestProcessor@138] - PrepRequestProcessor (sid:0) started, reconfigEnabled=false
zookeeper       | 2025-02-01 13:22:21,982 [myid:1] - INFO  [main:o.a.z.s.RequestThrottler@75] - zookeeper.request_throttler.shutdownTimeout = 10000 ms
zookeeper       | 2025-02-01 13:22:22,100 [myid:1] - INFO  [main:o.a.z.s.ContainerManager@84] - Using checkIntervalMs=60000 maxPerMinute=10000 maxNeverUsedIntervalMs=0
zookeeper       | 2025-02-01 13:22:22,109 [myid:1] - INFO  [main:o.a.z.a.ZKAuditProvider@42] - ZooKeeper audit is disabled.
spark-worker-1  | Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties
spark-worker-2  | Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties
spark-worker-1  | 25/02/01 13:22:22 INFO Worker: Started daemon with process name: 7@4af40e65ff3f
spark-worker-2  | 25/02/01 13:22:22 INFO Worker: Started daemon with process name: 7@a3d734f59d1f
spark-worker-1  | 25/02/01 13:22:22 INFO SignalUtils: Registering signal handler for TERM
spark-worker-1  | 25/02/01 13:22:22 INFO SignalUtils: Registering signal handler for HUP
spark-worker-1  | 25/02/01 13:22:22 INFO SignalUtils: Registering signal handler for INT
spark-worker-2  | 25/02/01 13:22:22 INFO SignalUtils: Registering signal handler for TERM
spark-worker-2  | 25/02/01 13:22:22 INFO SignalUtils: Registering signal handler for HUP
spark-worker-2  | 25/02/01 13:22:22 INFO SignalUtils: Registering signal handler for INT
kafka           | [2025-02-01 13:22:22,334] INFO Registered kafka:type=kafka.Log4jController MBean (kafka.utils.Log4jControllerRegistration$)
spark-master    | Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties
spark-master    | 25/02/01 13:22:22 INFO Master: Started daemon with process name: 7@ec044672a5ae
spark-master    | 25/02/01 13:22:22 INFO SignalUtils: Registering signal handler for TERM
spark-master    | 25/02/01 13:22:22 INFO SignalUtils: Registering signal handler for HUP
spark-master    | 25/02/01 13:22:22 INFO SignalUtils: Registering signal handler for INT
spark-worker-1  | 25/02/01 13:22:23 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
kafka           | [2025-02-01 13:22:23,155] INFO Setting -D jdk.tls.rejectClientInitiatedRenegotiation=true to disable client-initiated TLS renegotiation (org.apache.zookeeper.common.X509Util)
spark-worker-2  | 25/02/01 13:22:23 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
spark-master    | 25/02/01 13:22:23 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
spark-worker-1  | 25/02/01 13:22:23 INFO SecurityManager: Changing view acls to: NBuser
spark-worker-1  | 25/02/01 13:22:23 INFO SecurityManager: Changing modify acls to: NBuser
spark-worker-1  | 25/02/01 13:22:23 INFO SecurityManager: Changing view acls groups to: 
spark-worker-1  | 25/02/01 13:22:23 INFO SecurityManager: Changing modify acls groups to: 
spark-worker-1  | 25/02/01 13:22:23 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: NBuser; groups with view permissions: EMPTY; users with modify permissions: NBuser; groups with modify permissions: EMPTY
spark-worker-2  | 25/02/01 13:22:23 INFO SecurityManager: Changing view acls to: NBuser
spark-worker-2  | 25/02/01 13:22:23 INFO SecurityManager: Changing modify acls to: NBuser
spark-worker-2  | 25/02/01 13:22:23 INFO SecurityManager: Changing view acls groups to: 
spark-worker-2  | 25/02/01 13:22:23 INFO SecurityManager: Changing modify acls groups to: 
spark-worker-2  | 25/02/01 13:22:23 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: NBuser; groups with view permissions: EMPTY; users with modify permissions: NBuser; groups with modify permissions: EMPTY
kafka           | [2025-02-01 13:22:23,529] INFO Registered signal handlers for TERM, INT, HUP (org.apache.kafka.common.utils.LoggingSignalHandler)
kafka           | [2025-02-01 13:22:23,535] INFO starting (kafka.server.KafkaServer)
kafka           | [2025-02-01 13:22:23,538] INFO Connecting to zookeeper on zookeeper:2181 (kafka.server.KafkaServer)
spark-master    | 25/02/01 13:22:23 INFO SecurityManager: Changing view acls to: NBuser
spark-master    | 25/02/01 13:22:23 INFO SecurityManager: Changing modify acls to: NBuser
spark-master    | 25/02/01 13:22:23 INFO SecurityManager: Changing view acls groups to: 
spark-master    | 25/02/01 13:22:23 INFO SecurityManager: Changing modify acls groups to: 
spark-master    | 25/02/01 13:22:23 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: NBuser; groups with view permissions: EMPTY; users with modify permissions: NBuser; groups with modify permissions: EMPTY
kafka           | [2025-02-01 13:22:23,614] INFO [ZooKeeperClient Kafka server] Initializing a new session to zookeeper:2181. (kafka.zookeeper.ZooKeeperClient)
kafka           | [2025-02-01 13:22:23,648] INFO Client environment:zookeeper.version=3.6.4--d65253dcf68e9097c6e95a126463fd5fdeb4521c, built on 12/18/2022 18:10 GMT (org.apache.zookeeper.ZooKeeper)
kafka           | [2025-02-01 13:22:23,648] INFO Client environment:host.name=85922d0be45e (org.apache.zookeeper.ZooKeeper)
......
kafka           | [2025-02-01 13:22:23,791] INFO Opening socket connection to server zookeeper/172.18.0.2:2181. (org.apache.zookeeper.ClientCnxn)
kafka           | [2025-02-01 13:22:23,831] INFO Socket connection established, initiating session, client: /172.18.0.5:40864, server: zookeeper/172.18.0.2:2181 (org.apache.zookeeper.ClientCnxn)
zookeeper       | 2025-02-01 13:22:23,909 [myid:] - INFO  [SyncThread:0:o.a.z.s.p.FileTxnLog@285] - Creating new log file: log.1
kafka           | [2025-02-01 13:22:23,982] INFO Session establishment complete on server zookeeper/172.18.0.2:2181, session id = 0x100028f84150000, negotiated timeout = 18000 (org.apache.zookeeper.ClientCnxn)
kafka           | [2025-02-01 13:22:23,992] INFO [ZooKeeperClient Kafka server] Connected. (kafka.zookeeper.ZooKeeperClient)
spark-master    | 25/02/01 13:22:24 INFO Utils: Successfully started service 'sparkMaster' on port 7077.
spark-worker-1  | 25/02/01 13:22:24 INFO Utils: Successfully started service 'sparkWorker' on port 33353.
spark-worker-1  | 25/02/01 13:22:24 INFO Worker: Worker decommissioning not enabled.
spark-worker-2  | 25/02/01 13:22:24 INFO Utils: Successfully started service 'sparkWorker' on port 38881.
spark-worker-2  | 25/02/01 13:22:24 INFO Worker: Worker decommissioning not enabled.
spark-master    | 25/02/01 13:22:24 INFO Master: Starting Spark master at spark://spark-master:7077
spark-master    | 25/02/01 13:22:24 INFO Master: Running Spark version 3.4.1
spark-master    | 25/02/01 13:22:24 INFO JettyUtils: Start Jetty 0.0.0.0:8080 for MasterUI
kafka           | [2025-02-01 13:22:24,784] INFO Cluster ID = jWv5ooWdQ8eLKNH-rri-UA (kafka.server.KafkaServer)
kafka           | [2025-02-01 13:22:24,796] WARN No meta.properties file under dir /bitnami/kafka/data/meta.properties (kafka.server.BrokerMetadataCheckpoint)
spark-worker-1  | 25/02/01 13:22:24 INFO Worker: Starting Spark worker 172.18.0.6:33353 with 1 cores, 512.0 MiB RAM
spark-worker-2  | 25/02/01 13:22:24 INFO Worker: Starting Spark worker 172.18.0.7:38881 with 1 cores, 512.0 MiB RAM
spark-worker-2  | 25/02/01 13:22:24 INFO Worker: Running Spark version 3.4.1
spark-master    | 25/02/01 13:22:24 INFO Utils: Successfully started service 'MasterUI' on port 8080.
spark-worker-2  | 25/02/01 13:22:24 INFO Worker: Spark home: /usr/bin/spark-3.4.1-bin-hadoop3
spark-worker-1  | 25/02/01 13:22:24 INFO Worker: Running Spark version 3.4.1
spark-worker-1  | 25/02/01 13:22:24 INFO Worker: Spark home: /usr/bin/spark-3.4.1-bin-hadoop3
kafka           | [2025-02-01 13:22:24,959] INFO KafkaConfig values: 
kafka           |       advertised.listeners = PLAINTEXT://kafka:9092
......
kafka           |       transaction.state.log.num.partitions = 50
kafka           |       transaction.state.log.replication.factor = 1
kafka           |       transaction.state.log.segment.bytes = 104857600
kafka           |       transactional.id.expiration.ms = 604800000
kafka           |       unclean.leader.election.enable = false
kafka           |       unstable.api.versions.enable = false
......
kafka           |       zookeeper.ssl.endpoint.identification.algorithm = HTTPS
......
kafka           |  (kafka.server.KafkaConfig)
spark-worker-2  | 25/02/01 13:22:25 INFO ResourceUtils: ==============================================================
spark-worker-2  | 25/02/01 13:22:25 INFO ResourceUtils: No custom resources configured for spark.worker.
spark-worker-2  | 25/02/01 13:22:25 INFO ResourceUtils: ==============================================================
spark-worker-1  | 25/02/01 13:22:25 INFO ResourceUtils: ==============================================================
spark-worker-1  | 25/02/01 13:22:25 INFO ResourceUtils: No custom resources configured for spark.worker.
spark-worker-1  | 25/02/01 13:22:25 INFO ResourceUtils: ==============================================================
spark-master    | 25/02/01 13:22:25 INFO MasterWebUI: Bound MasterWebUI to 0.0.0.0, and started at http://ec044672a5ae:8080
spark-worker-1  | 25/02/01 13:22:25 INFO JettyUtils: Start Jetty 0.0.0.0:8081 for WorkerUI
spark-worker-2  | 25/02/01 13:22:25 INFO JettyUtils: Start Jetty 0.0.0.0:8081 for WorkerUI
kafka           | [2025-02-01 13:22:25,283] INFO [ThrottledChannelReaper-Fetch]: Starting (kafka.server.ClientQuotaManager$ThrottledChannelReaper)
kafka           | [2025-02-01 13:22:25,293] INFO [ThrottledChannelReaper-Produce]: Starting (kafka.server.ClientQuotaManager$ThrottledChannelReaper)
kafka           | [2025-02-01 13:22:25,297] INFO [ThrottledChannelReaper-Request]: Starting (kafka.server.ClientQuotaManager$ThrottledChannelReaper)
kafka           | [2025-02-01 13:22:25,308] INFO [ThrottledChannelReaper-ControllerMutation]: Starting (kafka.server.ClientQuotaManager$ThrottledChannelReaper)
spark-worker-1  | 25/02/01 13:22:25 INFO Utils: Successfully started service 'WorkerUI' on port 8081.
kafka           | [2025-02-01 13:22:25,417] INFO Loading logs from log dirs ArrayBuffer(/bitnami/kafka/data) (kafka.log.LogManager)
kafka           | [2025-02-01 13:22:25,431] INFO No logs found to be loaded in /bitnami/kafka/data (kafka.log.LogManager)
spark-worker-2  | 25/02/01 13:22:25 INFO Utils: Successfully started service 'WorkerUI' on port 8081.
kafka           | [2025-02-01 13:22:25,481] INFO Loaded 0 logs in 63ms (kafka.log.LogManager)
kafka           | [2025-02-01 13:22:25,493] INFO Starting log cleanup with a period of 300000 ms. (kafka.log.LogManager)
kafka           | [2025-02-01 13:22:25,508] INFO Starting log flusher with a default period of 9223372036854775807 ms. (kafka.log.LogManager)
spark-worker-1  | 25/02/01 13:22:25 INFO WorkerWebUI: Bound WorkerWebUI to 0.0.0.0, and started at http://4af40e65ff3f:8081
spark-worker-1  | 25/02/01 13:22:25 INFO Worker: Connecting to master spark-master:7077...
kafka           | [2025-02-01 13:22:25,571] INFO Starting the log cleaner (kafka.log.LogCleaner)
spark-master    | 25/02/01 13:22:25 INFO Master: I have been elected leader! New state: ALIVE
spark-worker-2  | 25/02/01 13:22:25 INFO WorkerWebUI: Bound WorkerWebUI to 0.0.0.0, and started at http://a3d734f59d1f:8081
spark-worker-2  | 25/02/01 13:22:25 INFO Worker: Connecting to master spark-master:7077...
kafka           | [2025-02-01 13:22:25,714] INFO [kafka-log-cleaner-thread-0]: Starting (kafka.log.LogCleaner$CleanerThread)
kafka           | [2025-02-01 13:22:25,768] INFO [feature-zk-node-event-process-thread]: Starting (kafka.server.FinalizedFeatureChangeListener$ChangeNotificationProcessorThread)
spark-worker-1  | 25/02/01 13:22:25 INFO TransportClientFactory: Successfully created connection to spark-master/172.18.0.4:7077 after 124 ms (2 ms spent in bootstraps)
spark-worker-2  | 25/02/01 13:22:25 INFO TransportClientFactory: Successfully created connection to spark-master/172.18.0.4:7077 after 81 ms (0 ms spent in bootstraps)
kafka           | [2025-02-01 13:22:25,833] INFO Feature ZK node at path: /feature does not exist (kafka.server.FinalizedFeatureChangeListener)
kafka           | [2025-02-01 13:22:25,889] INFO [zk-broker-1-to-controller-forwarding-channel-manager]: Starting (kafka.server.BrokerToControllerRequestThread)
spark-master    | 25/02/01 13:22:25 INFO Master: Registering worker 172.18.0.6:33353 with 1 cores, 512.0 MiB RAM
spark-master    | 25/02/01 13:22:25 INFO Master: Registering worker 172.18.0.7:38881 with 1 cores, 512.0 MiB RAM
spark-worker-1  | 25/02/01 13:22:26 INFO Worker: Successfully registered with master spark://spark-master:7077
spark-worker-2  | 25/02/01 13:22:26 INFO Worker: Successfully registered with master spark://spark-master:7077
kafka           | [2025-02-01 13:22:26,236] INFO Updated connection-accept-rate max connection creation rate to 2147483647 (kafka.network.ConnectionQuotas)
kafka           | [2025-02-01 13:22:26,251] INFO [SocketServer listenerType=ZK_BROKER, nodeId=1] Created data-plane acceptor and processors for endpoint : ListenerName(PLAINTEXT) (kafka.network.SocketServer)
......
kafka           | [2025-02-01 13:22:26,604] WARN [RequestSendThread controllerId=1] Controller 1's connection to broker kafka:9092 (id: 1 rack: null) was unsuccessful (kafka.controller.RequestSendThread)
kafka           | java.io.IOException: Connection to kafka:9092 (id: 1 rack: null) failed.
kafka           |       at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:70)
kafka           |       at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298)
kafka           |       at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251)
kafka           |       at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:127)
kafka           | [2025-02-01 13:22:26,610] INFO [Controller id=1, targetBrokerId=1] Client requested connection close from node 1 (org.apache.kafka.clients.NetworkClient)
kafka           | [2025-02-01 13:22:26,622] INFO Kafka version: 3.5.1 (org.apache.kafka.common.utils.AppInfoParser)
kafka           | [2025-02-01 13:22:26,622] INFO Kafka commitId: 2c6fb6c54472e90a (org.apache.kafka.common.utils.AppInfoParser)
kafka           | [2025-02-01 13:22:26,622] INFO Kafka startTimeMs: 1738416146615 (org.apache.kafka.common.utils.AppInfoParser)
kafka           | [2025-02-01 13:22:26,626] INFO [KafkaServer id=1] started (kafka.server.KafkaServer)
kafka           | [2025-02-01 13:22:26,626] INFO [Controller id=1] Starting the controller scheduler (kafka.controller.KafkaController)
kafka           | [2025-02-01 13:22:26,712] INFO [RequestSendThread controllerId=1] Controller 1 connected to kafka:9092 (id: 1 rack: null) for sending state change requests (kafka.controller.RequestSendThread)
kafka           | [2025-02-01 13:22:26,759] INFO [zk-broker-1-to-controller-alter-partition-channel-manager]: Recorded new controller, from now on will use node kafka:9092 (id: 1 rack: null) (kafka.server.BrokerToControllerRequestThread)
kafka           | [2025-02-01 13:22:26,812] INFO [zk-broker-1-to-controller-forwarding-channel-manager]: Recorded new controller, from now on will use node kafka:9092 (id: 1 rack: null) (kafka.server.BrokerToControllerRequestThread)
kafka           | [2025-02-01 13:22:31,627] INFO [Controller id=1] Processing automatic preferred replica leader election (kafka.controller.KafkaController)
kafka           | [2025-02-01 13:22:31,628] TRACE [Controller id=1] Checking need to trigger auto leader balancing (kafka.controller.KafkaController)

 

posted on 2025-02-01 20:24  ZhangZhihuiAAA  阅读(21)  评论(0)    收藏  举报
 
博客园  ©  2004-2025
浙公网安备 33010602011771号 浙ICP备2021040463号-3