Kafka集群搭建

1. 启动Server

http://kafka.apache.org/quickstart

For Kafka, a single broker is just a cluster of size one, so nothing much changes other than starting a few more broker instances.

# 下载kafka
[root@localhost kafka]# tar -xzf kafka_2.11-2.0.0.tgz
[root@localhost kafka]# cd kafka_2.11-2.0.0
[root@localhost kafka_2.11-2.0.0]# pwd
/usr/local/programs/kafka/kafka_2.11-2.0.0
[root@localhost kafka_2.11-2.0.0]# ls
bin  config  libs  LICENSE  logs  NOTICE  site-docs
[root@localhost kafka_2.11-2.0.0]#

# 启动zookeeper
[root@localhost kafka_2.11-2.0.0]# bin/zookeeper-server-start.sh config/zookeeper.properties &

# 启动一个Kafka Server
[root@localhost kafka_2.11-2.0.0]# bin/kafka-server-start.sh config/server.properties

# 创建包含多个broker（Kafka Server）的集群
[root@localhost kafka_2.11-2.0.0]# cp config/server.properties config/server-1.properties
[root@localhost kafka_2.11-2.0.0]# cp config/server.properties config/server-2.properties
[root@localhost kafka_2.11-2.0.0]# vim config/server-1.properties
    broker.id=1
    listeners=PLAINTEXT://:9093
    log.dirs=/tmp/kafka-logs-1
    zookeeper.connect=localhost:2181
[root@localhost kafka_2.11-2.0.0]# vim config/server-2.properties
    broker.id=2
    listeners=PLAINTEXT://:9094
    log.dirs=/tmp/kafka-logs-2
    zookeeper.connect=localhost:2181
[root@localhost kafka_2.11-2.0.0]# bin/kafka-server-start.sh config/server-1.properties &
[root@localhost kafka_2.11-2.0.0]# bin/kafka-server-start.sh config/server-2.properties &
[root@localhost kafka_2.11-2.0.0]# netstat -tnlp | grep 909
tcp6       0      0 :::9092                 :::*                    LISTEN      18663/java          
tcp6       0      0 :::9093                 :::*                    LISTEN      18998/java          
tcp6       0      0 :::9094                 :::*                    LISTEN      19332/java

# 创建一个主题
[root@localhost kafka_2.11-2.0.0]# bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test

# 查看主题
[root@localhost kafka_2.11-2.0.0]# bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic test
Topic:test      PartitionCount:1        ReplicationFactor:3     Configs:
        Topic: test     Partition: 0    Leader: 2       Replicas: 2,0,1 Isr: 2,1,0

# 发送消息
[root@localhost kafka_2.11-2.0.0]# bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test
>Hello World
>Hello Java
>^C

# 接收消息
[root@localhost kafka_2.11-2.0.0]# bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --from-beginning --topic test
Hello World
Hello Java

2. 安装JDK

http://www.oracle.com/technetwork/java/javase/downloads/index.html

3. Kafka Eagle

http://download.smartloli.org/

https://ke.smartloli.org/2.Install/2.Installing.html

https://ke.smartloli.org/3.Manuals/1.Dashboard.html

3.1. 设置KE_HOME环境变量

3.2. 修改配置文件

[root@localhost conf]# vim $KE_HOME/conf/system-config.properties

######################################
# multi zookeeper&kafka cluster list
######################################
kafka.eagle.zk.cluster.alias=cluster1
cluster1.zk.list=localhost:2181
#cluster2.zk.list=xdn10:2181,xdn11:2181,xdn12:2181

######################################
# zk client thread limit
######################################
kafka.zk.limit.size=25

######################################
# kafka eagle webui port
######################################
kafka.eagle.webui.port=8048

######################################
# kafka offset storage
######################################
kafka.eagle.offset.storage=kafka

######################################
# enable kafka metrics
######################################
kafka.eagle.metrics.charts=false

######################################
# alarm email configure
######################################
kafka.eagle.mail.enable=true
kafka.eagle.mail.sa=alert_sa
kafka.eagle.mail.username=alert_sa@163.com
kafka.eagle.mail.password=mqslimczkdqabbbg
kafka.eagle.mail.server.host=smtp.163.com
kafka.eagle.mail.server.port=25

######################################
# delete kafka topic token
######################################
kafka.eagle.topic.token=keadmin

######################################
# kafka sasl authenticate
######################################
kafka.eagle.sasl.enable=false
kafka.eagle.sasl.protocol=SASL_PLAINTEXT
kafka.eagle.sasl.mechanism=PLAIN

######################################
# kafka jdbc driver address
######################################
kafka.eagle.driver=com.mysql.jdbc.Driver
kafka.eagle.url=jdbc:mysql://10.123.52.189:3306/oh_coupon
kafka.eagle.username=devdb
kafka.eagle.password=d^V$0Fu!/6-<

3.3. 启动Eagle

[root@localhost kafka-eagle-web-1.2.3]# cd bin/
[root@localhost bin]# ls
ke.bat  ke.pid  ke.sh
[root@localhost bin]# chmod +x ke.sh
[root@localhost bin]# ./ke.sh start

4. 演示

http://192.168.101.5:8048/ke/