python连接Kafka实例

python依赖环境

sudo pip install kafka-python

如果没有 pip 命令，请安装 sudo yum install -y epel-release python2-pip

consumer服务

import time
from kafka import KafkaConsumer

config = {
    "topic": "test01",
    "groupID": "test",
    "username": "ops",
    "password": "ops-secret",
    'servers': ["kafka01:9092","kafka02:9092","kafka03:9092"]
}

consumer = KafkaConsumer(config["topic"],
                         group_id = config["groupID"],
                         bootstrap_servers = config["servers"],
                         security_protocol = "SASL_PLAINTEXT",
                         sasl_mechanism="SCRAM-SHA-512",
                         sasl_plain_username = config["username"],
                         sasl_plain_password = config["password"],
                         enable_auto_commit = False)

for msg in consumer:
    print("topic: {}, partition: {}, key: {}, value: {}, offset: {}".format(msg.topic, msg.partition, msg.key, msg.value, msg.offset))
    time.sleep(0.5)
    consumer.commit()

producer服务

import json
import time
import datetime
from kafka import KafkaProducer

config = {
    'username': "ops",
    'password': "ops-secret",
    'topic': "test01",
    'retries': 3,
    'batchSize': 1000,
    'bufferMemory': 33554432,
    'servers': ["kafka01:9092","kafka02:9092","kafka03:9092"]
}

producer = KafkaProducer(bootstrap_servers = config["servers"],
                         security_protocol = "SASL_PLAINTEXT",
                         sasl_mechanism = "SCRAM-SHA-512",
                         sasl_plain_username = config["username"],
                         sasl_plain_password = config["password"],
                         value_serializer=lambda m: json.dumps(m).encode(),
                         retries = config["retries"],
                         batch_size = config["batchSize"],
                         buffer_memory = config["bufferMemory"])

i = 1
while True:
    time.sleep(0.1)
    data = {'num': i, 'ts': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
    future = producer.send(config["topic"], data)
    result = future.get(timeout = 10)
    print("topic: {}, partition: {}, offset: {}, data: {}".format(result.topic, result.partition, result.offset, data))
    i += 1