librdkafka源码解析 Consumer
消息体定义：（消费者消息）
包含topic,partition,value(*key),消息长度,offset
typedef struct rd_kafka_message_s { rd_kafka_resp_err_t err; /**< Non-zero for error signaling. */ rd_kafka_topic_t *rkt; /**< Topic */ int32_t partition; /**< Partition */ void *payload; /**< Producer: original message payload. * Consumer: Depends on the value of \c err : * - \c err==0: Message payload. * - \c err!=0: Error string */ size_t len; /**< Depends on the value of \c err : * - \c err==0: Message payload length * - \c err!=0: Error string length */ void *key; /**< Depends on the value of \c err : * - \c err==0: Optional message key */ size_t key_len; /**< Depends on the value of \c err : * - \c err==0: Optional message key length*/ int64_t offset; /**< Consumer: * - Message offset (or offset for error * if \c err!=0 if applicable). * Producer, dr_msg_cb: * Message offset assigned by broker. * May be RD_KAFKA_OFFSET_INVALID * for retried messages when * idempotence is enabled. */ void *_private; /**< Consumer: * - rdkafka private pointer: DO NOT MODIFY * Producer: * - dr_msg_cb: * msg_opaque from produce() call or * RD_KAFKA_V_OPAQUE from producev(). */ } rd_kafka_message_t;
typedef struct rd_kafka_q_s rd_kafka_q_t; typedef struct rd_kafka_toppar_s rd_kafka_toppar_t; typedef struct rd_kafka_op_s rd_kafka_op_t;
/** * @struct Queue for rd_kafka_op_t*. * * @remark All readers of the queue must call rd_kafka_q_mark_served() * after reading the queue (while still holding the queue lock) to * clear the wakeup-sent flag. */ struct rd_kafka_q_s { mtx_t rkq_lock; cnd_t rkq_cond; struct rd_kafka_q_s *rkq_fwdq; /* Forwarded/Routed queue. * Used in place of this queue * for all operations. */ struct rd_kafka_op_tailq rkq_q; /* TAILQ_HEAD(, rd_kafka_op_s) */ int rkq_qlen; /* Number of entries in queue */ int64_t rkq_qsize; /* Size of all entries in queue */ int rkq_refcnt; int rkq_flags; #define RD_KAFKA_Q_F_ALLOCATED 0x1 /* Allocated: rd_free on destroy */ #define RD_KAFKA_Q_F_READY \ 0x2 /* Queue is ready to be used. \ * Flag is cleared on destroy */ #define RD_KAFKA_Q_F_FWD_APP \ 0x4 /* Queue is being forwarded by a call \ * to rd_kafka_queue_forward. */ #define RD_KAFKA_Q_F_YIELD \ 0x8 /* Have waiters return even if \ * no rko was enqueued. \ * This is used to wake up a waiter \ * by triggering the cond-var \ * but without having to enqueue \ * an op. */ rd_kafka_t *rkq_rk; struct rd_kafka_q_io *rkq_qio; /* FD-based application signalling */ /* Op serve callback (optional). * Mainly used for forwarded queues to use the original queue's * serve function from the forwarded position. * Shall return 1 if op was handled, else 0. */ rd_kafka_q_serve_cb_t *rkq_serve; void *rkq_opaque; #if ENABLE_DEVEL char rkq_name[64]; /* Debugging: queue name (FUNC:LINE) */ #else const char *rkq_name; /* Debugging: queue name (FUNC) */ #endif };
struct rd_kafka_op_s { TAILQ_ENTRY(rd_kafka_op_s) rko_link; rd_kafka_op_type_t rko_type; /* Internal op type */ rd_kafka_event_type_t rko_evtype; int rko_flags; /* See RD_KAFKA_OP_F_... above */ int32_t rko_version; rd_kafka_resp_err_t rko_err; rd_kafka_error_t *rko_error; int32_t rko_len; /* Depends on type, typically the * message length. */ rd_kafka_prio_t rko_prio; /**< In-queue priority. * Higher value means higher prio*/ rd_kafka_toppar_t *rko_rktp; /* * Generic fields */ /* Indicates request: enqueue reply on rko_replyq.q with .version. * .q is refcounted. */ rd_kafka_replyq_t rko_replyq; /* Original queue's op serve callback and opaque, if any. * Mainly used for forwarded queues to use the original queue's * serve function from the forwarded position. */ rd_kafka_q_serve_cb_t *rko_serve; void *rko_serve_opaque; rd_kafka_t *rko_rk; #if ENABLE_DEVEL const char *rko_source; /**< Where op was created */ #endif /* RD_KAFKA_OP_CB */ rd_kafka_op_cb_t *rko_op_cb; union { struct { rd_kafka_buf_t *rkbuf; rd_kafka_msg_t rkm; int evidx; } fetch; struct { rd_kafka_topic_partition_list_t *partitions; /** Require stable (txn-commited) offsets */ rd_bool_t require_stable; int do_free; /* free .partitions on destroy() */ } offset_fetch; struct { rd_kafka_topic_partition_list_t *partitions; void (*cb)(rd_kafka_t *rk, rd_kafka_resp_err_t err, rd_kafka_topic_partition_list_t *offsets, void *opaque); void *opaque; int silent_empty; /**< Fail silently if there are no * offsets to commit. */ rd_ts_t ts_timeout; char *reason; } offset_commit; struct { rd_kafka_topic_partition_list_t *topics; } subscribe; /* also used for GET_SUBSCRIPTION */ struct { rd_kafka_topic_partition_list_t *partitions; rd_kafka_assign_method_t method; } assign; /* also used for GET_ASSIGNMENT */ struct { rd_kafka_topic_partition_list_t *partitions; } rebalance; struct { const char *str; } rebalance_protocol; struct { char *str; } name; rd_kafka_consumer_group_metadata_t *cg_metadata; struct { int64_t offset; char *errstr; rd_kafka_msg_t rkm; rd_kafka_topic_t *rkt; int fatal; /**< This was a ERR__FATAL error that has * been translated to the fatal error * code. */ } err; /* used for ERR and CONSUMER_ERR */ struct { int throttle_time; int32_t nodeid; char *nodename; } throttle; struct { char *json; size_t json_len; } stats; struct { rd_kafka_buf_t *rkbuf; } xbuf; /* XMIT_BUF and RECV_BUF */ /* RD_KAFKA_OP_METADATA */ struct { rd_kafka_metadata_t *md; int force; /* force request regardless of outstanding * metadata requests. */ } metadata; struct { rd_kafka_topic_t *rkt; rd_kafka_msgq_t msgq; rd_kafka_msgq_t msgq2; int do_purge2; } dr; struct { int32_t nodeid; char nodename[RD_KAFKA_NODENAME_SIZE]; } node; struct { int64_t offset; char *reason; } offset_reset; struct { int64_t offset; struct rd_kafka_cgrp_s *rkcg; } fetch_start; /* reused for SEEK */ struct { int pause; int flag; } pause; struct { char fac[64]; int level; char *str; int ctx; } log; struct { rd_kafka_AdminOptions_t options; /**< Copy of user's * options */ rd_ts_t abs_timeout; /**< Absolute timeout * for this request. */ rd_kafka_timer_t tmr; /**< Timeout timer */ struct rd_kafka_enq_once_s *eonce; /**< Enqueue op * only once, * used to * (re)trigger * the request op * upon broker state * changes while * waiting for the * controller, or * due to .tmr * timeout. */ rd_list_t args; /**< Type depends on request, e.g. * rd_kafka_NewTopic_t for CreateTopics */ rd_kafka_buf_t *reply_buf; /**< Protocol reply, * temporary reference not * owned by this rko */ /**< Worker callbacks, see rdkafka_admin.c */ struct rd_kafka_admin_worker_cbs *cbs; /** Worker state */ enum { RD_KAFKA_ADMIN_STATE_INIT, RD_KAFKA_ADMIN_STATE_WAIT_BROKER, RD_KAFKA_ADMIN_STATE_WAIT_CONTROLLER, RD_KAFKA_ADMIN_STATE_WAIT_FANOUTS, RD_KAFKA_ADMIN_STATE_CONSTRUCT_REQUEST, RD_KAFKA_ADMIN_STATE_WAIT_RESPONSE, } state; int32_t broker_id; /**< Requested broker id to * communicate with. * Used for AlterConfigs, et.al, * that needs to speak to a * specific broker rather than * the controller. * See RD_KAFKA_ADMIN_TARGET_.. * for special values (coordinator, * fanout, etc). */ /** The type of coordinator to look up */ rd_kafka_coordtype_t coordtype; /** Which coordinator to look up */ char *coordkey; /** Application's reply queue */ rd_kafka_replyq_t replyq; rd_kafka_event_type_t reply_event_type; /** A collection of fanout child ops. */ struct { /** The type of request being fanned out. * This is used for the ADMIN_RESULT. */ rd_kafka_op_type_t reqtype; /** Worker callbacks, see rdkafka_admin.c */ struct rd_kafka_admin_fanout_worker_cbs *cbs; /** Number of outstanding requests remaining to * wait for. */ int outstanding; /** Incremental results from fanouts. * This list is pre-allocated to the number * of input objects and can thus be set * by index to retain original ordering. */ rd_list_t results; /** Reply event type */ rd_kafka_event_type_t reply_event_type; } fanout; /** A reference to the parent ADMIN_FANOUT op that * spawned this op, if applicable. NULL otherwise. */ struct rd_kafka_op_s *fanout_parent; } admin_request; struct { rd_kafka_op_type_t reqtype; /**< Request op type, * used for logging. */ rd_list_t args; /**< Args moved from the request op * when the result op is created. * * Type depends on request. */ char *errstr; /**< Error string, if rko_err * is set, else NULL. */ rd_list_t results; /**< Type depends on request type: * * (rd_kafka_topic_result_t *): * CreateTopics, DeleteTopics, * CreatePartitions. * * (rd_kafka_ConfigResource_t *): * AlterConfigs, DescribeConfigs */ void *opaque; /**< Application's opaque as set by * rd_kafka_AdminOptions_set_opaque */ /** A reference to the parent ADMIN_FANOUT op that * spawned this op, if applicable. NULL otherwise. */ struct rd_kafka_op_s *fanout_parent; } admin_result; struct { int flags; /**< purge_flags from rd_kafka_purge() */ } purge; /**< Mock cluster command */ struct { enum { RD_KAFKA_MOCK_CMD_TOPIC_SET_ERROR, RD_KAFKA_MOCK_CMD_TOPIC_CREATE, RD_KAFKA_MOCK_CMD_PART_SET_LEADER, RD_KAFKA_MOCK_CMD_PART_SET_FOLLOWER, RD_KAFKA_MOCK_CMD_PART_SET_FOLLOWER_WMARKS, RD_KAFKA_MOCK_CMD_BROKER_SET_UPDOWN, RD_KAFKA_MOCK_CMD_BROKER_SET_RTT, RD_KAFKA_MOCK_CMD_BROKER_SET_RACK, RD_KAFKA_MOCK_CMD_COORD_SET, RD_KAFKA_MOCK_CMD_APIVERSION_SET, } cmd; rd_kafka_resp_err_t err; /**< Error for: * TOPIC_SET_ERROR */ char *name; /**< For: * TOPIC_SET_ERROR * TOPIC_CREATE * PART_SET_FOLLOWER * PART_SET_FOLLOWER_WMARKS * BROKER_SET_RACK * COORD_SET (key_type) */ char *str; /**< For: * COORD_SET (key) */ int32_t partition; /**< For: * PART_SET_FOLLOWER * PART_SET_FOLLOWER_WMARKS * PART_SET_LEADER * APIVERSION_SET (ApiKey) */ int32_t broker_id; /**< For: * PART_SET_FOLLOWER * PART_SET_LEADER * BROKER_SET_UPDOWN * BROKER_SET_RACK * COORD_SET */ int64_t lo; /**< Low offset, for: * TOPIC_CREATE (part cnt) * PART_SET_FOLLOWER_WMARKS * BROKER_SET_UPDOWN * APIVERSION_SET (minver) * BROKER_SET_RTT */ int64_t hi; /**< High offset, for: * TOPIC_CREATE (repl fact) * PART_SET_FOLLOWER_WMARKS * APIVERSION_SET (maxver) */ } mock; struct { struct rd_kafka_broker_s *rkb; /**< Broker who's state * changed. */ /**< Callback to trigger on the op handler's thread. */ void (*cb)(struct rd_kafka_broker_s *rkb); } broker_monitor; struct { /** Consumer group metadata for send_offsets_to.. */ rd_kafka_consumer_group_metadata_t *cgmetadata; /** Consumer group id for AddOffsetsTo.. */ char *group_id; int timeout_ms; /**< Operation timeout */ rd_ts_t abs_timeout; /**< Absolute time */ /**< Offsets to commit */ rd_kafka_topic_partition_list_t *offsets; } txn; struct { /* This struct serves two purposes, the fields * with "Request:" are used for the async workers state * while the "Reply:" fields is a separate reply * rko that is enqueued for the caller upon * completion or failure. */ /** Request: Partitions to query. * Reply: Queried partitions with .err field set. */ rd_kafka_topic_partition_list_t *partitions; /** Request: Absolute timeout */ rd_ts_t ts_timeout; /** Request: Metadata query timer */ rd_kafka_timer_t query_tmr; /** Request: Timeout timer */ rd_kafka_timer_t timeout_tmr; /** Request: Enqueue op only once, used to (re)trigger * metadata cache lookups, topic refresh, timeout. */ struct rd_kafka_enq_once_s *eonce; /** Request: Caller's replyq */ rd_kafka_replyq_t replyq; /** Request: Number of metadata queries made. */ int query_cnt; /** Reply: Leaders (result) * (rd_kafka_partition_leader*) */ rd_list_t *leaders; /** Reply: Callback on completion (or failure) */ rd_kafka_op_cb_t *cb; /** Reply: Callback opaque */ void *opaque; } leaders; } rko_u; };
