rabbit相关
一、消息队列
1.1介绍
就是基础数据结构中的“先进先出”的一种数据机构。
1.2MQ解决问题
MQ一直存在，解决微服务框架。两个服务（服务端和客户端）调用方法：restful(http协议)，rpc（远程过程调用，一台机器调用另一台机器的函数或方法） 2.rpc：远程过程调用
-gRPC：谷歌出的，跨语言 3 不管用rpc或者restful来通信，涉及到同步，异步 4 消息队列解决的问题 -应用解耦 -流量消峰 -消息分发（发布订阅：观察者模式） -异步消息（celery就是对消息队列的封装） 5 rabbitmq,kafka -rabbitmq:吞吐量小，消息确认，订单，对消息可靠性有要求，就用它 -kafka：吞吐量高，注重高吞吐量，不注重消息的可靠性，数据量特别大（一般用于大数据） 6 电商、金融等对事务性要求很高的，可以考虑RabbitMQ 7 日志考虑Kafka
二、Rabbitmq
1、安装
1 原生安装 -安装扩展epel源 -yum -y install erlang -yum -y install rabbitmq-server -systemctl start rabbitmq-server 2 docker拉取 -docker pull rabbitmq:management（自动开启了web管理界面） -docker run -di --name rabbitmq -e RABBITMQ_DEFAULT_USER=admin -e RABBITMQ_DEFAULT_PASS=admin -p 15672:15672 -p 5672:5672 rabbitmq:management
# 5672：是rabbitmq的默认端口 15672：web管理界面的端口
2、创建用户再分配权限
4 创建用户 rabbitmqctl add_user aaa 123
5 分配权限 rabbitmqctl set_user_tags aaa administrator rabbitmqctl set_permissions -p "/" aaa ".*" ".*" ".*"
3、基本使用
生产者：
# 先安装包pika # pip3 install pika import pika # 拿到连接对象 # connection = pika.BlockingConnection(pika.ConnectionParameters('101.133.225.166')) # 有用户名密码的情况，拿连接对象用这个 credentials = pika.PlainCredentials("admin","admin") connection = pika.BlockingConnection(pika.ConnectionParameters('101.133.225.166',credentials=credentials)) # 拿到channel对象 channel = connection.channel() # 声明一个队列 channel.queue_declare(queue='hello') # 指定队列名字 # 生产者向队列中放一条消息 channel.basic_publish(exchange='', routing_key='hello', #就是队列名字 body='aaa js nb') print(" Sent 'Hello World!'") # 关闭连接 connection.close()
消费者：
import pika, sys, os def main(): #无验证密码情况用这个 # connection = pika.BlockingConnection(pika.ConnectionParameters(host='101.133.225.166')) #有验证密码用这个 credentials = pika.PlainCredentials("admin", "admin") connection = pika.BlockingConnection(pika.ConnectionParameters('101.133.225.166', credentials=credentials)) channel = connection.channel() channel.queue_declare(queue='hello') #获取消息队列 def callback(ch, method, properties, body): print(" [x] Received %r" % body) channel.basic_consume(queue='hello', on_message_callback=callback, auto_ack=True) channel.start_consuming() if __name__ == '__main__': main()
4、消息确认机制（消费者变化，主要是加这条代码确认：ch.basic_ack(delivery_tag=method.delivery_tag)）
生产者：# 先安装包pika
# pip3 install pika import pika # 拿到连接对象 # connection = pika.BlockingConnection(pika.ConnectionParameters('101.133.225.166')) # 有用户名密码的情况，拿到连接对象 credentials = pika.PlainCredentials("admin","admin") connection = pika.BlockingConnection(pika.ConnectionParameters('101.133.225.166',credentials=credentials)) # 拿到channel对象 channel = connection.channel() # 声明一个队列 channel.queue_declare(queue='aaa'，,durable=True) # 指定队列名字，并指明持久化
# 生产者向队列中放一条消息
channel.basic_publish(exchange='',
routing_key='aaa',
body='aaa jssss nb',
properties=pika.BasicProperties(
delivery_mode=2, # 此时声明消息持久化)
)
print(" aaa jssss nb'") # 关闭连接 connection.close()
消费者：
import pika, sys, os def main(): # connection = pika.BlockingConnection(pika.ConnectionParameters(host='101.133.225.166')) 有密码的情况 credentials = pika.PlainCredentials("admin", "admin") connection = pika.BlockingConnection(pika.ConnectionParameters('101.133.225.166', credentials=credentials)) channel = connection.channel()
channel.queue_declare(queue='aaa',durable=True) #此时声明队列持久化
def callback(ch, method, properties, body): print(" Received %r" % body)
# 真正的消息处理完了，再发确认
ch.basic_ack(delivery_tag=method.delivery_tag)
## 不会自动回复确认消息，
## auto_ack=True，队列收到确认，就会自动把消费过的消息删除,但一般不用这个，而是ch.basic_ack(delivery_tag=method.delivery_tag)
channel.basic_consume(queue='aaa', on_message_callback=callback, auto_ack=False)
channel.start_consuming()
if __name__ == '__main__': main()
4、持久化(消费者代码队列变，生产者代码消息改变)
#在声明队列时，指定持久化，消费者和生产者都需要修改 channel.queue_declare(queue='aaa',durable=True) # 声明消息持久化,只在生产者指定 #在发布消息的时候， properties=pika.BasicProperties( delivery_mode=2, # make message persistent )
5、闲置消费
谁闲置谁获取，没必要按照顺序一个一个来 消费者配置 channel.basic_qos(prefetch_count=1) ，生产者代码不变
import pika, sys, os def main(): # connection = pika.BlockingConnection(pika.ConnectionParameters(host='101.133.225.166')) #有用户名 credentials = pika.PlainCredentials("admin", "admin") connection = pika.BlockingConnection(pika.ConnectionParameters('101.133.225.166', credentials=credentials)) channel = connection.channel() channel.queue_declare(queue='lqz') def callback(ch, method, properties, body): import time time.sleep(50) print(" [x] Received %r" % body) # 真正的消息处理完了，再发确认 ch.basic_ack(delivery_tag=method.delivery_tag) ## 不会自动回复确认消息， ## auto_ack=True，队列收到确认，就会自动把消费过的消息删除 channel.basic_qos(prefetch_count=1) #####此时就是配置谁闲置谁获取，没必要按照顺序一个一个来 channel.basic_consume(queue='lqz', on_message_callback=callback, auto_ack=False) channel.start_consuming() if __name__ == '__main__': main()
6、消息订阅
发布者（生产者）：
import pika credentials = pika.PlainCredentials("admin", "admin") connection = pika.BlockingConnection(pika.ConnectionParameters('101.133.225.166', credentials=credentials)) channel = connection.channel() # 声明队列没有指定名字，指定了exchange channel.exchange_declare(exchange='logs', exchange_type='fanout') message = "info: Hello World!" channel.basic_publish(exchange='logs', routing_key='', body=message) print(" [x] Sent %r" % message) connection.close()
订阅者（消费者）：启动多次，会创建出多个队列，都绑定到了同一个exchange上
import pika credentials = pika.PlainCredentials("admin", "admin") connection = pika.BlockingConnection(pika.ConnectionParameters('101.133.225.166', credentials=credentials)) channel = connection.channel() channel.exchange_declare(exchange='logs', exchange_type='fanout') #这是result的queue会自动生成 result = channel.queue_declare(queue='', exclusive=True) queue_name = result.method.queue print(queue_name) #每个消费者的名字是不一样的 #将发布者的exchange绑定 channel.queue_bind(exchange='logs', queue=queue_name) print(' Waiting for logs. To exit press CTRL+C') def callback(ch, method, properties, body): print(" [x] %r" % body) channel.basic_consume( queue=queue_name, on_message_callback=callback, auto_ack=True) channel.start_consuming()
6.1发布订阅之routing关键字
发布者：
import pika credentials = pika.PlainCredentials("admin", "admin") connection = pika.BlockingConnection(pika.ConnectionParameters('101.133.225.166', credentials=credentials)) channel = connection.channel() # 声明队列没有指定名字，指定了exchange channel.exchange_declare(exchange='aaa123', exchange_type='direct') message = "info: asdfasdfasdfsadfasdf World!" #此时指定了关键字routing_key,代表只有订阅者订阅了bnb才能接收到这个消息 channel.basic_publish(exchange='lqz123', routing_key='bnb', body=message) print(" [x] Sent %r" % message) connection.close()
订阅者1：
import pika credentials = pika.PlainCredentials("admin", "admin") connection = pika.BlockingConnection(pika.ConnectionParameters('101.133.225.166', credentials=credentials)) channel = connection.channel() channel.exchange_declare(exchange='aaa123', exchange_type='direct') result = channel.queue_declare(queue='', exclusive=True) queue_name = result.method.queue print(queue_name) #此时代表订阅者订阅了routing_key为nb的消息频道 channel.queue_bind(exchange='aaa123', queue=queue_name,routing_key='nb') print(' [*] Waiting for logs. To exit press CTRL+C') def callback(ch, method, properties, body): print(" [x] %r" % body) channel.basic_consume( queue=queue_name, on_message_callback=callback, auto_ack=True) channel.start_consuming()
订阅者2：
import pika credentials = pika.PlainCredentials("admin", "admin") connection = pika.BlockingConnection(pika.ConnectionParameters('101.133.225.166', credentials=credentials)) channel = connection.channel() channel.exchange_declare(exchange='aaa123', exchange_type='direct') result = channel.queue_declare(queue='', exclusive=True) queue_name = result.method.queue print(queue_name) #此时订阅者监听了两个频道，routing_key，此时两个频道都能收到消息 channel.queue_bind(exchange='aaa123', queue=queue_name,routing_key='nb') channel.queue_bind(exchange='aaa123', queue=queue_name,routing_key='bnb') print(' [*] Waiting for logs. To exit press CTRL+C') def callback(ch, method, properties, body): print(" [x] %r" % body) channel.basic_consume( queue=queue_name, on_message_callback=callback, auto_ack=True) channel.start_consuming()
6.2模糊匹配： # 表示后面可以跟任意字符任意长度都能匹配 *表示后面只能跟一个英文单词
发布者：
import pika credentials = pika.PlainCredentials("admin", "admin") connection = pika.BlockingConnection(pika.ConnectionParameters('101.133.225.166', credentials=credentials)) channel = connection.channel() # 声明队列没有指定名字，指定了exchange channel.exchange_declare(exchange='m3', exchange_type='topic') message = "info: asdfasdfasdfsadfasdf World!" channel.basic_publish(exchange='m3', routing_key='aaa.ddxxx', body=message) print(" [x] Sent %r" % message) connection.close()
订阅者1：
import pika credentials = pika.PlainCredentials("admin", "admin") connection = pika.BlockingConnection(pika.ConnectionParameters('101.133.225.166', credentials=credentials)) channel = connection.channel() channel.exchange_declare(exchange='m3', exchange_type='topic') result = channel.queue_declare(queue='', exclusive=True) queue_name = result.method.queue print(queue_name) channel.queue_bind(exchange='m3', queue=queue_name,routing_key='aaa.*') #指定模糊匹配,此时不能收到 print(' [*] Waiting for logs. To exit press CTRL+C') def callback(ch, method, properties, body): print(" [x] %r" % body) channel.basic_consume( queue=queue_name, on_message_callback=callback, auto_ack=True) channel.start_consuming()
订阅者2：
import pika credentials = pika.PlainCredentials("admin", "admin") connection = pika.BlockingConnection(pika.ConnectionParameters('101.133.225.166', credentials=credentials)) channel = connection.channel() channel.exchange_declare(exchange='m3', exchange_type='topic') result = channel.queue_declare(queue='', exclusive=True) queue_name = result.method.queue print(queue_name) channel.queue_bind(exchange='m3', queue=queue_name,routing_key='aaa.#') #指定模糊匹配，此时可以收到 print(' [*] Waiting for logs. To exit press CTRL+C') def callback(ch, method, properties, body): print(" [x] %r" % body) channel.basic_consume( queue=queue_name, on_message_callback=callback, auto_ack=True) channel.start_consuming()
7、三种方法实现rpc
1、通过rabbitmq实现（例子：服务端的函数，客户端来调用）(不推荐)
服务端：
import pika credentials = pika.PlainCredentials("admin", "admin") connection = pika.BlockingConnection(pika.ConnectionParameters('101.133.225.166', credentials=credentials)) channel = connection.channel() channel.queue_declare(queue='rpc_queue') def fib(n): if n == 0: return 0 elif n == 1: return 1 else: return fib(n - 1) + fib(n - 2) def on_request(ch, method, props, body): n = int(body) print(" [.] fib(%s)" % n) response = fib(n) ch.basic_publish(exchange='', routing_key=props.reply_to, properties=pika.BasicProperties(correlation_id = \ props.correlation_id), body=str(response)) ch.basic_ack(delivery_tag=method.delivery_tag) channel.basic_qos(prefetch_count=1) channel.basic_consume(queue='rpc_queue', on_message_callback=on_request) print(" [x] Awaiting RPC requests") channel.start_consuming()
客户端：
import pika import uuid class FibonacciRpcClient(object): def __init__(self): self.credentials = pika.PlainCredentials("admin", "admin") self.connection = pika.BlockingConnection(pika.ConnectionParameters('101.133.225.166', credentials=self.credentials)) self.channel = self.connection.channel() result = self.channel.queue_declare(queue='', exclusive=True) self.callback_queue = result.method.queue self.channel.basic_consume( queue=self.callback_queue, on_message_callback=self.on_response, #调用函数on_response auto_ack=True) def on_response(self, ch, method, props, body): if self.corr_id == props.correlation_id: self.response = body def call(self, n): self.response = None self.corr_id = str(uuid.uuid4()) self.channel.basic_publish( exchange='', routing_key='rpc_queue', properties=pika.BasicProperties( reply_to=self.callback_queue, correlation_id=self.corr_id, ), body=str(n)) while self.response is None: self.connection.process_data_events() return int(self.response) fibonacci_rpc = FibonacciRpcClient() #此时实例化一个对象，就会执行init方法 print(" [x] Requesting fib(30)") response = fibonacci_rpc.call(10) # 外界看上去，就像调用本地的call()函数一样 print(" [.] Got %r" % response)
2、SimpleXMLRPCServer实现rpc（不推荐，比较慢，走XML协议）
服务端：
from xmlrpc.server import SimpleXMLRPCServer class RPCServer(object): def __init__(self): super(RPCServer, self).__init__() print(self) self.send_data = {'server:'+str(i): i for i in range(100)} self.recv_data = None def getObj(self): print('get data') return self.send_data def sendObj(self, data): print('send data') self.recv_data = data print(self.recv_data) # SimpleXMLRPCServer server = SimpleXMLRPCServer(('localhost',4242), allow_none=True) #前面是监听地址和端口 server.register_introspection_functions() server.register_instance(RPCServer()) server.serve_forever()
客户端：
import time from xmlrpc.client import ServerProxy # SimpleXMLRPCServer def xmlrpc_client(): print('xmlrpc client') c = ServerProxy('http://localhost:4242') data = {'client:'+str(i): i for i in range(100)} start = time.clock() for i in range(50): a=c.getObj() print(a) for i in range(50): c.sendObj(data) print('xmlrpc total time %s' % (time.clock() - start)) if __name__ == '__main__': xmlrpc_client()
3、ZeroRpc实现rpc（推荐，性能很高，走http协议）
服务端：
import zerorpc class RPCServer(object): def __init__(self): super(RPCServer, self).__init__() print(self) self.send_data = {'server:'+str(i): i for i in range(100)} self.recv_data = None def getObj(self): print('get data') return self.send_data def sendObj(self, data): print('send data') self.recv_data = data print(self.recv_data) # zerorpc s = zerorpc.Server(RPCServer()) s.bind('tcp://0.0.0.0:4243') s.run()
客户端：
import zerorpc import time # zerorpc def zerorpc_client(): print('zerorpc client') c = zerorpc.Client() c.connect('tcp://127.0.0.1:4243') data = {'client:'+str(i): i for i in range(100)} start = time.clock() for i in range(500): a=c.getObj() print(a) for i in range(500): c.sendObj(data) print('total time %s' % (time.clock() - start)) if __name__ == '__main__': zerorpc_client()
浙公网安备 33010602011771号