Python Message Queue package

Python Message Queue package

1. Official introduction to the snakemq
The GitHub project page of the snail kemq: https://github.com/dsiroky/snakemq
1. Pure python implementation, cross-platform

2. Automatic Reconnection

3. reliable transmission-configurable message mode and message timeout Mode

4. Two types of persistent/Temporary queues

5. support asynchronous -- poll ()

6. duplex rical-a single TCP connection can be used for duplex communication

7. Support for multiple databases-SQLite, MongoDB ......

8. brokerless-implementation principle similar to ZeroMQ

9. expansion module: RPC, bandwidth throttling

All of the above are official words, which need to be verified by yourself and encapsulated by yourself. It feels cute.

II. Key Issues

1. Automatic reconnection is supported. You do not need to write your own heartbeat logic. You only need to pay attention to sending and receiving.

2. Data Persistence is supported. If persistence starts, data is automatically sent after reconnection.

3. For data receiving, you only need to write a Receiving Method to add it to the callback list.

4. The data is sent here in bytes (Binary) type, so conversion is required. All the tests in the program are text strings, which are converted to bytes using str. encode ('utf-8') and then converted back upon receiving.

5. in terms of terminology, Connector: similar to socket TcpClient, Lisenter: similar to socket TcpServer. Each connector or listener is identified by an ident one by one. When sending and receiving data, you will know who the data is.

6. When using sqlite for persistence, You need to modify the source code, sqlite3.connect (filename, check_same_thread = False), to solve the problem of multi-thread access to sqlite. (Will it be deadlocked ?)

7. When persistence is started, if it is re-connected, it will be automatically sent to ensure reliability.

8. After receiving data for encapsulation purposes, I use callback to transmit the data.

Iii. Code

The custom log module is used in the code.

from common import nxloggerimport snakemqlogger as logger

Can be replaced with logging.

Callback class (callbacks. py ):

# -*- coding:utf-8 -*-'''synchronized callback'''class Callback(object):  def __init__(self):    self.callbacks = []   def add(self, func):    self.callbacks.append(func)   def remove(self, func):    self.callbacks.remove(func)   def __call__(self, *args, **kwargs):    for callback in self.callbacks:      callback(*args, **kwargs)

Connector class (snkemqconnector. py ):

# -*- coding:utf-8 -*-import threadingimport snakemqimport snakemq.linkimport snakemq.packeterimport snakemq.messagingimport snakemq.messagefrom snakemq.storage.sqlite import SqliteQueuesStoragefrom snakemq.message import FLAG_PERSISTENTfrom common.callbacks import Callback from common import nxloggerimport snakemqlogger as logger class SnakemqConnector(threading.Thread):     def __init__(self, snakemqident = None, remoteIp = "localhost", remotePort = 9090, persistent = False):         super(SnakemqConnector,self).__init__()         self.messaging = None         self.link = None         self.snakemqident = snakemqident         self.pktr = None         self.remoteIp = remoteIp         self.remotePort = remotePort         self.persistent = persistent         self.on_recv = Callback()         self._initConnector()      def run(self):         logger.info("connector start...")                  if self.link != None:              self.link.loop()          logger.info("connector end...")         def terminate(self):         logger.info("connetor terminating...")         if self.link != None:              self.link.stop()              self.link.cleanup()         logger.info("connetor terminated")      def on_recv_message(self, conn, ident, message):         try:              self.on_recv(ident, message.data.decode('utf-8'))#dispatch received data         except Exception as e:              logger.error("connector recv:{0}".format(e))              print(e)      '''send message to dest host named destIdent'''     def sendMsg(self, destIdent, byteseq):         msg = None         if self.persistent:              msg = snakemq.message.Message(byteseq, ttl=60, flags=FLAG_PERSISTENT)         else:              msg = snakemq.message.Message(byteseq, ttl=60)         if self.messaging == None:              logger.error("connector:messaging is not initialized, send message failed")              return         self.messaging.send_message(destIdent, msg)      '''         '''     def _initConnector(self):         try:              self.link = snakemq.link.Link()              self.link.add_connector((self.remoteIp, self.remotePort))               self.pktr = snakemq.packeter.Packeter(self.link)               if self.persistent:                  storage = SqliteQueuesStorage("SnakemqStorage.db")                  self.messaging = snakemq.messaging.Messaging(self.snakemqident, "", self.pktr, storage)              else:                  self.messaging = snakemq.messaging.Messaging(self.snakemqident, "", self.pktr)                           self.messaging.on_message_recv.add(self.on_recv_message)                      except Exception as e:              logger.error("connector:{0}".format(e))         finally:              logger.info("connector[{0}] loop ended...".format(self.snakemqident))

Listener class (snkemqlistener. py ):

# -*- coding:utf-8 -*-import threadingimport snakemqimport snakemq.linkimport snakemq.packeterimport snakemq.messagingimport snakemq.messagefrom common import nxloggerimport snakemqlogger as loggerfrom common.callbacks import Callbackclass SnakemqListener(threading.Thread):     def __init__(self, snakemqident = None, ip = "localhost", port = 9090, persistent = False):         super(SnakemqListener,self).__init__()         self.messaging = None         self.link = None         self.pktr = None         self.snakemqident = snakemqident         self.ip = ip;         self.port = port         self.connectors = {}         self.on_recv = Callback()         self.persistent = persistent         self._initlistener()      '''     thread run     '''     def run(self):         logger.info("listener start...")                  if self.link != None:              self.link.loop()          logger.info("listener end...")      '''     terminate snakemq listener thread     '''     def terminate(self):         logger.info("listener terminating...")         if self.link != None:              self.link.stop()              self.link.cleanup()         logger.info("listener terminated")      '''     receive message from host named ident     '''     def on_recv_message(self, conn, ident, message):         try:              self.on_recv(ident, message.data.decode('utf-8'))#dispatch received data              self.sendMsg('bob','hello,{0}'.format(ident).encode('utf-8'))         except Exception as e:              logger.error("listener recv:{0}".format(e))              print(e)      def on_drop_message(self, ident, message):         print("message dropped", ident, message)         logger.debug("listener:message dropped,ident:{0},message:{1}".format(ident, message))      '''client connect'''     def on_connect(self, ident):         logger.debug("listener:{0} connected".format(ident))         self.connectors[ident] = ident         self.sendMsg(ident, "hello".encode('utf-8'))      '''client disconnect'''     def on_disconnect(self, ident):         logger.debug("listener:{0} disconnected".format(ident))         if ident in self.connectors:              self.connectors.pop(ident)      '''     listen start loop     '''     def _initlistener(self):         try:              self.link = snakemq.link.Link()              self.link.add_listener((self.ip, self.port))               self.pktr = snakemq.packeter.Packeter(self.link)              self.pktr.on_connect.add(self.on_connect)              self.pktr.on_disconnect.add(self.on_disconnect)               if self.persistent:                  storage = SqliteQueuesStorage("SnakemqStorage.db")                  self.messaging = snakemq.messaging.Messaging(self.snakemqident, "", self.pktr, storage)              else:                  self.messaging = snakemq.messaging.Messaging(self.snakemqident, "", self.pktr)                           self.messaging.on_message_recv.add(self.on_recv_message)              self.messaging.on_message_drop.add(self.on_drop_message)          except Exception as e:              logger.error("listener:{0}".format(e))         finally:              logger.info("listener:loop ended...")     '''send message to dest host named destIdent'''     def sendMsg(self, destIdent, byteseq):         msg = None         if self.persistent:              msg = snakemq.message.Message(byteseq, ttl=60, flags=FLAG_PERSISTENT)         else:              msg = snakemq.message.Message(byteseq, ttl=60)         if self.messaging == None:              logger.error("listener:messaging is not initialized, send message failed")              return         self.messaging.send_message(destIdent, msg)

Test code connector (testsnkeconnector. py ):

Read a local 1 M file and send it to the listener. Then, the listener sends back a hello message.

From netComm. snakemq import snkemqconnectorimport timeimport sysimport osdef inclued (ident, data): print (data) if _ name _ = "_ main _": bob = snkemqconnector. snkemqconnector ('bob', "10.16.5.45", 4002, True) bob. on_recv.add (received) bob. start () try: with open ("testfile.txt", encoding = 'utf-8') as f: txt = f. read () for I in range (100): bob. sendMsg ("niess", txt. encode ('utf-8') time. sleep (0.1) failed t Exception as e: print (e) time. sleep (5) bob. terminate () test code listener (testsnkelistener. py): from netComm. snakemq import snkemqlistenerimport time def received (ident, data): filename = "log/recfile1_02.16.txt ". format (time. strftime ('% s', time. localtime () file = open (filename, 'w') file. writelines (data) file. close () if _ name _ = "_ main _": niess = snkemqlistener. snemqlistener ("niess", "10.16.5.45", 4002) niess. on_recv.add (received) niess. start () print ("niess start... ") time. sleep (60) niess. terminate () print ("niess end... ")