python zeromq rpc介紹

https://github.com/geoffwatts/zmqrpc

傳輸資料格式：bson
模型: 多線程
Client             Server   
------       ------------------------------          
client                        worker(thread)     
             \                /
cleint   --   queue    
             /                \
client                        worker(thread)

server.py

"""server: Implementing ZMQRPCServer class to export a user class via zmqrpc to ZMQRPC clients, and to arrange queued calls to server threads."""import sysif sys.version < '2.6':    sys.exit('ERROR: Sorry, python 2.6 is required for the way this module uses threading.')import zmqfrom bson import BSON   import threadingimport os, sys, tracebackfrom zmqrpc import ZMQRPCError, ZMQRPCRemoteErrorLISTEN=0CONNECT=1class ZMQRPCServer(object):    def _thread(self,context,worker_id,import_class,pid,serverid,counters,methods,target,stype,worker_args):        """        Worker thread for zmqrpc server - binds to zmq socket (target) and works ZMQRPCServer import_class.        Instantiated by work() threading        Handles BSON in/out, zmq REP to zmq QUEUE or REQ        """        socket = self.context.socket(zmq.REP)        job_count = 0        if stype == LISTEN:            socket.bind(target)        else:            socket.connect(target)        if worker_args:            nuclass = import_class(**worker_args)        else:            nuclass = import_class()                while True:            sockin = socket.recv()            message = BSON(sockin).decode()            result = None            fail = None            tb = None            method = str(message['method'])            args = message.get('args',[])            if self.export and (not method in self.export):                tb = "NameError: name '"+method+"' is not exported in ZMQRPC class '"+import_class.__name__+"'"                socket.send(BSON.encode({'fail':True,'result':None,'runner':None,'traceback':tb}))                return                            # Convert kwargs from unicode strings to 8bit strings                        if method == '__threadstatus__':                x = threading.current_thread()                socket.send(BSON.encode({'runner':None,'traceback':None,'fail':False,'result':{'id':serverid+':'+str(pid)+':'+str(x.name),'alive':x.is_alive(),'job_count':counters.get(x.name,0),'last_method':methods.get(x.name,''),}}))               else:                try:                    kwargs = {}                    for (k,v) in message.get('kwargs',{}).iteritems():                        kwargs[str(k)] = v                    job_count+=1                    counters[threading.currentThread().name] = job_count                    methods[threading.currentThread().name] = method                    runner = {'job_count':job_count,'thread':threading.currentThread().name,'method':import_class.__name__+'.'+method,}                                        # Find the method in the module, run it.                    try:                        if hasattr(nuclass,method):                            result = getattr(nuclass,method)(*args,**kwargs)                            fail = False                        else:                            fail = True                            tb = "NameError: name '"+method+"' is not defined in ZMQRPC class '"+import_class.__name__+"'"                    except:                        etype, evalue, etb = sys.exc_info()                        fail = True                        tb = "\n".join(traceback.format_exception(etype, evalue, etb))                    socket.send(BSON.encode({'fail':fail,'result':result,'runner':runner,'traceback':tb}))                except:                    etype, evalue, etb = sys.exc_info()                    fail = True                    tb = "\n".join(traceback.format_exception(etype, evalue, etb))                    socket.send(BSON.encode({'fail':fail,'result':None,'runner':None,'traceback':tb}))    def __init__(self,import_class,export=None):        """        Instantiate this class with your class to export via zmqrpc        """        self.iclass = import_class        self.pid = os.getpid()        self.serverid = os.uname()[1]        self.context = zmq.Context(1)        self.export = export    def work(self,workers=1,target="inproc://workers",stype=CONNECT,worker_args={}):        """        Call to spawn serverthreads that will then work forever.        stype: socket type, either zmqrpc.server.CONNECT or zmqrpc.server.LISTEN        target: zmq socket (eg: 'tcp://127.0.0.1:5000')        workers: number of worker threads to spwan        """        counters = {}        methods = {}        for i in range(0,workers):            thread = threading.Thread(target=self._thread, name='zmqrpc-'+str(i), args=(self.context,i,self.iclass,self.pid,self.serverid,counters,methods,target,stype,worker_args))            thread.start()                def queue(self,listen,bind='inproc://workers',thread=False):        """        Call to start a zmq queue device to disatch zmqrpc work.        listen: zmq socket to listen on for CLIENTS (eg: 'tcp://127.0.0.1:5 000')        target: zmq socket to listen on for worker threads (eg: 'tcp://127.0.0.1:6000')        workers: number of worker threads to spwan        """        def q(listen,worker_target):            self.workers = self.context.socket(zmq.XREQ)            self.workers.bind(worker_target);            self.clients = self.context.socket(zmq.XREP)            self.clients.bind(listen)             zmq.device(zmq.QUEUE, self.clients, self.workers)        if thread:            thread = threading.Thread(target=q, name='zmqrpc-queue', args=(listen,bind ))            thread.start()        else:            q(listen,bind)        

client.py

"""client: client class to export a class to an zmqrpc queue or client."""import zmqfrom bson import BSON   import os, sys, tracebackimport timefrom zmqrpc import ZMQRPCError, ZMQRPCRemoteErrorclass ZMQRPC(object):    """    ZMQRPC: client class to export a class to an zmqrpc queue or client.    """    def __init__(self,target,timeout=30):        """        Instantiate this class with a zmq target (eg 'tcp://127.0.0.1:5000') and a timeout (in seconds) for method calls.        Then call zmqrpc server exported methods from the class.        """                self._context = zmq.Context()        self._zmqsocket = self._context.socket(zmq.REQ)        # Connect to everything, or just one        if isinstance(target,list):            for t in target:                self._zmqsocket.connect(target)        else:            self._zmqsocket.connect(target)        self._socket = target        self._pollin = zmq.Poller()        self._pollin.register(self._zmqsocket,zmq.POLLIN)        self._pollout = zmq.Poller()        self._pollout.register(self._zmqsocket,zmq.POLLOUT)        self._timeout = timeout                self._lastrun = None    def _dorequest(self,msg,timeout=5):        """        _dorequest: Set up a BSON string and send zmq REQ to ZMQRPC target        """        # Set up bson message        bson = BSON.encode(msg)                # Send...        try:            self._pollout.poll(timeout=timeout*1000) # Poll for outbound send, then send            self._zmqsocket.send(bson,flags=zmq.NOBLOCK)        except:            raise ZMQRPCError('Request failure')        # Poll for inbound then rx        try:                    for i in range(0,timeout*100):                if len(self._pollin.poll(timeout=1)) > 0:                    break                time.sleep(0.01)            msg_in = self._zmqsocket.recv(flags=zmq.NOBLOCK)                except:            raise ZMQRPCError('Response timeout')                if msg_in == None:            raise ZMQRPCError('No response')            result = BSON(msg_in).decode()                self._lastrun = result.get('runner')                return result            def _debug_call(self,name,*args,**kwargs):        """        _debug_call: Convenience method to call _dorequest with pre-filled dict with method name, args, kwargs and timeout        """        return self._dorequest({'method':name,'args':args,'kwargs':kwargs},timeout=self._timeout)                   def __serverstatus__(self,max_nodes=1000):        """        __serverstatus__: Slightly hackish method to retreive threadstatus from all listening threads on a zmqrpc queue        """        results = {}        try:            for r in range(0,max_nodes):                res = self._dorequest({'method':'__threadstatus__'},timeout=self._timeout)[u'result']                id = res[u'id']                if results.has_key(id): break                del res[u'id']                results[id] = res        except:            raise ZMQRPCError('Error finding server threads')        return results                                     class RPC(object):        """        RPC: zmqrpc Remote procedure call class - encapsulates method calls to imported class        """        def __init__(self,name,fn,timeout,target):            self._name = name            self._fn = fn            self._timeout = timeout            self._socket = target                    def __call__(self,*args,**kwargs):            result = self._fn({'method':self._name,'args':args,'kwargs':kwargs},timeout=self._timeout)            if result['fail']:                raise ZMQRPCRemoteError(result['traceback']) #+"  RUNNER:"+str(result['runner']))            else:                return result['result']        def __repr__(self):            return '<zmqrpc method '+self._name+' to zmq socket '+self._socket+'>'                        def __getattr__(self,name):            return self.RPC(name,self._dorequest,timeout=self._timeout,target=self._socket) 

由於是使用zmq所以很容易把上面的模式改進為，多進程+多線程的模型
因為python中是鼓勵使用多進程的(多進程可使用多核)
模型二: 多進程+多線程
Client             Server   
------       ----------------------------------------------          
client                       Server1________worker(thread)
             \                /(process)    \___worker(thread)
cleint   --   queue    
             /                \                                      
client                       Server2________worker(thread)
                                (process)    \___worker(thread)

要使用模型二需要把傳輸協議由INPROC改為IPC