Modify the main loop in Python pyxmpp2 to improve its performance.

Modify the main loop in Python pyxmpp2 to improve its performance.

Introduction

Previously, the default mainloop of pyxmpp2 used by clubot is a main loop of poll. However, after clubot went online, the resource usage was very high. Using strace tracking, we found that clubot was continuously running poll, check the pyxmpp2 code and find that poll of pyxmpp2 uses the minimum timeout time when timeout blocking is used, and the minimum timeout time is always 0. Therefore, a non-blocking poll with no timeout is a waste of resources, I don't want to change the library code, so I wrote a more efficient epoll mainloop like the mainloop of poll.
Implementation

#! /Usr/bin/env python #-*-coding: UTF-8-*-# Author: cold # E-mail: wh_linux@126.com # Date: 13/01/06 10:41:31 # Desc: clubot epoll mainloop # from _ future _ import absolute_import, divisionimport selectfrom pyxmpp2.mainloop. interfaces import HandlerReady, PrepareAgainfrom pyxmpp2.mainloop. base import MainLoopBasefrom plugin. util import get_loggerclass EpollMainLoop (MainLoopBase): "" Main event loo P based on the epoll () syscall on Linux system "" READ_ONLY = (select. EPOLLIN | select. EPOLLPRI | select. EPOLLHUP | select. EPOLLERR | select. EPOLLET) READ_WRITE = READ_ONLY | select. EPOLLOUT def _ init _ (self, settings = None, handlers = None): self. epoll = select. epoll () self. _ handlers = {} self. _ unprepared_handlers ={} self. _ timeout = None self. _ exists_fd ={} self. logger = get_logger () MainLoop Base. _ init _ (self, settings, handlers) return def _ add_io_handler (self, handler): self. _ unprepared_handlers [handler] = None self. _ configure_io_handler (handler) def _ configure_io_handler (self, handler): if self. check_events (): return if handler in self. _ unprepared_handlers: old_fileno = self. _ unprepared_handlers [handler] prepared = self. _ prepare_io_handler (handler) else: old_fileno = None prepared = True fileno = handler. fileno () if old_fileno is not None and fileno! = Old_fileno: del self. _ handlers [old_fileno] self. _ exists. pop (old_fileno, None) self. epoll. unregister (old_fileno) if not prepared: self. _ unprepared_handlers [handler] = fileno if not fileno: return self. _ handlers [fileno] = handler events = 0 if handler. is_readable (): events | = self. READ_ONLY if handler. is_writable (): events | = self. READ_WRITE if events: if fileno in self. _ exists_fd: self. epoll. modify (fileno, events) else: self. _ exists_fd.update ({fileno: 1}) self. epoll. register (fileno, events) def _ prepare_io_handler (self, handler): ret = handler. prepare () if isinstance (ret, HandlerReady): del self. _ unprepared_handlers [handler] prepared = True elif isinstance (ret, PrepareAgain): if ret. timeout is not None: if self. _ timeout is not None: self. _ timeout = min (self. _ timeout, ret. timeout) else: self. _ timeout = ret. timeout prepared = False else: raise TypeError ("Unexpected result from prepare ()") return prepared def _ remove_io_handler (self, handler): if handler in self. _ unprepared_handlers: old_fileno = self. _ unprepared_handlers [handler] del self. _ unprepared_handlers [handler] else: old_fileno = handler. fileno () if old_fileno is not None: try: del self. _ handlers [old_fileno] self. _ exists. pop (old_fileno, None) self. epoll. unregister (old_fileno) Counter t KeyError: pass def loop_iteration (self, timeout = 60): next_timeout, sources_handled = self. _ call_timeout_handlers () if self. check_events (): return if self. _ quit: return sources_handled for handler in list (self. _ unprepared_handlers): self. _ configure_io_handler (handler) if self. _ timeout is not None: timeout = min (timeout, self. _ timeout) if next_timeout is not None: timeout = min (next_timeout, timeout) if timeout = 0: timeout + = 1 # non-blocking with timeout, cancel resource events = self. epoll. poll (timeout) for fd, flag in events: if flag & (select. EPOLLIN | select. EPOLLPRI | select. EPOLLET): self. _ handlers [fd]. handle_read () if flag & (select. EPOLLOUT | select. EPOLLET): self. _ handlers [fd]. handle_write () if flag & (select. EPOLLERR | select. EPOLLET): self. _ handlers [fd]. handle_err () if flag & (select. EPOLLHUP | select. EPOLLET): self. _ handlers [fd]. handle_hup () # if flag & select. EPOLLNVAL: # self. _ handlers [fd]. handle_nval () sources_handled + = 1 self. _ configure_io_handler (self. _ handlers [fd]) return sources_handled

Use

How to use the new mainloop? You only need to input

mainloop = EpollMainLoop(settings)client = Client(my_jid, [self, version_provider], settings, mainloop)

In this way, epoll will be used as the mainloop
Note:

Epoll is only supported in Linux