出現異常如下 RuntimeError: Second simultaneous read on fileno 8 detected. Unless you really know what you're doing, make sure that only one greenthread can read any particular socket. Consider using a pools.Pool. If you do know what you're doing and want to
disable this error, call eventlet.debug.hub_multiple_reader_prevention(False)
2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake File "/usr/lib/python2.7/dist-packages/memcache.py", line 862, in get2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake return self._get('get', key)2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake File "/usr/lib/python2.7/dist-packages/memcache.py", line 846, in _get2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake return _unsafe_get()2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake File "/usr/lib/python2.7/dist-packages/memcache.py", line 830, in _unsafe_get2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake rkey, flags, rlen, = self._expectvalue(server)2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake File "/usr/lib/python2.7/dist-packages/memcache.py", line 955, in _expectvalue2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake line = server.readline()2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake File "/usr/lib/python2.7/dist-packages/memcache.py", line 1125, in readline2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake data = recv(4096)2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake File "/usr/local/lib/python2.7/dist-packages/eventlet/greenio.py", line 249, in recv2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake timeout_exc=socket.timeout("timed out"))2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/__init__.py", line 117, in trampoline2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake listener = hub.add(hub.READ, fileno, current.switch)2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/poll.py", line 27, in add2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake listener = super(Hub, self).add(evtype, fileno, cb)2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/hub.py", line 126, in add2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake evtype, fileno, evtype))2013-03-08 18:27:27 17870 TRACE glance.store.chunkcache.fake RuntimeError: Second simultaneous read on fileno 8 detected. Unless you really know what you're doing, make sure that only one greenthread can read any particular socket. Consider using a pools.Pool. If you do know what you're doing and want to disable this error, call eventlet.debug.hub_multiple_reader_prevention(False)
除了這個之外再還有一堆的memcache.set返回0的情況。
出現問題時的大致模型如下
class CacheBackend(CacheBackendBase): ''' Cache in memcached servers. ''' def __init__(self): value_length = CONF.chunkcache_memcached_value_length servers = CONF.chunkcache_memcached_servers self.memcache = memcache.Client(servers, debug=1, server_max_value_length=value_length) @staticmethod def instance(): global CACHE_BACKEND if CACHE_BACKEND is None: CACHE_BACKEND = CacheBackend() return CACHE_BACKEND def get(self, key, default=None): result = self.memcache.get(str(key)) return result or default def exist(self, key): return self.get(str(key)) != None def set(self, key, value): result = self.memcache.set(str(key), value) if result == 0: raise exception.MemcacheSetError() return result def delete(self, key): if self.exist(key): result = self.memcache.delete(str(key)) if result == 0: raise exception.MemcacheDeleteError() return True def clear(self): # No need for clearing memcached. return True
經過同事交流,推測可能是因為fork之前有socket串連在或者是多個進程重用了一個socket導致,但是感覺都不太能講得通,對這塊確實不太瞭解,先記錄下來解決方案,有誰瞭解的求解釋~
下面是解決的模型,大致改動就是在每個set,get,delete處都new一個memcache client,
可以建立一個socket?不過看memcache模組,好像也是有socket緩衝的啊,會去緩衝中拿同個socket,
下面的disconnect_all加與不加都一樣,不會出現問題,而且在上面的模組中加上disconnect_all再重試也不起作用,
實在無法理解,但是確實解決了問題,多次測試都未複現上述問題。
class CacheBackend(CacheBackendBase): ''' Cache in memcached servers. ''' def __init__(self): self.value_length = CONF.chunkcache_memcached_value_length self.servers = CONF.chunkcache_memcached_servers @staticmethod def instance(): global CACHE_BACKEND if CACHE_BACKEND is None: CACHE_BACKEND = CacheBackend() return CACHE_BACKEND def get(self, key, default=None): client = _connect(self.servers, self.value_length) result = client.get(str(key)) client.disconnect_all() return result or default def exist(self, key): return self.get(str(key)) != None def set(self, key, value): client = _connect(self.servers, self.value_length) result = client.set(str(key), value) client.disconnect_all() if result == 0: raise exception.MemcacheSetError() return result def delete(self, key): client = _connect(self.servers, self.value_length) if self.exist(key): result = client.delete(str(key)) if result == 0: raise exception.MemcacheDeleteError() client.disconnect_all() return True def clear(self): # No need for clearing memcached. return Truedef _connect(servers, value_length=chunkstore.Store.CHUNKSIZE): return memcache.Client(servers, debug=1, server_max_value_length=value_length)
另外想解釋下,並不是單例或者維護一個client對象導致,去掉單例,問題一樣,而且並發是多進程並發的,也不應該存在什麼單例之類的問題。