Zulip server and web application. Open-source team chat that helps teams stay productive and focused.
import pickle
import socket
from functools import lru_cache
from typing import Any
from bmemcached.protocol import Protocol
from django_bmemcached.memcached import BMemcached
from zerver.lib import zstd_level9
def _enable_tcp_keepalive(sock: socket.socket) -> None:
# Turn on TCP keepalives, with aggressive timers, on a freshly
# opened connection to memcached.
#
# bmemcached keeps a long-lived connection per thread and exposes no
# hook to configure the underlying socket, so without this a
# connection silently reaped while idle -- e.g. by a Kubernetes CNI
# or a stateful firewall dropping the idle conntrack entry -- is only
# noticed on its next use. That surfaces as a spurious "Cannot query
# memcached" failure (the lost write makes the round-trip read return
# nothing), most visibly on the every-10s /health check.
#
# Keepalives both keep the idle connection fresh (so it is less
# likely to be reaped) and detect a genuinely dead peer promptly
# rather than on next use.
try:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
# Start probing after 60s idle, then every 10s, dropping the
# connection after 3 failed probes.
#
# KEEPIDLE is the constant that matters: a probe every 60s keeps
# the flow from looking idle, so it is not reaped in the first
# place. It must sit comfortably below the shortest idle timeout
# on the path (conntrack / firewall); 60s clears the common
# aggressive reapers (~1-5min) with margin. KEEPINTVL * KEEPCNT
# (~30s) only governs how fast a genuinely dead peer is noticed,
# and is kept loose enough not to misfire on a transient blip.
#
# These constants are only available on Linux; guard so dev
# environments on other platforms still work.
if hasattr(socket, "TCP_KEEPIDLE"):
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 60)
if hasattr(socket, "TCP_KEEPINTVL"):
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 10)
if hasattr(socket, "TCP_KEEPCNT"):
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 3)
except OSError: # nocoverage
# Best-effort: never let socket tuning break cache connectivity.
pass
_original_open_connection = Protocol._open_connection
def _open_connection_with_keepalive(self: Protocol) -> None:
# Only apply keepalives when we actually open a new TCP connection;
# _open_connection() is called before every command and short-circuits
# when a connection already exists. self.host is falsey for
# Unix-domain sockets, where the TCP options do not apply.
was_connected = self.connection is not None
_original_open_connection(self)
if not was_connected and self.connection is not None and self.host:
_enable_tcp_keepalive(self.connection)
Protocol._open_connection = _open_connection_with_keepalive
@lru_cache(None)
def _get_bmemcached(location: str, param_bytes: bytes) -> BMemcached:
params = pickle.loads(param_bytes) # noqa: S301
params["OPTIONS"]["compression"] = zstd_level9
return BMemcached(location, params)
def SingletonBMemcached(location: str, params: dict[str, Any]) -> BMemcached:
# Django instantiates the cache backend per-task to guard against
# thread safety issues, but BMemcached is already thread-safe and
# does its own per-thread pooling, so make sure we instantiate only
# one to avoid extra connections.
return _get_bmemcached(location, pickle.dumps(params))