LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-4123 [FIX] A credential-free REDIS_URL no longer connects anonymously (#2299)
ali6c0a69decf8d
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 9f5148370..aa03b577a 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -22,13 +22,18 @@ from utils.common_utils import CommonUtilsfrom utils.cors_origin import normalize_web_app_originfrom unstract.core.cache.redis_client import (+ apply_url_credentials,build_socketio_redis_url,ensure_tls_query_params,parse_db,+ parse_port,+ resolve_sentinel_master_check_hostname,resolve_ssl_cert_reqs,resolve_ssl_check_hostname,+ resolve_ssl_enabled,set_url_db_path,url_db_path,+ url_username_from_env,)# Django 5.0+ caps URLValidator at 2048 chars. S3 pre-signed URLs signed with@@ -112,12 +117,20 @@ GOOGLE_STORAGE_BASE_URL = os.environ.get("GOOGLE_STORAGE_BASE_URL")REDIS_USER = os.environ.get("REDIS_USER", "default")REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD", "")REDIS_HOST = os.environ.get("REDIS_HOST", "localhost")-REDIS_PORT = os.environ.get("REDIS_PORT", "6379")+# Through the shared parser, not a bare int() at the Sentinel call site below:+# a blank REDIS_PORT= — this repo's "leave the default" spelling — raised+# ValueError while Django settings were being imported, so the backend alone+# failed to start while every worker came up healthy on 6379.+REDIS_PORT = parse_port(os.environ.get("REDIS_PORT"), "REDIS_PORT", 6379)REDIS_DB = os.environ.get("REDIS_DB", "")# TLS to Redis (UN-4123). Off by default, so the in-cluster/local server is# untouched. `rediss://` is what actually selects TLS for both django-redis and# kombu; this flag only decides which scheme gets built.-REDIS_SSL = os.environ.get("REDIS_SSL", "false").strip().lower() == "true"+# Through the shared resolver, not a local `== "true"`: _TRUE_LITERALS accepts+# 1, yes and on, so the bare comparison read REDIS_SSL=1 as FALSE — the workers+# connected rediss:// while this cache built a redis:// LOCATION and skipped its+# CONNECTION_POOL_KWARGS, i.e. one endpoint with two TLS policies in one process.+REDIS_SSL = resolve_ssl_enabled()# Resolved by unstract.core, not re-read here: the raw value needs trimming,# lower-casing and validating, and a second copy of that logic is how this file# and create_redis_client came to hold two verification policies for one endpoint.@@ -552,18 +565,67 @@ REDIS_SENTINEL_MODE = (REDIS_SENTINEL_MASTER_NAME = os.environ.get("REDIS_SENTINEL_MASTER_NAME", "mymaster")if REDIS_SENTINEL_MODE:+ # This branch used to hold four divergences from create_redis_client, all+ # pre-existing. They are closed here because "the cache and the client reach+ # the same place" is not a property that can stop at a mode boundary — an+ # operator on Sentinel gets the same guarantee or the guarantee is a+ # half-truth. TestSentinelModeAgreesWithCore covers each one.+ #+ # The username comes from the shared resolver, not the module-level+ # REDIS_USER: that one defaults to "default", so this cache sent a+ # two-argument ACL AUTH where core sends the one-argument form, and it read+ # only REDIS_USER so the REDIS_USERNAME spelling platform-service ships was+ # ignored entirely.+ _sentinel_username = url_username_from_env()++ # 26379 — the Sentinel port — matching core's+ # _resolve_redis_env(default_port="26379"). The module-level REDIS_PORT+ # defaults to 6379, which is the standalone port, so an unset REDIS_PORT+ # pointed this cache at the wrong port while every other client found the+ # sentinels. The chart always sets it, which is why this stayed hidden.+ REDIS_PORT = parse_port(os.environ.get("REDIS_PORT"), "REDIS_PORT", 26379)+_sentinel_kwargs = {}if REDIS_PASSWORD:_sentinel_kwargs["password"] = REDIS_PASSWORD- if REDIS_USER:- _sentinel_kwargs["username"] = REDIS_USER+ if _sentinel_username:+ _sentinel_kwargs["username"] = _sentinel_username++ # TLS reached neither the discovery connections nor the master one, so a+ # TLS-only Sentinel deployment got a plaintext cache while core encrypted+ # both. Core builds them from one env dict for exactly this reason; the+ # same settings go to both here.+ _sentinel_pool_kwargs = {}+ if REDIS_SSL:+ _sentinel_tls = {"ssl": True, "ssl_cert_reqs": REDIS_SSL_CERT_REQS}+ if REDIS_SSL_CA_CERTS:+ _sentinel_tls["ssl_ca_certs"] = REDIS_SSL_CA_CERTS++ # The two planes answer by DIFFERENT rules, and copying one dict into+ # both got that wrong. Discovery connects to REDIS_HOST, a name whose+ # certificate can match, so it verifies like any other client. The+ # MASTER connects to whatever SENTINEL get-master-addr-by-name returns —+ # an IP that SentinelManagedConnection hands to SSLConnection as+ # server_hostname, which no DNS SAN covers and which changes on+ # failover. core turns checking off there unless the operator asked+ # explicitly; this cache has to agree, or it fails verification against+ # the same master that create_redis_client reaches.+ _sentinel_kwargs.update(_sentinel_tls)+ _sentinel_pool_kwargs.update(_sentinel_tls)+ if REDIS_SSL_CERT_REQS != "none":+ _sentinel_kwargs["ssl_check_hostname"] = REDIS_SSL_CHECK_HOSTNAME+ _sentinel_pool_kwargs["ssl_check_hostname"] = (+ resolve_sentinel_master_check_hostname()+ )_redis_db = parse_db(REDIS_DB, "REDIS_")# SocketIO connection manager (Kombu Sentinel URL format)_cred_prefix = ""- if REDIS_USER and REDIS_PASSWORD:- _cred_prefix = f"{quote(REDIS_USER, safe='')}:{quote(REDIS_PASSWORD, safe='')}@"+ if _sentinel_username and REDIS_PASSWORD:+ _cred_prefix = (+ f"{quote(_sentinel_username, safe='')}:{quote(REDIS_PASSWORD, safe='')}@"+ )elif REDIS_PASSWORD:_cred_prefix = f":{quote(REDIS_PASSWORD, safe='')}@"SOCKET_IO_MANAGER_URL = (@@ -572,7 +634,7 @@ if REDIS_SENTINEL_MODE:SOCKET_IO_TRANSPORT_OPTIONS = {"master_name": REDIS_SENTINEL_MASTER_NAME}# django-redis expects username in the LOCATION URL for ACL auth- _user_prefix = f"{REDIS_USER}@" if REDIS_USER else ""+ _user_prefix = f"{_sentinel_username}@" if _sentinel_username else ""CACHES = {"default": {"BACKEND": "django_redis.cache.RedisCache",@@ -581,8 +643,12 @@ if REDIS_SENTINEL_MODE:"CLIENT_CLASS": "django_redis.client.SentinelClient","CONNECTION_POOL_CLASS": "redis.sentinel.SentinelConnectionPool","CONNECTION_FACTORY": "django_redis.pool.SentinelConnectionFactory",- "SENTINELS": [(REDIS_HOST, int(REDIS_PORT))],+ "SENTINELS": [(REDIS_HOST, REDIS_PORT)],"SENTINEL_KWARGS": _sentinel_kwargs,+ # TLS for the MASTER connection. SENTINEL_KWARGS covers only the+ # discovery ones; core carries both, so a deployment with TLS on+ # got an encrypted discovery and a plaintext master.+ "CONNECTION_POOL_KWARGS": _sentinel_pool_kwargs,"DB": _redis_db,"PASSWORD": REDIS_PASSWORD,"SERIALIZER": "django_redis.serializers.json.JSONSerializer",@@ -643,23 +709,72 @@ else:# So the db has to travel in the URL path, or this cache silently sits on db 0# while every other service honours REDIS_DB: workers would RPUSH# log_history_queue to db N and the backend would LPOP an empty db 0.- # USERNAME and DB below are passed for readability and are DISCARDED by- # django-redis — they are not the mechanism for either. Auth stays password-only- # as the built-in `default` user (what a managed AUTH string is): a username- # would turn AUTH into its two-argument ACL form, and this cache never sends- # one. If a django-redis bump ever starts reading USERNAME, that becomes a real- # behaviour change rather than a silent one — which is what the assertions in- # tests/test_redis_settings_derivation.py pin.+ # DB below is passed for readability and is DISCARDED by django-redis — it is+ # not the mechanism. An ACL USERNAME is not passed through OPTIONS at all,+ # because django-redis would discard that too: it travels in the LOCATION,+ # the same route the URL branch uses. This used to say auth "stays+ # password-only as the built-in `default` user ... this cache never sends+ # one", which described the discrete path accurately and made it the one+ # consumer that could not reach an ACL endpoint — create_redis_client in the+ # same process sends the username, so where `default` is disabled the cache+ # alone failed. That is the identical divergence the URL branch was fixed+ # for, left standing on the path the module docstring calls primary._cache_options = {"CLIENT_CLASS": "django_redis.client.DefaultClient","SERIALIZER": "django_redis.serializers.json.JSONSerializer",}+ _cache_username = url_username_from_env()if not _redis_url:- # Credentials and db travel IN the URL in URL mode; passing them again- # through OPTIONS risks one of them winning over the other._cache_options["DB"] = _cache_db- _cache_options["USERNAME"] = REDIS_USER- _cache_options["PASSWORD"] = REDIS_PASSWORD+ _cache_location = f"{_scheme}://{REDIS_HOST}:{REDIS_PORT}/{_cache_db}"+ if _cache_username:+ # Only when a username must travel. Keeping the password in OPTIONS+ # otherwise is deliberate: OPTIONS["PASSWORD"] matches Django's+ # SafeExceptionReporterFilter and is masked in a settings dump,+ # while LOCATION is not. Pay that exposure only where django-redis+ # leaves no alternative.+ _cache_location = apply_url_credentials(+ _cache_location, REDIS_PASSWORD, _cache_username+ )+ else:+ _cache_options["PASSWORD"] = REDIS_PASSWORD+ else:+ # URL mode: the credentials go into the LOCATION, through the SAME helper+ # the Socket.IO URL uses. Two reasons it is not OPTIONS["PASSWORD"]:+ #+ # - django-redis 5.4.0 discards OPTIONS["USERNAME"], so an ACL username+ # could not reach this cache at all. It would authenticate as the+ # built-in `default` user while create_redis_client in the same+ # process authenticated as the configured one — and on an endpoint+ # where `default` is disabled, the cache alone would fail.+ # - a hand-rolled gate here diverged from the core one within a single+ # PR: it keyed on "@" being absent from the netloc, which the core+ # comment explicitly rejects, so a URL carrying only an ACL username+ # left this cache anonymous. One helper, one rule, no second copy to+ # keep in step.+ #+ # The helper returns the URL untouched when it already carries a+ # password, so a credential-bearing REDIS_URL behaves exactly as before.+ # The username comes from the SAME resolver the core client uses, not+ # from a hand-written os.environ.get here. Two reasons, and the second+ # was a live divergence:+ #+ # - the module-level REDIS_USER above defaults to "default", while+ # create_redis_client reads the variable with no default. Passing+ # the fallback would make this cache send the two-argument+ # `AUTH default <pw>` while the client in the same process sends the+ # one-argument form.+ # - the resolver accepts BOTH spellings, REDIS_USER and+ # REDIS_USERNAME. platform-service ships the second one, so a single+ # shared Redis-credential secret can easily inject it — and reading+ # only REDIS_USER left this cache authenticating as `default` while+ # the client beside it authenticated as the configured ACL user.+ #+ # Re-deriving the chain here is the same second copy this commit removed+ # for the "@" heuristic; asking core for it is what keeps the three+ # consumers in step by construction rather than by comment.+ _redis_url = apply_url_credentials(_redis_url, REDIS_PASSWORD, _cache_username)+ _cache_location = _redis_url# Gated on the EFFECTIVE scheme, not the flag. In URL mode TLS is carried by# the URL (and its query string), so a plaintext REDIS_URL left behind while# REDIS_SSL=true would otherwise hand ssl_cert_reqs to a plain@@ -697,11 +812,17 @@ else:# counters and the dashboard caches go with it, and during a rolling# deploy old pods read db 0 while new pods read db N. Drain# log_history_queue before cutting over.+ #+ # The message deliberately does NOT explain where the previous db came+ # from. It fires on both paths — discrete, where the cache sat on db 0+ # regardless because django-redis ignores OPTIONS['DB'], and URL mode,+ # where it came from the URL's own path — and the earlier wording gave+ # the discrete explanation to both. The db NUMBERS are right in every+ # case; only the stated reason was wrong on the URL path.logging.getLogger(__name__).warning(- "Django cache is moving from Redis db %s to db %s (REDIS_DB). In "- "discrete mode it previously sat on db 0 regardless, because "- "django-redis ignores OPTIONS['DB']. Anything already in db %s — "- "including log_history_queue — stays there.",+ "Django cache is moving from Redis db %s to db %s (REDIS_DB). "+ "Anything already in db %s — including log_history_queue — stays "+ "there.",_previous_db,_effective_db,_previous_db,@@ -710,8 +831,7 @@ else:CACHES = {"default": {"BACKEND": "django_redis.cache.RedisCache",- "LOCATION": _redis_url- or f"{_scheme}://{REDIS_HOST}:{REDIS_PORT}/{_cache_db}",+ "LOCATION": _cache_location,"OPTIONS": _cache_options,"KEY_FUNCTION": "utils.redis_cache.custom_key_function",}diff --git a/backend/backend/tests/test_redis_settings_derivation.py b/backend/backend/tests/test_redis_settings_derivation.pyindex 67d3aa0d7..7a474d707 100644--- a/backend/backend/tests/test_redis_settings_derivation.py+++ b/backend/backend/tests/test_redis_settings_derivation.py@@ -19,12 +19,81 @@ from __future__ import annotationsimport loggingimport pathlib+import re+from urllib.parse import urlsplitimport pytest_SETTINGS = pathlib.Path(__file__).resolve().parents[1] / "settings" / "base.py"+def _slice_bounds(source: str) -> dict[str, tuple[int, int]]:+ """Character ranges of base.py that _derive executes.++ Factored out so the coverage guard below can assert on the SAME ranges the+ harness runs, rather than a second description of them that could drift.+ """+ defs_start = source.index('REDIS_USER = os.environ.get("REDIS_USER"')+ defs_end = (+ source.index("\n", source.index('REDIS_URL = os.environ.get("REDIS_URL"')) + 1+ )+ imports_start = source.index("from unstract.core.cache.redis_client import (")+ imports_end = source.index(")\n", imports_start) + 2+ urllib_start = source.index("from urllib.parse import ")+ urllib_end = source.index("\n", urllib_start) + 1+ return {+ "urllib": (urllib_start, urllib_end),+ "imports": (imports_start, imports_end),+ "defs": (defs_start, defs_end),+ "block": (+ source.index("REDIS_SENTINEL_MODE = ("),+ source.index("SESSION_ENGINE ="),+ ),+ }+++def test_the_harness_covers_every_redis_line_in_the_settings_file():+ """The splice must not silently stop covering the code it claims to test.++ _derive executes two ranges out of base.py with a ~415-line gap between+ them, and anything in that gap is invisible to every assertion in this+ file. That is not theoretical: adding `REDIS_PASSWORD = ""` at base.py:251,+ or appending a CACHES["default"]["LOCATION"] override after SESSION_ENGINE,+ leaves all of these tests green while shipping a broken cache.++ The file already concedes the gap once — test_no_database_var_is_parsed_with+ _a_bare_int greps the source text because FILE_ACTIVE_CACHE_REDIS_DB sits+ outside both slices. A grep only covers the one pattern someone thought of;+ this covers the boundary itself, and fails naming the line that escaped.+ """+ source = _SETTINGS.read_text()+ bounds = _slice_bounds(source)+ covered = []+ for start, end in bounds.values():+ covered.append(range(start, end))++ offenders = []+ offset = 0+ for line in source.splitlines(keepends=True):+ if re.match(r"\s*(REDIS_|_redis|_cache|CACHES|SOCKET_IO)", line) and not any(+ offset in span for span in covered+ ):+ offenders.append((source[:offset].count("\n") + 1, line.strip()[:70]))+ offset += len(line)++ # Known and deliberate: these are asserted by source-text inspection+ # instead, because they are consumed far from the derivation block.+ allowed = {"FILE_ACTIVE_CACHE_REDIS_DB", "REDIS_DB_PORTAL"}+ offenders = [o for o in offenders if not any(a in o[1] for a in allowed)]++ assert not offenders, (+ "these Redis lines in base.py are OUTSIDE the ranges _derive executes, "+ "so no test in this file can observe them:\n"+ + "\n".join(f" base.py:{n}: {text}" for n, text in offenders)+ + "\nWiden the slice, or add the name to `allowed` with a reason."+ )++def _derive(**env: str) -> dict:"""Execute the standalone Redis block with the given env."""source = _SETTINGS.read_text()@@ -55,10 +124,17 @@ def _derive(**env: str) -> dict:# without it that branch raises NameError instead of logging, which is part of# why it went untested.ns: dict = {"__name__": "backend.settings.base"}+ # The urllib import is spliced from source for the same reason as the+ # unstract.core one: a hand-written copy must be remembered every time the+ # settings module starts using another name from it, and the symptom is a+ # NameError in every case rather than one clear failure.+ urllib_start = source.index("from urllib.parse import ")+ urllib_end = source.index("\n", urllib_start) + 1+prelude = (- "import logging\n"- "import os\n"- "from urllib.parse import quote\n" + source[imports_start:imports_end]+ "import logging\nimport os\n"+ + source[urllib_start:urllib_end]+ + source[imports_start:imports_end]) + source[defs_start:defs_end]import os as _os@@ -98,20 +174,35 @@ class TestDiscreteVars:derived = _derive(REDIS_HOST="h", REDIS_PASSWORD="s3cret")assert derived["CACHES"]["default"]["OPTIONS"]["PASSWORD"] == "s3cret"- def test_db_and_username_are_passed_through_options(self):- """Pins the stated invariant so a django-redis bump is visible.-- The comment beside this code says USERNAME is deliberately not honoured —- django-redis 5.4.0 discards it, so auth stays password-only as the- built-in `default` user. That holds by accident of the pinned version:- these assertions pin what the settings SEND, so if a bump starts reading- USERNAME the change is a deliberate one rather than a surprise.+ def test_an_acl_username_travels_in_the_location_not_options(self):+ """This used to assert OPTIONS["USERNAME"] == "alice" and call that the+ stated invariant — django-redis discards the key, so auth "stays+ password-only as the built-in `default` user".++ That made the discrete path the one consumer unable to reach an ACL+ endpoint: create_redis_client in the same process sends the username, so+ where `default` is disabled the cache alone failed. Passing a key the+ library throws away is not an invariant worth pinning; reaching the same+ server as everything else is. The LOCATION is the only route django-redis+ leaves, which is exactly why the URL branch already used it."""- options = _derive(REDIS_HOST="h", REDIS_DB="3", REDIS_USER="alice")["CACHES"][- "default"- ]["OPTIONS"]- assert options["DB"] == 3- assert options["USERNAME"] == "alice"+ derived = _derive(+ REDIS_HOST="h", REDIS_DB="3", REDIS_USER="alice", REDIS_PASSWORD="pw"+ )+ cache = derived["CACHES"]["default"]+ assert cache["OPTIONS"]["DB"] == 3+ assert "USERNAME" not in cache["OPTIONS"]+ parts = urlsplit(cache["LOCATION"])+ assert parts.username == "alice"+ assert parts.password == "pw"+ assert parts.hostname == "h"++ def test_no_username_keeps_the_password_out_of_the_location(self):+ """OPTIONS["PASSWORD"] is masked by Django's settings filter; LOCATION is+ not. Pay that exposure only where django-redis leaves no alternative."""+ cache = _derive(REDIS_HOST="h", REDIS_PASSWORD="pw")["CACHES"]["default"]+ assert cache["OPTIONS"]["PASSWORD"] == "pw"+ assert urlsplit(cache["LOCATION"]).password is Nonedef test_url_mode_does_not_duplicate_credentials_into_options(self):"""They travel in the URL; passing both risks one winning over the other."""@@ -130,7 +221,10 @@ class TestDiscreteVars:an empty db 0."""derived = _derive(REDIS_HOST="h", REDIS_DB="3")- assert derived["CACHES"]["default"]["LOCATION"].endswith("/3")+ # Full string, not endswith("/3"): the failure this guards is+ # same-endpoint-WRONG-db, and endswith is equally satisfied by+ # "redis://WRONGHOST:6379/3" or a changed port.+ assert derived["CACHES"]["default"]["LOCATION"] == "redis://h:6379/3"def test_ssl_switches_scheme_and_pool_kwargs(self):derived = _derive(REDIS_HOST="h", REDIS_SSL="true", REDIS_SSL_CA_CERTS="/ca.pem")@@ -438,3 +532,437 @@ class TestTheBackendUsesTheSharedParsers:"unstract.core.cache.redis_client.parse_db so the backend and the ""workers agree on a malformed value.")+++class TestUrlModeCacheCredentials:+ """The cache credentials travel in the LOCATION, through the shared helper.++ Not OPTIONS["PASSWORD"]: django-redis 5.4.0 discards OPTIONS["USERNAME"], so+ an ACL username could not reach this cache at all — it would authenticate as+ the built-in `default` user while create_redis_client in the same process+ used the configured one.++ A hand-rolled gate here also diverged from the core one within a single PR:+ it keyed on "@" being absent from the netloc, which the core comment+ explicitly rejects, leaving a username-only URL anonymous. One helper, one+ rule.+ """++ def _location(self, **env: str) -> str:+ return _derive(**env)["CACHES"]["default"]["LOCATION"]++ def test_the_password_fills_a_gap_the_url_leaves(self):+ assert (+ urlsplit(+ self._location(REDIS_URL="rediss://h:6380/0", REDIS_PASSWORD="s3cret")+ ).password+ == "s3cret"+ )++ def test_a_username_only_url_still_gets_the_password(self):+ """The shape the `@` heuristic got wrong."""+ parts = urlsplit(+ self._location(REDIS_URL="redis://alice@h:6379/0", REDIS_PASSWORD="s3cret")+ )+ assert parts.hostname == "h"+ assert parts.username == "alice"+ assert parts.password == "s3cret"++ def test_an_acl_username_reaches_the_cache(self):+ """django-redis discards OPTIONS["USERNAME"], so the LOCATION is the+ only lever. Without it the cache authenticates as `default` while the+ other consumers use the configured user.+ """+ parts = urlsplit(+ self._location(+ REDIS_URL="redis://h:6379/0", REDIS_PASSWORD="pw", REDIS_USER="alice"+ )+ )+ assert parts.username == "alice"+ assert parts.password == "pw"++ def test_a_url_carrying_credentials_is_left_alone(self):+ assert (+ urlsplit(+ self._location(+ REDIS_URL="rediss://:in-url@h:6380/0", REDIS_PASSWORD="s3cret"+ )+ ).password+ == "in-url"+ )++ def test_no_password_stays_anonymous(self):+ assert urlsplit(self._location(REDIS_URL="rediss://h:6380/0")).password is None++ def test_the_shipped_default_user_alone_adds_nothing(self):+ """REDIS_USER defaults to "default" in this module but not in+ create_redis_client; passing the fallback would make the two send+ different AUTH forms for the same configuration.++ The password must be NON-empty. With REDIS_PASSWORD="" the helper+ returns at its `if not password` guard before the username argument is+ read at all, so the test named for this choice could not fail on it —+ mutating the call to pass the module-level REDIS_USER left the suite+ green while changing a one-argument AUTH into a two-argument one.+ """+ assert (+ self._location(REDIS_URL="redis://h:6379/0", REDIS_PASSWORD="pw")+ == "redis://:pw@h:6379/0"+ )++ def test_the_cache_honours_the_second_username_spelling(self):+ """platform-service ships REDIS_USERNAME, the chart ships REDIS_USER.++ Reading only REDIS_USER left this cache authenticating as the built-in+ `default` while create_redis_client in the SAME process authenticated+ as the configured ACL user — and where `default` is disabled, only the+ cache fails.+ """+ assert (+ self._location(+ REDIS_URL="redis://h:6379/0", REDIS_PASSWORD="pw", REDIS_USERNAME="alice"+ )+ == "redis://alice:pw@h:6379/0"+ )++ def test_discrete_mode_still_uses_options(self):+ """URL mode moved to the LOCATION; the discrete path did not change."""+ options = _derive(REDIS_HOST="h", REDIS_PASSWORD="s3cret")["CACHES"]["default"][+ "OPTIONS"+ ]+ assert options["PASSWORD"] == "s3cret"+++class TestTheSslFlagParsesTheSameEverywhere:+ """REDIS_SSL decides TLS for the whole platform, so every consumer must+ read the same literals. A local `== "true"` here read `REDIS_SSL=1` as+ FALSE while unstract.core read it as True: the workers connected rediss://+ and this cache built a redis:// LOCATION and skipped its+ CONNECTION_POOL_KWARGS entirely — against a TLS-only managed endpoint the+ backend cache alone fails, and against one accepting both it authenticates+ in clear while everything else is encrypted.+ """++ @pytest.mark.parametrize("literal", ["true", "1", "yes", "on", "TRUE", " on "])+ def test_every_true_literal_switches_the_scheme(self, literal):+ derived = _derive(REDIS_HOST="h", REDIS_SSL=literal)+ assert derived["CACHES"]["default"]["LOCATION"].startswith("rediss://")+ assert derived["REDIS_SSL"] is True++ @pytest.mark.parametrize("literal", ["false", "0", "no", "off", "", " "])+ def test_every_false_and_blank_literal_leaves_it_plaintext(self, literal):+ derived = _derive(REDIS_HOST="h", REDIS_SSL=literal)+ assert derived["CACHES"]["default"]["LOCATION"].startswith("redis://")+ assert derived["REDIS_SSL"] is False++ def test_the_pool_kwargs_follow_the_same_flag(self):+ """The scheme and the verification settings must not disagree: the+ `if REDIS_SSL` gate guards both, so a literal one reader accepts and+ the other does not strips the cert settings as well as the scheme."""+ derived = _derive(REDIS_HOST="h", REDIS_SSL="1")+ assert derived["CACHES"]["default"]["OPTIONS"]["CONNECTION_POOL_KWARGS"]+++class TestTheBackendAgreesWithCoreOnEverySharedSetting:+ """A parity sweep, not another one-off.++ Four review rounds on this PR produced the same finding four times in+ different clothes: a setting was centralised in unstract.core and the+ parallel read in this settings module was left behind — the "@" credential+ heuristic, the REDIS_USERNAME spelling, blank-means-unset, and the SSL+ true-literals. Each was fixed with a test for that one setting, and the next+ round found the next one.++ This pins the PROPERTY instead: for a matrix of environments, what the+ Django cache ends up talking to must match what create_redis_client in the+ same process would. A future setting that is centralised on one side only+ fails here without anyone having to predict which setting it will be.+ """++ # Each case is what an operator actually writes, not a minimal pair.+ _CASES = [+ {"REDIS_HOST": "h"},+ {"REDIS_HOST": "h", "REDIS_PASSWORD": "pw"},+ {"REDIS_HOST": "h", "REDIS_PASSWORD": "pw", "REDIS_USER": "alice"},+ {"REDIS_HOST": "h", "REDIS_PASSWORD": "pw", "REDIS_USERNAME": "alice"},+ {"REDIS_HOST": "h", "REDIS_SSL": "true"},+ {"REDIS_HOST": "h", "REDIS_SSL": "1"},+ {"REDIS_HOST": "h", "REDIS_SSL": "yes", "REDIS_SSL_CERT_REQS": "none"},+ {"REDIS_URL": "redis://h:6379/0"},+ {"REDIS_URL": "redis://h:6379/0", "REDIS_PASSWORD": "pw"},+ {"REDIS_URL": "redis://h:6379/0", "REDIS_PASSWORD": "pw", "REDIS_USER": "alice"},+ {+ "REDIS_URL": "redis://h:6379/0",+ "REDIS_PASSWORD": "pw",+ "REDIS_USERNAME": "bob",+ },+ {"REDIS_URL": "redis://alice@h:6379/0", "REDIS_PASSWORD": "pw"},+ {"REDIS_URL": "redis://:inurl@h:6379/0", "REDIS_PASSWORD": "pw"},+ {"REDIS_URL": "redis://:@h:6379/0", "REDIS_PASSWORD": "pw"},+ {"REDIS_URL": "rediss://h:6380/0", "REDIS_PASSWORD": "pw"},+ {"REDIS_URL": "rediss://h:6380/0", "REDIS_PASSWORD": "p@ss/w:rd"},+ {"REDIS_URL": "redis://h:6379/0", "REDIS_DB": "3", "REDIS_PASSWORD": "pw"},+ # Username with NO password: neither side can honour it, so what the+ # sweep pins is that they fail the SAME way rather than one going+ # anonymous while the other crashes inside redis-py.+ {"REDIS_HOST": "h", "REDIS_USER": "alice"},+ {"REDIS_URL": "redis://h:6379/0", "REDIS_USERNAME": "alice"},+ {"REDIS_HOST": "h", "REDIS_PASSWORD": "pw", "REDIS_USER": ""},+ {"REDIS_HOST": "h", "REDIS_PASSWORD": "pw", "REDIS_SSL": ""},+ ]++ @staticmethod+ def _from_core(env: dict) -> dict:+ """What create_redis_client would connect with, for this environment."""+ import os++ from unstract.core.cache.redis_client import create_redis_client++ saved = {k: os.environ.get(k) for k in list(os.environ) if "REDIS" in k}+ for key in saved:+ os.environ.pop(key, None)+ os.environ.update(env)+ try:+ kwargs = create_redis_client().connection_pool.connection_kwargs+ cls = create_redis_client().connection_pool.connection_class.__name__+ finally:+ for key in list(os.environ):+ if "REDIS" in key:+ os.environ.pop(key, None)+ os.environ.update({k: v for k, v in saved.items() if v is not None})+ return {+ "host": kwargs.get("host"),+ "port": kwargs.get("port"),+ "db": int(kwargs.get("db") or 0),+ "username": kwargs.get("username") or None,+ "password": kwargs.get("password") or None,+ "tls": cls == "SSLConnection",+ }++ @staticmethod+ def _from_cache(env: dict) -> dict:+ """The same facts, resolved by django-redis ITSELF.++ Not by parsing the LOCATION string. django-redis's precedence — the+ LOCATION's userinfo wins, OPTIONS["PASSWORD"] only backfills — is true+ of 5.4.0, but hardcoding it here would be a second copy of library+ behaviour inside the very test whose job is to survive changes it cannot+ predict. A bump that reversed the precedence would leave this green+ while the cache diverged, which is the exact failure mode this class+ replaced. Asking ConnectionFactory means the rule is never restated.+ """+ from django_redis.pool import ConnectionFactory++ derived = _derive(**env)+ cache = derived["CACHES"]["default"]+ options = {+ k: v+ for k, v in cache["OPTIONS"].items()+ if k not in ("CLIENT_CLASS", "SERIALIZER")+ }+ # The factory memoises pools by LOCATION, so a stale entry from an+ # earlier case would answer for this one.+ ConnectionFactory._pools = {}+ client = ConnectionFactory(dict(options)).connect(cache["LOCATION"])+ pool = client.connection_pool+ kwargs = pool.connection_kwargs+ return {+ "host": kwargs.get("host"),+ "port": int(kwargs.get("port") or derived["REDIS_PORT"]),+ "db": int(kwargs.get("db") or 0),+ "username": kwargs.get("username") or None,+ "password": kwargs.get("password") or None,+ "tls": pool.connection_class.__name__ == "SSLConnection",+ }++ @pytest.mark.parametrize("env", _CASES, ids=lambda e: ",".join(sorted(e)))+ def test_the_cache_and_the_client_reach_the_same_place(self, env):+ assert self._from_cache(env) == self._from_core(env)+++class TestSentinelModeAgreesWithCore:+ """The parity property must not stop at a mode boundary.++ TestTheBackendAgreesWithCoreOnEverySharedSetting cannot cover Sentinel: its+ _from_core calls create_redis_client, which in Sentinel mode performs real+ discovery and blocks. So the comparison here is against the resolver rather+ than a built client — _resolve_redis_env with the same default_port core's+ Sentinel path passes — which is enough to pin the four facts that diverged.++ The branch was NOT untestable, which an earlier comment in base.py claimed:+ _derive's slice spans both branches, so it just needed an env that selects+ this one. "No test executes it" and "no test was written for it" are+ different statements, and only the second was true.+ """++ @staticmethod+ def _core(**env: str) -> dict:+ import os++ from unstract.core.cache.redis_client import _resolve_redis_env++ saved = {k: os.environ.get(k) for k in list(os.environ) if "REDIS" in k}+ for key in saved:+ os.environ.pop(key, None)+ os.environ.update(env)+ try:+ return _resolve_redis_env("REDIS_", default_port="26379")+ finally:+ for key in list(os.environ):+ if "REDIS" in key:+ os.environ.pop(key, None)+ os.environ.update({k: v for k, v in saved.items() if v is not None})++ def test_the_sentinel_port_default_matches_core(self):+ """REDIS_PORT unset pointed this cache at 6379 — the STANDALONE port —+ while every other client found the sentinels on 26379."""+ env = {"REDIS_SENTINEL_MODE": "true", "REDIS_HOST": "sent"}+ sentinels = _derive(**env)["CACHES"]["default"]["OPTIONS"]["SENTINELS"]+ assert sentinels == [("sent", self._core(**env)["port"])]+ assert sentinels == [("sent", 26379)]++ def test_an_explicit_port_still_wins(self):+ env = {"REDIS_SENTINEL_MODE": "true", "REDIS_HOST": "sent", "REDIS_PORT": "27000"}+ assert _derive(**env)["CACHES"]["default"]["OPTIONS"]["SENTINELS"] == [+ ("sent", 27000)+ ]++ def test_no_username_configured_means_no_username_sent(self):+ """REDIS_USER defaults to "default" at module scope, so this cache sent+ a two-argument ACL AUTH where core sends the one-argument form."""+ env = {+ "REDIS_SENTINEL_MODE": "true",+ "REDIS_HOST": "sent",+ "REDIS_PASSWORD": "pw",+ }+ cache = _derive(**env)["CACHES"]["default"]+ assert self._core(**env)["username"] is None+ assert "username" not in cache["OPTIONS"]["SENTINEL_KWARGS"]+ assert urlsplit(cache["LOCATION"]).username is None++ @pytest.mark.parametrize("spelling", ["REDIS_USER", "REDIS_USERNAME"])+ def test_both_username_spellings_reach_the_sentinel_cache(self, spelling):+ """platform-service ships REDIS_USERNAME; reading only REDIS_USER left+ this cache authenticating as `default` while core used the ACL user."""+ env = {+ "REDIS_SENTINEL_MODE": "true",+ "REDIS_HOST": "sent",+ "REDIS_PASSWORD": "pw",+ spelling: "alice",+ }+ derived = _derive(**env)+ cache = derived["CACHES"]["default"]+ assert self._core(**env)["username"] == "alice"+ assert cache["OPTIONS"]["SENTINEL_KWARGS"]["username"] == "alice"+ assert urlsplit(cache["LOCATION"]).username == "alice"+ assert urlsplit(derived["SOCKET_IO_MANAGER_URL"]).username == "alice"++ def test_tls_reaches_both_the_sentinels_and_the_master(self):+ """SENTINEL_KWARGS covers only discovery. Carrying TLS there alone would+ leave an encrypted discovery and a plaintext master; carrying it nowhere+ — which is what this branch did — leaves a plaintext cache against a+ TLS-only deployment while core encrypts both."""+ env = {+ "REDIS_SENTINEL_MODE": "true",+ "REDIS_HOST": "sent",+ "REDIS_SSL": "true",+ "REDIS_SSL_CERT_REQS": "none",+ }+ options = _derive(**env)["CACHES"]["default"]["OPTIONS"]+ assert self._core(**env)["ssl"] is True+ for where in ("SENTINEL_KWARGS", "CONNECTION_POOL_KWARGS"):+ assert options[where]["ssl"] is True, where+ assert options[where]["ssl_cert_reqs"] == "none", where++ def test_a_ca_reaches_both_too(self):+ env = {+ "REDIS_SENTINEL_MODE": "true",+ "REDIS_HOST": "sent",+ "REDIS_SSL": "true",+ "REDIS_SSL_CA_CERTS": "/etc/ssl/ca.pem",+ }+ options = _derive(**env)["CACHES"]["default"]["OPTIONS"]+ for where in ("SENTINEL_KWARGS", "CONNECTION_POOL_KWARGS"):+ assert options[where]["ssl_ca_certs"] == "/etc/ssl/ca.pem", where++ def test_no_tls_configured_adds_no_tls_keys(self):+ """Flag off must leave the Sentinel cache byte-identical to before."""+ options = _derive(REDIS_SENTINEL_MODE="true", REDIS_HOST="sent")["CACHES"][+ "default"+ ]["OPTIONS"]+ assert options["SENTINEL_KWARGS"] == {}+ assert options["CONNECTION_POOL_KWARGS"] == {}+++class TestSentinelTlsPlanesDifferCorrectly:+ """Discovery and master verify by DIFFERENT rules, and one dict served both.++ Discovery connects to REDIS_HOST — a name whose certificate can match — so+ it verifies like any other client. The master connects to whatever+ `SENTINEL get-master-addr-by-name` returns, which SentinelManagedConnection+ hands to SSLConnection as `server_hostname`: an IP no DNS SAN covers, and+ one that changes on failover. core turns checking off there unless asked+ explicitly, so this cache must too — otherwise it fails verification against+ the very master create_redis_client connects to.+ """++ @staticmethod+ def _options(**env: str) -> dict:+ return _derive(REDIS_SENTINEL_MODE="true", REDIS_HOST="sent", **env)["CACHES"][+ "default"+ ]["OPTIONS"]++ def test_the_master_does_not_verify_the_hostname_by_default(self):+ options = self._options(REDIS_SSL="true")+ assert options["SENTINEL_KWARGS"]["ssl_check_hostname"] is True+ assert options["CONNECTION_POOL_KWARGS"]["ssl_check_hostname"] is False++ def test_an_explicit_request_is_honoured_on_the_master(self):+ """An operator who asks for it still gets it — core says the same."""+ options = self._options(REDIS_SSL="true", REDIS_SSL_CHECK_HOSTNAME="true")+ assert options["CONNECTION_POOL_KWARGS"]["ssl_check_hostname"] is True++ def test_an_explicit_false_is_honoured_on_both(self):+ options = self._options(REDIS_SSL="true", REDIS_SSL_CHECK_HOSTNAME="false")+ assert options["SENTINEL_KWARGS"]["ssl_check_hostname"] is False+ assert options["CONNECTION_POOL_KWARGS"]["ssl_check_hostname"] is False++ def test_cert_reqs_none_turns_checking_off_on_both_planes(self):+ """redis-py 5.2.1 assigns check_hostname verbatim — it does NOT coerce+ the pair — so CERT_NONE with checking on reaches Python's ssl module,+ which rejects it. Verified against the pinned version."""+ options = self._options(REDIS_SSL="true", REDIS_SSL_CERT_REQS="none")+ assert "ssl_check_hostname" not in options["SENTINEL_KWARGS"]+ assert options["CONNECTION_POOL_KWARGS"]["ssl_check_hostname"] is False++ def test_the_master_matches_core_for_every_spelling(self):+ """Pinned to the resolver core itself uses, not to a second copy."""+ import os++ from unstract.core.cache.redis_client import (+ resolve_sentinel_master_check_hostname,+ )++ for raw in (None, "true", "false", "1", "off"):+ env = {+ "REDIS_SENTINEL_MODE": "true",+ "REDIS_HOST": "sent",+ "REDIS_SSL": "true",+ }+ if raw is not None:+ env["REDIS_SSL_CHECK_HOSTNAME"] = raw+ derived = _derive(**env)["CACHES"]["default"]["OPTIONS"]+ saved = {k: os.environ.get(k) for k in list(os.environ) if "REDIS" in k}+ for key in saved:+ os.environ.pop(key, None)+ os.environ.update(env)+ try:+ expected = resolve_sentinel_master_check_hostname()+ finally:+ for key in list(os.environ):+ if "REDIS" in key:+ os.environ.pop(key, None)+ os.environ.update({k: v for k, v in saved.items() if v is not None})+ assert derived["CONNECTION_POOL_KWARGS"]["ssl_check_hostname"] == expected, (+ raw+ )diff --git a/backend/sample.env b/backend/sample.envindex daea19134..ca5456167 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -60,7 +60,13 @@ REDIS_SENTINEL_MASTER_NAME=mymaster# 1. Discrete vars (what the Helm chart and these samples use). Set REDIS_SSL=true# alongside REDIS_HOST/REDIS_PORT. No URL-encoding to get wrong.# 2. REDIS_URL, where the SCHEME carries TLS and nothing else is needed:-# REDIS_URL=rediss://:<password>@<host>:6380/0?ssl_cert_reqs=required+# REDIS_URL=rediss://<host>:6380/0?ssl_cert_reqs=required+# REDIS_PASSWORD=<password>+# REDIS_USER= # blank — see below+# (The password is better kept OUT of the URL. A URL is printed into log+# lines, error messages and deployment tooling; a password in one travels+# with it. Credentials in the URL still work and still win, so an existing+# rediss://:<password>@host URL keeps behaving exactly as before.)# (6380 is an EXAMPLE, not a default — the TLS port is provider-specific:# Memorystore 6378, ElastiCache 6379, Azure Cache 6380. A wrong port# hangs the connection rather than erroring, so read it off the instance.)diff --git a/runner/sample.env b/runner/sample.envindex efdf245a1..23a9bc507 100644--- a/runner/sample.env+++ b/runner/sample.env@@ -55,7 +55,13 @@ REDIS_SENTINEL_MASTER_NAME=mymaster# 1. Discrete vars (what the Helm chart and these samples use). Set REDIS_SSL=true# alongside REDIS_HOST/REDIS_PORT. No URL-encoding to get wrong.# 2. REDIS_URL, where the SCHEME carries TLS and nothing else is needed:-# REDIS_URL=rediss://:<password>@<host>:6380/0?ssl_cert_reqs=required+# REDIS_URL=rediss://<host>:6380/0?ssl_cert_reqs=required+# REDIS_PASSWORD=<password>+# REDIS_USER= # blank — see below+# (The password is better kept OUT of the URL. A URL is printed into log+# lines, error messages and deployment tooling; a password in one travels+# with it. Credentials in the URL still work and still win, so an existing+# rediss://:<password>@host URL keeps behaving exactly as before.)# (6380 is an EXAMPLE, not a default — the TLS port is provider-specific:# Memorystore 6378, ElastiCache 6379, Azure Cache 6380. A wrong port# hangs the connection rather than erroring, so read it off the instance.)diff --git a/unstract/core/src/unstract/core/cache/redis_client.py b/unstract/core/src/unstract/core/cache/redis_client.pyindex dbf30ba50..ca6c43837 100644--- a/unstract/core/src/unstract/core/cache/redis_client.py+++ b/unstract/core/src/unstract/core/cache/redis_client.py@@ -136,15 +136,17 @@ def url_db_path(url: str) -> int | None:return None-def _parse_bool(raw: str, default: bool, name: str) -> bool:+def _parse_bool(raw: str | None, default: bool, name: str) -> bool:"""Parse a boolean env var; blank means UNSET, unknown warns and defaults.Blank-means-unset is this repo's own convention — `FOO=` in a sample.envmeans "leave the default", and every other variable in UN-4123 treats it thatway. `os.getenv` does not: it reports an empty string as SET, so a bare`os.getenv(...) == "true"` turns `FOO=` into False.++ Accepts None so callers can hand it env_chain's "nothing was set" directly."""- value = raw.strip().lower()+ value = (raw or "").strip().lower()if not value:return defaultif value in _TRUE_LITERALS:@@ -190,9 +192,7 @@ def resolve_ssl_cert_reqs(env_prefix: str = "REDIS_") -> str:endpoint, three verification policies. Case and stray whitespace had the sameeffect, since the "is verification off?" test is an equality check."""- raw = os.getenv(- f"{env_prefix}SSL_CERT_REQS", os.getenv("REDIS_SSL_CERT_REQS", "")- ).strip()+ raw = (env_chain(f"{env_prefix}SSL_CERT_REQS", "REDIS_SSL_CERT_REQS") or "").strip()value = raw.lower()if not value:return _DEFAULT_CERT_REQS@@ -208,6 +208,32 @@ def resolve_ssl_cert_reqs(env_prefix: str = "REDIS_") -> str:return value+def resolve_sentinel_master_check_hostname(env_prefix: str = "REDIS_") -> bool:+ """ssl_check_hostname for a Sentinel-managed MASTER connection.++ The master plane answers by a DIFFERENT rule than the discovery plane, and+ exporting it is what stops a consumer re-deriving it: a master connects to+ whatever `SENTINEL get-master-addr-by-name` returns, and+ SentinelManagedConnection assigns that address straight to ``self.host``,+ which SSLConnection passes as ``server_hostname``. It is an IP, so+ verification is checked against an address no DNS SAN covers — and pinning+ an IP SAN is not a workable answer, because the address changes on failover.++ So it is OFF unless the operator asked for it explicitly, matching+ _tls_kwargs(sentinel_master=True). Off as well when verification itself is+ off: the pinned redis-py (5.2.1) does NOT coerce that pair — it assigns+ check_hostname verbatim — so CERT_NONE with checking on reaches Python's ssl+ module, which rejects it.+ """+ raw, _ = env_chain_named(+ f"{env_prefix}SSL_CHECK_HOSTNAME", "REDIS_SSL_CHECK_HOSTNAME"+ )+ explicit = (raw or "").strip().lower() in _TRUE_LITERALS | _FALSE_LITERALS+ if not explicit or resolve_ssl_cert_reqs(env_prefix) == "none":+ return False+ return resolve_ssl_check_hostname(env_prefix)++def resolve_ssl_check_hostname(env_prefix: str = "REDIS_", default: bool = True) -> bool:"""{prefix}SSL_CHECK_HOSTNAME, falling back to REDIS_SSL_CHECK_HOSTNAME, then on.@@ -216,13 +242,10 @@ def resolve_ssl_check_hostname(env_prefix: str = "REDIS_", default: bool = True)FALSE, silently downgrading the Django cache to an encrypted butunauthenticated connection."""- return _parse_bool(- os.getenv(- f"{env_prefix}SSL_CHECK_HOSTNAME", os.getenv("REDIS_SSL_CHECK_HOSTNAME", "")- ),- default,- f"{env_prefix}SSL_CHECK_HOSTNAME",+ raw, name = env_chain_named(+ f"{env_prefix}SSL_CHECK_HOSTNAME", "REDIS_SSL_CHECK_HOSTNAME")+ return _parse_bool(raw, default, name)def url_cert_reqs(url: str) -> str | None:@@ -322,7 +345,8 @@ def ensure_tls_query_params("""Add the TLS settings a `rediss://` URL is missing, leaving present ones alone.Anything that hands a URL to a library that reads TLS out of the query string- needs this: kombu's KombuManager takes a URL and NOTHING else, and+ needs this: kombu's KombuManager reads TLS from the URL alone — its+ connection_options reach kombu.Connection, but the TLS settings do not — anddjango-redis's LOCATION is a string too (it also readsOPTIONS["CONNECTION_POOL_KWARGS"], but the query string wins on conflict).@@ -380,6 +404,156 @@ def _compose_redis_url(env: dict[str, Any]) -> str:return f"{scheme}://{credentials}{env['host']}:{env['port']}"+def env_chain(*names: str) -> str | None:+ """First NON-BLANK value among these env vars, else None.++ Blank means unset throughout this module — it is the "leave the default"+ spelling the sample recipes and values files use, and it is why a nested+ `os.getenv(prefixed, os.getenv(generic))` is wrong here: `os.getenv` reports+ a set-but-empty variable as set, so the blank shadows the level below it.+ """+ for name in names:+ value = os.getenv(name, "")+ # Emptiness is tested on the STRIPPED value; the RAW one is returned.+ # Stripping the return value silently truncated a password or username+ # that came from a file-mounted secret with a trailing newline, while+ # backend/settings/base.py read the same variable unstripped — so one+ # process authenticated two different ways against one endpoint.+ if value.strip():+ return value+ return None+++def env_chain_named(*names: str) -> tuple[str | None, str]:+ """env_chain, plus the name of the variable the value came from.++ So a warning about an unusable value can name the variable that is actually+ SET. Reporting the prefixed name for a value that came from the generic one+ sends an operator grepping for a key that does not exist in their config.+ """+ for name in names:+ value = os.getenv(name, "")+ if value.strip():+ return value, name+ return None, names[0]+++def parse_port(raw: str | None, var_name: str, default_port: str | int) -> int:+ """A port number; blank means unset, unparseable warns and falls back.++ parse_db already does this for the database and says why: an int() straight+ out of os.getenv raises at client construction with a message naming neither+ the variable nor the prefix, which in most consumers kills the process at+ import and in the two that catch broadly degrades to no-cache.+ """+ # Blank is unset, like everywhere else here. base.py hands this the raw env+ # value rather than env_chain's None, so without this a bare `REDIS_PORT=` —+ # the repo's own "leave the default" spelling — logged an ERROR at startup+ # in the backend and nowhere else.+ if raw is None or not raw.strip():+ return int(default_port)+ try:+ return int(raw)+ except ValueError:+ # error, not warning: the fallback is a PORT, and on a host serving both+ # 6379 and 6380 a typo lands the client on the other server rather than+ # failing — a wrong endpoint, reported as a wrong value.+ logger.error("Invalid %s=%r; using %s", var_name, raw, default_port)+ return int(default_port)+++def resolve_ssl_enabled(env_prefix: str = "REDIS_") -> bool:+ """{prefix}SSL, falling back to REDIS_SSL; blank means unset.++ Exported so the backend's settings module uses the SAME parse rather than+ its own `== "true"`. _TRUE_LITERALS accepts 1, yes and on, so the bare+ comparison read `REDIS_SSL=1` as FALSE — every create_redis_client consumer+ connected rediss:// while the Django cache built a redis:// LOCATION and+ skipped its CONNECTION_POOL_KWARGS entirely. One endpoint, one process, two+ TLS policies, which is the failure this module was centralised to prevent.+ """+ raw, name = env_chain_named(f"{env_prefix}SSL", "REDIS_SSL")+ return _parse_bool(raw, False, name)+++def url_username_from_env(env_prefix: str = "REDIS_") -> str | None:+ """The ACL username for a URL, in BOTH spellings that ship.++ `{prefix}USER` is what the chart writes; `{prefix}USERNAME` is what+ platform-service/sample.env writes and platform_service/env.py reads. One+ shared Redis-credential secret naturally injects whichever name its author+ picked, so a consumer that reads only one of them authenticates as a+ different user than the client beside it in the same process — and on an+ endpoint where the built-in `default` is disabled, only that one consumer+ fails.++ Empty means absent, matching the rest of this module: `REDIS_USER=` is the+ "leave it unset" spelling the sample recipes use, and `os.getenv(name,+ fallback)` would return that blank instead of falling through to the second+ spelling.+ """+ return env_chain(f"{env_prefix}USER", f"{env_prefix}USERNAME")+++def apply_url_credentials(url: str, password: str | None, username: str | None) -> str:+ """Put credentials into a URL that carries none, leaving one that does alone.++ KombuManager reads TLS out of the query string only, so the URL is where TLS+ has to go, and the credentials ride with it so ONE builder serves both the+ kombu and the redis-py callers. That is a choice, not a constraint:+ KombuManager forwards connection_options to kombu.Connection, which accepts+ userid= and password=.++ A URL written without credentials otherwise produces an anonymous publisher+ against an authenticated server, and Socket.IO events simply stop arriving.++ The password IS in the returned string. Smaller exposure than a values file,+ but not none, and it now has TWO landing places in Django settings:+ SOCKET_IO_MANAGER_URL, and CACHES["default"]["LOCATION"]. Neither name+ matches SafeExceptionReporterFilter's `API|TOKEN|KEY|SECRET|PASS|SIGNATURE|+ HTTP_COOKIE`, so neither is redacted on a technical-500 page, in+ `manage.py diffsettings`, or in any settings dump — whereas+ OPTIONS["PASSWORD"], where the cache password used to sit, IS masked.+ The LOCATION is also the key of django-redis's process-global pool dict.++ This is a deliberate trade, not an oversight: django-redis 5.4.0 discards+ OPTIONS["USERNAME"], so the LOCATION is the ONLY route by which an ACL+ username can reach that cache. Stated here because the call site in+ backend/settings/base.py is where someone will look for the risk and the+ reasoning lives on this side of the boundary. One more reason DEBUG must+ stay off.+ """+ if not password:+ return url+ parts = urlsplit(url)+ # Truthiness, not `is not None`: `redis://:@host` parses to password "", and+ # both redis-py (`if url.password:`) and kombu (`unquote(password or "") or+ # None`) read that as absent.+ if parts.password:+ return url+ # REBUILD the netloc; do not prepend to it. parts.netloc INCLUDES any+ # userinfo, so prepending on a username-only URL produced+ # `alice:pw@alice@host` — and userinfo splits on the LAST "@", so the+ # password parsed as "pw@alice". A wrong credential, not a missing one.+ #+ # A username already in the URL is reused VERBATIM rather than re-quoted: it+ # is percent-encoded already, and quoting it again turned `al%40ice` into+ # `al%2540ice`.+ userinfo, _, host_port = parts.netloc.rpartition("@")+ url_user = userinfo.partition(":")[0]+ user = url_user or (quote(str(username), safe="") if username else "")+ credentials = f"{user}:{quote(str(password), safe='')}@"+ return urlunsplit(+ (+ parts.scheme,+ credentials + host_port,+ parts.path,+ parts.query,+ parts.fragment,+ )+ )++def build_socketio_redis_url(env_prefix: str = "REDIS_") -> str:"""Redis URL for a Socket.IO/kombu client, TLS settings included (UN-4123).@@ -415,6 +589,9 @@ def build_socketio_redis_url(env_prefix: str = "REDIS_") -> str:ca_certs = env.get("ssl_ca_certs")url = env["url"] or _compose_redis_url(env)+ # _compose_redis_url already embeds them for the discrete path; a configured+ # URL may not carry any, and kombu can read them from nowhere else.+ url = apply_url_credentials(url, env.get("url_password"), env.get("url_username"))if not url.startswith(_TLS_SCHEME):return url@@ -431,13 +608,45 @@ def _resolve_redis_env(env_prefix: str, default_port: str = "6379", db_override: int | None = None) -> dict[str, Any]:"""Read common Redis env vars into a dict."""- host = os.getenv(f"{env_prefix}HOST", os.getenv("REDIS_HOST", "localhost"))- port = int(os.getenv(f"{env_prefix}PORT", os.getenv("REDIS_PORT", default_port)))- password = os.getenv(f"{env_prefix}PASSWORD", os.getenv("REDIS_PASSWORD"))- username = os.getenv(- f"{env_prefix}USER",- os.getenv(f"{env_prefix}USERNAME", os.getenv("REDIS_USER")),+ # Stripped explicitly: env_chain returns the RAW value so a credential keeps+ # any whitespace it was given, but a host must not. urlsplit() drops a+ # trailing newline from a URL host while redis.Redis(host=...) keeps it, so+ # a file-mounted REDIS_HOST ending in \n would fail DNS on the discrete path+ # and succeed on the URL one.+ host = (env_chain(f"{env_prefix}HOST", "REDIS_HOST") or "localhost").strip()+ port_raw, port_name = env_chain_named(f"{env_prefix}PORT", "REDIS_PORT")+ port = parse_port(port_raw, port_name, default_port)+ # BLANK MEANS UNSET, like parse_db and _parse_bool in this same module — a+ # nested os.getenv(prefixed, generic) takes the blank and never consults the+ # generic one. workers/sample.env ships `CACHE_REDIS_PASSWORD=` uncommented,+ # so an operator following this module's own recipe (REDIS_URL without+ # credentials + REDIS_PASSWORD) got url_password="" for every prefixed+ # client, apply_url_credentials no-opped, and the worker caches connected+ # ANONYMOUSLY — the exact failure that helper exists to eliminate,+ # reintroduced by an empty variable.+ password = env_chain(f"{env_prefix}PASSWORD", "REDIS_PASSWORD")+ username = env_chain(+ f"{env_prefix}USER", f"{env_prefix}USERNAME", "REDIS_USER", "REDIS_USERNAME")+ # An ACL username without a password is not a configuration Redis has. AUTH+ # takes either one argument or two, so redis-py packs `AUTH alice None` and+ # raises `DataError: Invalid input of type: 'NoneType'` on the FIRST command+ # — a type error from inside the library, nowhere near the values file that+ # caused it. Said plainly at configuration time instead, and the username is+ # dropped so this client degrades to the same anonymous connection the+ # Django cache already makes for this input (apply_url_credentials returns+ # the URL untouched when there is no password), rather than the two+ # disagreeing about a config neither can honour.+ if username and not password:+ logger.error(+ "%sUSER=%r is set but no password is; Redis has no one-argument ACL "+ "AUTH, so the username is being ignored. Set %sPASSWORD, or clear "+ "the username.",+ env_prefix,+ username,+ env_prefix,+ )+ username = Noneprefixed_db = os.getenv(f"{env_prefix}DB", "").strip()generic_db = os.getenv("REDIS_DB", "").strip()db = (@@ -449,10 +658,12 @@ def _resolve_redis_env(# turning TLS on platform-wide meant remembering CACHE_REDIS_SSL and# MANUAL_REVIEW_REDIS_SSL too — and a missed one fails as a plaintext client# talking to a TLS port, not as a config error.- ssl = (- os.getenv(f"{env_prefix}SSL", os.getenv("REDIS_SSL", "false")).strip().lower()- == "true"- )+ # BLANK MEANS UNSET here too. It did not, and the asymmetry was dangerous+ # rather than merely untidy: once password/user fell through a blank prefixed+ # value but TLS did not, a blank CACHE_REDIS_SSL beside REDIS_SSL=true gave+ # the worker cache a real AUTH over an UNENCRYPTED socket — the credential+ # this same commit taught it to inherit, now on the wire in clear.+ ssl = resolve_ssl_enabled(env_prefix)own_url = os.getenv(f"{env_prefix}URL", "").strip()generic_url = os.getenv("REDIS_URL", "").strip()result: dict[str, Any] = {@@ -499,6 +710,19 @@ def _resolve_redis_env(# docstring, sample.env or the chart state. For env_prefix="REDIS_" the two# levels are the same variable, so this reads identically there.explicit_db = prefixed_db if own_url else (prefixed_db or generic_db)+ # CREDENTIALS AT THE URL'S OWN LEVEL, by the same rule as the database above.+ # A prefix that brought its OWN url may point at a DIFFERENT endpoint, and+ # an anonymous one at that; letting the generic REDIS_PASSWORD reach across+ # would make that client send AUTH to a server with none, breaking a+ # connection that worked before. An INHERITED url is the same endpoint as+ # the generic one, so the full fallback chain applies. For env_prefix+ # "REDIS_" the two levels are the same variable and this reads identically.+ if own_url:+ result["url_password"] = env_chain(f"{env_prefix}PASSWORD")+ result["url_username"] = url_username_from_env(env_prefix)+ else:+ result["url_password"] = result["password"]+ result["url_username"] = result["username"]if result["url"] and explicit_db and db_override is None:result["db_from_prefix_env"] = parse_db(explicit_db, env_prefix)# Read OUTSIDE the `if ssl` below: URL mode carries TLS in the scheme and never@@ -508,8 +732,8 @@ def _resolve_redis_env(## Needed where the server's CA is not publicly trusted — notably Memorystore,# whose CA is Google-managed. ElastiCache and Azure chain to public CAs.- ca_certs = os.getenv(- f"{env_prefix}SSL_CA_CERTS", os.getenv("REDIS_SSL_CA_CERTS", "")+ ca_certs = (+ env_chain(f"{env_prefix}SSL_CA_CERTS", "REDIS_SSL_CA_CERTS") or "").strip()if ca_certs:result["ssl_ca_certs"] = ca_certs@@ -536,8 +760,14 @@ def _resolve_redis_env(# check_hostname is True while verify_mode is CERT_NONE. Decided from the# EFFECTIVE value — the URL's query string outranks the env var, and testing# the env var alone is what produced that ValueError on every connection.- raw_check_hostname = os.getenv(- f"{env_prefix}SSL_CHECK_HOSTNAME", os.getenv("REDIS_SSL_CHECK_HOSTNAME", "")+ # The SAME read as resolve_ssl_check_hostname above — this one decides+ # whether the value was EXPLICIT, and leaving it on the nested os.getenv+ # made a blank prefixed value fall through for the value but not for the+ # explicitness. On a Sentinel + TLS deployment that meant a global+ # REDIS_SSL_CHECK_HOSTNAME=true was honoured on the discovery connections+ # and silently dropped on the master one.+ raw_check_hostname, _ = env_chain_named(+ f"{env_prefix}SSL_CHECK_HOSTNAME", "REDIS_SSL_CHECK_HOSTNAME")# Whether the operator ASKED for a value, as opposed to inheriting the# default. Sentinel's master plane needs to know the difference — see@@ -546,8 +776,8 @@ def _resolve_redis_env(# verification back on for Sentinel masters, which is the breakage this# distinction exists to avoid.result["ssl_check_hostname_explicit"] = (- raw_check_hostname.strip().lower() in _TRUE_LITERALS | _FALSE_LITERALS- )+ raw_check_hostname or ""+ ).strip().lower() in _TRUE_LITERALS | _FALSE_LITERALSif effective_cert_reqs(result["url"], result["ssl_cert_reqs"]) == "none":result["ssl_check_hostname"] = Falseresult["ssl_check_hostname_explicit"] = False@@ -674,6 +904,8 @@ def _create_standalone_client(ssl_ca_certs=env.get("ssl_ca_certs"),ssl_cert_reqs=env.get("ssl_cert_reqs"),ssl_check_hostname=env.get("ssl_check_hostname"),+ password=env.get("url_password"),+ username=env.get("url_username"),)logger.info(@@ -715,6 +947,8 @@ def _create_client_from_url(ssl_ca_certs: str | None,ssl_cert_reqs: str | None = None,ssl_check_hostname: bool | None = None,+ password: str | None = None,+ username: str | None = None,) -> redis.Redis:"""Build a client from a full Redis URL.@@ -760,6 +994,34 @@ def _create_client_from_url(kwargs["db"] = db_overrideparts = urlsplit(url)+ # Credentials FILL A GAP the URL leaves; they never override one.+ # ConnectionPool.from_url ends with kwargs.update(url_options), so a URL+ # carrying credentials still wins.+ #+ # Without this, a URL written WITHOUT credentials plus a separately+ # configured {prefix}PASSWORD connects ANONYMOUSLY — and setting the two+ # apart is the configuration the on-prem recipe should be able to recommend,+ # because it keeps the password out of a URL that gets printed into error+ # messages, ArgoCD conditions and ExternalSecret templates. The endpoint+ # answers NOAUTH on the first command, which reads as a broken server rather+ # than a dropped password.+ # Gated on the PASSWORD, and the username rides with it. A username alone is+ # not a credential: values.yaml ships REDIS_USER: default, so filling it in+ # on its own would make redis-py send AUTH to the in-cluster server, which+ # has none — turning a working default deployment into a failing one.+ # Keyed on the URL's PASSWORD, not on the presence of "@". A URL may carry an+ # ACL username alone — redis://alice@host — and that @ is not evidence of a+ # password. Truthiness rather than `is None`: `redis://:@host` parses to ""+ # and redis-py's own parse_url sets the key only `if url.password`.+ #+ # No `not parts.username` guard below: from_url ends with+ # kwargs.update(url_options), so a URL username wins regardless — that guard+ # was unfalsifiable, which is where a later reader deletes something+ # load-bearing by mistake.+ if password and not parts.password:+ kwargs["password"] = password+ if username:+ kwargs["username"] = usernamelogger.info("Redis URL mode enabled. Connecting to %s:%s (tls=%s)",parts.hostname,diff --git a/unstract/core/tests/test_redis_client_config.py b/unstract/core/tests/test_redis_client_config.pyindex 63a0d191f..7411369df 100644--- a/unstract/core/tests/test_redis_client_config.py+++ b/unstract/core/tests/test_redis_client_config.py@@ -11,12 +11,18 @@ and writes against the wrong keyspace; an empty-string env var counts as "set" tNone of those raise at import, so they are asserted here instead."""+import loggingimport pathlibimport re+from urllib.parse import unquote, urlsplitimport pytestimport redisfrom unstract.core.cache.redis_client import (+ url_username_from_env,+ parse_port,+ env_chain_named,+ env_chain,_build_connection_kwargs,_resolve_redis_env,build_socketio_redis_url,@@ -263,19 +269,39 @@ class TestAuth:"s3cr3t")- def test_empty_prefixed_password_shadows_the_fallback(self, monkeypatch):- """Documents a trap rather than endorsing it.+ def test_empty_prefixed_password_falls_through_to_the_fallback(self, monkeypatch):+ """This is the revisit the previous version of this test asked for.- ``os.getenv(key, fallback)`` returns "" when the key exists but is empty,- so an empty CACHE_REDIS_PASSWORD suppresses REDIS_PASSWORD and the client- connects UNAUTHENTICATED. The Helm chart must therefore never render an- empty credential; this test fails if that behaviour ever changes, so the- chart-side guarantee can be revisited.+ It used to assert the opposite and said so: an empty+ CACHE_REDIS_PASSWORD suppressed REDIS_PASSWORD and the client connected+ UNAUTHENTICATED, which was tolerated because "the Helm chart must+ therefore never render an empty credential".++ That guarantee does not hold. workers/sample.env ships+ `CACHE_REDIS_PASSWORD=` uncommented, so an operator following this+ module's own managed-Redis recipe — REDIS_URL without credentials plus+ REDIS_PASSWORD — got no password on any prefixed client and a NOAUTH on+ first command. Blank now means unset here, the same rule parse_db and+ _parse_bool already document two functions away."""monkeypatch.setenv("REDIS_PASSWORD", "s3cr3t")monkeypatch.setenv("CACHE_REDIS_PASSWORD", "")kwargs = _kwargs(create_redis_client(env_prefix="CACHE_REDIS_"))- assert kwargs.get("password") is None+ assert kwargs["password"] == "s3cr3t"++ def test_a_prefixed_password_still_overrides(self, monkeypatch):+ """Blank falling through must not turn into the prefix being ignored."""+ monkeypatch.setenv("REDIS_PASSWORD", "s3cr3t")+ monkeypatch.setenv("CACHE_REDIS_PASSWORD", "other")+ kwargs = _kwargs(create_redis_client(env_prefix="CACHE_REDIS_"))+ assert kwargs["password"] == "other"++ def test_a_blank_password_on_a_prefixed_url_reaches_the_url(self, monkeypatch):+ """The recipe this PR documents, with the sample.env blank in place."""+ monkeypatch.setenv("REDIS_URL", "rediss://managed:6380/0")+ monkeypatch.setenv("REDIS_PASSWORD", "pw")+ monkeypatch.setenv("CACHE_REDIS_PASSWORD", "")+ assert urlsplit(build_socketio_redis_url("CACHE_REDIS_")).password == "pw"class TestSocketIoUrl:@@ -552,11 +578,25 @@ class TestContainerAllowlists:def _shared(cls) -> set[str]:"""Every REDIS_* env var redis_client.py reads, minus the excused ones."""source = cls._CLIENT.read_text()- names = set(re.findall(r'os\.getenv\(\s*"(REDIS_[A-Z_]+)"', source))- names |= {- f"REDIS_{suffix}"- for suffix in re.findall(r'os\.getenv\(\s*f"\{env_prefix\}([A-Z_]+)"', source)- }+ # Scans the ARGUMENTS of every env-reading call, not just os.getenv.+ # Keying on os.getenv alone meant that rewriting a read to go through+ # env_chain made the variable invisible here — the TLS set dropped out+ # of this guard silently, which is the same drift this class exists to+ # catch. Scoped to these call names rather than the whole file so a+ # variable merely NAMED in a docstring is not demanded of every+ # container.+ calls = re.findall(+ r"(?:os\.getenv|env_chain|env_chain_named)\("+ r"([^()]*(?:\([^()]*\)[^()]*)*)\)",+ source,+ )+ names: set[str] = set()+ for args in calls:+ names |= set(re.findall(r'"(REDIS_[A-Z_]+)"', args))+ names |= {+ f"REDIS_{suffix}"+ for suffix in re.findall(r'f"\{env_prefix\}([A-Z_]+)"', args)+ }return names - cls._NOT_FORWARDEDdef test_the_derived_set_is_not_empty(self):@@ -909,3 +949,352 @@ class TestDiscreteModeStillInheritsTheGenericDb:monkeypatch.setenv("REDIS_DB", "3")monkeypatch.setenv("CACHE_REDIS_DB", "1")assert _kwargs(create_redis_client("CACHE_REDIS_"))["db"] == 1+++class TestUrlModeCredentials:+ """A URL written without credentials must not connect anonymously.++ Keeping the password OUT of the URL is the safer configuration — a URL is+ printed into error messages, ArgoCD conditions and ExternalSecret templates,+ and a password in it travels to all three. That configuration is only usable+ if the separately-supplied credential is actually applied.+ """++ def test_the_configured_password_fills_a_gap_the_url_leaves(self, monkeypatch):+ monkeypatch.setenv("REDIS_URL", "rediss://h:6380/0")+ monkeypatch.setenv("REDIS_PASSWORD", "s3cret")+ assert _kwargs(create_redis_client())["password"] == "s3cret"++ def test_a_url_carrying_credentials_still_wins(self, monkeypatch):+ """ConnectionPool.from_url ends with kwargs.update(url_options)."""+ monkeypatch.setenv("REDIS_URL", "rediss://:in-url@h:6380/0")+ monkeypatch.setenv("REDIS_PASSWORD", "s3cret")+ assert _kwargs(create_redis_client())["password"] == "in-url"++ def test_a_username_password_pair_in_the_url_wins_too(self, monkeypatch):+ monkeypatch.setenv("REDIS_URL", "rediss://u:pw@h:6380/0")+ monkeypatch.setenv("REDIS_PASSWORD", "s3cret")+ monkeypatch.setenv("REDIS_USER", "alice")+ kwargs = _kwargs(create_redis_client())+ assert kwargs["password"] == "pw"+ assert kwargs["username"] == "u"++ def test_the_username_rides_with_the_password(self, monkeypatch):+ monkeypatch.setenv("REDIS_URL", "rediss://h:6380/0")+ monkeypatch.setenv("REDIS_PASSWORD", "s3cret")+ monkeypatch.setenv("REDIS_USER", "alice")+ assert _kwargs(create_redis_client())["username"] == "alice"++ def test_a_username_alone_is_not_a_credential(self, monkeypatch):+ """values.yaml ships REDIS_USER: default, and the in-cluster server has+ no auth. Filling in a username on its own would make redis-py send AUTH+ to it — turning a working default deployment into a failing one.+ """+ monkeypatch.setenv("REDIS_URL", "redis://h:6379/0")+ monkeypatch.setenv("REDIS_USER", "default")+ monkeypatch.setenv("REDIS_PASSWORD", "")+ kwargs = _kwargs(create_redis_client())+ assert kwargs.get("username") is None+ assert kwargs.get("password") is None++ def test_no_credentials_anywhere_stays_anonymous(self, monkeypatch):+ monkeypatch.setenv("REDIS_URL", "redis://h:6379/0")+ assert _kwargs(create_redis_client()).get("password") is None++ def test_the_socketio_url_gains_the_credentials_too(self, monkeypatch):+ """Kombu takes a URL and nothing else, so a separately-supplied password+ cannot reach it any other way. Without this the publisher is anonymous+ against an authenticated server and Socket.IO events simply stop.+ """+ monkeypatch.setenv("REDIS_URL", "rediss://h:6380/0")+ monkeypatch.setenv("REDIS_PASSWORD", "s3cret")+ # Assert on PARSED fields, not a substring: a substring check passes on+ # `rediss://alice:s3cret@alice@h:6380/0`, where the password actually+ # parses as "s3cret@alice".+ parts = urlsplit(build_socketio_redis_url())+ assert parts.password == "s3cret"+ assert parts.hostname == "h"++ def test_the_socketio_url_leaves_existing_credentials_alone(self, monkeypatch):+ """Parsed fields, not substrings — the same rule as the test above.++ `"in-url" in url` also passes on `rediss://:in-url@in-url@h:6380/0`,+ where the password parses as "in-url@in-url", and on+ `rediss://h:6380/0?note=in-url`, where there is no password at all.+ """+ monkeypatch.setenv("REDIS_URL", "rediss://:in-url@h:6380/0")+ monkeypatch.setenv("REDIS_PASSWORD", "s3cret")+ parts = urlsplit(build_socketio_redis_url())+ assert parts.password == "in-url"+ assert parts.hostname == "h"++ def test_an_empty_password_in_the_url_is_treated_as_absent(self, monkeypatch):+ """`redis://:@host` parses to password "", which redis-py's own+ parse_url reads as absent (`if url.password:`). Testing `is None` would+ decline to fill a gap the library agrees is a gap — and the shape is+ what Helm produces from `redis://:{{ .Values.password }}@host` when the+ value is empty.+ """+ monkeypatch.setenv("REDIS_URL", "redis://:@h:6379/0")+ monkeypatch.setenv("REDIS_PASSWORD", "s3cret")+ assert _kwargs(create_redis_client())["password"] == "s3cret"++ def test_a_username_only_url_still_gets_the_password(self, monkeypatch):+ """`redis://alice@host` carries an @ but NO password.++ Treating the @ as evidence of credentials dropped the separately+ supplied password and left the client unable to authenticate.+ """+ monkeypatch.setenv("REDIS_URL", "redis://alice@h:6379/0")+ monkeypatch.setenv("REDIS_PASSWORD", "s3cret")+ kwargs = _kwargs(create_redis_client())+ assert kwargs["password"] == "s3cret"+ assert kwargs["username"] == "alice"+++class TestUrlCredentialsAreResolvedAtTheUrlsLevel:+ """Same rule as the database: a prefix that brought its OWN url owns its+ own credentials.++ That url may point at a DIFFERENT endpoint, and an anonymous one; letting+ the generic REDIS_PASSWORD reach across would make the client send AUTH to+ a server that has none, breaking a connection that worked before.+ """++ def test_a_prefix_url_does_not_inherit_the_generic_password(self, monkeypatch):+ monkeypatch.setenv("CACHE_REDIS_URL", "redis://anon:6379/0")+ monkeypatch.setenv("REDIS_PASSWORD", "s3cret")+ assert _kwargs(create_redis_client("CACHE_REDIS_")).get("password") is None++ def test_the_prefixs_own_password_applies(self, monkeypatch):+ monkeypatch.setenv("CACHE_REDIS_URL", "redis://anon:6379/0")+ monkeypatch.setenv("CACHE_REDIS_PASSWORD", "own")+ monkeypatch.setenv("REDIS_PASSWORD", "s3cret")+ assert _kwargs(create_redis_client("CACHE_REDIS_"))["password"] == "own"++ def test_an_inherited_url_still_uses_the_generic_password(self, monkeypatch):+ """Same endpoint as the generic one, so the fallback chain applies."""+ monkeypatch.setenv("REDIS_URL", "redis://h:6379/0")+ monkeypatch.setenv("REDIS_PASSWORD", "s3cret")+ assert _kwargs(create_redis_client("CACHE_REDIS_"))["password"] == "s3cret"++ def test_the_socketio_url_follows_the_same_rule(self, monkeypatch):+ monkeypatch.setenv("CACHE_REDIS_URL", "rediss://anon:6379/0")+ monkeypatch.setenv("REDIS_PASSWORD", "s3cret")+ # A bare `"s3cret" not in url` also passes on "" and on any mangled+ # output, so the endpoint is pinned too.+ parts = urlsplit(build_socketio_redis_url("CACHE_REDIS_"))+ assert parts.password is None+ assert parts.hostname == "anon"+++class TestSocketIoUrlCredentials:+ """The Socket.IO builder must agree with create_redis_client on every shape.++ Its output is a STRING handed to kombu, so a malformed one authenticates+ with a wrong secret rather than none — and `socketio.Server` is constructed+ with logger=False, so the symptom is events silently stopping.+ """++ def test_a_username_only_url_keeps_its_host_and_gets_the_password(self, monkeypatch):+ """`parts.netloc` INCLUDES userinfo; prepending to it produced+ `alice:pw@alice@host`, and userinfo splits on the LAST @.+ """+ monkeypatch.setenv("REDIS_URL", "redis://alice@h:6379/0")+ monkeypatch.setenv("REDIS_PASSWORD", "s3cret")+ parts = urlsplit(build_socketio_redis_url())+ assert parts.hostname == "h"+ assert parts.username == "alice"+ assert parts.password == "s3cret"++ def test_an_encoded_username_is_not_double_encoded(self, monkeypatch):+ """It is percent-encoded already; quoting it again gave `al%2540ice`."""+ monkeypatch.setenv("REDIS_URL", "redis://al%40ice@h:6379/0")+ monkeypatch.setenv("REDIS_PASSWORD", "s3cret")+ # Asserted POSITIVELY: `"%2540" not in url` is equally satisfied by a+ # URL that dropped the username, or replaced it with REDIS_USER.+ assert urlsplit(build_socketio_redis_url()).username == "al%40ice"++ def test_a_password_with_url_metacharacters_is_encoded(self, monkeypatch):+ """Unencoded, `p@ss/w:rd` reparses with host "ss" — a DIFFERENT server."""+ monkeypatch.setenv("REDIS_URL", "rediss://h:6380/0")+ monkeypatch.setenv("REDIS_PASSWORD", "p@ss/w:rd")+ url = build_socketio_redis_url()+ assert urlsplit(url).hostname == "h"+ # Round-trip, not a spelling check. The hostname guard catches the+ # host half of a doubled userinfo but not the credential half:+ # `rediss://x:p%40ss%2Fw%3Ard@p%40ss%2Fw%3Ard@h:6380/0` has hostname+ # "h" and contains the substring, yet kombu would send+ # "p@ss/w:rd@p@ss/w:rd". unquote() is what kombu itself applies.+ assert unquote(urlsplit(url).password) == "p@ss/w:rd"++ def test_an_empty_password_in_the_url_is_treated_as_absent(self, monkeypatch):+ """`redis://:@host` parses to "", which redis-py and kombu both read as+ no password — so the separately supplied one must fill the gap.+ """+ monkeypatch.setenv("REDIS_URL", "redis://:@h:6379/0")+ monkeypatch.setenv("REDIS_PASSWORD", "s3cret")+ assert urlsplit(build_socketio_redis_url()).password == "s3cret"++ def test_a_username_alone_stays_anonymous(self, monkeypatch):+ """values.yaml ships REDIS_USER: default against an unauthenticated+ in-cluster server; adding credentials here would break it.+ """+ monkeypatch.setenv("REDIS_URL", "redis://h:6379/0")+ monkeypatch.setenv("REDIS_USER", "default")+ monkeypatch.setenv("REDIS_PASSWORD", "")+ assert build_socketio_redis_url() == "redis://h:6379/0"++ def test_the_prefix_username_spelling_is_honoured(self, monkeypatch):+ """{prefix}USERNAME is the compatibility spelling of {prefix}USER."""+ monkeypatch.setenv("CACHE_REDIS_URL", "redis://h:6379/0")+ monkeypatch.setenv("CACHE_REDIS_PASSWORD", "pw")+ monkeypatch.setenv("CACHE_REDIS_USERNAME", "alice")+ assert urlsplit(build_socketio_redis_url("CACHE_REDIS_")).username == "alice"+++class TestTheConsumersAgreeOnTheUsername:+ """The client, the Socket.IO URL and the Django cache must resolve the SAME+ ACL user. Each was previously pinned only to itself, so a change that made+ them disagree passed every test.+ """++ def test_a_url_username_outranks_the_configured_one(self, monkeypatch):+ """Both set at once is the only environment that tells the two apart.++ values.yaml ships REDIS_USER: default, so pairing it with a URL that+ names a different user is an ordinary config — and reversing the+ precedence made the client authenticate as `alice` while kombu and the+ Django cache authenticated as `default`. On an endpoint where `default`+ is disabled, Socket.IO events stop and nothing else does.+ """+ monkeypatch.setenv("REDIS_URL", "redis://alice@h:6379/0")+ monkeypatch.setenv("REDIS_PASSWORD", "pw")+ monkeypatch.setenv("REDIS_USER", "default")+ assert urlsplit(build_socketio_redis_url()).username == "alice"+ assert _kwargs(create_redis_client())["username"] == "alice"++ def test_a_password_only_url_takes_no_username_from_the_env(self, monkeypatch):+ """The `not parts.password` guard is load-bearing for the USERNAME.++ For the password it is not: from_url ends with kwargs.update(url_options)+ so the URL's password wins either way. Dropping the guard therefore+ looks harmless and is not — it lets REDIS_USER ride along, turning a+ one-argument `AUTH <pw>` into a two-argument `AUTH alice <pw>` while the+ Socket.IO URL for the same config still sends the one-argument form.+ """+ monkeypatch.setenv("REDIS_URL", "redis://:urlpw@h:6379/0")+ monkeypatch.setenv("REDIS_PASSWORD", "envpw")+ monkeypatch.setenv("REDIS_USER", "alice")+ kwargs = _kwargs(create_redis_client())+ assert kwargs["password"] == "urlpw"+ assert kwargs.get("username") is None+ assert urlsplit(build_socketio_redis_url()).username in (None, "")++ def test_the_username_env_var_has_two_spellings(self, monkeypatch):+ """REDIS_USER is what the chart writes; REDIS_USERNAME is what+ platform-service/sample.env writes. One shared credential secret can+ inject either, so a consumer that reads only one authenticates as a+ different user than the client beside it.+ """+ monkeypatch.delenv("REDIS_USER", raising=False)+ monkeypatch.setenv("REDIS_USERNAME", "alice")+ monkeypatch.setenv("REDIS_URL", "redis://h:6379/0")+ monkeypatch.setenv("REDIS_PASSWORD", "pw")+ assert urlsplit(build_socketio_redis_url()).username == "alice"+ assert _kwargs(create_redis_client())["username"] == "alice"++ def test_a_blank_user_falls_through_to_the_other_spelling(self, monkeypatch):+ """`REDIS_USER=` is the "leave it unset" spelling the recipes use, and+ `os.getenv(name, fallback)` would hand back the blank instead."""+ monkeypatch.setenv("REDIS_USER", "")+ monkeypatch.setenv("REDIS_USERNAME", "alice")+ monkeypatch.setenv("REDIS_URL", "redis://h:6379/0")+ monkeypatch.setenv("REDIS_PASSWORD", "pw")+ assert urlsplit(build_socketio_redis_url()).username == "alice"+++class TestBlankMeansUnsetForTlsToo:+ """The convention has to cover TLS, not just credentials.++ Once a blank prefixed PASSWORD fell through to the generic one but a blank+ prefixed SSL did not, `CACHE_REDIS_SSL=` beside `REDIS_SSL=true` gave the+ worker cache a real AUTH over an UNENCRYPTED socket — the credential the+ same change taught it to inherit, now in clear on the wire.+ """++ @pytest.mark.parametrize(+ "blank,generic,key,expected",+ [+ ("CACHE_REDIS_SSL", ("REDIS_SSL", "true"), "ssl", True),+ (+ "CACHE_REDIS_SSL_CERT_REQS",+ ("REDIS_SSL_CERT_REQS", "none"),+ "ssl_cert_reqs",+ "none",+ ),+ (+ "CACHE_REDIS_SSL_CHECK_HOSTNAME",+ ("REDIS_SSL_CHECK_HOSTNAME", "false"),+ "ssl_check_hostname",+ False,+ ),+ (+ "CACHE_REDIS_SSL_CA_CERTS",+ ("REDIS_SSL_CA_CERTS", "/etc/ca.pem"),+ "ssl_ca_certs",+ "/etc/ca.pem",+ ),+ ],+ )+ def test_a_blank_prefixed_tls_var_does_not_shadow(+ self, monkeypatch, blank, generic, key, expected+ ):+ monkeypatch.setenv(generic[0], generic[1])+ monkeypatch.setenv(blank, "")+ assert _resolve_redis_env("CACHE_REDIS_")[key] == expected+++class TestEnvChainHelpers:+ """The helpers the consumers share. Each had a one-line failure mode."""++ def test_env_chain_returns_the_raw_value_not_a_stripped_one(self, monkeypatch):+ """A file-mounted secret ends in a newline. Stripping the RETURN value+ truncated it here while backend/settings/base.py read the same variable+ unstripped, so one process authenticated two different ways."""+ monkeypatch.setenv("REDIS_PASSWORD", "s3cret\n")+ assert env_chain("REDIS_PASSWORD") == "s3cret\n"++ def test_env_chain_skips_whitespace_only_values(self, monkeypatch):+ monkeypatch.setenv("CACHE_REDIS_PASSWORD", " ")+ monkeypatch.setenv("REDIS_PASSWORD", "s3cret")+ assert env_chain("CACHE_REDIS_PASSWORD", "REDIS_PASSWORD") == "s3cret"++ def test_env_chain_named_reports_the_variable_that_was_set(self, monkeypatch):+ """So a warning cannot send an operator grepping for an unset key."""+ monkeypatch.delenv("CACHE_REDIS_PORT", raising=False)+ monkeypatch.setenv("REDIS_PORT", "6380")+ assert env_chain_named("CACHE_REDIS_PORT", "REDIS_PORT") == ("6380", "REDIS_PORT")++ def test_url_username_from_env_honours_the_same_blank_rule(self, monkeypatch):+ monkeypatch.setenv("REDIS_USER", " ")+ monkeypatch.setenv("REDIS_USERNAME", "alice")+ assert url_username_from_env() == "alice"++ @pytest.mark.parametrize("raw", [None, "", " "])+ def test_parse_port_treats_blank_as_unset(self, raw, caplog):+ """Quietly, too. base.py hands this the raw env value rather than+ env_chain's None, so treating "" as unparseable logged an ERROR at+ startup in the backend and nowhere else — for the repo's own+ "leave the default" spelling."""+ with caplog.at_level(logging.ERROR):+ assert parse_port(raw, "REDIS_PORT", 6379) == 6379+ assert caplog.text == ""++ def test_parse_port_falls_back_on_an_unparseable_value(self, caplog):+ with caplog.at_level(logging.ERROR):+ assert parse_port("638O", "REDIS_PORT", 6379) == 6379+ assert "REDIS_PORT" in caplog.text++ def test_parse_port_returns_an_int(self):+ assert parse_port("6380", "REDIS_PORT", 6379) == 6380diff --git a/workers/sample.env b/workers/sample.envindex 375a62d2a..b6c107857 100644--- a/workers/sample.env+++ b/workers/sample.env@@ -81,7 +81,13 @@ REDIS_SENTINEL_MASTER_NAME=mymaster# Managed / external Redis with TLS (UN-4123). Optional — unset keeps the plaintext# connection above. Either set REDIS_SSL=true beside REDIS_HOST/REDIS_PORT, or give a# full URL whose scheme carries TLS:-# REDIS_URL=rediss://:<password>@<host>:6380/0?ssl_cert_reqs=required+# REDIS_URL=rediss://<host>:6380/0?ssl_cert_reqs=required+# REDIS_PASSWORD=<password>+# REDIS_USER= # blank — see below+# (The password is better kept OUT of the URL. A URL is printed into log+# lines, error messages and deployment tooling; a password in one travels+# with it. Credentials in the URL still work and still win, so an existing+# rediss://:<password>@host URL keeps behaving exactly as before.)# A URL wins over the discrete vars. Auth is password-only as the built-in `default`# user (what a managed AUTH string is), so leave the username empty for a managed# endpoint. REDIS_SSL_CA_CERTS is only needed when the server's CA is not publicly