diff --git a/posthog/apps.py b/posthog/apps.py index 70e0593a6aaf..47d1264e7660 100644 --- a/posthog/apps.py +++ b/posthog/apps.py @@ -74,6 +74,17 @@ def ready(self): event="development server launched", properties={"git_rev": get_git_commit_short(), "git_branch": get_git_branch()}, ) + # Use HyperCache to provide flag definitions instead of per-process API polling. + # Falls back to the SDK's emergency API fetch (via personal_api_key) only when + # the cache is cold. In E2E testing personal_api_key is None, so a cold cache + # will result in no flag definitions being loaded — which is acceptable there. + if not posthoganalytics.disabled: + from posthog.feature_flags.sdk_cache_provider import HyperCacheFlagProvider + + posthoganalytics.flag_definition_cache_provider = HyperCacheFlagProvider( + team_id=int(os.environ.get("POSTHOG_SELF_TEAM_ID", "2")) + ) + # load feature flag definitions if not already loaded if not posthoganalytics.disabled and posthoganalytics.feature_flag_definitions() is None: posthoganalytics.load_feature_flags() diff --git a/posthog/feature_flags/__init__.py b/posthog/feature_flags/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/posthog/feature_flags/sdk_cache_provider.py b/posthog/feature_flags/sdk_cache_provider.py new file mode 100644 index 000000000000..9d5a1cad7892 --- /dev/null +++ b/posthog/feature_flags/sdk_cache_provider.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +import structlog +from posthoganalytics.flag_definition_cache import FlagDefinitionCacheData + +if TYPE_CHECKING: + from posthog.storage.hypercache import HyperCache + +logger = structlog.get_logger(__name__) + + +class HyperCacheFlagProvider: + """ + Read-only FlagDefinitionCacheProvider that reads flag definitions from the + existing HyperCache infrastructure instead of polling the API. + + The HyperCache is kept fresh via Django signals (when flags/cohorts change) + and periodic refresh tasks. This provider eliminates per-process API polling + by reading directly from the same Redis cache. + """ + + def __init__(self, team_id: int): + self._team_id = team_id + self._hypercache: Optional[HyperCache] = None + + def _get_hypercache(self): + """Lazily resolve the hypercache reference. + + The import is deferred because local_evaluation.py triggers a deep + import chain (cohort.util → hogql → api → ... → cohort.util) that + causes a circular ImportError when called during AppConfig.ready(). + By caching the reference after the first successful import, subsequent + calls skip the import entirely. + """ + if self._hypercache is None: + from posthog.models.feature_flag.local_evaluation import flag_definitions_hypercache + + self._hypercache = flag_definitions_hypercache + return self._hypercache + + def get_flag_definitions(self) -> Optional[FlagDefinitionCacheData]: + try: + data = self._get_hypercache().get_from_cache(self._team_id) + if data is not None: + # Defensive: ensure a valid FlagDefinitionCacheData even if + # the HyperCache shape drifts in the future. + return { + "flags": data.get("flags", []), + "group_type_mapping": data.get("group_type_mapping", {}), + "cohorts": data.get("cohorts", {}), + } + return None + except ImportError: + # Expected during Django startup — local_evaluation.py has a + # circular import chain through cohort.util that resolves once + # all modules finish loading. The SDK's next poll cycle will retry. + logger.debug("hypercache_flag_provider_import_pending", team_id=self._team_id) + return None + except Exception: + logger.exception("hypercache_flag_provider_read_error", team_id=self._team_id) + return None + + def should_fetch_flag_definitions(self) -> bool: + # Never poll the API — HyperCache handles all writes via Django signals + # and periodic refresh tasks + return False + + def on_flag_definitions_received(self, data: FlagDefinitionCacheData) -> None: + pass # No-op — should_fetch always returns False, so this is never called + + def shutdown(self) -> None: + pass # No-op — no locks or resources to release diff --git a/posthog/feature_flags/test_sdk_cache_provider.py b/posthog/feature_flags/test_sdk_cache_provider.py new file mode 100644 index 000000000000..4bb14ef3eae2 --- /dev/null +++ b/posthog/feature_flags/test_sdk_cache_provider.py @@ -0,0 +1,208 @@ +from unittest.mock import MagicMock, patch + +from django.test import SimpleTestCase + +from parameterized import parameterized +from posthoganalytics.client import Client + +from posthog.feature_flags.sdk_cache_provider import HyperCacheFlagProvider + + +class TestHyperCacheFlagProvider(SimpleTestCase): + def setUp(self): + self.provider = HyperCacheFlagProvider(team_id=2) + + def test_should_fetch_flag_definitions_always_returns_false(self): + assert self.provider.should_fetch_flag_definitions() is False + + def test_on_flag_definitions_received_is_noop(self): + self.provider.on_flag_definitions_received({"flags": [], "group_type_mapping": {}, "cohorts": {}}) + + def test_shutdown_is_noop(self): + self.provider.shutdown() + + @parameterized.expand( + [ + ( + "cache_hit", + { + "flags": [{"key": "test-flag", "active": True}], + "group_type_mapping": {"0": "company"}, + "cohorts": {"1": {"properties": []}}, + }, + None, + { + "flags": [{"key": "test-flag", "active": True}], + "group_type_mapping": {"0": "company"}, + "cohorts": {"1": {"properties": []}}, + }, + ), + ("cache_miss", None, None, None), + ( + "missing_keys_defaults", + {"flags": [{"key": "flag-1"}]}, + None, + {"flags": [{"key": "flag-1"}], "group_type_mapping": {}, "cohorts": {}}, + ), + ("exception", None, Exception("Redis connection failed"), None), + ] + ) + @patch("posthog.models.feature_flag.local_evaluation.flag_definitions_hypercache") + def test_get_flag_definitions(self, _name, cache_return, side_effect, expected, mock_hypercache): + # Reset cached reference so the mock is picked up + self.provider._hypercache = None + + if side_effect: + mock_hypercache.get_from_cache.side_effect = side_effect + else: + mock_hypercache.get_from_cache.return_value = cache_return + + result = self.provider.get_flag_definitions() + + if expected is None: + assert result is None + else: + assert result == expected + + @patch( + "posthog.feature_flags.sdk_cache_provider.HyperCacheFlagProvider._get_hypercache", + side_effect=ImportError("circular import"), + ) + def test_get_flag_definitions_returns_none_on_circular_import(self, _mock): + assert self.provider.get_flag_definitions() is None + + @patch("posthog.models.feature_flag.local_evaluation.flag_definitions_hypercache") + def test_caches_hypercache_reference(self, mock_hypercache): + self.provider._hypercache = None + mock_hypercache.get_from_cache.return_value = None + + self.provider.get_flag_definitions() + self.provider.get_flag_definitions() + + assert self.provider._hypercache is not None + + def test_implements_protocol(self): + from posthoganalytics.flag_definition_cache import FlagDefinitionCacheProvider + + assert isinstance(self.provider, FlagDefinitionCacheProvider) + + +SAMPLE_FLAGS = { + "flags": [ + {"id": 1, "key": "beta-feature", "active": True, "filters": {"groups": [{"rollout_percentage": 100}]}}, + {"id": 2, "key": "disabled-flag", "active": False, "filters": {}}, + ], + "group_type_mapping": {"0": "company", "1": "project"}, + "cohorts": {"10": {"properties": [{"key": "plan", "value": "enterprise"}]}}, +} + + +class TestSDKClientIntegration(SimpleTestCase): + """Test HyperCacheFlagProvider with a real posthoganalytics.Client.""" + + def _make_client(self, provider: HyperCacheFlagProvider) -> Client: + return Client( + project_api_key="test-key", + personal_api_key="test-personal-key", + host="http://localhost:8000", + flag_definition_cache_provider=provider, + poll_interval=99999, # prevent background polling + send=False, + enable_exception_autocapture=False, + ) + + def test_sdk_loads_flags_from_provider_instead_of_api(self): + mock_hypercache = MagicMock() + mock_hypercache.get_from_cache.return_value = SAMPLE_FLAGS + provider = HyperCacheFlagProvider(team_id=2) + provider._hypercache = mock_hypercache + + client = self._make_client(provider) + + with patch.object(client, "_fetch_feature_flags_from_api") as mock_api: + client._load_feature_flags() + + mock_api.assert_not_called() + + assert len(client.feature_flags) == 2 + flags_by_key: dict = client.feature_flags_by_key or {} + assert flags_by_key["beta-feature"]["active"] is True + assert client.group_type_mapping == {"0": "company", "1": "project"} + assert client.cohorts == {"10": {"properties": [{"key": "plan", "value": "enterprise"}]}} + + def test_sdk_falls_back_to_api_when_cache_is_empty_and_no_flags_loaded(self): + mock_hypercache = MagicMock() + mock_hypercache.get_from_cache.return_value = None + provider = HyperCacheFlagProvider(team_id=2) + provider._hypercache = mock_hypercache + + client = self._make_client(provider) + + with patch.object(client, "_fetch_feature_flags_from_api") as mock_api: + client._load_feature_flags() + + mock_api.assert_called_once() + + def test_sdk_skips_api_when_cache_empty_but_flags_already_loaded(self): + mock_hypercache = MagicMock() + provider = HyperCacheFlagProvider(team_id=2) + provider._hypercache = mock_hypercache + + client = self._make_client(provider) + + # First call: cache has data → loads flags + mock_hypercache.get_from_cache.return_value = SAMPLE_FLAGS + client._load_feature_flags() + assert len(client.feature_flags) == 2 + + # Second call: cache is empty → keeps existing flags, no API call + mock_hypercache.get_from_cache.return_value = None + + with patch.object(client, "_fetch_feature_flags_from_api") as mock_api: + client._load_feature_flags() + + mock_api.assert_not_called() + + # Flags from the first load are still there + assert len(client.feature_flags) == 2 + + def test_sdk_picks_up_flag_changes_on_next_poll(self): + mock_hypercache = MagicMock() + provider = HyperCacheFlagProvider(team_id=2) + provider._hypercache = mock_hypercache + + client = self._make_client(provider) + + # Initial load + mock_hypercache.get_from_cache.return_value = SAMPLE_FLAGS + client._load_feature_flags() + flags_by_key: dict = client.feature_flags_by_key or {} + assert flags_by_key["beta-feature"]["active"] is True + + # Flag changed in HyperCache (e.g., toggled off via Django admin) + updated_flags = { + "flags": [ + {"id": 1, "key": "beta-feature", "active": False, "filters": {}}, + ], + "group_type_mapping": {}, + "cohorts": {}, + } + mock_hypercache.get_from_cache.return_value = updated_flags + client._load_feature_flags() + + flags_by_key = client.feature_flags_by_key or {} + assert flags_by_key["beta-feature"]["active"] is False + assert len(client.feature_flags) == 1 + + def test_sdk_falls_back_to_api_when_provider_raises(self): + mock_hypercache = MagicMock() + mock_hypercache.get_from_cache.side_effect = Exception("Redis down") + provider = HyperCacheFlagProvider(team_id=2) + provider._hypercache = mock_hypercache + + client = self._make_client(provider) + + with patch.object(client, "_fetch_feature_flags_from_api") as mock_api: + client._load_feature_flags() + + mock_api.assert_called_once() diff --git a/pyproject.toml b/pyproject.toml index 894af244ffb4..cc3ddef10d61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ dependencies = [ "paramiko==3.4.1", "pillow==12.1.1", "protobuf~=5.29.6", - "posthoganalytics==7.9.0", + "posthoganalytics==7.9.12", "polars==1.37.1", "psycopg2-binary==2.9.10", "psycopg[binary]==3.2.4", diff --git a/uv.lock b/uv.lock index 4c60685f70d5..43e2cb2afc1c 100644 --- a/uv.lock +++ b/uv.lock @@ -5150,7 +5150,7 @@ requires-dist = [ { name = "pixelmatch", specifier = ">=0.3.1" }, { name = "playwright", specifier = "~=1.54.0" }, { name = "polars", specifier = "==1.37.1" }, - { name = "posthoganalytics", specifier = "==7.9.0" }, + { name = "posthoganalytics", specifier = "==7.9.12" }, { name = "protobuf", specifier = "~=5.29.6" }, { name = "psycopg", extras = ["binary"], specifier = "==3.2.4" }, { name = "psycopg2-binary", specifier = "==2.9.10" }, @@ -5280,7 +5280,7 @@ sentiment = [ [[package]] name = "posthoganalytics" -version = "7.9.0" +version = "7.9.12" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backoff" }, @@ -5290,9 +5290,9 @@ dependencies = [ { name = "six" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5b/8d/0a1550add57925ed91e3e9adf2402a39f7e2076e0f2febc9e06b03b6ec2e/posthoganalytics-7.9.0.tar.gz", hash = "sha256:0e96b852ed9ecd5e2a209164ad5c72159688bb9f1af5af95c412aba90e9521e3", size = 172634, upload-time = "2026-02-17T09:32:37.905Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5d/85/cb2fefa75536cf44f5d46a261fcf252caf78584136afa6051848b0cd522e/posthoganalytics-7.9.12.tar.gz", hash = "sha256:1fa0e017b3c8d3cd1b7cee4880bc00023f9f375b31f354385285ec51b375bcb7", size = 176416, upload-time = "2026-03-12T09:01:25.353Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/73/4c/7173de033621781e01ac362c5fe1e2ac6cd53d12261d804b1d1b8280cd8c/posthoganalytics-7.9.0-py3-none-any.whl", hash = "sha256:6aa46f5c79070cb92e8ddc088d8cc37186158e574b03720e26f3e2916e478006", size = 199301, upload-time = "2026-02-17T09:32:36.708Z" }, + { url = "https://files.pythonhosted.org/packages/d5/3b/ac2b8ad1a89cebf4022eddfbb76f8cc19886932f7a7069c0076989df2177/posthoganalytics-7.9.12-py3-none-any.whl", hash = "sha256:24c30dbf50b10d041d6443578bc5f40a7a4f0e6f7d4e00d701d4b3d82c4034ab", size = 202909, upload-time = "2026-03-12T09:01:23.593Z" }, ] [[package]]