From ff8a48af8405416a9091bc00e27cac7fff8b0da8 Mon Sep 17 00:00:00 2001 From: Ken Lewerentz Date: Fri, 3 Apr 2026 12:01:13 +0700 Subject: [PATCH 1/2] perf: add canary patterns to skip statistics during bulk operations Add context flags (skip_registrant_statistics, skip_program_statistics) that allow bulk operation callers to suppress expensive computed field recomputation. Add refresh_beneficiary_counts() on spp.program and refresh_statistics() on spp.cycle to recompute once at completion. Also replace bool(rec.program_membership_ids) with SQL EXISTS in _compute_has_members to avoid loading the full membership recordset. --- spp_programs/models/cycle.py | 10 ++ .../models/managers/cycle_manager_base.py | 4 +- .../models/managers/eligibility_manager.py | 3 +- spp_programs/models/programs.py | 27 +++- spp_programs/models/registrant.py | 8 ++ spp_programs/tests/__init__.py | 1 + spp_programs/tests/test_canary_patterns.py | 125 ++++++++++++++++++ 7 files changed, 173 insertions(+), 5 deletions(-) create mode 100644 spp_programs/tests/test_canary_patterns.py diff --git a/spp_programs/models/cycle.py b/spp_programs/models/cycle.py index d4ae4351..07c4a85b 100644 --- a/spp_programs/models/cycle.py +++ b/spp_programs/models/cycle.py @@ -275,6 +275,16 @@ def _compute_entitlements_count(self): entitlements_count = self.env["spp.entitlement"].search_count([("cycle_id", "=", rec.id)]) rec.entitlements_count = entitlements_count + def refresh_statistics(self): + """Refresh all cycle statistics after bulk operations. + + Call this after raw SQL inserts that bypass ORM dependency tracking + (e.g. bulk_create_memberships with skip_duplicates=True). + """ + self._compute_members_count() + self._compute_entitlements_count() + self._compute_total_entitlements_count() + @api.depends("entitlement_ids", "inkind_entitlement_ids") def _compute_total_entitlements_count(self): if not self.ids: diff --git a/spp_programs/models/managers/cycle_manager_base.py b/spp_programs/models/managers/cycle_manager_base.py index 22e6a0b1..c3914b96 100644 --- a/spp_programs/models/managers/cycle_manager_base.py +++ b/spp_programs/models/managers/cycle_manager_base.py @@ -326,8 +326,8 @@ def mark_import_as_done(self, cycle, msg): cycle.locked_reason = None cycle.message_post(body=msg) - # Update Statistics - cycle._compute_members_count() + # Refresh statistics after bulk operations + cycle.refresh_statistics() def mark_prepare_entitlement_as_done(self, cycle, msg): """Complete the preparation of entitlements. diff --git a/spp_programs/models/managers/eligibility_manager.py b/spp_programs/models/managers/eligibility_manager.py index 5392af4a..54dde250 100644 --- a/spp_programs/models/managers/eligibility_manager.py +++ b/spp_programs/models/managers/eligibility_manager.py @@ -165,8 +165,7 @@ def _import_registrants_async(self, new_beneficiaries, state="draft"): def mark_import_as_done(self): self.ensure_one() - self.program_id._compute_eligible_beneficiary_count() - self.program_id._compute_beneficiary_count() + self.program_id.refresh_beneficiary_counts() self.program_id.is_locked = False self.program_id.locked_reason = None diff --git a/spp_programs/models/programs.py b/spp_programs/models/programs.py index 0aa25ec9..1888ef05 100644 --- a/spp_programs/models/programs.py +++ b/spp_programs/models/programs.py @@ -187,8 +187,23 @@ def _check_unique_program_name(self): @api.depends("program_membership_ids") def _compute_has_members(self): + if self.env.context.get("skip_program_statistics"): + return + if not self.ids: + for rec in self: + rec.has_members = False + return + self.env.cr.execute( + """ + SELECT program_id FROM spp_program_membership + WHERE program_id IN %s + GROUP BY program_id + """, + (tuple(self.ids),), + ) + programs_with_members = {row[0] for row in self.env.cr.fetchall()} for rec in self: - rec.has_members = bool(rec.program_membership_ids) + rec.has_members = rec.id in programs_with_members @api.depends("compliance_manager_ids", "compliance_manager_ids.manager_ref_id") def _compute_has_compliance_criteria(self): @@ -273,6 +288,16 @@ def _compute_beneficiary_count(self): count = rec.count_beneficiaries(None)["value"] rec.update({"beneficiaries_count": count}) + def refresh_beneficiary_counts(self): + """Refresh all beneficiary statistics after bulk operations. + + Call this after raw SQL inserts that bypass ORM dependency tracking + (e.g. bulk_create_memberships with skip_duplicates=True). + """ + self._compute_beneficiary_count() + self._compute_eligible_beneficiary_count() + self._compute_has_members() + @api.depends("cycle_ids") def _compute_cycle_count(self): for rec in self: diff --git a/spp_programs/models/registrant.py b/spp_programs/models/registrant.py index 15dd4885..20a8cd31 100644 --- a/spp_programs/models/registrant.py +++ b/spp_programs/models/registrant.py @@ -43,6 +43,8 @@ def _compute_total_entitlements_count(self): @api.depends("program_membership_ids") def _compute_program_membership_count(self): """Batch-efficient program membership count using read_group.""" + if self.env.context.get("skip_registrant_statistics"): + return if not self: return @@ -66,6 +68,8 @@ def _compute_program_membership_count(self): @api.depends("entitlement_ids") def _compute_entitlements_count(self): """Batch-efficient entitlements count using _read_group.""" + if self.env.context.get("skip_registrant_statistics"): + return if not self: return @@ -89,6 +93,8 @@ def _compute_entitlements_count(self): @api.depends("cycle_ids") def _compute_cycle_count(self): """Batch-efficient cycle membership count using _read_group.""" + if self.env.context.get("skip_registrant_statistics"): + return if not self: return @@ -112,6 +118,8 @@ def _compute_cycle_count(self): @api.depends("inkind_entitlement_ids") def _compute_inkind_entitlements_count(self): """Batch-efficient in-kind entitlements count using _read_group.""" + if self.env.context.get("skip_registrant_statistics"): + return if not self: return diff --git a/spp_programs/tests/__init__.py b/spp_programs/tests/__init__.py index acb863ea..aef67f51 100644 --- a/spp_programs/tests/__init__.py +++ b/spp_programs/tests/__init__.py @@ -34,3 +34,4 @@ from . import test_cycle_auto_approve_fund_check from . import test_bulk_membership from . import test_keyset_pagination +from . import test_canary_patterns diff --git a/spp_programs/tests/test_canary_patterns.py b/spp_programs/tests/test_canary_patterns.py new file mode 100644 index 00000000..8008bbc5 --- /dev/null +++ b/spp_programs/tests/test_canary_patterns.py @@ -0,0 +1,125 @@ +# Part of OpenSPP. See LICENSE file for full copyright and licensing details. +"""Tests for Phase 8: Canary patterns for bulk operations. + +During bulk operations, expensive computed field recomputation should be +skipped via context flags and refreshed once at completion. +""" + +import uuid + +from odoo import fields +from odoo.tests import TransactionCase + + +class TestRegistrantCanaryFlags(TransactionCase): + """Test that registrant statistics skip recomputation with context flags.""" + + def setUp(self): + super().setUp() + self.program = self.env["spp.program"].create({"name": f"Test Program {uuid.uuid4().hex[:8]}"}) + self.partner = self.env["res.partner"].create({"name": "Test Registrant", "is_registrant": True}) + + def test_skip_registrant_statistics_skips_program_membership_count(self): + """With skip_registrant_statistics, _compute_program_membership_count should be a no-op.""" + self.partner.with_context(skip_registrant_statistics=True)._compute_program_membership_count() + # Value should remain at default (0) since compute was skipped + self.assertEqual(self.partner.program_membership_count, 0) + + def test_skip_registrant_statistics_skips_entitlements_count(self): + """With skip_registrant_statistics, _compute_entitlements_count should be a no-op.""" + self.partner.with_context(skip_registrant_statistics=True)._compute_entitlements_count() + self.assertEqual(self.partner.entitlements_count, 0) + + def test_skip_registrant_statistics_skips_cycle_count(self): + """With skip_registrant_statistics, _compute_cycle_count should be a no-op.""" + self.partner.with_context(skip_registrant_statistics=True)._compute_cycle_count() + self.assertEqual(self.partner.cycles_count, 0) + + def test_skip_registrant_statistics_skips_inkind_count(self): + """With skip_registrant_statistics, _compute_inkind_entitlements_count should be a no-op.""" + self.partner.with_context(skip_registrant_statistics=True)._compute_inkind_entitlements_count() + self.assertEqual(self.partner.inkind_entitlements_count, 0) + + def test_without_flag_computes_normally(self): + """Without the flag, compute methods should work normally.""" + self.env["spp.program.membership"].create( + { + "partner_id": self.partner.id, + "program_id": self.program.id, + "state": "draft", + } + ) + self.partner._compute_program_membership_count() + self.assertEqual(self.partner.program_membership_count, 1) + + +class TestProgramCanaryFlags(TransactionCase): + """Test that program statistics skip recomputation with context flags.""" + + def setUp(self): + super().setUp() + self.program = self.env["spp.program"].create({"name": f"Test Program {uuid.uuid4().hex[:8]}"}) + + def test_skip_program_statistics_skips_has_members(self): + """With skip_program_statistics, _compute_has_members should be a no-op.""" + self.program.with_context(skip_program_statistics=True)._compute_has_members() + self.assertFalse(self.program.has_members) + + def test_has_members_uses_sql_exists(self): + """_compute_has_members should detect members without loading the recordset.""" + partner = self.env["res.partner"].create({"name": "Registrant", "is_registrant": True}) + self.env["spp.program.membership"].create( + { + "partner_id": partner.id, + "program_id": self.program.id, + "state": "draft", + } + ) + self.program._compute_has_members() + self.assertTrue(self.program.has_members) + + +class TestRefreshMethods(TransactionCase): + """Test refresh_beneficiary_counts and refresh_statistics methods.""" + + def setUp(self): + super().setUp() + self.program = self.env["spp.program"].create({"name": f"Test Program {uuid.uuid4().hex[:8]}"}) + self.cycle = self.env["spp.cycle"].create( + { + "name": "Test Cycle", + "program_id": self.program.id, + "start_date": fields.Date.today(), + "end_date": fields.Date.today(), + } + ) + self.partners = self.env["res.partner"].create( + [{"name": f"Registrant {i}", "is_registrant": True} for i in range(5)] + ) + + def test_program_refresh_beneficiary_counts(self): + """refresh_beneficiary_counts should update all program statistics.""" + # Create memberships via SQL (bypassing ORM triggers) + for p in self.partners: + self.env["spp.program.membership"].bulk_create_memberships( + [{"partner_id": p.id, "program_id": self.program.id, "state": "enrolled"}], + skip_duplicates=True, + ) + + # Counts are stale (SQL insert bypassed ORM) + self.program.refresh_beneficiary_counts() + + self.assertEqual(self.program.beneficiaries_count, 5) + self.assertEqual(self.program.eligible_beneficiaries_count, 5) + self.assertTrue(self.program.has_members) + + def test_cycle_refresh_statistics(self): + """refresh_statistics should update cycle member and entitlement counts.""" + for p in self.partners: + self.env["spp.cycle.membership"].bulk_create_memberships( + [{"partner_id": p.id, "cycle_id": self.cycle.id, "state": "enrolled"}], + skip_duplicates=True, + ) + + self.cycle.refresh_statistics() + self.assertEqual(self.cycle.members_count, 5) From 8c49d838fe27d53da603c43d7f6b87294b198928 Mon Sep 17 00:00:00 2001 From: Edwin Gonzales Date: Fri, 17 Apr 2026 10:52:43 +0800 Subject: [PATCH 2/2] docs: bump version to 19.0.2.0.9, add changelog for canary patterns --- spp_programs/README.rst | 12 +++++++++ spp_programs/__manifest__.py | 2 +- spp_programs/readme/HISTORY.md | 6 +++++ spp_programs/static/description/index.html | 29 ++++++++++++++++------ 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/spp_programs/README.rst b/spp_programs/README.rst index 61b3d267..fc223252 100644 --- a/spp_programs/README.rst +++ b/spp_programs/README.rst @@ -254,6 +254,18 @@ Dependencies Changelog ========= +19.0.2.0.9 +~~~~~~~~~~ + +- Add context flags (``skip_registrant_statistics``, + ``skip_program_statistics``) to suppress expensive computed field + recomputation during bulk operations +- Add ``refresh_beneficiary_counts()`` on program and + ``refresh_statistics()`` on cycle for one-shot recomputation after + bulk operations +- Replace ``bool(rec.program_membership_ids)`` with SQL query in + ``_compute_has_members`` + 19.0.2.0.8 ~~~~~~~~~~ diff --git a/spp_programs/__manifest__.py b/spp_programs/__manifest__.py index 1de62a0c..b32ff972 100644 --- a/spp_programs/__manifest__.py +++ b/spp_programs/__manifest__.py @@ -4,7 +4,7 @@ "name": "OpenSPP Programs", "summary": "Manage programs, cycles, beneficiary enrollment, entitlements (cash and in-kind), payments, and fund tracking for social protection.", "category": "OpenSPP/Core", - "version": "19.0.2.0.8", + "version": "19.0.2.0.9", "sequence": 1, "author": "OpenSPP.org", "website": "https://github.com/OpenSPP/OpenSPP2", diff --git a/spp_programs/readme/HISTORY.md b/spp_programs/readme/HISTORY.md index 2e335856..bd3fbc33 100644 --- a/spp_programs/readme/HISTORY.md +++ b/spp_programs/readme/HISTORY.md @@ -1,3 +1,9 @@ +### 19.0.2.0.9 + +- Add context flags (`skip_registrant_statistics`, `skip_program_statistics`) to suppress expensive computed field recomputation during bulk operations +- Add `refresh_beneficiary_counts()` on program and `refresh_statistics()` on cycle for one-shot recomputation after bulk operations +- Replace `bool(rec.program_membership_ids)` with SQL query in `_compute_has_members` + ### 19.0.2.0.8 - Replace OFFSET pagination with NTILE-based ID-range batching in all async job dispatchers diff --git a/spp_programs/static/description/index.html b/spp_programs/static/description/index.html index a11c13ed..a9f08e62 100644 --- a/spp_programs/static/description/index.html +++ b/spp_programs/static/description/index.html @@ -658,6 +658,19 @@

Changelog

+

19.0.2.0.9

+
    +
  • Add context flags (skip_registrant_statistics, +skip_program_statistics) to suppress expensive computed field +recomputation during bulk operations
    • +
  • Add refresh_beneficiary_counts() on program and +refresh_statistics() on cycle for one-shot recomputation after +bulk operations
    • +
  • Replace bool(rec.program_membership_ids) with SQL query in +_compute_has_members
    • +
+
+

19.0.2.0.8

  • Replace OFFSET pagination with NTILE-based ID-range batching in all @@ -668,7 +681,7 @@

    19.0.2.0.8

    program and cycle
-
+

19.0.2.0.7

  • Bulk membership creation using raw SQL INSERT ON CONFLICT DO NOTHING @@ -677,7 +690,7 @@

    19.0.2.0.7

    _add_beneficiaries with bulk SQL path
-
+

19.0.2.0.6

  • Remove unused entitlement_base_model.py (dead code, never imported)
    • @@ -686,34 +699,34 @@

    19.0.2.0.6

    payment, and fund tests (172 → 492 tests)
-
+

19.0.2.0.5

  • Batch create entitlements and payments instead of one-by-one ORM creates
-
+

19.0.2.0.4

  • Fetch fund balance once per approval batch instead of per entitlement
-
+

19.0.2.0.3

  • Replace cycle computed fields (total_amount, entitlements_count, approval flags) with SQL aggregation queries
-
+

19.0.2.0.2

  • Add composite indexes for frequent query patterns on entitlements and program memberships
-
+

19.0.2.0.1

  • Replace Python-level uniqueness checks with SQL UNIQUE constraints for @@ -722,7 +735,7 @@

    19.0.2.0.1

    constraint creation
-
+

19.0.2.0.0

  • Initial migration to OpenSPP2