From e51003973a281a188e8cbbee9d872e973672266a Mon Sep 17 00:00:00 2001 From: cnathe Date: Mon, 2 Mar 2026 14:15:58 -0600 Subject: [PATCH 1/4] GitHub Issue #875: LuminexUpgradeCode.checkForMissingSummaryRows - java upgrade script for the Luminex case of missing summary background type rows for runs that have both summary and raw data --- .../labkey/luminex/LuminexDataHandler.java | 2 +- .../src/org/labkey/luminex/LuminexModule.java | 7 + .../labkey/luminex/LuminexUpgradeCode.java | 191 ++++++++++++++++++ 3 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 luminex/src/org/labkey/luminex/LuminexUpgradeCode.java diff --git a/luminex/src/org/labkey/luminex/LuminexDataHandler.java b/luminex/src/org/labkey/luminex/LuminexDataHandler.java index dd97666a8..378afdb7a 100644 --- a/luminex/src/org/labkey/luminex/LuminexDataHandler.java +++ b/luminex/src/org/labkey/luminex/LuminexDataHandler.java @@ -620,7 +620,7 @@ private Map getExistingAnalytes(ExpRun expRun) return existingAnalytes; } - private static class DataRowKey + public static class DataRowKey { private final long _dataId; private final int _analyteId; diff --git a/luminex/src/org/labkey/luminex/LuminexModule.java b/luminex/src/org/labkey/luminex/LuminexModule.java index 0a02dc171..a8979f7ab 100644 --- a/luminex/src/org/labkey/luminex/LuminexModule.java +++ b/luminex/src/org/labkey/luminex/LuminexModule.java @@ -23,6 +23,7 @@ import org.labkey.api.assay.AssayQCFlagColumn; import org.labkey.api.assay.AssayService; import org.labkey.api.data.Container; +import org.labkey.api.data.UpgradeCode; import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.exp.property.PropertyService; import org.labkey.api.module.DefaultModule; @@ -103,4 +104,10 @@ public Set getSchemaNames() LuminexSaveExclusionsForm.TestCase.class ); } + + @Override + public @Nullable UpgradeCode getUpgradeCode() + { + return new LuminexUpgradeCode(); + } } diff --git a/luminex/src/org/labkey/luminex/LuminexUpgradeCode.java b/luminex/src/org/labkey/luminex/LuminexUpgradeCode.java new file mode 100644 index 000000000..93007a5ea --- /dev/null +++ b/luminex/src/org/labkey/luminex/LuminexUpgradeCode.java @@ -0,0 +1,191 @@ +package org.labkey.luminex; + +import org.apache.commons.lang3.ArrayUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.assay.AssayService; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.BeanObjectFactory; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.ObjectFactory; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.UpgradeCode; +import org.labkey.api.data.statistics.MathStat; +import org.labkey.api.data.statistics.StatsService; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.MapDataIterator; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.security.User; +import org.labkey.api.util.GUID; +import org.labkey.luminex.model.LuminexDataRow; +import org.labkey.luminex.query.LuminexDataTable; +import org.labkey.luminex.query.LuminexProtocolSchema; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class LuminexUpgradeCode implements UpgradeCode +{ + private static final Logger _log = LogManager.getLogger(LuminexUpgradeCode.class); + + /** + * NOTE: this upgrade code is not called from a SQL upgrade script. It is meant to be ran manually from the admin console SQL Scripts page. + * GitHub Issue #875: Upgrade code to check for Luminex assay runs that have both summary and raw data but are missing summary rows. + */ + public static void checkForMissingSummaryRows(ModuleContext ctx) + { + if (ctx.isNewInstall()) + return; + + DbScope scope = LuminexProtocolSchema.getSchema().getScope(); + try (DbScope.Transaction tx = scope.ensureTransaction()) + { + // For any Luminex dataids (input files) that have both summary and raw data rows, + // find Luminex raw data rows (summary = false) that don't have a corresponding summary data row (summary = true) + SQLFragment missingSummaryRowsSql = new SQLFragment(""" + SELECT DISTINCT d.runid, dr_false.dataid, dr_false.analyteid, dr_false.type + FROM luminex.datarow dr_false + LEFT JOIN exp.data d ON d.rowid = dr_false.dataid + WHERE dr_false.summary = false + AND EXISTS (SELECT 1 FROM luminex.datarow WHERE dataid = dr_false.dataid AND summary = true) + AND EXISTS (SELECT 1 FROM luminex.datarow WHERE dataid = dr_false.dataid AND summary = false) + AND NOT EXISTS (SELECT 1 FROM luminex.datarow dr_true + WHERE dr_true.summary = true + AND dr_true.dataid = dr_false.dataid + AND dr_true.analyteid = dr_false.analyteid + AND dr_true.type = dr_false.type + ) + """); + + int missingSummaryRowCount = new SqlSelector(scope, new SQLFragment("SELECT COUNT(*) FROM (").append(missingSummaryRowsSql).append(")")).getObject(Integer.class); + if (missingSummaryRowCount == 0) + { + _log.info("No missing summary rows found for Luminex assay data."); + return; + } + + new SqlSelector(scope, missingSummaryRowsSql).forEach(rs -> { + int runid = rs.getInt("runid"); + int dataId = rs.getInt("dataid"); + int analyteId = rs.getInt("analyteid"); + String type = rs.getString("type"); + + ExpRun expRun = ExperimentService.get().getExpRun(runid); + if (expRun == null) + { + _log.warn("Could not find run for runid: " + runid + ", skipping missing summary row check for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type); + return; + } + + _log.info("Missing summary row for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type + " in run: " + expRun.getName() + " (" + expRun.getRowId() + ")"); + + // currently only inserting summary rows for Background (type = B) data rows + if (!"B".equals(type)) + { + _log.warn("...not inserting missing summary row for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type + " because type is not 'B' (Background)"); + return; + } + + // Query for existing raw data rows with the same dataId, analyteId, and type, and use those rows to calculate summary stats and well information for the new summary row that we will insert into the database + User user = ctx.getUpgradeUser(); + ExpProtocol protocol = expRun.getProtocol(); + LuminexDataTable tableInfo = ((LuminexProtocolSchema)AssayService.get().getProvider(protocol).createProtocolSchema(user, expRun.getContainer(), protocol, null)).createDataTable(null, false); + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Data"), dataId); + filter.addCondition(FieldKey.fromParts("Analyte"), analyteId); + filter.addCondition(FieldKey.fromParts("Type"), type); + Map existingRows = new HashMap<>(); + List fis = new ArrayList<>(); + List fiBkgds = new ArrayList<>(); + List wells = new ArrayList<>(); + LuminexDataRow newRow = new LuminexDataRow(); + newRow.setSummary(true); + for (Map databaseMap : new TableSelector(tableInfo, filter, null).getMapCollection()) + { + LuminexDataRow existingRow = BeanObjectFactory.Registry.getFactory(LuminexDataRow.class).fromMap(databaseMap); + existingRow._setExtraProperties(new CaseInsensitiveHashMap<>(databaseMap)); + existingRows.put(new LuminexDataHandler.DataRowKey(existingRow), existingRow); + + // keep track of well, FI, and FI Bkgd values from existing raw data rows to use in calculating summary stats for the new summary row + wells.add(existingRow.getWell()); + if (existingRow.getFi() != null) + fis.add(existingRow.getFi()); + if (existingRow.getFiBackground() != null) + fiBkgds.add(existingRow.getFiBackground()); + + // clone the following properties from the existing row to the newRow: + // dataid, analyteid, type, container, protocolid, description, wellrole, extraSpecimenInfo, + // specimenID, participantID, visitID, date, dilution, tittration, singlepointcontrol + // note: don't clone rowid, beadcount, summary, lsid + newRow.setData(existingRow.getData()); + newRow.setAnalyte(existingRow.getAnalyte()); + newRow.setType(existingRow.getType()); + newRow.setWellRole(existingRow.getWellRole()); + newRow.setContainer(existingRow.getContainer()); + newRow.setProtocol(existingRow.getProtocol()); + newRow.setDescription(existingRow.getDescription()); + newRow.setSpecimenID(existingRow.getSpecimenID()); + newRow.setParticipantID(existingRow.getParticipantID()); + newRow.setVisitID(existingRow.getVisitID()); + newRow.setDate(existingRow.getDate()); + newRow.setExtraSpecimenInfo(existingRow.getExtraSpecimenInfo()); + newRow.setDilution(existingRow.getDilution()); + newRow.setTitration(existingRow.getTitration()); + newRow.setSinglePointControl(existingRow.getSinglePointControl()); + + // we can clone stdev and cv from existing raw rows because LuminexDataHandler ensureSummaryStats() calculates them + newRow.setStdDev(existingRow.getStdDev()); + newRow.setCv(existingRow.getCv()); + } + + // Calculate FI and FI-BKGD values for the new summary row based on the existing raw data rows with the same dataId, analyteId, and type. + // similar to LuminexDataHandler ensureSummaryStats() + StatsService service = StatsService.get(); + MathStat statsFi = service.getStats(ArrayUtils.toPrimitive(fis.toArray(new Double[0]))); + newRow.setFi(Math.abs(statsFi.getMean())); + newRow.setFiString(newRow.getFi().toString()); + MathStat statsFiBkgd = service.getStats(ArrayUtils.toPrimitive(fiBkgds.toArray(new Double[0]))); + newRow.setFiBackground(Math.abs(statsFiBkgd.getMean())); + newRow.setFiBackgroundString(newRow.getFiBackground().toString()); + + // Calculate well to be a comma-separated list of wells from the existing raw data rows + newRow.setWell(String.join(",", wells)); + + // Generate an LSID for the new summary row + Lsid.LsidBuilder builder = new Lsid.LsidBuilder(LuminexAssayProvider.LUMINEX_DATA_ROW_LSID_PREFIX,""); + newRow.setLsid(builder.setObjectId(GUID.makeGUID()).toString()); + + // Insert the new summary row into the database. + // similar to LuminexDataHandler saveDataRows() + LuminexImportHelper helper = new LuminexImportHelper(); + Map row = new CaseInsensitiveHashMap<>(); + ObjectFactory f = ObjectFactory.Registry.getFactory(LuminexDataRow.class); + row.putAll(f.toMap(newRow, null)); + try + { + OntologyManager.insertTabDelimited(tableInfo, expRun.getContainer(), user, helper, MapDataIterator.of(List.of(row)).getDataIterator(new DataIteratorContext()), true, _log, null); + String comment = "Inserted missing summary row for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type; + ExperimentService.get().auditRunEvent(user, protocol, expRun, null, "LuminexUpgradeCode.checkForMissingSummaryRows: " + comment, null); + _log.info("..." + comment); + } + catch (BatchValidationException e) + { + _log.warn("...failed to insert missing summary row for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type, e); + } + }); + + tx.commit(); + } + } +} From 6ccdad3b0186e2c7a5ce8b5f42f6b22bf6b043ff Mon Sep 17 00:00:00 2001 From: cnathe Date: Thu, 26 Mar 2026 08:14:23 -0500 Subject: [PATCH 2/4] - updates to handle scenario with multiple standards for a given runid/analyteid/dataid/type combination --- .../labkey/luminex/LuminexUpgradeCode.java | 166 ++++++++++-------- 1 file changed, 93 insertions(+), 73 deletions(-) diff --git a/luminex/src/org/labkey/luminex/LuminexUpgradeCode.java b/luminex/src/org/labkey/luminex/LuminexUpgradeCode.java index 93007a5ea..b389c4bcb 100644 --- a/luminex/src/org/labkey/luminex/LuminexUpgradeCode.java +++ b/luminex/src/org/labkey/luminex/LuminexUpgradeCode.java @@ -41,8 +41,8 @@ public class LuminexUpgradeCode implements UpgradeCode private static final Logger _log = LogManager.getLogger(LuminexUpgradeCode.class); /** - * NOTE: this upgrade code is not called from a SQL upgrade script. It is meant to be ran manually from the admin console SQL Scripts page. * GitHub Issue #875: Upgrade code to check for Luminex assay runs that have both summary and raw data but are missing summary rows. + * NOTE: this upgrade code is not called from a SQL upgrade script. It is meant to be run manually from the admin console SQL Scripts page. */ public static void checkForMissingSummaryRows(ModuleContext ctx) { @@ -98,94 +98,114 @@ AND NOT EXISTS (SELECT 1 FROM luminex.datarow dr_true return; } - // Query for existing raw data rows with the same dataId, analyteId, and type, and use those rows to calculate summary stats and well information for the new summary row that we will insert into the database + // Query for existing raw data rows with the same dataId, analyteId, and type + StatsService service = StatsService.get(); User user = ctx.getUpgradeUser(); ExpProtocol protocol = expRun.getProtocol(); LuminexDataTable tableInfo = ((LuminexProtocolSchema)AssayService.get().getProvider(protocol).createProtocolSchema(user, expRun.getContainer(), protocol, null)).createDataTable(null, false); SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Data"), dataId); filter.addCondition(FieldKey.fromParts("Analyte"), analyteId); filter.addCondition(FieldKey.fromParts("Type"), type); - Map existingRows = new HashMap<>(); - List fis = new ArrayList<>(); - List fiBkgds = new ArrayList<>(); - List wells = new ArrayList<>(); - LuminexDataRow newRow = new LuminexDataRow(); - newRow.setSummary(true); + + // keep track of the set of wells for the given dataId/analyteId/type/standard combinations + record WellGroupKey(long dataId, int analyteId, String type, Object standard) {} + Map> rowsByWellGroup = new HashMap<>(); for (Map databaseMap : new TableSelector(tableInfo, filter, null).getMapCollection()) { LuminexDataRow existingRow = BeanObjectFactory.Registry.getFactory(LuminexDataRow.class).fromMap(databaseMap); existingRow._setExtraProperties(new CaseInsensitiveHashMap<>(databaseMap)); - existingRows.put(new LuminexDataHandler.DataRowKey(existingRow), existingRow); - - // keep track of well, FI, and FI Bkgd values from existing raw data rows to use in calculating summary stats for the new summary row - wells.add(existingRow.getWell()); - if (existingRow.getFi() != null) - fis.add(existingRow.getFi()); - if (existingRow.getFiBackground() != null) - fiBkgds.add(existingRow.getFiBackground()); - - // clone the following properties from the existing row to the newRow: - // dataid, analyteid, type, container, protocolid, description, wellrole, extraSpecimenInfo, - // specimenID, participantID, visitID, date, dilution, tittration, singlepointcontrol - // note: don't clone rowid, beadcount, summary, lsid - newRow.setData(existingRow.getData()); - newRow.setAnalyte(existingRow.getAnalyte()); - newRow.setType(existingRow.getType()); - newRow.setWellRole(existingRow.getWellRole()); - newRow.setContainer(existingRow.getContainer()); - newRow.setProtocol(existingRow.getProtocol()); - newRow.setDescription(existingRow.getDescription()); - newRow.setSpecimenID(existingRow.getSpecimenID()); - newRow.setParticipantID(existingRow.getParticipantID()); - newRow.setVisitID(existingRow.getVisitID()); - newRow.setDate(existingRow.getDate()); - newRow.setExtraSpecimenInfo(existingRow.getExtraSpecimenInfo()); - newRow.setDilution(existingRow.getDilution()); - newRow.setTitration(existingRow.getTitration()); - newRow.setSinglePointControl(existingRow.getSinglePointControl()); - - // we can clone stdev and cv from existing raw rows because LuminexDataHandler ensureSummaryStats() calculates them - newRow.setStdDev(existingRow.getStdDev()); - newRow.setCv(existingRow.getCv()); - } - // Calculate FI and FI-BKGD values for the new summary row based on the existing raw data rows with the same dataId, analyteId, and type. - // similar to LuminexDataHandler ensureSummaryStats() - StatsService service = StatsService.get(); - MathStat statsFi = service.getStats(ArrayUtils.toPrimitive(fis.toArray(new Double[0]))); - newRow.setFi(Math.abs(statsFi.getMean())); - newRow.setFiString(newRow.getFi().toString()); - MathStat statsFiBkgd = service.getStats(ArrayUtils.toPrimitive(fiBkgds.toArray(new Double[0]))); - newRow.setFiBackground(Math.abs(statsFiBkgd.getMean())); - newRow.setFiBackgroundString(newRow.getFiBackground().toString()); - - // Calculate well to be a comma-separated list of wells from the existing raw data rows - newRow.setWell(String.join(",", wells)); - - // Generate an LSID for the new summary row - Lsid.LsidBuilder builder = new Lsid.LsidBuilder(LuminexAssayProvider.LUMINEX_DATA_ROW_LSID_PREFIX,""); - newRow.setLsid(builder.setObjectId(GUID.makeGUID()).toString()); - - // Insert the new summary row into the database. - // similar to LuminexDataHandler saveDataRows() - LuminexImportHelper helper = new LuminexImportHelper(); - Map row = new CaseInsensitiveHashMap<>(); - ObjectFactory f = ObjectFactory.Registry.getFactory(LuminexDataRow.class); - row.putAll(f.toMap(newRow, null)); - try - { - OntologyManager.insertTabDelimited(tableInfo, expRun.getContainer(), user, helper, MapDataIterator.of(List.of(row)).getDataIterator(new DataIteratorContext()), true, _log, null); - String comment = "Inserted missing summary row for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type; - ExperimentService.get().auditRunEvent(user, protocol, expRun, null, "LuminexUpgradeCode.checkForMissingSummaryRows: " + comment, null); - _log.info("..." + comment); + WellGroupKey groupKey = new WellGroupKey( + existingRow.getData(), + existingRow.getAnalyte(), + existingRow.getType(), + existingRow._getExtraProperties().get("Standard") + ); + rowsByWellGroup.computeIfAbsent(groupKey, k -> new ArrayList<>()).add(existingRow); } - catch (BatchValidationException e) + + // calculate summary stats and well information for the new summary rows that we will insert into the database + for (Map.Entry> wellGroupEntry : rowsByWellGroup.entrySet()) { - _log.warn("...failed to insert missing summary row for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type, e); + WellGroupKey groupKey = wellGroupEntry.getKey(); + LuminexDataRow newRow = new LuminexDataRow(); + newRow.setSummary(true); + newRow.setData(groupKey.dataId); + newRow.setAnalyte(groupKey.analyteId); + newRow.setType(groupKey.type); + + List fis = new ArrayList<>(); + List fiBkgds = new ArrayList<>(); + List wells = new ArrayList<>(); + for (LuminexDataRow existingRow : wellGroupEntry.getValue()) + { + // keep track of well, FI, and FI Bkgd values from existing raw data rows to use in calculating summary stats for the new summary row + wells.add(existingRow.getWell()); + if (existingRow.getFi() != null) + fis.add(existingRow.getFi()); + if (existingRow.getFiBackground() != null) + fiBkgds.add(existingRow.getFiBackground()); + + // clone the following properties from the existing row to the newRow: + // extraProperties, container, protocolid, description, wellrole, extraSpecimenInfo, + // specimenID, participantID, visitID, date, dilution, tittration, singlepointcontrol + // note: don't clone rowid, beadcount, lsid + newRow._setExtraProperties(existingRow._getExtraProperties()); + newRow.setWellRole(existingRow.getWellRole()); + newRow.setContainer(existingRow.getContainer()); + newRow.setProtocol(existingRow.getProtocol()); + newRow.setDescription(existingRow.getDescription()); + newRow.setSpecimenID(existingRow.getSpecimenID()); + newRow.setParticipantID(existingRow.getParticipantID()); + newRow.setVisitID(existingRow.getVisitID()); + newRow.setDate(existingRow.getDate()); + newRow.setExtraSpecimenInfo(existingRow.getExtraSpecimenInfo()); + newRow.setDilution(existingRow.getDilution()); + newRow.setTitration(existingRow.getTitration()); + newRow.setSinglePointControl(existingRow.getSinglePointControl()); + + // we can clone stdev and cv from existing raw rows because LuminexDataHandler ensureSummaryStats() calculates them + newRow.setStdDev(existingRow.getStdDev()); + newRow.setCv(existingRow.getCv()); + } + + // Calculate FI and FI-BKGD values for the new summary row based on the existing raw data rows with the same dataId, analyteId, type, and standard. + // similar to LuminexDataHandler ensureSummaryStats() + MathStat statsFi = service.getStats(ArrayUtils.toPrimitive(fis.toArray(new Double[0]))); + newRow.setFi(Math.abs(statsFi.getMean())); + newRow.setFiString(newRow.getFi().toString()); + MathStat statsFiBkgd = service.getStats(ArrayUtils.toPrimitive(fiBkgds.toArray(new Double[0]))); + newRow.setFiBackground(Math.abs(statsFiBkgd.getMean())); + newRow.setFiBackgroundString(newRow.getFiBackground().toString()); + + // Calculate well to be a comma-separated list of wells from the existing raw data rows + newRow.setWell(String.join(",", wells)); + + // Generate an LSID for the new summary row + Lsid.LsidBuilder builder = new Lsid.LsidBuilder(LuminexAssayProvider.LUMINEX_DATA_ROW_LSID_PREFIX,""); + newRow.setLsid(builder.setObjectId(GUID.makeGUID()).toString()); + + // Insert the new summary row into the database. + // similar to LuminexDataHandler saveDataRows() + LuminexImportHelper helper = new LuminexImportHelper(); + Map row = new CaseInsensitiveHashMap<>(newRow._getExtraProperties()); + ObjectFactory f = ObjectFactory.Registry.getFactory(LuminexDataRow.class); + row.putAll(f.toMap(newRow, null)); + try + { + OntologyManager.insertTabDelimited(tableInfo, expRun.getContainer(), user, helper, MapDataIterator.of(List.of(row)).getDataIterator(new DataIteratorContext()), true, _log, null); + String comment = "Inserted missing summary row for Luminex runId: " + runid + ", dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type + ", standard: " + groupKey.standard; + ExperimentService.get().auditRunEvent(user, protocol, expRun, null, "LuminexUpgradeCode.checkForMissingSummaryRows: " + comment, null); + _log.info("..." + comment); + } + catch (BatchValidationException e) + { + _log.warn("...failed to insert missing summary row for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type + ", standard: " + groupKey.standard, e); + } } }); tx.commit(); } } -} +} \ No newline at end of file From 0a0e380962b67269d5bf4d22bb86a6f2f06cdbfa Mon Sep 17 00:00:00 2001 From: cnathe Date: Thu, 26 Mar 2026 08:15:50 -0500 Subject: [PATCH 3/4] - add explicit date filter so we only check for assay runs imported after the issue was deployed --- luminex/src/org/labkey/luminex/LuminexUpgradeCode.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/luminex/src/org/labkey/luminex/LuminexUpgradeCode.java b/luminex/src/org/labkey/luminex/LuminexUpgradeCode.java index b389c4bcb..5c8228661 100644 --- a/luminex/src/org/labkey/luminex/LuminexUpgradeCode.java +++ b/luminex/src/org/labkey/luminex/LuminexUpgradeCode.java @@ -58,7 +58,8 @@ public static void checkForMissingSummaryRows(ModuleContext ctx) SELECT DISTINCT d.runid, dr_false.dataid, dr_false.analyteid, dr_false.type FROM luminex.datarow dr_false LEFT JOIN exp.data d ON d.rowid = dr_false.dataid - WHERE dr_false.summary = false + WHERE d.created > '2025-02-17' -- NOTE: GitHub Issue 875 only applies to runs imported after this date + AND dr_false.summary = false AND EXISTS (SELECT 1 FROM luminex.datarow WHERE dataid = dr_false.dataid AND summary = true) AND EXISTS (SELECT 1 FROM luminex.datarow WHERE dataid = dr_false.dataid AND summary = false) AND NOT EXISTS (SELECT 1 FROM luminex.datarow dr_true From f9c9d4a6fbf884700b5c05e3cd9e039ef3b7b3b9 Mon Sep 17 00:00:00 2001 From: cnathe Date: Thu, 26 Mar 2026 09:26:27 -0500 Subject: [PATCH 4/4] Claude CR feedback - revert change to LuminexDataHandler, upgrade code SQLFragment updates for MSSQL compatibility --- .../labkey/luminex/LuminexDataHandler.java | 2 +- .../labkey/luminex/LuminexUpgradeCode.java | 78 +++++++++++-------- 2 files changed, 45 insertions(+), 35 deletions(-) diff --git a/luminex/src/org/labkey/luminex/LuminexDataHandler.java b/luminex/src/org/labkey/luminex/LuminexDataHandler.java index 378afdb7a..dd97666a8 100644 --- a/luminex/src/org/labkey/luminex/LuminexDataHandler.java +++ b/luminex/src/org/labkey/luminex/LuminexDataHandler.java @@ -620,7 +620,7 @@ private Map getExistingAnalytes(ExpRun expRun) return existingAnalytes; } - public static class DataRowKey + private static class DataRowKey { private final long _dataId; private final int _analyteId; diff --git a/luminex/src/org/labkey/luminex/LuminexUpgradeCode.java b/luminex/src/org/labkey/luminex/LuminexUpgradeCode.java index 5c8228661..5d44838d7 100644 --- a/luminex/src/org/labkey/luminex/LuminexUpgradeCode.java +++ b/luminex/src/org/labkey/luminex/LuminexUpgradeCode.java @@ -1,7 +1,6 @@ package org.labkey.luminex; import org.apache.commons.lang3.ArrayUtils; -import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.labkey.api.assay.AssayService; import org.labkey.api.collections.CaseInsensitiveHashMap; @@ -13,6 +12,7 @@ import org.labkey.api.data.SqlSelector; import org.labkey.api.data.TableSelector; import org.labkey.api.data.UpgradeCode; +import org.labkey.api.data.dialect.SqlDialect; import org.labkey.api.data.statistics.MathStat; import org.labkey.api.data.statistics.StatsService; import org.labkey.api.dataiterator.DataIteratorContext; @@ -27,6 +27,7 @@ import org.labkey.api.query.FieldKey; import org.labkey.api.security.User; import org.labkey.api.util.GUID; +import org.labkey.api.util.logging.LogHelper; import org.labkey.luminex.model.LuminexDataRow; import org.labkey.luminex.query.LuminexDataTable; import org.labkey.luminex.query.LuminexProtocolSchema; @@ -38,7 +39,7 @@ public class LuminexUpgradeCode implements UpgradeCode { - private static final Logger _log = LogManager.getLogger(LuminexUpgradeCode.class); + private static final Logger LOG = LogHelper.getLogger(LuminexUpgradeCode.class, "Luminex upgrade code"); /** * GitHub Issue #875: Upgrade code to check for Luminex assay runs that have both summary and raw data but are missing summary rows. @@ -54,26 +55,28 @@ public static void checkForMissingSummaryRows(ModuleContext ctx) { // For any Luminex dataids (input files) that have both summary and raw data rows, // find Luminex raw data rows (summary = false) that don't have a corresponding summary data row (summary = true) - SQLFragment missingSummaryRowsSql = new SQLFragment(""" - SELECT DISTINCT d.runid, dr_false.dataid, dr_false.analyteid, dr_false.type - FROM luminex.datarow dr_false - LEFT JOIN exp.data d ON d.rowid = dr_false.dataid - WHERE d.created > '2025-02-17' -- NOTE: GitHub Issue 875 only applies to runs imported after this date - AND dr_false.summary = false - AND EXISTS (SELECT 1 FROM luminex.datarow WHERE dataid = dr_false.dataid AND summary = true) - AND EXISTS (SELECT 1 FROM luminex.datarow WHERE dataid = dr_false.dataid AND summary = false) - AND NOT EXISTS (SELECT 1 FROM luminex.datarow dr_true - WHERE dr_true.summary = true - AND dr_true.dataid = dr_false.dataid - AND dr_true.analyteid = dr_false.analyteid - AND dr_true.type = dr_false.type - ) - """); - - int missingSummaryRowCount = new SqlSelector(scope, new SQLFragment("SELECT COUNT(*) FROM (").append(missingSummaryRowsSql).append(")")).getObject(Integer.class); + // NOTE: the d.created date filter is because the GitHub Issue 875 only applies to runs imported after this date + SqlDialect dialect = LuminexProtocolSchema.getSchema().getSqlDialect(); + SQLFragment missingSummaryRowsSql = new SQLFragment(""" + SELECT DISTINCT d.runid, dr_false.dataid, dr_false.analyteid, dr_false.type + FROM luminex.datarow dr_false + LEFT JOIN exp.data d ON d.rowid = dr_false.dataid + WHERE d.created > '2025-02-17' + AND dr_false.summary = """).append(dialect.getBooleanFALSE()).append("\n").append(""" + AND EXISTS (SELECT 1 FROM luminex.datarow WHERE dataid = dr_false.dataid AND summary = """).append(dialect.getBooleanTRUE()).append(")\n").append(""" + AND EXISTS (SELECT 1 FROM luminex.datarow WHERE dataid = dr_false.dataid AND summary = """).append(dialect.getBooleanFALSE()).append(")\n").append(""" + AND NOT EXISTS (SELECT 1 FROM luminex.datarow dr_true + WHERE dr_true.summary = """).append(dialect.getBooleanTRUE()).append("\n").append(""" + AND dr_true.dataid = dr_false.dataid + AND dr_true.analyteid = dr_false.analyteid + AND dr_true.type = dr_false.type + ) + """); + + int missingSummaryRowCount = new SqlSelector(scope, new SQLFragment("SELECT COUNT(*) FROM (").append(missingSummaryRowsSql).append(") as subq")).getObject(Integer.class); if (missingSummaryRowCount == 0) { - _log.info("No missing summary rows found for Luminex assay data."); + LOG.info("No missing summary rows found for Luminex assay data."); return; } @@ -86,16 +89,16 @@ AND NOT EXISTS (SELECT 1 FROM luminex.datarow dr_true ExpRun expRun = ExperimentService.get().getExpRun(runid); if (expRun == null) { - _log.warn("Could not find run for runid: " + runid + ", skipping missing summary row check for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type); + LOG.warn("Could not find run for runid: " + runid + ", skipping missing summary row check for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type); return; } - _log.info("Missing summary row for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type + " in run: " + expRun.getName() + " (" + expRun.getRowId() + ")"); + LOG.info("Missing summary row for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type + " in run: " + expRun.getName() + " (" + expRun.getRowId() + ")"); // currently only inserting summary rows for Background (type = B) data rows if (!"B".equals(type)) { - _log.warn("...not inserting missing summary row for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type + " because type is not 'B' (Background)"); + LOG.warn("...not inserting missing summary row for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type + " because type is not 'B' (Background)"); return; } @@ -109,7 +112,7 @@ AND NOT EXISTS (SELECT 1 FROM luminex.datarow dr_true filter.addCondition(FieldKey.fromParts("Type"), type); // keep track of the set of wells for the given dataId/analyteId/type/standard combinations - record WellGroupKey(long dataId, int analyteId, String type, Object standard) {} + record WellGroupKey(long dataId, int analyteId, String type, String standard) {} Map> rowsByWellGroup = new HashMap<>(); for (Map databaseMap : new TableSelector(tableInfo, filter, null).getMapCollection()) { @@ -120,7 +123,7 @@ record WellGroupKey(long dataId, int analyteId, String type, Object standard) {} existingRow.getData(), existingRow.getAnalyte(), existingRow.getType(), - existingRow._getExtraProperties().get("Standard") + (String) existingRow._getExtraProperties().get("Standard") ); rowsByWellGroup.computeIfAbsent(groupKey, k -> new ArrayList<>()).add(existingRow); } @@ -172,12 +175,18 @@ record WellGroupKey(long dataId, int analyteId, String type, Object standard) {} // Calculate FI and FI-BKGD values for the new summary row based on the existing raw data rows with the same dataId, analyteId, type, and standard. // similar to LuminexDataHandler ensureSummaryStats() - MathStat statsFi = service.getStats(ArrayUtils.toPrimitive(fis.toArray(new Double[0]))); - newRow.setFi(Math.abs(statsFi.getMean())); - newRow.setFiString(newRow.getFi().toString()); - MathStat statsFiBkgd = service.getStats(ArrayUtils.toPrimitive(fiBkgds.toArray(new Double[0]))); - newRow.setFiBackground(Math.abs(statsFiBkgd.getMean())); - newRow.setFiBackgroundString(newRow.getFiBackground().toString()); + if (!fis.isEmpty()) + { + MathStat statsFi = service.getStats(ArrayUtils.toPrimitive(fis.toArray(new Double[0]))); + newRow.setFi(Math.abs(statsFi.getMean())); + newRow.setFiString(newRow.getFi().toString()); + } + if (!fiBkgds.isEmpty()) + { + MathStat statsFiBkgd = service.getStats(ArrayUtils.toPrimitive(fiBkgds.toArray(new Double[0]))); + newRow.setFiBackground(Math.abs(statsFiBkgd.getMean())); + newRow.setFiBackgroundString(newRow.getFiBackground().toString()); + } // Calculate well to be a comma-separated list of wells from the existing raw data rows newRow.setWell(String.join(",", wells)); @@ -192,16 +201,17 @@ record WellGroupKey(long dataId, int analyteId, String type, Object standard) {} Map row = new CaseInsensitiveHashMap<>(newRow._getExtraProperties()); ObjectFactory f = ObjectFactory.Registry.getFactory(LuminexDataRow.class); row.putAll(f.toMap(newRow, null)); + row.put("summary", true); // make sure the extra properties value from the raw row didn't override the summary setting try { - OntologyManager.insertTabDelimited(tableInfo, expRun.getContainer(), user, helper, MapDataIterator.of(List.of(row)).getDataIterator(new DataIteratorContext()), true, _log, null); + OntologyManager.insertTabDelimited(tableInfo, expRun.getContainer(), user, helper, MapDataIterator.of(List.of(row)).getDataIterator(new DataIteratorContext()), true, LOG, null); String comment = "Inserted missing summary row for Luminex runId: " + runid + ", dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type + ", standard: " + groupKey.standard; ExperimentService.get().auditRunEvent(user, protocol, expRun, null, "LuminexUpgradeCode.checkForMissingSummaryRows: " + comment, null); - _log.info("..." + comment); + LOG.info("..." + comment); } catch (BatchValidationException e) { - _log.warn("...failed to insert missing summary row for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type + ", standard: " + groupKey.standard, e); + LOG.warn("...failed to insert missing summary row for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type + ", standard: " + groupKey.standard, e); } } });