From eecb4770514f4d576728e695362a4cff81cdf69a Mon Sep 17 00:00:00 2001 From: d-bernat Date: Wed, 17 Apr 2024 19:52:11 +0200 Subject: [PATCH] fix: Investigate the reason of differences in events x enrollments in analytics query[2.40-DHIS2-16227] --- .../data/JdbcEnrollmentAnalyticsManager.java | 44 +++++++++++++------ .../data/EnrollmentAnalyticsManagerTest.java | 34 +++++++------- 2 files changed, 48 insertions(+), 30 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 51dec038f13b..1c2b01eb5cfd 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -92,7 +92,9 @@ public class JdbcEnrollmentAnalyticsManager extends AbstractJdbcEventAnalyticsMa private static final String ANALYTICS_EVENT = "analytics_event_"; - private static final String ORDER_BY_EXECUTION_DATE = "order by executiondate "; + private static final String DIRECTION_PLACEHOLDER = "#DIRECTION_PLACEHOLDER"; + private static final String ORDER_BY_EXECUTION_DATE = + "order by executiondate " + DIRECTION_PLACEHOLDER + ", created " + DIRECTION_PLACEHOLDER; private static final String LIMIT_1 = "limit 1"; @@ -517,8 +519,9 @@ protected ColumnAndAlias getCoordinateColumn(final QueryItem item, final String + colName + " is not null " + psCondition - + ORDER_BY_EXECUTION_DATE - + createOrderTypeAndOffset(item.getProgramStageOffset()) + + createOrderType(item.getProgramStageOffset()) + + " " + + createOffset(item.getProgramStageOffset()) + " " + LIMIT_1 + " )", @@ -569,8 +572,9 @@ protected String getColumn(final QueryItem item, final String suffix) { + getExecutionDateFilter( item.getRepeatableStageParams().getStartDate(), item.getRepeatableStageParams().getEndDate()) - + ORDER_BY_EXECUTION_DATE - + createOrderTypeAndOffset(item.getProgramStageOffset()) + + createOrderType(item.getProgramStageOffset()) + + " " + + createOffset(item.getProgramStageOffset()) + getLimit(item.getRepeatableStageParams().getCount()) + " ) as t1)"; } @@ -591,8 +595,9 @@ protected String getColumn(final QueryItem item, final String suffix) { + getExecutionDateFilter( item.getRepeatableStageParams().getStartDate(), item.getRepeatableStageParams().getEndDate()) - + ORDER_BY_EXECUTION_DATE - + createOrderTypeAndOffset(item.getProgramStageOffset()) + + createOrderType(item.getProgramStageOffset()) + + " " + + createOffset(item.getProgramStageOffset()) + " " + LIMIT_1 + " )"; @@ -618,8 +623,9 @@ protected String getColumn(final QueryItem item, final String suffix) { + "and ps = '" + item.getProgramStage().getUid() + "' " - + ORDER_BY_EXECUTION_DATE - + createOrderTypeAndOffset(item.getProgramStageOffset()) + + createOrderType(item.getProgramStageOffset()) + + " " + + createOffset(item.getProgramStageOffset()) + " " + LIMIT_1 + " )"; @@ -690,14 +696,26 @@ protected AnalyticsType getAnalyticsType() { return AnalyticsType.ENROLLMENT; } - private String createOrderTypeAndOffset(int offset) { + private String createOffset(int offset) { + if (offset == 0) { + return EMPTY; + } + + if (offset < 0) { + return "offset " + (-1 * offset); + } else { + return "offset " + (offset - 1); + } + } + + private String createOrderType(int offset) { if (offset == 0) { - return "desc"; + return ORDER_BY_EXECUTION_DATE.replace(DIRECTION_PLACEHOLDER, "desc"); } if (offset < 0) { - return "desc offset " + (-1 * offset); + return ORDER_BY_EXECUTION_DATE.replace(DIRECTION_PLACEHOLDER, "desc"); } else { - return "asc offset " + (offset - 1); + return ORDER_BY_EXECUTION_DATE.replace(DIRECTION_PLACEHOLDER, "asc"); } } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsManagerTest.java index 69df258b56d5..3bc4e7c6c94c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsManagerTest.java @@ -206,7 +206,7 @@ private void verifyWithProgramStageAndNumericDataElement(ValueType valueType) { + programA.getUid() + ".pi = ax.pi and \"fWIAEtYVEGk\" is not null and ps = '" + programStage.getUid() - + "' order by executiondate desc limit 1 )"; + + "' order by executiondate desc, created desc limit 1 )"; if (valueType == ValueType.NUMBER) { subSelect = subSelect + " as \"fWIAEtYVEGk\""; @@ -259,7 +259,7 @@ private void verifyWithRepeatableProgramStageAndDataElement(ValueType valueType) + programUid + ".pi = ax.pi and ps = '" + repeatableProgramStage.getUid() - + "' order by executiondate desc offset 1 limit 1 ) " + + "' order by executiondate desc, created desc offset 1 limit 1 ) " + "as \"" + programStageUid + "[-1]." @@ -274,7 +274,7 @@ private void verifyWithRepeatableProgramStageAndDataElement(ValueType valueType) + programUid + ".pi = ax.pi and ps = '" + programStageUid - + "' order by executiondate desc offset 1 limit 1 )) " + + "' order by executiondate desc, created desc offset 1 limit 1 )) " + "as \"" + programStageUid + "[-1]." @@ -306,7 +306,7 @@ void verifyWithProgramStageAndTextualDataElementAndFilter() { + programA.getUid() + ".pi = ax.pi and \"fWIAEtYVEGk\" is not null and ps = '" + programStage.getUid() - + "' order by executiondate desc limit 1 )"; + + "' order by executiondate desc, created desc limit 1 )"; String expected = "ax.\"quarterly\",ax.\"ou\"," @@ -357,7 +357,7 @@ void verifyWithProgramStageAndNumericDataElementAndFilter2() { + programA.getUid() + ".pi = ax.pi and \"fWIAEtYVEGk\" is not null and ps = '" + programStage.getUid() - + "' order by executiondate desc limit 1 )"; + + "' order by executiondate desc, created desc limit 1 )"; String expected = "ax.\"quarterly\",ax.\"ou\"," @@ -407,7 +407,7 @@ void verifyGetEnrollmentsWithMissingValueEqFilter() { + programA.getUid() + ".pi = ax.pi and \"fWIAEtYVEGk\" is not null and ps = '" + programStage.getUid() - + "' order by executiondate desc limit 1 )"; + + "' order by executiondate desc, created desc limit 1 )"; String expected = subSelect + " is null"; @@ -426,7 +426,7 @@ void verifyGetEnrollmentsWithMissingValueNeqFilter() { + programA.getUid() + ".pi = ax.pi and \"fWIAEtYVEGk\" is not null and ps = '" + programStage.getUid() - + "' order by executiondate desc limit 1 )"; + + "' order by executiondate desc, created desc limit 1 )"; String expected = subSelect + " is not null"; testIt( @@ -444,7 +444,7 @@ void verifyGetEnrollmentsWithMissingValueAndNumericValuesInFilter() { + programA.getUid() + ".pi = ax.pi and \"fWIAEtYVEGk\" is not null and ps = '" + programStage.getUid() - + "' order by executiondate desc limit 1 )"; + + "' order by executiondate desc, created desc limit 1 )"; String numericValues = String.join(OPTION_SEP, "10", "11", "12"); String expected = @@ -472,7 +472,7 @@ void verifyGetEnrollmentsWithoutMissingValueAndNumericValuesInFilter() { + programA.getUid() + ".pi = ax.pi and \"fWIAEtYVEGk\" is not null and ps = '" + programStage.getUid() - + "' order by executiondate desc limit 1 )"; + + "' order by executiondate desc, created desc limit 1 )"; String numericValues = String.join(OPTION_SEP, "10", "11", "12"); String expected = subSelect + " in (" + String.join(",", numericValues.split(OPTION_SEP)) + ")"; @@ -491,7 +491,7 @@ void verifyGetEnrollmentsWithOnlyMissingValueInFilter() { + programA.getUid() + ".pi = ax.pi and \"fWIAEtYVEGk\" is not null and ps = '" + programStage.getUid() - + "' order by executiondate desc limit 1 )"; + + "' order by executiondate desc, created desc limit 1 )"; String expected = subSelect + " is null"; String unexpected = "(" + subSelect + " in ("; @@ -729,7 +729,7 @@ void verifyGetColumnOfTypeCoordinateAndWithProgramStages() { + dataElementA.getUid() + "\" is not null and ps = '" + programStage.getUid() - + "' order by executiondate desc limit 1 )")); + + "' order by executiondate desc, created desc limit 1 )")); } @Test @@ -763,7 +763,7 @@ void verifyGetColumnOfTypeCoordinateAndWithProgramStagesAndParamsWithReferenceTy + programB.getUid() + ".pi = ax.pi and ps = '" + repeatableProgramStage.getUid() - + "' and executiondate >= '2022-01-01' and executiondate <= '2022-01-31' order by executiondate desc LIMIT 100 ) as t1)")); + + "' and executiondate >= '2022-01-01' and executiondate <= '2022-01-31' order by executiondate desc, created desc LIMIT 100 ) as t1)")); } @Test @@ -795,7 +795,7 @@ void verifyGetColumnOfTypeCoordinateAndWithProgramStagesAndParamsWithNumberTypeV + programB.getUid() + ".pi = ax.pi and ps = '" + repeatableProgramStage.getUid() - + "' order by executiondate desc limit 1 )")); + + "' order by executiondate desc, created desc limit 1 )")); } @Test @@ -834,8 +834,8 @@ void verifyGetCoordinateColumnAndNoProgramStage() { + "and " + colName + " is not null " - + "order by executiondate " - + "desc limit 1 )")); + + "order by executiondate desc, created desc " + + " limit 1 )")); } @Test @@ -877,8 +877,8 @@ void verifyGetCoordinateColumnWithProgramStage() { + " is not null " + "and ps = '" + item.getProgramStage().getUid() - + "' order by executiondate " - + "desc limit 1 )")); + + "' order by executiondate desc, created desc " + + " limit 1 )")); } @Test