From aee986987983b8cfc6f4ecf4d33fbaca5ed126c8 Mon Sep 17 00:00:00 2001 From: "nikita.smirnov" Date: Tue, 27 Aug 2024 19:05:08 +0400 Subject: [PATCH] [TH2-5234] Added messageIdsLookupLimit option --- README.md | 2 +- .../entities/configuration/Configuration.kt | 7 +++++++ .../requests/SseMessageSearchRequest.kt | 18 ++++++++++++------ .../handlers/SearchMessagesHandler.kt | 15 +++++++++++---- .../handlers/messages/MessageExtractor.kt | 19 ++++--------------- .../th2/rptdataprovider/server/HttpServer.kt | 2 +- 6 files changed, 36 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 804536a5..bdde59b7 100644 --- a/README.md +++ b/README.md @@ -296,7 +296,7 @@ spec: # Release notes ## 5.13.0 -* Provided ability to limit `messageIds` request by `lookupLimitDays` argument +* Provided ability to limit `messageIds` request by `lookupLimitDays` argument or `messageIdsLookupLimit` option * Updated: * th2 gradle plugin: `0.1.1` * common: `5.14.0-dev` diff --git a/src/main/kotlin/com/exactpro/th2/rptdataprovider/entities/configuration/Configuration.kt b/src/main/kotlin/com/exactpro/th2/rptdataprovider/entities/configuration/Configuration.kt index 5a6cfbcf..d4d88831 100644 --- a/src/main/kotlin/com/exactpro/th2/rptdataprovider/entities/configuration/Configuration.kt +++ b/src/main/kotlin/com/exactpro/th2/rptdataprovider/entities/configuration/Configuration.kt @@ -48,6 +48,7 @@ class CustomConfigurationClass { val messageUnpackerOutputMessageBuffer: Int = 100 val messageFilterOutputMessageBuffer: Int = 100 val messageMergerOutputMessageBuffer: Int = 10 + val messageIdsLookupLimit: Int = 60 * 30 * 1_000 val codecResponseTimeout: Int = 6_000 val codecPendingBatchLimit: Int = 16 @@ -177,6 +178,12 @@ class Configuration(customConfiguration: CustomConfigurationClass) { "10" ) + val messageIdsLookupLimit: Variable = Variable( + "messageIdsLookupLimit", + customConfiguration.messageIdsLookupLimit.toString(), + "1800000" + ) + val codecResponseTimeout: Variable = Variable( "codecResponseTimeout", customConfiguration.codecResponseTimeout.toString(), "6000" diff --git a/src/main/kotlin/com/exactpro/th2/rptdataprovider/entities/requests/SseMessageSearchRequest.kt b/src/main/kotlin/com/exactpro/th2/rptdataprovider/entities/requests/SseMessageSearchRequest.kt index 6cd37081..4f200ad1 100644 --- a/src/main/kotlin/com/exactpro/th2/rptdataprovider/entities/requests/SseMessageSearchRequest.kt +++ b/src/main/kotlin/com/exactpro/th2/rptdataprovider/entities/requests/SseMessageSearchRequest.kt @@ -39,7 +39,7 @@ data class SseMessageSearchRequest( val endTimestamp: Instant?, val resultCountLimit: Int?, val attachedEvents: Boolean, - val lookupLimitDays: Int?, + val lookupLimit: Long?, val resumeFromIdsList: List, val includeProtocols: List?, val excludeProtocols: List?, @@ -70,7 +70,7 @@ data class SseMessageSearchRequest( resultCountLimit = parameters["resultCountLimit"]?.firstOrNull()?.toInt(), attachedEvents = parameters["attachedEvents"]?.firstOrNull()?.toBoolean() ?: false, - lookupLimitDays = parameters["lookupLimitDays"]?.firstOrNull()?.toInt(), + lookupLimit = parameters["lookupLimitDays"]?.firstOrNull()?.toLong()?.run { this * 24 * 60 * 60 * 1_000 }, includeProtocols = parameters["includeProtocols"], excludeProtocols = parameters["excludeProtocols"], @@ -127,7 +127,7 @@ data class SseMessageSearchRequest( attachedEvents = false, - lookupLimitDays = null, + lookupLimit = null, includeProtocols = null, @@ -165,13 +165,19 @@ data class SseMessageSearchRequest( } private fun checkLookupLimitDays() { - if (lookupLimitDays != null && endTimestamp != null) { + if (lookupLimit != null && endTimestamp != null) { throw InvalidRequestException( - "endTimestamp: $endTimestamp must be null if lookupLimitDays: $lookupLimitDays isn't null" + "endTimestamp: $endTimestamp must be null if lookupLimit: $lookupLimit isn't null" ) } } + private fun checkNoEndTimestamp() { + if (endTimestamp != null) { + throw InvalidRequestException("endTimestamp: $endTimestamp must be null") + } + } + private fun checkStartPoint() { if (startTimestamp == null && resumeFromIdsList.isEmpty()) throw InvalidRequestException("One of the 'startTimestamp' or 'messageId' must not be null") @@ -211,7 +217,7 @@ data class SseMessageSearchRequest( fun checkIdsRequest() { checkStartPoint() - checkEndTimestamp() + checkNoEndTimestamp() checkStreamList() checkTimestampAndId() checkLookupLimitDays() diff --git a/src/main/kotlin/com/exactpro/th2/rptdataprovider/handlers/SearchMessagesHandler.kt b/src/main/kotlin/com/exactpro/th2/rptdataprovider/handlers/SearchMessagesHandler.kt index 222e959d..799452ba 100644 --- a/src/main/kotlin/com/exactpro/th2/rptdataprovider/handlers/SearchMessagesHandler.kt +++ b/src/main/kotlin/com/exactpro/th2/rptdataprovider/handlers/SearchMessagesHandler.kt @@ -127,7 +127,10 @@ abstract class SearchMessagesHandler( } } - suspend fun getIds(request: SseMessageSearchRequest): Map> { + suspend fun getIds(request: SseMessageSearchRequest, lookupLimit: Long): Map> { + require(request.startTimestamp != null && request.endTimestamp == null) { + "startTimestamp must be not null and endTimestamp be null in request: $request" + } searchMessageRequests.inc() val resumeId = request.resumeFromIdsList.firstOrNull() val messageId = resumeId?.let { @@ -143,8 +146,8 @@ abstract class SearchMessagesHandler( request.copy(startTimestamp = resumeId.timestamp) } ?: request - val before = getIds(resultRequest, messageId, TimeRelation.BEFORE) - val after = getIds(resultRequest, messageId, TimeRelation.AFTER) + val before = getIds(resultRequest, messageId, lookupLimit, TimeRelation.BEFORE) + val after = getIds(resultRequest, messageId, lookupLimit, TimeRelation.AFTER) return mapOf( TimeRelationMapper.toHttp(TimeRelation.BEFORE) to before, @@ -161,9 +164,13 @@ abstract class SearchMessagesHandler( private suspend fun getIds( request: SseMessageSearchRequest, messageId: StoredMessageId?, + lookupLimit: Long, searchDirection: TimeRelation ): MutableList { - val resultRequest = request.copy(searchDirection = searchDirection) + val resultRequest = request.copy( + searchDirection = searchDirection, + lookupLimit = request.lookupLimit ?: lookupLimit + ) val pipelineStatus = PipelineStatus() diff --git a/src/main/kotlin/com/exactpro/th2/rptdataprovider/handlers/messages/MessageExtractor.kt b/src/main/kotlin/com/exactpro/th2/rptdataprovider/handlers/messages/MessageExtractor.kt index 4c51f3fd..bd06fb43 100644 --- a/src/main/kotlin/com/exactpro/th2/rptdataprovider/handlers/messages/MessageExtractor.kt +++ b/src/main/kotlin/com/exactpro/th2/rptdataprovider/handlers/messages/MessageExtractor.kt @@ -42,7 +42,6 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.launch import java.time.Instant -import java.time.temporal.ChronoUnit class MessageExtractor( @@ -184,25 +183,15 @@ class MessageExtractor( startTimestamp?.let { builder.timestampFrom().isGreaterThanOrEqualTo(it) } endTimestamp?.let { builder.timestampTo().isLessThan(it) } - if (startTimestamp != null && - endTimestamp == null && - lookupLimitDays != null - ) { - builder.timestampTo().isLessThan( - startTimestamp.plus(lookupLimitDays.toLong(), ChronoUnit.DAYS) - ) + if (startTimestamp != null && endTimestamp == null && lookupLimit != null) { + builder.timestampTo().isLessThan(startTimestamp.plusMillis(lookupLimit)) } } else { startTimestamp?.let { builder.timestampTo().isLessThanOrEqualTo(it) } endTimestamp?.let { builder.timestampFrom().isGreaterThan(it) } - if (startTimestamp != null && - endTimestamp == null && - lookupLimitDays != null - ) { - builder.timestampFrom().isGreaterThan( - startTimestamp.minus(lookupLimitDays.toLong(), ChronoUnit.DAYS) - ) + if (startTimestamp != null && endTimestamp == null && lookupLimit != null) { + builder.timestampFrom().isGreaterThan(startTimestamp.minusMillis(lookupLimit)) } } } diff --git a/src/main/kotlin/com/exactpro/th2/rptdataprovider/server/HttpServer.kt b/src/main/kotlin/com/exactpro/th2/rptdataprovider/server/HttpServer.kt index a635e1e2..7891dcfa 100644 --- a/src/main/kotlin/com/exactpro/th2/rptdataprovider/server/HttpServer.kt +++ b/src/main/kotlin/com/exactpro/th2/rptdataprovider/server/HttpServer.kt @@ -445,7 +445,7 @@ class HttpServer( queryParametersMap, messageFiltersPredicateFactory.getEmptyPredicate(), ).also(SseMessageSearchRequest<*, *>::checkIdsRequest) - searchMessagesHandler.getIds(request) + searchMessagesHandler.getIds(request, configuration.messageIdsLookupLimit.value.toLong()) } }