From 5e365cdae394772b5f073e3efd634bb0dee63c1d Mon Sep 17 00:00:00 2001 From: Vakhtang Donadze <51744864+vdonadze@users.noreply.github.com> Date: Tue, 12 Sep 2023 13:16:53 +0000 Subject: [PATCH] [TH2-5025] update CR converted to support new urlPath structure (conversion from v2 to v2.2) (#32) Co-authored-by: Vakhtang Donadze Co-authored-by: andrey.shulika Co-authored-by: Oleg --- build.gradle | 31 ++- config/detekt/detekt.yml | 16 +- gradle.properties | 2 +- .../th2/converter/ConverterApplication.kt | 18 +- .../controllers/ConverterController.kt | 45 ++-- .../th2/converter/conversion/Converter.kt | 211 +++++++++------ .../th2/converter/conversion/LinksInserter.kt | 6 +- .../conversion/LocalFilesConverter.kt | 9 +- .../th2/converter/fun/ConvertibleBoxSpecV1.kt | 6 +- .../th2/converter/fun/ConvertibleBoxSpecV2.kt | 21 +- .../converter/fun/ConvertibleBoxSpecV2x2.kt | 29 +++ .../fun/ExtendedSettingsExtensions.kt | 177 ++++++++++++- .../th2/converter/model/Th2Resource.kt | 12 +- .../th2/converter/util/ProjectConstants.kt | 14 +- .../{RepositoryUtils.kt => Repository.kt} | 26 ++ .../th2/converter/util/SchemaVersion.kt | 29 +++ src/test/kotlin/ConverterTest.kt | 243 +++++++++++++++--- src/test/resources/V1/act.yml | 6 + src/test/resources/V1/check1.yml | 9 +- src/test/resources/V1/fix-server.yml | 3 + src/test/resources/V1/script.yml | 9 + src/test/resources/V2/act.yml | 6 + src/test/resources/V2/check1.yml | 10 +- src/test/resources/V2/fix-server.yml | 3 + src/test/resources/V2/script.yml | 8 + src/test/resources/V2_2/act.yml | 65 +++++ src/test/resources/V2_2/check1.yml | 80 ++++++ src/test/resources/V2_2/fix-server.yml | 105 ++++++++ src/test/resources/V2_2/script.yml | 44 ++++ 29 files changed, 1053 insertions(+), 190 deletions(-) create mode 100644 src/main/kotlin/com/exactpro/th2/converter/fun/ConvertibleBoxSpecV2x2.kt rename src/main/kotlin/com/exactpro/th2/converter/util/{RepositoryUtils.kt => Repository.kt} (83%) create mode 100644 src/main/kotlin/com/exactpro/th2/converter/util/SchemaVersion.kt create mode 100644 src/test/resources/V2_2/act.yml create mode 100644 src/test/resources/V2_2/check1.yml create mode 100644 src/test/resources/V2_2/fix-server.yml create mode 100644 src/test/resources/V2_2/script.yml diff --git a/build.gradle b/build.gradle index e67dbbc..403f5dd 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,6 @@ +import com.github.jk1.license.filter.LicenseBundleNormalizer +import com.github.jk1.license.render.JsonReportRenderer + plugins { id "org.jetbrains.kotlin.jvm" version "${kotlin_version}" id "org.jetbrains.kotlin.kapt" version "${kotlin_version}" @@ -11,6 +14,8 @@ plugins { id 'signing' id 'distribution' id "org.owasp.dependencycheck" version "${owaspVersion}" + id 'com.github.jk1.dependency-license-report' version '2.5' + id "de.undercouch.download" version "5.4.0" } ext { @@ -18,7 +23,7 @@ ext { fabric8_version = '6.6.2' jackson_databind_version = '2.15.2' infra_repo_version = "1.2.4" - resource_model_version = '1.3.3' + resource_model_version = '1.4.0' slf4j_api_version = '2.0.5' slf4j2_impl_version = '2.19.0' junit_jupiter_version = '5.9.2' @@ -80,7 +85,6 @@ dependencies { implementation "org.apache.commons:commons-text:$commons_text_version" implementation group: 'com.fasterxml.jackson.module', name: 'jackson-module-kotlin', version: jackson_databind_version - testImplementation "org.junit.jupiter:junit-jupiter:$junit_jupiter_version" testImplementation "org.junit.jupiter:junit-jupiter:$junit_jupiter_version" testImplementation group: 'org.jetbrains.kotlin', name: 'kotlin-test-junit5', version: kotlin_version @@ -253,4 +257,25 @@ configurations { compileClasspath { resolutionStrategy.activateDependencyLocking() } -} \ No newline at end of file +} + +licenseReport { + def licenseNormalizerBundlePath = "$buildDir/license-normalizer-bundle.json" + + if (!file(licenseNormalizerBundlePath).exists()) { + download.run { + src 'https://raw.githubusercontent.com/th2-net/.github/main/license-compliance/gradle-license-report/license-normalizer-bundle.json' + dest "$buildDir/license-normalizer-bundle.json" + overwrite false + } + } + + filters = [ + new LicenseBundleNormalizer(licenseNormalizerBundlePath, false) + ] + renderers = [ + new JsonReportRenderer('licenses.json', false), + ] + excludeOwnGroup = false + allowedLicensesFile = new URL("https://raw.githubusercontent.com/th2-net/.github/main/license-compliance/gradle-license-report/allowed-licenses.json") +} diff --git a/config/detekt/detekt.yml b/config/detekt/detekt.yml index b3d80fa..851a769 100644 --- a/config/detekt/detekt.yml +++ b/config/detekt/detekt.yml @@ -106,6 +106,10 @@ complexity: - 'run' - 'use' - 'with' + CyclomaticComplexMethod: + active: true + excludes: + - '**/fun/ExtendedSettingsExtensions**' LabeledExpression: active: false ignoredLabels: [] @@ -114,7 +118,9 @@ complexity: threshold: 600 LongMethod: active: true - threshold: 65 + threshold: 75 + excludes: + - '**/fun/ExtendedSettingsExtensions**' LongParameterList: active: true functionThreshold: 6 @@ -547,6 +553,8 @@ style: - '0' - '1' - '2' + - '8080' + - '80' ignoreHashCodeFunction: true ignorePropertyDeclaration: false ignoreLocalVariableDeclaration: false @@ -599,11 +607,13 @@ style: active: false ReturnCount: active: true - max: 2 + max: 3 excludedFunctions: 'equals' excludeLabeled: false excludeReturnFromLambda: true excludeGuardClauses: false + excludes: + - '**/fun/ExtendedSettingsExtensions**' SafeCast: active: true SerialVersionUIDInSerializableClass: @@ -612,7 +622,7 @@ style: active: true ThrowsCount: active: true - max: 2 + max: 3 excludeGuardClauses: false TrailingWhitespace: active: false diff --git a/gradle.properties b/gradle.properties index e4eb848..bf790e7 100644 --- a/gradle.properties +++ b/gradle.properties @@ -14,7 +14,7 @@ # limitations under the License. # -release_version=1.3.6 +release_version=1.4.0 description = 'cr schema converter' kotlin.code.style=official diff --git a/src/main/kotlin/com/exactpro/th2/converter/ConverterApplication.kt b/src/main/kotlin/com/exactpro/th2/converter/ConverterApplication.kt index 494f7aa..ed1c91b 100644 --- a/src/main/kotlin/com/exactpro/th2/converter/ConverterApplication.kt +++ b/src/main/kotlin/com/exactpro/th2/converter/ConverterApplication.kt @@ -17,6 +17,7 @@ package com.exactpro.th2.converter import com.exactpro.th2.converter.conversion.LocalFilesConverter +import com.exactpro.th2.converter.util.SchemaVersion import io.micronaut.runtime.Micronaut import mu.KotlinLogging @@ -24,6 +25,14 @@ open class ConverterApplication private val logger = KotlinLogging.logger {} +private const val LOCAL_USAGE: String = + "Usage: local " + +private const val ARGUMENTS_SIZE = 4 +private const val SCHEMA_INDEX = 1 +private const val CURRENT_VERSION_INDEX = 2 +private const val TARGET_VERSION_INDEX = 3 + fun main(args: Array) { when (val mode = if (args.isNotEmpty()) args[0] else "server") { "server" -> { @@ -40,10 +49,17 @@ fun main(args: Array) { } } "local" -> { - LocalFilesConverter(args[1], args[2]).convert() + require(args.size == ARGUMENTS_SIZE) { LOCAL_USAGE } + val currentVersion: SchemaVersion = getEnumValue(args[CURRENT_VERSION_INDEX]) + val targetVersion: SchemaVersion = getEnumValue(args[TARGET_VERSION_INDEX]) + LocalFilesConverter(args[SCHEMA_INDEX], currentVersion, targetVersion).convert() } else -> { logger.error("Mode: {} is not supported", mode) } } } + +private fun getEnumValue(name: String): SchemaVersion = + SchemaVersion.values().find { it.name.equals(name, ignoreCase = true) } + ?: error("unknown schema version $name") diff --git a/src/main/kotlin/com/exactpro/th2/converter/controllers/ConverterController.kt b/src/main/kotlin/com/exactpro/th2/converter/controllers/ConverterController.kt index 2657bf4..ec0ede9 100644 --- a/src/main/kotlin/com/exactpro/th2/converter/controllers/ConverterController.kt +++ b/src/main/kotlin/com/exactpro/th2/converter/controllers/ConverterController.kt @@ -21,13 +21,13 @@ import com.exactpro.th2.converter.controllers.errors.BadRequestException import com.exactpro.th2.converter.controllers.errors.ErrorCode import com.exactpro.th2.converter.conversion.Converter.convertFromGit import com.exactpro.th2.converter.conversion.Converter.convertFromRequest -import com.exactpro.th2.converter.util.ProjectConstants -import com.exactpro.th2.converter.util.ProjectConstants.PROPAGATION_DENY -import com.exactpro.th2.converter.util.ProjectConstants.PROPAGATION_RULE -import com.exactpro.th2.converter.util.ProjectConstants.SOURCE_BRANCH +import com.exactpro.th2.converter.util.RepositoryUtils.PROPAGATION_DENY +import com.exactpro.th2.converter.util.RepositoryUtils.PROPAGATION_RULE +import com.exactpro.th2.converter.util.RepositoryUtils.SOURCE_BRANCH import com.exactpro.th2.converter.util.RepositoryUtils.schemaExists import com.exactpro.th2.converter.util.RepositoryUtils.updateRepository import com.exactpro.th2.converter.util.RepositoryUtils.updateSchemaK8sPropagation +import com.exactpro.th2.converter.util.SchemaVersion import com.exactpro.th2.infrarepo.git.GitterContext import com.exactpro.th2.infrarepo.repo.Repository import com.exactpro.th2.infrarepo.repo.RepositoryResource @@ -43,16 +43,16 @@ import org.eclipse.jgit.errors.EntryExistsException @Produces(MediaType.APPLICATION_JSON) class ConverterController { - @Post("convert/{schemaName}/{targetVersion}") + @Post("convert/{schemaName}/{currentVersion}/{targetVersion}") fun convertInSameBranch( schemaName: String, - targetVersion: String + currentVersion: SchemaVersion, + targetVersion: SchemaVersion ): ConversionSummary { - checkRequestedVersion(targetVersion) val gitterContext = GitterContext.getContext(ApplicationConfig.git) checkSourceSchema(schemaName, gitterContext) val gitter = gitterContext.getGitter(schemaName) - val conversionResult = convertFromGit(targetVersion, gitter) + val conversionResult = convertFromGit(currentVersion, targetVersion, gitter) val currentResponse = conversionResult.summary if (currentResponse.hasErrors()) { return currentResponse @@ -60,7 +60,6 @@ class ConverterController { try { gitter.lock() - Repository.removeLinkResources(gitter) updateRepository(conversionResult.convertedResources, gitter, Repository::update) conversionResult.summary.commitRef = gitter.commitAndPush("Schema conversion") } finally { @@ -69,17 +68,18 @@ class ConverterController { return conversionResult.summary } - @Post("convert/{sourceSchemaName}/{newSchemaName}/{targetVersion}") + @Post("convert/{sourceSchemaName}/{newSchemaName}/{currentVersion}/{targetVersion}") fun convertInNewBranch( sourceSchemaName: String, newSchemaName: String, - targetVersion: String + currentVersion: SchemaVersion, + targetVersion: SchemaVersion ): ConversionSummary { - checkRequestedVersion(targetVersion) val gitterContext: GitterContext = GitterContext.getContext(ApplicationConfig.git) checkSourceSchema(sourceSchemaName, gitterContext) val sourceBranchGitter = gitterContext.getGitter(sourceSchemaName) - val conversionResult = convertFromGit(targetVersion, sourceBranchGitter) + val newBranchGitter = gitterContext.getGitter(newSchemaName) + val conversionResult = convertFromGit(currentVersion, targetVersion, sourceBranchGitter, newBranchGitter) val currentResponse = conversionResult.summary if (currentResponse.hasErrors()) { return currentResponse @@ -94,12 +94,10 @@ class ConverterController { sourceBranchGitter.unlock() } - val newBranchGitter = gitterContext.getGitter(newSchemaName) try { newBranchGitter.lock() newBranchGitter.createBranch(sourceSchemaName) updateSchemaK8sPropagation(PROPAGATION_RULE, newBranchGitter) - Repository.removeLinkResources(newBranchGitter) updateRepository(conversionResult.convertedResources, newBranchGitter, Repository::update) conversionResult.summary.commitRef = newBranchGitter.commitAndPush("Schema conversion") } catch (e: EntryExistsException) { @@ -113,13 +111,13 @@ class ConverterController { return conversionResult.summary } - @Get("convert/{targetVersion}") + @Get("convert/{currentVersion}/{targetVersion}") fun convertRequestedResources( - targetVersion: String, + currentVersion: SchemaVersion, + targetVersion: SchemaVersion, @Body resources: Set ): ConversionResult { - checkRequestedVersion(targetVersion) - return convertFromRequest(targetVersion, resources) + return convertFromRequest(currentVersion, targetVersion, resources) } @Get("test") @@ -127,15 +125,6 @@ class ConverterController { return "Conversion API is working !" } - private fun checkRequestedVersion(version: String) { - if (version !in ProjectConstants.ACCEPTED_VERSIONS) { - throw BadRequestException( - ErrorCode.VERSION_NOT_ALLOWED, - "Conversion to specified version: '$version' is not supported" - ) - } - } - private fun checkSourceSchema(schemaName: String, gitterContext: GitterContext) { if (!schemaExists(schemaName, gitterContext)) { throw BadRequestException( diff --git a/src/main/kotlin/com/exactpro/th2/converter/conversion/Converter.kt b/src/main/kotlin/com/exactpro/th2/converter/conversion/Converter.kt index 73149fb..21d9a8b 100644 --- a/src/main/kotlin/com/exactpro/th2/converter/conversion/Converter.kt +++ b/src/main/kotlin/com/exactpro/th2/converter/conversion/Converter.kt @@ -24,130 +24,160 @@ import com.exactpro.th2.converter.controllers.errors.ErrorCode import com.exactpro.th2.converter.`fun`.Convertible import com.exactpro.th2.converter.`fun`.ConvertibleBoxSpecV1 import com.exactpro.th2.converter.`fun`.ConvertibleBoxSpecV2 +import com.exactpro.th2.converter.`fun`.ConvertibleBoxSpecV2x2 import com.exactpro.th2.converter.model.Th2Resource import com.exactpro.th2.converter.util.Mapper.YAML_MAPPER -import com.exactpro.th2.converter.util.ProjectConstants.API_VERSION_V1 -import com.exactpro.th2.converter.util.ProjectConstants.SHORT_API_VERSION_V2 +import com.exactpro.th2.converter.util.RepositoryContext +import com.exactpro.th2.converter.util.SchemaVersion import com.exactpro.th2.infrarepo.ResourceType import com.exactpro.th2.infrarepo.git.Gitter import com.exactpro.th2.infrarepo.repo.Repository import com.exactpro.th2.infrarepo.repo.RepositoryResource import com.exactpro.th2.model.latest.box.Spec import com.exactpro.th2.model.v1.box.SpecV1 +import com.exactpro.th2.model.v2.SpecV2 import com.fasterxml.jackson.module.kotlin.convertValue object Converter { - fun convertFromGit( - version: String, - gitter: Gitter + currentVersion: SchemaVersion, + targetVersion: SchemaVersion, + sourceBranchGitter: Gitter, + newBranchGitter: Gitter? = null ): ConversionResult { val summary = ConversionSummary() - val convertedResources: List - - when (version) { - SHORT_API_VERSION_V2 -> { - val boxesToConvert: Set - val links: Set - try { - gitter.lock() - boxesToConvert = HashSet(Repository.getAllBoxesAndStores(gitter)) - links = HashSet(Repository.getResourcesByKind(gitter, ResourceType.Th2Link)) - } finally { - gitter.unlock() - } - convertedResources = convert(boxesToConvert, API_VERSION_V1, summary) + val repositoryContext = RepositoryContext.load(sourceBranchGitter) - val linksInserter = LinksInserter() - linksInserter.insertLinksIntoBoxes(convertedResources, links) - linksInserter.addErrorsToSummary(summary) - return ConversionResult(summary, convertedResources) - } - else -> throw BadRequestException( - ErrorCode.VERSION_NOT_ALLOWED, - "Conversion to specified version: '$version' is not supported" - ) + if (!validateCurrentSchemaVersion(repositoryContext.allResources, currentVersion.apiVersion, summary)) { + return ConversionResult(summary, emptyList()) } + + val gitter = newBranchGitter ?: sourceBranchGitter + return processSwitching(currentVersion, targetVersion, repositoryContext, summary, gitter) } fun convertFromRequest( - targetVersion: String, + currentVersion: SchemaVersion, + targetVersion: SchemaVersion, resources: Set ): ConversionResult { val summary = ConversionSummary() - val convertedResources: List - - when (targetVersion) { - SHORT_API_VERSION_V2 -> { - val linkKind = ResourceType.Th2Link.kind() - val boxKinds = setOf( - ResourceType.Th2Box.kind(), - ResourceType.Th2CoreBox.kind(), - ResourceType.Th2Estore.kind(), - ResourceType.Th2Mstore.kind() - ) - - val boxesToConvert = resources.filterTo(HashSet()) { boxKinds.contains(it.kind) } - convertedResources = convert(boxesToConvert, API_VERSION_V1, summary) - - val links = resources.filterTo(HashSet()) { it.kind.equals(linkKind) } - val linksInserter = LinksInserter() - linksInserter.insertLinksIntoBoxes(convertedResources, links) - linksInserter.addErrorsToSummary(summary) - } - else -> throw BadRequestException( - ErrorCode.VERSION_NOT_ALLOWED, - "Conversion to specified version: '$targetVersion' is not supported" - ) + val linkKind = ResourceType.Th2Link.kind() + val boxKinds = setOf( + ResourceType.Th2Box.kind(), + ResourceType.Th2CoreBox.kind(), + ResourceType.Th2Estore.kind(), + ResourceType.Th2Mstore.kind() + ) + + val boxes = resources.filterTo(HashSet()) { boxKinds.contains(it.kind) } + val links = resources.filterTo(HashSet()) { it.kind.equals(linkKind) } + + val repositoryContext = RepositoryContext(boxes, links) + if (!validateCurrentSchemaVersion(repositoryContext.allResources, currentVersion.apiVersion, summary)) { + return ConversionResult(summary, emptyList()) } - return ConversionResult(summary, convertedResources) + + return processSwitching(currentVersion, targetVersion, repositoryContext, summary) } fun convertLocal( - version: String, + currentVersion: SchemaVersion, + targetVersion: SchemaVersion, gitter: Gitter ): ConversionResult { val summary = ConversionSummary() - val convertedResources: List - when (version) { - SHORT_API_VERSION_V2 -> { - val boxesToConvert = Repository.getAllBoxesAndStores(gitter, false) - val links = Repository.getResourcesByKind(gitter, ResourceType.Th2Link, false) - convertedResources = convert(boxesToConvert, API_VERSION_V1, summary) + val boxes = Repository.getAllBoxesAndStores(gitter, false) + val links = Repository.getResourcesByKind(gitter, ResourceType.Th2Link, false) + val repositoryContext = RepositoryContext(boxes, links) + + if (!validateCurrentSchemaVersion(repositoryContext.allResources, currentVersion.apiVersion, summary)) { + return ConversionResult(summary, emptyList()) + } + + return processSwitching(currentVersion, targetVersion, repositoryContext, summary) + } + + private fun processSwitching( + currentVersion: SchemaVersion, + targetVersion: SchemaVersion, + repositoryContext: RepositoryContext, + summary: ConversionSummary, + gitter: Gitter? = null, + ): ConversionResult { + when (targetVersion) { + SchemaVersion.V1 -> throw BadRequestException( + ErrorCode.VERSION_NOT_ALLOWED, + "Conversion to specified version: '${SchemaVersion.V1}' is not supported" + ) + SchemaVersion.V2 -> { + if (currentVersion != SchemaVersion.V1) { + throw BadRequestException( + ErrorCode.VERSION_NOT_ALLOWED, + """Conversion to v2 is only allowed from v1.specified. 2 + |specified current version: '$currentVersion' is not supported + """.trimMargin() + ) + } + val convertedResources = convert(repositoryContext.boxes, summary) val linksInserter = LinksInserter() - linksInserter.insertLinksIntoBoxes(convertedResources, links) + linksInserter.insertLinksIntoBoxes(convertedResources, repositoryContext.links) linksInserter.addErrorsToSummary(summary) + if (gitter != null) { + Repository.removeLinkResources(gitter) + } return ConversionResult(summary, convertedResources) } - else -> throw BadRequestException( - ErrorCode.VERSION_NOT_ALLOWED, - "Conversion to specified version: '$version' is not supported" - ) + + SchemaVersion.V2_2 -> { + when (currentVersion) { + SchemaVersion.V1 -> { + var convertedResources = convert(repositoryContext.boxes, summary) + val linksInserter = LinksInserter() + linksInserter.insertLinksIntoBoxes(convertedResources, repositoryContext.links) + linksInserter.addErrorsToSummary(summary) + if (gitter != null) { + Repository.removeLinkResources(gitter) + } + val repositoryResourcesV2 = convertedResources.map(Th2Resource::toRepositoryResource) + convertedResources = convert(repositoryResourcesV2, summary) + return ConversionResult(summary, convertedResources) + } + + SchemaVersion.V2 -> { + val convertedResources = convert(repositoryContext.boxes, summary) + return ConversionResult(summary, convertedResources) + } + + else -> { + throw BadRequestException( + ErrorCode.VERSION_NOT_ALLOWED, + """Conversion to v2-2 is only allowed from v1 or v2.specified. + |specified current version: '$currentVersion' is not supported + """.trimMargin() + ) + } + } + } } } private inline fun convert( - resources: Set, - fromVersion: String, + resources: Collection, summary: ConversionSummary ): List { val convertedResources: MutableList = ArrayList() for (resource in resources) { - if (!resource.apiVersion.equals(fromVersion)) { - summary.errorMessages.add( - ErrorMessage( - resource.metadata.name, - "Resource must be of version: $fromVersion" - ) - ) - continue - } - try { val specFrom: From = YAML_MAPPER.convertValue(resource.spec) - val resourceFrom = Th2Resource(resource.apiVersion, resource.kind, resource.metadata, wrap(specFrom)) + val resourceFrom = Th2Resource( + SchemaVersion.fromApiVersion(resource.apiVersion), + resource.kind, + resource.metadata, + wrap(specFrom) + ) convertedResources.add(resourceFrom.toNextVersion()) summary.convertedResourceNames.add(resource.metadata.name) } catch (e: Exception) { @@ -157,10 +187,31 @@ object Converter { return convertedResources } + private fun validateCurrentSchemaVersion( + resources: Set, + currentVersion: String, + summary: ConversionSummary + ): Boolean { + var isValid = true + resources.forEach { resource -> + if (currentVersion != resource.apiVersion) { + isValid = false + summary.errorMessages.add( + ErrorMessage( + resource.metadata.name, + "Resource api version ${resource.version}. is different requested api version: $currentVersion" + ) + ) + } + } + return isValid + } + private fun wrap(specFrom: From): Convertible { return when (specFrom) { is SpecV1 -> ConvertibleBoxSpecV1(specFrom) - is Spec -> ConvertibleBoxSpecV2(specFrom) + is SpecV2 -> ConvertibleBoxSpecV2(specFrom) + is Spec -> ConvertibleBoxSpecV2x2(specFrom) else -> throw AssertionError("Provided spec class is not supported for conversion") } } diff --git a/src/main/kotlin/com/exactpro/th2/converter/conversion/LinksInserter.kt b/src/main/kotlin/com/exactpro/th2/converter/conversion/LinksInserter.kt index 1671aeb..566e712 100644 --- a/src/main/kotlin/com/exactpro/th2/converter/conversion/LinksInserter.kt +++ b/src/main/kotlin/com/exactpro/th2/converter/conversion/LinksInserter.kt @@ -22,10 +22,10 @@ import com.exactpro.th2.converter.`fun`.ConvertibleBoxSpecV2 import com.exactpro.th2.converter.model.Th2Resource import com.exactpro.th2.converter.util.Mapper.YAML_MAPPER import com.exactpro.th2.infrarepo.repo.RepositoryResource -import com.exactpro.th2.model.latest.box.Spec import com.exactpro.th2.model.latest.link.LinkEndpoint import com.exactpro.th2.model.v1.link.LinkSpecV1 import com.exactpro.th2.model.v1.link.MultiDictionary +import com.exactpro.th2.model.v2.SpecV2 import com.fasterxml.jackson.module.kotlin.convertValue import com.fasterxml.jackson.module.kotlin.readValue import java.util.regex.Pattern @@ -122,7 +122,7 @@ class LinksInserter { resToLinkMap.forEach { (key, value) -> if (convertedResourcesMap.containsKey(key)) { val resource = convertedResourcesMap[key] - val spec: Spec = YAML_MAPPER.convertValue(resource!!.spec) + val spec: SpecV2 = YAML_MAPPER.convertValue(resource!!.spec) val mqPinMap = spec.pins?.mq?.subscribers?.associateBy { it.name } val grpcPinMap = spec.pins?.grpc?.client?.associateBy { it.name } value.forEach { (key, value) -> @@ -144,7 +144,7 @@ class LinksInserter { } } - private fun insertDictionariesAsAliases(spec: Spec, multiDictionaries: MutableList) { + private fun insertDictionariesAsAliases(spec: SpecV2, multiDictionaries: MutableList) { val customConfigStr = YAML_MAPPER.writeValueAsString(spec.customConfig) val patternStr: StringBuilder = StringBuilder() val dictionary: MutableMap = HashMap() diff --git a/src/main/kotlin/com/exactpro/th2/converter/conversion/LocalFilesConverter.kt b/src/main/kotlin/com/exactpro/th2/converter/conversion/LocalFilesConverter.kt index e2ae3f7..bf79753 100644 --- a/src/main/kotlin/com/exactpro/th2/converter/conversion/LocalFilesConverter.kt +++ b/src/main/kotlin/com/exactpro/th2/converter/conversion/LocalFilesConverter.kt @@ -19,19 +19,24 @@ package com.exactpro.th2.converter.conversion import com.exactpro.th2.converter.config.ApplicationConfig import com.exactpro.th2.converter.model.Th2Resource import com.exactpro.th2.converter.util.RepositoryUtils.updateRepository +import com.exactpro.th2.converter.util.SchemaVersion import com.exactpro.th2.infrarepo.ResourceType import com.exactpro.th2.infrarepo.git.GitterContext import com.exactpro.th2.infrarepo.repo.Repository import com.exactpro.th2.infrarepo.repo.RepositoryResource import mu.KotlinLogging -class LocalFilesConverter(private val schema: String, private val version: String) { +class LocalFilesConverter( + private val schema: String, + private val currentVersion: SchemaVersion, + private val targetVersion: SchemaVersion +) { private val logger = KotlinLogging.logger {} private val gitterContext = GitterContext.getContext(ApplicationConfig.git) fun convert() { val gitter = gitterContext.getGitter(schema) - val result = Converter.convertLocal(version, gitter) + val result = Converter.convertLocal(currentVersion, targetVersion, gitter) val dictionaries = Repository.getResourcesByKind(gitter, ResourceType.Th2Dictionary, false) if (result.summary.hasErrors()) { logger.error("Conversion for schema {} failed due to following errors", schema) diff --git a/src/main/kotlin/com/exactpro/th2/converter/fun/ConvertibleBoxSpecV1.kt b/src/main/kotlin/com/exactpro/th2/converter/fun/ConvertibleBoxSpecV1.kt index fa8216d..7667931 100644 --- a/src/main/kotlin/com/exactpro/th2/converter/fun/ConvertibleBoxSpecV1.kt +++ b/src/main/kotlin/com/exactpro/th2/converter/fun/ConvertibleBoxSpecV1.kt @@ -16,7 +16,6 @@ package com.exactpro.th2.converter.`fun` -import com.exactpro.th2.model.latest.box.Spec import com.exactpro.th2.model.latest.box.pins.GrpcClient import com.exactpro.th2.model.latest.box.pins.GrpcSection import com.exactpro.th2.model.latest.box.pins.GrpcServer @@ -26,17 +25,18 @@ import com.exactpro.th2.model.latest.box.pins.MqSubscriber import com.exactpro.th2.model.latest.box.pins.PinSpec import com.exactpro.th2.model.v1.box.SpecV1 import com.exactpro.th2.model.v1.box.pins.PinType +import com.exactpro.th2.model.v2.SpecV2 class ConvertibleBoxSpecV1(val spec: SpecV1) : Convertible { override fun toNextVersion(): Convertible { return ConvertibleBoxSpecV2( - Spec( + SpecV2( spec.imageName, spec.imageVersion, spec.type, spec.versionRange, spec.customConfig, - spec.extendedSettings?.toExtendedSettings(), + spec.extendedSettings?.toExtendedSettingsV2(), convertPins().takeIf { it.isNotEmpty() }, spec.prometheus, spec.loggingConfig, diff --git a/src/main/kotlin/com/exactpro/th2/converter/fun/ConvertibleBoxSpecV2.kt b/src/main/kotlin/com/exactpro/th2/converter/fun/ConvertibleBoxSpecV2.kt index bcbe2d1..b416aa2 100644 --- a/src/main/kotlin/com/exactpro/th2/converter/fun/ConvertibleBoxSpecV2.kt +++ b/src/main/kotlin/com/exactpro/th2/converter/fun/ConvertibleBoxSpecV2.kt @@ -17,10 +17,27 @@ package com.exactpro.th2.converter.`fun` import com.exactpro.th2.model.latest.box.Spec +import com.exactpro.th2.model.v2.SpecV2 -class ConvertibleBoxSpecV2(val spec: Spec) : Convertible { +class ConvertibleBoxSpecV2(val spec: SpecV2) : Convertible { override fun toNextVersion(): Convertible { - throw AssertionError("This is the latest version. Further conversions are not supported") + return ConvertibleBoxSpecV2x2( + Spec( + spec.imageName, + spec.imageVersion, + spec.type, + spec.versionRange, + spec.customConfig, + spec.extendedSettings?.toExtendedSettings(), + spec.pins, + spec.prometheus, + spec.loggingConfig, + spec.mqRouter, + spec.grpcRouter, + spec.cradleManager, + spec.disabled + ) + ) } override fun getSpecObject(): Any { diff --git a/src/main/kotlin/com/exactpro/th2/converter/fun/ConvertibleBoxSpecV2x2.kt b/src/main/kotlin/com/exactpro/th2/converter/fun/ConvertibleBoxSpecV2x2.kt new file mode 100644 index 0000000..5ad6ddb --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/converter/fun/ConvertibleBoxSpecV2x2.kt @@ -0,0 +1,29 @@ +/* + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.converter.`fun` + +import com.exactpro.th2.model.latest.box.Spec + +class ConvertibleBoxSpecV2x2(val spec: Spec) : Convertible { + override fun toNextVersion(): Convertible { + throw AssertionError("This is the latest version. Further conversions are not supported") + } + + override fun getSpecObject(): Any { + return spec + } +} diff --git a/src/main/kotlin/com/exactpro/th2/converter/fun/ExtendedSettingsExtensions.kt b/src/main/kotlin/com/exactpro/th2/converter/fun/ExtendedSettingsExtensions.kt index dda4ed3..ca62eaf 100644 --- a/src/main/kotlin/com/exactpro/th2/converter/fun/ExtendedSettingsExtensions.kt +++ b/src/main/kotlin/com/exactpro/th2/converter/fun/ExtendedSettingsExtensions.kt @@ -27,34 +27,42 @@ import com.exactpro.th2.model.v1.box.extendedsettings.ServiceV1 import com.exactpro.th2.model.v1.box.extendedsettings.ServiceV1.ServiceType.ClusterIP import com.exactpro.th2.model.v1.box.extendedsettings.ServiceV1.ServiceType.LoadBalancer import com.exactpro.th2.model.v1.box.extendedsettings.ServiceV1.ServiceType.NodePort +import com.exactpro.th2.model.v2.ClusterIpConfigV2 +import com.exactpro.th2.model.v2.ExtendedSettingsV2 +import com.exactpro.th2.model.v2.LoadBalancerConfigV2 +import com.exactpro.th2.model.v2.NodePortConfigV2 +import com.exactpro.th2.model.v2.ServiceV2 -fun ServiceV1.toService(): Service { - val nodePort: MutableList = ArrayList() - val clusterIP: MutableList = ArrayList() - val loadBalancer: MutableList = ArrayList() +fun ServiceV1.toServiceV2(): ServiceV2 { + val nodePort: MutableList = ArrayList() + val clusterIP: MutableList = ArrayList() + val loadBalancer: MutableList = ArrayList() val addToRelevantList: (ServiceEndpoint) -> Unit = when (type) { NodePort -> { { - nodePort.add(NodePortConfig(it.name, it.targetPort, it.nodePort ?: -1)) + nodePort.add(NodePortConfigV2(it.name, it.targetPort, it.nodePort ?: -1)) } } + ClusterIP -> { { - clusterIP.add(ClusterIpConfig(it.name, it.targetPort)) + clusterIP.add(ClusterIpConfigV2(it.name, it.targetPort)) } } + LoadBalancer -> { { - loadBalancer.add(LoadBalancerConfig(it.name, it.targetPort)) + loadBalancer.add(LoadBalancerConfigV2(it.name, it.targetPort)) } } + null -> { _ -> } } endpoints?.forEach(addToRelevantList) - return Service( + return ServiceV2( enabled, nodePort.takeIf { it.isNotEmpty() }, clusterIP.takeIf { it.isNotEmpty() }, @@ -63,7 +71,158 @@ fun ServiceV1.toService(): Service { ) } -fun ExtendedSettingsV1.toExtendedSettings(): ExtendedSettings { +fun ServiceV2.toService(): Service { + if (ingress == null) { + return Service( + enabled, + this.nodePort?.map { NodePortConfig(it.name, it.containerPort, it.exposedPort) }, + this.clusterIP?.map { ClusterIpConfig(it.name, it.containerPort) }, + this.loadBalancer?.map { LoadBalancerConfig(it.name, it.containerPort) } + ) + } + assert(ingress!!.urlPaths!!.size == 1) { + "Service can't be upgraded to version v2-2. 'ingress.urlPaths' must contain 1 item" + } + + val multiConfigErrorMessage = + "Service can't be upgraded to next version. only one out of nodePort, clusterIP or loadBalancer is allowed" + val urlPath = ingress!!.urlPaths!![0] + val port8080 = 8080 + val port80 = 80 + if (nodePort != null) { + assert(clusterIP == null) { multiConfigErrorMessage } + assert(loadBalancer == null) { multiConfigErrorMessage } + val nodePortsV2 = nodePort!!.map { NodePortConfig(it.name, it.containerPort, it.exposedPort) }.toMutableList() + if (nodePort!!.size == 1) { + val firstPort = nodePortsV2[0] + nodePortsV2[0] = NodePortConfig(firstPort.name, firstPort.containerPort, firstPort.exposedPort, urlPath) + } else { + var contains8080 = false + var contains80 = false + nodePortsV2.forEachIndexed breaking@{ index, nodePortConfig -> + if (nodePortConfig.containerPort == port8080) { + val port = nodePortsV2[index] + nodePortsV2[index] = NodePortConfig(port.name, port.containerPort, port.exposedPort, urlPath) + contains8080 = true + return@breaking + } + } + if (!contains8080) { + nodePortsV2.forEachIndexed breaking@{ index, nodePortConfig -> + if (nodePortConfig.containerPort == port80) { + val port = nodePortsV2[index] + nodePortsV2[index] = NodePortConfig(port.name, port.containerPort, port.exposedPort, urlPath) + contains80 = true + return@breaking + } + } + } + if (!contains80 && !contains8080) { + val firstPort = nodePortsV2[0] + nodePortsV2[0] = NodePortConfig(firstPort.name, firstPort.containerPort, firstPort.exposedPort, urlPath) + } + } + return Service( + enabled, + nodePort = nodePortsV2 + ) + } else if (clusterIP != null) { + assert(nodePort == null) { multiConfigErrorMessage } + assert(loadBalancer == null) { multiConfigErrorMessage } + val clusterIPV2 = clusterIP!!.map { ClusterIpConfig(it.name, it.containerPort) }.toMutableList() + if (clusterIP!!.size == 1) { + val firstPort = clusterIPV2[0] + clusterIPV2[0] = ClusterIpConfig(firstPort.name, firstPort.containerPort, urlPath) + } else { + var contains8080 = false + var contains80 = false + clusterIPV2.forEachIndexed breaking@{ index, nodePortConfig -> + if (nodePortConfig.containerPort == port8080) { + val port = clusterIPV2[index] + clusterIPV2[index] = ClusterIpConfig(port.name, port.containerPort, urlPath) + contains8080 = true + return@breaking + } + } + if (!contains8080) { + clusterIPV2.forEachIndexed breaking@{ index, nodePortConfig -> + if (nodePortConfig.containerPort == port80) { + val port = clusterIPV2[index] + clusterIPV2[index] = ClusterIpConfig(port.name, port.containerPort, urlPath) + contains80 = true + return@breaking + } + } + } + if (!contains80 && !contains8080) { + val firstPort = clusterIPV2[0] + clusterIPV2[0] = ClusterIpConfig(firstPort.name, firstPort.containerPort, urlPath) + } + } + return Service( + enabled, + clusterIP = clusterIPV2 + ) + } else if (loadBalancer != null) { + assert(clusterIP == null) { multiConfigErrorMessage } + assert(nodePort == null) { multiConfigErrorMessage } + val loadBalancerV2 = loadBalancer!!.map { LoadBalancerConfig(it.name, it.containerPort) }.toMutableList() + if (loadBalancer!!.size == 1) { + val firstPort = loadBalancerV2[0] + loadBalancerV2[0] = LoadBalancerConfig(firstPort.name, firstPort.containerPort, urlPath) + } else { + var contains8080 = false + var contains80 = false + loadBalancerV2.forEachIndexed breaking@{ index, nodePortConfig -> + if (nodePortConfig.containerPort == port8080) { + val port = loadBalancerV2[index] + loadBalancerV2[index] = LoadBalancerConfig(port.name, port.containerPort, urlPath) + contains8080 = true + return@breaking + } + } + if (!contains8080) { + loadBalancerV2.forEachIndexed breaking@{ index, nodePortConfig -> + if (nodePortConfig.containerPort == port80) { + val port = loadBalancerV2[index] + loadBalancerV2[index] = LoadBalancerConfig(port.name, port.containerPort, urlPath) + contains80 = true + return@breaking + } + } + } + if (!contains80 && !contains8080) { + val firstPort = loadBalancerV2[0] + loadBalancerV2[0] = LoadBalancerConfig(firstPort.name, firstPort.containerPort, urlPath) + } + } + return Service( + enabled, + loadBalancer = loadBalancerV2 + ) + } + throw IllegalArgumentException( + "When ingress.urlPath is specified, one of nodePort, clusterIP or loadBalancer should also be specified " + ) +} + +fun ExtendedSettingsV1.toExtendedSettingsV2(): ExtendedSettingsV2 { + return ExtendedSettingsV2( + envVariables, + sharedMemory, + replicas, + k8sProbes, + externalBox, + hostAliases, + hostNetwork, + nodeSelector, + mounting, + resources, + service?.toServiceV2() + ) +} + +fun ExtendedSettingsV2.toExtendedSettings(): ExtendedSettings { return ExtendedSettings( envVariables, sharedMemory, diff --git a/src/main/kotlin/com/exactpro/th2/converter/model/Th2Resource.kt b/src/main/kotlin/com/exactpro/th2/converter/model/Th2Resource.kt index 5c3deca..57c712a 100644 --- a/src/main/kotlin/com/exactpro/th2/converter/model/Th2Resource.kt +++ b/src/main/kotlin/com/exactpro/th2/converter/model/Th2Resource.kt @@ -17,26 +17,32 @@ package com.exactpro.th2.converter.model import com.exactpro.th2.converter.`fun`.Convertible -import com.exactpro.th2.converter.util.ProjectConstants.API_VERSION_V2 +import com.exactpro.th2.converter.util.SchemaVersion +import com.exactpro.th2.infrarepo.repo.RepositoryResource import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.annotation.JsonProperty import io.fabric8.kubernetes.api.model.ObjectMeta data class Th2Resource( - val apiVersion: String, + @JsonIgnore val schemaVersion: SchemaVersion, val kind: String, val metadata: ObjectMeta, @JsonIgnore var specWrapper: Convertible, ) { + @JsonProperty + val apiVersion = schemaVersion.apiVersion + @JsonProperty val spec = specWrapper.getSpecObject() fun toNextVersion(): Th2Resource { return Th2Resource( - API_VERSION_V2, + schemaVersion.nextVersion!!, kind, metadata, specWrapper.toNextVersion() ) } + + fun toRepositoryResource() = RepositoryResource(apiVersion, kind, metadata, spec) } diff --git a/src/main/kotlin/com/exactpro/th2/converter/util/ProjectConstants.kt b/src/main/kotlin/com/exactpro/th2/converter/util/ProjectConstants.kt index 70bdf0e..bb21023 100644 --- a/src/main/kotlin/com/exactpro/th2/converter/util/ProjectConstants.kt +++ b/src/main/kotlin/com/exactpro/th2/converter/util/ProjectConstants.kt @@ -16,16 +16,4 @@ package com.exactpro.th2.converter.util -object ProjectConstants { - const val API_VERSION_V1 = "th2.exactpro.com/v1" - const val API_VERSION_V2 = "th2.exactpro.com/v2" - - const val SHORT_API_VERSION_V2 = "v2" - - val ACCEPTED_VERSIONS = listOf(SHORT_API_VERSION_V2) - - const val PROPAGATION_RULE = "rule" - const val PROPAGATION_DENY = "deny" - - const val SOURCE_BRANCH = "master" -} +object ProjectConstants diff --git a/src/main/kotlin/com/exactpro/th2/converter/util/RepositoryUtils.kt b/src/main/kotlin/com/exactpro/th2/converter/util/Repository.kt similarity index 83% rename from src/main/kotlin/com/exactpro/th2/converter/util/RepositoryUtils.kt rename to src/main/kotlin/com/exactpro/th2/converter/util/Repository.kt index 0539d45..5c0e43e 100644 --- a/src/main/kotlin/com/exactpro/th2/converter/util/RepositoryUtils.kt +++ b/src/main/kotlin/com/exactpro/th2/converter/util/Repository.kt @@ -20,6 +20,7 @@ import com.exactpro.th2.converter.controllers.errors.ErrorCode import com.exactpro.th2.converter.controllers.errors.ServiceException import com.exactpro.th2.converter.model.Th2Resource import com.exactpro.th2.infrarepo.InconsistentRepositoryStateException +import com.exactpro.th2.infrarepo.ResourceType import com.exactpro.th2.infrarepo.git.Gitter import com.exactpro.th2.infrarepo.git.GitterContext import com.exactpro.th2.infrarepo.repo.Repository @@ -28,6 +29,11 @@ import mu.KotlinLogging object RepositoryUtils { + const val PROPAGATION_RULE = "rule" + const val PROPAGATION_DENY = "deny" + + const val SOURCE_BRANCH = "master" + private val logger = KotlinLogging.logger { } fun updateRepository( @@ -105,3 +111,23 @@ object RepositoryUtils { return branches.contains(schemaName) } } + +data class RepositoryContext( + val boxes: Set, + val links: Set +) { + val allResources = boxes.plus(links) + + companion object { + fun load(gitter: Gitter): RepositoryContext { + gitter.lock() + try { + val boxes = HashSet(Repository.getAllBoxesAndStores(gitter)) + val links = HashSet(Repository.getResourcesByKind(gitter, ResourceType.Th2Link)) + return RepositoryContext(boxes, links) + } finally { + gitter.unlock() + } + } + } +} diff --git a/src/main/kotlin/com/exactpro/th2/converter/util/SchemaVersion.kt b/src/main/kotlin/com/exactpro/th2/converter/util/SchemaVersion.kt new file mode 100644 index 0000000..c0b1755 --- /dev/null +++ b/src/main/kotlin/com/exactpro/th2/converter/util/SchemaVersion.kt @@ -0,0 +1,29 @@ +/* + * Copyright 2022-2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.converter.util + +enum class SchemaVersion(val apiVersion: String, val nextVersion: SchemaVersion?) { + V2_2("th2.exactpro.com/v2-2", null), + V2("th2.exactpro.com/v2", V2_2), + V1("th2.exactpro.com/v1", V2); + + companion object { + fun fromApiVersion(apiVersion: String) = SchemaVersion.values().first { schemaVersion -> + schemaVersion.apiVersion == apiVersion + } + } +} diff --git a/src/test/kotlin/ConverterTest.kt b/src/test/kotlin/ConverterTest.kt index aa6b249..f8862ff 100644 --- a/src/test/kotlin/ConverterTest.kt +++ b/src/test/kotlin/ConverterTest.kt @@ -14,24 +14,23 @@ * limitations under the License. */ -import com.exactpro.th2.converter.controllers.errors.BadRequestException import com.exactpro.th2.converter.conversion.Converter import com.exactpro.th2.converter.`fun`.ConvertibleBoxSpecV2 +import com.exactpro.th2.converter.`fun`.ConvertibleBoxSpecV2x2 import com.exactpro.th2.converter.model.Th2Resource import com.exactpro.th2.converter.util.Mapper.YAML_MAPPER +import com.exactpro.th2.converter.util.SchemaVersion import com.exactpro.th2.infrarepo.repo.RepositoryResource -import com.exactpro.th2.model.latest.box.Spec import com.exactpro.th2.model.latest.box.pins.PinSpec import com.exactpro.th2.model.v1.box.SpecV1 import com.exactpro.th2.model.v1.box.pins.PinSpecV1 import com.exactpro.th2.model.v1.box.pins.PinType +import com.exactpro.th2.model.v2.SpecV2 import com.fasterxml.jackson.module.kotlin.convertValue import io.fabric8.kubernetes.api.model.ObjectMeta import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertDoesNotThrow -import org.junit.jupiter.api.assertThrows import java.io.File -import java.util.* import kotlin.test.DefaultAsserter.assertTrue import kotlin.test.assertEquals import kotlin.test.assertNotNull @@ -52,6 +51,8 @@ internal class ConverterTest { private val check1V1 = readV1RepoResource(CHECK1) + private val check1V2 = readV2RepoResource(CHECK1) + private val fixServerV1 = readV1RepoResource(FIX_SERVER) private val fixServerV2 = readV2RepoResource(FIX_SERVER) @@ -81,7 +82,7 @@ internal class ConverterTest { */ @Test fun testPinsConversionToV2FromRequest() { - val convertedTh2ResList = Converter.convertFromRequest("v2", repoV1BoxesFullSet) + val convertedTh2ResList = Converter.convertFromRequest(SchemaVersion.V1, SchemaVersion.V2, repoV1BoxesFullSet) .convertedResources .sortedBy { it.metadata.name } @@ -159,15 +160,19 @@ internal class ConverterTest { val sampleResSet = setOf(scriptV1, links1, links2) val failMessage = "script conversion v1 -> v2 failed" - val actualConvertedList = Converter.convertFromRequest("v2", sampleResSet).convertedResources + val actualConvertedList = Converter.convertFromRequest( + SchemaVersion.V1, + SchemaVersion.V2, + sampleResSet + ).convertedResources val actualConvertedScript = actualConvertedList .find { it.metadata.name == "script" } as Th2Resource - var expectedConvertedScriptSpec: Spec? = null + var expectedConvertedScriptSpec: SpecV2? = null assertDoesNotThrow("$failMessage: Spec structure or naming is incorrect") { expectedConvertedScriptSpec = YAML_MAPPER.convertValue(scriptV2.spec) } val expectedConvertedScript = Th2Resource( - scriptV2.apiVersion, + SchemaVersion.fromApiVersion(scriptV2.apiVersion), scriptV2.kind, scriptV2.metadata, ConvertibleBoxSpecV2(expectedConvertedScriptSpec!!) @@ -182,21 +187,6 @@ internal class ConverterTest { ) } - /** - * Tests Converter.convertFromRequest(targetVersion, resources). - * Tests scenarios where we specify unsupported version - * And when we specify the supported one - */ - @Test - fun testConvertFromRequestTargetVersion() { - assertThrows("Function didn't throw Exception when specified unsupported version") { - Converter.convertFromRequest("v4", repoV1BoxesFullSet) - } - assertDoesNotThrow("Function threw Exception when specified supported version") { - Converter.convertFromRequest("v2", repoV1BoxesFullSet) - } - } - /** * Tests Converter.convertFromRequest(targetVersion, resources). * From V1 to V2. @@ -204,15 +194,25 @@ internal class ConverterTest { */ @Test fun testServiceConversionToV2FromRequest() { - val convertedResMap = convertFromRequestAsMap("v2", setOf(actV1, fixServerV1, check1V1)) + val convertedResMap = convertFromRequestAsMap( + SchemaVersion.V1, + SchemaVersion.V2, + setOf(actV1, fixServerV1, check1V1) + ) val actSpec = boxSpecV2FromMap(convertedResMap, "act") val actualActService = YAML_MAPPER.writeValueAsString(actSpec.spec.extendedSettings?.service) val expectedActService = """ enabled: true nodePort: + - name: port2 + containerPort: 80 + exposedPort: 30742 - name: grpc containerPort: 8080 exposedPort: 30741 + ingress: + urlPaths: + - /path1 """.trimIndent().plus("\n") assertEquals( @@ -228,6 +228,9 @@ internal class ConverterTest { clusterIP: - name: other containerPort: 8080 + ingress: + urlPaths: + - /path4 """.trimIndent().plus("\n") assertEquals( @@ -241,10 +244,13 @@ internal class ConverterTest { val expectedCheck1Service = """ enabled: true loadBalancer: - - name: grpc - containerPort: 8080 - - name: test + - name: port2 containerPort: 8081 + - name: grpc + containerPort: 80 + ingress: + urlPaths: + - /path2 """.trimIndent().plus("\n") assertEquals( @@ -264,7 +270,7 @@ internal class ConverterTest { val repoV1AllResourcesSet = HashSet(repoV1BoxesFullSet) repoV1AllResourcesSet.add(links1) repoV1AllResourcesSet.add(links2) - val convertedResMap = convertFromRequestAsMap("v2", repoV1AllResourcesSet) + val convertedResMap = convertFromRequestAsMap(SchemaVersion.V1, SchemaVersion.V2, repoV1AllResourcesSet) val actualFixServerSpec = boxSpecV2FromMap(convertedResMap, "fix-server") val actualFixServerLinks = actualFixServerSpec.spec.pins?.mq?.subscribers @@ -312,7 +318,7 @@ internal class ConverterTest { @Test fun testV2DictionaryLinksInBoxes() { val repoV1ResourceSet = setOf(scriptV1, dictionaryLinks) - val convertedResMap = convertFromRequestAsMap("v2", repoV1ResourceSet) + val convertedResMap = convertFromRequestAsMap(SchemaVersion.V1, SchemaVersion.V2, repoV1ResourceSet) val expectedScriptSpec = readV2ResourceSpec(scriptV2) val expectedScriptCustomCfg: MutableMap? = expectedScriptSpec.customConfig @@ -327,11 +333,182 @@ internal class ConverterTest { ) } + /** + * v2 conversion to v2-2. check that ingress urlPaths are inserted into service ports + */ + @Test + fun testV2toV2x2Ingress() { + val convertedResMap = convertFromRequestAsMap( + SchemaVersion.V2, + SchemaVersion.V2_2, + setOf(actV2, check1V2, fixServerV2, scriptV2) + ) + val actSpec = boxSpecV2x2FromMap(convertedResMap, "act") + val actualActService = YAML_MAPPER.writeValueAsString(actSpec.spec.extendedSettings?.service) + val expectedActService = """ + enabled: true + nodePort: + - name: port2 + containerPort: 80 + exposedPort: 30742 + - name: grpc + containerPort: 8080 + exposedPort: 30741 + urlPath: /path1 + """.trimIndent().plus("\n") + assertEquals( + expectedActService, + actualActService, + "Service conversion in act v2 -> v2-2 failed" + ) + + val fixServerSpec = boxSpecV2x2FromMap(convertedResMap, "fix-server") + val actualFixServerService = YAML_MAPPER.writeValueAsString(fixServerSpec.spec.extendedSettings?.service) + val expectedFixServerService = """ + enabled: true + clusterIP: + - name: other + containerPort: 8080 + urlPath: /path4 + """.trimIndent().plus("\n") + + assertEquals( + expectedFixServerService, + actualFixServerService, + "Service conversion in fix-server v2 -> v2-2 failed" + ) + + val check1Spec = boxSpecV2x2FromMap(convertedResMap, "check1") + val actualCheck1Service = YAML_MAPPER.writeValueAsString(check1Spec.spec.extendedSettings?.service) + val expectedCheck1Service = """ + enabled: true + loadBalancer: + - name: port2 + containerPort: 8081 + - name: grpc + containerPort: 80 + urlPath: /path2 + """.trimIndent().plus("\n") + + assertEquals( + expectedCheck1Service, + actualCheck1Service, + "Service conversion in check1 v2 -> v2-2 failed" + ) + + val scriptSpec = boxSpecV2x2FromMap(convertedResMap, "script") + val actualScriptService = YAML_MAPPER.writeValueAsString(scriptSpec.spec.extendedSettings?.service) + val expectedScriptService = """ + enabled: false + clusterIP: + - name: port + containerPort: 8081 + urlPath: /path3 + - name: port2 + containerPort: 8083 + """.trimIndent().plus("\n") + + assertEquals( + expectedScriptService, + actualScriptService, + "Service conversion in check1 v2 -> v2-2 failed" + ) + } + + /** + * v1 conversion to v2-2. check that ingress urlPaths are inserted into service ports + */ + @Test + fun testV1toV2x2Ingress() { + val convertedResMap = convertFromRequestAsMap( + SchemaVersion.V1, + SchemaVersion.V2_2, + setOf(actV1, check1V1, fixServerV1, scriptV1) + ) + val actSpec = boxSpecV2x2FromMap(convertedResMap, "act") + val actualActService = YAML_MAPPER.writeValueAsString(actSpec.spec.extendedSettings?.service) + val expectedActService = """ + enabled: true + nodePort: + - name: port2 + containerPort: 80 + exposedPort: 30742 + - name: grpc + containerPort: 8080 + exposedPort: 30741 + urlPath: /path1 + """.trimIndent().plus("\n") + assertEquals( + expectedActService, + actualActService, + "Service conversion in act v1 -> v2-2 failed" + ) + + val fixServerSpec = boxSpecV2x2FromMap(convertedResMap, "fix-server") + val actualFixServerService = YAML_MAPPER.writeValueAsString(fixServerSpec.spec.extendedSettings?.service) + val expectedFixServerService = """ + enabled: true + clusterIP: + - name: other + containerPort: 8080 + urlPath: /path4 + """.trimIndent().plus("\n") + + assertEquals( + expectedFixServerService, + actualFixServerService, + "Service conversion in fix-server v1 -> v2-2 failed" + ) + + val check1Spec = boxSpecV2x2FromMap(convertedResMap, "check1") + val actualCheck1Service = YAML_MAPPER.writeValueAsString(check1Spec.spec.extendedSettings?.service) + val expectedCheck1Service = """ + enabled: true + loadBalancer: + - name: port2 + containerPort: 8081 + - name: grpc + containerPort: 80 + urlPath: /path2 + """.trimIndent().plus("\n") + + assertEquals( + expectedCheck1Service, + actualCheck1Service, + "Service conversion in check1 v1 -> v2-2 failed" + ) + + val scriptSpec = boxSpecV2x2FromMap(convertedResMap, "script") + val actualScriptService = YAML_MAPPER.writeValueAsString(scriptSpec.spec.extendedSettings?.service) + val expectedScriptService = """ + enabled: false + clusterIP: + - name: port + containerPort: 8081 + urlPath: /path3 + - name: port2 + containerPort: 8083 + """.trimIndent().plus("\n") + + assertEquals( + expectedScriptService, + actualScriptService, + "Service conversion in check1 v1 -> v2-2 failed" + ) + } + private fun boxSpecV2FromMap(convertedResMap: Map, resName: String) = convertedResMap[resName]?.specWrapper as ConvertibleBoxSpecV2 - private fun convertFromRequestAsMap(toVersion: String, resSet: Set): Map { - val convertedTh2ResList = Converter.convertFromRequest(toVersion, resSet).convertedResources + private fun boxSpecV2x2FromMap(convertedResMap: Map, resName: String) = + convertedResMap[resName]?.specWrapper as ConvertibleBoxSpecV2x2 + + private fun convertFromRequestAsMap( + currentVersion: SchemaVersion, + targetVersion: SchemaVersion, + resSet: Set + ): Map { + val convertedTh2ResList = Converter.convertFromRequest(currentVersion, targetVersion, resSet).convertedResources return convertedTh2ResList.associateBy { it.metadata.name } } @@ -342,8 +519,8 @@ internal class ConverterTest { ) } - private fun readV2ResourceSpec(res: RepositoryResource): Spec { - return YAML_MAPPER.convertValue(res.spec, Spec::class.java) + private fun readV2ResourceSpec(res: RepositoryResource): SpecV2 { + return YAML_MAPPER.convertValue(res.spec, SpecV2::class.java) } private fun readV1RepoResource(fileName: String): RepositoryResource { diff --git a/src/test/resources/V1/act.yml b/src/test/resources/V1/act.yml index f42c0c9..94c3908 100644 --- a/src/test/resources/V1/act.yml +++ b/src/test/resources/V1/act.yml @@ -50,9 +50,15 @@ spec: enabled: true type: NodePort endpoints: + - name: 'port2' + targetPort: 80 + nodePort: 30742 - name: 'grpc' targetPort: 8080 nodePort: 30741 + ingress: + urlPaths: + - '/path1' envVariables: JAVA_TOOL_OPTIONS: "-XX:+ExitOnOutOfMemoryError" resources: diff --git a/src/test/resources/V1/check1.yml b/src/test/resources/V1/check1.yml index cf15261..8bd4fe5 100644 --- a/src/test/resources/V1/check1.yml +++ b/src/test/resources/V1/check1.yml @@ -54,10 +54,13 @@ spec: enabled: true type: LoadBalancer endpoints: - - name: grpc - targetPort: 8080 - - name: test + - name: port2 targetPort: 8081 + - name: grpc + targetPort: 80 + ingress: + urlPaths: + - '/path2' envVariables: JAVA_TOOL_OPTIONS: '-XX:+ExitOnOutOfMemoryError -XX:+UseContainerSupport -XX:MaxRAMPercentage=85' resources: diff --git a/src/test/resources/V1/fix-server.yml b/src/test/resources/V1/fix-server.yml index 6f3cb47..9a59bbd 100644 --- a/src/test/resources/V1/fix-server.yml +++ b/src/test/resources/V1/fix-server.yml @@ -82,6 +82,9 @@ spec: endpoints: - name: 'other' targetPort: 8080 + ingress: + urlPaths: + - '/path4' envVariables: JAVA_TOOL_OPTIONS: | -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=1099 -Dcom.sun.management.jmxremote.rmi.port=1099 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1 -XX:+ExitOnOutOfMemoryError -XX:+UseContainerSupport -XX:MaxRAMPercentage=85 diff --git a/src/test/resources/V1/script.yml b/src/test/resources/V1/script.yml index eb83ce4..b8da965 100644 --- a/src/test/resources/V1/script.yml +++ b/src/test/resources/V1/script.yml @@ -27,3 +27,12 @@ spec: enabled: true service: enabled: false + type: ClusterIP + endpoints: + - name: 'port' + targetPort: 8081 + - name: 'port2' + targetPort: 8083 + ingress: + urlPaths: + - '/path3' diff --git a/src/test/resources/V2/act.yml b/src/test/resources/V2/act.yml index 441577e..d60891f 100644 --- a/src/test/resources/V2/act.yml +++ b/src/test/resources/V2/act.yml @@ -19,9 +19,15 @@ spec: service: enabled: true nodePort: + - name: port2 + containerPort: 80 + exposedPort: 30742 - name: grpc containerPort: 8080 exposedPort: 30741 + ingress: + urlPaths: + - '/path1' pins: mq: subscribers: diff --git a/src/test/resources/V2/check1.yml b/src/test/resources/V2/check1.yml index ee56278..1f67fa7 100644 --- a/src/test/resources/V2/check1.yml +++ b/src/test/resources/V2/check1.yml @@ -27,10 +27,14 @@ spec: memory: 100Mi service: enabled: true - nodePort: + loadBalancer: + - name: port2 + containerPort: 8081 - name: grpc - containerPort: 8080 - exposedPort: 30742 + containerPort: 80 + ingress: + urlPaths: + - '/path2' pins: mq: subscribers: diff --git a/src/test/resources/V2/fix-server.yml b/src/test/resources/V2/fix-server.yml index db2f8b5..0a8c3c0 100644 --- a/src/test/resources/V2/fix-server.yml +++ b/src/test/resources/V2/fix-server.yml @@ -71,6 +71,9 @@ spec: clusterIP: - name: other containerPort: 8080 + ingress: + urlPaths: + - '/path4' pins: mq: subscribers: diff --git a/src/test/resources/V2/script.yml b/src/test/resources/V2/script.yml index 84441a2..5704189 100644 --- a/src/test/resources/V2/script.yml +++ b/src/test/resources/V2/script.yml @@ -36,3 +36,11 @@ spec: enabled: true service: enabled: false + clusterIP: + - name: port + containerPort: 8081 + - name: port2 + containerPort: 8083 + ingress: + urlPaths: + - '/path3' diff --git a/src/test/resources/V2_2/act.yml b/src/test/resources/V2_2/act.yml new file mode 100644 index 0000000..46560ce --- /dev/null +++ b/src/test/resources/V2_2/act.yml @@ -0,0 +1,65 @@ +apiVersion: th2.exactpro.com/v2 +kind: Th2Box +metadata: + name: act +spec: + imageName: ghcr.io/th2-net/th2-act-template-j + imageVersion: 3.5.02123 + type: th2-act + extendedSettings: + envVariables: + JAVA_TOOL_OPTIONS: '-XX:+ExitOnOutOfMemoryError' + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 50m + memory: 110Mi + service: + enabled: true + nodePort: + - name: port2 + containerPort: 80 + exposedPort: 30742 + - name: grpc + containerPort: 8080 + exposedPort: 30741 + urlPath: '/path1' + pins: + mq: + subscribers: + - name: from_codec + attributes: + - first + - oe + - subscribe + - parsed + - raw + - att + publishers: + - name: to_send + attributes: + - raw + - publish + - test + - name: pub_test + attributes: + - raw + - publish + grpc: + client: + - name: to_check1 + serviceClass: com.exactpro.th2.check1.grpc.Check1Service + linkTo: + - box: check1 + pin: server + - box: check1 + pin: server2 + server: + - name: server + serviceClasses: + - com.exactpro.th2.act.grpc.ActService + - name: server2 + serviceClasses: + - com.exactpro.th2.act.grpc.ActService diff --git a/src/test/resources/V2_2/check1.yml b/src/test/resources/V2_2/check1.yml new file mode 100644 index 0000000..7909ccf --- /dev/null +++ b/src/test/resources/V2_2/check1.yml @@ -0,0 +1,80 @@ +apiVersion: th2.exactpro.com/v2 +kind: Th2Box +metadata: + name: check1 +spec: + imageName: ghcr.io/th2-net/th2-check1 + imageVersion: 3.9.0-dev-1553192360 + type: th2-check1 + customConfig: + message-cache-size: 1000 + cleanup-older-than: 10 + cleanup-time-unit: SECONDS + rule-execution-timeout: '7000' + time-precision: PT0.000000001S + decimal-precision: 0.001 + extendedSettings: + envVariables: + JAVA_TOOL_OPTIONS: >- + -XX:+ExitOnOutOfMemoryError -XX:+UseContainerSupport + -XX:MaxRAMPercentage=85 + resources: + limits: + cpu: 200m + memory: 600Mi + requests: + cpu: 110m + memory: 100Mi + service: + enabled: true + nodePort: + - name: port2 + containerPort: 8081 + exposedPort: 30745 + - name: grpc + containerPort: 80 + exposedPort: 30746 + urlPath: '/path2' + pins: + mq: + subscribers: + - name: from_codec + attributes: + - subscribe + - parsed + linkTo: + - box: act + pin: to_send + - name: sub_check_test + attributes: + - subscribe + - parsed + linkTo: + - box: act + pin: pub_test + publishers: + - name: check_pub + attributes: + - raw + - publish + grpc: + server: + - name: server + serviceClasses: + - com.exactpro.th2.check1.grpc.Check1Service + - name: server2 + serviceClasses: + - com.exactpro.th2.check1.grpc.Check1Service + loggingConfig: > + log4j.rootLogger=INFO, CON + + log4j.appender.CON=org.apache.log4j.ConsoleAppender + + log4j.appender.CON.layout=org.apache.log4j.PatternLayout + + log4j.appender.CON.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} + %-6p [%-15t] %c - %m%n + + log4j.logger.com.exactpro.th2.readlog.impl.RegexpContentParser=INFO + + log4j.logger.com.exactpro.th2=INFO diff --git a/src/test/resources/V2_2/fix-server.yml b/src/test/resources/V2_2/fix-server.yml new file mode 100644 index 0000000..56a99d5 --- /dev/null +++ b/src/test/resources/V2_2/fix-server.yml @@ -0,0 +1,105 @@ +apiVersion: th2.exactpro.com/v2 +kind: Th2Box +metadata: + name: fix-server +spec: + imageName: ghcr.io/th2-net/th2-conn-fix + imageVersion: 3.4.2 + type: th2-conn + customConfig: + enableMessageSendingEvent: true + maxMessageBatchSize: 1000 + session-alias: fix-server + workspace: /home/sailfish/workspace + type: 'th2_service:FIX_Server' + name: fix-server + settings: + autoStartable: true + expectedTimeOfStarting: 2000 + invertStoredMessageTypes: false + performDump: false + persistMessages: true + waitingTimeBeforeStarting: 0 + allowUnknownMsgFields: false + beginString: FIXT.1.1 + checkLatency: false + defaultApplVerID: 9 + duplicateTagsAllowed: false + endTime: '00:00:00' + fileStorePath: store/fix/sessions + logHeartbeats: false + maxLatency: 120 + microsecondsInTimeStampFields: false + millisecondsInTimeStampFields: true + orderingFields: false + receiveLimit: 0 + rejectInvalidMessage: false + resetOnDisconnect: false + resetOnLogout: false + senderCompID: FGW + startTime: '00:00:00' + targetCompID: DEMO-CONN1 + useLocalTime: false + useSSL: false + validateFieldsHaveValues: true + validateFieldsOutOfOrder: true + validateFieldsOutOfRange: true + validateSequenceNumbers: true + validateUserDefinedFields: true + applicationClassName: com.exactpro.sf.services.fix.ServerApplication + keepMessagesInMemory: true + socketAcceptPort: 8080 + extendedSettings: + envVariables: + JAVA_TOOL_OPTIONS: > + -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=1099 + -Dcom.sun.management.jmxremote.rmi.port=1099 + -Dcom.sun.management.jmxremote.ssl=false + -Dcom.sun.management.jmxremote.authenticate=false + -Dcom.sun.management.jmxremote.local.only=false + -Djava.rmi.server.hostname=127.0.0.1 -XX:+ExitOnOutOfMemoryError + -XX:+UseContainerSupport -XX:MaxRAMPercentage=85 + resources: + limits: + cpu: 1000m + memory: 2000Mi + requests: + cpu: 20m + memory: 100Mi + service: + enabled: true + loadBalancer: + - name: other + containerPort: 8080 + urlPath: '/path4' + pins: + mq: + subscribers: + - name: fix_to_send + attributes: + - send + - raw + - subscribe + linkTo: + - box: check1 + pin: check_pub + - box: act + pin: pub_test + filters: + - metadata: + - expectedValue: fix-server + fieldName: session_alias + operation: EQUAL + publishers: + - name: in_raw + attributes: + - first + - raw + - publish + - store + - name: out_raw + attributes: + - second + - raw + - publish + - store diff --git a/src/test/resources/V2_2/script.yml b/src/test/resources/V2_2/script.yml new file mode 100644 index 0000000..364435d --- /dev/null +++ b/src/test/resources/V2_2/script.yml @@ -0,0 +1,44 @@ +apiVersion: th2.exactpro.com/v2 +kind: Th2Box +metadata: + name: script +spec: + imageName: dev-script + imageVersion: dev-script + type: th2-script + customConfig: + single: "${dictionary_link:multi2}" + dictionaryMap: + dict_for_a: "${dictionary_link:multi1}" + dict_for_b: "${dictionary_link:multi2}" + subStructure: + genericFixDictionary: "${dictionary_link:multi1}" + subStructure: + genericFixDictionary: "${dictionary_link:multi2}" + + dictionaries: + LEVEL1: "${dictionary_link:level1-dict}" + MAIN: "${dictionary_link:main-dict}" + pins: + grpc: + client: + - name: to_act + serviceClass: com.exactpro.th2.act.grpc.ActService + linkTo: + - box: act + pin: server + - box: act + pin: server2 + - name: to_check1 + serviceClass: com.exactpro.th2.check1.grpc.Check1Service + extendedSettings: + externalBox: + enabled: true + service: + enabled: false + clusterIP: + - name: port + containerPort: 8081 + urlPath: '/path3' + - name: por2 + containerPort: 8083 \ No newline at end of file